def watch(): db = session.get_a_trading_db_mysql_session() r = session.get_a_redis_connection() logger.info('Reporting for duty.') try: while True: logger.info('Scanning for abnormalities.') check_profit(db) check_open_pl(db) check_ticktimes(db) check_position(db) check_btc_net_assets(db) check_spreads_are_normal(db) check_fv_not_stagnant(db) check_fv_predictor_key_set(r) # for some reason this commit is needed to pick up changes on # subsequent queries. session.commit_mysql_session(db) logger.info('Going to sleep for %s seconds.' % TICK_SLEEP) time.sleep(TICK_SLEEP) finally: db.remove()
def record(self, datum_type, numeric_value=None, string_value=None, meta_data={}, order=None): datum = Datum( datum_type, numeric_value=numeric_value, string_value=string_value, meta_data=meta_data, order=order, ) if not hasattr(self, 'db') and not hasattr(self, 'external_logger'): raise Exception( 'DatumRecorder must be created before you can record') if self.db: self.db.add(datum) commit_mysql_session(self.db) elif self.external_logger: self.external_logger.info(datum) else: # we aren't recording events. pass
def trades_consumer_function(message, db): subprocess.call(['touch', 'monit/heartbeat/trades_consumer.txt']) trade_json = json.loads(message) timestamp = epoch(trade_json['timestamp']).datetime price_currency = trade_json.get('price_currency', 'USD') volume_currency = trade_json.get('volume_currency', 'BTC') t = Trade( price=Money(trade_json['price'], price_currency), volume=Money(trade_json['volume'], volume_currency), exchange=unicode(trade_json['exchange']), timestamp=timestamp, exchange_trade_id=unicode(trade_json['trade_id']), ) db.add(t) try: session.commit_mysql_session(db) except exc.IntegrityError as e: # We will get a duplicate entry error from the database if we happen to stop # the bot after we write the entry but before we acknowlege it to the queue. # This will cause an infinite loop of errors where we keep failing to write the # same entry. In this case we can successfully ack the message because we # already have it stored. if 'Duplicate entry' in str(e): return else: raise e
def exception_retry_loop(harness, sentry, db): success = False backoff_seconds = BACKOFF_SECONDS_START while not success and not warm_shutdown_flag: success = handle_exception(harness, sentry) if not success: logger.info( tc.colored( 'handle_exception failed, backing off for %s' % (humanize_seconds(backoff_seconds)), 'red', )) gentle_sleep(backoff_seconds) backoff_seconds *= BACKOFF_MULTIPLIER else: logger.info( tc.colored( 'handle_exception succeeded, full speed ahead', 'green', )) return # This is necessary to flush local db cache and pick up changes other processes # have made. For example, if this is a balance mismatch which was fixed by the # withdrawal tool. session.commit_mysql_session(db)
def withdraw(exchange_name, target_exchange_name, amount_str): db = session.get_a_trading_db_mysql_session() try: exchange_data = make_exchange_data_from_key(exchange_name, db) target_exchange_data = make_exchange_data_from_key(target_exchange_name, db) target_exchange = make_exchange_from_key(target_exchange_name) amount = Money.loads(amount_str) addr = target_exchange.current_deposit_address exchange_data.record_withdrawal(target_exchange_data, amount, addr) session.commit_mysql_session(db) logger.info(tc.colored("Recorded %s withdrawal from %s" % (amount, exchange_name), "green")) finally: db.remove()
def withdraw_fiat(exchange_name, target_exchange_name, amount_str, deposit_amount_str, transaction_details): db = session.get_a_trading_db_mysql_session() try: exchange_data = make_exchange_data_from_key(exchange_name, db) target_exchange_data = make_exchange_data_from_key(target_exchange_name, db) amount = Money.loads(amount_str) if deposit_amount_str: deposit_amount = Money.loads(deposit_amount_str) exchange_data.record_fiat_withdrawal(target_exchange_data, amount, deposit_amount=deposit_amount, transaction_details=transaction_details) else: exchange_data.record_fiat_withdrawal(target_exchange_data, amount, transaction_details=transaction_details) session.commit_mysql_session(db) logger.info(tc.colored("Recorded %s withdrawal from %s" % (amount, exchange_name), "green")) finally: db.remove()
def bitcoin_rebalance(db, exchange_data, algo, execute=False): algo.log('------- Bitcoin Balancer Report -----------', (), 'yellow') if exchange_data.name not in conf.withdrawals_enabled: algo.log('Withdrawals not enabled for %s, not balancing', exchange_data.name, 'yellow') return exchanges = get_deposit_enabled_exchanges(db) destination_exchange_data = get_destination_exchange( algo, exchange_data, exchanges) if destination_exchange_data is None: return elif destination_exchange_data == exchange_data: algo.log('%s not sending to self', exchange_data.name, 'yellow') return source_exchange = make_exchange_from_key(exchange_data.name) destination_exchange = make_exchange_from_key( destination_exchange_data.name) transfer_amount = TRANSFER_UNIT + create_uniquifier() algo.log( 'Sending %s from %s to %s', (transfer_amount, source_exchange.name, destination_exchange.name), 'yellow') if execute: # Send the bitcoins and record the transactions. deposit_address, transaction_hash, exchange_withdrawal_id = send_btc_to_exchange( source_exchange, destination_exchange, transfer_amount) transactions = exchange_data.record_withdrawal( destination_exchange_data, transfer_amount, deposit_address, transaction_hash, exchange_withdrawal_id, ) commit_mysql_session(db) else: algo.log('Not Sending Because of NO EXECUTE', (), 'yellow')
def run(): db = session.get_a_trading_db_mysql_session() DatumRecorder().create(db=db) logger.info('Reporting for duty.') try: while True: current_time = Delorean().datetime try: update_tx_hashes(db) except Exception as e: logger.exception('Error while updating transaction hashes') # Checking the exact minute amounts works because the shoebox is ticking every 1m # This may not work 100% if the shoebox tick takes any noticable amount of time # to run. In that case we might tick at the end of xx:59 and the beginning of xx:01, # and these tasks wouldn't run. We'll keep an eye on it and may have to add some # persistance later. # every 5 minutes if current_time.minute % 5 == 0: try: get_breaking_bitcoin_news() except Exception: logger.exception('Error while getting breaking news') # every hour if current_time.minute == 0: money_moving() manual_btc_withdrawals(db) # end of day UTC if current_time.hour == 0 and current_time.minute == 0: notify_revenue(db) heartbeat(SHOEBOX_HEARTBEAT_KEY) session.commit_mysql_session(db) logger.info('Going to sleep for %s.' % humanize_seconds(TICK_SLEEP)) time.sleep(TICK_SLEEP) finally: db.remove()
def transaction_complete(exchange_name, currency): db = session.get_a_trading_db_mysql_session() try: exchange_data = make_exchange_data_from_key(exchange_name, db) tr = db.query(Transaction).filter_by(exchange=exchange_data).filter_by(_amount_currency=currency).filter_by(transaction_status=Transaction.IN_TRANSIT).order_by(Transaction.time_created).first() if tr: tr.complete() session.commit_mysql_session(db) if tr.transaction_type == Transaction.DEPOSIT: action = "deposit to" elif tr.transaction_type == Transaction.WITHDRAWL: action = "withdrawal from" logger.info(tc.colored("Recorded %s %s %s" % (tr.amount, action, exchange_name), "green")) else: logger.info(tc.colored("No Transaction of that currency found", "red")) finally: db.remove()
def run(): db = session.get_a_trading_db_mysql_session() try: logger.info('Reporting for duty.') while True: audit_bmo_accounts(db) audit_boa_accounts(db) session.commit_mysql_session(db) logger.info('Going to sleep for %s.' % humanize_seconds(TICK_SLEEP)) time.sleep(TICK_SLEEP) finally: db.remove()
def exchange_volumes_consumer_function(message, db): subprocess.call(['touch', 'monit/heartbeat/exchange_volumes_consumer.txt']) exchange_volume_json = json.loads(message) timestamp = epoch(exchange_volume_json['timestamp']).datetime exchange = exchange_volume_json['exchange_name'] exch_vol_money = Money(exchange_volume_json['volume'], 'BTC') t = ExchangeVolume( exchange_volume=exch_vol_money, exchange=exchange, timestamp=timestamp, ) db.add(t) session.commit_mysql_session(db)
def wind_down(exchange_name, strategy_params, execute=False): db = session.get_a_trading_db_mysql_session() try: strategy = Strategy( make_exchange_from_key(exchange_name), db, debug=False, backtest=False, execute=execute, params=strategy_params, ) harness = Harness(strategy, db) harness.wind_down() finally: session.commit_mysql_session(db) db.remove()
def record(self, event_type, exchange_name='', data={}): event = Event(event_type, exchange_name, data) if not hasattr(self, 'db') and not hasattr(self, 'external_logger'): # we didn't call create. we aren't recording events return if self.db: self.db.add(event) commit_mysql_session(self.db) elif self.external_logger: self.external_logger.info( '[EVENT] %s : %s : %s -- %s' % (event.exchange_name, event.event_type, str( event.time_created), json.dumps(event.data))) else: # we aren't recording events. pass
def orderbook_consumer_function(message, db): subprocess.call(["touch", "monit/heartbeat/orderbook_consumer.txt"]) ob = json.loads(message) assert len(ob.keys()) == 2 exchange_name = list(set(ob.keys()) - set(['timestamp'])).pop() timestamp = ob['timestamp'] orderbook_data = ob[exchange_name] orderbook = Orderbook( exchange_name, orderbook=orderbook_data, timestamp=epoch(timestamp).datetime, ) db.add(orderbook) session.commit_mysql_session(db)
def write_trades_to_db(our_exchange_id, trades_list): db = session.get_a_gds_db_mysql_session() try: for i in range(len(trades_list)): trade = Trade( trades_list[i][1], trades_list[i][2], our_exchange_id, trades_list[i][0], None, source='BITCOINCHARTS' ) db.add(trade) if i % 30 == 0: session.commit_mysql_session(db) session.commit_mysql_session(db) finally: db.remove()
def live_run(configuration): strategy_name = configuration['platform']['strategy'] is_builtin_strategy = configuration['platform']['builtin'] execute = configuration['platform']['execute'] logger.info('live_run(%s, %s)' % (strategy_name, execute)) db = session.get_a_trading_db_mysql_session() harness = Harness(db, configuration) strategy_class = get_strategy_class(strategy_name, is_builtin_strategy) strategy = strategy_class(db, harness, configuration['strategy']) strategy.set_up() harness.strategy = strategy if execute: EventRecorder().create(db=db) DatumRecorder().create(db=db) else: EventRecorder().create() DatumRecorder().create() sentry = Sentry(configuration['platform']['sentry']) heartbeat = Heartbeat(configuration['platform']['heartbeat']) try: tick_count = 0 while True: try: tick_start = Delorean().epoch print '\n\n%s' % strategy.name if warm_shutdown_flag: return # This takes us into the finally block. # Initial audit. This is done inside the main loop so that our robust # exception catching kicks in on initial audit failures. if harness.audit is True and tick_count == 0: # We try a fast audit (no wind down) since the bots usually start # from a clean slate. try: harness.full_audit(wind_down=False) except AuditException: logger.info( 'Bot was not cleanly shut down, winding down and auditing', ) harness.full_audit(wind_down=True) # Regular audits. if (harness.audit is True and tick_count > 0 and tick_count % harness.audit_tick == 0): harness.full_audit() else: harness.tick() harness.post_tick(tick_count) tick_profiling.record_tick_data(tick_start, strategy.name) tick_profiling.record_tick_block_data( strategy, tick_count, strategy.name, ) heartbeat.heartbeat(strategy.name) except Exception as e: sentry.captureException() logger.exception( tc.colored( '[%s] %s' % (e.__class__.__name__, e.message), 'red', )) exception_retry_loop(harness, sentry, db) finally: session.commit_mysql_session(db) tick_count += 1 if harness.strategy_complete() is True: break else: gentle_sleep(harness.sleep_time_to_next_tick()) finally: warm_shutdown(harness, db, sentry, execute) session.commit_mysql_session(db) db.remove() if restart_flag: restart()
def manual_accounting(exchange_name, order_id, actor, execute=False): if actor not in EXPECTED_ACTORS: logger.warning(tc.colored(ACTOR_WARNING, color='yellow')) db = session.get_a_trading_db_mysql_session() try: exchange = make_exchange_from_key(exchange_name) exchange_data = make_exchange_data_from_key(exchange_name, db) vol_currency_key = '%s_total' % exchange.volume_currency.lower() try: exchange.cancel_order(order_id) except CancelOrderNotFoundError: pass details = exchange.get_order_details(order_id) unit_price = details['fiat_total'] / details[vol_currency_key].amount try: order = db.query(Order)\ .filter(Order.exchange_order_id == order_id)\ .filter(Order._exchange_name == exchange.name)\ .one() order.time_created = epoch(details['time_created']).naive action = 'Updated' except sqlalchemy.orm.exc.NoResultFound: order = Order( actor, details['type'], details[vol_currency_key], unit_price, exchange, order_id, ) order.time_created = epoch(details['time_created']).naive order.exchange_rate = Money('1', exchange.currency).to('USD').amount action = 'Added' position_change, position_change_no_fees = order.was_eaten(details) old_balance = exchange_data.balance[exchange.volume_currency] for currency_code, position in position_change.iteritems(): exchange_data.position[currency_code] += position exchange_data.balance[currency_code] += position logger.info( 'Order: %s for %.4f @ %.2f', order.order_type, order.volume, order.price, ) for trade in order.trades: logger.info( 'Trade: %s for %.4f @ %.2f', trade.trade_type, trade.volume, trade.price, ) if execute: db.add(order) db.add(exchange_data) session.commit_mysql_session(db) logger.info( tc.colored( '%s order #%s and its %s trade(s)' % (action, order_id, len(details['trades'])), 'green', )) logger.info( tc.colored( 'Updated balance from %s to %s' % (old_balance, exchange_data.balance[exchange.volume_currency]), 'green', )) else: logger.info( tc.colored( 'pass --execute to save this order to the db', 'red', )) finally: db.remove()