def watch():
db = session.get_a_trading_db_mysql_session()
r = session.get_a_redis_connection()
logger.info('Reporting for duty.')
try:
while True:
logger.info('Scanning for abnormalities.')
check_profit(db)
check_open_pl(db)
check_ticktimes(db)
check_position(db)
check_btc_net_assets(db)
check_spreads_are_normal(db)
check_fv_not_stagnant(db)
check_fv_predictor_key_set(r)
# for some reason this commit is needed to pick up changes on
# subsequent queries.
session.commit_mysql_session(db)
logger.info('Going to sleep for %s seconds.' % TICK_SLEEP)
time.sleep(TICK_SLEEP)
finally:
db.remove()
def record(self,
datum_type,
numeric_value=None,
string_value=None,
meta_data={},
order=None):
datum = Datum(
datum_type,
numeric_value=numeric_value,
string_value=string_value,
meta_data=meta_data,
order=order,
)
if not hasattr(self, 'db') and not hasattr(self, 'external_logger'):
raise Exception(
'DatumRecorder must be created before you can record')
if self.db:
self.db.add(datum)
commit_mysql_session(self.db)
elif self.external_logger:
self.external_logger.info(datum)
else:
# we aren't recording events.
pass
def trades_consumer_function(message, db):
subprocess.call(['touch', 'monit/heartbeat/trades_consumer.txt'])
trade_json = json.loads(message)
timestamp = epoch(trade_json['timestamp']).datetime
price_currency = trade_json.get('price_currency', 'USD')
volume_currency = trade_json.get('volume_currency', 'BTC')
t = Trade(
price=Money(trade_json['price'], price_currency),
volume=Money(trade_json['volume'], volume_currency),
exchange=unicode(trade_json['exchange']),
timestamp=timestamp,
exchange_trade_id=unicode(trade_json['trade_id']),
)
db.add(t)
try:
session.commit_mysql_session(db)
except exc.IntegrityError as e:
# We will get a duplicate entry error from the database if we happen to stop
# the bot after we write the entry but before we acknowlege it to the queue.
# This will cause an infinite loop of errors where we keep failing to write the
# same entry. In this case we can successfully ack the message because we
# already have it stored.
if 'Duplicate entry' in str(e):
return
else:
raise e
def exception_retry_loop(harness, sentry, db):
success = False
backoff_seconds = BACKOFF_SECONDS_START
while not success and not warm_shutdown_flag:
success = handle_exception(harness, sentry)
if not success:
logger.info(
tc.colored(
'handle_exception failed, backing off for %s' %
(humanize_seconds(backoff_seconds)),
'red',
))
gentle_sleep(backoff_seconds)
backoff_seconds *= BACKOFF_MULTIPLIER
else:
logger.info(
tc.colored(
'handle_exception succeeded, full speed ahead',
'green',
))
return
# This is necessary to flush local db cache and pick up changes other processes
# have made. For example, if this is a balance mismatch which was fixed by the
# withdrawal tool.
session.commit_mysql_session(db)
def withdraw(exchange_name, target_exchange_name, amount_str):
db = session.get_a_trading_db_mysql_session()
try:
exchange_data = make_exchange_data_from_key(exchange_name, db)
target_exchange_data = make_exchange_data_from_key(target_exchange_name, db)
target_exchange = make_exchange_from_key(target_exchange_name)
amount = Money.loads(amount_str)
addr = target_exchange.current_deposit_address
exchange_data.record_withdrawal(target_exchange_data, amount, addr)
session.commit_mysql_session(db)
logger.info(tc.colored("Recorded %s withdrawal from %s" % (amount, exchange_name), "green"))
finally:
db.remove()
def withdraw_fiat(exchange_name, target_exchange_name, amount_str, deposit_amount_str, transaction_details):
db = session.get_a_trading_db_mysql_session()
try:
exchange_data = make_exchange_data_from_key(exchange_name, db)
target_exchange_data = make_exchange_data_from_key(target_exchange_name, db)
amount = Money.loads(amount_str)
if deposit_amount_str:
deposit_amount = Money.loads(deposit_amount_str)
exchange_data.record_fiat_withdrawal(target_exchange_data, amount, deposit_amount=deposit_amount, transaction_details=transaction_details)
else:
exchange_data.record_fiat_withdrawal(target_exchange_data, amount, transaction_details=transaction_details)
session.commit_mysql_session(db)
logger.info(tc.colored("Recorded %s withdrawal from %s" % (amount, exchange_name), "green"))
finally:
db.remove()
def bitcoin_rebalance(db, exchange_data, algo, execute=False):
algo.log('------- Bitcoin Balancer Report -----------', (), 'yellow')
if exchange_data.name not in conf.withdrawals_enabled:
algo.log('Withdrawals not enabled for %s, not balancing',
exchange_data.name, 'yellow')
return
exchanges = get_deposit_enabled_exchanges(db)
destination_exchange_data = get_destination_exchange(
algo, exchange_data, exchanges)
if destination_exchange_data is None:
return
elif destination_exchange_data == exchange_data:
algo.log('%s not sending to self', exchange_data.name, 'yellow')
return
source_exchange = make_exchange_from_key(exchange_data.name)
destination_exchange = make_exchange_from_key(
destination_exchange_data.name)
transfer_amount = TRANSFER_UNIT + create_uniquifier()
algo.log(
'Sending %s from %s to %s',
(transfer_amount, source_exchange.name, destination_exchange.name),
'yellow')
if execute:
# Send the bitcoins and record the transactions.
deposit_address, transaction_hash, exchange_withdrawal_id = send_btc_to_exchange(
source_exchange, destination_exchange, transfer_amount)
transactions = exchange_data.record_withdrawal(
destination_exchange_data,
transfer_amount,
deposit_address,
transaction_hash,
exchange_withdrawal_id,
)
commit_mysql_session(db)
else:
algo.log('Not Sending Because of NO EXECUTE', (), 'yellow')
def run():
db = session.get_a_trading_db_mysql_session()
DatumRecorder().create(db=db)
logger.info('Reporting for duty.')
try:
while True:
current_time = Delorean().datetime
try:
update_tx_hashes(db)
except Exception as e:
logger.exception('Error while updating transaction hashes')
# Checking the exact minute amounts works because the shoebox is ticking every 1m
# This may not work 100% if the shoebox tick takes any noticable amount of time
# to run. In that case we might tick at the end of xx:59 and the beginning of xx:01,
# and these tasks wouldn't run. We'll keep an eye on it and may have to add some
# persistance later.
# every 5 minutes
if current_time.minute % 5 == 0:
try:
get_breaking_bitcoin_news()
except Exception:
logger.exception('Error while getting breaking news')
# every hour
if current_time.minute == 0:
money_moving()
manual_btc_withdrawals(db)
# end of day UTC
if current_time.hour == 0 and current_time.minute == 0:
notify_revenue(db)
heartbeat(SHOEBOX_HEARTBEAT_KEY)
session.commit_mysql_session(db)
logger.info('Going to sleep for %s.' %
humanize_seconds(TICK_SLEEP))
time.sleep(TICK_SLEEP)
finally:
db.remove()
def transaction_complete(exchange_name, currency):
db = session.get_a_trading_db_mysql_session()
try:
exchange_data = make_exchange_data_from_key(exchange_name, db)
tr = db.query(Transaction).filter_by(exchange=exchange_data).filter_by(_amount_currency=currency).filter_by(transaction_status=Transaction.IN_TRANSIT).order_by(Transaction.time_created).first()
if tr:
tr.complete()
session.commit_mysql_session(db)
if tr.transaction_type == Transaction.DEPOSIT:
action = "deposit to"
elif tr.transaction_type == Transaction.WITHDRAWL:
action = "withdrawal from"
logger.info(tc.colored("Recorded %s %s %s" % (tr.amount, action, exchange_name), "green"))
else:
logger.info(tc.colored("No Transaction of that currency found", "red"))
finally:
db.remove()
def run():
db = session.get_a_trading_db_mysql_session()
try:
logger.info('Reporting for duty.')
while True:
audit_bmo_accounts(db)
audit_boa_accounts(db)
session.commit_mysql_session(db)
logger.info('Going to sleep for %s.' %
humanize_seconds(TICK_SLEEP))
time.sleep(TICK_SLEEP)
finally:
db.remove()
def exchange_volumes_consumer_function(message, db):
subprocess.call(['touch', 'monit/heartbeat/exchange_volumes_consumer.txt'])
exchange_volume_json = json.loads(message)
timestamp = epoch(exchange_volume_json['timestamp']).datetime
exchange = exchange_volume_json['exchange_name']
exch_vol_money = Money(exchange_volume_json['volume'], 'BTC')
t = ExchangeVolume(
exchange_volume=exch_vol_money,
exchange=exchange,
timestamp=timestamp,
)
db.add(t)
session.commit_mysql_session(db)
def wind_down(exchange_name, strategy_params, execute=False):
db = session.get_a_trading_db_mysql_session()
try:
strategy = Strategy(
make_exchange_from_key(exchange_name),
db,
debug=False,
backtest=False,
execute=execute,
params=strategy_params,
)
harness = Harness(strategy, db)
harness.wind_down()
finally:
session.commit_mysql_session(db)
db.remove()
def record(self, event_type, exchange_name='', data={}):
event = Event(event_type, exchange_name, data)
if not hasattr(self, 'db') and not hasattr(self, 'external_logger'):
# we didn't call create. we aren't recording events
return
if self.db:
self.db.add(event)
commit_mysql_session(self.db)
elif self.external_logger:
self.external_logger.info(
'[EVENT] %s : %s : %s -- %s' %
(event.exchange_name, event.event_type, str(
event.time_created), json.dumps(event.data)))
else:
# we aren't recording events.
pass
def orderbook_consumer_function(message, db):
subprocess.call(["touch", "monit/heartbeat/orderbook_consumer.txt"])
ob = json.loads(message)
assert len(ob.keys()) == 2
exchange_name = list(set(ob.keys()) - set(['timestamp'])).pop()
timestamp = ob['timestamp']
orderbook_data = ob[exchange_name]
orderbook = Orderbook(
exchange_name,
orderbook=orderbook_data,
timestamp=epoch(timestamp).datetime,
)
db.add(orderbook)
session.commit_mysql_session(db)
def write_trades_to_db(our_exchange_id, trades_list):
db = session.get_a_gds_db_mysql_session()
try:
for i in range(len(trades_list)):
trade = Trade(
trades_list[i][1],
trades_list[i][2],
our_exchange_id,
trades_list[i][0],
None,
source='BITCOINCHARTS'
)
db.add(trade)
if i % 30 == 0:
session.commit_mysql_session(db)
session.commit_mysql_session(db)
finally:
db.remove()
def live_run(configuration):
strategy_name = configuration['platform']['strategy']
is_builtin_strategy = configuration['platform']['builtin']
execute = configuration['platform']['execute']
logger.info('live_run(%s, %s)' % (strategy_name, execute))
db = session.get_a_trading_db_mysql_session()
harness = Harness(db, configuration)
strategy_class = get_strategy_class(strategy_name, is_builtin_strategy)
strategy = strategy_class(db, harness, configuration['strategy'])
strategy.set_up()
harness.strategy = strategy
if execute:
EventRecorder().create(db=db)
DatumRecorder().create(db=db)
else:
EventRecorder().create()
DatumRecorder().create()
sentry = Sentry(configuration['platform']['sentry'])
heartbeat = Heartbeat(configuration['platform']['heartbeat'])
try:
tick_count = 0
while True:
try:
tick_start = Delorean().epoch
print '\n\n%s' % strategy.name
if warm_shutdown_flag:
return # This takes us into the finally block.
# Initial audit. This is done inside the main loop so that our robust
# exception catching kicks in on initial audit failures.
if harness.audit is True and tick_count == 0:
# We try a fast audit (no wind down) since the bots usually start
# from a clean slate.
try:
harness.full_audit(wind_down=False)
except AuditException:
logger.info(
'Bot was not cleanly shut down, winding down and auditing',
)
harness.full_audit(wind_down=True)
# Regular audits.
if (harness.audit is True and tick_count > 0
and tick_count % harness.audit_tick == 0):
harness.full_audit()
else:
harness.tick()
harness.post_tick(tick_count)
tick_profiling.record_tick_data(tick_start, strategy.name)
tick_profiling.record_tick_block_data(
strategy,
tick_count,
strategy.name,
)
heartbeat.heartbeat(strategy.name)
except Exception as e:
sentry.captureException()
logger.exception(
tc.colored(
'[%s] %s' % (e.__class__.__name__, e.message),
'red',
))
exception_retry_loop(harness, sentry, db)
finally:
session.commit_mysql_session(db)
tick_count += 1
if harness.strategy_complete() is True:
break
else:
gentle_sleep(harness.sleep_time_to_next_tick())
finally:
warm_shutdown(harness, db, sentry, execute)
session.commit_mysql_session(db)
db.remove()
if restart_flag:
restart()
def manual_accounting(exchange_name, order_id, actor, execute=False):
if actor not in EXPECTED_ACTORS:
logger.warning(tc.colored(ACTOR_WARNING, color='yellow'))
db = session.get_a_trading_db_mysql_session()
try:
exchange = make_exchange_from_key(exchange_name)
exchange_data = make_exchange_data_from_key(exchange_name, db)
vol_currency_key = '%s_total' % exchange.volume_currency.lower()
try:
exchange.cancel_order(order_id)
except CancelOrderNotFoundError:
pass
details = exchange.get_order_details(order_id)
unit_price = details['fiat_total'] / details[vol_currency_key].amount
try:
order = db.query(Order)\
.filter(Order.exchange_order_id == order_id)\
.filter(Order._exchange_name == exchange.name)\
.one()
order.time_created = epoch(details['time_created']).naive
action = 'Updated'
except sqlalchemy.orm.exc.NoResultFound:
order = Order(
actor,
details['type'],
details[vol_currency_key],
unit_price,
exchange,
order_id,
)
order.time_created = epoch(details['time_created']).naive
order.exchange_rate = Money('1',
exchange.currency).to('USD').amount
action = 'Added'
position_change, position_change_no_fees = order.was_eaten(details)
old_balance = exchange_data.balance[exchange.volume_currency]
for currency_code, position in position_change.iteritems():
exchange_data.position[currency_code] += position
exchange_data.balance[currency_code] += position
logger.info(
'Order: %s for %.4f @ %.2f',
order.order_type,
order.volume,
order.price,
)
for trade in order.trades:
logger.info(
'Trade: %s for %.4f @ %.2f',
trade.trade_type,
trade.volume,
trade.price,
)
if execute:
db.add(order)
db.add(exchange_data)
session.commit_mysql_session(db)
logger.info(
tc.colored(
'%s order #%s and its %s trade(s)' %
(action, order_id, len(details['trades'])),
'green',
))
logger.info(
tc.colored(
'Updated balance from %s to %s' %
(old_balance,
exchange_data.balance[exchange.volume_currency]),
'green',
))
else:
logger.info(
tc.colored(
'pass --execute to save this order to the db',
'red',
))
finally:
db.remove()
