def do_backfill(self):
#
self.logger.info("Run EXO backfill from {0}".format(
self.args.backfill))
assetindex = AssetIndexMongo(MONGO_CONNSTR, MONGO_EXO_DB)
exostorage = EXOStorage(MONGO_CONNSTR, MONGO_EXO_DB)
futures_limit = 3
options_limit = 20
# datasource = DataSourceMongo(mongo_connstr, mongo_db_name, assetindex, futures_limit, options_limit, exostorage)
datasource = DataSourceSQL(SQL_HOST, SQL_USER, SQL_PASS, assetindex,
futures_limit, options_limit, exostorage)
exos = exostorage.exo_list(exo_filter=self.args.instrument + '_',
return_names=True)
exo_start_dates = {}
exec_time, decision_time = AssetIndexMongo.get_exec_time(
self.args.backfill, self.asset_info)
current_time = decision_time
if len(exos) > 0:
for exo_name in exos:
series = exostorage.load_series(exo_name)[0]
if series is not None:
last_date = series.index[-1] + timedelta(days=1)
exec_time, decision_time = AssetIndexMongo.get_exec_time(
last_date, self.asset_info)
self.logger.info(
'Updating existing {0} series from: {1}'.format(
exo_name, decision_time))
exo_start_dates[exo_name] = decision_time
else:
self.logger.info('Updating new EXO series from: {0}'.format(
self.args.backfill))
exec_time, decision_time = AssetIndexMongo.get_exec_time(
self.args.backfill, self.asset_info)
exec_time_end, decision_time_end = AssetIndexMongo.get_exec_time(
datetime.now(), self.asset_info)
while current_time <= decision_time_end:
self.logger.info("Backfilling: {0}".format(current_time))
self.run_exo_calc(datasource,
current_time,
args.instrument,
backfill_dict=exo_start_dates)
current_time += timedelta(days=1)
exec_time += timedelta(days=1)
def on_new_quote(self, appclass, appname, data):
if data.mtype != MsgQuoteNotification.mtype:
return
# Check data integrity
if not self.check_quote_data(appname, appclass, data):
return
exec_time, decision_time = AssetIndexMongo.get_exec_time(
datetime.now(), self.asset_info)
start_time = time.time()
quote_date = data['date']
symbol = appname
if quote_date > decision_time:
# TODO: Check to avoid dupe launch
# Run first EXO calculation for this day
self.logger.info(
"Run EXO calculation, at decision time: {0}".format(
decision_time))
assetindex = AssetIndexMongo(MONGO_CONNSTR, MONGO_EXO_DB)
exostorage = EXOStorage(MONGO_CONNSTR, MONGO_EXO_DB)
futures_limit = 3
options_limit = 10
#datasource = DataSourceMongo(mongo_connstr, mongo_db_name, assetindex, futures_limit, options_limit, exostorage)
#datasource = DataSourceSQL(SQL_HOST, SQL_USER, SQL_PASS, assetindex, futures_limit, options_limit, exostorage)
#
# Test DB temporary credentials
#
tmp_mongo_connstr = 'mongodb://*****:*****@10.0.1.2/tmldb_test?authMechanism=SCRAM-SHA-1'
tmp_mongo_db = 'tmldb_test'
datasource = DataSourceHybrid(SQL_HOST, SQL_USER, SQL_PASS,
assetindex, tmp_mongo_connstr,
tmp_mongo_db, futures_limit,
options_limit, exostorage)
# Run EXO calculation
self.run_exo_calc(datasource,
decision_time,
symbol,
backfill_dict=None)
end_time = time.time()
self.signalapp.send(
MsgStatus('OK',
'EXO processed for {0} at {1}'.format(
symbol, quote_date),
context={
'instrument': symbol,
'date': quote_date,
'exec_time': end_time - start_time
},
notify=True))
else:
self.logger.debug("Waiting next decision time")
def build_smartexo(self, start_date, **smartexo_kwargs):
self.clear_smartexo()
logging.info(
"Starting EXO calculation process from: {0}".format(start_date))
if self.smartexo_class.ASSET_LIST is None:
warnings.warn(
"You must define ASSET_LIST inside SmartEXO class. Aborting..."
)
return
for ticker in self.smartexo_class.ASSET_LIST:
logging.info("Processing: " + ticker)
currdate = start_date
enddate = datetime.combine(datetime.now().date(), dttime(0, 0, 0))
while currdate <= enddate:
start_time = time.time()
date = currdate
asset_info = self.assetindex.get_instrument_info(ticker)
exec_time_end, decision_time_end = AssetIndexMongo.get_exec_time(
date, asset_info)
logging.info("\t\tRun on {0}".format(decision_time_end))
with self.smartexo_class(ticker, 0, decision_time_end,
self.datasource,
**smartexo_kwargs) as exo_engine:
try:
asset_list = exo_engine.ASSET_LIST
# Checking if current symbol is present in EXO class ASSET_LIST
if asset_list is not None:
if ticker not in asset_list:
# Skipping assets which are not in the list
continue
except AttributeError:
warnings.warn(
"EXO class {0} doesn't contain ASSET_LIST attribute filter, calculating all assets"
.format(self.smartexo_class))
# Load EXO information from mongo
exo_engine.load()
exo_engine.calculate()
end_time = time.time()
currdate += timedelta(days=1)
logging.debug("Elapsed: {0}".format(end_time - start_time))
logging.info('Done')
def process_quote(self, last_bar_time):
dtnow = self.date_now()
exec_time, decision_time = AssetIndexMongo.get_exec_time(
dtnow, self.asset_info)
if self.last_quote_date is None:
self.last_quote_date = self.get_last_quote_date()
quote_status = 'IDLE'
if self.is_quote_delayed(last_bar_time):
if self.last_minute != dtnow.minute:
logging.info('Quote delayed more than {0} minutes '
'for {1} LastBarTimeDB: {2} Now: {3}'.format(
self.args.delay, self.args.instrument,
last_bar_time, dtnow))
self.signalapp.send(
MsgStatus('DELAY',
'Quote delayed more than {0} minutes '
'for {1} LastBarTimeDB: {2} Now: {3}'.format(
self.args.delay, self.args.instrument,
last_bar_time, dtnow),
notify=True))
quote_status = 'DELAY'
# Fire new quote notification if last_bar_time > decision_time
if self.last_quote_date.date() != last_bar_time.date(
) and last_bar_time > decision_time:
if quote_status != 'DELAY':
quote_status = 'RUN'
# Reporting current status
self.signalapp.send(
MsgStatus('RUN',
'Processing new bar {0}'.format(last_bar_time)))
logging.info(
'Running new bar. Bar time: {0}'.format(last_bar_time))
self.last_quote_date = last_bar_time
context = {
'last_bar_time': last_bar_time,
'now': dtnow,
'last_run_date': self.last_quote_date,
'decision_time': decision_time,
'execution_time': exec_time,
'instrument': self.args.instrument,
'quote_status': quote_status,
}
logging.debug('Current context:\n {0}'.format(
self.pprinter.pformat(context)))
self.signalapp.send(
MsgQuoteNotification(self.args.instrument, last_bar_time,
context))
self.set_last_quote_state(context, update=False)
else:
context = {
'last_bar_time': last_bar_time,
'now': dtnow,
'last_run_date': self.last_quote_date,
'decision_time': decision_time,
'execution_time': exec_time,
'instrument': self.args.instrument,
'quote_status': quote_status,
}
if self.last_quote_date == NULL_DATE:
# If quote is absent is status_quotes collection, insert new
self.set_last_quote_state(context, update=False)
else:
self.set_last_quote_state(context, update=True)
# Log initial information:
if self.last_minute == -1:
logging.debug('Current context:\n {0}'.format(
self.pprinter.pformat(context)))
elif self.last_minute != dtnow.minute:
logging.debug('Last bar time {0}'.format(last_bar_time))
#self.signalapp.send(MsgStatus('IDLE', 'Last bar time {0}'.format(last_bar_time), context))
self.last_minute = dtnow.minute
enddate = datetime.combine(datetime.now().date(), dttime(12, 45, 0))
currdate = base_date
instruments = ['CL', 'ES', 'NG', 'ZC', 'ZS', 'ZW', 'ZN']
directions = [1] #[1, -1]
# for i in range(100):
while currdate <= enddate:
start_time = time.time()
# date = base_date + timedelta(days=i)
date = currdate
for ticker in instruments:
asset_info = assetindex.get_instrument_info(ticker)
exec_time_end, decision_time_end = AssetIndexMongo.get_exec_time(
date, asset_info)
for dir in directions:
with SmartEXOichimokuFutures(
ticker,
dir,
exec_time_end,
datasource,
log_file_path=DEBUG) as exo_engine:
# Load EXO information from mongo
exo_engine.load()
exo_engine.calculate()
end_time = time.time()
currdate += timedelta(days=1)
