def do_backfill(self): # self.logger.info("Run EXO backfill from {0}".format( self.args.backfill)) assetindex = AssetIndexMongo(MONGO_CONNSTR, MONGO_EXO_DB) exostorage = EXOStorage(MONGO_CONNSTR, MONGO_EXO_DB) futures_limit = 3 options_limit = 20 # datasource = DataSourceMongo(mongo_connstr, mongo_db_name, assetindex, futures_limit, options_limit, exostorage) datasource = DataSourceSQL(SQL_HOST, SQL_USER, SQL_PASS, assetindex, futures_limit, options_limit, exostorage) exos = exostorage.exo_list(exo_filter=self.args.instrument + '_', return_names=True) exo_start_dates = {} exec_time, decision_time = AssetIndexMongo.get_exec_time( self.args.backfill, self.asset_info) current_time = decision_time if len(exos) > 0: for exo_name in exos: series = exostorage.load_series(exo_name)[0] if series is not None: last_date = series.index[-1] + timedelta(days=1) exec_time, decision_time = AssetIndexMongo.get_exec_time( last_date, self.asset_info) self.logger.info( 'Updating existing {0} series from: {1}'.format( exo_name, decision_time)) exo_start_dates[exo_name] = decision_time else: self.logger.info('Updating new EXO series from: {0}'.format( self.args.backfill)) exec_time, decision_time = AssetIndexMongo.get_exec_time( self.args.backfill, self.asset_info) exec_time_end, decision_time_end = AssetIndexMongo.get_exec_time( datetime.now(), self.asset_info) while current_time <= decision_time_end: self.logger.info("Backfilling: {0}".format(current_time)) self.run_exo_calc(datasource, current_time, args.instrument, backfill_dict=exo_start_dates) current_time += timedelta(days=1) exec_time += timedelta(days=1)
def on_new_quote(self, appclass, appname, data): if data.mtype != MsgQuoteNotification.mtype: return # Check data integrity if not self.check_quote_data(appname, appclass, data): return exec_time, decision_time = AssetIndexMongo.get_exec_time( datetime.now(), self.asset_info) start_time = time.time() quote_date = data['date'] symbol = appname if quote_date > decision_time: # TODO: Check to avoid dupe launch # Run first EXO calculation for this day self.logger.info( "Run EXO calculation, at decision time: {0}".format( decision_time)) assetindex = AssetIndexMongo(MONGO_CONNSTR, MONGO_EXO_DB) exostorage = EXOStorage(MONGO_CONNSTR, MONGO_EXO_DB) futures_limit = 3 options_limit = 10 #datasource = DataSourceMongo(mongo_connstr, mongo_db_name, assetindex, futures_limit, options_limit, exostorage) #datasource = DataSourceSQL(SQL_HOST, SQL_USER, SQL_PASS, assetindex, futures_limit, options_limit, exostorage) # # Test DB temporary credentials # tmp_mongo_connstr = 'mongodb://*****:*****@10.0.1.2/tmldb_test?authMechanism=SCRAM-SHA-1' tmp_mongo_db = 'tmldb_test' datasource = DataSourceHybrid(SQL_HOST, SQL_USER, SQL_PASS, assetindex, tmp_mongo_connstr, tmp_mongo_db, futures_limit, options_limit, exostorage) # Run EXO calculation self.run_exo_calc(datasource, decision_time, symbol, backfill_dict=None) end_time = time.time() self.signalapp.send( MsgStatus('OK', 'EXO processed for {0} at {1}'.format( symbol, quote_date), context={ 'instrument': symbol, 'date': quote_date, 'exec_time': end_time - start_time }, notify=True)) else: self.logger.debug("Waiting next decision time")
def build_smartexo(self, start_date, **smartexo_kwargs): self.clear_smartexo() logging.info( "Starting EXO calculation process from: {0}".format(start_date)) if self.smartexo_class.ASSET_LIST is None: warnings.warn( "You must define ASSET_LIST inside SmartEXO class. Aborting..." ) return for ticker in self.smartexo_class.ASSET_LIST: logging.info("Processing: " + ticker) currdate = start_date enddate = datetime.combine(datetime.now().date(), dttime(0, 0, 0)) while currdate <= enddate: start_time = time.time() date = currdate asset_info = self.assetindex.get_instrument_info(ticker) exec_time_end, decision_time_end = AssetIndexMongo.get_exec_time( date, asset_info) logging.info("\t\tRun on {0}".format(decision_time_end)) with self.smartexo_class(ticker, 0, decision_time_end, self.datasource, **smartexo_kwargs) as exo_engine: try: asset_list = exo_engine.ASSET_LIST # Checking if current symbol is present in EXO class ASSET_LIST if asset_list is not None: if ticker not in asset_list: # Skipping assets which are not in the list continue except AttributeError: warnings.warn( "EXO class {0} doesn't contain ASSET_LIST attribute filter, calculating all assets" .format(self.smartexo_class)) # Load EXO information from mongo exo_engine.load() exo_engine.calculate() end_time = time.time() currdate += timedelta(days=1) logging.debug("Elapsed: {0}".format(end_time - start_time)) logging.info('Done')
def process_quote(self, last_bar_time): dtnow = self.date_now() exec_time, decision_time = AssetIndexMongo.get_exec_time( dtnow, self.asset_info) if self.last_quote_date is None: self.last_quote_date = self.get_last_quote_date() quote_status = 'IDLE' if self.is_quote_delayed(last_bar_time): if self.last_minute != dtnow.minute: logging.info('Quote delayed more than {0} minutes ' 'for {1} LastBarTimeDB: {2} Now: {3}'.format( self.args.delay, self.args.instrument, last_bar_time, dtnow)) self.signalapp.send( MsgStatus('DELAY', 'Quote delayed more than {0} minutes ' 'for {1} LastBarTimeDB: {2} Now: {3}'.format( self.args.delay, self.args.instrument, last_bar_time, dtnow), notify=True)) quote_status = 'DELAY' # Fire new quote notification if last_bar_time > decision_time if self.last_quote_date.date() != last_bar_time.date( ) and last_bar_time > decision_time: if quote_status != 'DELAY': quote_status = 'RUN' # Reporting current status self.signalapp.send( MsgStatus('RUN', 'Processing new bar {0}'.format(last_bar_time))) logging.info( 'Running new bar. Bar time: {0}'.format(last_bar_time)) self.last_quote_date = last_bar_time context = { 'last_bar_time': last_bar_time, 'now': dtnow, 'last_run_date': self.last_quote_date, 'decision_time': decision_time, 'execution_time': exec_time, 'instrument': self.args.instrument, 'quote_status': quote_status, } logging.debug('Current context:\n {0}'.format( self.pprinter.pformat(context))) self.signalapp.send( MsgQuoteNotification(self.args.instrument, last_bar_time, context)) self.set_last_quote_state(context, update=False) else: context = { 'last_bar_time': last_bar_time, 'now': dtnow, 'last_run_date': self.last_quote_date, 'decision_time': decision_time, 'execution_time': exec_time, 'instrument': self.args.instrument, 'quote_status': quote_status, } if self.last_quote_date == NULL_DATE: # If quote is absent is status_quotes collection, insert new self.set_last_quote_state(context, update=False) else: self.set_last_quote_state(context, update=True) # Log initial information: if self.last_minute == -1: logging.debug('Current context:\n {0}'.format( self.pprinter.pformat(context))) elif self.last_minute != dtnow.minute: logging.debug('Last bar time {0}'.format(last_bar_time)) #self.signalapp.send(MsgStatus('IDLE', 'Last bar time {0}'.format(last_bar_time), context)) self.last_minute = dtnow.minute
enddate = datetime.combine(datetime.now().date(), dttime(12, 45, 0)) currdate = base_date instruments = ['CL', 'ES', 'NG', 'ZC', 'ZS', 'ZW', 'ZN'] directions = [1] #[1, -1] # for i in range(100): while currdate <= enddate: start_time = time.time() # date = base_date + timedelta(days=i) date = currdate for ticker in instruments: asset_info = assetindex.get_instrument_info(ticker) exec_time_end, decision_time_end = AssetIndexMongo.get_exec_time( date, asset_info) for dir in directions: with SmartEXOichimokuFutures( ticker, dir, exec_time_end, datasource, log_file_path=DEBUG) as exo_engine: # Load EXO information from mongo exo_engine.load() exo_engine.calculate() end_time = time.time() currdate += timedelta(days=1)