def save_daily_df(self, base_dir='data'): logging.info('upload_daily_df') self.daily_trade_started = False t_1 = datetime.datetime.utcnow() df_daily = self.aggregations.get_daily_df() t_2 = datetime.datetime.utcnow() dt_21 = t_2 - t_1 logging.info('[save_daily_df] {s} seconds took to get daily_df'.format( s=dt_21.seconds)) if not os.path.exists(base_dir): os.mkdir(base_dir) df_daily.to_csv('{base_dir}/daily.csv'.format(base_dir=base_dir)) return df_daily
def on_daily_trade_end(self, base_dir='data'): logging.info('on_daily_trade_end') self.daily_trade_started = False t_1 = datetime.datetime.utcnow() df_minute = self.aggregations.get_minute_df() t_2 = datetime.datetime.utcnow() dt_21 = t_2 - t_1 logging.info( '{s} seconds took to get minute_df'.format(s=dt_21.seconds)) df_daily = self.aggregations.get_daily_df() t_3 = datetime.datetime.utcnow() dt_32 = t_3 - t_2 logging.info( '{s} seconds took to get daily_df'.format(s=dt_32.seconds)) if not os.path.exists(base_dir): os.mkdir(base_dir) df_minute.to_csv('{base_dir}/minute.csv'.format(base_dir=base_dir)) df_daily.to_csv('{base_dir}/daily.csv'.format(base_dir=base_dir)) self.aggregations.clean()
def get_minute_df(self, print_log=True): if print_log: logging.info('Aggregations.get_minute_df for {l_s} symbols'.format( l_s=len(self.aggregation_per_symbol))) df = pd.DataFrame(columns=BarWithTime.get_minute_tuple_names()) for symbol, aggregation in self.aggregation_per_symbol.items(): t_1 = datetime.datetime.utcnow() df_ = aggregation.get_minute_df() t_2 = datetime.datetime.utcnow() dt_21 = t_2 - t_1 logging.info( '{s} seconds {ms} microseconds took to get minute_df for symbol {symbol}' .format(s=dt_21.seconds, ms=dt_21.microseconds, symbol=symbol)) df = df.append(df_) t_3 = datetime.datetime.utcnow() dt_32 = t_3 - t_2 logging.info( '{s} seconds {ms} microseconds took to append for symbol {symbol}' .format(s=dt_32.seconds, ms=dt_32.microseconds, symbol=symbol)) return df.set_index('datetime')
def on_daily_trade_end(self, base_dir='data'): logging.info('on_daily_trade_end') self.daily_trade_started = False self.aggregations.clean()
def on_daily_trade_start(self): logging.info('on_daily_trade_start') self.daily_trade_started = True
def log_heartbeat(): while True: logging.info("us_finance_streaming_data_miner: heartbeat message.") time.sleep(30 * 60)
def run(forcerun): cfg = config.load('config.us.yaml') tz = config.get_tz(cfg) polygon_run = PolygonAggregationsRun(subscription_id=os.getenv( 'FINANCE_STREAM_INTRADAY_PUBSUB_SUBSCRIPTION_ID')) while True: dt = us_finance_streaming_data_miner.util.time.get_utcnow().astimezone( tz) dt_str = str(dt.date()) if dt.weekday() >= 5: logging.info( 'skipping the routing during weekend, weekday: {weekday} for {dt_str}' .format(weekday=dt.weekday(), dt_str=dt_str)) time.sleep(60 * 60) continue logging.info('checking if run for {date_str} should be done'.format( date_str=dt_str)) if not forcerun and us_finance_streaming_data_miner.history.history.did_run_today( cfg): logging.info( 'run for {date_str} is already done'.format(date_str=dt_str)) time.sleep(10 * 60) continue t_market_open = config.get_market_open(cfg) while True: t_cur = us_finance_streaming_data_miner.util.time.get_utcnow( ).astimezone(tz).time() logging.info( cfg, 'checking if the schedule time for {dt_str} has reached'. format(dt_str=dt_str)) if forcerun or t_cur > t_market_open: polygon_run.on_daily_trade_start() break logging.info( cfg, 'schedule time {t_run_after} not yet reached at {t_cur}'. format(t_run_after=t_market_open, t_cur=t_cur)) time.sleep(10 * 60) if forcerun: time.sleep(70) t_ingest_end = config.get_market_ingest_end(cfg) while True: t_cur = us_finance_streaming_data_miner.util.time.get_utcnow( ).astimezone(tz).time() logging.info( cfg, 'checking if the schedule time for {dt_str} has reached'. format(dt_str=dt_str)) logging.info(polygon_run.get_status_string()) if forcerun or t_cur > t_ingest_end: polygon_run.save_daily_df() daily_upload.upload() polygon_run.on_daily_trade_end() us_finance_streaming_data_miner.history.history.on_run(cfg) break logging.info( cfg, 'schedule time {t_run_after} not yet reached at {t_cur}'. format(t_run_after=t_ingest_end, t_cur=t_cur)) time.sleep(10 * 60) if forcerun: # forcerun runs only once break
def run(subscription_id, shard_id, shard_size): logging.info('starting the job: {dt_str}'.format(dt_str=datetime.datetime.now())) _ = BinanceAggregationsRun(shard_id = shard_id, shard_size = shard_size, subscription_id = subscription_id)