def save_daily_df(self, base_dir='data'):
logging.info('upload_daily_df')
self.daily_trade_started = False
t_1 = datetime.datetime.utcnow()
df_daily = self.aggregations.get_daily_df()
t_2 = datetime.datetime.utcnow()
dt_21 = t_2 - t_1
logging.info('[save_daily_df] {s} seconds took to get daily_df'.format(
s=dt_21.seconds))
if not os.path.exists(base_dir):
os.mkdir(base_dir)
df_daily.to_csv('{base_dir}/daily.csv'.format(base_dir=base_dir))
return df_daily
def on_daily_trade_end(self, base_dir='data'):
logging.info('on_daily_trade_end')
self.daily_trade_started = False
t_1 = datetime.datetime.utcnow()
df_minute = self.aggregations.get_minute_df()
t_2 = datetime.datetime.utcnow()
dt_21 = t_2 - t_1
logging.info(
'{s} seconds took to get minute_df'.format(s=dt_21.seconds))
df_daily = self.aggregations.get_daily_df()
t_3 = datetime.datetime.utcnow()
dt_32 = t_3 - t_2
logging.info(
'{s} seconds took to get daily_df'.format(s=dt_32.seconds))
if not os.path.exists(base_dir):
os.mkdir(base_dir)
df_minute.to_csv('{base_dir}/minute.csv'.format(base_dir=base_dir))
df_daily.to_csv('{base_dir}/daily.csv'.format(base_dir=base_dir))
self.aggregations.clean()
def get_minute_df(self, print_log=True):
if print_log:
logging.info('Aggregations.get_minute_df for {l_s} symbols'.format(
l_s=len(self.aggregation_per_symbol)))
df = pd.DataFrame(columns=BarWithTime.get_minute_tuple_names())
for symbol, aggregation in self.aggregation_per_symbol.items():
t_1 = datetime.datetime.utcnow()
df_ = aggregation.get_minute_df()
t_2 = datetime.datetime.utcnow()
dt_21 = t_2 - t_1
logging.info(
'{s} seconds {ms} microseconds took to get minute_df for symbol {symbol}'
.format(s=dt_21.seconds, ms=dt_21.microseconds, symbol=symbol))
df = df.append(df_)
t_3 = datetime.datetime.utcnow()
dt_32 = t_3 - t_2
logging.info(
'{s} seconds {ms} microseconds took to append for symbol {symbol}'
.format(s=dt_32.seconds, ms=dt_32.microseconds, symbol=symbol))
return df.set_index('datetime')
def on_daily_trade_end(self, base_dir='data'):
logging.info('on_daily_trade_end')
self.daily_trade_started = False
self.aggregations.clean()
def on_daily_trade_start(self):
logging.info('on_daily_trade_start')
self.daily_trade_started = True
def log_heartbeat():
while True:
logging.info("us_finance_streaming_data_miner: heartbeat message.")
time.sleep(30 * 60)
def run(forcerun):
cfg = config.load('config.us.yaml')
tz = config.get_tz(cfg)
polygon_run = PolygonAggregationsRun(subscription_id=os.getenv(
'FINANCE_STREAM_INTRADAY_PUBSUB_SUBSCRIPTION_ID'))
while True:
dt = us_finance_streaming_data_miner.util.time.get_utcnow().astimezone(
tz)
dt_str = str(dt.date())
if dt.weekday() >= 5:
logging.info(
'skipping the routing during weekend, weekday: {weekday} for {dt_str}'
.format(weekday=dt.weekday(), dt_str=dt_str))
time.sleep(60 * 60)
continue
logging.info('checking if run for {date_str} should be done'.format(
date_str=dt_str))
if not forcerun and us_finance_streaming_data_miner.history.history.did_run_today(
cfg):
logging.info(
'run for {date_str} is already done'.format(date_str=dt_str))
time.sleep(10 * 60)
continue
t_market_open = config.get_market_open(cfg)
while True:
t_cur = us_finance_streaming_data_miner.util.time.get_utcnow(
).astimezone(tz).time()
logging.info(
cfg, 'checking if the schedule time for {dt_str} has reached'.
format(dt_str=dt_str))
if forcerun or t_cur > t_market_open:
polygon_run.on_daily_trade_start()
break
logging.info(
cfg, 'schedule time {t_run_after} not yet reached at {t_cur}'.
format(t_run_after=t_market_open, t_cur=t_cur))
time.sleep(10 * 60)
if forcerun:
time.sleep(70)
t_ingest_end = config.get_market_ingest_end(cfg)
while True:
t_cur = us_finance_streaming_data_miner.util.time.get_utcnow(
).astimezone(tz).time()
logging.info(
cfg, 'checking if the schedule time for {dt_str} has reached'.
format(dt_str=dt_str))
logging.info(polygon_run.get_status_string())
if forcerun or t_cur > t_ingest_end:
polygon_run.save_daily_df()
daily_upload.upload()
polygon_run.on_daily_trade_end()
us_finance_streaming_data_miner.history.history.on_run(cfg)
break
logging.info(
cfg, 'schedule time {t_run_after} not yet reached at {t_cur}'.
format(t_run_after=t_ingest_end, t_cur=t_cur))
time.sleep(10 * 60)
if forcerun:
# forcerun runs only once
break
def run(subscription_id, shard_id, shard_size):
logging.info('starting the job: {dt_str}'.format(dt_str=datetime.datetime.now()))
_ = BinanceAggregationsRun(shard_id = shard_id, shard_size = shard_size, subscription_id = subscription_id)