def add_missing_data():
logger.info('Add missing minute data.')
EquityMinDAO().add_missing_data('SVXY')
EquityRealTimeDAO().add_missing_data('SVXY')
EquityMinDAO().add_missing_data('SPY')
EquityRealTimeDAO().add_missing_data('SPY')
logger.info('Completed.')
class RealTimeDataCollector(object):
def __init__(self):
self.symbols = ['SVXY', 'SPY'] # , 'VIX', 'SVXY', 'UVXY']
self.equity_realtime_dao = EquityRealTimeDAO()
@property
def logger(self):
return LoggerFactory.create_daily_logger(
__name__, PathMgr.get_log_path('realtime'))
def collect_data(self, symbol):
self.logger.info('ingest data for %s...' % symbol)
us_dt = datetime.datetime.now(pytz.timezone('US/Eastern'))
if us_dt.hour == 9 and us_dt.minute == 30 and us_dt.second == 0:
us_dt = datetime.datetime(us_dt.year, us_dt.month, us_dt.day, 9,
30, 1)
price = None
try:
price = SINAScraper().get_current_data(symbol)
except Exception as e:
self.logger.exception(str(e))
if price is None:
try:
# price = MarketWatchScraper().get_current_data(symbol)
price = YahooScraper().get_current_data(symbol)
except Exception as e:
self.logger.exception(str(e))
if price is not None:
self.logger.info('save record into database...')
self.equity_realtime_dao.insert(symbol, us_dt, price)
else:
self.logger.info('ingest record failed at %s...' % us_dt)
def run(self):
self.logger.info("start...")
while True:
if TradeTime.is_market_open():
try:
for symbol in self.symbols:
self.collect_data(symbol)
self.logger.info('Check missing data...')
count = self.equity_realtime_dao.add_missing_data_in_real_time(
symbol)
if count is not None and count > 0:
self.logger.info(
'Filled {} records...'.format(count))
except Exception as e:
self.logger.error('Trace: ' + traceback.format_exc())
self.logger.error(str(e))
time.sleep(1)
else:
self.logger.info('market not open...')
time.sleep(10)
def add_missing_data_to_realtime_from_min(self, date, symbol='SVXY'):
start_time = datetime.datetime.fromordinal(date.toordinal())
end_time = start_time + datetime.timedelta(days=1)
rows = EquityMinDAO().get_time_and_price(symbol=symbol,
start_time=start_time,
end_time=end_time)
rows = map(list, rows)
rows[0][0] = rows[0][0] + datetime.timedelta(seconds=1)
for row in rows:
EquityRealTimeDAO().insert(symbol, row[0], row[1])
def plot_yahoo_equity_line():
real_data = EquityRealTimeDAO().get_min_time_and_price('XIV', datetime.datetime(2018, 1, 23, 9, 30, 0), datetime.datetime(2018, 1, 23, 16, 30, 0))
min_data = EquityMinDAO().get_time_and_price('XIV', datetime.datetime(2018, 1, 23, 9, 30, 0), datetime.datetime(2018, 1, 23, 16, 30, 0))
time = map(lambda x: x[0], real_data)
real_values = map(lambda x: x[1], real_data)
min_values = map(lambda x: x[1], min_data)
fig, ax = plt.subplots()
ax.plot(time, min_values, label='min')
ax.plot(time, real_values, label='real')
plt.legend(bbox_to_anchor=(1.05, 1), loc=8, borderaxespad=0.)
plt.show()
def realtimedata_to_csv():
records = \
EquityRealTimeDAO().get_min_time_and_price('SVXY',
datetime.datetime(2018, 05, 29, 0, 0, 0),
datetime.datetime(2018, 05, 30, 0, 0, 0),
)
lines = map(lambda x: '%s,%s' % (x[0], x[1]), records[1:])
content = '\n'.join(lines)
write_path = PathMgr.get_data_path(
'quantopian_daily_min/realtime_%s%s.csv' % ('SVXY', '20180529'))
write_to_file(write_path, content)
def GET(self):
svxy_integrity_p, svxy_results = EquityRealTimeDAO().validate_integrity_for_real_time_data('SVXY')
spy_integrity_p, spy_results = EquityRealTimeDAO().validate_integrity_for_real_time_data('SPY')
return render.realtime_data_status(svxy_integrity_p, svxy_results, spy_integrity_p, spy_results)
def save_minute_data_to_csv():
logger.info('Save minute data.')
EquityMinDAO().save_to_csv()
EquityRealTimeDAO().save_to_csv()
logger.info('Completed.')
def __init__(self):
self.symbols = ['SVXY', 'SPY'] # , 'VIX', 'SVXY', 'UVXY']
self.equity_realtime_dao = EquityRealTimeDAO()