def add_missing_data(): logger.info('Add missing minute data.') EquityMinDAO().add_missing_data('SVXY') EquityRealTimeDAO().add_missing_data('SVXY') EquityMinDAO().add_missing_data('SPY') EquityRealTimeDAO().add_missing_data('SPY') logger.info('Completed.')
class RealTimeDataCollector(object): def __init__(self): self.symbols = ['SVXY', 'SPY'] # , 'VIX', 'SVXY', 'UVXY'] self.equity_realtime_dao = EquityRealTimeDAO() @property def logger(self): return LoggerFactory.create_daily_logger( __name__, PathMgr.get_log_path('realtime')) def collect_data(self, symbol): self.logger.info('ingest data for %s...' % symbol) us_dt = datetime.datetime.now(pytz.timezone('US/Eastern')) if us_dt.hour == 9 and us_dt.minute == 30 and us_dt.second == 0: us_dt = datetime.datetime(us_dt.year, us_dt.month, us_dt.day, 9, 30, 1) price = None try: price = SINAScraper().get_current_data(symbol) except Exception as e: self.logger.exception(str(e)) if price is None: try: # price = MarketWatchScraper().get_current_data(symbol) price = YahooScraper().get_current_data(symbol) except Exception as e: self.logger.exception(str(e)) if price is not None: self.logger.info('save record into database...') self.equity_realtime_dao.insert(symbol, us_dt, price) else: self.logger.info('ingest record failed at %s...' % us_dt) def run(self): self.logger.info("start...") while True: if TradeTime.is_market_open(): try: for symbol in self.symbols: self.collect_data(symbol) self.logger.info('Check missing data...') count = self.equity_realtime_dao.add_missing_data_in_real_time( symbol) if count is not None and count > 0: self.logger.info( 'Filled {} records...'.format(count)) except Exception as e: self.logger.error('Trace: ' + traceback.format_exc()) self.logger.error(str(e)) time.sleep(1) else: self.logger.info('market not open...') time.sleep(10)
def add_missing_data_to_realtime_from_min(self, date, symbol='SVXY'): start_time = datetime.datetime.fromordinal(date.toordinal()) end_time = start_time + datetime.timedelta(days=1) rows = EquityMinDAO().get_time_and_price(symbol=symbol, start_time=start_time, end_time=end_time) rows = map(list, rows) rows[0][0] = rows[0][0] + datetime.timedelta(seconds=1) for row in rows: EquityRealTimeDAO().insert(symbol, row[0], row[1])
def plot_yahoo_equity_line(): real_data = EquityRealTimeDAO().get_min_time_and_price('XIV', datetime.datetime(2018, 1, 23, 9, 30, 0), datetime.datetime(2018, 1, 23, 16, 30, 0)) min_data = EquityMinDAO().get_time_and_price('XIV', datetime.datetime(2018, 1, 23, 9, 30, 0), datetime.datetime(2018, 1, 23, 16, 30, 0)) time = map(lambda x: x[0], real_data) real_values = map(lambda x: x[1], real_data) min_values = map(lambda x: x[1], min_data) fig, ax = plt.subplots() ax.plot(time, min_values, label='min') ax.plot(time, real_values, label='real') plt.legend(bbox_to_anchor=(1.05, 1), loc=8, borderaxespad=0.) plt.show()
def realtimedata_to_csv(): records = \ EquityRealTimeDAO().get_min_time_and_price('SVXY', datetime.datetime(2018, 05, 29, 0, 0, 0), datetime.datetime(2018, 05, 30, 0, 0, 0), ) lines = map(lambda x: '%s,%s' % (x[0], x[1]), records[1:]) content = '\n'.join(lines) write_path = PathMgr.get_data_path( 'quantopian_daily_min/realtime_%s%s.csv' % ('SVXY', '20180529')) write_to_file(write_path, content)
def GET(self): svxy_integrity_p, svxy_results = EquityRealTimeDAO().validate_integrity_for_real_time_data('SVXY') spy_integrity_p, spy_results = EquityRealTimeDAO().validate_integrity_for_real_time_data('SPY') return render.realtime_data_status(svxy_integrity_p, svxy_results, spy_integrity_p, spy_results)
def save_minute_data_to_csv(): logger.info('Save minute data.') EquityMinDAO().save_to_csv() EquityRealTimeDAO().save_to_csv() logger.info('Completed.')
def __init__(self): self.symbols = ['SVXY', 'SPY'] # , 'VIX', 'SVXY', 'UVXY'] self.equity_realtime_dao = EquityRealTimeDAO()