class MinDataCollector(object):
def __init__(self):
self.logger = Logger(__name__, PathMgr.get_log_path('minutes'))
self.alpha_vantage = AlphaVantage()
self.equity_min_dao = EquityMinDAO()
self.symbols = [
'SVXY', 'SPY', 'SPX', 'VIX', 'UVXY', 'QQQ', 'QLD', 'SSO', 'TLT',
'UBT'
]
def collect_data(self):
for symbol in self.symbols:
self.logger.info('ingest data for %s...' % symbol)
equities = self.alpha_vantage.get_minutes_equites(symbol)
self.logger.info('save %s records into database...' % symbol)
self.equity_min_dao.insert(equities)
time.sleep(1)
def run(self):
self.logger.info("start...")
while True:
if TradeTime.is_market_open() or datetime.datetime.now(
).minute == 0:
try:
self.collect_data()
except Exception as e:
self.logger.error('Trace: ' + traceback.format_exc())
self.logger.error(str(e))
else:
time.sleep(1)
def add_missing_data():
logger.info('Add missing minute data.')
EquityMinDAO().add_missing_data('SVXY')
EquityRealTimeDAO().add_missing_data('SVXY')
EquityMinDAO().add_missing_data('SPY')
EquityRealTimeDAO().add_missing_data('SPY')
logger.info('Completed.')
def __init__(self):
self.logger = Logger(__name__, PathMgr.get_log_path('minutes'))
self.alpha_vantage = AlphaVantage()
self.equity_min_dao = EquityMinDAO()
self.symbols = [
'SVXY', 'SPY', 'SPX', 'VIX', 'UVXY', 'QQQ', 'QLD', 'SSO', 'TLT',
'UBT'
]
def save_to_db(self, code):
records = self.load_log(code)
count = 240
for i in range(len(records)/count):
sub_records = records[i*count: (i+1)*count]
print sub_records[0].tradeTime
EquityMinDAO().insert(sub_records)
def save_to_db(self,
code,
from_time=datetime.datetime(1990, 1, 1, 0, 0, 0)):
records = self.load_csv(code)
records = filter(lambda x: x is not None and x.tradeTime > from_time,
records)
count = 390
for i in range(len(records) / count):
sub_records = records[i * count:(i + 1) * count]
# sub_records = filter(lambda x: x is not None, sub_records)
print sub_records[0].tradeTime
EquityMinDAO().insert(sub_records)
left_count = len(records) % count
if left_count > 0:
left_records = records[-left_count:]
print left_records[0].tradeTime
EquityMinDAO().insert(left_records)
class MinDataCollectorSVXY(object):
def __init__(self):
self.logger = Logger(__name__, PathMgr.get_log_path('minutes'))
self.alpha_vantage = AlphaVantage()
self.equity_min_dao = EquityMinDAO()
def collect_data(self):
symbol = 'SVXY'
self.logger.info('ingest data for %s...' % symbol)
equities = self.alpha_vantage.get_minutes_equites(symbol)
self.logger.info('save %s records into database...' % symbol)
self.equity_min_dao.insert(equities)
def run(self):
self.collect_data()
self.logger.info('Check missing data...')
count = self.equity_min_dao.add_missing_data_in_real_time()
if count is not None and count > 0:
self.logger.info('Filled {} records...'.format(count))
def add_missing_data_to_realtime_from_min(self, date, symbol='SVXY'):
start_time = datetime.datetime.fromordinal(date.toordinal())
end_time = start_time + datetime.timedelta(days=1)
rows = EquityMinDAO().get_time_and_price(symbol=symbol,
start_time=start_time,
end_time=end_time)
rows = map(list, rows)
rows[0][0] = rows[0][0] + datetime.timedelta(seconds=1)
for row in rows:
EquityRealTimeDAO().insert(symbol, row[0], row[1])
def plot_yahoo_equity_line():
real_data = EquityRealTimeDAO().get_min_time_and_price('XIV', datetime.datetime(2018, 1, 23, 9, 30, 0), datetime.datetime(2018, 1, 23, 16, 30, 0))
min_data = EquityMinDAO().get_time_and_price('XIV', datetime.datetime(2018, 1, 23, 9, 30, 0), datetime.datetime(2018, 1, 23, 16, 30, 0))
time = map(lambda x: x[0], real_data)
real_values = map(lambda x: x[1], real_data)
min_values = map(lambda x: x[1], min_data)
fig, ax = plt.subplots()
ax.plot(time, min_values, label='min')
ax.plot(time, real_values, label='real')
plt.legend(bbox_to_anchor=(1.05, 1), loc=8, borderaxespad=0.)
plt.show()
def agg1mtodaily(symbol, start_time=datetime.datetime(1971, 1, 1, 0, 0, 0), end_time=datetime.datetime(9999, 1, 1, 0, 0, 0)):
records = EquityMinDAO().get_records(symbol, start_time, end_time)
sub_records = []
combined_records = []
for record in records:
sub_records.append(record)
if record[0].hour == 15:
sub_records.append(record)
combined_record = AGG30Min.combine_records(symbol, sub_records)
combined_records.append(combined_record)
sub_records = []
YahooEquityDAO().save_from_equities(combined_records)
def agg1to30(symbol, start_time=datetime.datetime(1971, 1, 1, 0, 0, 0), end_time=datetime.datetime(9999, 1, 1, 0, 0, 0)):
records = EquityMinDAO().get_records(symbol, start_time, end_time)
sub_records = []
combined_records = []
for record in records:
sub_records.append(record)
if record[0].minute%30 == 0:
sub_records.append(record)
combined_record = AGG30Min.combine_records(symbol, sub_records)
combined_records.append(combined_record)
sub_records = []
Equity30MinDAO().insert(combined_records)
def load_log(self, symbol, date):
file_name = '%s%s.log' % (symbol, date.strftime('%Y%m%d'))
path = PathMgr.get_data_path('quantopian_daily_min/%s' % file_name)
content = read_file_to_string(path)
lines = content.split('\n')
filtered_lines = filter(lambda x: len(x) > 100, lines)
lines = map(lambda x: string_fetch(x, 'PRINT ', ''), filtered_lines)
close_list_str = ','.join(lines)
prices_list = map(float, close_list_str.split(','))
datetimes = TradeTime.generate_datetimes(date, date)
equities = map(lambda x, y: Equity(symbol, x, y, y, y, y), datetimes,
prices_list)
EquityMinDAO().insert(equities)
def GET(self):
symbols = ['SPX', 'VIX', 'SVXY', 'UVXY', 'SPY', 'QQQ', 'QLD', 'SSO', 'TLT', 'UBT']
dao = EquityMinDAO()
status_list = map(lambda x: [x, dao.validate_integrity_for_min_data(x)], symbols)
return render.min_data_status(status_list)
def save_minute_data_to_csv():
logger.info('Save minute data.')
EquityMinDAO().save_to_csv()
EquityRealTimeDAO().save_to_csv()
logger.info('Completed.')
def __init__(self):
self.logger = Logger(__name__, PathMgr.get_log_path('minutes'))
self.alpha_vantage = AlphaVantage()
self.equity_min_dao = EquityMinDAO()
def remove_market_open_data_for_min(self):
EquityMinDAO().remove_market_open_records()