def ingest_all_historical_etf(date_from = '1993-01-29', date_to=None, symbols=None):
if symbols is None:
symbols = Symbols.get_all_symbols()
date_to = date_to or datetime.date.today().strftime("%Y-%m-%d")
logger = Logger(__name__, PathMgr.get_log_path())
for symbol in symbols:
logger.info('ingest for %s...' % symbol)
path = PathMgr.get_historical_etf_path(symbol)
content = YahooScraper.download_quote2(symbol, date_from, date_to)
write_to_file(path, content)
time.sleep(1)
def get_missing_records_symbols(self):
conn = self.get_connection()
cursor = conn.cursor()
missing_symbols = []
last_trade_date = TradeTime.get_latest_trade_date()
for symbol in Symbols.get_all_symbols():
end_date = self.get_end_date_by_symbol(symbol, cursor)
if end_date < last_trade_date:
missing_symbols.append(symbol)
conn.close()
return missing_symbols
def get_start_end_date_by_symbols(self):
reversed_yahoo_symbol_mapping = Symbols.get_reversed_yahoo_symbol_mapping(
)
conn = self.get_connection()
cursor = conn.cursor()
records = []
for symbol in Symbols.get_all_symbols():
start_date = self.get_start_date_by_symbol(symbol, cursor)
end_date = self.get_end_date_by_symbol(symbol, cursor)
records.append([
Symbols.get_mapped_symbol(symbol,
reversed_yahoo_symbol_mapping),
start_date, end_date
])
records.sort()
conn.close()
return records
def insert_all(self):
for symbol in Symbols.get_all_symbols():
self.logger.info('insert data for %s...' % symbol)
path = PathMgr.get_historical_etf_path(symbol)
df = pd.read_csv(path)
self.insert(symbol, df)
def save_all(self, symbols=Symbols.get_all_symbols()):
for symbol in symbols:
self.logger.info('save data for %s...' % symbol)
path = PathMgr.get_historical_etf_path(symbol)
df = pd.read_csv(path)
self.save(symbol, df)