def test_get_record_dates(self):
symbols = [
{"symbol": "C6L.SI", "price_updated_at": None},
{"symbol": "ABC.SI", "price_updated_at": None},
]
balance_sheets = [
{"symbol_obj": "C6L.SI", "date": datetime(1994, 4, 1), "inventories": 1000},
{"symbol_obj": "C6L.SI", "date": datetime(1995, 4, 1), "inventories": 2000},
{"symbol_obj": "ABC.SI", "date": datetime(2000, 4, 1), "inventories": 10000},
]
with db.transaction():
Symbol.insert_many(symbols).execute()
BalanceSheet.insert_many(balance_sheets).execute()
dates = [r.date for r in query.get_record_dates("balance_sheet", "C6L.SI")]
self.assertEqual(dates, [datetime(1994, 4, 1), datetime(1995, 4, 1)])
initialize.init()
logger = logging.getLogger(__name__)
for report_type in wsj.FINANCIAL_REPORT_TYPES:
logger.info("Will update {0} data of all symbols not up to date on {1}.".format(report_type, end_date))
data_type = report_type.replace('-', '_')
symbol_update_dates = {s.symbol: s.updated_at for s in get_outdated_symbols(data_type, end_date, "stock")}
for symbol in symbol_update_dates:
try:
data = wsj.get_financial_data(symbol, "annual", report_type)
except MinerException as e:
logger.warning("Could not find updated {0} data of {1} due to {2}. Skip.".format(report_type, symbol, e.__class__.__name__))
continue
else:
records = list(csv_rows_to_records(symbol, data))
if symbol_update_dates[symbol]:
logger.info("Filter out {0} records of {1} that already exist".format(report_type, symbol))
existing_record_dates = set(r.date for r in get_record_dates(data_type, symbol))
records = [r for r in records if r["date"] not in existing_record_dates]
if records:
update_fundamentals(data_type, symbol, records, end_date)
else:
logger.info("No new {0} data for {1}.".format(report_type, symbol))
logger.info("Finished updating {0} data.".format(report_type))