def test_update_fundamentals_delete_old(self): symbols = [ {"symbol": "C6L.SI"}, {"symbol": "ABC.SI"}, ] prices = [ {"symbol_obj": "C6L.SI", "date": datetime(2012, 12, 21), "open": 0, "close": 0, "high": 0, "low": 0, "volume": 0, "adj_close": 99.9}, {"symbol_obj": "ABC.SI", "date": datetime(2012, 12, 21), "open": 0, "close": 0, "high": 0, "low": 0, "volume": 0, "adj_close": 669.5}, ] with db.transaction(): Symbol.insert_many(symbols).execute() Price.insert_many(prices).execute() symbol = "C6L.SI" records = [ {"symbol_obj": "C6L.SI", "date": datetime(2012, 12, 21), "open": 0, "close": 0, "high": 0, "low": 0, "volume": 0, "adj_close": 100.0}, {"symbol_obj": "C6L.SI", "date": datetime(2012, 12, 22), "open": 0, "close": 0, "high": 0, "low": 0, "volume": 0, "adj_close": 120.0}, ] query.update_fundamentals("price", symbol, records, datetime(2012, 12, 22), delete_old=True) updated_data = list(Price.select().dicts()) for p in updated_data: del p["id"] self.assertEqual(updated_data, prices[1:] + records)
def test_update_fundamentals(self): symbols = [ {"symbol": "C6L.SI"}, {"symbol": "ABC.SI"} ] balance_sheets = [ {"symbol_obj": "C6L.SI", "date": datetime(1994, 4, 1), "inventories": 1000}, {"symbol_obj": "C6L.SI", "date": datetime(1995, 4, 1), "inventories": 2000}, ] with db.transaction(): Symbol.insert_many(symbols).execute() BalanceSheet.insert_many(balance_sheets).execute() symbol = "C6L.SI" records = [ {"symbol_obj": "C6L.SI", "date": datetime(1996, 4, 1), "inventories": 4000}, {"symbol_obj": "C6L.SI", "date": datetime(1997, 4, 1), "inventories": 8000}, ] end_date = datetime(2012, 12, 22) query.update_fundamentals("balance_sheet", symbol, records, end_date) updated_data = list(BalanceSheet.select( BalanceSheet.symbol_obj, BalanceSheet.date, BalanceSheet.inventories ).dicts()) markers = list(Symbol.select( Symbol.symbol, Symbol.balance_sheet_updated_at ).dicts()) self.assertEqual(updated_data, balance_sheets + records) self.assertEqual(markers, [ {"symbol": "C6L.SI", "balance_sheet_updated_at": end_date}, {"symbol": "ABC.SI", "balance_sheet_updated_at": None}, ])
from fa.database.query import get_outdated_symbols, update_fundamentals import initialize from settings import * """ Download data from internet to database """ initialize.init() logger = logging.getLogger(__name__) logger.info("Will update historical prices of all symbols not up to date on {0}.".format(end_date)) all_symbols = [s.symbol for s in get_outdated_symbols("price", end_date)] # do the download in chunks of size 8 to prevent overloading servers for symbols in partition(all_symbols, 8): data = yahoo.get_historical_data(symbols, start_date, end_date) for symbol, csv_string in data.items(): if csv_string: try: records = csv_string_to_records(symbol, csv_string, strict=True) update_fundamentals("price", symbol, records, end_date, delete_old=True) except csv.Error as e: logger.exception(e) logger.error("csv of {0} is malformed.".format(symbol)) else: logger.warning("Could not find updated historical prices of {0}. Skip.".format(symbol)) logger.info("Finished updating historical prices.")
initialize.init() logger = logging.getLogger(__name__) for report_type in wsj.FINANCIAL_REPORT_TYPES: logger.info("Will update {0} data of all symbols not up to date on {1}.".format(report_type, end_date)) data_type = report_type.replace('-', '_') symbol_update_dates = {s.symbol: s.updated_at for s in get_outdated_symbols(data_type, end_date, "stock")} for symbol in symbol_update_dates: try: data = wsj.get_financial_data(symbol, "annual", report_type) except MinerException as e: logger.warning("Could not find updated {0} data of {1} due to {2}. Skip.".format(report_type, symbol, e.__class__.__name__)) continue else: records = list(csv_rows_to_records(symbol, data)) if symbol_update_dates[symbol]: logger.info("Filter out {0} records of {1} that already exist".format(report_type, symbol)) existing_record_dates = set(r.date for r in get_record_dates(data_type, symbol)) records = [r for r in records if r["date"] not in existing_record_dates] if records: update_fundamentals(data_type, symbol, records, end_date) else: logger.info("No new {0} data for {1}.".format(report_type, symbol)) logger.info("Finished updating {0} data.".format(report_type))