def test_get_outdated_symbols(self):
symbols = [
{"symbol": "C6L.SI", "price_updated_at": None},
{"symbol": "J7X.SI", "price_updated_at": datetime(2012, 12, 21)},
{"symbol": "ABC.SI", "price_updated_at": datetime(2013, 1, 1)},
]
with db.transaction():
Symbol.insert_many(symbols).execute()
result = list(query.get_outdated_symbols("price", datetime(2013, 1, 1)).dicts())
self.assertEqual(result, [
{"symbol": "C6L.SI", "updated_at": None},
{"symbol": "J7X.SI", "updated_at": datetime(2012, 12, 21)},
])
def test_get_outdated_symbols_category(self):
symbols = [
{"symbol": "C6L.SI", "category": "market_index"},
{"symbol": "J7X.SI", "category": "market_index"},
{"symbol": "ABC.SI", "category": "stock"},
]
with db.transaction():
Symbol.insert_many(symbols).execute()
result = list(query.get_outdated_symbols("price", datetime(2013, 1, 1), "market_index").dicts())
self.assertEqual(result, [
{"symbol": "C6L.SI", "updated_at": None},
{"symbol": "J7X.SI", "updated_at": None},
])
from fa.miner import yahoo
from fa.piping import csv_string_to_records
from fa.util import partition
from fa.database.query import get_outdated_symbols, update_fundamentals
import initialize
from settings import *
""" Download data from internet to database """
initialize.init()
logger = logging.getLogger(__name__)
logger.info("Will update historical prices of all symbols not up to date on {0}.".format(end_date))
all_symbols = [s.symbol for s in get_outdated_symbols("price", end_date)]
# do the download in chunks of size 8 to prevent overloading servers
for symbols in partition(all_symbols, 8):
data = yahoo.get_historical_data(symbols, start_date, end_date)
for symbol, csv_string in data.items():
if csv_string:
try:
records = csv_string_to_records(symbol, csv_string, strict=True)
update_fundamentals("price", symbol, records, end_date, delete_old=True)
except csv.Error as e:
logger.exception(e)
logger.error("csv of {0} is malformed.".format(symbol))
else:
logger.warning("Could not find updated historical prices of {0}. Skip.".format(symbol))
from fa.piping import csv_rows_to_records
import initialize
from settings import *
""" Download data from internet to database """
initialize.init()
logger = logging.getLogger(__name__)
for report_type in wsj.FINANCIAL_REPORT_TYPES:
logger.info("Will update {0} data of all symbols not up to date on {1}.".format(report_type, end_date))
data_type = report_type.replace('-', '_')
symbol_update_dates = {s.symbol: s.updated_at for s in get_outdated_symbols(data_type, end_date, "stock")}
for symbol in symbol_update_dates:
try:
data = wsj.get_financial_data(symbol, "annual", report_type)
except MinerException as e:
logger.warning("Could not find updated {0} data of {1} due to {2}. Skip.".format(report_type, symbol, e.__class__.__name__))
continue
else:
records = list(csv_rows_to_records(symbol, data))
if symbol_update_dates[symbol]:
logger.info("Filter out {0} records of {1} that already exist".format(report_type, symbol))
existing_record_dates = set(r.date for r in get_record_dates(data_type, symbol))
records = [r for r in records if r["date"] not in existing_record_dates]