def test_get_financial_data(self):
with patch("fa.miner.wsj._get_report_url", MagicMock(return_value="http://foo")) as mock_get_report_url, \
patch("fa.miner.wsj.strict_get", MagicMock(return_value="html")) as mock_strict_get, \
patch("fa.miner.wsj._scrape_html", MagicMock(return_value="raw_data")) as mock_scrape_html, \
patch("fa.miner.wsj.preprocess", MagicMock(return_value="items")) as mock_preprocess, \
patch("fa.miner.wsj.transpose_items", MagicMock(return_value="csv rows")) as mock_transpose_items:
result = wsj.get_financial_data("C6L.SI", "annual", "balance-sheet")
mock_get_report_url.assert_called_once_with("C6L.SI", "annual", "balance-sheet")
mock_strict_get.assert_called_once_with("http://foo", wsj._test_for_not_found)
mock_scrape_html.assert_called_once_with("html")
mock_preprocess.assert_called_once_with("raw_data", "C6L.SI", "annual", "balance-sheet")
mock_transpose_items.assert_called_once_with("items")
self.assertEqual(result, "csv rows")
""" Download data from internet to database """
initialize.init()
logger = logging.getLogger(__name__)
for report_type in wsj.FINANCIAL_REPORT_TYPES:
logger.info("Will update {0} data of all symbols not up to date on {1}.".format(report_type, end_date))
data_type = report_type.replace('-', '_')
symbol_update_dates = {s.symbol: s.updated_at for s in get_outdated_symbols(data_type, end_date, "stock")}
for symbol in symbol_update_dates:
try:
data = wsj.get_financial_data(symbol, "annual", report_type)
except MinerException as e:
logger.warning("Could not find updated {0} data of {1} due to {2}. Skip.".format(report_type, symbol, e.__class__.__name__))
continue
else:
records = list(csv_rows_to_records(symbol, data))
if symbol_update_dates[symbol]:
logger.info("Filter out {0} records of {1} that already exist".format(report_type, symbol))
existing_record_dates = set(r.date for r in get_record_dates(data_type, symbol))
records = [r for r in records if r["date"] not in existing_record_dates]
if records:
update_fundamentals(data_type, symbol, records, end_date)
else:
logger.info("No new {0} data for {1}.".format(report_type, symbol))
