def test_get_financial_data(self): with patch("fa.miner.wsj._get_report_url", MagicMock(return_value="http://foo")) as mock_get_report_url, \ patch("fa.miner.wsj.strict_get", MagicMock(return_value="html")) as mock_strict_get, \ patch("fa.miner.wsj._scrape_html", MagicMock(return_value="raw_data")) as mock_scrape_html, \ patch("fa.miner.wsj.preprocess", MagicMock(return_value="items")) as mock_preprocess, \ patch("fa.miner.wsj.transpose_items", MagicMock(return_value="csv rows")) as mock_transpose_items: result = wsj.get_financial_data("C6L.SI", "annual", "balance-sheet") mock_get_report_url.assert_called_once_with("C6L.SI", "annual", "balance-sheet") mock_strict_get.assert_called_once_with("http://foo", wsj._test_for_not_found) mock_scrape_html.assert_called_once_with("html") mock_preprocess.assert_called_once_with("raw_data", "C6L.SI", "annual", "balance-sheet") mock_transpose_items.assert_called_once_with("items") self.assertEqual(result, "csv rows")
""" Download data from internet to database """ initialize.init() logger = logging.getLogger(__name__) for report_type in wsj.FINANCIAL_REPORT_TYPES: logger.info("Will update {0} data of all symbols not up to date on {1}.".format(report_type, end_date)) data_type = report_type.replace('-', '_') symbol_update_dates = {s.symbol: s.updated_at for s in get_outdated_symbols(data_type, end_date, "stock")} for symbol in symbol_update_dates: try: data = wsj.get_financial_data(symbol, "annual", report_type) except MinerException as e: logger.warning("Could not find updated {0} data of {1} due to {2}. Skip.".format(report_type, symbol, e.__class__.__name__)) continue else: records = list(csv_rows_to_records(symbol, data)) if symbol_update_dates[symbol]: logger.info("Filter out {0} records of {1} that already exist".format(report_type, symbol)) existing_record_dates = set(r.date for r in get_record_dates(data_type, symbol)) records = [r for r in records if r["date"] not in existing_record_dates] if records: update_fundamentals(data_type, symbol, records, end_date) else: logger.info("No new {0} data for {1}.".format(report_type, symbol))