def get_financial_data(symbol, timeframe, report_type):
""" Returns financial data in an iterator of csv rows (tuples). First row will be the headers.
symbol: e.g. "C6L.SI"
timeframe: "annual" or "quarter"
report_type: any string in FINANCIAL_REPORT_TYPES
"""
logger.info("getting {0} {1} data of {2}".format(timeframe, report_type, symbol))
url = _get_report_url(symbol, timeframe, report_type)
html = strict_get(url, _test_for_not_found)
try:
raw_data = _scrape_html(html)
except (IndexError, AttributeError, KeyError) as e:
msg = "failed to scrape raw data out from the html content of {0} {1} {2}".format(symbol, timeframe, report_type)
logger.exception(e)
logger.error(msg)
raise ScrapingError(msg) from e
try: # exception can only be triggered in transpose_items because preprocess returns a generator
items = preprocess(raw_data, symbol, timeframe, report_type)
return transpose_items(items)
except (AssertionError, ValueError) as e:
msg = "failed to preprocess the raw data of {0} {1} {2}".format(symbol, timeframe, report_type)
logger.exception(e)
logger.error(msg)
raise PreprocessingError(msg) from e
def test_strict_get(self):
class MockResponse(object):
def __init__(self, url):
""" HTTP response from accessing the URL """
self.status_code = 200
self.text = url[7:]
with patch("fa.miner.http.requests.get", MagicMock(side_effect=MockResponse)) as mock_get:
data = http.strict_get("http://foo")
mock_get.assert_called_once_with("http://foo")
self.assertEqual(data, "foo")
def test_strict_get_test_for_error(self):
test_for_error = lambda r: "404" in r.url
class MockResponse(object):
def __init__(self, url):
""" HTTP response from accessing the URL """
self.status_code = 200
self.text = "bar"
self.reason = ''
self.url = url
with patch("fa.miner.http.requests.get", MagicMock(side_effect=MockResponse)):
data = http.strict_get("http://foo", test_for_error)
self.assertRaises(GetError, http.strict_get, "http://foo/404.html", test_for_error)
self.assertEqual(data, "bar")
def get_historical_data(symbols, start_date, end_date):
""" Returns {"symbol": "historical data in csv string"}:
(blank string if there is an error.)
>>> get_historical_data(("C6L.SI", "ZZZZZZ",), datetime(2004, 3, 1), datetime(2014, 3, 1))
{"C6L.SI": "...", "ZZZZZZ": ''}
"""
logger.info("getting historical data of {0} from {1} to {2}".format(','.join(symbols), start_date, end_date))
abcdef = _get_abcdef(start_date, end_date)
result = {}
for s in symbols:
url = HISTORICAL_DATA_API_URL_TEMPLATE.format(symbol=s, **abcdef)
try:
result[s] = strict_get(url)
except GetError:
result[s] = ''
return result
