def test_get_historical_data_exception_handling(self):
def fake_get(url):
symbol = url[36:39]
if symbol == "ZZZ":
raise GetError()
else:
return symbol + ".csv"
with patch("fa.miner.yahoo.strict_get", MagicMock(side_effect=fake_get)):
result = yahoo.get_historical_data(("C6L.SI", "ZZZ"), datetime(2004, 3, 1), datetime(2014, 3, 1))
self.assertEqual(result, {"C6L.SI": "C6L.csv", "ZZZ": ''})
def test_get_historical_data(self):
def fake_get(url):
symbol = url[36:39]
return symbol + ".csv"
with patch("fa.miner.yahoo.strict_get", MagicMock(side_effect=fake_get)) as mock_strict_get:
result = yahoo.get_historical_data(("C6L.SI", "ZZZ"), datetime(2004, 3, 1), datetime(2014, 3, 1))
mock_strict_get.assert_has_calls([
call("http://ichart.yahoo.com/table.csv?s=C6L.SI&a=2&b=1&c=2004&d=2&e=1&f=2014&g=d&ignore=.csv"),
call("http://ichart.yahoo.com/table.csv?s=ZZZ&a=2&b=1&c=2004&d=2&e=1&f=2014&g=d&ignore=.csv"),
])
self.assertEqual(result, {"C6L.SI": "C6L.csv", "ZZZ": "ZZZ.csv"})
from fa.database.query import get_outdated_symbols, update_fundamentals
import initialize
from settings import *
""" Download data from internet to database """
initialize.init()
logger = logging.getLogger(__name__)
logger.info("Will update historical prices of all symbols not up to date on {0}.".format(end_date))
all_symbols = [s.symbol for s in get_outdated_symbols("price", end_date)]
# do the download in chunks of size 8 to prevent overloading servers
for symbols in partition(all_symbols, 8):
data = yahoo.get_historical_data(symbols, start_date, end_date)
for symbol, csv_string in data.items():
if csv_string:
try:
records = csv_string_to_records(symbol, csv_string, strict=True)
update_fundamentals("price", symbol, records, end_date, delete_old=True)
except csv.Error as e:
logger.exception(e)
logger.error("csv of {0} is malformed.".format(symbol))
else:
logger.warning("Could not find updated historical prices of {0}. Skip.".format(symbol))
logger.info("Finished updating historical prices.")
