def test_valid_and_non_valid_dates(self, mock_urlopen):
test_data = test_utils.load_test_data()
scraper = Scraper('yahoo')
mock_urlopen = mock_urlopen.return_value.__enter__.return_value
mock_urlopen.status = 200
mock_urlopen.read.return_value = load_test_data()
dt_1 = datetime.datetime(2019, 8, 26)
dt_2 = datetime.datetime(2019, 8, 23)
dt_3 = datetime.datetime(2019, 9, 2)
dt_4 = datetime.datetime(2019, 8, 27)
dt_5 = datetime.datetime(2019, 9, 4)
dates = (dt_1, dt_2, dt_3, dt_4, dt_5)
data, errors = scraper.scrape_eq_multiple_dates(self.ticker, dates)
self.assertEqual(len(data), 3)
self.assertEqual(len(errors), 2)
# data checks
for d in data:
self.assertIsInstance(d[1], EquityData)
for i, date in enumerate((dt_1, dt_2, dt_4)):
self.assertEqual(data[i][0], date.date())
expected_data = test_utils.get_test_data(test_data, self.ticker,
date)
self.assertEqual(data[i][1], expected_data,
msg=f'res: {data[i][1]} != ex: {expected_data}')
# error checks
for i, date in enumerate((dt_3, dt_5)):
self.assertIsInstance(errors[i], InvalidDateError)
self.assertEqual(str(date.date()), str(errors[i]))
def test_scraper_empty_date_list_input(self, mock_urlopen):
scraper = Scraper('yahoo')
mock_urlopen = mock_urlopen.return_value.__enter__.return_value
mock_urlopen.status = 200
mock_urlopen.read.return_value = load_test_data()
with self.assertRaises(EmptyDateListError):
_, _ = scraper.scrape_eq_multiple_dates(self.ticker, [])
def test_scrape_invalid_ticker(self, mock_urlopen):
ticker = 'AMZNN'
dt = datetime.datetime(2019, 8, 23)
scraper = Scraper('yahoo')
mock_urlopen_context = mock_urlopen.return_value.__enter__.return_value
mock_urlopen_context.read.return_value = b''
with self.assertRaises(InvalidTickerError):
results = scraper.scrape_equity_data(ticker, dt)
def test_scrape_invalid_ticker(self, mock_urlopen):
ticker = 'AMZNN'
dt = datetime.datetime(2019, 8, 23)
scraper = Scraper('yahoo')
mock_urlopen = mock_urlopen.return_value.__enter__.return_value
mock_urlopen.read.return_value = b''
with self.assertRaises(InvalidTickerError):
_, _ = scraper.scrape_eq_multiple_dates(ticker, [dt])
def test_scrape_invalid_date(self, mock_urlopen):
ticker = 'AMZN'
dt = datetime.datetime(2019, 9, 3)
scraper = Scraper('yahoo')
mock_urlopen_context = mock_urlopen.return_value.__enter__.return_value
mock_urlopen_context.status = 200
mock_urlopen_context.read.return_value = load_test_data()
with self.assertRaises(InvalidDateError):
results = scraper.scrape_equity_data(ticker, dt)
def run(self, database):
"""
Initialises MarketData class with scraper and data adapter.
Args:
database: namedtuple('Database', ['conn_string', 'source']).
"""
self._init = True
self._scraper = Scraper('yahoo')
da = data_adapter.get_adapter(database.source)
self._database = da.connect(database.conn_string)
def test_scrape_returns_correct_equity_data(self, mock_urlopen):
ticker, dt, expected_data = test_utils.get_expected_equity_data()
scraper = Scraper('yahoo')
mock_urlopen_context = mock_urlopen.return_value.__enter__.return_value
mock_urlopen_context.status = 200
mock_urlopen_context.read.return_value = load_test_data()
results = scraper.scrape_equity_data(ticker, dt)
self.assertIsInstance(results, EquityData)
self.assertEqual(results, expected_data,
msg=f'res: {results} != ex: {expected_data}')
def test_scrape_equity_data_with_date_and_time(self, mock_urlopen):
ticker, dt, expected_data = test_utils.get_expected_equity_data()
scraper = Scraper('yahoo')
dt = datetime.datetime(dt.year, dt.month, dt.day, 19, 35, 33)
mock_urlopen_context = mock_urlopen.return_value.__enter__.return_value
mock_urlopen_context.status = 200
mock_urlopen_context.read.return_value = load_test_data()
results = scraper.scrape_equity_data(ticker, dt)
self.assertIsInstance(results, EquityData)
self.assertEqual(results, expected_data,
msg=f'res: {results} != ex: {expected_data}')
def test_scraper_single_date(self, mock_urlopen):
ticker, dt, expected_data = test_utils.get_expected_equity_data()
scraper = Scraper('yahoo')
mock_urlopen = mock_urlopen.return_value.__enter__.return_value
mock_urlopen.status = 200
mock_urlopen.read.return_value = load_test_data()
data, errors = scraper.scrape_eq_multiple_dates(ticker, [dt])
self.assertEqual(len(data), 1)
self.assertEqual(len(errors), 0)
self.assertIsInstance(data[0][1], EquityData)
self.assertEqual(data[0][0], dt.date())
self.assertEqual(data[0][1], expected_data,
msg=f'res: {data[0][1]} != ex: {expected_data}')
def test_no_valid_dates(self, mock_urlopen):
scraper = Scraper('yahoo')
mock_urlopen = mock_urlopen.return_value.__enter__.return_value
mock_urlopen.status = 200
mock_urlopen.read.return_value = load_test_data()
dt_1 = datetime.datetime(2019, 9, 2)
dt_2 = datetime.datetime(2019, 9, 4)
dates = (dt_1, dt_2)
data, errors = scraper.scrape_eq_multiple_dates(self.ticker, dates)
self.assertEqual(len(data), 0)
self.assertEqual(len(errors), 2)
for i, error in enumerate(errors):
self.assertIsInstance(error, InvalidDateError)
self.assertEqual(str(dates[i].date()), str(error))
def test_invalid_source_scrape_error(self):
with self.assertRaises(InvalidSourceError):
scraper = Scraper('google')
def test_scrape_from_valid_source(self):
scraper = Scraper('yahoo')
self.assertIsInstance(scraper, Scraper)
self.assertEqual(scraper.source, 'yahoo')
class MarketData:
Database = namedtuple('Database', ['conn_string', 'source'])
_init = False
# NOTE(steve): this method will be used to initialise all
# the dependencies before the user can use the application
# this is where we will throw dependency errors as well
# TODO(steve): the DataAdapter should be passed into the
# MarketData class not a connection string to connect to
# the database???
def run(self, database):
"""
Initialises MarketData class with scraper and data adapter.
Args:
database: namedtuple('Database', ['conn_string', 'source']).
"""
self._init = True
self._scraper = Scraper('yahoo')
da = data_adapter.get_adapter(database.source)
self._database = da.connect(database.conn_string)
# NOTE(steve): this method will be used to clean up
# all the dependency e.g. closing of the database
# after the app is closed
def close(self):
"""Ensures dependent objects are properly cleaned up."""
self._init = False
self._database.close()
# TODO(steve): should turn this into a decorator
def _check_initialised(self):
"""Checks if class is initialised."""
if not self._init:
raise NotInitialisedError('Call run method first!')
def add_security(self, ticker):
"""
Add security into market data.
Args:
ticker: Yahoo ticker for the security to be added.
"""
self._check_initialised()
self._database.insert_securities([ticker])
def get_securities_list(self):
"""
Returns list of securities currently stored.
Returns:
List of security tickers.
"""
self._check_initialised()
return self._database.get_securities_list()
def get_equity_data(self, ticker, dt):
"""
Returns equity data object (open, high, low, close, adj_close, volume)
for a selected ticker (Yahoo) and date.
Args:
ticker: Yahoo ticker.
dt: date of equity data.
Returns:
Equity data object (open, high, low, close, adj_close, volume)
for the selected ticker and date if available.
Raises:
InvalidTickerError: Security not in market data.
InvalidDateError: No equity data for date provided.
"""
self._check_initialised()
data = self._database.get_equity_data(ticker, dt)
return data
def get_equity_data_series(self, ticker):
"""
Return equity data for all available dates for the selected ticker.
Args:
ticker: Yahoo ticker.
Returns:
Equity data series sorted by date (newest to oldest) as a list of
tuples (date, equity_data).
Raises:
InvalidTickerError: Security not in market data.
"""
self._check_initialised()
data = self._database.get_equity_data_series(ticker)
return data
def get_latest_equity_data(self, ticker):
"""
Return the most recent equity data object for the selected ticker.
Args:
ticker: Yahoo ticker.
Returns:
Most recent equity data object by date (open, high, low, close,
adj_close, volume) for the selected ticker.
Raises:
InvalidTickerError: Security not in market data.
NoDataError: No data availabe for selected security.
"""
self._check_initialised()
data = self.get_equity_data_series(ticker)
if len(data) > 0:
return data[0]
else:
raise NoDataError(ticker)
def update_market_data(self, ticker, dt):
"""
Updates market data for the selected security and date.
Args:
ticker: Yahoo ticker.
dt: date of equity data.
Raise:
InvalidTickerError: Security not in market data.
"""
self._check_initialised()
if ticker in self._database.get_securities_list():
data = self._scraper.scrape_equity_data(ticker, dt)
self._database.update_market_data(ticker, (dt, data))
else:
raise InvalidTickerError(ticker)
def bulk_update_market_data(self, ticker, date_list):
"""
Updates market data for a selected security over a
specified number of dates in list.
Args:
ticker: Yahoo ticker.
date_list: List of dates to update market data on.
Returns:
A list of errors or an empty list if no errors.
Raise:
InvalidTickerError: Security not in market data.
"""
self._check_initialised()
if ticker in self._database.get_securities_list():
data, errors = self._scraper.scrape_eq_multiple_dates(
ticker, date_list)
self._database.bulk_update_market_data(ticker, data)
return errors
else:
raise InvalidTickerError(ticker)