def test_valid_and_non_valid_dates(self, mock_urlopen): test_data = test_utils.load_test_data() scraper = Scraper('yahoo') mock_urlopen = mock_urlopen.return_value.__enter__.return_value mock_urlopen.status = 200 mock_urlopen.read.return_value = load_test_data() dt_1 = datetime.datetime(2019, 8, 26) dt_2 = datetime.datetime(2019, 8, 23) dt_3 = datetime.datetime(2019, 9, 2) dt_4 = datetime.datetime(2019, 8, 27) dt_5 = datetime.datetime(2019, 9, 4) dates = (dt_1, dt_2, dt_3, dt_4, dt_5) data, errors = scraper.scrape_eq_multiple_dates(self.ticker, dates) self.assertEqual(len(data), 3) self.assertEqual(len(errors), 2) # data checks for d in data: self.assertIsInstance(d[1], EquityData) for i, date in enumerate((dt_1, dt_2, dt_4)): self.assertEqual(data[i][0], date.date()) expected_data = test_utils.get_test_data(test_data, self.ticker, date) self.assertEqual(data[i][1], expected_data, msg=f'res: {data[i][1]} != ex: {expected_data}') # error checks for i, date in enumerate((dt_3, dt_5)): self.assertIsInstance(errors[i], InvalidDateError) self.assertEqual(str(date.date()), str(errors[i]))
def test_scraper_empty_date_list_input(self, mock_urlopen): scraper = Scraper('yahoo') mock_urlopen = mock_urlopen.return_value.__enter__.return_value mock_urlopen.status = 200 mock_urlopen.read.return_value = load_test_data() with self.assertRaises(EmptyDateListError): _, _ = scraper.scrape_eq_multiple_dates(self.ticker, [])
def test_scrape_invalid_ticker(self, mock_urlopen): ticker = 'AMZNN' dt = datetime.datetime(2019, 8, 23) scraper = Scraper('yahoo') mock_urlopen = mock_urlopen.return_value.__enter__.return_value mock_urlopen.read.return_value = b'' with self.assertRaises(InvalidTickerError): _, _ = scraper.scrape_eq_multiple_dates(ticker, [dt])
def test_scraper_single_date(self, mock_urlopen): ticker, dt, expected_data = test_utils.get_expected_equity_data() scraper = Scraper('yahoo') mock_urlopen = mock_urlopen.return_value.__enter__.return_value mock_urlopen.status = 200 mock_urlopen.read.return_value = load_test_data() data, errors = scraper.scrape_eq_multiple_dates(ticker, [dt]) self.assertEqual(len(data), 1) self.assertEqual(len(errors), 0) self.assertIsInstance(data[0][1], EquityData) self.assertEqual(data[0][0], dt.date()) self.assertEqual(data[0][1], expected_data, msg=f'res: {data[0][1]} != ex: {expected_data}')
def test_no_valid_dates(self, mock_urlopen): scraper = Scraper('yahoo') mock_urlopen = mock_urlopen.return_value.__enter__.return_value mock_urlopen.status = 200 mock_urlopen.read.return_value = load_test_data() dt_1 = datetime.datetime(2019, 9, 2) dt_2 = datetime.datetime(2019, 9, 4) dates = (dt_1, dt_2) data, errors = scraper.scrape_eq_multiple_dates(self.ticker, dates) self.assertEqual(len(data), 0) self.assertEqual(len(errors), 2) for i, error in enumerate(errors): self.assertIsInstance(error, InvalidDateError) self.assertEqual(str(dates[i].date()), str(error))
class MarketData: Database = namedtuple('Database', ['conn_string', 'source']) _init = False # NOTE(steve): this method will be used to initialise all # the dependencies before the user can use the application # this is where we will throw dependency errors as well # TODO(steve): the DataAdapter should be passed into the # MarketData class not a connection string to connect to # the database??? def run(self, database): """ Initialises MarketData class with scraper and data adapter. Args: database: namedtuple('Database', ['conn_string', 'source']). """ self._init = True self._scraper = Scraper('yahoo') da = data_adapter.get_adapter(database.source) self._database = da.connect(database.conn_string) # NOTE(steve): this method will be used to clean up # all the dependency e.g. closing of the database # after the app is closed def close(self): """Ensures dependent objects are properly cleaned up.""" self._init = False self._database.close() # TODO(steve): should turn this into a decorator def _check_initialised(self): """Checks if class is initialised.""" if not self._init: raise NotInitialisedError('Call run method first!') def add_security(self, ticker): """ Add security into market data. Args: ticker: Yahoo ticker for the security to be added. """ self._check_initialised() self._database.insert_securities([ticker]) def get_securities_list(self): """ Returns list of securities currently stored. Returns: List of security tickers. """ self._check_initialised() return self._database.get_securities_list() def get_equity_data(self, ticker, dt): """ Returns equity data object (open, high, low, close, adj_close, volume) for a selected ticker (Yahoo) and date. Args: ticker: Yahoo ticker. dt: date of equity data. Returns: Equity data object (open, high, low, close, adj_close, volume) for the selected ticker and date if available. Raises: InvalidTickerError: Security not in market data. InvalidDateError: No equity data for date provided. """ self._check_initialised() data = self._database.get_equity_data(ticker, dt) return data def get_equity_data_series(self, ticker): """ Return equity data for all available dates for the selected ticker. Args: ticker: Yahoo ticker. Returns: Equity data series sorted by date (newest to oldest) as a list of tuples (date, equity_data). Raises: InvalidTickerError: Security not in market data. """ self._check_initialised() data = self._database.get_equity_data_series(ticker) return data def get_latest_equity_data(self, ticker): """ Return the most recent equity data object for the selected ticker. Args: ticker: Yahoo ticker. Returns: Most recent equity data object by date (open, high, low, close, adj_close, volume) for the selected ticker. Raises: InvalidTickerError: Security not in market data. NoDataError: No data availabe for selected security. """ self._check_initialised() data = self.get_equity_data_series(ticker) if len(data) > 0: return data[0] else: raise NoDataError(ticker) def update_market_data(self, ticker, dt): """ Updates market data for the selected security and date. Args: ticker: Yahoo ticker. dt: date of equity data. Raise: InvalidTickerError: Security not in market data. """ self._check_initialised() if ticker in self._database.get_securities_list(): data = self._scraper.scrape_equity_data(ticker, dt) self._database.update_market_data(ticker, (dt, data)) else: raise InvalidTickerError(ticker) def bulk_update_market_data(self, ticker, date_list): """ Updates market data for a selected security over a specified number of dates in list. Args: ticker: Yahoo ticker. date_list: List of dates to update market data on. Returns: A list of errors or an empty list if no errors. Raise: InvalidTickerError: Security not in market data. """ self._check_initialised() if ticker in self._database.get_securities_list(): data, errors = self._scraper.scrape_eq_multiple_dates( ticker, date_list) self._database.bulk_update_market_data(ticker, data) return errors else: raise InvalidTickerError(ticker)