class StockScraper(object): """Used to scrape stock data from a variety of sources on the web. This class should be used by the client code instead of the specific scrapers. """ def __init__(self, config): ms_username = config.get('Credentials', 'morningstar_username') ms_password = config.get('Credentials', 'morningstar_password') # we only want stocks with market cap > 50,000 # TODO: add this value to the config file! self._google_scraper = GoogleScraper(50000000) self._asx_scraper = AsxScraper() self._ms_scraper = MorningStarScraper(ms_username, ms_password) self._ms_scraper.login() def scrape_stock_profiles(self): """Scrapes and returns a list of stock profiles from various sources on the web. """ stock_profiles = [] # using Google stock data as our base, populate all the stock profiles for stock in self._google_scraper.scrape_stock_profiles(): try: stock.sector = self._asx_scraper.sector(stock.symbol) except KeyError: # The Google list sometimes has delisted companies, so if a # sector can't be found, it's probably been delisted. continue # TODO: add these values into the config file? if 'Utilities' in stock.sector \ or 'Financ' in stock.sector \ or 'Banks' in stock.sector \ or 'Real Estate' in stock.sector: continue # Merge in stock data from MorningStar # Note that we keep Google's 'market cap', as it's more up-to-date try: ms_stock = self._ms_scraper.scrape_stock_profile(stock.symbol) except FieldMissingException as e: print(ms_stock.symbol, str(e)) # if an attribute can't be found, we can't really do anything # other than just continue. some companies don't have 'return # on capital' available. continue stock.return_on_capital = ms_stock.return_on_capital stock.ebit = ms_stock.ebit stock.total_debt = ms_stock.total_debt stock.cash = ms_stock.cash stock_profiles.append(stock) return stock_profiles
class StockScraper(object): """Used to scrape stock data from a variety of sources on the web. This class should be used by the client code instead of the specific scrapers. """ def __init__(self, config): ms_username = config.get('Credentials', 'morningstar_username') ms_password = config.get('Credentials', 'morningstar_password') # we only want stocks with market cap > 50,000 # TODO: add this value to the config file! self._google_scraper = GoogleScraper(50000000) self._asx_scraper = AsxScraper() self._ms_scraper = MorningStarScraper(ms_username, ms_password) self._ms_scraper.login() def scrape_stock_profiles(self): """Scrapes and returns a list of stock profiles from various sources on the web. """ stock_profiles = [] # using Google stock data as our base, populate all the stock profiles for stock in self._google_scraper.scrape_stock_profiles(): try: stock.sector = self._asx_scraper.sector(stock.symbol) except KeyError: # The Google list sometimes has delisted companies, so if a # sector can't be found, it's probably been delisted. continue # TODO: add these values into the config file? if 'Utilities' in stock.sector \ or 'Financ' in stock.sector \ or 'Banks' in stock.sector \ or 'Real Estate' in stock.sector: continue # Merge in stock data from MorningStar # Note that we keep Google's 'market cap', as it's more up-to-date try: ms_stock = self._ms_scraper.scrape_stock_profile(stock.symbol) except FieldMissingException as e: print(ms_stock.symbol, str(e)) # if an attribute can't be found, we can't really do anything # other than just continue. some companies don't have 'return # on capital' available. continue stock.return_on_capital = ms_stock.return_on_capital stock.ebit = ms_stock.ebit stock.total_debt = ms_stock.total_debt stock.cash = ms_stock.cash stock_profiles.append(stock) return stock_profiles
def test_scrape(self): scraper = MorningStarScraper('username', 'password') stock = scraper.scrape_stock_profile('PXS') self.assertEqual(stock.symbol, 'PXS') self.assertEqual(stock.title, 'Pharmaxis Ltd') self.assertEqual(stock.return_on_capital, 0.36) self.assertEqual(stock.ebit, 19010000) self.assertEqual(stock.market_cap, 69000000) self.assertEqual(stock.total_debt, 10893000) self.assertEqual(stock.cash, 54138000) self.assertEqual(stock.earnings_yield, 0.7381091050281499)
def test_scrape(self): scraper = MorningStarScraper('username', 'password') stock = scraper.scrape_stock_profile('PXS') self.assertEqual(stock.symbol, 'PXS') self.assertEqual(stock.title, 'Pharmaxis Ltd') self.assertEqual(stock.return_on_capital, 0.36) self.assertEqual(stock.ebit, 19010000) self.assertEqual(stock.market_cap, 69000000) self.assertEqual(stock.total_debt, 10893000) self.assertEqual(stock.cash, 54138000) self.assertEqual(stock.earnings_yield, 0.7381091050281499)