def retrieveOverviewData(storage_dir, headings = None):
store = Storage(storage_dir)
xls = XLSio(store)
scraper = WSJscraper(store)
xls.loadWorkbook("ASXListedCompanies")
tickers = xls.getTickers()
new_data = {}
for ticker in tickers:
scraper.load_overview(ticker)
try:
new_data[ticker] = scraper.keyStockData()
except Exception:
print("Problem with: " + ticker)
xls.updateTable(new_data)
xls.saveAs("StockSummary")
class Test_LoadingPages(unittest.TestCase):
def setUp(self):
self.ticker = "SRV"
self.type = "overview"
self.data_dir = "..\\testData\\"
self.store = Storage(self.data_dir)
self.scraper = WSJscraper(self.store)
def test_StoresSoupObjectOnPageLoad(self):
self.scraper.load_overview(self.ticker)
expected = BeautifulSoup(open(self.store.html(self.ticker, "overview")), "lxml")
self.assertIsInstance(self.scraper.overview, BeautifulSoup)
self.assertEqual(self.scraper.overview.title, expected.title)
def test_ChecksForValidYears(self):
years = [u'2015', u' ', u'012', u'.3']
self.assertRaises(InsufficientDataError, self.scraper.check_years, years = years)
self.assertIsNone(self.scraper.check_years([u'2015', u'2014', u'2013']))
class Test_LoadingPages(unittest.TestCase):
def setUp(self):
self.ticker = "SRV"
self.type = "overview"
self.data_dir = "..\\testData\\"
self.store = Storage(self.data_dir)
self.scraper = WSJscraper(self.store)
def test_StoresSoupObjectOnPageLoad(self):
self.scraper.load_overview(self.ticker)
expected = BeautifulSoup(
open(self.store.html(self.ticker, "overview")), "lxml")
self.assertIsInstance(self.scraper.overview, BeautifulSoup)
self.assertEqual(self.scraper.overview.title, expected.title)
def test_ChecksForValidYears(self):
years = [u'2015', u' ', u'012', u'.3']
self.assertRaises(InsufficientDataError,
self.scraper.check_years,
years=years)
self.assertIsNone(self.scraper.check_years([u'2015', u'2014',
u'2013']))