def stock_scrape_task(*args, **kwargs):
currency = args[0].upper()
segment = " ".join(args[1:])
service = kwargs.get('service')
session = Session()
scraped = 0
# TODO: not so pretty but what to do when there's no universal ticker naming scheme
prefix_wrapper = {'SEK': 'STO'}
prefix = prefix_wrapper.get(currency, None)
try:
result = session.query(NasdaqCompany.ticker)\
.filter(NasdaqCompany.segment == segment)\
.filter(NasdaqCompany.currency == currency)
for r in result:
# delete the record first if already existing
try:
session.query(StockDomain).filter(
StockDomain.ticker == r.ticker).delete()
session.commit()
except Exception as e:
LOGGER.exception("failed to delete stock ticker '{}'".format(
r.ticker))
session.rollback()
# fetch the updated quote from interweb
try:
if prefix is not None:
ticker = "{}:{}".format(prefix, r.ticker)
else:
ticker = r.ticker
quote = service.get_quote(ticker)
stock = StockDomain()
stock.from_google_finance_quote(quote)
session.add(stock)
session.commit()
except Exception as e:
LOGGER.exception(
"failed to fetch and store stock ticker '{}'".format(
r.ticker))
else:
scraped += 1
# arbitrary sleep, avoid getting us blocked, rate-limited etc
time.sleep(3)
except Exception as e:
LOGGER.exception("failed to scrape stocks")
return "Failed to scrape stocks"
else:
return "Done scraping segment '{segment}' currency '{currency}' - scraped {scraped} companies".format(
segment=segment, currency=currency, scraped=scraped)
finally:
session.close()
def add_quote_hint(*args, **kwargs):
provider = args[0]
dst_ticker = args[1]
src_text = " ".join(args[2:])
session = Session()
try:
hint = ProviderHints(provider=provider, src=src_text, dst=dst_ticker)
session.add(hint)
session.commit()
return "Added hint"
except Exception as e:
LOGGER.exception("failed to add hint")
finally:
session.close()