def stock_scrape_task(*args, **kwargs): currency = args[0].upper() segment = " ".join(args[1:]) service = kwargs.get('service') session = Session() scraped = 0 # TODO: not so pretty but what to do when there's no universal ticker naming scheme prefix_wrapper = {'SEK': 'STO'} prefix = prefix_wrapper.get(currency, None) try: result = session.query(NasdaqCompany.ticker)\ .filter(NasdaqCompany.segment == segment)\ .filter(NasdaqCompany.currency == currency) for r in result: # delete the record first if already existing try: session.query(StockDomain).filter( StockDomain.ticker == r.ticker).delete() session.commit() except Exception as e: LOGGER.exception("failed to delete stock ticker '{}'".format( r.ticker)) session.rollback() # fetch the updated quote from interweb try: if prefix is not None: ticker = "{}:{}".format(prefix, r.ticker) else: ticker = r.ticker quote = service.get_quote(ticker) stock = StockDomain() stock.from_google_finance_quote(quote) session.add(stock) session.commit() except Exception as e: LOGGER.exception( "failed to fetch and store stock ticker '{}'".format( r.ticker)) else: scraped += 1 # arbitrary sleep, avoid getting us blocked, rate-limited etc time.sleep(3) except Exception as e: LOGGER.exception("failed to scrape stocks") return "Failed to scrape stocks" else: return "Done scraping segment '{segment}' currency '{currency}' - scraped {scraped} companies".format( segment=segment, currency=currency, scraped=scraped) finally: session.close()
def add_quote_hint(*args, **kwargs): provider = args[0] dst_ticker = args[1] src_text = " ".join(args[2:]) session = Session() try: hint = ProviderHints(provider=provider, src=src_text, dst=dst_ticker) session.add(hint) session.commit() return "Added hint" except Exception as e: LOGGER.exception("failed to add hint") finally: session.close()