def list_quote_hint(*args, **kwargs): provider = args[0] session = Session() try: result = [] for hint in session.query(ProviderHints).filter(ProviderHints.provider == provider).all(): result.append("Provider: {}, Ticker: {}, Free-text: {}".format(hint.provider, hint.dst, hint.src)) if len(result) > 0: return result else: return "no hints found" except Exception as e: LOGGER.exception("failed to list hints") return "broken" finally: session.close()
def ticker_hint(provider, ticker): session = None try: session = Session() hint_ticker = session.query(ProviderHints).filter( and_(ProviderHints.provider == provider, ProviderHints.src == ticker)) \ .one_or_none() if hint_ticker: return hint_ticker.dst else: return ticker except Exception as e: LOGGER.exception("failed to query hints", e) return ticker finally: if session: session.close()
def remove_quote_hint(*args, **kwargs): provider = args[0] dst_ticker = args[1] session = Session() try: hint = session.query(ProviderHints).filter(and_(ProviderHints.provider == provider, ProviderHints.dst == dst_ticker)).one_or_none() if hint: session.delete(hint) session.commit() return "Removed hint" else: return "No matching hint to remove" except Exception as e: LOGGER.exception("failed to remove hint") finally: session.close()
def stock_analytics_top(*args, **kwargs): rv = [] if len(args) < 2: return ["Error: need moar args"] try: count = int(args[0]) except ValueError: rv.append("Error: {} is not a number sherlock".format(args[0])) count = 5 try: sort_field_name = args[1] filter_by = getattr(StockDomain, sort_field_name) except AttributeError: return ["Error: '{}' is not a valid field".format(args[1])] sort_descending = len(args) == 3 and args[2] == "desc" session = Session() if sort_descending: tmp = getattr(StockDomain, sort_field_name) order_by = getattr(tmp, "desc")() else: order_by = filter_by try: result = session.query(StockDomain)\ .filter(filter_by != 0.0)\ .order_by(order_by).limit(count) rv.extend(["Top {}: Ticker: {}, Name: {}, Value: {}".format(i + 1, x.ticker, x.name, getattr(x, sort_field_name)) for i, x in enumerate(result)]) except Exception as e: LOGGER.exception("failed to query stockdomain for top '{}'".format(sort_field_name)) finally: session.close() if len(rv) == 0: rv.append("Nothing found") return rv
def add_quote_hint(*args, **kwargs): provider = args[0] dst_ticker = args[1] src_text = " ".join(args[2:]) session = Session() try: hint = ProviderHints(provider=provider, src=src_text, dst=dst_ticker) session.add(hint) session.commit() return "Added hint" except Exception as e: LOGGER.exception("failed to add hint") finally: session.close()
def get_articles(matches=(), sender=None): req = requests.get("https://www.avanza.se/placera/telegram.plc.html") req.raise_for_status() tree = html.fromstring(req.content) items = tree.xpath( '//ul[@class="feedArticleList XSText"]/li[@class="item"]/a') for item in items: url_path = item.attrib["href"] absolute_url = "https://avanza.se{}".format(url_path) headline_date = item.find('span').text_content().lstrip().rstrip() headline = item.find('div').text_content() headline_split = headline.split(":") if len(headline_split) == 1: headline_topic = "unknown" headline_message = headline_split[0].lstrip().rstrip() else: headline_topic = headline_split[0].lower().lstrip().rstrip() headline_message = headline_split[1].lstrip().rstrip() if len(matches) == 0 or headline_topic in matches: session = Session() article = NewsArticle(headline_topic, absolute_url, headline_message, headline_date) try: if not NewsArticleSeen.is_seen(session, article, sender): yield article NewsArticleSeen.mark_as_seen(session, article, sender) session.commit() except Exception as e: LOGGER.exception( "Something went wrong when fetching news articles") finally: session.close()
def stock_scrape_task(*args, **kwargs): currency = args[0].upper() segment = " ".join(args[1:]) service = kwargs.get('service') session = Session() scraped = 0 # TODO: not so pretty but what to do when there's no universal ticker naming scheme prefix_wrapper = {'SEK': 'STO'} prefix = prefix_wrapper.get(currency, None) try: result = session.query(NasdaqCompany.ticker)\ .filter(NasdaqCompany.segment == segment)\ .filter(NasdaqCompany.currency == currency) for r in result: # delete the record first if already existing try: session.query(StockDomain).filter( StockDomain.ticker == r.ticker).delete() session.commit() except Exception as e: LOGGER.exception("failed to delete stock ticker '{}'".format( r.ticker)) session.rollback() # fetch the updated quote from interweb try: if prefix is not None: ticker = "{}:{}".format(prefix, r.ticker) else: ticker = r.ticker quote = service.get_quote(ticker) stock = StockDomain() stock.from_google_finance_quote(quote) session.add(stock) session.commit() except Exception as e: LOGGER.exception( "failed to fetch and store stock ticker '{}'".format( r.ticker)) else: scraped += 1 # arbitrary sleep, avoid getting us blocked, rate-limited etc time.sleep(3) except Exception as e: LOGGER.exception("failed to scrape stocks") return "Failed to scrape stocks" else: return "Done scraping segment '{segment}' currency '{currency}' - scraped {scraped} companies".format( segment=segment, currency=currency, scraped=scraped) finally: session.close()
def scrape_stats(*args, **kwargs): session = Session() result = session.query(NasdaqCompany.segment, func.count(NasdaqCompany.segment)).group_by(NasdaqCompany.segment)\ .all() return "Scraped: {}".format(", ".join( ["{k}={v}".format(k=x[0], v=x[1]) for x in result]))
def nasdaq_scraper_task(*args, **kwargs): session = Session() nasdaq_scraper = NasdaqIndexScraper() try: session.query(NasdaqCompany).delete() session.commit() except Exception as e: LOGGER.exception("Failed to delete all records") session.rollback() else: try: rv = [] for index in nasdaq_scraper.indexes.keys(): rv.extend(nasdaq_scraper.scrape(index)) session.add_all(rv) session.commit() return "Scraped {} companies from Nasdaq".format(len(rv)) except Exception as e: LOGGER.exception("Failed to fetch and store nasdaq companies") session.rollback() finally: session.close()
def setUp(self): self.ircbot = FakeIrcBot() self.service = FakeQuoteService() self.session = Session() create_tables()
class TestCommand(unittest.TestCase): def setUp(self): self.ircbot = FakeIrcBot() self.service = FakeQuoteService() self.session = Session() create_tables() def tearDown(self): drop_tables() self.session.close() def __cmd_wrap(self, *args): """ test helper """ factory = QuoteServiceFactory() factory.providers = {"fakeprovider": FakeQuoteService} return root_command.execute(*args, command_args={ "service_factory": factory, "instance": self.ircbot }) def test_quote_get_command(self): command = ["quote", "get", "fakeprovider", "aapl"] res = self.__cmd_wrap(*command) self.assertEquals("Here's your fake quote for aapl", res) def test_quote_get_fresh_command(self): class FakeQuoteIsNotFresh(object): def is_fresh(self): return False class FakeQuoteIsFresh(object): def is_fresh(self): return True def __str__(self): return "I'm fresh" class FakeQuoteServiceLocal(object): def get_quote(self, ticker): if ticker == "not_fresh": return FakeQuoteIsNotFresh() else: return FakeQuoteIsFresh() factory = QuoteServiceFactory() factory.providers = {"fakeprovider": FakeQuoteServiceLocal} command = ["quote", "get_fresh", "fakeprovider", "not_fresh"] res = root_command.execute(*command, command_args={ "service_factory": factory, "instance": self.ircbot }) self.assertIsNone(res) command = ["quote", "get_fresh", "fakeprovider", "fresh"] res = root_command.execute(*command, command_args={ "service_factory": factory, "instance": self.ircbot }) self.assertEquals("I'm fresh", str(res)) def test_lucky_quote_and_quick_get_command(self): class FakeQuote(object): def __init__(self, data={}): self.data = data def is_empty(self): return len(self.data.items()) == 0 def __str__(self): return "Ticker: {}".format(self.data.get("ticker")) class FakeQuoteServiceLocal(object): def get_quote(self, ticker): if ticker == "fancyticker": return FakeQuote() else: return FakeQuote(data={"ticker": "AWESOMO"}) def search(self, query): return GoogleFinanceSearchResult( result={ "matches": [{ "t": "AWESOMO", "e": "Foo Market", "n": "Foo Company" }] }) factory = QuoteServiceFactory() factory.providers = {"fakeprovider": FakeQuoteServiceLocal} command = ["q", "gl", "fakeprovider", "fancyticker"] res = root_command.execute(*command, command_args={ "service_factory": factory, "instance": self.ircbot }) self.assertEquals("Ticker: AWESOMO", str(res)) factory.providers = {"avanza": FakeQuoteServiceLocal} command = ["qq", "fancyticker"] res = root_command.execute(*command, command_args={ "service_factory": factory, "instance": self.ircbot }) self.assertEquals("Ticker: AWESOMO", str(res)) def test_quote_get_command_invalid_input(self): command = ["quote", "get", "invalid-provider", "aapl"] res = self.__cmd_wrap(*command) self.assertEquals("No such provider 'invalid-provider'", res) def test_quote_search_command(self): command = ["quote", "search", "fakeprovider", "foobar"] res = self.__cmd_wrap(*command) self.assertIn("Ticker: FOO, Market: Foo Market, Name: Foo Company", res) def test_quote_search_command_invalid_input(self): command = ["quote", "search", "invalid-provider", "foobar"] res = self.__cmd_wrap(*command) self.assertIn("No such provider 'invalid-provider", res) def test_quote_search_command_none_type_response(self): command = ["quote", "search", "fakeprovider", "none-type-response"] res = self.__cmd_wrap(*command) self.assertIn("Response from provider 'fakeprovider' broken", res) def test_execute_help_command(self): command = ["help"] res = self.__cmd_wrap(*command) self.assertIn("quote (q) get <provider> <ticker>", res) self.assertIn("quote (q) search <provider> <ticker>", res) def test_execute_scheduler_ticker_commands(self): # blank state command = ["scheduler", "command", "get"] res = self.__cmd_wrap(*command) self.assertEquals("No commands added", res) # add command command = [ "scheduler", "command", "add", "quote", "get", "google", "foobar" ] res = self.__cmd_wrap(*command) self.assertEquals("Added command: quote get google foobar", res) # add command again and fail gracefully command = [ "scheduler", "command", "add", "quote", "get", "google", "foobar" ] res = self.__cmd_wrap(*command) self.assertEquals("Command already in list", res) # verify command is there command = ["scheduler", "command", "get"] res = self.__cmd_wrap(*command) self.assertIn("Command: quote get google foobar", res) # remove command command = [ "scheduler", "command", "remove", "quote", "get", "google", "foobar" ] res = self.__cmd_wrap(*command) self.assertEquals("Removed command: quote get google foobar", res) # verify command is not there command = ["scheduler", "command", "get"] res = self.__cmd_wrap(*command) self.assertEquals("No commands added", res) # remove it again and fail gracefully command = [ "scheduler", "command", "remove", "quote", "get", "google", "foobar" ] res = self.__cmd_wrap(*command) self.assertEquals("Command not in list", res) def test_execute_scheduler_interval_command(self): # default state command = ["scheduler", "interval", "get"] res = self.__cmd_wrap(*command) self.assertEquals("Interval: 3600 seconds", res) # update interval command = ["scheduler", "interval", "set", "60"] res = self.__cmd_wrap(*command) self.assertEquals("New interval: 60 seconds", res) # get updated state command = ["scheduler", "interval", "get"] res = self.__cmd_wrap(*command) self.assertEquals("Interval: 60 seconds", res) # set garbage input command = ["scheduler", "interval", "set", "horseshit"] res = self.__cmd_wrap(*command) self.assertEquals( "Can't set interval from garbage input, must be of an int", res) def test_execute_scheduler_toggle_command(self): command = ["scheduler", "enable"] res = self.__cmd_wrap(*command) self.assertEquals("Scheduler: enabled", res) self.assertTrue(self.ircbot.scheduler) command = ["scheduler", "disable"] res = self.__cmd_wrap(*command) self.assertEquals("Scheduler: disabled", res) self.assertFalse(self.ircbot.scheduler) def test_execute_unknown_command(self): command = ["hi", "stockbot"] res = self.__cmd_wrap(*command) self.assertEquals(None, res) @vcr.use_cassette('mock/vcr_cassettes/nasdaq/scraper.yaml') def test_execute_scrape_nasdaq(self): command = ["scrape", "nasdaq"] res = self.__cmd_wrap(*command) self.assertEquals("Scraped 657 companies from Nasdaq", res) command = ["scrape", "stats"] res = self.__cmd_wrap(*command) self.assertEquals( "Scraped: nordic large cap=201, nordic mid cap=219, nordic small cap=237", res) @patch('time.sleep') @vcr.use_cassette('mock/vcr_cassettes/google/quote/scrape_large_cap.yaml') def test_execute_nonblocking_scrape_stocks(self, sleep_mock): # Mock sleep in the scrape task sleep_mock.return_value = False companies = [ NasdaqCompany(name="AAK", ticker="AAK", currency="SEK", category="bla", segment="nordic large cap"), NasdaqCompany(name="ABB Ltd", ticker="ABB", currency="SEK", category="bla", segment="nordic large cap") ] self.session.add_all(companies) self.session.commit() command = ["scrape", "stocks", "sek", "nordic", "large", "cap"] root_command.execute( *command, command_args={'service': GoogleFinanceQueryService()}, callback=self.ircbot.callback) self.assertEquals("Task started", self.ircbot.callback_args[0]) for t in threading.enumerate(): if t.name == "thread-sek_nordic_large_cap": t.join() self.assertEquals( "Done scraping segment 'nordic large cap' currency 'SEK' - scraped 2 companies", self.ircbot.callback_args[0]) for c in companies: row = self.session.query(StockDomain).filter( StockDomain.ticker == c.ticker).first() self.assertNotEquals(None, row) def test_execute_analytics_fields(self): command = ["fundamental", "fields"] res = self.__cmd_wrap(*command) self.assertEquals( "Fields: id, name, ticker, net_profit_margin_last_q, net_profit_margin_last_y, operating_margin_last_q, operating_margin_last_y, ebitd_margin_last_q, ebitd_margin_last_y, roaa_last_q, roaa_last_y, roae_last_q, roae_last_y, market_cap, price_to_earnings, beta, earnings_per_share, dividend_yield, latest_dividend", res) def test_execute_analytics_top(self): # TODO: fix test data command = ["fundamental", "top", "5", "net_profit_margin_last_q"] res = self.__cmd_wrap(*command) self.assertEquals(["Nothing found"], res) command = ["fundamental", "top", "foobar", "net_profit_margin_last_q"] res = self.__cmd_wrap(*command) self.assertEquals(["Error: foobar is not a number sherlock"], res) command = ["fundamental", "top", "5", "this_field_doesnt_exist"] res = self.__cmd_wrap(*command) self.assertEquals( ["Error: 'this_field_doesnt_exist' is not a valid field"], res) def test_quote_hints(self): command = ["quote", "hint", "list", "yahoo"] res = self.__cmd_wrap(*command) self.assertEqual("no hints found", res) command = ["quote", "hint", "add", "yahoo", "foo", "bar"] res = self.__cmd_wrap(*command) self.assertEqual("Added hint", res) command = ["quote", "hint", "list", "yahoo"] res = self.__cmd_wrap(*command) self.assertEqual(["Provider: yahoo, Ticker: foo, Free-text: bar"], res) command = ["quote", "hint", "remove", "yahoo", "foo"] res = self.__cmd_wrap(*command) self.assertEqual("Removed hint", res) command = ["quote", "hint", "remove", "yahoo", "foo"] res = self.__cmd_wrap(*command) self.assertEqual("No matching hint to remove", res) command = ["quote", "hint", "list", "yahoo"] res = self.__cmd_wrap(*command) self.assertEqual("no hints found", res)