def list_quote_hint(*args, **kwargs):
provider = args[0]
session = Session()
try:
result = []
for hint in session.query(ProviderHints).filter(ProviderHints.provider == provider).all():
result.append("Provider: {}, Ticker: {}, Free-text: {}".format(hint.provider, hint.dst, hint.src))
if len(result) > 0:
return result
else:
return "no hints found"
except Exception as e:
LOGGER.exception("failed to list hints")
return "broken"
finally:
session.close()
def ticker_hint(provider, ticker):
session = None
try:
session = Session()
hint_ticker = session.query(ProviderHints).filter(
and_(ProviderHints.provider == provider, ProviderHints.src == ticker)) \
.one_or_none()
if hint_ticker:
return hint_ticker.dst
else:
return ticker
except Exception as e:
LOGGER.exception("failed to query hints", e)
return ticker
finally:
if session:
session.close()
def remove_quote_hint(*args, **kwargs):
provider = args[0]
dst_ticker = args[1]
session = Session()
try:
hint = session.query(ProviderHints).filter(and_(ProviderHints.provider == provider, ProviderHints.dst == dst_ticker)).one_or_none()
if hint:
session.delete(hint)
session.commit()
return "Removed hint"
else:
return "No matching hint to remove"
except Exception as e:
LOGGER.exception("failed to remove hint")
finally:
session.close()
def stock_analytics_top(*args, **kwargs):
rv = []
if len(args) < 2:
return ["Error: need moar args"]
try:
count = int(args[0])
except ValueError:
rv.append("Error: {} is not a number sherlock".format(args[0]))
count = 5
try:
sort_field_name = args[1]
filter_by = getattr(StockDomain, sort_field_name)
except AttributeError:
return ["Error: '{}' is not a valid field".format(args[1])]
sort_descending = len(args) == 3 and args[2] == "desc"
session = Session()
if sort_descending:
tmp = getattr(StockDomain, sort_field_name)
order_by = getattr(tmp, "desc")()
else:
order_by = filter_by
try:
result = session.query(StockDomain)\
.filter(filter_by != 0.0)\
.order_by(order_by).limit(count)
rv.extend(["Top {}: Ticker: {}, Name: {}, Value: {}".format(i + 1, x.ticker, x.name,
getattr(x, sort_field_name))
for i, x in enumerate(result)])
except Exception as e:
LOGGER.exception("failed to query stockdomain for top '{}'".format(sort_field_name))
finally:
session.close()
if len(rv) == 0:
rv.append("Nothing found")
return rv
def add_quote_hint(*args, **kwargs):
provider = args[0]
dst_ticker = args[1]
src_text = " ".join(args[2:])
session = Session()
try:
hint = ProviderHints(provider=provider, src=src_text, dst=dst_ticker)
session.add(hint)
session.commit()
return "Added hint"
except Exception as e:
LOGGER.exception("failed to add hint")
finally:
session.close()
def get_articles(matches=(), sender=None):
req = requests.get("https://www.avanza.se/placera/telegram.plc.html")
req.raise_for_status()
tree = html.fromstring(req.content)
items = tree.xpath(
'//ul[@class="feedArticleList XSText"]/li[@class="item"]/a')
for item in items:
url_path = item.attrib["href"]
absolute_url = "https://avanza.se{}".format(url_path)
headline_date = item.find('span').text_content().lstrip().rstrip()
headline = item.find('div').text_content()
headline_split = headline.split(":")
if len(headline_split) == 1:
headline_topic = "unknown"
headline_message = headline_split[0].lstrip().rstrip()
else:
headline_topic = headline_split[0].lower().lstrip().rstrip()
headline_message = headline_split[1].lstrip().rstrip()
if len(matches) == 0 or headline_topic in matches:
session = Session()
article = NewsArticle(headline_topic, absolute_url,
headline_message, headline_date)
try:
if not NewsArticleSeen.is_seen(session, article, sender):
yield article
NewsArticleSeen.mark_as_seen(session, article, sender)
session.commit()
except Exception as e:
LOGGER.exception(
"Something went wrong when fetching news articles")
finally:
session.close()
def stock_scrape_task(*args, **kwargs):
currency = args[0].upper()
segment = " ".join(args[1:])
service = kwargs.get('service')
session = Session()
scraped = 0
# TODO: not so pretty but what to do when there's no universal ticker naming scheme
prefix_wrapper = {'SEK': 'STO'}
prefix = prefix_wrapper.get(currency, None)
try:
result = session.query(NasdaqCompany.ticker)\
.filter(NasdaqCompany.segment == segment)\
.filter(NasdaqCompany.currency == currency)
for r in result:
# delete the record first if already existing
try:
session.query(StockDomain).filter(
StockDomain.ticker == r.ticker).delete()
session.commit()
except Exception as e:
LOGGER.exception("failed to delete stock ticker '{}'".format(
r.ticker))
session.rollback()
# fetch the updated quote from interweb
try:
if prefix is not None:
ticker = "{}:{}".format(prefix, r.ticker)
else:
ticker = r.ticker
quote = service.get_quote(ticker)
stock = StockDomain()
stock.from_google_finance_quote(quote)
session.add(stock)
session.commit()
except Exception as e:
LOGGER.exception(
"failed to fetch and store stock ticker '{}'".format(
r.ticker))
else:
scraped += 1
# arbitrary sleep, avoid getting us blocked, rate-limited etc
time.sleep(3)
except Exception as e:
LOGGER.exception("failed to scrape stocks")
return "Failed to scrape stocks"
else:
return "Done scraping segment '{segment}' currency '{currency}' - scraped {scraped} companies".format(
segment=segment, currency=currency, scraped=scraped)
finally:
session.close()
def scrape_stats(*args, **kwargs):
session = Session()
result = session.query(NasdaqCompany.segment, func.count(NasdaqCompany.segment)).group_by(NasdaqCompany.segment)\
.all()
return "Scraped: {}".format(", ".join(
["{k}={v}".format(k=x[0], v=x[1]) for x in result]))
def nasdaq_scraper_task(*args, **kwargs):
session = Session()
nasdaq_scraper = NasdaqIndexScraper()
try:
session.query(NasdaqCompany).delete()
session.commit()
except Exception as e:
LOGGER.exception("Failed to delete all records")
session.rollback()
else:
try:
rv = []
for index in nasdaq_scraper.indexes.keys():
rv.extend(nasdaq_scraper.scrape(index))
session.add_all(rv)
session.commit()
return "Scraped {} companies from Nasdaq".format(len(rv))
except Exception as e:
LOGGER.exception("Failed to fetch and store nasdaq companies")
session.rollback()
finally:
session.close()
def setUp(self):
self.ircbot = FakeIrcBot()
self.service = FakeQuoteService()
self.session = Session()
create_tables()
class TestCommand(unittest.TestCase):
def setUp(self):
self.ircbot = FakeIrcBot()
self.service = FakeQuoteService()
self.session = Session()
create_tables()
def tearDown(self):
drop_tables()
self.session.close()
def __cmd_wrap(self, *args):
""" test helper """
factory = QuoteServiceFactory()
factory.providers = {"fakeprovider": FakeQuoteService}
return root_command.execute(*args,
command_args={
"service_factory": factory,
"instance": self.ircbot
})
def test_quote_get_command(self):
command = ["quote", "get", "fakeprovider", "aapl"]
res = self.__cmd_wrap(*command)
self.assertEquals("Here's your fake quote for aapl", res)
def test_quote_get_fresh_command(self):
class FakeQuoteIsNotFresh(object):
def is_fresh(self):
return False
class FakeQuoteIsFresh(object):
def is_fresh(self):
return True
def __str__(self):
return "I'm fresh"
class FakeQuoteServiceLocal(object):
def get_quote(self, ticker):
if ticker == "not_fresh":
return FakeQuoteIsNotFresh()
else:
return FakeQuoteIsFresh()
factory = QuoteServiceFactory()
factory.providers = {"fakeprovider": FakeQuoteServiceLocal}
command = ["quote", "get_fresh", "fakeprovider", "not_fresh"]
res = root_command.execute(*command,
command_args={
"service_factory": factory,
"instance": self.ircbot
})
self.assertIsNone(res)
command = ["quote", "get_fresh", "fakeprovider", "fresh"]
res = root_command.execute(*command,
command_args={
"service_factory": factory,
"instance": self.ircbot
})
self.assertEquals("I'm fresh", str(res))
def test_lucky_quote_and_quick_get_command(self):
class FakeQuote(object):
def __init__(self, data={}):
self.data = data
def is_empty(self):
return len(self.data.items()) == 0
def __str__(self):
return "Ticker: {}".format(self.data.get("ticker"))
class FakeQuoteServiceLocal(object):
def get_quote(self, ticker):
if ticker == "fancyticker":
return FakeQuote()
else:
return FakeQuote(data={"ticker": "AWESOMO"})
def search(self, query):
return GoogleFinanceSearchResult(
result={
"matches": [{
"t": "AWESOMO",
"e": "Foo Market",
"n": "Foo Company"
}]
})
factory = QuoteServiceFactory()
factory.providers = {"fakeprovider": FakeQuoteServiceLocal}
command = ["q", "gl", "fakeprovider", "fancyticker"]
res = root_command.execute(*command,
command_args={
"service_factory": factory,
"instance": self.ircbot
})
self.assertEquals("Ticker: AWESOMO", str(res))
factory.providers = {"avanza": FakeQuoteServiceLocal}
command = ["qq", "fancyticker"]
res = root_command.execute(*command,
command_args={
"service_factory": factory,
"instance": self.ircbot
})
self.assertEquals("Ticker: AWESOMO", str(res))
def test_quote_get_command_invalid_input(self):
command = ["quote", "get", "invalid-provider", "aapl"]
res = self.__cmd_wrap(*command)
self.assertEquals("No such provider 'invalid-provider'", res)
def test_quote_search_command(self):
command = ["quote", "search", "fakeprovider", "foobar"]
res = self.__cmd_wrap(*command)
self.assertIn("Ticker: FOO, Market: Foo Market, Name: Foo Company",
res)
def test_quote_search_command_invalid_input(self):
command = ["quote", "search", "invalid-provider", "foobar"]
res = self.__cmd_wrap(*command)
self.assertIn("No such provider 'invalid-provider", res)
def test_quote_search_command_none_type_response(self):
command = ["quote", "search", "fakeprovider", "none-type-response"]
res = self.__cmd_wrap(*command)
self.assertIn("Response from provider 'fakeprovider' broken", res)
def test_execute_help_command(self):
command = ["help"]
res = self.__cmd_wrap(*command)
self.assertIn("quote (q) get <provider> <ticker>", res)
self.assertIn("quote (q) search <provider> <ticker>", res)
def test_execute_scheduler_ticker_commands(self):
# blank state
command = ["scheduler", "command", "get"]
res = self.__cmd_wrap(*command)
self.assertEquals("No commands added", res)
# add command
command = [
"scheduler", "command", "add", "quote", "get", "google", "foobar"
]
res = self.__cmd_wrap(*command)
self.assertEquals("Added command: quote get google foobar", res)
# add command again and fail gracefully
command = [
"scheduler", "command", "add", "quote", "get", "google", "foobar"
]
res = self.__cmd_wrap(*command)
self.assertEquals("Command already in list", res)
# verify command is there
command = ["scheduler", "command", "get"]
res = self.__cmd_wrap(*command)
self.assertIn("Command: quote get google foobar", res)
# remove command
command = [
"scheduler", "command", "remove", "quote", "get", "google",
"foobar"
]
res = self.__cmd_wrap(*command)
self.assertEquals("Removed command: quote get google foobar", res)
# verify command is not there
command = ["scheduler", "command", "get"]
res = self.__cmd_wrap(*command)
self.assertEquals("No commands added", res)
# remove it again and fail gracefully
command = [
"scheduler", "command", "remove", "quote", "get", "google",
"foobar"
]
res = self.__cmd_wrap(*command)
self.assertEquals("Command not in list", res)
def test_execute_scheduler_interval_command(self):
# default state
command = ["scheduler", "interval", "get"]
res = self.__cmd_wrap(*command)
self.assertEquals("Interval: 3600 seconds", res)
# update interval
command = ["scheduler", "interval", "set", "60"]
res = self.__cmd_wrap(*command)
self.assertEquals("New interval: 60 seconds", res)
# get updated state
command = ["scheduler", "interval", "get"]
res = self.__cmd_wrap(*command)
self.assertEquals("Interval: 60 seconds", res)
# set garbage input
command = ["scheduler", "interval", "set", "horseshit"]
res = self.__cmd_wrap(*command)
self.assertEquals(
"Can't set interval from garbage input, must be of an int", res)
def test_execute_scheduler_toggle_command(self):
command = ["scheduler", "enable"]
res = self.__cmd_wrap(*command)
self.assertEquals("Scheduler: enabled", res)
self.assertTrue(self.ircbot.scheduler)
command = ["scheduler", "disable"]
res = self.__cmd_wrap(*command)
self.assertEquals("Scheduler: disabled", res)
self.assertFalse(self.ircbot.scheduler)
def test_execute_unknown_command(self):
command = ["hi", "stockbot"]
res = self.__cmd_wrap(*command)
self.assertEquals(None, res)
@vcr.use_cassette('mock/vcr_cassettes/nasdaq/scraper.yaml')
def test_execute_scrape_nasdaq(self):
command = ["scrape", "nasdaq"]
res = self.__cmd_wrap(*command)
self.assertEquals("Scraped 657 companies from Nasdaq", res)
command = ["scrape", "stats"]
res = self.__cmd_wrap(*command)
self.assertEquals(
"Scraped: nordic large cap=201, nordic mid cap=219, nordic small cap=237",
res)
@patch('time.sleep')
@vcr.use_cassette('mock/vcr_cassettes/google/quote/scrape_large_cap.yaml')
def test_execute_nonblocking_scrape_stocks(self, sleep_mock):
# Mock sleep in the scrape task
sleep_mock.return_value = False
companies = [
NasdaqCompany(name="AAK",
ticker="AAK",
currency="SEK",
category="bla",
segment="nordic large cap"),
NasdaqCompany(name="ABB Ltd",
ticker="ABB",
currency="SEK",
category="bla",
segment="nordic large cap")
]
self.session.add_all(companies)
self.session.commit()
command = ["scrape", "stocks", "sek", "nordic", "large", "cap"]
root_command.execute(
*command,
command_args={'service': GoogleFinanceQueryService()},
callback=self.ircbot.callback)
self.assertEquals("Task started", self.ircbot.callback_args[0])
for t in threading.enumerate():
if t.name == "thread-sek_nordic_large_cap":
t.join()
self.assertEquals(
"Done scraping segment 'nordic large cap' currency 'SEK' - scraped 2 companies",
self.ircbot.callback_args[0])
for c in companies:
row = self.session.query(StockDomain).filter(
StockDomain.ticker == c.ticker).first()
self.assertNotEquals(None, row)
def test_execute_analytics_fields(self):
command = ["fundamental", "fields"]
res = self.__cmd_wrap(*command)
self.assertEquals(
"Fields: id, name, ticker, net_profit_margin_last_q, net_profit_margin_last_y, operating_margin_last_q, operating_margin_last_y, ebitd_margin_last_q, ebitd_margin_last_y, roaa_last_q, roaa_last_y, roae_last_q, roae_last_y, market_cap, price_to_earnings, beta, earnings_per_share, dividend_yield, latest_dividend",
res)
def test_execute_analytics_top(self):
# TODO: fix test data
command = ["fundamental", "top", "5", "net_profit_margin_last_q"]
res = self.__cmd_wrap(*command)
self.assertEquals(["Nothing found"], res)
command = ["fundamental", "top", "foobar", "net_profit_margin_last_q"]
res = self.__cmd_wrap(*command)
self.assertEquals(["Error: foobar is not a number sherlock"], res)
command = ["fundamental", "top", "5", "this_field_doesnt_exist"]
res = self.__cmd_wrap(*command)
self.assertEquals(
["Error: 'this_field_doesnt_exist' is not a valid field"], res)
def test_quote_hints(self):
command = ["quote", "hint", "list", "yahoo"]
res = self.__cmd_wrap(*command)
self.assertEqual("no hints found", res)
command = ["quote", "hint", "add", "yahoo", "foo", "bar"]
res = self.__cmd_wrap(*command)
self.assertEqual("Added hint", res)
command = ["quote", "hint", "list", "yahoo"]
res = self.__cmd_wrap(*command)
self.assertEqual(["Provider: yahoo, Ticker: foo, Free-text: bar"], res)
command = ["quote", "hint", "remove", "yahoo", "foo"]
res = self.__cmd_wrap(*command)
self.assertEqual("Removed hint", res)
command = ["quote", "hint", "remove", "yahoo", "foo"]
res = self.__cmd_wrap(*command)
self.assertEqual("No matching hint to remove", res)
command = ["quote", "hint", "list", "yahoo"]
res = self.__cmd_wrap(*command)
self.assertEqual("no hints found", res)