def process_queue(self, worker_id):
"""Continuously processes tasks on the queue."""
# Create a new logs instance (with its own httplib2 instance) so that
# there is a separate one for each thread.
logs = Logs("twitter-listener-worker-%s" % worker_id,
to_cloud=self.logs_to_cloud)
logs.debug("Started worker thread: %s" % worker_id)
while not self.stop_event.is_set():
try:
data = self.queue.get(block=True, timeout=QUEUE_TIMEOUT_S)
start_time = time()
self.handle_data(logs, data)
self.queue.task_done()
end_time = time()
qsize = self.queue.qsize()
logs.debug("Worker %s took %.f ms with %d tasks remaining." %
(worker_id, end_time - start_time, qsize))
except Empty:
logs.debug("Worker %s timed out on an empty queue." %
worker_id)
continue
except Exception:
# The main loop doesn't catch and report exceptions from
# background threads, so do that here.
logs.catch()
logs.debug("Stopped worker thread: %s" % worker_id)
class Trading:
"""A helper for making stock trades."""
def __init__(self, logs_to_cloud):
self.logs = Logs(name="trading", to_cloud=logs_to_cloud)
self.alpaca = AlpacaConnector(logs_to_cloud)
self.trail_percent = float(getenv('TRAIL_PERCENT'))
self.limit_percent = float(
getenv('LIMIT_PERCENT')
) # The fraction of the stock price at which to set order limits.
self.cash_hold = float(
getenv('CASH_HOLD')
) # The amount of cash in dollars to hold from being spent.
self.max_position = float(
getenv('MAX_POSITION')) # Max position to take for each trade
def make_trades(self, companies):
"""Executes trades for the specified companies based on sentiment."""
# Determine whether the markets are open.
is_market_open = self.alpaca.get_market_status()
if not is_market_open:
self.logs.error("Not trading while market is closed")
return False
# Filter for any strategies resulting in trades.
actionable_strategies = []
for company in companies:
strategy = self.get_strategy(company, is_market_open)
if strategy["action"] != "hold":
actionable_strategies.append(strategy)
else:
self.logs.warn("Dropping strategy: %s" % strategy)
if not actionable_strategies:
self.logs.warn("No actionable strategies for trading.")
return False
# Calculate the budget per strategy.
balance = self.alpaca.get_balance()
budget = self.get_budget(balance, len(actionable_strategies))
if not budget:
self.logs.warn("No budget for trading: %s %s %s" %
(budget, balance, actionable_strategies))
return False
self.logs.debug("Using budget: %s x $%s" %
(len(actionable_strategies), budget))
# Handle trades for each strategy.
success = True
for strategy in actionable_strategies:
ticker = strategy["ticker"]
action = strategy["action"]
# Execute the strategy. ##BEEP
if action == "bull":
self.logs.info("Bull: %s %s" % (ticker, budget))
success = success and self.bull(ticker, budget)
elif action == "bear":
self.logs.info("Bear: %s %s" % (ticker, budget))
success = success and self.bear(ticker, budget)
else:
self.logs.error("Unknown strategy: %s" % strategy)
return success
def get_strategy(self, company, is_market_open):
"""Determines the strategy for trading a company based on sentiment and
market status.
"""
ticker = company["ticker"]
sentiment = company["sentiment"]
strategy = {}
strategy["name"] = company["name"]
if "root" in company:
strategy["root"] = company["root"]
strategy["sentiment"] = company["sentiment"]
strategy["ticker"] = ticker
if "exchange" in company:
strategy["exchange"] = company["exchange"]
# Don't do anything with blacklisted stocks.
if ticker in TICKER_BLACKLIST:
strategy["action"] = "hold"
strategy["reason"] = "blacklist"
return strategy
# TODO: Figure out some strategy for the markets closed case.
# Don't trade unless the markets are open or are about to open.
if not is_market_open:
strategy["action"] = "hold"
strategy["reason"] = "market closed"
return strategy
# Can't trade without sentiment.
if sentiment == 0:
strategy["action"] = "hold"
strategy["reason"] = "neutral sentiment"
return strategy
# Determine bull or bear based on sentiment direction.
if sentiment > 0:
strategy["action"] = "bull"
strategy["reason"] = "positive sentiment"
return strategy
else: # sentiment < 0
strategy[
"action"] = "hold" #typically "bear", but i dont want to short anything
strategy["reason"] = "negative sentiment"
return strategy
def get_budget(self, balance, num_strategies):
"""Calculates the budget per company based on the available balance."""
if num_strategies <= 0:
self.logs.warn("No budget without strategies.")
return 0.0
return round(
min(self.max_position,
max(0.0, balance - self.cash_hold) / num_strategies), 2)
def get_buy_limit(self, price):
"""Calculates the limit price for a buy (or cover) order."""
return round((1 + self.limit_percent / 100) * price, 2)
def get_sell_limit(self, price):
"""Calculates the limit price for a sell (or short) order."""
return round((1 - self.limit_percent / 100) * price, 2)
def get_quantity(self, ticker, budget):
"""Calculates the quantity of a stock based on the current market price
and a maximum budget.
"""
# Calculate the quantity based on the current price and the budget.
price = self.alpaca.get_last_price(ticker)
if price == -1:
self.logs.error("Failed to determine price for: %s" % ticker)
return (None, None)
# Use maximum possible quantity within the budget.
quantity = int(budget // price)
self.logs.debug("Determined quantity %s for %s at $%s within $%s." %
(quantity, ticker, price, budget))
return (quantity, price)
def bull(self, ticker, budget):
"""Executes the bullish strategy on the specified stock within the
specified budget: Buy now at market rate and sell at market rate at
close.
"""
# Calculate the quantity.
quantity, price = self.get_quantity(ticker, budget)
if not quantity:
self.logs.warn(f'Cannot trade quantity = {quantity}')
return False
# Buy the stock now.
self.logs.info(f'Trying to buy {quantity} units of {ticker}')
buy_limit = self.get_buy_limit(price)
buy_status, buy_order_id = self.alpaca.submit_market_buy(
ticker, quantity, buy_limit)
if buy_status not in self.alpaca.positive_statuses:
return False
# TODO: Do this properly by checking the order status API and using
# retries with exponential backoff.
#wait for stock order to be filled
timeout = time.time() + 60 * 10
check_interval = 15
while self.alpaca.get_order_status(
buy_order_id) != 'filled' and time.time() > timeout:
time.sleep(check_interval) #order not filled for 10 minutes
check_interval *= 1.1
if self.alpaca.get_order_status(buy_order_id) != 'filled':
self.logs.warn(
f'Not able to fill buy for {ticker}. Cancelling order')
self.alpaca.cancel_order(buy_order_id)
return False
self.logs.warn(
f'Trying to place trailing stop sell order for {ticker}')
#Create trailing_stop_order
sell_order_status, sell_order_id = self.alpaca.submit_trailing_stop(
ticker, quantity, self.trail_percent)
if sell_order_status == False:
self.logs.error(
f'Not able to place trailing_stop sell order for {ticker}. Order status: {sell_order_status} | Order ID: {sell_order_id}'
)
return True
def bear(self, ticker, budget):
return False
"""Executes the bearish strategy on the specified stock within the
specified budget: Sell short at market rate and buy to cover at market
rate at close.
"""
"""
class Trading:
"""A helper for making stock trades."""
def __init__(self, logs_to_cloud):
self.logs = Logs(name="trading", to_cloud=logs_to_cloud)
def make_trades(self, companies):
"""Executes trades for the specified companies based on sentiment."""
# Determine whether the markets are open.
market_status = self.get_market_status()
if not market_status:
self.logs.error("Not trading without market status.")
return False
# Filter for any strategies resulting in trades.
actionable_strategies = []
market_status = self.get_market_status()
for company in companies:
strategy = self.get_strategy(company, market_status)
if strategy["action"] != "hold":
actionable_strategies.append(strategy)
else:
self.logs.warn("Dropping strategy: %s" % strategy)
if not actionable_strategies:
self.logs.warn("No actionable strategies for trading.")
return False
# Calculate the budget per strategy.
balance = self.get_balance()
budget = self.get_budget(balance, len(actionable_strategies))
if not budget:
self.logs.warn("No budget for trading: %s %s %s" %
(budget, balance, actionable_strategies))
return False
self.logs.debug("Using budget: %s x $%s" %
(len(actionable_strategies), budget))
# Handle trades for each strategy.
success = True
for strategy in actionable_strategies:
ticker = strategy["ticker"]
action = strategy["action"]
# Execute the strategy.
if action == "bull":
self.logs.info("Bull: %s %s" % (ticker, budget))
success = success and self.bull(ticker, budget)
elif action == "bear":
self.logs.info("Bear: %s %s" % (ticker, budget))
success = success and self.bear(ticker, budget)
else:
self.logs.error("Unknown strategy: %s" % strategy)
return success
def get_strategy(self, company, market_status):
"""Determines the strategy for trading a company based on sentiment and
market status.
"""
ticker = company["ticker"]
sentiment = company["sentiment"]
strategy = {}
strategy["name"] = company["name"]
if "root" in company:
strategy["root"] = company["root"]
strategy["sentiment"] = company["sentiment"]
strategy["ticker"] = ticker
if "exchange" in company:
strategy["exchange"] = company["exchange"]
# Don't do anything with blacklisted stocks.
if ticker in TICKER_BLACKLIST:
strategy["action"] = "hold"
strategy["reason"] = "blacklist"
return strategy
# TODO: Figure out some strategy for the markets closed case.
# Don't trade unless the markets are open or are about to open.
if market_status != "open" and market_status != "pre":
strategy["action"] = "hold"
strategy["reason"] = "market closed"
return strategy
# Can't trade without sentiment.
if sentiment == 0:
strategy["action"] = "hold"
strategy["reason"] = "neutral sentiment"
return strategy
# Determine bull or bear based on sentiment direction.
if sentiment > 0:
strategy["action"] = "bull"
strategy["reason"] = "positive sentiment"
return strategy
else: # sentiment < 0
strategy[
"action"] = "hold" #typically "bear", but i dont want to short anything
strategy["reason"] = "negative sentiment"
return strategy
def get_budget(self, balance, num_strategies):
"""Calculates the budget per company based on the available balance."""
if num_strategies <= 0:
self.logs.warn("No budget without strategies.")
return 0.0
return round(
min(
MAX_POSITION,
max(0.0, balance - CASH_HOLD) / num_strategies /
NUM_TRADES_PER_DAY), 2)
def get_market_status(self):
"""Finds out whether the markets are open right now."""
clock_url = TRADEKING_API_URL % "market/clock"
response = self.make_request(url=clock_url)
if not response:
self.logs.error("No clock response.")
return None
try:
clock_response = response["response"]
current = clock_response["status"]["current"]
except KeyError:
self.logs.error("Malformed clock response: %s" % response)
return None
if current not in ["pre", "open", "after", "close"]:
self.logs.error("Unknown market status: %s" % current)
return None
self.logs.debug("Current market status: %s" % current)
return current
def get_historical_prices(self, ticker, timestamp):
"""Finds the last price at or before a timestamp and at EOD."""
# Start with today's quotes.
quotes = self.get_day_quotes(ticker, timestamp)
if not quotes:
self.logs.warn("No quotes for day: %s" % timestamp)
return None
# Depending on where we land relative to the trading day, pick the
# right quote and EOD quote.
first_quote = quotes[0]
first_quote_time = first_quote["time"]
last_quote = quotes[-1]
last_quote_time = last_quote["time"]
if timestamp < first_quote_time:
self.logs.debug("Using previous quote.")
previous_day = self.get_previous_day(timestamp)
previous_quotes = self.get_day_quotes(ticker, previous_day)
if not previous_quotes:
self.logs.error("No quotes for previous day: %s" %
previous_day)
return None
quote_at = previous_quotes[-1]
quote_eod = last_quote
elif timestamp >= first_quote_time and timestamp <= last_quote_time:
self.logs.debug("Using closest quote.")
# Walk through the quotes until we stepped over the timestamp.
previous_quote = first_quote
for quote in quotes:
quote_time = quote["time"]
if quote_time > timestamp:
break
previous_quote = quote
quote_at = previous_quote
quote_eod = last_quote
else: # timestamp > last_quote_time
self.logs.debug("Using last quote.")
quote_at = last_quote
next_day = self.get_next_day(timestamp)
next_quotes = self.get_day_quotes(ticker, next_day)
if not next_quotes:
self.logs.error("No quotes for next day: %s" % next_day)
return None
quote_eod = next_quotes[-1]
self.logs.debug("Using quotes: %s %s" % (quote_at, quote_eod))
return {"at": quote_at["price"], "eod": quote_eod["price"]}
def get_historical_prices_T_min(self, ticker, timestamp):
"""Finds the last price at or before a timestamp and at EOD."""
T = SELL_AFTER_MIN
# Start with today's quotes.
quotes = self.get_day_quotes(ticker, timestamp)
if not quotes:
self.logs.warn("No quotes for day: %s" % timestamp)
return None
# Depending on where we land relative to the trading day, pick the
# right quote and T min later quote (or end of day if end of day is less than T min away).
first_quote = quotes[0]
first_quote_time = first_quote["time"]
last_quote = quotes[-1]
last_quote_time = last_quote["time"]
if timestamp < first_quote_time:
self.logs.debug("Using previous quote.")
previous_day = self.get_previous_day(timestamp)
previous_quotes = self.get_day_quotes(ticker, previous_day)
if not previous_quotes:
self.logs.error("No quotes for previous day: %s" %
previous_day)
return None
quote_at = previous_quotes[-1]
quote_eod = quotes[T] #the quote T minutes into the day
elif timestamp >= first_quote_time and timestamp <= last_quote_time:
self.logs.debug("Using closest quote.")
# Walk through the quotes until we stepped over the timestamp.
previous_quote = first_quote
for i, quote in enumerate(quotes):
quote_time = quote["time"]
if quote_time > timestamp:
break
previous_quote = quote
quote_at = previous_quote
if i + T < len(quotes):
quote_eod = quotes[i + T]
else:
quote_eod = quotes[-1]
else: # timestamp > last_quote_time
self.logs.debug("Using last quote.")
quote_at = last_quote
next_day = self.get_next_day(timestamp)
next_quotes = self.get_day_quotes(ticker, next_day)
if not next_quotes:
self.logs.error("No quotes for next day: %s" % next_day)
return None
quote_eod = next_quotes[T]
self.logs.debug("Using quotes: %s %s" % (quote_at, quote_eod))
return {"at": quote_at["price"], "eod": quote_eod["price"]}
def get_day_quotes(self, ticker, timestamp):
"""Collects all quotes from the day of the market timestamp."""
polygon_client = PolygonClient(POLYGON_API_KEY)
quotes = []
time.sleep(
15
) ## TODO: Pay polygon for increased api usage and remove this timer
# The timestamp is expected in market time.
day_str = timestamp.strftime("%Y-%m-%d")
response = polygon_client.stocks_equities_aggregates(
ticker, 1, "minute", day_str, day_str)
if not response or response.status != "OK" or not response.results:
self.logs.error(
"Failed to request historical data for %s on %s: %s" %
(ticker, timestamp, response))
return None
for result in response.results:
try:
# Parse and convert the current minute's timestamp.
minute_timestamp = result["t"] / 1000
minute_market_time = self.utc_to_market_time(
datetime.fromtimestamp(minute_timestamp))
# Use the price at the beginning of the minute.
price = result["o"]
if not price or price < 0:
self.logs.warn("Invalid price: %s" % price)
continue
quote = {"time": minute_market_time, "price": price}
quotes.append(quote)
except (KeyError, TypeError, ValueError) as e:
self.logs.warn("Failed to parse result: %s" % e)
return quotes
def is_trading_day(self, timestamp):
"""Tests whether markets are open on a given day."""
# Markets are closed on holidays.
if timestamp in UnitedStates():
self.logs.debug("Identified holiday: %s" % timestamp)
return False
# Markets are closed on weekends.
if timestamp.weekday() in [5, 6]:
self.logs.debug("Identified weekend: %s" % timestamp)
return False
# Otherwise markets are open.
return True
def get_previous_day(self, timestamp):
"""Finds the previous trading day."""
previous_day = timestamp - timedelta(days=1)
# Walk backwards until we hit a trading day.
while not self.is_trading_day(previous_day):
previous_day -= timedelta(days=1)
self.logs.debug("Previous trading day for %s: %s" %
(timestamp, previous_day))
return previous_day
def get_next_day(self, timestamp):
"""Finds the next trading day."""
next_day = timestamp + timedelta(days=1)
# Walk forward until we hit a trading day.
while not self.is_trading_day(next_day):
next_day += timedelta(days=1)
self.logs.debug("Next trading day for %s: %s" % (timestamp, next_day))
return next_day
def utc_to_market_time(self, timestamp):
"""Converts a UTC timestamp to local market time."""
utc_time = utc.localize(timestamp)
market_time = utc_time.astimezone(MARKET_TIMEZONE)
return market_time
def market_time_to_utc(self, timestamp):
"""Converts a timestamp in local market time to UTC."""
market_time = MARKET_TIMEZONE.localize(timestamp)
utc_time = market_time.astimezone(utc)
return utc_time
def as_market_time(self, year, month, day, hour=0, minute=0, second=0):
"""Creates a timestamp in market time."""
market_time = datetime(year, month, day, hour, minute, second)
return MARKET_TIMEZONE.localize(market_time)
def make_request(self, url, method="GET", body="", headers=None):
"""Makes a request to the TradeKing API."""
consumer = Consumer(key=TRADEKING_CONSUMER_KEY,
secret=TRADEKING_CONSUMER_SECRET)
token = Token(key=TRADEKING_ACCESS_TOKEN,
secret=TRADEKING_ACCESS_TOKEN_SECRET)
client = Client(consumer, token)
body_bytes = body.encode("utf-8")
self.logs.debug("TradeKing request: %s %s %s %s" %
(url, method, body_bytes, headers))
response, content = client.request(url,
method=method,
body=body_bytes,
headers=headers)
self.logs.debug("TradeKing response: %s %s" % (response, content))
try:
return loads(content)
except ValueError:
self.logs.error("Failed to decode JSON response: %s" % content)
return None
def xml_tostring(self, xml):
"""Generates a string representation of the XML."""
return tostring(xml, encoding="utf-8").decode("utf-8")
def fixml_buy_now(self, ticker, quantity, limit):
self.logs.info(
f'Making a trade: Buying {quantity} units of {ticker} at limit price of {limit}'
)
"""Generates the FIXML for a buy order."""
fixml = Element("FIXML")
fixml.set("xmlns", FIXML_NAMESPACE)
order = SubElement(fixml, "Order")
order.set("TmInForce", "0") # Day order
order.set("Typ", "2") # Limit
order.set("Side", "1") # Buy
order.set("Px", "%.2f" % limit) # Limit price
order.set("Acct", TRADEKING_ACCOUNT_NUMBER)
instrmt = SubElement(order, "Instrmt")
instrmt.set("SecTyp", "CS") # Common stock
instrmt.set("Sym", ticker)
ord_qty = SubElement(order, "OrdQty")
ord_qty.set("Qty", str(quantity))
return self.xml_tostring(fixml)
def fixml_sell_eod(self, ticker, quantity, limit):
"""Generates the FIXML for a sell order."""
self.logs.info(
f'Placing EoD order: Selling {quantity} units of {ticker} at limit price of {limit}'
)
fixml = Element("FIXML")
fixml.set("xmlns", FIXML_NAMESPACE)
order = SubElement(fixml, "Order")
order.set("TmInForce", "7") # Market on close
order.set("Typ", "2") # Limit
order.set("Side", "2") # Sell
order.set("Px", "%.2f" % limit) # Limit price
order.set("Acct", TRADEKING_ACCOUNT_NUMBER)
instrmt = SubElement(order, "Instrmt")
instrmt.set("SecTyp", "CS") # Common stock
instrmt.set("Sym", ticker)
ord_qty = SubElement(order, "OrdQty")
ord_qty.set("Qty", str(quantity))
return self.xml_tostring(fixml)
def fixml_short_now(self, ticker, quantity, limit):
"""Generates the FIXML for a sell short order."""
fixml = Element("FIXML")
fixml.set("xmlns", FIXML_NAMESPACE)
order = SubElement(fixml, "Order")
order.set("TmInForce", "0") # Day order
order.set("Typ", "2") # Limit
order.set("Side", "5") # Sell short
order.set("Px", "%.2f" % limit) # Limit price
order.set("Acct", TRADEKING_ACCOUNT_NUMBER)
instrmt = SubElement(order, "Instrmt")
instrmt.set("SecTyp", "CS") # Common stock
instrmt.set("Sym", ticker)
ord_qty = SubElement(order, "OrdQty")
ord_qty.set("Qty", str(quantity))
return self.xml_tostring(fixml)
def fixml_cover_eod(self, ticker, quantity, limit):
"""Generates the FIXML for a sell to cover order."""
fixml = Element("FIXML")
fixml.set("xmlns", FIXML_NAMESPACE)
order = SubElement(fixml, "Order")
order.set("TmInForce", "7") # Market on close
order.set("Typ", "2") # Limit
order.set("Side", "1") # Buy
order.set("Px", "%.2f" % limit) # Limit price
order.set("AcctTyp", "5") # Cover
order.set("Acct", TRADEKING_ACCOUNT_NUMBER)
instrmt = SubElement(order, "Instrmt")
instrmt.set("SecTyp", "CS") # Common stock
instrmt.set("Sym", ticker)
ord_qty = SubElement(order, "OrdQty")
ord_qty.set("Qty", str(quantity))
return self.xml_tostring(fixml)
def get_buy_limit(self, price):
"""Calculates the limit price for a buy (or cover) order."""
return round((1 + LIMIT_FRACTION) * price, 2)
def get_sell_limit(self, price):
"""Calculates the limit price for a sell (or short) order."""
return round((1 - LIMIT_FRACTION) * price, 2)
def get_balance(self):
"""Finds the cash balance in dollars available to spend."""
balances_url = TRADEKING_API_URL % ("accounts/%s" %
TRADEKING_ACCOUNT_NUMBER)
response = self.make_request(url=balances_url)
if not response:
self.logs.error("No balances response.")
return 0
try:
balances = response["response"]
money = balances["accountbalance"]["money"]
cash_str = money["cashavailable"]
uncleareddeposits_str = money["uncleareddeposits"]
except KeyError:
self.logs.error("Malformed balances response: %s" % response)
return 0
try:
cash = float(cash_str)
uncleareddeposits = float(uncleareddeposits_str)
return cash - uncleareddeposits
except ValueError:
self.logs.error("Malformed number in response: %s" % money)
return 0
def get_last_price(self, ticker):
"""Finds the last trade price for the specified stock."""
quotes_url = TRADEKING_API_URL % "market/ext/quotes"
quotes_url += "?symbols=%s" % ticker
quotes_url += "&fids=last,date,symbol,exch_desc,name"
response = self.make_request(url=quotes_url)
if not response:
self.logs.error("No quotes response for %s: %s" %
(ticker, response))
return None
try:
quotes = response["response"]
quote = quotes["quotes"]["quote"]
last_str = quote["last"]
except KeyError:
self.logs.error("Malformed quotes response: %s" % response)
return None
self.logs.debug("Quote for %s: %s" % (ticker, quote))
try:
last = float(last_str)
except ValueError:
self.logs.error("Malformed last for %s: %s" % (ticker, last_str))
return None
if last > 0:
return last
else:
self.logs.error("Bad quote for: %s" % ticker)
return None
def get_order_url(self):
"""Gets the TradeKing URL for placing orders."""
url_path = "accounts/%s/orders" % TRADEKING_ACCOUNT_NUMBER
if not USE_REAL_MONEY:
url_path += "/preview"
return TRADEKING_API_URL % url_path
def get_quantity(self, ticker, budget):
"""Calculates the quantity of a stock based on the current market price
and a maximum budget.
"""
# Calculate the quantity based on the current price and the budget.
price = self.get_last_price(ticker)
if not price:
self.logs.error("Failed to determine price for: %s" % ticker)
return (None, None)
# Use maximum possible quantity within the budget.
quantity = int(budget // price)
self.logs.debug("Determined quantity %s for %s at $%s within $%s." %
(quantity, ticker, price, budget))
return (quantity, price)
def bull(self, ticker, budget):
"""Executes the bullish strategy on the specified stock within the
specified budget: Buy now at market rate and sell at market rate at
close.
"""
# Calculate the quantity.
quantity, price = self.get_quantity(ticker, budget)
if not quantity:
self.logs.warn("Not trading without quantity.")
return False
# Buy the stock now.
buy_limit = self.get_buy_limit(price)
buy_fixml = self.fixml_buy_now(ticker, quantity, buy_limit)
if not self.make_order_request(buy_fixml):
return False
# Sell the stock at close.
sell_limit = self.get_sell_limit(price)
sell_fixml = self.fixml_sell_eod(ticker, quantity, sell_limit)
# TODO: Do this properly by checking the order status API and using
# retries with exponential backoff.
# Wait until the previous order has been executed.
Timer(ORDER_DELAY_S, self.make_order_request, [sell_fixml]).start()
return True
def bear(self, ticker, budget):
"""Executes the bearish strategy on the specified stock within the
specified budget: Sell short at market rate and buy to cover at market
rate at close.
"""
# Calculate the quantity.
quantity, price = self.get_quantity(ticker, budget)
if not quantity:
self.logs.warn("Not trading without quantity.")
return False
# Short the stock now.
short_limit = self.get_sell_limit(price)
short_fixml = self.fixml_short_now(ticker, quantity, short_limit)
if not self.make_order_request(short_fixml):
return False
# Cover the short at close.
cover_limit = self.get_buy_limit(price)
cover_fixml = self.fixml_cover_eod(ticker, quantity, cover_limit)
# TODO: Do this properly by checking the order status API and using
# retries with exponential backoff.
# Wait until the previous order has been executed.
Timer(ORDER_DELAY_S, self.make_order_request, [cover_fixml]).start()
return True
def make_order_request(self, fixml):
"""Executes an order defined by FIXML and verifies the response."""
response = self.make_request(url=self.get_order_url(),
method="POST",
body=fixml,
headers=FIXML_HEADERS)
if not response:
self.logs.error("No order response for: %s" % fixml)
return False
try:
order_response = response["response"]
error = order_response["error"]
except KeyError:
self.logs.error("Malformed order response: %s" % response)
return False
# The error field indicates whether the order succeeded.
error = order_response["error"]
if error != "Success":
self.logs.error("Error in order response: %s %s" %
(error, order_response))
return False
return True
class Main:
"""A wrapper for the main application logic and retry loop."""
def __init__(self):
self.logs = Logs(name="main", to_cloud=LOGS_TO_CLOUD)
self.twitter = Twitter(logs_to_cloud=LOGS_TO_CLOUD)
self.logs.info("I'm running")
self.twitter.test_tweet()
def twitter_callback(self, tweet):
"""Analyzes tweets, trades stocks, and tweets about it."""
# Initialize the Analysis, Logs, Trading, and Twitter instances inside
# the callback to create separate httplib2 instances per thread.
analysis = Analysis(logs_to_cloud=LOGS_TO_CLOUD)
logs = Logs(name="main-callback", to_cloud=LOGS_TO_CLOUD)
# Analyze the tweet.
companies = analysis.find_companies(tweet)
logs.info("Using companies: %s" % companies)
if not companies:
return
# Trade stocks.
trading = Trading(logs_to_cloud=LOGS_TO_CLOUD)
trading.make_trades(companies)
# Tweet about it.
twitter = Twitter(logs_to_cloud=LOGS_TO_CLOUD)
twitter.tweet(companies, tweet)
def run_session(self):
"""Runs a single streaming session. Logs and cleans up after
exceptions.
"""
self.logs.info("Starting new session.")
try:
self.twitter.start_streaming(self.twitter_callback)
except:
self.logs.catch()
finally:
self.twitter.stop_streaming()
self.logs.info("Ending session.")
def backoff(self, tries):
"""Sleeps an exponential number of seconds based on the number of
tries.
"""
delay = BACKOFF_STEP_S * pow(2, tries)
self.logs.warn("Waiting for %.1f seconds." % delay)
sleep(delay)
def run(self):
"""Runs the main retry loop with exponential backoff."""
tries = 0
while True:
# The session blocks until an error occurs.
self.run_session()
# Remember the first time a backoff sequence starts.
now = datetime.now()
if tries == 0:
self.logs.debug("Starting first backoff sequence.")
backoff_start = now
# Reset the backoff sequence if the last error was long ago.
if (now - backoff_start).total_seconds() > BACKOFF_RESET_S:
self.logs.debug("Starting new backoff sequence.")
tries = 0
backoff_start = now
# Give up after the maximum number of tries.
if tries >= MAX_TRIES:
self.logs.warn("Exceeded maximum retry count.")
break
# Wait according to the progression of the backoff sequence.
self.backoff(tries)
# Increment the number of tries for the next error.
tries += 1
class AlpacaConnector():
def __init__(self, logs_to_cloud):
if getenv('USE_REAL_MONEY') == 'YES':
self.url = 'https://api.alpaca.markets'
self.__key_id = getenv('APCA_API_KEY_ID')
self.__secret_key = getenv('APCA_API_SECRET_KEY')
else:
self.url = 'https://paper-api.alpaca.markets'
self.__key_id = getenv('APCA_PAPER_KEY_ID')
self.__secret_key = getenv('APCA_PAPER_SECRET_KEY')
self.API = alpaca_trade_api.REST(self.__key_id,
self.__secret_key,
base_url=self.url)
self.polygon = self.API.polygon
self.logs = Logs(name="alpaca trading", to_cloud=logs_to_cloud)
self.positive_statuses = set([
'new', 'accepted', 'pending_new', 'accepted_for_bidding',
'calculated', 'partially_filled', 'filled'
])
# TODO: ADD PRE and POST MARKET FUNCTIONALITY
def get_market_status(self):
clock = self.API.get_clock()
return clock.is_open
def get_balance(self):
account = self.API.get_account()
if hasattr(account, 'cash'):
return float(account.cash)
self.logs.warn(f'Not able to get account balance')
return 0
def get_last_price(self, ticker):
ticker.replace('$', '')
quote = self.polygon.last_quote(ticker)
self.logs.debug(f'Quote for {ticker}: {quote.bidprice}')
if hasattr(quote, 'bidprice'):
return quote.bidprice
else:
self.logs.warn(f'Not able to retrieve last price for {ticker}')
return -1
def submit_market_buy(self, symbol, qty, limit):
self.logs.info(
f'Making a trade: Buying {qty} units of {symbol} at limit price of {limit}'
)
response = self.API.submit_order(symbol=symbol,
qty=str(qty),
side='buy',
type='limit',
time_in_force='day',
limit_price=str(limit))
self.logs.info(f'Order response: {response}')
if hasattr(response, 'status'):
return (response.status, response.id)
else:
self.logs.warn(f'Not able to place order for {symbol}')
return (False, "-1")
def submit_trailing_stop(self, symbol, qty, trail_percent):
response = self.API.submit_order(symbol=symbol,
qty=str(qty),
side='sell',
type='trailing_stop',
time_in_force='gtc',
trail_percent=str(trail_percent))
if hasattr(response, 'status'):
self.logs.info(
f'Trailing stop order status: {response.status}. Order ID: {response.id}'
)
return (response.status, response.id)
else:
self.logs.warn(
f'Not able to place trailing_stop sell order for {symbol}. QTY: {qty}. t_pct: {trail_percent}'
)
return (False, "-1")
def cancel_order(self, order_id):
#In case the order doesn't fill correctly or the market closes or something
try:
self.API.cancel_order(order_id)
if self.get_order_status(order_id) == 'canceled':
return True
except alpaca_trade_api.rest.APIError:
self.logs.warn(
f'Not able to cancel order with order_id: {order_id}')
return False
def get_order_status(self, order_id):
return self.API.get_order(order_id).status
class Twitter:
"""A helper for talking to Twitter APIs."""
def __init__(self, logs_to_cloud):
self.logs_to_cloud = logs_to_cloud
self.logs = Logs(name="twitter", to_cloud=self.logs_to_cloud)
self.twitter_auth = OAuthHandler(TWITTER_CONSUMER_KEY,
TWITTER_CONSUMER_SECRET)
self.twitter_auth.set_access_token(TWITTER_ACCESS_TOKEN,
TWITTER_ACCESS_TOKEN_SECRET)
self.twitter_api = API(auth_handler=self.twitter_auth,
retry_count=API_RETRY_COUNT,
retry_delay=API_RETRY_DELAY_S,
retry_errors=API_RETRY_ERRORS,
wait_on_rate_limit=True,
wait_on_rate_limit_notify=True)
self.twitter_listener = None
def start_streaming(self, callback):
"""Starts streaming tweets and returning data to the callback."""
self.twitter_listener = TwitterListener(
callback=callback, logs_to_cloud=self.logs_to_cloud)
twitter_stream = Stream(self.twitter_auth, self.twitter_listener)
self.logs.debug("Starting stream.")
twitter_stream.filter(follow=INFLUENCER_USER_IDS)
# If we got here because of an API error, raise it.
if self.twitter_listener and self.twitter_listener.get_error_status():
raise Exception("Twitter API error: %s" %
self.twitter_listener.get_error_status())
def stop_streaming(self):
"""Stops the current stream."""
if not self.twitter_listener:
self.logs.warn("No stream to stop.")
return
self.logs.debug("Stopping stream.")
self.twitter_listener.stop_queue()
self.twitter_listener = None
def test_tweet(self):
x = randint(1, 100)
self.twitter_api.update_status(f'Beep {x} Beep')
return 9
def tweet(self, companies, tweet):
"""Posts a tweet listing the companies, their ticker symbols, and a
quote of the original tweet.
"""
link = self.get_tweet_link(tweet)
text = self.make_tweet_text(companies, link)
self.logs.info("Tweeting: %s" % text)
self.twitter_api.update_status(text)
def make_tweet_text(self, companies, link):
"""Generates the text for a tweet."""
# Find all distinct company names.
names = []
for company in companies:
name = company["name"]
if name not in names:
names.append(name)
# Collect the ticker symbols and sentiment scores for each name.
tickers = {}
sentiments = {}
for name in names:
tickers[name] = []
for company in companies:
if company["name"] == name:
ticker = company["ticker"]
tickers[name].append(ticker)
sentiment = company["sentiment"]
# Assuming the same sentiment for each ticker.
sentiments[name] = sentiment
# Create lines for each name with sentiment emoji and ticker symbols.
lines = []
for name in names:
sentiment_str = self.get_sentiment_emoji(sentiments[name])
tickers_str = " ".join(["$%s" % t for t in tickers[name]])
line = "%s %s %s" % (name, sentiment_str, tickers_str)
lines.append(line)
# Combine the lines and ellipsize if necessary.
lines_str = "\n".join(lines)
size = len(lines_str) + 1 + len(link)
if size > MAX_TWEET_SIZE:
self.logs.warn("Ellipsizing lines: %s" % lines_str)
lines_size = MAX_TWEET_SIZE - len(link) - 2
lines_str = "%s\u2026" % lines_str[:lines_size]
# Combine the lines with the link.
text = "%s\n%s" % (lines_str, link)
return text
def get_sentiment_emoji(self, sentiment):
"""Returns the emoji matching the sentiment."""
if not sentiment:
return EMOJI_SHRUG
if sentiment > 0:
return EMOJI_THUMBS_UP
if sentiment < 0:
return EMOJI_THUMBS_DOWN
self.logs.warn("Unknown sentiment: %s" % sentiment)
return EMOJI_SHRUG
def get_tweet(self, tweet_id):
"""Looks up metadata for a single tweet."""
# Use tweet_mode=extended so we get the full text.
print(tweet_id)
status = self.twitter_api.get_status(tweet_id, tweet_mode="extended")
if not status:
self.logs.error("Bad status response: %s" % status)
return None
# Use the raw JSON, just like the streaming API.
return status._json
# def get_all_tweets(self):
# """Looks up metadata for the most recent Influencer tweets."""
# tweets = []
# # Only the 3,200 most recent tweets are available through the API.
# for status in Cursor(self.twitter_api.user_timeline,
# user_id=INFLUENCER_USER_ID,
# exclude_replies=True).items():
# # Extract the quoted influencer tweet, if available.
# try:
# quoted_tweet_id = status.quoted_status_id
# except AttributeError:
# self.logs.warn('Skipping tweet: %s' % status)
# continue
# # Get the tweet details and add it to the list.
# quoted_tweet = status._json #self.get_tweet(quoted_tweet_id)
# tweets.append(quoted_tweet)
# self.logs.debug("Got tweets: %s" % tweets)
# return tweets
def get_tweet_text(self, tweet):
"""Returns the full text of a tweet."""
# The format for getting at the full text is different depending on
# whether the tweet came through the REST API or the Streaming API:
# https://dev.twitter.com/overview/api/upcoming-changes-to-tweets
try:
if "extended_tweet" in tweet:
self.logs.debug("Decoding extended tweet from Streaming API.")
return tweet["extended_tweet"]["full_text"]
elif "full_text" in tweet:
self.logs.debug("Decoding extended tweet from REST API.")
return tweet["full_text"]
else:
self.logs.debug("Decoding short tweet.")
return tweet["text"]
except KeyError:
self.logs.error("Malformed tweet: %s" % tweet)
return None
def get_tweet_link(self, tweet):
"""Creates the link URL to a tweet."""
if not tweet:
self.logs.error("No tweet to get link.")
return None
try:
screen_name = tweet["user"]["screen_name"]
id_str = tweet["id_str"]
except KeyError:
self.logs.error("Malformed tweet for link: %s" % tweet)
return None
link = TWEET_URL % (screen_name, id_str)
return link
class TwitterListener(StreamListener):
"""A listener class for handling streaming Twitter data."""
def __init__(self, callback, logs_to_cloud):
self.logs_to_cloud = logs_to_cloud
self.logs = Logs(name="twitter-listener", to_cloud=self.logs_to_cloud)
self.callback = callback
self.error_status = None
self.start_queue()
def start_queue(self):
"""Creates a queue and starts the worker threads."""
self.queue = Queue()
self.stop_event = Event()
self.logs.debug("Starting %s worker threads." % NUM_THREADS)
self.workers = []
for worker_id in range(NUM_THREADS):
worker = Thread(target=self.process_queue, args=[worker_id])
worker.daemon = True
worker.start()
self.workers.append(worker)
def stop_queue(self):
"""Shuts down the queue and worker threads."""
# First stop the queue.
if self.queue:
self.logs.debug("Stopping queue.")
self.queue.join()
else:
self.logs.warn("No queue to stop.")
# Then stop the worker threads.
if self.workers:
self.logs.debug("Stopping %d worker threads." % len(self.workers))
self.stop_event.set()
for worker in self.workers:
# Block until the thread terminates.
worker.join()
else:
self.logs.warn("No worker threads to stop.")
def process_queue(self, worker_id):
"""Continuously processes tasks on the queue."""
# Create a new logs instance (with its own httplib2 instance) so that
# there is a separate one for each thread.
logs = Logs("twitter-listener-worker-%s" % worker_id,
to_cloud=self.logs_to_cloud)
logs.debug("Started worker thread: %s" % worker_id)
while not self.stop_event.is_set():
try:
data = self.queue.get(block=True, timeout=QUEUE_TIMEOUT_S)
start_time = time()
self.handle_data(logs, data)
self.queue.task_done()
end_time = time()
qsize = self.queue.qsize()
logs.debug("Worker %s took %.f ms with %d tasks remaining." %
(worker_id, end_time - start_time, qsize))
except Empty:
logs.debug("Worker %s timed out on an empty queue." %
worker_id)
continue
except Exception:
# The main loop doesn't catch and report exceptions from
# background threads, so do that here.
logs.catch()
logs.debug("Stopped worker thread: %s" % worker_id)
def on_error(self, status):
"""Handles any API errors."""
self.logs.error("Twitter error: %s" % status)
self.error_status = status
self.stop_queue()
return False
def get_error_status(self):
"""Returns the API error status, if there was one."""
return self.error_status
def on_data(self, data):
"""Puts a task to process the new data on the queue."""
# Stop streaming if requested.
if self.stop_event.is_set():
return False
# Put the task on the queue and keep streaming.
self.queue.put(data)
return True
def handle_data(self, logs, data):
"""Sanity-checks and extracts the data before sending it to the
callback.
"""
try:
tweet = loads(data)
except ValueError:
logs.error("Failed to decode JSON data: %s" % data)
return
try:
user_id_str = tweet["user"]["id_str"]
screen_name = tweet["user"]["screen_name"]
except KeyError:
logs.error("Malformed tweet: %s" % tweet)
return
# We're only interested in tweets from the influencer him/herself, so skip the
# rest.
if user_id_str not in INFLUENCER_USER_IDS:
# logs.debug("Skipping tweet from user: %s (%s)" %
# (screen_name, user_id_str))
logs.debug(
f'Skipping tweet from user: {screen_name}. The {user_id_str} is not in {INFLUENCER_USER_IDS}'
)
return
logs.info("Examining tweet: %s" % tweet)
# Call the callback.
self.callback(tweet)