def __init__(self, symbols):
self.ts = TimeSeries(key="84WDI082Z0HOREL6", output_format='pandas', indexing_type='date')
sns.set()
self.MEAN_ERROR = 0.1
self.symbols = symbols
self.stocks_data = None
self.retrieve_stock_values()
self.low_risk_range = (0, 10)
self.medium_risk_range = (10, 20)
self.high_risk_range = (20, 999)
def __init__(self, api_key):
self.ts = TimeSeries(key=api_key, output_format='pandas', indexing_type='date')
self.cc = CryptoCurrencies(key=api_key, output_format='pandas', indexing_type='date')
self.nasdaq_stockfile = f'{self.stock_filename}-{datetime.datetime.fromtimestamp(time.time()).strftime("%Y%m%d")}.csv'
self.stocks = []
os.makedirs("stock_estimations", exist_ok=True)
self._download_nasdaq_stocks()
with open(self.nasdaq_stockfile, "r") as file:
csv_content = csv.reader(file, delimiter=',', quotechar='"')
next(csv_content, None)
for row in csv_content:
row = list(filter(None, row))
self.stocks.append(Stock.from_nasdaq(row))
class StockDatabase(object):
"""Download all the available values of known stock"""
nasdaq_url = "https://old.nasdaq.com/screening/companies-by-name.aspx?letter=0&exchange=nasdaq&render=download"
stock_filename = "stock_estimations/companylist"
def __init__(self, api_key):
self.ts = TimeSeries(key=api_key, output_format='pandas', indexing_type='date')
self.cc = CryptoCurrencies(key=api_key, output_format='pandas', indexing_type='date')
self.nasdaq_stockfile = f'{self.stock_filename}-{datetime.datetime.fromtimestamp(time.time()).strftime("%Y%m%d")}.csv'
self.stocks = []
os.makedirs("stock_estimations", exist_ok=True)
self._download_nasdaq_stocks()
with open(self.nasdaq_stockfile, "r") as file:
csv_content = csv.reader(file, delimiter=',', quotechar='"')
next(csv_content, None)
for row in csv_content:
row = list(filter(None, row))
self.stocks.append(Stock.from_nasdaq(row))
def generate_stock_history(self, stock, time_series="day", output_size="full", interval=None, is_crypto=False):
if time_series == "intraday":
data, _ = self.ts.get_intraday(symbol=stock.symbol, outputsize=output_size, interval=interval) if not is_crypto else self.cc.get_digital_currency_intraday(symbol=stock.symbol, output_size=output_size)
if time_series == "day":
data, _ = self.ts.get_daily(symbol=stock.symbol, outputsize=output_size) if not is_crypto else self.cc.get_digital_currency_daily(symbol=stock.symbol, market="USD")
if time_series == "week":
data, _ = self.ts.get_weekly(symbol=stock.symbol) if not is_crypto else self.cc.get_digital_currency_weekly(symbol=stock.symbol, market="USD")
if time_series == "month":
data, _ = self.ts.get_monthly(symbol=stock.symbol) if not is_crypto else self.cc.get_digital_currency_monthly(symbol=stock.symbol, market="USD")
directory = f'stock_estimations/{stock.symbol}/{datetime.datetime.today().strftime("%d-%m-%Y")}'
filename = f'{directory}/{stock.symbol}-prices.csv'
Path(directory).mkdir(parents=True, exist_ok=True)
with open(filename, 'w+') as out:
for index, row in data.iterrows():
out.write(f'{index.replace("-", "")},{row.iloc[0]}\n')
return filename
def _download_nasdaq_stocks(self):
yesterday = datetime.datetime.now() - datetime.timedelta(days=1)
yesterday_file = f'{self.stock_filename}-{yesterday.strftime("%Y%m%d")}.csv'
if os.path.exists(yesterday_file):
os.remove(yesterday_file)
if os.path.exists(self.nasdaq_stockfile):
return
else:
urllib.request.urlretrieve(self.nasdaq_url, self.nasdaq_stockfile)
class SuggestionEngine(object):
"""
Basic suggestion using stock volatility and mean returns
"""
def __init__(self, symbols):
self.ts = TimeSeries(key="84WDI082Z0HOREL6", output_format='pandas', indexing_type='date')
sns.set()
self.MEAN_ERROR = 0.1
self.symbols = symbols
self.stocks_data = None
self.retrieve_stock_values()
self.low_risk_range = (0, 10)
self.medium_risk_range = (10, 20)
self.high_risk_range = (20, 999)
def suggest(self):
"""
Suggest a stock
:return:
"""
if self.stocks_data.empty:
self.retrieve_stock_values()
returns = self.stocks_data.pct_change()
mean_daily_returns = returns.mean()
#
volatilities = returns.std()
# https://www.fool.com/knowledge-center/how-to-calculate-annualized-volatility.aspx
combine = pd.DataFrame({'retour': mean_daily_returns * 252,
'volatilite du stock': volatilities * 252})
ret = {
"buy": {
"low": [],
"medium": [],
"high": []
},
"sell": [],
"result": None
}
x = combine["volatilite du stock"].values[:, np.newaxis]
y = combine["retour"].values
model = LinearRegression()
model.fit(x, y)
linear_reg = model.predict(x)
# y = ax + b
combine["Distance"] = y - linear_reg
# https://seaborn.pydata.org/generated/seaborn.jointplot.html
g = sns.jointplot("volatilite du stock", "retour", data=combine, kind="reg", height=7)
for i in range(combine.shape[0]):
# Under linear regression
if combine.iloc[i, 2] - self.MEAN_ERROR > 0:
ret["sell"].append(self.symbols[i])
else:
# Check risk threshold
risk = combine.iloc[i, 1]
if self.low_risk_range[0] <= risk <= self.low_risk_range[1]:
ret["buy"]["low"].append(self.symbols[i])
elif self.medium_risk_range[0] <= risk <= self.medium_risk_range[1]:
ret["buy"]["medium"].append(self.symbols[i])
else:
ret["buy"]["high"].append(self.symbols[i])
plt.annotate(self.symbols[i], (combine.iloc[i, 1], combine.iloc[i, 0]))
filename = f'{next(tempfile._get_candidate_names())}-plot.png'
plt.savefig(filename)
with open(filename, 'rb') as image_file:
ret["result"] = base64.b64encode(image_file.read()).decode('utf-8')
print(ret)
os.remove(filename)
return ret
def retrieve_stock_values(self):
"""
Retrieve stocks values using AlphaVantage wrapper
Limit to MAX_STOCKS stocks name at a time
:return:
"""
filenames = []
for symbol in self.symbols:
data, _ = self.ts.get_daily(symbol=symbol, outputsize="full")
filename = f'{next(tempfile._get_candidate_names())}.csv'
filenames.append(filename)
data.to_csv(path_or_buf=filename)
dfs = [pd.read_csv(file)[['date', '4. close']] for file in filenames]
self.stocks_data = reduce(lambda left, right: pd.merge(left, right, on='date'), dfs).iloc[:, 1:]
[os.remove(f) for f in filenames]
return True
def testTradingReloadSavedSimulateSmallInventory(self):
"""
Test trading by loading data from a user for multiple days
User have a small inventory
"""
symbol = "AAPL"
user = User(None)
start_capital = 1500
model = None
# Create a fake user
email = f"{''.join(random.choices(string.ascii_uppercase + string.digits, k=10))}@test.com"
password = ''.join(
random.choices(string.ascii_uppercase + string.digits, k=10))
user.create_user("Test", "Test", email, password)
user.login(email, password)
self.assertIsNotNone(user.bearer,
"Couldn't connect as fake user on API")
print(f"[*] Testing for user '{email}' with password '{password}'")
# User has 1500 invested and 1500 available
self.assertTrue(user.update_balance(start_capital))
self.assertTrue(user.update_invested_balance(start_capital))
self.assertEqual(0, user.balance,
"User balance wasn't updated properly")
self.assertEqual(start_capital, user.invested_balance,
"User invested balance wasn't updated properly")
# Load model
save_file = os.path.join(os.getcwd(), "trade_model.pkl")
with open(save_file, "rb") as model_file:
model = pickle.load(model_file)
self.assertIsNotNone(model, "Couldn't load sklearn model")
print("[+] Model loaded !")
ts = TimeSeries(API_KEY, output_format="pandas")
data, _ = ts.get_daily(symbol=symbol, outputsize="full")
self.assertIsNotNone(data, "Couldn't retrieve data from AlphaVantage")
parameters = [data['4. close'].tolist(), data['5. volume'].tolist()]
minmax = MinMaxScaler(feature_range=(100,
200)).fit(np.array(parameters).T)
trend = data['4. close'].tolist()
scaled_parameters = minmax.transform(np.array(parameters).T).T.tolist()
agent = TradingAgent(model,
timeseries=scaled_parameters,
real_trend=trend,
minmax=minmax,
symbol=symbol)
# User has a small inventory on said stock
stock_price = data.iloc[0:-120].tail(1)["4. close"].tolist()[0]
print(f"Stock {symbol} worth {stock_price}/ea")
user.update_balance(start_capital + (stock_price * 2))
user.update_invested_balance(start_capital + (stock_price * 2))
user.update_inventory(symbol, 2, stock_price, "buy")
# Load user inventory and balance -120
agent.load_user_info(user)
print("[+] User info loaded !")
# Feed 20 days from 100 days ago (so D-120 to D-100)
agent.load_window(data.iloc[0:-120])
# Run actions from 100 days to today
data = data.tail(100)
buys = []
sells = []
for i in range(100):
print(
f"Day n°{i} : Invested : {user.invested_balance} | Inventory : {user.inventory}"
)
last_data = data.iloc[i]
self.assertIsNotNone(last_data)
action = agent.trade([
last_data["4. close"].tolist(),
last_data["5. volume"].tolist()
])
if action is not None:
if action['action'] == 'buy':
buys.append(i)
else:
sells.append(i)
# Create figure to have multiple plots
fig = plt.figure(figsize=(15, 5))
plt.plot(data['4. close'],
color='r',
lw=2.0,
label=f'Valeur du stock {symbol}')
plt.plot(data['4. close'],
'^',
markersize=8,
color='m',
label="Signal d'achat",
markevery=buys)
plt.plot(data['4. close'],
'v',
markersize=8,
color='k',
label='Signal de vente',
markevery=sells)
print(
f"Started at {start_capital}, now sitting at total {user.balance + user.invested_balance}"
)
invest = (((user.balance + user.invested_balance) - start_capital) /
start_capital) * 100
plt.title(
f'{symbol} - Simulation pour user sur 100 jours (Gain total : {round(invest)}%)'
)
plt.legend()
plt.savefig(f'{DEST_PLOT_DIR}/{symbol}-user-simulated-trade')
plt.close(fig)
def testTradingManually(self):
"""
Test trading stocks analysis and manually verify results
"""
ts = TimeSeries(API_KEY, output_format="pandas")
# Containing old historical data (picked from https://en.wikipedia.org/wiki/List_of_S%26P_500_companies)
training_stocks = [
"AMD", "FSV", "ROKU", "TMUS", "FTNT", "KHC", "TRIP", "TXN", "LYFT",
"FSV", "SINA", "BIIB"
]
# Interesting stocks to tests onto
up_trends = ["MSFT", "AAPL", "FB", "ADBE", "CTXS"]
down_trends = ["ALGN", "MYL", "AAL"]
test_stocks = up_trends + down_trends
skip = 1
layer_size = 500
# 1500 dollars should always be enough
initial_money = 1500
output_size = 3
model = Model(79, layer_size, output_size)
model_initialized = False
agent = None
# We need to train our model first
for stock in training_stocks:
print(f"Training on {stock} with {initial_money}$")
data, _ = ts.get_daily(symbol=stock, outputsize="full")
self.assertIsNotNone(data,
"Couldn't retrieve data from Alphavantage")
# Take a full year of trading
data = data.tail(253)
print(data)
trend = data['4. close'].tolist()
parameters = [
data['4. close'].tolist(), data['5. volume'].tolist()
]
minmax = MinMaxScaler(feature_range=(100, 200)).fit(
np.array(parameters).T)
scaled_parameters = minmax.transform(
np.array(parameters).T).T.tolist()
if not model_initialized:
agent = TradingAgent(model, scaled_parameters, skip,
initial_money, trend, minmax)
# Enable debug mode for test purpose
agent.es.debug = True
model_initialized = True
else:
# Feed new training data to model without recreating one
agent.change_data(scaled_parameters, skip, initial_money,
trend, minmax)
# Use 100 epoch for best result
agent.fit(iterations=100)
# Model is trained, now test on non-trained stocks
for stock in test_stocks:
data, _ = ts.get_daily(symbol=stock, outputsize="full")
self.assertIsNotNone(data,
"Couldn't retrieve data from Alphavantage")
# Take a full year of trading
data = data.tail(253)
data['5. volume'] = data['5. volume'].astype(int)
trend = data['4. close'].tolist()
parameters = [
data['4. close'].tolist(), data['5. volume'].tolist()
]
minmax = MinMaxScaler(feature_range=(100, 200)).fit(
np.array(parameters).T)
scaled_parameters = minmax.transform(
np.array(parameters).T).T.tolist()
agent.change_data(scaled_parameters, skip, initial_money, trend,
minmax)
buys, sells, total_gain, invest = agent.test_trade()
# Create figure to have multiple plots
fig = plt.figure(figsize=(15, 5))
plt.plot(data['4. close'],
color='r',
lw=2.0,
label=f'Valeur du stock {stock}')
plt.plot(data['4. close'],
'^',
markersize=8,
color='m',
label="Signal d'achat",
markevery=buys)
plt.plot(data['4. close'],
'v',
markersize=8,
color='k',
label='Signal de vente',
markevery=sells)
plt.title(f'{stock} - Gain total : {invest}%')
plt.legend()
plt.savefig(f'{DEST_PLOT_DIR}/{stock}')
plt.close(fig)
agent.save_model()
def trade(bearer):
"""
Trade for a user
"""
ts = TimeSeries("84WDI082Z0HOREL6", output_format="pandas")
# Load user
user = User(bearer)
suggest_engine = SuggestionEngine(symbols=["MSFT", "AAPL", "AMD", "CSCO"])
suggestion = suggest_engine.suggest()
# Load model
model = None
save_file = os.path.join(os.getcwd(), "trade_model.pkl")
ret = {
"suggested": {
"buy": [],
"sell": [],
},
"checked_symbols": [],
"actions": []
}
with open(save_file, "rb") as model_file:
model = pickle.load(model_file)
# Append suggested buy trade to list of checks
print(f"User has risk {user.get_risk()}")
symbols_to_check = []
if len(suggestion["buy"][user.get_risk()]):
symbols_to_check.extend(suggestion["buy"][user.get_risk()])
ret["suggested"]["buy"] = suggestion["buy"][user.get_risk()]
if len(suggestion["sell"]):
symbols_to_check.extend(suggestion["sell"])
ret["suggested"]["sell"] = suggestion["sell"]
if len(user.inventory):
symbols_to_check.extend(
[item["stock"]["symbol"] for item in user.inventory])
# Run a check on all the user inventory + suggested trade offers (both sell and buy)
ret["checked_symbols"] = symbols_to_check
time.sleep(60)
for symbol in symbols_to_check:
print(f"TradingAgent: Checking {symbol}...")
data, _ = ts.get_daily(symbol=symbol, outputsize="compact")
parameters = [data['4. close'].tolist(), data['5. volume'].tolist()]
minmax = MinMaxScaler(feature_range=(100,
200)).fit(np.array(parameters).T)
trend = data['4. close'].tolist()
scaled_parameters = minmax.transform(np.array(parameters).T).T.tolist()
agent = TradingAgent(model,
timeseries=scaled_parameters,
real_trend=trend,
minmax=minmax,
symbol=symbol)
# Load user inventory
agent.load_user_info(user)
# Feed 20 days
agent.load_window(data.iloc[0:-20])
last_data = data.tail(1)
print(f"Data loaded for {symbol}, taking decision...")
action = agent.trade([
last_data["4. close"].tolist()[0],
last_data["5. volume"].tolist()[0]
])
if action is not None:
ret["actions"].append(action)
print(f"Action for {symbol} : {action}")
return ret