def run(predict):
samples = read_data()
mapping = {'stop': 0, 'warning': 1, 'go': 2, 'stopLeft': 3, 'warningLeft': 4, 'goLeft': 5}
X, labels = next(generator(samples)(1))
p_labels = predict(X)
print(p_labels)
for img, l, p_l in zip(X, labels, p_labels):
final_image = cv2.cvtColor(img, cv2.COLOR_YUV2BGR)
indx = np.argmax(p_l)
label = list(mapping.keys())[list(mapping.values()).index(indx)]
cv2.imshow(label, final_image)
cv2.waitKey(0)
def job_function():
for element in SUPPORTED_INSTRUMENTS:
instrument = element["instrument"]
period = element["period"]
point = element["point"]
last_learn_time = read_time(LAST_LEARN_FILE, instrument, period)
if ((datetime.now() - last_learn_time).total_seconds() / 60 / 60) > LEARN_PERIOD_HOURS:
filename = datasets.get_daily_dataset_file(instrument, period)
if datasets.daily_dataset_exists(instrument, period):
logger.info("job_function: Start learning %s %s", instrument, period)
x, y = model.read_data([filename], instrument, period, point)
model.train_model(x, y, instrument, period, verbose=0)
write_time(LAST_LEARN_FILE, instrument, period)
else:
logger.info("job_function: Daily dataset %s not found. Waiting...", str(filename))
def search(term):
output = []
term = term.lower()
data = model.read_data()
temp = term.split()
if(len(temp)>0):
for datum in data:
dat = datum[1].lower()
if term in dat:
output.append(datum)
continue
hit_count = 0
for word in temp:
if(word in dat):
hit_count += 1
if(hit_count>2):
output.append(datum)
return output
def check_metrics(_model: Model):
samples = read_data()
batch_size = 100
gen = generator(samples)
epoch = 0
y_true = np.array([])
y_pred = np.array([])
while epoch < 10:
x, labels = next(gen(batch_size))
p_labels = model_predict()(x)
y_true = np.append(y_true, np.argmax(labels, axis=1))
y_pred = np.append(y_pred,np.argmax(p_labels, axis=1))
print(epoch)
epoch += 1
precision, recall, fscore, support = score(y_true, y_pred)
print('precision: {}'.format(precision))
print('recall: {}'.format(recall))
print('fscore: {}'.format(fscore))
print('support: {}'.format(support))
def simulate(stocks,
forecast_out=7,
start_day=2,
start_month=1,
start_year=2017,
end_day=None,
end_month=None,
end_year=None,
start_money=1000,
plot=False,
deposits=None,
loss_threshold=0,
buy_threshold=0,
verbose=False):
'''
stocks - A list of stocks to test and forecast with
forecast_out - the number of days to forecast over
start_day - day of month to start simulation
start_month - month to start simulation
start_year - year to start simulation
end_day - day of month to end simulation
end_month - month to end simulation
end_year - year to end simulation
start_money - the amount of money to start the simulation with
plot - if True, plots percentage increase compared to market
'''
first_start = time.time()
if (verbose):
print("STOCKS BEING LOOKED AT: {}\n".format(stocks))
print("STARTING WITH: ${}".format(start_money))
# Make start date if none given
if (start_day == 2 and start_month == 1 or start_year == 2017):
if (verbose):
print("DEFAULT START DATE")
start_date = dt(start_year, start_month, start_day)
if (end_day == None or end_month == None or start_year == None):
if (verbose):
print("DEFAULT END DATE OF TODAY")
end_date = dt.now()
else:
end_date = dt(end_year, end_month, end_day)
if (deposits is not None):
start_money = 50
# Initialize directory for plotting, log files, etc.
file_name = initialize_directory(stocks, start_date, end_date, start_money,
forecast_out, loss_threshold,
buy_threshold, verbose)
# Initializes plotting if plot
if (plot):
fig, ax = plt.subplots()
start_market = read_data("^GSPC", start_date,
dt.now())["close"].values[0]
# To keep track of total earnings
total_spent = 0 # Do these even matter?
total_sold = 0
dates = 1
if (verbose):
print("START DATE: {}".format(start_date))
end_hold = end_hold_date(forecast_out, start_day, start_month, start_year)
new_money = np.copy(start_money)
pred_percentages, mkt_percentages = ([] for i in range(2))
last_bought = True
while (end_hold.date() < end_date.date()):
if (verbose):
print("START DATE: \t{}".format(start_date))
print("END HOLD DATE: \t{}".format(end_hold))
print()
start = time.time()
# Test each stock, hold on to relevant data
if (verbose):
print("\nTESTING")
best, vol_best, correlations, stock_corr_averages = test(
stocks, forecast_out, start_date, verbose)
# Grab data from testing results
best_combination = best_combinations(best, vol_best)
try:
num = best[best_combination[0]]
vnum = vol_best[best_combination[1]]
except:
num = 0
vnum = 0
if (verbose):
print(best_combination)
print("BEST MEAN PRICE COMBINATION: {} WITH {} BESTS".format(
best_combination[0], num))
print("BEST MEAN VOLUME COMBINATION: {} WITH {} BESTS".format(
best_combination[1], vnum))
# Get list of stocks that had high correlation on average
d = {
k: v
for k, v in filter(lambda t: t[1] > 0.15,
stock_corr_averages.items())
}
good_stocks = list(d.keys())
# Forecast stocks and get good buys
stock_pred = forecast(good_stocks, forecast_out, start_date,
best_combination, buy_threshold, verbose)
write_predictions(file_name, dates, stock_pred)
buy_stocks = stock_pred[stock_pred['Good Buy']]['Stock'].values
# Do buying
total_spent, total_sold, new_money, last_bought = buy(
buy_stocks, start_date, end_hold, new_money, file_name,
last_bought, loss_threshold, verbose)
# Print out results from trading cycle
denominator = 999999999 if (total_spent == 0) else total_spent
if (verbose):
print("AFTER CYCLE {}".format(dates))
print("DIFFERENCE: {}\nPERCENTAGE: {}".format(
total_sold - total_spent,
100 * (total_sold / denominator - 1)))
print("NEW TOTAL MONEY: {}\n\n".format(new_money))
# Plot agains market
if (plot):
end_market = read_data("^GSPC", end_hold,
dt.now())["close"].values[0]
if (verbose):
print("END HOLD PERIOD: {}".format(end_hold))
print("\nEND MARKET HOLD: {}".format(end_market))
print("START MARKET: {}".format(start_market))
print()
mkt_percentages.append(100 * (end_market / start_market - 1))
# Plot percent from model
pred_percentages.append(100 * (new_money / start_money - 1))
# Write data
write_cycle(file_name, dates, new_money, total_sold, total_spent,
denominator, start_date, end_hold, start)
# Update days for next trading cycle
start_date = end_hold
end_hold = end_hold_date(forecast_out, end_hold.day, end_hold.month,
end_hold.year)
if (deposits is not None):
deposit_money, deposits = new_deposit(deposits, start_date,
file_name)
new_money = new_money + float(deposit_money)
start_money = start_money + float(deposit_money)
dates += 1
# Print out final results
if (verbose):
print("START DATE: {}, END DATE: {}".format(start_date, end_hold))
print("TOTAL SPEND: {}\nTOTAL SOLD: {}\nDIFFERENCE: {}".format(
total_spent, total_sold, total_sold - total_spent))
print(
"INITIAL INVESTMENT: {}, MONEY NOW: {}, PERCENT CHANGE: {}".format(
start_money, new_money, 100 * (new_money / start_money - 1)))
percent = 100 * (new_money / start_money - 1)
if (plot):
ax.plot(mkt_percentages, marker='s', color='k', label='Market')
ax.plot(pred_percentages, marker='p', color='g', label='Prediction')
ax.set(title='Predicition vs. Market for {} Day Forecasting'.format(
forecast_out),
xlabel='Trade Cycle',
ylabel='Percent Growth')
ax.legend(loc='best')
#plt.show()
path = "/home/dr/Projects/multi_model_stock_forecasting/results/"
path += "simulation_result_{}_{}_stocks.png".format(
forecast_out, len(stocks))
#plt.savefig(path)
plt.savefig(file_name + ".png")
plt.close()
# Final write to file
write_final(file_name, dates, new_money, start_money, first_start,
time.time(), pred_percentages, mkt_percentages)
return percent
def buy(buy_stocks,
start_date,
end_hold,
new_money,
file_name,
last_bought,
threshold=0,
verbose=False):
bought_stocks = {}
num_no_buys = 0
total_spent = 0
total_sold = 0
old_money = np.copy(new_money)
old_total_spent = np.copy(total_spent)
old_total_sold = np.copy(total_sold)
#print("STOCKS TO BUY: {}".format(buy_stocks))
while (num_no_buys < len(buy_stocks)):
for bs in buy_stocks:
bs_dat = read_data(bs, start_date, dt.now())['close']
#if(verbose):
# print("Before hold: {} at {}".format(bs, bs_dat[0]))
if (bs_dat.values[0] < new_money):
total_spent += bs_dat.values[0]
new_money = new_money - bs_dat.values[0]
if (bs not in bought_stocks):
bought_stocks[bs] = [1, bs_dat.values[0], 0]
else:
bought_stocks[bs][0] += 1
num_no_buys = 0
else:
num_no_buys += 1
if (verbose):
print("STOCKS PURCHASED")
for key, val in bought_stocks.items():
print(key, val)
for bs, vals in bought_stocks.items():
hold_dat = read_data(bs, end_hold, dt.now())['close']
bought_stocks[bs][2] = hold_dat.values[0]
total_sold += hold_dat.values[0] * vals[0]
new_money += hold_dat.values[0] * vals[0]
money = (hold_dat.values[0] - vals[1]) * vals[0]
# Prints relevant details
if (verbose):
print("BOUGHT {} OF {} AT {}".format(vals[0], bs, vals[1]))
print("AFTER HOLD: {}".format(hold_dat.values[0]))
print("MONEY MADE FROM PURCHASE OF {} IS: {} WAS {}".format(
bs, money, "GOOD" if money > 0 else "BAD"))
print("LAST BOUGHT: {}".format(last_bought))
print("OLD MONEY: {} NEW MONEY: {}".format(old_money, new_money))
# Doesn't buy if last buy was bad
# This buy good and last buy good
if ((new_money >= (1 - threshold) * old_money) and (last_bought)):
#print("GOOD GOOD")
#print("SHOULD BE HERE")
last_bought = True
write_buys(file_name, bought_stocks)
return total_spent, total_sold, new_money, last_bought
# This buy good and last buy bad
elif ((new_money > (1 - threshold) * old_money) and not (last_bought)):
#print("GOOD BAD")
last_bought = True
with open(file_name, 'a+') as fout:
fout.write("\nLAST CYCLE LOST MONEY. THIS CYCLE MADE MONEY")
fout.write("\nMAKING PURCHASES NEXT CYCLE\n\n")
fout.close()
return old_total_spent, old_total_sold, old_money, last_bought
# This buy bad and last buy good
elif ((new_money <= (1 - threshold) * old_money) and (last_bought)):
#print("BAD GOOD")
last_bought = False
with open(file_name, 'a+') as fout:
fout.write("\nTHIS CYCLE LOST MONEY")
fout.write("\nNOT PURCHASING NEXT CYCLE\n\n")
fout.close()
return total_spent, total_sold, new_money, last_bought
# This buy bad and last buy bad
elif ((new_money <= (1 - threshold) * old_money) and not (last_bought)):
#print("BAD BAD")
last_bought = False
with open(file_name, 'a+') as fout:
fout.write("\nLAST CYCLE LOST MONEY")
fout.write("\nNO PURCHASES MADE THIS CYCLE")
fout.write("\nNO PURCHASES WILL BE MADE NEXT CYCLE")
fout.close()
return old_total_spent, old_total_sold, old_money, last_bought
# Undefined case
else:
raise Exception("UNDEFINED BEHAVIOR IN STOCK PURCHASING")
exit(1)
import pandas as pd
import numpy as np
import pickle
from model import read_data, strip_bio, load_glove, uniGram
EMBEDDING_FILE = "glove.840B.300d.txt"
raw = read_data("train.txt")
raw_withoutBIO = strip_bio(raw, "ner") # ner, pos, tokens, short_ner
raw_withoutBIO = raw_withoutBIO.drop(columns=["short_ner"])
extra_data = pd.read_pickle("conll2003_combined.pkl")
test_file = read_data("test.txt")
raw_withoutBIO = pd.concat([raw_withoutBIO, extra_data, test_file],
ignore_index=True)
all_words = uniGram(raw_withoutBIO, "tokens").keys()
all_words = [x.lower() for x in all_words]
print(len(all_words))
all_words = list(set(all_words))
print(len(all_words))
embedding = load_glove(EMBEDDING_FILE)
word_dict = dict()
matrix = np.zeros([len(all_words), 300])
print("start scanning the smaller words")
unknown_count = 0
for i in range(len(all_words)):
word_dict[all_words[i]] = i
try:
matrix[i] = embedding[all_words[i]]
except:
def data_table():
data = read_data() # Shows only 50 samples
return render_template(
'index.html', tables=[data.to_html(classes='table table-striped')])
#!/home/slamphear/miniconda2/bin/python
import cgi
import sys
import model as md
import json
import os
sql_path = os.path.join(os.path.dirname(__file__), 'data/data.sqlite')
# -----------
# boiler plate
# -----------
fs = cgi.FieldStorage()
sys.stdout.write("Content-Type: application/json")
sys.stdout.write("\n")
sys.stdout.write("\n")
# -----------
d = md.read_data(filename=sql_path)
sys.stderr.write(json.dumps(d, indent=1))
result = json.dumps(d, indent=1)
# -----------
# boiler plate
# -----------
sys.stdout.write(result)
sys.stdout.write("\n")
sys.stdout.close()
#!/u/p/r/pradap/miniconda/bin/python
import cgi
import sys
import model as md
import json
import os
sql_path = os.path.join(os.path.dirname(__file__), 'data/data.sqlite')
# -----------
# boiler plate
# -----------
fs = cgi.FieldStorage()
sys.stdout.write("Content-Type: application/json")
sys.stdout.write("\n")
sys.stdout.write("\n")
checked_data = fs.getvalue('checked_options')
# -----------
# checked_data = request.form['checked_options']
# print checked_data
d = md.read_data(filename=sql_path, filter_label_str=checked_data)
result = json.dumps(d)
# -----------
# boiler plate
# -----------
sys.stdout.write(result)
sys.stdout.write("\n")
sys.stdout.close()
#!/usr/bin/python
import cgi
import model as md
import json
fs = cgi.FieldStorage()
print ("""Content-Type: application/json\n
""")
d = md.read_data()
result = json.dumps(d, indent=1)
print(result)
print("\n")
#!/usr/bin/python
import cgi
import model as md
import json
import os
fs = cgi.FieldStorage()
print ("""Content-Type: application/json\n
""")
checked_data = fs.getvalue('checked_options')
d = md.read_data(filter_label_str=checked_data)
result = json.dumps(d)
print(result)
print("\n")
def put(put_stocks, start_date, end_hold, new_money, file_name, last_put, verbose=False):
putted_stocks = {}
num_no_puts = 0
total_spent = 0
total_sold = 0
old_money = np.copy(new_money)
old_total_spent = np.copy(total_spent)
old_total_sold = np.copy(total_sold)
#print("STOCKS TO BUY: {}".format(buy_stocks))
while(num_no_puts < len(put_stocks)):
for ps in put_stocks:
ps_dat = read_data(ps, start_date, dt.now())['close']
#if(verbose):
# print("Before hold: {} at {}".format(bs, bs_dat[0]))
if(ps_dat.values[0] < new_money):
total_spent += ps_dat.values[0]
new_money -= ps_dat.values[0]
if(ps not in putted_stocks):
putted_stocks[ps] = [1, ps_dat.values[0], 0]
else:
putted_stocks[ps][0] += 1
num_no_puts = 0
else:
num_no_puts += 1
print("\n\nPUT STOCKS: {}".format(put_stocks))
print("LAST PUT: {}".format(last_put))
if(verbose):
print("STOCKS PUT")
for key, val in putted_stocks.items():
print(key, val)
for ps, vals in putted_stocks.items():
hold_dat = read_data(ps, end_hold, dt.now())['close']
putted_stocks[ps][2] = hold_dat.values[0]
total_sold += hold_dat.values[0]*vals[0]
new_money += hold_dat.values[0]*vals[0]
print("PUT LINE: {}, SELL LINE: {}".format(vals[1], hold_dat.values[0]))
money = (vals[1] - hold_dat.values[0])*vals[0]
# Prints relevant details
if(verbose):
print("PUT {} OF {} AT {}".format(vals[0], ps, vals[1]))
print("AFTER HOLD: {}".format(hold_dat.values[0]))
print("MONEY MADE FROM PUT OF {} IS: {} WAS {}".format(ps, money,
"GOOD" if money>0 else "BAD"))
print("LAST PUT: {}".format(last_put))
print("OLD MONEY: {} NEW MONEY: {}".format(old_money, new_money))
# Doesn't buy if last buy was bad
# This buy good and last buy good
print("\n\nNEW PUT MONAYYY: {}".format(new_money))
if((new_money >= old_money) and (last_put)):
print("GOOD GOOD")
print("SHOULD BE HERE")
last_put = True
write_puts(file_name, putted_stocks)
return total_spent, total_sold, new_money, last_put
# This buy good and last buy bad
elif((new_money > old_money) and not(last_put)):
print("GOOD BAD")
last_put = True
with open(file_name, 'a+') as fout:
fout.write("\nLAST CYCLE LOST MONEY. THIS CYCLE MADE MONEY")
fout.write("\nMAKING PUTS NEXT CYCLE\n\n")
fout.close()
return old_total_spent, old_total_sold, old_money, last_put
# This buy bad and last buy good
elif((new_money <= old_money) and (last_put)):
print("BAD GOOD")
last_put = False
with open(file_name, 'a+') as fout:
fout.write("\nTHIS CYCLE LOST MONEY")
fout.write("\nNOT PUTTING NEXT CYCLE\n\n")
fout.close()
return total_spent, total_sold, new_money, last_put
# This buy bad and last buy bad
elif((new_money <= old_money) and not(last_put)):
print("BAD BAD")
last_put = False
with open(file_name, 'a+') as fout:
fout.write("\nLAST CYCLE LOST MONEY")
fout.write("\nNO PUTS MADE THIS CYCLE")
fout.write("\nNO PUTS WILL BE MADE NEXT CYCLE")
fout.close()
return old_total_spent, old_total_sold, old_money, last_put
# Undefined case
else:
raise Exception("UNDEFINED BEHAVIOR IN STOCK PURCHASING")
exit(1)
