def main() -> None:
"""Runs the program, plots raw and normalized dataset. If the weights are not null, we draw them as a red line."""
data = pd.read_csv("./data.csv")
_, ax = plt.subplots(1, 2)
ax[0].scatter(data=data, x="km", y="price")
ax[0].set_title("Raw dataset")
theta0, theta1 = get_weights()
if (theta0 != 0 and theta1 != 0):
x = data["km"]
y = theta0 + theta1 * x
ax[0].plot(x, y, 'r')
prepare_data(data)
ax[1].set_title("Normalized dataset")
ax[1].scatter(data=data, x="km", y="price")
plt.show()
def main() -> None:
"""Run the training and save the weights."""
data = pd.read_csv("./data.csv")
max_km, max_price = prepare_data(data)
data = data.values
features = data[:, 0]
targets = data[:, 1]
learning_rate = 0.1
epochs = 1000
batch_size = data.shape[0]
errors = []
theta0, theta1 = (0.0, 0.0)
for epoch in range(1, epochs + 1):
for b in range(0, data.shape[0], batch_size):
theta0, theta1 = train(features[b:b + batch_size],
targets[b:b + batch_size], theta0, theta1,
learning_rate)
avg_error = cost(data[:, 0], data[:, 1], theta0, theta1)
errors.append(avg_error)
print("Epoch {:4}/{:4}, average error: {:.6f}".format(
epoch, epochs, avg_error))
plt.plot(np.array(errors))
plt.show()
theta0 *= max_price
theta1 *= (max_price / max_km)
print("Theta0: {:.4f}".format(theta0))
print("Theta1: {:.4f}".format(theta1))
save_weights(theta0, theta1)
def run_net(opt):
psnrs = []
cnn.eval()
for i, (images, labels) in enumerate(tst_dloader):
images, labels = functions.prepare_data(
images,
labels,
pixels_shuffle=opt.pixels_shuffle,
use_cuda=use_cuda)
outputs = cnn(images)
outputs.data = torch.clamp(outputs.data, min=0, max=1)
psnr = compute_psnr(outputs, labels, opt)
psnrs.append(psnr)
if (opt.plot):
if (opt.pixels_shuffle):
images = functions.resize_lr(
images, (outputs.size()[2], outputs.size()[3]), use_cuda)
functions.plot_images(torch.cat(
(labels.data, images.data, outputs.data), 0),
use_cuda=use_cuda)
else:
functions.plot_images(torch.cat(
(labels.data, images.data, outputs.data), 0),
use_cuda=use_cuda)
return psnrs
def run_net(plot_images=False):
psnrs = []
cnn.eval()
for i, (images, labels) in enumerate(tst_loader):
images, labels = functions.prepare_data(images,
labels,
use_cuda=use_cuda)
outputs = cnn(images)
cleans = images - outputs # res
cleans = cleans.clamp(min=0, max=1)
psnr = compute_psnr(cleans, labels)
psnrs.append(psnr)
if (plot_images):
functions.plot_images(torch.cat(
(labels.data, images.data, cleans.data), 0),
use_cuda,
num_cols=3)
return psnrs
################################################################################
# Sidebar
st.sidebar.markdown(
"Look up ticker symbols [here](https://finance.yahoo.com/lookup)")
ticker_symbol = st.sidebar.text_input('Stock ticker symbol', value='AAPL')
start_date = st.sidebar.date_input("Start day", datetime.date(2010, 8, 14))
end_date = st.sidebar.date_input("End day", datetime.date(2020, 8, 14))
# Main Window
st.title("Stock Price Chart")
# Function calls to get data and make image
df_ticker = functions.get_stock_data(ticker_symbol, start_date, end_date)
stock_prices = functions.prepare_data(df_ticker['Close'])
fig = functions.make_picture(stock_prices,
img=img,
x_width_image=x_width_image,
horizon_height=horizon_height)
st.pyplot(fig=fig, bbox_inches='tight')
time.sleep(
1) # workaound, see https://github.com/streamlit/streamlit/issues/1294
plt.close(fig)
gc.collect()
st.markdown(
"Suggestions [welcome](https://github.com/dhaitz/stock-art). Image source: [mamunurpics](https://www.pexels.com/@mamunurpics). Inspired by [stoxart](https://www.stoxart.com)."
)
# workaround to open in wide mode (https://github.com/streamlit/streamlit/issues/314#issuecomment-579274365)
def train_lstm(data=None,
col_date='date',
n_days=1,
split_fac=0.7,
plot=True,
n_units=100,
drop_fac=0.3,
n_epochs=15,
n_batch=10,
split_size=0.2,
model=None):
from tensorflow import keras
"""
Train a LSTM neural network for prediction
The parameters for the NN architecture and training have to be specified:
n_units = 100
drop_fac = 0.2
n_epochs = 15
n_batch = 10
"""
# Do the train - test splitting with normalized data
X_train, y_train, X_test, y_test = f.prepare_data(data=data,
split_fac=split_fac,
LSTM=True)
X_train = np.asarray(X_train)
split = len(y_train)
print(
f'Train test split, train = {split}, Shape of X_Train/Test: {X_train.shape}'
)
# Reshape the arrays for keras
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1))
print(f'Reshaping X_Train/Test: {X_train.shape}')
if model == None:
# Build the network
inputs = keras.layers.Input(shape=(X_train.shape[1], X_train.shape[2]))
x = keras.layers.LSTM(n_units, return_sequences=True)(inputs)
x = keras.layers.Dropout(drop_fac)(x)
x = keras.layers.LSTM(n_units)(x)
output = keras.layers.Dense(1, activation='linear')(x)
model = keras.models.Model(inputs=inputs, outputs=output)
model.compile(optimizer='adam', loss='mse')
print(model.summary())
history = model.fit(X_train,
y_train,
epochs=n_epochs,
batch_size=n_batch,
validation_split=split_size)
predictions = model.predict(X_test)
pred = []
for p in predictions:
pred.append(p[0])
data_new = pd.DataFrame()
data_new[col_date] = data[col_date].iloc[split:-1]
data_new['True'] = y_test
data_new['Pred'] = np.array(pred)
title = 'LSTM for N days=' + str(n_days)
f.interactive_plot(data=data_new, title=title)
return model, predictions
if __name__ == "__main__":
""" MAIN PROGRAM """
# Read and normalize the data
stock_file = 'data/stock.csv'
volume_file = 'data/stock_volume.csv'
stock = pd.read_csv(stock_file)
volume = pd.read_csv(volume_file)
stock_norm = f.normalize(data=stock)
one_stock = f.individual_stock(price=stock, volume=volume, col='AAPL')
one_stock = f.trading_window(data=one_stock)
# Do the train - test splitting with normalized data
X_train, y_train, X_test, y_test = f.prepare_data(data=one_stock,
split_fac=0.7,
LSTM=True)
split = len(y_train)
X_train = np.asarray(X_train)
print(
f'Train test split, train = {split}, Shape of X_Train/Test: {X_train.shape}'
)
# Reshape the arrays
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1))
print(f'Reshaping X_Train/Test: {X_train.shape}')
# Deisgn the network
'''
print(volume.head())
print(stock.isnull().sum())
print(volume.isnull().sum())
print(stock.info())
print(volume['AAPL'].mean())
print(volume[volume.columns[-1]].max())
print(stock.describe())
print(volume.describe())
'''
one_stock = f.individual_stock(price=stock, volume=volume, col='AAPL')
one_stock = f.trading_window(data=one_stock)
# Do the train - test splitting with normalize data
X_train, y_train, X_test, y_test = f.prepare_data(data=one_stock,
split_fac=0.65)
split = len(y_train)
print(f'Train test split, train = {split}')
# Show some plots
#f.show_plot(X_train, 'Training data')
#f.show_plot(X_test, 'Testing data')
# Do a linear regression with Ridge to avoid overfitting
regression_model = Ridge(alpha=1.2, fit_intercept=False)
#regression_model = LinearRegression()
regression_model.fit(X_train, y_train)
#Ridge(alpha=0.8, copy_X=True, fit_intercept=True, max_iter=None, normalize=False, random_state=None, solver='auto', tol=0.0001)
#----------------------------------------------------------------------------------------------------
# Read config
#------------
config = ConfigParser.ConfigParser()
config.read('keras.cfg')
s_exp = float(config.get('PARAMETERS', 'sig_xsec_times_eff')) * float(
config.get('PARAMETERS', 'lumi'))
b_exp = float(config.get('PARAMETERS', 'bkg_xsec_times_eff')) * float(
config.get('PARAMETERS', 'lumi'))
#----------------------------------------------------------------------------------------------------
# Make train and test dataframes
#-------------------------------
# Split dataset into test and training set
X_train, X_test, Y_train, Y_test = fcn.prepare_data()
#----------------------------------------------------------------------------------------------------
# Define the network (!)
#-----------------------
model = keras.models.Sequential()
model.add(
keras.layers.Dense(23, input_shape=(X_train.shape[1], ),
activation='relu'))
model.add(keras.layers.Dense(2, activation='softmax'))
# Define callbacks
monitor_variable = 'val_loss'
if float(config.get('KERAS', 'validation_split')) == 0.:
monitor_variable = 'loss'
cb = keras.callbacks.EarlyStopping(monitor=monitor_variable,
########################################
print("Choose one of the following vectorization methods: ")
print("* Bag Of Words\n* TF-IDF\n* Word Embeddings\n")
method = input()
# The vectorizer we will be using
if method == "Word Embeddings":
word2vector(df,file_name)
elif method == "Bag Of Words" or method == "TF-IDF":
if method == "Bag Of Words":
vectorizer = CountVectorizer(max_features=1000)
else:
vectorizer = TfidfVectorizer(max_features=1000)
corpus = prepare_data(df)
# To actually create the vectorizer, we simply need to call fit on the text
# Learn the vocabulary dictionary and return term-document matrix.
vector = vectorizer.fit_transform(corpus)
# vectorizer.vocabulary_ is a dictionary. The number of elements it holds is
# the same as the total number of words in our vocabulary. Each key repre-
# sents a word from the vocabulary and its value is that word's index in
# the vector.
# print(vectorizer.vocabulary_,"\n")
# Array mapping from feature integer indices to feature name
# print(vectorizer.get_feature_names())
pkl_path = os.getcwd() + '/pkl/' # the directory where the .pkl will be stored