def upload():
from tensorflow.keras.preprocessing.image import ImageDataGenerator
train_datagen = ImageDataGenerator(rescale = 1./255,shear_range = 0.2,zoom_range = 0.2,horizontal_flip = True)
test_datagen = ImageDataGenerator(rescale = 1)
x_train = train_datagen.flow_from_directory(r'C:\Users\Washifa\Desktop\dataset\train_set',target_size=(64,64),batch_size=32,class_mode='binary')
x_test = test_datagen.flow_from_directory(r'C:\Users\Washifa\Desktop\dataset\test_set',target_size=(64,64),batch_size=32,class_mode='binary')
print(x_train.class_indices)
import tensorflow as tf
from tensorflow.python.keras.layers import Dense
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Convolution2D
from tensorflow.keras.layers import MaxPooling2D
from tensorflow.keras.layers import Flatten
model = Sequential()
model.add(Convolution2D(32,(3,3),input_shape = (64,64,3)))
model.add(MaxPooling2D(pool_size = (2,2)))
model.add(Flatten())
model.add(Dense(units = 128,kernel_initializer = 'uniform'))
model.add(Dense(units = 1,kernel_initializer = 'uniform'))
model.compile(loss='binary_crossentropy',optimizer = "adam",metrics = ["accuracy"])
model.fit_generator(x_train,validation_data=x_test, steps_per_epoch=10)
if request.method == 'POST':
f = request.files['image']
print("current path")
basepath = os.path.dirname(__file__)
print("current path", basepath)
filepath = os.path.join(basepath,'uploads\image',f.filename)
print("upload folder is ", filepath)
f.save(filepath)
test_datagen = ImageDataGenerator(rescale = 1./255)
vals = ['cancer','no cancer']
print(vals)
test_generator = test_datagen.flow_from_directory(r'C:\Users\Washifa\Desktop\pavithra\uploads',target_size =(64,64),class_mode ='categorical',batch_size = 32)
print(test_generator)
pred = model.predict_generator(test_generator)
print(pred)
y=str(vals[np.argmax(pred)])
os.remove(filepath)
return y
# the use of the Adam optimizer with learning rate 0.0001
opt = keras.optimizers.SGD(lr=0.0001)
# setting early stopping
earlystop = EarlyStopping(patience=10,
monitor='val_loss',
restore_best_weights=True)
callbacks = [earlystop]
# compiling the model
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=["accuracy"])
# fitting the model
model.fit(train_gen, validation_data=val_gen, epochs=50, callbacks=callbacks)
# saving the model
model.save("ConvLSTM_model")
# creating predictions with the created model
predictions = model.predict_generator(test_gen)
# changing the 4 predictions in a list of only the highest predictions
y_pred = final_pred(predictions)
# creating a list with the true predictions
y_true = generate_ytrue(test_paths)
# creating the right numbers for the confusion matrix
conf_matrix = metrics(y_true, y_pred)
class RECOGNIZE_FLOWERS():
def __init__(self):
self.num_epochs = 3
self.batch_size = 40
self.input_height = 500
self.input_width = 500
self.channels = 3
self.input_shape = (self.input_height, self.input_width, self.channels)
self.train_dir = "Flower_images/data/train"
self.train_files = []
for i in range(0, 200):
self.train_files.append(str(i) + ".jpg")
self.labels = pd.read_csv("Flower_images/data/train.csv")
self.train_labels = self.labels[:280]
self.train_labels_dict = {
i: j
for i, j in zip(self.train_labels["image_id"],
self.train_labels["category"])
}
self.validation_files = []
for i in range(200, 280):
self.validation_files.append(str(i) + ".jpg")
self.steps_per_epoch = 5
def buildAndCompileModel(self):
self.model = Sequential()
self.model.add(
Conv2D(16, (3, 3), padding='same', input_shape=self.input_shape))
self.model.add(Activation('relu'))
self.model.add(Conv2D(16, (3, 3)))
self.model.add(Activation('relu'))
self.model.add(Conv2D(16, (3, 3)))
self.model.add(Activation('relu'))
self.model.add(Flatten())
self.model.add(Dense(256))
self.model.add(Activation('relu'))
self.model.add(Dense(102, activation='sigmoid'))
self.model.compile(optimizer='adam',
loss="binary_crossentropy",
metrics=["accuracy"])
def train_model(self):
training_generator = self.image_datagenerator(self.train_files)
validation_generator = self.image_datagenerator(self.validation_files)
self.model.fit_generator(training_generator,
steps_per_epoch=self.steps_per_epoch,
validation_data=validation_generator,
validation_steps=2,
epochs=self.num_epochs)
self.model.predict_generator(self.validation_generator, verbose=1)
def image_datagenerator(self, input_filenames):
input_files = []
for i in input_filenames:
input_files.append(self.train_dir + "/" + i)
counter = 0
random.shuffle(input_files)
while True:
images = np.zeros(
(self.batch_size, self.input_width, self.input_height, 3))
labels = []
if counter + self.batch_size >= len(input_files):
counter = 0
for i in range(self.batch_size):
img = str(input_files[counter + i])
images[i] = np.array(Image.open(img)) / 255.0
file_number = img.replace("Flower_images/data/train/",
"").replace(".jpg", "")
labels.append(self.train_labels_dict[int(file_number)])
yield (images, labels)
counter += self.batch_size
validation_data=test_gen)
##### PLOTTING LOSS ######
plt.plot(history.history['loss'], label='train')
plt.plot(history.history['val_loss'], label='test')
plt.legend()
plt.show()
score = model.evaluate_generator(test_gen, verbose=1)
print()
print('Test loss:', score[0])
print('Test accuracy:', score[1])
print()
###### RESHAPE ACTUAL DATA #######
actual_train = reshape_actual(train_gen)
predictions_train = model.predict_generator(train_gen, verbose=0)
##### RSME FOR TRAIN #####
rmse_train = math.sqrt(
mean_squared_error(actual_train[:], predictions_train[:]))
print()
print(rmse_train)
######PLOT TRAIN######
plot_them_graphs(actual_train, predictions_train, "train", ticker)
###### TEST DATA ######
actual_test = reshape_actual(test_gen)
predictions_test = model.predict_generator(test_gen, verbose=0)
rmse_test = math.sqrt(mean_squared_error(actual_test[:], predictions_test[:]))
def train(ticker):
#Initializing the Paths
create_dir(model_dir + '/' + ticker)
output = {}
present_date = date.today()
prev_date = date.today() - timedelta(days=5457)
dataset = get_stock_data(ticker,
start_date=prev_date,
end_date=present_date,
drop_na=True).reset_index(drop=True)
dataset.to_csv(model_dir + '/' + ticker + '/' + str(ticker) + '.csv',
index=False)
scaler = MinMaxScaler(feature_range=(0, 1))
scaled = scaler.fit_transform(dataset.values)
pickle.dump(scaler, open(model_dir + '/' + ticker + '/scaler.pkl', 'wb'))
##### TRAIN TEST SPLITTING #####
'''
train_gen = TimeseriesGenerator(scaled, scaled[:,0:4], start_index = 0, end_index = int(len(scaled) * 0.90), length = 1, batch_size = 256)
test_gen = TimeseriesGenerator(scaled, scaled[:,0:4], start_index = int(len(scaled) * 0.90), end_index = int(len(scaled) - 1), length = 1, batch_size = 256)
'''
train_gen = TimeseriesGenerator(scaled,
scaled[:, :4],
start_index=0,
end_index=int(len(scaled) * 0.85),
length=5,
batch_size=16)
test_gen = TimeseriesGenerator(scaled,
scaled[:, :4],
start_index=int(len(scaled) * 0.85),
end_index=int(len(scaled) - 1),
length=5,
batch_size=16)
##### MODEL CREATION ######
'''
model = Sequential()
model.add(Conv1D(18, kernel_size=3, activation='relu', padding = 'valid', strides=1, input_shape=(1,18), data_format='channels_first'))
model.add(Conv1D(18, kernel_size=3, activation='relu', padding = 'valid', strides=1))
#model.add(MaxPooling1D(pool_size=2))
model.add(LSTM(100, activation = 'tanh', recurrent_activation = 'sigmoid', unroll = False, use_bias = True, recurrent_dropout = 0, return_sequences=True))
model.add(Dropout(0.4))
model.add(LSTM(100, activation = 'tanh', recurrent_activation = 'sigmoid', unroll = False, use_bias = True, recurrent_dropout = 0,return_sequences=True))
model.add(Dropout(0.4))
model.add(LSTM(100, activation = 'tanh', recurrent_activation = 'sigmoid', unroll = False, use_bias = True, recurrent_dropout = 0,return_sequences= True))
model.add(Dropout(0.2))
model.add(Flatten())
model.add(Dense(4, activation = 'linear'))
adadelta = Adadelta(learning_rate=1.0, rho=0.95)
model.compile(loss= 'mae', optimizer = adadelta, metrics=[tf.keras.metrics.MeanAbsolutePercentageError()])
#model.summary()
'''
from tensorflow.keras.regularizers import l1
model = Sequential()
model.add(
Conv1D(18,
kernel_size=3,
activation='relu',
padding='valid',
strides=1,
input_shape=(5, 18),
data_format='channels_first'))
model.add(
Conv1D(18,
kernel_size=4,
activation='relu',
padding='valid',
strides=1))
#model.add(MaxPooling1D(pool_size=2))
model.add(
LSTM(100,
activation='tanh',
recurrent_activation='sigmoid',
unroll=False,
use_bias=True,
recurrent_dropout=0,
return_sequences=True,
kernel_regularizer=l1(0.001)))
model.add(Dropout(0.4))
model.add(
LSTM(50,
activation='tanh',
recurrent_activation='sigmoid',
unroll=False,
use_bias=True,
recurrent_dropout=0,
return_sequences=True,
activity_regularizer=l1(0.001)))
model.add(Dropout(0.4))
model.add(
LSTM(25,
activation='tanh',
recurrent_activation='sigmoid',
unroll=False,
use_bias=True,
recurrent_dropout=0,
return_sequences=True,
activity_regularizer=l1(0.001)))
model.add(Dropout(0.2))
model.add(Flatten())
model.add(Dense(4, activation='linear'))
adadelta = Adadelta(learning_rate=1.0, rho=0.95)
model.compile(loss='mse',
optimizer=adadelta,
metrics=[tf.keras.metrics.RootMeanSquaredError()])
model.summary()
##### TRAINING #####
my_callbacks = [
#tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', patience=4, factor=0.2, min_lr=0.001),
tf.keras.callbacks.ModelCheckpoint(
filepath=model_dir + "/" + ticker + "/model" + ".h5",
save_weights_only=True,
save_best_only=True,
monitor="val_root_mean_squared_error",
mode="min"),
]
history = model.fit_generator(train_gen,
epochs=300,
verbose=0,
shuffle=True,
validation_data=test_gen,
callbacks=my_callbacks)
##### PLOTTING LOSS ######
'''plt.plot(history.history['loss'], label='train')
plt.plot(history.history['val_loss'], label='test')
plt.legend()
plt.show()
score = model.evaluate_generator(test_gen, verbose = 1)
print()
print('Test loss:', score[0])
print('Test accuracy:', score[1])
print()'''
###### RESHAPE ACTUAL DATA #######
actual_train = reshape_actual(train_gen)
predictions_train = model.predict_generator(train_gen, verbose=0)
print(predictions_train)
##### RSME FOR TRAIN #####
rmse_train = math.sqrt(
mean_squared_error(actual_train[:], predictions_train[:]))
print(rmse_train)
###### TEST DATA ######
actual_test = reshape_actual(test_gen)
predictions_test = model.predict_generator(test_gen, verbose=0)
rmse_test = math.sqrt(
mean_squared_error(actual_test[:], predictions_test[:]))
print(rmse_test)
output["Accuracy"] = {
"Train": round(rmse_train * 100, 2),
"Test": round(rmse_test * 100, 2)
}
###### PLOT TEST ######
output["Train"] = plot_them_graphs(actual_train, predictions_train,
"train", ticker, scaler)
output["Test"] = plot_them_graphs(actual_test, predictions_test, "test",
ticker, scaler)
##### SAVE IT!!!!!! #####
model_json = model.to_json()
with open(model_dir + "/" + ticker + "/model" + ".json", "w") as json_file:
json_file.write(model_json)
#model.save_weights(model_dir + "/" + ticker + "/model" + ".h5")
print("Saved model to disk")
data = {"name": ticker, "date": present_date.strftime("%d-%b-%Y")}
with open(model_dir + "/" + ticker + '/data.json', 'w') as outfile:
json.dump(data, outfile)
return output
test = pd.read_csv('{}/test.csv'.format(DATA_PATH))
preds = [[0] * 6] * len(test)
if os.path.exists('../input/prostate-cancer-grade-assessment/test_images'):
subm_datagen = DataGenPanda(
imgs_path='{}/test_images'.format(DATA_PATH),
df=test,
batch_size=1,
mode='predict',
shuffle=False,
aug=None,
seq_len=SEQ_LEN,
img_size=IMG_SIZE,
n_classes=6
)
preds = model.predict_generator(subm_datagen)
print('preds done, total:', len(preds))
else:
print('preds are zeros')
test['isup_grade'] = np.argmax(preds, axis=1)
test.drop('data_provider', axis=1, inplace=True)
test.to_csv('submission.csv', index=False)
print('submission saved')
# In[ ]:
])
model.summary()
model.compile(optimizer=Adam(lr=0.001),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
def scheduler(epoch):
if epoch <= 2:
return 0.001
elif epoch > 2 and epoch <= 15:
return 0.0001
else:
return 0.00001
lr_callbacks = tf.keras.callbacks.LearningRateScheduler(scheduler)
# from keras.models import load_model
# model = load_model('model.h5')
model.fit_generator(train,
epochs=50,
callbacks=[lr_callbacks])
model.save("model.h5")
model.evaluate_generator(train)
model.predict_generator(train)
class LSTM_model:
'''LSTM model'''
def __init__(self, config, datasource):
self.config = config
self.opt = None
self.checkpoint = None
self.tensorboard = None
self.early_stopping = None
self.x_train = None
self.x_test = None
self.y_train = None
self.y_test = None
self.weights = None
self.train_data_gen = None
self.test_data_gen = None
self.datasource = datasource
self.set_params()
def set_params(self):
'''prepares the hypermarameters to be used by the model'''
if self.config == 1:
self.model_params = {
'optimizer': Adam,
'epochs': 150,
'learning_rate': 0.001,
'decay': 1e-6,
'dropout': 0.5,
'SEQ_LEN': 90,
'NAME': f"LSTM-{self.config}-\
{time.strftime('%Y-%m-%d %H-%M-%S')}",
'patience': 100,
'lstm_neurons': [256, 256, 128],
'shuffle': True,
'batch_size': 32,
'steps_per_epoch': 50
}
elif self.config == 2:
self.model_params = {
'optimizer': Adam,
'epochs': 150,
'learning_rate': 0.001,
'decay': 1e-6,
'dropout': 0.5,
'SEQ_LEN': 10,
'NAME': f"LSTM-{self.config}-\
{time.strftime('%Y-%m-%d %H-%M-%S')}",
'patience': 100,
'lstm_neurons': [256, 256, 128],
'shuffle': True,
'batch_size': 32,
'steps_per_epoch': 50
}
else:
assert 0, "Bad Config creation: " + self.config.name
@staticmethod
def get_weights(dependent_var):
'''calculate classification weights'''
classes, cnt = np.unique(dependent_var, return_counts=True,
axis=0)
classes = classes.astype(int)
weights = 1/(cnt/cnt.sum())
weights = weights/weights.sum()
return dict(zip(classes, weights))
def add_optimizer(self):
'''add optimiser to be used by LSTM'''
self.opt = self.model_params['optimizer'](
lr=self.model_params['learning_rate'],
decay=self.model_params['decay'], clipnorm=1.)
def create_model(self):
'''create LSTM model'''
self.model = Sequential()
self.model.add(LSTM(units=self.model_params['lstm_neurons'][0],
return_sequences=True,
input_shape=(self.model_params['SEQ_LEN'],
self.train_generator.df.shape[1]-3)))
self.model.add(Dropout(self.model_params['dropout']))
self.model.add(BatchNormalization())
if len(self.model_params['lstm_neurons']) > 1:
for i in self.model_params['lstm_neurons'][1:]:
self.model.add(LSTM(units=i, return_sequences=True if i !=
self.model_params['lstm_neurons'][-1]
else False))
self.model.add(Dropout(self.model_params['dropout']))
self.model.add(BatchNormalization())
self.model.add(Dense(32, activation='relu'))
self.model.add(Dropout(self.model_params['dropout']))
self.model.add(Flatten())
self.model.add(Dense(3, activation='softmax'))
self.add_optimizer()
self.model.compile(loss='sparse_categorical_crossentropy',
optimizer=self.opt, metrics=['accuracy'])
self.model.summary()
'''
self.tensorboard = TensorBoard(log_dir=f"./LSTM_Models/LSTM_logs/\
{self.model_params['NAME']}")'''
# run tensorboard from the console with next comment to follow training
# tensorboard --logdir=LSTM_Models/LSTM_logs/
self.checkpoint = ModelCheckpoint('./LSTM_Models/Models/LSTM_T1-Best-'
+ self.datasource,
monitor='val_accuracy', verbose=1,
save_weights_only=True,
save_best_only=True, mode='max')
self.early_stopping =\
EarlyStopping(monitor='val_loss',
patience=self.model_params['patience'])
def load_model(self):
'''loading a trained model'''
self.create_model()
self.model.load_weights('./LSTM_Models/Models/LSTM_T1-Best-'
+ self.datasource)
self.model.compile(loss='sparse_categorical_crossentropy',
optimizer=self.opt, metrics=['accuracy'])
def input_data(self):
'''timeseries data creation'''
self.weights = self.get_weights(self.train_generator.df.iloc[:, -1])
self.ttl_batches = int((len(self.train_generator.df) -
self.model_params['SEQ_LEN']) /
self.model_params['batch_size'])
def train(self, train_gen, validation_gen):
'''train on the dataset provided'''
self.train_generator = train_gen
self.validation_generator = validation_gen
self.input_data()
self.create_model()
history = \
self.model.fit(
x=self.train_generator,
epochs=self.model_params['epochs'],
shuffle=self.model_params['shuffle'],
validation_data=self.validation_generator,
steps_per_epoch=self.model_params['steps_per_epoch'],
validation_steps=self.model_params['steps_per_epoch'],
class_weight=self.weights,
callbacks=[# self.tensorboard,
self.checkpoint,
self.early_stopping])
print(history)
def test(self, start_date=dt.datetime.today() -
dt.timedelta(days=15), end_date=dt.datetime.today()):
'''test and return confusion_matrix and classification_report'''
self.set_params()
data = dataset(self.datasource,
max_date=end_date) # TEST
data.prepare_test_dataset()
self.test_data_gen = datagen(data.sdf,
gen_length=self.model_params['SEQ_LEN'],
start_date=start_date)
self.x_train =\
self.test_data_gen.df.loc[self.test_data_gen.df.date.between(
start_date, end_date)].iloc[:, :-1]
self.y_train =\
self.test_data_gen.df.loc[self.test_data_gen.df.date.between(
start_date, end_date)].iloc[:, -1]
self.load_model()
y_lstm_pred =\
self.model.predict_generator(self.test_data_gen,
steps=len(self.test_data_gen))
y_lstm_pred = np.argmax(y_lstm_pred, axis=1)
self.test_data_gen.result = self.test_data_gen.result[:-10]
print("LSTM: Predictions have finished")
cm_lstm = confusion_matrix(self.test_data_gen.result,
y_lstm_pred)
o_acc = np.around(np.sum(np.diag(cm_lstm)) / np.sum(cm_lstm)*100, 1)
plt.title(f'Confusion Matrix \n Accuracy={o_acc}%', size=18)
sn.heatmap(cm_lstm, fmt=".0f", annot=True, cbar=False,
annot_kws={"size": 15}, xticklabels=['Sell', 'Hold', 'Buy'],
yticklabels=['Sell', 'Hold', 'Buy'])
plt.xlabel('Predicted Label', size=15)
plt.ylabel('True Label', size=15)
print(np.diag(cm_lstm).sum())
cr_lstm = classification_report(self.test_data_gen.result,
y_lstm_pred)
print(cr_lstm)
return cm_lstm, cr_lstm
def predict(self, start_date=dt.datetime.today().date(),
end_date=dt.datetime.today().date()):
self.set_params()
data = dataset(self.datasource,
min_date=start_date, max_date=end_date) # TEST
print("The dataset is", len(data), "datapoints long")
data.prepare_test_dataset()
self.test_data_gen = datagen(data.sdf,
gen_length=self.model_params['SEQ_LEN'],
start_date=start_date)
print(self.test_data_gen.df.head())
self.x_train =\
self.test_data_gen.df.loc[self.test_data_gen.df.date.between(
start_date, end_date)].iloc[:, :-1]
print(self.x_train.columns)
self.y_train =\
self.test_data_gen.df.loc[self.test_data_gen.df.date.between(
start_date, end_date)].iloc[:, -1]
self.load_model()
y_lstm_pred =\
self.model.predict_generator(self.test_data_gen,
steps=len(self.test_data_gen))
return self.test_data_gen.ticks, y_lstm_pred
def predict2(self, start_date=dt.datetime.today().date() -
dt.timedelta(days=120),
end_date=dt.datetime.today().date()):
self.set_params()
data = dataset(self.datasource,
min_date=start_date, max_date=end_date)
print("The dataset is", len(data), "datapoints long")
data.prepare_test_dataset()
self.x_train = data.sdf.drop(['target'], axis=1)
self.x_train = np.array(self.x_train.iloc[-90:, :])
self.load_model()
predictions = pd.DataFrame(columns=['sell', 'hold', 'buy'])
results = pd.DataFrame(columns=['sym', 'date', 'target'])
for sym in data.sdf.symbol.unique():
temp = data.sdf.loc[data.sdf.symbol == sym]
for date in temp.date[90:]:
results =\
results.append({'sym': sym, 'date': date,
'target':
temp.loc[temp.date == date,
['adjusted_close']].values[0][0]},
ignore_index=True)
self.x_train = temp.loc[temp.date <= date].iloc[-90:, :]\
.drop(['symbol', 'date', 'target'], axis=1)
self.x_train = np.array(self.x_train)
self.x_train =\
np.reshape(self.x_train,
(1, self.x_train.shape[0],
self.x_train.shape[1])).astype('float32')
predictions = predictions.append(pd.DataFrame(
self.model.predict(self.x_train),
columns=['sell', 'hold', 'buy']), ignore_index=True)
results = pd.concat([results, predictions], axis=1)
return results
