def upload(): from tensorflow.keras.preprocessing.image import ImageDataGenerator train_datagen = ImageDataGenerator(rescale = 1./255,shear_range = 0.2,zoom_range = 0.2,horizontal_flip = True) test_datagen = ImageDataGenerator(rescale = 1) x_train = train_datagen.flow_from_directory(r'C:\Users\Washifa\Desktop\dataset\train_set',target_size=(64,64),batch_size=32,class_mode='binary') x_test = test_datagen.flow_from_directory(r'C:\Users\Washifa\Desktop\dataset\test_set',target_size=(64,64),batch_size=32,class_mode='binary') print(x_train.class_indices) import tensorflow as tf from tensorflow.python.keras.layers import Dense from tensorflow.keras import Sequential from tensorflow.keras.layers import Convolution2D from tensorflow.keras.layers import MaxPooling2D from tensorflow.keras.layers import Flatten model = Sequential() model.add(Convolution2D(32,(3,3),input_shape = (64,64,3))) model.add(MaxPooling2D(pool_size = (2,2))) model.add(Flatten()) model.add(Dense(units = 128,kernel_initializer = 'uniform')) model.add(Dense(units = 1,kernel_initializer = 'uniform')) model.compile(loss='binary_crossentropy',optimizer = "adam",metrics = ["accuracy"]) model.fit_generator(x_train,validation_data=x_test, steps_per_epoch=10) if request.method == 'POST': f = request.files['image'] print("current path") basepath = os.path.dirname(__file__) print("current path", basepath) filepath = os.path.join(basepath,'uploads\image',f.filename) print("upload folder is ", filepath) f.save(filepath) test_datagen = ImageDataGenerator(rescale = 1./255) vals = ['cancer','no cancer'] print(vals) test_generator = test_datagen.flow_from_directory(r'C:\Users\Washifa\Desktop\pavithra\uploads',target_size =(64,64),class_mode ='categorical',batch_size = 32) print(test_generator) pred = model.predict_generator(test_generator) print(pred) y=str(vals[np.argmax(pred)]) os.remove(filepath) return y
# the use of the Adam optimizer with learning rate 0.0001 opt = keras.optimizers.SGD(lr=0.0001) # setting early stopping earlystop = EarlyStopping(patience=10, monitor='val_loss', restore_best_weights=True) callbacks = [earlystop] # compiling the model model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=["accuracy"]) # fitting the model model.fit(train_gen, validation_data=val_gen, epochs=50, callbacks=callbacks) # saving the model model.save("ConvLSTM_model") # creating predictions with the created model predictions = model.predict_generator(test_gen) # changing the 4 predictions in a list of only the highest predictions y_pred = final_pred(predictions) # creating a list with the true predictions y_true = generate_ytrue(test_paths) # creating the right numbers for the confusion matrix conf_matrix = metrics(y_true, y_pred)
class RECOGNIZE_FLOWERS(): def __init__(self): self.num_epochs = 3 self.batch_size = 40 self.input_height = 500 self.input_width = 500 self.channels = 3 self.input_shape = (self.input_height, self.input_width, self.channels) self.train_dir = "Flower_images/data/train" self.train_files = [] for i in range(0, 200): self.train_files.append(str(i) + ".jpg") self.labels = pd.read_csv("Flower_images/data/train.csv") self.train_labels = self.labels[:280] self.train_labels_dict = { i: j for i, j in zip(self.train_labels["image_id"], self.train_labels["category"]) } self.validation_files = [] for i in range(200, 280): self.validation_files.append(str(i) + ".jpg") self.steps_per_epoch = 5 def buildAndCompileModel(self): self.model = Sequential() self.model.add( Conv2D(16, (3, 3), padding='same', input_shape=self.input_shape)) self.model.add(Activation('relu')) self.model.add(Conv2D(16, (3, 3))) self.model.add(Activation('relu')) self.model.add(Conv2D(16, (3, 3))) self.model.add(Activation('relu')) self.model.add(Flatten()) self.model.add(Dense(256)) self.model.add(Activation('relu')) self.model.add(Dense(102, activation='sigmoid')) self.model.compile(optimizer='adam', loss="binary_crossentropy", metrics=["accuracy"]) def train_model(self): training_generator = self.image_datagenerator(self.train_files) validation_generator = self.image_datagenerator(self.validation_files) self.model.fit_generator(training_generator, steps_per_epoch=self.steps_per_epoch, validation_data=validation_generator, validation_steps=2, epochs=self.num_epochs) self.model.predict_generator(self.validation_generator, verbose=1) def image_datagenerator(self, input_filenames): input_files = [] for i in input_filenames: input_files.append(self.train_dir + "/" + i) counter = 0 random.shuffle(input_files) while True: images = np.zeros( (self.batch_size, self.input_width, self.input_height, 3)) labels = [] if counter + self.batch_size >= len(input_files): counter = 0 for i in range(self.batch_size): img = str(input_files[counter + i]) images[i] = np.array(Image.open(img)) / 255.0 file_number = img.replace("Flower_images/data/train/", "").replace(".jpg", "") labels.append(self.train_labels_dict[int(file_number)]) yield (images, labels) counter += self.batch_size
validation_data=test_gen) ##### PLOTTING LOSS ###### plt.plot(history.history['loss'], label='train') plt.plot(history.history['val_loss'], label='test') plt.legend() plt.show() score = model.evaluate_generator(test_gen, verbose=1) print() print('Test loss:', score[0]) print('Test accuracy:', score[1]) print() ###### RESHAPE ACTUAL DATA ####### actual_train = reshape_actual(train_gen) predictions_train = model.predict_generator(train_gen, verbose=0) ##### RSME FOR TRAIN ##### rmse_train = math.sqrt( mean_squared_error(actual_train[:], predictions_train[:])) print() print(rmse_train) ######PLOT TRAIN###### plot_them_graphs(actual_train, predictions_train, "train", ticker) ###### TEST DATA ###### actual_test = reshape_actual(test_gen) predictions_test = model.predict_generator(test_gen, verbose=0) rmse_test = math.sqrt(mean_squared_error(actual_test[:], predictions_test[:]))
def train(ticker): #Initializing the Paths create_dir(model_dir + '/' + ticker) output = {} present_date = date.today() prev_date = date.today() - timedelta(days=5457) dataset = get_stock_data(ticker, start_date=prev_date, end_date=present_date, drop_na=True).reset_index(drop=True) dataset.to_csv(model_dir + '/' + ticker + '/' + str(ticker) + '.csv', index=False) scaler = MinMaxScaler(feature_range=(0, 1)) scaled = scaler.fit_transform(dataset.values) pickle.dump(scaler, open(model_dir + '/' + ticker + '/scaler.pkl', 'wb')) ##### TRAIN TEST SPLITTING ##### ''' train_gen = TimeseriesGenerator(scaled, scaled[:,0:4], start_index = 0, end_index = int(len(scaled) * 0.90), length = 1, batch_size = 256) test_gen = TimeseriesGenerator(scaled, scaled[:,0:4], start_index = int(len(scaled) * 0.90), end_index = int(len(scaled) - 1), length = 1, batch_size = 256) ''' train_gen = TimeseriesGenerator(scaled, scaled[:, :4], start_index=0, end_index=int(len(scaled) * 0.85), length=5, batch_size=16) test_gen = TimeseriesGenerator(scaled, scaled[:, :4], start_index=int(len(scaled) * 0.85), end_index=int(len(scaled) - 1), length=5, batch_size=16) ##### MODEL CREATION ###### ''' model = Sequential() model.add(Conv1D(18, kernel_size=3, activation='relu', padding = 'valid', strides=1, input_shape=(1,18), data_format='channels_first')) model.add(Conv1D(18, kernel_size=3, activation='relu', padding = 'valid', strides=1)) #model.add(MaxPooling1D(pool_size=2)) model.add(LSTM(100, activation = 'tanh', recurrent_activation = 'sigmoid', unroll = False, use_bias = True, recurrent_dropout = 0, return_sequences=True)) model.add(Dropout(0.4)) model.add(LSTM(100, activation = 'tanh', recurrent_activation = 'sigmoid', unroll = False, use_bias = True, recurrent_dropout = 0,return_sequences=True)) model.add(Dropout(0.4)) model.add(LSTM(100, activation = 'tanh', recurrent_activation = 'sigmoid', unroll = False, use_bias = True, recurrent_dropout = 0,return_sequences= True)) model.add(Dropout(0.2)) model.add(Flatten()) model.add(Dense(4, activation = 'linear')) adadelta = Adadelta(learning_rate=1.0, rho=0.95) model.compile(loss= 'mae', optimizer = adadelta, metrics=[tf.keras.metrics.MeanAbsolutePercentageError()]) #model.summary() ''' from tensorflow.keras.regularizers import l1 model = Sequential() model.add( Conv1D(18, kernel_size=3, activation='relu', padding='valid', strides=1, input_shape=(5, 18), data_format='channels_first')) model.add( Conv1D(18, kernel_size=4, activation='relu', padding='valid', strides=1)) #model.add(MaxPooling1D(pool_size=2)) model.add( LSTM(100, activation='tanh', recurrent_activation='sigmoid', unroll=False, use_bias=True, recurrent_dropout=0, return_sequences=True, kernel_regularizer=l1(0.001))) model.add(Dropout(0.4)) model.add( LSTM(50, activation='tanh', recurrent_activation='sigmoid', unroll=False, use_bias=True, recurrent_dropout=0, return_sequences=True, activity_regularizer=l1(0.001))) model.add(Dropout(0.4)) model.add( LSTM(25, activation='tanh', recurrent_activation='sigmoid', unroll=False, use_bias=True, recurrent_dropout=0, return_sequences=True, activity_regularizer=l1(0.001))) model.add(Dropout(0.2)) model.add(Flatten()) model.add(Dense(4, activation='linear')) adadelta = Adadelta(learning_rate=1.0, rho=0.95) model.compile(loss='mse', optimizer=adadelta, metrics=[tf.keras.metrics.RootMeanSquaredError()]) model.summary() ##### TRAINING ##### my_callbacks = [ #tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', patience=4, factor=0.2, min_lr=0.001), tf.keras.callbacks.ModelCheckpoint( filepath=model_dir + "/" + ticker + "/model" + ".h5", save_weights_only=True, save_best_only=True, monitor="val_root_mean_squared_error", mode="min"), ] history = model.fit_generator(train_gen, epochs=300, verbose=0, shuffle=True, validation_data=test_gen, callbacks=my_callbacks) ##### PLOTTING LOSS ###### '''plt.plot(history.history['loss'], label='train') plt.plot(history.history['val_loss'], label='test') plt.legend() plt.show() score = model.evaluate_generator(test_gen, verbose = 1) print() print('Test loss:', score[0]) print('Test accuracy:', score[1]) print()''' ###### RESHAPE ACTUAL DATA ####### actual_train = reshape_actual(train_gen) predictions_train = model.predict_generator(train_gen, verbose=0) print(predictions_train) ##### RSME FOR TRAIN ##### rmse_train = math.sqrt( mean_squared_error(actual_train[:], predictions_train[:])) print(rmse_train) ###### TEST DATA ###### actual_test = reshape_actual(test_gen) predictions_test = model.predict_generator(test_gen, verbose=0) rmse_test = math.sqrt( mean_squared_error(actual_test[:], predictions_test[:])) print(rmse_test) output["Accuracy"] = { "Train": round(rmse_train * 100, 2), "Test": round(rmse_test * 100, 2) } ###### PLOT TEST ###### output["Train"] = plot_them_graphs(actual_train, predictions_train, "train", ticker, scaler) output["Test"] = plot_them_graphs(actual_test, predictions_test, "test", ticker, scaler) ##### SAVE IT!!!!!! ##### model_json = model.to_json() with open(model_dir + "/" + ticker + "/model" + ".json", "w") as json_file: json_file.write(model_json) #model.save_weights(model_dir + "/" + ticker + "/model" + ".h5") print("Saved model to disk") data = {"name": ticker, "date": present_date.strftime("%d-%b-%Y")} with open(model_dir + "/" + ticker + '/data.json', 'w') as outfile: json.dump(data, outfile) return output
test = pd.read_csv('{}/test.csv'.format(DATA_PATH)) preds = [[0] * 6] * len(test) if os.path.exists('../input/prostate-cancer-grade-assessment/test_images'): subm_datagen = DataGenPanda( imgs_path='{}/test_images'.format(DATA_PATH), df=test, batch_size=1, mode='predict', shuffle=False, aug=None, seq_len=SEQ_LEN, img_size=IMG_SIZE, n_classes=6 ) preds = model.predict_generator(subm_datagen) print('preds done, total:', len(preds)) else: print('preds are zeros') test['isup_grade'] = np.argmax(preds, axis=1) test.drop('data_provider', axis=1, inplace=True) test.to_csv('submission.csv', index=False) print('submission saved') # In[ ]:
]) model.summary() model.compile(optimizer=Adam(lr=0.001), loss='sparse_categorical_crossentropy', metrics=['accuracy']) def scheduler(epoch): if epoch <= 2: return 0.001 elif epoch > 2 and epoch <= 15: return 0.0001 else: return 0.00001 lr_callbacks = tf.keras.callbacks.LearningRateScheduler(scheduler) # from keras.models import load_model # model = load_model('model.h5') model.fit_generator(train, epochs=50, callbacks=[lr_callbacks]) model.save("model.h5") model.evaluate_generator(train) model.predict_generator(train)
class LSTM_model: '''LSTM model''' def __init__(self, config, datasource): self.config = config self.opt = None self.checkpoint = None self.tensorboard = None self.early_stopping = None self.x_train = None self.x_test = None self.y_train = None self.y_test = None self.weights = None self.train_data_gen = None self.test_data_gen = None self.datasource = datasource self.set_params() def set_params(self): '''prepares the hypermarameters to be used by the model''' if self.config == 1: self.model_params = { 'optimizer': Adam, 'epochs': 150, 'learning_rate': 0.001, 'decay': 1e-6, 'dropout': 0.5, 'SEQ_LEN': 90, 'NAME': f"LSTM-{self.config}-\ {time.strftime('%Y-%m-%d %H-%M-%S')}", 'patience': 100, 'lstm_neurons': [256, 256, 128], 'shuffle': True, 'batch_size': 32, 'steps_per_epoch': 50 } elif self.config == 2: self.model_params = { 'optimizer': Adam, 'epochs': 150, 'learning_rate': 0.001, 'decay': 1e-6, 'dropout': 0.5, 'SEQ_LEN': 10, 'NAME': f"LSTM-{self.config}-\ {time.strftime('%Y-%m-%d %H-%M-%S')}", 'patience': 100, 'lstm_neurons': [256, 256, 128], 'shuffle': True, 'batch_size': 32, 'steps_per_epoch': 50 } else: assert 0, "Bad Config creation: " + self.config.name @staticmethod def get_weights(dependent_var): '''calculate classification weights''' classes, cnt = np.unique(dependent_var, return_counts=True, axis=0) classes = classes.astype(int) weights = 1/(cnt/cnt.sum()) weights = weights/weights.sum() return dict(zip(classes, weights)) def add_optimizer(self): '''add optimiser to be used by LSTM''' self.opt = self.model_params['optimizer']( lr=self.model_params['learning_rate'], decay=self.model_params['decay'], clipnorm=1.) def create_model(self): '''create LSTM model''' self.model = Sequential() self.model.add(LSTM(units=self.model_params['lstm_neurons'][0], return_sequences=True, input_shape=(self.model_params['SEQ_LEN'], self.train_generator.df.shape[1]-3))) self.model.add(Dropout(self.model_params['dropout'])) self.model.add(BatchNormalization()) if len(self.model_params['lstm_neurons']) > 1: for i in self.model_params['lstm_neurons'][1:]: self.model.add(LSTM(units=i, return_sequences=True if i != self.model_params['lstm_neurons'][-1] else False)) self.model.add(Dropout(self.model_params['dropout'])) self.model.add(BatchNormalization()) self.model.add(Dense(32, activation='relu')) self.model.add(Dropout(self.model_params['dropout'])) self.model.add(Flatten()) self.model.add(Dense(3, activation='softmax')) self.add_optimizer() self.model.compile(loss='sparse_categorical_crossentropy', optimizer=self.opt, metrics=['accuracy']) self.model.summary() ''' self.tensorboard = TensorBoard(log_dir=f"./LSTM_Models/LSTM_logs/\ {self.model_params['NAME']}")''' # run tensorboard from the console with next comment to follow training # tensorboard --logdir=LSTM_Models/LSTM_logs/ self.checkpoint = ModelCheckpoint('./LSTM_Models/Models/LSTM_T1-Best-' + self.datasource, monitor='val_accuracy', verbose=1, save_weights_only=True, save_best_only=True, mode='max') self.early_stopping =\ EarlyStopping(monitor='val_loss', patience=self.model_params['patience']) def load_model(self): '''loading a trained model''' self.create_model() self.model.load_weights('./LSTM_Models/Models/LSTM_T1-Best-' + self.datasource) self.model.compile(loss='sparse_categorical_crossentropy', optimizer=self.opt, metrics=['accuracy']) def input_data(self): '''timeseries data creation''' self.weights = self.get_weights(self.train_generator.df.iloc[:, -1]) self.ttl_batches = int((len(self.train_generator.df) - self.model_params['SEQ_LEN']) / self.model_params['batch_size']) def train(self, train_gen, validation_gen): '''train on the dataset provided''' self.train_generator = train_gen self.validation_generator = validation_gen self.input_data() self.create_model() history = \ self.model.fit( x=self.train_generator, epochs=self.model_params['epochs'], shuffle=self.model_params['shuffle'], validation_data=self.validation_generator, steps_per_epoch=self.model_params['steps_per_epoch'], validation_steps=self.model_params['steps_per_epoch'], class_weight=self.weights, callbacks=[# self.tensorboard, self.checkpoint, self.early_stopping]) print(history) def test(self, start_date=dt.datetime.today() - dt.timedelta(days=15), end_date=dt.datetime.today()): '''test and return confusion_matrix and classification_report''' self.set_params() data = dataset(self.datasource, max_date=end_date) # TEST data.prepare_test_dataset() self.test_data_gen = datagen(data.sdf, gen_length=self.model_params['SEQ_LEN'], start_date=start_date) self.x_train =\ self.test_data_gen.df.loc[self.test_data_gen.df.date.between( start_date, end_date)].iloc[:, :-1] self.y_train =\ self.test_data_gen.df.loc[self.test_data_gen.df.date.between( start_date, end_date)].iloc[:, -1] self.load_model() y_lstm_pred =\ self.model.predict_generator(self.test_data_gen, steps=len(self.test_data_gen)) y_lstm_pred = np.argmax(y_lstm_pred, axis=1) self.test_data_gen.result = self.test_data_gen.result[:-10] print("LSTM: Predictions have finished") cm_lstm = confusion_matrix(self.test_data_gen.result, y_lstm_pred) o_acc = np.around(np.sum(np.diag(cm_lstm)) / np.sum(cm_lstm)*100, 1) plt.title(f'Confusion Matrix \n Accuracy={o_acc}%', size=18) sn.heatmap(cm_lstm, fmt=".0f", annot=True, cbar=False, annot_kws={"size": 15}, xticklabels=['Sell', 'Hold', 'Buy'], yticklabels=['Sell', 'Hold', 'Buy']) plt.xlabel('Predicted Label', size=15) plt.ylabel('True Label', size=15) print(np.diag(cm_lstm).sum()) cr_lstm = classification_report(self.test_data_gen.result, y_lstm_pred) print(cr_lstm) return cm_lstm, cr_lstm def predict(self, start_date=dt.datetime.today().date(), end_date=dt.datetime.today().date()): self.set_params() data = dataset(self.datasource, min_date=start_date, max_date=end_date) # TEST print("The dataset is", len(data), "datapoints long") data.prepare_test_dataset() self.test_data_gen = datagen(data.sdf, gen_length=self.model_params['SEQ_LEN'], start_date=start_date) print(self.test_data_gen.df.head()) self.x_train =\ self.test_data_gen.df.loc[self.test_data_gen.df.date.between( start_date, end_date)].iloc[:, :-1] print(self.x_train.columns) self.y_train =\ self.test_data_gen.df.loc[self.test_data_gen.df.date.between( start_date, end_date)].iloc[:, -1] self.load_model() y_lstm_pred =\ self.model.predict_generator(self.test_data_gen, steps=len(self.test_data_gen)) return self.test_data_gen.ticks, y_lstm_pred def predict2(self, start_date=dt.datetime.today().date() - dt.timedelta(days=120), end_date=dt.datetime.today().date()): self.set_params() data = dataset(self.datasource, min_date=start_date, max_date=end_date) print("The dataset is", len(data), "datapoints long") data.prepare_test_dataset() self.x_train = data.sdf.drop(['target'], axis=1) self.x_train = np.array(self.x_train.iloc[-90:, :]) self.load_model() predictions = pd.DataFrame(columns=['sell', 'hold', 'buy']) results = pd.DataFrame(columns=['sym', 'date', 'target']) for sym in data.sdf.symbol.unique(): temp = data.sdf.loc[data.sdf.symbol == sym] for date in temp.date[90:]: results =\ results.append({'sym': sym, 'date': date, 'target': temp.loc[temp.date == date, ['adjusted_close']].values[0][0]}, ignore_index=True) self.x_train = temp.loc[temp.date <= date].iloc[-90:, :]\ .drop(['symbol', 'date', 'target'], axis=1) self.x_train = np.array(self.x_train) self.x_train =\ np.reshape(self.x_train, (1, self.x_train.shape[0], self.x_train.shape[1])).astype('float32') predictions = predictions.append(pd.DataFrame( self.model.predict(self.x_train), columns=['sell', 'hold', 'buy']), ignore_index=True) results = pd.concat([results, predictions], axis=1) return results