def train(): slices = s3.list_slices() ids = list(labels.keys()) np.random.shuffle(ids) split = int(math.ceil(len(ids) * 0.7)) train = segment(slices, ids[:split]) test = segment(slices, ids[split:]) training_generator = generate(train) validation_generator = generate(test) s3_upload = UploadCheckpoint() early_stop = EarlyStopping(patience=EARLY_STOPPING_PATIENCE) filepath = 'models/' + MODEL_NAME checkpoint = ModelCheckpoint(filepath, monitor='val_acc', save_best_only=True, mode='max') callbacks_list = [checkpoint, s3_upload, early_stop] model = cnn() model.fit_generator(generator=training_generator, steps_per_epoch=len(train) // BATCH_SIZE, validation_data=validation_generator, validation_steps=len(test) // BATCH_SIZE, epochs=EPOCHS, callbacks=callbacks_list)
def _setup_cnn(self, X, X_seq_len, is_training): """ Sets up the convolutional layers. Args: X: input sequence X_seq_len: inputs' sequence lengths is_training: whether the model is training, a boolean Returns: outputs_squeezed: the layer's outputs shaped [batch_size, height, width] new_seq_len: shaped [batch_size] which are - either the new sequence lengths (after taking into account subsampling) if config.use_dynamic_lengths is True - or the old sequence lengths """ X_expanded = tf.expand_dims(X, axis=3) with tf.variable_scope("cnn"): cnn = CNN(self.config) output, new_seq_len = cnn(X_expanded, X_seq_len, is_training) output_squeezed = tf.squeeze(output, axis=2) return output_squeezed, new_seq_len
def _setup_cnn(self, X, X_seq_len, is_training): X_expanded = tf.expand_dims(X, axis=3) with tf.variable_scope("cnn"): cnn = CNN(self.config) output, new_seq_len = cnn(X_expanded, X_seq_len, is_training) output_squeezed = tf.squeeze(output, axis=2) return output_squeezed, new_seq_len
def initialize_model(TIME_PERIODS, input_shape, num_sensors, num_classes, x_train, y_train_hot, x_test, y_train, y_test): model = models.cnn(time_periods=TIME_PERIODS, input_shape=input_shape, number_of_sensors=num_sensors, number_of_classes=num_classes, optimizer='adam', filters=100, kernel_size=3) callbacks_list = [ tf.keras.callbacks.ModelCheckpoint( filepath='../best_models/best_model.{epoch:02d}-{val_loss:.2f}.h5', monitor='val_loss', save_best_only=True), tf.keras.callbacks.EarlyStopping(monitor='acc', patience=1) ] # TODO Train the model # Enable validation to use ModelCheckpoint and EarlyStopping callbacks. history = model.fit(x_train, y_train_hot, batch_size=80, epochs=50, callbacks=callbacks_list, validation_split=0.2, verbose=1) plots.show_model_history(history) # plot metrics plt.plot(history.history['acc']) plt.show() plt.plot(history.history['mean_squared_error']) plt.show() y_pred_train, max_y_pred_train = train_model(model, x_train, y_train) y_pred_test, max_y_pred_test = test_model(model, x_test, y_test) return y_pred_train, max_y_pred_train, y_pred_test, max_y_pred_test
[input_data_scaled_no_dummies, input_data_dummy], axis=1) input_data, _ = get_cnn_input_output(input_data, np.zeros_like(input_data), time_steps=time_steps) price_data = price_data[-len(input_data):] # split to train,test and cross validation input_train, input_test, input_cv, price_train, price_test, price_cv = \ train_test_validation_split([input_data, price_data], split=split) # get dims _, input_dim, _, _ = np.shape(input_train) # forward-propagation x, y, logits, y_, learning_r, drop_out = cnn(input_dim, 3, time_steps=time_steps, filter=[1, 1]) # tf cost and optimizer price_h = tf.placeholder(tf.float32, [None, 1]) signals = tf.constant([[1., -1., 0.]]) cost = (tf.reduce_mean(y_ * signals * price_h * 100)) # profit function train_step = tf.train.AdamOptimizer(learning_r).minimize(-cost) # init session cost_hist_train, cost_hist_test, value_hist_train, value_hist_test, value_hist_cv, value_hist_train_ma, \ value_hist_test_ma, value_hist_cv_ma, step, step_hist, saving_score = [ ], [], [], [], [], [], [], [], 0, [], 0.05 saver = tf.train.Saver() init = tf.global_variables_initializer() sess = tf.Session()
#model.compile(loss=keras.losses.categorical_crossentropy, # optimizer=keras.optimizers.Adam(), # metrics=['accuracy']) class AccuracyHistory(keras.callbacks.Callback): def on_train_begin(self, logs={}): self.acc = [] def on_epoch_end(self, batch, logs={}): self.acc.append(logs.get('acc')) history = AccuracyHistory() model = models.cnn() model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_validation, y_validation), callbacks=[history]) score = model.evaluate(x_validation, y_validation, verbose=0) print('Test loss:', score[0]) print('Test accuracy:', score[1]) plt.plot(range(1, epochs+1), history.acc) plt.xlabel('Epochs') plt.ylabel('Accuracy') plt.show()
def train_step(train_data_gen, num_epochs, num_stock_size, num_iters): if hp._gpu: physical_devices = tf.config.experimental.list_physical_devices('GPU') if len(physical_devices) > 0: for k in range(len(physical_devices)): tf.config.experimental.set_memory_growth( physical_devices[k], True) print( 'memory growth:', tf.config.experimental.get_memory_growth( physical_devices[k])) else: print("Not enough GPU hardware devices available") M = hp.M optimizer = tf.keras.optimizers.Adam(learning_rate=hp.lr) if hp.model_type == "dnn": model = models.dnn() elif hp.model_type == "cnn": model = models.cnn() elif hp.model_type == "rnn": model = models.rnn() elif hp.model_type == "cond_dnn": model = models.cond_dnn() else: print("model type error") if hp._task == "class": loss_fn = tf.keras.losses.BinaryCrossentropy() else: loss_fn = tf.keras.losses.MeanSquaredError() #model.summary() train_losses = [] val_losses = [] for epoch in range(num_epochs): epoch_loss_avg = tf.keras.metrics.Mean() val_loss_avg = tf.keras.metrics.Mean() for i in range(num_stock_size): data_ = next(train_data_gen) #data_[M+N] train_data = data_[:M, :] test_data = data_[M:, :] for j in range(num_iters): x_batch, y_batch = make_batch(train_data, hp.n_features) with tf.GradientTape() as tape: y_ = model(x_batch) loss_value = loss_fn(y_batch, y_) grads = tape.gradient(loss_value, model.trainable_variables) optimizer.apply_gradients(zip(grads, model.trainable_variables)) epoch_loss_avg(loss_value) num_test_iters = num_iters // 4 for j in range(num_test_iters): x_batch, y_batch = make_batch(test_data, hp.n_features, _test=True) y_ = model(x_batch) val_loss_value = loss_fn(y_batch, y_) val_loss_avg(val_loss_value) train_losses.append(loss_value) val_losses.append(val_loss_value) print( "Epoch {:03d}: , Number of stock time {:03d}, Loss: {:.5f}".format( epoch, i, epoch_loss_avg.result())) print("Val_Loss: {:.3f}".format(val_loss_avg.result())) if hp.b_loss_plot: losses = [train_losses, val_losses] utils.plot_loss(losses) return model
output_data, time_steps=time_steps) price_data = price_data[-len(input_data):] # split to train, test and cross validation input_train, input_test, input_cv, output_train, output_test, output_cv, price_train, price_test, price_cv = \ train_test_validation_split( [input_data, output_data, price_data], split=split) # get dims _, input_dim, _, _ = np.shape(input_data) _, output_dim = np.shape(output_data) # forward-propagation x, y, logits, y_, learning_r, drop_out = cnn(input_dim, output_dim, time_steps=time_steps, filter=[3, 6]) # tf cost and optimizer cost = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y)) train_step = tf.train.AdamOptimizer(learning_r).minimize(cost) # init session cost_hist_train, cost_hist_test, value_hist_train, value_hist_test, value_hist_cv, value_hist_train_ma, \ value_hist_test_ma, value_hist_cv_ma, step, step_hist, saving_score = [ ], [], [], [], [], [], [], [], 0, [], 0.05 saver = tf.train.Saver() init = tf.global_variables_initializer() sess = tf.Session() sess.run(init)
Y = train[target_columns] embedding_matrix, X, X_ = tp.embedded_glove_matrix(train, test, 'comment_text', glove_path, embed_size, max_features, max_length) inp = Input(shape=(max_length, )) x = None if model_type == 'lstm-cnn': x = models.lstm_cnn(max_features, embed_size, embedding_matrix, inp) elif model_type == 'lstm': x = models.lstm(max_features, embed_size, embedding_matrix, inp) elif model_type == 'cnn': x = models.cnn(max_features, embed_size, embedding_matrix, inp) model = Model(inputs=inp, outputs=x) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) model.fit(X, Y, batch_size=batch_size, epochs=epochs) if save_weights: model.save_weights('checkpoint.csv') sub = model.predict(X_) temp = test.copy() temp[target_columns] = model.predict(X_) temp.to_csv('output.csv', index=False)
client_headers = { "content-type": "binary tensor", "site": SITE, "mode": MODE, "val_type": None, # determined later in code "step": None, # determined later in code } if MODE == "federated": client_headers["val_type"] = "gradients" elif MODE == "weightavg" or MODE == "cyclic": client_headers["val_type"] = "weights" #################### MODEL #################### model = cnn(k_init, n_channels=1, n_classes=10) model.save_weights(str(WEIGHT_DIR / "init_weights.h5")) with open(str(MODEL_PATH), 'w') as f: json.dump(model.to_json(), f) opt = tf.optimizers.Adam(learning_rate=LEARNING_RATE) # set up metrics train_acc = tf.keras.metrics.Accuracy(name="train_acc") train_loss = tf.keras.metrics.Mean(name="train_loss") val_acc = tf.keras.metrics.Accuracy(name="val_acc") val_loss = tf.keras.metrics.Mean(name="val_loss") #################### LOAD DATA ####################
use_augm=params.use_augm) ##samp_per_epoch= params.samp_per_epoch ## validation_generator = DataGenerator(image_tr, labels_val, coords_val, index_val, params.channels, ## params.patch_size, params.batch_size, dim, shuffle=True) # validation_generator = DataGenerator(image_tr, labels_val, coords_val, index_val, params.channels, # params.patch_size, params.batch_size, dim, # samp_per_epoch = params.samp_epoch_val, shuffle=True, # use_augm = params.use_augm) # Define optimazer optimizer = Adam(lr=params.learning_rate) # Define model if params.model == "custom": model = cnn(img_shape=dim, nb_classes=params.classes) else: model = DeepLabVersions(dim, params) ##plot_model(model, to_file=os.path.join(model_k,'model.png'), show_shapes=True) cl_ind = [x for x in range(params.classes)] losses = { "cl_output": categorical_focal_loss(depth=np.int(params.classes + 1), alpha=[ratio.tolist()], class_indexes=cl_ind) } #losses = {"cl_output": categorical_focal_ignoring_last_label(alpha=0.25,gamma=2)}
def get_model(input_shape): return cnn(input_shape=input_shape)
def main(): args = parse_args() ## Create an output dir output_dir_path = args.od + args.en if not os.path.exists(output_dir_path): os.makedirs(output_dir_path) dir_name=output_dir_path tb_dirname = output_dir_path + '/tb_logdir' else: counter=1 dir_name = output_dir_path new_dir_name = dir_name while os.path.exists(new_dir_name): new_dir_name = dir_name + "_" + str(counter) counter +=1 os.makedirs(new_dir_name) dir_name=new_dir_name tb_dirname = dir_name + "/tb_logdir" print("===>> Output folder = {}".format(dir_name)) args.cuda = not args.no_cuda and torch.cuda.is_available() torch.manual_seed(args.seed) if args.cuda: torch.backends.cudnn.benchmark = True torch.cuda.manual_seed(args.seed) loaders = Cifar10Loaders() train_loader = loaders.train_loader() test_loader = loaders.test_loader() if args.netqat: base_model = cnn() model=cnn(qat_mode=True) ## Loading checkpoints here checkpoint = torch.load(args.load_ckpt) ##making sure that the checkpoint was loaded from disk correctly base_model.load_state_dict(checkpoint['model_state_dict'],strict=True) base_model_sd = base_model.state_dict() ##renaming the keys only to match the model with qat nodes. Only convs and BNs's will be changed base_model_sd_new = map_ckpt_names(base_model_sd) ##Updating the values of keys for qat model for k in base_model_sd_new.keys(): try: model.state_dict()[k].copy_(base_model_sd_new[k]) print("{} successfully loaded".format(k)) except: print("{} didnt load".format(k)) else: model=cnn() ## Instantiate tensorboard logs writer = SummaryWriter(tb_dirname) images, labels = iter(train_loader).next() img_grid = torchvision.utils.make_grid(images) writer.add_image('cifar-10', img_grid) #writer.add_graph(model,images) if args.cuda: model = model.cuda() if args.parallel: model = nn.DataParallel(model,device_ids=range(torch.cuda.device_count())) criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.wd) best_test_accuracy=0 print("===>> Training started") optimizer.load_state_dict(checkpoint['optimizer_state_dict']) epoch = checkpoint['epoch'] loss = checkpoint['loss'] print("===>>> Checkpoint loaded successfully from {} at epoch {} ".format(args.load_ckpt,epoch)) print(model) for epoch in range(args.start_epoch, args.start_epoch + args.num_epochs): running_loss=0.0 start=time.time() model.train() for i, data in enumerate(train_loader,0): inputs, labels = data if args.cuda: inputs = inputs.cuda() labels = labels.cuda() optimizer.zero_grad() outputs = model(inputs) loss = criterion(outputs,labels) loss.backward() optimizer.step() running_loss +=loss.item() if i %1000 == 999: writer.add_scalar('train_loss',running_loss/1000, epoch * len(train_loader) + i) writer.add_scalar('learning_rate',optimizer.param_groups[0]['lr'],epoch * len(train_loader) + i) if epoch > 0 and epoch % args.lrdt == 0: print("===>> decaying learning rate at epoch {}".format(epoch)) for param_group in optimizer.param_groups: param_group['lr'] = param_group['lr'] * 0.94 running_loss /= len(train_loader) end = time.time() test_accuracy = calculate_accuracy(model,test_loader) writer.add_scalar('test_accuracy',test_accuracy, epoch) ## Adding histograms after every epoch for tag, value in model.named_parameters(): tag = tag.replace('.','/') writer.add_histogram(tag,value.data.cpu().numpy(),epoch) print("Epoch: {0} | Loss: {1} | Test accuracy: {2}| Time Taken (sec): {3} ".format(epoch+1, np.around(running_loss,6), test_accuracy, np.around((end-start),4))) best_ckpt_filename = dir_name + "/ckpt_" + str(epoch) +'.pth' ##Save the best checkpoint if test_accuracy > best_test_accuracy: best_test_accuracy = test_accuracy torch.save({ 'epoch': epoch, 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer.state_dict(), 'loss': running_loss, }, best_ckpt_filename) writer.close() print("Training finished")
y = np.loadtxt(open('data/training_labels.csv'), delimiter=',') val = np.loadtxt(open('data/validation_data.csv'), delimiter=',') val_y = np.loadtxt(open('data/validation_labels.csv'), delimiter=',') #x = normalize(x) space = PCA() X_full = np.concatenate((X, val)) y_full = np.concatenate((y, val_y)) X_full_normal = normalize(X_full) space.fit(X_full_normal) X_reduced = space.transform(X_full_normal) (real, pred) = run_nn(X_reduced, y_full) print_results(pred, real) X_train, X_test, y_train, y_test = train_test_split(X_reduced, y, test_size=0.1) #X_res = X_train.reshape((X_train.shape[0], 12750, 1)) val_red = space.transform(val) model = cnn() scores = cross_val_score(model, X_reduced, y) #model.fit(X_train, y_train, epochs=10) #pred = model.predict(X_test) roc_curve(pred, y_test) np.savetxt("predictions_cnn.csv", pred)
""" import pandas as pd import preprocessingfile as preprocess import models data = 'pc2.csv' original_data, original_X, original_Y, combined_training_data, x_train1, x_train2, x_train, x_test, x_val, y_train1, y_train2, y_train, y_test, y_val = preprocess.my_sdp_preprocessor( data) all_data = [ original_data, original_X, original_Y, combined_training_data, x_train1, x_train2, x_train, x_test, x_val, y_train1, y_train2, y_train, y_test, y_val ] cnn_clf = models.cnn(*all_data) svm_clf = models.svm(*all_data) rf_clf = models.random_forest(*all_data) nn_clf = models.NN(*all_data) from sklearn.metrics import * def print_accuracy(model): #nn,cnn,svm,clf if (model == nn_clf): y_pred_on_val = model.predict(x_val) > 0.5 y_pred_on_test = model.predict(x_test) > 0.5 elif (model == cnn_clf): x_val_matrix = x_val.values x_val1 = x_val_matrix.reshape(x_val_matrix.shape[0], 1, len(x_val.columns), 1)