def prep_student(dataset, verbose, alpha, temperature): # if student is not saved beforehand, train and save it model_path = '{}/models/student/'.format(dataset) if not isfile(model_path + 'model/model.h5') or not isfile(model_path + 'model/model.json'): print('Student model doesnt exist, training it...') # load dataset and logits _dataset = get_data(dataset) x_train, y_train, x_test, y_test = _dataset.get_data() train_features = np.load( '{}/models/teacher/npy/train_logits.npy'.format(dataset)) test_features = np.load( '{}/models/teacher/npy/test_logits.npy'.format(dataset)) # normalized output with temperature shape=(num_samples,num_classes) y_train_soft = softmax(train_features / temperature) y_test_soft = softmax(test_features / temperature) # concatenated output labels=(num_samples,2*num_classes) y_train_new = np.concatenate([y_train, y_train_soft], axis=1) y_test_new = np.concatenate([y_test, y_test_soft], axis=1) # build student model student = get_model(_dataset, 'distillation', is_dropout=True) # remove softmax student.layers.pop() # get features logits = student.layers[-1].output # normal softmax output probs = Activation('softmax')(logits) # softmax output with temperature logits_T = Lambda(lambda x: x / temperature)(logits) probs_T = Activation('softmax')(logits_T) # concatanete output = concatenate([probs, probs_T]) # This is our new student model student = Model(student.input, output) compile_model(student, loss=distillation_loss(_dataset.num_classes, alpha), metrics=[acc_distillation(_dataset.num_classes)]) # create a new dataset with generated data dataset_s = DatasetCls(x_train, y_train_new, x_test, y_test_new, dataset_name=dataset) # train student student = train(dataset_s, student, PARAMS[dataset]['epochs'] * 2, PARAMS[dataset]['batch_size'], log_dir=model_path, callbacks=[ early_stop(patience=PARAMS[dataset]['patience'], monitor='val_loss', verbose=verbose) ], verbose=verbose) # save output files save_model_outputs(student, _dataset, model_path) K.clear_session()
def classification_model(self, num_dense_layers=2, num_dense_units=256, dropout_rate=0.2, num_classes=7, pooling='avg', name='default_resnet_classification_model', input_shape = 256, save_to = "", lr=1e-4 **kwargs): """ Returns a classifier model using the correct backbone. """ if isinstance(input_shape, int): if K.image_data_format() == 'channels_last': input_shape = (input_shape, input_shape, 3) else: input_shape = (3, input_shape, input_shape) else : t = keras.layers.Input(shape = input_shape) #input shape is processed in base_model base_model = self.build_base_model(input_tensor=t) x = base_model.ouput #output is softmax output of last dense layer output = default_classification_model(input_tensor=x, input_shape=base_model.ouput_shape[1:], num_classes=num_classes, num_dense_layers=num_dense_layers, num_dense_units=num_dense_units, dropout_rate=dropout_rate, pooling=pooling, kernel_regularizer='L2' ) model = keras.models.Model(inputs = t, outputs=output, name=name) if save_to: save_model_to_run(model, save_to) compile_model(model=model, num_classes=num_classes, metrics='acc', loss='ce', lr=lr) return model def validate(self): """ Checks whether the backbone string is correct. """ allowed_backbones = ['resnet50', ] if self.backbone_name not in allowed_backbones: raise ValueError('Backbone (\'{}\') not in allowed backbones ({}).'.format(self.backbone_name, allowed_backbones))
def classification_model(self, num_dense_layers=2, num_dense_units=256, dropout_rate=0.2, num_classes=7, pooling='avg', name='default_resnet_classification_model', input_shape = 256, save_to = "", lr=1e-4, kernel_regularizer=None ): """ Returns a classifier model using the correct backbone. """ if isinstance(input_shape, int): if K.image_data_format() == 'channels_last': input_shape = (input_shape, input_shape, 3) else: input_shape = (3, input_shape, input_shape) t = keras.layers.Input(shape = input_shape) #input shape is processed in base_model base_model = self.build_base_model(input_tensor=t) x = base_model.ouput #output is softmax output of last dense layer output = default_classification_model(input_tensor=x, input_shape=base_model.ouput_shape[1:], num_classes=num_classes, num_dense_layers=num_dense_layers, num_dense_units=num_dense_units, dropout_rate=dropout_rate, pooling=pooling, kernel_regularizer=kernel_ ) model = keras.models.Model(inputs = t, outputs=output, name=name) if save_to: save_model_to_run(model, save_to) compile_model(model=model, num_classes=num_classes, metrics='acc', loss='ce', lr=lr) return model
def get_model(self) -> Model: """Get selected model. Returns ------- Model Keras model object, compiled. """ # Build the model optimizer = RMSprop(self.learning_rate) if self.model_type == "unet": model = models.unet( self.input_shape, self.classes, self.loss, ) elif self.model_type == "unet_large": model = models.unet_large( self.input_shape, self.classes, self.loss, ) elif self.model_type == "fcdensenet": model = models.fcdensenet( self.input_shape, self.classes, self.loss, ) elif self.model_type == "fcn_small": model = models.fcn_small(self.input_shape, self.classes, self.loss) if self.preload_weights: logger.info( "=====================================================") logger.info(f"Using weights from {self.preload_weights}") logger.info( "=====================================================") model.load_weights(self.preload_weights) if config.FINE_TUNE_AT > 0: for layer in model.layers[:config.FINE_TUNE_AT]: layer.trainable = False logger.info( "=====================================================") logger.info(f"Fine tuning from %d layer" % config.FINE_TUNE_AT) logger.info( "=====================================================") model = models.compile_model(model, self.loss, optimizer) model.run_eagerly = True model.summary() return model
def run(self): model_id = self.job['model'] categories = self.job['categories'] filename, architecture = get_model(model_id) cat_dict, categories_id = request_categories(categories) path = prepare_dataset(categories_id, filename) model, gen = compile_model(architecture, categories_id) train_generator, test_generator, image_files, class_indices = create_gens( path, gen) train_accuracy, train_loss, val_accuracy, val_loss = start_training( model, train_generator, test_generator, image_files, filename) create_labels(cat_dict, filename, class_indices) api.update_job( self.job['id'], { 'done': 1, 'train_accuracy': train_accuracy, 'train_loss': train_loss, 'val_accuracy': val_accuracy, 'val_loss': val_loss })
def main( train_x, train_y, patch_rows, patch_cols, batch_size=64, epochs=1000, model="h_unet", name_suffix="suffix", ): tmp_model_path = os.path.join( 'cache', name_suffix + '{epoch:02d}-{val_loss:.2f}_ckpt_best.hdf5') early_stopping_patience = 15 model_args = { 'optimizer': 'Nadam', 'input_shape': (patch_rows, patch_cols, 13), 'base_depth': 64, 'lr': 0.0001 } # reduce base_depth to 32 if using vanilla unet if model == 'unet': model_args['base_depth'] = 32 # (1064, 13, 900, 900) print(train_x.shape) print(train_y.shape) # create generators for training and validation training_gen = DataGenerator(train_x[100:], train_y[100], batch_size=batch_size, img_rows=patch_rows, img_cols=patch_cols, horizontal_flip=True, vertical_flip=True, swap_axis=True) validation_gen = DataGenerator( train_x[0:100, :, :, :], train_y[0:100, :, :, :], batch_size=batch_size, ) n_train_ims = train_x.shape[0] - 100 n_val_ims = 100 monitor = 'val_loss' print() print("<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>") print(" BEGINNING MODEL TRAINING") print(" MODEL ARCHITECTURE: {}".format(model)) print(" OPTIMIZER: {}".format(model_args['optimizer'])) print(" INPUT SHAPE: {}".format(model_args['input_shape'])) print(" BATCH SIZE: {}".format(batch_size)) print(" LEARNING RATE: {}".format(model_args['lr'])) print("<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>") print() callbax = [] callbax.append( ReduceLROnPlateau(factor=0.2, patience=3, verbose=1, min_delta=0.01)) # callbax.append(ModelCheckpoint(tmp_model_path, monitor=monitor)) callbax.append(TerminateOnMetricNaN('precision')) callbax.append( EarlyStopping(monitor=monitor, patience=early_stopping_patience, mode='auto')) callbax.append(TensorBoard(log_dir='tensorboard')) lf = jaccard_coef_loss am = [precision, recall, jaccard_coef_int] original_model, parallel_model = compile_model(arch=model, loss_func=lf, additional_metrics=am, verbose=True, **model_args) callbax.append(MyCbk(original_model, tmp_model_path)) # plot_model(model, to_file='scheme.png', show_shapes=True) # model = load_model('cache/8_1000_Atlanta_nadir44_catid_1030010003CCD70076-0.47_ckpt_best.hdf5', custom_objects={ # 'jaccard_coef_loss': jaccard_coef_loss, # 'jaccard_coef_int': jaccard_coef_int, # 'precision': precision, # 'recall': recall}) parallel_model.fit_generator(training_gen, validation_data=validation_gen, validation_steps=np.floor(n_val_ims / batch_size), steps_per_epoch=400, epochs=epochs, workers=8, callbacks=callbax) original_model.save(os.path.join('cache', 'model' + name_suffix + '.hdf5')) print("<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>") print(" MODEL TRAINING COMPLETE! ") print(" Model located at {}".format(name_suffix)) print("<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>")
word_emb_dim) #construct student models for different stages with strategy.scope(): model_1 = models.construct_bilstm_student_model( distil_tokenizer.word_index, word_emb, max_seq_length, bilstm_hidden_size, config.hidden_size, dropout_rate, dense_act_func, len(label_list), stage=1, do_NER=do_NER) model_1 = models.compile_model(model_1, strategy, stage=1) model_2 = models.construct_bilstm_student_model( distil_tokenizer.word_index, word_emb, max_seq_length, bilstm_hidden_size, config.hidden_size, dropout_rate, dense_act_func, len(label_list), stage=2, do_NER=do_NER) model_2 = models.compile_model(model_2, strategy, stage=2) #get shared layers for student models
print(y_test.shape) x_train = x_train.astype('float32') / 255 x_test = x_test.astype('float32') / 255 n_classes = len(np.unique(y_train)) #model = models.get_model(width, height, channels, n_classes, hl=6, max_pool=False, nnodes=220) model = models.get_vgg_model2(width, height, channels, n_classes, hl=6, max_pool=False, nnodes=220) model = models.compile_model(model) n_epochs = 50 history = model.fit(x_train, y_train, batch_size=20, epochs=n_epochs) test_scores = model.evaluate(x_test, y_test, verbose=0) test_scores.append( history.history['sparse_categorical_accuracy'][n_epochs - 1]) results.append(test_scores) tf.keras.backend.clear_session() iteration_end_time = time.time() iteration_time = (iteration_end_time - iteration_start_time) / 60 total_elapsed_time = (iteration_end_time - start_time) / 60 est_complete = (total_elapsed_time / (idx + 1)) * (num_iterations - idx) print(f'Iteration Time: {iteration_time}') print(f'Total Elapsed Time: {total_elapsed_time}') print(f'Est. Time Remaining: {est_complete}')
import numpy as np BATCH_SIZE = 1000 TXT_DIM = 4092 MODEL_PATH = "./models/main_{}.h5".format(TXT_DIM) """ change file paths here """ txt_train = np.load('../Data/txt_train_trunc_{}.npy'.format(TXT_DIM)) txt_val = np.load('../Data/txt_val_trunc_{}.npy'.format(TXT_DIM)) img_train = np.load('../Data/img_train.npy') img_val = np.load('../Data/img_val.npy') model = create_model(TXT_DIM) print(model.count_params()) compile_model(model) print("done") history = model.fit(x=[txt_train,img_train], y=[np.ones((txt_train.shape[0],1024))], \ validation_data = ([txt_val, img_val],[np.ones((1000,1024))]), \ batch_size = BATCH_SIZE, epochs = 50, verbose = 2, shuffle = True) # Plot history import matplotlib.pyplot as plt plt.plot(history.history['loss'], label='loss (testing data)') plt.plot(history.history['val_loss'], label='loss (validation data)') plt.title('Training losses') plt.ylabel('loss value') plt.xlabel('No. epoch') plt.legend(loc="upper left")
# Split train data into input (X) and output (Y) variables. X_train = train[:, 1:3197] y_train = train[:, 0] # Split test data into input (X) and output (Y) variables. X_test = test[:, 1:3197] y_test = test[:, 0] # Normalize train and test features X_train, X_test = normalize_data(X_train, X_test) # Create model. model = build_model(gpus, units, dropout) # Compile model. model = compile_model(model, lr_rate) # Fit model. model = fit_model(model, loss_patience, X_train, y_train, X_test, y_test) # Evaluate training data on the model. validate_data("Train", X_train, y_train) # Evaluate test data on the model. validate_data("Test", X_test, y_test) # Predict our test dataset. predictions = model.predict(X_test) # Output our test dataset for visualization. print_predictions(predictions, print_results)