def build_model(cls, n_features): n_labels = len(cls.label_names) input_shape = (None, n_features) model_input = Input(input_shape, name='input') layer = model_input layer = LSTM(units=128, dropout=0.05, recurrent_dropout=0.35, return_sequences=True, input_shape=input_shape, name='lstm_1')(layer) layer = LSTM(units=32, dropout=0.05, recurrent_dropout=0.35, return_sequences=False, name='lstm_2')(layer) layer = Dense(units=n_labels, activation='softmax')(layer) model_output = layer model = Model(model_input, model_output) model.compile(loss='categorical_crossentropy', optimizer=Adam(), metrics=[ 'accuracy', TopKCategoricalAccuracy(3, name='top3-accuracy') ]) return model
def build_model(cls, n_features): n_labels = len(cls.label_names) print('Building model...') input_shape = (None, n_features) model_input = Input(input_shape, name='input') layer = model_input for i in range(N_LAYERS): layer = Convolution1D(filters=CONV_FILTER_COUNT, kernel_size=FILTER_LENGTH, name='convolution_' + str(i + 1))(layer) layer = BatchNormalization(momentum=0.9)(layer) layer = Activation('relu')(layer) layer = MaxPooling1D(2)(layer) layer = Dropout(0.5)(layer) layer = TimeDistributed(Dense(n_labels))(layer) time_distributed_merge_layer = Lambda( function=lambda x: K.mean(x, axis=1), output_shape=lambda shape: (shape[0], ) + shape[2:], name='output_merged') layer = time_distributed_merge_layer(layer) layer = Activation('softmax', name='output_realtime')(layer) model_output = layer model = Model(model_input, model_output) model.compile(loss='categorical_crossentropy', optimizer=Adam(learning_rate=0.001), metrics=[ 'accuracy', TopKCategoricalAccuracy(3, name='top3-accuracy') ]) return model
def perform_grid_search(opt, lr, bs, model): # Just cross tunning every parameter with each other with 3 nested for loops. for batch in bs: train_batches = prepare(training_data, batch_size=batch) test_batches = prepare(test_data, batch_size=batch) for optimizer in opt: for learning_rate in lr: model.compile(optimizer = optimizer(learning_rata = learning_rate), loss = 'categorical_crossentropy', metrics = ['accuracy', TopKCategoricalAccuracy(k=3)]) model_history = model.fit(train_batches, validation_data=test_batches, epochs = 50, callbacks = [early_stopping]) plot_train_and_validation_results(model_history.history['loss'], model_history.history['val_loss'], 'Training and Validation Loss', range(0, 51, 10) ) plot_train_and_validation_results(model_history.history['accuracy'], model_history.history['val_accuracy'], 'Training and Validation Accuracy', range(0, 51, 10) ) plot_train_and_validation_results(model_history.history['top_k_categorical_accuracy'], model_history.history['val_top_k_categorical_accuracy'], 'Top 3 Training and Validation Accuracy', range(0, 51, 10))
def run(args, use_gpu=True): # saving save_path = os.path.join(os.getcwd(), 'models') if not os.path.isdir(save_path): os.mkdir(save_path) model = lipnext(inputDim=256, hiddenDim=512, nClasses=args.nClasses, frameLen=29, alpha=args.alpha) #model = tf.keras.Sequential([ # tf.keras.layers.Flatten(), # tf.keras.layers.Dense(args.nClasses) # ]) if args.train == True: mode = "train" else: mode = "test" # train_list = glob.glob("./test_tfrecord_ACTUALLY_color/*.tfrecords") # val_list = glob.glob("./test_tfrecord_ACTUALLY_color/*.tfrecords") # test_list = glob.glob("./test_tfrecord_ACTUALLY_color/*.tfrecords") #train_list = glob.glob("/mnt/disks/data/dataset/lipread_tfrecords/*/train/*.tfrecords") #val_list = glob.glob("/mnt/disks/data/dataset/lipread_tfrecords/*/val/*.tfrecords") #test_list = glob.glob("/mnt/disks/data/dataset/lipread_tfrecords/*/test/*.tfrecords") with open(args.labels) as f: labels = f.read().splitlines() train_list = [] val_list = [] test_list = [] for word in labels: # print(word) train_list.extend(glob.glob(args.dataset + word + '/train/*.tfrecords')) val_list.extend(glob.glob(args.dataset + word + '/val/*.tfrecords')) test_list.extend(glob.glob(args.dataset + word + '/test/*.tfrecords')) # randomly shuffle *_list np.random.shuffle(train_list) np.random.shuffle(val_list) np.random.shuffle(test_list) if mode == "train": dataset = tf.data.TFRecordDataset(train_list) val_dataset = tf.data.TFRecordDataset(val_list) else: dataset = tf.data.TFRecordDataset(test_list) #test_list) # print("raw_dataset: ", dataset) if mode == "train": dataset = dataset.map(_parse_function) val_dataset = val_dataset.map(_parse_function) #check_dataset(dataset, "train_test_images", 'RGB') dataset = dataset.map(_train_preprocess_function) val_dataset = val_dataset.map(_test_preprocess_function) dataset = dataset.map( lambda x, y: _normalize_function(x, y, args.nClasses)) val_dataset = val_dataset.map( lambda x, y: _normalize_function(x, y, args.nClasses)) dataset = dataset.batch(args.batch_size, drop_remainder=True) val_dataset = val_dataset.batch(args.batch_size, drop_remainder=True) # check_dataset(dataset) dataset = dataset.map(lambda x, y: (x[:, :, ::2, ::2, :], y)) val_dataset = val_dataset.map(lambda x, y: (x[:, :, ::2, ::2, :], y)) # check_dataset(dataset) # dataset = dataset.map(lambda x, y: ( tf.reshape(x, [-1, x.shape[2], x.shape[3], x.shape[4]]), y)) # val_dataset = val_dataset.map(lambda x, y: ( tf.reshape(x, [-1, x.shape[2], x.shape[3], x.shape[4]]), y)) else: dataset = dataset.map(_parse_function) #check_dataset(dataset, "test_test_images", 'RGB') dataset = dataset.map(_test_preprocess_function) dataset = dataset.map( lambda x, y: _normalize_function(x, y, args.nClasses)) dataset = dataset.batch(1, drop_remainder=True) dataset = dataset.map(lambda x, y: (x[:, :, ::2, ::2, :], y)) #check_dataset(dataset, "test_test_images_processed", 'L') model.compile(optimizer=Adam(learning_rate=args.lr), loss=CategoricalCrossentropy(from_logits=True), metrics=[ 'accuracy', TopKCategoricalAccuracy(3), keras.metrics.CategoricalAccuracy() ]) run_dir = args.save_path + datetime.now().strftime("%Y%m%d-%H%M%S") print(run_dir, "_-----------------------------------------------") callbacks = [ # Interrupt training if `val_loss` stops improving for over 2 epochs # tf.keras.callbacks.EarlyStopping(patience=2, monitor='val_loss'), # Learning rate scheduler tf.keras.callbacks.LearningRateScheduler( lambda e: lr_scheduler(e, args.lr, args.lr_sleep_epochs)), # Save checkpoints tf.keras.callbacks.ModelCheckpoint(filepath=run_dir + '/runs/{epoch}/checkpoint', save_weights_only=True, save_best_only=True, monitor='val_categorical_accuracy', mode='max'), # Write TensorBoard logs to `./logs` directory tf.keras.callbacks.TensorBoard(log_dir=run_dir + '/logs', profile_batch=0) ] file_writer = tf.summary.create_file_writer(run_dir + '/logs/metrics') file_writer.set_as_default() if args.checkpoint: print("Loading model from: ", args.checkpoint) #model.load_weights(args.checkpoint) status = model.load_weights(args.checkpoint).expect_partial() print(f'STATUS: {status.assert_existing_objects_matched()}') #model = tf.keras.models.load_model(args.checkpoint) else: print("Model training from scratch -") def rep_dataset(): for i in range(10): image = next(iter(dataset)) yield [image[0]] #return [dataset.__iter__().next()[0]] print(f' REP DATASET: {rep_dataset()}') if mode == "train": #model.evaluate(val_dataset) model.fit(dataset, epochs=args.epochs, callbacks=callbacks, validation_data=val_dataset) #assert False '''model.fit(dataset, epochs=1, callbacks=callbacks, steps_per_epoch=1) #model.save('../saved_model') #tf.keras.models.save_model(model, '../saved_model') #model.save_weights(args.save_path +'/final_weights/Conv3D_model') # Convert the model. converter = tf.lite.TFLiteConverter.from_keras_model(model) converter.optimizations = [tf.lite.Optimize.DEFAULT] # This ensures that if any ops can't be quantized, the converter throws an error converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8] # These set the input and output tensors to uint8 converter.inference_input_type = tf.float32 # uint8 converter.inference_output_type = tf.uint8 # uint8 # And this sets the representative dataset so we can quantize the activations converter.representative_dataset = rep_dataset print('\n\nstarting conversion\n\n') tflite_model = converter.convert() print('\n\nmodel converted\n\n') # Save the TF Lite model. with tf.io.gfile.GFile('model_quantized_float.tflite', 'wb') as f: f.write(tflite_model)''' else: model.evaluate(dataset)
cnn4.add(Flatten()) cnn4.add(Dense(512, activation='relu')) cnn4.add(BatchNormalization()) cnn4.add(Dropout(0.5)) cnn4.add(Dense(128, activation='relu')) cnn4.add(BatchNormalization()) cnn4.add(Dropout(0.5)) cnn4.add(Dense(14, activation='softmax')) cnn4.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy', TopKCategoricalAccuracy(k=3)]) train_datagen = ImageDataGenerator(rotation_range=30., shear_range=0.2, zoom_range=0.2, width_shift_range=0.2, height_shift_range=0.2, horizontal_flip=True, rescale=1. / 255) test_datagen = ImageDataGenerator(rescale=1. / 255) train_iterator = train_datagen.flow_from_directory( base_path + 'train/', class_mode='categorical', target_size=(200, 200), batch_size=16)
x = model_resnet.output x = Dense(512, activation='elu', kernel_regularizer=l2(0.001))(x) y = Dense(14, activation='softmax', name='img')(x) final_model = Model(inputs=model_resnet.input, outputs=y) print(final_model.summary()) opt = SGD(lr=0.0001, momentum=0.9, nesterov=True) final_model.compile( optimizer=opt, loss={'img': 'categorical_crossentropy'}, metrics={'img': ['accuracy', TopKCategoricalAccuracy(k=3)]}) train_datagen = ImageDataGenerator(rotation_range=30., shear_range=0.2, zoom_range=0.2, width_shift_range=0.2, height_shift_range=0.2, horizontal_flip=True, rescale=1. / 255, validation_split=0.2) train_iterator = train_datagen.flow_from_directory( base_path, class_mode='categorical', target_size=(224, 224), subset='training')
#input_shape, output_shape) for layer in model.layers[:-2]: layer.trainable = False for layer in model.layers[-2:]: layer.trainable = True model_checkpoint = ModelCheckpoint(filepath='weights_{}.hdf5'.format( args.experiment), monitor='val_loss') early_stopping = EarlyStopping(monitor='val_loss', patience=6) reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3) callbacks = [model_checkpoint, early_stopping, reduce_lr] top3_metric = TopKCategoricalAccuracy(k=3, name='top_3_cat_acc') precision_metric = Precision() recall_metric = Recall() metrics = ['accuracy', top3_metric, precision_metric, recall_metric] model.compile(optimizer=Adam(0.0001, 0.9, 0.99), loss='categorical_crossentropy', metrics=metrics) history = model.fit_generator(train_flow, steps_per_epoch=params['batches_per_epoch'], epochs=args.max_epochs, validation_data=valid_flow, workers=4, callbacks=callbacks) #history = model.fit_generator(train_flow, steps_per_epoch=params['batches_per_epoch'], epochs=4, # validation_data=valid_flow, workers=4, callbacks=callbacks)
def train_model(dataset, pretrained=False): """ Train Tensorflow-Keras model """ LOGGER.info('\n---------------- Training Starting -----------------') LOGGER.info("Model Name: {}".format(Config['model_name'])) LOGGER.info('\nData, Configuration, & Model Parameters') LOGGER.info('-----------------------------------------') LOGGER.info("Seed: {}".format(Config['seed'])) LOGGER.info("Train/Test split: {}".format(Config['train_eval_split'])) LOGGER.info("Max sequence: {}".format(Config['max_sequence_size'])) LOGGER.info("Max labels: {}".format(Config['max_label_size'])) LOGGER.info("Embedding dim: {}".format(Config['embedding_dim'])) LOGGER.info("Binary threshold: {}".format(Config['binary_threshold'])) LOGGER.info("Batch size: {}".format(Config['batch_size'])) LOGGER.info("N epochs: {}".format(Config['n_epochs'])) LOGGER.info("Eval @ k: {}".format(Config['evaluation@k'])) LOGGER.info("Patience: {}".format(Config['patience'])) LOGGER.info("Encoer type: {}".format(Config['model_encoder'])) if Config['model_encoder'] == 'cnns': LOGGER.info("Num hidden units: {}".format( Config['cnns']['hidden_units_size'])) LOGGER.info("Learning rate: {}".format( Config['cnns']['learning_rate'])) LOGGER.info("Dropout: {}".format(Config['cnns']['spatial_dropout'])) if Config['model_encoder'] == 'grus': LOGGER.info("Num hidden units: {}".format( Config['grus']['hidden_units_size'])) LOGGER.info("Num hidden layers: {}".format( Config['grus']['n_hidden_layers'])) LOGGER.info("Learning rate: {}".format( Config['grus']['learning_rate'])) LOGGER.info("Dropout: {}".format(Config['grus']['spatial_dropout'])) if pretrained: LOGGER.info('Loading Pretrained Model: {}'.format( Config['pretrained_model'])) # Retrieve training / validation data train_X = dataset['training_dataset']['train_X'] train_y = dataset['training_dataset']['train_y'] dev_X = dataset['evaluation_dataset']['eval_X'] dev_y = dataset['evaluation_dataset']['eval_y'] # For training final model train_X = np.concatenate((train_X, dev_X)) train_y = np.concatenate((train_y, dev_y)) train_generator = BatchGenerator(train_X, train_y, batch_size=Config['batch_size']) val_generator = BatchGenerator(dev_X, dev_y, batch_size=Config['batch_size']) # Initialize the model if pretrained: LOGGER.info('\nLoading Model: {}'.format(BASE + Config['model_loc'])) model = tf.saved_model.load(BASE + Config['model_loc']) else: word_vectors = np.array([np.array(x) for x in dataset['word_vectors']]) # Model class from model.py model = LWAN(n_classes=train_y.shape[1], emb_weights=word_vectors) optimizer = Adam(learning_rate=Config['cnns']['learning_rate']) loss_fn = BinaryCrossentropy() #metric = BinaryAccuracy() metric = TopKCategoricalAccuracy() model.compile(optimizer, loss_fn, metric) # Start Training LOGGER.info('\n ------ Fitting model ------ ') ckpt_path = os.path.join(BASE + Config['model_location'], Config['model_name'], Config['timestamp'], 'checkpoints', 'model-{epoch:02d}-{val_loss:.2f}') early_stopping = EarlyStopping(monitor='val_loss', patience=Config['patience'], restore_best_weights=True) model_checkpoint = ModelCheckpoint(filepath=ckpt_path, monitor='val_loss', mode='auto', save_freq='epoch', verbose=1, save_best_only=True, save_weights_only=False) start_time = time.time() # start training fit_history = model.fit(train_generator, validation_data=val_generator, workers=os.cpu_count(), epochs=Config['n_epochs'], callbacks=[early_stopping, model_checkpoint]) total_time = time.time() - start_time LOGGER.info('\nTotal Training Time: {} secs'.format(total_time)) # Record final loss LOGGER.info('\n ----- Final Loss & Best Epoch ----- ') best_epoch = np.argmin(fit_history.history['val_loss']) + 1 n_epochs = len(fit_history.history['val_loss']) val_loss_per_epoch = '- ' + ' '.join( '-' if fit_history.history['val_loss'][i] < np. min(fit_history.history['val_loss'][:i]) else '+' for i in range(1, len(fit_history.history['val_loss']))) LOGGER.info('Val loss per epoch: {}\n'.format(val_loss_per_epoch)) LOGGER.info('\nBest epoch: {}/{}'.format(best_epoch, n_epochs)) # Save model LOGGER.info('\n ----- Saving Final Model ------ ') best_val_loss = fit_history.history['val_loss'][best_epoch - 1] model_path = os.path.join( BASE + Config['model_location'], Config['model_name'], Config['timestamp'], '{}_FINAL_epoch_{:02d}_loss_{:.5f}'.format(Config['model_name'], best_epoch, best_val_loss)) LOGGER.info('Model Path: {}'.format(model_path)) tf.saved_model.save(model, model_path)
q_aware_model = tf.keras.models.clone_model( model, clone_function=apply_quantization_to_dense, ) adam = optimizers.Adam(learning_rate=lr, beta_1=0.9, beta_2=0.99, epsilon=None, decay=1e-5, amsgrad=False) q_aware_model.compile(optimizer=adam, loss='categorical_crossentropy', metrics=[ TopKCategoricalAccuracy(k=1, name='accuracy'), TopKCategoricalAccuracy(k=3, name='top3_accuracy'), TopKCategoricalAccuracy(k=5, name='top5_accuracy') ]) # Define callbacks model_folder = os.path.dirname(model_path) model_filename = os.path.basename(model_path) output_filename = model_filename[:model_filename.index('.hdf5' )] + '_quant.hdf5' checkpoint = ModelCheckpoint(filepath=os.path.join(model_folder, output_filename), save_best_only=True, monitor='val_accuracy', save_weights_only=False, verbose=0)
def run(args, use_gpu=True): # saving save_path = os.path.join(os.getcwd(), 'models') if not os.path.isdir(save_path): os.mkdir(save_path) model = lipnext(inputDim=256, hiddenDim=512, nClasses=args.nClasses, frameLen=29, alpha=args.alpha) #model = tf.keras.Sequential([ # tf.keras.layers.Flatten(), # tf.keras.layers.Dense(1) #]) if args.train == True: mode = "train" else: mode = "test" # train_list = glob.glob("./test_tfrecord_ACTUALLY_color/*.tfrecords") # val_list = glob.glob("./test_tfrecord_ACTUALLY_color/*.tfrecords") # test_list = glob.glob("./test_tfrecord_ACTUALLY_color/*.tfrecords") #train_list = glob.glob("/mnt/disks/data/dataset/lipread_tfrecords/*/train/*.tfrecords") #val_list = glob.glob("/mnt/disks/data/dataset/lipread_tfrecords/*/val/*.tfrecords") #test_list = glob.glob("/mnt/disks/data/dataset/lipread_tfrecords/*/test/*.tfrecords") with open(args.labels) as f: labels = f.read().splitlines() train_list = [] val_list = [] test_list = [] for word in labels: print(word) train_list.extend(glob.glob(args.dataset + word + '/train/*.tfrecords')) val_list.extend(glob.glob(args.dataset + word + '/val/*.tfrecords')) test_list.extend(glob.glob(args.dataset + word + '/test/*.tfrecords')) # randomly shuffle *_list np.random.shuffle(train_list) np.random.shuffle(val_list) np.random.shuffle(test_list) if mode == "train": dataset = tf.data.TFRecordDataset(train_list) val_dataset = tf.data.TFRecordDataset(val_list) else: dataset = tf.data.TFRecordDataset(test_list) print("raw_dataset: ", dataset) if mode == "train": dataset = dataset.map(_parse_function) val_dataset = val_dataset.map(_parse_function) #check_dataset(dataset, "train_test_images", 'RGB') dataset = dataset.map(_train_preprocess_function) val_dataset = val_dataset.map(_test_preprocess_function) #check_dataset(dataset, "train_test_images_processed", 'L') dataset = dataset.map( lambda x, y: _normalize_function(x, y, args.nClasses)) val_dataset = val_dataset.map( lambda x, y: _normalize_function(x, y, args.nClasses)) dataset = dataset.shuffle(500).batch(args.batch_size, drop_remainder=True) val_dataset = val_dataset.batch(args.batch_size, drop_remainder=True) else: dataset = dataset.map(_parse_function) #check_dataset(dataset, "test_test_images", 'RGB') dataset = dataset.map(_test_preprocess_function) dataset = dataset.map( lambda x, y: _normalize_function(x, y, args.nClasses)) dataset = dataset.batch(args.batch_size) #check_dataset(dataset, "test_test_images_processed", 'L') model.compile(optimizer=Adam(learning_rate=args.lr), loss=CategoricalCrossentropy(from_logits=True), metrics=[ 'accuracy', TopKCategoricalAccuracy(3), keras.metrics.CategoricalAccuracy() ]) run_dir = args.save_path + datetime.now().strftime("%Y%m%d-%H%M%S") callbacks = [ # Interrupt training if `val_loss` stops improving for over 2 epochs # tf.keras.callbacks.EarlyStopping(patience=2, monitor='val_loss'), # Learning rate scheduler tf.keras.callbacks.LearningRateScheduler( lambda e: lr_scheduler(e, args.lr)), # Save checkpoints tf.keras.callbacks.ModelCheckpoint( filepath=run_dir + '/checkpoints/Conv3D_model_{epoch}', #save_weights_only=True, save_best_only=True, monitor='val_loss'), # Write TensorBoard logs to `./logs` directory tf.keras.callbacks.TensorBoard(log_dir=run_dir + '/logs') ] file_writer = tf.summary.create_file_writer(run_dir + '/logs/metrics') file_writer.set_as_default() if args.checkpoint: print("Loading model from: ", args.checkpoint) #model.load_weights(args.checkpoint) model = tf.keras.models.load_model(args.checkpoint) else: print("Model training from scratch") if mode == "train": model.fit(dataset, epochs=args.epochs, callbacks=callbacks, validation_data=val_dataset) #model.save_weights(args.save_path +'/final_weights/Conv3D_model') else: model.evaluate(dataset)