def main(_): # get experiment folder and create dir for plots exp_folder = os.path.join(FLAGS.exp_root, FLAGS.exp_name, 'exp{}'.format(FLAGS.exp_nr)) test_folder = os.path.join(exp_folder, 'test') tf.io.gfile.mkdir(test_folder) # get experiment FLAGS TRAINING_FLAGS = yaml.safe_load( tf.io.gfile.GFile(os.path.join(exp_folder, 'FLAGS.yml'), 'r')) # get dataset test_set, test_labels = get_dataset( '.', TRAINING_FLAGS['num_feat'], TRAINING_FLAGS['slice_length'], type='test', return_sequences=TRAINING_FLAGS['return_sequences']) sequence_length = test_set.shape[1] feature_dim = test_set.shape[2] if TRAINING_FLAGS['model'] == 'tcn': model = get_tcn( sequence_length, feature_dim, nb_filters=TRAINING_FLAGS['num_filters'], nb_stacks=TRAINING_FLAGS['num_stacks'], use_skip_connections=TRAINING_FLAGS['use_skip_connections'], use_batch_norm=TRAINING_FLAGS['bn'], return_sequences=False, #TRAINING_FLAGS['return_sequences'], dilation_stages=TRAINING_FLAGS['dilation_stages']) elif TRAINING_FLAGS['model'] == 'cnn': model = get_cnn((sequence_length, feature_dim)) else: assert False, 'Unknown model!' model(tf.zeros((1, sequence_length, feature_dim))) model.load_weights(os.path.join(exp_folder, 'model.h5')) model.compile() model.summary() #if tcn, we have to cut off the model above the strided slice since it is not supported in NNTool, we perform the last Dense layer as a matrix product if TRAINING_FLAGS['model'] == 'tcn': model = tf.keras.Model( inputs=[model.input], outputs=[model.get_layer(name='reshape_1').output]) model.summary() converter = tf.lite.TFLiteConverter.from_keras_model(model) # Convert the model to the TensorFlow Lite format with quantization tflite_model_name = 'quant_model' quantize = True if (quantize): def representative_dataset(): for i in range(100): yield [test_set[i].reshape(1, sequence_length, feature_dim)] # Set the optimization flag. converter.optimizations = [tf.lite.Optimize.DEFAULT] # Enforce full-int8 quantization converter.target_spec.supported_ops = [ tf.lite.OpsSet.TFLITE_BUILTINS_INT8 ] converter.inference_input_type = tf.uint8 # or tf.uint8 converter.inference_output_type = tf.uint8 # or tf.uint8 # Provide a representative dataset to ensure we quantize correctly. converter.representative_dataset = representative_dataset tflite_model = converter.convert() model_path = os.path.join(exp_folder, tflite_model_name + '.tflite') open(model_path, 'wb').write(tflite_model)
def main(do_test_run=False, resume_checkpoint=None, run_method='alternate_train_and_validate', **kwargs): """This is the main entry of the script. It's main responsibility is to create an Engine object using the default config that is modified by the kwargs given to this function. By default it creates an Engine with the no preloaded weights for the model and optimizer. When `resume_checkpoint` is set to the path to a checkpoint file, this checkpoint file will be used. By default, the `Engine#alternate_train_and_validate` method will be called. This can be modified by the `run_method` parameter of this function. """ config.update(kwargs) if os.path.exists(config['pwd']): if input(f"{config['pwd']} already exists, are you sure? [y/N]: " ) != 'y': return print(f"config =\n{config}") cnn_model = get_cnn(config) classifier_model = get_classifier(config) detective_model = get_detective(config) if resume_checkpoint is not None: info(f"loading state_dict for the engine from {resume_checkpoint} ...") the_state_dict = torch.load(resume_checkpoint) cnn_model.load_state_dict(the_state_dict['cnn_model']) classifier_model.load_state_dict(the_state_dict['classifier_model']) detective_model.load_state_dict(the_state_dict['detective_model']) info(f"loaded state_dict for the engine from {resume_checkpoint}.") engine = Engine( cnn_model=cnn_model, cnn_optimizer=(torch.optim.Adam, { 'lr': 3e-4 }), classifier_model=classifier_model, classifier_optimizer=(torch.optim.Adam, { 'lr': 3e-4 }), classifier_loss_fn=nn.BCEWithLogitsLoss(), detective_model=detective_model, detective_optimizer=(torch.optim.Adam, { 'lr': 3e-4 }), detective_loss_fn=nn.SmoothL1Loss(), max_epochs=config['max_epochs'], loader_pair=get_loader_pair(config), plugins=[ Timestamp(), TrainingMetrics( classifier_metrics={ # 'loss': loss, # 'acc': acc, }, detective_metrics={ 'loss': loss, }, residual_factor=max(1 - config['train_batch_size'] / 10000, 0), ), ValidationMetrics( classifier_metrics={ # 'loss': loss, # 'acc': acc, # 'std': std, # 'auc': auc, }, detective_metrics={ 'loss': loss, 'std': std, }, ), ReduceLROnPlateau(), Checkpoint(), Messages(), ], device=config['device'], pwd=config['pwd'], ) if do_test_run: engine.__getattribute__(run_method)(test_run=True) engine.reset() engine.__getattribute__(run_method)()
def main(_): tf.config.experimental_run_functions_eagerly(FLAGS.debug) # get experiment folder if FLAGS.exp_folder == '': exp_folder, exp_name = get_experiment_folder() else: exp_folder = FLAGS.exp_folder # save FLAGS to yml tf.io.gfile.makedirs(exp_folder) yaml.dump( FLAGS.flag_values_dict(), tf.io.gfile.GFile(os.path.join(exp_folder, 'FLAGS.yml'), 'w'), ) # get dataset train_set, train_labels = get_dataset( FLAGS.data_root, FLAGS.num_feat, FLAGS.slice_length, return_sequences=FLAGS.return_sequences) # optimizer optimizer = tf.keras.optimizers.Adam(learning_rate=FLAGS.lr) # loss function loss_func = sparse_categorical_crossentropy sequence_length = train_set.shape[1] feature_dim = train_set.shape[2] if FLAGS.model == 'tcn': model = get_tcn(sequence_length, feature_dim, nb_filters=FLAGS.num_filters, nb_stacks=FLAGS.num_stacks, use_skip_connections=FLAGS.use_skip_connections, use_batch_norm=FLAGS.bn, return_sequences=FLAGS.return_sequences, dilation_stages=FLAGS.dilation_stages) elif FLAGS.model == 'cnn': model = get_cnn((sequence_length, feature_dim)) METRICS = [ tf.keras.metrics.TruePositives(name='tp'), tf.keras.metrics.FalsePositives(name='fp'), tf.keras.metrics.TrueNegatives(name='tn'), tf.keras.metrics.FalseNegatives(name='fn'), tf.keras.metrics.BinaryAccuracy(name='accuracy'), tf.keras.metrics.Precision(name='precision'), tf.keras.metrics.Recall(name='recall'), tf.keras.metrics.AUC(name='auc'), ] #wrap binary metric to make them work with sparse categorical crossentropy wrapped_metrics = list(map(lambda m: MetricWrapper(m), METRICS)) callbacks = [ tf.keras.callbacks.TensorBoard(log_dir=exp_folder, write_graph=True), tf.keras.callbacks.EarlyStopping(patience=FLAGS.patience), ModelCheckpoint(filepath=os.path.join(exp_folder, 'model.h5'), save_best_only=True), tf.keras.callbacks.ReduceLROnPlateau(monitor="val_loss", factor=0.5, patience=100) ] if FLAGS.return_sequences: model.compile( optimizer=optimizer, loss=loss_func, metrics=['accuracy'], #add metrics if wished sample_weight_mode='temporal') class_weights = class_weight.compute_class_weight( 'balanced', classes=[0, 1], y=train_labels.flatten()) #temporal sample weighting: give higher weight to timesteps that are labelled as 1 (rare class) sample_weight = train_labels * class_weights[1] + class_weights[ 0] - train_labels * class_weights[0] train_labels = np.expand_dims( train_labels, axis=-1 ) #necessary because tf complains otherwise that data is not temporal # train model history = model.fit(train_set, train_labels, epochs=FLAGS.epochs, batch_size=FLAGS.batch_size, validation_split=0.25, callbacks=callbacks, sample_weight=sample_weight) else: model.compile(optimizer=optimizer, loss=loss_func, metrics=['accuracy']) #use class weights to counteract class imbalance class_weights = class_weight.compute_class_weight('balanced', classes=[0, 1], y=train_labels) class_weights = {i: class_weights[i] for i in range(2)} history = model.fit(train_set, train_labels, epochs=FLAGS.epochs, batch_size=FLAGS.batch_size, validation_split=0.25, callbacks=callbacks, class_weight=class_weights) # save history to yaml yaml.dump(history.history, tf.io.gfile.GFile(os.path.join(exp_folder, 'history.yml'), 'w'))
def main(_): # get experiment folder and create dir for plots exp_folder = os.path.join(FLAGS.exp_root, FLAGS.exp_name, 'exp{}'.format(FLAGS.exp_nr)) test_folder = os.path.join(exp_folder, 'test') tf.io.gfile.mkdir(test_folder) # get experiment FLAGS TRAINING_FLAGS = yaml.safe_load( tf.io.gfile.GFile(os.path.join(exp_folder, 'FLAGS.yml'), 'r')) # get dataset test_set, test_labels = get_dataset( '.', TRAINING_FLAGS['num_feat'], TRAINING_FLAGS['slice_length'], type='test', return_sequences=TRAINING_FLAGS['return_sequences']) sequence_length = test_set.shape[1] feature_dim = test_set.shape[2] if TRAINING_FLAGS['model'] == 'tcn': model = get_tcn( sequence_length, feature_dim, nb_filters=TRAINING_FLAGS['num_filters'], nb_stacks=TRAINING_FLAGS['num_stacks'], use_skip_connections=TRAINING_FLAGS['use_skip_connections'], use_batch_norm=TRAINING_FLAGS['bn'], return_sequences=TRAINING_FLAGS['return_sequences'], dilation_stages=TRAINING_FLAGS['dilation_stages']) elif TRAINING_FLAGS['model'] == 'cnn': model = get_cnn((sequence_length, feature_dim)) else: assert False, 'Unknown model!' model(tf.zeros((1, sequence_length, feature_dim))) model.load_weights(os.path.join(exp_folder, 'model.h5')) model.compile() model.summary() # print(model.count_params()) # weights = model.get_layer('dense').get_weights() # kernel = weights[0] # bias = weights[1] # scale = 128 / max(kernel.min(), kernel.max(), bias.min(), bias.max()) # kernel_scaled = (kernel * scale).astype('int8') # bias_scaled = (bias * scale).astype('int8') converter = tf.lite.TFLiteConverter.from_keras_model(model) # Convert the model to the TensorFlow Lite format with quantization tflite_model_name = 'quant_model' quantize = True if (quantize): def representative_dataset(): for i in range(100): yield [test_set[i].reshape(1, sequence_length, feature_dim)] # Set the optimization flag. converter.optimizations = [tf.lite.Optimize.DEFAULT] # Enforce full-int8 quantization converter.target_spec.supported_ops = [ tf.lite.OpsSet.TFLITE_BUILTINS_INT8 ] converter.inference_input_type = tf.uint8 # or tf.uint8 converter.inference_output_type = tf.uint8 # or tf.uint8 # Provide a representative dataset to ensure we quantize correctly. converter.representative_dataset = representative_dataset tflite_model = converter.convert() model_path = os.path.join('/tmp', tflite_model_name + '.tflite') open(model_path, 'wb').write(tflite_model) tflite_interpreter = tf.lite.Interpreter(model_path=model_path) tflite_interpreter.allocate_tensors() input_details = tflite_interpreter.get_input_details() output_details = tflite_interpreter.get_output_details() predictions = [] for i in range(len(test_set)): val_batch = test_set[i] val_batch = np.expand_dims(val_batch, axis=0).astype(input_details[0]["dtype"]) tflite_interpreter.set_tensor(input_details[0]['index'], val_batch) tflite_interpreter.allocate_tensors() tflite_interpreter.invoke() output = tflite_interpreter.get_tensor(output_details[0]['index']) predictions += [output] METRICS = [ tf.keras.metrics.TruePositives(name='tp'), tf.keras.metrics.FalsePositives(name='fp'), tf.keras.metrics.TrueNegatives(name='tn'), tf.keras.metrics.FalseNegatives(name='fn'), tf.keras.metrics.BinaryAccuracy(name='accuracy'), tf.keras.metrics.Precision(name='precision'), tf.keras.metrics.Recall(name='recall'), tf.keras.metrics.AUC(name='auc'), ] wrapped_metrics = list(map(lambda m: MetricWrapper(m, dims=2), METRICS)) predictions = np.stack(predictions).squeeze() res = {} for m in wrapped_metrics: m.update_state(y_true=test_labels, y_pred=predictions) res[m.name] = m.result().numpy() with open(os.path.join(exp_folder, 'test/quant_metrics.p'), 'wb') as handle: pickle.dump(res, handle, protocol=pickle.HIGHEST_PROTOCOL) pprint.pprint(res)
def main(_): # get experiment folder and create dir for plots exp_folder = os.path.join(FLAGS.exp_root, FLAGS.exp_name, 'exp{}'.format(FLAGS.exp_nr)) test_folder = os.path.join(exp_folder, 'test') tf.io.gfile.mkdir(test_folder) # get experiment FLAGS TRAINING_FLAGS = yaml.safe_load( tf.io.gfile.GFile(os.path.join(exp_folder, 'FLAGS.yml'), 'r') ) # get dataset test_set, test_labels = get_dataset(TRAINING_FLAGS['data_root'], TRAINING_FLAGS['num_feat'], TRAINING_FLAGS['slice_length'], type='test', return_sequences=TRAINING_FLAGS['return_sequences']) sequence_length = test_set.shape[1] feature_dim = test_set.shape[2] if TRAINING_FLAGS['model'] == 'tcn': model = get_tcn(sequence_length, feature_dim, nb_filters=TRAINING_FLAGS['num_filters'], nb_stacks=TRAINING_FLAGS['num_stacks'], use_skip_connections=TRAINING_FLAGS['use_skip_connections'], use_batch_norm=TRAINING_FLAGS['bn'], return_sequences=TRAINING_FLAGS['return_sequences'], dilation_stages=TRAINING_FLAGS['dilation_stages']) elif TRAINING_FLAGS['model'] == 'cnn': model = get_cnn((sequence_length, feature_dim)) else: assert False, 'Unknown model!' #evaluate the model to make sure it is built model(tf.zeros((1, sequence_length, feature_dim))) model.load_weights(os.path.join(exp_folder, 'model.h5')) model.compile() #count the numper of parameters in the model parameters = model.count_params() pred_labels = model.predict(test_set, batch_size=64, use_multiprocessing=True, workers=8) #define metrics METRICS = [ tf.keras.metrics.TruePositives(name='tp'), tf.keras.metrics.FalsePositives(name='fp'), tf.keras.metrics.TrueNegatives(name='tn'), tf.keras.metrics.FalseNegatives(name='fn'), tf.keras.metrics.BinaryAccuracy(name='accuracy'), tf.keras.metrics.Precision(name='precision'), tf.keras.metrics.Recall(name='recall'), tf.keras.metrics.AUC(name='auc'), ] #wrap metrics to be compatible with categorical output wrapped_metrics = list(map(lambda m: MetricWrapper(m, dims=2), METRICS)) res = {} for m in wrapped_metrics: m.update_state(y_true=test_labels, y_pred=pred_labels) res[m.name] = m.result().numpy() res['parameters'] = parameters #store metrics in pickle file with open(os.path.join(exp_folder, 'test/metrics.p'), 'wb') as handle: pickle.dump(res, handle, protocol=pickle.HIGHEST_PROTOCOL) pprint.pprint(res)