def main(_):
# get experiment folder and create dir for plots
exp_folder = os.path.join(FLAGS.exp_root, FLAGS.exp_name,
'exp{}'.format(FLAGS.exp_nr))
test_folder = os.path.join(exp_folder, 'test')
tf.io.gfile.mkdir(test_folder)
# get experiment FLAGS
TRAINING_FLAGS = yaml.safe_load(
tf.io.gfile.GFile(os.path.join(exp_folder, 'FLAGS.yml'), 'r'))
# get dataset
test_set, test_labels = get_dataset(
'.',
TRAINING_FLAGS['num_feat'],
TRAINING_FLAGS['slice_length'],
type='test',
return_sequences=TRAINING_FLAGS['return_sequences'])
sequence_length = test_set.shape[1]
feature_dim = test_set.shape[2]
if TRAINING_FLAGS['model'] == 'tcn':
model = get_tcn(
sequence_length,
feature_dim,
nb_filters=TRAINING_FLAGS['num_filters'],
nb_stacks=TRAINING_FLAGS['num_stacks'],
use_skip_connections=TRAINING_FLAGS['use_skip_connections'],
use_batch_norm=TRAINING_FLAGS['bn'],
return_sequences=False, #TRAINING_FLAGS['return_sequences'],
dilation_stages=TRAINING_FLAGS['dilation_stages'])
elif TRAINING_FLAGS['model'] == 'cnn':
model = get_cnn((sequence_length, feature_dim))
else:
assert False, 'Unknown model!'
model(tf.zeros((1, sequence_length, feature_dim)))
model.load_weights(os.path.join(exp_folder, 'model.h5'))
model.compile()
model.summary()
#if tcn, we have to cut off the model above the strided slice since it is not supported in NNTool, we perform the last Dense layer as a matrix product
if TRAINING_FLAGS['model'] == 'tcn':
model = tf.keras.Model(
inputs=[model.input],
outputs=[model.get_layer(name='reshape_1').output])
model.summary()
converter = tf.lite.TFLiteConverter.from_keras_model(model)
# Convert the model to the TensorFlow Lite format with quantization
tflite_model_name = 'quant_model'
quantize = True
if (quantize):
def representative_dataset():
for i in range(100):
yield [test_set[i].reshape(1, sequence_length, feature_dim)]
# Set the optimization flag.
converter.optimizations = [tf.lite.Optimize.DEFAULT]
# Enforce full-int8 quantization
converter.target_spec.supported_ops = [
tf.lite.OpsSet.TFLITE_BUILTINS_INT8
]
converter.inference_input_type = tf.uint8 # or tf.uint8
converter.inference_output_type = tf.uint8 # or tf.uint8
# Provide a representative dataset to ensure we quantize correctly.
converter.representative_dataset = representative_dataset
tflite_model = converter.convert()
model_path = os.path.join(exp_folder, tflite_model_name + '.tflite')
open(model_path, 'wb').write(tflite_model)
def main(do_test_run=False,
resume_checkpoint=None,
run_method='alternate_train_and_validate',
**kwargs):
"""This is the main entry of the script.
It's main responsibility is to create an Engine object using the default
config that is modified by the kwargs given to this function.
By default it creates an Engine with the no preloaded weights for the model
and optimizer. When `resume_checkpoint` is set to the path to a checkpoint
file, this checkpoint file will be used.
By default, the `Engine#alternate_train_and_validate` method will be called.
This can be modified by the `run_method` parameter of this function.
"""
config.update(kwargs)
if os.path.exists(config['pwd']):
if input(f"{config['pwd']} already exists, are you sure? [y/N]: "
) != 'y':
return
print(f"config =\n{config}")
cnn_model = get_cnn(config)
classifier_model = get_classifier(config)
detective_model = get_detective(config)
if resume_checkpoint is not None:
info(f"loading state_dict for the engine from {resume_checkpoint} ...")
the_state_dict = torch.load(resume_checkpoint)
cnn_model.load_state_dict(the_state_dict['cnn_model'])
classifier_model.load_state_dict(the_state_dict['classifier_model'])
detective_model.load_state_dict(the_state_dict['detective_model'])
info(f"loaded state_dict for the engine from {resume_checkpoint}.")
engine = Engine(
cnn_model=cnn_model,
cnn_optimizer=(torch.optim.Adam, {
'lr': 3e-4
}),
classifier_model=classifier_model,
classifier_optimizer=(torch.optim.Adam, {
'lr': 3e-4
}),
classifier_loss_fn=nn.BCEWithLogitsLoss(),
detective_model=detective_model,
detective_optimizer=(torch.optim.Adam, {
'lr': 3e-4
}),
detective_loss_fn=nn.SmoothL1Loss(),
max_epochs=config['max_epochs'],
loader_pair=get_loader_pair(config),
plugins=[
Timestamp(),
TrainingMetrics(
classifier_metrics={
# 'loss': loss,
# 'acc': acc,
},
detective_metrics={
'loss': loss,
},
residual_factor=max(1 - config['train_batch_size'] / 10000, 0),
),
ValidationMetrics(
classifier_metrics={
# 'loss': loss,
# 'acc': acc,
# 'std': std,
# 'auc': auc,
},
detective_metrics={
'loss': loss,
'std': std,
},
),
ReduceLROnPlateau(),
Checkpoint(),
Messages(),
],
device=config['device'],
pwd=config['pwd'],
)
if do_test_run:
engine.__getattribute__(run_method)(test_run=True)
engine.reset()
engine.__getattribute__(run_method)()
def main(_):
tf.config.experimental_run_functions_eagerly(FLAGS.debug)
# get experiment folder
if FLAGS.exp_folder == '':
exp_folder, exp_name = get_experiment_folder()
else:
exp_folder = FLAGS.exp_folder
# save FLAGS to yml
tf.io.gfile.makedirs(exp_folder)
yaml.dump(
FLAGS.flag_values_dict(),
tf.io.gfile.GFile(os.path.join(exp_folder, 'FLAGS.yml'), 'w'),
)
# get dataset
train_set, train_labels = get_dataset(
FLAGS.data_root,
FLAGS.num_feat,
FLAGS.slice_length,
return_sequences=FLAGS.return_sequences)
# optimizer
optimizer = tf.keras.optimizers.Adam(learning_rate=FLAGS.lr)
# loss function
loss_func = sparse_categorical_crossentropy
sequence_length = train_set.shape[1]
feature_dim = train_set.shape[2]
if FLAGS.model == 'tcn':
model = get_tcn(sequence_length,
feature_dim,
nb_filters=FLAGS.num_filters,
nb_stacks=FLAGS.num_stacks,
use_skip_connections=FLAGS.use_skip_connections,
use_batch_norm=FLAGS.bn,
return_sequences=FLAGS.return_sequences,
dilation_stages=FLAGS.dilation_stages)
elif FLAGS.model == 'cnn':
model = get_cnn((sequence_length, feature_dim))
METRICS = [
tf.keras.metrics.TruePositives(name='tp'),
tf.keras.metrics.FalsePositives(name='fp'),
tf.keras.metrics.TrueNegatives(name='tn'),
tf.keras.metrics.FalseNegatives(name='fn'),
tf.keras.metrics.BinaryAccuracy(name='accuracy'),
tf.keras.metrics.Precision(name='precision'),
tf.keras.metrics.Recall(name='recall'),
tf.keras.metrics.AUC(name='auc'),
]
#wrap binary metric to make them work with sparse categorical crossentropy
wrapped_metrics = list(map(lambda m: MetricWrapper(m), METRICS))
callbacks = [
tf.keras.callbacks.TensorBoard(log_dir=exp_folder, write_graph=True),
tf.keras.callbacks.EarlyStopping(patience=FLAGS.patience),
ModelCheckpoint(filepath=os.path.join(exp_folder, 'model.h5'),
save_best_only=True),
tf.keras.callbacks.ReduceLROnPlateau(monitor="val_loss",
factor=0.5,
patience=100)
]
if FLAGS.return_sequences:
model.compile(
optimizer=optimizer,
loss=loss_func,
metrics=['accuracy'], #add metrics if wished
sample_weight_mode='temporal')
class_weights = class_weight.compute_class_weight(
'balanced', classes=[0, 1], y=train_labels.flatten())
#temporal sample weighting: give higher weight to timesteps that are labelled as 1 (rare class)
sample_weight = train_labels * class_weights[1] + class_weights[
0] - train_labels * class_weights[0]
train_labels = np.expand_dims(
train_labels, axis=-1
) #necessary because tf complains otherwise that data is not temporal
# train model
history = model.fit(train_set,
train_labels,
epochs=FLAGS.epochs,
batch_size=FLAGS.batch_size,
validation_split=0.25,
callbacks=callbacks,
sample_weight=sample_weight)
else:
model.compile(optimizer=optimizer,
loss=loss_func,
metrics=['accuracy'])
#use class weights to counteract class imbalance
class_weights = class_weight.compute_class_weight('balanced',
classes=[0, 1],
y=train_labels)
class_weights = {i: class_weights[i] for i in range(2)}
history = model.fit(train_set,
train_labels,
epochs=FLAGS.epochs,
batch_size=FLAGS.batch_size,
validation_split=0.25,
callbacks=callbacks,
class_weight=class_weights)
# save history to yaml
yaml.dump(history.history,
tf.io.gfile.GFile(os.path.join(exp_folder, 'history.yml'), 'w'))
def main(_):
# get experiment folder and create dir for plots
exp_folder = os.path.join(FLAGS.exp_root, FLAGS.exp_name,
'exp{}'.format(FLAGS.exp_nr))
test_folder = os.path.join(exp_folder, 'test')
tf.io.gfile.mkdir(test_folder)
# get experiment FLAGS
TRAINING_FLAGS = yaml.safe_load(
tf.io.gfile.GFile(os.path.join(exp_folder, 'FLAGS.yml'), 'r'))
# get dataset
test_set, test_labels = get_dataset(
'.',
TRAINING_FLAGS['num_feat'],
TRAINING_FLAGS['slice_length'],
type='test',
return_sequences=TRAINING_FLAGS['return_sequences'])
sequence_length = test_set.shape[1]
feature_dim = test_set.shape[2]
if TRAINING_FLAGS['model'] == 'tcn':
model = get_tcn(
sequence_length,
feature_dim,
nb_filters=TRAINING_FLAGS['num_filters'],
nb_stacks=TRAINING_FLAGS['num_stacks'],
use_skip_connections=TRAINING_FLAGS['use_skip_connections'],
use_batch_norm=TRAINING_FLAGS['bn'],
return_sequences=TRAINING_FLAGS['return_sequences'],
dilation_stages=TRAINING_FLAGS['dilation_stages'])
elif TRAINING_FLAGS['model'] == 'cnn':
model = get_cnn((sequence_length, feature_dim))
else:
assert False, 'Unknown model!'
model(tf.zeros((1, sequence_length, feature_dim)))
model.load_weights(os.path.join(exp_folder, 'model.h5'))
model.compile()
model.summary()
# print(model.count_params())
# weights = model.get_layer('dense').get_weights()
# kernel = weights[0]
# bias = weights[1]
# scale = 128 / max(kernel.min(), kernel.max(), bias.min(), bias.max())
# kernel_scaled = (kernel * scale).astype('int8')
# bias_scaled = (bias * scale).astype('int8')
converter = tf.lite.TFLiteConverter.from_keras_model(model)
# Convert the model to the TensorFlow Lite format with quantization
tflite_model_name = 'quant_model'
quantize = True
if (quantize):
def representative_dataset():
for i in range(100):
yield [test_set[i].reshape(1, sequence_length, feature_dim)]
# Set the optimization flag.
converter.optimizations = [tf.lite.Optimize.DEFAULT]
# Enforce full-int8 quantization
converter.target_spec.supported_ops = [
tf.lite.OpsSet.TFLITE_BUILTINS_INT8
]
converter.inference_input_type = tf.uint8 # or tf.uint8
converter.inference_output_type = tf.uint8 # or tf.uint8
# Provide a representative dataset to ensure we quantize correctly.
converter.representative_dataset = representative_dataset
tflite_model = converter.convert()
model_path = os.path.join('/tmp', tflite_model_name + '.tflite')
open(model_path, 'wb').write(tflite_model)
tflite_interpreter = tf.lite.Interpreter(model_path=model_path)
tflite_interpreter.allocate_tensors()
input_details = tflite_interpreter.get_input_details()
output_details = tflite_interpreter.get_output_details()
predictions = []
for i in range(len(test_set)):
val_batch = test_set[i]
val_batch = np.expand_dims(val_batch,
axis=0).astype(input_details[0]["dtype"])
tflite_interpreter.set_tensor(input_details[0]['index'], val_batch)
tflite_interpreter.allocate_tensors()
tflite_interpreter.invoke()
output = tflite_interpreter.get_tensor(output_details[0]['index'])
predictions += [output]
METRICS = [
tf.keras.metrics.TruePositives(name='tp'),
tf.keras.metrics.FalsePositives(name='fp'),
tf.keras.metrics.TrueNegatives(name='tn'),
tf.keras.metrics.FalseNegatives(name='fn'),
tf.keras.metrics.BinaryAccuracy(name='accuracy'),
tf.keras.metrics.Precision(name='precision'),
tf.keras.metrics.Recall(name='recall'),
tf.keras.metrics.AUC(name='auc'),
]
wrapped_metrics = list(map(lambda m: MetricWrapper(m, dims=2), METRICS))
predictions = np.stack(predictions).squeeze()
res = {}
for m in wrapped_metrics:
m.update_state(y_true=test_labels, y_pred=predictions)
res[m.name] = m.result().numpy()
with open(os.path.join(exp_folder, 'test/quant_metrics.p'),
'wb') as handle:
pickle.dump(res, handle, protocol=pickle.HIGHEST_PROTOCOL)
pprint.pprint(res)
def main(_):
# get experiment folder and create dir for plots
exp_folder = os.path.join(FLAGS.exp_root, FLAGS.exp_name,
'exp{}'.format(FLAGS.exp_nr))
test_folder = os.path.join(exp_folder, 'test')
tf.io.gfile.mkdir(test_folder)
# get experiment FLAGS
TRAINING_FLAGS = yaml.safe_load(
tf.io.gfile.GFile(os.path.join(exp_folder, 'FLAGS.yml'), 'r')
)
# get dataset
test_set, test_labels = get_dataset(TRAINING_FLAGS['data_root'], TRAINING_FLAGS['num_feat'], TRAINING_FLAGS['slice_length'], type='test',
return_sequences=TRAINING_FLAGS['return_sequences'])
sequence_length = test_set.shape[1]
feature_dim = test_set.shape[2]
if TRAINING_FLAGS['model'] == 'tcn':
model = get_tcn(sequence_length, feature_dim,
nb_filters=TRAINING_FLAGS['num_filters'],
nb_stacks=TRAINING_FLAGS['num_stacks'],
use_skip_connections=TRAINING_FLAGS['use_skip_connections'],
use_batch_norm=TRAINING_FLAGS['bn'],
return_sequences=TRAINING_FLAGS['return_sequences'],
dilation_stages=TRAINING_FLAGS['dilation_stages'])
elif TRAINING_FLAGS['model'] == 'cnn':
model = get_cnn((sequence_length, feature_dim))
else:
assert False, 'Unknown model!'
#evaluate the model to make sure it is built
model(tf.zeros((1, sequence_length, feature_dim)))
model.load_weights(os.path.join(exp_folder, 'model.h5'))
model.compile()
#count the numper of parameters in the model
parameters = model.count_params()
pred_labels = model.predict(test_set, batch_size=64, use_multiprocessing=True, workers=8)
#define metrics
METRICS = [
tf.keras.metrics.TruePositives(name='tp'),
tf.keras.metrics.FalsePositives(name='fp'),
tf.keras.metrics.TrueNegatives(name='tn'),
tf.keras.metrics.FalseNegatives(name='fn'),
tf.keras.metrics.BinaryAccuracy(name='accuracy'),
tf.keras.metrics.Precision(name='precision'),
tf.keras.metrics.Recall(name='recall'),
tf.keras.metrics.AUC(name='auc'),
]
#wrap metrics to be compatible with categorical output
wrapped_metrics = list(map(lambda m: MetricWrapper(m, dims=2), METRICS))
res = {}
for m in wrapped_metrics:
m.update_state(y_true=test_labels, y_pred=pred_labels)
res[m.name] = m.result().numpy()
res['parameters'] = parameters
#store metrics in pickle file
with open(os.path.join(exp_folder, 'test/metrics.p'), 'wb') as handle:
pickle.dump(res, handle, protocol=pickle.HIGHEST_PROTOCOL)
pprint.pprint(res)