def start_training(config, profile_step):
use_horovod = horovod_util.is_enabled()
print("use_horovod:", use_horovod)
if use_horovod:
hvd = horovod_util.setup()
rank = hvd.rank()
local_rank = hvd.local_rank()
else:
rank = 0
local_rank = -1
ModelClass = config.NETWORK_CLASS
network_kwargs = {key.lower(): val for key, val in config.NETWORK.items()}
train_dataset = setup_dataset(config, "train", rank, local_rank)
print("train dataset num:", train_dataset.num_per_epoch)
validation_dataset = setup_dataset(config, "validation", rank, local_rank)
print("validation dataset num:", validation_dataset.num_per_epoch)
graph = tf.Graph()
with graph.as_default():
if config.TASK == Tasks.OBJECT_DETECTION:
model = ModelClass(
classes=train_dataset.classes,
num_max_boxes=train_dataset.num_max_boxes,
is_debug=config.IS_DEBUG,
**network_kwargs,
)
else:
model = ModelClass(
classes=train_dataset.classes,
is_debug=config.IS_DEBUG,
**network_kwargs,
)
is_training_placeholder = tf.compat.v1.placeholder(
tf.bool, name="is_training_placeholder")
images_placeholder, labels_placeholder = model.placeholders()
output = model.inference(images_placeholder, is_training_placeholder)
loss = model.loss(output, labels_placeholder)
opt = model.optimizer()
if use_horovod:
# add Horovod Distributed Optimizer
opt = hvd.DistributedOptimizer(opt)
train_op = model.train(loss, opt)
metrics_ops_dict, metrics_update_op = model.metrics(
output, labels_placeholder)
# TODO(wakisaka): Deal with many networks.
model.summary(output, labels_placeholder)
summary_op = tf.compat.v1.summary.merge_all()
metrics_summary_op = executor.metrics_summary_op(metrics_ops_dict)
init_op = tf.compat.v1.global_variables_initializer()
reset_metrics_op = tf.compat.v1.local_variables_initializer()
if use_horovod:
# add Horovod broadcasting variables from rank 0 to all
bcast_global_variables_op = hvd.broadcast_global_variables(0)
saver = tf.compat.v1.train.Saver(
max_to_keep=config.KEEP_CHECKPOINT_MAX)
with file_io.File(os.path.join(environment.EXPERIMENT_DIR,
"pretrain_vars.txt"),
mode="w") as f:
train_vars = tf.compat.v1.trainable_variables()
f.write("[\n")
f.write("".join(" '%s',\n" % var.name for var in train_vars))
f.write("]\n")
if config.IS_PRETRAIN:
all_vars = tf.compat.v1.global_variables()
pretrain_var_list = [
var for var in all_vars
if var.name.startswith(tuple(config.PRETRAIN_VARS))
]
print("pretrain_vars", [var.name for var in pretrain_var_list])
pretrain_saver = tf.compat.v1.train.Saver(pretrain_var_list,
name="pretrain_saver")
if use_horovod:
# For distributed training
session_config = tf.compat.v1.ConfigProto(
gpu_options=tf.compat.v1.GPUOptions(
allow_growth=True, visible_device_list=str(hvd.local_rank())))
else:
# TODO(wakisaka): For debug.
# session_config = tf.ConfigProto(
# gpu_options=tf.GPUOptions(
# allow_growth=True,
# per_process_gpu_memory_fraction=0.1
# )
# )
session_config = tf.compat.v1.ConfigProto(
) # tf.ConfigProto(log_device_placement=True)
# TODO(wakisaka): XLA JIT
# session_config.graph_options.optimizer_options.global_jit_level = tf.OptimizerOptions.ON_1
sess = tf.compat.v1.Session(graph=graph, config=session_config)
sess.run([init_op, reset_metrics_op])
executor.save_pb_file(sess, environment.CHECKPOINTS_DIR)
if rank == 0:
train_writer = tf.compat.v1.summary.FileWriter(
environment.TENSORBOARD_DIR + "/train", sess.graph)
val_writer = tf.compat.v1.summary.FileWriter(
environment.TENSORBOARD_DIR + "/validation")
if config.IS_PRETRAIN:
print("------- Load pretrain data ----------")
pretrain_saver.restore(
sess, os.path.join(config.PRETRAIN_DIR, config.PRETRAIN_FILE))
# for recovery
ckpt = tf.train.get_checkpoint_state(environment.CHECKPOINTS_DIR)
if ckpt and ckpt.model_checkpoint_path:
print("--------- Restore last checkpoint -------------")
saver.restore(sess, ckpt.model_checkpoint_path)
# saver.recover_last_checkpoints(ckpt.model_checkpoint_path)
last_step = sess.run(model.global_step)
# TODO(wakisaka): tensorflow v1.3 remain previous event log in tensorboard.
# https://github.com/tensorflow/tensorflow/blob/r1.3/tensorflow/python/training/supervisor.py#L1072
train_writer.add_session_log(SessionLog(status=SessionLog.START),
global_step=last_step + 1)
val_writer.add_session_log(SessionLog(status=SessionLog.START),
global_step=last_step + 1)
print("recovered. last step", last_step)
if use_horovod:
# broadcast variables from rank 0 to all other processes
sess.run(bcast_global_variables_op)
last_step = sess.run(model.global_step)
# Calculate max steps. The priority of config.MAX_EPOCHS is higher than config.MAX_STEPS.
if "MAX_EPOCHS" in config:
max_steps = int(train_dataset.num_per_epoch / config.BATCH_SIZE *
config.MAX_EPOCHS)
if max_steps < 1:
print(
"The max_steps is less than 1, consider reduce BATCH_SIZE. exit.",
file=sys.stderr)
sys.exit(1)
else:
max_steps = config.MAX_STEPS
if max_steps < 1:
print(
"The max_steps is less than 1, consider set MAX_STEPS greater than 0. exit.",
file=sys.stderr)
sys.exit(1)
progbar = Progbar(max_steps)
if rank == 0:
progbar.update(last_step)
for step in range(last_step, max_steps):
images, labels = train_dataset.feed()
feed_dict = {
is_training_placeholder: True,
images_placeholder: images,
labels_placeholder: labels,
}
# Runtime statistics for develop.
if step == profile_step:
options = tf.compat.v1.RunOptions(
trace_level=tf.compat.v1.RunOptions.FULL_TRACE)
run_meta = tf.compat.v1.RunMetadata()
else:
options = None
run_meta = None
if step * ((step + 1) % config.SUMMARISE_STEPS) == 0 and rank == 0:
sess.run(reset_metrics_op)
_, summary, _ = sess.run(
[train_op, summary_op, metrics_update_op],
feed_dict=feed_dict,
options=options,
run_metadata=run_meta,
)
# train_writer.add_run_metadata(run_metadata, "step: {}".format(step + 1))
train_writer.add_summary(summary, step + 1)
metrics_summary = sess.run(metrics_summary_op)
train_writer.add_summary(metrics_summary, step + 1)
train_writer.flush()
else:
sess.run([train_op],
feed_dict=feed_dict,
options=options,
run_metadata=run_meta)
if step == profile_step:
executor.profile_train_step(step, sess, run_meta)
to_be_saved = step == 0 or (
step +
1) == max_steps or (step + 1) % config.SAVE_CHECKPOINT_STEPS == 0
if to_be_saved and rank == 0:
_save_checkpoint(saver, sess, model.global_step)
if step == 0 or (step + 1) % config.TEST_STEPS == 0:
# init metrics values
sess.run(reset_metrics_op)
test_step_size = int(
math.ceil(validation_dataset.num_per_epoch /
config.BATCH_SIZE))
for test_step in range(test_step_size):
images, labels = validation_dataset.feed()
feed_dict = {
is_training_placeholder: False,
images_placeholder: images,
labels_placeholder: labels,
}
if test_step % config.SUMMARISE_STEPS == 0:
summary, _ = sess.run([summary_op, metrics_update_op],
feed_dict=feed_dict)
if rank == 0:
val_writer.add_summary(summary, step + 1)
val_writer.flush()
else:
sess.run([metrics_update_op], feed_dict=feed_dict)
metrics_summary = sess.run(metrics_summary_op)
if rank == 0:
val_writer.add_summary(metrics_summary, step + 1)
val_writer.flush()
if rank == 0:
progbar.update(step + 1)
# training loop end.
train_dataset.close()
validation_dataset.close()
print("Done")
test_accuracy = tf.keras.metrics.CategoricalAccuracy()
test_loss = tf.keras.metrics.CategoricalCrossentropy(from_logits=True)
from tensorflow.keras.utils import Progbar
model = model_mnist(0) # ！！！
source = []
for i in range(4): # ！！！
source.append(
tf.keras.models.load_model('./Models/ens/model_' + str(i) + '.h5'))
for epoch in range(num_epochs):
train_dataset = get_training_dataset()
print("\nepoch {}/{}".format(epoch + 1, num_epochs))
pb_i = Progbar(x_train.shape[0],
stateful_metrics=['loss', 'acc'],
verbose=2)
for x, y in train_dataset:
i = np.random.choice([0, 2, 3]) # ！！！
# PGD Adversarial Training
# adv_train(model,x,y,adv_params='pgd_76_10_8',random_init=True) #！！！
# Ensemble Adversarial Training
## x_adv = generate_adversarial_examples(x, bounds, source[i], 'pgd_76_76_1', random_init=False)
## normal_train(model,x,y)
# ens_train(source[i],model,x,y,adv_params='pgd_76_76_1',random_init=False) #！！！
# Projection Regularization
pr_train(source[i], model, x, y, alpha=0.3, beta=10,
sigma=sigma) # ！！！
def eval_results(ds_name, encoder, paths):
"""The main function for executing network training. It loads the specified
dataset iterator, saliency model, and helper classes. Training is then
performed in a new session by iterating over all batches for a number of
epochs. After validation on an independent set, the model is saved and
the training history is updated.
Args:
ds_name (str): Denotes the dataset to be used during training.
paths (dict, str): A dictionary with all path elements.
"""
w_filename_template = "/%s_%s_%s_weights.h5" # [encoder]_[ds_name]_[loss_fn_name]_weights.h5
(eval_ds, n_eval) = data.load_eval_dataset(ds_name, paths["data"])
print(">> Preparing model with encoder %s..." % encoder)
model = MyModel(encoder, ds_name, "train")
if "trained_weights" in paths:
if os.path.exists(paths["trained_weights"]):
weights_path = paths["trained_weights"]
else:
raise ValueError("could not find the specified weights file.\n specified weights: %s"%paths["trained_weights"])
else:
weights_path = paths["weights"] + w_filename_template % (encoder, ds_name, loss_fn_name)
if os.path.exists(weights_path):
print("Weights are loaded!\n %s"%weights_path)
else:
download.download_pretrained_weights(paths["weights"], encoder, "salicon", loss_fn_name)
model.load_weights(weights_path)
del weights_path
model.summary()
# Preparing
metrics = config.PARAMS["metrics"]
print("\n>> Start evaluating model on %s..." % ds_name.upper())
print(("Evaluation details:" +
"\n{0:<4}Metrics: {2}").format(" ", loss_fn_name, ", ".join(metrics), **config.PARAMS))
print("_" * 65)
eval_progbar = Progbar(n_eval)
categorical = config.SPECS[ds_name].get("categorical", False)
cat_metrics = {}
for eval_x, eval_fixs, eval_y_true, eval_ori_sizes, eval_filenames in eval_ds:
eval_y_pred = test_step(eval_x, model)
for pred, y_true, fixs, filename, ori_size in zip(eval_y_pred, eval_fixs, eval_y_true, eval_filenames.numpy(), eval_ori_sizes):
pred = tf.expand_dims(data.postprocess_saliency_map(pred, ori_size), axis=0)
fixs = tf.expand_dims(fixs, axis=0)
y_true = tf.expand_dims(y_true, axis=0)
met_vals = _calc_metrics(metrics, y_true, fixs, pred)
if categorical:
cat = "/".join(filename.decode("utf-8").split("/")[:-1])
if not cat in cat_metrics:
cat_metrics[cat] = {}
for name in metrics:
cat_metrics[cat][name] = {"sum":0, "count": 0}
for name, value in met_vals.items():
cat_metrics[cat][name]["sum"] += value
cat_metrics[cat][name]["count"] += 1
eval_progbar.add(eval_x.shape[0], met_vals.items())
for cat, cat_met in cat_metrics.items():
to_print = []
for name, value in cat_met.items():
_mean = value["sum"]/value["count"]
to_print.append("{}: {}".format(name, ('%.4f' if _mean > 1e-3 else '%.4e') % _mean))
print('Results ({}): {}'.format(cat, " - ".join(to_print)))
def validation_loop(sess,
g,
n_batches,
chars=None,
val_gen=None,
tb_writer=None):
Loss = []
Cer = []
Wer = []
Predictions = []
progbar = Progbar(target=n_batches, verbose=1, stateful_metrics=['t'])
print('Starting validation Loop')
for i in range(n_batches):
x, y = val_gen.next()
if len(x) == 1: x = x[0]
if len(y) == 1: y = y[0]
# -- Autoregressive inference
preds = np.zeros((config.batch_size, config.maxlen), np.int32)
tile_preds = config.test_aug_times
# -- For train graph feed in the previous step's predictions manually for the next
if not 'infer' in config.graph_type:
prev_inp = np.tile(
preds, [config.test_aug_times, 1]) if tile_preds else preds
feed_dict = {g.x: x, g.prev: prev_inp, g.y: y}
enc = sess.run(g.enc, feed_dict)
if type(enc) is list:
for enc_tens, enc_val in zip(g.enc, enc):
feed_dict[enc_tens] = enc_val
else:
feed_dict[g.enc] = enc
for j in range(config.maxlen):
_preds, loss, cer = sess.run([g.preds, g.mean_loss, g.cer],
feed_dict)
preds[:, j] = _preds[:, j]
prev_inp = np.tile(
preds, [config.test_aug_times, 1]) if tile_preds else preds
feed_dict[g.prev] = prev_inp
# if all samples in batch predict the pad symbol (char_id==0)
if np.sign(preds[:, j]).sum() == 0:
if g.tb_sum is not None:
tb_sum = sess.run(g.tb_sum, {
g.x: x,
g.prev: prev_inp,
g.y: y
})
break
# -- Autoregression loop is built into the beam search graph
else:
feed_dict = {g.x: x, g.y: y}
enc = sess.run(g.enc, feed_dict)
if type(enc) is list:
for enc_tens, enc_val in zip(g.enc, enc):
feed_dict[enc_tens] = enc_val
else:
feed_dict[g.enc] = enc
_preds, loss, cer = sess.run([g.preds, g.mean_loss, g.cer],
feed_dict)
preds = _preds
# use last loss
gt_sents = [''.join([chars[cid] for cid in prr]).strip() for prr in y]
gt_words = [sent.split('-') for sent in gt_sents]
def decode_preds_to_chars(decoding):
return ''.join([chars[cid] for cid in decoding]).strip()
pred_sentences = [decode_preds_to_chars(prr) for prr in preds]
pred_words = [sent.split('-') for sent in pred_sentences]
edists = [
rel_edist(gt, dec_str)
for gt, dec_str in zip(gt_words, pred_words)
]
wer = np.mean(edists)
# -- Write tb_summaries if any
if g.tb_sum is not None:
if wer == 0:
tb_writer.add_summary(tb_sum, i)
if config.print_predictions:
print()
for gts, prs, wr in zip(gt_sents, pred_sentences, edists):
print('(wer={:.1f}) {} --> {}'.format(wr * 100, gts, prs))
progbar.update(i + 1, [('cer', cer), ('wer', wer)])
Wer.append(wer)
Predictions.append(pred_sentences)
Cer.append(cer)
Loss.append(loss)
return np.average(Loss), np.average(Cer), np.average(Wer), Predictions[0]
def main(_):
with open(os.path.join(FLAGS.configuration_dir, "config.json")) as f:
configuration = json.load(f)
add_conf_dir_to_paths(configuration, FLAGS.configuration_dir)
vocab_file = configuration["general"]["vocab_file"]
input_data_file = configuration["pretraining"]["data-generation"][
"input_data_file"]
num_shards = configuration["pretraining"]["data-generation"]["num_shards"]
random_seed = configuration["pretraining"]["data-generation"][
"random_seed"]
max_seq_length = configuration["general"]["max_seq_length"]
dupe_factor = configuration["pretraining"]["data-generation"][
"dupe_factor"]
max_predictions_per_seq = configuration["pretraining"]["data-generation"][
"max_predictions_per_seq"]
masked_lm_prob = configuration["pretraining"]["data-generation"][
"masked_lm_prob"]
output_data_dir = configuration["pretraining"]["data-generation"][
"output_data_dir"]
bert_config_file = configuration["general"]["bert_config_file"]
bert_config = modeling.BertConfig.from_json_file(bert_config_file)
tf.logging.set_verbosity(tf.logging.INFO)
tokenizer = tokenization.SpaceTokenizer(vocab=vocab_file,
vocab_size=bert_config.vocab_size)
input_files = [input_data_file]
output_files = [
os.path.join(output_data_dir, "train_{0:04d}.tfrecord".format(i))
for i in range(num_shards)
]
tf.logging.info("*** Reading from input files ***")
for input_file in input_files:
tf.logging.info(" %s", input_file)
tf.logging.info("*** Writing to output files ***")
for output_file in output_files:
tf.logging.info(" %s", output_file)
writers = []
for output_file in output_files:
writers.append(tf.python_io.TFRecordWriter(output_file))
rng = random.Random(random_seed)
bar = Progbar(num_shards)
instances_written = 0
all_examples = []
for i in range(num_shards):
bar.add(1)
examples, instances = create_training_instances(
(i, num_shards), input_files, writers, tokenizer, max_seq_length,
dupe_factor, masked_lm_prob, max_predictions_per_seq, rng)
instances_written += instances
all_examples.extend(examples)
rng.shuffle(all_examples)
for example in all_examples[:FLAGS.num_show_examples]:
write_instance_to_example_files(example, None, tokenizer,
max_seq_length,
max_predictions_per_seq, True)
tf.logging.info("Wrote %d instances.\n", instances_written)
for writer in writers:
writer.close()
def train(training_data, validation_data, cfg):
x_train, y_train, y_train_cat = training_data
x_test, y_test, y_test_cat = validation_data
test_indices = np.arange(len(x_test))
# get the training data generator. We are not using validation generator because the
# data is already loaded in memory and we don't have to perform any extra operation
# apart from loading the validation images and validation labels.
ds = DataGenerator(x_train, y_train_cat, batch_size=cfg.batch_size)
enqueuer = OrderedEnqueuer(ds, use_multiprocessing=True)
enqueuer.start(workers=multiprocessing.cpu_count())
train_ds = enqueuer.get()
# get the total number of training and validation steps
nb_train_steps = int(np.ceil(len(x_train) / cfg.batch_size))
nb_test_steps = int(np.ceil(len(x_test) / cfg.batch_size))
global total_steps
total_steps = nb_train_steps * cfg.num_epochs
# get the optimizer
# SGD with cosine lr is causing NaNs. Need to investigate more
optim = optimizers.Adam(learning_rate=0.001)
# checkpoint prefix
checkpoint_prefix = os.path.join(cfg.save_dir_path, "ckpt")
checkpoint = tf.train.Checkpoint(optimizer=optim, model=model)
checkpoint_manager = tf.train.CheckpointManager(
checkpoint, directory=cfg.save_dir_path, max_to_keep=10)
# check for previous checkpoints, if any
checkpoint.restore(checkpoint_manager.latest_checkpoint)
if checkpoint_manager.latest_checkpoint:
print("Checkpoint restored from {}".format(
checkpoint_manager.latest_checkpoint))
starting_epoch = checkpoint.save_counter.numpy()
else:
print("Initializing from scratch.")
starting_epoch = 0
# sanity check for epoch number. For example, if someone restored a checkpoint
# from 15th epoch and want to train for two more epochs, then we need to explicitly
# encode this logic in the for loop
if cfg.num_epochs <= starting_epoch:
cfg.num_epochs += starting_epoch
for epoch in range(starting_epoch, cfg.num_epochs):
pbar = Progbar(target=nb_train_steps, interval=0.5, width=30)
# Train for an epoch and keep track of
# loss and accracy for each batch.
for bno, (images, labels) in enumerate(train_ds):
if bno == nb_train_steps:
break
# Get the batch data
clean, aug1, aug2 = images
loss_value, y_pred_clean = train_step(clean, aug1, aug2, labels,
optim)
# Record batch loss and batch accuracy
train_loss(loss_value)
train_accuracy(labels, y_pred_clean)
pbar.update(bno + 1)
# Validate after each epoch
for bno in range(nb_test_steps):
# Get the indices for the current batch
indices = test_indices[bno * cfg.batch_size:(bno + 1) *
cfg.batch_size]
# Get the data
images, labels = x_test[indices], y_test_cat[indices]
# Get the predicitions and loss for this batch
loss_value, y_pred = validate_step(images, labels)
# Record batch loss and accuracy
test_loss(loss_value)
test_accuracy(labels, y_pred)
# get training and validataion stats
# after one epoch is completed
loss = train_loss.result()
acc = train_accuracy.result()
val_loss = test_loss.result()
val_acc = test_accuracy.result()
# record values in the history object
history.update([loss, acc], [val_loss, val_acc])
# print loss values and accuracy values for each epoch
# for both training as well as validation sets
print(f"""Epoch: {epoch+1}
train_loss: {loss:.6f} train_acc: {acc*100:.2f}%
test_loss: {val_loss:.6f} test_acc: {val_acc*100:.2f}%\n""")
## wandb logging
wandb.log({
'Epoch': epoch,
'train loss': loss,
'train acc': acc,
'test loss': val_loss,
'test acc': val_acc
})
# get the model progress
improved, stop_training = es.check_progress(val_loss)
# check if performance of model has imporved or not
if improved:
print("Saving model checkpoint.")
checkpoint.save(checkpoint_prefix)
if stop_training:
break
# plot and save progression
history.plot_and_save(initial_epoch=starting_epoch)
# Reset the losses and accuracy
train_loss.reset_states()
train_accuracy.reset_states()
test_loss.reset_states()
test_accuracy.reset_states()
print("")
print("*" * 78)
print("")
def load_weights_from_tf_checkpoint(model, checkpoint_file, background_label):
print('Load weights from tensorflow checkpoint')
progbar = Progbar(target=len(model.layers))
reader = tf.compat.v1.train.NewCheckpointReader(checkpoint_file)
# pprint.pprint(reader.debug_string().decode("utf-8")) #类型是str
for index, layer in enumerate(model.layers):
progbar.update(current=index)
print("layer.name", layer.name)
print("layer", layer)
if isinstance(layers, MobileNetV3Extractor):
for index, llayers in enumerate(layers.layers):
progbar.update(current=index)
print("llayers.name", llayers.name)
if isinstance(llayers, layers.SeparableConv2D):
depthwise = reader.get_tensor(
'{}/depthwise_weights'.format(llayers.name))
pointwise = reader.get_tensor(
'{}/pointwise_weights'.format(llayers.name))
if K.image_data_format() == 'channels_first':
depthwise = convert_kernel(depthwise)
pointwise = convert_kernel(pointwise)
llayers.set_weights([depthwise, pointwise])
elif isinstance(llayers, layers.Conv2D):
weights = reader.get_tensor('{}/weights'.format(
llayers.name))
if K.image_data_format() == 'channels_first':
weights = convert_kernel(weights)
llayers.set_weights([weights])
elif isinstance(llayers, layers.BatchNormalization):
beta = reader.get_tensor('{}/beta'.format(llayers.name))
gamma = reader.get_tensor('{}/gamma'.format(llayers.name))
moving_mean = reader.get_tensor('{}/moving_mean'.format(
llayers.name))
moving_variance = reader.get_tensor(
'{}/moving_variance'.format(llayers.name))
llayers.set_weights(
[gamma, beta, moving_mean, moving_variance])
elif isinstance(llayers, layers.Dense):
weights = reader.get_tensor('{}/weights'.format(
llayers.name))
biases = reader.get_tensor('{}/biases'.format(
llayers.name))
if background_label:
llayers.set_weights([weights, biases])
else:
llayers.set_weights([weights[:, 1:], biases[1:]])
elif isinstance(layer, layers.SeparableConv2D):
depthwise = reader.get_tensor('{}/depthwise_weights'.format(
layer.name))
pointwise = reader.get_tensor('{}/pointwise_weights'.format(
layer.name))
if K.image_data_format() == 'channels_first':
depthwise = convert_kernel(depthwise)
pointwise = convert_kernel(pointwise)
layer.set_weights([depthwise, pointwise])
elif isinstance(layer, layers.Conv2D):
weights = reader.get_tensor('{}/weights'.format(layer.name))
if K.image_data_format() == 'channels_first':
weights = convert_kernel(weights)
layer.set_weights([weights])
elif isinstance(layer, layers.BatchNormalization):
beta = reader.get_tensor('{}/beta'.format(layer.name))
gamma = reader.get_tensor('{}/gamma'.format(layer.name))
moving_mean = reader.get_tensor('{}/moving_mean'.format(
layer.name))
moving_variance = reader.get_tensor('{}/moving_variance'.format(
layer.name))
layer.set_weights([gamma, beta, moving_mean, moving_variance])
elif isinstance(layer, layers.Dense):
weights = reader.get_tensor('{}/weights'.format(layer.name))
biases = reader.get_tensor('{}/biases'.format(layer.name))
if background_label:
layer.set_weights([weights, biases])
else:
layer.set_weights([weights[:, 1:], biases[1:]])
def represent(self, molecules):
"""
provides bag of bonds representation for input molecules.
Parameters
----------
molecules: chemml.chem.Molecule object or array
If list, it must be a list of chemml.chem.Molecule objects, otherwise we raise a ValueError.
In addition, all the molecule objects must provide the XYZ information. Please make sure the XYZ geometry has been
stored or optimized in advance.
Returns
-------
features: pandas data frame, shape: (n_molecules, max_length_of_combinations)
The bag of bond features.
"""
if isinstance(molecules, (list, np.ndarray)):
molecules = np.array(molecules)
elif isinstance(molecules, Molecule):
molecules = np.array([molecules])
else:
msg = "The input molecules must be a chemml.chem.Molecule object or a list of objects."
raise ValueError(msg)
if molecules.ndim > 1:
msg = "The molecule must be a chemml.chem.Molecule object or a list of objects."
raise ValueError(msg)
# pool of processes
if self.n_jobs == -1:
self.n_jobs = cpu_count()
pool = Pool(processes=self.n_jobs)
# Create an iterator
# http://stackoverflow.com/questions/312443/how-do-you-split-a-list-into-evenly-sized-chunks
def chunks(l, n):
"""Yield successive n-sized chunks from l."""
for i in range(0, len(l), n):
yield l[i:i + n]
# find size of each batch
batch_size = int(len(molecules) / self.n_jobs)
if batch_size == 0:
batch_size = 1
molecule_chunks = chunks(molecules, batch_size)
# MAP: CM in parallel
map_function = partial(self._represent)
if self.verbose:
print('featurizing molecules in batches of %i ...' % batch_size)
pbar = Progbar(len(molecules), width=50)
bbs_info = []
for tensors in pool.imap(map_function, molecule_chunks):
pbar.add(len(tensors[0]))
bbs_info.append(tensors)
print('Merging batch features ... ', end='')
else:
bbs_info = pool.map(map_function, molecule_chunks)
if self.verbose:
print('[DONE]')
# REDUCE: Concatenate the obtained tensors
pool.close()
pool.join()
return self.concat_mol_features(bbs_info)
def model_fit(model,
train,
test,
epochs=1,
metrics=[],
desc=None,
debug=False):
"""Custom implementation of tf.keras.models.Model.fit.
See https://github.com/tensorflow/tensorflow/issues/39448
for why this is necessary (how the f**k has this not been fixed?)
Parameters
----------
model : tf.keras.models.Model
Model to train.
train : tf.data.Dataset
Train dataset; should be batched and shuffled already if required.
test : tf.data.Dataset
Test dataset
Keyword Args
------------
epochs : int
Number of epochs to run.
metrics : [callable(tf.Tensor, tf.Tensor) -> float]
List of tensorflow metrics to evaluate.
desc : str
Description for display.
debug : bool
Whether to log debug information from optimizer.get_debug_summary().
"""
strategy = tf.distribute.get_strategy()
# Distribute datasets to replicas
train = strategy.experimental_distribute_dataset(train)
test = strategy.experimental_distribute_dataset(test)
start_time = time.time()
# Train
def _train_step(batch):
x, y = batch
with tf.GradientTape() as tape:
y_hat = model(x, training=True)
loss = model.compiled_loss(y, y_hat)
grads = tape.gradient(loss, model.trainable_variables)
model.optimizer.apply_gradients(zip(grads, model.trainable_variables))
for m in metrics:
m.update_state(y, y_hat)
return loss
@tf.function
def train_step(batch):
losses = strategy.run(_train_step, args=(batch, ))
return strategy.reduce(tf.distribute.ReduceOp.SUM, losses, axis=None)
# Test
def _test_step(batch):
x, y = batch
y_hat = model(x, training=False)
loss = model.compiled_loss(y, y_hat)
for m in metrics:
m.update_state(y, y_hat)
return loss
@tf.function
def test_step(batch):
losses = strategy.run(_test_step, args=(batch, ))
return strategy.reduce(tf.distribute.ReduceOp.SUM, losses, axis=None)
# Train/test loop
def run_loop(dataset, step, callback=None):
losses = []
times = []
for batch in dataset:
losses.append(step(batch).numpy())
times.append(time.time() - start_time)
if callback is not None:
callback()
metric_values = [m.result() for m in metrics]
for m in metrics:
m.reset_states()
return losses, times, metric_values
# List of stats to log
stats = {
"batch_loss": [],
"batch_time": [],
"loss": [],
"val_loss": [],
"epoch_time": [],
}
for m in metrics:
stats[m.name] = []
stats["val_" + m.name] = []
# Debug
if debug:
trace = []
def log_debug():
trace.append(
model.optimizer.get_debug_summary(model.trainable_variables))
else:
log_debug = None
# Epoch loop
pbar = Progbar(epochs, unit_name='epoch')
for _ in range(epochs):
train_loss, train_time, train_metrics = run_loop(train,
train_step,
callback=log_debug)
stats["batch_loss"] += train_loss
stats["batch_time"] += train_time
stats["loss"].append(np.mean(train_loss))
stats["epoch_time"].append(time.time() - start_time)
for m, val in zip(metrics, train_metrics):
stats[m.name].append(val)
test_loss, test_time, test_metrics = run_loop(test, test_step)
stats["val_loss"].append(np.mean(test_loss))
for m, val in zip(metrics, test_metrics):
stats["val_" + m.name].append(val)
pbar.add(1,
values=[("train", stats["loss"][-1]),
("val", stats["val_loss"][-1])])
res = {k: np.array(v, dtype=np.float32) for k, v in stats.items()}
if debug:
res.update(model.optimizer.aggregate_debug_data(trace))
return res
def matrix(self, lastfm, tags=None, dim=3, save_to=None):
''' Computes a n-dimensional matrix where the (i_1, ... ,i_n)-th entry contains the number of tracks having all the i_1-th, ..., i_n-th tags (where the i's are the indexes in self.m_tags).
Notes
-----
To optimize performance, values are computed only with indexes in increasing order (which means, we only compute the number of tracks having tag-0
and tag-1, not vice-versa). This is something to keep in mind when indexing the matrix.
To optimize memory, the matrix is saved in sparse format. DOK is the preferred sparse format for building and indexing, while COO is the preferred
sparse format to perform mathematical operations).
The dimension of the matrix captures the kind of queries which you will be able to perform. A matrix of dim=2 on tracks=['rock', 'pop', 'hip-hop'] will
capture how many tracks have tags rock and pop, or pop and hip-hop, but not rock, pop and hip-hop at the same time.
A matrix of dim=len(tags) will fully describe the database (or the subset of the database having the given tags).
A matrix of dim>len(tags) will be rather pointless (but we won't prevent you from doing it).
Parameters
----------
lastfm: LastFm, LastFm2Pandas
Instance of tags database. Using LastFm2Pandas is strongly recommended here.
tags: list
List of tags to use. If None, all the tags will be used.
dim: int
The dimension of the matrix.
save_to: str
Filename or full path of the .npz file to save matrix and matrix tags. Use to load_from in the future.
'''
# initialize matrix tags
if tags is None:
tags = lastfm.get_tags()
else:
tags = [tag for tag in tags if tag in lastfm.get_tags()
] # possibly purge inexistent tags
# initialize matrix
matrix = sparse.DOK(
(len(tags), ) * dim, dtype=np.int32
) # sparse dict-of-keys matrix (for easy creation, awful for calculations)
# compute total number of steps to comatplotlibetion (see http://www.iosrjournals.org/iosr-jm/papers/Vol8-issue3/A0830110.pdf)
n_steps = crazysum(n=len(tags), s=3, k=dim - 1)
# check whether a progress bar is needed
verbose = n_steps > 100
if verbose:
progbar = Progbar(n_steps) # instantiate progress bar
def count_intersect_tags(tags):
tids_list = [lastfm.with_tag(tag) for tag in tags]
tids_list.sort(key=len, reverse=True)
tids = set(
tids_list.pop()
) # start with shortest list of tids to improve performance; convert to set to be able to intersect
for _ in range(len(tids_list)):
tids = tids.intersection(tids_list.pop(
)) # intersections performed from shortest list to longest
return len(tids) # how many tids have all tags
def count_intersect_tags_recursive(
tags_idxs, dim
): # recursively iterate count_intersect_tags dim times; avoid repetitions such as 'rock AND pop AND folk' vs. 'rock AND folk AND pop' vs. 'folk AND pop AND rock'
if dim >= 1:
for i in range(tags_idxs[-1] + 1):
count_intersect_tags_recursive(tags_idxs + (i, ), dim - 1)
else:
matrix[tags_idxs] = count_intersect_tags(
np.take(tags, tags_idxs)) # add count to sparse matrix
if verbose:
progbar.add(1)
# instantiate recursive loop
for i in range(len(tags)):
count_intersect_tags_recursive((i, ), dim - 1)
matrix = matrix.to_coo() # convert to coordinate matrix
if save_to is not None:
# save matrix
sparse.save_npz(save_to, matrix.to_coo(
)) # default to compressed format (i.e. sparse format)
# save matrix tags in serialized format
with open(os.path.splitext(save_to)[0] + '.nfo', 'wb') as f:
pickle.dump(tags, f)
return matrix, tags
def fit_generator(self,
generator=None,
batch_size=None,
epochs=1,
verbose=True,
logdir=None,
ckptdir=None,
n_recent=100,
initial_epoch=0,
steps_per_epoch=None,
generator_val=None,
batch_size_val=None,
freq_val=10,
**kwargs):
# Initialize training
if initial_epoch == 0:
self.sess.run(self.init)
if steps_per_epoch is None:
steps_per_epoch = 1
# Progress
freq_prog = 100
progbar = Progbar(
steps_per_epoch,
stateful_metrics=['loss', 'val loss', 'accuracy', 'val accuracy'])
# Summary (Tensorboard)
freq_log = 100
if not logdir is None:
logdir = os.path.join(
logdir, datetime.strftime(datetime.utcnow(), "%Y%m%d_%H%M%S"))
flag_log = True
train_writer = tf.summary.FileWriter(logdir,
self.graph,
flush_secs=5)
else:
flag_log = False
# Checkpoint
freq_ckpt = freq_val
if not ckptdir is None:
flag_ckpt = True
fn_ckpt = os.path.join(ckptdir, 'WE_Label')
# Save best model
else:
flag_ckpt = False
def check_point(step):
self.saver.save(self.sess, fn_ckpt, global_step=step)
def load_best():
self.saver.restore(self.sess, tf.train.latest_checkpoint(ckptdir))
# History
history = {
'train': deque(maxlen=n_recent),
'val': deque(maxlen=n_recent)
}
def is_plateau():
h = history['val'] if history['val'] else history['train']
return len(h) == h.maxlen and h[0] == max(h)
best = deque(maxlen=1)
def check_best(loss):
if not best or loss <= best[0]:
best.append(loss)
return True
else:
return False
# Validation
if not generator_val is None:
flag_val = True
if batch_size_val is None:
batch_size_val = batch_size
else:
flag_val = False
# Stopping
min_lr = 1E-5
# We must initialize all variables before we use them.
self.sess.run(self.init)
print('Initialized')
for step in range(1, steps_per_epoch + 1):
def should(freq):
return step == 1 or step % freq == 0
batch_x, batch_y = self._preprocess_batch(*generator(batch_size))
feed_dict = {self.x: batch_x, self.y: batch_y}
# We perform one update step by evaluating the optimizer op (including it
# in the list of returned values for session.run()
# Also, evaluate the merged op to get all summaries from the returned
# "summary" variable. Feed metadata variable to session for visualizing
# the graph in TensorBoard.
_, loss, accuracy = self.sess.run(
[self.optimizer, self.loss, self.accuracy],
feed_dict=feed_dict)
if flag_val and should(freq_val):
batch_val_x, batch_val_y = self._preprocess_batch(
*generator_val(batch_size_val))
feed_dict = {self.x: batch_val_x, self.y: batch_val_y}
loss_val, accuracy_val = self.sess.run(
[self.loss, self.accuracy], feed_dict=feed_dict)
history['val'].append(-loss_val)
else:
history['train'].append(-loss)
if verbose and should(freq_prog):
progbar.update(step, [
('loss', np.mean([loss])),
('val loss', np.mean([loss_val])),
('accuracy', np.mean([accuracy])),
('val accuracy', np.mean([accuracy_val])),
])
if flag_log and should(freq_log):
for x in (self.sess.run(self.loss_summary,
feed_dict={self.loss: loss}),
self.sess.run(self.accuracy_summary,
feed_dict={self.accuracy: accuracy})):
train_writer.add_summary(x, step)
if flag_val:
for x in (self.sess.run(self.loss_val_summary,
feed_dict={self.loss: loss_val}),
self.sess.run(
self.accuracy_val_summary,
feed_dict={self.accuracy: accuracy_val})):
train_writer.add_summary(x, step)
if flag_ckpt and should(freq_ckpt):
val_check = loss_val if flag_val else loss
if check_best(val_check):
print('\rSave BEST model at step {0} ({1:.3f})'.format(
step, val_check),
end='',
flush=True)
check_point(step)
if is_plateau():
print('\nPlateau reached.'.format(step))
# Load best model
load_best()
# Try reduce learning rate
self.learning_rate = self.learning_rate * 0.5
print('\nReduce lr to {}'.format(
self.sess.run(self.learning_rate)))
# Flush history
[x.clear() for x in history.values()]
if self.sess.run(self.learning_rate) < min_lr:
print('\nLearning rate {0} reaches limit: < {1}.'.format(
self.sess.run(self.learning_rate), min_lr))
break
# Make sure to load the best model
load_best()
# Get fit embedding
self.V, self.Uw, self.Ub = self.sess.run(
[self.embeddings, self.weights, self.biases], feed_dict={})
# Combine weight and bias (intercept) for U (label embedding)
self.U = np.concatenate((self.Uw, self.Ub[..., np.newaxis]), axis=1)
def val(sess, val_vars):
sess.run([val_vars.it.initializer, tf.local_variables_initializer()])
val_progbar = Progbar(None, stateful_metrics=['loss', 'acc', 'num_word',])
preds = []
preds_b = []
preds_m = []
preds_s = []
preds_d = []
gts = []
indices = tf.argmax(val_vars.logits, 1)
feed_dict = [val_vars.loss_metric, val_vars.acc_metric, val_vars.num_word_metric, indices, val_vars.labels]
if FLAGS.bmsd:
unshift = 1 if FLAGS.zero_based else 0
indices_b = tf.argmax(val_vars.logits_b, 1)+unshift
indices_m = tf.argmax(val_vars.logits_m, 1)+unshift
indices_s = tf.argmax(val_vars.logits_s, 1)+unshift
indices_d = tf.argmax(val_vars.logits_d, 1)+unshift
feed_dict.append(indices_b)
feed_dict.append(indices_m)
feed_dict.append(indices_s)
feed_dict.append(indices_d)
try:
step = 0
while True:
if FLAGS.bmsd:
(loss, _), (acc, _), (num_word, _), pred, gt, pred_b, pred_m, pred_s, pred_d = sess.run(feed_dict)
preds_b.append(pred_b)
preds_m.append(pred_m)
preds_s.append(pred_s)
preds_d.append(pred_d)
else:
(loss, _), (acc, _), (num_word, _), pred, gt = sess.run(feed_dict)
val_progbar.update(step, [('loss', loss), ('acc', acc), ('num_word', num_word), ])
preds.append(pred)
gts.append(gt)
except tf.errors.OutOfRangeError:
logger.info('[*] Validation loss: %.4f, acc: %.4f, num_word: %.2f' % (loss, acc, num_word))
if FLAGS.bmsd:
flat_preds = list(itertools.chain(*preds))
flat_gts = list(itertools.chain(*gts))
flat_preds_b = list(itertools.chain(*preds_b))
flat_preds_m = list(itertools.chain(*preds_m))
flat_preds_s = list(itertools.chain(*preds_s))
flat_preds_d = list(itertools.chain(*preds_d))
score, base_scores = evaluate(flat_gts, flat_preds)
print('\n[!] Label Score: %.4f' % (score))
bmsd_score, bmsd_scores = evaluate_bmsd(flat_gts, flat_preds_b, flat_preds_m, flat_preds_s, flat_preds_d)
print('BMSD Score: %.4f' % (bmsd_score))
final_score = 0.0
for r, s1, s2 in zip([1.0,1.2,1.3,1.4], base_scores, bmsd_scores):
final_score += r * max(s1, s2)
final_score /= 4.0
print('[!] Final Score: %.4f' % (final_score))
lfirst_score, lfirst_scores = evaluate_lfirst(flat_gts, flat_preds, flat_preds_b, flat_preds_m, flat_preds_s, flat_preds_d)
print('[!] Label First Score: %.4f' % (lfirst_score))
else:
flat_preds = list(itertools.chain(*preds))
flat_gts = list(itertools.chain(*gts))
score, base_scores = evaluate(flat_gts, flat_preds)
print('\n[!] Label Score: %.4f' % (score))
return score
def main():
hvd.init()
n_epochs = 10
batch_size = 5
step = len(im) // batch_size
params = parse_args(PARSER.parse_args())
optimizer = tf.keras.optimizers.Adam(learning_rate=params.learning_rate)
ce_loss = tf.keras.metrics.Mean(name='ce_loss')
f1_loss = tf.keras.metrics.Mean(name='dice_loss')
checkpoint = tf.train.Checkpoint(optimizer=optimizer, model=model)
pb_i = Progbar(step, stateful_metrics=metrics_names)
count = 0
for epoch in range(n_epochs):
if count >= step:
count = 0
features = im[epoch * batch_size:(epoch * batch_size) + batch_size]
features = np.reshape(features,
(len(features), features[0].shape[1],
features[0].shape[2], features[0].shape[0]))
features = features.astype('float32')
labels = lb[epoch * batch_size:(epoch * batch_size) + batch_size]
labels = np.reshape(
labels, (len(labels), labels[0].shape[0], labels[0].shape[1], 1))
labels = labels.astype('float32')
print(features.shape, labels.shape)
print('Epoch {} out of epochs {}'.format(epoch, n_epochs))
for i, (features_, labels_) in enumerate(zip(features, labels)):
with tf.GradientTape() as tape:
output_map = model(features)
crossentropy_loss, dice_loss = partial_losses(
output_map, labels)
added_losses = tf.add(crossentropy_loss,
dice_loss,
name='total_loss_ref')
values = [('Xent', crossentropy_loss),
('added_losses', added_losses)]
pb_i.add(1, values=values)
# calculate the gradients using our tape and then update the
# model weights
tape = hvd.DistributedGradientTape(tape)
gradients = tape.gradient(added_losses, model.trainable_variables)
optimizer.apply_gradients(zip(gradients,
model.trainable_variables))
# Calculate something wrong here
# val_total_loss = 0
# val_total_acc = 0
# total_val_num = 0
# for bIdx, (val_X, val_y) in enumerate(val_batch):
# if bIdx >= features.shape[0]:
# break
# y_pred = model(val_X, training=False)
print('Xen: ', crossentropy_loss, dice_loss, added_losses)
# Initial Data class
Data = Data_Generator(C)
print('Start training')
neg_flag = 0
# The start of
################################ Remove the data from last iteration
for big_epoch_num in range(num_epochs):
###########################################################################
#First CNN
###########################################################################
first_loss = np.zeros(num_first_epochs)
final_epoch_num = 0
for epoch_num in range(num_first_epochs):
progbar = Progbar(Fov_num)
print('Epoch of the First CNN {}/{}'.format(epoch_num + 1,
num_first_epochs))
print('There are %d FOVs in total' % (Fov_num))
loss = 0
count = 0
change_matrix = 0
first_data_generator = itertools.cycle(
Data.data_generator(FOV_Path, LABEL_Path, IMAGE_Path, epoch_num,
big_epoch_num))
while count <= Fov_num:
try:
## 设计思想，读取一次 训练一个batch，使用itertools， 一旦停止即进行下一个 for循环
# print('The %d epoch of the first CNN in %d big epoch'%(count, big_epoch_num))
Fov, Label, Image, x, y, z = next(first_data_generator)
def main():
# ------------------------------------------------------------------------------------
# Defining random seeds
random_seed = 42
tf.random.set_seed(random_seed)
np.random.seed(random_seed)
rn.seed(random_seed)
# ------------------------------------------------------------------------------------
# Loading data
(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()
height = 28
width = 28
n_channel = 1
x_train = x_train.astype('float32') / 255.
x_test = x_test.astype('float32') / 255.
x_train = x_train.reshape(x_train.shape[0], height, width, n_channel)
x_test = x_test.reshape(x_test.shape[0], height, width, n_channel)
# ------------------------------------------------------------------------------------
# Quantise the input data in q levels
q_levels = 2
x_train_quantised = quantise(x_train, q_levels)
x_test_quantised = quantise(x_test, q_levels)
# ------------------------------------------------------------------------------------
# Creating input stream using tf.data API
batch_size = 128
train_buf = 60000
train_dataset = tf.data.Dataset.from_tensor_slices(
(x_train_quantised / (q_levels - 1),
x_train_quantised.astype('int32')))
train_dataset = train_dataset.shuffle(buffer_size=train_buf)
train_dataset = train_dataset.batch(batch_size)
test_dataset = tf.data.Dataset.from_tensor_slices((x_test_quantised / (q_levels - 1),
x_test_quantised.astype('int32')))
test_dataset = test_dataset.batch(batch_size)
# ------------------------------------------------------------------------------------
# Create PixelCNN model
inputs = keras.layers.Input(shape=(height, width, n_channel))
x = MaskedConv2D(mask_type='A', filters=128, kernel_size=7, strides=1)(inputs)
for i in range(15):
x = ResidualBlock(h=64)(x)
x = keras.layers.Activation(activation='relu')(x)
x = keras.layers.Conv2D(filters=128, kernel_size=1, strides=1)(x)
x = keras.layers.Activation(activation='relu')(x)
x = keras.layers.Conv2D(filters=q_levels, kernel_size=1, strides=1)(x)
pixelcnn = keras.Model(inputs=inputs, outputs=x)
# ------------------------------------------------------------------------------------
# Prepare optimizer and loss function
lr_decay = 0.999995
learning_rate = 1e-3
optimizer = keras.optimizers.Adam(lr=learning_rate)
compute_loss = keras.losses.CategoricalCrossentropy(from_logits=True)
# ------------------------------------------------------------------------------------
@tf.function
def train_step(batch_x, batch_y):
with tf.GradientTape() as ae_tape:
logits = pixelcnn(batch_x, training=True)
loss = compute_loss(tf.squeeze(tf.one_hot(batch_y, q_levels)), logits)
gradients = ae_tape.gradient(loss, pixelcnn.trainable_variables)
gradients, _ = tf.clip_by_global_norm(gradients, 1.0)
optimizer.apply_gradients(zip(gradients, pixelcnn.trainable_variables))
return loss
# ------------------------------------------------------------------------------------
# Training loop
n_epochs = 100
n_iter = int(np.ceil(x_train_quantised.shape[0] / batch_size))
for epoch in range(n_epochs):
progbar = Progbar(n_iter)
print('Epoch {:}/{:}'.format(epoch + 1, n_epochs))
for i_iter, (batch_x, batch_y) in enumerate(train_dataset):
optimizer.lr = optimizer.lr * lr_decay
loss = train_step(batch_x, batch_y)
progbar.add(1, values=[('loss', loss)])
# ------------------------------------------------------------------------------------
# Test set performance
test_loss = []
for batch_x, batch_y in test_dataset:
logits = pixelcnn(batch_x, training=False)
# Calculate cross-entropy (= negative log-likelihood)
loss = compute_loss(tf.squeeze(tf.one_hot(batch_y, q_levels)), logits)
test_loss.append(loss)
print('nll : {:} nats'.format(np.array(test_loss).mean()))
print('bits/dim : {:}'.format(np.array(test_loss).mean() / np.log(2)))
# ------------------------------------------------------------------------------------
# Generating new images
samples = np.zeros((100, height, width, n_channel), dtype='float32')
for i in range(height):
for j in range(width):
logits = pixelcnn(samples)
next_sample = tf.random.categorical(logits[:, i, j, :], 1)
samples[:, i, j, 0] = (next_sample.numpy() / (q_levels - 1))[:, 0]
fig = plt.figure(figsize=(10, 10))
for i in range(100):
ax = fig.add_subplot(10, 10, i + 1)
ax.matshow(samples[i, :, :, 0], cmap=matplotlib.cm.binary)
plt.xticks(np.array([]))
plt.yticks(np.array([]))
plt.show()
# ------------------------------------------------------------------------------------
# Filling occluded images
occlude_start_row = 14
num_generated_images = 10
samples = np.copy(x_test_quantised[0:num_generated_images, :, :, :])
samples = samples / (q_levels - 1)
samples[:, occlude_start_row:, :, :] = 0
fig = plt.figure(figsize=(10, 10))
for i in range(10):
ax = fig.add_subplot(1, 10, i + 1)
ax.matshow(samples[i, :, :, 0], cmap=matplotlib.cm.binary)
plt.xticks(np.array([]))
plt.yticks(np.array([]))
for i in range(occlude_start_row, height):
for j in range(width):
logits = pixelcnn(samples)
next_sample = tf.random.categorical(logits[:, i, j, :], 1)
samples[:, i, j, 0] = (next_sample.numpy() / (q_levels - 1))[:, 0]
fig = plt.figure(figsize=(10, 10))
for i in range(10):
ax = fig.add_subplot(1, 10, i + 1)
ax.matshow(samples[i, :, :, 0], cmap=matplotlib.cm.binary)
plt.xticks(np.array([]))
plt.yticks(np.array([]))
plt.show()
def represent(self, molecules):
"""
provides coulomb matrix representation for input molecules.
Parameters
----------
molecules: chemml.chem.Molecule object or array
If list, it must be a list of chemml.chem.Molecule objects, otherwise we raise a ValueError.
In addition, all the molecule objects must provide the XYZ information. Please make sure the XYZ geometry has been
stored or optimized in advance.
Returns
-------
Pandas DataFrame
A data frame with same number of rows as number of molecules will be returned.
The exact shape of the dataframe depends on the type of CM as follows:
- shape of Unsorted_Matrix (UM): (n_molecules, max_n_atoms**2)
- shape of Unsorted_Triangular (UT): (n_molecules, max_n_atoms*(max_n_atoms+1)/2)
- shape of eigenspectrums (E): (n_molecules, max_n_atoms)
- shape of Sorted_Coulomb (SC): (n_molecules, max_n_atoms*(max_n_atoms+1)/2)
- shape of Random_Coulomb (RC): (n_molecules, nPerm * max_n_atoms * (max_n_atoms+1)/2)
"""
# check input molecules
if isinstance(molecules, (list, np.ndarray)):
molecules = np.array(molecules)
elif isinstance(molecules, Molecule):
molecules = np.array([molecules])
else:
msg = "The molecule must be a chemml.chem.Molecule object or a list of objects."
raise ValueError(msg)
if molecules.ndim > 1:
msg = "The molecule must be a chemml.chem.Molecule object or a list of objects."
raise ValueError(msg)
self.n_molecules_ = molecules.shape[0]
# max number of atoms based on the list of molecules
if self.max_n_atoms_ == 'auto':
try:
self.max_n_atoms_ = max(
[m.xyz.atomic_numbers.shape[0] for m in molecules])
except:
msg = "The xyz representation of molecules is not available."
raise ValueError(msg)
# pool of processes
if self.n_jobs == -1:
self.n_jobs = cpu_count()
pool = Pool(processes=self.n_jobs)
# Create an iterator
# http://stackoverflow.com/questions/312443/how-do-you-split-a-list-into-evenly-sized-chunks
def chunks(l, n):
"""Yield successive n-sized chunks from l."""
for i in range(0, len(l), n):
yield l[i:i + n]
# find size of each batch
batch_size = int(len(molecules) / self.n_jobs)
if batch_size == 0:
batch_size = 1
molecule_chunks = chunks(molecules, batch_size)
# MAP: CM in parallel
map_function = partial(self._represent)
if self.verbose:
print('featurizing molecules in batches of %i ...' % batch_size)
pbar = Progbar(len(molecules), width=50)
tensor_list = []
for tensors in pool.imap(map_function, molecule_chunks):
pbar.add(len(tensors[0]))
tensor_list.append(tensors)
print('Merging batch features ... ', end='')
else:
tensor_list = pool.map(map_function, molecule_chunks)
if self.verbose:
print('[DONE]')
# REDUCE: Concatenate the obtained tensors
pool.close()
pool.join()
return pd.concat(tensor_list, axis=0, ignore_index=True)
def tensorise_molecules(molecules,
max_degree=5,
max_atoms=None,
n_jobs=-1,
batch_size=3000,
verbose=True):
"""
Takes a list of molecules and provides tensor representation of atom and bond features.
This representation is based on the "convolutional networks on graphs for learning molecular fingerprints" by
David Duvenaud et al., NIPS 2015.
Parameters
----------
molecules: chemml.chem.Molecule object or array
If list, it must be a list of chemml.chem.Molecule objects, otherwise we raise a ValueError.
In addition, all the molecule objects must provide the SMILES representation.
We try to create the SMILES representation if it's not available.
max_degree: int, optional (default=5)
The maximum number of neighbour per atom that each molecule can have
(to which all molecules will be padded), use 'None' for auto
max_atoms: int, optional (default=None)
The maximum number of atoms per molecule (to which all
molecules will be padded), use 'None' for auto
n_jobs: int, optional(default=-1)
The number of parallel processes. If -1, uses all the available processes.
batch_size: int, optional(default=3000)
The number of molecules per process, bigger chunksize is preffered as each process will preallocate np.arrays
verbose: bool, optional(default=True)
The verbosity of messages.
Notes
-----
It is not recommended to set max_degree to `None`/auto when
using `NeuralGraph` layers. Max_degree determines the number of
trainable parameters and is essentially a hyperparameter.
While models can be rebuilt using different `max_atoms`, they cannot
be rebuild for different values of `max_degree`, as the architecture
will be different.
For organic molecules `max_degree=5` is a good value (Duvenaud et. al, 2015)
Returns
-------
atoms: array
An atom feature array of shape (molecules, max_atoms, atom_features)
bonds: array
A bonds array of shape (molecules, max_atoms, max_degree)
edges: array
A connectivity array of shape (molecules, max_atoms, max_degree, bond_features)
"""
# TODO:
# - fix python keyboardinterrupt bug:
# https://noswap.com/blog/python-multiprocessing-keyboardinterrupt
# - replace progbar with proper logging
# molecules
if isinstance(molecules, list) or isinstance(molecules, np.ndarray):
molecules = np.array(molecules)
elif isinstance(molecules, Molecule):
molecules = np.array([molecules])
else:
msg = "The input molecules must be a chemml.chem.Molecule object or a list of objects."
raise ValueError(msg)
# pool of processes
if n_jobs == -1:
n_jobs = cpu_count()
pool = Pool(processes=n_jobs)
# Create an iterator
#http://stackoverflow.com/questions/312443/how-do-you-split-a-list-into-evenly-sized-chunks
def chunks(l, n):
"""Yield successive n-sized chunks from l."""
for i in range(0, len(l), n):
yield l[i:i + n]
molecule_chunks = chunks(molecules, batch_size)
# MAP: Tensorise in parallel
map_function = partial(tensorise_molecules_singlecore,
max_degree=max_degree,
max_atoms=max_atoms)
if verbose:
print('Tensorising molecules in batches of %i ...' % batch_size)
pbar = Progbar(len(molecules), width=50)
tensor_list = []
for tensors in pool.imap(map_function, molecule_chunks):
pbar.add(tensors[0].shape[0])
tensor_list.append(tensors)
print('Merging batch tensors ... ', end='')
else:
tensor_list = pool.map(map_function, molecule_chunks)
if verbose:
print('[DONE]')
# REDUCE: Concatenate the obtained tensors
pool.close()
pool.join()
return concat_mol_tensors(tensor_list,
match_degree=max_degree != None,
match_max_atoms=max_atoms != None)
def start_training(config):
use_horovod = horovod_util.is_enabled()
print("use_horovod:", use_horovod)
if use_horovod:
hvd = horovod_util.setup()
rank = hvd.rank()
else:
rank = 0
ModelClass = config.NETWORK_CLASS
network_kwargs = {key.lower(): val for key, val in config.NETWORK.items()}
if "train_validation_saving_size".upper() in config.DATASET.keys():
use_train_validation_saving = config.DATASET.TRAIN_VALIDATION_SAVING_SIZE > 0
else:
use_train_validation_saving = False
if use_train_validation_saving:
top_train_validation_saving_set_accuracy = 0
train_dataset = setup_dataset(config, "train", rank)
print("train dataset num:", train_dataset.num_per_epoch)
if use_train_validation_saving:
train_validation_saving_dataset = setup_dataset(config, "train_validation_saving", rank)
print("train_validation_saving dataset num:", train_validation_saving_dataset.num_per_epoch)
validation_dataset = setup_dataset(config, "validation", rank)
print("validation dataset num:", validation_dataset.num_per_epoch)
graph = tf.Graph()
with graph.as_default():
if ModelClass.__module__.startswith("lmnet.networks.object_detection"):
model = ModelClass(
classes=train_dataset.classes,
num_max_boxes=train_dataset.num_max_boxes,
is_debug=config.IS_DEBUG,
**network_kwargs,
)
else:
model = ModelClass(
classes=train_dataset.classes,
is_debug=config.IS_DEBUG,
**network_kwargs,
)
global_step = tf.Variable(0, name="global_step", trainable=False)
is_training_placeholder = tf.placeholder(tf.bool, name="is_training_placeholder")
images_placeholder, labels_placeholder = model.placeholderes()
output = model.inference(images_placeholder, is_training_placeholder)
if ModelClass.__module__.startswith("lmnet.networks.object_detection"):
loss = model.loss(output, labels_placeholder, global_step)
else:
loss = model.loss(output, labels_placeholder)
opt = model.optimizer(global_step)
if use_horovod:
# add Horovod Distributed Optimizer
opt = hvd.DistributedOptimizer(opt)
train_op = model.train(loss, opt, global_step)
metrics_ops_dict, metrics_update_op = model.metrics(output, labels_placeholder)
# TODO(wakisaka): Deal with many networks.
model.summary(output, labels_placeholder)
summary_op = tf.summary.merge_all()
metrics_summary_op, metrics_placeholders = executor.prepare_metrics(metrics_ops_dict)
init_op = tf.global_variables_initializer()
reset_metrics_op = tf.local_variables_initializer()
if use_horovod:
# add Horovod broadcasting variables from rank 0 to all
bcast_global_variables_op = hvd.broadcast_global_variables(0)
if use_train_validation_saving:
saver = tf.train.Saver(max_to_keep=1)
else:
saver = tf.train.Saver(max_to_keep=config.KEEP_CHECKPOINT_MAX)
if config.IS_PRETRAIN:
all_vars = tf.global_variables()
pretrain_var_list = [
var for var in all_vars if var.name.startswith(tuple(config.PRETRAIN_VARS))
]
print("pretrain_vars", [
var.name for var in pretrain_var_list
])
pretrain_saver = tf.train.Saver(pretrain_var_list, name="pretrain_saver")
if use_horovod:
# For distributed training
session_config = tf.ConfigProto(
gpu_options=tf.GPUOptions(
allow_growth=True,
visible_device_list=str(hvd.local_rank())
)
)
else:
# TODO(wakisaka): For debug.
# session_config = tf.ConfigProto(
# gpu_options=tf.GPUOptions(
# allow_growth=True,
# per_process_gpu_memory_fraction=0.1
# )
# )
session_config = tf.ConfigProto() # tf.ConfigProto(log_device_placement=True)
# TODO(wakisaka): XLA JIT
# session_config.graph_options.optimizer_options.global_jit_level = tf.OptimizerOptions.ON_1
sess = tf.Session(graph=graph, config=session_config)
sess.run([init_op, reset_metrics_op])
if rank == 0:
train_writer = tf.summary.FileWriter(environment.TENSORBOARD_DIR + "/train", sess.graph)
if use_train_validation_saving:
train_val_saving_writer = tf.summary.FileWriter(environment.TENSORBOARD_DIR + "/train_validation_saving")
val_writer = tf.summary.FileWriter(environment.TENSORBOARD_DIR + "/validation")
if config.IS_PRETRAIN:
print("------- Load pretrain data ----------")
pretrain_saver.restore(sess, os.path.join(config.PRETRAIN_DIR, config.PRETRAIN_FILE))
sess.run(tf.assign(global_step, 0))
last_step = 0
# for recovery
ckpt = tf.train.get_checkpoint_state(environment.CHECKPOINTS_DIR)
if ckpt and ckpt.model_checkpoint_path:
print("--------- Restore last checkpoint -------------")
saver.restore(sess, ckpt.model_checkpoint_path)
# saver.recover_last_checkpoints(ckpt.model_checkpoint_path)
last_step = sess.run(global_step)
# TODO(wakisaka): tensorflow v1.3 remain previous event log in tensorboard.
# https://github.com/tensorflow/tensorflow/blob/r1.3/tensorflow/python/training/supervisor.py#L1072
train_writer.add_session_log(SessionLog(status=SessionLog.START), global_step=last_step + 1)
val_writer.add_session_log(SessionLog(status=SessionLog.START), global_step=last_step + 1)
print("recovered. last step", last_step)
if use_horovod:
# broadcast variables from rank 0 to all other processes
sess.run(bcast_global_variables_op)
last_step = sess.run(global_step)
# Calculate max steps. The priority of config.MAX_EPOCHS is higher than config.MAX_STEPS.
if "MAX_EPOCHS" in config:
max_steps = int(train_dataset.num_per_epoch / config.BATCH_SIZE * config.MAX_EPOCHS)
else:
max_steps = config.MAX_STEPS
progbar = Progbar(max_steps)
if rank == 0:
progbar.update(last_step)
for step in range(last_step, max_steps):
images, labels = train_dataset.feed()
feed_dict = {
is_training_placeholder: True,
images_placeholder: images,
labels_placeholder: labels,
}
if step * ((step + 1) % config.SUMMARISE_STEPS) == 0 and rank == 0:
# Runtime statistics for develop.
# run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
# run_metadata = tf.RunMetadata()
sess.run(reset_metrics_op)
_, summary, _ = sess.run(
[train_op, summary_op, metrics_update_op], feed_dict=feed_dict,
# options=run_options,
# run_metadata=run_metadata,
)
# train_writer.add_run_metadata(run_metadata, "step: {}".format(step + 1))
train_writer.add_summary(summary, step + 1)
metrics_values = sess.run(list(metrics_ops_dict.values()))
metrics_feed_dict = {placeholder: value for placeholder, value in zip(metrics_placeholders, metrics_values)}
metrics_summary, = sess.run(
[metrics_summary_op], feed_dict=metrics_feed_dict,
)
train_writer.add_summary(metrics_summary, step + 1)
train_writer.flush()
else:
sess.run([train_op], feed_dict=feed_dict)
to_be_saved = step == 0 or (step + 1) == max_steps or (step + 1) % config.SAVE_CHECKPOINT_STEPS == 0
if to_be_saved and rank == 0:
if use_train_validation_saving:
sess.run(reset_metrics_op)
train_validation_saving_step_size = int(math.ceil(train_validation_saving_dataset.num_per_epoch
/ config.BATCH_SIZE))
print("train_validation_saving_step_size", train_validation_saving_step_size)
current_train_validation_saving_set_accuracy = 0
for train_validation_saving_step in range(train_validation_saving_step_size):
print("train_validation_saving_step", train_validation_saving_step)
images, labels = train_validation_saving_dataset.feed()
feed_dict = {
is_training_placeholder: False,
images_placeholder: images,
labels_placeholder: labels,
}
if train_validation_saving_step % config.SUMMARISE_STEPS == 0:
summary, _ = sess.run([summary_op, metrics_update_op], feed_dict=feed_dict)
train_val_saving_writer.add_summary(summary, step + 1)
train_val_saving_writer.flush()
else:
sess.run([metrics_update_op], feed_dict=feed_dict)
metrics_values = sess.run(list(metrics_ops_dict.values()))
metrics_feed_dict = {
placeholder: value for placeholder, value in zip(metrics_placeholders, metrics_values)
}
metrics_summary, = sess.run(
[metrics_summary_op], feed_dict=metrics_feed_dict,
)
train_val_saving_writer.add_summary(metrics_summary, step + 1)
train_val_saving_writer.flush()
current_train_validation_saving_set_accuracy = sess.run(metrics_ops_dict["accuracy"])
if current_train_validation_saving_set_accuracy > top_train_validation_saving_set_accuracy:
top_train_validation_saving_set_accuracy = current_train_validation_saving_set_accuracy
print("New top train_validation_saving accuracy is: ", top_train_validation_saving_set_accuracy)
_save_checkpoint(saver, sess, global_step, step)
else:
_save_checkpoint(saver, sess, global_step, step)
if step == 0:
# check create pb on only first step.
minimal_graph = tf.graph_util.convert_variables_to_constants(
sess,
sess.graph.as_graph_def(add_shapes=True),
["output"],
)
pb_name = "minimal_graph_with_shape_{}.pb".format(step + 1)
pbtxt_name = "minimal_graph_with_shape_{}.pbtxt".format(step + 1)
tf.train.write_graph(minimal_graph, environment.CHECKPOINTS_DIR, pb_name, as_text=False)
tf.train.write_graph(minimal_graph, environment.CHECKPOINTS_DIR, pbtxt_name, as_text=True)
if step == 0 or (step + 1) % config.TEST_STEPS == 0:
# init metrics values
sess.run(reset_metrics_op)
test_step_size = int(math.ceil(validation_dataset.num_per_epoch / config.BATCH_SIZE))
for test_step in range(test_step_size):
images, labels = validation_dataset.feed()
feed_dict = {
is_training_placeholder: False,
images_placeholder: images,
labels_placeholder: labels,
}
if test_step % config.SUMMARISE_STEPS == 0:
summary, _ = sess.run([summary_op, metrics_update_op], feed_dict=feed_dict)
if rank == 0:
val_writer.add_summary(summary, step + 1)
val_writer.flush()
else:
sess.run([metrics_update_op], feed_dict=feed_dict)
metrics_values = sess.run(list(metrics_ops_dict.values()))
metrics_feed_dict = {
placeholder: value for placeholder, value in zip(metrics_placeholders, metrics_values)
}
metrics_summary, = sess.run(
[metrics_summary_op], feed_dict=metrics_feed_dict,
)
if rank == 0:
val_writer.add_summary(metrics_summary, step + 1)
val_writer.flush()
if rank == 0:
progbar.update(step + 1)
# training loop end.
print("Done")
step = ckpt.step.numpy()
val_metric = ckpt.metric.numpy()
total_val_sample = 5000
progbar = None
start_time = time.perf_counter()
# Start training
for image, cat_true, mask_true in train_dataset:
ckpt.step.assign_add(1)
step += 1
# On epoch start
epoch_step = (step % TRAINING_PARAMETERS['steps_per_epoch']) + 1
if epoch_step == 1:
print("Epoch {}/{}".format((step // TRAINING_PARAMETERS['steps_per_epoch']) + 1, TRAINING_PARAMETERS['num_epoch']))
progbar = Progbar(TRAINING_PARAMETERS['steps_per_epoch'], interval=1, stateful_metrics=['train_acc', 'train_meaniou'])
total_loss, l_cate, l_mask = train_step(model,
optimizer,
loss_fn,
image,
cat_true,
mask_true,
train_acc,
train_meaniou)
values = [('train_loss', total_loss),
('train_cat_loss', l_cate),
('train_mask_loss', l_mask),
('train_acc', train_acc.result()),
('train_meaniou', train_meaniou.result())]
progbar.update(epoch_step, values)
char = TimeDistributed(Flatten())(maxpool_out)
char = Dropout(0.5)(char)
output = concatenate([words, casing, char])
output = Bidirectional(
LSTM(200, return_sequences=True, dropout=0.50,
recurrent_dropout=0.25))(output)
output = TimeDistributed(Dense(len(label2Idx), activation='softmax'))(output)
model = Model(inputs=[words_input, casing_input, character_input],
outputs=[output])
model.compile(loss='sparse_categorical_crossentropy', optimizer='nadam')
model.summary()
# plot_model(model, to_file='model.png')
for epoch in range(epochs):
print("Epoch %d/%d" % (epoch, epochs))
a = Progbar(len(train_batch_len))
for i, batch in enumerate(iterate_minibatches(train_batch,
train_batch_len)):
labels, tokens, casing, char = batch
model.train_on_batch([tokens, casing, char], labels)
a.update(i)
a.update(i + 1)
print(' ')
model.save("models/model.h5")
# Performance on dev dataset
predLabels, correctLabels = tag_dataset(dev_batch)
pre_dev, rec_dev, f1_dev = compute_f1(predLabels, correctLabels, idx2Label)
print("Dev-Data: Prec: %.3f, Rec: %.3f, F1: %.3f" % (pre_dev, rec_dev, f1_dev))
d["business_id"] = "b_" + d["business_id"]
d["review_id"] = "r_" + d["review_id"]
review_data_raw[d["review_id"]] = d
# Create NX graph
G = nx.Graph()
# Maintain a list of users, businesses and reviews in graphs
# these are sets to make inclusion checking fast
users_in_graph = set()
business_in_graph = set()
reviews_in_graph = set()
# Create business nodes
print("\nAdding businesses to graph")
p = Progbar(len(business_data_raw))
for ii, b in enumerate(business_data_raw.values()):
p.update(ii)
if filter_state != "false" and b["state"].lower() != filter_state:
continue
G.add_node(b["business_id"], ntype="business")
# Add business to set
business_in_graph.add(b["business_id"])
# Create review links
print("\nAdding reviews to graph")
p = Progbar(len(review_data_raw))
for ii, r in enumerate(review_data_raw.values()):
p.update(ii)
if r["business_id"] not in business_in_graph:
#else:
#best_loss2 = np.min(r_curr_loss2)
start_time = time.time()
class_mapping_inv = {v: k for k, v in class_mapping.items()}
# Start training process
print('Starting training')
start_time = time.time()
for epoch_num in range(num_epochs):
progbar = Progbar(epoch_length)
print('Epoch {}/{}'.format(epoch_num + 1, num_epochs))
iter_num_aux = 0
iter_num = 0
i = True
training = True
validate = False
#while(i):
try:
if(training):
if len(rpn_accuracy_rpn_monitor) == len(train_imgs) and C.verbose:
mean_overlapping_bboxes = float(sum(rpn_accuracy_rpn_monitor)) / len(rpn_accuracy_rpn_monitor)
rpn_accuracy_rpn_monitor = []
def predict(self,
x,
batch_size=None,
verbose=0,
steps=1,
callbacks=None,
max_queue_size=10,
workers=1,
use_multiprocessing=False):
"""
Model predicting on data yielded (generator).
A predict() abstration function of TensorFlow 2 using the encoder and decoder models
:param: See tensorflow.keras.Model.predict()
:return: A numpy array(s) of predictions.
References:
Tal Weiss
Deep Spelling
Medium: https://machinelearnings.co/deep-spelling-9ffef96a24f6
Github: https://github.com/MajorTal/DeepSpell
Vu Tran
Sequence-to-Sequence Learning for Spelling Correction
Github: https://github.com/vuptran/deep-spell-checkr
"""
try:
enqueuer = GeneratorEnqueuer(
x, use_multiprocessing=use_multiprocessing)
enqueuer.start(workers=workers, max_queue_size=max_queue_size)
output_generator = enqueuer.get()
steps_done = 0
if verbose == 1:
print("Model Predict")
progbar = Progbar(target=steps)
predicts = []
while steps_done < steps:
x = next(output_generator)[0]
batch_size = len(x)
# Encode the input as state vectors
encoder_out, state_h, state_c = self.encoder.predict(x)
dec_state = tf.concat([state_h, state_c], axis=-1)
# Create batch of empty target sequences of length 1 character and populate
# the first element of target sequence with the # start-of-sequence character
target = np.zeros((batch_size, 1, self.tokenizer.vocab_size))
target[:, 0, self.tokenizer.SOS] = 1.0
# Sampling loop for a batch of sequences
decoded_tokens = [''] * batch_size
for _ in range(self.tokenizer.maxlen):
# `char_probs` has shape (batch_size, 1, nb_target_chars)
char_probs, dec_state = self.decoder.predict(
[encoder_out, dec_state, target])
# Reset the target sequences.
target = np.zeros(
(batch_size, 1, self.tokenizer.vocab_size))
# Sample next character using argmax or multinomial mode
sampled_chars = []
for i in range(batch_size):
next_index = char_probs[i].argmax(axis=-1)
next_char = self.tokenizer.decode([next_index])
decoded_tokens[i] += next_char
sampled_chars.append(next_char)
# Update target sequence with index of next character
target[i, 0, next_index] = 1.0
stop_char = set(sampled_chars)
if len(stop_char) == 1 and stop_char.pop(
) == self.tokenizer.EOS_TK:
break
# Sampling finished
predicts.extend(
[self.tokenizer.remove_tokens(x) for x in decoded_tokens])
steps_done += 1
if verbose == 1:
progbar.update(steps_done)
finally:
enqueuer.stop()
return predicts
def find_n_high(ds_name, encoder, paths, n, metric, negate=False):
"""The main function for executing network training. It loads the specified
dataset iterator, saliency model, and helper classes. Training is then
performed in a new session by iterating over all batches for a number of
epochs. After validation on an independent set, the model is saved and
the training history is updated.
Args:
ds_name (str): Denotes the dataset to be used during training.
paths (dict, str): A dictionary with all path elements.
"""
w_filename_template = "/%s_%s_%s_weights.h5" # [encoder]_[ds_name]_[loss_fn_name]_weights.h5
(eval_ds, n_eval) = data.load_eval_dataset(ds_name, paths["data"])
print(">> Preparing model with encoder %s..." % encoder)
model = MyModel(encoder, ds_name, "train")
if "trained_weights" in paths:
if os.path.exists(paths["trained_weights"]):
weights_path = paths["trained_weights"]
else:
raise ValueError("could not find the specified weights file.\n specified weights: %s"%paths["trained_weights"])
else:
weights_path = paths["weights"] + w_filename_template % (encoder, ds_name, loss_fn_name)
if os.path.exists(weights_path):
print("Weights are loaded!\n %s"%weights_path)
else:
download.download_pretrained_weights(paths["weights"], encoder, "salicon", loss_fn_name)
model.load_weights(weights_path)
del weights_path
model.summary()
# Preparing
print("\n>> Start finding %d %s results for model on %s..." % (n, "worst" if negate else "best",ds_name.upper()))
print(("Evaluation details:" +
"\n{0:<4}Metric: {1}").format(" ", metric))
print("_" * 65)
eval_progbar = Progbar(n_eval)
min_heap = []
count = 0
sign = -1 if negate else 1
for eval_x, eval_fixs, eval_y_true, eval_ori_sizes, eval_filenames in eval_ds:
eval_y_pred = test_step(eval_x, model)
for pred, y_true, fixs, filename, ori_size in zip(eval_y_pred, eval_fixs, eval_y_true, eval_filenames.numpy(), eval_ori_sizes):
pred = tf.expand_dims(data.postprocess_saliency_map(pred, ori_size), axis=0)
fixs = tf.expand_dims(fixs, axis=0)
y_true = tf.expand_dims(y_true, axis=0)
score = _calc_metrics([metric], y_true, fixs, pred)[metric].numpy() * sign
if count < n:
count+=1
heapq.heappush(min_heap, (score, filename.decode("utf-8")))
else:
heapq.heappushpop(min_heap, (score, filename.decode("utf-8")))
eval_progbar.add(eval_x.shape[0])
min_heap.sort(reverse=True)
for s, n in min_heap:
print(s, n)
def predict_generator(self,
generator,
steps,
max_queue_size=10,
workers=1,
use_multiprocessing=False,
verbose=0):
"""Generates predictions for the input samples from a data generator.
The generator should return the same kind of data as accepted by `predict_on_batch`.
generator = DataGenerator class that returns:
x = Input data as a 3D Tensor (batch_size, max_input_len, dim_features)
x_len = 1D array with the length of each data in batch_size
# Arguments
generator: Generator yielding batches of input samples
or an instance of Sequence (tensorflow.keras.utils.Sequence)
object in order to avoid duplicate data
when using multiprocessing.
steps:
Total number of steps (batches of samples)
to yield from `generator` before stopping.
max_queue_size:
Maximum size for the generator queue.
workers: Maximum number of processes to spin up
when using process based threading
use_multiprocessing: If `True`, use process based threading.
Note that because this implementation relies on multiprocessing,
you should not pass non picklable arguments to the generator
as they can't be passed easily to children processes.
verbose:
verbosity mode, 0 or 1.
# Returns
A numpy array(s) of predictions.
# Raises
ValueError: In case the generator yields
data in an invalid format.
"""
self.model_pred._make_predict_function()
is_sequence = isinstance(generator, Sequence)
allab_outs = []
steps_done = 0
enqueuer = None
try:
if is_sequence:
enqueuer = OrderedEnqueuer(
generator, use_multiprocessing=use_multiprocessing)
else:
enqueuer = GeneratorEnqueuer(
generator, use_multiprocessing=use_multiprocessing)
enqueuer.start(workers=workers, max_queue_size=max_queue_size)
output_generator = enqueuer.get()
if verbose == 1:
progbar = Progbar(target=steps)
while steps_done < steps:
x = next(output_generator)
outs = self.predict_on_batch(x)
if not isinstance(outs, list):
outs = [outs]
for i, out in enumerate(outs):
encode = [
valab_out for valab_out in out if valab_out != -1
]
allab_outs.append([int(c) for c in encode])
steps_done += 1
if verbose == 1:
progbar.update(steps_done)
finally:
if enqueuer is not None:
enqueuer.stop()
return allab_outs
