def build_optimizer(args,loss_op,num_train_items):
# Optimizer.
banner_print("Building optimizer.")
global_step = tf.train.get_or_create_global_step()
lr_init = float(RANKS*args.lr_init)
if args.use_clr:
import clr
if RANK == 0:
print("Using Cyclic LR with initial LR: ", lr_init)
step_sz = num_train_items/args.batch_size
max_steps = args.epochs*(num_train_items/args.batch_size)
lr_decay = tf.train.exponential_decay(lr_init, global_step, max_steps*10, 0.9, staircase=False)
learning_rate = clr.cyclic_learning_rate(global_step=global_step, learning_rate=lr_decay, max_lr=100*lr_decay,
step_size=2*step_sz, mode='triangular', gamma=.999)
else:
learning_rate = lr_init
if RANK == 0:
print("Using constant LR: ", learning_rate)
optimizer = tf.train.AdamOptimizer(learning_rate)
#optimizer = tf.train.GradientDescentOptimizer(learning_rate)
#optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate)
if args.hvd:
import horovod.tensorflow as hvd
compression = hvd.Compression.fp16 if args.use_fp16 else hvd.Compression.none
optimizer = hvd.DistributedOptimizer(optimizer, use_locking=False, compression=compression, op=hvd.Average)
optimizer._learning_rate = tf.cast(learning_rate,tf.float32)
else:
optimizer._learning_rate = tf.cast(learning_rate,tf.float32)
step_op = optimizer.minimize(loss_op,global_step)
# Return the optimizer and the step_op
return optimizer, step_op
def test_triangular2(self):
step = 5
lr = 0.01
max_lr = 0.1
step_size = 20.
cyclic_lr = learning_rate_decay.cyclic_learning_rate(
step, lr, max_lr, step_size, mode='triangular2')
expected = self.np_cyclic_learning_rate(step,
lr,
max_lr,
step_size,
mode='triangular2')
self.assertAllClose(self.evaluate(cyclic_lr), expected, 1e-6)
def __init__(self, model, preds, labels, lr, num_u, num_v,
association_nam):
norm = num_u * num_v / float((num_u * num_v - association_nam) * 2)
preds_sub = preds
labels_sub = labels
pos_weight = float(num_u * num_v - association_nam) / (association_nam)
global_step = tf.Variable(0, trainable=False)
self.optimizer = tf.train.AdamOptimizer(
learning_rate=cyclic_learning_rate(global_step=global_step,
learning_rate=lr * 0.1,
max_lr=lr,
mode='exp_range',
gamma=.995))
self.cost = norm * tf.reduce_mean(
tf.nn.weighted_cross_entropy_with_logits(
logits=preds_sub, targets=labels_sub, pos_weight=pos_weight))
self.opt_op = self.optimizer.minimize(
self.cost,
global_step=global_step,
)
self.grads_vars = self.optimizer.compute_gradients(self.cost)
Theta1 = tf.Variable(tf.random_uniform([21, 15], -1, 1), name="Theta1")
Theta2 = tf.Variable(tf.random_uniform([15, 15], -1, 1), name="Theta2")
Theta3 = tf.Variable(tf.random_uniform([15, 3], -1, 1), name="Theta3")
Bias1 = tf.Variable(tf.zeros([15]), name="Bias1")
Bias2 = tf.Variable(tf.zeros([15]), name="Bias2")
Bias3 = tf.Variable(tf.zeros([3]), name="Bias3")
A2 = tf.sigmoid(tf.matmul(x_, Theta1) + Bias1)
A3 = tf.sigmoid(tf.matmul(A2, Theta2) + Bias2)
A4 = tf.sigmoid(tf.matmul(A3, Theta3) + Bias3)
cost = tf.reduce_mean(((y_ * tf.log(A4)) + ((1 - y_) * tf.log(1.0 - A4))) * -1)
gs = tf.train.create_global_step()
optimizer = tf.train.AdamOptimizer(learning_rate=clr.cyclic_learning_rate(
global_step=gs, mode='triangular2')).minimize(cost)
input_X = pd.read_excel(r'C:\Users\dwije\Downloads\data_normalized.xlsx',
sheet_name='upinput_norm')
input_Y = pd.read_excel(r'C:\Users\dwije\Downloads\data_normalized.xlsx',
sheet_name='upout') #Loading the output of size mx1
test_input = pd.read_excel(r'C:\Users\dwije\Downloads\data_normalized.xlsx',
sheet_name='test_norm')
test_out = pd.read_excel(r'C:\Users\dwije\Downloads\data_normalized.xlsx',
sheet_name='testout2').to_numpy().flatten()
print(test_out.shape)
init = tf.global_variables_initializer()
sess = tf.Session()
writer = tf.summary.FileWriter("./logs/xor_logs", sess.graph)
def run_experiment(config_path):
config = parse_config("config", config_path)
temp_model_path = config_path + ".model"
if "random_seed" in config:
random.seed(config["random_seed"])
numpy.random.seed(config["random_seed"])
# To print everything in config - not needed for now
# for key, val in config.items():
# print(str(key) + ": " + str(val))
data_train, data_dev, data_test = None, None, None
if config["path_train"] != None and len(config["path_train"]) > 0:
data_train = read_input_files(config["path_train"], config["max_train_sent_length"])
if config["path_dev"] != None and len(config["path_dev"]) > 0:
data_dev = read_input_files(config["path_dev"])
if config["path_test"] != None and len(config["path_test"]) > 0:
data_test = []
for path_test in config["path_test"].strip().split(":"):
data_test += read_input_files(path_test)
model = MLTModel(config)
model.build_vocabs(data_train, data_dev, data_test, config["preload_vectors"])
model.construct_network()
model.initialize_session()
if config["preload_vectors"] != None:
model.preload_word_embeddings(config["preload_vectors"])
merged_summary = tf.summary.merge_all()
writer = tf.summary.FileWriter("output", model.session.graph)
print("parameter_count: " + str(model.get_parameter_count()))
print("parameter_count_without_word_embeddings: " + str(model.get_parameter_count_without_word_embeddings()))
if data_train != None:
model_selector = config["model_selector"].split(":")[0]
model_selector_type = config["model_selector"].split(":")[1]
best_selector_value = 0.0
best_epoch = -1
# learningrate = config["learningrate"]
sess = tf.Session()
for epoch in range(config["epochs"]):
print("EPOCH: " + str(epoch))
learningrate = sess.run(clr.cyclic_learning_rate(epoch, learning_rate=0.8, max_lr=1.2, mode='triangular2'))
print("current_learningrate: " + str(learningrate))
random.shuffle(data_train)
results_train = process_sentences(writer, merged_summary, epoch, data_train, model, is_training=True, learningrate=learningrate, config=config, name="train")
if data_dev != None:
results_dev = process_sentences(epoch, data_dev, model, is_training=False, learningrate=0.0, config=config, name="dev")
if math.isnan(results_dev["dev_cost_sum"]) or math.isinf(results_dev["dev_cost_sum"]):
raise ValueError("Cost is NaN or Inf. Exiting.")
if (epoch == 0 or (model_selector_type == "high" and results_dev[model_selector] > best_selector_value)
or (model_selector_type == "low" and results_dev[model_selector] < best_selector_value)):
best_epoch = epoch
best_selector_value = results_dev[model_selector]
model.saver.save(model.session, temp_model_path, latest_filename=os.path.basename(temp_model_path)+".checkpoint")
print("best_epoch: " + str(best_epoch))
if config["stop_if_no_improvement_for_epochs"] > 0 and (epoch - best_epoch) >= config["stop_if_no_improvement_for_epochs"]:
break
if (epoch - best_epoch) > 3:
learningrate *= config["learningrate_decay"]
while config["garbage_collection"] == True and gc.collect() > 0:
pass
if data_dev != None and best_epoch >= 0:
# loading the best model so far
model.saver.restore(model.session, temp_model_path)
os.remove(temp_model_path+".checkpoint")
os.remove(temp_model_path+".data-00000-of-00001")
os.remove(temp_model_path+".index")
os.remove(temp_model_path+".meta")
if config["save"] is not None and len(config["save"]) > 0:
model.save(config["save"])
if config["path_test"] is not None:
i = 0
for path_test in config["path_test"].strip().split(":"):
data_test = read_input_files(path_test)
results_test = process_sentences(epoch, data_test, model, is_training=False, learningrate=0.0, config=config, name="test"+str(i))
i += 1
writer.close()
def train(input_train, output_train, strain_train, input_validate, output_validate, strain_validate):
print
if GENERATE_DATA:
pass
# print("Generating Input Data")
# input_train = np.zeros((NUM_SAMPLES_TRAIN, NUM_STRANDS, NUM_MARKERS, NUM_AMINO_ACIDS))
# for strain_counter in range(NUM_SAMPLES_TRAIN):
# for strand_counter in range(NUM_STRANDS):
# for snp_counter in range(NUM_MARKERS):
# rand_aa = random.randint(0, 19)
# input_train[strain_counter][strand_counter][snp_counter][rand_aa] = 1.0
#
# input_validate = np.zeros((NUM_SAMPLES_VALIDATE, NUM_STRANDS, NUM_MARKERS, NUM_AMINO_ACIDS))
# for strain_counter in range(NUM_SAMPLES_VALIDATE):
# for strand_counter in range(NUM_STRANDS):
# for snp_counter in range(NUM_MARKERS):
# rand_aa = random.randint(0, 19)
# input_validate[strain_counter][strand_counter][snp_counter][rand_aa] = 1.0
#
# print("Generating Output Data")
# output_train = np.random.rand(NUM_SAMPLES_TRAIN, NUM_TRAITS) * 5
# output_validate = np.random.rand(NUM_SAMPLES_VALIDATE, NUM_TRAITS) * 5
epoch = tf.Variable(0, trainable=False, dtype=tf.int32)
epoch_add_op = epoch.assign(epoch + 1)
loaded_epoch = tf.placeholder(dtype=tf.int32)
epoch_load_op = epoch.assign(loaded_epoch)
# learning_rate = tf.Variable(LEARN_RATE, trainable=False, dtype=tf.float32)
learning_rate = cyclic_learning_rate(global_step=epoch, mode='triangular2')
keep_prob_dense = tf.placeholder(dtype=tf.float32)
L2_param = tf.constant(L2_WEIGHT, dtype=tf.float32)
#input_variable = tf.placeholder(dtype=tf.float32, shape=[None, 2, NUM_MARKERS, 20])
input_variable = tf.placeholder(dtype=tf.float32, shape=[None, NUM_PCA])
output_variable = tf.placeholder(dtype=tf.float32, shape=[None, NUM_TRAITS])
output_prediction_variable = model_residual(input_variable, keep_prob_dense)
# cost = cost_function_sse(output_prediction_variable, output_variable)
scaling = tf.placeholder(dtype=tf.float32, shape=[None, NUM_TRAITS])
cost = cost_function_mse(output_prediction_variable, output_variable, scaling)
total_parameters = 0
L2_cost = 0
counter = 0
for variable in tf.trainable_variables():
shape = variable.get_shape()
L2_cost += tf.nn.l2_loss(variable)
variable_parameters = 1
for dim in shape:
variable_parameters *= dim.value
total_parameters += variable_parameters
counter += 1
print "Num Variables = ", counter, "\tTotal parameters = ", total_parameters
print
L2_cost = tf.multiply(L2_param, L2_cost)
cost = tf.add(cost, L2_cost)
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate, epsilon=1.0e-8)
grads_and_vars = optimizer.compute_gradients(cost)
if CLIP > 0:
grads_and_vars = [(tf.clip_by_value(grad, -1 * CLIP, CLIP), var) for grad, var in grads_and_vars]
if NOISE > 0:
grads_and_vars = [(i[0] + tf.random_normal(shape=tf.shape(i[0]), mean=0, stddev=NOISE), i[1]) for i in
grads_and_vars]
train_op = optimizer.apply_gradients(grads_and_vars)
saver = tf.train.Saver()
print("Running Session")
with tf.Session() as sess:
if LOAD_TRAINING_MODEL:
print("Reading model parameters from %s" % PATH_SAVE_TRAIN)
saver.restore(sess, tf.train.latest_checkpoint(PATH_SAVE_TRAIN))
tag = PATH_SAVE + "loss_summary.txt"
loss_file = open(tag)
previous_best = np.inf
counter = 1
for line in loss_file:
line = line.split()
lv = float(line[3])
if lv < previous_best:
previous_best = lv
counter += 1
loss_file.close()
sess.run(epoch_load_op, feed_dict={loaded_epoch: counter})
print "current epoch =", epoch.eval(session=sess)
print "best validation error =", previous_best
else:
sess.run(tf.global_variables_initializer())
previous_best = np.inf
best_accuracy_total = 0.0
if TRAIT_TARGET == "CANNABINOIDS" or TRAIT_TARGET == "TERPENES" or TRAIT_TARGET == "ALL":
scale_train = np.array([[SCALE] + [1]*(NUM_TRAITS-1), ] * BATCH_SIZE)
scale_val = np.array([[SCALE] + [1]*(NUM_TRAITS-1), ] * len(input_validate))
elif TRAIT_TARGET == "SPECIFIC":
scale_train = np.array([[SCALE] , ] * BATCH_SIZE)
scale_val = np.array([[SCALE] , ] * len(input_validate))
while True:
sess.run(epoch_add_op)
loss_train = 0
loss_train_l2 = 0
shuffle_in_unison_3(input_train, output_train, strain_train)
for batch_num in xrange(0, len(input_train), BATCH_SIZE):
if batch_num + BATCH_SIZE > len(input_train):
break
input_train_batch = input_train[batch_num:batch_num + BATCH_SIZE]
output_train_batch = output_train[batch_num:batch_num + BATCH_SIZE]
_, loss, loss_l2, output_pred_train = sess.run(
[train_op, cost, L2_cost, output_prediction_variable],
feed_dict={output_variable: output_train_batch,
input_variable: input_train_batch,
scaling: scale_train,
keep_prob_dense: KEEP_PROB_DENSE})
loss_train += loss
loss_train_l2 += loss_l2
loss_train -= loss_train_l2
loss_validate, loss_validate_l2, output_pred_validate = sess.run(
[cost, L2_cost, output_prediction_variable],
feed_dict={output_variable: output_validate,
input_variable: input_validate,
scaling: scale_val,
keep_prob_dense: 1.0})
loss_validate -= loss_validate_l2
print epoch.eval(session=sess), "\t", loss_train, "\t", loss_validate
tag = PATH_SAVE_TRAIN + "best_train_loss_model"
saver.save(sess, tag)
if loss_validate < previous_best:
total_percent_real = np.sum(output_validate, axis=1)
for i in range(len(total_percent_real)):
if total_percent_real[i] == 0.0:
total_percent_real[i] = 0.00001
total_percent_pred = np.sum(output_pred_validate, axis=1)
diff_total = abs(total_percent_real - total_percent_pred)
accuracy_total = (1.0 - diff_total / total_percent_real) * 100
tot = sum(accuracy_total)
accuracy_mean_total = tot / len(accuracy_total)
if accuracy_mean_total > best_accuracy_total:
best_accuracy_total = accuracy_mean_total
previous_best = loss_validate
tag = PATH_SAVE_VAL + "best_val_loss_model"
saver.save(sess, tag)
if epoch.eval(session=sess) % 25 == 0:
# diff = abs(output_validate - output_pred_validate)
# accuracy = (1.0 - diff / output_validate) * 100
# print accuracy
# exit()
total_percent_real = np.sum(output_validate, axis=1)
total_percent_pred = np.sum(output_pred_validate, axis=1)
diff_total = abs(total_percent_real - total_percent_pred)
accuracy_total = (1.0 - diff_total / total_percent_real) * 100
tot = sum(accuracy_total)
accuracy_mean_total = tot / len(accuracy_total)
if accuracy_mean_total > best_accuracy_total:
best_accuracy_total = accuracy_mean_total
print "\t\t", best_accuracy_total, "\t", accuracy_mean_total
if epoch.eval(session=sess) % EPOCH_CHECK == 0:
tag = PATH_SAVE + "loss_summary.txt"
string = str(loss_train) + " " + str(loss_validate) + "\n"
with open(tag, "a") as myfile:
myfile.write(string)
if epoch.eval(session=sess) == EPOCH_LIMIT:
break
string = str(TRAIT_TARGET) + "\t" + str(BATCH_SIZE) + "\t" + str(NUM_LAYERS_DENSE) + "\t" + str(
NUM_NODES) + "\t" + str(LEARN_RATE) + "\t" + str(L2_WEIGHT) + "\t" + str(
epoch.eval(session=sess)) + "\t" + str(TEN_FOLD_VALIDATION_COUNTER) + "\t" + str(
best_accuracy) + "\t" + str(DENSE_BATCH_NORM_FLAG) + "\t" + str(KEEP_PROB_DENSE) + "\n"
with open(PATH_SAVE_FINAL, "a") as myfile:
myfile.write(string)
def model_fn(features, labels, mode, params):
with tf.variable_scope('model', reuse=tf.AUTO_REUSE):
# Model Definition (resnet_3D)
model_output_ops = resnet_3d(
inputs=features[
'x'], # Input: Concatenated Patches (dimensions=128cube,channel=61-63; defined by 'patch_size')
num_res_units=2,
num_classes=NUM_CLASSES,
filters=(16, 32, 64, 128, 256),
strides=((1, 1, 1), (2, 2, 2), (2, 2, 2), (2, 2, 2), (2, 2, 2)),
mode=mode,
activation=tf.nn.relu6,
kernel_initializer=tf.initializers.variance_scaling(
distribution='uniform'),
bias_initializer=tf.zeros_initializer(),
kernel_regularizer=tf.contrib.layers.l2_regularizer(1e-3))
# Prediction Mode
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(
mode=mode,
predictions=model_output_ops,
export_outputs={
'out': tf.estimator.export.PredictOutput(model_output_ops)
})
# Loss Function
one_hot_labels = tf.reshape(tf.one_hot(labels['y'], depth=NUM_CLASSES),
[-1, NUM_CLASSES])
loss = tf.losses.softmax_cross_entropy(
onehot_labels=one_hot_labels, logits=model_output_ops['logits'])
global_step = tf.train.get_global_step()
# Learning Rate
if (LR_MODE == 'eLR'):
# Exponential Learning Rate Decay [ learning_rate = learning_rate * decay_rate ^ (global_step / decay_steps) ]
learning_rate = tf.train.exponential_decay(
eLR_INITIAL,
global_step,
decay_steps=eLRDECAY_STEPS,
decay_rate=eLRDECAY_RATE,
staircase=True)
elif (LR_MODE == 'CLR'):
# Cyclic Learning Rate
# >> cycle = floor( 1 + global_step / ( 2 * step_size ) )
# >> x = abs( global_step / step_size - 2 * cycle + 1 )
# >> clr = learning_rate + ( max_lr - learning_rate ) * max( 0 , 1 - x )
learning_rate = cyclic_learning_rate(global_step=global_step,
learning_rate=CLR_MINLR,
max_lr=CLR_MAXLR,
step_size=CLR_STEPSIZE,
gamma=CLR_GAMMA,
mode=CLR_MODE)
# Optimizer
if params["opt"] == 'adam':
optimiser = tf.train.AdamOptimizer(learning_rate=learning_rate,
epsilon=1e-5)
optimiser = tf.contrib.estimator.TowerOptimizer(optimiser)
elif params["opt"] == 'momentum':
optimiser = tf.train.MomentumOptimizer(learning_rate=learning_rate,
momentum=0.9)
optimiser = tf.contrib.estimator.TowerOptimizer(optimiser)
elif params["opt"] == 'rmsprop':
optimiser = tf.train.RMSPropOptimizer(learning_rate=learning_rate,
momentum=0.9)
optimiser = tf.contrib.estimator.TowerOptimizer(optimiser)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_op = optimiser.minimize(loss, global_step=global_step)
# Custom Image Summaries (TensorBoard)
my_image_summaries = {}
my_image_summaries['CT_Patch'] = features['x'][0, 32, :, :, 0]
expected_output_size = [1, PATCH, PATCH, 1] # [B, W, H, C]
[
tf.summary.image(name, tf.reshape(image, expected_output_size))
for name, image in my_image_summaries.items()
]
# Track Metrics
acc = tf.metrics.accuracy
prec = tf.metrics.precision
auc = tf.metrics.auc
eval_metric_ops = {
"accuracy": acc(labels['y'], model_output_ops['y_']),
"precision": prec(labels['y'], model_output_ops['y_']),
"auc": prec(labels['y'], model_output_ops['y_'])
}
# Return EstimatorSpec Object
return tf.estimator.EstimatorSpec(mode=mode,
predictions=model_output_ops,
loss=loss,
train_op=train_op,
eval_metric_ops=eval_metric_ops)
def train_test(FLAGS):
dataset_tools = import_module('tools.' + FLAGS.dataset)
train_images, train_labels = dataset_tools.get_data('train')
if FLAGS.target_dataset is not None:
target_dataset_tools = import_module('tools.' + FLAGS.target_dataset)
train_images_unlabeled, train_images_label = target_dataset_tools.get_data(
FLAGS.target_dataset_split)
else:
train_images_unlabeled, train_images_label = dataset_tools.get_data(
'unlabeled')
architecture = getattr(semisup.architectures, FLAGS.architecture)
num_labels = dataset_tools.NUM_LABELS
image_shape = dataset_tools.IMAGE_SHAPE
# Sample labeled training subset.
seed = FLAGS.sup_seed if FLAGS.sup_seed != -1 else None
sup_by_label = semisup.sample_by_label(train_images, train_labels,
FLAGS.sup_per_class, num_labels,
seed)
# Sample unlabeled training subset.
if FLAGS.unsup_samples > -1:
num_unlabeled = len(train_images_unlabeled)
assert FLAGS.unsup_samples <= num_unlabeled, (
'Chose more unlabeled samples ({})'
' than there are in the '
'unlabeled batch ({}).'.format(FLAGS.unsup_samples, num_unlabeled))
#TODO: make smaple slections per classs :done
#unsup_by_label = semisup.sample_by_label(train_images_unlabeled, train_images_label,
# FLAGS.unsup_samples/num_labels+num_labels, num_labels,
# seed)
rng = np.random.RandomState(seed=seed)
train_images_unlabeled = train_images_unlabeled[rng.choice(
num_unlabeled, FLAGS.unsup_samples, False)]
graph = tf.Graph()
with graph.as_default():
with tf.device(
tf.train.replica_device_setter(FLAGS.ps_tasks,
merge_devices=True)):
# Set up inputs.
t_unsup_images = semisup.create_input(train_images_unlabeled, None,
FLAGS.unsup_batch_size)
t_sup_images, t_sup_labels = semisup.create_per_class_inputs(
sup_by_label, FLAGS.sup_per_batch)
#print(t_sup_images.shape)
#with tf.Session() as sess: print (t_sup_images.eval().shape)
if FLAGS.remove_classes:
t_sup_images = tf.slice(t_sup_images, [
0, 0, 0, 0
], [FLAGS.sup_per_batch *
(num_labels - FLAGS.remove_classes)] + image_shape)
# Resize if necessary.
if FLAGS.new_size > 0:
new_shape = [FLAGS.new_size, FLAGS.new_size, image_shape[-1]]
else:
new_shape = None
# Apply augmentation
if FLAGS.augmentation:
# TODO(haeusser) generalize augmentation
def _random_invert(inputs1, _):
inputs = tf.cast(inputs1, tf.float32)
inputs = tf.image.adjust_brightness(
inputs, tf.random_uniform((1, 1), 0.0, 0.5))
inputs = tf.image.random_contrast(inputs, 0.3, 1)
# inputs = tf.image.per_image_standardization(inputs)
inputs = tf.image.random_hue(inputs, 0.05)
inputs = tf.image.random_saturation(inputs, 0.5, 1.1)
def f1():
return tf.abs(inputs) #annotations
def f2():
return tf.abs(inputs1)
return tf.cond(tf.less(tf.random_uniform([], 0.0, 1), 0.5),
f1, f2)
augmentation_function = _random_invert
else:
augmentation_function = None
# Create function that defines the network.
model_function = partial(
architecture,
new_shape=new_shape,
img_shape=image_shape,
augmentation_function=augmentation_function,
batch_norm_decay=FLAGS.batch_norm_decay,
emb_size=FLAGS.emb_size)
# Set up semisup model.
model = semisup.SemisupModel(model_function, num_labels,
image_shape)
# Compute embeddings and logits.
t_sup_emb = model.image_to_embedding(t_sup_images)
t_sup_logit = model.embedding_to_logit(t_sup_emb)
# Add losses.
if FLAGS.unsup_samples != 0:
t_unsup_emb = model.image_to_embedding(t_unsup_images)
visit_weight_envelope_steps = (
FLAGS.walker_weight_envelope_steps
if FLAGS.visit_weight_envelope_steps == -1 else
FLAGS.visit_weight_envelope_steps)
visit_weight_envelope_delay = (
FLAGS.walker_weight_envelope_delay
if FLAGS.visit_weight_envelope_delay == -1 else
FLAGS.visit_weight_envelope_delay)
visit_weight = apply_envelope(
type=FLAGS.visit_weight_envelope,
step=model.step,
final_weight=FLAGS.visit_weight,
growing_steps=visit_weight_envelope_steps,
delay=visit_weight_envelope_delay)
walker_weight = apply_envelope(
type=FLAGS.walker_weight_envelope,
step=model.step,
final_weight=FLAGS.walker_weight,
growing_steps=FLAGS.walker_weight_envelope_steps, # pylint:disable=line-too-long
delay=FLAGS.walker_weight_envelope_delay)
tf.summary.scalar('Weights_Visit', visit_weight)
tf.summary.scalar('Weights_Walker', walker_weight)
model.add_semisup_loss(t_sup_emb,
t_unsup_emb,
t_sup_labels,
visit_weight=visit_weight,
walker_weight=walker_weight)
model.add_logit_loss(t_sup_logit,
t_sup_labels,
weight=FLAGS.logit_weight)
# Set up learning rate
if FLAGS.learning_rate_type is None:
t_learning_rate = tf.maximum(
tf.train.exponential_decay(FLAGS.learning_rate,
model.step,
FLAGS.decay_steps,
FLAGS.decay_factor,
staircase=True),
FLAGS.minimum_learning_rate)
elif FLAGS.learning_rate_type == 'exp2':
t_learning_rate = tf.maximum(
cyclic_learning_rate(model.step,
FLAGS.minimum_learning_rate,
FLAGS.maximum_learning_rate,
FLAGS.learning_rate_cycle_step,
mode='exp_range',
gamma=0.9999),
cyclic_learning_rate(model.step,
FLAGS.minimum_learning_rate,
FLAGS.learning_rate,
FLAGS.learning_rate_cycle_step,
mode='triangular',
gamma=0.9994))
else:
t_learning_rate = tf.maximum(
cyclic_learning_rate(model.step,
FLAGS.minimum_learning_rate,
FLAGS.learning_rate,
FLAGS.learning_rate_cycle_step,
mode='triangular',
gamma=0.9994),
FLAGS.minimum_learning_rate)
# Create training operation and start the actual training loop.
train_op = model.create_train_op(t_learning_rate)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# config.log_device_placement = True
saver = tf_saver.Saver(max_to_keep=FLAGS.max_checkpoints,
keep_checkpoint_every_n_hours=FLAGS.
keep_checkpoint_every_n_hours) # pylint:disable=line-too-long
final_loss = slim.learning.train(
train_op,
logdir=FLAGS.logdir + '/train',
save_summaries_secs=FLAGS.save_summaries_secs,
save_interval_secs=FLAGS.save_interval_secs,
master=FLAGS.master,
is_chief=(FLAGS.task == 0),
startup_delay_steps=(FLAGS.task * 20),
log_every_n_steps=FLAGS.log_every_n_steps,
session_config=config,
trace_every_n_steps=1000,
saver=saver,
number_of_steps=FLAGS.max_steps,
#session_wrapper=tf_debug.LocalCLIDebugWrapperSession
)
print(final_loss)
return final_loss
