def evaluate_sess_sample(sess,
model_spec,
num_steps,
writer=None,
params=None):
"""Train the model on `num_steps` batches.
Args:
sess: (tf.Session) current session
model_spec: (dict) contains the graph operations or nodes needed for training
num_steps: (int) train for this number of batches
writer: (tf.summary.FileWriter) writer for summaries. Is None if we don't log anything
params: (Params) hyperparameters
"""
update_metrics = model_spec['update_metrics']
eval_metrics = model_spec['metrics']
global_step = tf.train.get_global_step()
# Load the evaluation dataset into the pipeline and initialize the metrics init op
# sess.run(model_spec['iterator_init_op'])
sess.run(model_spec['metrics_init_op'])
for temp_query_id in range(int(num_steps)):
sess.run(update_metrics)
# Get the values of the metrics
metrics_values = {k: v[0] for k, v in eval_metrics.items()}
metrics_val = sess.run(metrics_values)
expanded_metrics_val = get_expaned_metrics(metrics_val)
metrics_string = " ; ".join("{}: {:05.4f}".format(k, v)
for k, v in expanded_metrics_val.items())
logging.info("- Eval metrics: " + metrics_string)
# Add summaries manually to writer at global_step_val
return expanded_metrics_val['loss']
def evaluate_sess(sess, model_spec, num_steps, writer=None, params=None):
"""Train the model on `num_steps` batches.
Args:
sess: (tf.Session) current session
model_spec: (dict) contains the graph operations or nodes needed for training
num_steps: (int) train for this number of batches
writer: (tf.summary.FileWriter) writer for summaries. Is None if we don't log anything
params: (Params) hyperparameters
"""
update_metrics = model_spec['update_metrics']
eval_metrics = model_spec['metrics']
global_step = tf.train.get_global_step()
# Load the evaluation dataset into the pipeline and initialize the metrics init op
# sess.run([model_spec['iterator_init_op'], model_spec['metrics_init_op']])
sess.run(model_spec['iterator_init_op'])
sess.run(model_spec['metrics_init_op'])
if params.save_predictions:
# save the predictions and lable_qid to files
prediction_list = []
label_list = []
# compute metrics over the dataset
for temp_query_id in range(int(num_steps)):
# prediction_per_query, label_per_query, height = sess.run([predictions, labels, model_spec["height"]])
# logging.info("- height per query: \n" + str(height))
prediction_per_query, label_per_query, label_gains, _ = sess.run([model_spec["predictions"], \
model_spec["labels"], model_spec["label_gains"], update_metrics])
prediction_list.extend([v[0] for v in prediction_per_query.tolist()])
# prediction_string = "\n".join(str(v[0]) for v in prediction_per_query.tolist())
# logging.info("- prediction_per_query: \n" + str(prediction_string))
label_per_query_list = label_per_query.tolist()
label_gains_list = label_gains.tolist()
# label_per_query_list_string = "\n".join(str(v[0]) for v in label_per_query_list)
# logging.warning("- label_per_query_list_string: \n" + label_per_query_list_string)
# label_gains_list_string = "\n".join(str(v[0]) for v in label_gains_list)
# logging.info("- label_gains_list: \n" + label_gains_list_string)
label_list.extend(['{} qid:{} 1:{}'.format(int(label_per_query_list[i][0]), \
temp_query_id, \
label_gains_list[i][0]) \
for i in range(0, len(label_per_query_list))])
save_predictions_to_file(prediction_list, "./prediction_output")
# tensorflow mess up test input orders
save_predictions_to_file(label_list, "./label_output")
else:
# only update metrics
for temp_query_id in range(int(num_steps)):
sess.run(update_metrics)
# Get the values of the metrics
metrics_values = {k: v[0] for k, v in eval_metrics.items()}
metrics_val = sess.run(metrics_values)
expanded_metrics_val = get_expaned_metrics(metrics_val, params.top_ks)
metrics_string = " ; ".join("{}: {:05.3f}".format(k, v) for k, v in expanded_metrics_val.items())
logging.info("- Eval metrics: " + metrics_string)
# Add summaries manually to writer at global_step_val
if writer is not None:
global_step_val = sess.run(global_step)
for tag, val in expanded_metrics_val.items():
summ = tf.Summary(value=[tf.Summary.Value(tag=tag, simple_value=val)])
writer.add_summary(summ, global_step_val)
return expanded_metrics_val
def take_train_samples_sess(sess, model_spec, num_steps, params, sorted_index):
"""Train the model on `num_steps` batches.
Args:
sess: (tf.Session) current session
model_spec: (dict) contains the graph operations or nodes needed for training
num_steps: (int) train for this number of batches
writer: (tf.summary.FileWriter) writer for summaries. Is None if we don't log anything
params: (Params) hyperparameters
# """
# logging.info('sorted_index:\n {}'.format(sorted_index))
update_metrics = model_spec['update_metrics']
eval_metrics = model_spec['metrics']
# global_step = tf.train.get_global_step()
# Load the evaluation dataset into the pipeline and initialize the metrics init op
# sess.run(model_spec['iterator_init_op'])
sess.run(model_spec['metrics_init_op'])
count = 0
with tf.python_io.TFRecordWriter(
os.path.join(params.data_dir, 'sample-temp.tfrecords')) as writer:
# compute metrics over the dataset
for temp_query_id in range(int(num_steps) + 1):
features, labels, _ = sess.run([model_spec["features"], \
model_spec["labels"], update_metrics])
# whether save the sample
if temp_query_id not in sorted_index:
continue
for (feature, label) in zip(features, labels):
image_raw = feature.tostring()
label_raw = np.argmax(label)
# logging.info('------------------------------------------------')
count += 1
# logging.info(type(feature))
# logging.info(feature.shape)
# logging.info(label_raw)
# logging.info('------------------------------------------------')
example = tf.train.Example(features=tf.train.Features(
feature={
'height': _int64_feature(int(params.height)),
'width': _int64_feature(int(params.width)),
'depth': _int64_feature(int(params.depth)),
'label': _int64_feature(int(label_raw)),
'image_raw': _bytes_feature(image_raw)
}))
writer.write(example.SerializeToString())
# Get the values of the metrics
metrics_values = {k: v[0] for k, v in eval_metrics.items()}
metrics_val = sess.run(metrics_values)
expanded_metrics_val = get_expaned_metrics(metrics_val)
metrics_string = " ; ".join("{}: {:05.4f}".format(k, v)
for k, v in expanded_metrics_val.items())
logging.info("- take_train_samples_sess metrics: " + metrics_string)
# logging.info('*********************************************{}'.format(count))
return expanded_metrics_val
def train_sess(sess, model_spec, num_steps, writer, params):
"""Train the model on `num_steps` batches
Args:
sess: (tf.Session) current session
model_spec: (dict) contains the graph operations or nodes needed for training
num_steps: (int) train for this number of batches
writer: (tf.summary.FileWriter) writer for summaries
params: (Params) hyperparameters
"""
# Get relevant graph operations or nodes needed for training
loss = model_spec['loss']
train_op = model_spec['train_op']
update_metrics = model_spec['update_metrics']
metrics = model_spec['metrics']
summary_op = model_spec['summary_op']
global_step = tf.train.get_global_step()
# Load the training dataset into the pipeline and initialize the metrics local variables
# sess.run(model_spec['iterator_init_op'])
sess.run(model_spec['metrics_init_op'])
# Use tqdm for progress bar
t = trange(int(num_steps))
epoch_loss = []
for i in t:
# Evaluate summaries for tensorboard only once in a while
if i == params.save_summary_steps - 1:
# if i % params.save_summary_steps == 0:
# Perform a mini-batch update
_, _, loss_val, summ, global_step_val = sess.run(
[train_op, update_metrics, loss, summary_op, global_step])
# Write summaries for tensorboard
writer.add_summary(summ, global_step_val)
else:
_, _, loss_val = sess.run([train_op, update_metrics, loss])
# Log the loss in the tqdm progress bar
# t.set_postfix(loss='{:05.3f}'.format(loss_val))
epoch_loss.append(loss_val)
# logging.info('loss_val: {}'.format(epoch_loss))
metrics_values = {k: v[0] for k, v in metrics.items()}
metrics_val = sess.run(metrics_values)
expanded_metrics_val = get_expaned_metrics(metrics_val)
metrics_string = " ; ".join("{}: {:05.4f}".format(k, v)
for k, v in expanded_metrics_val.items())
logging.info("- Train metrics: " + metrics_string)
return epoch_loss
def train_initial_sess(sess, model_spec, num_steps, writer, params, \
old_index, numbers_of_selections, sums_of_reward, arm_weights, max_upper_bound, old_loss_val):
"""Train the model on `num_steps` batches
Args:
sess: (tf.Session) current session
model_spec: (dict) contains the graph operations or nodes needed for training
num_steps: (int) train for this number of batches
writer: (tf.summary.FileWriter) writer for summaries
params: (Params) hyperparameters
"""
# Get relevant graph operations or nodes needed for training
loss = model_spec['loss']
# train_op = model_spec['train_op']
update_metrics = model_spec['update_metrics']
metrics = model_spec['metrics']
summary_op = model_spec['summary_op']
# reward = model_spec['reward']
global_step = tf.train.get_global_step()
# Load the training dataset into the pipeline and initialize the metrics local variables
# sess.run(model_spec['iterator_init_op'])
sess.run(model_spec['metrics_init_op'])
# Use tqdm for progress bar
t = trange(int(num_steps))
epoch_loss = []
index = 0
# for i in t:
# # Evaluate summaries for tensorboard only once in a while
# if i == params.save_summary_steps - 1:
# # if i % params.save_summary_steps == 0:
# # Perform a mini-batch update
# _, _, loss_val, summ, global_step_val = sess.run([train_op, update_metrics, loss,
# summary_op, global_step])
# # Write summaries for tensorboard
# writer.add_summary(summ, global_step_val)
# else:
# _, _, loss_val = sess.run([train_op, update_metrics, loss])
# # Log the loss in the tqdm progress bar
# # t.set_postfix(loss='{:05.3f}'.format(loss_val))
# epoch_loss.append(loss_val)
for i in t:
# Evaluate summaries for tensorboard only once in a while
if i == params.save_summary_steps - 1:
# if i % params.save_summary_steps == 0:
# Perform a mini-batch update
_, loss_val, summ, global_step_val = sess.run(
[update_metrics, loss, summary_op, global_step])
# Write summaries for tensorboard
writer.add_summary(summ, global_step_val)
else:
_, loss_val = sess.run([update_metrics, loss])
# Log the loss in the tqdm progress bar
# t.set_postfix(loss='{:05.3f}'.format(loss_val))
epoch_loss.append(loss_val)
#################################################################################
if 'retrain' in params.loss_fn:
# calculate the rewards of the (10) clusters --> pick and update the action variable
global_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope='model/rewards')
action_var = [v for v in global_vars if 'action' in v.name][0]
action_var_value = sess.run([action_var])
# logging.info('****action_var at train_sess:\n {}'.format(action_var_value))
# logging.info('*****old_index is {}, loss_val is {}'.format(old_index, loss_val))
sums_of_reward[old_index] += loss_val - old_loss_val
old_loss_val = loss_val
index = old_index
# index, numbers_of_selections, arm_weights, \
# max_upper_bound = rl_weights(params, numbers_of_selections, \
# sums_of_reward, max_upper_bound, \
# t, arm_weights)
# logging.info('numbers_of_selections at i:\n {}'.format(numbers_of_selections))
# total_reward += reward
# index = 2
assign_op = action_var.assign([index])
sess.run(assign_op)
action_var_value = sess.run([action_var])
# logging.info('******updated action_var at train_sess:\n {}'.format(action_var_value))
#################################################################################
metrics_values = {k: v[0] for k, v in metrics.items()}
metrics_val = sess.run(metrics_values)
expanded_metrics_val = get_expaned_metrics(metrics_val)
metrics_string = " ; ".join("{}: {:05.4f}".format(k, v)
for k, v in expanded_metrics_val.items())
# logging.info("- Train metrics: " + metrics_string)
# logging.info("-- Reward: \n")
# logging.info(reward)
return epoch_loss, index, sums_of_reward, arm_weights, max_upper_bound, old_loss_val
def evaluate_on_train_sess(sess, model_spec, num_steps, params=None):
"""Train the model on `num_steps` batches.
Args:
sess: (tf.Session) current session
model_spec: (dict) contains the graph operations or nodes needed for training
num_steps: (int) train for this number of batches
writer: (tf.summary.FileWriter) writer for summaries. Is None if we don't log anything
params: (Params) hyperparameters
"""
update_metrics = model_spec['update_metrics']
eval_metrics = model_spec['metrics']
# global_step = tf.train.get_global_step()
# Load the evaluation dataset into the pipeline and initialize the metrics init op
# sess.run(model_spec['iterator_init_op'])
sess.run(model_spec['metrics_init_op'])
if params.save_predictions:
# save the predictions and lable_qid to files
prediction_list = []
label_list = []
# compute metrics over the dataset
for temp_query_id in range(int(num_steps)):
prediction_per_query, label_per_query, _ = sess.run([model_spec["predictions"], \
model_spec["labels"], update_metrics])
prediction_list.extend(
[v[0] for v in prediction_per_query.tolist()])
label_per_query_list = label_per_query.tolist()
label_list.extend(['{} id:{}'.format(int(label_per_query_list[i][0]), \
temp_query_id) \
for i in range(0, len(label_per_query_list))])
save_predictions_to_file(prediction_list, "./prediction_output")
# tensorflow mess up test input orders
save_predictions_to_file(label_list, "./label_output")
else:
# only update metrics
for temp_query_id in range(int(num_steps)):
# # masks_weights=[v for v in tf.trainable_variables() if 'model/mask' in v.name]
# # # loss = eval_metrics['loss']
# # # update_masks = [cal_gradient(loss, weight) for weight in masks_weights]
# # sess.run(update_masks)
# # sess.run(model_spec['masks'])
# masks = sess.run(model_spec['masks'])
# print('Printing masks')
# print(masks)
sess.run(model_spec['masks'])
# mweights3_2=[v for v in tf.trainable_variables() if 'model/mask/mweights3_2' in v.name]
# mweights3_2 = sess.run(mweights3_2)
# print('Printing mweights3_2')
# print(mweights3_2)
sess.run(update_metrics)
masks = [v for v in tf.trainable_variables() if 'model/mask' in v.name]
update_masks = [tf.assign(nm, m/float(num_steps)) for (nm, m) in \
zip(masks, masks)]
sess.run(update_masks)
# Get the values of the metrics
metrics_values = {k: v[0] for k, v in eval_metrics.items()}
metrics_val = sess.run(metrics_values)
expanded_metrics_val = get_expaned_metrics(metrics_val)
metrics_string = " ; ".join("{}: {:05.4f}".format(k, v)
for k, v in expanded_metrics_val.items())
logging.info("- metrics on full train: " + metrics_string)
return expanded_metrics_val