def evaluate_sess_sample(sess, model_spec, num_steps, writer=None, params=None): """Train the model on `num_steps` batches. Args: sess: (tf.Session) current session model_spec: (dict) contains the graph operations or nodes needed for training num_steps: (int) train for this number of batches writer: (tf.summary.FileWriter) writer for summaries. Is None if we don't log anything params: (Params) hyperparameters """ update_metrics = model_spec['update_metrics'] eval_metrics = model_spec['metrics'] global_step = tf.train.get_global_step() # Load the evaluation dataset into the pipeline and initialize the metrics init op # sess.run(model_spec['iterator_init_op']) sess.run(model_spec['metrics_init_op']) for temp_query_id in range(int(num_steps)): sess.run(update_metrics) # Get the values of the metrics metrics_values = {k: v[0] for k, v in eval_metrics.items()} metrics_val = sess.run(metrics_values) expanded_metrics_val = get_expaned_metrics(metrics_val) metrics_string = " ; ".join("{}: {:05.4f}".format(k, v) for k, v in expanded_metrics_val.items()) logging.info("- Eval metrics: " + metrics_string) # Add summaries manually to writer at global_step_val return expanded_metrics_val['loss']
def evaluate_sess(sess, model_spec, num_steps, writer=None, params=None): """Train the model on `num_steps` batches. Args: sess: (tf.Session) current session model_spec: (dict) contains the graph operations or nodes needed for training num_steps: (int) train for this number of batches writer: (tf.summary.FileWriter) writer for summaries. Is None if we don't log anything params: (Params) hyperparameters """ update_metrics = model_spec['update_metrics'] eval_metrics = model_spec['metrics'] global_step = tf.train.get_global_step() # Load the evaluation dataset into the pipeline and initialize the metrics init op # sess.run([model_spec['iterator_init_op'], model_spec['metrics_init_op']]) sess.run(model_spec['iterator_init_op']) sess.run(model_spec['metrics_init_op']) if params.save_predictions: # save the predictions and lable_qid to files prediction_list = [] label_list = [] # compute metrics over the dataset for temp_query_id in range(int(num_steps)): # prediction_per_query, label_per_query, height = sess.run([predictions, labels, model_spec["height"]]) # logging.info("- height per query: \n" + str(height)) prediction_per_query, label_per_query, label_gains, _ = sess.run([model_spec["predictions"], \ model_spec["labels"], model_spec["label_gains"], update_metrics]) prediction_list.extend([v[0] for v in prediction_per_query.tolist()]) # prediction_string = "\n".join(str(v[0]) for v in prediction_per_query.tolist()) # logging.info("- prediction_per_query: \n" + str(prediction_string)) label_per_query_list = label_per_query.tolist() label_gains_list = label_gains.tolist() # label_per_query_list_string = "\n".join(str(v[0]) for v in label_per_query_list) # logging.warning("- label_per_query_list_string: \n" + label_per_query_list_string) # label_gains_list_string = "\n".join(str(v[0]) for v in label_gains_list) # logging.info("- label_gains_list: \n" + label_gains_list_string) label_list.extend(['{} qid:{} 1:{}'.format(int(label_per_query_list[i][0]), \ temp_query_id, \ label_gains_list[i][0]) \ for i in range(0, len(label_per_query_list))]) save_predictions_to_file(prediction_list, "./prediction_output") # tensorflow mess up test input orders save_predictions_to_file(label_list, "./label_output") else: # only update metrics for temp_query_id in range(int(num_steps)): sess.run(update_metrics) # Get the values of the metrics metrics_values = {k: v[0] for k, v in eval_metrics.items()} metrics_val = sess.run(metrics_values) expanded_metrics_val = get_expaned_metrics(metrics_val, params.top_ks) metrics_string = " ; ".join("{}: {:05.3f}".format(k, v) for k, v in expanded_metrics_val.items()) logging.info("- Eval metrics: " + metrics_string) # Add summaries manually to writer at global_step_val if writer is not None: global_step_val = sess.run(global_step) for tag, val in expanded_metrics_val.items(): summ = tf.Summary(value=[tf.Summary.Value(tag=tag, simple_value=val)]) writer.add_summary(summ, global_step_val) return expanded_metrics_val
def take_train_samples_sess(sess, model_spec, num_steps, params, sorted_index): """Train the model on `num_steps` batches. Args: sess: (tf.Session) current session model_spec: (dict) contains the graph operations or nodes needed for training num_steps: (int) train for this number of batches writer: (tf.summary.FileWriter) writer for summaries. Is None if we don't log anything params: (Params) hyperparameters # """ # logging.info('sorted_index:\n {}'.format(sorted_index)) update_metrics = model_spec['update_metrics'] eval_metrics = model_spec['metrics'] # global_step = tf.train.get_global_step() # Load the evaluation dataset into the pipeline and initialize the metrics init op # sess.run(model_spec['iterator_init_op']) sess.run(model_spec['metrics_init_op']) count = 0 with tf.python_io.TFRecordWriter( os.path.join(params.data_dir, 'sample-temp.tfrecords')) as writer: # compute metrics over the dataset for temp_query_id in range(int(num_steps) + 1): features, labels, _ = sess.run([model_spec["features"], \ model_spec["labels"], update_metrics]) # whether save the sample if temp_query_id not in sorted_index: continue for (feature, label) in zip(features, labels): image_raw = feature.tostring() label_raw = np.argmax(label) # logging.info('------------------------------------------------') count += 1 # logging.info(type(feature)) # logging.info(feature.shape) # logging.info(label_raw) # logging.info('------------------------------------------------') example = tf.train.Example(features=tf.train.Features( feature={ 'height': _int64_feature(int(params.height)), 'width': _int64_feature(int(params.width)), 'depth': _int64_feature(int(params.depth)), 'label': _int64_feature(int(label_raw)), 'image_raw': _bytes_feature(image_raw) })) writer.write(example.SerializeToString()) # Get the values of the metrics metrics_values = {k: v[0] for k, v in eval_metrics.items()} metrics_val = sess.run(metrics_values) expanded_metrics_val = get_expaned_metrics(metrics_val) metrics_string = " ; ".join("{}: {:05.4f}".format(k, v) for k, v in expanded_metrics_val.items()) logging.info("- take_train_samples_sess metrics: " + metrics_string) # logging.info('*********************************************{}'.format(count)) return expanded_metrics_val
def train_sess(sess, model_spec, num_steps, writer, params): """Train the model on `num_steps` batches Args: sess: (tf.Session) current session model_spec: (dict) contains the graph operations or nodes needed for training num_steps: (int) train for this number of batches writer: (tf.summary.FileWriter) writer for summaries params: (Params) hyperparameters """ # Get relevant graph operations or nodes needed for training loss = model_spec['loss'] train_op = model_spec['train_op'] update_metrics = model_spec['update_metrics'] metrics = model_spec['metrics'] summary_op = model_spec['summary_op'] global_step = tf.train.get_global_step() # Load the training dataset into the pipeline and initialize the metrics local variables # sess.run(model_spec['iterator_init_op']) sess.run(model_spec['metrics_init_op']) # Use tqdm for progress bar t = trange(int(num_steps)) epoch_loss = [] for i in t: # Evaluate summaries for tensorboard only once in a while if i == params.save_summary_steps - 1: # if i % params.save_summary_steps == 0: # Perform a mini-batch update _, _, loss_val, summ, global_step_val = sess.run( [train_op, update_metrics, loss, summary_op, global_step]) # Write summaries for tensorboard writer.add_summary(summ, global_step_val) else: _, _, loss_val = sess.run([train_op, update_metrics, loss]) # Log the loss in the tqdm progress bar # t.set_postfix(loss='{:05.3f}'.format(loss_val)) epoch_loss.append(loss_val) # logging.info('loss_val: {}'.format(epoch_loss)) metrics_values = {k: v[0] for k, v in metrics.items()} metrics_val = sess.run(metrics_values) expanded_metrics_val = get_expaned_metrics(metrics_val) metrics_string = " ; ".join("{}: {:05.4f}".format(k, v) for k, v in expanded_metrics_val.items()) logging.info("- Train metrics: " + metrics_string) return epoch_loss
def train_initial_sess(sess, model_spec, num_steps, writer, params, \ old_index, numbers_of_selections, sums_of_reward, arm_weights, max_upper_bound, old_loss_val): """Train the model on `num_steps` batches Args: sess: (tf.Session) current session model_spec: (dict) contains the graph operations or nodes needed for training num_steps: (int) train for this number of batches writer: (tf.summary.FileWriter) writer for summaries params: (Params) hyperparameters """ # Get relevant graph operations or nodes needed for training loss = model_spec['loss'] # train_op = model_spec['train_op'] update_metrics = model_spec['update_metrics'] metrics = model_spec['metrics'] summary_op = model_spec['summary_op'] # reward = model_spec['reward'] global_step = tf.train.get_global_step() # Load the training dataset into the pipeline and initialize the metrics local variables # sess.run(model_spec['iterator_init_op']) sess.run(model_spec['metrics_init_op']) # Use tqdm for progress bar t = trange(int(num_steps)) epoch_loss = [] index = 0 # for i in t: # # Evaluate summaries for tensorboard only once in a while # if i == params.save_summary_steps - 1: # # if i % params.save_summary_steps == 0: # # Perform a mini-batch update # _, _, loss_val, summ, global_step_val = sess.run([train_op, update_metrics, loss, # summary_op, global_step]) # # Write summaries for tensorboard # writer.add_summary(summ, global_step_val) # else: # _, _, loss_val = sess.run([train_op, update_metrics, loss]) # # Log the loss in the tqdm progress bar # # t.set_postfix(loss='{:05.3f}'.format(loss_val)) # epoch_loss.append(loss_val) for i in t: # Evaluate summaries for tensorboard only once in a while if i == params.save_summary_steps - 1: # if i % params.save_summary_steps == 0: # Perform a mini-batch update _, loss_val, summ, global_step_val = sess.run( [update_metrics, loss, summary_op, global_step]) # Write summaries for tensorboard writer.add_summary(summ, global_step_val) else: _, loss_val = sess.run([update_metrics, loss]) # Log the loss in the tqdm progress bar # t.set_postfix(loss='{:05.3f}'.format(loss_val)) epoch_loss.append(loss_val) ################################################################################# if 'retrain' in params.loss_fn: # calculate the rewards of the (10) clusters --> pick and update the action variable global_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='model/rewards') action_var = [v for v in global_vars if 'action' in v.name][0] action_var_value = sess.run([action_var]) # logging.info('****action_var at train_sess:\n {}'.format(action_var_value)) # logging.info('*****old_index is {}, loss_val is {}'.format(old_index, loss_val)) sums_of_reward[old_index] += loss_val - old_loss_val old_loss_val = loss_val index = old_index # index, numbers_of_selections, arm_weights, \ # max_upper_bound = rl_weights(params, numbers_of_selections, \ # sums_of_reward, max_upper_bound, \ # t, arm_weights) # logging.info('numbers_of_selections at i:\n {}'.format(numbers_of_selections)) # total_reward += reward # index = 2 assign_op = action_var.assign([index]) sess.run(assign_op) action_var_value = sess.run([action_var]) # logging.info('******updated action_var at train_sess:\n {}'.format(action_var_value)) ################################################################################# metrics_values = {k: v[0] for k, v in metrics.items()} metrics_val = sess.run(metrics_values) expanded_metrics_val = get_expaned_metrics(metrics_val) metrics_string = " ; ".join("{}: {:05.4f}".format(k, v) for k, v in expanded_metrics_val.items()) # logging.info("- Train metrics: " + metrics_string) # logging.info("-- Reward: \n") # logging.info(reward) return epoch_loss, index, sums_of_reward, arm_weights, max_upper_bound, old_loss_val
def evaluate_on_train_sess(sess, model_spec, num_steps, params=None): """Train the model on `num_steps` batches. Args: sess: (tf.Session) current session model_spec: (dict) contains the graph operations or nodes needed for training num_steps: (int) train for this number of batches writer: (tf.summary.FileWriter) writer for summaries. Is None if we don't log anything params: (Params) hyperparameters """ update_metrics = model_spec['update_metrics'] eval_metrics = model_spec['metrics'] # global_step = tf.train.get_global_step() # Load the evaluation dataset into the pipeline and initialize the metrics init op # sess.run(model_spec['iterator_init_op']) sess.run(model_spec['metrics_init_op']) if params.save_predictions: # save the predictions and lable_qid to files prediction_list = [] label_list = [] # compute metrics over the dataset for temp_query_id in range(int(num_steps)): prediction_per_query, label_per_query, _ = sess.run([model_spec["predictions"], \ model_spec["labels"], update_metrics]) prediction_list.extend( [v[0] for v in prediction_per_query.tolist()]) label_per_query_list = label_per_query.tolist() label_list.extend(['{} id:{}'.format(int(label_per_query_list[i][0]), \ temp_query_id) \ for i in range(0, len(label_per_query_list))]) save_predictions_to_file(prediction_list, "./prediction_output") # tensorflow mess up test input orders save_predictions_to_file(label_list, "./label_output") else: # only update metrics for temp_query_id in range(int(num_steps)): # # masks_weights=[v for v in tf.trainable_variables() if 'model/mask' in v.name] # # # loss = eval_metrics['loss'] # # # update_masks = [cal_gradient(loss, weight) for weight in masks_weights] # # sess.run(update_masks) # # sess.run(model_spec['masks']) # masks = sess.run(model_spec['masks']) # print('Printing masks') # print(masks) sess.run(model_spec['masks']) # mweights3_2=[v for v in tf.trainable_variables() if 'model/mask/mweights3_2' in v.name] # mweights3_2 = sess.run(mweights3_2) # print('Printing mweights3_2') # print(mweights3_2) sess.run(update_metrics) masks = [v for v in tf.trainable_variables() if 'model/mask' in v.name] update_masks = [tf.assign(nm, m/float(num_steps)) for (nm, m) in \ zip(masks, masks)] sess.run(update_masks) # Get the values of the metrics metrics_values = {k: v[0] for k, v in eval_metrics.items()} metrics_val = sess.run(metrics_values) expanded_metrics_val = get_expaned_metrics(metrics_val) metrics_string = " ; ".join("{}: {:05.4f}".format(k, v) for k, v in expanded_metrics_val.items()) logging.info("- metrics on full train: " + metrics_string) return expanded_metrics_val