def EmbedQueryTargetData(query_records, target_records, config): """Embeds the full set of query_records and target_records. Args: query_records: List of Strings, paths to tfrecord datasets to use as queries. target_records: List of Strings, paths to tfrecord datasets to use as targets. config: A T object describing training config. Returns: query_sequences_to_data: A dict holding 'embeddings' and 'images' target_sequences_to_data: A dict holding 'embeddings' and 'images' """ batch_size = config.data.embed_batch_size # Choose an estimator based on training strategy. estimator = get_estimator(config, FLAGS.checkpointdir) # Choose a checkpoint path to restore. checkpointdir = FLAGS.checkpointdir checkpoint_path = os.path.join(checkpointdir, 'model.ckpt-%s' % FLAGS.checkpoint_iter) # Embed num_sequences query sequences, store embeddings and image strings in # query_sequences_to_data. num_query_sequences = FLAGS.num_query_sequences num_target_sequences = FLAGS.num_target_sequences query_sequences_to_data = {} for (view_embeddings, view_raw_image_strings, seqname) in estimator.inference(query_records, checkpoint_path, batch_size, num_sequences=num_query_sequences): query_sequences_to_data[seqname] = { 'embeddings': view_embeddings, 'images': view_raw_image_strings, } if (query_records == target_records) and (num_query_sequences == num_target_sequences): target_sequences_to_data = query_sequences_to_data else: # Embed num_sequences target sequences, store embeddings and image strings # in sequences_to_data. target_sequences_to_data = {} for (view_embeddings, view_raw_image_strings, seqname) in estimator.inference( target_records, checkpoint_path, batch_size, num_sequences=num_target_sequences): target_sequences_to_data[seqname] = { 'embeddings': view_embeddings, 'images': view_raw_image_strings, } return query_sequences_to_data, target_sequences_to_data
def main(_): """Runs main training loop.""" # Parse config dict from yaml config files / command line flags. config = util.ParseConfigsToLuaTable( FLAGS.config_paths, FLAGS.model_params, save=True, logdir=FLAGS.logdir) # Choose an estimator based on training strategy. estimator = get_estimator(config, FLAGS.logdir) # Run training estimator.train()
def main(_): """Runs main training loop.""" # Parse config dict from yaml config files / command line flags. config = util.ParseConfigsToLuaTable( FLAGS.config_paths, FLAGS.model_params, save=True, logdir=FLAGS.logdir) # Choose an estimator based on training strategy. estimator = get_estimator(config, FLAGS.logdir) # Run training estimator.train()
def main(_): """Runs main labeled eval loop.""" # Parse config dict from yaml config files / command line flags. config = util.ParseConfigsToLuaTable(FLAGS.config_paths, FLAGS.model_params) # Choose an estimator based on training strategy. checkpointdir = FLAGS.checkpointdir estimator = get_estimator(config, checkpointdir) # Get data configs. image_attr_keys = config.data.labeled.image_attr_keys label_attr_keys = config.data.labeled.label_attr_keys embedding_size = config.embedding_size num_views = config.data.num_views k_list = config.val.recall_at_k_list batch_size = config.data.batch_size # Get either labeled validation or test tables. labeled_tables = get_labeled_tables(config) def input_fn_by_view(view_index): """Returns an input_fn for use with a tf.Estimator by view.""" def input_fn(): # Get raw labeled images. (preprocessed_images, labels, tasks) = data_providers.labeled_data_provider( labeled_tables, estimator.preprocess_data, view_index, image_attr_keys, label_attr_keys, batch_size=batch_size) return { 'batch_preprocessed': preprocessed_images, 'tasks': tasks, 'classification_labels': labels, }, None return input_fn # If evaluating a specific checkpoint, do that. if FLAGS.checkpoint_iter: checkpoint_path = os.path.join('%s/model.ckpt-%s' % (checkpointdir, FLAGS.checkpoint_iter)) evaluate_once(estimator, input_fn_by_view, batch_size, checkpoint_path, label_attr_keys, embedding_size, num_views, k_list) else: for checkpoint_path in tf.contrib.training.checkpoints_iterator( checkpointdir): evaluate_once(estimator, input_fn_by_view, batch_size, checkpoint_path, label_attr_keys, embedding_size, num_views, k_list)
def EmbedQueryTargetData(query_records, target_records, config): """Embeds the full set of query_records and target_records. Args: query_records: List of Strings, paths to tfrecord datasets to use as queries. target_records: List of Strings, paths to tfrecord datasets to use as targets. config: A T object describing training config. Returns: query_sequences_to_data: A dict holding 'embeddings' and 'images' target_sequences_to_data: A dict holding 'embeddings' and 'images' """ batch_size = config.data.embed_batch_size # Choose an estimator based on training strategy. estimator = get_estimator(config, FLAGS.checkpointdir) # Choose a checkpoint path to restore. checkpointdir = FLAGS.checkpointdir checkpoint_path = os.path.join(checkpointdir, 'model.ckpt-%s' % FLAGS.checkpoint_iter) # Embed num_sequences query sequences, store embeddings and image strings in # query_sequences_to_data. num_query_sequences = FLAGS.num_query_sequences num_target_sequences = FLAGS.num_target_sequences query_sequences_to_data = {} for (view_embeddings, view_raw_image_strings, seqname) in estimator.inference( query_records, checkpoint_path, batch_size, num_sequences=num_query_sequences): query_sequences_to_data[seqname] = { 'embeddings': view_embeddings, 'images': view_raw_image_strings, } if (query_records == target_records) and ( num_query_sequences == num_target_sequences): target_sequences_to_data = query_sequences_to_data else: # Embed num_sequences target sequences, store embeddings and image strings # in sequences_to_data. target_sequences_to_data = {} for (view_embeddings, view_raw_image_strings, seqname) in estimator.inference( target_records, checkpoint_path, batch_size, num_sequences=num_target_sequences): target_sequences_to_data[seqname] = { 'embeddings': view_embeddings, 'images': view_raw_image_strings, } return query_sequences_to_data, target_sequences_to_data
def main(_): """Runs main labeled eval loop.""" # Parse config dict from yaml config files / command line flags. config = util.ParseConfigsToLuaTable(FLAGS.config_paths, FLAGS.model_params) # Choose an estimator based on training strategy. checkpointdir = FLAGS.checkpointdir estimator = get_estimator(config, checkpointdir) # Get data configs. image_attr_keys = config.data.labeled.image_attr_keys label_attr_keys = config.data.labeled.label_attr_keys embedding_size = config.embedding_size num_views = config.data.num_views k_list = config.val.recall_at_k_list batch_size = config.data.batch_size # Get either labeled validation or test tables. labeled_tables = get_labeled_tables(config) def input_fn_by_view(view_index): """Returns an input_fn for use with a tf.Estimator by view.""" def input_fn(): # Get raw labeled images. (preprocessed_images, labels, tasks) = data_providers.labeled_data_provider( labeled_tables, estimator.preprocess_data, view_index, image_attr_keys, label_attr_keys, batch_size=batch_size) return { 'batch_preprocessed': preprocessed_images, 'tasks': tasks, 'classification_labels': labels, }, None return input_fn # If evaluating a specific checkpoint, do that. if FLAGS.checkpoint_iter: checkpoint_path = os.path.join( '%s/model.ckpt-%s' % (checkpointdir, FLAGS.checkpoint_iter)) evaluate_once( estimator, input_fn_by_view, batch_size, checkpoint_path, label_attr_keys, embedding_size, num_views, k_list) else: for checkpoint_path in tf.contrib.training.checkpoints_iterator( checkpointdir): evaluate_once( estimator, input_fn_by_view, batch_size, checkpoint_path, label_attr_keys, embedding_size, num_views, k_list)
def main(_): """Runs main eval loop.""" # Parse config dict from yaml config files / command line flags. logdir = FLAGS.logdir config = util.ParseConfigsToLuaTable(FLAGS.config_paths, FLAGS.model_params) # Choose an estimator based on training strategy. estimator = get_estimator(config, logdir) # Wait for the first checkpoint file to be written. while not tf.train.latest_checkpoint(logdir): tf.logging.info('Waiting for a checkpoint file...') time.sleep(10) # Run validation. while True: estimator.evaluate()
def main(_): """Runs main eval loop.""" # Parse config dict from yaml config files / command line flags. logdir = FLAGS.logdir config = util.ParseConfigsToLuaTable(FLAGS.config_paths, FLAGS.model_params) # Choose an estimator based on training strategy. estimator = get_estimator(config, logdir) # Wait for the first checkpoint file to be written. while not tf.train.latest_checkpoint(logdir): tf.logging.info('Waiting for a checkpoint file...') time.sleep(10) # Run validation. while True: estimator.evaluate()
def SameSequenceVideos(query_records, config, height, width): """Generate same sequence, cross-view imitation videos.""" batch_size = config.data.embed_batch_size # Choose an estimator based on training strategy. estimator = get_estimator(config, FLAGS.checkpointdir) # Choose a checkpoint path to restore. checkpointdir = FLAGS.checkpointdir checkpoint_path = os.path.join(checkpointdir, 'model.ckpt-%s' % FLAGS.checkpoint_iter) # Embed num_sequences query sequences, store embeddings and image strings in # query_sequences_to_data. sequences_to_data = {} for (view_embeddings, view_raw_image_strings, seqname) in estimator.inference( query_records, checkpoint_path, batch_size, num_sequences=FLAGS.num_query_sequences): sequences_to_data[seqname] = { 'embeddings': view_embeddings, 'images': view_raw_image_strings, } # Loop over query videos. qview = FLAGS.query_view tview = FLAGS.target_view for task_i, data_i in sequences_to_data.iteritems(): ims = data_i['images'] embs = data_i['embeddings'] query_embs = SmoothEmbeddings(embs[qview]) query_ims = ims[qview] target_embs = SmoothEmbeddings(embs[tview]) target_ims = ims[tview] tf.logging.info('Generating %s imitating %s video.' % (task_i, task_i)) vid_name = 'q%sv%s_im%sv%s' % (task_i, qview, task_i, tview) vid_name = vid_name.replace('/', '_') GenerateImitationVideo(vid_name, query_ims, query_embs, target_ims, target_embs, height, width)
def evaluate_once( config, checkpointdir, validation_records, checkpoint_path, batch_size, num_views): """Evaluates and reports the validation alignment.""" # Choose an estimator based on training strategy. estimator = get_estimator(config, checkpointdir) # Embed all validation sequences. seqname_to_embeddings = {} for (view_embeddings, _, seqname) in estimator.inference( validation_records, checkpoint_path, batch_size): seqname_to_embeddings[seqname] = view_embeddings # Compute and report alignment statistics. ckpt_step = int(checkpoint_path.split('-')[-1]) summary_dir = os.path.join(FLAGS.outdir, 'alignment_summaries') summary_writer = tf.summary.FileWriter(summary_dir) compute_average_alignment( seqname_to_embeddings, num_views, summary_writer, ckpt_step)
def evaluate_once(config, checkpointdir, validation_records, checkpoint_path, batch_size, num_views): """Evaluates and reports the validation alignment.""" # Choose an estimator based on training strategy. estimator = get_estimator(config, checkpointdir) # Embed all validation sequences. seqname_to_embeddings = {} for (view_embeddings, _, seqname) in estimator.inference(validation_records, checkpoint_path, batch_size): seqname_to_embeddings[seqname] = view_embeddings # Compute and report alignment statistics. ckpt_step = int(checkpoint_path.split('-')[-1]) summary_dir = os.path.join(FLAGS.outdir, 'alignment_summaries') summary_writer = tf.summary.FileWriter(summary_dir) compute_average_alignment(seqname_to_embeddings, num_views, summary_writer, ckpt_step)
def main(_): """Runs main eval loop.""" # Parse config dict from yaml config files / command line flags. logdir = FLAGS.logdir config = util.ParseConfigsToLuaTable(FLAGS.config_paths, FLAGS.model_params) # Choose an estimator based on training strategy. estimator = get_estimator(config, logdir) # Wait for the first checkpoint file to be written. while not tf.train.latest_checkpoint(logdir): tf.logging.info('Waiting for a checkpoint file...') time.sleep(10) ckpt = tf.train.get_checkpoint_state(logdir) for val1, val2, val3 in estimator.inference(FLAGS.tfrecords, ckpt.model_checkpoint_path, FLAGS.batch_size, model_name=FLAGS.model_name): break
def SameSequenceVideos(query_records, config, height, width): """Generate same sequence, cross-view imitation videos.""" batch_size = config.data.embed_batch_size # Choose an estimator based on training strategy. estimator = get_estimator(config, FLAGS.checkpointdir) # Choose a checkpoint path to restore. checkpointdir = FLAGS.checkpointdir checkpoint_path = os.path.join(checkpointdir, 'model.ckpt-%s' % FLAGS.checkpoint_iter) # Embed num_sequences query sequences, store embeddings and image strings in # query_sequences_to_data. sequences_to_data = {} for (view_embeddings, view_raw_image_strings, seqname) in estimator.inference( query_records, checkpoint_path, batch_size, num_sequences=FLAGS.num_query_sequences): sequences_to_data[seqname] = { 'embeddings': view_embeddings, 'images': view_raw_image_strings, } # Loop over query videos. qview = FLAGS.query_view tview = FLAGS.target_view for task_i, data_i in sequences_to_data.iteritems(): ims = data_i['images'] embs = data_i['embeddings'] query_embs = SmoothEmbeddings(embs[qview]) query_ims = ims[qview] target_embs = SmoothEmbeddings(embs[tview]) target_ims = ims[tview] tf.logging.info('Generating %s imitating %s video.' % (task_i, task_i)) vid_name = 'q%sv%s_im%sv%s' % (task_i, qview, task_i, tview) vid_name = vid_name.replace('/', '_') GenerateImitationVideo(vid_name, query_ims, query_embs, target_ims, target_embs, height, width)
def main(_): """Runs main labeled eval loop.""" # Parse config dict from yaml config files / command line flags. config = util.ParseConfigsToLuaTable(FLAGS.config_paths, FLAGS.model_params) # Choose an estimator based on training strategy. checkpointdir = FLAGS.checkpointdir checkpoint_path = os.path.join( '%s/model.ckpt-%s' % (checkpointdir, FLAGS.checkpoint_iter)) estimator = get_estimator(config, checkpointdir) # Get records to embed. validation_dir = FLAGS.embedding_records validation_records = util.GetFilesRecursively(validation_dir) sequences_to_data = {} for (view_embeddings, view_raw_image_strings, seqname) in estimator.inference( validation_records, checkpoint_path, config.data.embed_batch_size, num_sequences=FLAGS.num_sequences): sequences_to_data[seqname] = { 'embeddings': view_embeddings, 'images': view_raw_image_strings, } all_embeddings = np.zeros((0, config.embedding_size)) all_ims = [] all_seqnames = [] num_embeddings = FLAGS.num_embed # Concatenate all views from all sequences into a big flat list. for seqname, data in sequences_to_data.iteritems(): embs = data['embeddings'] ims = data['images'] for v in range(config.data.num_views): for (emb, im) in zip(embs[v], ims[v]): all_embeddings = np.append(all_embeddings, [emb], axis=0) all_ims.append(im) all_seqnames.append(seqname) # Choose N indices uniformly from all images. random_indices = range(all_embeddings.shape[0]) random.shuffle(random_indices) viz_indices = random_indices[:num_embeddings] # Extract embs. viz_embs = np.array(all_embeddings[viz_indices]) # Extract and decode ims. viz_ims = list(np.array(all_ims)[viz_indices]) decoded_ims = [] sprite_dim = FLAGS.sprite_dim for i, im in enumerate(viz_ims): if i % 100 == 0: print('Decoding image %d/%d.' % (i, num_embeddings)) nparr_i = np.fromstring(str(im), np.uint8) img_np = cv2.imdecode(nparr_i, 1) img_np = img_np[..., [2, 1, 0]] img_np = imresize(img_np, [sprite_dim, sprite_dim, 3]) decoded_ims.append(img_np) decoded_ims = np.array(decoded_ims) # Extract sequence names. outdir = FLAGS.outdir # The embedding variable, which needs to be stored # Note this must a Variable not a Tensor! embedding_var = tf.Variable(viz_embs, name='viz_embs') with tf.Session() as sess: sess.run(embedding_var.initializer) summary_writer = tf.summary.FileWriter(outdir) config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = embedding_var.name # Comment out if you don't want sprites embedding.sprite.image_path = os.path.join(outdir, 'sprite.png') embedding.sprite.single_image_dim.extend( [decoded_ims.shape[1], decoded_ims.shape[1]]) projector.visualize_embeddings(summary_writer, config) saver = tf.train.Saver([embedding_var]) saver.save(sess, os.path.join(outdir, 'model2.ckpt'), 1) sprite = images_to_sprite(decoded_ims) imsave(os.path.join(outdir, 'sprite.png'), sprite)
def main(_): """Runs main labeled eval loop.""" # Parse config dict from yaml config files / command line flags. config = util.ParseConfigsToLuaTable(FLAGS.config_paths, FLAGS.model_params) # Choose an estimator based on training strategy. checkpointdir = FLAGS.checkpointdir checkpoint_path = os.path.join('%s/model.ckpt-%s' % (checkpointdir, FLAGS.checkpoint_iter)) estimator = get_estimator(config, checkpointdir) # Get records to embed. validation_dir = FLAGS.embedding_records validation_records = util.GetFilesRecursively(validation_dir) sequences_to_data = {} for (view_embeddings, view_raw_image_strings, seqname) in estimator.inference(validation_records, checkpoint_path, config.data.embed_batch_size, num_sequences=FLAGS.num_sequences): sequences_to_data[seqname] = { 'embeddings': view_embeddings, 'images': view_raw_image_strings, } all_embeddings = np.zeros((0, config.embedding_size)) all_ims = [] all_seqnames = [] num_embeddings = FLAGS.num_embed # Concatenate all views from all sequences into a big flat list. for seqname, data in sequences_to_data.iteritems(): embs = data['embeddings'] ims = data['images'] for v in range(config.data.num_views): for (emb, im) in zip(embs[v], ims[v]): all_embeddings = np.append(all_embeddings, [emb], axis=0) all_ims.append(im) all_seqnames.append(seqname) # Choose N indices uniformly from all images. random_indices = range(all_embeddings.shape[0]) random.shuffle(random_indices) viz_indices = random_indices[:num_embeddings] # Extract embs. viz_embs = np.array(all_embeddings[viz_indices]) # Extract and decode ims. viz_ims = list(np.array(all_ims)[viz_indices]) decoded_ims = [] sprite_dim = FLAGS.sprite_dim for i, im in enumerate(viz_ims): if i % 100 == 0: print('Decoding image %d/%d.' % (i, num_embeddings)) nparr_i = np.fromstring(str(im), np.uint8) img_np = cv2.imdecode(nparr_i, 1) img_np = img_np[..., [2, 1, 0]] img_np = imresize(img_np, [sprite_dim, sprite_dim, 3]) decoded_ims.append(img_np) decoded_ims = np.array(decoded_ims) # Extract sequence names. outdir = FLAGS.outdir # The embedding variable, which needs to be stored # Note this must a Variable not a Tensor! embedding_var = tf.Variable(viz_embs, name='viz_embs') with tf.Session() as sess: sess.run(embedding_var.initializer) summary_writer = tf.summary.FileWriter(outdir) config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = embedding_var.name # Comment out if you don't want sprites embedding.sprite.image_path = os.path.join(outdir, 'sprite.png') embedding.sprite.single_image_dim.extend( [decoded_ims.shape[1], decoded_ims.shape[1]]) projector.visualize_embeddings(summary_writer, config) saver = tf.train.Saver([embedding_var]) saver.save(sess, os.path.join(outdir, 'model2.ckpt'), 1) sprite = images_to_sprite(decoded_ims) imsave(os.path.join(outdir, 'sprite.png'), sprite)