def test_build_subnetwork(self, builder_params, want_name): with tf.Graph().as_default() as g, self.test_session(graph=g) as sess: data = np.concatenate([ np.ones((1, _IMAGE_DIM, _IMAGE_DIM, 1)), 2. * np.ones( (1, _IMAGE_DIM, _IMAGE_DIM, 1)) ]) features = {"x": tf.constant(data)} labels = tf.constant([0, 1]) training = True mode = tf.estimator.ModeKeys.TRAIN head = tf.contrib.estimator.binary_classification_head( loss_reduction=tf.losses.Reduction.SUM) ensemble = None name = None subnetwork = None builders = [] for builder_param in builder_params: builders.append( _builder(checkpoint_dir=self.test_subdirectory, **builder_param)) for idx, builder in enumerate(builders): name = builder.name # Pass the subnetworks of previous builders to the next builder. with tf.variable_scope("subnetwork_{}".format(idx)): subnetwork = builder.build_subnetwork( features=features, logits_dimension=head.logits_dimension, training=training, iteration_step=tf.train.get_or_create_global_step(), summary=_FakeSummary(), previous_ensemble=ensemble) logits = subnetwork.logits weighted_subnetworks = [] if ensemble: logits += ensemble.logits weighted_subnetworks = ensemble.weighted_subnetworks ensemble = adanet.Ensemble( weighted_subnetworks=weighted_subnetworks + [ adanet.WeightedSubnetwork( name=None, logits=logits, weight=None, subnetwork=subnetwork) ], logits=logits, bias=0.) estimator_spec = head.create_estimator_spec( features=features, labels=labels, mode=mode, train_op_fn=lambda loss: tf.no_op(), logits=ensemble.logits) sess.run(tf.global_variables_initializer()) train_op = builders[-1].build_subnetwork_train_op( subnetwork, estimator_spec.loss, var_list=None, labels=labels, iteration_step=tf.train.get_or_create_global_step(), summary=_FakeSummary(), previous_ensemble=ensemble) for _ in range(10): sess.run(train_op) self.assertEqual(want_name, name) self.assertGreater(sess.run(estimator_spec.loss), 0.0)
def training_graph(opts, training_data): train_graph = tf.Graph() with train_graph.as_default(): dataset, train_iterator, placeholders = training_data.get_dataset( opts, is_training=True) infeed = ipu_infeed_queue.IPUInfeedQueue( dataset, "training_dataset_infeed", 0) with ipu_scope('/device:IPU:0'): def comp_fn(): def body(total_loss_, sum_rmse_metric, *args, **kwargs): data_tensors = args observed_ratings = data_tensors[0] loss, rmse_metric, apply_grads_ = graph_builder(opts, observed_ratings=observed_ratings, learning_rate=placeholders["learning_rate"], type='TRAIN') with tf.control_dependencies([apply_grads_]): return total_loss_ + loss, sum_rmse_metric + rmse_metric return loops.repeat(opts.batches_per_step, body, [tf.constant(0, tf.float32), tf.constant(0, tf.float32)], infeed) total_loss, sum_rmse_metric = ipu_compiler.compile(comp_fn, []) rmse = sum_rmse_metric / opts.batches_per_step loss = total_loss / opts.batches_per_step tf.summary.scalar("loss", loss) tf.summary.scalar("learning_rate", placeholders["learning_rate"]) tf.summary.scalar("RMSE/train", rmse) if opts.compiler_report: ipu_ops.ipu_compile_summary('compile_summary', loss) train_summary = tf.summary.merge_all() train_saver = tf.train.Saver() ipu_utils.move_variable_initialization_to_cpu() train_init = tf.global_variables_initializer() train_writer = tf.summary.FileWriter( opts.logs_path + '/train', graph=train_graph, flush_secs=30) ipu_options = util.get_config(opts, profiling=opts.compiler_report) ipu_utils.configure_ipu_system(ipu_options) train_sess = tf.Session(graph=train_graph) return GraphOps(train_graph, train_sess, train_init, [loss, train_summary, rmse], placeholders, infeed, train_saver, train_writer)
def benchmark_model(self, warmup_runs, bm_runs, num_threads, trace_filename=None): """Benchmark model.""" if self.tensorrt: print('Using tensorrt ', self.tensorrt) graphdef = self.freeze_model() if num_threads > 0: print('num_threads for benchmarking: {}'.format(num_threads)) sess_config = tf.ConfigProto( intra_op_parallelism_threads=num_threads, inter_op_parallelism_threads=1) else: sess_config = tf.ConfigProto() # rewriter_config_pb2.RewriterConfig.OFF sess_config.graph_options.rewrite_options.dependency_optimization = 2 if self.use_xla: sess_config.graph_options.optimizer_options.global_jit_level = ( tf.OptimizerOptions.ON_2) with tf.Graph().as_default(), tf.Session(config=sess_config) as sess: inputs = tf.placeholder(tf.float32, name='input', shape=self.inputs_shape) output = self.build_model(inputs) img = np.random.uniform(size=self.inputs_shape) sess.run(tf.global_variables_initializer()) if self.tensorrt: fetches = [inputs.name] + [i.name for i in output] goutput = self.convert_tr(graphdef, fetches) inputs, output = goutput[0], goutput[1:] if not self.use_xla: # Don't use tf.group because XLA removes the whole graph for tf.group. output = tf.group(*output) else: output = tf.add_n([tf.reduce_sum(x) for x in output]) output_name = [output.name] input_name = inputs.name graphdef = tf.graph_util.convert_variables_to_constants( sess, sess.graph_def, output_name) with tf.Graph().as_default(), tf.Session(config=sess_config) as sess: tf.import_graph_def(graphdef, name='') for i in range(warmup_runs): start_time = time.time() sess.run(output_name, feed_dict={input_name: img}) logging.info('Warm up: {} {:.4f}s'.format( i, time.time() - start_time)) print('Start benchmark runs total={}'.format(bm_runs)) start = time.perf_counter() for i in range(bm_runs): sess.run(output_name, feed_dict={input_name: img}) end = time.perf_counter() inference_time = (end - start) / bm_runs print('Per batch inference time: ', inference_time) print('FPS: ', self.batch_size / inference_time) if trace_filename: run_options = tf.RunOptions() run_options.trace_level = tf.RunOptions.FULL_TRACE run_metadata = tf.RunMetadata() sess.run(output_name, feed_dict={input_name: img}, options=run_options, run_metadata=run_metadata) logging.info('Dumping trace to %s', trace_filename) trace_dir = os.path.dirname(trace_filename) if not tf.io.gfile.exists(trace_dir): tf.io.gfile.makedirs(trace_dir) with tf.io.gfile.GFile(trace_filename, 'w') as trace_file: trace = timeline.Timeline( step_stats=run_metadata.step_stats) trace_file.write( trace.generate_chrome_trace_format(show_memory=True))
def main(): with tf.Graph().as_default(): with tf.Session() as sess: np.random.seed(seed=SEED) if USE_SPLIT_DATASET: dataset_tmp = facenet.get_dataset(DATA_DIR) train_set, test_set = split_dataset(dataset_tmp, MIN_NROF_IMAGES_PER_CLASS, NROF_BATCHES_PER_EPOCH) if MODE == 'TRAIN': dataset = train_set elif MODE == 'CLASSIFY': dataset = test_set else: dataset = facenet.get_dataset(DATA_DIR) # Check that there are at least one training image per class for cls in dataset: assert ( len(cls.image_paths) > 0, 'There must be at least one image for each class in the dataset' ) paths, labels = facenet.get_image_paths_and_labels(dataset) print('Number of classes: %d' % len(dataset)) print('Number of images: %d' % len(paths)) # Load the model print('Loading feature extraction model') facenet.load_model(MODEL) # Get input and output tensors images_placeholder = tf.get_default_graph().get_tensor_by_name( "input:0") embeddings = tf.get_default_graph().get_tensor_by_name( "embeddings:0") phase_train_placeholder = tf.get_default_graph( ).get_tensor_by_name("phase_train:0") embedding_size = embeddings.get_shape()[1] # Run forward pass to calculate embeddings print('Calculating features for images') nrof_images = len(paths) nrof_batches_per_epoch = int( math.ceil(1.0 * nrof_images / BATCH_SIZE)) emb_array = np.zeros((nrof_images, embedding_size)) for i in range(nrof_batches_per_epoch): start_index = i * BATCH_SIZE end_index = min((i + 1) * BATCH_SIZE, nrof_images) paths_batch = paths[start_index:end_index] images = facenet.load_data(paths_batch, False, False, IMAGE_SIZE) feed_dict = { images_placeholder: images, phase_train_placeholder: False } emb_array[start_index:end_index, :] = sess.run( embeddings, feed_dict=feed_dict) classifier_filename_exp = os.path.expanduser(CLASSIFIER_FILENAME) if MODE == 'TRAIN': # Train classifier print('Training classifier') model = SVC(kernel='linear', probability=True) model.fit(emb_array, labels) # Create a list of class names class_names = [cls.name.replace('_', ' ') for cls in dataset] # Saving classifier model with open(classifier_filename_exp, 'wb') as outfile: pickle.dump((model, class_names), outfile) print('Saved classifier model to file "%s"' % classifier_filename_exp) elif MODE == 'CLASSIFY': # Classify images print('Testing classifier') with open(classifier_filename_exp, 'rb') as infile: (model, class_names) = pickle.load(infile) print('Loaded classifier model from file "%s"' % classifier_filename_exp) predictions = model.predict_proba(emb_array) best_class_indices = np.argmax(predictions, axis=1) best_class_probabilities = predictions[ np.arange(len(best_class_indices)), best_class_indices] for i in range(len(best_class_indices)): print('%4d %s: %.3f' % (i, class_names[best_class_indices[i]], best_class_probabilities[i])) accuracy = np.mean(np.equal(best_class_indices, labels)) print('Accuracy: %.3f' % accuracy)
def train(self, sampling: str = 'user uniform', epochs: int = 5, batch_size: int = 256, epoch_sample_limit: int = None, model_path: str = None): assert isinstance(sampling, str) assert isinstance(epochs, int) assert isinstance(batch_size, int) with tf.Graph().as_default(): u, i, j = self.build_graph() self.__saver = tf.train.Saver() if epoch_sample_limit is not None: assert isinstance(epoch_sample_limit, int) self.epoch_sample_limit = epoch_sample_limit batch_limit = self.epoch_sample_limit // batch_size + 1 self.__sess = tf.Session(config=self.tf_config) sampler = None if sampling == 'user uniform': sampler = self._uniform_user_sampling with self.__sess.as_default(): self.__sess.run(tf.global_variables_initializer()) if model_path is not None: assert isinstance(model_path, str) tprint( "Initialize weights with the previous trained model") self.import_embeddings(model_path) tprint( 'Training parameters: lu=%.6f, li=%.6f, lj=%.6f, lb=%.6f' % (self.lu, self.li, self.lj, self.lb)) tprint('Learning rate is %.6f, regularization mode is %s' % (self.lr, self.mode)) tprint( 'Training for %d epochs of %d batches using %s sampler' % (epochs, batch_limit, sampling)) if self.fue is not None: tprint('Initialize user embeddings') self.__sess.run(tf.assign(self.__ue, self.fue)) if self.fie is not None: tprint('Initialize item embeddings') self.__sess.run(tf.assign(self.__ie, self.fie)) if self.fib is not None: tprint('Initialize item biases') self.__sess.run(tf.assign(self.__ib, self.fib.ravel())) for eid in range(epochs): total_time = 0 bno = 1 for ub, ib, jb in sampler(batch_size): t1 = time.time() _, loss = self.__sess.run([self.solver, self.obj], feed_dict={ u: ub, i: ib, j: jb }) t2 = time.time() - t1 sys.stderr.write( '\rEpoch=%3d, batch=%6d, loss=%8.4f, time=%4.4fs' % (eid + 1, bno, loss, t2)) total_time += t2 bno += 1 if bno == batch_limit: break sys.stderr.write(' ... total time collapse %8.4fs' % (total_time)) sys.stderr.flush() print() self.fue = self.__sess.run(self.__ue) self.fie = self.__sess.run(self.__ie) self.fib = self.__sess.run(tf.reshape(self.__ib, (-1, 1)))
def testMultipleConvMaskAdded(self, pruning_method): tf.reset_default_graph() g = tf.Graph() with g.as_default(): number_of_layers = 5 kernel_size = [3, 3] base_depth = 4 depth_step = 7 input_tensor = tf.ones((8, self.height, self.width, base_depth)) top_layer = input_tensor for ix in range(number_of_layers): units = base_depth + (ix + 1) * depth_step top_layer = pruning_layers.sparse_conv2d( x=top_layer, units=units, kernel_size=kernel_size, is_training=False, sparsity_technique=pruning_method) if pruning_method == 'variational_dropout': theta_logsigma2 = tf.get_collection( vd.layers.THETA_LOGSIGMA2_COLLECTION) self.assertLen(theta_logsigma2, number_of_layers) utils.add_vd_pruning_summaries(theta_logsigma2, threshold=3.0) dkl_loss_1 = utils.variational_dropout_dkl_loss( reg_scalar=1, start_reg_ramp_up=0, end_reg_ramp_up=1000, warm_up=False, use_tpu=False) dkl_loss_1 = tf.reshape(dkl_loss_1, [1]) dkl_loss_2 = utils.variational_dropout_dkl_loss( reg_scalar=5, start_reg_ramp_up=0, end_reg_ramp_up=1000, warm_up=False, use_tpu=False) dkl_loss_2 = tf.reshape(dkl_loss_2, [1]) for ix in range(number_of_layers): self.assertListEqual(theta_logsigma2[ix][0].get_shape().as_list(), [ kernel_size[0], kernel_size[1], base_depth + ix * depth_step, base_depth + (ix + 1) * depth_step ]) init_op = tf.global_variables_initializer() with self.test_session() as sess: sess.run(init_op) if pruning_method == 'variational_dropout': loss_1, loss_2 = sess.run([dkl_loss_1, dkl_loss_2]) self.assertGreater(loss_2, loss_1) elif pruning_method == 'l0_regularization': theta_logalpha = tf.get_collection( l0.layers.THETA_LOGALPHA_COLLECTION) self.assertLen(theta_logalpha, number_of_layers) utils.add_l0_summaries(theta_logalpha) l0_norm_loss_1 = utils.l0_regularization_loss( reg_scalar=1, start_reg_ramp_up=0, end_reg_ramp_up=1000, warm_up=False, use_tpu=False) l0_norm_loss_1 = tf.reshape(l0_norm_loss_1, [1]) l0_norm_loss_2 = utils.l0_regularization_loss( reg_scalar=5, start_reg_ramp_up=0, end_reg_ramp_up=1000, warm_up=False, use_tpu=False) l0_norm_loss_2 = tf.reshape(l0_norm_loss_2, [1]) for ix in range(number_of_layers): self.assertListEqual(theta_logalpha[ix][0].get_shape().as_list(), [ kernel_size[0], kernel_size[1], base_depth + ix * depth_step, base_depth + (ix + 1) * depth_step ]) init_op = tf.global_variables_initializer() with self.test_session() as sess: sess.run(init_op) loss_1, loss_2 = sess.run([l0_norm_loss_1, l0_norm_loss_2]) self.assertGreater(loss_2, loss_1) else: mask = tf.get_collection(core.MASK_COLLECTION) for ix in range(number_of_layers): self.assertListEqual(mask[ix].get_shape().as_list(), [ kernel_size[0], kernel_size[1], base_depth + ix * depth_step, base_depth + (ix + 1) * depth_step ])
import os os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' import pickle import tensorflow.compat.v1 as tf import numpy as np from PIL import Image import matplotlib.pyplot as plt '''시작!''' with tf.Graph().as_default(): '''기록, 저장 결로 설정''' logs_path = 'logs_noise_cancel_machine' save_path = 'save_noise_cancel_machine' '''학습 데이터 준비''' with open('./data/noise_canceling_dataset.pickle', 'rb') as f: (x_train, y_train), (x_test, y_test) = pickle.load(f) '''신경망 구성''' # 네트워크 파라미터 설정 img_size = 256 regulation = 0.01 learning_rate = 0.001 batch_size = 10 dropout_rate = 0.1 # Placeholder 선언 input_data = tf.placeholder(tf.float32, [None, img_size, img_size, 3], name='input_data') input_label = tf.placeholder(tf.float32, [None, img_size, img_size, 3], name='input_label') # 가변 파라미터 설정 W1 = tf.Variable(tf.random_normal([3, 3, 3, 128],
def __init__(self, A, X, Y, num_hidden_feat, learning_rate=5e-2, gamma=1e-3, idx_gpu="/gpu:2"): self.num_hidden_feat = num_hidden_feat self.learning_rate = learning_rate self.gamma = gamma with tf.Graph().as_default() as g: self.graph = g with tf.device(idx_gpu): # definition of constant matrices self.A = convert_coo_to_sparse_tensor(A.tocoo()) self.X = tf.constant(X, dtype=tf.float32) self.Y = tf.constant(Y, dtype=tf.float32) self.W0 = tf.get_variable( "W0", shape=[X.shape[1], self.num_hidden_feat], initializer=tf.keras.layers.xavier_initializer(), ) self.W1 = tf.get_variable( "W1", shape=[self.num_hidden_feat, Y.shape[1]], initializer=tf.keras.layers.xavier_initializer(), ) # placeholder definition self.idx_nodes = tf.placeholder(tf.int32) self.keep_prob = tf.placeholder(tf.float32) # model definition self.l_input = tf.nn.dropout(self.X, self.keep_prob) self.X0_tilde = tf.sparse_tensor_dense_matmul( self.A, self.l_input) self.X0 = tf.matmul(self.X0_tilde, self.W0) self.X0 = tf.nn.relu(self.X0) self.X0 = tf.nn.dropout(self.X0, self.keep_prob) self.X1_tilde = tf.sparse_tensor_dense_matmul(self.A, self.X0) self.logits = tf.matmul(self.X1_tilde, self.W1) self.l_out = tf.gather(self.logits, self.idx_nodes) self.c_Y = tf.gather(self.Y, self.idx_nodes) # loss function definition self.l2_reg = tf.nn.l2_loss(self.W0) self.data_loss = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=self.l_out, labels=self.c_Y)) self.loss = self.data_loss + self.gamma * self.l2_reg # solver definition self.optimizer = tf.train.AdamOptimizer( learning_rate=self.learning_rate) self.opt_step = self.optimizer.minimize(self.loss) # predictions and accuracy extraction self.c_predictions = tf.argmax(tf.nn.softmax(self.l_out), 1) self.accuracy = tf.contrib.metrics.accuracy( self.c_predictions, tf.argmax(self.c_Y, 1)) # gradients computation self.trainable_variables = tf.trainable_variables() self.var_grad = tf.gradients(self.loss, tf.trainable_variables()) self.norm_grad = frobenius_norm( tf.concat([tf.reshape(g, [-1]) for g in self.var_grad], 0)) # session creation config = tf.ConfigProto(allow_soft_placement=True) config.gpu_options.allow_growth = True self.session = tf.Session(config=config) # session initialization init = tf.global_variables_initializer() self.session.run(init)
def build_graph(self): """Builds the neural network graph.""" # define graph self.g = tf.Graph() with self.g.as_default(): self.sess = tf.Session() self.x = tf.placeholder(shape=[None, 24, 24, 3], dtype=tf.float32) self.y = tf.placeholder(shape=[None, NUM_CLASSES], dtype=tf.float32) # conv1 with tf.variable_scope('conv1') as scope: kernel = variable_on_cpu('weights', shape=[5, 5, 3, 64]) conv = tf.nn.conv2d(self.x, kernel, [1, 1, 1, 1], padding='SAME') biases = variable_on_cpu('biases', [64]) pre_activation = tf.nn.bias_add(conv, biases) conv1 = tf.nn.relu(pre_activation, name=scope.name) # pool1 pool1 = tf.nn.max_pool(conv1, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool1') # norm1 norm1 = tf.nn.lrn(pool1, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75, name='norm1') # conv2 with tf.variable_scope('conv2') as scope: kernel = variable_on_cpu('weights', shape=[5, 5, 64, 64]) conv = tf.nn.conv2d(norm1, kernel, [1, 1, 1, 1], padding='SAME') biases = variable_on_cpu('biases', [64]) pre_activation = tf.nn.bias_add(conv, biases) conv2 = tf.nn.relu(pre_activation, name=scope.name) # norm2 norm2 = tf.nn.lrn(conv2, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75, name='norm2') # pool2 pool2 = tf.nn.max_pool(norm2, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool2') # local3 with tf.variable_scope('local3') as scope: # Move everything into depth so we can perform a single matrix multiply. # images.get_shape().as_list()[0] = batchsize # reshape = tf.keras.layers.Flatten()(pool2) reshape = tf.reshape(pool2, [tf.shape(self.x)[0], 6 * 6 * 64]) dim = reshape.get_shape()[1].value weights = variable_on_cpu('weights', shape=[dim, 384]) biases = variable_on_cpu('biases', [384]) local3 = tf.nn.relu(tf.matmul(reshape, weights) + biases, name=scope.name) # local4 with tf.variable_scope('local4') as scope: weights = variable_on_cpu('weights', shape=[384, 192]) biases = variable_on_cpu('biases', [192]) self.local4 = tf.nn.relu(tf.matmul(local3, weights) + biases, name=scope.name) # linear layer(WX + b), # We don't apply softmax here because # tf.nn.sparse_softmax_cross_entropy_with_logits # accepts the unscaled logits # and performs the softmax internally for efficiency. with tf.variable_scope('softmax_linear') as scope: weights = variable_on_cpu('weights', [192, NUM_CLASSES]) biases = variable_on_cpu('biases', [NUM_CLASSES]) self.softmax_linear = tf.add(tf.matmul(self.local4, weights), biases, name=scope.name) print('loading the network ...') saver_network = tf.train.Saver() # Restores from checkpoint saver_network.restore( self.sess, os.path.join(PATH_MODEL, 'model.ckpt-100000')) print('Graph successfully loaded.')
def convert(checkpoint_from_path, checkpoint_to_path, num_heads, name_replacements, permutations, exclude_patterns=None): """Migrates the names of variables within a checkpoint. Args: checkpoint_from_path: Path to source checkpoint to be read in. checkpoint_to_path: Path to checkpoint to be written out. num_heads: The number of heads of the model. name_replacements: A list of tuples of the form (match_str, replace_str) describing variable names to adjust. permutations: A list of tuples of the form (match_str, permutation) describing permutations to apply to given variables. Note that match_str should match the original variable name, not the replaced one. exclude_patterns: A list of string patterns to exclude variables from checkpoint conversion. Returns: A dictionary that maps the new variable names to the Variable objects. A dictionary that maps the old variable names to the new variable names. """ with tf.Graph().as_default(): tf.logging.info("Reading checkpoint_from_path %s", checkpoint_from_path) reader = tf.train.NewCheckpointReader(checkpoint_from_path) name_shape_map = reader.get_variable_to_shape_map() new_variable_map = {} conversion_map = {} for var_name in name_shape_map: if exclude_patterns and _has_exclude_patterns(var_name, exclude_patterns): continue # Get the original tensor data. tensor = reader.get_tensor(var_name) # Look up the new variable name, if any. new_var_name = _bert_name_replacement(var_name, name_replacements) # See if we need to reshape the underlying tensor. new_shape = None if num_heads > 0: new_shape = _get_new_shape(new_var_name, tensor.shape, num_heads) if new_shape: tf.logging.info("Veriable %s has a shape change from %s to %s", var_name, tensor.shape, new_shape) tensor = np.reshape(tensor, new_shape) # See if we need to permute the underlying tensor. permutation = _get_permutation(var_name, permutations) if permutation: tensor = np.transpose(tensor, permutation) # Create a new variable with the possibly-reshaped or transposed tensor. var = tf.Variable(tensor, name=var_name) # Save the variable into the new variable map. new_variable_map[new_var_name] = var # Keep a list of converter variables for sanity checking. if new_var_name != var_name: conversion_map[var_name] = new_var_name saver = tf.train.Saver(new_variable_map) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) tf.logging.info("Writing checkpoint_to_path %s", checkpoint_to_path) saver.save(sess, checkpoint_to_path) tf.logging.info("Summary:") tf.logging.info(" Converted %d variable name(s).", len(new_variable_map)) tf.logging.info(" Converted: %s", str(conversion_map))
def train(self, env_fn, hparams, simulated, save_continuously, epoch, sampling_temp=1.0, num_env_steps=None, env_step_multiplier=1, eval_env_fn=None, report_fn=None, model_save_fn=None): assert sampling_temp == 1.0 or hparams.learning_rate == 0.0, \ "Sampling with non-1 temperature does not make sense during training." if not save_continuously: # We do not save model, as that resets frames that we need at restarts. # But we need to save at the last step, so we set it very high. hparams.save_models_every_epochs = 1000000 if simulated: simulated_str = "sim" else: simulated_str = "real" name_scope = "ppo_{}{}".format(simulated_str, epoch + 1) event_dir = os.path.join(self.base_event_dir, "ppo_summaries", str(epoch) + simulated_str) with tf.Graph().as_default(): with tf.name_scope(name_scope): with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE): env = env_fn(in_graph=True) (train_summary_op, eval_summary_op, initializers) = (_define_train( env, hparams, eval_env_fn, sampling_temp, distributional_size=self._distributional_size, distributional_subscale=self._distributional_subscale, distributional_threshold=self. _distributional_threshold, epoch=epoch if simulated else -1, frame_stack_size=self.frame_stack_size, force_beginning_resets=simulated)) if num_env_steps is None: iteration_increment = hparams.epochs_num else: iteration_increment = int( math.ceil(num_env_steps / (env.batch_size * hparams.epoch_length))) iteration_increment *= env_step_multiplier self._num_completed_iterations += iteration_increment restarter = Restarter("policy", self.agent_model_dir, self._num_completed_iterations) if restarter.should_skip: return if hparams.lr_decay_in_final_epoch: if epoch != self.total_num_epochs - 1: # Extend the warmup period to the end of this epoch. hparams.learning_rate_warmup_steps = restarter.target_global_step else: if self._lr_decay_start is None: # Stop the warmup at the beginning of this epoch. self._lr_decay_start = \ restarter.target_global_step - iteration_increment hparams.learning_rate_warmup_steps = self._lr_decay_start _run_train(hparams, event_dir, self.agent_model_dir, restarter, train_summary_op, eval_summary_op, initializers, epoch, report_fn=report_fn, model_save_fn=model_save_fn)
def __init__(self, config, batch_size, checkpoint_dir_or_path=None, var_name_substitutions=None, session_target='', **sample_kwargs): if tf.gfile.IsDirectory(checkpoint_dir_or_path): checkpoint_path = tf.train.latest_checkpoint( checkpoint_dir_or_path) else: checkpoint_path = checkpoint_dir_or_path self._config = copy.deepcopy(config) self._config.data_converter.set_mode('infer') self._config.hparams.batch_size = batch_size with tf.Graph().as_default(): model = self._config.model model.build(self._config.hparams, self._config.data_converter.output_depth, is_training=False) # Input placeholders self._temperature = tf.placeholder(tf.float32, shape=()) if self._config.hparams.z_size: self._z_input = tf.placeholder( tf.float32, shape=[batch_size, self._config.hparams.z_size]) else: self._z_input = None if self._config.data_converter.control_depth > 0: self._c_input = tf.placeholder( tf.float32, shape=[None, self._config.data_converter.control_depth]) else: self._c_input = None self._inputs = tf.placeholder( tf.float32, shape=[ batch_size, None, self._config.data_converter.input_depth ]) self._controls = tf.placeholder( tf.float32, shape=[ batch_size, None, self._config.data_converter.control_depth ]) self._inputs_length = tf.placeholder( tf.int32, shape=[batch_size] + list(self._config.data_converter.length_shape)) self._max_length = tf.placeholder(tf.int32, shape=()) # Outputs self._outputs, self._decoder_results = model.sample( batch_size, max_length=self._max_length, z=self._z_input, c_input=self._c_input, temperature=self._temperature, **sample_kwargs) if self._config.hparams.z_size: q_z = model.encode(self._inputs, self._inputs_length, self._controls) self._mu = q_z.loc self._sigma = q_z.scale.diag self._z = q_z.sample() var_map = None if var_name_substitutions is not None: var_map = {} for v in tf.global_variables(): var_name = v.name[:-2] # Strip ':0' suffix. for pattern, substitution in var_name_substitutions: var_name = re.sub(pattern, substitution, var_name) if var_name != v.name[:-2]: tf.logging.info('Renaming `%s` to `%s`.', v.name[:-2], var_name) var_map[var_name] = v # Restore graph self._sess = tf.Session(target=session_target) saver = tf.train.Saver(var_map) if (os.path.exists(checkpoint_path) and tarfile.is_tarfile(checkpoint_path)): tf.logging.info('Unbundling checkpoint.') with tempfile.TemporaryDirectory() as temp_dir: tar = tarfile.open(checkpoint_path) tar.extractall(temp_dir) # Assume only a single checkpoint is in the directory. for name in tar.getnames(): if name.endswith('.index'): checkpoint_path = os.path.join( temp_dir, name[0:-6]) break saver.restore(self._sess, checkpoint_path) else: saver.restore(self._sess, checkpoint_path)
def main(unused_argv): train_steps = FLAGS.train_steps num_episode = FLAGS.num_episode trainee_model = FLAGS.trainee_model best_validation_loss = 10 batch_size = 1000 train_size = 50000 valid_size = 10000 test_batch_num = 10 if trainee_model == 'resnet': test_size = 1000 else: test_size = 10000 fashion_mnist = keras.datasets.fashion_mnist (train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data() train_images = train_images / 255.0 test_images = test_images / 255.0 # Different input tensor shape for different trainee models. if trainee_model == 'mlp': num_pixels = train_images.shape[1] * train_images.shape[2] x_train = train_images.reshape(train_images.shape[0], num_pixels).astype('float32') train_set = x_train[0:train_size, :] train_label = train_labels[0:train_size].astype('int64') valid_set = x_train[train_size:, :] valid_label = train_labels[train_size:].astype('int64') test_set = test_images.reshape(test_images.shape[0], num_pixels).astype('float32') test_label = test_labels.astype('int64') elif trainee_model == 'cnn' or trainee_model == 'resnet': x_train = train_images.reshape(train_images.shape[0], train_images.shape[1], train_images.shape[2], 1).astype('float32') train_set = x_train[0:train_size, :, :, :] train_label = train_labels[0:train_size].astype('int64') valid_set = x_train[train_size:, :, :, :] valid_label = train_labels[train_size:].astype('int64') test_set = test_images.reshape(test_images.shape[0], test_images.shape[1], test_images.shape[2], 1).astype('float32') test_label = test_labels.astype('int64') init_observation = np.array([1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]) observation_dim = init_observation.shape[0] adaptive_tuner_sess = tf.Session() adaptive_tuner = AdaptiveTuner( sess=adaptive_tuner_sess, n_features=observation_dim, n_actions=1, actor_lr=FLAGS.actor_learning_rate, critic_lr=FLAGS.critic_learning_rate, proposed_learning_rate=FLAGS.default_learning_rate) # Log the training process. running_reward = 0 valid_loss_matrix = np.zeros((num_episode, train_steps)) train_loss_matrix = np.zeros((num_episode, train_steps)) valid_accuracy_matrix = np.zeros((num_episode, train_steps)) test_loss_matrix = np.zeros((num_episode, train_steps)) test_accuracy_matrix = np.zeros((num_episode, train_steps)) reward_matrix = np.zeros((num_episode, train_steps)) observation_matrix = np.zeros((num_episode, train_steps, observation_dim)) learning_rate_matrix = np.zeros((num_episode, train_steps)) running_reward_array = np.zeros(num_episode) gnn = tf.Graph() with gnn.as_default(): # Prepare train/valid/test split for Fashion Mnist dataset = tf.data.Dataset.from_tensor_slices( (train_set, train_label)).repeat().batch(batch_size) train_iter = tf.data.make_one_shot_iterator(dataset) feature, label = train_iter.get_next() valid_dataset = tf.data.Dataset.from_tensor_slices( (valid_set, valid_label)).repeat().batch(valid_size) valid_iter = tf.data.make_one_shot_iterator(valid_dataset) valid_feature, valid_label = valid_iter.get_next() test_dataset = tf.data.Dataset.from_tensor_slices( (test_set, test_label)).repeat().batch(test_size) test_iter = tf.data.make_one_shot_iterator(test_dataset) test_feature, test_label = test_iter.get_next() learned_learning_rate = tf.placeholder(tf.float32, shape=[]) # Build trainee model. if trainee_model == 'mlp': train_logits = build_mlp_model(feature, reuse=False) valid_logits = build_mlp_model(valid_feature, reuse=True) test_logits = build_mlp_model(test_feature, reuse=True) elif trainee_model == 'cnn': train_logits = build_cnn_model(feature, reuse=False) valid_logits = build_cnn_model(valid_feature, reuse=True) test_logits = build_cnn_model(test_feature, reuse=True) elif trainee_model == 'resnet': resnet_size = 18 resnet_18 = resnet_model.FastCifar10Model( resnet_size=resnet_size, data_format='channels_first') train_logits = resnet_18(feature, True) train_logits = tf.cast(train_logits, tf.float32) valid_logits = resnet_18(valid_feature, False) valid_logits = tf.cast(valid_logits, tf.float32) test_logits = resnet_18(test_feature, False) test_logits = tf.cast(test_logits, tf.float32) else: raise ValueError('Wrong trainee_model flag value.') prediction = tf.nn.softmax(train_logits) valid_prediction = tf.nn.softmax(valid_logits) test_prediction = tf.nn.softmax(test_logits) if trainee_model == 'resnet': w1 = tf.get_default_graph().get_tensor_by_name( 'resnet_model/dense/kernel:0') w2 = tf.get_default_graph().get_tensor_by_name( 'resnet_model/dense/bias:0') else: w1 = tf.get_default_graph().get_tensor_by_name('w1/kernel:0') w2 = tf.get_default_graph().get_tensor_by_name('w2/kernel:0') mean_w1, var_w1 = tf.nn.moments(w1, axes=[0, 1]) if trainee_model == 'resnet': mean_w2, var_w2 = tf.nn.moments(w2, axes=[0]) else: mean_w2, var_w2 = tf.nn.moments(w2, axes=[0, 1]) loss = tf.losses.sparse_softmax_cross_entropy( labels=label, logits=train_logits) train_op = tf.train.AdamOptimizer( learning_rate=learned_learning_rate).minimize(loss) if trainee_model == 'resnet': update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) train_op = tf.group([train_op, update_ops]) valid_loss = tf.losses.sparse_softmax_cross_entropy( labels=valid_label, logits=valid_logits) valid_accuracy = compute_accuracy(valid_prediction, valid_label) test_loss = tf.losses.sparse_softmax_cross_entropy( labels=test_label, logits=test_logits) test_accuracy = compute_accuracy(test_prediction, test_label) for i_episode in range(num_episode): obs_t, action_t, reward_t, obs_t_1 = [], [], [], [] best_target_valid_loss = 10.0 with tf.Session(config=tf.ConfigProto(), graph=gnn) as sess: sess.run(tf.global_variables_initializer()) observation = init_observation prev_prediction = np.random.rand(batch_size, 10) prediction_log_var_ema = 0 prediction_change_log_var_ema = 0 reward = 0 num_reward = 0 average_reward = 0 reward_sum = 0 keep_lr_interval = 1 exp_decay = 0.8 action = 0.0 track_reward = [] adaptive_tuner.proposed_learning_rate = FLAGS.default_learning_rate for i in range(train_steps): if keep_lr_interval == 1 or i > train_steps - 10: action, action_multiplier = adaptive_tuner.choose_action(observation) keep_lr_interval = FLAGS.keep_lr_interval learning_rate_matrix[i_episode, i] = action _, loss_value, prediction_value, valid_loss_value, valid_accuracy_value, mean_w1_value, var_w1_value, mean_w2_value, var_w2_value = ( sess.run([ train_op, loss, prediction, valid_loss, valid_accuracy, mean_w1, var_w1, mean_w2, var_w2 ], feed_dict={learned_learning_rate: action})) if valid_loss_value < best_target_valid_loss: best_target_valid_loss = valid_loss_value if trainee_model == 'resnet': total_test_loss = 0.0 total_test_accuracy = 0.0 for _ in range(test_batch_num): batch_test_loss_value, batch_test_accuracy_value = sess.run( [test_loss, test_accuracy]) total_test_loss += batch_test_loss_value total_test_accuracy += batch_test_accuracy_value test_loss_value = total_test_loss / test_batch_num test_accuracy_value = total_test_accuracy / test_batch_num else: test_loss_value, test_accuracy_value = sess.run( [test_loss, test_accuracy]) test_loss_matrix[i_episode, i] = test_loss_value test_accuracy_matrix[i_episode, i] = test_accuracy_value train_loss_matrix[i_episode, i] = loss_value valid_loss_matrix[i_episode, i] = valid_loss_value valid_accuracy_matrix[i_episode, i] = valid_accuracy_value diff_prediction = prediction_value - prev_prediction if prediction_log_var_ema == 0: prediction_log_var_ema = np.log(np.var(prediction_value)) else: prediction_log_var_ema = exp_decay * prediction_log_var_ema + ( 1 - exp_decay) * np.log(np.var(prediction_value) + LOG_BASE) if prediction_change_log_var_ema == 0: prediction_change_log_var_ema = np.log(np.var(diff_prediction)) else: prediction_change_log_var_ema = exp_decay * prediction_change_log_var_ema + ( 1 - exp_decay) * np.log(np.var(diff_prediction) + LOG_BASE) # Collect all state observations. new_observation = np.array([ valid_loss_value, prediction_log_var_ema, prediction_change_log_var_ema, loss_value, mean_w1_value, var_w1_value, mean_w2_value, var_w2_value, action ]) observation_matrix[i_episode, i, :] = observation # Different reward functions. if FLAGS.reward_baseline == 'Fixed': reward = FLAGS.fixed_baseline_value - valid_loss_value elif FLAGS.reward_baseline == 'Average': reward_sum += valid_loss_value num_reward += 1 average_reward = float(reward_sum) / num_reward reward = average_reward - valid_loss_value elif FLAGS.reward_baseline == 'Exponential': if average_reward == 0: average_reward = valid_loss_value else: average_reward = average_reward * exp_decay + ( valid_loss_value * (1 - exp_decay)) reward = average_reward - valid_loss_value elif FLAGS.reward_baseline == 'Ratio': reward = FLAGS.reward_numerator / valid_loss_value - FLAGS.fixed_baseline_value else: raise ValueError('Wrong reward_baseline flag value.') reward_matrix[i_episode, i] = reward track_reward.append(reward) obs_t.append(observation) action_t.append(np.squeeze(action_multiplier)) reward_t.append(reward) obs_t_1.append(new_observation) observation = new_observation prev_prediction = prediction_value else: keep_lr_interval -= 1 sess.run([train_op], feed_dict={learned_learning_rate: action}) if i == train_steps - 1: current_best_validation_loss = valid_loss_matrix[i_episode, i] if current_best_validation_loss < best_validation_loss: best_validation_loss = current_best_validation_loss obs_t_ts, action_t_ts, reward_t_ts, obs_t_1_ts = np.vstack( obs_t), np.vstack(action_t), np.vstack(reward_t), np.vstack( obs_t_1) value_t = adaptive_tuner.get_value(obs_t_1_ts) td_reward = reward_t_ts + GAMMA * value_t adaptive_tuner.update(obs_t_ts, action_t_ts, td_reward) obs_t, action_t, reward_t, obs_t_1 = [], [], [], [] episode_reward = sum(track_reward) if running_reward == 0: running_reward = episode_reward else: running_reward = running_reward * 0.99 + episode_reward * 0.01 running_reward_array[i_episode] = running_reward
def generate_text(self, input_text): model_name = '124M_TRAINED' #String, which model to use seed = None #Integer seed for random number generators, fix seed to reproduce results nsamples = 1 #Number of samples to return total batch_size = 1 #Number of batches (only affects speed/memory). Must divide nsamples. length = 150 #Number of tokens in generated text, if None (default), is determined by model hyperparameters temperature = 1 #Float value controlling randomness in boltzmann # distribution. Lower temperature results in less random completions. As the # temperature approaches zero, the model will become deterministic and # repetitive. Higher temperature results in more random completions. top_k = 40 # Integer value controlling diversity. 1 means only 1 word is # considered for each step (token), resulting in deterministic completions, # while 40 means 40 words are considered at each step. 0 (default) is a # special setting meaning no restrictions. 40 generally is a good value. top_p = 1 #:models_dir : path to parent folder containing model subfolders (i.e. contains the <model_name> folder) self.response = "" if batch_size is None: batch_size = 1 assert nsamples % batch_size == 0 # this is going to give us our current location where our generator.py is cur_path = os.path.dirname(__file__) + "/models/" + model_name enc = encoder.get_encoder(model_name, os.path.dirname(__file__) + "/models") hparams = model.default_hparams() with open(os.path.join(cur_path + '/hparams.json')) as f: hparams.override_from_dict(json.load(f)) if length is None: length = hparams.n_ctx // 2 elif length > hparams.n_ctx: raise ValueError("Can't get samples longer than window size: %s" % hparams.n_ctx) with tf.Session(graph=tf.Graph()) as sess: context = tf.placeholder(tf.int32, [batch_size, None]) np.random.seed(seed) tf.set_random_seed(seed) output = sample.sample_sequence(hparams=hparams, length=length, context=context, batch_size=batch_size, temperature=temperature, top_k=top_k, top_p=top_p) saver = tf.train.Saver() ckpt = tf.train.latest_checkpoint(cur_path) saver.restore(sess, ckpt) context_tokens = enc.encode(input_text) generated = 0 for _ in range(nsamples // batch_size): out = sess.run(output, feed_dict={ context: [context_tokens for _ in range(batch_size)] })[:, len(context_tokens):] for i in range(batch_size): generated += 1 # our response/generated text text = enc.decode(out[i]) self.response = text # return generated text return self.response
def testCompressionOpInterface(self): with tf.Graph().as_default(): with self.cached_session() as sess: compression_hparams = ("name=cifar10_compression," "begin_compression_step=1000," "end_compression_step=120000," "compression_frequency=10," "compression_option=1," "update_option=0") global_step = tf.compat.v1.get_variable("global_step", initializer=30) c = compression_op.CompressionOp( spec=compression_op.CompressionOp.get_default_hparams().parse( compression_hparams), global_step=global_step) # Need to add initial value for a_matrix so that we would know what # to expect back. a_matrix_init = np.array([[1.0, 1.0, 1.0], [1.0, 0, 0], [1.0, 0, 0]]) a_matrix = tf.compat.v1.get_variable( "a_matrix", initializer=a_matrix_init.astype(np.float32), dtype=tf.float32) matrix_compressor = compression_op.LowRankDecompMatrixCompressor( spec=compression_op.LowRankDecompMatrixCompressor .get_default_hparams().parse("num_rows=3,num_cols=3,rank=200")) [a_matrix_compressed, a_matrix_update_op] = c.get_apply_compression_op( a_matrix, matrix_compressor, scope="my_scope") tf.compat.v1.global_variables_initializer().run() self.assertAllEqual( np.all(np.abs(np.linalg.norm(c.a_matrix_tfvar.eval())) < 0.00001), False) self.assertAllEqual( np.all(np.abs(np.linalg.norm(c.b_matrix_tfvar.eval())) < 0.00001), True) self.assertAllEqual( np.all(np.abs(np.linalg.norm(c.c_matrix_tfvar.eval())) < 0.00001), True) tf.compat.v1.assign(global_step, 1001).eval() sess.run(a_matrix_update_op) a_matrix_compressed.eval() self.assertEqual(c._global_step.eval(), 1001) self.assertAlmostEqual(c.alpha.eval(), 0.99) self.assertEqual(c._last_alpha_update_step.eval(), 1001) self.assertAllEqual( np.array([ np.linalg.norm(c.a_matrix_tfvar.eval()), np.linalg.norm(c.b_matrix_tfvar.eval()), np.linalg.norm(c.c_matrix_tfvar.eval()) ]) > 0, [True, True, True]) self.assertAllEqual( np.all(np.abs(np.linalg.norm(c.b_matrix_tfvar.eval())) < 0.00001), False) self.assertAllEqual( np.all(np.abs(np.linalg.norm(c.c_matrix_tfvar.eval())) < 0.00001), False) [b_matrix, c_matrix] = matrix_compressor.static_matrix_compressor(a_matrix_init) # since the matrices may match up to signs, we take absolute values. self.assertAllEqual( np.linalg.norm(np.abs(b_matrix) - np.abs(c.b_matrix_tfvar.eval())) < 0.00001, True) self.assertAllEqual( np.linalg.norm(np.abs(c_matrix) - np.abs(c.c_matrix_tfvar.eval())) < 0.00001, True) self.assertAllEqual( np.all(np.abs(np.linalg.norm(c.b_matrix_tfvar.eval())) < 0.00001), False) self.assertAllEqual( np.all(np.abs(np.linalg.norm(c.c_matrix_tfvar.eval())) < 0.00001), False) print("before 1002 step, c.alpha is ", c.alpha.eval()) tf.compat.v1.assign(global_step, 1001).eval() sess.run(a_matrix_update_op) a_matrix_compressed.eval() print("after 1002 step, c.alpha is ", c.alpha.eval()) self.assertEqual(c._global_step.eval(), 1001) self.assertAlmostEqual(c.alpha.eval(), 0.99) self.assertEqual(c._last_alpha_update_step.eval(), 1001) self.assertAllEqual( np.all([ np.linalg.norm(c.a_matrix_tfvar.eval()), np.linalg.norm(c.b_matrix_tfvar.eval()), np.linalg.norm(c.c_matrix_tfvar.eval()) ]) > 0, True) print("before 2000 step, alpha is ", c.alpha.eval()) tf.compat.v1.assign(global_step, 2000).eval() a_matrix_update_op.eval() a_matrix_compressed.eval() print("after 2000 step, alpha is ", c.alpha.eval()) self.assertEqual(c._global_step.eval(), 2000) self.assertAlmostEqual(c.alpha.eval(), 0.98) self.assertEqual(c._last_alpha_update_step.eval(), 2000) self.assertAllEqual( np.array([ np.linalg.norm(c.a_matrix_tfvar.eval()), np.linalg.norm(c.b_matrix_tfvar.eval()), np.linalg.norm(c.c_matrix_tfvar.eval()) ]) > 0, [True, True, True])
if not checkpoint: raise ValueError( "No checkpoint file found in: {}".format(checkpoint)) saver.restore(sess, checkpoint) #sess.run(tf.initialize_all_variables()) ''' for op in tf.get_default_graph().get_operations(): print(op.name, op.op_def.name) for var in tf.global_variables(): print (var.name) #if "convolutional_alexnet/conv1/weights" in var.name: if "detection/biases" in var.name: print(sess.run(var)) ''' # extract frozed graph # http://workpiles.com/2016/07/tensorflow-protobuf-dump/ frozen_graph_def = tf.graph_util.convert_variables_to_constants( sess, tf.get_default_graph().as_graph_def(), ["upsample/final_result"]) frozen_graph = tf.Graph() with frozen_graph.as_default(): tf.import_graph_def(frozen_graph_def) tf.train.write_graph(frozen_graph_def, model_save_dir, 'whole_model_scale' + str(args.scale) + '.pb', as_text=False)
def train(images, labels, ckpt_path, dropout=False): """ This function contains the loop that actually trains the model. :param images: a numpy array with the input data :param labels: a numpy array with the output labels :param ckpt_path: a path (including name) where model checkpoints are saved :param dropout: Boolean, whether to use dropout or not :return: True if everything went well """ # Check training data assert len(images) == len(labels) assert images.dtype == np.float32 assert labels.dtype == np.int32 # Set default TF graph with tf.Graph().as_default(): global_step = tf.Variable(0, trainable=False) # Declare data placeholder train_data_node = _input_placeholder() # Create a placeholder to hold labels train_labels_shape = (FLAGS.batch_size,) train_labels_node = tf.placeholder(tf.int32, shape=train_labels_shape) print("Done Initializing Training Placeholders") # Build a Graph that computes the logits predictions from the placeholder if FLAGS.deeper: logits = inference_deeper(train_data_node, dropout=dropout) else: logits = inference(train_data_node, dropout=dropout) # Calculate loss loss = loss_fun(logits, train_labels_node) # Build a Graph that trains the model with one batch of examples and # updates the model parameters. train_op = train_op_fun(loss, global_step) # Create a saver. saver = tf.train.Saver(tf.global_variables()) print("Graph constructed and saver created") # Build an initialization operation to run below. init = tf.global_variables_initializer() # Create and init sessions sess = tf.Session(config=tf.ConfigProto(log_device_placement=FLAGS.log_device_placement)) #NOLINT(long-line) sess.run(init) print("Session ready, beginning training loop") # Initialize the number of batches data_length = len(images) nb_batches = math.ceil(data_length / FLAGS.batch_size) for step in xrange(FLAGS.max_steps): # for debug, save start time start_time = time.time() # Current batch number batch_nb = step % nb_batches # Current batch start and end indices start, end = utils.batch_indices(batch_nb, data_length, FLAGS.batch_size) # Prepare dictionnary to feed the session with feed_dict = {train_data_node: images[range(start, end)], train_labels_node: labels[range(start, end)]} # Run training step _, loss_value = sess.run([train_op, loss], feed_dict=feed_dict) # Compute duration of training step duration = time.time() - start_time # Sanity check assert not np.isnan(loss_value), 'Model diverged with loss = NaN' # Echo loss once in a while if step % 100 == 0: num_examples_per_step = FLAGS.batch_size examples_per_sec = num_examples_per_step / duration sec_per_batch = float(duration) format_str = ('%s: step %d, loss = %.2f (%.1f examples/sec; %.3f ' 'sec/batch)') print (format_str % (datetime.now(), step, loss_value, examples_per_sec, sec_per_batch)) # Save the model checkpoint periodically. if step % 1000 == 0 or (step + 1) == FLAGS.max_steps: saver.save(sess, ckpt_path, global_step=step) return True
def model_inference(model_fn, model_dir, checkpoint_path, data_fn, hparams, examples_path, output_dir, summary_writer, master, preprocess_examples, shuffle_examples): """Runs inference for the given examples.""" tf.logging.info('model_dir=%s', model_dir) tf.logging.info('checkpoint_path=%s', checkpoint_path) tf.logging.info('examples_path=%s', examples_path) tf.logging.info('output_dir=%s', output_dir) estimator = train_util.create_estimator(model_fn, model_dir, hparams, master=master) transcription_data = functools.partial( data_fn, examples=examples_path, preprocess_examples=preprocess_examples, is_training=False, shuffle_examples=shuffle_examples, skip_n_initial_records=0) input_fn = infer_util.labels_to_features_wrapper(transcription_data) start_time = time.time() infer_times = [] num_frames = [] file_num = 0 all_metrics = collections.defaultdict(list) for predictions in estimator.predict(input_fn, checkpoint_path=checkpoint_path, yield_single_examples=False): # Remove batch dimension for convenience. for k in predictions.keys(): if predictions[k].shape[0] != 1: raise ValueError( 'All predictions must have batch size 1, but shape of ' '{} was: {}'.format(k, +predictions[k].shape[0])) predictions[k] = predictions[k][0] end_time = time.time() infer_time = end_time - start_time infer_times.append(infer_time) num_frames.append(predictions['frame_predictions'].shape[0]) tf.logging.info( 'Infer time %f, frames %d, frames/sec %f, running average %f', infer_time, num_frames[-1], num_frames[-1] / infer_time, np.sum(num_frames) / np.sum(infer_times)) tf.logging.info('Scoring sequence %s', predictions['sequence_ids']) sequence_prediction = music_pb2.NoteSequence.FromString( predictions['sequence_predictions']) sequence_label = music_pb2.NoteSequence.FromString( predictions['sequence_labels']) # Make filenames UNIX-friendly. filename_chars = six.ensure_text(predictions['sequence_ids'], 'utf-8') filename_chars = [c if c.isalnum() else '_' for c in filename_chars] filename_safe = ''.join(filename_chars).rstrip() filename_safe = '{:04d}_{}'.format(file_num, filename_safe[:200]) file_num += 1 output_file = os.path.join(output_dir, filename_safe + '.mid') tf.logging.info('Writing inferred midi file to %s', output_file) midi_io.sequence_proto_to_midi_file(sequence_prediction, output_file) label_output_file = os.path.join(output_dir, filename_safe + '_label.mid') tf.logging.info('Writing label midi file to %s', label_output_file) midi_io.sequence_proto_to_midi_file(sequence_label, label_output_file) # Also write a pianoroll showing acoustic model output vs labels. pianoroll_output_file = os.path.join(output_dir, filename_safe + '_pianoroll.png') tf.logging.info('Writing acoustic logit/label file to %s', pianoroll_output_file) # Calculate frames based on the sequence. Includes any postprocessing done # to turn raw onsets/frames predictions into the final sequence. # TODO(fjord): This work is duplicated in metrics.py. sequence_frame_predictions = sequences_lib.sequence_to_pianoroll( sequence_prediction, frames_per_second=data.hparams_frames_per_second(hparams), min_pitch=constants.MIN_MIDI_PITCH, max_pitch=constants.MAX_MIDI_PITCH).active with tf.gfile.GFile(pianoroll_output_file, mode='w') as f: imageio.imwrite(f, infer_util.posterior_pianoroll_image( predictions['onset_probs'], predictions['onset_labels'], predictions['frame_probs'], predictions['frame_labels'], sequence_frame_predictions), format='png') # Update histogram and current scalar for metrics. with tf.Graph().as_default(), tf.Session().as_default(): for k, v in predictions.items(): if not k.startswith('metrics/'): continue all_metrics[k].extend(v) histogram_name = k + '_histogram' metric_summary = tf.summary.histogram(histogram_name, all_metrics[k]) summary_writer.add_summary(metric_summary.eval(), global_step=file_num) scalar_name = k metric_summary = tf.summary.scalar(scalar_name, np.mean(all_metrics[k])) summary_writer.add_summary(metric_summary.eval(), global_step=file_num) summary_writer.flush() start_time = time.time() # Write final mean values for all metrics. with tf.Graph().as_default(), tf.Session().as_default(): for k, v in all_metrics.items(): final_scalar_name = 'final/' + k metric_summary = tf.summary.scalar(final_scalar_name, np.mean(all_metrics[k])) summary_writer.add_summary(metric_summary.eval()) summary_writer.flush()
def run_train(scope): """Trains a network. Args: scope: the scope of variables in this function """ with tf.Graph().as_default(): with tf.device(tf.train.replica_device_setter(FLAGS.ps_tasks)): to_gray = True if 'sem' in FLAGS.network_id: to_gray = False batch_frames, batch_labels = get_samples(to_gray, 'train') batch_hmg_prediction, _ = predict_homography( batch_frames, network_id=FLAGS.network_id, is_training=True, scope=scope) if FLAGS.loss == 'hier_l2': for level in range(FLAGS.num_level): delta_level = FLAGS.num_level - level - 1 scale = 2**delta_level l2 = tf.losses.mean_squared_error( batch_labels / scale, batch_hmg_prediction[level]) slim.summaries.add_scalar_summary(l2, 'l2%d' % delta_level, 'losses') elif FLAGS.loss == 'hier_ld': for level in range(FLAGS.num_level): delta_level = FLAGS.num_level - level - 1 scale = 2**delta_level diff = tf.reshape( batch_labels / scale - batch_hmg_prediction[level], [FLAGS.batch_size, 4, 2]) l2d = tf.reduce_mean( tf.sqrt(tf.reduce_sum(tf.square(diff), 2))) tf.losses.add_loss(l2d) slim.summaries.add_scalar_summary(l2d, 'l2%d' % delta_level, 'losses') else: l2 = tf.losses.mean_squared_error( batch_labels, batch_hmg_prediction[FLAGS.num_level - 1]) slim.summaries.add_scalar_summary(slim.losses.get_total_loss(), 'loss', 'losses') global_step = slim.get_or_create_global_step() learning_rate_decay = tf.train.exponential_decay( learning_rate=FLAGS.learning_rate, global_step=global_step, decay_steps=FLAGS.lr_decay_steps, decay_rate=FLAGS.lr_decay_rate, staircase=True) optimizer = tf.train.AdamOptimizer(learning_rate_decay) is_chief = (FLAGS.task == 0) train_op = slim.learning.create_train_op( slim.losses.get_total_loss(), optimizer=optimizer) saver = tf.train.Saver(max_to_keep=20) if FLAGS.level_wise == 0: variables_to_restore = [] for i in range(0, FLAGS.num_level - 1): variables = slim.get_variables(scope='%s/level%d' % (scope, i)) variables_to_restore = variables_to_restore + variables init_fn = slim.assign_from_checkpoint_fn( FLAGS.model_path, variables_to_restore) elif 'sem' in FLAGS.network_id: variables_to_restore = slim.get_variables(scope='vgg_16') init_fn = slim.assign_from_checkpoint_fn( FLAGS.vgg_model_path, variables_to_restore) else: init_fn = None slim.learning.train(train_op=train_op, logdir=FLAGS.train_dir, save_summaries_secs=60, save_interval_secs=600, saver=saver, number_of_steps=FLAGS.max_step, master=FLAGS.master, is_chief=is_chief, init_fn=init_fn)
def AdaBatch(gpu_id, input_reader, model_type, training_epochs, batch_size, lr_boundaries, lr_values, optimizer_type, update_method, warm_up_period, s_e=100.0, pretrain=0, log_dir="log"): if not os.path.exists(log_dir): os.makedirs(log_dir) text_log = [] text_log.append( "epoch, time(s), learning rate, minibatch loss, minibatch error, test loss, test error" ) num_train_images = input_reader.num_train_images num_val_images = input_reader.num_val_images num_label = input_reader.num_classes image_shape = [input_reader.width, input_reader.height, input_reader.depth] train_batch_patcher = patcher.BatchPatcher(num_train_images, batch_size, num_label, s_e=s_e, update_method=update_method) validation_batch_patcher = patcher.BatchPatcher( num_val_images, batch_size, num_label, update_method=update_method) config = tf.ConfigProto(allow_soft_placement=True) config.gpu_options.visible_device_list = str(gpu_id) config.gpu_options.allow_growth = True graph = tf.Graph() with graph.as_default(): with tf.device('/gpu:' + str(gpu_id)): with tf.Session(config=config) as sess: # Input Graph Generation ############################################################################# t_ids, t_images, t_labels = input_reader.data_read(batch_size, train=True) v_ids, v_images, v_labels = input_reader.data_read(batch_size, train=False) # Model Graph Construction ########################################################################### if model_type == "DenseNet-25-12": model = DenseNet(25, 12, image_shape, num_label, batch_size, batch_size) elif model_type == "WideResNet16-8": model = WideResNet(16, 8, image_shape, num_label, batch_size, batch_size) train_loss_op, train_accuracy_op, train_op, _, train_distance_op = model.build_train_op( lr_boundaries, lr_values, optimizer_type) test_loss_op, test_accuracy_op, _ = model.build_test_op() # Data load in memeory ############################################################################### print("start to load data set.") coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) train_batch_patcher.bulk_load_in_memory( sess, t_ids, t_images, t_labels) validation_batch_patcher.bulk_load_in_memory( sess, v_ids, v_images, v_labels) start_time = time.time() # Model Initialization ########################################################################### # init params: we share the initial epochs. See paper. if pretrain != 0: start_time = time.time() saver = tf.train.Saver() file_dir = "init_weight/" + input_reader.dataset_name + "/" + model_type + "_" + optimizer_type + "_lr=" + str( lr_values[0]) + "_e=" + str(pretrain) + "/" minus_start_time = 0 with open(file_dir + "text_log.csv") as f: for line in f: print(line, end="") text_log.append(line.rstrip()) minus_start_time = line.split(",")[1] start_time = start_time - float(minus_start_time) saver.restore(sess, file_dir + "model.ckpt") for i in range(train_batch_patcher.num_iters_per_epoch): ids, images, labels = train_batch_patcher.get_init_mini_batch( i) distance = sess.run(train_distance_op, feed_dict={ model.train_image_placeholder: images, model.train_label_placeholder: labels }) train_batch_patcher.update_prob_table(ids, distance) print(train_batch_patcher.prob_table.table) print("shared weight is successfully loaded") else: sess.run(tf.global_variables_initializer()) # Traing Process ##################################################################################### for epoch in range(pretrain, training_epochs): if epoch < warm_up_period: is_warm_up = True else: is_warm_up = False # (1) Mini-batch loss and error along with netowrk updates avg_mini_loss = 0.0 avg_mini_acc = 0.0 for i in range(train_batch_patcher.num_iters_per_epoch): # Next batch depends on the method: {Ada_Boundary, Ada-Hard, Ada-Uniform} ids, images, labels = train_batch_patcher.get_next_mini_batch( num_of_sample=batch_size, is_warm_up=is_warm_up) mini_loss, mini_acc, _, distance = sess.run( [ train_loss_op, train_accuracy_op, train_op, train_distance_op ], feed_dict={ model.train_image_placeholder: images, model.train_label_placeholder: labels }) train_batch_patcher.update_prob_table(ids, distance) avg_mini_loss += mini_loss avg_mini_acc += mini_acc avg_mini_loss /= train_batch_patcher.num_iters_per_epoch avg_mini_acc /= train_batch_patcher.num_iters_per_epoch # (2) Compute training loss and error avg_train_loss = 0.0 avg_train_acc = 0.0 for i in range(train_batch_patcher.num_iters_per_epoch): ids, images, labels = train_batch_patcher.get_init_mini_batch( i) train_loss, train_acc = sess.run( [test_loss_op, test_accuracy_op], feed_dict={ model.test_image_placeholder: images, model.test_label_placeholder: labels }) avg_train_loss += train_loss avg_train_acc += train_acc avg_train_loss /= train_batch_patcher.num_iters_per_epoch avg_train_acc /= train_batch_patcher.num_iters_per_epoch # (3) Validation (or test) loss and error avg_val_loss = 0.0 avg_val_acc = 0.0 for i in range( validation_batch_patcher.num_iters_per_epoch): ids, images, labels = validation_batch_patcher.get_init_mini_batch( i) val_loss, val_acc = sess.run( [test_loss_op, test_accuracy_op], feed_dict={ model.test_image_placeholder: images, model.test_label_placeholder: labels }) avg_val_loss += val_loss avg_val_acc += val_acc avg_val_loss /= validation_batch_patcher.num_iters_per_epoch avg_val_acc /= validation_batch_patcher.num_iters_per_epoch # Log Writing #################################################################################### cur_lr = sess.run(model.learning_rate) print((epoch + 1), ", ", int(time.time() - start_time), ", ", cur_lr, ", ", avg_mini_loss, ", ", (1.0 - avg_mini_acc), ", ", avg_train_loss, ", ", (1.0 - avg_train_acc), ", ", avg_val_loss, ", ", (1.0 - avg_val_acc)) text_log.append( str(epoch + 1) + ", " + str(int(time.time() - start_time)) + ", " + str(cur_lr) + ", " + str(avg_mini_loss) + ", " + str(1.0 - avg_mini_acc) + ", " + str(avg_train_loss) + ", " + str(1.0 - avg_train_acc) + ", " + str(avg_val_loss) + ", " + str(1.0 - avg_val_acc)) coord.request_stop() coord.join(threads) sess.close() # Log Flushing f = open(log_dir + "/text_log.csv", "w") for text in text_log: f.write(text + "\n") f.close()
def __init__(self, bert_config, tokenizer, **kwargs): device = get_device(**kwargs) _graph = tf.Graph() with _graph.as_default(): with tf.device(device): self.X = tf.placeholder(tf.int32, [None, None]) self.segment_ids = tf.placeholder(tf.int32, [None, None]) self.top_p = tf.placeholder(tf.float32, None) self.top_k = tf.placeholder(tf.int32, None) self.k = tf.placeholder(tf.int32, None) self.temperature = tf.placeholder(tf.float32, None) self.indices = tf.placeholder(tf.int32, [None, None]) self.MASK = tf.placeholder(tf.int32, [None, None]) self._tokenizer = tokenizer self.model = modeling.BertModel( config=bert_config, is_training=False, input_ids=self.X, input_mask=self.MASK, use_one_hot_embeddings=False, ) self.logits = self.model.get_pooled_output() output_layer = self.model.get_sequence_output() embedding = self.model.get_embedding_table() with tf.variable_scope('cls/predictions'): with tf.variable_scope('transform'): input_tensor = tf.layers.dense( output_layer, units=bert_config.hidden_size, activation=modeling.get_activation( bert_config.hidden_act), kernel_initializer=modeling.create_initializer( bert_config.initializer_range), ) input_tensor = modeling.layer_norm(input_tensor) output_bias = tf.get_variable( 'output_bias', shape=[bert_config.vocab_size], initializer=tf.zeros_initializer(), ) logits = tf.matmul(input_tensor, embedding, transpose_b=True) self._logits = tf.nn.bias_add(logits, output_bias) self._log_softmax = tf.nn.log_softmax(self._logits) logits = tf.gather_nd(self._logits, self.indices) logits = logits / self.temperature def necleus(): return top_p_logits(logits, self.top_p) def select_k(): return top_k_logits(logits, self.top_k) logits = tf.cond(self.top_p > 0, necleus, select_k) self.samples = tf.multinomial(logits, num_samples=self.k, output_dtype=tf.int32) self._sess = generate_session(_graph, **kwargs) self._sess.run(tf.global_variables_initializer()) var_lists = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='bert') cls = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='cls') self._saver = tf.train.Saver(var_list=var_lists + cls) attns = _extract_attention_weights( bert_config.num_hidden_layers, tf.get_default_graph()) self.attns = attns
# here in case it was not downloaded or put in the correct location. if not os.path.exists(MODEL_FILE): opener = urllib.request.URLopener() opener.retrieve( 'http://download.tensorflow.org/models/object_detection/' + MODEL_FILE, MODEL_FILE) tar_file = tarfile.open(MODEL_FILE) for file in tar_file.getmembers(): file_name = os.path.basename(file.name) if 'frozen_inference_graph.pb' in file_name: tar_file.extract(file, os.getcwd()) # Path to frozen detection graph. This is the actual model that is used for the object detection. PATH_TO_FROZEN_GRAPH = MODEL_NAME + '/frozen_inference_graph.pb' detection_graph = tf.Graph() with detection_graph.as_default(): od_graph_def = tf.GraphDef() with tf.gfile.GFile(PATH_TO_FROZEN_GRAPH, 'rb') as fid: serialized_graph = fid.read() od_graph_def.ParseFromString(serialized_graph) tf.import_graph_def(od_graph_def, name='') # Create the mapping from label index to human-readable object label (e.g. 'Person') PATH_TO_LABELS = os.path.join('object_detection', 'data', 'mscoco_label_map.pbtxt') #PATH_TO_LABELS='/home/pi/final_project/v1/fa20-cs498it-lab1-master/src/adas/object_detection/data/mscoco_label_map.pbtxt' #categories = label_map_util.create_categories_from_labelmap(PATH_TO_LABELS, use_display_name=True sys.path.append('') category_index = label_map_util.create_category_index_from_labelmap(
def train_collect_eval(collect_env, eval_env, test_env, policy_class, run_agent_fn=run_env.run_env, train_fn=train_q.train_q, do_collect_eval=True, file_patterns='', get_data_fn=input_data.get_data, onpolicy=True, num_collect=100, num_eval=100, num_test=100, data_format='tfrecord', eval_frequency=5, root_dir=None, task=0, master='', ps_tasks=0): """Runs synchronous train, collect, eval loop. This loop instantiates the policy instance from policy_class. The policy manages its own tf.Session. The train function may create its own session for the purpose of updating its variables. train_fn reuses graph created by policy, to avoid having to configure the same neural net twice (one for policy and one for training.) Args: collect_env: (gym.Env) Gym environment to collect data from (and train the policy on). eval_env: (gym.Env) Gym environment to evaluate the policy on. Can be another instance of collect_env, or a different environment if one wishes to evaluate generalization capability. The only constraint is that the action and observation spaces have to be equivalent. If None, eval_env is not evaluated. test_env: (gym.Env) Another environment to evaluate on. Either another instance of collect_env, or a different environment to evaluate generalization. policy_class: Policy class that we want to train. run_agent_fn: (Optional) Python function that executes the interaction of the policy with the environment. Defaults to run_env.run_env. train_fn: (Optional) Python function that trains the policy. Defaults to train_q.train_q. do_collect_eval: If True, performs data collection using the trained policy. file_patterns: (str) Comma-separated regex of file patterns to train on. This is used to instantiate the file-backed "replay buffer". get_data_fn: (Optional) Python function that fetches data from files. onpolicy: (bool) If True, appends data from policy_collect directory. num_collect: (int) Number of episodes to collect & evaluate from collect_env. num_eval: (int) Number of episodes to collect & evaluate from eval_env. num_test: (int) Number of episodes to collect & evaluate from test_env. data_format: (string) File extension of input data files. eval_frequency: (int) How many times we run eval/test vs. collect. Evaluating is costly compared to training, so we can speed up iteration time by not evaluating every time we collect. root_dir: (str) Root directory for this training trial. Training directory, eval directory are subdirectories of root_dir. task: (int) Optional worker task for distributed training. Defaults to solo master task on a single machine master: (int) Optional address of master worker. Specify this when doing distributed training. ps_tasks: (int) Optional number of parameter-server tasks. Used only for distributed TF training jobs. Raises: ValueError: If ps_tasks > 0 (implies distributed training) while do_collect_eval is set to True. """ # Spaces do not implement `==` operator. Convert to strings to check # compatibility between training & eval env representation. if ((collect_env and eval_env) and (str(collect_env.observation_space), str(collect_env.action_space)) != (str(eval_env.observation_space), str(eval_env.action_space))): raise ValueError('Collect and Eval environments have incompatible ' 'observation or action dimensions.') if ps_tasks > 0 and do_collect_eval: raise ValueError( 'Collecting data not supported by distributed training jobs') if onpolicy: file_patterns += ',' + os.path.join(root_dir, 'policy_collect', '*.%s' % data_format) train_dir = os.path.join(root_dir, 'train') it = 0 while True: tf.reset_default_graph() # Re-fresh the source of data. with tf.Graph().as_default(): with tf.device(tf.train.replica_device_setter(ps_tasks)): policy = policy_class() if train_fn: dataset = get_data_fn(file_patterns=file_patterns) step, done = train_fn(dataset, policy, log_dir=train_dir, reuse=True, task=task, master=master) else: step, done = 0, True if train_fn: tf.logging.info('Evaluating policy at step %d' % step) ckpt = tf.train.latest_checkpoint(train_dir) tf.logging.info('Restoring model variables from %s' % ckpt) policy.restore(ckpt) if ckpt: step = int(ckpt.split('.ckpt-')[-1]) if onpolicy: run_agent_fn(collect_env, policy=policy, global_step=step, root_dir=root_dir, task=task, num_episodes=num_collect, tag='collect') if it % eval_frequency == 0: if eval_env: run_agent_fn(eval_env, policy=policy, global_step=step, root_dir=root_dir, task=task, explore_schedule=None, num_episodes=num_eval, tag='eval') if test_env: run_agent_fn(test_env, policy=policy, global_step=step, root_dir=root_dir, task=task, explore_schedule=None, num_episodes=num_test, tag='test') it += 1 if done: tf.logging.info('Train-Collect-Eval completed.') break
def generate_encoded_samples(self, data_dir, tmp_dir, dataset_split): """Generate samples of the encoded frames with possible extra data. By default this function just encodes the numpy array returned as "frame" from `self.generate_samples` into a PNG image. Override this function to get other encodings on disk. Args: data_dir: final data directory. Typically only used in this method to copy over user-supplied vocab files if there are extra fields needing them. tmp_dir: temporary directory that you can use for downloading and scratch. dataset_split: problem.DatasetSplit, which data split to generate samples for (for example, training and evaluation). Yields: Sample: dict<str feature_name, feature value> which is in disk encoding. Raises: ValueError: if the frame has a different number of channels than required. """ writer = None with tf.Graph().as_default(): image_t = tf.placeholder(dtype=tf.uint8, shape=(None, None, None)) encoded_image_t = tf.image.encode_png(image_t) with tf.Session() as sess: for features in self.generate_samples(data_dir, tmp_dir, dataset_split): unencoded_frame = features.pop("frame") self.validate_frame(unencoded_frame) height, width, _ = unencoded_frame.shape encoded_frame = sess.run( encoded_image_t, feed_dict={image_t: unencoded_frame}) features["image/encoded"] = [encoded_frame] features["image/format"] = ["png"] features["image/height"] = [height] features["image/width"] = [width] has_debug_image = "image/debug" in features if has_debug_image: unencoded_debug = features.pop("image/debug") encoded_debug = sess.run( encoded_image_t, feed_dict={image_t: unencoded_debug}) features["image/encoded_debug"] = [encoded_debug] if self.debug_dump_frames_path: # Defer creating debug writer until we know debug_dump_frames_path. if writer is None: if not tf.gfile.Exists( self.debug_dump_frames_path): tf.gfile.MkDir(self.debug_dump_frames_path) writer = debug_video_writer_factory( self.debug_dump_frames_path) img = unencoded_debug if has_debug_image else unencoded_frame encoded_img = encoded_debug if has_debug_image else encoded_frame writer.write(img, encoded_img) yield features if self.debug_dump_frames_path: writer.finish_to_disk()
#import tensorflow as tf import tensorflow.compat.v1 as tf tf.disable_v2_behavior() from build_cnn import build_cnn from cnn_utils import load, predict from std_mnist import X_test_centered, y_test # 2019.12.07 add ## Define random seed random_seed = 123 np.random.seed(random_seed) ## create a new graph ## and build the model g2 = tf.Graph() with g2.as_default(): tf.set_random_seed(random_seed) ## build the graph build_cnn() ## saver: saver = tf.train.Saver() np.set_printoptions(precision=2, suppress=True) with tf.Session(graph=g2) as sess: load(saver, sess, epoch=20, path='./model/') print(predict(sess, X_test_centered[:10], return_proba=False))
def train(create_tensor_dict_fn, create_model_fn, train_config, master, task, num_clones, worker_replicas, clone_on_cpu, ps_tasks, worker_job_name, is_chief, train_dir, graph_hook_fn=None): """Training function for detection models. Args: create_tensor_dict_fn: a function to create a tensor input dictionary. create_model_fn: a function that creates a DetectionModel and generates losses. train_config: a train_pb2.TrainConfig protobuf. master: BNS name of the TensorFlow master to use. task: The task id of this training instance. num_clones: The number of clones to run per machine. worker_replicas: The number of work replicas to train with. clone_on_cpu: True if clones should be forced to run on CPU. ps_tasks: Number of parameter server tasks. worker_job_name: Name of the worker job. is_chief: Whether this replica is the chief replica. train_dir: Directory to write checkpoints and training summaries to. graph_hook_fn: Optional function that is called after the inference graph is built (before optimization). This is helpful to perform additional changes to the training graph such as adding FakeQuant ops. The function should modify the default graph. Raises: ValueError: If both num_clones > 1 and train_config.sync_replicas is true. """ detection_model = create_model_fn() data_augmentation_options = [ preprocessor_builder.build(step) for step in train_config.data_augmentation_options] with tf.Graph().as_default(): # Build a configuration specifying multi-GPU and multi-replicas. deploy_config = model_deploy.DeploymentConfig( num_clones=num_clones, clone_on_cpu=clone_on_cpu, replica_id=task, num_replicas=worker_replicas, num_ps_tasks=ps_tasks, worker_job_name=worker_job_name) # Place the global step on the device storing the variables. with tf.device(deploy_config.variables_device()): global_step = slim.create_global_step() if num_clones != 1 and train_config.sync_replicas: raise ValueError('In Synchronous SGD mode num_clones must ', 'be 1. Found num_clones: {}'.format(num_clones)) batch_size = train_config.batch_size // num_clones if train_config.sync_replicas: batch_size //= train_config.replicas_to_aggregate with tf.device(deploy_config.inputs_device()): input_queue = create_input_queue( batch_size, create_tensor_dict_fn, train_config.batch_queue_capacity, train_config.num_batch_queue_threads, train_config.prefetch_queue_capacity, data_augmentation_options) # Gather initial summaries. # TODO(rathodv): See if summaries can be added/extracted from global tf # collections so that they don't have to be passed around. summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES)) global_summaries = set([]) model_fn = functools.partial(_create_losses, create_model_fn=create_model_fn, train_config=train_config) clones = model_deploy.create_clones(deploy_config, model_fn, [input_queue]) first_clone_scope = clones[0].scope if graph_hook_fn: with tf.device(deploy_config.variables_device()): graph_hook_fn() # Gather update_ops from the first clone. These contain, for example, # the updates for the batch_norm variables created by model_fn. update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, first_clone_scope) with tf.device(deploy_config.optimizer_device()): training_optimizer, optimizer_summary_vars = optimizer_builder.build( train_config.optimizer) for var in optimizer_summary_vars: tf.summary.scalar(var.op.name, var, family='LearningRate') sync_optimizer = None if train_config.sync_replicas: training_optimizer = tf.train.SyncReplicasOptimizer( training_optimizer, replicas_to_aggregate=train_config.replicas_to_aggregate, total_num_replicas=worker_replicas) sync_optimizer = training_optimizer with tf.device(deploy_config.optimizer_device()): regularization_losses = (None if train_config.add_regularization_loss else []) total_loss, grads_and_vars = model_deploy.optimize_clones( clones, training_optimizer, regularization_losses=regularization_losses) total_loss = tf.check_numerics(total_loss, 'LossTensor is inf or nan.') # Optionally multiply bias gradients by train_config.bias_grad_multiplier. if train_config.bias_grad_multiplier: biases_regex_list = ['.*/biases'] grads_and_vars = variables_helper.multiply_gradients_matching_regex( grads_and_vars, biases_regex_list, multiplier=train_config.bias_grad_multiplier) # Optionally freeze some layers by setting their gradients to be zero. if train_config.freeze_variables: grads_and_vars = variables_helper.freeze_gradients_matching_regex( grads_and_vars, train_config.freeze_variables) # Optionally clip gradients if train_config.gradient_clipping_by_norm > 0: with tf.name_scope('clip_grads'): grads_and_vars = slim.learning.clip_gradient_norms( grads_and_vars, train_config.gradient_clipping_by_norm) # Create gradient updates. grad_updates = training_optimizer.apply_gradients(grads_and_vars, global_step=global_step) update_ops.append(grad_updates) update_op = tf.group(*update_ops, name='update_barrier') with tf.control_dependencies([update_op]): train_tensor = tf.identity(total_loss, name='train_op') # Add summaries. for model_var in slim.get_model_variables(): global_summaries.add(tf.summary.histogram('ModelVars/' + model_var.op.name, model_var)) for loss_tensor in tf.losses.get_losses(): global_summaries.add(tf.summary.scalar('Losses/' + loss_tensor.op.name, loss_tensor)) global_summaries.add( tf.summary.scalar('Losses/TotalLoss', tf.losses.get_total_loss())) # Add the summaries from the first clone. These contain the summaries # created by model_fn and either optimize_clones() or _gather_clone_loss(). summaries |= set(tf.get_collection(tf.GraphKeys.SUMMARIES, first_clone_scope)) summaries |= global_summaries # Merge all summaries together. summary_op = tf.summary.merge(list(summaries), name='summary_op') # Soft placement allows placing on CPU ops without GPU implementation. session_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) # Save checkpoints regularly. keep_checkpoint_every_n_hours = train_config.keep_checkpoint_every_n_hours saver = tf.train.Saver( keep_checkpoint_every_n_hours=keep_checkpoint_every_n_hours) # Create ops required to initialize the model from a given checkpoint. init_fn = None if train_config.fine_tune_checkpoint: if not train_config.fine_tune_checkpoint_type: # train_config.from_detection_checkpoint field is deprecated. For # backward compatibility, fine_tune_checkpoint_type is set based on # from_detection_checkpoint. if train_config.from_detection_checkpoint: train_config.fine_tune_checkpoint_type = 'detection' else: train_config.fine_tune_checkpoint_type = 'classification' var_map = detection_model.restore_map( fine_tune_checkpoint_type=train_config.fine_tune_checkpoint_type, load_all_detection_checkpoint_vars=( train_config.load_all_detection_checkpoint_vars)) available_var_map = (variables_helper. get_variables_available_in_checkpoint( var_map, train_config.fine_tune_checkpoint, include_global_step=False)) init_saver = tf.train.Saver(available_var_map) def initializer_fn(sess): init_saver.restore(sess, train_config.fine_tune_checkpoint) init_fn = initializer_fn slim.learning.train( train_tensor, logdir=train_dir, master=master, is_chief=is_chief, session_config=session_config, startup_delay_steps=train_config.startup_delay_steps, init_fn=init_fn, summary_op=summary_op, number_of_steps=( train_config.num_steps if train_config.num_steps else None), save_summaries_secs=120, sync_optimizer=sync_optimizer, saver=saver)
def evaluate(self, input_fn, checkpoint_path=None): if not tf.train.latest_checkpoint(checkpoint_path): raise ValueError("Could not find trained model at %s" % checkpoint_path) with tf.Graph().as_default(): features, labels = self._get_features_and_labels_from_input_fn( input_fn, ModeKeys.EVAL) spec, model = self._get_model_spec(features, labels, ModeKeys.EVAL) # Track the average loss in default eval_metric_ops = spec.eval_metric_ops or {} if model_fn_lib.LOSS_METRIC_KEY not in eval_metric_ops: loss_metric = tf.metrics.mean(spec.loss) eval_metric_ops[model_fn_lib.LOSS_METRIC_KEY] = loss_metric # Create the real eval op update_ops, eval_dict = _extract_metric_update_ops(eval_metric_ops) update_ops.extend(model._train_ops) eval_op = tf.group(*update_ops) # Also track the global step if tf.GraphKeys.GLOBAL_STEP in eval_dict: raise ValueError( 'Metric with name `global_step` is not allowed, because ' 'Estimator already defines a default metric with the ' 'same name.') eval_dict[tf.GraphKeys.GLOBAL_STEP] = \ tf.train.get_or_create_global_step() # Prepare the session creator. scaffold = tf.train.Scaffold() session_creator = tf.train.ChiefSessionCreator( scaffold=scaffold, checkpoint_dir=checkpoint_path) # Prepare hooks all_hooks = list(spec.evaluation_hooks) or [] final_ops_hook = tf.train.FinalOpsHook(eval_dict) all_hooks.append(final_ops_hook) # Evaluate over dataset self._bridge.connect() try: with tf.train.MonitoredSession(session_creator=session_creator, hooks=all_hooks) as sess: iter_id = 0 while not sess.should_stop(): self._bridge.start(iter_id) logging.debug('after bridge start.') start_time = time.time() sess.run(eval_op) end_time = time.time() metrics.emit_timer(name="iter_timer", value=end_time - start_time, tags={}) logging.debug('after session run.') self._bridge.commit() logging.debug('after bridge commit.') iter_id += 1 finally: self._bridge.terminate() # Print result logging.info('Metrics for iteration %d: %s', iter_id, _dict_to_str(final_ops_hook.final_ops_values)) return final_ops_hook.final_ops_values
def testApplyCompression(self): with tf.Graph().as_default(): with self.cached_session(): compression_hparams = ("name=cifar10_compression," "begin_compression_step=1000," "end_compression_step=120000," "compression_frequency=100," "compression_option=1") compression_op_spec = ( compression_op.CompressionOp.get_default_hparams().parse( compression_hparams)) compressor_spec = ( compression_op.LowRankDecompMatrixCompressor.get_default_hparams() .parse("num_rows=5,num_cols=5,rank=200")) matrix_compressor = compression_op.LowRankDecompMatrixCompressor( spec=compressor_spec) global_step = tf.compat.v1.get_variable("global_step", initializer=30) apply_comp = compression_op.ApplyCompression( scope="default_scope", compression_spec=compression_op_spec, compressor=matrix_compressor, global_step=global_step) # Need to add initial value for a_matrix so that we would know what # to expect back. a_matrix_init = np.outer(np.array([1., 2., 3.]), np.array([4., 5., 6.])) a_matrix = tf.compat.v1.get_variable( "a_matrix", initializer=a_matrix_init.astype(np.float32), dtype=tf.float32) a_matrix_compressed = apply_comp.apply_compression( a_matrix, scope="first_compressor") c = apply_comp._compression_ops[0] a_matrix2 = tf.compat.v1.get_variable( "a_matrix2", initializer=a_matrix_init.astype(np.float32), dtype=tf.float32) _ = apply_comp.apply_compression(a_matrix2, scope="second_compressor") c2 = apply_comp._compression_ops[1] _ = apply_comp.all_update_op() tf.compat.v1.global_variables_initializer().run() _ = a_matrix_compressed.eval() self.assertEqual(c._global_step.eval(), 30) self.assertEqual(c.alpha.eval(), 1.0) self.assertEqual(c2.alpha.eval(), 1.0) self.assertEqual(c._last_alpha_update_step.eval(), -1) self.assertAllEqual( np.array([ np.linalg.norm(c.a_matrix_tfvar.eval()), np.linalg.norm(c.b_matrix_tfvar.eval()), np.linalg.norm(c.c_matrix_tfvar.eval()) ]) > 0, [True, False, False]) self.assertAllEqual( np.all(np.abs(np.linalg.norm(c.a_matrix_tfvar.eval())) < 0.00001), False) self.assertAllEqual( np.all(np.abs(np.linalg.norm(c.b_matrix_tfvar.eval())) < 0.00001), True) self.assertAllEqual( np.all(np.abs(np.linalg.norm(c.c_matrix_tfvar.eval())) < 0.00001), True) tf.compat.v1.assign(global_step, 1001).eval() # apply_comp_update_op.run() apply_comp._all_update_op.run() _ = a_matrix_compressed.eval() self.assertEqual(c._global_step.eval(), 1001) self.assertAlmostEqual(c.alpha.eval(), 0.99) self.assertEqual(c._last_alpha_update_step.eval(), 1001) self.assertAllEqual( np.array([ np.linalg.norm(c.a_matrix_tfvar.eval()), np.linalg.norm(c.b_matrix_tfvar.eval()), np.linalg.norm(c.c_matrix_tfvar.eval()) ]) > 0, [True, True, True]) self.assertAllEqual( np.all(np.abs(np.linalg.norm(c.b_matrix_tfvar.eval())) < 0.00001), False) self.assertAllEqual( np.all(np.abs(np.linalg.norm(c.c_matrix_tfvar.eval())) < 0.00001), False) [b_matrix, c_matrix] = matrix_compressor.static_matrix_compressor(a_matrix_init) self.assertAllEqual( np.linalg.norm(np.abs(b_matrix) - np.abs(c.b_matrix_tfvar.eval())) < 0.00001, True) self.assertAllEqual( np.linalg.norm(np.abs(c_matrix) - np.abs(c.c_matrix_tfvar.eval())) < 0.00001, True) self.assertAllEqual( np.all(np.abs(np.linalg.norm(c.b_matrix_tfvar.eval())) < 0.00001), False) self.assertAllEqual( np.all(np.abs(np.linalg.norm(c.c_matrix_tfvar.eval())) < 0.00001), False) tf.compat.v1.assign(global_step, 1001).eval() apply_comp._all_update_op.run() _ = a_matrix_compressed.eval() self.assertEqual(c._global_step.eval(), 1001) self.assertAlmostEqual(c.alpha.eval(), 0.99) self.assertEqual(c._last_alpha_update_step.eval(), 1001) self.assertAllEqual( np.array([ np.linalg.norm(c.a_matrix_tfvar.eval()), np.linalg.norm(c.b_matrix_tfvar.eval()), np.linalg.norm(c.c_matrix_tfvar.eval()) ]) > 0, [True, True, True]) tf.compat.v1.assign(global_step, 2000).eval() apply_comp._all_update_op.run() _ = a_matrix_compressed.eval() self.assertEqual(c._global_step.eval(), 2000) self.assertAlmostEqual(c.alpha.eval(), 0.98) self.assertAlmostEqual(c2.alpha.eval(), 0.98) self.assertEqual(c._last_alpha_update_step.eval(), 2000) self.assertAllEqual( np.array([ np.linalg.norm(c.a_matrix_tfvar.eval()), np.linalg.norm(c.b_matrix_tfvar.eval()), np.linalg.norm(c.c_matrix_tfvar.eval()) ]) > 0, [True, True, True])
def train(self, input_fn, checkpoint_path=None, save_checkpoint_steps=None, save_checkpoint_secs=None): if self._cluster_spec is not None: device_fn = tf.train.replica_device_setter( worker_device="/job:worker/task:%d" % self._worker_rank, merge_devices=True, cluster=self._cluster_spec) cluster_def = self._cluster_spec.as_cluster_def() local_address = self._cluster_spec.job_tasks('worker')[ self._worker_rank] server = tf.train.Server(tf.train.ClusterSpec( {'local': { 0: local_address }}), job_name='local', task_index=0) target = 'grpc://' + local_address else: device_fn = None cluster_def = None target = None config = tf.ConfigProto(cluster_def=cluster_def) config.inter_op_parallelism_threads = 4 config.intra_op_parallelism_threads = 4 config.experimental.share_session_state_in_clusterspec_propagation \ = True tf.config.set_soft_device_placement(False) with tf.Graph().as_default() as g: with tf.device(device_fn): features, labels = self._get_features_and_labels_from_input_fn( input_fn, ModeKeys.TRAIN) spec, _ = self._get_model_spec(features, labels, ModeKeys.TRAIN) # Explicitly add a Saver if not tf.get_collection(tf.GraphKeys.SAVERS): saver = tf.train.Saver( sharded=True, defer_build=True, save_relative_paths=True) # Must set for portability tf.add_to_collection(tf.GraphKeys.SAVERS, saver) self._bridge.connect() try: with tf.train.MonitoredTrainingSession( master=target, config=config, is_chief=(self._worker_rank == 0), checkpoint_dir=checkpoint_path, save_checkpoint_steps=save_checkpoint_steps, save_checkpoint_secs=save_checkpoint_secs, hooks=spec.training_hooks) as sess: iter_id = 0 while not sess.should_stop(): self._bridge.start(iter_id) logging.debug('after bridge start.') sess.run(spec.train_op, feed_dict={}) logging.debug('after session run.') self._bridge.commit() logging.debug('after bridge commit.') iter_id += 1 if self._cluster_spec is not None: self._cheif_barriar(is_chief=(self._worker_rank == 0)) finally: self._bridge.terminate() return self
def main(unused_argv): # pylint:disable=invalid-name # Reason: # Following variables have their name consider to be invalid by pylint so # we disable the warning. # - Variable that is class del unused_argv use_gaussian_pretrained_model = FLAGS.use_gaussian_pretrained_model gen_ckpt_dir = FLAGS.gen_ckpt_dir inception_ckpt_dir = FLAGS.inception_ckpt_dir # TF init tf.reset_default_graph() # - generative model graph_gan = tf.Graph() with graph_gan.as_default(): sess_gan = tf.Session(graph=graph_gan) if use_gaussian_pretrained_model: saver_gan = tf.train.import_meta_graph( os.path.join(gen_ckpt_dir, '..', 'infer', 'infer.meta')) saver_gan.restore(sess_gan, os.path.join(gen_ckpt_dir, 'model.ckpt')) else: saver_gan = tf.train.import_meta_graph( os.path.join(gen_ckpt_dir, 'infer.meta')) saver_gan.restore(sess_gan, os.path.join(gen_ckpt_dir, 'model.ckpt')) # - classifier (inception) graph_class = tf.Graph() with graph_class.as_default(): sess_class = tf.Session(graph=graph_class) saver_class = tf.train.import_meta_graph( os.path.join(inception_ckpt_dir, 'infer.meta')) saver_class.restore( sess_class, os.path.join(inception_ckpt_dir, 'best_acc-103005')) # Generate: Tensor symbols z = graph_gan.get_tensor_by_name('z:0') G_z = graph_gan.get_tensor_by_name('G_z:0')[:, :, 0] # G_z_spec = graph_gan.get_tensor_by_name('G_z_spec:0') # Classification: Tensor symbols x = graph_class.get_tensor_by_name('x:0') scores = graph_class.get_tensor_by_name('scores:0') # Sample something AND classify them output_dir = FLAGS.latent_dir tf.gfile.MakeDirs(output_dir) np.random.seed(19260817) total_per_label = FLAGS.total_per_label top_per_label = FLAGS.top_per_label group_by_label = [[] for _ in range(10)] batch_size = 200 hidden_dim = 100 with tqdm(desc='min label count', unit=' #', total=total_per_label) as pbar: label_count = [0] * 10 last_min_label_count = 0 while True: min_label_count = min(label_count) pbar.update(min_label_count - last_min_label_count) last_min_label_count = min_label_count if use_gaussian_pretrained_model: _z = np.random.randn(batch_size, hidden_dim) else: _z = (np.random.rand(batch_size, hidden_dim) * 2.) - 1. # _G_z, _G_z_spec = sess_gan.run([G_z, G_z_spec], {z: _z}) _G_z = sess_gan.run(G_z, {z: _z}) _x = _G_z _scores = sess_class.run(scores, {x: _x}) _max_scores = np.max(_scores, axis=1) _labels = np.argmax(_scores, axis=1) for i in range(batch_size): label = _labels[i] group_by_label[label].append((_max_scores[i], (_z[i], _G_z[i]))) label_count[label] += 1 if len(group_by_label[label]) >= top_per_label * 2: # remove unneeded tails group_by_label[label].sort(key=operator.itemgetter(0), reverse=True) group_by_label[label] = group_by_label[label][:top_per_label] if last_min_label_count >= total_per_label: break for label in range(10): group_by_label[label].sort(key=operator.itemgetter(0), reverse=True) group_by_label[label] = group_by_label[label][:top_per_label] # output a few samples as image image_output_dir = os.path.join(output_dir, 'sample_iamge') tf.gfile.MakeDirs(image_output_dir) for label in range(10): group_by_label[label].sort(key=operator.itemgetter(0), reverse=True) index = 0 for confidence, ( _, this_G_z, ) in group_by_label[label][:10]: output_basename = 'predlabel=%d_index=%02d_confidence=%.6f' % ( label, index, confidence) wavfile.write( filename=os.path.join( image_output_dir, output_basename + '_sound.wav'), rate=16000, data=this_G_z) # Make Numpy arrays and save everything as an npz file array_label, array_z, array_G_z = [], [], [] for label in range(10): for _, blob in group_by_label[label]: this_z, this_G_z = blob[:2] array_label.append(label) array_z.append(this_z) array_G_z.append(this_G_z) array_label = np.array(array_label, dtype='i') array_z = np.array(array_z) array_G_z = np.array(array_G_z) np.savez( os.path.join(output_dir, 'data_train.npz'), label=array_label, z=array_z, G_z=array_G_z, )