def higher_order_tensors(step): # We're not limited to passing scalar tensors to the summary # operation. If we pass a rank-1 or rank-2 tensor, it'll be visualized # as a table in TensorBoard. (For higher-ranked tensors, you'll see # just a 2D slice of the data.) # # To demonstrate this, let's create a multiplication table. # First, we'll create the table body, a `step`-by-`step` array of # strings. numbers = tf.range(step) numbers_row = tf.expand_dims(numbers, 0) # shape: [1, step] numbers_column = tf.expand_dims(numbers, 1) # shape: [step, 1] products = tf.matmul(numbers_column, numbers_row) # shape: [step, step] table_body = tf.as_string(products) # Next, we'll create a header row and column, and a little # multiplication sign to put in the corner. bold_numbers = tf.string_join(['**', tf.as_string(numbers), '**']) bold_row = tf.expand_dims(bold_numbers, 0) bold_column = tf.expand_dims(bold_numbers, 1) corner_cell = tf.constant(u'\u00d7'.encode('utf-8')) # MULTIPLICATION SIGN # Now, we have to put the pieces together. Using `axis=0` stacks # vertically; using `axis=1` juxtaposes horizontally. table_body_and_top_row = tf.concat([bold_row, table_body], axis=0) table_left_column = tf.concat([[[corner_cell]], bold_column], axis=0) table_full = tf.concat([table_left_column, table_body_and_top_row], axis=1) # The result, `table_full`, is a rank-2 string tensor of shape # `[step + 1, step + 1]`. We can pass it directly to the summary, and # we'll get a nicely formatted table in TensorBoard. tf.summary.text('multiplication_table', table_full)
def make_status_message(model): """Makes a string `Tensor` of training status.""" return tf.string_join( [ 'Starting train step: current_image_id: ', tf.as_string(model.current_image_id), ', progress: ', tf.as_string(model.progress), ', num_blocks: {}'.format( model.num_blocks), ', batch_size: {}'.format(model.batch_size) ], name='status_message')
def testFloat(self): float_inputs_ = [0, 1, -1, 0.5, 0.25, 0.125, float("INF"), float("NAN"), float("-INF")] with self.test_session(): for dtype in (tf.float32, tf.float64): input_ = tf.placeholder(dtype) output = tf.as_string(input_, shortest=True) result = output.eval(feed_dict={input_: float_inputs_}) s = lambda strs: [x.decode("ascii") for x in strs] self.assertAllEqual(s(result), ["%g" % x for x in float_inputs_]) output = tf.as_string(input_, scientific=True) result = output.eval(feed_dict={input_: float_inputs_}) self.assertAllEqual(s(result), ["%e" % x for x in float_inputs_]) output = tf.as_string(input_) result = output.eval(feed_dict={input_: float_inputs_}) self.assertAllEqual(s(result), ["%f" % x for x in float_inputs_]) output = tf.as_string(input_, width=3) result = output.eval(feed_dict={input_: float_inputs_}) self.assertAllEqual(s(result), ["%3f" % x for x in float_inputs_]) output = tf.as_string(input_, width=3, fill="0") result = output.eval(feed_dict={input_: float_inputs_}) self.assertAllEqual(s(result), ["%03f" % x for x in float_inputs_]) output = tf.as_string(input_, width=3, fill="0", shortest=True) result = output.eval(feed_dict={input_: float_inputs_}) self.assertAllEqual(s(result), ["%03g" % x for x in float_inputs_]) output = tf.as_string(input_, precision=10, width=3) result = output.eval(feed_dict={input_: float_inputs_}) self.assertAllEqual(s(result), ["%03.10f" % x for x in float_inputs_]) output = tf.as_string(input_, precision=10, width=3, fill="0", shortest=True) result = output.eval(feed_dict={input_: float_inputs_}) self.assertAllEqual(s(result), ["%03.10g" % x for x in float_inputs_]) with self.assertRaisesOpError("Cannot select both"): output = tf.as_string(input_, scientific=True, shortest=True) output.eval(feed_dict={input_: float_inputs_}) with self.assertRaisesOpError("Fill string must be one or fewer"): output = tf.as_string(input_, fill="ab") output.eval(feed_dict={input_: float_inputs_})
def op(name, images, max_outputs=3, display_name=None, description=None, collections=None): """Create an image summary op for use in a TensorFlow graph. Arguments: name: A unique name for the generated summary node. images: A `Tensor` representing pixel data with shape `[k, w, h, c]`, where `k` is the number of images, `w` and `h` are the width and height of the images, and `c` is the number of channels, which should be 1, 3, or 4. Any of the dimensions may be statically unknown (i.e., `None`). max_outputs: Optional `int` or rank-0 integer `Tensor`. At most this many images will be emitted at each step. When more than `max_outputs` many images are provided, the first `max_outputs` many images will be used and the rest silently discarded. display_name: Optional name for this summary in TensorBoard, as a constant `str`. Defaults to `name`. description: Optional long-form description for this summary, as a constant `str`. Markdown is supported. Defaults to empty. collections: Optional list of graph collections keys. The new summary op is added to these collections. Defaults to `[Graph Keys.SUMMARIES]`. Returns: A TensorFlow summary op. """ if display_name is None: display_name = name summary_metadata = metadata.create_summary_metadata( display_name=display_name, description=description) with tf.name_scope(name), \ tf.control_dependencies([tf.assert_rank(images, 4), tf.assert_type(images, tf.uint8), tf.assert_non_negative(max_outputs)]): limited_images = images[:max_outputs] encoded_images = tf.map_fn(tf.image.encode_png, limited_images, dtype=tf.string, name='encode_each_image') image_shape = tf.shape(images) dimensions = tf.stack([tf.as_string(image_shape[1], name='width'), tf.as_string(image_shape[2], name='height')], name='dimensions') tensor = tf.concat([dimensions, encoded_images], axis=0) return tf.summary.tensor_summary(name='image_summary', tensor=tensor, collections=collections, summary_metadata=summary_metadata)
def simple_example(step): # Text summaries log arbitrary text. This can be encoded with ASCII or # UTF-8. Here's a simple example, wherein we greet the user on each # step: step_string = tf.as_string(step) greeting = tf.string_join(['Hello from step ', step_string, '!']) tf.summary.text('greeting', greeting)
def main(_): path_to_image_file = FLAGS.image path_to_restore_checkpoint_file = FLAGS.restore_checkpoint image = tf.image.decode_jpeg(tf.read_file(path_to_image_file), channels=3) image = tf.reshape(image, [64, 64, 3]) image = tf.image.convert_image_dtype(image, dtype=tf.float32) image = tf.multiply(tf.subtract(image, 0.5), 2) image = tf.image.resize_images(image, [54, 54]) images = tf.reshape(image, [1, 54, 54, 3]) length_logits, digits_logits = Model.inference(images, drop_rate=0.0) length_predictions = tf.argmax(length_logits, axis=1) digits_predictions = tf.argmax(digits_logits, axis=2) digits_predictions_string = tf.reduce_join(tf.as_string(digits_predictions), axis=1) with tf.Session() as sess: restorer = tf.train.Saver() restorer.restore(sess, path_to_restore_checkpoint_file) length_predictions_val, digits_predictions_string_val = sess.run([length_predictions, digits_predictions_string]) length_prediction_val = length_predictions_val[0] digits_prediction_string_val = digits_predictions_string_val[0] print 'length: %d' % length_prediction_val print 'digits: %s' % digits_prediction_string_val
def generate_run(self, run_name, include_graph): """Create a run with a text summary, metadata, and optionally a graph.""" tf.reset_default_graph() k1 = tf.constant(math.pi, name='k1') k2 = tf.constant(math.e, name='k2') result = (k1 ** k2) - k1 expected = tf.constant(20.0, name='expected') error = tf.abs(result - expected, name='error') message_prefix_value = 'error ' * 1000 true_length = len(message_prefix_value) assert true_length > self._MESSAGE_PREFIX_LENGTH_LOWER_BOUND, true_length message_prefix = tf.constant(message_prefix_value, name='message_prefix') error_message = tf.string_join([message_prefix, tf.as_string(error, name='error_string')], name='error_message') summary_message = tf.summary.text('summary_message', error_message) sess = tf.Session() writer = tf.summary.FileWriter(os.path.join(self.logdir, run_name)) if include_graph: writer.add_graph(sess.graph) options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) run_metadata = tf.RunMetadata() s = sess.run(summary_message, options=options, run_metadata=run_metadata) writer.add_summary(s) writer.add_run_metadata(run_metadata, self._METADATA_TAG) writer.close()
def wrapped_preprocessing_fn(inputs): outputs = preprocessing.preprocessing_fn(inputs) for key in outputs: if outputs[key].dtype == tf.bool: outputs[key] = tft.string_to_int(tf.as_string(outputs[key]), vocab_filename='vocab_' + key) return outputs
def _read_from_disk_temporal( fpath, nframes, num_samples=25, optical_flow_frames=10, start_frame=0, file_prefix='', file_zero_padding=4, file_index=1, dataset_dir='', step=None): duration = nframes if step is None: if num_samples == 1: step = tf.random_uniform([1], 0, nframes-optical_flow_frames-1, dtype='int32')[0] else: step = tf.cast((duration-tf.constant(optical_flow_frames)) / (tf.constant(num_samples)), 'int32') allimgs = [] with tf.variable_scope('read_flow_video'): for i in range(num_samples): if num_samples == 1: i = 1 # so that the random step value can be used with tf.variable_scope('read_flow_image'): flow_img = [] for j in range(optical_flow_frames): with tf.variable_scope('read_flow_channels'): for dr in ['x', 'y']: prefix = file_prefix + '_' if file_prefix else '' impath = tf.string_join([ tf.constant(dataset_dir + '/'), fpath, tf.constant('/'), prefix, '%s_' % dr, tf.as_string(start_frame + i * step + file_index + j, width=file_zero_padding, fill='0'), tf.constant('.jpg')]) img_str = tf.read_file(impath) flow_img.append(img_str) allimgs.append(flow_img) return allimgs
def _read_from_disk_spatial(fpath, nframes, num_samples=25, start_frame=0, file_prefix='', file_zero_padding=4, file_index=1, dataset_dir='', step=None): duration = nframes if step is None: if num_samples == 1: step = tf.random_uniform([1], 0, nframes, dtype='int32')[0] else: step = tf.cast((duration-tf.constant(1)) / (tf.constant(num_samples-1)), 'int32') allimgs = [] with tf.variable_scope('read_rgb_video'): for i in range(num_samples): if num_samples == 1: i = 1 # so that the random step value can be used with tf.variable_scope('read_rgb_image'): prefix = file_prefix + '_' if file_prefix else '' impath = tf.string_join([ tf.constant(dataset_dir + '/'), fpath, tf.constant('/'), prefix, tf.as_string(start_frame + i * step + file_index, width=file_zero_padding, fill='0'), tf.constant('.jpg')]) img_str = tf.read_file(impath) allimgs.append(img_str) return allimgs
def _replace_empty_string_with_random_number(string_tensor): """Returns string unchanged if non-empty, and random string tensor otherwise. The random string is an integer 0 and 2**63 - 1, casted as string. Args: string_tensor: A tf.tensor of dtype string. Returns: out_string: A tf.tensor of dtype string. If string_tensor contains the empty string, out_string will contain a random integer casted to a string. Otherwise string_tensor is returned unchanged. """ empty_string = tf.constant('', dtype=tf.string, name='EmptyString') random_source_id = tf.as_string( tf.random_uniform(shape=[], maxval=2**63 - 1, dtype=tf.int64)) out_string = tf.cond( tf.equal(string_tensor, empty_string), true_fn=lambda: random_source_id, false_fn=lambda: string_tensor) return out_string
def testStateSaverScopeNames(self): batch_size = tf.constant(2) sqss_scope_name = "unique_scope_name_for_sqss" num_unroll = 2 length = 3 key = tf.string_join(["key_", tf.as_string(tf.cast( 10000 * tf.random_uniform(()), tf.int32))]) padded_length = 4 sequences = {"seq1": np.random.rand(padded_length, 5), "seq2": np.random.rand(padded_length, 4, 2)} context = {"context1": [3, 4]} initial_states = {"state1": np.random.rand(6, 7), "state2": np.random.rand(8)} state_saver = tf.contrib.training.SequenceQueueingStateSaver( batch_size=batch_size, num_unroll=num_unroll, input_length=length, input_key=key, input_sequences=sequences, input_context=context, initial_states=initial_states, name=sqss_scope_name) prefetch_op = state_saver.prefetch_op next_batch = state_saver.next_batch self.assertTrue(state_saver.barrier.barrier_ref.name.startswith( "%s/" % sqss_scope_name)) self.assertTrue(prefetch_op.name.startswith("%s/" % sqss_scope_name)) self.assertTrue(next_batch.key.name.startswith("%s/" % sqss_scope_name))
def testLargeInt(self): # Cannot use values outside -128..127 for test, because we're also # testing int8 s = lambda strs: [x.decode("ascii") for x in strs] with self.test_session(): input_ = tf.placeholder(tf.int32) int_inputs_ = [np.iinfo(np.int32).min, np.iinfo(np.int32).max] output = tf.as_string(input_) result = output.eval(feed_dict={input_: int_inputs_}) self.assertAllEqual(s(result), ["%d" % x for x in int_inputs_]) input_ = tf.placeholder(tf.int64) int_inputs_ = [np.iinfo(np.int64).min, np.iinfo(np.int64).max] output = tf.as_string(input_) result = output.eval(feed_dict={input_: int_inputs_}) self.assertAllEqual(s(result), ["%d" % x for x in int_inputs_])
def testBool(self): bool_inputs_ = [False, True] s = lambda strs: [x.decode("ascii") for x in strs] with self.test_session(): for dtype in (tf.bool,): input_ = tf.placeholder(dtype) output = tf.as_string(input_) result = output.eval(feed_dict={input_: bool_inputs_}) self.assertAllEqual(s(result), ["false", "true"])
def setUp(self): super(BatchSequencesWithStatesTest, self).setUp() self.value_length = 4 self.batch_size = 2 self.key = tf.string_join(["key_", tf.as_string(tf.cast( 10000 * tf.random_uniform(()), tf.int32))]) self.sequences = {"seq1": np.random.rand(self.value_length, 5), "seq2": np.random.rand(self.value_length, 4, 2)} self.context = {"context1": [3, 4]} self.initial_states = {"state1": np.random.rand(6, 7), "state2": np.random.rand(8)}
def main(_): # Create the log_dir if not exist. if not tf.gfile.Exists(FLAGS.train_log_dir): tf.gfile.MakeDirs(FLAGS.train_log_dir) # Shard the model to different parameter servers. with tf.device(tf.train.replica_device_setter(FLAGS.ps_tasks)): # Create the input dataset. with tf.name_scope('inputs'): images, labels = data_provider.provide_data( FLAGS.image_file_patterns, FLAGS.batch_size, FLAGS.patch_size) # Define the model. with tf.name_scope('model'): model = _define_model(images, labels) # Add image summary. tfgan.eval.add_stargan_image_summaries( model, num_images=len(FLAGS.image_file_patterns) * FLAGS.batch_size, display_diffs=True) # Define the model loss. loss = tfgan.stargan_loss(model) # Define the train ops. with tf.name_scope('train_ops'): train_ops = _define_train_ops(model, loss) # Define the train steps. train_steps = _define_train_step() # Define a status message. status_message = tf.string_join( [ 'Starting train step: ', tf.as_string(tf.train.get_or_create_global_step()) ], name='status_message') # Train the model. tfgan.gan_train( train_ops, FLAGS.train_log_dir, get_hooks_fn=tfgan.get_sequential_train_hooks(train_steps), hooks=[ tf.train.StopAtStepHook(num_steps=FLAGS.max_number_of_steps), tf.train.LoggingTensorHook([status_message], every_n_iter=10) ], master=FLAGS.master, is_chief=FLAGS.task == 0)
def testInt(self): # Cannot use values outside -128..127 for test, because we're also # testing int8 int_inputs_ = [0, -1, 1, -128, 127, -101, 101, -0] s = lambda strs: [x.decode("ascii") for x in strs] with self.test_session(): for dtype in (tf.int32, tf.int64, tf.int8): input_ = tf.placeholder(dtype) output = tf.as_string(input_) result = output.eval(feed_dict={input_: int_inputs_}) self.assertAllEqual(s(result), ["%d" % x for x in int_inputs_]) output = tf.as_string(input_, width=3) result = output.eval(feed_dict={input_: int_inputs_}) self.assertAllEqual(s(result), ["%3d" % x for x in int_inputs_]) output = tf.as_string(input_, width=3, fill="0") result = output.eval(feed_dict={input_: int_inputs_}) self.assertAllEqual(s(result), ["%03d" % x for x in int_inputs_]) with self.assertRaisesOpError("scientific and shortest"): output = tf.as_string(input_, scientific=True) output.eval(feed_dict={input_: int_inputs_}) with self.assertRaisesOpError("scientific and shortest"): output = tf.as_string(input_, shortest=True) output.eval(feed_dict={input_: int_inputs_}) with self.assertRaisesOpError("precision not supported"): output = tf.as_string(input_, precision=0) output.eval(feed_dict={input_: int_inputs_})
def markdown_table(step): # The text summary can also contain Markdown, including Markdown # tables. Markdown tables look like this: # # | hello | there | # |-------|-------| # | this | is | # | a | table | # # The leading and trailing pipes in each row are optional, and the text # doesn't actually have to be neatly aligned, so we can create these # pretty easily. Let's do so. header_row = 'Pounds of chocolate | Happiness' chocolate = tf.range(step) happiness = tf.square(chocolate + 1) chocolate_column = tf.as_string(chocolate) happiness_column = tf.as_string(happiness) table_rows = tf.string_join([chocolate_column, " | ", happiness_column]) table_body = tf.reduce_join(table_rows, separator='\n') table = tf.string_join([header_row, "---|---", table_body], separator='\n') preamble = 'We conducted an experiment and found the following data:\n\n' result = tf.string_join([preamble, table]) tf.summary.text('chocolate_study', result)
def main(_): if not tf.gfile.Exists(FLAGS.train_log_dir): tf.gfile.MakeDirs(FLAGS.train_log_dir) with tf.device(tf.train.replica_device_setter(FLAGS.ps_tasks)): with tf.name_scope('inputs'): images_x, images_y = data_provider.provide_custom_data( [FLAGS.image_set_x_file_pattern, FLAGS.image_set_y_file_pattern], batch_size=FLAGS.batch_size, patch_size=FLAGS.patch_size) # Set batch size for summaries. images_x.set_shape([FLAGS.batch_size, None, None, None]) images_y.set_shape([FLAGS.batch_size, None, None, None]) # Define CycleGAN model. cyclegan_model = _define_model(images_x, images_y) # Define CycleGAN loss. cyclegan_loss = tfgan.cyclegan_loss( cyclegan_model, cycle_consistency_loss_weight=FLAGS.cycle_consistency_loss_weight, tensor_pool_fn=tfgan.features.tensor_pool) # Define CycleGAN train ops. train_ops = _define_train_ops(cyclegan_model, cyclegan_loss) # Training train_steps = tfgan.GANTrainSteps(1, 1) status_message = tf.string_join( [ 'Starting train step: ', tf.as_string(tf.train.get_or_create_global_step()) ], name='status_message') if not FLAGS.max_number_of_steps: return tfgan.gan_train( train_ops, FLAGS.train_log_dir, get_hooks_fn=tfgan.get_sequential_train_hooks(train_steps), hooks=[ tf.train.StopAtStepHook(num_steps=FLAGS.max_number_of_steps), tf.train.LoggingTensorHook([status_message], every_n_iter=10) ], master=FLAGS.master, is_chief=FLAGS.task == 0)
def scaled_dot_product_attention(q, k, v, mask, collec): """Calculate the attention weights. q, k, v must have matching leading dimensions. k, v must have matching penultimate dimension, i.e.: seq_len_k = seq_len_v. The mask has different shapes depending on its type(padding or look ahead) but it must be broadcastable for addition. Args: q: query shape == (..., seq_len_q, depth) k: key shape == (..., seq_len_k, depth) v: value shape == (..., seq_len_v, depth_v) mask: Float tensor with shape broadcastable to (..., seq_len_q, seq_len_k). Defaults to None. Returns: output, attention_weights """ # q_length=tf.norm(q,ord=2, axis=3,keepdims=True) # k_length=tf.norm(k,ord=2, axis=3,keepdims=True) matmul_qk = tf.matmul(q, k, transpose_b=True) # (..., seq_len_q, seq_len_k) # matmul_qk=q_length*k_length print() # scale matmul_qk dk = tf.cast(tf.shape(k)[-1], tf.float32) scaled_attention_logits = matmul_qk / tf.math.sqrt(dk) # add the mask to the scaled tensor. if mask is not None: scaled_attention_logits += (mask * -1e9) # softmax is normalized on the last axis (seq_len_k) so that the scores # add up to 1. # if is_training==True: # collect_scope='train' # else: # collect_scope='eval' # print(is_training) select = tf.constant(["train", "eval"]) # collect_scope_sum=tf.convert_to_tensor([collect_scope],tf.) # summary_op = tf.summary.text("is_training", tf.strings.as_string(is_training),collections=['train']) # summary_op1 = tf.summary.text("collect_scope", collect_scope,collections=['train']) def z_norm(a, axis=-1, scale=1): return (a - tf.reduce_mean(a, axis=axis, keepdims=True)) / ( tf.math.reduce_std(a, axis=axis, keepdims=True) * scale + 1e-10) attention_weights = tf.nn.softmax(scaled_attention_logits, axis=3) # (..., seq_len_q, seq_len_k) d_seq = tf.cast(tf.shape(k)[-2], tf.float32) # scaled_attention_weights_withinq=tf.nn.softmax(tf.reduce_sum(scaled_attention_logits,axis=-1,keep_dims=True),axis=2)/ d_seq # scaled_attention_weights_betwq=tf.nn.softmax(tf.reduce_sum(scaled_attention_logits,axis=[2,3],keep_dims=True),axis=1)/ tf.math.square(d_seq) # withinqbatch_k=tf.layers.BatchNormalization(2) # betwqbatch_q=tf.layers.BatchNormalization(1) last_dim = tf.reduce_mean(scaled_attention_logits, axis=-1, keep_dims=True) last2_dim = tf.reduce_mean(scaled_attention_logits, axis=[2, 3], keep_dims=True) scaled_attention_weights_withinq = tf.nn.softmax(z_norm(last_dim, 2, 2), axis=2) scaled_attention_weights_betwq = tf.nn.softmax(z_norm(last2_dim, 1, 2), axis=1) print(scaled_attention_weights_withinq.get_shape()\ ,"scaled_attention_weights_withinq.get_shape(),") weights_different_entry = tf.expand_dims(attention_weights[0, :, :, :], 3) weights_different_q = tf.expand_dims( scaled_attention_weights_withinq[:, :, :, 0], 3) weights_different_Heads = tf.expand_dims( scaled_attention_weights_betwq[:, :, :, 0], 0) # attention_weights_horizontal=tf.nn.softmax(tf.reduce_sum(scaled_attention_logits,axis=[2],keep_dims=True),axis=1) # scaled_attention_weights_betwq_summary=tf.reduce_mean(attention_weights,axis=0) # scaled_attention_weights_withinq_1st=tf.expand_dims(attention_weights[0,:,:,:],axis=3) # attention_weights_horizontal=tf.reduce_sum(attention_weights,axis=2) # attention_weights_horizontal_row=tf.expand_dims(attention_weights_horizontal,axis=3) # scaled_attention_weights_betwq_1st=tf.expand_dims(scaled_attention_weights_betwq[:,:,:,0],axis=0) # def for_train(): # tf.summary.histogram("train scaled_attention_weights_betwq_summary",scaled_attention_weights_betwq_summary,collections=collec) tf.summary.image("weights_different_entry_[H,S,S,1]",\ weights_different_entry,max_outputs=8,collections=collec) tf.summary.image("weights_different_q_[B,H,S,1]",\ weights_different_q,max_outputs=8,collections=collec) tf.summary.image("weights_different_Heads_[1,B,H,1]",\ weights_different_Heads,max_outputs=8,collections=collec) tf.summary.text("weights_first_portion_Heads_[1,B,H,1]",\ tf.as_string(weights_different_Heads[0,1,:,0]),collections=collec) tf.summary.tensor_summary('weights_first_portion_Heads_[1,B,H,1]',\ weights_different_Heads[0,1,:,0],collections=collec) # tf.summary.image("[]",attention_weights_horizontal_row,max_outputs=8,collections=collec) # return True # def for_eval(): # tf.summary.histogram("evalscaled_attention_weights_betwq_summary",scaled_attention_weights_betwq_summary,collections=['eval']) # tf.summary.image("evalscaled_attention_weights_withinq",scaled_attention_weights_withinq_1st,max_outputs=8,collections=['eval']) # tf.summary.image("evalscaled_attention_weights_betwq",scaled_attention_weights_betwq_1st,max_outputs=8,collections=['eval']) # tf.summary.image("evalattention_weights_horizontal_row",attention_weights_horizontal_row,max_outputs=8,collections=['eval']) # return True # tf.cond(is_training,for_train,for_eval) # for i in range(tf.shape(k)[1]): # tf.summary.scalar('attention_heads'+str(i), scaled_attention_weights_betwq_summary[i], collections=[collect_scope]) # attention_weights=attention_weights*scaled_attention_weights_withinq # attention_weights=attention_weights*scaled_attention_weights_betwq output = tf.matmul(attention_weights, v) # (..., seq_len_q, depth_v) return output, attention_weights
def run(logdir, run_name, wave_name, wave_constructor): """Generate wave data of the given form. The provided function `wave_constructor` should accept a scalar tensor of type float32, representing the frequency (in Hz) at which to construct a wave, and return a tensor of shape [1, _samples(), `n`] representing audio data (for some number of channels `n`). Waves will be generated at frequencies ranging from A4 to A5. Arguments: logdir: the top-level directory into which to write summary data run_name: the name of this run; will be created as a subdirectory under logdir wave_name: the name of the wave being generated wave_constructor: see above """ tf.reset_default_graph() tf.set_random_seed(0) # On each step `i`, we'll set this placeholder to `i`. This allows us # to know "what time it is" at each step. step_placeholder = tf.placeholder(tf.float32, shape=[]) # We want to linearly interpolate a frequency between A4 (440 Hz) and # A5 (880 Hz). with tf.name_scope('compute_frequency'): f_min = 440.0 f_max = 880.0 t = step_placeholder / (FLAGS.steps - 1) frequency = f_min * (1.0 - t) + f_max * t # Let's log this frequency, just so that we can make sure that it's as # expected. tf.summary.scalar('frequency', frequency) # Now, we pass this to the wave constructor to get our waveform. Doing # so within a name scope means that any summaries that the wave # constructor produces will be namespaced. with tf.name_scope(wave_name): waveform = wave_constructor(frequency) # We also have the opportunity to annotate each audio clip with a # label. This is a good place to include the frequency, because it'll # be visible immediately next to the audio clip. with tf.name_scope('compute_labels'): samples = tf.shape(waveform)[0] wave_types = tf.tile(["*Wave type:* `%s`." % wave_name], [samples]) frequencies = tf.string_join([ "*Frequency:* ", tf.tile([tf.as_string(frequency, precision=2)], [samples]), " Hz.", ]) samples = tf.string_join([ "*Sample:* ", tf.as_string(tf.range(samples) + 1), " of ", tf.as_string(samples), ".", ]) labels = tf.string_join([wave_types, frequencies, samples], separator=" ") # We can place a description next to the summary in TensorBoard. This # is a good place to explain what the summary represents, methodology # for creating it, etc. Let's include the source code of the function # that generated the wave. source = '\n'.join(' %s' % line.rstrip() for line in inspect.getsourcelines(wave_constructor)[0]) description = ("A wave of type `%r`, generated via:\n\n%s" % (wave_name, source)) # Here's the crucial piece: we interpret this result as audio. summary.op('waveform', waveform, FLAGS.sample_rate, labels=labels, display_name=wave_name, description=description) # Now, we can collect up all the summaries and begin the run. summ = tf.summary.merge_all() sess = tf.Session() writer = tf.summary.FileWriter(os.path.join(logdir, run_name)) writer.add_graph(sess.graph) sess.run(tf.global_variables_initializer()) for step in xrange(FLAGS.steps): s = sess.run(summ, feed_dict={step_placeholder: float(step)}) writer.add_summary(s, global_step=step) writer.close()
rnn = tf.keras.layers.RNN(cell) semantics = rnn(embeddedTokens) prediction = tf.keras.layers.Dense(1, activation='sigmoid')(semantics) lossOp = tf.losses.mean_squared_error(label, prediction) minimizeLossOp = tf.train.AdamOptimizer(0.005).minimize(lossOp) tf.summary.histogram("embeddings", embeddings) tf.summary.histogram("embeddedTokens", embeddedTokens) tf.summary.histogram("semantics", semantics) tf.summary.scalar("loss", lossOp) predictionsViz = tf.concat( [text[:10], tf.as_string(label[:10]), tf.as_string(prediction[:10])], axis=1) tf.summary.text("predictions", predictionsViz) summaryOp = tf.summary.merge_all() with tf.Session() as sess: now = datetime.now().isoformat() writerTrain = tf.summary.FileWriter("logs/{}/train".format(now), sess.graph) writerTest = tf.summary.FileWriter("logs/{}/test".format(now), sess.graph) sess.run( [tf.global_variables_initializer(), tf.local_variables_initializer()]) epochs = 10 runnr = 1
def main(_): if not tf.gfile.Exists(FLAGS.train_log_dir): tf.gfile.MakeDirs(FLAGS.train_log_dir) # Force all input processing onto CPU in order to reserve the GPU for # the forward inference and back-propagation. with tf.name_scope('inputs'): with tf.device('/cpu:0'): images, one_hot_labels, _ = data_provider.provide_data( 'train', FLAGS.batch_size, FLAGS.dataset_dir, num_threads=4) generator_fn = networks.unconditional_generator noise_fn = tf.random_normal([FLAGS.batch_size, FLAGS.noise_dims]) gan_model = tfgan.gan_model( generator_fn=generator_fn, discriminator_fn=networks.unconditional_discriminator, real_data=images, generator_inputs=noise_fn) tfgan.eval.add_gan_model_image_summaries(gan_model, FLAGS.grid_size) # Get the GANLoss tuple. You can pass a custom function, use one of the # already-implemented losses from the losses library, or use the defaults. with tf.name_scope('loss'): gan_loss = tfgan.gan_loss(gan_model, gradient_penalty_weight=1.0, mutual_information_penalty_weight=0.0, add_summaries=True) tfgan.eval.add_regularization_loss_summaries(gan_model) # Get the GANTrain ops using custom optimizers. with tf.name_scope('train'): gen_lr, dis_lr = _learning_rate(FLAGS.gan_type) train_ops = tfgan.gan_train_ops( gan_model, gan_loss, generator_optimizer=tf.train.AdamOptimizer(gen_lr, 0.5), discriminator_optimizer=tf.train.AdamOptimizer(dis_lr, 0.5), summarize_gradients=True, aggregation_method=tf.AggregationMethod.EXPERIMENTAL_ACCUMULATE_N) # Run the alternating training loop. Skip it if no steps should be taken # (used for graph construction tests). status_message = tf.string_join([ 'Starting train step: ', tf.as_string(tf.train.get_or_create_global_step()) ], name='status_message') if FLAGS.max_number_of_steps == 0: return gan_plotter_hook = PlotGanImageHook(gan_model=gan_model, path=os.path.join( os.sep, "tmp", "gan_output"), every_n_iter=100, batch_size=FLAGS.batch_size) tfgan.gan_train( train_ops, hooks=[ gan_plotter_hook, tf.train.StopAtStepHook(num_steps=FLAGS.max_number_of_steps), tf.train.LoggingTensorHook([status_message], every_n_iter=100) ], logdir=FLAGS.train_log_dir, get_hooks_fn=tfgan.get_joint_train_hooks())
def main(_): if not tf.gfile.Exists(FLAGS.train_log_dir): tf.gfile.MakeDirs(FLAGS.train_log_dir) with tf.device(tf.train.replica_device_setter(FLAGS.ps_tasks)): # Put input pipeline on CPU to reserve GPU for training. with tf.name_scope('inputs'), tf.device('/cpu:0'): images = data_provider.provide_data('train', FLAGS.batch_size, dataset_dir=FLAGS.dataset_dir, patch_size=FLAGS.patch_size) # Manually define a GANModel tuple. This is useful when we have custom # code to track variables. Note that we could replace all of this with a # call to `tfgan.gan_model`, but we don't in order to demonstrate some of # TFGAN's flexibility. with tf.variable_scope('generator') as gen_scope: reconstructions, _, prebinary = networks.compression_model( images, num_bits=FLAGS.bits_per_patch, depth=FLAGS.model_depth) gan_model = _get_gan_model(generator_inputs=images, generated_data=reconstructions, real_data=images, generator_scope=gen_scope) summaries.add_reconstruction_summaries(images, reconstructions, prebinary) tfgan.eval.add_gan_model_summaries(gan_model) # Define the GANLoss tuple using standard library functions. with tf.name_scope('loss'): gan_loss = tfgan.gan_loss( gan_model, generator_loss_fn=tfgan.losses.least_squares_generator_loss, discriminator_loss_fn=tfgan.losses. least_squares_discriminator_loss, add_summaries=FLAGS.weight_factor > 0) # Define the standard pixel loss. l1_pixel_loss = tf.norm(gan_model.real_data - gan_model.generated_data, ord=1) # Modify the loss tuple to include the pixel loss. Add summaries as well. gan_loss = tfgan.losses.combine_adversarial_loss( gan_loss, gan_model, l1_pixel_loss, weight_factor=FLAGS.weight_factor) # Get the GANTrain ops using the custom optimizers and optional # discriminator weight clipping. with tf.name_scope('train_ops'): gen_lr, dis_lr = _lr(FLAGS.generator_lr, FLAGS.discriminator_lr) gen_opt, dis_opt = _optimizer(gen_lr, dis_lr) train_ops = tfgan.gan_train_ops( gan_model, gan_loss, generator_optimizer=gen_opt, discriminator_optimizer=dis_opt, summarize_gradients=True, colocate_gradients_with_ops=True, aggregation_method=tf.AggregationMethod. EXPERIMENTAL_ACCUMULATE_N) tf.summary.scalar('generator_lr', gen_lr) tf.summary.scalar('discriminator_lr', dis_lr) # Determine the number of generator vs discriminator steps. train_steps = tfgan.GANTrainSteps( generator_train_steps=1, discriminator_train_steps=int(FLAGS.weight_factor > 0)) # Run the alternating training loop. Skip it if no steps should be taken # (used for graph construction tests). status_message = tf.string_join([ 'Starting train step: ', tf.as_string(tf.train.get_or_create_global_step()) ], name='status_message') if FLAGS.max_number_of_steps == 0: return tfgan.gan_train( train_ops, FLAGS.train_log_dir, tfgan.get_sequential_train_hooks(train_steps), hooks=[ tf.train.StopAtStepHook(num_steps=FLAGS.max_number_of_steps), tf.train.LoggingTensorHook([status_message], every_n_iter=10) ], master=FLAGS.master, is_chief=FLAGS.task == 0)
def main(_): with tf.Session() as sess: tf.set_random_seed(4285) epochs = 1 batch_size = 4 # must divide dataset size (some strange error occurs if not) image_size = 128 tfrecords_file_in = 'data/val-001-118287.tfrecords' # ''/data/cvg/lukas/datasets/coco/2017_training/tfrecords_l2mix_flip_tile_10-L2nn_4285/' filedir_out = '../logs/test/test_tfrecords_with_tile_10L2nn' tile_filedir_in = '/data/cvg/lukas/datasets/coco/2017_training/clustering_224x224_4285/' tile_filedir_out = '~/results/knn_results/' reader = tf.TFRecordReader() read_fn = lambda name: read_record(name, reader, image_size) filename, train_images, t1_10nn_str, t2_10nn_str, t3_10nn_str, t4_10nn_str, \ t1_10nn_strs, t2_10nn_strs, t3_10nn_strs, t4_10nn_strs = \ get_pipeline(tfrecords_file_in, batch_size, epochs, read_fn) t1_10nn_str = tf.reshape(tf.sparse.to_dense(t1_10nn_str), (batch_size, 10)) t2_10nn_str = tf.reshape(tf.sparse.to_dense(t2_10nn_str), (batch_size, 10)) t3_10nn_str = tf.reshape(tf.sparse.to_dense(t3_10nn_str), (batch_size, 10)) t4_10nn_str = tf.reshape(tf.sparse.to_dense(t4_10nn_str), (batch_size, 10)) t1_10nn_strs = tf.reshape(tf.sparse.to_dense(t1_10nn_strs), (batch_size, 10)) t2_10nn_strs = tf.reshape(tf.sparse.to_dense(t2_10nn_strs), (batch_size, 10)) t3_10nn_strs = tf.reshape(tf.sparse.to_dense(t3_10nn_strs), (batch_size, 10)) t4_10nn_strs = tf.reshape(tf.sparse.to_dense(t4_10nn_strs), (batch_size, 10)) underscore = tf.constant("_") # t1 filetype = tf.constant("_t1.jpg") for nn_id in range(batch_size): t2_gather = tf.gather(t1_10nn_str, nn_id) t2_one_nn = tf.as_string(t2_gather) t2_gathers = tf.gather(t1_10nn_strs, nn_id) t2_one_nns = tf.as_string(t2_gathers) postfix = underscore + t2_one_nns + filetype fname = get_filename(t2_one_nn, postfix) t1_10nn_fnames = fname if nn_id == 0 else tf.concat( axis=0, values=[t1_10nn_fnames, fname]) # t2 filetype = tf.constant("_t2.jpg") for nn_id in range(batch_size): t2_gather = tf.gather(t2_10nn_str, nn_id) t2_one_nn = tf.as_string(t2_gather) t2_gathers = tf.gather(t2_10nn_strs, nn_id) t2_one_nns = tf.as_string(t2_gathers) postfix = underscore + t2_one_nns + filetype fname = get_filename(t2_one_nn, postfix) t2_10nn_fnames = fname if nn_id == 0 else tf.concat( axis=0, values=[t2_10nn_fnames, fname]) # t3 filetype = tf.constant("_t3.jpg") for nn_id in range(batch_size): t2_gather = tf.gather(t3_10nn_str, nn_id) t2_one_nn = tf.as_string(t2_gather) t2_gathers = tf.gather(t3_10nn_strs, nn_id) t2_one_nns = tf.as_string(t2_gathers) postfix = underscore + t2_one_nns + filetype fname = get_filename(t2_one_nn, postfix) t3_10nn_fnames = fname if nn_id == 0 else tf.concat( axis=0, values=[t3_10nn_fnames, fname]) # t4 filetype = tf.constant("_t4.jpg") for nn_id in range(batch_size): t2_gather = tf.gather(t4_10nn_str, nn_id) t2_one_nn = tf.as_string(t2_gather) t2_gathers = tf.gather(t4_10nn_strs, nn_id) t2_one_nns = tf.as_string(t2_gathers) postfix = underscore + t2_one_nns + filetype fname = get_filename(t2_one_nn, postfix) t4_10nn_fnames = fname if nn_id == 0 else tf.concat( axis=0, values=[t4_10nn_fnames, fname]) # [('000000000927_1.jpg', 0.03125), ('000000568135_2.jpg', 19095.953), ('000000187857_1.jpg', 23359.39), # ('000000521998_2.jpg', 23557.688), ('000000140816_1.jpg', 24226.852), ('000000015109_1.jpg', 25191.469), # ('000000525567_1.jpg', 25484.93), ('000000377422_1.jpg', 25654.125), ('000000269815_2.jpg', 26794.836), # ('000000345617_2.jpg', 26872.812)] ######################################################################################################## sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer()) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord=coord) max = 5 cnt = 0 try: while not coord.should_stop(): print('t1_10nn_str type: %s' % type(t1_10nn_str)) fns, t_imgs, t1_fns, t2_fns, t3_fns, t4_fns = sess.run([ filename, train_images, t1_10nn_fnames, t2_10nn_fnames, t3_10nn_fnames, t4_10nn_fnames ]) print('fns.shape: %s' % str(fns.shape)) print('t1_fns.shape: %s' % str(t1_fns.shape)) for i in range(batch_size): print('ITERATION [%d] >>>>>>' % i) fname = fns[i].decode("utf-8") t_img = t_imgs[i] name = os.path.join(filedir_out, fname) print('save I_ref to %s...' % name) imsave(name, t_img) f_o = os.path.join(tile_filedir_out, 'I_ref_' + fname) print('cp %s %s' % (name, f_o)) t1_10nn = [e.decode("utf-8") for e in t1_fns[i]] t2_10nn = [e.decode("utf-8") for e in t2_fns[i]] t3_10nn = [e.decode("utf-8") for e in t3_fns[i]] t4_10nn = [e.decode("utf-8") for e in t4_fns[i]] print('I_ref: %s' % fname) print('t1 10-NN:') # % str(t1_10nn)) for j in range(10): t_f = os.path.join(tile_filedir_in, 't1') t_f = os.path.join(t_f, t1_10nn[j]) t_o = os.path.join(tile_filedir_out, 't1', str(j + 1) + '_' + t1_10nn[j]) print('cp %s %s' % (t_f, t_o)) print('-----') print('t2 10-NN:') # %s' % str(t2_10nn)) for j in range(10): t_f = os.path.join(tile_filedir_in, 't2') t_f = os.path.join(t_f, t2_10nn[j]) t_o = os.path.join(tile_filedir_out, 't2', str(j + 1) + '_' + t2_10nn[j]) print('cp %s %s' % (t_f, t_o)) print('-----') print('t3 10-NN:') # %s' % str(t3_10nn)) for j in range(10): t_f = os.path.join(tile_filedir_in, 't3') t_f = os.path.join(t_f, t3_10nn[j]) t_o = os.path.join(tile_filedir_out, 't3', str(j + 1) + '_' + t3_10nn[j]) print('cp %s %s' % (t_f, t_o)) print('-----') print('t4 10-NN:') # %s' % str(t4_10nn)) for j in range(10): t_f = os.path.join(tile_filedir_in, 't4') t_f = os.path.join(t_f, t4_10nn[j]) t_o = os.path.join(tile_filedir_out, 't4', str(j + 1) + '_' + t4_10nn[j]) print('cp %s %s' % (t_f, t_o)) print('-----') print('ITERATION [%d] <<<<<<' % i) cnt = cnt + 1 if cnt >= max: break except Exception as e: if hasattr( e, 'message' ) and 'is closed and has insufficient elements' in e.message: print('Done training -- epoch limit reached') else: print('Exception here, ending training..') tb = traceback.format_exc() print('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>') print(e) print(tb) print('<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<') finally: # When done, ask the threads to stop. coord.request_stop() coord.join(threads)
def make_sort_stage(self, ready_to_sort): """ :param ready_to_sort: :return: (id_and_count, record_id, intermediate_name, superchunk_num_records, superchunk_matrix) + rest_of_input, log_event """ bpp = persona_ops.buffer_pair_pool( size=0, bound=False, name="local_read_sort_buffer_list_pool") self.log.info("order by is '{ob}'".format(ob=self.order_by)) if self.order_by == location_value: self.log.info("sorting by location") sort_op = partial(persona_ops.agd_sort, buffer_pair_pool=bpp, name="agd_sort_results") else: raise Exception("not supported") sort_op = partial(persona_ops.agd_sort_metadata, buffer_pair_pool=bpp, name="agd_sort_metadata") for id_and_count, components in ready_to_sort: output_buffer_handless, num_recordss, first_ordinals, record_ids = components[: 4] rest_of_inputs = components[4:] # need to just pick the top things rest_of_input = tuple(a[0] for a in rest_of_inputs) record_id = record_ids[0] first_ordinal = first_ordinals[0] first_ordinal_str = tf.as_string(first_ordinal, name="first_ordinal_conversion") # this filename is guaranteed to be unique because of the ordinal (unique among this dataset) and the extension (so it doesn't conflict with existing chunk files) # otherwise when a request is resubmitted, the cleanup from the merge stage may overlap with the new files created! random_gen = tf.as_string( tf.random_uniform(dtype=tf.int32, maxval=2**20, shape=(), name="random_intermediate_name_gen"), name="random_intermediate_value_to_string") intermediate_name = tf.string_join( (record_id, first_ordinal_str, random_gen, intermediate_extension), separator="_", name="intermediate_filename") # TODO not sure if this axis=1 is correct unstack_handles = tf.unstack(output_buffer_handless, axis=1, name="buffers_unstack") key_handles = unstack_handles[0] # output_buffer_handless[:,0,:] other_handles = tf.stack(unstack_handles[1:], axis=1) # output_buffer_handless[:,1:,:] # first column is always the correct one, due to self.extended_columns order superchunk_matrix, superchunk_num_records = sort_op( num_records=num_recordss, sort_key_handles=key_handles, column_handles=other_handles) if self.log_goodput: with tf.control_dependencies((superchunk_num_records, )): ts = gate.unix_timestamp(name="sort_tail_timestamp") log_event = (gate.log_events( item_names=("id", "time", "record_id", "num_records"), directory=self.log_directory, event_name="sort_tail", name="sort_tail_event_logger", components=(slice_id(id_and_count), ts, record_id, superchunk_num_records)), ) else: log_event = () yield (id_and_count, record_id, intermediate_name, superchunk_num_records, superchunk_matrix) + rest_of_input, log_event
def run(logdir, run_name, wave_name, wave_constructor): """Generate wave data of the given form. The provided function `wave_constructor` should accept a scalar tensor of type float32, representing the frequency (in Hz) at which to construct a wave, and return a tensor of shape [1, _samples(), `n`] representing audio data (for some number of channels `n`). Waves will be generated at frequencies ranging from A4 to A5. Arguments: logdir: the top-level directory into which to write summary data run_name: the name of this run; will be created as a subdirectory under logdir wave_name: the name of the wave being generated wave_constructor: see above """ tf.compat.v1.reset_default_graph() tf.compat.v1.set_random_seed(0) # On each step `i`, we'll set this placeholder to `i`. This allows us # to know "what time it is" at each step. step_placeholder = tf.compat.v1.placeholder(tf.float32, shape=[]) # We want to linearly interpolate a frequency between A4 (440 Hz) and # A5 (880 Hz). with tf.name_scope('compute_frequency'): f_min = 440.0 f_max = 880.0 t = step_placeholder / (FLAGS.steps - 1) frequency = f_min * (1.0 - t) + f_max * t # Let's log this frequency, just so that we can make sure that it's as # expected. tf.compat.v1.summary.scalar('frequency', frequency) # Now, we pass this to the wave constructor to get our waveform. Doing # so within a name scope means that any summaries that the wave # constructor produces will be namespaced. with tf.name_scope(wave_name): waveform = wave_constructor(frequency) # We also have the opportunity to annotate each audio clip with a # label. This is a good place to include the frequency, because it'll # be visible immediately next to the audio clip. with tf.name_scope('compute_labels'): samples = tf.shape(input=waveform)[0] wave_types = tf.tile(["*Wave type:* `%s`." % wave_name], [samples]) frequencies = tf.strings.join([ "*Frequency:* ", tf.tile([tf.as_string(frequency, precision=2)], [samples]), " Hz.", ]) samples = tf.strings.join([ "*Sample:* ", tf.as_string(tf.range(samples) + 1), " of ", tf.as_string(samples), ".", ]) labels = tf.strings.join([wave_types, frequencies, samples], separator=" ") # We can place a description next to the summary in TensorBoard. This # is a good place to explain what the summary represents, methodology # for creating it, etc. Let's include the source code of the function # that generated the wave. source = '\n'.join(' %s' % line.rstrip() for line in inspect.getsourcelines(wave_constructor)[0]) description = ("A wave of type `%r`, generated via:\n\n%s" % (wave_name, source)) # Here's the crucial piece: we interpret this result as audio. summary.op('waveform', waveform, FLAGS.sample_rate, labels=labels, display_name=wave_name, description=description) # Now, we can collect up all the summaries and begin the run. summ = tf.compat.v1.summary.merge_all() sess = tf.compat.v1.Session() writer = tf.summary.FileWriter(os.path.join(logdir, run_name)) writer.add_graph(sess.graph) sess.run(tf.compat.v1.global_variables_initializer()) for step in xrange(FLAGS.steps): s = sess.run(summ, feed_dict={step_placeholder: float(step)}) writer.add_summary(s, global_step=step) writer.close()
def model_fn(self, features, labels, mode, params): """Model function used in the estimator. Args: features (Tensor): Input features to the model. labels (Tensor): Labels tensor for training and evaluation. mode (ModeKeys): Specifies if training, evaluation or prediction. params (HParams): hyperparameters. Returns: (EstimatorSpec): Model to be run by Estimator. """ is_training = mode == tf.estimator.ModeKeys.TRAIN if mode == tf.estimator.ModeKeys.PREDICT: ids = { id: features.pop(id) for id in self.parser.model_out_format if id[:-1] not in self.model.out_node_names } # ids = {key: features[key] for key in features if key != 'features'} predictions = self.model.forward(features, params=params, is_training=is_training) predictions_out = self.model.get_predictions_out( features, predictions, ids, self.parser.model_out_format) # export_outputs = {'predict_output': tf.estimator.export.PredictOutput(predictions_out)} return tf.estimator.EstimatorSpec( mode, predictions=predictions_out) #, export_outputs=export_outputs) predictions = self.model.forward(features, params=params, is_training=is_training) if predictions is not None and predictions.shape.ndims == 1: predictions = predictions[:, np.newaxis] if labels is not None and labels.shape.ndims == 1: labels = labels[:, np.newaxis] loss = self.model.get_loss(labels, predictions) eval_metric_ops = self.model.get_eval_metric_ops(labels, predictions) if mode == tf.estimator.ModeKeys.EVAL: return tf.estimator.EstimatorSpec(mode, loss=loss, eval_metric_ops=eval_metric_ops) if DEBUG: compare = tf.concat([ tf.as_string(labels), tf.as_string(predictions), tf.as_string(tf.cast(labels, dtype=tf.float32) - predictions) ], axis=1) hook = \ [tf.estimator.LoggingTensorHook({'result': compare}, every_n_iter=1)] else: hook = None if mode == tf.estimator.ModeKeys.TRAIN: self.model.add_summary(labels, predictions, eval_metric_ops) update_ops = tf.compat.v1.get_collection( tf.compat.v1.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): train_op = self.model.get_train_op_fn(loss, params) return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op, training_hooks=hook)
def get_accuracy(self, path_to_checkpoint, path_to_tfrecords_file, num_examples, global_step): """ input: path_to_checkpoint => model checkpoint path path_to_tfrecords_file => tfrecords path num_samples => number of samples to be measured funct: evaluates the accuracy of the predicted values of the samples output: returns the total accuracy of the model on the sample data given """ batch_size = 128 num_batches = num_examples / batch_size needs_include_length = False with tf.Graph().as_default(): # gets the batches of the evaluting data image_batch, length_batch, digits_batch = DataPreprocessor.build_batch(path_to_tfrecords_file, num_examples=num_examples, batch_size=batch_size, shuffled=False) length_logits, digits_logits = CNNModel.get_inference(image_batch, drop_rate=0.0) length_predictions = tf.argmax(length_logits, axis=1) digits_predictions = tf.argmax(digits_logits, axis=2) soft = tf.nn.softmax(digits_logits) coverage = tf.reduce_max(soft, reduction_indices=2) proba = tf.reduce_mean(coverage, axis=1) ones = 0.8*tf.ones_like(proba) mask = tf.greater(proba, ones) # if length and batch to be concatenated then concatenates if needs_include_length: labels = tf.concat([tf.reshape(length_batch, [-1, 1]), digits_batch], axis=1) predictions = tf.concat([tf.reshape(length_predictions, [-1, 1]), digits_predictions], axis=1) else: labels = digits_batch predictions = digits_predictions labels_string = tf.reduce_join(tf.as_string(labels), axis=1) predictions_string = tf.reduce_join(tf.as_string(predictions), axis=1) labels_mask_string = tf.boolean_mask(labels_string, mask) predictions_mask_string = tf.boolean_mask(predictions_string, mask) coverage_size = tf.size(predictions_mask_string)/tf.size(predictions_string) # determining the accuracy of the evaluating data accuracy, update_accuracy = tf.metrics.accuracy( labels=labels_string, predictions=predictions_string ) # determining the accuracy mask of the evaluating data accuracy_mask, update_accuracy_mask = tf.metrics.accuracy( labels=labels_mask_string, predictions=predictions_mask_string ) tf.summary.image('image', image_batch) tf.summary.scalar('accuracy', accuracy) tf.summary.histogram('variables', tf.concat([tf.reshape(var, [-1]) for var in tf.trainable_variables()], axis=0)) summary = tf.summary.merge_all() with tf.Session() as sess: sess.run([tf.global_variables_initializer(), tf.local_variables_initializer()]) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) restorer = tf.train.Saver() restorer.restore(sess, path_to_checkpoint) for _ in xrange(num_batches): sess.run([update_accuracy, update_accuracy_mask]) accuracy_val, summary_val = sess.run([accuracy, summary]) accuracy_mask_val, coverage_size_val = sess.run([accuracy_mask, coverage_size]) self.summary_writer.add_summary(summary_val, global_step=global_step) coord.request_stop() coord.join(threads) return accuracy_val, accuracy_mask_val, coverage_size_val
def dilated_cnn_model_pam4(features, labels, mode): num_classes = 4 samples_per_label = 16 input_layer = tf.reshape(features["x"], [-1, samples_per_label, 1]) # dilated convolutional layer conv1 = tf.layers.conv1d(inputs=input_layer, filters=4, kernel_size=8, padding='same', dilation_rate=2, activation=tf.nn.relu, name="dilated_conv_1") # max-pooling layer pool1 = tf.layers.max_pooling1d(inputs=conv1, pool_size=2, strides=2, name="max_pool_1") # dilated convolutional layer conv2_filters = 16 conv2 = tf.layers.conv1d(inputs=pool1, filters=conv2_filters, kernel_size=4, padding='same', dilation_rate=2, activation=tf.nn.relu, name="dilated_conv_2") # max-pooling layer pool2 = tf.layers.max_pooling1d(inputs=conv2, pool_size=2, strides=2, name="max_pool_2") # write kernel for TensorBoard visualization kernel = tf.get_collection(tf.GraphKeys.VARIABLES, 'dilated_conv_2/kernel')[0] kernel_txt = tf.Print(tf.as_string(kernel), [tf.as_string(kernel)], message='kernel_txt', name='kernel_txt') tf.summary.text('kernel_txt', kernel_txt) # flatten features pool2_flat = tf.reshape(pool2, [-1, 4 * conv2_filters]) dense = tf.layers.dense(inputs=pool2_flat, units=16, activation=tf.nn.relu) dropout = tf.layers.dropout(inputs=dense, rate=0.4, training=mode == tf.estimator.ModeKeys.TRAIN) logits = tf.layers.dense(inputs=dropout, units=num_classes) predictions = { "classes": tf.argmax(input=logits, axis=1), "probabilities": tf.nn.softmax(logits, name="softmax_tensor") } if mode == tf.estimator.ModeKeys.PREDICT: return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions) loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits) if mode == tf.estimator.ModeKeys.TRAIN: optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001) train_op = optimizer.minimize(loss=loss, global_step=tf.train.get_global_step()) return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op) eval_metric_ops = { "accuracy": tf.metrics.accuracy(labels=labels, predictions=predictions["classes"]) } return tf.estimator.EstimatorSpec(mode=mode, loss=loss, eval_metric_ops=eval_metric_ops)
def train(hparams): """Trains an MNIST GAN. Args: hparams: An HParams instance containing the hyperparameters for training. """ if not tf.io.gfile.exists(hparams.train_log_dir): tf.io.gfile.makedirs(hparams.train_log_dir) # Force all input processing onto CPU in order to reserve the GPU for # the forward inference and back-propagation. with tf.compat.v1.name_scope('inputs'), tf.device('/cpu:0'): images, one_hot_labels = data_provider.provide_data( 'train', hparams.batch_size, num_parallel_calls=4) # Define the GANModel tuple. Optionally, condition the GAN on the label or # use an InfoGAN to learn a latent representation. if hparams.gan_type == 'unconditional': gan_model = tfgan.gan_model( generator_fn=networks.unconditional_generator, discriminator_fn=networks.unconditional_discriminator, real_data=images, generator_inputs=tf.random.normal( [hparams.batch_size, hparams.noise_dims])) elif hparams.gan_type == 'conditional': noise = tf.random.normal([hparams.batch_size, hparams.noise_dims]) gan_model = tfgan.gan_model( generator_fn=networks.conditional_generator, discriminator_fn=networks.conditional_discriminator, real_data=images, generator_inputs=(noise, one_hot_labels)) elif hparams.gan_type == 'infogan': cat_dim, cont_dim = 10, 2 generator_fn = functools.partial(networks.infogan_generator, categorical_dim=cat_dim) discriminator_fn = functools.partial(networks.infogan_discriminator, categorical_dim=cat_dim, continuous_dim=cont_dim) unstructured_inputs, structured_inputs = util.get_infogan_noise( hparams.batch_size, cat_dim, cont_dim, hparams.noise_dims) gan_model = tfgan.infogan_model( generator_fn=generator_fn, discriminator_fn=discriminator_fn, real_data=images, unstructured_generator_inputs=unstructured_inputs, structured_generator_inputs=structured_inputs) tfgan.eval.add_gan_model_image_summaries(gan_model, hparams.grid_size) # Get the GANLoss tuple. You can pass a custom function, use one of the # already-implemented losses from the losses library, or use the defaults. with tf.compat.v1.name_scope('loss'): if hparams.gan_type == 'infogan': gan_loss = tfgan.gan_loss( gan_model, generator_loss_fn=tfgan.losses.modified_generator_loss, discriminator_loss_fn=tfgan.losses.modified_discriminator_loss, mutual_information_penalty_weight=1.0, add_summaries=True) else: gan_loss = tfgan.gan_loss(gan_model, add_summaries=True) tfgan.eval.add_regularization_loss_summaries(gan_model) # Get the GANTrain ops using custom optimizers. with tf.compat.v1.name_scope('train'): gen_lr, dis_lr = _learning_rate(hparams.gan_type) train_ops = tfgan.gan_train_ops( gan_model, gan_loss, generator_optimizer=tf.compat.v1.train.AdamOptimizer(gen_lr, 0.5), discriminator_optimizer=tf.compat.v1.train.AdamOptimizer( dis_lr, 0.5), summarize_gradients=True, aggregation_method=tf.AggregationMethod.EXPERIMENTAL_ACCUMULATE_N) # Run the alternating training loop. Skip it if no steps should be taken # (used for graph construction tests). status_message = tf.strings.join([ 'Starting train step: ', tf.as_string(tf.compat.v1.train.get_or_create_global_step()) ], name='status_message') if hparams.max_number_of_steps == 0: return tfgan.gan_train( train_ops, hooks=[ tf.estimator.StopAtStepHook(num_steps=hparams.max_number_of_steps), tf.estimator.LoggingTensorHook([status_message], every_n_iter=10) ], logdir=hparams.train_log_dir, get_hooks_fn=tfgan.get_joint_train_hooks(), save_checkpoint_secs=60)
def main(_): if not tf.gfile.Exists(FLAGS.train_log_dir): tf.gfile.MakeDirs(FLAGS.train_log_dir) with tf.device(tf.train.replica_device_setter(FLAGS.ps_tasks)): # Get real and distorted images. with tf.device('/cpu:0'), tf.name_scope('inputs'): real_images = data_provider.provide_data( 'train', FLAGS.batch_size, dataset_dir=FLAGS.dataset_dir, patch_size=FLAGS.patch_size) distorted_images = _distort_images( real_images, downscale_size=int(FLAGS.patch_size / 2), upscale_size=FLAGS.patch_size) # Create a GANModel tuple. gan_model = tfgan.gan_model( generator_fn=networks.generator, discriminator_fn=networks.discriminator, real_data=real_images, generator_inputs=distorted_images) tfgan.eval.add_image_comparison_summaries( gan_model, num_comparisons=3, display_diffs=True) tfgan.eval.add_gan_model_image_summaries(gan_model, grid_size=3) # Define the GANLoss tuple using standard library functions. with tf.name_scope('losses'): gan_loss = tfgan.gan_loss( gan_model, generator_loss_fn=tfgan.losses.least_squares_generator_loss, discriminator_loss_fn=tfgan.losses.least_squares_discriminator_loss) # Define the standard L1 pixel loss. l1_pixel_loss = tf.norm(gan_model.real_data - gan_model.generated_data, ord=1) / FLAGS.patch_size ** 2 # Modify the loss tuple to include the pixel loss. Add summaries as well. gan_loss = tfgan.losses.combine_adversarial_loss( gan_loss, gan_model, l1_pixel_loss, weight_factor=FLAGS.weight_factor) with tf.name_scope('train_ops'): # Get the GANTrain ops using the custom optimizers and optional # discriminator weight clipping. gen_lr, dis_lr = _lr(FLAGS.generator_lr, FLAGS.discriminator_lr) gen_opt, dis_opt = _optimizer(gen_lr, dis_lr) train_ops = tfgan.gan_train_ops( gan_model, gan_loss, generator_optimizer=gen_opt, discriminator_optimizer=dis_opt, summarize_gradients=True, colocate_gradients_with_ops=True, aggregation_method=tf.AggregationMethod.EXPERIMENTAL_ACCUMULATE_N, transform_grads_fn=tf.contrib.training.clip_gradient_norms_fn(1e3)) tf.summary.scalar('generator_lr', gen_lr) tf.summary.scalar('discriminator_lr', dis_lr) # Use GAN train step function if using adversarial loss, otherwise # only train the generator. train_steps = tfgan.GANTrainSteps( generator_train_steps=1, discriminator_train_steps=int(FLAGS.weight_factor > 0)) # Run the alternating training loop. Skip it if no steps should be taken # (used for graph construction tests). status_message = tf.string_join( ['Starting train step: ', tf.as_string(tf.train.get_or_create_global_step())], name='status_message') if FLAGS.max_number_of_steps == 0: return tfgan.gan_train( train_ops, FLAGS.train_log_dir, get_hooks_fn=tfgan.get_sequential_train_hooks(train_steps), hooks=[tf.train.StopAtStepHook(num_steps=FLAGS.max_number_of_steps), tf.train.LoggingTensorHook([status_message], every_n_iter=10)], master=FLAGS.master, is_chief=FLAGS.task == 0)
def evaluate(self, path_to_restore_model_checkpoint_file, path_to_restore_defender_checkpoint_file, path_to_tfrecords_file, num_examples, global_step, defend_layer, attacker_type): batch_size = 32 num_batches = num_examples // batch_size needs_include_length = False with tf.Graph().as_default(): image_batch, length_batch, digits_batch = Donkey.build_batch( path_to_tfrecords_file, num_examples=num_examples, batch_size=batch_size, shuffled=False) with tf.variable_scope('model'): length_logits, digits_logits, hidden_out = Model.inference( image_batch, drop_rate=0.0, is_training=False, defend_layer=defend_layer) with tf.variable_scope('defender'): recovered = Attacker.recover_hidden(attacker_type, hidden_out, is_training=False, defend_layer=defend_layer) ssim = tf.reduce_mean( tf.abs(tf.image.ssim(image_batch, recovered, max_val=2))) length_predictions = tf.argmax(length_logits, axis=1) digits_predictions = tf.argmax(digits_logits, axis=2) if needs_include_length: labels = tf.concat( [tf.reshape(length_batch, [-1, 1]), digits_batch], axis=1) predictions = tf.concat([ tf.reshape(length_predictions, [-1, 1]), digits_predictions ], axis=1) else: labels = digits_batch predictions = digits_predictions labels_string = tf.reduce_join(tf.as_string(labels), axis=1) predictions_string = tf.reduce_join(tf.as_string(predictions), axis=1) accuracy, update_accuracy = tf.metrics.accuracy( labels=labels_string, predictions=predictions_string) tf.summary.image('image', image_batch, max_outputs=20) tf.summary.image('recovered', recovered, max_outputs=20) tf.summary.scalar('ssim', ssim) tf.summary.scalar('accuracy', accuracy) tf.summary.histogram( 'variables', tf.concat([ tf.reshape(var, [-1]) for var in tf.trainable_variables() ], axis=0)) summary = tf.summary.merge_all() with tf.Session() as sess: sess.run([ tf.global_variables_initializer(), tf.local_variables_initializer() ]) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) model_saver = tf.train.Saver(var_list=tf.get_collection( tf.GraphKeys.TRAINABLE_VARIABLES, scope='model')) defender_saver = tf.train.Saver(var_list=tf.get_collection( tf.GraphKeys.TRAINABLE_VARIABLES, scope='defender')) model_saver.restore(sess, path_to_restore_model_checkpoint_file) print("Evaluation model restored from {}".format( path_to_restore_model_checkpoint_file)) defender_saver.restore( sess, path_to_restore_defender_checkpoint_file) for _ in range(num_batches): sess.run(update_accuracy) accuracy_val, summary_val = sess.run([accuracy, summary]) self.summary_writer.add_summary(summary_val, global_step=global_step) coord.request_stop() coord.join(threads) return accuracy_val
def main(_): if not tf.gfile.Exists(FLAGS.train_log_dir): tf.gfile.MakeDirs(FLAGS.train_log_dir) with tf.device(tf.train.replica_device_setter(FLAGS.ps_tasks)): # Put input pipeline on CPU to reserve GPU for training. with tf.name_scope('inputs'), tf.device('/cpu:0'): images = data_provider.provide_data( 'train', FLAGS.batch_size, dataset_dir=FLAGS.dataset_dir, patch_size=FLAGS.patch_size) # Manually define a GANModel tuple. This is useful when we have custom # code to track variables. Note that we could replace all of this with a # call to `tfgan.gan_model`, but we don't in order to demonstrate some of # TFGAN's flexibility. with tf.variable_scope('generator') as gen_scope: reconstructions, _, prebinary = networks.compression_model( images, num_bits=FLAGS.bits_per_patch, depth=FLAGS.model_depth) gan_model = _get_gan_model( generator_inputs=images, generated_data=reconstructions, real_data=images, generator_scope=gen_scope) summaries.add_reconstruction_summaries(images, reconstructions, prebinary) tfgan.eval.add_gan_model_summaries(gan_model) # Define the GANLoss tuple using standard library functions. with tf.name_scope('loss'): gan_loss = tfgan.gan_loss( gan_model, generator_loss_fn=tfgan.losses.least_squares_generator_loss, discriminator_loss_fn=tfgan.losses.least_squares_discriminator_loss, add_summaries=FLAGS.weight_factor > 0) # Define the standard pixel loss. l1_pixel_loss = tf.norm(gan_model.real_data - gan_model.generated_data, ord=1) # Modify the loss tuple to include the pixel loss. Add summaries as well. gan_loss = tfgan.losses.combine_adversarial_loss( gan_loss, gan_model, l1_pixel_loss, weight_factor=FLAGS.weight_factor) # Get the GANTrain ops using the custom optimizers and optional # discriminator weight clipping. with tf.name_scope('train_ops'): gen_lr, dis_lr = _lr(FLAGS.generator_lr, FLAGS.discriminator_lr) gen_opt, dis_opt = _optimizer(gen_lr, dis_lr) train_ops = tfgan.gan_train_ops( gan_model, gan_loss, generator_optimizer=gen_opt, discriminator_optimizer=dis_opt, summarize_gradients=True, colocate_gradients_with_ops=True, aggregation_method=tf.AggregationMethod.EXPERIMENTAL_ACCUMULATE_N) tf.summary.scalar('generator_lr', gen_lr) tf.summary.scalar('discriminator_lr', dis_lr) # Determine the number of generator vs discriminator steps. train_steps = tfgan.GANTrainSteps( generator_train_steps=1, discriminator_train_steps=int(FLAGS.weight_factor > 0)) # Run the alternating training loop. Skip it if no steps should be taken # (used for graph construction tests). status_message = tf.string_join( ['Starting train step: ', tf.as_string(tf.train.get_or_create_global_step())], name='status_message') if FLAGS.max_number_of_steps == 0: return tfgan.gan_train( train_ops, FLAGS.train_log_dir, tfgan.get_sequential_train_hooks(train_steps), hooks=[tf.train.StopAtStepHook(num_steps=FLAGS.max_number_of_steps), tf.train.LoggingTensorHook([status_message], every_n_iter=10)], master=FLAGS.master, is_chief=FLAGS.task == 0)
def load_image(id): filename = tf.constant("MNIST_images/", tf.string) + tf.as_string(id) + tf.constant(".png") img_string = tf.read_file(filename) img = tf.image.decode_image(img_string) return img
saver = tf.train.Saver(max_to_keep=2) # Remember the training_op we want to run by adding it to a collection. tf.add_to_collection('train_step', train_step) tf.add_to_collection('all_weights', allweights) tf.add_to_collection('all_biasses', allbiases) #for visualization in tensorboard, merge all the summaries tf.summary.scalar('cross_entropy', cross_entropy) tf.summary.scalar('accuracy', accuracy) merged = tf.summary.merge_all() model_version = tf.as_string([ tf.cast(W1.shape, tf.float32), tf.cast(W2.shape, tf.float32), tf.cast(W3.shape, tf.float32), tf.cast(W4.shape, tf.float32), tf.cast(W5.shape, tf.float32), tf.concat([tf.cast(W6.shape, tf.float32), [0, 0]], 0), tf.concat([tf.cast(W7.shape, tf.float32), [0, 0]], 0) ]) model_version = tf.summary.text("Model version", model_version) train_writer = tf.summary.FileWriter('tensorboard_train', sess.graph) test_writer = tf.summary.FileWriter('tensorboard_test') #at the beginning add info about model structure test_writer.add_summary(model_version.eval()) train_writer.add_summary(model_version.eval()) sess.run(init_op) #initilize all variables
def main(unused_argv): ''' check path ''' if FLAGS.checkpoint_dir == '' or not os.path.exists(FLAGS.checkpoint_dir): raise ValueError('invalid data directory {}'.format( FLAGS.checkpoint_dir)) checkpoint_dir = os.path.join(FLAGS.checkpoint_dir, '') if FLAGS.output_dir == '': raise ValueError('invalid output directory {}'.format( FLAGS.output_dir)) elif not os.path.exists(FLAGS.output_dir): os.makedirs(FLAGS.output_dir) event_log_dir = os.path.join(FLAGS.output_dir, '') checkpoint_path = os.path.join(FLAGS.output_dir, 'model.ckpt') ''' setup summaries ''' summ = Summaries() ''' setup the game environment ''' filenames = glob.glob( os.path.join(FLAGS.data_dir, 'test-{}'.format(FLAGS.sampling_rate), '*.mat')) assert (len(filenames) > 0), "invalid file names" game_env = Env(FLAGS.decay) game_actions = list(game_env.actions.keys()) ''' setup agent ''' stateDim = [FLAGS.num_chans, FLAGS.num_points] input_dim = [1] + stateDim s_placeholder = tf.placeholder(tf.float32, input_dim, 's_placeholder') network = Model(FLAGS.batch_size, len(game_actions), FLAGS.num_chans, FLAGS.sampling_rate, \ FLAGS.num_filters, FLAGS.num_recurs, FLAGS.pooling_stride, name = "network") state_placeholder = tf.placeholder(tf.float32, input_dim, 'state_placeholder') action_tensor_greedy = greedy(state_placeholder, network) ''' setup the testing process ''' episode_reward_placeholder = tf.placeholder(tf.float32, [], "episode_reward_placeholder") # summ.register('stats', 'episode_reward', episode_reward_placeholder) ''' gathering summary operators ''' test_stats_op = tf.summary.text('episode_reward', tf.as_string(episode_reward_placeholder)) ''' setup the testing process ''' saver = tf.train.Saver(network.get_all_variables()) config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) assert (FLAGS.gpus != ''), 'invalid GPU specification' config.gpu_options.visible_device_list = FLAGS.gpus with tf.Session(config=config) as sess: sess.run([ tf.global_variables_initializer(), tf.local_variables_initializer() ]) ckpt = tf.train.get_checkpoint_state(checkpoint_dir) if ckpt and ckpt.model_checkpoint_path: # Restores from checkpoint. saver.restore(sess, ckpt.model_checkpoint_path) # Assuming model_checkpoint_path looks something like: # /my-favorite-path/imagenet_train/model.ckpt-0, # extract global_step from it. # global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1] # print('Successfully loaded model from %s at step = %s.' % # (ckpt.model_checkpoint_path, global_step)) print('Successfully loaded model from %s.' % ckpt.model_checkpoint_path) else: print('No checkpoint file found') return for filename in filenames: measured_rt = { 'index': [], 'value': [], 'title': 'measured RT (original)' } predicted_rt = { 'index': [], 'value': [], 'title': 'predicted RT (original)' } fb, _ = os.path.splitext(filename) fb = os.path.basename(fb) f_event_log_dir = os.path.join(event_log_dir, fb) if not os.path.exists(f_event_log_dir): os.makedirs(f_event_log_dir) writer = tf.summary.FileWriter(f_event_log_dir, tf.get_default_graph()) print("file name: {}".format(filename)) game_env.reset(filename) episode_reward = 0 count = 0 initial_stage = True while True: print("Evaluation step: {}".format(count)) if initial_stage: action = np.random.randint(0, len(game_actions)) initial_stage = False else: action = action_index print("action -> {}".format(game_actions[action])) state, reward, terminal = game_env.step(game_actions[action]) # game over? if terminal: break episode_reward += reward if not terminal: action_index = sess.run(action_tensor_greedy, feed_dict={ state_placeholder: np.expand_dims(state, axis=0) }) action_index = np.squeeze(action_index) # print('state -> {}'.format(state)) # print('action_index -> {}'.format(action_index)) else: action_index = 0 if game_env.measured_rt: measured_rt['index'].append(count) measured_rt['value'].append(game_env.measured_rt) predicted_rt['index'].append(count) predicted_rt['value'].append(game_env.predicted_rt) count += 1 stats_str = sess.run( test_stats_op, feed_dict={episode_reward_placeholder: episode_reward}) writer.add_summary(stats_str, count) @tfmpl.figure_tensor def draw_line(measured_rt, predicted_rt): fig = tfmpl.create_figures(1, figsize=(16, 8))[0] ax = fig.add_subplot(1, 2, 1) for item in ([ax.title, ax.xaxis.label, ax.yaxis.label] + \ ax.get_xticklabels() + ax.get_yticklabels()): item.set_fontsize(20) # ax.axis('off') ax.plot(measured_rt['index'], measured_rt['value'], 'b') ax.set_title(measured_rt['title'], fontsize=24) ax = fig.add_subplot(1, 2, 2) for item in ([ax.title, ax.xaxis.label, ax.yaxis.label] + \ ax.get_xticklabels() + ax.get_yticklabels()): item.set_fontsize(20) ax.plot(predicted_rt['index'], predicted_rt['value'], 'r') ax.set_title(predicted_rt['title'], fontsize=24) fig.tight_layout() return fig image_tensor = draw_line(measured_rt, predicted_rt) image_summary = tf.summary.image('test', image_tensor) image_str = sess.run(image_summary) writer.add_summary(image_str, global_step=0) measured_rt_ma = measured_rt.copy() predicted_rt_ma = predicted_rt.copy() import matlab.engine eng = matlab.engine.start_matlab("-nojvm -nodisplay") m_rt = matlab.double(measured_rt_ma['value']) p_rt = matlab.double(predicted_rt_ma['value']) m_rt = eng.movmean(m_rt, 11) p_rt = eng.movmean(p_rt, 11) m_rt = np.array(m_rt) m_rt = np.reshape(m_rt, -1) measured_rt_ma['value'] = list(m_rt) measured_rt_ma['title'] = "measured RT (after moving average)" p_rt = np.array(p_rt) p_rt = np.reshape(p_rt, -1) predicted_rt_ma['value'] = list(p_rt) predicted_rt_ma['title'] = "predicted RT (after moving average)" image_tensor_ma = draw_line(measured_rt_ma, predicted_rt_ma) image_summary_ma = tf.summary.image('test_ma', image_tensor_ma) image_str_ma = sess.run(image_summary_ma) writer.add_summary(image_str_ma, global_step=1) io.savemat(os.path.join(f_event_log_dir, "stats.mat"), { "measured_rt": measured_rt, 'predicted_rt': predicted_rt }) eng.quit() writer.close()
def main(_): if not tf.gfile.Exists(FLAGS.train_log_dir): tf.gfile.MakeDirs(FLAGS.train_log_dir) with tf.device(tf.train.replica_device_setter(FLAGS.ps_tasks)): # Force all input processing onto CPU in order to reserve the GPU for # the forward inference and back-propagation. with tf.name_scope('inputs'): with tf.device('/cpu:0'): images, one_hot_labels, _, _ = data_provider.provide_data( FLAGS.batch_size, FLAGS.dataset_dir) # Define the GANModel tuple. noise = tf.random_normal([FLAGS.batch_size, 64]) if FLAGS.conditional: generator_fn = networks.conditional_generator discriminator_fn = networks.conditional_discriminator generator_inputs = (noise, one_hot_labels) else: generator_fn = networks.generator discriminator_fn = networks.discriminator generator_inputs = noise gan_model = tfgan.gan_model(generator_fn, discriminator_fn, real_data=images, generator_inputs=generator_inputs) tfgan.eval.add_gan_model_image_summaries(gan_model) # Get the GANLoss tuple. Use the selected GAN loss functions. # (joelshor): Put this block in `with tf.name_scope('loss'):` when # cl/171610946 goes into the opensource release. gan_loss = tfgan.gan_loss(gan_model, gradient_penalty_weight=1.0, add_summaries=True) # Get the GANTrain ops using the custom optimizers and optional # discriminator weight clipping. with tf.name_scope('train'): gen_lr, dis_lr = _learning_rate() gen_opt, dis_opt = _optimizer(gen_lr, dis_lr, FLAGS.use_sync_replicas) train_ops = tfgan.gan_train_ops( gan_model, gan_loss, generator_optimizer=gen_opt, discriminator_optimizer=dis_opt, summarize_gradients=True, colocate_gradients_with_ops=True, aggregation_method=tf.AggregationMethod. EXPERIMENTAL_ACCUMULATE_N) tf.summary.scalar('generator_lr', gen_lr) tf.summary.scalar('discriminator_lr', dis_lr) # Run the alternating training loop. Skip it if no steps should be taken # (used for graph construction tests). sync_hooks = ([ gen_opt.make_session_run_hook(FLAGS.task == 0), dis_opt.make_session_run_hook(FLAGS.task == 0) ] if FLAGS.use_sync_replicas else []) status_message = tf.string_join([ 'Starting train step: ', tf.as_string(tf.train.get_or_create_global_step()) ], name='status_message') if FLAGS.max_number_of_steps == 0: return sess_config = tf.ConfigProto( inter_op_parallelism_threads=FLAGS.inter_op_parallelism_threads, intra_op_parallelism_threads=FLAGS.intra_op_parallelism_threads) tfgan.gan_train( train_ops, hooks=([ tf.train.StopAtStepHook(num_steps=FLAGS.max_number_of_steps), tf.train.LoggingTensorHook([status_message], every_n_iter=10) ] + sync_hooks), logdir=FLAGS.train_log_dir, master=FLAGS.master, is_chief=FLAGS.task == 0, config=sess_config)
def build_model(self): user_click_item_list_idx = tf.string_to_hash_bucket_fast(tf.as_string(self.user_click_item_list), self.ITEM_MOD) # embed_init = tf.nn.embedding_lookup(self.item_embedding, user_click_item_list_idx) # # User Embedding Layer with tf.name_scope("user_tower"): with tf.name_scope('user_embedding'): user_item_click_avg_embed = self.get_seq_embedding(self.item_embedding, user_click_item_list_idx, self.user_click_item_list_len, self.item_embedding_size, 'sum') # gru_cell = tf.nn.rnn_cell.GRUCell(self.item_embedding_size) # # # initial_state = tf.nn.rnn_cell.zero_state(self.batch_size, dtype=tf.float32) # output, state = dynamic_rnn(cell=gru_cell, inputs=embed_init, dtype=tf.float32, # sequence_length=self.user_click_item_list_len) gender_one_hot = tf.one_hot(self.gender, self.GENDER_CNT) client_type_one_hot = tf.one_hot(self.client_type, self.CLIENT_TYPE_CNT) user_embed_concat = tf.concat( [user_item_click_avg_embed, gender_one_hot, client_type_one_hot], axis=-1) with tf.name_scope('layers'): bn = tf.layers.batch_normalization(inputs=user_embed_concat, name='user_bn', ) user_layer_1 = tf.layers.dense(bn, 1024, activation=tf.nn.tanh, name='user_first', kernel_initializer=tf.glorot_normal_initializer()) user_layer_2 = tf.layers.dense(user_layer_1, 512, activation=tf.nn.tanh, name='user_second', kernel_initializer=tf.glorot_normal_initializer()) user_layer_3 = tf.layers.dense(user_layer_2, self.item_embedding_size, activation=tf.nn.tanh, name='user_final', kernel_initializer=tf.glorot_normal_initializer()) self.user_embedding_final = user_layer_3 with tf.name_scope("item_tower"): # self.item_idx = self.item_table.lookup(self.item_vobabulary) # self.item_cate_mapping_idx = self.cate_table.lookup(self.item_cate_mapping) # self.item_tag_mapping_idx = self.tag_table.lookup(self.item_tag_mapping) pos_item_idx = tf.string_to_hash_bucket_fast(tf.as_string(self.pos_item), self.ITEM_MOD) pos_cate_idx = tf.string_to_hash_bucket_fast(tf.as_string(self.pos_cate), self.CATE_MOD) pos_tag_idx = tf.string_to_hash_bucket_fast(tf.as_string(self.pos_tag), self.TAG_MOD) neg_item_idx = tf.string_to_hash_bucket_fast(tf.as_string(self.neg_item), self.ITEM_MOD) neg_cate_idx = tf.string_to_hash_bucket_fast(tf.as_string(self.neg_cate), self.CATE_MOD) neg_tag_idx = tf.string_to_hash_bucket_fast(tf.as_string(self.neg_tag), self.TAG_MOD) pos_item_id_embed = tf.nn.embedding_lookup(self.item_embedding, pos_item_idx) pos_cate_id_embed = tf.nn.embedding_lookup(self.cate_embedding, pos_cate_idx) pos_tag_id_embed = tf.nn.embedding_lookup(self.tag_embedding, pos_tag_idx) pos_key_word_embed = self.get_seq_embedding(self.key_word_embedding, self.pos_key, self.pos_item_key_len, self.key_word_embedding_size, 'avg') self.pos_embed_concat = tf.concat( [pos_item_id_embed, pos_cate_id_embed, pos_tag_id_embed, pos_key_word_embed], axis=-1) neg_item_id_embed = tf.nn.embedding_lookup(self.item_embedding, neg_item_idx) neg_cate_id_embed = tf.nn.embedding_lookup(self.cate_embedding, neg_cate_idx) neg_tag_id_embed = tf.nn.embedding_lookup(self.tag_embedding, neg_tag_idx) neg_key_word_embed = self.get_seq_embedding(self.key_word_embedding, self.neg_key, self.neg_item_key_len, self.key_word_embedding_size, 'avg') self.neg_embed_concat = tf.concat( [neg_item_id_embed, neg_cate_id_embed, neg_tag_id_embed, neg_key_word_embed], axis=-1) self.item_concat = tf.concat([self.pos_embed_concat, self.neg_embed_concat], axis=0) with tf.name_scope('item_layers'): bn = tf.layers.batch_normalization(inputs=self.item_concat, name='item_bn', ) item_layer_1 = tf.layers.dense(bn, self.item_embedding_size, activation=tf.nn.tanh, name='item_first', kernel_initializer=tf.glorot_normal_initializer()) # item_layer_2 = tf.layers.dense(item_layer_1, self.item_embedding_size, activation=tf.nn.tanh, # name='item_second', # kernel_initializer=tf.glorot_normal_initializer()) # item_layer_3 = tf.layers.dense(item_layer_2, self.item_embedding_size, activation=tf.nn.tanh, # name='item_third', # kernel_initializer=tf.glorot_normal_initializer()) self.item_embed_output = item_layer_1 self.item_embed_split = tf.split(self.item_embed_output, 2, 0) self.pos_embed_final = tf.squeeze(self.item_embed_split[0]) self.neg_embed_final = tf.squeeze(self.item_embed_split[1])
def slot_map_fn(x): x = tf.as_string(x) x = tf.strings.to_hash_bucket_fast(x, nslots) return x
def evaluate(self, path_to_checkpoint, path_to_tfrecords_file, num_examples, global_step): batch_size = 128 num_batches = num_examples // batch_size needs_include_length = False with tf.Graph().as_default(): image_batch, length_batch, digits_batch = build_batch( path_to_tfrecords_file, num_examples=num_examples, batch_size=batch_size, shuffled=False) length_logits, digits_logits = Model.inference(image_batch, drop_rate=0.0) length_predictions = tf.argmax(length_logits, axis=1) digits_predictions = tf.argmax(digits_logits, axis=2) if needs_include_length: labels = tf.concat( [tf.reshape(length_batch, [-1, 1]), digits_batch], axis=1) predictions = tf.concat([ tf.reshape(length_predictions, [-1, 1]), digits_predictions ], axis=1) else: labels = digits_batch predictions = digits_predictions labels_string = tf.reduce_join(tf.as_string(labels), axis=1) predictions_string = tf.reduce_join(tf.as_string(predictions), axis=1) accuracy, update_accuracy = tf.metrics.accuracy( labels=labels_string, predictions=predictions_string) tf.summary.image('image', image_batch) tf.summary.scalar('accuracy', accuracy) tf.summary.histogram( 'variables', tf.concat([ tf.reshape(var, [-1]) for var in tf.trainable_variables() ], axis=0)) summary = tf.summary.merge_all() with tf.Session() as sess: sess.run([ tf.global_variables_initializer(), tf.local_variables_initializer() ]) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) restorer = tf.train.Saver() restorer.restore(sess, path_to_checkpoint) for _ in range(num_batches): sess.run(update_accuracy) accuracy_val, summary_val = sess.run([accuracy, summary]) self.summary_writer.add_summary(summary_val, global_step=global_step) coord.request_stop() coord.join(threads) return accuracy_val
def sample_fn(spec): """Return a composite tensor sample given `spec`. Args: spec: A TensorSpec, SparseTensorSpec, etc. Returns: A tensor or SparseTensor. Raises: NotImplementedError: If `outer_dims` is not statically known and a SparseTensor is requested. """ if isinstance(spec, tf.SparseTensorSpec): outer_shape = tf.get_static_value(outer_dims) if outer_dims is not None and outer_shape is None: raise NotImplementedError( "outer_dims must be statically known, got: {}".format( outer_dims)) shape = tf.TensorShape(outer_shape or []).concatenate(spec.shape) if shape.num_elements() == 0 or tf.compat.dimension_value( shape[0]) == 0: return tf.SparseTensor(indices=tf.zeros([0, shape.rank], dtype=tf.int64), values=tf.zeros([0], dtype=spec.dtype), dense_shape=shape) indices_spec = BoundedTensorSpec( dtype=tf.int64, shape=[7, shape.rank], minimum=[0] * shape.rank, maximum=[x - 1 for x in shape.as_list()]) values_dtype = tf.int32 if spec.dtype == tf.string else spec.dtype values_spec = BoundedTensorSpec(dtype=values_dtype, shape=[7], minimum=0, maximum=shape.as_list()[-1] - 1) values_sample = sample_bounded_spec(values_spec, seed=seed_stream()) if spec.dtype == tf.string: values_sample = tf.as_string(values_sample) return tf.sparse.reorder( tf.SparseTensor(indices=sample_bounded_spec( indices_spec, seed=seed_stream()), values=values_sample, dense_shape=shape)) elif isinstance(spec, (TensorSpec, BoundedTensorSpec)): if spec.dtype == tf.string: sample_spec = BoundedTensorSpec(spec.shape, tf.int32, minimum=0, maximum=10) return tf.as_string( sample_bounded_spec(sample_spec, outer_dims=outer_dims, seed=seed_stream())) else: return sample_bounded_spec(BoundedTensorSpec.from_spec(spec), outer_dims=outer_dims, seed=seed_stream()) else: raise TypeError("Spec type not supported: '{}'".format(spec))
def decode(data, items): out = {} # Arguments: # data: Can be 3-col or 4-col CSV. A 3-col would look like "filepath # nframes class_id", 4-col will be similar for Charades like dataset # items: The different items to be returned. with tf.name_scope('decode_video'): if modality == 'rgb' or \ modality.startswith('flow') or \ modality.startswith('rgb+flow') or \ modality.startswith('pose'): image_buffer, label, fpath, frame_sublist, start_frame = reader.read( data) # stacking required due to the way queues in main train loop work # image_buffer = tf.stack([tf.stack(_decode_from_string(el, modality)) for # el in image_buffer]) image_lst = [] image_hts = [] image_wds = [] for im_buf in image_buffer: temp = _decode_from_string(im_buf, modality) image_lst += temp[0] image_hts.append(temp[1]) image_wds.append(temp[2]) image_buffer = tf.stack(image_lst) im_ht = tf.reduce_max(image_hts) im_wd = tf.reduce_max(image_wds) # image_buffer = tf.stack([ # _decode_from_string(el, modality)[0] for el in image_buffer]) else: logging.error('Unknown modality %s\n' % modality) # since my code gives a 0-1 image, change it back out['image'] = tf.cast(image_buffer * 255.0, tf.uint8) if 'pose' in items: if pose_dataset_dir is None: out['pose'] = [ -tf.ones([ num_pose_keypoints * 3, ], dtype=tf.int64) ] else: out['pose'] = [ read_json_pose( tf.string_join([ pose_dataset_dir, '/', fpath, '/', 'image_', tf.as_string(frame_sublist_i + 1, width=5, fill='0'), '.json' ])) for frame_sublist_i in tf.unstack(frame_sublist) ] if 'objects' in items: if objects_dataset_dir is None: out['objects'] = [] else: out['objects'] = [ tf.read_file( tf.string_join([ objects_dataset_dir, '/', fpath, '/', 'image_', tf.as_string(frame_sublist_i + 1, width=5, fill='0'), '.txt' ])) for frame_sublist_i in tf.unstack(frame_sublist) ] out['action_label'] = label # The following is the original image size on disk, # on which pose etc would have been computed out['im_wd'] = tf.cast(im_wd, tf.int64) out['im_ht'] = tf.cast(im_ht, tf.int64) return [out[el] for el in items]
def build_model(self): self.user_click_item_list_idx = tf.string_to_hash_bucket_fast( tf.as_string(self.user_click_item_list), self.ITEM_CNT) self.target_item_idx = tf.string_to_hash_bucket_fast( tf.as_string(self.target_item_list), self.ITEM_CNT) target_cate_idx = tf.string_to_hash_bucket_fast( tf.as_string(self.target_cate_list), self.CATE_CNT) target_tag_idx = tf.string_to_hash_bucket_fast( tf.as_string(self.target_tag_list), self.TAG_CNT) # # User Embedding Layer with tf.name_scope('user_embedding'): # user_item_click_sum_embed = self.get_seq_embedding(self.item_embedding, # self.user_click_item_list_idx, # self.user_click_item_list_len, # self.item_embedding_size, # "sum") self.user_item_click_attention_embed = self.get_attention_embedding( self.item_embedding, self.user_click_item_list_idx, self.user_click_item_list_len, self.target_item_idx, self.item_embedding_size, "sum") # one-hot feature gender_one_hot = tf.tile( tf.expand_dims(tf.one_hot(self.gender, self.GENDER_CNT), 1), [1, tf.shape(self.user_item_click_attention_embed)[1], 1]) client_type_one_hot = tf.tile( tf.expand_dims( tf.one_hot(self.client_type, self.CLIENT_TYPE_CNT), 1), [1, tf.shape(self.user_item_click_attention_embed)[1], 1]) # concat embedding self.user_embed_concat = tf.concat([ self.user_item_click_attention_embed, gender_one_hot, client_type_one_hot ], axis=-1) with tf.name_scope('layers'): user_layer_1 = tf.layers.dense( inputs=self.user_embed_concat, units=1024, activation=tf.nn.tanh, name='user_first', kernel_initializer=tf.initializers.glorot_normal(seed=1234), use_bias=False) user_layer_2 = tf.layers.dense( inputs=user_layer_1, units=512, activation=tf.nn.tanh, name='user_second', kernel_initializer=tf.initializers.glorot_normal(seed=1234), use_bias=False) user_layer_3 = tf.layers.dense( inputs=user_layer_2, units=128, activation=tf.nn.tanh, name='user_final', kernel_initializer=tf.initializers.glorot_normal(seed=1234), use_bias=False) # user参数写入summary # user_first_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, 'user_first') # user_second_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, 'user_second') # user_final_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, 'user_final') # tf.summary.histogram("user_layer_1_weights", user_first_vars[0]) # # tf.summary.histogram("user_layer_1_biases", user_first_vars[1]) # tf.summary.histogram("user_layer_1_output", user_layer_1) # # tf.summary.histogram("user_layer_2_weights", user_second_vars[0]) # # tf.summary.histogram("user_layer_2_biases", user_second_vars[1]) # tf.summary.histogram("user_layer_2_output", user_layer_2) # # # tf.summary.histogram("user_layer_3_weights", user_final_vars[0]) # # tf.summary.histogram("user_layer_3_biases", user_final_vars[1]) # tf.summary.histogram("user_layer_3_output", user_layer_3) self.user_embedding_final = user_layer_3 with tf.name_scope("item_tower"): # if self.mode == "train": item_id_embed = tf.nn.embedding_lookup(self.item_embedding, self.target_item_idx) cate_id_embed = tf.nn.embedding_lookup(self.cate_embedding, target_cate_idx) tag_id_embed = tf.nn.embedding_lookup(self.tag_embedding, target_tag_idx) target_embed_concat = tf.concat( [item_id_embed, cate_id_embed, tag_id_embed], axis=-1) with tf.name_scope('item_layers'): item_layer_1 = tf.layers.dense( inputs=target_embed_concat, units=256, activation=tf.nn.tanh, name='item_first', kernel_initializer=tf.initializers.glorot_normal(seed=1234), use_bias=False) item_layer_2 = tf.layers.dense( inputs=item_layer_1, units=128, activation=tf.nn.tanh, name='item_second', kernel_initializer=tf.initializers.glorot_normal(seed=1234), use_bias=False) # item参数写入summary # item_first_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, 'item_first') # tf.summary.histogram("item_layer_1_weights", item_first_vars[0]) # # tf.summary.histogram("item_layer_1_biases", item_first_vars[1]) # tf.summary.histogram("item_layer_1_output", item_layer_1) self.item_embeding_final = item_layer_2 # ## 计算logits # self.user_embedding_final_expand = tf.expand_dims(self.user_embedding_final, 1) # self.item_embeding_final = tf.transpose(self.item_embeding_final, perm=[0, 2, 1]) self.logits = tf.reduce_sum( tf.multiply(self.user_embedding_final, self.item_embeding_final), -1) # saved_model 输出 self.output_score = tf.nn.sigmoid(self.logits) tensor_info_output_score = tf.saved_model.utils.build_tensor_info( self.output_score) self.saved_model_outputs["logits"] = tensor_info_output_score
def main(_): if not tf.gfile.Exists(FLAGS.train_log_dir): tf.gfile.MakeDirs(FLAGS.train_log_dir) with tf.device(tf.train.replica_device_setter(FLAGS.ps_tasks)): # Force all input processing onto CPU in order to reserve the GPU for # the forward inference and back-propagation. with tf.name_scope('inputs'): with tf.device('/cpu:0'): images, one_hot_labels, _, _ = data_provider.provide_data( FLAGS.batch_size, FLAGS.dataset_dir) # Define the GANModel tuple. noise = tf.random_normal([FLAGS.batch_size, 64]) if FLAGS.conditional: generator_fn = networks.conditional_generator discriminator_fn = networks.conditional_discriminator generator_inputs = (noise, one_hot_labels) else: generator_fn = networks.generator discriminator_fn = networks.discriminator generator_inputs = noise gan_model = tfgan.gan_model( generator_fn, discriminator_fn, real_data=images, generator_inputs=generator_inputs) tfgan.eval.add_gan_model_image_summaries(gan_model) # Get the GANLoss tuple. Use the selected GAN loss functions. # (joelshor): Put this block in `with tf.name_scope('loss'):` when # cl/171610946 goes into the opensource release. gan_loss = tfgan.gan_loss(gan_model, gradient_penalty_weight=1.0, add_summaries=True) # Get the GANTrain ops using the custom optimizers and optional # discriminator weight clipping. with tf.name_scope('train'): gen_lr, dis_lr = _learning_rate() gen_opt, dis_opt = _optimizer(gen_lr, dis_lr, FLAGS.use_sync_replicas) train_ops = tfgan.gan_train_ops( gan_model, gan_loss, generator_optimizer=gen_opt, discriminator_optimizer=dis_opt, summarize_gradients=True, colocate_gradients_with_ops=True, aggregation_method=tf.AggregationMethod.EXPERIMENTAL_ACCUMULATE_N) tf.summary.scalar('generator_lr', gen_lr) tf.summary.scalar('discriminator_lr', dis_lr) # Run the alternating training loop. Skip it if no steps should be taken # (used for graph construction tests). sync_hooks = ([gen_opt.make_session_run_hook(FLAGS.task == 0), dis_opt.make_session_run_hook(FLAGS.task == 0)] if FLAGS.use_sync_replicas else []) status_message = tf.string_join( ['Starting train step: ', tf.as_string(tf.train.get_or_create_global_step())], name='status_message') if FLAGS.max_number_of_steps == 0: return sess_config = tf.ConfigProto( inter_op_parallelism_threads=FLAGS.inter_op_parallelism_threads, intra_op_parallelism_threads=FLAGS.intra_op_parallelism_threads) tfgan.gan_train( train_ops, hooks=( [tf.train.StopAtStepHook(num_steps=FLAGS.max_number_of_steps), tf.train.LoggingTensorHook([status_message], every_n_iter=10)] + sync_hooks), logdir=FLAGS.train_log_dir, master=FLAGS.master, is_chief=FLAGS.task == 0, config=sess_config)
def main(_): if not tf.gfile.Exists(FLAGS.train_log_dir): tf.gfile.MakeDirs(FLAGS.train_log_dir) # Force all input processing onto CPU in order to reserve the GPU for # the forward inference and back-propagation. with tf.name_scope('inputs'): with tf.device('/cpu:0'): images, one_hot_labels, _ = data_provider.provide_data( 'train', FLAGS.batch_size, FLAGS.dataset_dir, num_threads=4) # Define the GANModel tuple. Optionally, condition the GAN on the label or # use an InfoGAN to learn a latent representation. if FLAGS.gan_type == 'unconditional': gan_model = tfgan.gan_model( generator_fn=networks.unconditional_generator, discriminator_fn=networks.unconditional_discriminator, real_data=images, generator_inputs=tf.random_normal( [FLAGS.batch_size, FLAGS.noise_dims])) elif FLAGS.gan_type == 'conditional': noise = tf.random_normal([FLAGS.batch_size, FLAGS.noise_dims]) gan_model = tfgan.gan_model( generator_fn=networks.conditional_generator, discriminator_fn=networks.conditional_discriminator, real_data=images, generator_inputs=(noise, one_hot_labels)) elif FLAGS.gan_type == 'infogan': cat_dim, cont_dim = 10, 2 generator_fn = functools.partial( networks.infogan_generator, categorical_dim=cat_dim) discriminator_fn = functools.partial( networks.infogan_discriminator, categorical_dim=cat_dim, continuous_dim=cont_dim) unstructured_inputs, structured_inputs = util.get_infogan_noise( FLAGS.batch_size, cat_dim, cont_dim, FLAGS.noise_dims) gan_model = tfgan.infogan_model( generator_fn=generator_fn, discriminator_fn=discriminator_fn, real_data=images, unstructured_generator_inputs=unstructured_inputs, structured_generator_inputs=structured_inputs) tfgan.eval.add_gan_model_image_summaries(gan_model, FLAGS.grid_size) # Get the GANLoss tuple. You can pass a custom function, use one of the # already-implemented losses from the losses library, or use the defaults. with tf.name_scope('loss'): mutual_information_penalty_weight = (1.0 if FLAGS.gan_type == 'infogan' else 0.0) gan_loss = tfgan.gan_loss( gan_model, gradient_penalty_weight=1.0, mutual_information_penalty_weight=mutual_information_penalty_weight, add_summaries=True) tfgan.eval.add_regularization_loss_summaries(gan_model) # Get the GANTrain ops using custom optimizers. with tf.name_scope('train'): gen_lr, dis_lr = _learning_rate(FLAGS.gan_type) train_ops = tfgan.gan_train_ops( gan_model, gan_loss, generator_optimizer=tf.train.AdamOptimizer(gen_lr, 0.5), discriminator_optimizer=tf.train.AdamOptimizer(dis_lr, 0.5), summarize_gradients=True, aggregation_method=tf.AggregationMethod.EXPERIMENTAL_ACCUMULATE_N) # Run the alternating training loop. Skip it if no steps should be taken # (used for graph construction tests). status_message = tf.string_join( ['Starting train step: ', tf.as_string(tf.train.get_or_create_global_step())], name='status_message') if FLAGS.max_number_of_steps == 0: return tfgan.gan_train( train_ops, hooks=[tf.train.StopAtStepHook(num_steps=FLAGS.max_number_of_steps), tf.train.LoggingTensorHook([status_message], every_n_iter=10)], logdir=FLAGS.train_log_dir, get_hooks_fn=tfgan.get_joint_train_hooks())
def main(_): if FLAGS.train_log_dir is None: timestamp = datetime.now().strftime("%Y-%m-%d_%H%M%S") logdir = "{}/{}-{}".format('data_out', "CycleGanEst", timestamp) else: logdir = FLAGS.train_log_dir if not tf.gfile.Exists(logdir): tf.gfile.MakeDirs(logdir) with tf.device(tf.train.replica_device_setter(FLAGS.ps_tasks)): with tf.name_scope('inputs'): if FLAGS.use_dataset == 'synth': # Generated two channels. First channel image, second channel is mask. images_healthy, images_cancer = data_provider.provide_synth_dataset( batch_size=FLAGS.batch_size, img_size=(FLAGS.height, FLAGS.width)) elif FLAGS.use_dataset == 'cbis': # Generated two channels. First channel image, second channel is mask. images_healthy, images_cancer = data_provider.provide_cbis_dataset( [FLAGS.image_x_file, FLAGS.image_y_file], batch_size=FLAGS.batch_size, img_size=(FLAGS.height, FLAGS.width)) else: images_healthy, images_cancer = data_provider.provide_custom_datasets( [FLAGS.image_set_x_file_pattern, FLAGS.image_set_y_file_pattern], batch_size=FLAGS.batch_size, img_size=(FLAGS.height, FLAGS.width)) # Define CycleGAN model. print("images healthy", images_healthy.get_shape()) print("images cancer", images_cancer.get_shape()) cyclegan_model = _define_model(images_healthy, images_cancer, FLAGS.include_masks) # Define CycleGAN loss. if FLAGS.gan_type == 'lsgan': print("Using lsgan") generator_loss_fn = _args_to_gan_model(tfgan.losses.wargs.least_squares_generator_loss) discriminator_loss_fn = _args_to_gan_model(tfgan.losses.wargs.least_squares_discriminator_loss) elif FLAGS.gan_type == 'hinge': print("Using hinge") generator_loss_fn = _args_to_gan_model(mygan.hinge_generator_loss) discriminator_loss_fn = _args_to_gan_model(mygan.hinge_discriminator_loss) else: raise ValueError("Unknown gan type.") cyclegan_loss = tfgan.cyclegan_loss( cyclegan_model, cycle_consistency_loss_weight=FLAGS.cycle_consistency_loss_weight, generator_loss_fn=generator_loss_fn, discriminator_loss_fn=discriminator_loss_fn, cycle_consistency_loss_fn=functools.partial( mygan.cycle_consistency_loss, lambda_identity=FLAGS.loss_identity_lambda), tensor_pool_fn=tfgan.features.tensor_pool) # Define CycleGAN train ops. train_ops = _define_train_ops(cyclegan_model, cyclegan_loss) # Training train_steps = tfgan.GANTrainSteps(1, 1) status_message = tf.string_join( ['Starting train step: ', tf.as_string(tf.train.get_or_create_global_step())], name='status_message') if not FLAGS.max_number_of_steps: return # To avoid problems with GPU memmory. config = tf.ConfigProto() config.gpu_options.allow_growth = True # Do not assign whole gpu memory, just use it on the go # If a operation is not define it the default device, let it execute in another. config.allow_soft_placement = True hooks = [ tf.train.StopAtStepHook(num_steps=FLAGS.max_number_of_steps), tf.train.LoggingTensorHook([status_message], every_n_iter=10), ] if FLAGS.checkpoint_hook_steps > 0: chkpt_hook = tf.train.CheckpointSaverHook( checkpoint_dir=os.path.join(logdir, 'chook'), save_steps=FLAGS.checkpoint_hook_steps, saver=tf.train.Saver(max_to_keep=300)) hooks.append(chkpt_hook) tfgan.gan_train( train_ops, logdir, hooks=hooks, get_hooks_fn=tfgan.get_sequential_train_hooks(train_steps), master=FLAGS.master, is_chief=FLAGS.task == 0, config=config)
def testTopKTerminatedHypsOp(self): with self.session(use_gpu=False) as sess: b_size = 8 num_beams = 2 num_hyps_per_beam = b_size / num_beams seq_len = 6 scores = tf.random_uniform([b_size, 5], seed=12345) atten_probs = tf.random_uniform([b_size, 3], seed=12345) src_seq_lengths = [3, 3] best_scores = tf.zeros([num_beams]) cumulative_scores = tf.zeros([b_size]) in_scores = tf.zeros([seq_len, b_size]) in_hyps = tf.zeros([seq_len, b_size], dtype=tf.int32) in_prev_hyps = tf.zeros([seq_len, b_size], dtype=tf.int32) in_done_hyps = tf.as_string(tf.zeros([seq_len, b_size], dtype=tf.int32)) in_atten_probs = tf.zeros([seq_len, b_size, 3]) (out_best_scores_0, out_cumulative_scores_0, out_scores_0, out_hyps_0, out_prev_hyps_0, out_done_hyps_0, out_atten_probs_0, _) = py_x_ops.beam_search_step( scores, atten_probs, best_scores, cumulative_scores, in_scores, in_hyps, in_prev_hyps, in_done_hyps, in_atten_probs, [], 0, eos_id=2, beam_size=3.0, num_hyps_per_beam=num_hyps_per_beam) outputs = py_x_ops.beam_search_step( scores, atten_probs, out_best_scores_0, out_cumulative_scores_0, out_scores_0, out_hyps_0, out_prev_hyps_0, out_done_hyps_0, out_atten_probs_0, [], 1, eos_id=2, beam_size=3.0, num_hyps_per_beam=num_hyps_per_beam) # Get the topk terminated hyps. in_done_hyps = outputs[5] topk_hyps = py_x_ops.top_k_terminated_hyps( in_done_hyps, src_seq_lengths, k=2, num_hyps_per_beam=num_hyps_per_beam, length_normalization=0.2, coverage_penalty=0.2, target_seq_length_ratio=1.0) seq_ids, seq_lens, seq_scores = py_x_ops.unpack_hyp( tf.reshape(topk_hyps, [-1]), max_seq_length=5) k1, k2, k3, k4 = sess.run([topk_hyps, seq_ids, seq_lens, seq_scores]) print(np.array_repr(k1)) assert k1.size == 4 expected_top1_for_beam_0 = """ beam_id: 0 ids: 1 ids: 2 scores: 0.86230338 scores: 0.65504861 atten_vecs { prob: 0.45372832 prob: 0.86230338 prob: 0.65504861 } atten_vecs { prob: 0.45372832 prob: 0.86230338 prob: 0.65504861 } normalized_score: 1.002714 """ expected_top2_for_beam_1 = """ beam_id: 1 ids: 3 ids: 2 scores: 0.38127339 scores: 0.57700801 atten_vecs { prob: 0.38612545 prob: 0.42067075 prob: 0.84442794 } atten_vecs { prob: 0.18693292 prob: 0.17821217 prob: 0.66380036 } normalized_score: 0.480028 """ self._SameHyp(expected_top1_for_beam_0, k1[0, 0]) self._SameHyp(expected_top2_for_beam_1, k1[1, 1]) self.assertAllClose( k2, [[1, 2, 0, 0, 0], [4, 2, 0, 0, 0], [4, 2, 0, 0, 0], [3, 2, 0, 0, 0]]) self.assertAllClose(k3, [2, 2, 2, 2]) self.assertAllClose(k4, [1.002714, 0.684296, 0.522484, 0.480028])
def evaluate(self, path_to_checkpoint, image_eval, length_eval, digits_eval, global_step): batch_size = 128 needs_include_length = False accuracy_val = 0.0 with tf.Graph().as_default(): image_batch = tf.placeholder(tf.float32, shape=[ None, self.image_size, self.image_size, self.num_channels ]) length_batch = tf.placeholder(tf.int32, shape=[None]) digits_batch = tf.placeholder(tf.int32, shape=[None, self.digits_nums]) num_examples = image_eval.shape[0] num_batches = num_examples / batch_size # length_logits, digits_logits = Model.inference(image_batch, drop_rate=0.0) length_logits, digits_logits = Model.forward(image_batch, 1.0) length_predictions = tf.argmax(length_logits, axis=1) digits_predictions = tf.argmax(digits_logits, axis=2) if needs_include_length: labels = tf.concat( [tf.reshape(length_batch, [-1, 1]), digits_batch], axis=1) predictions = tf.concat([ tf.reshape(length_predictions, [-1, 1]), digits_predictions ], axis=1) else: labels = digits_batch predictions = digits_predictions #correct_pre = tf.equal(tf.argmax(labels,axis=1),tf.argmax(predictions,axis=1)) #accuracy = tf.reduce_mean(tf.cast(correct_pre, tf.float32)) labels_string = tf.reduce_join(tf.as_string(labels), axis=1) predictions_string = tf.reduce_join(tf.as_string(predictions), axis=1) correct_pre = tf.equal(labels_string, predictions_string) accuracy = tf.reduce_mean(tf.cast(correct_pre, tf.float32)) tf.summary.image('image', image_batch) tf.summary.scalar('accuracy', accuracy) tf.summary.histogram( 'variables', tf.concat([ tf.reshape(var, [-1]) for var in tf.trainable_variables() ], axis=0)) summary = tf.summary.merge_all() with tf.Session() as sess: sess.run([ tf.global_variables_initializer(), tf.local_variables_initializer() ]) restorer = tf.train.Saver() restorer.restore(sess, path_to_checkpoint) for _ in range(math.floor(num_examples / batch_size)): image_batch_input, length_batch_input, digits_batch_input = Donkey.build_batch( image_eval, length_eval, digits_eval, batch_size=batch_size) feed_dict = { image_batch: image_batch_input, length_batch: length_batch_input, digits_batch: digits_batch_input } accuracy_step, summary_val = sess.run([accuracy, summary], feed_dict=feed_dict) accuracy_val += accuracy_step self.summary_writer.add_summary(summary_val, global_step=global_step) accuracy_val = accuracy_val / math.floor( num_examples / batch_size) return accuracy_val * 100
def testComplex(self): float_inputs_ = [0, 1, -1, 0.5, 0.25, 0.125, complex("INF"), complex("NAN"), complex("-INF")] complex_inputs_ = [(x + (x + 1) * 1j) for x in float_inputs_] with self.test_session(): for dtype in (tf.complex64,): input_ = tf.placeholder(dtype) def clean_nans(s_l): return [s.decode("ascii").replace("-nan", "nan") for s in s_l] output = tf.as_string(input_, shortest=True) result = output.eval(feed_dict={input_: complex_inputs_}) self.assertAllEqual(clean_nans(result), ["(%g,%g)" % (x.real, x.imag) for x in complex_inputs_]) output = tf.as_string(input_, scientific=True) result = output.eval(feed_dict={input_: complex_inputs_}) self.assertAllEqual(clean_nans(result), ["(%e,%e)" % (x.real, x.imag) for x in complex_inputs_]) output = tf.as_string(input_) result = output.eval(feed_dict={input_: complex_inputs_}) self.assertAllEqual(clean_nans(result), ["(%f,%f)" % (x.real, x.imag) for x in complex_inputs_]) output = tf.as_string(input_, width=3) result = output.eval(feed_dict={input_: complex_inputs_}) self.assertAllEqual(clean_nans(result), ["(%03f,%03f)" % (x.real, x.imag) for x in complex_inputs_]) output = tf.as_string(input_, width=3, fill="0", shortest=True) result = output.eval(feed_dict={input_: complex_inputs_}) self.assertAllEqual(clean_nans(result), ["(%03g,%03g)" % (x.real, x.imag) for x in complex_inputs_]) output = tf.as_string(input_, precision=10, width=3) result = output.eval(feed_dict={input_: complex_inputs_}) self.assertAllEqual(clean_nans(result), ["(%03.10f,%03.10f)" % (x.real, x.imag) for x in complex_inputs_]) output = tf.as_string(input_, precision=10, width=3, fill="0", shortest=True) result = output.eval(feed_dict={input_: complex_inputs_}) self.assertAllEqual(clean_nans(result), ["(%03.10g,%03.10g)" % (x.real, x.imag) for x in complex_inputs_]) with self.assertRaisesOpError("Cannot select both"): output = tf.as_string(input_, scientific=True, shortest=True) output.eval(feed_dict={input_: complex_inputs_})
def testTrainWithSparseTensorAndDenseFeaturesLayer(self, agent_class): obs_spec = { 'dense': tensor_spec.BoundedTensorSpec(dtype=tf.float32, shape=[3], minimum=-10.0, maximum=10.0), 'sparse_terms': tf.SparseTensorSpec(dtype=tf.string, shape=[4]), 'sparse_frequencies': tf.SparseTensorSpec(dtype=tf.float32, shape=[4]), } cat_column = ( tf.compat.v2.feature_column.categorical_column_with_hash_bucket( 'sparse_terms', hash_bucket_size=5)) weighted_cat_column = ( tf.compat.v2.feature_column.weighted_categorical_column( cat_column, weight_feature_key='sparse_frequencies')) feature_columns = [ tf.compat.v2.feature_column.numeric_column('dense', [3]), tf.compat.v2.feature_column.embedding_column( weighted_cat_column, 3), ] dense_features_layer = tf.compat.v2.keras.layers.DenseFeatures( feature_columns) time_step_spec = ts.time_step_spec(obs_spec) q_net = q_network.QNetwork(time_step_spec.observation, self._action_spec, preprocessing_combiner=dense_features_layer) agent = agent_class(time_step_spec, self._action_spec, q_network=q_net, optimizer=tf.compat.v1.train.AdamOptimizer()) observations = tensor_spec.sample_spec_nest(obs_spec, outer_dims=[5, 2]) # sparse_terms and sparse_frequencies must be defined on matching indices. observations['sparse_terms'] = tf.SparseTensor( indices=observations['sparse_frequencies'].indices, values=tf.as_string( tf.math.round(observations['sparse_frequencies'].values)), dense_shape=observations['sparse_frequencies'].dense_shape) if not tf.executing_eagerly(): # Mimic unknown inner dims on the SparseTensor def _unknown_inner_shape(t): if not isinstance(t, tf.SparseTensor): return t return tf.SparseTensor( indices=t.indices, values=t.values, dense_shape=tf.compat.v1.placeholder_with_default( t.dense_shape, shape=t.dense_shape.shape)) observations = tf.nest.map_structure(_unknown_inner_shape, observations) self.assertIsNone( tf.get_static_value(observations['sparse_terms'].dense_shape)) time_step = ts.restart(observations, batch_size=[5, 2]) action_step = tensor_spec.sample_spec_nest(self._action_spec, outer_dims=[5, 2]) p_step = policy_step.PolicyStep(action=action_step, state=(), info=()) traj = trajectory.from_transition(time_step, p_step, time_step) loss_info = agent.train(traj) self.evaluate(tf.compat.v1.global_variables_initializer()) loss_info = self.evaluate(loss_info) self.assertGreater(loss_info.loss, 0)
def init_predict_graph(self): """ init predict model graph :return: """ # split 1-D String dense Tensor to words SparseTensor self.input_sentences = tf.placeholder(dtype=tf.string, shape=[None], name='input_sentences') sparse_words = tf.string_split(self.input_sentences, delimiter=' ') # slice SparseTensor valid_indices = tf.less(sparse_words.indices, tf.constant([self.num_steps], dtype=tf.int64)) valid_indices = tf.reshape(tf.split(valid_indices, [1, 1], axis=1)[1], [-1]) valid_sparse_words = tf.sparse_retain(sparse_words, valid_indices) excess_indices = tf.greater_equal(sparse_words.indices, tf.constant([self.num_steps], dtype=tf.int64)) excess_indices = tf.reshape(tf.split(excess_indices, [1, 1], axis=1)[1], [-1]) excess_sparse_words = tf.sparse_retain(sparse_words, excess_indices) # compute sentences lengths int_values = tf.ones(shape=tf.shape(valid_sparse_words.values), dtype=tf.int64) int_valid_sparse_words = tf.SparseTensor(indices=valid_sparse_words.indices, values=int_values, dense_shape=valid_sparse_words.dense_shape) input_sentences_lengths = tf.sparse_reduce_sum(int_valid_sparse_words, axis=1) # sparse to dense default_padding_word = self.data_utils._START_VOCAB[0] words = tf.sparse_to_dense(sparse_indices=valid_sparse_words.indices, output_shape=[valid_sparse_words.dense_shape[0], self.num_steps], sparse_values=valid_sparse_words.values, default_value=default_padding_word) # dict words to ids with open(os.path.join(self.vocab_path, 'words_vocab.txt'), encoding='utf-8', mode='rt') as data_file: words_table_list = [line.strip() for line in data_file if line.strip()] words_table_tensor = tf.constant(words_table_list, dtype=tf.string) words_table = lookup.index_table_from_tensor(mapping=words_table_tensor, default_value=self.data_utils._START_VOCAB_ID[3]) # words_table = lookup.index_table_from_file(os.path.join(vocab_path, 'words_vocab.txt'), default_value=3) words_ids = words_table.lookup(words) # blstm model predict with tf.variable_scope('model', reuse=None): logits = self.sequence_labeling_model.inference(words_ids, input_sentences_lengths, self.num_classes, is_training=False) if self.use_crf: logits = tf.reshape(logits, shape=[-1, self.num_steps, self.num_classes]) transition_params = tf.get_variable("transitions", [self.num_classes, self.num_classes]) input_sentences_lengths = tf.to_int32(input_sentences_lengths) predict_labels_ids, sequence_scores = crf.crf_decode(logits, transition_params, input_sentences_lengths) predict_labels_ids = tf.to_int64(predict_labels_ids) sequence_scores = tf.reshape(sequence_scores, shape=[-1, 1]) normalized_sequence_scores = self.tensorflow_utils.score_normalize(sequence_scores) predict_scores = tf.matmul(normalized_sequence_scores, tf.ones(shape=[1, self.num_steps], dtype=tf.float32)) else: props = tf.nn.softmax(logits) max_prop_values, max_prop_indices = tf.nn.top_k(props, k=1) predict_labels_ids = tf.reshape(max_prop_indices, shape=[-1, self.num_steps]) predict_labels_ids = tf.to_int64(predict_labels_ids) predict_scores = tf.reshape(max_prop_values, shape=[-1, self.num_steps]) predict_scores = tf.as_string(predict_scores, precision=3) # dict ids to labels with open(os.path.join(self.vocab_path, 'labels_vocab.txt'), encoding='utf-8', mode='rt') as data_file: labels_table_list = [line.strip() for line in data_file if line.strip()] labels_table_tensor = tf.constant(labels_table_list, dtype=tf.string) labels_table = lookup.index_to_string_table_from_tensor(mapping=labels_table_tensor, default_value=self.default_label) # labels_table = lookup.index_to_string_table_from_file(os.path.join(vocab_path, 'labels_vocab.txt'), default_value='O') predict_labels = labels_table.lookup(predict_labels_ids) sparse_predict_labels = self.tensorflow_utils.sparse_concat(predict_labels, valid_sparse_words, excess_sparse_words, self.default_label) sparse_predict_scores = self.tensorflow_utils.sparse_concat(predict_scores, valid_sparse_words, excess_sparse_words, '0.0') self.format_predict_labels = self.tensorflow_utils.sparse_string_join(sparse_predict_labels, 'predict_labels') self.format_predict_scores = self.tensorflow_utils.sparse_string_join(sparse_predict_scores, 'predict_scores') saver = tf.train.Saver() tables_init_op = tf.tables_initializer() self.sess = tf.Session() self.sess.run(tables_init_op) ckpt = tf.train.get_checkpoint_state(self.checkpoint_path) if ckpt and ckpt.model_checkpoint_path: print('read model from {}'.format(ckpt.model_checkpoint_path)) saver.restore(self.sess, ckpt.model_checkpoint_path) else: print('No checkpoint file found at %s' % self.checkpoint_path) return
def train(args): tf.reset_default_graph() session = tf.Session() # Random permutation features, labels = extract_data(args) np.random.seed([111]) shuffled = np.random.permutation(np.arange(len(labels))) features_shuffled = features[shuffled] labels_shuffled = labels[shuffled] labels_shuffled = utils.one_hot_encode(labels_shuffled) x = tf.placeholder("float", [None, hparams.n_steps, hparams.n_input], name="x") y = tf.placeholder("float", [None, hparams.n_classes], name="y") bias = tf.Variable(tf.random_normal([hparams.n_classes]), name="bias") weight = tf.Variable(tf.truncated_normal( [hparams.n_hidden, hparams.n_classes], stddev=0.1), name="weights") # Inference prediction = RNN(x, weight, bias) # Loss and optimizer loss_f = tf.reduce_mean(-tf.reduce_sum(y * tf.log(prediction)), name="lost_f") optimizer = tf.train.AdamOptimizer(earning_rate=hparams.learning_rate, name="optimizer").minimize(loss_f) # Evaluate correct_pred = tf.equal(tf.argmax(prediction, 1), tf.argmax(y, 1), name="correct_pred") accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32), name="accuracy") confusion_matrix = tf.as_string(tf.confusion_matrix(tf.argmax(prediction, 1), tf.argmax(y, 1), num_classes=hparams.n_classes, name="confusion_matrix")) # Summaries accuracy_summary = tf.summary.scalar('accuracy_summary', accuracy) loss_summary = tf.summary.scalar('loss_summary', loss_f) confusion_matrix_summary = tf.summary.text("confusion_matrix_summary", confusion_matrix) # Initializing the variables init = tf.global_variables_initializer() session.run(init) writer = tf.summary.FileWriter('./graphs', session.graph) step = 0 for itr in range(hparams.training_iters): print("%s epoch / %s" % (int(itr+1), hparams.training_iters)) # Shuffle training data shuffled = np.random.permutation(np.arange(len(labels_shuffled))) features_shuffled = features_shuffled[shuffled] labels_shuffled = labels_shuffled[shuffled] cutoff = int(len(labels_shuffled)*0.9) # Select validation data tr_features, val_features = features_shuffled[:cutoff], features_shuffled[cutoff:] tr_labels, val_labels = labels_shuffled[:cutoff], labels_shuffled[cutoff:] num_batches = int(len(tr_labels)/hparams.batch_size) + 1 for i in range(num_batches): # Select train data min_index = i * hparams.batch_size max_index = np.min( [len(tr_features), ((i+1) * hparams.batch_size)]) batch_x = tr_features[min_index:max_index] batch_y = tr_labels[min_index:max_index] session.run([optimizer, loss_f], feed_dict={x: batch_x, y: batch_y}) # Calculate accuracy acc = session.run(accuracy, feed_dict={x: batch_x, y: batch_y}) # Calculate loss loss = session.run(loss_f, feed_dict={x: batch_x, y: batch_y}) print("batch " + str(i+1) + " / " + str(num_batches) + ", Traning Loss= " + "{:.6f}".format(loss) + ", Training Accuracy= " + "{:.5f}".format(acc)) acc_s, loss_s = session.run([accuracy_summary, loss_summary], feed_dict={ x: batch_x, y: batch_y}) # Add summaries writer.add_summary(acc_s, step) writer.add_summary(loss_s, step) step += 1 print('Validation accuracy: ', round(session.run(accuracy, feed_dict={x: val_features, y: val_labels}), 3)) print('Validation predictions: ', session.run(prediction, feed_dict={x: val_features, y: val_labels})) print('Validation correct predictions: ', session.run(correct_pred, feed_dict={x: val_features, y: val_labels})) # Calculate confusion matrix con_mat = session.run(confusion_matrix_summary, feed_dict={ x: val_features, y: val_labels}) # Add summary writer.add_summary(con_mat, (num_batches*(itr+1))) # Save model saver = tf.train.Saver() save_path = saver.save(session, "./model.ckpt") print("Model saved in path: %s" % save_path)
def main(): """Main function.""" # Setup logging.basicConfig(level=LOGLEVEL, format=LOG_FORMAT) params, config = setup() LOGGER.info("Using parameters:\n%s", pformat(params)) LOGGER.info("Using configurations:\n%s", pformat(config)) # ================================== Data ================================== # Load training data train_x, _ = load_training_data(params, config) # ================================= Model ================================== # Build model model = Model(params) if params['is_accompaniment']: train_c = tf.expand_dims( train_x[..., params['condition_track_idx']], -1) train_nodes = model( x=train_x, c=train_c, mode='train', params=params, config=config) else: train_nodes = model( x=train_x, mode='train', params=params, config=config) # Log number of parameters in the model def get_n_params(var_list): """Return the number of variables in a variable list.""" return int(np.sum([np.product( [x.value for x in var.get_shape()]) for var in var_list])) LOGGER.info("Number of trainable parameters in {}: {:,}".format( model.name, get_n_params(tf.trainable_variables(model.name)))) for component in model.components: LOGGER.info("Number of trainable parameters in {}: {:,}".format( component.name, get_n_params(tf.trainable_variables( model.name + '/' + component.name)))) # ================================ Sampler ================================= if config['save_samples_steps'] > 0: # Get sampler inputs sample_x, sample_y, sample_z = load_or_create_samples(params, config) # Create sampler configurations sampler_config = { 'result_dir': config['sample_dir'], 'suffix': tf.as_string(train_nodes['gen_step']), 'image_grid': config['sample_grid'], 'colormap': np.array(config['colormap']).T, 'midi': config['midi'], 'collect_save_arrays_op': config['save_array_samples'], 'collect_save_images_op': config['save_image_samples'], 'collect_save_pianorolls_op': config['save_pianoroll_samples']} # Get prediction nodes placeholder_z = tf.placeholder(tf.float32, shape=sample_z.shape) placeholder_y = None if params['is_accompaniment']: c_shape = np.append(sample_x.shape[:-1], 1) placeholder_c = tf.placeholder(tf.float32, shape=c_shape) predict_nodes = model( z=placeholder_z, y=placeholder_y, c=placeholder_c, mode='predict', params=params, config=sampler_config) else: predict_nodes = model( z=placeholder_z, y=placeholder_y, mode='predict', params=params, config=sampler_config) # Get sampler op sampler_op = tf.group([ predict_nodes[key] for key in ( 'save_arrays_op', 'save_images_op', 'save_pianorolls_op') if key in predict_nodes]) sampler_op_no_pianoroll = tf.group([ predict_nodes[key] for key in ('save_arrays_op', 'save_images_op') if key in predict_nodes]) # ================================ Metrics ================================= if config['evaluate_steps'] > 0: binarized = tf.round(.5 * (predict_nodes['fake_x'] + 1.)) save_metric_ops = get_save_metric_ops( binarized, params['beat_resolution'], train_nodes['gen_step'], config['eval_dir']) save_metrics_op = tf.group(save_metric_ops) # ========================== Training Preparation ========================== # Get tensorflow session config tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True # Training hooks global_step = tf.train.get_global_step() steps_per_iter = config['n_dis_updates_per_gen_update'] + 1 hooks = [tf.train.NanTensorHook(train_nodes['loss'])] # Tensor logger tensor_logger = { 'step': train_nodes['gen_step'], 'gen_loss': train_nodes['gen_loss'], 'dis_loss': train_nodes['dis_loss']} step_logger = open(os.path.join(config['log_dir'], 'step.log'), 'w') # ======================= Monitored Training Session ======================= LOGGER.info("Training start.") with tf.train.MonitoredTrainingSession( save_checkpoint_steps=config['save_checkpoint_steps'] * steps_per_iter, save_summaries_steps=config['save_summaries_steps'] * steps_per_iter, checkpoint_dir=config['model_dir'], log_step_count_steps=0, hooks=hooks, config=tf_config) as sess: # Get global step value step = tf.train.global_step(sess, global_step) if step == 0: step_logger.write('# step, gen_loss, dis_loss\n') # ============================== Training ============================== if step >= config['steps']: LOGGER.info("Global step has already exceeded total steps.") step_logger.close() return # Training iteration while step < config['steps']: # Train the discriminator if step < 10: n_dis_updates = 10 * config['n_dis_updates_per_gen_update'] else: n_dis_updates = config['n_dis_updates_per_gen_update'] for _ in range(n_dis_updates): sess.run(train_nodes['train_ops']['dis']) # Train the generator log_loss_steps = config['log_loss_steps'] or 100 if (step + 1) % log_loss_steps == 0: step, _, tensor_logger_values = sess.run([ train_nodes['gen_step'], train_nodes['train_ops']['gen'], tensor_logger]) # Logger if config['log_loss_steps'] > 0: LOGGER.info("step={}, {}".format( tensor_logger_values['step'], ', '.join([ '{}={: 8.4E}'.format(key, value) for key, value in tensor_logger_values.items() if key != 'step']))) step_logger.write("{}, {: 10.6E}, {: 10.6E}\n".format( tensor_logger_values['step'], tensor_logger_values['gen_loss'], tensor_logger_values['dis_loss'])) else: step, _ = sess.run([ train_nodes['gen_step'], train_nodes['train_ops']['gen']]) # Run sampler if ((config['save_samples_steps'] > 0) and (step % config['save_samples_steps'] == 0)): LOGGER.info("Running sampler") feed_dict_sampler = {placeholder_z: sample_z} if params['is_accompaniment']: feed_dict_sampler[placeholder_c] = np.expand_dims( sample_x[..., params['condition_track_idx']], -1) if step < 3000: sess.run( sampler_op_no_pianoroll, feed_dict=feed_dict_sampler) else: sess.run(sampler_op, feed_dict=feed_dict_sampler) # Run evaluation if ((config['evaluate_steps'] > 0) and (step % config['evaluate_steps'] == 0)): LOGGER.info("Running evaluation") feed_dict_evaluation = { placeholder_z: scipy.stats.truncnorm.rvs(-2, 2, size=( np.prod(config['sample_grid']), params['latent_dim']))} if params['is_accompaniment']: feed_dict_evaluation[placeholder_c] = np.expand_dims( sample_x[..., params['condition_track_idx']], -1) sess.run(save_metrics_op, feed_dict=feed_dict_evaluation) # Stop training if stopping criterion suggests if sess.should_stop(): break LOGGER.info("Training end") step_logger.close()
def main(_): if not tf.gfile.Exists(FLAGS.train_log_dir): tf.gfile.MakeDirs(FLAGS.train_log_dir) with tf.device(tf.train.replica_device_setter(FLAGS.ps_tasks)): # Get real and distorted images. with tf.device('/cpu:0'), tf.name_scope('inputs'): real_images = data_provider.provide_data( 'train', FLAGS.batch_size, dataset_dir=FLAGS.dataset_dir, patch_size=FLAGS.patch_size) distorted_images = _distort_images(real_images, downscale_size=int( FLAGS.patch_size / 2), upscale_size=FLAGS.patch_size) # Create a GANModel tuple. gan_model = tfgan.gan_model(generator_fn=networks.generator, discriminator_fn=networks.discriminator, real_data=real_images, generator_inputs=distorted_images) tfgan.eval.add_image_comparison_summaries(gan_model, num_comparisons=3, display_diffs=True) tfgan.eval.add_gan_model_image_summaries(gan_model, grid_size=3) # Define the GANLoss tuple using standard library functions. with tf.name_scope('losses'): gan_loss = tfgan.gan_loss( gan_model, generator_loss_fn=tfgan.losses.least_squares_generator_loss, discriminator_loss_fn=tfgan.losses. least_squares_discriminator_loss) # Define the standard L1 pixel loss. l1_pixel_loss = tf.norm( gan_model.real_data - gan_model.generated_data, ord=1) / FLAGS.patch_size**2 # Modify the loss tuple to include the pixel loss. Add summaries as well. gan_loss = tfgan.losses.combine_adversarial_loss( gan_loss, gan_model, l1_pixel_loss, weight_factor=FLAGS.weight_factor) with tf.name_scope('train_ops'): # Get the GANTrain ops using the custom optimizers and optional # discriminator weight clipping. gen_lr, dis_lr = _lr(FLAGS.generator_lr, FLAGS.discriminator_lr) gen_opt, dis_opt = _optimizer(gen_lr, dis_lr) train_ops = tfgan.gan_train_ops( gan_model, gan_loss, generator_optimizer=gen_opt, discriminator_optimizer=dis_opt, summarize_gradients=True, colocate_gradients_with_ops=True, aggregation_method=tf.AggregationMethod. EXPERIMENTAL_ACCUMULATE_N, transform_grads_fn=tf.contrib.training.clip_gradient_norms_fn( 1e3)) tf.summary.scalar('generator_lr', gen_lr) tf.summary.scalar('discriminator_lr', dis_lr) # Use GAN train step function if using adversarial loss, otherwise # only train the generator. train_steps = tfgan.GANTrainSteps( generator_train_steps=1, discriminator_train_steps=int(FLAGS.weight_factor > 0)) # Run the alternating training loop. Skip it if no steps should be taken # (used for graph construction tests). status_message = tf.string_join([ 'Starting train step: ', tf.as_string(tf.train.get_or_create_global_step()) ], name='status_message') if FLAGS.max_number_of_steps == 0: return tfgan.gan_train( train_ops, FLAGS.train_log_dir, get_hooks_fn=tfgan.get_sequential_train_hooks(train_steps), hooks=[ tf.train.StopAtStepHook(num_steps=FLAGS.max_number_of_steps), tf.train.LoggingTensorHook([status_message], every_n_iter=10) ], master=FLAGS.master, is_chief=FLAGS.task == 0)
def main(_): if not tf.gfile.Exists(FLAGS.job_dir): tf.gfile.MakeDirs(FLAGS.job_dir) sess_config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True)) with tf.name_scope('inputs'): with tf.device('/cpu:0'): correct, wrong, context, label = train_input_fn( data_dir=FLAGS.data_dir) gan_model = tfgan.gan_model( generator_fn=functools.partial(generator_fn, weight_decay=FLAGS.weight_decay), discriminator_fn=functools.partial(discriminator_fn, weight_decay=FLAGS.weight_decay), real_data=correct, generator_inputs=(tf.random_normal([FLAGS.batch_size, FLAGS.z_dim]), context, wrong), check_shapes=False) my_summary_image('1_input', gan_model.real_data) my_summary_image('2_fake', gan_model.generated_data[0]) my_summary_image('3_wrong', gan_model.generator_inputs[2]) for variable in tf.global_variables(): tf.summary.histogram(variable.op.name, variable) with tf.name_scope('loss'): g_loss = tfgan.gan_loss(gan_model, generator_loss_fn=generator_loss, discriminator_loss_fn=discriminator_loss, add_summaries=True) tfgan.eval.add_regularization_loss_summaries(gan_model) global_step = tf.train.get_or_create_global_step() generator_lr = tf.train.exponential_decay(FLAGS.generator_lr, global_step, FLAGS.decay_steps, 0.5, staircase=True) discriminator_lr = tf.train.exponential_decay(FLAGS.discriminator_lr, global_step, FLAGS.decay_steps, 0.5, staircase=True) tf.summary.scalar('learning_rate/generator', generator_lr) tf.summary.scalar('learning_rate/discriminator', discriminator_lr) with tf.name_scope('train'): train_ops = tfgan.gan_train_ops( gan_model, g_loss, generator_optimizer=tf.train.AdamOptimizer(generator_lr, 0.5), discriminator_optimizer=tf.train.AdamOptimizer( discriminator_lr, 0.5), summarize_gradients=True) status_message = tf.string_join([ 'Starting train step: ', tf.as_string(tf.train.get_or_create_global_step()) ], name='status_message') if FLAGS.max_steps == 0: return tfgan.gan_train(train_ops, hooks=[ tf.train.StopAtStepHook(num_steps=FLAGS.max_steps), tf.train.LoggingTensorHook([status_message], every_n_iter=100) ], logdir=FLAGS.job_dir, get_hooks_fn=tfgan.get_joint_train_hooks(), config=sess_config, save_checkpoint_secs=FLAGS.save_checkpoint_secs)
def testStateSaverWithTwoSimpleSteps(self): with self.test_session() as sess: batch_size_value = 2 batch_size = tf.constant(batch_size_value) num_unroll = 2 length = 3 key = tf.string_join(["key_", tf.as_string(tf.cast( 10000 * tf.random_uniform(()), tf.int32))]) padded_length = 4 sequences = {"seq1": np.random.rand(padded_length, 5), "seq2": np.random.rand(padded_length, 4, 2)} context = {"context1": [3, 4]} initial_states = {"state1": np.random.rand(6, 7), "state2": np.random.rand(8)} state_saver = tf.contrib.training.SequenceQueueingStateSaver( batch_size=batch_size, num_unroll=num_unroll, input_length=length, input_key=key, input_sequences=sequences, input_context=context, initial_states=initial_states) initial_key_value_0, _ = sess.run((key, state_saver.prefetch_op)) initial_key_value_1, _ = sess.run((key, state_saver.prefetch_op)) initial_key_value_0 = initial_key_value_0.decode("ascii") initial_key_value_1 = initial_key_value_1.decode("ascii") # Step 1 next_batch = state_saver.next_batch (key_value, next_key_value, seq1_value, seq2_value, context1_value, state1_value, state2_value, length_value, _, _) = sess.run( (next_batch.key, next_batch.next_key, next_batch.sequences["seq1"], next_batch.sequences["seq2"], next_batch.context["context1"], next_batch.state("state1"), next_batch.state("state2"), next_batch.length, next_batch.save_state("state1", next_batch.state("state1") + 1), next_batch.save_state("state2", next_batch.state("state2") - 1))) expected_first_keys = set( ("00000_of_00002:%s" % x).encode("ascii") for x in (initial_key_value_0, initial_key_value_1)) expected_second_keys = set( ("00001_of_00002:%s" % x).encode("ascii") for x in (initial_key_value_0, initial_key_value_1)) expected_final_keys = set( ("STOP:%s" % x).encode("ascii") for x in (initial_key_value_0, initial_key_value_1)) self.assertEqual(set(key_value), expected_first_keys) self.assertEqual(set(next_key_value), expected_second_keys) self.assertAllEqual( context1_value, np.tile(context["context1"], (batch_size_value, 1))) self.assertAllEqual( seq1_value, np.tile(sequences["seq1"][np.newaxis, 0:2, :], (batch_size_value, 1, 1))) self.assertAllEqual( seq2_value, np.tile(sequences["seq2"][np.newaxis, 0:2, :, :], (batch_size_value, 1, 1, 1))) self.assertAllEqual( state1_value, np.tile(initial_states["state1"], (batch_size_value, 1, 1))) self.assertAllEqual( state2_value, np.tile(initial_states["state2"], (batch_size_value, 1))) self.assertAllEqual(length_value, [2, 2]) # Step 2 (key_value, next_key_value, seq1_value, seq2_value, context1_value, state1_value, state2_value, length_value, _, _) = sess.run( (next_batch.key, next_batch.next_key, next_batch.sequences["seq1"], next_batch.sequences["seq2"], next_batch.context["context1"], next_batch.state("state1"), next_batch.state("state2"), next_batch.length, next_batch.save_state("state1", next_batch.state("state1") + 1), next_batch.save_state("state2", next_batch.state("state2") - 1))) self.assertEqual(set(key_value), expected_second_keys) self.assertEqual(set(next_key_value), expected_final_keys) self.assertAllEqual( context1_value, np.tile(context["context1"], (batch_size_value, 1))) self.assertAllEqual( seq1_value, np.tile(sequences["seq1"][np.newaxis, 2:4, :], (batch_size_value, 1, 1))) self.assertAllEqual( seq2_value, np.tile(sequences["seq2"][np.newaxis, 2:4, :, :], (batch_size_value, 1, 1, 1))) self.assertAllEqual( state1_value, 1 + np.tile(initial_states["state1"], (batch_size_value, 1, 1))) self.assertAllEqual( state2_value, -1 + np.tile(initial_states["state2"], (batch_size_value, 1))) self.assertAllEqual(length_value, [1, 1]) # Finished. Let's make sure there's nothing left in the barrier. self.assertEqual(0, state_saver.barrier.ready_size().eval())