def testIntegerSummaries(self): step = training_util.create_global_step() writer = self.create_db_writer() def adder(x, y): state_ops.assign_add(step, 1) summary_ops.generic('x', x) summary_ops.generic('y', y) sum_ = x + y summary_ops.generic('sum', sum_) return sum_ with summary_ops.always_record_summaries(): with writer.as_default(): self.assertEqual(5, adder(int64(2), int64(3)).numpy()) six.assertCountEqual( self, [1, 1, 1], get_all(self.db, 'SELECT step FROM Tensors WHERE dtype IS NOT NULL')) six.assertCountEqual(self, ['x', 'y', 'sum'], get_all(self.db, 'SELECT tag_name FROM Tags')) x_id = get_one(self.db, 'SELECT tag_id FROM Tags WHERE tag_name = "x"') y_id = get_one(self.db, 'SELECT tag_id FROM Tags WHERE tag_name = "y"') sum_id = get_one(self.db, 'SELECT tag_id FROM Tags WHERE tag_name = "sum"') with summary_ops.always_record_summaries(): with writer.as_default(): self.assertEqual(9, adder(int64(4), int64(5)).numpy()) six.assertCountEqual( self, [1, 1, 1, 2, 2, 2], get_all(self.db, 'SELECT step FROM Tensors WHERE dtype IS NOT NULL')) six.assertCountEqual(self, [x_id, y_id, sum_id], get_all(self.db, 'SELECT tag_id FROM Tags')) self.assertEqual(2, get_tensor(self.db, x_id, 1)) self.assertEqual(3, get_tensor(self.db, y_id, 1)) self.assertEqual(5, get_tensor(self.db, sum_id, 1)) self.assertEqual(4, get_tensor(self.db, x_id, 2)) self.assertEqual(5, get_tensor(self.db, y_id, 2)) self.assertEqual(9, get_tensor(self.db, sum_id, 2)) six.assertCountEqual( self, ['experiment'], get_all(self.db, 'SELECT experiment_name FROM Experiments')) six.assertCountEqual(self, ['run'], get_all(self.db, 'SELECT run_name FROM Runs')) six.assertCountEqual(self, ['user'], get_all(self.db, 'SELECT user_name FROM Users'))
def testFlushFunction(self): logdir = self.get_temp_dir() writer = summary_ops.create_file_writer(logdir, max_queue=999999, flush_millis=999999) with writer.as_default(), summary_ops.always_record_summaries(): summary_ops.scalar('scalar', 2.0, step=1) flush_op = summary_ops.flush() with self.cached_session() as sess: sess.run(summary_ops.summary_writer_initializer_op()) get_total = lambda: len( summary_test_util.events_from_logdir(logdir)) # Note: First tf.Event is always file_version. self.assertEqual(1, get_total()) sess.run(summary_ops.all_summary_ops()) self.assertEqual(1, get_total()) sess.run(flush_op) self.assertEqual(2, get_total()) # Test "writer" parameter sess.run(summary_ops.all_summary_ops()) sess.run(summary_ops.flush(writer=writer)) self.assertEqual(3, get_total()) sess.run(summary_ops.all_summary_ops()) sess.run(summary_ops.flush(writer=writer._resource)) # pylint:disable=protected-access self.assertEqual(4, get_total())
def testWriterInitAndClose(self): logdir = self.get_temp_dir() get_total = lambda: len(summary_test_util.events_from_logdir(logdir)) with summary_ops.always_record_summaries(): writer = summary_ops.create_file_writer(logdir, max_queue=100, flush_millis=1000000) self.assertEqual(1, get_total()) # file_version Event # Calling init() again while writer is open has no effect writer.init() self.assertEqual(1, get_total()) try: # Not using .as_default() to avoid implicit flush when exiting writer.set_as_default() summary_ops.scalar('one', 1.0, step=1) self.assertEqual(1, get_total()) # Calling .close() should do an implicit flush writer.close() self.assertEqual(2, get_total()) # Calling init() on a closed writer should start a new file time.sleep(1.1) # Ensure filename has a different timestamp writer.init() files = sorted(gfile.Glob(os.path.join(logdir, '*tfevents*'))) self.assertEqual(2, len(files)) get_total = lambda: len( summary_test_util.events_from_file(files[1])) self.assertEqual(1, get_total()) # file_version Event summary_ops.scalar('two', 2.0, step=2) writer.close() self.assertEqual(2, get_total()) finally: # Clean up by resetting default writer summary_ops.create_file_writer(None).set_as_default()
def testWriterInitAndClose(self): logdir = self.get_temp_dir() with summary_ops.always_record_summaries(): writer = summary_ops.create_file_writer( logdir, max_queue=100, flush_millis=1000000) with writer.as_default(): summary_ops.scalar('one', 1.0, step=1) with self.cached_session() as sess: sess.run(summary_ops.summary_writer_initializer_op()) get_total = lambda: len(summary_test_util.events_from_logdir(logdir)) self.assertEqual(1, get_total()) # file_version Event # Running init() again while writer is open has no effect sess.run(writer.init()) self.assertEqual(1, get_total()) sess.run(summary_ops.all_summary_ops()) self.assertEqual(1, get_total()) # Running close() should do an implicit flush sess.run(writer.close()) self.assertEqual(2, get_total()) # Running init() on a closed writer should start a new file time.sleep(1.1) # Ensure filename has a different timestamp sess.run(writer.init()) sess.run(summary_ops.all_summary_ops()) sess.run(writer.close()) files = sorted(gfile.Glob(os.path.join(logdir, '*tfevents*'))) self.assertEqual(2, len(files)) self.assertEqual(2, len(summary_test_util.events_from_file(files[1])))
def testWriterInitAndClose(self): logdir = self.get_temp_dir() with summary_ops.always_record_summaries(): writer = summary_ops.create_file_writer(logdir, max_queue=100, flush_millis=1000000) with writer.as_default(): summary_ops.scalar('one', 1.0, step=1) with self.cached_session() as sess: sess.run(summary_ops.summary_writer_initializer_op()) get_total = lambda: len( summary_test_util.events_from_logdir(logdir)) self.assertEqual(1, get_total()) # file_version Event # Running init() again while writer is open has no effect sess.run(writer.init()) self.assertEqual(1, get_total()) sess.run(summary_ops.all_summary_ops()) self.assertEqual(1, get_total()) # Running close() should do an implicit flush sess.run(writer.close()) self.assertEqual(2, get_total()) # Running init() on a closed writer should start a new file time.sleep(1.1) # Ensure filename has a different timestamp sess.run(writer.init()) sess.run(summary_ops.all_summary_ops()) sess.run(writer.close()) files = sorted(gfile.Glob(os.path.join(logdir, '*tfevents*'))) self.assertEqual(2, len(files)) self.assertEqual(2, len(summary_test_util.events_from_file(files[1])))
def testWriterInitAndClose(self): logdir = self.get_temp_dir() get_total = lambda: len(summary_test_util.events_from_logdir(logdir)) with summary_ops.always_record_summaries(): writer = summary_ops.create_file_writer( logdir, max_queue=100, flush_millis=1000000) self.assertEqual(1, get_total()) # file_version Event # Calling init() again while writer is open has no effect writer.init() self.assertEqual(1, get_total()) try: # Not using .as_default() to avoid implicit flush when exiting writer.set_as_default() summary_ops.scalar('one', 1.0, step=1) self.assertEqual(1, get_total()) # Calling .close() should do an implicit flush writer.close() self.assertEqual(2, get_total()) # Calling init() on a closed writer should start a new file time.sleep(1.1) # Ensure filename has a different timestamp writer.init() files = sorted(gfile.Glob(os.path.join(logdir, '*tfevents*'))) self.assertEqual(2, len(files)) get_total = lambda: len(summary_test_util.events_from_file(files[1])) self.assertEqual(1, get_total()) # file_version Event summary_ops.scalar('two', 2.0, step=2) writer.close() self.assertEqual(2, get_total()) finally: # Clean up by resetting default writer summary_ops.create_file_writer(None).set_as_default()
def train(training_dataset, preprocessing_type, base_model, optimizer, summary_dir, ckpt_dir, logging_every_n_steps, summary_every_n_steps, save_every_n_steps, restore_ckpt_file_path): # 获取 pretained model variables = base_model.variables + [tf.train.get_or_create_global_step()] saver = model_saver.Saver(variables) if restore_ckpt_file_path is not None: saver.restore(restore_ckpt_file_path) if tf.train.latest_checkpoint(ckpt_dir) is not None: saver.restore(tf.train.latest_checkpoint(ckpt_dir)) # writer = tf.summary.FileWriter(train_dir, flush_secs=100) writer = summary.create_file_writer(summary_dir, flush_millis=100000) for i in range(configs['epoches']): tf.compat.v1.logging.info('epoch %d starting...' % (i + 1)) start_time = time.time() with writer.as_default(), summary.always_record_summaries(): train_one_epoch(dataset=training_dataset, base_model=base_model, optimizer=optimizer, preprocessing_type=preprocessing_type, logging_every_n_steps=logging_every_n_steps, summary_every_n_steps=summary_every_n_steps, save_path=ckpt_dir, saver=saver, save_every_n_steps=save_every_n_steps) tf.set_random_seed(1) end_time = time.time() tf.compat.v1.logging.info('epoch %d training finished, costing %d seconds...' % (i, end_time - start_time))
def testGraphSummary(self): training_util.get_or_create_global_step() name = 'hi' graph = graph_pb2.GraphDef(node=(node_def_pb2.NodeDef(name=name),)) with summary_ops.always_record_summaries(): with self.create_db_writer().as_default(): summary_ops.graph(graph) six.assertCountEqual(self, [name], get_all(self.db, 'SELECT node_name FROM Nodes'))
def testGraphSummary(self): training_util.get_or_create_global_step() name = 'hi' graph = graph_pb2.GraphDef(node=(node_def_pb2.NodeDef(name=name), )) with summary_ops.always_record_summaries(): with self.create_db_writer().as_default(): summary_ops.graph(graph) six.assertCountEqual(self, [name], get_all(self.db, 'SELECT node_name FROM Nodes'))
def testEagerMemory(self): training_util.get_or_create_global_step() logdir = self.get_temp_dir() with summary_ops.create_file_writer( logdir, max_queue=0, name='t0').as_default(), summary_ops.always_record_summaries(): summary_ops.generic('tensor', 1, '') summary_ops.scalar('scalar', 2.0) summary_ops.histogram('histogram', [1.0]) summary_ops.image('image', [[[[1.0]]]]) summary_ops.audio('audio', [[1.0]], 1.0, 1)
def testSummaryName(self): logdir = self.get_temp_dir() writer = summary_ops.create_file_writer(logdir, max_queue=0) with writer.as_default(), summary_ops.always_record_summaries(): summary_ops.scalar('scalar', 2.0, step=1) with self.cached_session() as sess: sess.run(summary_ops.summary_writer_initializer_op()) sess.run(summary_ops.all_summary_ops()) events = summary_test_util.events_from_logdir(logdir) self.assertEqual(2, len(events)) self.assertEqual('scalar', events[1].summary.value[0].tag)
def testDbURIOpen(self): tmpdb_path = os.path.join(self.get_temp_dir(), 'tmpDbURITest.sqlite') tmpdb_uri = six.moves.urllib_parse.urljoin('file:', tmpdb_path) tmpdb_writer = summary_ops.create_db_writer(tmpdb_uri, 'experimentA', 'run1', 'user1') with summary_ops.always_record_summaries(): with tmpdb_writer.as_default(): summary_ops.scalar('t1', 2.0) tmpdb = sqlite3.connect(tmpdb_path) num = get_one(tmpdb, 'SELECT count(*) FROM Tags WHERE tag_name = "t1"') self.assertEqual(num, 1) tmpdb.close()
def testSummaryGlobalStep(self): step = training_util.get_or_create_global_step() logdir = tempfile.mkdtemp() with summary_ops.create_file_writer( logdir, max_queue=0, name='t2').as_default(), summary_ops.always_record_summaries(): summary_ops.scalar('scalar', 2.0, step=step) events = summary_test_util.events_from_logdir(logdir) self.assertEqual(len(events), 2) self.assertEqual(events[1].summary.value[0].tag, 'scalar')
def testMaxQueue(self): logs = tempfile.mkdtemp() with summary_ops.create_file_writer( logs, max_queue=1, flush_millis=999999, name='lol').as_default(), summary_ops.always_record_summaries(): get_total = lambda: len(summary_test_util.events_from_logdir(logs)) # Note: First tf.Event is always file_version. self.assertEqual(1, get_total()) summary_ops.scalar('scalar', 2.0, step=1) self.assertEqual(1, get_total()) # Should flush after second summary since max_queue = 1 summary_ops.scalar('scalar', 2.0, step=2) self.assertEqual(3, get_total())
def testMaxQueue(self): logs = tempfile.mkdtemp() with summary_ops.create_file_writer( logs, max_queue=1, flush_millis=999999, name='lol').as_default( ), summary_ops.always_record_summaries(): get_total = lambda: len(summary_test_util.events_from_logdir(logs)) # Note: First tf.compat.v1.Event is always file_version. self.assertEqual(1, get_total()) summary_ops.scalar('scalar', 2.0, step=1) self.assertEqual(1, get_total()) # Should flush after second summary since max_queue = 1 summary_ops.scalar('scalar', 2.0, step=2) self.assertEqual(3, get_total())
def testSummaryGlobalStep(self): training_util.get_or_create_global_step() logdir = self.get_temp_dir() writer = summary_ops.create_file_writer(logdir, max_queue=0) with writer.as_default(), summary_ops.always_record_summaries(): summary_ops.scalar('scalar', 2.0) with self.cached_session() as sess: sess.run(variables.global_variables_initializer()) sess.run(summary_ops.summary_writer_initializer_op()) step, _ = sess.run( [training_util.get_global_step(), summary_ops.all_summary_ops()]) events = summary_test_util.events_from_logdir(logdir) self.assertEqual(2, len(events)) self.assertEqual(step, events[1].step)
def testSummaryOps(self): training_util.get_or_create_global_step() logdir = tempfile.mkdtemp() with summary_ops.create_file_writer( logdir, max_queue=0, name='t0').as_default(), summary_ops.always_record_summaries(): summary_ops.generic('tensor', 1, '') summary_ops.scalar('scalar', 2.0) summary_ops.histogram('histogram', [1.0]) summary_ops.image('image', [[[[1.0]]]]) summary_ops.audio('audio', [[1.0]], 1.0, 1) # The working condition of the ops is tested in the C++ test so we just # test here that we're calling them correctly. self.assertTrue(gfile.Exists(logdir))
def testDefunSummarys(self): training_util.get_or_create_global_step() logdir = tempfile.mkdtemp() with summary_ops.create_file_writer( logdir, max_queue=0, name='t1').as_default(), summary_ops.always_record_summaries(): @function.defun def write(): summary_ops.scalar('scalar', 2.0) write() events = summary_test_util.events_from_logdir(logdir) self.assertEqual(len(events), 2) self.assertEqual(events[1].summary.value[0].simple_value, 2.0)
def testWriterFlush(self): logdir = self.get_temp_dir() get_total = lambda: len(summary_test_util.events_from_logdir(logdir)) with summary_ops.always_record_summaries(): writer = summary_ops.create_file_writer( logdir, max_queue=100, flush_millis=1000000) self.assertEqual(1, get_total()) # file_version Event with writer.as_default(): summary_ops.scalar('one', 1.0, step=1) self.assertEqual(1, get_total()) writer.flush() self.assertEqual(2, get_total()) summary_ops.scalar('two', 2.0, step=2) # Exiting the "as_default()" should do an implicit flush of the "two" tag self.assertEqual(3, get_total())
def testSummaryOps(self): logdir = self.get_temp_dir() writer = summary_ops.create_file_writer(logdir, max_queue=0) with writer.as_default(), summary_ops.always_record_summaries(): summary_ops.generic('tensor', 1, step=1) summary_ops.scalar('scalar', 2.0, step=1) summary_ops.histogram('histogram', [1.0], step=1) summary_ops.image('image', [[[[1.0]]]], step=1) summary_ops.audio('audio', [[1.0]], 1.0, 1, step=1) with self.cached_session() as sess: sess.run(summary_ops.summary_writer_initializer_op()) sess.run(summary_ops.all_summary_ops()) # The working condition of the ops is tested in the C++ test so we just # test here that we're calling them correctly. self.assertTrue(gfile.Exists(logdir))
def testWriterFlush(self): logdir = self.get_temp_dir() with summary_ops.always_record_summaries(): writer = summary_ops.create_file_writer( logdir, max_queue=100, flush_millis=1000000) with writer.as_default(): summary_ops.scalar('one', 1.0, step=1) with self.cached_session() as sess: sess.run(summary_ops.summary_writer_initializer_op()) get_total = lambda: len(summary_test_util.events_from_logdir(logdir)) self.assertEqual(1, get_total()) # file_version Event sess.run(summary_ops.all_summary_ops()) self.assertEqual(1, get_total()) sess.run(writer.flush()) self.assertEqual(2, get_total())
def testMaxQueue(self): logdir = self.get_temp_dir() writer = summary_ops.create_file_writer( logdir, max_queue=1, flush_millis=999999) with writer.as_default(), summary_ops.always_record_summaries(): summary_ops.scalar('scalar', 2.0, step=1) with self.cached_session() as sess: sess.run(summary_ops.summary_writer_initializer_op()) get_total = lambda: len(summary_test_util.events_from_logdir(logdir)) # Note: First tf.Event is always file_version. self.assertEqual(1, get_total()) sess.run(summary_ops.all_summary_ops()) self.assertEqual(1, get_total()) # Should flush after second summary since max_queue = 1 sess.run(summary_ops.all_summary_ops()) self.assertEqual(3, get_total())
def testWriterFlush(self): logdir = self.get_temp_dir() get_total = lambda: len(summary_test_util.events_from_logdir(logdir)) with summary_ops.always_record_summaries(): writer = summary_ops.create_file_writer(logdir, max_queue=100, flush_millis=1000000) self.assertEqual(1, get_total()) # file_version Event with writer.as_default(): summary_ops.scalar('one', 1.0, step=1) self.assertEqual(1, get_total()) writer.flush() self.assertEqual(2, get_total()) summary_ops.scalar('two', 2.0, step=2) # Exiting the "as_default()" should do an implicit flush of the "two" tag self.assertEqual(3, get_total())
def testMaxQueue(self): logdir = self.get_temp_dir() writer = summary_ops.create_file_writer( logdir, max_queue=1, flush_millis=999999) with writer.as_default(), summary_ops.always_record_summaries(): summary_ops.scalar('scalar', 2.0, step=1) with self.cached_session() as sess: sess.run(summary_ops.summary_writer_initializer_op()) get_total = lambda: len(summary_test_util.events_from_logdir(logdir)) # Note: First tf.compat.v1.Event is always file_version. self.assertEqual(1, get_total()) sess.run(summary_ops.all_summary_ops()) self.assertEqual(1, get_total()) # Should flush after second summary since max_queue = 1 sess.run(summary_ops.all_summary_ops()) self.assertEqual(3, get_total())
def train( training_dataset, preprocessing_type, base_model, optimizer, logging_every_n_steps, save_every_n_steps, summary_every_n_steps, train_dir, ckpt_dir, restore_ckpt_file_path, ): # 获取 pretrained model variables = base_model.variables + [tf.train.get_or_create_global_step()] saver = eager_saver.Saver(variables) # 命令行指定 ckpt file if restore_ckpt_file_path is not None: saver.restore(restore_ckpt_file_path) # 当前 logs_dir 中的预训练模型,用于继续训练 if tf.train.latest_checkpoint(ckpt_dir) is not None: saver.restore(tf.train.latest_checkpoint(ckpt_dir)) train_writer = tf.contrib.summary.create_file_writer(train_dir, flush_millis=100000) for i in range(CONFIG['epochs']): tf_logging.info('epoch %d starting...' % (i + 1)) start = time.time() with train_writer.as_default(), summary.always_record_summaries(): train_one_epoch( dataset=training_dataset, base_model=base_model, optimizer=optimizer, preprocessing_type=preprocessing_type, logging_every_n_steps=logging_every_n_steps, summary_every_n_steps=summary_every_n_steps, saver=saver, save_every_n_steps=save_every_n_steps, save_path=ckpt_dir, ) tf.set_random_seed(1) train_end = time.time() tf_logging.info('epoch %d training finished, costing %d seconds...' % (i + 1, train_end - start))
def testSharedName(self): logdir = self.get_temp_dir() with summary_ops.always_record_summaries(): # Create with default shared name (should match logdir) writer1 = summary_ops.create_file_writer(logdir) with writer1.as_default(): summary_ops.scalar('one', 1.0, step=1) # Create with explicit logdir shared name (should be same resource/file) shared_name = 'logdir:' + logdir writer2 = summary_ops.create_file_writer(logdir, name=shared_name) with writer2.as_default(): summary_ops.scalar('two', 2.0, step=2) # Create with different shared name (should be separate resource/file) writer3 = summary_ops.create_file_writer(logdir, name='other') with writer3.as_default(): summary_ops.scalar('three', 3.0, step=3) with self.cached_session() as sess: # Run init ops across writers sequentially to avoid race condition. # TODO(nickfelt): fix race condition in resource manager lookup or create sess.run(writer1.init()) sess.run(writer2.init()) time.sleep(1.1) # Ensure filename has a different timestamp sess.run(writer3.init()) sess.run(summary_ops.all_summary_ops()) sess.run([writer1.flush(), writer2.flush(), writer3.flush()]) event_files = iter( sorted(gfile.Glob(os.path.join(logdir, '*tfevents*')))) # First file has tags "one" and "two" events = summary_test_util.events_from_file(next(event_files)) self.assertEqual('brain.Event:2', events[0].file_version) tags = [e.summary.value[0].tag for e in events[1:]] self.assertItemsEqual(['one', 'two'], tags) # Second file has tag "three" events = summary_test_util.events_from_file(next(event_files)) self.assertEqual('brain.Event:2', events[0].file_version) tags = [e.summary.value[0].tag for e in events[1:]] self.assertItemsEqual(['three'], tags) # No more files self.assertRaises(StopIteration, lambda: next(event_files))
def testSharedName(self): logdir = self.get_temp_dir() with summary_ops.always_record_summaries(): # Create with default shared name (should match logdir) writer1 = summary_ops.create_file_writer(logdir) with writer1.as_default(): summary_ops.scalar('one', 1.0, step=1) # Create with explicit logdir shared name (should be same resource/file) shared_name = 'logdir:' + logdir writer2 = summary_ops.create_file_writer(logdir, name=shared_name) with writer2.as_default(): summary_ops.scalar('two', 2.0, step=2) # Create with different shared name (should be separate resource/file) writer3 = summary_ops.create_file_writer(logdir, name='other') with writer3.as_default(): summary_ops.scalar('three', 3.0, step=3) with self.cached_session() as sess: # Run init ops across writers sequentially to avoid race condition. # TODO(nickfelt): fix race condition in resource manager lookup or create sess.run(writer1.init()) sess.run(writer2.init()) time.sleep(1.1) # Ensure filename has a different timestamp sess.run(writer3.init()) sess.run(summary_ops.all_summary_ops()) sess.run([writer1.flush(), writer2.flush(), writer3.flush()]) event_files = iter(sorted(gfile.Glob(os.path.join(logdir, '*tfevents*')))) # First file has tags "one" and "two" events = summary_test_util.events_from_file(next(event_files)) self.assertEqual('brain.Event:2', events[0].file_version) tags = [e.summary.value[0].tag for e in events[1:]] self.assertItemsEqual(['one', 'two'], tags) # Second file has tag "three" events = summary_test_util.events_from_file(next(event_files)) self.assertEqual('brain.Event:2', events[0].file_version) tags = [e.summary.value[0].tag for e in events[1:]] self.assertItemsEqual(['three'], tags) # No more files self.assertRaises(StopIteration, lambda: next(event_files))
def testSummaryGraphModeCond(self): with ops.Graph().as_default(), self.cached_session(): training_util.get_or_create_global_step() logdir = tempfile.mkdtemp() with summary_ops.create_file_writer( logdir, max_queue=0, name='t2').as_default(), summary_ops.always_record_summaries(): summary_ops.initialize() training_util.get_or_create_global_step().initializer.run() def f(): summary_ops.scalar('scalar', 2.0) return constant_op.constant(True) pred = array_ops.placeholder(dtypes.bool) x = control_flow_ops.cond(pred, f, lambda: constant_op.constant(False)) x.eval(feed_dict={pred: True}) events = summary_test_util.events_from_logdir(logdir) self.assertEqual(len(events), 2) self.assertEqual(events[1].summary.value[0].tag, 'cond/scalar')
def testFlushFunction(self): logs = tempfile.mkdtemp() writer = summary_ops.create_file_writer( logs, max_queue=999999, flush_millis=999999, name='lol') with writer.as_default(), summary_ops.always_record_summaries(): get_total = lambda: len(summary_test_util.events_from_logdir(logs)) # Note: First tf.Event is always file_version. self.assertEqual(1, get_total()) summary_ops.scalar('scalar', 2.0, step=1) summary_ops.scalar('scalar', 2.0, step=2) self.assertEqual(1, get_total()) summary_ops.flush() self.assertEqual(3, get_total()) # Test "writer" parameter summary_ops.scalar('scalar', 2.0, step=3) summary_ops.flush(writer=writer) self.assertEqual(4, get_total()) summary_ops.scalar('scalar', 2.0, step=4) summary_ops.flush(writer=writer._resource) # pylint:disable=protected-access self.assertEqual(5, get_total())
def testFlushFunction(self): logs = tempfile.mkdtemp() writer = summary_ops.create_file_writer(logs, max_queue=999999, flush_millis=999999, name='lol') with writer.as_default(), summary_ops.always_record_summaries(): get_total = lambda: len(summary_test_util.events_from_logdir(logs)) # Note: First tf.compat.v1.Event is always file_version. self.assertEqual(1, get_total()) summary_ops.scalar('scalar', 2.0, step=1) summary_ops.scalar('scalar', 2.0, step=2) self.assertEqual(1, get_total()) summary_ops.flush() self.assertEqual(3, get_total()) # Test "writer" parameter summary_ops.scalar('scalar', 2.0, step=3) summary_ops.flush(writer=writer) self.assertEqual(4, get_total()) summary_ops.scalar('scalar', 2.0, step=4) summary_ops.flush(writer=writer._resource) # pylint:disable=protected-access self.assertEqual(5, get_total())
def testSharedName(self): logdir = self.get_temp_dir() with summary_ops.always_record_summaries(): # Create with default shared name (should match logdir) writer1 = summary_ops.create_file_writer(logdir) with writer1.as_default(): summary_ops.scalar('one', 1.0, step=1) summary_ops.flush() # Create with explicit logdir shared name (should be same resource/file) shared_name = 'logdir:' + logdir writer2 = summary_ops.create_file_writer(logdir, name=shared_name) with writer2.as_default(): summary_ops.scalar('two', 2.0, step=2) summary_ops.flush() # Create with different shared name (should be separate resource/file) time.sleep(1.1) # Ensure filename has a different timestamp writer3 = summary_ops.create_file_writer(logdir, name='other') with writer3.as_default(): summary_ops.scalar('three', 3.0, step=3) summary_ops.flush() event_files = iter( sorted(gfile.Glob(os.path.join(logdir, '*tfevents*')))) # First file has tags "one" and "two" events = iter(summary_test_util.events_from_file(next(event_files))) self.assertEqual('brain.Event:2', next(events).file_version) self.assertEqual('one', next(events).summary.value[0].tag) self.assertEqual('two', next(events).summary.value[0].tag) self.assertRaises(StopIteration, lambda: next(events)) # Second file has tag "three" events = iter(summary_test_util.events_from_file(next(event_files))) self.assertEqual('brain.Event:2', next(events).file_version) self.assertEqual('three', next(events).summary.value[0].tag) self.assertRaises(StopIteration, lambda: next(events)) # No more files self.assertRaises(StopIteration, lambda: next(event_files))
def testSharedName(self): logdir = self.get_temp_dir() with summary_ops.always_record_summaries(): # Create with default shared name (should match logdir) writer1 = summary_ops.create_file_writer(logdir) with writer1.as_default(): summary_ops.scalar('one', 1.0, step=1) summary_ops.flush() # Create with explicit logdir shared name (should be same resource/file) shared_name = 'logdir:' + logdir writer2 = summary_ops.create_file_writer(logdir, name=shared_name) with writer2.as_default(): summary_ops.scalar('two', 2.0, step=2) summary_ops.flush() # Create with different shared name (should be separate resource/file) time.sleep(1.1) # Ensure filename has a different timestamp writer3 = summary_ops.create_file_writer(logdir, name='other') with writer3.as_default(): summary_ops.scalar('three', 3.0, step=3) summary_ops.flush() event_files = iter(sorted(gfile.Glob(os.path.join(logdir, '*tfevents*')))) # First file has tags "one" and "two" events = iter(summary_test_util.events_from_file(next(event_files))) self.assertEqual('brain.Event:2', next(events).file_version) self.assertEqual('one', next(events).summary.value[0].tag) self.assertEqual('two', next(events).summary.value[0].tag) self.assertRaises(StopIteration, lambda: next(events)) # Second file has tag "three" events = iter(summary_test_util.events_from_file(next(event_files))) self.assertEqual('brain.Event:2', next(events).file_version) self.assertEqual('three', next(events).summary.value[0].tag) self.assertRaises(StopIteration, lambda: next(events)) # No more files self.assertRaises(StopIteration, lambda: next(event_files))
def testFlushFunction(self): logdir = self.get_temp_dir() writer = summary_ops.create_file_writer( logdir, max_queue=999999, flush_millis=999999) with writer.as_default(), summary_ops.always_record_summaries(): summary_ops.scalar('scalar', 2.0, step=1) flush_op = summary_ops.flush() with self.cached_session() as sess: sess.run(summary_ops.summary_writer_initializer_op()) get_total = lambda: len(summary_test_util.events_from_logdir(logdir)) # Note: First tf.Event is always file_version. self.assertEqual(1, get_total()) sess.run(summary_ops.all_summary_ops()) self.assertEqual(1, get_total()) sess.run(flush_op) self.assertEqual(2, get_total()) # Test "writer" parameter sess.run(summary_ops.all_summary_ops()) sess.run(summary_ops.flush(writer=writer)) self.assertEqual(3, get_total()) sess.run(summary_ops.all_summary_ops()) sess.run(summary_ops.flush(writer=writer._resource)) # pylint:disable=protected-access self.assertEqual(4, get_total())
return optimizer.minimize(loss=lambda: loss_func(model, inputs, labels, **kwargs), global_step=tf.train.get_or_create_global_step()) train_accuracy = tfe.metrics.Accuracy() epochs = 1000 logdir = '../logs/iris-eager' acc = 0.0 _loss = 0. # File writer for tensorboard visualization. writer = summary.create_file_writer(logdir=logdir) with writer.as_default(): with summary.always_record_summaries(): print('\n{}'.format(75 * '-')) print('Training started: {:%c}\n'.format(datetime.now())) start = time.time() for epoch in range(epochs): # Loop through each data batches. for X_batch, y_batch in tfe.Iterator(train_data): # Run the training step. train_step(model, optimizer, loss, X_batch, y_batch) # Estimate loss _loss = loss(model, X_batch, y_batch) tf.summary.scalar('loss', _loss) # Estimate accuracy. y_pred = tf.argmax(model(X_batch), axis=1,
def testShouldRecordSummary(self): self.assertFalse(summary_ops.should_record_summaries()) with summary_ops.always_record_summaries(): self.assertTrue(summary_ops.should_record_summaries())