def testSelectEverthingDetail(self):
ops.reset_default_graph()
dev = '/gpu:0' if test.is_gpu_available() else '/cpu:0'
outfile = os.path.join(test.get_temp_dir(), 'dump')
opts = (builder(builder.trainable_variables_parameter())
.with_file_output(outfile)
.with_accounted_types(['.*'])
.select(['micros', 'bytes', 'params', 'float_ops', 'occurrence',
'device', 'op_types', 'input_shapes']).build())
config = config_pb2.ConfigProto()
with session.Session(config=config) as sess, ops.device(dev):
x = lib.BuildSmallModel()
sess.run(variables.global_variables_initializer())
run_meta = config_pb2.RunMetadata()
_ = sess.run(x,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE),
run_metadata=run_meta)
model_analyzer.profile(
sess.graph, run_meta, options=opts)
with gfile.Open(outfile, 'r') as f:
# pylint: disable=line-too-long
outputs = f.read().split('\n')
self.assertEqual(outputs[0],
'node name | # parameters | # float_ops | requested bytes | total execution time | accelerator execution time | cpu execution time | assigned devices | op types | op count (run|defined) | input shapes')
for o in outputs[1:]:
if o.find('Conv2D ') > 0:
metrics = o[o.find('(') +1: o.find(')')].split(',')
# Make sure time is profiled.
gap = 1 if test.is_gpu_available() else 2
for i in range(3, 6, gap):
mat = re.search('(.*)[um]s/(.*)[um]s', metrics[i])
self.assertGreater(float(mat.group(1)), 0.0)
self.assertGreater(float(mat.group(2)), 0.0)
# Make sure device is profiled.
if test.is_gpu_available():
self.assertTrue(metrics[6].find('gpu') > 0)
self.assertFalse(metrics[6].find('cpu') > 0)
else:
self.assertFalse(metrics[6].find('gpu') > 0)
self.assertTrue(metrics[6].find('cpu') > 0)
# Make sure float_ops is profiled.
mat = re.search('(.*)k/(.*)k flops', metrics[1].strip())
self.assertGreater(float(mat.group(1)), 0.0)
self.assertGreater(float(mat.group(2)), 0.0)
# Make sure op_count is profiled.
self.assertEqual(metrics[8].strip(), '1/1|1/1')
# Make sure input_shapes is profiled.
self.assertEqual(metrics[9].strip(), '0:2x6x6x3|1:3x3x3x6')
if o.find('DW (3x3x3x6') > 0:
metrics = o[o.find('(') +1: o.find(')')].split(',')
mat = re.search('(.*)/(.*) params', metrics[1].strip())
self.assertGreater(float(mat.group(1)), 0.0)
self.assertGreater(float(mat.group(2)), 0.0)
def testAllocationHistory(self):
if not test.is_gpu_available(cuda_only=True):
return
gpu_dev = test.gpu_device_name()
ops.reset_default_graph()
with ops.device(gpu_dev):
_, run_meta = _run_model()
mm = _extract_node(run_meta, 'MatMul')['gpu:0'][0]
mm_allocs = mm.memory[0].allocation_records
# has allocation and deallocation.
self.assertEqual(len(mm_allocs), 2)
# first allocated.
self.assertGreater(mm_allocs[1].alloc_micros, mm_allocs[0].alloc_micros)
self.assertGreater(mm_allocs[0].alloc_bytes, 0)
# Then deallocated.
self.assertLess(mm_allocs[1].alloc_bytes, 0)
# All memory deallocated.
self.assertEqual(mm_allocs[0].alloc_bytes + mm_allocs[1].alloc_bytes, 0)
rand = _extract_node(
run_meta, 'random_normal/RandomStandardNormal')['gpu:0'][0]
random_allocs = rand.memory[0].allocation_records
# random normal must allocated first since matmul depends on it.
self.assertLess(random_allocs[0].alloc_micros, mm.all_start_micros)
# deallocates the memory after matmul started.
self.assertGreater(random_allocs[1].alloc_micros, mm.all_start_micros)
def testLoopGPU(self):
if not test.is_gpu_available():
return
ops.reset_default_graph()
with ops.device('/gpu:0'):
tfprof_node, run_meta = _run_loop_model()
# The while-loop caused a node to appear 4 times in scheduling.
ret = _extract_node(run_meta,
'rnn/while/rnn/basic_rnn_cell/basic_rnn_cell/MatMul')
self.assertEqual(len(ret['/job:localhost/replica:0/task:0/gpu:0']), 4)
total_cpu_execs = 0
for node in ret['/job:localhost/replica:0/task:0/gpu:0']:
total_cpu_execs += node.op_end_rel_micros
ret = _extract_node(
run_meta,
'rnn/while/rnn/basic_rnn_cell/basic_rnn_cell/MatMul:MatMul')
self.assertGreaterEqual(len(ret['/gpu:0/stream:all']), 4)
total_accelerator_execs = 0
for node in ret['/gpu:0/stream:all']:
total_accelerator_execs += node.op_end_rel_micros
mm_node = lib.SearchTFProfNode(
tfprof_node,
'rnn/while/rnn/basic_rnn_cell/basic_rnn_cell/MatMul')
self.assertEqual(mm_node.run_count, 4)
self.assertEqual(mm_node.accelerator_exec_micros, total_accelerator_execs)
self.assertEqual(mm_node.cpu_exec_micros, total_cpu_execs)
self.assertEqual(mm_node.exec_micros,
total_cpu_execs + total_accelerator_execs)
def clear_session():
global _SESSION
global _LEARNING_PHASE
reset_default_graph()
reset_uids()
_SESSION = None
_LEARNING_PHASE = tf.placeholder(dtype='uint8', name='keras_learning_phase')
def _train(self, checkpoint_path, layout_optimizer=False, restore=False):
ops.reset_default_graph()
graph = ops.get_default_graph()
with session.Session(
config=get_config(layout_optimizer), graph=graph) as sess:
batch = 2
height = 6
width = 7
input_channels = 3
shape = [batch, height, width, input_channels]
image = array_ops.placeholder(dtype='float32', shape=shape)
conv1 = conv_layers.conv2d(image, 32, [3, 3])
conv2 = conv_layers.conv2d(conv1, 32, [3, 3])
optimizer = gradient_descent.GradientDescentOptimizer(0.01)
loss = math_ops.reduce_mean(conv2)
train_op = optimizer.minimize(loss)
saver = saver_lib.Saver(write_version=saver_pb2.SaverDef.V2)
if restore:
saver.restore(sess, checkpoint_path)
else:
sess.run(variables.global_variables_initializer())
np.random.seed(0)
for _ in range(2):
image_val = np.random.rand(*shape).astype(np.float32)
sess.run([loss, train_op], feed_dict={image: image_val})
if restore:
all_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)
all_vars_values = [var.eval(session=sess) for var in all_vars]
return all_vars_values
else:
saver.save(sess, checkpoint_path)
def testSelectEverything(self):
ops.reset_default_graph()
opts = model_analyzer.TRAINABLE_VARS_PARAMS_STAT_OPTIONS.copy()
outfile = os.path.join(test.get_temp_dir(), 'dump')
opts['output'] = 'file:outfile=' + outfile
opts['account_type_regexes'] = ['.*']
opts['select'] = [
'params', 'float_ops', 'occurrence', 'device', 'op_types',
'input_shapes'
]
with session.Session() as sess, ops.device('/cpu:0'):
x = lib.BuildSmallModel()
sess.run(variables.global_variables_initializer())
run_meta = config_pb2.RunMetadata()
_ = sess.run(x,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE),
run_metadata=run_meta)
model_analyzer.print_model_analysis(
sess.graph, run_meta, tfprof_options=opts)
with gfile.Open(outfile, 'r') as f:
# pylint: disable=line-too-long
self.assertEqual(
'node name | # parameters | # float_ops | assigned devices | op types | op count (run|defined) | input shapes\n_TFProfRoot (--/451 params, --/10.44k flops, _kTFScopeParent, --/7|--/35, )\n Conv2D (0/0 params, 5.83k/5.83k flops, /job:localhost/replica:0/task:0/cpu:0, /job:localhost/replica:0/task:0/cpu:0|Conv2D, 1/1|1/1, 0:2x6x6x3|1:3x3x3x6)\n Conv2D_1 (0/0 params, 4.61k/4.61k flops, /job:localhost/replica:0/task:0/cpu:0, /job:localhost/replica:0/task:0/cpu:0|Conv2D, 1/1|1/1, 0:2x3x3x6|1:2x2x6x12)\n DW (3x3x3x6, 162/162 params, 0/0 flops, /job:localhost/replica:0/task:0/cpu:0, /job:localhost/replica:0/task:0/cpu:0|VariableV2|_trainable_variables, 1/2|1/10, )\n DW/Assign (0/0 params, 0/0 flops, Assign, 0/0|1/1, 0:3x3x3x6|1:3x3x3x6)\n DW/Initializer (0/0 params, 0/0 flops, _kTFScopeParent, 0/0|1/7, )\n DW/Initializer/random_normal (0/0 params, 0/0 flops, Add, 0/0|1/6, 0:3x3x3x6|1:1)\n DW/Initializer/random_normal/RandomStandardNormal (0/0 params, 0/0 flops, RandomStandardNormal, 0/0|1/1, 0:4)\n DW/Initializer/random_normal/mean (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW/Initializer/random_normal/mul (0/0 params, 0/0 flops, Mul, 0/0|1/1, 0:3x3x3x6|1:1)\n DW/Initializer/random_normal/shape (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW/Initializer/random_normal/stddev (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW/read (0/0 params, 0/0 flops, /job:localhost/replica:0/task:0/cpu:0, /job:localhost/replica:0/task:0/cpu:0|Identity, 1/1|1/1, 0:3x3x3x6)\n DW2 (2x2x6x12, 288/288 params, 0/0 flops, /job:localhost/replica:0/task:0/cpu:0, /job:localhost/replica:0/task:0/cpu:0|VariableV2|_trainable_variables, 1/2|1/10, )\n DW2/Assign (0/0 params, 0/0 flops, Assign, 0/0|1/1, 0:2x2x6x12|1:2x2x6x12)\n DW2/Initializer (0/0 params, 0/0 flops, _kTFScopeParent, 0/0|1/7, )\n DW2/Initializer/random_normal (0/0 params, 0/0 flops, Add, 0/0|1/6, 0:2x2x6x12|1:1)\n DW2/Initializer/random_normal/RandomStandardNormal (0/0 params, 0/0 flops, RandomStandardNormal, 0/0|1/1, 0:4)\n DW2/Initializer/random_normal/mean (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW2/Initializer/random_normal/mul (0/0 params, 0/0 flops, Mul, 0/0|1/1, 0:2x2x6x12|1:1)\n DW2/Initializer/random_normal/shape (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW2/Initializer/random_normal/stddev (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW2/read (0/0 params, 0/0 flops, /job:localhost/replica:0/task:0/cpu:0, /job:localhost/replica:0/task:0/cpu:0|Identity, 1/1|1/1, 0:2x2x6x12)\n ScalarW (1, 1/1 params, 0/0 flops, VariableV2|_trainable_variables, 0/0|1/10, )\n ScalarW/Assign (0/0 params, 0/0 flops, Assign, 0/0|1/1, 0:1|1:1)\n ScalarW/Initializer (0/0 params, 0/0 flops, _kTFScopeParent, 0/0|1/7, )\n ScalarW/Initializer/random_normal (0/0 params, 0/0 flops, Add, 0/0|1/6, 0:1|1:1)\n ScalarW/Initializer/random_normal/RandomStandardNormal (0/0 params, 0/0 flops, RandomStandardNormal, 0/0|1/1, 0:0)\n ScalarW/Initializer/random_normal/mean (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n ScalarW/Initializer/random_normal/mul (0/0 params, 0/0 flops, Mul, 0/0|1/1, 0:1|1:1)\n ScalarW/Initializer/random_normal/shape (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n ScalarW/Initializer/random_normal/stddev (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n ScalarW/read (0/0 params, 0/0 flops, Identity, 0/0|1/1, 0:1)\n init (0/0 params, 0/0 flops, NoOp, 0/0|1/1, 0:1|1:3x3x3x6|2:2x2x6x12)\n zeros (0/0 params, 0/0 flops, /job:localhost/replica:0/task:0/cpu:0, /job:localhost/replica:0/task:0/cpu:0|Const, 1/1|1/1, )\n',
f.read())
def testSimpleCodeView(self):
ops.reset_default_graph()
opts = model_analyzer.TRAINABLE_VARS_PARAMS_STAT_OPTIONS.copy()
outfile = os.path.join(test.get_temp_dir(), 'dump')
opts['output'] = 'file:outfile=' + outfile
opts['account_type_regexes'] = ['.*']
opts['show_name_regexes'] = ['.*model_analyzer_testlib.*']
opts['account_displayed_op_only'] = False
# TODO(xpan): Test 'micros'. Since the execution time changes each run,
# it's a bit difficult to test it now.
opts['select'] = [
'bytes', 'params', 'float_ops', 'num_hidden_ops', 'device',
'input_shapes'
]
with session.Session() as sess:
x = lib.BuildSmallModel()
sess.run(variables.global_variables_initializer())
run_meta = config_pb2.RunMetadata()
_ = sess.run(x,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE),
run_metadata=run_meta)
model_analyzer.print_model_analysis(
sess.graph, run_meta, tfprof_cmd='code', tfprof_options=opts)
with gfile.Open(outfile, 'r') as f:
# pylint: disable=line-too-long
self.assertEqual(
'node name | output bytes | # parameters | # float_ops | assigned devices | input',
f.read()[0:80])
def testTimeline(self):
ops.reset_default_graph()
opts = model_analyzer.TRAINABLE_VARS_PARAMS_STAT_OPTIONS.copy()
outfile = os.path.join(test.get_temp_dir(), 'timeline')
opts['output'] = 'timeline:outfile=' + outfile
opts['account_type_regexes'] = ['.*']
opts['max_depth'] = 100000
opts['step'] = 0
with session.Session() as sess:
x = lib.BuildFullModel()
sess.run(variables.global_variables_initializer())
run_meta = config_pb2.RunMetadata()
_ = sess.run(
x,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE),
run_metadata=run_meta)
_ = model_analyzer.print_model_analysis(
sess.graph, run_meta, tfprof_cmd='graph', tfprof_options=opts)
with gfile.Open(outfile, 'r') as f:
# Test that a json file is created.
# TODO(xpan): tfprof Timeline isn't quite correct on Windows.
# Investigate why.
if os.name != 'nt':
self.assertLess(1000, len(f.read()))
else:
self.assertLess(1, len(f.read()))
def testAdvisor(self):
ops.reset_default_graph()
with session.Session() as sess:
x = lib.BuildFullModel()
sess.run(variables.global_variables_initializer())
run_meta = config_pb2.RunMetadata()
_ = sess.run(
x,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE),
run_metadata=run_meta)
advice_pb = model_analyzer.advise(sess.graph, run_meta)
self.assertTrue('AcceleratorUtilizationChecker' in advice_pb.checkers)
self.assertTrue('ExpensiveOperationChecker' in advice_pb.checkers)
self.assertTrue('OperationChecker' in advice_pb.checkers)
checker = advice_pb.checkers['AcceleratorUtilizationChecker']
if test.is_gpu_available():
self.assertGreater(len(checker.reports), 0)
else:
self.assertEqual(len(checker.reports), 0)
checker = advice_pb.checkers['ExpensiveOperationChecker']
self.assertGreater(len(checker.reports), 0)
def testSimpleCodeView(self):
ops.reset_default_graph()
outfile = os.path.join(test.get_temp_dir(), 'dump')
# TODO(xpan): Test 'micros'. Since the execution time changes each run,
# it's a bit difficult to test it now.
opts = (builder(builder.trainable_variables_parameter())
.with_file_output(outfile)
.with_accounted_types(['.*'])
.with_node_names(show_name_regexes=['.*model_analyzer_testlib.*'])
.account_displayed_op_only(False)
.select(['bytes', 'params', 'float_ops', 'num_hidden_ops', 'device',
'input_shapes']).build())
with session.Session() as sess:
x = lib.BuildSmallModel()
sess.run(variables.global_variables_initializer())
run_meta = config_pb2.RunMetadata()
_ = sess.run(x,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE),
run_metadata=run_meta)
model_analyzer.profile(
sess.graph, run_meta, cmd='code', options=opts)
with gfile.Open(outfile, 'r') as f:
# pylint: disable=line-too-long
self.assertEqual(
'node name | output bytes | # parameters | # float_ops | assigned devices | input',
f.read()[0:80])
def testCodeViewLeafGraphNode(self):
ops.reset_default_graph()
opts = model_analyzer.TRAINABLE_VARS_PARAMS_STAT_OPTIONS.copy()
opts['account_type_regexes'] = ['.*']
opts['account_displayed_op_only'] = False
opts['select'] = [
'bytes', 'params', 'float_ops', 'device'
]
opts['output'] = 'none'
with session.Session() as sess:
x = lib.BuildSmallModel()
sess.run(variables.global_variables_initializer())
run_meta = config_pb2.RunMetadata()
_ = sess.run(x,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE),
run_metadata=run_meta)
tfprof_node = model_analyzer.print_model_analysis(
sess.graph, run_meta, tfprof_cmd='code', tfprof_options=opts)
leaf = tfprof_node
while leaf.children:
self.assertEqual(0, len(leaf.graph_nodes))
leaf = leaf.children[0]
self.assertEqual(1, len(leaf.graph_nodes))
def testCodeViewLeafGraphNode(self):
ops.reset_default_graph()
opts = (builder(builder.trainable_variables_parameter())
.with_empty_output()
.with_accounted_types(['.*'])
.account_displayed_op_only(False)
.select(['bytes', 'params', 'float_ops', 'device']).build())
with session.Session() as sess:
x = lib.BuildSmallModel()
sess.run(variables.global_variables_initializer())
run_meta = config_pb2.RunMetadata()
_ = sess.run(x,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE),
run_metadata=run_meta)
tfprof_node = model_analyzer.profile(
sess.graph, run_meta, cmd='code', options=opts)
leaf = tfprof_node
while leaf.children:
self.assertEqual(0, len(leaf.graph_nodes))
leaf = leaf.children[0]
self.assertEqual(1, len(leaf.graph_nodes))
def testTimeline(self):
ops.reset_default_graph()
outfile = os.path.join(test.get_temp_dir(), 'timeline')
opts = (builder(builder.trainable_variables_parameter())
.with_max_depth(100000)
.with_step(0)
.with_timeline_output(outfile)
.with_accounted_types(['.*']).build())
with session.Session() as sess:
x = lib.BuildFullModel()
sess.run(variables.global_variables_initializer())
run_meta = config_pb2.RunMetadata()
_ = sess.run(
x,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE),
run_metadata=run_meta)
_ = model_analyzer.profile(
sess.graph, run_meta, cmd='graph', options=opts)
with gfile.Open(outfile, 'r') as f:
# Test that a json file is created.
# TODO(xpan): tfprof Timeline isn't quite correct on Windows.
# Investigate why.
if os.name != 'nt':
self.assertLess(1000, len(f.read()))
else:
self.assertLess(1, len(f.read()))
def testTrainRegressorNonInMemory(self):
ops.reset_default_graph()
expected_first, expected_second, expected_third = (
self._get_expected_ensembles_for_regression())
with self.test_session() as sess:
# Train without train_in_memory mode.
with sess.graph.as_default():
train_op, ensemble_serialized = self._get_train_op_and_ensemble(
boosted_trees._create_regression_head(label_dimension=1),
run_config.RunConfig(),
is_classification=False,
train_in_memory=False)
_, serialized = sess.run([train_op, ensemble_serialized])
# Validate the trained ensemble.
ensemble_proto = boosted_trees_pb2.TreeEnsemble()
ensemble_proto.ParseFromString(serialized)
self.assertProtoEquals(expected_first, ensemble_proto)
# Run one more time and validate the trained ensemble.
_, serialized = sess.run([train_op, ensemble_serialized])
ensemble_proto = boosted_trees_pb2.TreeEnsemble()
ensemble_proto.ParseFromString(serialized)
self.assertProtoEquals(expected_second, ensemble_proto)
# Third round training and validation.
_, serialized = sess.run([train_op, ensemble_serialized])
ensemble_proto = boosted_trees_pb2.TreeEnsemble()
ensemble_proto.ParseFromString(serialized)
self.assertProtoEquals(expected_third, ensemble_proto)
def load_policy(cls, policy_dict_path, tf_generator, network_config=None):
"""
For when we only need to load a policy for the forward pass. For instance, to run on the robot from
a checkpointed policy.
"""
from tensorflow.python.framework import ops
ops.reset_default_graph() # we need to destroy the default graph before re_init or checkpoint won't restore.
pol_dict = pickle.load(open(policy_dict_path, "rb"))
tf_map = tf_generator(dim_input=pol_dict['deg_obs'], dim_output=pol_dict['deg_action'],
batch_size=1, network_config=network_config)
sess = tf.Session()
init_op = tf.initialize_all_variables()
sess.run(init_op)
saver = tf.train.Saver()
check_file = pol_dict['checkpoint_path_tf']
saver.restore(sess, check_file)
device_string = pol_dict['device_string']
cls_init = cls(pol_dict['deg_action'], tf_map.get_input_tensor(), tf_map.get_output_op(), np.zeros((1,)),
sess, device_string)
cls_init.chol_pol_covar = pol_dict['chol_pol_covar']
cls_init.scale = pol_dict['scale']
cls_init.bias = pol_dict['bias']
cls_init.x_idx = pol_dict['x_idx']
return cls_init
def setUp(self):
ops.reset_default_graph()
self.scalar_int_feed = array_ops.placeholder(dtypes_lib.int32, ())
self.unk_int64_feed = array_ops.placeholder(dtypes_lib.int64, (None,))
self.vec3_str_feed = array_ops.placeholder(dtypes_lib.string, (3,))
self.sparse_c = sparse_tensor.SparseTensor(
indices=[[0]],
values=[1.0],
dense_shape=[1])
self._coord = coordinator.Coordinator()
# Make capacity very large so we can feed all the inputs in the
# main thread without blocking
input_queue = data_flow_ops.PaddingFIFOQueue(
5000,
dtypes=[dtypes_lib.int32, dtypes_lib.int64, dtypes_lib.string],
shapes=[(), (None,), (3,)])
self._input_enqueue_op = input_queue.enqueue(
(self.scalar_int_feed, self.unk_int64_feed, self.vec3_str_feed))
self.scalar_int, self.unk_int64, self.vec3_str = input_queue.dequeue()
self._threads = None
self._close_op = input_queue.close()
self._sess = None
def testDropoutWrapperWithSeed(self):
keep_some = 0.5
random_seed.set_random_seed(2)
## Use parallel_iterations = 1 in both calls to
## _testDropoutWrapper to ensure the (per-time step) dropout is
## consistent across both calls. Otherwise the seed may not end
## up being munged consistently across both graphs.
res_standard_1 = self._testDropoutWrapper(
input_keep_prob=keep_some,
output_keep_prob=keep_some,
state_keep_prob=keep_some,
seed=10,
parallel_iterations=1)
# Clear away the graph and the test session (which keeps variables around)
ops.reset_default_graph()
self._ClearCachedSession()
random_seed.set_random_seed(2)
res_standard_2 = self._testDropoutWrapper(
input_keep_prob=keep_some,
output_keep_prob=keep_some,
state_keep_prob=keep_some,
seed=10,
parallel_iterations=1)
self.assertAllClose(res_standard_1[0], res_standard_2[0])
self.assertAllClose(res_standard_1[1].c, res_standard_2[1].c)
self.assertAllClose(res_standard_1[1].h, res_standard_2[1].h)
def testTrackPersistentBytes(self):
ops.reset_default_graph()
a = array_ops.constant(np.ones((100, 100)))
b = array_ops.constant(np.ones((100, 100)))
c = a * b
with session.Session() as sess:
run_options = config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE)
run_metadata = config_pb2.RunMetadata()
sess.run(c, options=run_options, run_metadata=run_metadata)
options = option_builder.ProfileOptionBuilder.time_and_memory()
options['min_bytes'] = 0
options['select'] = ('bytes', 'peak_bytes', 'output_bytes',
'residual_bytes')
ret = model_analyzer.profile(
sess.graph, run_meta=run_metadata, cmd='scope', options=options)
run_metadata = config_pb2.RunMetadata()
sess.run(c, options=run_options, run_metadata=run_metadata)
ret2 = model_analyzer.profile(
sess.graph, run_meta=run_metadata, cmd='scope', options=options)
n = lib.SearchTFProfNode(ret, 'mul')
n2 = lib.SearchTFProfNode(ret2, 'mul')
self.assertGreater(n.peak_bytes, 0)
self.assertGreater(n.output_bytes, 0)
self.assertGreater(n.residual_bytes, 0)
self.assertEqual(n.peak_bytes, n2.peak_bytes)
self.assertEqual(n.output_bytes, n2.output_bytes)
self.assertEqual(n.residual_bytes, n2.residual_bytes)
def testDebuggedSessionRunWorksWithDelayedDebugServerStartup(self):
"""Test debugged Session.run() tolerates delayed debug server startup."""
ops.reset_default_graph()
# Start a debug server asynchronously, with a certain amount of delay.
(debug_server_port, _, _, server_thread,
debug_server) = grpc_debug_test_server.start_server_on_separate_thread(
server_start_delay_sec=2.0, dump_to_filesystem=False)
with self.test_session() as sess:
a_init = constant_op.constant(42.0, name="a_init")
a = variables.Variable(a_init, name="a")
def watch_fn(fetches, feeds):
del fetches, feeds
return framework.WatchOptions(debug_ops=["DebugIdentity"])
sess = grpc_wrapper.GrpcDebugWrapperSession(
sess, "localhost:%d" % debug_server_port, watch_fn=watch_fn)
sess.run(a.initializer)
self.assertAllClose(
[42.0], debug_server.debug_tensor_values["a_init:0:DebugIdentity"])
debug_server.stop_server().wait()
server_thread.join()
def testOOM(self):
if not test.is_gpu_available():
return
ops.reset_default_graph()
with ops.device('/device:GPU:0'):
a = random_ops.random_normal([1, 10000, 20000], name='test_random1')
b = random_ops.random_normal([30000, 10000, 1], name='test_random2')
c = a * b
try:
with session.Session() as sess:
sess.run(c, options=config_pb2.RunOptions(
report_tensor_allocations_upon_oom=True))
except Exception as e: # pylint: disable=broad-except
exception_str = '%s' % e
# This trace reports allocations for to random tensor.
self.assertTrue(
'OOM when allocating tensor with shape[30000,10000,20000]' in
exception_str)
mat = re.search('(.*)GiB from test_random2/RandomStandardNormal',
exception_str)
self.assertGreater(float(mat.group(1)), 0.0)
mat = re.search('(.*)MiB from test_random1/RandomStandardNormal',
exception_str)
self.assertGreater(float(mat.group(1)), 0.0)
def testDistributedOOM(self):
if not test.is_gpu_available():
return
ops.reset_default_graph()
workers, _ = test_util.create_local_cluster(2, 0)
with ops.device('/job:worker/replica:0/task:0/gpu:0'):
a = random_ops.random_normal([1, 10000, 20000], name='test_random1')
with ops.device('/job:worker/replica:0/task:1/gpu:0'):
b = random_ops.random_normal([30000, 10000, 1], name='test_random2')
c = a * b
try:
with session.Session(workers[1].target) as sess:
sess.run(c, options=config_pb2.RunOptions(
report_tensor_allocations_upon_oom=True))
except Exception as e: # pylint: disable=broad-except
exception_str = '%s' % e
# test_random2 is reported because it's allocated in worker 1.
self.assertTrue('Current usage from device: '
'/job:worker/replica:0/task:1/device:GPU:0, '
'allocator: GPU_0_bfc' in exception_str)
mat = re.search('(.*)GiB from test_random2/RandomStandardNormal',
exception_str)
self.assertGreater(float(mat.group(1)), 0.0)
# test_random1 is not reported because it's allocated in worker 0.
mat = re.search('(.*)MiB from test_random1/RandomStandardNormal',
exception_str)
self.assertTrue(mat is None)
def testComplexCodeView(self):
ops.reset_default_graph()
outfile = os.path.join(test.get_temp_dir(), 'dump')
opts = (builder(builder.trainable_variables_parameter())
.with_file_output(outfile)
.with_accounted_types(['.*'])
.with_node_names(show_name_regexes=
['.*model_analyzer_testlib.py.*'])
.account_displayed_op_only(False)
.select(['params', 'float_ops']).build())
with profile_context.ProfileContext(test.get_temp_dir(),
trace_steps=[],
dump_steps=[]) as pctx:
with session.Session() as sess:
x = lib.BuildFullModel()
sess.run(variables.global_variables_initializer())
pctx.trace_next_step()
_ = sess.run(x)
tfprof_node = pctx.profiler.profile_python(options=opts)
# pylint: disable=line-too-long
with gfile.Open(outfile, 'r') as f:
lines = f.read().split('\n')
result = '\n'.join([l[:min(len(l), 80)] for l in lines])
self.assertEqual(
compat.as_bytes(
'node name | # parameters | # float_ops\n_TFProfRoot (--/2.84k params, --/168.86k flops)\n model_analyzer_testlib.py:63:BuildFullModel (0/1.80k params, 0/45.37k flops)\n model_analyzer_testlib.py:40:BuildSmallModel (0/0 params, 0/0 flops)\n model_analyzer_testlib.py:44:BuildSmallModel (0/4 params, 0/8 flops)\n model_analyzer_testlib.py:48:BuildSmallModel (0/648 params, 0/1.30k flops)\n model_analyzer_testlib.py:49:BuildSmallModel (0/0 params, 0/23.33k flops)\n model_analyzer_testlib.py:53:BuildSmallModel (0/1.15k params, 0/2.30k flops)\n model_analyzer_testlib.py:54:BuildSmallModel (0/0 params, 0/18.43k flops)\n model_analyzer_testlib.py:63:BuildFullModel (gradient) (0/0 params, 0/67.39k f\n model_analyzer_testlib.py:49:BuildSmallModel (gradient) (0/0 params, 0/46.66\n model_analyzer_testlib.py:54:BuildSmallModel (gradient) (0/0 params, 0/20.74\n model_analyzer_testlib.py:67:BuildFullModel (0/1.04k params, 0/18.58k flops)\n model_analyzer_testlib.py:67:BuildFullModel (gradient) (0/0 params, 0/37.00k f\n model_analyzer_testlib.py:69:BuildFullModel (0/0 params, 0/0 flops)\n model_analyzer_testlib.py:70:BuildFullModel (0/0 params, 0/258 flops)\n model_analyzer_testlib.py:70:BuildFullModel (gradient) (0/0 params, 0/129 flop\n model_analyzer_testlib.py:72:BuildFullModel (0/0 params, 0/141 flops)\n'
), compat.as_bytes(lib.CheckAndRemoveDoc(result)))
self.assertLess(0, tfprof_node.total_exec_micros)
self.assertEqual(2844, tfprof_node.total_parameters)
self.assertEqual(168863, tfprof_node.total_float_ops)
self.assertEqual(8, len(tfprof_node.children))
self.assertEqual('_TFProfRoot', tfprof_node.name)
self.assertEqual(
'model_analyzer_testlib.py:63:BuildFullModel',
tfprof_node.children[0].name)
self.assertEqual(
'model_analyzer_testlib.py:63:BuildFullModel (gradient)',
tfprof_node.children[1].name)
self.assertEqual(
'model_analyzer_testlib.py:67:BuildFullModel',
tfprof_node.children[2].name)
self.assertEqual(
'model_analyzer_testlib.py:67:BuildFullModel (gradient)',
tfprof_node.children[3].name)
self.assertEqual(
'model_analyzer_testlib.py:69:BuildFullModel',
tfprof_node.children[4].name)
self.assertEqual(
'model_analyzer_testlib.py:70:BuildFullModel',
tfprof_node.children[5].name)
self.assertEqual(
'model_analyzer_testlib.py:70:BuildFullModel (gradient)',
tfprof_node.children[6].name)
self.assertEqual(
'model_analyzer_testlib.py:72:BuildFullModel',
tfprof_node.children[7].name)
def testSelectEverything(self):
ops.reset_default_graph()
opts = model_analyzer.TRAINABLE_VARS_PARAMS_STAT_OPTIONS
outfile = os.path.join(test.get_temp_dir(), 'dump')
opts['output'] = 'file:outfile=' + outfile
opts['account_type_regexes'] = ['.*']
opts['select'] = [
'bytes', 'params', 'float_ops', 'occurrence',
'device', 'op_types'
]
with session.Session() as sess, ops.device('/cpu:0'):
x = lib.BuildSmallModel()
sess.run(variables.global_variables_initializer())
run_meta = config_pb2.RunMetadata()
_ = sess.run(x,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE),
run_metadata=run_meta)
model_analyzer.print_model_analysis(
sess.graph, run_meta, tfprof_options=opts)
with gfile.Open(outfile, 'r') as f:
# pylint: disable=line-too-long
self.assertEqual(
'node name | # parameters | # float_ops | output bytes | assigned devices | op types\n_TFProfRoot (--/451 params, --/10.44k flops, --/5.28KB, _kTFScopeParent)\n Conv2D (0/0 params, 5.83k/5.83k flops, 432B/432B, /job:localhost/replica:0/task:0/cpu:0, /job:localhost/replica:0/task:0/cpu:0|Conv2D)\n Conv2D_1 (0/0 params, 4.61k/4.61k flops, 384B/384B, /job:localhost/replica:0/task:0/cpu:0, /job:localhost/replica:0/task:0/cpu:0|Conv2D)\n DW (3x3x3x6, 162/162 params, 0/0 flops, 648B/1.30KB, /job:localhost/replica:0/task:0/cpu:0, /job:localhost/replica:0/task:0/cpu:0|VariableV2|_trainable_variables)\n DW/Assign (0/0 params, 0/0 flops, 0B/0B, Assign)\n DW/Initializer (0/0 params, 0/0 flops, 0B/0B, _kTFScopeParent)\n DW/Initializer/random_normal (0/0 params, 0/0 flops, 0B/0B, Add)\n DW/Initializer/random_normal/RandomStandardNormal (0/0 params, 0/0 flops, 0B/0B, RandomStandardNormal)\n DW/Initializer/random_normal/mean (0/0 params, 0/0 flops, 0B/0B, Const)\n DW/Initializer/random_normal/mul (0/0 params, 0/0 flops, 0B/0B, Mul)\n DW/Initializer/random_normal/shape (0/0 params, 0/0 flops, 0B/0B, Const)\n DW/Initializer/random_normal/stddev (0/0 params, 0/0 flops, 0B/0B, Const)\n DW/read (0/0 params, 0/0 flops, 648B/648B, /job:localhost/replica:0/task:0/cpu:0, /job:localhost/replica:0/task:0/cpu:0|Identity)\n DW2 (2x2x6x12, 288/288 params, 0/0 flops, 1.15KB/2.30KB, /job:localhost/replica:0/task:0/cpu:0, /job:localhost/replica:0/task:0/cpu:0|VariableV2|_trainable_variables)\n DW2/Assign (0/0 params, 0/0 flops, 0B/0B, Assign)\n DW2/Initializer (0/0 params, 0/0 flops, 0B/0B, _kTFScopeParent)\n DW2/Initializer/random_normal (0/0 params, 0/0 flops, 0B/0B, Add)\n DW2/Initializer/random_normal/RandomStandardNormal (0/0 params, 0/0 flops, 0B/0B, RandomStandardNormal)\n DW2/Initializer/random_normal/mean (0/0 params, 0/0 flops, 0B/0B, Const)\n DW2/Initializer/random_normal/mul (0/0 params, 0/0 flops, 0B/0B, Mul)\n DW2/Initializer/random_normal/shape (0/0 params, 0/0 flops, 0B/0B, Const)\n DW2/Initializer/random_normal/stddev (0/0 params, 0/0 flops, 0B/0B, Const)\n DW2/read (0/0 params, 0/0 flops, 1.15KB/1.15KB, /job:localhost/replica:0/task:0/cpu:0, /job:localhost/replica:0/task:0/cpu:0|Identity)\n ScalarW (1, 1/1 params, 0/0 flops, 0B/0B, VariableV2|_trainable_variables)\n ScalarW/Assign (0/0 params, 0/0 flops, 0B/0B, Assign)\n ScalarW/Initializer (0/0 params, 0/0 flops, 0B/0B, _kTFScopeParent)\n ScalarW/Initializer/random_normal (0/0 params, 0/0 flops, 0B/0B, Add)\n ScalarW/Initializer/random_normal/RandomStandardNormal (0/0 params, 0/0 flops, 0B/0B, RandomStandardNormal)\n ScalarW/Initializer/random_normal/mean (0/0 params, 0/0 flops, 0B/0B, Const)\n ScalarW/Initializer/random_normal/mul (0/0 params, 0/0 flops, 0B/0B, Mul)\n ScalarW/Initializer/random_normal/shape (0/0 params, 0/0 flops, 0B/0B, Const)\n ScalarW/Initializer/random_normal/stddev (0/0 params, 0/0 flops, 0B/0B, Const)\n ScalarW/read (0/0 params, 0/0 flops, 0B/0B, Identity)\n init (0/0 params, 0/0 flops, 0B/0B, NoOp)\n zeros (0/0 params, 0/0 flops, 864B/864B, /job:localhost/replica:0/task:0/cpu:0, /job:localhost/replica:0/task:0/cpu:0|Const)\n',
f.read())
def testBasic(self):
base_path = test.test_src_dir_path(SESSION_BUNDLE_PATH)
ops.reset_default_graph()
sess, meta_graph_def = session_bundle.load_session_bundle_from_path(
base_path,
target="",
config=config_pb2.ConfigProto(device_count={"CPU": 2}))
self.assertTrue(sess)
asset_path = os.path.join(base_path, constants.ASSETS_DIRECTORY)
with sess.as_default():
path1, path2 = sess.run(["filename1:0", "filename2:0"])
self.assertEqual(
compat.as_bytes(os.path.join(asset_path, "hello1.txt")), path1)
self.assertEqual(
compat.as_bytes(os.path.join(asset_path, "hello2.txt")), path2)
collection_def = meta_graph_def.collection_def
signatures_any = collection_def[constants.SIGNATURES_KEY].any_list.value
self.assertEquals(len(signatures_any), 1)
signatures = manifest_pb2.Signatures()
signatures_any[0].Unpack(signatures)
self._checkRegressionSignature(signatures, sess)
self._checkNamedSignatures(signatures, sess)
def setUp(self):
ops.reset_default_graph()
dim = 1
num = 3
with ops.name_scope('some_scope'):
# Basically from 0 to dim*num-1.
flat_data = math_ops.linspace(0.0, dim * num - 1, dim * num)
bias = variables.Variable(
array_ops.reshape(flat_data, (num, dim)), name='bias')
save = saver.Saver([bias])
with self.test_session() as sess:
variables.global_variables_initializer().run()
self.bundle_file = os.path.join(test.get_temp_dir(), 'bias_checkpoint')
save.save(sess, self.bundle_file)
self.new_class_vocab_file = os.path.join(
test.test_src_dir_path(_TESTDATA_PATH), 'keyword_new.txt')
self.old_class_vocab_file = os.path.join(
test.test_src_dir_path(_TESTDATA_PATH), 'keyword.txt')
self.init_val = 42
def _init_val_initializer(shape, dtype=None, partition_info=None):
del dtype, partition_info # Unused by this unit-testing initializer.
return array_ops.tile(
constant_op.constant([[self.init_val]], dtype=dtypes.float32), shape)
self.initializer = _init_val_initializer
def test_graph_replace_gradients(self):
ops.reset_default_graph()
w = variables.VariableV1(0.0, name="w")
y = math_ops.multiply(math_ops.multiply(w, w, name="mul1"), w, name="mul2")
g = gradients_impl.gradients(y, w, name="grad")[0]
# Extract the operations.
replacement_ts = {w.value(): g}
original_mul1_grad = (ops.get_default_graph().
get_operation_by_name("grad/mul1_grad/Mul_1"))
# Should not raise exception.
res = ge.graph_replace(g, replacement_ts, dst_scope="res")
# Extract the operations after graph_replace.
result_mul1_grad = (ops.get_default_graph().
get_operation_by_name("res/grad/mul1_grad/Mul_1"))
# Make sure _original_ops are as expected.
self.assertEqual(original_mul1_grad._original_op.name, u"mul1")
self.assertEqual(result_mul1_grad._original_op.name, u"res/mul1")
self.assertNotEqual(res.name, g.name)
with session.Session() as sess:
sess.run(variables.global_variables_initializer())
g_val, res_val = sess.run([g, res])
self.assertNear(g_val, 0.0, ERROR_TOLERANCE)
self.assertNear(res_val, 0.0, ERROR_TOLERANCE)
def testBasics(self):
ops.reset_default_graph()
outfile = os.path.join(test.get_temp_dir(), "dump")
opts = builder(builder.time_and_memory()
).with_file_output(outfile).build()
x = lib.BuildFullModel()
profile_str = None
profile_step100 = os.path.join(test.get_temp_dir(), "profile_100")
with profile_context.ProfileContext(test.get_temp_dir()) as pctx:
pctx.add_auto_profiling("op", options=opts, profile_steps=[15, 50, 100])
with session.Session() as sess:
self.evaluate(variables.global_variables_initializer())
total_steps = 101
for i in range(total_steps):
self.evaluate(x)
if i == 14 or i == 49:
self.assertTrue(gfile.Exists(outfile))
gfile.Remove(outfile)
if i == 99:
self.assertTrue(gfile.Exists(profile_step100))
with gfile.Open(outfile, "r") as f:
profile_str = f.read()
gfile.Remove(outfile)
self.assertEqual(set([15, 50, 100]), set(pctx.get_profiles("op").keys()))
with lib.ProfilerFromFile(
os.path.join(test.get_temp_dir(), "profile_100")) as profiler:
profiler.profile_operations(options=opts)
with gfile.Open(outfile, "r") as f:
self.assertEqual(profile_str, f.read())
def testSelectEverything(self):
ops.reset_default_graph()
outfile = os.path.join(test.get_temp_dir(), 'dump')
opts = (builder(builder.trainable_variables_parameter())
.with_file_output(outfile)
.with_accounted_types(['.*'])
.select(['params', 'float_ops', 'occurrence', 'device', 'op_types',
'input_shapes']).build())
rewriter_config = rewriter_config_pb2.RewriterConfig(
disable_model_pruning=True)
graph_options = config_pb2.GraphOptions(rewrite_options=rewriter_config)
config = config_pb2.ConfigProto(graph_options=graph_options)
with session.Session(config=config) as sess, ops.device('/device:CPU:0'):
x = lib.BuildSmallModel()
sess.run(variables.global_variables_initializer())
run_meta = config_pb2.RunMetadata()
_ = sess.run(x,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE),
run_metadata=run_meta)
model_analyzer.profile(
sess.graph, run_meta, options=opts)
def generate_testdata(self):
ops.reset_default_graph()
sess = session.Session()
placeholder = array_ops.placeholder(dtypes.string)
summary_tensor = text_summary.text_summary('message', placeholder)
vector_summary = text_summary.text_summary('vector', placeholder)
run_names = ['fry', 'leela']
for run_name in run_names:
subdir = os.path.join(self.logdir, run_name)
writer = summary.FileWriter(subdir)
writer.add_graph(sess.graph)
step = 0
for gem in GEMS:
message = run_name + ' *loves* ' + gem
feed_dict = {placeholder: message}
summ = sess.run(summary_tensor, feed_dict=feed_dict)
writer.add_summary(summ, global_step=step)
step += 1
vector_message = ['one', 'two', 'three', 'four']
summ = sess.run(vector_summary, feed_dict={placeholder: vector_message})
writer.add_summary(summ)
writer.close()
def testAutoProfiling(self):
ops.reset_default_graph()
time_dir = os.path.join(test.get_temp_dir(), 'time')
memory_dir = os.path.join(test.get_temp_dir(), 'memory')
profile_dir = os.path.join(test.get_temp_dir(), 'dir/dir2/profile')
# TODO(xpan): Should we create parent directory for them?
gfile.MkDir(time_dir)
gfile.MkDir(memory_dir)
time_opts = (builder(builder.time_and_memory())
.with_file_output(os.path.join(time_dir, 'profile'))
.select(['micros']).build())
memory_opts = (builder(builder.time_and_memory())
.with_file_output(os.path.join(memory_dir, 'profile'))
.select(['bytes']).build())
time_steps = [2, 3]
memory_steps = [1, 3]
dump_steps = [3, 4]
x = lib.BuildSmallModel()
with profile_context.ProfileContext(profile_dir,
trace_steps=[1, 2, 3],
dump_steps=[3, 4]) as pctx:
pctx.add_auto_profiling('scope', time_opts, time_steps)
pctx.add_auto_profiling('scope', memory_opts, memory_steps)
self._trainLoop(x, 10, time_dir, time_steps,
memory_dir, memory_steps, profile_dir, dump_steps)
def model(X_train, Y_train, X_dev, Y_dev, learning_rate, lambd, layers_dims):
ops.reset_default_graph()
X = tf.placeholder(tf.float32, shape=(dim, None), name='X')
Y = tf.placeholder(tf.float32, shape=(10, None), name='Y')
parameters = mm.initialize_parameters(layers_dims)
ZL = mm.forward_propagation(X, parameters)
train_prediction = tf.nn.softmax(ZL)
loss = mm.loss(tf.transpose(Y), tf.transpose(ZL), lambd, parameters)
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(loss)
correct_prediction = tf.equal(tf.argmax(ZL), tf.argmax(Y))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
# Save the loss on training and dev set after each epoch, as well as the dev accuracy.
loss_train = []
loss_dev = []
with tf.Session() as session:
# This is a one-time operation which ensures the parameters get initialized as
# we described in the graph: random weights for the matrix, zeros for the
# biases.
tf.global_variables_initializer().run()
print('Initialized')
for epoch in range(num_epochs):
epoch_cost = 0.
num_minibatches = int(train_size / minibatch_size)
mini_batches = mm.random_mini_batches(X_train, Y_train,
minibatch_size)
for step, mini_batch in enumerate(mini_batches):
# Run the computations. We tell .run() that we want to run the optimizer,
# and get the loss value and the training predictions returned as numpy
# arrays.
_, l, predictions = session.run(
[optimizer, loss, train_prediction],
feed_dict={
X: mini_batch[0],
Y: mini_batch[1]
})
# if (step % 100 == 0):
# print('Loss at step %d: %f' % (step, l))
# Calculate the correct predictions
loss_train.append(str(session.run(loss, {X: X_train, Y: Y_train})))
loss_dev.append(str(session.run(loss, {X: X_dev, Y: Y_dev})))
print('----- epoch: {0} -----'.format(epoch + 1))
print('Loss train = ' +
str(session.run(loss, {
X: X_train,
Y: Y_train
})))
print('Loss dev = ' + str(session.run(loss, {X: X_dev, Y: Y_dev})))
print('Accuracy train ' +
str(accuracy.eval({
X: X_train,
Y: Y_train
})))
print('Accuracy dev ' + str(accuracy.eval({X: X_dev, Y: Y_dev})))
train_accuracy = accuracy.eval({X: X_train, Y: Y_train})
dev_accuracy = accuracy.eval({X: X_dev, Y: Y_dev})
print(accuracy.eval({X: X_train, Y: Y_train}))
print(accuracy.eval({X: X_dev, Y: Y_dev}))
mm.plot_results(loss_train, loss_dev, learning_rate, train_accuracy,
dev_accuracy)
# -*- coding: utf-8 -*-
# TensorFlow Production Example (Evaluating)
#----------------------------------
#
# We pull together everything and create an example
# of best tensorflow production tips
#
# The example we will productionalize is the spam/ham RNN
# from the RNN Chapter.
import os
import re
import numpy as np
import tensorflow as tf
from tensorflow.python.framework import ops
ops.reset_default_graph()
tf.app.flags.DEFINE_string("storage_folder", "temp",
"Where to store model and data.")
tf.app.flags.DEFINE_boolean('model_file', False, 'Model file location.')
tf.app.flags.DEFINE_boolean('run_unit_tests', False, 'If true, run tests.')
FLAGS = tf.app.flags.FLAGS
# Create a text cleaning function
def clean_text(text_string):
text_string = re.sub(r'([^\s\w]|_|[0-9])+', '', text_string)
text_string = " ".join(text_string.split())
text_string = text_string.lower()
return (text_string)
def model(X_train,
Y_train,
X_test,
Y_test,
learning_rate=0.009,
num_epochs=100,
minibatch_size=64,
print_cost=True):
"""
Implements a three-layer ConvNet in Tensorflow:
CONV2D -> RELU -> MAXPOOL -> CONV2D -> RELU -> MAXPOOL -> FLATTEN -> FULLYCONNECTED
Arguments:
X_train -- training set, of shape (None, 64, 64, 3)
Y_train -- test set, of shape (None, n_y = 6)
X_test -- training set, of shape (None, 64, 64, 3)
Y_test -- test set, of shape (None, n_y = 6)
learning_rate -- learning rate of the optimization
num_epochs -- number of epochs of the optimization loop
minibatch_size -- size of a minibatch
print_cost -- True to print the cost every 100 epochs
Returns:
train_accuracy -- real number, accuracy on the train set (X_train)
test_accuracy -- real number, testing accuracy on the test set (X_test)
parameters -- parameters learnt by the model. They can then be used to predict.
"""
ops.reset_default_graph(
) # to be able to rerun the model without overwriting tf variables
tf.set_random_seed(1) # to keep results consistent (tensorflow seed)
seed = 3 # to keep results consistent (numpy seed)
(m, n_H0, n_W0, n_C0) = X_train.shape
n_y = Y_train.shape[1]
costs = [] # To keep track of the cost
# Create Placeholders of the correct shape
### START CODE HERE ### (1 line)
X, Y = create_placeholders(n_H0, n_W0, n_C0, n_y)
### END CODE HERE ###
# Initialize parameters
### START CODE HERE ### (1 line)
parameters = initialize_parameters()
### END CODE HERE ###
# Forward propagation: Build the forward propagation in the tensorflow graph
### START CODE HERE ### (1 line)
Z3 = forward_propagation(X, parameters)
### END CODE HERE ###
# Cost function: Add cost function to tensorflow graph
### START CODE HERE ### (1 line)
cost = compute_cost(Z3, Y)
### END CODE HERE ###
# Backpropagation: Define the tensorflow optimizer. Use an AdamOptimizer that minimizes the cost.
### START CODE HERE ### (1 line)
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)
### END CODE HERE ###
# Initialize all the variables globally
init = tf.global_variables_initializer()
# Start the session to compute the tensorflow graph
with tf.Session() as sess:
# Run the initialization
sess.run(init)
# Do the training loop
for epoch in range(num_epochs):
minibatch_cost = 0.
num_minibatches = int(
m / minibatch_size
) # number of minibatches of size minibatch_size in the train set
seed = seed + 1
minibatches = random_mini_batches(X_train, Y_train, minibatch_size,
seed)
for minibatch in minibatches:
# Select a minibatch
(minibatch_X, minibatch_Y) = minibatch
# IMPORTANT: The line that runs the graph on a minibatch.
# Run the session to execute the optimizer and the cost, the feedict should contain a minibatch for (X,Y).
### START CODE HERE ### (1 line)
_, temp_cost = sess.run([optimizer, cost],
feed_dict={
X: minibatch_X,
Y: minibatch_Y
})
### END CODE HERE ###
minibatch_cost += temp_cost / num_minibatches
# Print the cost every epoch
if print_cost == True and epoch % 5 == 0:
print("Cost after epoch %i: %f" % (epoch, minibatch_cost))
if print_cost == True and epoch % 1 == 0:
costs.append(minibatch_cost)
# plot the cost
plt.plot(np.squeeze(costs))
plt.ylabel('cost')
plt.xlabel('iterations (per tens)')
plt.title("Learning rate =" + str(learning_rate))
plt.show()
# Calculate the correct predictions
predict_op = tf.argmax(Z3, 1)
correct_prediction = tf.equal(predict_op, tf.argmax(Y, 1))
# Calculate accuracy on the test set
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
print(accuracy)
train_accuracy = accuracy.eval({X: X_train, Y: Y_train})
test_accuracy = accuracy.eval({X: X_test, Y: Y_test})
print("Train Accuracy:", train_accuracy)
print("Test Accuracy:", test_accuracy)
return train_accuracy, test_accuracy, parameters
def yolo(X_pretrain,
Y_pretrain,
X_pretest,
Y_pretest,
X_train,
Y_train,
X_test,
Y_test,
learning_rate=0.009,
num_epochs=100,
minibatch_size=64,
print_cost=True):
ops.reset_default_graph()
tf.set_random_seed(1)
seed = 3
mp, hp, wp, cp = X_pretrain.shape
yp = Y_pretrain.shape[1]
pcosts = []
# Run pre-training
Xp, Yp = create_placeholders(hp, wp, cp, yp)
parameters = initializeParameters()
Z_fin_pre = forward_propagation(Xp, parameters, 'classification')
cost = binaryClassificationCost(Z_fin_pre, Yp)
optimizer = tf.train.AdamOptimizer(
learning_rate=learning_rate).minimize(cost)
init = tf.global_variables_initializer()
print('init completed')
with tf.Session() as sess:
sess.run(init)
for epoch in range(num_epochs):
print('epoch ', epoch)
_, c = sess.run([optimizer, cost],
feed_dict={
Xp: X_pretrain,
Yp: Y_pretrain
})
print('Cost after epoch %i: %f' % (epoch, c))
predict_op = tf.argmax(Z_fin_pre, 1)
correct_prediction = tf.equal(predict_op, tf.argmax(Yp, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
train_accuracy = accuracy.eval({Xp: X_pretrain, Yp: Y_pretrain})
test_accuracy = accuracy.eval({Xp: X_pretest, Yp: Y_pretest})
print('Train accuracy', train_accuracy)
print('Test accuracy', test_accuracy)
#ops.reset_default_graph() # Not sure about this
m, nh0, nw0, nc0 = X_train.shape
ny = Y_train.shape[1]
print('ny = ', ny)
costs = []
X, Y = create_placeholders(nh0, nw0, nc0, ny)
Z_fin = forward_propagation(X, parameters, 'detection')
dcost = detectionCost(Z_fin, Y)
optimizer2 = tf.train.AdamOptimizer(
learning_rate=learning_rate).minimize(dcost)
init = tf.global_variables_initializer() # Do we need to set this again?
print('init2 completed')
with tf.Session() as sess:
sess.run(init)
for epoch in range(num_epochs):
minibatch_cost = 0.
num_minibatches = int(m / minibatch_size)
seed = seed + 1
# Temporary fix
_, c = sess.run([optimizer2, dcost],
feed_dict={
X: X_train,
Y: Y_train
})
'''
Let's save mini-batching for later - not tryna debug this now, plus until we get the memory
error worked out, the point is probably moot - I was only able to fit a few hundred examples in.
minibatches = random_mini_batches(X_train, Y_train, minibatch_size, seed)
Scratch that, I'm trying it with 10 examples...
for minibatch in minibatches:
(minibatch_X, minibatch_Y) = minibatch
_, temp_cost = sess.run([optimizer2, dcost],
feed_dict = {X: minibatch_X, Y: minibatch_Y})
minibatch_cost += temp_cost / num_minibatches
'''
if print_cost == True and epoch % 5 == 0:
print("Cost after epoch %i: %f" % (epoch, c))
def train_mynetwork(x1_train_set,
x1_test_set,
y_train_set,
y_test_set,
learning_rate_base=0.001,
beta_reg=0.01,
num_epochs=150,
minibatch_size=64,
print_cost=True):
ops.reset_default_graph()
tf.set_random_seed(1)
seed = 1
(m, n_x1) = x1_train_set.shape
(m, n_y) = y_train_set.shape
costs = []
costs_dev = []
train_acc = []
val_acc = []
correct_prediction = 0
# Create Placeholders of shape (n_x, n_y)
x1, y, isTraining = create_placeholders(n_x1, n_y)
# Initialize parameters
parameters = initialize_parameters()
with tf.name_scope("network"):
joint_layer, l2_loss = mynetwork(x1, parameters, isTraining)
global_step = tf.Variable(0, trainable=False)
learning_rate = tf.train.exponential_decay(learning_rate_base,
global_step,
30 * m / minibatch_size,
0.5,
staircase=True)
with tf.name_scope("optimization"):
# network optimization
cost, optimizer = mynetwork_optimaization(joint_layer, y, l2_loss,
beta_reg, learning_rate,
global_step)
with tf.name_scope("metrics"):
# Calculate the correct predictions
joint_layerT = tf.transpose(joint_layer)
yT = tf.transpose(y)
correct_prediction = tf.equal(tf.argmax(joint_layerT), tf.argmax(yT))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
# Initialize all the variables
init = tf.global_variables_initializer()
saver = tf.train.Saver()
# Start the session to compute the tensorflow graph
with tf.Session() as sess:
# Run the initialization
sess.run(init)
# Do the training loop
for epoch in range(num_epochs + 1):
epoch_cost = 0. # Defines a cost related to an epoch
epoch_acc = 0.
num_minibatches = int(
m / minibatch_size
) # number of minibatches of size minibatch_size in the train set
seed = seed + 1
minibatches = random_mini_batches_standard(x1_train_set,
y_train_set,
minibatch_size, seed)
for minibatch in minibatches:
# Select a minibatch
(batch_x1, batch_y) = minibatch
# IMPORTANT: The line that runs the graph on a minibatch.
# Run the session to execute the "optimizer" and the "cost", the feedict should contain a minibatch for (X,Y).
_, minibatch_cost, minibatch_acc = sess.run(
[optimizer, cost, accuracy],
feed_dict={
x1: batch_x1,
y: batch_y,
isTraining: True
})
epoch_cost += minibatch_cost / (num_minibatches + 1)
epoch_acc += minibatch_acc / (num_minibatches + 1)
feature, epoch_cost_dev, epoch_acc_dev = sess.run(
[joint_layerT, cost, accuracy],
feed_dict={
x1: x1_test_set,
y: y_test_set,
isTraining: False
})
# Print the cost every epoch
if print_cost == True and epoch % 50 == 0:
print(
"epoch %i: Train_loss: %f, Val_loss: %f, Train_acc: %f, Val_acc: %f"
% (epoch, epoch_cost, epoch_cost_dev, epoch_acc,
epoch_acc_dev))
if print_cost == True and epoch % 5 == 0:
costs.append(epoch_cost)
train_acc.append(epoch_acc)
costs_dev.append(epoch_cost_dev)
val_acc.append(epoch_acc_dev)
# plot the cost
plt.plot(np.squeeze(costs))
plt.plot(np.squeeze(costs_dev))
plt.ylabel('cost')
plt.xlabel('iterations (per tens)')
plt.title("Learning rate =" + str(learning_rate))
plt.show()
# plot the accuracy
plt.plot(np.squeeze(train_acc))
plt.plot(np.squeeze(val_acc))
plt.ylabel('accuracy')
plt.xlabel('iterations (per tens)')
plt.title("Learning rate =" + str(learning_rate))
plt.show()
# lets save the parameters in a variable
parameters = sess.run(parameters)
print("Parameters have been trained!")
print("save model")
save_path = saver.save(
sess, "D:\Python_Project\MDL-RS/save_LiDAR/model.ckpt")
print("save model:{0} Finished".format(save_path))
return parameters, val_acc, feature
def _testBucketBySequenceLength(self, allow_small_batch):
ops.reset_default_graph()
# All inputs must be identical lengths across tuple index.
# The input reader will get input_length from the first tuple
# entry.
data_len = 4
labels_len = 3
input_pairs = [(length, ([np.int64(length)] * data_len,
[str(length).encode("ascii")] * labels_len))
for length in (1, 3, 4, 5, 6, 10)]
lengths = array_ops.placeholder(dtypes_lib.int32, ())
data = array_ops.placeholder(dtypes_lib.int64, (data_len,))
labels = array_ops.placeholder(dtypes_lib.string, (labels_len,))
batch_size = 8
bucket_boundaries = [3, 4, 5, 10]
# Make capacity very large so we can feed all the inputs in the
# main thread without blocking
input_queue = data_flow_ops.FIFOQueue(
5000, (dtypes_lib.int32, dtypes_lib.int64, dtypes_lib.string), (
(), (data_len,), (labels_len,)))
input_enqueue_op = input_queue.enqueue((lengths, data, labels))
lengths_t, data_t, labels_t = input_queue.dequeue()
close_input_op = input_queue.close()
(out_lengths_t, data_and_labels_t) = (bucket_ops.bucket_by_sequence_length(
input_length=lengths_t,
tensors=[data_t, labels_t],
batch_size=batch_size,
bucket_boundaries=bucket_boundaries,
allow_smaller_final_batch=allow_small_batch,
num_threads=10))
expected_batch_size = None if allow_small_batch else batch_size
self.assertEqual(out_lengths_t.get_shape().as_list(), [expected_batch_size])
self.assertEqual(data_and_labels_t[0].get_shape().as_list(),
[expected_batch_size, data_len])
self.assertEqual(data_and_labels_t[1].get_shape().as_list(),
[expected_batch_size, labels_len])
def _read_test(sess):
for _ in range(50):
(out_lengths, (data, labels)) = sess.run(
(out_lengths_t, data_and_labels_t))
if allow_small_batch:
self.assertEqual(data_len, data.shape[1])
self.assertEqual(labels_len, labels.shape[1])
self.assertGreaterEqual(batch_size, out_lengths.shape[0])
self.assertGreaterEqual(batch_size, data.shape[0])
self.assertGreaterEqual(batch_size, labels.shape[0])
else:
self.assertEqual((batch_size, data_len), data.shape)
self.assertEqual((batch_size, labels_len), labels.shape)
self.assertEqual((batch_size,), out_lengths.shape)
for (lr, dr, tr) in zip(out_lengths, data, labels):
# Make sure length matches data (here it's the same value).
self.assertEqual(dr[0], lr)
# Make sure data & labels match.
self.assertEqual(dr[0], int(tr[0].decode("ascii")))
# Make sure for each row, data came from the same bucket.
self.assertEqual(
_which_bucket(bucket_boundaries, dr[0]),
_which_bucket(bucket_boundaries, dr[1]))
with self.test_session() as sess:
coord = coordinator.Coordinator()
# Feed the inputs, then close the input thread.
for _ in range(50 * batch_size + 100):
which = random.randint(0, len(input_pairs) - 1)
length, pair = input_pairs[which]
sess.run(input_enqueue_op,
feed_dict={lengths: length,
data: pair[0],
labels: pair[1]})
sess.run(close_input_op)
# Start the queue runners
threads = queue_runner_impl.start_queue_runners(coord=coord)
# Read off the top of the bucket and ensure correctness of output
_read_test(sess)
coord.request_stop()
coord.join(threads)
def testDifferentAssignGraph(self):
with ops.Graph().as_default():
v = resource_variable_ops.ResourceVariable(1.0)
ops.reset_default_graph()
v.assign(
2.0) # Note: this fails if we run convert_to_tensor on not the
def tearDown(self):
ops.reset_default_graph()
def setUp(self): ops.reset_default_graph()
def model(X_train, Y_train, X_test, Y_test, learning_rate = 0.0001,
num_epochs = 1500, minibatch_size = 32, print_cost = True):
"""
Implements a three-layer tensorflow neural network: LINEAR->RELU->LINEAR->RELU->LINEAR->SOFTMAX.
Arguments:
X_train -- training set, of shape (input size = 12288, number of training examples = 1080)
Y_train -- test set, of shape (output size = 6, number of training examples = 1080)
X_test -- training set, of shape (input size = 12288, number of training examples = 120)
Y_test -- test set, of shape (output size = 6, number of test examples = 120)
learning_rate -- learning rate of the optimization
num_epochs -- number of epochs of the optimization loop
minibatch_size -- size of a minibatch
print_cost -- True to print the cost every 100 epochs
Returns:
parameters -- parameters learnt by the model. They can then be used to predict.
"""
ops.reset_default_graph() # to be able to rerun the model without overwriting tf variables
tf.set_random_seed(1) # to keep consistent results
seed = 3 # to keep consistent results
(n_x, m) = X_train.shape # (n_x: input size, m : number of examples in the train set)
n_y = Y_train.shape[0] # n_y : output size
costs = [] # To keep track of the cost
# Create Placeholders of shape (n_x, n_y)
### START CODE HERE ### (1 line)
X, Y = create_placeholders(n_x, n_y)
### END CODE HERE ###
# Initialize parameters
### START CODE HERE ### (1 line)
parameters = initialize_parameters()
### END CODE HERE ###
# Forward propagation: Build the forward propagation in the tensorflow graph
### START CODE HERE ### (1 line)
Z3 = forward_propagation(X, parameters)
### END CODE HERE ###
# Cost function: Add cost function to tensorflow graph
### START CODE HERE ### (1 line)
cost = compute_cost(Z3, Y)
### END CODE HERE ###
# Backpropagation: Define the tensorflow optimizer. Use an AdamOptimizer.
### START CODE HERE ### (1 line)
optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(cost)
### END CODE HERE ###
# Initialize all the variables
init = tf.global_variables_initializer()
# Start the session to compute the tensorflow graph
with tf.Session() as sess:
# Run the initialization
sess.run(init)
# Do the training loop
for epoch in range(num_epochs):
epoch_cost = 0. # Defines a cost related to an epoch
num_minibatches = int(m / minibatch_size) # number of minibatches of size minibatch_size in the train set
seed = seed + 1
minibatches = random_mini_batches(X_train, Y_train, minibatch_size, seed)
for minibatch in minibatches:
# Select a minibatch
(minibatch_X, minibatch_Y) = minibatch
# IMPORTANT: The line that runs the graph on a minibatch.
# Run the session to execute the "optimizer" and the "cost", the feedict should contain a minibatch for (X,Y).
### START CODE HERE ### (1 line)
_ , minibatch_cost = sess.run([optimizer, cost], feed_dict={X: minibatch_X, Y: minibatch_Y})
### END CODE HERE ###
epoch_cost += minibatch_cost / num_minibatches
# Print the cost every epoch
if print_cost == True and epoch % 100 == 0:
print ("Cost after epoch %i: %f" % (epoch, epoch_cost))
if print_cost == True and epoch % 5 == 0:
costs.append(epoch_cost)
# plot the cost
plt.plot(np.squeeze(costs))
plt.ylabel('cost')
plt.xlabel('iterations (per tens)')
plt.title("Learning rate =" + str(learning_rate))
plt.show()
# lets save the parameters in a variable
parameters = sess.run(parameters)
print ("Parameters have been trained!")
# Calculate the correct predictions
correct_prediction = tf.equal(tf.argmax(Z3), tf.argmax(Y))
# Calculate accuracy on the test set
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
print ("Train Accuracy:", accuracy.eval({X: X_train, Y: Y_train}))
print ("Test Accuracy:", accuracy.eval({X: X_test, Y: Y_test}))
return parameters
lbl_ind = lbl
###############print('\tInputs: ',dat )
###############print('\n\tOutput:',lbl)
# Define hyperparameters
D = 1 # Dimensionality of the data. Since our data is 1-D this would be 1
num_unrollings = 50 # Number of time steps you look into the future.
batch_size = 500 # Number of samples in a batch
num_nodes = [
200, 200, 150
] # Number of hidden nodes in each layer of the deep LSTM stack we're using
n_layers = len(num_nodes) # number of layers, which are 3
dropout = 0.2 # dropout amount
ops.reset_default_graph(
) # This is important in case you run this multiple times
# Define input and output palceholders
# Input data.
train_inputs, train_outputs = [], []
# You unroll the input over time defining placeholders for each time step
for ui in range(num_unrollings):
train_inputs.append(
tf.placeholder(tf.float32,
shape=[batch_size, D],
name='train_inputs_%d' % ui))
train_outputs.append(
tf.placeholder(tf.float32,
shape=[batch_size, 1],
for k in range(1, len(vec)):
coords = coords + "\t" + str(vec[k])
file_vectors.write(str(coords) + "\n")
#metadata :
file_metadata.write(
str(labels_categories[i]) + "\t" + str(labels_clusters[i]) + "\t" +
clusters_purity[labels_clusters[i]][0] + "\t" +
str(clusters_purity[labels_clusters[i]][1]) + "\t" +
str(nb_articles_in_cluster[labels_clusters[i]]) + "\t" +
documents_display[i] + "\n")
#........................ : preparations des meta-fichiers pour data viz avec tensorflow projector
#On se place dans le dossier de notre tensorboard
TENSORBOARD_FILES_PATH = "projections"
#Tensorflow Placeholders
ops.reset_default_graph() # ajout..util dans le notebook
tf.compat.v1.disable_eager_execution()
X_init = tf.compat.v1.placeholder(tf.float32,
shape=(len(documents),
len(features_using_tf_idf)),
name="embedding_tf")
X = tf.Variable(X_init) # tf.Variable(X_init)
#Initialize
init = tf.compat.v1.global_variables_initializer()
#Start Tensorflow Session
#x=tf.get_default_graph().get_tensor_by_name("embedding_tf") #Ajout
sess = tf.compat.v1.Session()
sess.run(init, feed_dict={X_init: vectors})
#Instance of Saver, save the graph.
saver = tf.compat.v1.train.Saver()
writer = tf.compat.v1.summary.FileWriter(TENSORBOARD_FILES_PATH, sess.graph)
def setUp(self):
np.random.seed(1)
ops.reset_default_graph()
def model(X_train,
Y_train,
X_test,
Y_test,
learning_rate,
num_epochs,
mini_batch_size,
print_cost=True):
#print(X_train.shape)
#print(Y_train.shape)
#print(X_test.shape)
#print(Y_test.shape)
ops.reset_default_graph()
(n_x, m) = X_train.shape
seed = 3
#print(nx)
#print(m)
n_y = Y_train.shape[0]
#print(n_y)
costs = []
#print(n_x)
#print(n_y)
X, Y = create_placeholders(n_x, n_y)
parameters = initialize_parameters()
Z3 = forward_propagation(X, parameters)
cost = compute_cost(Z3, Y)
print(str(cost))
optimizer = tf.train.AdamOptimizer(
learning_rate=learning_rate).minimize(cost)
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
for epoch in range(num_epochs):
epoch_cost = 0
num_mini_batches = (int)(m / mini_batch_size)
#print(str(num_mini_batches))
seed = seed + 1
minibatches = random_mini_batches(X_train, Y_train,
mini_batch_size, seed)
for minibatch in minibatches:
(minibatch_X, minibatch_Y) = minibatch
_, minibatch_cost = sess.run([optimizer, cost],
feed_dict={
X: minibatch_X,
Y: minibatch_Y
})
epoch_cost += minibatch_cost / num_mini_batches
if print_cost == True and epoch % 100 == 0:
print("Cost after epoch %i: %f" % (epoch, epoch_cost))
if print_cost == True and epoch % 5 == 0:
costs.append(epoch_cost)
plt.plot(np.squeeze(costs))
plt.ylabel('cost')
plt.xlabel('iterations (per tens)')
plt.title("Learning rate =" + str(learning_rate))
plt.show()
parameters = sess.run(parameters)
# Calculate the correct predictions
correct_prediction = tf.equal(tf.argmax(Z3), tf.argmax(Y))
# Calculate accuracy on the test set
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
print("Train Accuracy:", accuracy.eval({X: X_train, Y: Y_train}))
print("Test Accuracy:", accuracy.eval({X: X_test, Y: Y_test}))
def reset_graph():
yield
ops.reset_default_graph()
def __init__(self, lr, batch_size, util_train, util_test, campaign,
reg_lambda):
# hyperparameters
self.lr = lr
self.batch_size = batch_size
self.util_train = util_train
self.util_test = util_test
self.reg_lambda = reg_lambda
self.emb_size = 20
self.train_data_amt = util_train.get_data_amt()
self.test_data_amt = util_test.get_data_amt()
# output dir
model_name = "{}_{}_{}".format(self.lr, self.reg_lambda,
self.batch_size)
self.output_dir = "output/dwpp/{}/{}/".format(campaign, model_name)
if not os.path.exists(self.output_dir):
os.makedirs(self.output_dir)
# reset graph
ops.reset_default_graph()
# field params
self.field_sizes = self.util_train.feat_sizes
self.field_num = len(self.field_sizes)
# placeholders
tfc.disable_eager_execution()
self.X = [
tfc.sparse_placeholder(tf.float32)
for i in range(0, self.field_num)
]
self.z = tfc.placeholder(tf.float32, [None, 1])
self.b = tfc.placeholder(tf.float32, [None, 1])
self.y = tfc.placeholder(tf.float32, [None, 1])
self.all_prices = tfc.placeholder(tf.float32, [None, 300])
# embedding layer
self.var_map = {
'embed_0':
tf.Variable(
tfc.truncated_normal([self.field_sizes[0], 1],
dtype=tf.float32))
}
# for truncated
for i in range(1, self.field_num):
self.var_map['embed_%d' % i] = tf.Variable(
tfc.truncated_normal([self.field_sizes[i], self.emb_size],
dtype=tf.float32))
# after embedding
w0 = [self.var_map['embed_%d' % i] for i in range(self.field_num)]
self.dense_input = tf.concat([
tfc.sparse_tensor_dense_matmul(self.X[i], w0[i])
for i in range(self.field_num)
], 1)
self.layer1 = tfc.layers.dense(self.dense_input,
50,
activation=tf.nn.relu)
self.layer2 = tfc.layers.dense(self.layer1, 30, activation=tf.nn.relu)
self.u = tfc.layers.dense(self.layer2, 1, activation=tf.nn.relu)
self.sigma = 1 # tf.get_variable('sigma', [], dtype=tf.float32)
self.pz = tf.exp(-(self.z - self.u) * (self.z - self.u) /
(2 * self.sigma * self.sigma)) / self.sigma
self.p_all = tf.exp(-(self.all_prices - self.u) *
(self.all_prices - self.u) /
(2 * self.sigma * self.sigma)) / self.sigma
self.w_all = tf.cumsum(self.p_all, axis=1)
idx_b = tf.stack([
tf.reshape(tf.range(tf.shape(self.b)[0]), (-1, 1)),
tf.cast(self.b - 1, tf.int32)
],
axis=-1)
self.wb = tf.gather_nd(self.w_all, idx_b)
self.loss = tf.losses.mean_squared_error(
self.z * self.y, self.u * self.y) + tf.losses.mean_squared_error(
self.wb * (1 - self.y),
tf.zeros_like(self.wb) * (1 - self.y))
self.optimizer = tfc.train.GradientDescentOptimizer(self.lr)
self.train_step = self.optimizer.minimize(self.loss)
# session initialization
config = tfc.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tfc.Session(config=config)
tfc.global_variables_initializer().run(session=self.sess)
# pre-activation: http://arxiv.org/abs/1603.05027
# wrapping convolutions and batch_norm
def conv_pre(l_in, num_outputs, kernel_size, scope, stride=1):
l_norm = batch_norm(l_in)
l_relu = relu(l_norm)
return convolution2d(l_relu, num_outputs=num_outputs, kernel_size=kernel_size,
stride=stride, activation_fn=None, scope=scope)
# easy to use pool function
def pool(l_in, scope, kernel_size=(3, 3)):
return max_pool2d(l_in, kernel_size=kernel_size, scope=scope) # (3, 3) has shown to work better than (2, 2)
# hyperameters of the model
height, width, channels = IMAGE_SHAPE
# resetting the graph ...
reset_default_graph()
# Setting up placeholder, this is where your data enters the graph!
x_image_pl = tf.placeholder(tf.float32, [None, height, width, channels], name="x_image_pl")
x_margin_pl = tf.placeholder(tf.float32, [None, NUM_FEATURES], name="x_margin_pl")
x_shape_pl = tf.placeholder(tf.float32, [None, NUM_FEATURES], name="x_shape_pl")
x_texture_pl = tf.placeholder(tf.float32, [None, NUM_FEATURES], name="x_texture_pl")
is_training_pl = tf.placeholder(tf.bool, name="is_training_pl")
# Building the layers of the neural network
# we define the variable scope, so we more easily can recognise our variables later
## IMAGE
#l_conv1_a = conv(x_image_pl, 16, (5, 5), scope="l_conv1_a")
#l_pool1 = pool(l_conv1_a, scope="l_pool1")
#l_conv2_a = conv(l_pool1, 16, (5, 5), scope="l_conv2_a")
def tearDown(self):
ops.reset_default_graph()
super(ProfileAnalyzerPrintSourceTest, self).tearDown()
def model(X_train,
Y_train,
X_test,
Y_test,
learning_rate=0.0001,
num_epochs=1500,
minibatch_size=32,
print_cost=True,
is_plot=True):
"""
实现3层神经网络
num_epochs -整个训练集的遍历次数
"""
ops.reset_default_graph()
tf.set_random_seed(1)
seed = 3
(n_x, m) = X_train.shape
n_y = Y_train.shape[0]
costs = []
X, Y = create_placeholder(n_x, n_y)
parameters = initialize_parameters()
Z3 = forward_propagation(X, parameters)
cost = compute_cost(Z3, Y)
#backward propagation,optimize with adam
optimizer = tf.train.AdamOptimizer(
learning_rate=learning_rate).minimize(cost)
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
for epoch in range(num_epochs):
epoch_cost = 0
num_minibatches = int(m / minibatch_size)
seed = seed + 1
minibatches = tf_utils.random_mini_batches(X_train, Y_train,
minibatch_size, seed)
for minibatch in minibatches:
(minibatch_X, minibatch_Y) = minibatch
_, minibatch_cost = sess.run([optimizer, cost],
feed_dict={
X: minibatch_X,
Y: minibatch_Y
})
epoch_cost = epoch_cost + minibatch_cost / num_minibatches
if epoch % 5 == 0:
costs.append(epoch_cost)
if print_cost == True and epoch % 100 == 0:
print("epoch=" + str(epoch) + " epoch_cost=" +
str(epoch_cost))
parameters = sess.run(parameters) #save learned parameters
correct_prediction = tf.equal(tf.argmax(Z3), tf.argmax(Y))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
print("Train set accuracy:", accuracy.eval({X: X_train, Y: Y_train}))
print("Test set accuracy:", accuracy.eval({X: X_test, Y: Y_test}))
return parameters
def model(X_train,
Y_train,
X_dev,
Y_dev,
X_test,
Y_test,
learning_rate=0.009,
num_epochs=200,
minibatch_size=10,
print_cost=True):
"""
Implements a three-layer ConvNet in Tensorflow:
CONV2D -> RELU -> MAXPOOL -> CONV2D -> RELU -> MAXPOOL -> FLATTEN -> FULLYCONNECTED
Arguments:
X_train -- training set, of shape (None, 64, 64, 3)
Y_train -- test set, of shape (None, n_y = 6)
X_test -- training set, of shape (None, 64, 64, 3)
Y_test -- test set, of shape (None, n_y = 6)
learning_rate -- learning rate of the optimization
num_epochs -- number of epochs of the optimization loop
minibatch_size -- size of a minibatch
print_cost -- True to print the cost every 100 epochs
Returns:
train_accuracy -- real number, accuracy on the train set (X_train)
test_accuracy -- real number, testing accuracy on the test set (X_test)
parameters -- parameters learnt by the model. They can then be used to predict.
"""
ops.reset_default_graph(
) # to be able to rerun the model without overwriting tf variables
tf.set_random_seed(1) # to keep results consistent (tensorflow seed)
seed = 3 # to keep results consistent (numpy seed)
(m, n_H0, n_W0, n_C0) = X_train.shape
n_y = Y_train.shape[1]
costs = [] # To keep track of the cost
# Create Placeholders of the correct shape
X, Y = create_placeholders(n_H0, n_W0, n_C0, n_y)
# Initialize parameters
parameters = initialize_parameters()
# Forward propagation: Build the forward propagation in the tensorflow graph
Z3 = forward_propagation(X, parameters)['Z3']
# Get network layers (for CAM later)
inputL = forward_propagation(X, parameters)['input']
Z1 = forward_propagation(X, parameters)['Z1']
Z2 = forward_propagation(X, parameters)['Z2']
W_fc = [
v for v in tf.trainable_variables()
if v.name == "fully_connected/weights:0"
][0]
# Cost function: Add cost function to tensorflow graph
cost = compute_cost(Z3, Y)
# Backpropagation: Define the tensorflow optimizer. Use an AdamOptimizer that minimizes the cost.
optimizer = tf.train.AdamOptimizer(
learning_rate=learning_rate).minimize(cost)
# Initialize all the variables globally
init = tf.global_variables_initializer()
# Start the session to compute the tensorflow graph
with tf.Session() as sess:
# Run the initialization
sess.run(init)
# Do the training loop
for epoch in range(num_epochs):
minibatch_cost = 0.
num_minibatches = int(
m / minibatch_size
) # number of minibatches of size minibatch_size in the train set
seed = seed + 1
minibatches = rmb.random_mini_batches(X_train, Y_train,
minibatch_size, seed)
# Peform mini-batch gradient descent
for minibatch in minibatches:
# Select a minibatch
(minibatch_X, minibatch_Y) = minibatch
_, temp_cost = sess.run([optimizer, cost],
feed_dict={
X: minibatch_X,
Y: minibatch_Y
})
minibatch_cost += temp_cost / num_minibatches
# Print the cost every epoch
if print_cost == True and epoch % 5 == 0:
print("Cost after epoch %i: %f" % (epoch, minibatch_cost))
if print_cost == True and epoch % 1 == 0:
costs.append(minibatch_cost)
# # Plot the cost
# plt.plot(np.squeeze(costs))
# plt.ylabel('cost')
# plt.xlabel('iterations (per tens)')
# plt.title("Learning rate =" + str(learning_rate))
# plt.show()
# Calculate RMSE on the dev set
# Maybe ROUND to nearest 10 as final output
rmse = tf.sqrt(tf.reduce_mean(tf.square(Z3 - Y)))
train_rmse = rmse.eval({X: X_train, Y: Y_train})
dev_rmse = rmse.eval({X: X_dev, Y: Y_dev})
test_rmse = rmse.eval({X: X_test, Y: Y_test})
print("Train RMSE:", train_rmse)
print("Dev RMSE:", dev_rmse)
print("Test RMSE:", test_rmse)
# Make CAM map
inputimg = sess.run(inputL, feed_dict={X: testimg})
outval = sess.run(Z3, feed_dict={X: testimg})
camval = sess.run(Z1, feed_dict={X: testimg})
cweights = W_fc
# Plot original Image
plt.matshow(inputimg[0, :, :, 0], cmap=plt.get_cmap('gray'))
plt.title("Input image")
plt.colorbar()
plt.show()
# Plot class activation maps
predlabel = np.argmax(outval)
predweights = cweights[:, 0]
camsum = np.zeros((camval.shape[1], camval.shape[2]))
for j in range(camval.shape[3]):
camsum = camsum + predweights[j] * camval[0, :, :, j]
camavg = camsum / camval.shape[3]
# Plot
fig, ax = plt.subplots()
im = ax.matshow(camavg.eval(session=sess),
cmap=plt.get_cmap('inferno'))
ax.set_title("[%d] prob is %.3f" % (0, outval[0, 0]), size=10)
ax.xaxis.set_major_locator(plt.NullLocator())
ax.yaxis.set_major_locator(plt.NullLocator())
plt.draw()
cbar_ax = fig.add_axes([0.85, 0.15, 0.05, 0.7])
fig.colorbar(im, cax=cbar_ax)
plt.show()
return train_rmse, dev_rmse, test_rmse, parameters, costs
def setUp(self): self._ClearCachedSession() random.seed(random_seed.DEFAULT_GRAPH_SEED) np.random.seed(random_seed.DEFAULT_GRAPH_SEED) ops.reset_default_graph() ops.get_default_graph().seed = random_seed.DEFAULT_GRAPH_SEED
def tearDown(self):
# Tear down temporary dump directory.
if os.path.isdir(self._dump_root):
shutil.rmtree(self._dump_root)
ops.reset_default_graph()
def forecast():
ops.reset_default_graph()
# tf.reset_default_graph()
modelnn = Model(learning_rate, num_layers, df_log.shape[1], size_layer,
df_log.shape[1], dropout_rate)
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
date_ori = pd.to_datetime(df.iloc[:, 0]).tolist()
pbar = tqdm(range(epoch), desc='train loop')
last_cost = 0.0
for i in pbar:
init_value = np.zeros((1, num_layers * 2 * size_layer))
total_loss, total_acc = [], []
for k in range(0, df_train.shape[0] - 1, timestamp):
index = min(k + timestamp, df_train.shape[0] - 1)
batch_x = np.expand_dims(df_train.iloc[k:index, :].values, axis=0)
batch_y = df_train.iloc[k + 1:index + 1, :].values
logits, last_state, _, loss = sess.run(
[
modelnn.logits, modelnn.last_state, modelnn.optimizer,
modelnn.cost
],
feed_dict={
modelnn.X: batch_x,
modelnn.Y: batch_y,
modelnn.hidden_layer: init_value,
},
)
init_value = last_state
total_loss.append(loss)
total_acc.append(calculate_accuracy(batch_y[:, 0], logits[:, 0]))
current_cost = np.mean(total_loss)
pbar.set_postfix(cost=current_cost, acc=np.mean(total_acc))
if current_cost == last_cost:
break
last_cost = current_cost
future_day = test_size
output_predict = np.zeros(
(df_train.shape[0] + future_day, df_train.shape[1]))
output_predict[0] = df_train.iloc[0]
upper_b = (df_train.shape[0] // timestamp) * timestamp
init_value = np.zeros((1, num_layers * 2 * size_layer))
for k in range(0, (df_train.shape[0] // timestamp) * timestamp, timestamp):
out_logits, last_state = sess.run(
[modelnn.logits, modelnn.last_state],
feed_dict={
modelnn.X: np.expand_dims(df_train.iloc[k:k + timestamp],
axis=0),
modelnn.hidden_layer: init_value,
},
)
init_value = last_state
output_predict[k + 1:k + timestamp + 1] = out_logits
if upper_b != df_train.shape[0]:
out_logits, last_state = sess.run(
[modelnn.logits, modelnn.last_state],
feed_dict={
modelnn.X: np.expand_dims(df_train.iloc[upper_b:], axis=0),
modelnn.hidden_layer: init_value,
},
)
output_predict[upper_b + 1:df_train.shape[0] + 1] = out_logits
future_day -= 1
date_ori.append(date_ori[-1] + timedelta(days=1))
init_value = last_state
for i in range(future_day):
o = output_predict[-future_day - timestamp + i:-future_day + i]
out_logits, last_state = sess.run(
[modelnn.logits, modelnn.last_state],
feed_dict={
modelnn.X: np.expand_dims(o, axis=0),
modelnn.hidden_layer: init_value,
},
)
init_value = last_state
output_predict[-future_day + i] = out_logits[-1]
date_ori.append(date_ori[-1] + timedelta(days=1))
output_predict = minmax.inverse_transform(output_predict)
deep_future = anchor(output_predict[:, 0], 0.4)
return deep_future
def model(X_train,
Y_train,
X_test,
Y_test,
op,
file=None,
learning_rate=0.002,
num_epochs=101,
minibatch_size=64,
print_cost=True):
ops.reset_default_graph(
) # to be able to rerun the model without overwriting tf variables
tf.set_random_seed(1) # to keep consistent results
seed = 3 # to keep consistent results
(
n_x, m
) = X_train.shape # (n_x: input size, m : number of examples in the train set)
n_y = Y_train.shape[0] # n_y : output size
costs = [] # To keep track of the cost
# Create Placeholders of shape (n_x, n_y)
X, Y = create_placeholders(n_x, n_y)
# Initialize parameters
parameters = initialize_parameters()
keep_prob = tf.placeholder(tf.float32)
training = tf.placeholder(tf.bool)
# Forward propagation: Build the forward propagation in the tensorflow graph
Zf = forward_propagation(X,
parameters,
keep_prob=keep_prob,
training=training)
# Cost function: Add cost function to tensorflow graph
cost = compute_cost(Zf, Y)
# Backpropagation: Define the tensorflow optimizer. Use an AdamOptimizer.
optimizer = tf.train.GradientDescentOptimizer(
learning_rate=learning_rate).minimize(cost)
# Initialize all the variables
init = tf.global_variables_initializer()
#Initialize saver
saver = tf.train.Saver()
# Start the session to compute the tensorflow graph
with tf.Session() as sess:
if op == 'train':
# Run the initialization
sess.run(init)
# Do the training loop
for epoch in range(num_epochs):
epoch_cost = 0. # Defines a cost related to an epoch
validation_cost = 0
num_minibatches = int(
m / minibatch_size
) # number of minibatches of size minibatch_size in the train set
seed = seed + 1
minibatches = random_mini_batches(X_train, Y_train,
minibatch_size, seed)
for minibatch in minibatches:
# Select a minibatch
(minibatch_X, minibatch_Y) = minibatch
# Run the session to execute the "optimizer" and the "cost", the feedict should contain a minibatch for (X,Y).
_, minibatch_cost = sess.run(
[optimizer, cost],
feed_dict={
X: minibatch_X,
Y: minibatch_Y,
keep_prob: 0.8,
training: True
})
epoch_cost += minibatch_cost / num_minibatches
minibatch_validation_cost = sess.run(cost,
feed_dict={
X: X_test,
Y: Y_test,
keep_prob: 0.8,
training: True
})
validation_cost += minibatch_validation_cost
# Print the cost every epoch
train_summary(epoch + 1, epoch_cost, validation_cost, Zf, X, Y,
X_train, Y_train, X_test, Y_test, keep_prob,
training)
if print_cost == True and epoch % 5 == 0:
costs.append(epoch_cost)
saver.save(sess, './model/model')
# plot the cost
plt.plot(np.squeeze(costs))
plt.ylabel('cost')
plt.xlabel('iterations (per tens)')
plt.title("Learning rate =" + str(learning_rate))
# plt.show()
# lets save the parameters in a variable
parameters = sess.run(parameters)
# print("Parameters have been trained!")
# Calculate the correct predictions
correct_prediction = tf.equal(tf.argmax(Zf), tf.argmax(Y))
# Calculate accuracy on the test set
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
print(
"Train Accuracy:",
accuracy.eval({
X: X_train,
Y: Y_train,
keep_prob: 1,
training: False
}))
print(
"Test Accuracy:",
accuracy.eval({
X: X_test,
Y: Y_test,
keep_prob: 1,
training: False
}))
return parameters
else:
label_dic = {
0: 'airplane',
1: 'automobile',
2: 'bird',
3: 'cat',
4: 'deer',
5: 'dog',
6: 'frog',
7: 'horse',
8: 'ship',
9: 'truck'
}
#Load model
img = misc.imread(file)
if not img.shape == (32, 32, 3):
print("shape_issue")
else:
img = img.reshape((3072, 1))
saver.restore(sess, "./model/model")
print(label_dic[sess.run(tf.argmax(Zf),
feed_dict={
X: img,
keep_prob: 1,
training: False
})[0]])
def train_mynetwork(x_train, x_test, train_x, test_x, y_train, y_test, L_train, L_test, learning_rate_base = 0.001, beta_reg = 0.001, num_epochs = 200, minibatch_size = 32, print_cost = True):
ops.reset_default_graph()
tf.set_random_seed(1)
seed = 1
(m, n_x) = x_train.shape
(m, n_y) = y_train.shape
(m, n_x1) = train_x.shape
costs = []
costs_dev = []
train_acc = []
val_acc = []
x_in, x_in1, y_in, lap_train, isTraining = create_placeholders(n_x, n_x1, n_y)
parameters = initialize_parameters()
with tf.name_scope("network"):
x_out, l2_loss = mynetwork(x_in, x_in1, parameters, lap_train, isTraining)
global_step = tf.Variable(0, trainable=False)
learning_rate = tf.train.exponential_decay(learning_rate_base, global_step, 50 * m/minibatch_size, 0.5, staircase = True)
with tf.name_scope("optimization"):
cost, optimizer = mynetwork_optimaization(x_out, y_in, l2_loss, beta_reg, learning_rate, global_step)
with tf.name_scope("metrics"):
joint_layerT = tf.transpose(x_out)
yT = tf.transpose(y_in)
correct_prediction = tf.equal(tf.argmax(joint_layerT), tf.argmax(yT))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
# Do the training loop
for epoch in range(num_epochs + 1):
epoch_cost = 0. # Defines a cost related to an epoch
epoch_acc = 0.
num_minibatches = int(m / minibatch_size) # number of minibatches of size minibatch_size in the train set
seed = seed + 1
minibatches = random_mini_batches_GCN1(x_train, train_x, y_train, L_train, minibatch_size, seed)
for minibatch in minibatches:
# Select a minibatch
(batch_x, batch_x1, batch_y, batch_l) = minibatch
# IMPORTANT: The line that runs the graph on a minibatch.
# Run the session to execute the "optimizer" and the "cost", the feedict should contain a minibatch for (X,Y).
_, minibatch_cost, minibatch_acc = sess.run([optimizer, cost, accuracy], feed_dict={x_in: batch_x, x_in1: batch_x1, y_in: batch_y, lap_train: batch_l, isTraining: True})
epoch_cost += minibatch_cost / (num_minibatches+ 1)
epoch_acc += minibatch_acc / (num_minibatches + 1)
if print_cost == True and (epoch) % 50 == 0:
features, epoch_cost_dev, epoch_acc_dev = sess.run([x_out, cost, accuracy], feed_dict={x_in: x_test, x_in1: test_x, y_in: y_test, lap_train: L_test, isTraining: False})
print ("epoch %i: Train_loss: %f, Val_loss: %f, Train_acc: %f, Val_acc: %f" % (epoch, epoch_cost, epoch_cost_dev, epoch_acc, epoch_acc_dev))
if print_cost == True and epoch % 5 == 0:
costs.append(epoch_cost)
train_acc.append(epoch_acc)
costs_dev.append(epoch_cost_dev)
val_acc.append(epoch_acc_dev)
# plot the cost
plt.plot(np.squeeze(costs))
plt.plot(np.squeeze(costs_dev))
plt.ylabel('cost')
plt.xlabel('iterations (per tens)')
plt.title("Learning rate =" + str(learning_rate))
plt.show()
# plot the accuracy
plt.plot(np.squeeze(train_acc))
plt.plot(np.squeeze(val_acc))
plt.ylabel('accuracy')
plt.xlabel('iterations (per tens)')
plt.title("Learning rate =" + str(learning_rate))
plt.show()
# lets save the parameters in a variable
parameters = sess.run(parameters)
print ("Parameters have been trained!")
return parameters, val_acc, features
def model(X_train, Y_train, X_test, Y_test, learning_rate = 0.009,
num_epochs = 50, minibatch_size = 128,
num_lstm_units = 100, news_per_hour = 10, set_verbosity=True):
if(set_verbosity):
print ("Learning rate: " +str(learning_rate))
ops.reset_default_graph() # to be able to rerun the model without overwriting tf variables
m =int(len(X_train))
costs_train = [] # To keep track of the cost
costs_test = [] # To keep track of the cost
accuracy_train = [] # To keep track of the cost
accuracy_test = [] # To keep track of the cost
# Create Placeholders of the correct shape
X, Y = MyModel.create_placeholders()
# Forward propagation: Build the forward propagation in the tensorflow graph
prediction = MyModel.forward_propagation(X)
# Cost function: Add cost function to tensorflow graph
cost = MyModel.compute_cost(prediction, Y)
# Backpropagation: Define the tensorflow optimizer. Use an AdamOptimizer that minimizes the cost.
optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(cost)
#grads = tf.train.AdamOptimizer(learning_rate = learning_rate).compute_gradients(cost)
# Initialize all the variables globally
init = tf.global_variables_initializer()
#This is for computing the test accuracy every epoch
predict_op = tf.to_float(prediction > 0.5)
correct_prediction = tf.equal(predict_op, Y)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
# Start the session to compute the tensorflow graph
with tf.Session() as sess:
# Run the initialization
if(set_verbosity):
print('initialization')
sess.run(init)
# Do the training loop
for epoch in range(num_epochs):
minibatch_cost = 0.
num_minibatches = int(m / minibatch_size) # number of minibatches of size minibatch_size in the train set
if(num_minibatches == 0):
num_minibatches = 1
minibatches = MyModel.random_mini_batches(X_train, Y_train, minibatch_size)
for minibatch in minibatches:
# Select a minibatch
(minibatch_X, minibatch_Y) = minibatch
minibatch_X = np.asarray(minibatch_X)
minibatch_Y = np.asarray(minibatch_Y).reshape((len(minibatch_Y), 1))
# Run the session to execute the optimizer and the cost, the feedict should contain a minibatch for (X,Y).
_ , temp_cost = sess.run([optimizer, cost], feed_dict={X: minibatch_X, Y: minibatch_Y})
# weights = tf.get_default_graph().get_tensor_by_name('dense_1/kernel:0')
# print(sess.run(weights))
minibatch_cost += temp_cost / num_minibatches
# Print the cost every epoch
if epoch % 1 == 0:
trainCost = sess.run(cost, feed_dict={X: np.asarray(X_train), Y: np.asarray(Y_train).reshape((len(Y_train), 1))})
costs_train.append(trainCost)
testCost = sess.run(cost, feed_dict={X: np.asarray(X_test), Y: np.asarray(Y_test).reshape((len(Y_test), 1))})
costs_test.append(testCost)
if set_verbosity and epoch % 5 == 0:
print('miniCost ='+str(minibatch_cost))
testAccuracy = str(sess.run(accuracy, feed_dict={X: np.asarray(X_test), Y: np.asarray(Y_test).reshape((len(Y_test), 1))}))
trainAccuracy = str(sess.run(accuracy, feed_dict={X: np.asarray(X_train), Y: np.asarray(Y_train).reshape((len(Y_train), 1))}))
accuracy_train.append(float(trainAccuracy))
accuracy_test.append(float(testAccuracy))
print("Epoch "+str(epoch)+": \tTrain cost: "+str(trainCost)+" \tTest cost: "+str(testCost)+" \tTrain Accuracy: "+str(trainAccuracy)+" \tTest accuracy: "+str(testAccuracy))
# Calculate accuracy on the test set
predict_op = tf.to_float(prediction > 0.5)
correct_prediction = tf.equal(predict_op, Y)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
train_accuracy = sess.run(accuracy, feed_dict={X: np.asarray(X_train), Y: np.asarray(Y_train).reshape((len(Y_train), 1))})
test_accuracy = sess.run(accuracy, feed_dict={X: np.asarray(X_test), Y: np.asarray(Y_test).reshape((len(Y_test), 1))})
if(set_verbosity):
# Calculate the correct predictions
plt.plot(np.arange(0, len(accuracy_train)*5, 5), accuracy_train,'b')
plt.plot(np.arange(0, len(accuracy_train)*5, 5), accuracy_test,'r')
plt.ylabel('accuracy')
plt.xlabel('epochs')
plt.title("Learning rate =" + str(learning_rate))
plt.show()
# plot the cost
plt.plot(range(0,len(costs_train)),costs_train,'b',costs_test,'r')
plt.ylabel('cost')
plt.xlabel('epochs')
plt.title("Learning rate =" + str(learning_rate))
plt.show()
print("Train Accuracy:", train_accuracy)
print("Test Accuracy:", test_accuracy)
return (train_accuracy, test_accuracy)
def setUp(self):
super(CreateGraphDebugInfoDefTest, self).setUp()
ops.reset_default_graph()
def model(X_train,
Y_train,
X_test,
starting_learning_rate=0.0001,
num_epochs=1500,
minibatch_size=128,
print_cost=True):
# input -- X_train (training set), Y_train(training labels), X_test (test set), Y_test (test labels),
# output -- trained parameters
ops.reset_default_graph(
) # to be able to rerun the model without overwriting tf variables
tf.set_random_seed(3)
seed = 3
(nf, ns) = X_train.shape
nt = Y_train.shape[0]
costs = []
# create placeholders of shape (nf, nt)
X, Y = create_placeholders(nf, nt)
# initialize parameters
parameters = initialize_parameters(nf=nf, ln1=100, ln2=50, ln3=25, nt=nt)
# forward propagation: build the forward propagation in the tensorflow graph
Z4 = forward_propagation(X, parameters)
# cost function: add cost function to tensorflow graph
cost = compute_cost(Z4, Y, parameters, 0.005)
# Use learning rate decay
global_step = tf.Variable(0, trainable=False)
learning_rate = tf.train.exponential_decay(starting_learning_rate,
global_step,
1000,
0.95,
staircase=True)
# backpropagation: define the tensorflow optimizer, AdamOptimizer is used.
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
trainer = optimizer.minimize(cost, global_step=global_step)
# initialize all the variables
init = tf.global_variables_initializer()
# start the session to compute the tensorflow graph
with tf.Session() as sess:
# run the initialization
sess.run(init)
# do the training loop
for epoch in range(num_epochs):
epoch_cost = 0.
num_minibatches = int(ns / minibatch_size)
seed = seed + 1
minibatches = random_mini_batches(X_train, Y_train, minibatch_size,
seed)
for minibatch in minibatches:
# select a minibatch
(minibatch_X, minibatch_Y) = minibatch
# run the session to execute the "optimizer" and the "cost", the feedict contains a minibatch for (X,Y).
_, minibatch_cost = sess.run([trainer, cost],
feed_dict={
X: minibatch_X,
Y: minibatch_Y
})
epoch_cost += minibatch_cost / num_minibatches
# print the cost every epoch
if print_cost == True and (epoch + 1) % 5 == 0:
print("Cost after epoch %i: %f" % (epoch + 1, epoch_cost))
costs.append(epoch_cost)
parameters = sess.run(parameters)
print("Parameters have been trained!")
# calculate the correct predictions
correct_prediction = tf.equal(tf.argmax(Z4), tf.argmax(Y))
# calculate accuracy on the test set
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
print("Train Accuracy:", accuracy.eval({X: X_train, Y: Y_train}))
return parameters
def model(X_train, Y_train, X_test, Y_test, learning_rate=0.009,
num_epochs=100, minibatch_size=64, print_cost=True):
"""
Arguments:
X_train -- training set, of shape (None, 64, 64, 3)
Y_train -- test set, of shape (None, n_y = 6)
X_test -- training set, of shape (None, 64, 64, 3)
Y_test -- test set, of shape (None, n_y = 6)
learning_rate -- learning rate of the optimization
num_epochs -- number of epochs of the optimization loop
minibatch_size -- size of a minibatch
print_cost -- True to print the cost every 100 epochs
Returns:
train_accuracy -- real number, accuracy on the train set (X_train)
test_accuracy -- real number, testing accuracy on the test set (X_test)
parameters -- parameters learnt by the model. They can then be used to predict.
"""
ops.reset_default_graph() # to be able to rerun the model without overwriting tf variables
tf.set_random_seed(1) # to keep results consistent (tensorflow seed)
seed = 3 # to keep results consistent (numpy seed)
m, n_H0, n_W0, n_C0 = X_train.shape
n_y = Y_train.shape[1]
costs = []
X, Y = create_placeholders(n_H0, n_W0, n_C0, n_y)
parameters = initialize_parameters()
Z3 = forward_propagation(X, parameters)
cost = compute_cost(Z3, Y)
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
init = tf.global_variables_initializer()
with tf.Session() as session:
session.run(init)
for i in range(num_epochs):
minibatch_cost = 0.
num_minibatches = int(m / minibatch_size)
seed = seed + 1
minibatches = random_mini_batches(X_train, Y_train, minibatch_size, seed)
for minibatch in minibatches:
(minibatch_X, minibatch_Y) = minibatch
_, minibatch_cost = session.run([optimizer, cost], feed_dict={X: minibatch_X, Y: minibatch_Y})
minibatch_cost += minibatch_cost / num_minibatches
if print_cost is True and i % 5 == 0:
print("Cost after epoch %i: %f" % (i, minibatch_cost))
if print_cost is True:
costs.append(minibatch_cost)
# 绘制代价函数
plt.plot(np.squeeze(costs))
plt.ylabel('cost')
plt.xlabel('iterations (per tens)')
plt.title("Learning rate =" + str(learning_rate))
plt.show()
# Calculate the correct predictions
predict_op = tf.argmax(Z3, 1)
correct_prediction = tf.equal(predict_op, tf.argmax(Y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
print(accuracy)
print("Train Accuracy:", accuracy.eval({X: X_train, Y: Y_train}))
print("Test Accuracy:", accuracy.eval({X: X_test, Y: Y_test}))
return parameters
def model(X_train, Y_train, X_dev, Y_dev, learning_rate = 0.0005, num_epochs = 150, print_cost = True):
ops.reset_default_graph()
(n_x, m) = X_train.shape
n_y = Y_train.shape[0]
costs = []
#Create placeholders
X, Y = create_placeholders(n_x, n_y)
#Initialize params
parameters = initialize_parameters(n_x)
#Run forward prop
Z5 = forward_propogation(X, parameters)
#compute cost
cost = compute_cost(Z5, Y)
#Run back prop
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
#Initialize all variables
init = tf.global_variables_initializer()
#Start session
with tf.Session() as sess:
sess.run(init)
for epoch in range(num_epochs):
epoch_cost = 0
#TODO: implement minibatches
_ , curr_cost = sess.run([optimizer, cost], feed_dict = {X: X_train, Y:Y_train})
epoch_cost += curr_cost
# Print the cost every epoch
if print_cost == True and epoch % 10 == 0:
print ("Cost after epoch %i: %f" % (epoch, epoch_cost))
if print_cost == True and epoch % 5 == 0:
costs.append(epoch_cost)
#Plot cost
plt.plot(np.squeeze(costs))
plt.ylabel('cost')
plt.xlabel('iterations (per tens)')
plt.title("Learning rate =" + str(learning_rate))
plt.show()
#Save trained parameters
parameters = sess.run(parameters)
print ("Parameters have been trained!")
# Calculate the correct predictions
correct_prediction = tf.equal(tf.argmax(Z5), tf.argmax(Y))
# Calculate accuracy on the dev set
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
print ("Train Accuracy:", accuracy.eval({X: X_train, Y: Y_train}))
print ("Dev Accuracy:", accuracy.eval({X: X_dev, Y: Y_dev}))
return parameters
