def _annotated_graph(self): graph = ops.Graph() with graph.as_default(): random_seed.set_random_seed(2) current_activation = variable_scope.get_variable( name='start', shape=[1, 2, 2, 5]) conv_filter = variable_scope.get_variable(name='filter', shape=[5, 5, 5, 5]) for layer_number in range(3): with variable_scope.variable_scope( 'layer_{}'.format(layer_number)): after_conv = nn.conv2d(current_activation, conv_filter, [1, 1, 1, 1], 'SAME') current_activation = 2. * after_conv current_activation.op._set_attr( '_recompute_hint', # The value of the attribute does not matter; just that the key # exists in the op's attributes. attr_value_pb2.AttrValue(i=1)) current_activation += 5. current_activation.op._set_attr( '_recompute_hint', attr_value_pb2.AttrValue(i=0)) current_activation = nn.relu(current_activation) current_activation.op._set_attr( '_recompute_hint', attr_value_pb2.AttrValue(i=1)) loss = math_ops.reduce_mean(current_activation) optimizer = train.AdamOptimizer(0.001) train_op = optimizer.minimize(loss) init_op = variables.global_variables_initializer() return graph, init_op, train_op
def __init__(self, model, state_manager=None, optimizer=None, model_dir=None, config=None, head_type=ts_head_lib.TimeSeriesRegressionHead): """Initialize the Estimator. Args: model: The time series model to wrap (inheriting from TimeSeriesModel). state_manager: The state manager to use, or (by default) PassthroughStateManager if none is needed. optimizer: The optimization algorithm to use when training, inheriting from tf.train.Optimizer. Defaults to Adam with step size 0.02. model_dir: See `Estimator`. config: See `Estimator`. head_type: The kind of head to use for the model (inheriting from `TimeSeriesRegressionHead`). """ input_statistics_generator = math_utils.InputStatisticsFromMiniBatch( dtype=model.dtype, num_features=model.num_features) if state_manager is None: if isinstance(model, ar_model.ARModel): state_manager = state_management.FilteringOnlyStateManager() else: state_manager = state_management.PassthroughStateManager() if optimizer is None: optimizer = train.AdamOptimizer(0.02) self._model = model ts_regression_head = head_type( model=model, state_manager=state_manager, optimizer=optimizer, input_statistics_generator=input_statistics_generator) model_fn = ts_regression_head.create_estimator_spec super(TimeSeriesRegressor, self).__init__( model_fn=model_fn, model_dir=model_dir, config=config)
def _GetMetaGraph(self, batch_size=14, image_dim=12, optimizer_scope_name=''): """A simple layered graph with conv, an intermediate op, and a ReLU.""" graph = ops.Graph() with graph.as_default(): random_seed.set_random_seed(1) current_activation = variable_scope.get_variable( name='start', shape=[batch_size, image_dim, image_dim, 5]) conv_filter = variable_scope.get_variable(name='filter', shape=[5, 5, 5, 5]) for layer_number in range(10): with variable_scope.variable_scope( 'layer_{}'.format(layer_number)): after_conv = nn.conv2d(current_activation, conv_filter, [1, 1, 1, 1], 'SAME') current_activation = 2. * after_conv current_activation = nn.relu(current_activation) loss = math_ops.reduce_mean(current_activation) with ops.name_scope(optimizer_scope_name): optimizer = train.AdamOptimizer(0.001) train_op = optimizer.minimize(loss) init_op = variables.global_variables_initializer() metagraph = train.export_meta_graph() return (metagraph, init_op.name, train_op.name, loss.name)
def _RunGraphWithConfig(self, config, batch_size=14, image_dim=12): """Run a simple layered graph with conv, an intermediate op, and a ReLU.""" graph = ops.Graph() with graph.as_default(): random_seed.set_random_seed(1) current_activation = variable_scope.get_variable( name='start', shape=[batch_size, image_dim, image_dim, 5]) conv_filter = variable_scope.get_variable(name='filter', shape=[5, 5, 5, 5]) for layer_number in range(10): with variable_scope.variable_scope( 'layer_{}'.format(layer_number)): after_conv = nn.conv2d(current_activation, conv_filter, [1, 1, 1, 1], 'SAME') current_activation = 2. * after_conv current_activation = nn.relu(current_activation) loss = math_ops.reduce_mean(current_activation) optimizer = train.AdamOptimizer(0.001) train_op = optimizer.minimize(loss) init_op = variables.global_variables_initializer() with session.Session(config=config, graph=graph) as sess: sess.run(init_op) sess.run(train_op) sess.run(train_op) return sess.run(loss)
def _configure_optimizer(learning_rate, opt_type='adam'): if opt_type == 'adadelta': optimizer = training.AdadeltaOptimizer(learning_rate, rho=FLAGS.adadelta_rho, epsilon=FLAGS.opt_epsilon) elif opt_type == 'adagrad': optimizer = training.AdagradOptimizer( learning_rate, initial_accumulator_value=FLAGS.adagrad_initial_accumulator_value) elif opt_type == 'adam': optimizer = training.AdamOptimizer(learning_rate, ) elif opt_type == 'ftrl': optimizer = training.FtrlOptimizer( learning_rate, learning_rate_power=FLAGS.ftrl_learning_rate_power, initial_accumulator_value=FLAGS.ftrl_initial_accumulator_value, l1_regularization_strength=FLAGS.ftrl_l1, l2_regularization_strength=FLAGS.ftrl_l2) elif opt_type == 'momentum': optimizer = training.MomentumOptimizer(learning_rate, momentum=FLAGS.momentum, name='Momentum') elif opt_type == 'rmsprop': optimizer = training.RMSPropOptimizer(learning_rate, decay=FLAGS.rmsprop_decay, momentum=FLAGS.rmsprop_momentum, epsilon=FLAGS.opt_epsilon) elif opt_type == 'sgd': optimizer = training.GradientDescentOptimizer(learning_rate) else: raise ValueError('Optimizer [%s] was not recognized', FLAGS.optimizer) return optimizer
def __init__(self, model, state_manager=None, optimizer=None, model_dir=None, config=None): """Initialize the Estimator. Args: model: The time series model to wrap (inheriting from TimeSeriesModel). state_manager: The state manager to use, or (by default) PassthroughStateManager if none is needed. optimizer: The optimization algorithm to use when training, inheriting from tf.train.Optimizer. Defaults to Adam with step size 0.02. model_dir: See `Estimator`. config: See `Estimator`. """ input_statistics_generator = math_utils.InputStatisticsFromMiniBatch( dtype=model.dtype, num_features=model.num_features) if state_manager is None: state_manager = state_management.PassthroughStateManager() if optimizer is None: optimizer = train.AdamOptimizer(0.02) self._model = model model_fn = model_utils.make_model_fn( model, state_manager, optimizer, input_statistics_generator=input_statistics_generator) super(_TimeSeriesRegressor, self).__init__( model_fn=model_fn, model_dir=model_dir, config=config)
def testCloneBatchNorm(self): g = ops.Graph() with g.as_default(): np.random.seed(1234) x_value = np.random.random([2, 5, 5, 3]) w_value = np.random.random([3, 3, 3, 2]) is_training_t = array_ops.placeholder(dtypes.bool, name='is_training_t') x_t = array_ops.constant(x_value, dtype=dtypes.float32, name='x_t') y_t = conv2d( x_t, 2, [3, 3], kernel_initializer=init_ops.constant_initializer(w_value)) y_t = batch_norm(y_t, training=is_training_t) optimizer_t = train.AdamOptimizer() optimize_t = optimizer_t.minimize(math_ops.reduce_sum(y_t)) with self.test_session(use_gpu=True) as sess: sess.run(variables.global_variables_initializer()) y_test_1 = sess.run(y_t, feed_dict={is_training_t: False}) sess.run(optimize_t, feed_dict={is_training_t: True}) y_test_2 = sess.run(y_t, feed_dict={is_training_t: False}) is_training = array_ops.placeholder(dtypes.bool, name='is_training') x = array_ops.constant(np.zeros([2, 5, 5, 3]), dtype=dtypes.float32, name='x') y = conv2d( x, 2, [3, 3], kernel_initializer=init_ops.constant_initializer(w_value)) y = batch_norm(y, training=is_training) x_new = array_ops.constant(x_value, dtype=dtypes.float32, name='x') y_out = meta_graph.clone(y, "copy", replace={x: x_new}) optimizer = train.AdamOptimizer() optimize = optimizer.minimize(math_ops.reduce_sum(y_out)) with self.test_session(use_gpu=True) as sess: sess.run(variables.global_variables_initializer()) y_out_1 = sess.run(y_out, feed_dict={is_training: False}) y_out_2 = sess.run(y_out, feed_dict={is_training: False}) sess.run(optimize, feed_dict={is_training: True}) y_out_3 = sess.run(y_out, feed_dict={is_training: False}) self.assertAllClose(y_out_1, y_out_2) self.assertTrue(np.abs(y_out_1 - y_out_3).max() > 1e-6) self.assertAllClose(y_test_1, y_out_1) self.assertAllClose(y_test_2, y_out_3)
def model_fn(features, labels, mode, params): """The model_fn argument for creating an Estimator.""" model = Model(params["data_format"]) image = features if isinstance(image, dict): image = features["image"] if mode == estimator.ModeKeys.PREDICT: logits = model(image, training=False) predictions = { "classes": math_ops.argmax(logits, axis=1), "probabilities": nn.softmax(logits), } return estimator.EstimatorSpec( mode=estimator.ModeKeys.PREDICT, predictions=predictions, export_outputs={ "classify": estimator.export.PredictOutput(predictions) }) elif mode == estimator.ModeKeys.TRAIN: optimizer = train.AdamOptimizer(learning_rate=1e-4) logits = model(image, training=True) loss = losses.sparse_softmax_cross_entropy(labels=labels, logits=logits) return estimator.EstimatorSpec(mode=estimator.ModeKeys.TRAIN, loss=loss, train_op=optimizer.minimize( loss, train.get_or_create_global_step())) elif mode == estimator.ModeKeys.EVAL: logits = model(image, training=False) loss = losses.sparse_softmax_cross_entropy(labels=labels, logits=logits) return estimator.EstimatorSpec( mode=estimator.ModeKeys.EVAL, loss=loss, eval_metric_ops={ "accuracy": ops.metrics.accuracy(labels=labels, predictions=math_ops.argmax(logits, axis=1)), })
def _stub_model_fn(): return ts_head_lib.TimeSeriesRegressionHead( model=_StubModel(), state_manager=state_management.PassthroughStateManager(), optimizer=train.AdamOptimizer(0.001)).create_estimator_spec
def _stub_model_fn(): return model_utils.make_model_fn( model=_StubModel(), state_manager=state_management.PassthroughStateManager(), optimizer=train.AdamOptimizer(0.001))