def test_construct_tower_with_transfer_learning( self, transfer_learning_type=transfer_learning_spec_pb2.TransferLearningSpec. NO_TRANSFER_LEARNING): # convolutions and then flatten plate. architecture = np.array([1, 3, 34]) str_signature = "_1334" input_tensor = tf.zeros([100, 32, 32, 3]) tower_name = "test_tower" transfer_learning_spec = transfer_learning_spec_pb2.TransferLearningSpec( transfer_learning_type=transfer_learning_type) phoenix_spec = phoenix_spec_pb2.PhoenixSpec( problem_type=phoenix_spec_pb2.PhoenixSpec.CNN, transfer_learning_spec=transfer_learning_spec) _ = architecture_utils.construct_tower( phoenix_spec=phoenix_spec, input_tensor=input_tensor, tower_name=tower_name, architecture=architecture, is_training=True, lengths=None, logits_dimension=10, hparams=hp.HParams(), model_directory=self.get_temp_dir(), is_frozen=False, dropout_rate=None) tensors = architecture_utils.get_tower_variables(tower_name) for tensor in tensors: if (transfer_learning_type == transfer_learning_spec_pb2. TransferLearningSpec.NO_TRANSFER_LEARNING): self.assertEndsWith(tensor.op.name, str_signature) else: self.assertNotEndsWith(tensor.op.name, str_signature)
def _create_checkpoint(self, towers, trial_id): with self.test_session(graph=tf.Graph()) as sess: architecture = np.array([1, 3, 34]) input_tensor = tf.zeros([100, 32, 32, 3]) phoenix_spec = phoenix_spec_pb2.PhoenixSpec( problem_type=phoenix_spec_pb2.PhoenixSpec.CNN) dirname = os.path.join(flags.FLAGS.test_tmpdir, str(trial_id)) if dirname and not tf.io.gfile.exists(dirname): tf.io.gfile.makedirs(dirname) for tower in towers: _ = architecture_utils.construct_tower( phoenix_spec=phoenix_spec, input_tensor=input_tensor, tower_name=str(tower) + '_0', architecture=architecture, is_training=True, lengths=None, logits_dimension=10, hparams=hp.HParams(), model_directory=dirname, is_frozen=False, dropout_rate=None) architecture_utils.set_number_of_towers(tower, 1) architecture_utils.set_number_of_towers('replay_generator', 0) directory = flags.FLAGS.test_tmpdir saver = tf.compat.v1.train.Saver() sess.run(tf.compat.v1.global_variables_initializer()) sess.run(tf.compat.v1.local_variables_initializer()) saver.save(sess, os.path.join(directory, str(trial_id)) + '/ckpt')
def test_construct_network(self, dropout, expected_logits): # Force graph mode with tf.compat.v1.Graph().as_default(): tf.random.set_seed(1234) # convolutions and then flatten plate. architecture = np.array([1, 3, 34]) input_tensor = tf.compat.v1.placeholder(dtype=tf.float32, shape=[None, 32, 32, 3], name="input") phoenix_spec = phoenix_spec_pb2.PhoenixSpec( problem_type=phoenix_spec_pb2.PhoenixSpec.CNN) tower_spec = architecture_utils.construct_tower( phoenix_spec=phoenix_spec, input_tensor=input_tensor, tower_name="test_tower", architecture=architecture, is_training=True, lengths=None, logits_dimension=10, hparams=hp.HParams(), model_directory=self.get_temp_dir(), is_frozen=False, dropout_rate=dropout) np.random.seed(42) test_input = np.random.random([1, 32, 32, 3]) with tf.compat.v1.Session() as sess: sess.run([ tf.compat.v1.global_variables_initializer(), tf.compat.v1.local_variables_initializer() ]) logits_val = sess.run(tower_spec.logits_spec.logits, feed_dict={input_tensor: test_input}) self.assertAllClose(expected_logits, logits_val, rtol=1e-3)
def test_init_variables(self, new_architecture, expected_output, new_tower_name="test_tower"): # Force graph mode with tf.compat.v1.Graph().as_default(): directory = self.get_temp_dir() architecture = np.array([1, 3, 34]) phoenix_spec = phoenix_spec_pb2.PhoenixSpec( problem_type=phoenix_spec_pb2.PhoenixSpec.CNN) with self.test_session(graph=tf.Graph()) as sess: input_tensor = tf.zeros([100, 32, 32, 3]) _ = architecture_utils.construct_tower( phoenix_spec=phoenix_spec, input_tensor=input_tensor, tower_name="test_tower", architecture=architecture, is_training=True, lengths=None, logits_dimension=10, model_directory=self.get_temp_dir(), hparams=hp.HParams(), is_frozen=False, dropout_rate=None) saver = tf.compat.v1.train.Saver() sess.run(tf.compat.v1.global_variables_initializer()) sess.run(tf.compat.v1.local_variables_initializer()) saver.save(sess, directory + "/ckpt") with self.test_session(graph=tf.Graph()) as sess: input_tensor = tf.zeros([100, 32, 32, 3]) _ = architecture_utils.construct_tower( phoenix_spec=phoenix_spec, input_tensor=input_tensor, tower_name=new_tower_name, architecture=new_architecture, is_training=True, lengths=None, logits_dimension=10, hparams=hp.HParams(), model_directory=self.get_temp_dir(), is_frozen=False, dropout_rate=None) snapshotting_variables = architecture_utils.init_variables( tf.train.latest_checkpoint(directory), "Phoenix/test_tower", "Phoenix/{}".format(new_tower_name)) self.assertCountEqual(snapshotting_variables, expected_output)
def _create_new_architecture(self, features, input_layer_fn, shared_input_tensor, architecture, run_config, my_id, is_training, shared_lengths, hparams, logits_dimension, dropout_rate, prev_trial, trials): logging.info("Creating new architecture: ") logging.info(architecture) input_tensor = shared_input_tensor lengths = shared_lengths if not self._phoenix_spec.is_input_shared: lengths_feature_name = self._phoenix_spec.lengths_feature_name if isinstance(features, dict) and lengths_feature_name not in features: lengths_feature_name = "" input_tensor, lengths = input_layer_fn( features=features, is_training=is_training, scope_name="Phoenix/" + self.generator_name() + "_0/Input", lengths_feature_name=lengths_feature_name) self._save_architecture(architecture, run_config.model_dir, my_id) tower_spec = architecture_utils.construct_tower( phoenix_spec=self._phoenix_spec, input_tensor=input_tensor, tower_name=self.generator_name() + "_0", architecture=architecture, is_training=is_training, lengths=lengths, logits_dimension=logits_dimension, is_frozen=False, hparams=hparams, model_directory=run_config.model_dir, dropout_rate=dropout_rate, allow_auxiliary_head=self._allow_auxiliary_head) logits_specs = [tower_spec.logits_spec] apply_snapshot = ( self._phoenix_spec.transfer_learning_spec.transfer_learning_type == transfer_learning_spec_pb2.TransferLearningSpec .SNAPSHOT_TRANSFER_LEARNING) if prev_trial and prev_trial > 0 and apply_snapshot: architecture_utils.init_variables( checkpoint=tf.train.latest_checkpoint( architecture_utils.DirectoryHandler.trial_dir( self._get_trial_from_id(prev_trial, trials))), original_scope="Phoenix/{}_0".format(self.generator_name()), new_scope="Phoenix/{}_0".format(self.generator_name())) architecture_utils.set_number_of_towers(self.generator_name(), 1) return logits_specs, [tower_spec.architecture]
def test_import_tower(self, shared_input): np.random.seed(42) test_input = np.random.random([1, 32, 32, 3]) # Force graph mode with tf.compat.v1.Graph().as_default(): directory = self.get_temp_dir() architecture = np.array([1, 3, 34]) phoenix_spec = phoenix_spec_pb2.PhoenixSpec( problem_type=phoenix_spec_pb2.PhoenixSpec.CNN) phoenix_spec.is_input_shared = shared_input features = {} shared_input_tensor = None with self.test_session(graph=tf.Graph()) as sess: input_tensor_1 = tf.compat.v1.placeholder( dtype=tf.float32, shape=[None, 32, 32, 3], name="input_1") tf.random.set_seed(1234) tower_spec_1 = architecture_utils.construct_tower( phoenix_spec=phoenix_spec, input_tensor=input_tensor_1, tower_name="test_tower", architecture=architecture, is_training=True, lengths=None, logits_dimension=10, hparams=hp.HParams(), model_directory=self.get_temp_dir(), is_frozen=False, dropout_rate=None) saver = tf.compat.v1.train.Saver() sess.run(tf.compat.v1.global_variables_initializer()) sess.run(tf.compat.v1.local_variables_initializer()) logits_val_1 = sess.run(tower_spec_1.logits_spec.logits, feed_dict={input_tensor_1: test_input}) saver.save(sess, directory + "/ckpt") with self.test_session(graph=tf.Graph()) as sess: input_tensor_2 = tf.compat.v1.placeholder( dtype=tf.float32, shape=[None, 32, 32, 3], name="input_2") if shared_input: shared_input_tensor = input_tensor_2 def _input_layer_fn(features, is_training, scope_name="Phoenix/Input", lengths_feature_name=None): del features, is_training, scope_name, lengths_feature_name return None, None else: features = {"x": input_tensor_2} def _input_layer_fn(features, is_training, scope_name="Phoenix/Input", lengths_feature_name=None): del is_training, lengths_feature_name with tf.compat.v1.variable_scope(scope_name): return tf.cast(features["x"], dtype=tf.float32), None tf.random.set_seed(1234) tower_spec_2 = architecture_utils.import_tower( features=features, input_layer_fn=_input_layer_fn, phoenix_spec=phoenix_spec, shared_input_tensor=shared_input_tensor, original_tower_name="test_tower", new_tower_name="imported_tower", model_directory=directory, new_model_directory=self.get_temp_dir(), is_training=True, logits_dimension=10, shared_lengths=None, force_snapshot=False, force_freeze=False) sess.run(tf.compat.v1.global_variables_initializer()) sess.run(tf.compat.v1.local_variables_initializer()) logits_val_2 = sess.run(tower_spec_2.logits_spec.logits, feed_dict={input_tensor_2: test_input}) self.assertAllClose(logits_val_1, logits_val_2, rtol=1e-3)
def create_model_spec(self, features, params, learning_rate_spec, use_tpu, train_logits_specs, eval_logits_spec, labels, mode, lengths, loss_fn, predictions_fn, optimizer_fn=None): """Creates model spec for all tasks.""" is_training = mode == tf.estimator.ModeKeys.TRAIN if not optimizer_fn and is_training: optimizer_fn = _get_optimizer_fn( optimizer=params.optimizer, learning_rate=learning_rate_spec["learning_rate"], use_tpu=use_tpu, exponential_decay_steps=learning_rate_spec.get( "exponential_decay_steps", -1), exponential_decay_rate=learning_rate_spec.get( "exponential_decay_rate", -1), lr_warmup_steps=self._phoenix_spec.learning_spec. lr_warmup_steps) train_op_fn = eval_op_fn if is_training: train_op_fn = functools.partial( _train_op_fn, optimizer_fn=optimizer_fn, use_synchronous_optimizer=self._phoenix_spec. use_synchronous_optimizer, l2_regularization=learning_rate_spec.get( "l2_regularization", -1), gradient_max_norm=learning_rate_spec.get( "gradient_max_norm", -1)) # One label / One task - Old Phoenix behavior if (self._phoenix_spec.multi_task_spec is None or len(self._phoenix_spec.multi_task_spec) <= 1): with tf.compat.v1.variable_scope("Phoenix/Trainer"): weights = 1.0 if (self._phoenix_spec.weight_feature_name and mode != tf.estimator.ModeKeys.PREDICT): weights = features[self._phoenix_spec.weight_feature_name] task_spec = self._create_task_spec( labels=labels, weights=weights, train_logits_specs=train_logits_specs, eval_logits_spec=eval_logits_spec, train_op_fn=train_op_fn, name="Trainer", mode=mode, loss_fn=loss_fn) train_op = None if is_training: train_op = _merge_train_op_list( task_spec.train_op_list, self._ensemble_spec.no_train_speedup) # create predictions here. predictions = predictions_fn( eval_logits_spec.logits, mode=mode, temperature=self._phoenix_spec.temperature) return ModelSpec(loss=task_spec.loss, train_op=train_op, predictions=predictions, train_hooks=task_spec.train_hooks_list) # MultiTask. New Phoenix Behavior # Details in: # go/phoenix-multitask num_weights_in_labels = 0 for task_spec in self._phoenix_spec.multi_task_spec: if not task_spec.weight_is_a_feature: num_weights_in_labels += 1 # In predict mode we don't have labels if labels: assert (len(labels) == len(self._phoenix_spec.multi_task_spec) + num_weights_in_labels) if len(train_logits_specs) > 1: logging.warning( "Using ensembling in a multi-task training. If there is " "no task that restrict the searchable logits from " "rotating, then training is going to produce bad " "non-aligned ensembles.") primary_task = None task_spec_list = [] for task_spec in self._phoenix_spec.multi_task_spec: logits_spec = train_logits_specs[0] # Build tower on top of searched model for the specific task if task_spec.architecture: tower_architecture = [ blocks.BlockType[block_type] for block_type in task_spec.architecture ] task_tower_spec = architecture_utils.construct_tower( phoenix_spec=self._phoenix_spec, input_tensor=tf.nn.relu(logits_spec.logits), tower_name="task_{}_tower".format(task_spec.label_name), architecture=np.array(tower_architecture), is_training=is_training, lengths=lengths, logits_dimension=task_spec.number_of_classes, is_frozen=False, # TODO(b/172564129): add dropouts. dropout_rate=None) # Ignore auxiliary heads for task towers, if any. task_logits = task_tower_spec.logits_spec.logits else: with tf.compat.v1.variable_scope( "Phoenix/task_{}_tower".format(task_spec.label_name)): task_logits = self._add_projection_if_needed( logits_spec.logits, task_spec.number_of_classes) # Replace the base tower logits with those of the task tower. logits_spec = logits_spec._replace(logits=task_logits) if logits_spec.aux_logits: aux_logits = self._add_projection_if_needed( logits_spec.aux_logits, task_spec.number_of_classes) logits_spec = logits_spec._replace(aux_logits=aux_logits) with tf.compat.v1.variable_scope("Phoenix/trainer_{}".format( task_spec.label_name)): if mode != tf.estimator.ModeKeys.PREDICT: task_labels = labels[task_spec.label_name] else: task_labels = None weights = 1.0 if (task_spec.weight_feature_name and mode != tf.estimator.ModeKeys.PREDICT): if task_spec.weight_is_a_feature: weights = features[task_spec.weight_feature_name] else: weights = labels[task_spec.weight_feature_name] task = self._create_task_spec(labels=task_labels, weights=weights, train_logits_specs=[logits_spec], eval_logits_spec=logits_spec, train_op_fn=train_op_fn, name=task_spec.label_name, mode=mode, loss_fn=loss_fn) task_spec_list.append(task) if task_spec.label_name == self._phoenix_spec.primary_task_name: primary_task = task if not self._phoenix_spec.primary_task_name: primary_task = task_spec_list[0] model_spec_predictions = {} for task in task_spec_list: task_predictions = predictions_fn( task.logits, mode=mode, temperature=self._phoenix_spec.temperature) for prediction_key, prediction_value in task_predictions.items(): prediction_key_name = prediction_key + "/" + task.name model_spec_predictions[prediction_key_name] = prediction_value if task.name == primary_task.name: model_spec_predictions[prediction_key] = prediction_value logging.info(model_spec_predictions) train_op_list = [] train_hooks_list = [] for task in task_spec_list: train_op_list.extend(task.train_op_list) train_hooks_list.extend(task.train_hooks_list) train_op = None if is_training: train_op = _merge_train_op_list( train_op_list, self._ensemble_spec.no_train_speedup) return ModelSpec(loss=primary_task.loss, train_op=train_op, predictions=model_spec_predictions, train_hooks=train_hooks_list)