def _create_feed_dicts(self, generator, training): """Create feed dicts for use in fitting or prediction. Parameters ---------- generator: Generator the feed dict generator that was passed to fit_generator() or predict_on_generator() training: bool True during training, False during prediction """ train_value = 1.0 if training else 0.0 if self.queue_installed: while True: yield {self._training_placeholder: train_value} else: for d in generator: feed_dict = {} for key, value in d.items(): if isinstance(key, Input): value = _ensure_value_shape(value, key) if tfe.in_eager_mode(): value = tf.cast(value, key.dtype) feed_dict[key] = value else: feed_dict[key] = value if not tfe.in_eager_mode(): feed_dict[self._training_placeholder] = train_value yield feed_dict
def test_copy_layers(self): """Test copying layers.""" tg = dc.models.TensorGraph() features = Feature(shape=(None, 10)) dense = Dense(10, in_layers=features, biases_initializer=tf.random_normal_initializer) constant = Constant(10.0) output = dense + constant tg.add_output(output) tg.set_loss(output) tg.fit_generator([]) replacements = {constant: Constant(20.0)} copy = output.copy(replacements, tg) assert isinstance(copy, Add) assert isinstance(copy.in_layers[0], Dense) assert isinstance(copy.in_layers[0].in_layers[0], Feature) assert copy.in_layers[1] == replacements[constant] variables = tg.get_layer_variables(dense) with tg._get_tf("Graph").as_default(): if tfe.in_eager_mode(): values = [v.numpy() for v in variables] else: values = tg.session.run(variables) for v1, v2 in zip(values, copy.in_layers[0].variable_values): assert np.array_equal(v1, v2)
def _run_graph(self, outputs, feed_dict, training): """Run the calculations in the graph to compute some outputs. In graph mode, this just calls session.run(). In eager mode, it executes all required layers to compute the output. Parameters ---------- outputs: list of Layers the output layers to compute feed_dict: dict maps input layers to values training: bool whether this is being executed in training mode """ if not tfe.in_eager_mode(): return self.session.run(outputs, feed_dict) def run_layers(layer, tensors): if layer in tensors: return tensors[layer] inputs = [run_layers(input, tensors) for input in layer.in_layers] tensor = layer.create_tensor( in_layers=inputs, set_tensors=False, training=training) tensors[layer] = tensor return tensor tensors = feed_dict.copy() return [run_layers(o, tensors) for o in outputs]
def test_copy_layers(self): """Test copying layers.""" tg = dc.models.TensorGraph() features = Feature(shape=(None, 10)) dense = Dense( 10, in_layers=features, biases_initializer=tf.random_normal_initializer) constant = Constant(10.0) output = dense + constant tg.add_output(output) tg.set_loss(output) tg.fit_generator([]) replacements = {constant: Constant(20.0)} copy = output.copy(replacements, tg) assert isinstance(copy, Add) assert isinstance(copy.in_layers[0], Dense) assert isinstance(copy.in_layers[0].in_layers[0], Feature) assert copy.in_layers[1] == replacements[constant] variables = tg.get_layer_variables(dense) with tg._get_tf("Graph").as_default(): if tfe.in_eager_mode(): values = [v.numpy() for v in variables] else: values = tg.session.run(variables) for v1, v2 in zip(values, copy.in_layers[0].variable_values): assert np.array_equal(v1, v2)
def get_layer_variables(self, layer): """Get the list of trainable variables in a layer of the graph.""" if tfe.in_eager_mode(): return layer.variables if not self.built: self.build() with self._get_tf("Graph").as_default(): if layer.variable_scope == '': return [] return tf.get_collection( tf.GraphKeys.TRAINABLE_VARIABLES, scope=layer.variable_scope)
def get_variables(self): """Get the list of all trainable variables in the graph.""" if not self.built: self.build() if tfe.in_eager_mode(): variables = [] for layer in self.layers.values(): variables += layer.variables return variables else: with self._get_tf("Graph").as_default(): return tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
def _create_feed_dicts(self, generator, training): """Create feed dicts for use in fitting or prediction. Parameters ---------- generator: Generator the feed dict generator that was passed to fit_generator() or predict_on_generator() training: bool True during training, False during prediction """ if tfe.in_eager_mode(): for d in generator: feed_dict = {} for key, value in d.items(): if isinstance(key, Input): # Add or remove dimensions of size 1 to match the shape of the layer. try: value_dims = len(value.shape) layer_dims = len(key.shape) if value_dims < layer_dims: if all(i == 1 for i in key.shape[value_dims:]): value = tf.reshape(value, list(value.shape) + [1] * (layer_dims - value_dims)) if value_dims > layer_dims: if all(i == 1 for i in value.shape[layer_dims:]): value = tf.reshape(value, value.shape[:layer_dims]) except: pass feed_dict[key] = tf.cast(value, key.dtype) else: feed_dict[key] = value yield feed_dict else: train_value = 1.0 if training else 0.0 if self.queue_installed: while True: yield {self._training_placeholder: train_value} for d in generator: feed_dict = dict(d) feed_dict[self._training_placeholder] = train_value yield feed_dict
def fit_generator(self, feed_dict_generator, max_checkpoints_to_keep=5, checkpoint_interval=1000, restore=False, submodel=None): """Train this model on data from a generator. Parameters ---------- feed_dict_generator: generator this should generate batches, each represented as a dict that maps Layers to values. max_checkpoints_to_keep: int the maximum number of checkpoints to keep. Older checkpoints are discarded. checkpoint_interval: int the frequency at which to write checkpoints, measured in training steps. Set this to 0 to disable automatic checkpointing. restore: bool if True, restore the model from the most recent checkpoint and continue training from there. If False, retrain the model from scratch. submodel: Submodel an alternate training objective to use. This should have been created by calling create_submodel(). Returns ------- the average loss over the most recent checkpoint interval """ if not self.built: self.build() with self._get_tf("Graph").as_default(): time1 = time.time() loss = self.loss if submodel is not None and submodel.loss is not None: loss = submodel.loss if tfe.in_eager_mode(): # In eager mode we want an optimizer and a function to compute the # gradient of the loss. submodel_vars = None if submodel is None: optimizer = self._get_tf("Optimizer") else: optimizer = submodel.create_optimizer() if submodel.layers is not None: submodel_vars = set() for layer in submodel.layers: for var in layer.variables: submodel_vars.add(var) val_grad_fn = tfe.implicit_value_and_gradients( lambda x: self._run_graph([loss], x, True)[0]) else: # In graph mode we want a training operation. if submodel is None: train_op = self._get_tf('train_op') else: train_op = submodel.get_train_op() if checkpoint_interval > 0: saver = tf.train.Saver( self.get_variables(), max_to_keep=max_checkpoints_to_keep, save_relative_paths=True) if restore: self.restore() avg_loss, n_averaged_batches = 0.0, 0.0 n_samples = 0 n_enqueued = [0] final_sample = [None] if self.queue_installed: enqueue_thread = threading.Thread( target=_enqueue_batch, args=(self, feed_dict_generator, self._get_tf("Graph"), self.session, n_enqueued, final_sample)) enqueue_thread.start() for feed_dict in self._create_feed_dicts(feed_dict_generator, True): if self.queue_installed: # Don't let this thread get ahead of the enqueue thread, since if # we try to read more batches than the total number that get queued, # this thread will hang indefinitely. while n_enqueued[0] <= n_samples: if n_samples == final_sample[0]: break time.sleep(0) if n_samples == final_sample[0]: break n_samples += 1 should_log = (self.tensorboard and n_samples % self.tensorboard_log_frequency == 0) if tfe.in_eager_mode(): value, grads_and_vars = val_grad_fn(feed_dict) if submodel_vars is not None: grads_and_vars = [ x for x in grads_and_vars if x[1] in submodel_vars ] optimizer.apply_gradients(grads_and_vars) avg_loss += value else: fetches = [train_op, loss.out_tensor] if should_log: fetches.append(self._get_tf("summary_op")) fetched_values = self.session.run(fetches, feed_dict=feed_dict) if should_log: self._log_tensorboard(fetched_values[2]) avg_loss += fetched_values[1] n_averaged_batches += 1 self.global_step += 1 if checkpoint_interval > 0 and self.global_step % checkpoint_interval == checkpoint_interval - 1: saver.save(self.session, self.save_file, global_step=self.global_step) avg_loss = float(avg_loss) / n_averaged_batches logger.info('Ending global_step %d: Average loss %g' % (self.global_step, avg_loss)) avg_loss, n_averaged_batches = 0.0, 0.0 if n_averaged_batches > 0: avg_loss = float(avg_loss) / n_averaged_batches if checkpoint_interval > 0: if n_averaged_batches > 0: logger.info('Ending global_step %d: Average loss %g' % (self.global_step, avg_loss)) saver.save(self.session, self.save_file, global_step=self.global_step) time2 = time.time() logger.info("TIMING: model fitting took %0.3f s" % (time2 - time1)) return avg_loss
def create_variable(value, dtype=None, name=None, trainable=True): """Create a tf.Variable or tfe.Variable, depending on the current mode.""" if tfe.in_eager_mode(): return tfe.Variable(value, dtype=dtype, name=name, trainable=trainable) else: return tf.Variable(value, dtype=dtype, name=name, trainable=trainable)
def build(self): if self.built: return if tfe.in_eager_mode(): # In eager mode, we need to execute every layer once to ensure its variables # have been created. def build_layers(layer, tensors): if layer in tensors: return tensors[layer] inputs = [build_layers(input, tensors) for input in layer.in_layers] if isinstance(layer, Input): # We can't execute Input layers in eager mode, since they would try # to create placeholders. Instead create a tensor of the correct # size and type. shape = [1 if s is None else s for s in layer.shape] tensor = tf.zeros(shape, layer.dtype) else: with tf.name_scope(layer.name): tensor = layer.create_tensor(in_layers=inputs, set_tensors=False) tensors[layer] = tensor return tensor tensors = {} with self._get_tf("Graph").as_default(): # Build the layers. build_layers(self.loss, tensors) for output in self.outputs: build_layers(output, tensors) for variance in self.variances: build_layers(variance, tensors) for submodel in self.submodels: build_layers(submodel.loss, tensors) # Initialize variables. for layer in self.layers.values(): if layer.variable_values is not None: for var, val in zip(layer.variables, layer.variable_values): var.assign(val) self.session = None self._training_placeholder = None self.built = True return # In graph mode we need to create the computation graph. with self._get_tf("Graph").as_default(): self._training_placeholder = tf.placeholder(dtype=tf.float32, shape=()) if self.random_seed is not None: tf.set_random_seed(self.random_seed) self._install_queue() for layer in self.topsort(): with tf.name_scope(layer.name): layer.create_tensor(training=self._training_placeholder) self.rnn_initial_states += layer.rnn_initial_states self.rnn_final_states += layer.rnn_final_states self.rnn_zero_states += layer.rnn_zero_states layer.add_summary_to_tg() self.session = tf.Session(config=self.configproto) self.built = True # Ensure all training operators have been created. self._get_tf('train_op') for submodel in self.submodels: train_op = submodel.get_train_op() # Initialize variables. self.session.run(tf.global_variables_initializer()) for layer in self.layers.values(): if layer.variable_values is not None: variables = self.get_layer_variables(layer) for var, val in zip(variables, layer.variable_values): self.session.run(var.assign(val)) for layer in self.layers.values(): if layer.tensorboard: self.tensorboard = True tf.summary.scalar("loss", self.loss.out_tensor) for layer in self.layers.values(): if layer.tensorboard: tf.summary.tensor_summary(layer.name, layer.out_tensor) if self.tensorboard: writer = self._get_tf("FileWriter") writer.add_graph(self._get_tf("Graph")) writer.close() # As a sanity check, make sure all tensors have the correct shape. for layer in self.layers.values(): try: assert list(layer.shape) == layer.out_tensor.get_shape().as_list( ), '%s: Expected shape %s does not match actual shape %s' % ( layer.name, layer.shape, layer.out_tensor.get_shape().as_list()) except NotImplementedError: pass
def _predict(self, generator, transformers, outputs, uncertainty): """ Predict outputs for data provided by a generator. This is the private implementation of prediction. Do not call it directly. Instead call one of the public prediction methods. Parameters ---------- generator: Generator Generator that constructs feed dictionaries for TensorGraph. transformers: list List of dc.trans.Transformers. outputs: object If outputs is None, then will assume outputs = self.outputs. If outputs is a Layer/Tensor, then will evaluate and return as a single ndarray. If outputs is a list of Layers/Tensors, will return a list of ndarrays. uncertainty: bool specifies whether this is being called as part of estimating uncertainty. If True, it sets the training flag so that dropout will be enabled, and returns the values of the uncertainty outputs. Returns: y_pred: numpy ndarray of shape (n_samples, n_classes*n_tasks) """ if not self.built: self.build() if outputs is None: outputs = self.outputs elif not isinstance(outputs, collections.Sequence): outputs = [outputs] if uncertainty: if len(self.variances) == 0: raise ValueError('This model cannot compute uncertainties') if len(self.variances) != len(outputs): raise ValueError( 'The number of variances must exactly match the number of outputs') tensors = outputs + self.variances else: tensors = outputs with self._get_tf("Graph").as_default(): # Gather results for each output results = [[] for out in tensors] n_samples = 0 n_enqueued = [0] final_sample = [None] if self.queue_installed: enqueue_thread = threading.Thread( target=_enqueue_batch, args=(self, generator, self._get_tf("Graph"), self.session, n_enqueued, final_sample)) enqueue_thread.start() for feed_dict in self._create_feed_dicts(generator, uncertainty): if self.queue_installed: # Don't let this thread get ahead of the enqueue thread, since if # we try to read more batches than the total number that get queued, # this thread will hang indefinitely. while n_enqueued[0] <= n_samples: if n_samples == final_sample[0]: break time.sleep(0) if n_samples == final_sample[0]: break n_samples += 1 feed_results = self._run_graph(tensors, feed_dict, uncertainty) if tfe.in_eager_mode(): feed_results = [f.numpy() for f in feed_results] if len(feed_results) > 1: if len(transformers): raise ValueError("Does not support transformations " "for multiple outputs.") elif len(feed_results) == 1: result = undo_transforms(feed_results[0], transformers) feed_results = [result] for ind, result in enumerate(feed_results): results[ind].append(result) final_results = [] for result_list in results: final_results.append(np.concatenate(result_list, axis=0)) # If only one output, just return array if len(final_results) == 1: return final_results[0] elif uncertainty: return zip(final_results[:len(outputs)], final_results[len(outputs):]) else: return final_results
def make_estimator(self, feature_columns, weight_column=None, metrics={}, model_dir=None, config=None): """Construct a Tensorflow Estimator from this model. tf.estimator.Estimator is the standard Tensorflow API for representing models. This method provides interoperability between DeepChem and other Tensorflow based tools by allowing any model to be used an Estimator. Once this method returns, the Estimator it created is independent of the model it was created from. They do not share tensors, variables, save files, or any other resources. The Estimator is a self contained object with its own methods for training, evaluation, prediction, checkpointing, etc. Parameters ---------- feature_columns: list of tf.feature_column objects this describes the input features to the models. There must be one entry for each Feature layer in this model's features field. weight_column: tf.feature_column or None if this model includes a Weights layer, this describes the input weights. Otherwise, this should be None. metrics: map metrics that should be computed in calls to evaluate(). For each entry, the key is the name to report for the metric, and the value is a function of the form f(labels, predictions, weights) that returns the tensors for computing the metric. Any of the functions in tf.metrics can be used, as can other functions that satisfy the same interface. model_dir: str the directory in which the Estimator should save files. If None, this defaults to the model's model_dir. config: RunConfig configuration options for the Estimator """ # Check the inputs. if tfe.in_eager_mode(): raise ValueError('make_estimator() is not supported in eager mode') if len(feature_columns) != len(self.features): raise ValueError( 'This model requires %d feature column(s)' % len(self.features)) if len(self.labels) != 1: raise ValueError( 'Can only create an Estimator from a model with exactly one Label input' ) if len(self.task_weights) > 1: raise ValueError( 'Cannot create an Estimator from a model with multiple Weight inputs') if weight_column is None: if len(self.task_weights) > 0: raise ValueError('This model requires a weight column') else: if len(self.task_weights) == 0: raise ValueError( 'Cannot specify weight_column for a model with no Weight inputs') if model_dir is None: model_dir = self.model_dir # Define a function that recursively creates tensors from layers. def create_tensors(layer, tensors, training): if layer in tensors: return tensors[layer] inputs = [ create_tensors(in_layer, tensors, training) for in_layer in layer.in_layers ] tensor = layer.create_tensor( in_layers=inputs, set_tensors=False, training=training) tensors[layer] = tensor layer.add_summary_to_tg(tensor) return tensor # Define the model function. def model_fn(features, labels, mode): # Define the inputs. tensors = self.create_estimator_inputs(feature_columns, weight_column, features, labels, mode) for layer, tensor in tensors.items(): layer.add_summary_to_tg(tensor) # Create the correct outputs, based on the mode. if mode == tf.estimator.ModeKeys.PREDICT: predictions = {} for i, output in enumerate(self.outputs): predictions[i] = create_tensors(output, tensors, 0) return tf.estimator.EstimatorSpec(mode, predictions=predictions) if mode == tf.estimator.ModeKeys.EVAL: loss = create_tensors(self.loss, tensors, 0) predictions = create_tensors(self.outputs[0], tensors, 0) if len(self.task_weights) == 0: weights = None else: weights = tensors[self.task_weights[0]] eval_metric_ops = {} for name, function in metrics.items(): eval_metric_ops[name] = function(tensors[self.labels[0]], predictions, weights) return tf.estimator.EstimatorSpec( mode, loss=loss, eval_metric_ops=eval_metric_ops) if mode == tf.estimator.ModeKeys.TRAIN: loss = create_tensors(self.loss, tensors, 1) global_step = tf.train.get_global_step() optimizer = self.optimizer._create_optimizer(global_step) train_op = optimizer.minimize(loss, global_step=global_step) return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op) raise ValueError('Unknown mode') # Create the Estimator. return tf.estimator.Estimator( model_fn=model_fn, model_dir=model_dir, config=config)
def make_estimator(self, feature_columns, weight_column=None, metrics={}, model_dir=None, config=None): """Construct a Tensorflow Estimator from this model. tf.estimator.Estimator is the standard Tensorflow API for representing models. This method provides interoperability between DeepChem and other Tensorflow based tools by allowing any model to be used an Estimator. Once this method returns, the Estimator it created is independent of the model it was created from. They do not share tensors, variables, save files, or any other resources. The Estimator is a self contained object with its own methods for training, evaluation, prediction, checkpointing, etc. Parameters ---------- feature_columns: list of tf.feature_column objects this describes the input features to the models. There must be one entry for each Feature layer in this model's features field. weight_column: tf.feature_column or None if this model includes a Weights layer, this describes the input weights. Otherwise, this should be None. metrics: map metrics that should be computed in calls to evaluate(). For each entry, the key is the name to report for the metric, and the value is a function of the form f(labels, predictions, weights) that returns the tensors for computing the metric. Any of the functions in tf.metrics can be used, as can other functions that satisfy the same interface. model_dir: str the directory in which the Estimator should save files. If None, this defaults to the model's model_dir. config: RunConfig configuration options for the Estimator """ # Check the inputs. if tfe.in_eager_mode(): raise ValueError('make_estimator() is not supported in eager mode') if len(feature_columns) != len(self.features): raise ValueError( 'This model requires %d feature column(s)' % len(self.features)) if len(self.labels) != 1: raise ValueError( 'Can only create an Estimator from a model with exactly one Label input' ) if len(self.task_weights) > 1: raise ValueError( 'Cannot create an Estimator from a model with multiple Weight inputs') if weight_column is None: if len(self.task_weights) > 0: raise ValueError('This model requires a weight column') else: if len(self.task_weights) == 0: raise ValueError( 'Cannot specify weight_column for a model with no Weight inputs') if model_dir is None: model_dir = self.model_dir # Define a function that recursively creates tensors from layers. def create_tensors(layer, tensors, training): if layer in tensors: return tensors[layer] inputs = [ create_tensors(in_layer, tensors, training) for in_layer in layer.in_layers ] tensor = layer.create_tensor( in_layers=inputs, set_tensors=False, training=training) tensors[layer] = tensor vars = tf.get_collection( tf.GraphKeys.TRAINABLE_VARIABLES, scope=layer.name) layer.add_summary_to_tg(tensor, vars) return tensor # Define the model function. def model_fn(features, labels, mode): # Define the inputs. tensors = self.create_estimator_inputs(feature_columns, weight_column, features, labels, mode) for layer, tensor in tensors.items(): layer.add_summary_to_tg(tensor, []) # Create the correct outputs, based on the mode. if mode == tf.estimator.ModeKeys.PREDICT: predictions = {} for i, output in enumerate(self.outputs): predictions[i] = create_tensors(output, tensors, 0) return tf.estimator.EstimatorSpec(mode, predictions=predictions) if mode == tf.estimator.ModeKeys.EVAL: loss = create_tensors(self.loss, tensors, 0) predictions = create_tensors(self.outputs[0], tensors, 0) if len(self.task_weights) == 0: weights = None else: weights = tensors[self.task_weights[0]] eval_metric_ops = {} for name, function in metrics.items(): eval_metric_ops[name] = function(tensors[self.labels[0]], predictions, weights) return tf.estimator.EstimatorSpec( mode, loss=loss, eval_metric_ops=eval_metric_ops) if mode == tf.estimator.ModeKeys.TRAIN: loss = create_tensors(self.loss, tensors, 1) global_step = tf.train.get_global_step() optimizer = self.optimizer._create_optimizer(global_step) train_op = optimizer.minimize(loss, global_step=global_step) return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op) raise ValueError('Unknown mode') # Create the Estimator. return tf.estimator.Estimator( model_fn=model_fn, model_dir=model_dir, config=config)
def build(self): if self.built: return if tfe.in_eager_mode(): # In eager mode, we need to execute every layer once to ensure its variables # have been created. def build_layers(layer, tensors): if layer in tensors: return tensors[layer] inputs = [build_layers(input, tensors) for input in layer.in_layers] if isinstance(layer, Input): # We can't execute Input layers in eager mode, since they would try # to create placeholders. Instead create a tensor of the correct # size and type. shape = [1 if s is None else s for s in layer.shape] tensor = tf.zeros(shape, layer.dtype) else: with tf.name_scope(layer.name): tensor = layer.create_tensor(in_layers=inputs, set_tensors=False) tensors[layer] = tensor return tensor tensors = {} with self._get_tf("Graph").as_default(): # Build the layers. build_layers(self.loss, tensors) for output in self.outputs: build_layers(output, tensors) for variance in self.variances: build_layers(variance, tensors) for submodel in self.submodels: build_layers(submodel.loss, tensors) # Initialize variables. for layer in self.layers.values(): if layer.variable_values is not None: for var, val in zip(layer.variables, layer.variable_values): var.assign(val) self.session = None self._training_placeholder = None self.built = True return # In graph mode we need to create the computation graph. with self._get_tf("Graph").as_default(): self._training_placeholder = tf.placeholder(dtype=tf.float32, shape=()) if self.random_seed is not None: tf.set_random_seed(self.random_seed) self._install_queue() self.built = True for layer in self.topsort(): with tf.name_scope(layer.name): layer.create_tensor(training=self._training_placeholder) self.rnn_initial_states += layer.rnn_initial_states self.rnn_final_states += layer.rnn_final_states self.rnn_zero_states += layer.rnn_zero_states layer.add_summary_to_tg(layer.out_tensor, self.get_layer_variables(layer)) self.session = tf.Session(config=self.configproto) # Ensure all training operators have been created. self._get_tf('train_op') for submodel in self.submodels: train_op = submodel.get_train_op() # Initialize variables. self.session.run(tf.global_variables_initializer()) for layer in self.layers.values(): if layer.variable_values is not None: variables = self.get_layer_variables(layer) for var, val in zip(variables, layer.variable_values): self.session.run(var.assign(val)) for layer in self.layers.values(): if layer.tensorboard: self.tensorboard = True tf.summary.scalar("loss", self.loss.out_tensor) for layer in self.layers.values(): if layer.tensorboard: tf.summary.tensor_summary(layer.name, layer.out_tensor) if self.tensorboard: writer = self._get_tf("FileWriter") writer.add_graph(self._get_tf("Graph")) writer.close() # As a sanity check, make sure all tensors have the correct shape. for layer in self.layers.values(): try: assert list(layer.shape) == layer.out_tensor.get_shape().as_list( ), '%s: Expected shape %s does not match actual shape %s' % ( layer.name, layer.shape, layer.out_tensor.get_shape().as_list()) except NotImplementedError: pass