def _create_feed_dicts(self, generator, training):
"""Create feed dicts for use in fitting or prediction.
Parameters
----------
generator: Generator
the feed dict generator that was passed to fit_generator() or predict_on_generator()
training: bool
True during training, False during prediction
"""
train_value = 1.0 if training else 0.0
if self.queue_installed:
while True:
yield {self._training_placeholder: train_value}
else:
for d in generator:
feed_dict = {}
for key, value in d.items():
if isinstance(key, Input):
value = _ensure_value_shape(value, key)
if tfe.in_eager_mode():
value = tf.cast(value, key.dtype)
feed_dict[key] = value
else:
feed_dict[key] = value
if not tfe.in_eager_mode():
feed_dict[self._training_placeholder] = train_value
yield feed_dict
def _create_feed_dicts(self, generator, training):
"""Create feed dicts for use in fitting or prediction.
Parameters
----------
generator: Generator
the feed dict generator that was passed to fit_generator() or predict_on_generator()
training: bool
True during training, False during prediction
"""
train_value = 1.0 if training else 0.0
if self.queue_installed:
while True:
yield {self._training_placeholder: train_value}
else:
for d in generator:
feed_dict = {}
for key, value in d.items():
if isinstance(key, Input):
value = _ensure_value_shape(value, key)
if tfe.in_eager_mode():
value = tf.cast(value, key.dtype)
feed_dict[key] = value
else:
feed_dict[key] = value
if not tfe.in_eager_mode():
feed_dict[self._training_placeholder] = train_value
yield feed_dict
def test_copy_layers(self):
"""Test copying layers."""
tg = dc.models.TensorGraph()
features = Feature(shape=(None, 10))
dense = Dense(10,
in_layers=features,
biases_initializer=tf.random_normal_initializer)
constant = Constant(10.0)
output = dense + constant
tg.add_output(output)
tg.set_loss(output)
tg.fit_generator([])
replacements = {constant: Constant(20.0)}
copy = output.copy(replacements, tg)
assert isinstance(copy, Add)
assert isinstance(copy.in_layers[0], Dense)
assert isinstance(copy.in_layers[0].in_layers[0], Feature)
assert copy.in_layers[1] == replacements[constant]
variables = tg.get_layer_variables(dense)
with tg._get_tf("Graph").as_default():
if tfe.in_eager_mode():
values = [v.numpy() for v in variables]
else:
values = tg.session.run(variables)
for v1, v2 in zip(values, copy.in_layers[0].variable_values):
assert np.array_equal(v1, v2)
def _run_graph(self, outputs, feed_dict, training):
"""Run the calculations in the graph to compute some outputs.
In graph mode, this just calls session.run(). In eager mode, it executes
all required layers to compute the output.
Parameters
----------
outputs: list of Layers
the output layers to compute
feed_dict: dict
maps input layers to values
training: bool
whether this is being executed in training mode
"""
if not tfe.in_eager_mode():
return self.session.run(outputs, feed_dict)
def run_layers(layer, tensors):
if layer in tensors:
return tensors[layer]
inputs = [run_layers(input, tensors) for input in layer.in_layers]
tensor = layer.create_tensor(
in_layers=inputs, set_tensors=False, training=training)
tensors[layer] = tensor
return tensor
tensors = feed_dict.copy()
return [run_layers(o, tensors) for o in outputs]
def test_copy_layers(self):
"""Test copying layers."""
tg = dc.models.TensorGraph()
features = Feature(shape=(None, 10))
dense = Dense(
10, in_layers=features, biases_initializer=tf.random_normal_initializer)
constant = Constant(10.0)
output = dense + constant
tg.add_output(output)
tg.set_loss(output)
tg.fit_generator([])
replacements = {constant: Constant(20.0)}
copy = output.copy(replacements, tg)
assert isinstance(copy, Add)
assert isinstance(copy.in_layers[0], Dense)
assert isinstance(copy.in_layers[0].in_layers[0], Feature)
assert copy.in_layers[1] == replacements[constant]
variables = tg.get_layer_variables(dense)
with tg._get_tf("Graph").as_default():
if tfe.in_eager_mode():
values = [v.numpy() for v in variables]
else:
values = tg.session.run(variables)
for v1, v2 in zip(values, copy.in_layers[0].variable_values):
assert np.array_equal(v1, v2)
def _run_graph(self, outputs, feed_dict, training):
"""Run the calculations in the graph to compute some outputs.
In graph mode, this just calls session.run(). In eager mode, it executes
all required layers to compute the output.
Parameters
----------
outputs: list of Layers
the output layers to compute
feed_dict: dict
maps input layers to values
training: bool
whether this is being executed in training mode
"""
if not tfe.in_eager_mode():
return self.session.run(outputs, feed_dict)
def run_layers(layer, tensors):
if layer in tensors:
return tensors[layer]
inputs = [run_layers(input, tensors) for input in layer.in_layers]
tensor = layer.create_tensor(
in_layers=inputs, set_tensors=False, training=training)
tensors[layer] = tensor
return tensor
tensors = feed_dict.copy()
return [run_layers(o, tensors) for o in outputs]
def get_layer_variables(self, layer):
"""Get the list of trainable variables in a layer of the graph."""
if tfe.in_eager_mode():
return layer.variables
if not self.built:
self.build()
with self._get_tf("Graph").as_default():
if layer.variable_scope == '':
return []
return tf.get_collection(
tf.GraphKeys.TRAINABLE_VARIABLES, scope=layer.variable_scope)
def get_layer_variables(self, layer):
"""Get the list of trainable variables in a layer of the graph."""
if tfe.in_eager_mode():
return layer.variables
if not self.built:
self.build()
with self._get_tf("Graph").as_default():
if layer.variable_scope == '':
return []
return tf.get_collection(
tf.GraphKeys.TRAINABLE_VARIABLES, scope=layer.variable_scope)
def get_variables(self):
"""Get the list of all trainable variables in the graph."""
if not self.built:
self.build()
if tfe.in_eager_mode():
variables = []
for layer in self.layers.values():
variables += layer.variables
return variables
else:
with self._get_tf("Graph").as_default():
return tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
def get_variables(self):
"""Get the list of all trainable variables in the graph."""
if not self.built:
self.build()
if tfe.in_eager_mode():
variables = []
for layer in self.layers.values():
variables += layer.variables
return variables
else:
with self._get_tf("Graph").as_default():
return tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
def _create_feed_dicts(self, generator, training):
"""Create feed dicts for use in fitting or prediction.
Parameters
----------
generator: Generator
the feed dict generator that was passed to fit_generator() or predict_on_generator()
training: bool
True during training, False during prediction
"""
if tfe.in_eager_mode():
for d in generator:
feed_dict = {}
for key, value in d.items():
if isinstance(key, Input):
# Add or remove dimensions of size 1 to match the shape of the layer.
try:
value_dims = len(value.shape)
layer_dims = len(key.shape)
if value_dims < layer_dims:
if all(i == 1 for i in key.shape[value_dims:]):
value = tf.reshape(value,
list(value.shape) + [1] *
(layer_dims - value_dims))
if value_dims > layer_dims:
if all(i == 1 for i in value.shape[layer_dims:]):
value = tf.reshape(value, value.shape[:layer_dims])
except:
pass
feed_dict[key] = tf.cast(value, key.dtype)
else:
feed_dict[key] = value
yield feed_dict
else:
train_value = 1.0 if training else 0.0
if self.queue_installed:
while True:
yield {self._training_placeholder: train_value}
for d in generator:
feed_dict = dict(d)
feed_dict[self._training_placeholder] = train_value
yield feed_dict
def fit_generator(self,
feed_dict_generator,
max_checkpoints_to_keep=5,
checkpoint_interval=1000,
restore=False,
submodel=None):
"""Train this model on data from a generator.
Parameters
----------
feed_dict_generator: generator
this should generate batches, each represented as a dict that maps
Layers to values.
max_checkpoints_to_keep: int
the maximum number of checkpoints to keep. Older checkpoints are discarded.
checkpoint_interval: int
the frequency at which to write checkpoints, measured in training steps.
Set this to 0 to disable automatic checkpointing.
restore: bool
if True, restore the model from the most recent checkpoint and continue training
from there. If False, retrain the model from scratch.
submodel: Submodel
an alternate training objective to use. This should have been created by
calling create_submodel().
Returns
-------
the average loss over the most recent checkpoint interval
"""
if not self.built:
self.build()
with self._get_tf("Graph").as_default():
time1 = time.time()
loss = self.loss
if submodel is not None and submodel.loss is not None:
loss = submodel.loss
if tfe.in_eager_mode():
# In eager mode we want an optimizer and a function to compute the
# gradient of the loss.
submodel_vars = None
if submodel is None:
optimizer = self._get_tf("Optimizer")
else:
optimizer = submodel.create_optimizer()
if submodel.layers is not None:
submodel_vars = set()
for layer in submodel.layers:
for var in layer.variables:
submodel_vars.add(var)
val_grad_fn = tfe.implicit_value_and_gradients(
lambda x: self._run_graph([loss], x, True)[0])
else:
# In graph mode we want a training operation.
if submodel is None:
train_op = self._get_tf('train_op')
else:
train_op = submodel.get_train_op()
if checkpoint_interval > 0:
saver = tf.train.Saver(
self.get_variables(),
max_to_keep=max_checkpoints_to_keep,
save_relative_paths=True)
if restore:
self.restore()
avg_loss, n_averaged_batches = 0.0, 0.0
n_samples = 0
n_enqueued = [0]
final_sample = [None]
if self.queue_installed:
enqueue_thread = threading.Thread(
target=_enqueue_batch,
args=(self, feed_dict_generator, self._get_tf("Graph"),
self.session, n_enqueued, final_sample))
enqueue_thread.start()
for feed_dict in self._create_feed_dicts(feed_dict_generator, True):
if self.queue_installed:
# Don't let this thread get ahead of the enqueue thread, since if
# we try to read more batches than the total number that get queued,
# this thread will hang indefinitely.
while n_enqueued[0] <= n_samples:
if n_samples == final_sample[0]:
break
time.sleep(0)
if n_samples == final_sample[0]:
break
n_samples += 1
should_log = (self.tensorboard and
n_samples % self.tensorboard_log_frequency == 0)
if tfe.in_eager_mode():
value, grads_and_vars = val_grad_fn(feed_dict)
if submodel_vars is not None:
grads_and_vars = [
x for x in grads_and_vars if x[1] in submodel_vars
]
optimizer.apply_gradients(grads_and_vars)
avg_loss += value
else:
fetches = [train_op, loss.out_tensor]
if should_log:
fetches.append(self._get_tf("summary_op"))
fetched_values = self.session.run(fetches, feed_dict=feed_dict)
if should_log:
self._log_tensorboard(fetched_values[2])
avg_loss += fetched_values[1]
n_averaged_batches += 1
self.global_step += 1
if checkpoint_interval > 0 and self.global_step % checkpoint_interval == checkpoint_interval - 1:
saver.save(self.session, self.save_file, global_step=self.global_step)
avg_loss = float(avg_loss) / n_averaged_batches
logger.info('Ending global_step %d: Average loss %g' %
(self.global_step, avg_loss))
avg_loss, n_averaged_batches = 0.0, 0.0
if n_averaged_batches > 0:
avg_loss = float(avg_loss) / n_averaged_batches
if checkpoint_interval > 0:
if n_averaged_batches > 0:
logger.info('Ending global_step %d: Average loss %g' %
(self.global_step, avg_loss))
saver.save(self.session, self.save_file, global_step=self.global_step)
time2 = time.time()
logger.info("TIMING: model fitting took %0.3f s" % (time2 - time1))
return avg_loss
def create_variable(value, dtype=None, name=None, trainable=True):
"""Create a tf.Variable or tfe.Variable, depending on the current mode."""
if tfe.in_eager_mode():
return tfe.Variable(value, dtype=dtype, name=name, trainable=trainable)
else:
return tf.Variable(value, dtype=dtype, name=name, trainable=trainable)
def build(self):
if self.built:
return
if tfe.in_eager_mode():
# In eager mode, we need to execute every layer once to ensure its variables
# have been created.
def build_layers(layer, tensors):
if layer in tensors:
return tensors[layer]
inputs = [build_layers(input, tensors) for input in layer.in_layers]
if isinstance(layer, Input):
# We can't execute Input layers in eager mode, since they would try
# to create placeholders. Instead create a tensor of the correct
# size and type.
shape = [1 if s is None else s for s in layer.shape]
tensor = tf.zeros(shape, layer.dtype)
else:
with tf.name_scope(layer.name):
tensor = layer.create_tensor(in_layers=inputs, set_tensors=False)
tensors[layer] = tensor
return tensor
tensors = {}
with self._get_tf("Graph").as_default():
# Build the layers.
build_layers(self.loss, tensors)
for output in self.outputs:
build_layers(output, tensors)
for variance in self.variances:
build_layers(variance, tensors)
for submodel in self.submodels:
build_layers(submodel.loss, tensors)
# Initialize variables.
for layer in self.layers.values():
if layer.variable_values is not None:
for var, val in zip(layer.variables, layer.variable_values):
var.assign(val)
self.session = None
self._training_placeholder = None
self.built = True
return
# In graph mode we need to create the computation graph.
with self._get_tf("Graph").as_default():
self._training_placeholder = tf.placeholder(dtype=tf.float32, shape=())
if self.random_seed is not None:
tf.set_random_seed(self.random_seed)
self._install_queue()
for layer in self.topsort():
with tf.name_scope(layer.name):
layer.create_tensor(training=self._training_placeholder)
self.rnn_initial_states += layer.rnn_initial_states
self.rnn_final_states += layer.rnn_final_states
self.rnn_zero_states += layer.rnn_zero_states
layer.add_summary_to_tg()
self.session = tf.Session(config=self.configproto)
self.built = True
# Ensure all training operators have been created.
self._get_tf('train_op')
for submodel in self.submodels:
train_op = submodel.get_train_op()
# Initialize variables.
self.session.run(tf.global_variables_initializer())
for layer in self.layers.values():
if layer.variable_values is not None:
variables = self.get_layer_variables(layer)
for var, val in zip(variables, layer.variable_values):
self.session.run(var.assign(val))
for layer in self.layers.values():
if layer.tensorboard:
self.tensorboard = True
tf.summary.scalar("loss", self.loss.out_tensor)
for layer in self.layers.values():
if layer.tensorboard:
tf.summary.tensor_summary(layer.name, layer.out_tensor)
if self.tensorboard:
writer = self._get_tf("FileWriter")
writer.add_graph(self._get_tf("Graph"))
writer.close()
# As a sanity check, make sure all tensors have the correct shape.
for layer in self.layers.values():
try:
assert list(layer.shape) == layer.out_tensor.get_shape().as_list(
), '%s: Expected shape %s does not match actual shape %s' % (
layer.name, layer.shape, layer.out_tensor.get_shape().as_list())
except NotImplementedError:
pass
def _predict(self, generator, transformers, outputs, uncertainty):
"""
Predict outputs for data provided by a generator.
This is the private implementation of prediction. Do not call it directly.
Instead call one of the public prediction methods.
Parameters
----------
generator: Generator
Generator that constructs feed dictionaries for TensorGraph.
transformers: list
List of dc.trans.Transformers.
outputs: object
If outputs is None, then will assume outputs = self.outputs.
If outputs is a Layer/Tensor, then will evaluate and return as a
single ndarray. If outputs is a list of Layers/Tensors, will return a list
of ndarrays.
uncertainty: bool
specifies whether this is being called as part of estimating uncertainty.
If True, it sets the training flag so that dropout will be enabled, and
returns the values of the uncertainty outputs.
Returns:
y_pred: numpy ndarray of shape (n_samples, n_classes*n_tasks)
"""
if not self.built:
self.build()
if outputs is None:
outputs = self.outputs
elif not isinstance(outputs, collections.Sequence):
outputs = [outputs]
if uncertainty:
if len(self.variances) == 0:
raise ValueError('This model cannot compute uncertainties')
if len(self.variances) != len(outputs):
raise ValueError(
'The number of variances must exactly match the number of outputs')
tensors = outputs + self.variances
else:
tensors = outputs
with self._get_tf("Graph").as_default():
# Gather results for each output
results = [[] for out in tensors]
n_samples = 0
n_enqueued = [0]
final_sample = [None]
if self.queue_installed:
enqueue_thread = threading.Thread(
target=_enqueue_batch,
args=(self, generator, self._get_tf("Graph"), self.session,
n_enqueued, final_sample))
enqueue_thread.start()
for feed_dict in self._create_feed_dicts(generator, uncertainty):
if self.queue_installed:
# Don't let this thread get ahead of the enqueue thread, since if
# we try to read more batches than the total number that get queued,
# this thread will hang indefinitely.
while n_enqueued[0] <= n_samples:
if n_samples == final_sample[0]:
break
time.sleep(0)
if n_samples == final_sample[0]:
break
n_samples += 1
feed_results = self._run_graph(tensors, feed_dict, uncertainty)
if tfe.in_eager_mode():
feed_results = [f.numpy() for f in feed_results]
if len(feed_results) > 1:
if len(transformers):
raise ValueError("Does not support transformations "
"for multiple outputs.")
elif len(feed_results) == 1:
result = undo_transforms(feed_results[0], transformers)
feed_results = [result]
for ind, result in enumerate(feed_results):
results[ind].append(result)
final_results = []
for result_list in results:
final_results.append(np.concatenate(result_list, axis=0))
# If only one output, just return array
if len(final_results) == 1:
return final_results[0]
elif uncertainty:
return zip(final_results[:len(outputs)], final_results[len(outputs):])
else:
return final_results
def fit_generator(self,
feed_dict_generator,
max_checkpoints_to_keep=5,
checkpoint_interval=1000,
restore=False,
submodel=None):
"""Train this model on data from a generator.
Parameters
----------
feed_dict_generator: generator
this should generate batches, each represented as a dict that maps
Layers to values.
max_checkpoints_to_keep: int
the maximum number of checkpoints to keep. Older checkpoints are discarded.
checkpoint_interval: int
the frequency at which to write checkpoints, measured in training steps.
Set this to 0 to disable automatic checkpointing.
restore: bool
if True, restore the model from the most recent checkpoint and continue training
from there. If False, retrain the model from scratch.
submodel: Submodel
an alternate training objective to use. This should have been created by
calling create_submodel().
Returns
-------
the average loss over the most recent checkpoint interval
"""
if not self.built:
self.build()
with self._get_tf("Graph").as_default():
time1 = time.time()
loss = self.loss
if submodel is not None and submodel.loss is not None:
loss = submodel.loss
if tfe.in_eager_mode():
# In eager mode we want an optimizer and a function to compute the
# gradient of the loss.
submodel_vars = None
if submodel is None:
optimizer = self._get_tf("Optimizer")
else:
optimizer = submodel.create_optimizer()
if submodel.layers is not None:
submodel_vars = set()
for layer in submodel.layers:
for var in layer.variables:
submodel_vars.add(var)
val_grad_fn = tfe.implicit_value_and_gradients(
lambda x: self._run_graph([loss], x, True)[0])
else:
# In graph mode we want a training operation.
if submodel is None:
train_op = self._get_tf('train_op')
else:
train_op = submodel.get_train_op()
if checkpoint_interval > 0:
saver = tf.train.Saver(
self.get_variables(),
max_to_keep=max_checkpoints_to_keep,
save_relative_paths=True)
if restore:
self.restore()
avg_loss, n_averaged_batches = 0.0, 0.0
n_samples = 0
n_enqueued = [0]
final_sample = [None]
if self.queue_installed:
enqueue_thread = threading.Thread(
target=_enqueue_batch,
args=(self, feed_dict_generator, self._get_tf("Graph"),
self.session, n_enqueued, final_sample))
enqueue_thread.start()
for feed_dict in self._create_feed_dicts(feed_dict_generator, True):
if self.queue_installed:
# Don't let this thread get ahead of the enqueue thread, since if
# we try to read more batches than the total number that get queued,
# this thread will hang indefinitely.
while n_enqueued[0] <= n_samples:
if n_samples == final_sample[0]:
break
time.sleep(0)
if n_samples == final_sample[0]:
break
n_samples += 1
should_log = (self.tensorboard and
n_samples % self.tensorboard_log_frequency == 0)
if tfe.in_eager_mode():
value, grads_and_vars = val_grad_fn(feed_dict)
if submodel_vars is not None:
grads_and_vars = [
x for x in grads_and_vars if x[1] in submodel_vars
]
optimizer.apply_gradients(grads_and_vars)
avg_loss += value
else:
fetches = [train_op, loss.out_tensor]
if should_log:
fetches.append(self._get_tf("summary_op"))
fetched_values = self.session.run(fetches, feed_dict=feed_dict)
if should_log:
self._log_tensorboard(fetched_values[2])
avg_loss += fetched_values[1]
n_averaged_batches += 1
self.global_step += 1
if checkpoint_interval > 0 and self.global_step % checkpoint_interval == checkpoint_interval - 1:
saver.save(self.session, self.save_file, global_step=self.global_step)
avg_loss = float(avg_loss) / n_averaged_batches
logger.info('Ending global_step %d: Average loss %g' %
(self.global_step, avg_loss))
avg_loss, n_averaged_batches = 0.0, 0.0
if n_averaged_batches > 0:
avg_loss = float(avg_loss) / n_averaged_batches
if checkpoint_interval > 0:
if n_averaged_batches > 0:
logger.info('Ending global_step %d: Average loss %g' %
(self.global_step, avg_loss))
saver.save(self.session, self.save_file, global_step=self.global_step)
time2 = time.time()
logger.info("TIMING: model fitting took %0.3f s" % (time2 - time1))
return avg_loss
def make_estimator(self,
feature_columns,
weight_column=None,
metrics={},
model_dir=None,
config=None):
"""Construct a Tensorflow Estimator from this model.
tf.estimator.Estimator is the standard Tensorflow API for representing models.
This method provides interoperability between DeepChem and other Tensorflow
based tools by allowing any model to be used an Estimator.
Once this method returns, the Estimator it created is independent of the model
it was created from. They do not share tensors, variables, save files, or any
other resources. The Estimator is a self contained object with its own methods
for training, evaluation, prediction, checkpointing, etc.
Parameters
----------
feature_columns: list of tf.feature_column objects
this describes the input features to the models. There must be one entry
for each Feature layer in this model's features field.
weight_column: tf.feature_column or None
if this model includes a Weights layer, this describes the input weights.
Otherwise, this should be None.
metrics: map
metrics that should be computed in calls to evaluate(). For each entry,
the key is the name to report for the metric, and the value is a function
of the form f(labels, predictions, weights) that returns the tensors for
computing the metric. Any of the functions in tf.metrics can be used, as
can other functions that satisfy the same interface.
model_dir: str
the directory in which the Estimator should save files. If None, this
defaults to the model's model_dir.
config: RunConfig
configuration options for the Estimator
"""
# Check the inputs.
if tfe.in_eager_mode():
raise ValueError('make_estimator() is not supported in eager mode')
if len(feature_columns) != len(self.features):
raise ValueError(
'This model requires %d feature column(s)' % len(self.features))
if len(self.labels) != 1:
raise ValueError(
'Can only create an Estimator from a model with exactly one Label input'
)
if len(self.task_weights) > 1:
raise ValueError(
'Cannot create an Estimator from a model with multiple Weight inputs')
if weight_column is None:
if len(self.task_weights) > 0:
raise ValueError('This model requires a weight column')
else:
if len(self.task_weights) == 0:
raise ValueError(
'Cannot specify weight_column for a model with no Weight inputs')
if model_dir is None:
model_dir = self.model_dir
# Define a function that recursively creates tensors from layers.
def create_tensors(layer, tensors, training):
if layer in tensors:
return tensors[layer]
inputs = [
create_tensors(in_layer, tensors, training)
for in_layer in layer.in_layers
]
tensor = layer.create_tensor(
in_layers=inputs, set_tensors=False, training=training)
tensors[layer] = tensor
layer.add_summary_to_tg(tensor)
return tensor
# Define the model function.
def model_fn(features, labels, mode):
# Define the inputs.
tensors = self.create_estimator_inputs(feature_columns, weight_column,
features, labels, mode)
for layer, tensor in tensors.items():
layer.add_summary_to_tg(tensor)
# Create the correct outputs, based on the mode.
if mode == tf.estimator.ModeKeys.PREDICT:
predictions = {}
for i, output in enumerate(self.outputs):
predictions[i] = create_tensors(output, tensors, 0)
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
if mode == tf.estimator.ModeKeys.EVAL:
loss = create_tensors(self.loss, tensors, 0)
predictions = create_tensors(self.outputs[0], tensors, 0)
if len(self.task_weights) == 0:
weights = None
else:
weights = tensors[self.task_weights[0]]
eval_metric_ops = {}
for name, function in metrics.items():
eval_metric_ops[name] = function(tensors[self.labels[0]], predictions,
weights)
return tf.estimator.EstimatorSpec(
mode, loss=loss, eval_metric_ops=eval_metric_ops)
if mode == tf.estimator.ModeKeys.TRAIN:
loss = create_tensors(self.loss, tensors, 1)
global_step = tf.train.get_global_step()
optimizer = self.optimizer._create_optimizer(global_step)
train_op = optimizer.minimize(loss, global_step=global_step)
return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)
raise ValueError('Unknown mode')
# Create the Estimator.
return tf.estimator.Estimator(
model_fn=model_fn, model_dir=model_dir, config=config)
def _predict(self, generator, transformers, outputs, uncertainty):
"""
Predict outputs for data provided by a generator.
This is the private implementation of prediction. Do not call it directly.
Instead call one of the public prediction methods.
Parameters
----------
generator: Generator
Generator that constructs feed dictionaries for TensorGraph.
transformers: list
List of dc.trans.Transformers.
outputs: object
If outputs is None, then will assume outputs = self.outputs.
If outputs is a Layer/Tensor, then will evaluate and return as a
single ndarray. If outputs is a list of Layers/Tensors, will return a list
of ndarrays.
uncertainty: bool
specifies whether this is being called as part of estimating uncertainty.
If True, it sets the training flag so that dropout will be enabled, and
returns the values of the uncertainty outputs.
Returns:
y_pred: numpy ndarray of shape (n_samples, n_classes*n_tasks)
"""
if not self.built:
self.build()
if outputs is None:
outputs = self.outputs
elif not isinstance(outputs, collections.Sequence):
outputs = [outputs]
if uncertainty:
if len(self.variances) == 0:
raise ValueError('This model cannot compute uncertainties')
if len(self.variances) != len(outputs):
raise ValueError(
'The number of variances must exactly match the number of outputs')
tensors = outputs + self.variances
else:
tensors = outputs
with self._get_tf("Graph").as_default():
# Gather results for each output
results = [[] for out in tensors]
n_samples = 0
n_enqueued = [0]
final_sample = [None]
if self.queue_installed:
enqueue_thread = threading.Thread(
target=_enqueue_batch,
args=(self, generator, self._get_tf("Graph"), self.session,
n_enqueued, final_sample))
enqueue_thread.start()
for feed_dict in self._create_feed_dicts(generator, uncertainty):
if self.queue_installed:
# Don't let this thread get ahead of the enqueue thread, since if
# we try to read more batches than the total number that get queued,
# this thread will hang indefinitely.
while n_enqueued[0] <= n_samples:
if n_samples == final_sample[0]:
break
time.sleep(0)
if n_samples == final_sample[0]:
break
n_samples += 1
feed_results = self._run_graph(tensors, feed_dict, uncertainty)
if tfe.in_eager_mode():
feed_results = [f.numpy() for f in feed_results]
if len(feed_results) > 1:
if len(transformers):
raise ValueError("Does not support transformations "
"for multiple outputs.")
elif len(feed_results) == 1:
result = undo_transforms(feed_results[0], transformers)
feed_results = [result]
for ind, result in enumerate(feed_results):
results[ind].append(result)
final_results = []
for result_list in results:
final_results.append(np.concatenate(result_list, axis=0))
# If only one output, just return array
if len(final_results) == 1:
return final_results[0]
elif uncertainty:
return zip(final_results[:len(outputs)], final_results[len(outputs):])
else:
return final_results
def make_estimator(self,
feature_columns,
weight_column=None,
metrics={},
model_dir=None,
config=None):
"""Construct a Tensorflow Estimator from this model.
tf.estimator.Estimator is the standard Tensorflow API for representing models.
This method provides interoperability between DeepChem and other Tensorflow
based tools by allowing any model to be used an Estimator.
Once this method returns, the Estimator it created is independent of the model
it was created from. They do not share tensors, variables, save files, or any
other resources. The Estimator is a self contained object with its own methods
for training, evaluation, prediction, checkpointing, etc.
Parameters
----------
feature_columns: list of tf.feature_column objects
this describes the input features to the models. There must be one entry
for each Feature layer in this model's features field.
weight_column: tf.feature_column or None
if this model includes a Weights layer, this describes the input weights.
Otherwise, this should be None.
metrics: map
metrics that should be computed in calls to evaluate(). For each entry,
the key is the name to report for the metric, and the value is a function
of the form f(labels, predictions, weights) that returns the tensors for
computing the metric. Any of the functions in tf.metrics can be used, as
can other functions that satisfy the same interface.
model_dir: str
the directory in which the Estimator should save files. If None, this
defaults to the model's model_dir.
config: RunConfig
configuration options for the Estimator
"""
# Check the inputs.
if tfe.in_eager_mode():
raise ValueError('make_estimator() is not supported in eager mode')
if len(feature_columns) != len(self.features):
raise ValueError(
'This model requires %d feature column(s)' % len(self.features))
if len(self.labels) != 1:
raise ValueError(
'Can only create an Estimator from a model with exactly one Label input'
)
if len(self.task_weights) > 1:
raise ValueError(
'Cannot create an Estimator from a model with multiple Weight inputs')
if weight_column is None:
if len(self.task_weights) > 0:
raise ValueError('This model requires a weight column')
else:
if len(self.task_weights) == 0:
raise ValueError(
'Cannot specify weight_column for a model with no Weight inputs')
if model_dir is None:
model_dir = self.model_dir
# Define a function that recursively creates tensors from layers.
def create_tensors(layer, tensors, training):
if layer in tensors:
return tensors[layer]
inputs = [
create_tensors(in_layer, tensors, training)
for in_layer in layer.in_layers
]
tensor = layer.create_tensor(
in_layers=inputs, set_tensors=False, training=training)
tensors[layer] = tensor
vars = tf.get_collection(
tf.GraphKeys.TRAINABLE_VARIABLES, scope=layer.name)
layer.add_summary_to_tg(tensor, vars)
return tensor
# Define the model function.
def model_fn(features, labels, mode):
# Define the inputs.
tensors = self.create_estimator_inputs(feature_columns, weight_column,
features, labels, mode)
for layer, tensor in tensors.items():
layer.add_summary_to_tg(tensor, [])
# Create the correct outputs, based on the mode.
if mode == tf.estimator.ModeKeys.PREDICT:
predictions = {}
for i, output in enumerate(self.outputs):
predictions[i] = create_tensors(output, tensors, 0)
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
if mode == tf.estimator.ModeKeys.EVAL:
loss = create_tensors(self.loss, tensors, 0)
predictions = create_tensors(self.outputs[0], tensors, 0)
if len(self.task_weights) == 0:
weights = None
else:
weights = tensors[self.task_weights[0]]
eval_metric_ops = {}
for name, function in metrics.items():
eval_metric_ops[name] = function(tensors[self.labels[0]], predictions,
weights)
return tf.estimator.EstimatorSpec(
mode, loss=loss, eval_metric_ops=eval_metric_ops)
if mode == tf.estimator.ModeKeys.TRAIN:
loss = create_tensors(self.loss, tensors, 1)
global_step = tf.train.get_global_step()
optimizer = self.optimizer._create_optimizer(global_step)
train_op = optimizer.minimize(loss, global_step=global_step)
return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)
raise ValueError('Unknown mode')
# Create the Estimator.
return tf.estimator.Estimator(
model_fn=model_fn, model_dir=model_dir, config=config)
def create_variable(value, dtype=None, name=None, trainable=True):
"""Create a tf.Variable or tfe.Variable, depending on the current mode."""
if tfe.in_eager_mode():
return tfe.Variable(value, dtype=dtype, name=name, trainable=trainable)
else:
return tf.Variable(value, dtype=dtype, name=name, trainable=trainable)
def build(self):
if self.built:
return
if tfe.in_eager_mode():
# In eager mode, we need to execute every layer once to ensure its variables
# have been created.
def build_layers(layer, tensors):
if layer in tensors:
return tensors[layer]
inputs = [build_layers(input, tensors) for input in layer.in_layers]
if isinstance(layer, Input):
# We can't execute Input layers in eager mode, since they would try
# to create placeholders. Instead create a tensor of the correct
# size and type.
shape = [1 if s is None else s for s in layer.shape]
tensor = tf.zeros(shape, layer.dtype)
else:
with tf.name_scope(layer.name):
tensor = layer.create_tensor(in_layers=inputs, set_tensors=False)
tensors[layer] = tensor
return tensor
tensors = {}
with self._get_tf("Graph").as_default():
# Build the layers.
build_layers(self.loss, tensors)
for output in self.outputs:
build_layers(output, tensors)
for variance in self.variances:
build_layers(variance, tensors)
for submodel in self.submodels:
build_layers(submodel.loss, tensors)
# Initialize variables.
for layer in self.layers.values():
if layer.variable_values is not None:
for var, val in zip(layer.variables, layer.variable_values):
var.assign(val)
self.session = None
self._training_placeholder = None
self.built = True
return
# In graph mode we need to create the computation graph.
with self._get_tf("Graph").as_default():
self._training_placeholder = tf.placeholder(dtype=tf.float32, shape=())
if self.random_seed is not None:
tf.set_random_seed(self.random_seed)
self._install_queue()
self.built = True
for layer in self.topsort():
with tf.name_scope(layer.name):
layer.create_tensor(training=self._training_placeholder)
self.rnn_initial_states += layer.rnn_initial_states
self.rnn_final_states += layer.rnn_final_states
self.rnn_zero_states += layer.rnn_zero_states
layer.add_summary_to_tg(layer.out_tensor,
self.get_layer_variables(layer))
self.session = tf.Session(config=self.configproto)
# Ensure all training operators have been created.
self._get_tf('train_op')
for submodel in self.submodels:
train_op = submodel.get_train_op()
# Initialize variables.
self.session.run(tf.global_variables_initializer())
for layer in self.layers.values():
if layer.variable_values is not None:
variables = self.get_layer_variables(layer)
for var, val in zip(variables, layer.variable_values):
self.session.run(var.assign(val))
for layer in self.layers.values():
if layer.tensorboard:
self.tensorboard = True
tf.summary.scalar("loss", self.loss.out_tensor)
for layer in self.layers.values():
if layer.tensorboard:
tf.summary.tensor_summary(layer.name, layer.out_tensor)
if self.tensorboard:
writer = self._get_tf("FileWriter")
writer.add_graph(self._get_tf("Graph"))
writer.close()
# As a sanity check, make sure all tensors have the correct shape.
for layer in self.layers.values():
try:
assert list(layer.shape) == layer.out_tensor.get_shape().as_list(
), '%s: Expected shape %s does not match actual shape %s' % (
layer.name, layer.shape, layer.out_tensor.get_shape().as_list())
except NotImplementedError:
pass
