def test_tensorboard(self):
"""Test creating an Estimator from a TensorGraph that logs information to TensorBoard."""
n_samples = 10
n_features = 3
n_tasks = 2
# Create a dataset and an input function for processing it.
np.random.seed(123)
X = np.random.rand(n_samples, n_features)
y = np.zeros((n_samples, n_tasks))
dataset = dc.data.NumpyDataset(X, y)
def input_fn(epochs):
x, y, weights = dataset.make_iterator(batch_size=n_samples,
epochs=epochs).get_next()
return {'x': x, 'weights': weights}, y
# Create a TensorGraph model.
model = dc.models.TensorGraph()
features = layers.Feature(shape=(None, n_features))
dense = layers.Dense(out_channels=n_tasks, in_layers=features)
dense.set_summary('histogram')
model.add_output(dense)
labels = layers.Label(shape=(None, n_tasks))
loss = layers.ReduceMean(layers.L2Loss(in_layers=[labels, dense]))
model.set_loss(loss)
# Create an estimator from it.
x_col = tf.feature_column.numeric_column('x', shape=(n_features, ))
estimator = model.make_estimator(feature_columns=[x_col])
# Train the model.
estimator.train(input_fn=lambda: input_fn(100))
def __init__(self,
input_tokens,
output_tokens,
max_output_length,
encoder_layers=4,
decoder_layers=4,
embedding_dimension=512,
dropout=0.0,
reverse_input=True,
variational=False,
annealing_start_step=5000,
annealing_final_step=10000,
**kwargs):
"""Construct a SeqToSeq model.
In addition to the following arguments, this class also accepts all the keyword arguments
from TensorGraph.
Parameters
----------
input_tokens: list
a list of all tokens that may appear in input sequences
output_tokens: list
a list of all tokens that may appear in output sequences
max_output_length: int
the maximum length of output sequence that may be generated
encoder_layers: int
the number of recurrent layers in the encoder
decoder_layers: int
the number of recurrent layers in the decoder
embedding_dimension: int
the width of the embedding vector. This also is the width of all
recurrent layers.
dropout: float
the dropout probability to use during training
reverse_input: bool
if True, reverse the order of input sequences before sending them into
the encoder. This can improve performance when working with long sequences.
variational: bool
if True, train the model as a variational autoencoder. This adds random
noise to the encoder, and also constrains the embedding to follow a unit
Gaussian distribution.
annealing_start_step: int
the step (that is, batch) at which to begin turning on the constraint term
for KL cost annealing
annealing_final_step: int
the step (that is, batch) at which to finish turning on the constraint term
for KL cost annealing
"""
super(SeqToSeq, self).__init__(
use_queue=False, **kwargs) # TODO can we make it work with the queue?
if SeqToSeq.sequence_end not in input_tokens:
input_tokens = input_tokens + [SeqToSeq.sequence_end]
if SeqToSeq.sequence_end not in output_tokens:
output_tokens = output_tokens + [SeqToSeq.sequence_end]
self._input_tokens = input_tokens
self._output_tokens = output_tokens
self._input_dict = dict((x, i) for i, x in enumerate(input_tokens))
self._output_dict = dict((x, i) for i, x in enumerate(output_tokens))
self._max_output_length = max_output_length
self._embedding_dimension = embedding_dimension
self._annealing_final_step = annealing_final_step
self._annealing_start_step = annealing_start_step
self._features = self._create_features()
self._labels = layers.Label(shape=(None, None, len(output_tokens)))
self._gather_indices = layers.Feature(
shape=(self.batch_size, 2), dtype=tf.int32)
self._reverse_input = reverse_input
self._variational = variational
self.embedding = self._create_encoder(encoder_layers, dropout)
self.output = self._create_decoder(decoder_layers, dropout)
self.set_loss(self._create_loss())
self.add_output(self.output)
def _create_features(self):
return layers.Feature(
shape=(self.batch_size, self._max_output_length,
len(self._input_tokens)))
def _create_features(self): return layers.Feature(shape=(None, None, len(self._input_tokens)))
# Train a model to predict how well sequences will work for RNA interference.
import deepchem as dc
import deepchem.models.tensorgraph.layers as layers
import tensorflow as tf
import matplotlib.pyplot as plot
# Build the model.
model = dc.models.TensorGraph(model_dir='rnai')
features = layers.Feature(shape=(None, 21, 4))
labels = layers.Label(shape=(None, 1))
prev = features
for i in range(2):
prev = layers.Conv1D(filters=10,
kernel_size=10,
activation=tf.nn.relu,
padding='same',
in_layers=prev)
prev = layers.Dropout(dropout_prob=0.3, in_layers=prev)
output = layers.Dense(out_channels=1,
activation_fn=tf.sigmoid,
in_layers=layers.Flatten(prev))
model.add_output(output)
loss = layers.ReduceMean(layers.L2Loss(in_layers=[labels, output]))
model.set_loss(loss)
# Load the data.
train = dc.data.DiskDataset('train_siRNA')
valid = dc.data.DiskDataset('valid_siRNA')
if f.endswith('.TIF'):
for row, blur in zip(rows, blurs):
fname = f.replace('_F1', '_F%d' % blur).replace('_A', '_%s' % row)
files.append(os.path.join(image_dir, fname))
labels.append(os.path.join(label_dir, f))
loader = dc.data.ImageLoader()
dataset = loader.featurize(files, labels)
splitter = dc.splits.RandomSplitter()
train_dataset, valid_dataset, test_dataset = splitter.train_valid_test_split(
dataset, seed=123)
# Create the model.
learning_rate = dc.models.optimizers.ExponentialDecay(0.01, 0.9, 250)
model = dc.models.TensorGraph(learning_rate=learning_rate,
model_dir='models/segmentation')
features = layers.Feature(shape=(None, 520, 696, 1)) / 255.0
labels = layers.Label(shape=(None, 520, 696, 1)) / 255.0
# Downsample three times.
conv1 = layers.Conv2D(16, kernel_size=5, stride=2, in_layers=features)
conv2 = layers.Conv2D(32, kernel_size=5, stride=2, in_layers=conv1)
conv3 = layers.Conv2D(64, kernel_size=5, stride=2, in_layers=conv2)
# Do a 1x1 convolution.
conv4 = layers.Conv2D(64, kernel_size=1, stride=1, in_layers=conv3)
# Upsample three times.
concat1 = layers.Concat(in_layers=[conv3, conv4], axis=3)
deconv1 = layers.Conv2DTranspose(32,
kernel_size=5,
stride=2,
in_layers=concat1)
concat2 = layers.Concat(in_layers=[conv2, deconv1], axis=3)
deconv2 = layers.Conv2DTranspose(16,
def __init__(self, **kwargs):
"""Construct a GAN.
This class accepts all the keyword arguments from TensorGraph.
"""
super(GAN, self).__init__(use_queue=False, **kwargs)
# Create the inputs.
self.noise_input = layers.Feature(shape=self.get_noise_input_shape())
self.data_inputs = []
for shape in self.get_data_input_shapes():
self.data_inputs.append(layers.Feature(shape=shape))
self.conditional_inputs = []
for shape in self.get_conditional_input_shapes():
self.conditional_inputs.append(layers.Feature(shape=shape))
# Create the generator.
self.generator = self.create_generator(self.noise_input,
self.conditional_inputs)
if not isinstance(self.generator, Sequence):
raise ValueError('create_generator() must return a list of Layers')
if len(self.generator) != len(self.data_inputs):
raise ValueError(
'The number of generator outputs must match the number of data inputs'
)
for g, d in zip(self.generator, self.data_inputs):
if g.shape != d.shape:
raise ValueError(
'The shapes of the generator outputs must match the shapes of the data inputs'
)
for g in self.generator:
self.add_output(g)
# Create the discriminator.
self.discrim_train = self.create_discriminator(self.data_inputs,
self.conditional_inputs)
# Make a copy of the discriminator that takes the generator's output as
# its input.
replacements = {}
for g, d in zip(self.generator, self.data_inputs):
replacements[d] = g
for c in self.conditional_inputs:
replacements[c] = c
self.discrim_gen = self.discrim_train.copy(replacements, shared=True)
# Make a list of all layers in the generator and discriminator.
def add_layers_to_set(layer, layers):
if layer not in layers:
layers.add(layer)
for i in layer.in_layers:
add_layers_to_set(i, layers)
gen_layers = set()
for layer in self.generator:
add_layers_to_set(layer, gen_layers)
discrim_layers = set()
add_layers_to_set(self.discrim_train, discrim_layers)
discrim_layers -= gen_layers
# Create submodels for training the generator and discriminator.
gen_loss = self.create_generator_loss(self.discrim_gen)
discrim_loss = self.create_discriminator_loss(self.discrim_train,
self.discrim_gen)
self.generator_submodel = self.create_submodel(
layers=gen_layers, loss=gen_loss)
self.discriminator_submodel = self.create_submodel(
layers=discrim_layers, loss=discrim_loss)
def __init__(self, n_generators=1, n_discriminators=1, **kwargs):
"""Construct a GAN.
In addition to the parameters listed below, this class accepts all the
keyword arguments from TensorGraph.
Parameters
----------
n_generators: int
the number of generators to include
n_discriminators: int
the number of discriminators to include
"""
super(GAN, self).__init__(use_queue=False, **kwargs)
self.n_generators = n_generators
self.n_discriminators = n_discriminators
# Create the inputs.
self.noise_input = layers.Feature(shape=self.get_noise_input_shape())
self.data_inputs = []
for shape in self.get_data_input_shapes():
self.data_inputs.append(layers.Feature(shape=shape))
self.conditional_inputs = []
for shape in self.get_conditional_input_shapes():
self.conditional_inputs.append(layers.Feature(shape=shape))
# Create the generators.
self.generators = []
for i in range(n_generators):
generator = self.create_generator(self.noise_input,
self.conditional_inputs)
if not isinstance(generator, Sequence):
raise ValueError(
'create_generator() must return a list of Layers')
if len(generator) != len(self.data_inputs):
raise ValueError(
'The number of generator outputs must match the number of data inputs'
)
for g, d in zip(generator, self.data_inputs):
if g.shape != d.shape:
raise ValueError(
'The shapes of the generator outputs must match the shapes of the data inputs'
)
for g in generator:
self.add_output(g)
self.generators.append(generator)
# Create the discriminators.
self.discrim_train = []
self.discrim_gen = []
for i in range(n_discriminators):
discrim_train = self.create_discriminator(self.data_inputs,
self.conditional_inputs)
self.discrim_train.append(discrim_train)
# Make a copy of the discriminator that takes each generator's output as
# its input.
for generator in self.generators:
replacements = {}
for g, d in zip(generator, self.data_inputs):
replacements[d] = g
for c in self.conditional_inputs:
replacements[c] = c
discrim_gen = discrim_train.copy(replacements, shared=True)
self.discrim_gen.append(discrim_gen)
# Make a list of all layers in the generators and discriminators.
def add_layers_to_set(layer, layers):
if layer not in layers:
layers.add(layer)
for i in layer.in_layers:
add_layers_to_set(i, layers)
gen_layers = set()
for generator in self.generators:
for layer in generator:
add_layers_to_set(layer, gen_layers)
discrim_layers = set()
for discriminator in self.discrim_train:
add_layers_to_set(discriminator, discrim_layers)
discrim_layers -= gen_layers
# Compute the loss functions.
gen_losses = [self.create_generator_loss(d) for d in self.discrim_gen]
discrim_losses = []
for i in range(n_discriminators):
for j in range(n_generators):
discrim_losses.append(
self.create_discriminator_loss(
self.discrim_train[i],
self.discrim_gen[i * n_generators + j]))
if n_generators == 1 and n_discriminators == 1:
total_gen_loss = gen_losses[0]
total_discrim_loss = discrim_losses[0]
else:
# Create learnable weights for the generators and discriminators.
gen_alpha = layers.Variable(np.ones((1, n_generators)))
gen_weights = layers.SoftMax(gen_alpha)
discrim_alpha = layers.Variable(np.ones((1, n_discriminators)))
discrim_weights = layers.SoftMax(discrim_alpha)
# Compute the weighted errors
weight_products = layers.Reshape(
(n_generators * n_discriminators, ),
in_layers=layers.Reshape(
(n_discriminators, 1), in_layers=discrim_weights) *
layers.Reshape((1, n_generators), in_layers=gen_weights))
total_gen_loss = layers.WeightedError(
(layers.Stack(gen_losses, axis=0), weight_products))
total_discrim_loss = layers.WeightedError(
(layers.Stack(discrim_losses, axis=0), weight_products))
gen_layers.add(gen_alpha)
discrim_layers.add(gen_alpha)
discrim_layers.add(discrim_alpha)
# Add an entropy term to the loss.
entropy = -(layers.ReduceSum(layers.Log(gen_weights)) /
n_generators + layers.ReduceSum(
layers.Log(discrim_weights)) / n_discriminators)
total_discrim_loss += entropy
# Create submodels for training the generators and discriminators.
self.generator_submodel = self.create_submodel(layers=gen_layers,
loss=total_gen_loss)
self.discriminator_submodel = self.create_submodel(
layers=discrim_layers, loss=total_discrim_loss)
# Train a model to predict transcription factor binding, based on both
# sequence and chromatin accessibility.
import deepchem as dc
import deepchem.models.tensorgraph.layers as layers
import tensorflow as tf
import numpy as np
# Build the model.
model = dc.models.TensorGraph(batch_size=1000, model_dir='chromatin')
features = layers.Feature(shape=(None, 101, 4))
accessibility = layers.Feature(shape=(None, 1))
labels = layers.Label(shape=(None, 1))
weights = layers.Weights(shape=(None, 1))
prev = features
for i in range(3):
prev = layers.Conv1D(filters=15, kernel_size=10, activation=tf.nn.relu, padding='same', in_layers=prev)
prev = layers.Dropout(dropout_prob=0.5, in_layers=prev)
prev = layers.Concat([layers.Flatten(prev), accessibility])
logits = layers.Dense(out_channels=1, in_layers=prev)
output = layers.Sigmoid(logits)
model.add_output(output)
loss = layers.SigmoidCrossEntropy(in_layers=[labels, logits])
weighted_loss = layers.WeightedError(in_layers=[loss, weights])
model.set_loss(weighted_loss)
# Load the data.
train = dc.data.DiskDataset('train_dataset')
valid = dc.data.DiskDataset('valid_dataset')
import tensorflow as tf
import deepchem.models.tensorgraph.layers as layers
from tensorflow.examples.tutorials.mnist import input_data
# Read dataset
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
# Transfor dataset into format readable by deepchem
train_dataset = dc.data.NumpyDataset(mnist.train.images, mnist.train.labels)
test_dataset = dc.data.NumpyDataset(mnist.test.images, mnist.test.labels)
model = dc.models.TensorGraph(model_dir='mnist')
# Images in MNIST are 28x28, flattened 784
# None means that the input can be of any dimension - we can use it as variable batch size
feature = layers.Feature(shape=(None, 784))
# 0..9 digits
label = layers.Label(shape=(None, 10))
# Reshape flattened layer to matrix to use it with convolution
make_image = layers.Reshape(shape=(None, 28, 28), in_layers=feature)
conv2d_1 = layers.Conv2D(num_outputs=32,
activation_fn=tf.nn.relu,
in_layers=make_image)
conv2d_2 = layers.Conv2D(num_outputs=64,
activation_fn=tf.nn.relu,
in_layers=conv2d_1)
flatten = layers.Flatten(in_layers=conv2d_2)
dense1 = layers.Dense(out_channels=1024,
activation_fn=tf.nn.relu,
image_dir = 'BBBC005_v1_images'
files = []
labels = []
for f in os.listdir(image_dir):
if f.endswith('.TIF'):
files.append(os.path.join(image_dir, f))
labels.append(int(re.findall('_C(.*?)_', f)[0]))
loader = dc.data.ImageLoader()
dataset = loader.featurize(files, np.array(labels))
splitter = dc.splits.RandomSplitter()
train_dataset, valid_dataset, test_dataset = splitter.train_valid_test_split(dataset, seed=123)
# Create the model.
learning_rate = dc.models.tensorgraph.optimizers.ExponentialDecay(0.001, 0.9, 250)
model = dc.models.TensorGraph(learning_rate=learning_rate, model_dir='models/model')
features = layers.Feature(shape=(None, 520, 696))
labels = layers.Label(shape=(None,))
prev_layer = features
for num_outputs in [16, 32, 64, 128, 256]:
prev_layer = layers.Conv2D(num_outputs, kernel_size=5, stride=2, in_layers=prev_layer)
output = layers.Dense(1, in_layers=layers.Flatten(prev_layer))
model.add_output(output)
loss = layers.ReduceSum(layers.L2Loss(in_layers=(output, labels)))
model.set_loss(loss)
if not os.path.exists('./models'):
os.mkdir('models')
if not os.path.exists('./models/model'):
os.mkdir('models/model')
if not RETRAIN:
def create_model():
"""
Create our own MNIST model from scratch
:return:
:rtype:
"""
mnist = input_data.read_data_sets("MNIST_DATA/", one_hot=True)
# the layers from deepchem are the building blocks of what we will use to make our deep learning architecture
# now we wrap our dataset into a NumpyDataset
train_dataset = dc.data.NumpyDataset(mnist.train.images,
mnist.train.labels)
test_dataset = dc.data.NumpyDataset(mnist.test.images, mnist.test.labels)
# we will create a model that will take an input, add multiple layers, where each layer takes input from the
# previous layers.
model = dc.models.TensorGraph(model_dir='mnist')
# 784 corresponds to an image of size 28 X 28
# 10 corresponds to the fact that there are 10 possible digits (0-9)
# the None indicates that we can accept any size input (e.g. an empty array or 500 items each with 784 features)
# our data is also categorical so we must one hot encode, set single array element to 1 and the rest to 0
feature = layers.Feature(shape=(None, 784))
labels = layers.Label(shape=(None, 10))
# in order to apply convolutional layers to our input, we convert flat vector of 785 to 28X28
# in_layers means it takes our feature layer as an input
make_image = layers.Reshape(shape=(None, 28, 28), in_layers=feature)
# now that we have reshaped the input, we pass to convolution layers
conv2d_1 = layers.Conv2D(num_outputs=32,
activation_fn=tf.nn.relu,
in_layers=make_image)
conv2d_2 = layers.Conv2D(num_outputs=64,
activation_fn=tf.nn.relu,
in_layers=conv2d_1)
# we want to end by applying fully connected (Dense) layers to the outputs of our convolutional layer
# but first, we must flatten the layer from a 2d matrix to a 1d vector
flatten = layers.Flatten(in_layers=conv2d_2)
dense1 = layers.Dense(out_channels=1024,
activation_fn=tf.nn.relu,
in_layers=flatten)
# note that this is final layer so out_channels of 10 represents the 10 outputs and no activation_fn
dense2 = layers.Dense(out_channels=10,
activation_fn=None,
in_layers=dense1)
# next we want to connect this output to a loss function, so we can train the output
# compute the value of loss function for every sample then average of all samples to get final loss (ReduceMean)
smce = layers.SoftMaxCrossEntropy(in_layers=[labels, dense2])
loss = layers.ReduceMean(in_layers=smce)
model.set_loss(loss)
# for MNIST we want the probability that a given sample represents one of the 10 digits
# we can achieve this using a softmax function to get the probabilities, then cross entropy to get the labels
output = layers.SoftMax(in_layers=dense2)
model.add_output(output)
# if our model takes long to train, reduce nb_epoch to 1
model.fit(train_dataset, nb_epoch=10)
# our metric is accuracy, the fraction of labels that are accurately predicted
metric = dc.metrics.Metric(dc.metrics.accuracy_score)
train_scores = model.evaluate(train_dataset, [metric])
test_scores = model.evaluate(test_dataset, [metric])
print('train_scores', train_scores)
print('test_scores', test_scores)
