def test_reduce_mean(self):
"""Test that ReduceMean can be invoked."""
batch_size = 10
n_features = 5
in_tensor = np.random.rand(batch_size, n_features)
with self.session() as sess:
in_tensor = tf.convert_to_tensor(in_tensor, dtype=tf.float32)
out_tensor = ReduceMean()(in_tensor)
out_tensor = out_tensor.eval()
assert isinstance(out_tensor, np.float32)
def test_single_task_classifier(self):
n_data_points = 20
n_features = 2
X = np.random.rand(n_data_points, n_features)
y = [[0, 1] for x in range(n_data_points)]
dataset = NumpyDataset(X, y)
features = Feature(shape=(None, n_features))
dense = Dense(out_channels=2, in_layers=[features])
output = SoftMax(in_layers=[dense])
label = Label(shape=(None, 2))
smce = SoftMaxCrossEntropy(in_layers=[label, dense])
loss = ReduceMean(in_layers=[smce])
tg = dc.models.TensorGraph(learning_rate=0.01)
tg.add_output(output)
tg.set_loss(loss)
tg.fit(dataset, nb_epoch=1000)
prediction = np.squeeze(tg.predict_on_batch(X))
assert_true(np.all(np.isclose(prediction, y, atol=0.4)))
def test_invoke_model_eager(self):
"""Test invoking the model with __call__() in eager mode."""
with context.eager_mode():
with tfe.IsolateTest():
batch_size = 5
tg = dc.models.TensorGraph(batch_size=batch_size)
features = Feature(shape=(None, 10))
dense = Dense(10, in_layers=features)
loss = ReduceMean(in_layers=dense)
tg.add_output(dense)
tg.set_loss(loss)
input = np.random.rand(batch_size, 10).astype(np.float32)
# We should get the same result with either predict_on_batch() or __call__().
output1 = tg.predict_on_batch(input)
output2 = tg(input)
assert np.allclose(output1, output2.numpy())
def test_multi_task_regressor(self):
n_data_points = 20
n_features = 2
X = np.random.rand(n_data_points, n_features)
y1 = np.expand_dims(np.array([0.5 for x in range(n_data_points)]),
axis=-1)
y2 = np.expand_dims(np.array([-0.5 for x in range(n_data_points)]),
axis=-1)
X = NumpyDataset(X)
ys = [NumpyDataset(y1), NumpyDataset(y2)]
databag = Databag()
features = Feature(shape=(None, n_features))
databag.add_dataset(features, X)
outputs = []
losses = []
for i in range(2):
label = Label(shape=(None, 1))
dense = Dense(out_channels=1, in_layers=[features])
loss = ReduceSquareDifference(in_layers=[dense, label])
outputs.append(dense)
losses.append(loss)
databag.add_dataset(label, ys[i])
total_loss = ReduceMean(in_layers=losses)
tg = dc.models.TensorGraph(learning_rate=0.01)
for output in outputs:
tg.add_output(output)
tg.set_loss(total_loss)
tg.fit_generator(
databag.iterbatches(epochs=1000,
batch_size=tg.batch_size,
pad_batches=True))
predictions = tg.predict_on_generator(databag.iterbatches())
for i in range(2):
y_real = ys[i].X
y_pred = predictions[i]
assert_true(np.all(np.isclose(y_pred, y_real, atol=1.5)))
def test_mnist(self):
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
train = dc.data.NumpyDataset(mnist.train.images, mnist.train.labels)
valid = dc.data.NumpyDataset(mnist.validation.images,
mnist.validation.labels)
# Images are square 28x28 (batch, height, width, channel)
feature = Feature(shape=(None, 784), name="Feature")
make_image = Reshape(shape=(-1, 28, 28, 1), in_layers=[feature])
conv2d_1 = Conv2d(num_outputs=32, in_layers=[make_image])
maxpool_1 = MaxPool(in_layers=[conv2d_1])
conv2d_2 = Conv2d(num_outputs=64, in_layers=[maxpool_1])
maxpool_2 = MaxPool(in_layers=[conv2d_2])
flatten = Flatten(in_layers=[maxpool_2])
dense1 = Dense(out_channels=1024,
activation_fn=tf.nn.relu,
in_layers=[flatten])
dense2 = Dense(out_channels=10, in_layers=[dense1])
label = Label(shape=(None, 10), name="Label")
smce = SoftMaxCrossEntropy(in_layers=[label, dense2])
loss = ReduceMean(in_layers=[smce])
output = SoftMax(in_layers=[dense2])
tg = dc.models.TensorGraph(model_dir='/tmp/mnist',
batch_size=1000,
use_queue=True)
tg.add_output(output)
tg.set_loss(loss)
tg.fit(train, nb_epoch=2)
prediction = np.squeeze(tg.predict_proba_on_batch(valid.X))
fpr = dict()
tpr = dict()
roc_auc = dict()
for i in range(10):
fpr[i], tpr[i], thresh = roc_curve(valid.y[:, i], prediction[:, i])
roc_auc[i] = auc(fpr[i], tpr[i])
assert_true(roc_auc[i] > 0.99)
def test_multi_task_classifier(self):
n_data_points = 20
n_features = 2
X = np.random.rand(n_data_points, n_features)
y1 = np.array([[0, 1] for x in range(n_data_points)])
y2 = np.array([[1, 0] for x in range(n_data_points)])
X = NumpyDataset(X)
ys = [NumpyDataset(y1), NumpyDataset(y2)]
databag = Databag()
features = Feature(shape=(None, n_features))
databag.add_dataset(features, X)
outputs = []
entropies = []
for i in range(2):
label = Label(shape=(None, 2))
dense = Dense(out_channels=2, in_layers=[features])
output = SoftMax(in_layers=[dense])
smce = SoftMaxCrossEntropy(in_layers=[label, dense])
entropies.append(smce)
outputs.append(output)
databag.add_dataset(label, ys[i])
total_loss = ReduceMean(in_layers=entropies)
tg = dc.models.TensorGraph(learning_rate=0.01)
for output in outputs:
tg.add_output(output)
tg.set_loss(total_loss)
tg.fit_generator(
databag.iterbatches(epochs=1000,
batch_size=tg.batch_size,
pad_batches=True))
predictions = tg.predict_on_generator(databag.iterbatches())
for i in range(2):
y_real = ys[i].X
y_pred = predictions[i]
assert_true(np.all(np.isclose(y_pred, y_real, atol=0.6)))
def test_shared_layer(self):
n_data_points = 20
n_features = 2
X = np.random.rand(n_data_points, n_features)
y1 = np.array([[0, 1] for x in range(n_data_points)])
X = NumpyDataset(X)
ys = [NumpyDataset(y1)]
databag = Databag()
features = Feature(shape=(None, n_features))
databag.add_dataset(features, X)
outputs = []
label = Label(shape=(None, 2))
dense1 = Dense(out_channels=2, in_layers=[features])
dense2 = dense1.shared(in_layers=[features])
output1 = SoftMax(in_layers=[dense1])
output2 = SoftMax(in_layers=[dense2])
smce = SoftMaxCrossEntropy(in_layers=[label, dense1])
outputs.append(output1)
outputs.append(output2)
databag.add_dataset(label, ys[0])
total_loss = ReduceMean(in_layers=[smce])
tg = dc.models.TensorGraph(learning_rate=0.01)
for output in outputs:
tg.add_output(output)
tg.set_loss(total_loss)
tg.fit_generator(
databag.iterbatches(epochs=1,
batch_size=tg.batch_size,
pad_batches=True))
prediction = tg.predict_on_generator(databag.iterbatches())
assert_true(np.all(np.isclose(prediction[0], prediction[1],
atol=0.01)))
def test_compute_model_performance_multitask_classifier(self):
n_data_points = 20
n_features = 1
n_tasks = 2
n_classes = 2
X = np.ones(shape=(n_data_points // 2, n_features)) * -1
X1 = np.ones(shape=(n_data_points // 2, n_features))
X = np.concatenate((X, X1))
class_1 = np.array([[0.0, 1.0] for x in range(int(n_data_points / 2))])
class_0 = np.array([[1.0, 0.0] for x in range(int(n_data_points / 2))])
y1 = np.concatenate((class_0, class_1))
y2 = np.concatenate((class_1, class_0))
y = np.stack([y1, y2], axis=1)
dataset = NumpyDataset(X, y)
features = Feature(shape=(None, n_features))
label = Label(shape=(None, n_tasks, n_classes))
dense = Dense(out_channels=n_tasks * n_classes, in_layers=[features])
logits = Reshape(shape=(None, n_tasks, n_classes), in_layers=dense)
output = SoftMax(in_layers=[logits])
smce = SoftMaxCrossEntropy(in_layers=[label, logits])
total_loss = ReduceMean(in_layers=smce)
tg = dc.models.TensorGraph(learning_rate=0.01,
batch_size=n_data_points)
tg.add_output(output)
tg.set_loss(total_loss)
tg.fit(dataset, nb_epoch=1000)
metric = dc.metrics.Metric(dc.metrics.roc_auc_score,
np.mean,
mode="classification")
scores = tg.evaluate_generator(tg.default_generator(dataset), [metric],
labels=[label],
per_task_metrics=True)
scores = list(scores[1].values())
# Loosening atol to see if tests stop failing sporadically
assert_true(np.all(np.isclose(scores, [1.0, 1.0], atol=0.50)))
def __init__(self,
frag1_num_atoms=70,
frag2_num_atoms=634,
complex_num_atoms=701,
max_num_neighbors=12,
batch_size=24,
atom_types=[
6, 7., 8., 9., 11., 12., 15., 16., 17., 20., 25., 30., 35.,
53., -1.
],
radial=[[
1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0, 5.5, 6.0, 6.5, 7.0,
7.5, 8.0, 8.5, 9.0, 9.5, 10.0, 10.5, 11.0, 11.5, 12.0
], [0.0, 4.0, 8.0], [0.4]],
layer_sizes=[32, 32, 16],
learning_rate=0.001,
**kwargs):
"""Implements an Atomic Convolution Model.
Implements the atomic convolutional networks as introduced in
https://arxiv.org/abs/1703.10603. The atomic convolutional networks
function as a variant of graph convolutions. The difference is that the
"graph" here is the nearest neighbors graph in 3D space. The
AtomicConvModel leverages these connections in 3D space to train models
that learn to predict energetic state starting from the spatial
geometry of the model.
Params
------
frag1_num_atoms: int
Number of atoms in first fragment
frag2_num_atoms: int
Number of atoms in sec
max_num_neighbors: int
Maximum number of neighbors possible for an atom. Recall neighbors
are spatial neighbors.
atom_types: list
List of atoms recognized by model. Atoms are indicated by their
nuclear numbers.
radial: list
TODO: add description
layer_sizes: list
TODO: add description
learning_rate: float
Learning rate for the model.
"""
# TODO: Turning off queue for now. Safe to re-activate?
super(AtomicConvModel, self).__init__(use_queue=False, **kwargs)
self.complex_num_atoms = complex_num_atoms
self.frag1_num_atoms = frag1_num_atoms
self.frag2_num_atoms = frag2_num_atoms
self.max_num_neighbors = max_num_neighbors
self.batch_size = batch_size
self.atom_types = atom_types
rp = [x for x in itertools.product(*radial)]
self.frag1_X = Feature(shape=(batch_size, frag1_num_atoms, 3))
self.frag1_nbrs = Feature(
shape=(batch_size, frag1_num_atoms, max_num_neighbors))
self.frag1_nbrs_z = Feature(
shape=(batch_size, frag1_num_atoms, max_num_neighbors))
self.frag1_z = Feature(shape=(batch_size, frag1_num_atoms))
self.frag2_X = Feature(shape=(batch_size, frag2_num_atoms, 3))
self.frag2_nbrs = Feature(
shape=(batch_size, frag2_num_atoms, max_num_neighbors))
self.frag2_nbrs_z = Feature(
shape=(batch_size, frag2_num_atoms, max_num_neighbors))
self.frag2_z = Feature(shape=(batch_size, frag2_num_atoms))
self.complex_X = Feature(shape=(batch_size, complex_num_atoms, 3))
self.complex_nbrs = Feature(
shape=(batch_size, complex_num_atoms, max_num_neighbors))
self.complex_nbrs_z = Feature(
shape=(batch_size, complex_num_atoms, max_num_neighbors))
self.complex_z = Feature(shape=(batch_size, complex_num_atoms))
frag1_conv = AtomicConvolution(
atom_types=self.atom_types,
radial_params=rp,
boxsize=None,
in_layers=[self.frag1_X, self.frag1_nbrs, self.frag1_nbrs_z])
frag2_conv = AtomicConvolution(
atom_types=self.atom_types,
radial_params=rp,
boxsize=None,
in_layers=[self.frag2_X, self.frag2_nbrs, self.frag2_nbrs_z])
complex_conv = AtomicConvolution(
atom_types=self.atom_types,
radial_params=rp,
boxsize=None,
in_layers=[self.complex_X, self.complex_nbrs, self.complex_nbrs_z])
score = AtomicConvScore(
self.atom_types,
layer_sizes,
in_layers=[
frag1_conv, frag2_conv, complex_conv, self.frag1_z, self.frag2_z,
self.complex_z
])
self.label = Label(shape=(None, 1))
loss = ReduceMean(in_layers=L2Loss(in_layers=[score, self.label]))
self.add_output(score)
self.set_loss(loss)
flatten = Flatten()
tg.add_layer(flatten, parents=[conv2d_2])
dense1 = Dense(out_channels=1024, activation_fn=tf.nn.relu)
tg.add_layer(dense1, parents=[flatten])
dense2 = Dense(out_channels=10)
tg.add_layer(dense2, parents=[dense1])
label = Input(shape=(None, 10))
tg.add_label(label)
smce = SoftMaxCrossEntropy()
tg.add_layer(smce, parents=[label, dense2])
loss = ReduceMean()
tg.add_layer(loss, parents=[smce])
tg.set_loss(loss)
output = SoftMax()
tg.add_layer(output, parents=[dense2])
tg.add_output(output)
tg.fit(train, nb_epoch=2)
tg.fit(train, nb_epoch=2)
tg.save()
# tg = TensorGraph.load_from_dir("/tmp/mnist")
from sklearn.metrics import roc_curve, auc
import numpy as np
def atomic_conv_model(frag1_num_atoms=70,
frag2_num_atoms=634,
complex_num_atoms=701,
max_num_neighbors=12,
batch_size=24,
at=[
6, 7., 8., 9., 11., 12., 15., 16., 17., 20., 25.,
30., 35., 53., -1.
],
radial=[[
1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0, 5.5, 6.0,
6.5, 7.0, 7.5, 8.0, 8.5, 9.0, 9.5, 10.0, 10.5, 11.0,
11.5, 12.0
], [0.0, 4.0, 8.0], [0.4]],
layer_sizes=[32, 32, 16],
learning_rate=0.001):
rp = [x for x in itertools.product(*radial)]
frag1_X = Feature(shape=(batch_size, frag1_num_atoms, 3))
frag1_nbrs = Feature(shape=(batch_size, frag1_num_atoms,
max_num_neighbors))
frag1_nbrs_z = Feature(shape=(batch_size, frag1_num_atoms,
max_num_neighbors))
frag1_z = Feature(shape=(batch_size, frag1_num_atoms))
frag2_X = Feature(shape=(batch_size, frag2_num_atoms, 3))
frag2_nbrs = Feature(shape=(batch_size, frag2_num_atoms,
max_num_neighbors))
frag2_nbrs_z = Feature(shape=(batch_size, frag2_num_atoms,
max_num_neighbors))
frag2_z = Feature(shape=(batch_size, frag2_num_atoms))
complex_X = Feature(shape=(batch_size, complex_num_atoms, 3))
complex_nbrs = Feature(shape=(batch_size, complex_num_atoms,
max_num_neighbors))
complex_nbrs_z = Feature(shape=(batch_size, complex_num_atoms,
max_num_neighbors))
complex_z = Feature(shape=(batch_size, complex_num_atoms))
frag1_conv = AtomicConvolution(
atom_types=at,
radial_params=rp,
boxsize=None,
in_layers=[frag1_X, frag1_nbrs, frag1_nbrs_z])
frag2_conv = AtomicConvolution(
atom_types=at,
radial_params=rp,
boxsize=None,
in_layers=[frag2_X, frag2_nbrs, frag2_nbrs_z])
complex_conv = AtomicConvolution(
atom_types=at,
radial_params=rp,
boxsize=None,
in_layers=[complex_X, complex_nbrs, complex_nbrs_z])
score = AtomicConvScore(at,
layer_sizes,
in_layers=[
frag1_conv, frag2_conv, complex_conv, frag1_z,
frag2_z, complex_z
])
label = Label(shape=(None, 1))
loss = ReduceMean(in_layers=L2Loss(in_layers=[score, label]))
def feed_dict_generator(dataset, batch_size, epochs=1, pad_batches=True):
def replace_atom_types(z):
def place_holder(i):
if i in at:
return i
return -1
return np.array([place_holder(x) for x in z])
for epoch in range(epochs):
for ind, (F_b, y_b, w_b, ids_b) in enumerate(
dataset.iterbatches(batch_size,
deterministic=True,
pad_batches=pad_batches)):
N = complex_num_atoms
N_1 = frag1_num_atoms
N_2 = frag2_num_atoms
M = max_num_neighbors
orig_dict = {}
batch_size = F_b.shape[0]
num_features = F_b[0][0].shape[1]
frag1_X_b = np.zeros((batch_size, N_1, num_features))
for i in range(batch_size):
frag1_X_b[i] = F_b[i][0]
orig_dict[frag1_X] = frag1_X_b
frag2_X_b = np.zeros((batch_size, N_2, num_features))
for i in range(batch_size):
frag2_X_b[i] = F_b[i][3]
orig_dict[frag2_X] = frag2_X_b
complex_X_b = np.zeros((batch_size, N, num_features))
for i in range(batch_size):
complex_X_b[i] = F_b[i][6]
orig_dict[complex_X] = complex_X_b
frag1_Nbrs = np.zeros((batch_size, N_1, M))
frag1_Z_b = np.zeros((batch_size, N_1))
for i in range(batch_size):
z = replace_atom_types(F_b[i][2])
frag1_Z_b[i] = z
frag1_Nbrs_Z = np.zeros((batch_size, N_1, M))
for atom in range(N_1):
for i in range(batch_size):
atom_nbrs = F_b[i][1].get(atom, "")
frag1_Nbrs[i,
atom, :len(atom_nbrs)] = np.array(atom_nbrs)
for j, atom_j in enumerate(atom_nbrs):
frag1_Nbrs_Z[i, atom, j] = frag1_Z_b[i, atom_j]
orig_dict[frag1_nbrs] = frag1_Nbrs
orig_dict[frag1_nbrs_z] = frag1_Nbrs_Z
orig_dict[frag1_z] = frag1_Z_b
frag2_Nbrs = np.zeros((batch_size, N_2, M))
frag2_Z_b = np.zeros((batch_size, N_2))
for i in range(batch_size):
z = replace_atom_types(F_b[i][5])
frag2_Z_b[i] = z
frag2_Nbrs_Z = np.zeros((batch_size, N_2, M))
for atom in range(N_2):
for i in range(batch_size):
atom_nbrs = F_b[i][4].get(atom, "")
frag2_Nbrs[i,
atom, :len(atom_nbrs)] = np.array(atom_nbrs)
for j, atom_j in enumerate(atom_nbrs):
frag2_Nbrs_Z[i, atom, j] = frag2_Z_b[i, atom_j]
orig_dict[frag2_nbrs] = frag2_Nbrs
orig_dict[frag2_nbrs_z] = frag2_Nbrs_Z
orig_dict[frag2_z] = frag2_Z_b
complex_Nbrs = np.zeros((batch_size, N, M))
complex_Z_b = np.zeros((batch_size, N))
for i in range(batch_size):
z = replace_atom_types(F_b[i][8])
complex_Z_b[i] = z
complex_Nbrs_Z = np.zeros((batch_size, N, M))
for atom in range(N):
for i in range(batch_size):
atom_nbrs = F_b[i][7].get(atom, "")
complex_Nbrs[i, atom, :len(atom_nbrs)] = np.array(
atom_nbrs)
for j, atom_j in enumerate(atom_nbrs):
complex_Nbrs_Z[i, atom, j] = complex_Z_b[i, atom_j]
orig_dict[complex_nbrs] = complex_Nbrs
orig_dict[complex_nbrs_z] = complex_Nbrs_Z
orig_dict[complex_z] = complex_Z_b
orig_dict[label] = np.reshape(y_b, newshape=(batch_size, 1))
yield orig_dict
tg = TensorGraph(batch_size=batch_size,
mode=str("regression"),
model_dir=str("/tmp/atom_conv"),
learning_rate=learning_rate)
tg.add_output(score)
tg.set_loss(loss)
return tg, feed_dict_generator, label
def build_graph(self):
# Build placeholders
self.atom_features = Feature(shape=(None, self.n_atom_feat))
self.pair_features = Feature(shape=(None, self.n_pair_feat))
self.atom_split = Feature(shape=(None, ), dtype=tf.int32)
self.atom_to_pair = Feature(shape=(None, 2), dtype=tf.int32)
message_passing = MessagePassing(self.T,
message_fn='enn',
update_fn='gru',
n_hidden=self.n_hidden,
in_layers=[
self.atom_features,
self.pair_features,
self.atom_to_pair
])
atom_embeddings = Dense(self.n_hidden, in_layers=[message_passing])
mol_embeddings = SetGather(
self.M,
self.batch_size,
n_hidden=self.n_hidden,
in_layers=[atom_embeddings, self.atom_split])
dense1 = Dense(out_channels=2 * self.n_hidden,
activation_fn=tf.nn.relu,
in_layers=[mol_embeddings])
n_tasks = self.n_tasks
weights = Weights(shape=(None, n_tasks))
if self.mode == 'classification':
n_classes = self.n_classes
labels = Label(shape=(None, n_tasks, n_classes))
logits = Reshape(shape=(None, n_tasks, n_classes),
in_layers=[
Dense(in_layers=dense1,
out_channels=n_tasks * n_classes)
])
logits = TrimGraphOutput([logits, weights])
output = SoftMax(logits)
self.add_output(output)
loss = SoftMaxCrossEntropy(in_layers=[labels, logits])
weighted_loss = WeightedError(in_layers=[loss, weights])
self.set_loss(weighted_loss)
else:
labels = Label(shape=(None, n_tasks))
output = Reshape(
shape=(None, n_tasks),
in_layers=[Dense(in_layers=dense1, out_channels=n_tasks)])
output = TrimGraphOutput([output, weights])
self.add_output(output)
if self.uncertainty:
log_var = Reshape(
shape=(None, n_tasks),
in_layers=[Dense(in_layers=dense1, out_channels=n_tasks)])
log_var = TrimGraphOutput([log_var, weights])
var = Exp(log_var)
self.add_variance(var)
diff = labels - output
weighted_loss = weights * (diff * diff / var + log_var)
weighted_loss = ReduceSum(ReduceMean(weighted_loss, axis=[1]))
else:
weighted_loss = ReduceSum(
L2Loss(in_layers=[labels, output, weights]))
self.set_loss(weighted_loss)
def build_graph(self):
"""
Building graph structures:
"""
self.atom_features = Feature(shape=(None, self.number_atom_features))
self.degree_slice = Feature(shape=(None, 2), dtype=tf.int32)
self.membership = Feature(shape=(None, ), dtype=tf.int32)
self.deg_adjs = []
for i in range(0, 10 + 1):
deg_adj = Feature(shape=(None, i + 1), dtype=tf.int32)
self.deg_adjs.append(deg_adj)
in_layer = self.atom_features
for layer_size, dropout in zip(self.graph_conv_layers, self.dropout):
gc1_in = [in_layer, self.degree_slice, self.membership
] + self.deg_adjs
gc1 = GraphConv(layer_size,
activation_fn=tf.nn.relu,
in_layers=gc1_in)
batch_norm1 = BatchNorm(in_layers=[gc1])
if dropout > 0.0:
batch_norm1 = Dropout(dropout, in_layers=batch_norm1)
gp_in = [batch_norm1, self.degree_slice, self.membership
] + self.deg_adjs
in_layer = GraphPool(in_layers=gp_in)
dense = Dense(out_channels=self.dense_layer_size,
activation_fn=tf.nn.relu,
in_layers=[in_layer])
batch_norm3 = BatchNorm(in_layers=[dense])
if self.dropout[-1] > 0.0:
batch_norm3 = Dropout(self.dropout[-1], in_layers=batch_norm3)
readout = GraphGather(
batch_size=self.batch_size,
activation_fn=tf.nn.tanh,
in_layers=[batch_norm3, self.degree_slice, self.membership] +
self.deg_adjs)
n_tasks = self.n_tasks
weights = Weights(shape=(None, n_tasks))
if self.mode == 'classification':
n_classes = self.n_classes
labels = Label(shape=(None, n_tasks, n_classes))
logits = Reshape(shape=(None, n_tasks, n_classes),
in_layers=[
Dense(in_layers=readout,
out_channels=n_tasks * n_classes)
])
logits = TrimGraphOutput([logits, weights])
output = SoftMax(logits)
self.add_output(output)
loss = SoftMaxCrossEntropy(in_layers=[labels, logits])
weighted_loss = WeightedError(in_layers=[loss, weights])
self.set_loss(weighted_loss)
else:
labels = Label(shape=(None, n_tasks))
output = Reshape(
shape=(None, n_tasks),
in_layers=[Dense(in_layers=readout, out_channels=n_tasks)])
output = TrimGraphOutput([output, weights])
self.add_output(output)
if self.uncertainty:
log_var = Reshape(
shape=(None, n_tasks),
in_layers=[Dense(in_layers=readout, out_channels=n_tasks)])
log_var = TrimGraphOutput([log_var, weights])
var = Exp(log_var)
self.add_variance(var)
diff = labels - output
weighted_loss = weights * (diff * diff / var + log_var)
weighted_loss = ReduceSum(ReduceMean(weighted_loss, axis=[1]))
else:
weighted_loss = ReduceSum(
L2Loss(in_layers=[labels, output, weights]))
self.set_loss(weighted_loss)
def build_graph(self):
"""Building graph structures:
Features => DAGLayer => DAGGather => Classification or Regression
"""
self.atom_features = Feature(shape=(None, self.n_atom_feat))
self.parents = Feature(shape=(None, self.max_atoms, self.max_atoms),
dtype=tf.int32)
self.calculation_orders = Feature(shape=(None, self.max_atoms),
dtype=tf.int32)
self.calculation_masks = Feature(shape=(None, self.max_atoms),
dtype=tf.bool)
self.membership = Feature(shape=(None, ), dtype=tf.int32)
self.n_atoms = Feature(shape=(), dtype=tf.int32)
dag_layer1 = DAGLayer(n_graph_feat=self.n_graph_feat,
n_atom_feat=self.n_atom_feat,
max_atoms=self.max_atoms,
layer_sizes=self.layer_sizes,
dropout=self.dropout,
batch_size=self.batch_size,
in_layers=[
self.atom_features, self.parents,
self.calculation_orders,
self.calculation_masks, self.n_atoms
])
dag_gather = DAGGather(n_graph_feat=self.n_graph_feat,
n_outputs=self.n_outputs,
max_atoms=self.max_atoms,
layer_sizes=self.layer_sizes_gather,
dropout=self.dropout,
in_layers=[dag_layer1, self.membership])
n_tasks = self.n_tasks
weights = Weights(shape=(None, n_tasks))
if self.mode == 'classification':
n_classes = self.n_classes
labels = Label(shape=(None, n_tasks, n_classes))
logits = Reshape(shape=(None, n_tasks, n_classes),
in_layers=[
Dense(in_layers=dag_gather,
out_channels=n_tasks * n_classes)
])
output = SoftMax(logits)
self.add_output(output)
loss = SoftMaxCrossEntropy(in_layers=[labels, logits])
weighted_loss = WeightedError(in_layers=[loss, weights])
self.set_loss(weighted_loss)
else:
labels = Label(shape=(None, n_tasks))
output = Reshape(
shape=(None, n_tasks),
in_layers=[Dense(in_layers=dag_gather, out_channels=n_tasks)])
self.add_output(output)
if self.uncertainty:
log_var = Reshape(shape=(None, n_tasks),
in_layers=[
Dense(in_layers=dag_gather,
out_channels=n_tasks)
])
var = Exp(log_var)
self.add_variance(var)
diff = labels - output
weighted_loss = weights * (diff * diff / var + log_var)
weighted_loss = ReduceSum(ReduceMean(weighted_loss, axis=[1]))
else:
weighted_loss = ReduceSum(
L2Loss(in_layers=[labels, output, weights]))
self.set_loss(weighted_loss)
def __init__(self,
n_tasks,
n_features,
layer_sizes=[1000],
weight_init_stddevs=0.02,
bias_init_consts=1.0,
weight_decay_penalty=0.0,
weight_decay_penalty_type="l2",
dropouts=0.5,
activation_fns=tf.nn.relu,
uncertainty=False,
**kwargs):
"""Create a MultitaskRegressor.
In addition to the following arguments, this class also accepts all the keywork arguments
from TensorGraph.
Parameters
----------
n_tasks: int
number of tasks
n_features: int
number of features
layer_sizes: list
the size of each dense layer in the network. The length of this list determines the number of layers.
weight_init_stddevs: list or float
the standard deviation of the distribution to use for weight initialization of each layer. The length
of this list should equal len(layer_sizes)+1. The final element corresponds to the output layer.
Alternatively this may be a single value instead of a list, in which case the same value is used for every layer.
bias_init_consts: list or float
the value to initialize the biases in each layer to. The length of this list should equal len(layer_sizes)+1.
The final element corresponds to the output layer. Alternatively this may be a single value instead of a list,
in which case the same value is used for every layer.
weight_decay_penalty: float
the magnitude of the weight decay penalty to use
weight_decay_penalty_type: str
the type of penalty to use for weight decay, either 'l1' or 'l2'
dropouts: list or float
the dropout probablity to use for each layer. The length of this list should equal len(layer_sizes).
Alternatively this may be a single value instead of a list, in which case the same value is used for every layer.
activation_fns: list or object
the Tensorflow activation function to apply to each layer. The length of this list should equal
len(layer_sizes). Alternatively this may be a single value instead of a list, in which case the
same value is used for every layer.
uncertainty: bool
if True, include extra outputs and loss terms to enable the uncertainty
in outputs to be predicted
"""
super(MultitaskRegressor, self).__init__(**kwargs)
self.n_tasks = n_tasks
self.n_features = n_features
n_layers = len(layer_sizes)
if not isinstance(weight_init_stddevs, collections.Sequence):
weight_init_stddevs = [weight_init_stddevs] * (n_layers + 1)
if not isinstance(bias_init_consts, collections.Sequence):
bias_init_consts = [bias_init_consts] * (n_layers + 1)
if not isinstance(dropouts, collections.Sequence):
dropouts = [dropouts] * n_layers
if not isinstance(activation_fns, collections.Sequence):
activation_fns = [activation_fns] * n_layers
if uncertainty:
if any(d == 0.0 for d in dropouts):
raise ValueError(
'Dropout must be included in every layer to predict uncertainty'
)
# Add the input features.
mol_features = Feature(shape=(None, n_features))
prev_layer = mol_features
# Add the dense layers
for size, weight_stddev, bias_const, dropout, activation_fn in zip(
layer_sizes, weight_init_stddevs, bias_init_consts, dropouts,
activation_fns):
layer = Dense(in_layers=[prev_layer],
out_channels=size,
activation_fn=activation_fn,
weights_initializer=TFWrapper(
tf.truncated_normal_initializer,
stddev=weight_stddev),
biases_initializer=TFWrapper(tf.constant_initializer,
value=bias_const))
if dropout > 0.0:
layer = Dropout(dropout, in_layers=[layer])
prev_layer = layer
self.neural_fingerprint = prev_layer
# Compute the loss function for each label.
output = Reshape(shape=(-1, n_tasks, 1),
in_layers=[
Dense(in_layers=[prev_layer],
out_channels=n_tasks,
weights_initializer=TFWrapper(
tf.truncated_normal_initializer,
stddev=weight_init_stddevs[-1]),
biases_initializer=TFWrapper(
tf.constant_initializer,
value=bias_init_consts[-1]))
])
self.add_output(output)
labels = Label(shape=(None, n_tasks, 1))
weights = Weights(shape=(None, n_tasks, 1))
if uncertainty:
log_var = Reshape(
shape=(-1, n_tasks, 1),
in_layers=[
Dense(in_layers=[prev_layer],
out_channels=n_tasks,
weights_initializer=TFWrapper(
tf.truncated_normal_initializer,
stddev=weight_init_stddevs[-1]),
biases_initializer=TFWrapper(tf.constant_initializer,
value=0.0))
])
var = Exp(log_var)
self.add_variance(var)
diff = labels - output
weighted_loss = weights * (diff * diff / var + log_var)
weighted_loss = ReduceSum(ReduceMean(weighted_loss, axis=[1, 2]))
else:
weighted_loss = ReduceSum(
L2Loss(in_layers=[labels, output, weights]))
if weight_decay_penalty != 0.0:
weighted_loss = WeightDecay(weight_decay_penalty,
weight_decay_penalty_type,
in_layers=[weighted_loss])
self.set_loss(weighted_loss)
def __init__(self,
img_rows=512,
img_cols=512,
filters=[64, 128, 256, 512, 1024],
**kwargs):
super(UNet, self).__init__(use_queue=False, **kwargs)
self.img_cols = img_cols
self.img_rows = img_rows
self.filters = filters
input = Feature(shape=(None, self.img_rows, self.img_cols, 3))
labels = Label(shape=(None, self.img_rows * self.img_cols))
conv1 = Conv2D(num_outputs=self.filters[0],
kernel_size=3,
activation='relu',
padding='same',
in_layers=[input])
conv1 = Conv2D(num_outputs=self.filters[0],
kernel_size=3,
activation='relu',
padding='same',
in_layers=[conv1])
pool1 = MaxPool2D(ksize=[1, 2, 2, 1], in_layers=[conv1])
conv2 = Conv2D(num_outputs=self.filters[1],
kernel_size=3,
activation='relu',
padding='same',
in_layers=[pool1])
conv2 = Conv2D(num_outputs=self.filters[1],
kernel_size=3,
activation='relu',
padding='same',
in_layers=[conv2])
pool2 = MaxPool2D(ksize=[1, 2, 2, 1], in_layers=[conv2])
conv3 = Conv2D(num_outputs=self.filters[2],
kernel_size=3,
activation='relu',
padding='same',
in_layers=[pool2])
conv3 = Conv2D(num_outputs=self.filters[2],
kernel_size=3,
activation='relu',
padding='same',
in_layers=[conv3])
pool3 = MaxPool2D(ksize=[1, 2, 2, 1], in_layers=[conv3])
conv4 = Conv2D(num_outputs=self.filters[3],
kernel_size=3,
activation='relu',
padding='same',
in_layers=[pool3])
conv4 = Conv2D(num_outputs=self.filters[3],
kernel_size=3,
activation='relu',
padding='same',
in_layers=[conv4])
pool4 = MaxPool2D(ksize=[1, 2, 2, 1], in_layers=[conv4])
conv5 = Conv2D(num_outputs=self.filters[4],
kernel_size=3,
activation='relu',
padding='same',
in_layers=[pool4])
conv5 = Conv2D(num_outputs=self.filters[4],
kernel_size=3,
activation='relu',
padding='same',
in_layers=[conv5])
up6 = Conv2DTranspose(num_outputs=self.filters[3],
kernel_size=2,
stride=2,
in_layers=[conv5])
concat6 = Concat(in_layers=[conv4, up6], axis=3)
conv6 = Conv2D(num_outputs=self.filters[3],
kernel_size=3,
activation='relu',
padding='same',
in_layers=[concat6])
conv6 = Conv2D(num_outputs=self.filters[3],
kernel_size=3,
activation='relu',
padding='same',
in_layers=[conv6])
up7 = Conv2DTranspose(num_outputs=self.filters[2],
kernel_size=2,
stride=2,
in_layers=[conv6])
concat7 = Concat(in_layers=[conv3, up7], axis=3)
conv7 = Conv2D(num_outputs=self.filters[2],
kernel_size=3,
activation='relu',
padding='same',
in_layers=[concat7])
conv7 = Conv2D(num_outputs=self.filters[2],
kernel_size=3,
activation='relu',
padding='same',
in_layers=[conv7])
up8 = Conv2DTranspose(num_outputs=self.filters[1],
kernel_size=2,
stride=2,
in_layers=[conv7])
concat8 = Concat(in_layers=[conv2, up8], axis=3)
conv8 = Conv2D(num_outputs=self.filters[1],
kernel_size=3,
activation='relu',
padding='same',
in_layers=[concat8])
conv8 = Conv2D(num_outputs=self.filters[1],
kernel_size=3,
activation='relu',
padding='same',
in_layers=[conv8])
up9 = Conv2DTranspose(num_outputs=self.filters[0],
kernel_size=2,
stride=2,
in_layers=[conv8])
concat9 = Concat(in_layers=[conv1, up9], axis=3)
conv9 = Conv2D(num_outputs=self.filters[0],
kernel_size=3,
activation='relu',
padding='same',
in_layers=[concat9])
conv9 = Conv2D(num_outputs=self.filters[0],
kernel_size=3,
activation='relu',
padding='same',
in_layers=[conv9])
conv10 = Conv2D(num_outputs=1,
kernel_size=1,
activation='sigmoid',
in_layers=[conv9])
loss = SoftMaxCrossEntropy(in_layers=[labels, conv10])
loss = ReduceMean(in_layers=[loss])
self.set_loss(loss)
self.add_output(conv10)
activation_fn=tf.nn.tanh,
in_layers=[dense1, degree_slice, membership] + deg_adjs)
# in this model, multilabel (15 precursors) shall be classified
# using the trained featuret vector
cost15 = []
for ts in range(ntask):
label_t = label15[ts]
classification_t = Dense(out_channels=2, in_layers=[out1])
softmax_t = SoftMax(in_layers=[classification_t])
tg.add_output(softmax_t)
cost_t = SoftMaxCrossEntropy(in_layers=[label_t, classification_t])
cost15.append(cost_t)
# The loss function is the average of the 15 crossentropy
loss = ReduceMean(in_layers=cost15)
tg.set_loss(loss)
def data_generator(dataset, n_epoch=1, predict=False):
for ee in range(n_epoch):
if not predict:
print('Starting epoch %i' % ee)
for ind, (X_b, y_b, w_b, ids_b) in enumerate(
dataset.iterbatches(n_batch,
pad_batches=True,
deterministic=True)):
fd = {}
for ts, label_t in enumerate(label15):
fd[label_t] = to_one_hot(y_b[:, ts])
mol = ConvMol.agglomerate_mols(X_b)