def add_placeholders(self, graph, name_scopes):
"""Adds all placeholders for this model."""
# Create placeholders
placeholder_scope = TensorflowGraph.get_placeholder_scope(
graph, name_scopes)
labels, weights = [], []
n_features = self.n_features
with placeholder_scope:
self.mol_features = tf.placeholder(tf.float32,
shape=[None, n_features],
name='mol_features')
for task in range(self.n_tasks):
weights.append(
tf.identity(
tf.placeholder(tf.float32,
shape=[
None,
],
name='weights_%d' % task)))
labels.append(
tf.identity(
tf.placeholder(tf.float32,
shape=[
None,
],
name='labels_%d' % task)))
return self.mol_features, labels, weights
def construct_task_feed_dict(self, this_task, X_b, y_b=None, w_b=None, ids_b=None):
"""Construct a feed dictionary from minibatch data.
TODO(rbharath): ids_b is not used here. Can we remove it?
Args:
X_b: np.ndarray of shape (batch_size, n_features)
y_b: np.ndarray of shape (batch_size, n_tasks)
w_b: np.ndarray of shape (batch_size, n_tasks)
ids_b: List of length (batch_size) with datapoint identifiers.
"""
orig_dict = {}
orig_dict["mol_features"] = X_b
n_samples = len(X_b)
for task in range(self.n_tasks):
if (this_task == task) and y_b is not None:
#orig_dict["labels_%d" % task] = np.reshape(y_b[:, task], (n_samples, 1))
orig_dict["labels_%d" % task] = np.reshape(y_b[:, task], (n_samples,))
else:
# Dummy placeholders
#orig_dict["labels_%d" % task] = np.zeros((n_samples, 1))
orig_dict["labels_%d" % task] = np.zeros((n_samples,))
if (this_task == task) and w_b is not None:
#orig_dict["weights_%d" % task] = np.reshape(w_b[:, task], (n_samples, 1))
orig_dict["weights_%d" % task] = np.reshape(w_b[:, task], (n_samples,))
else:
# Dummy placeholders
#orig_dict["weights_%d" % task] = np.zeros((n_samples, 1))
orig_dict["weights_%d" % task] = np.zeros((n_samples,))
return TensorflowGraph.get_feed_dict(orig_dict)
def construct_feed_dict(self, X_b, y_b=None, w_b=None, ids_b=None):
"""Construct a feed dictionary from minibatch data.
TODO(rbharath): ids_b is not used here. Can we remove it?
Args:
X_b: np.ndarray of shape (batch_size, n_features)
y_b: np.ndarray of shape (batch_size, n_tasks)
w_b: np.ndarray of shape (batch_size, n_tasks)
ids_b: List of length (batch_size) with datapoint identifiers.
"""
orig_dict = {}
orig_dict["mol_features"] = X_b
for task in range(self.n_tasks):
if y_b is not None:
orig_dict["labels_%d" % task] = to_one_hot(y_b[:, task])
else:
# Dummy placeholders
orig_dict["labels_%d" % task] = np.squeeze(
to_one_hot(np.zeros((self.batch_size, ))))
if w_b is not None:
orig_dict["weights_%d" % task] = w_b[:, task]
else:
# Dummy placeholders
orig_dict["weights_%d" % task] = np.ones((self.batch_size, ))
return TensorflowGraph.get_feed_dict(orig_dict)
def build(self, graph, name_scopes, training):
"""Constructs the graph architecture as specified in its config.
This method creates the following Placeholders:
mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape
batch_size x n_features.
"""
n_features = self.n_features
placeholder_scope = TensorflowGraph.get_placeholder_scope(
graph, name_scopes)
with graph.as_default():
with placeholder_scope:
self.mol_features = tf.placeholder(tf.float32,
shape=[None, n_features],
name='mol_features')
layer_sizes = self.layer_sizes
weight_init_stddevs = self.weight_init_stddevs
bias_init_consts = self.bias_init_consts
dropouts = self.dropouts
lengths_set = {
len(layer_sizes),
len(weight_init_stddevs),
len(bias_init_consts),
len(dropouts),
}
assert len(
lengths_set) == 1, 'All layer params must have same length.'
n_layers = lengths_set.pop()
assert n_layers > 0, 'Must have some layers defined.'
prev_layer = self.mol_features
prev_layer_size = n_features
for i in range(n_layers):
layer = tf.nn.relu(
model_ops.fully_connected_layer(
tensor=prev_layer,
size=layer_sizes[i],
weight_init=tf.truncated_normal(
shape=[prev_layer_size, layer_sizes[i]],
stddev=weight_init_stddevs[i]),
bias_init=tf.constant(value=bias_init_consts[i],
shape=[layer_sizes[i]])))
layer = model_ops.dropout(layer, dropouts[i])
prev_layer = layer
prev_layer_size = layer_sizes[i]
output = []
for task in range(self.n_tasks):
output.append(
tf.squeeze(
model_ops.fully_connected_layer(
tensor=prev_layer,
size=layer_sizes[i],
weight_init=tf.truncated_normal(
shape=[prev_layer_size, 1],
stddev=weight_init_stddevs[i]),
bias_init=tf.constant(value=bias_init_consts[i],
shape=[1]))))
return output
def construct_task_feed_dict(self,
this_task,
X_b,
y_b=None,
w_b=None,
ids_b=None):
"""Construct a feed dictionary from minibatch data.
TODO(rbharath): ids_b is not used here. Can we remove it?
Args:
X_b: np.ndarray of shape (batch_size, n_features)
y_b: np.ndarray of shape (batch_size, n_tasks)
w_b: np.ndarray of shape (batch_size, n_tasks)
ids_b: List of length (batch_size) with datapoint identifiers.
"""
orig_dict = {}
orig_dict["mol_features"] = X_b
n_samples = len(X_b)
for task in range(self.n_tasks):
if (this_task == task) and y_b is not None:
#orig_dict["labels_%d" % task] = np.reshape(y_b[:, task], (n_samples, 1))
orig_dict["labels_%d" % task] = np.reshape(y_b[:, task], (n_samples,))
else:
# Dummy placeholders
#orig_dict["labels_%d" % task] = np.zeros((n_samples, 1))
orig_dict["labels_%d" % task] = np.zeros((n_samples,))
if (this_task == task) and w_b is not None:
#orig_dict["weights_%d" % task] = np.reshape(w_b[:, task], (n_samples, 1))
orig_dict["weights_%d" % task] = np.reshape(w_b[:, task], (n_samples,))
else:
# Dummy placeholders
#orig_dict["weights_%d" % task] = np.zeros((n_samples, 1))
orig_dict["weights_%d" % task] = np.zeros((n_samples,))
return TensorflowGraph.get_feed_dict(orig_dict)
def build(self, graph, name_scopes, training):
"""Constructs the graph architecture of IRV as described in:
https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2750043/
"""
placeholder_scope = TensorflowGraph.get_placeholder_scope(graph,
name_scopes)
K = self.K
with graph.as_default():
output = []
with placeholder_scope:
self.features = tf.placeholder(
tf.float32, shape=[None, self.n_features], name='mol_features')
with tf.name_scope('variable'):
V = tf.Variable(tf.constant([0.01, 1.]), name="vote", dtype=tf.float32)
W = tf.Variable(tf.constant([1., 1.]), name="w", dtype=tf.float32)
b = tf.Variable(tf.constant([0.01]), name="b", dtype=tf.float32)
b2 = tf.Variable(tf.constant([0.01]), name="b2", dtype=tf.float32)
for count in range(self.n_tasks):
similarity = self.features[:, 2 * K * count:(2 * K * count + K)]
ys = tf.to_int32(
self.features[:, (2 * K * count + K):2 * K * (count + 1)])
R = b + W[0] * similarity + W[1] * tf.constant(
np.arange(K) + 1, dtype=tf.float32)
R = tf.sigmoid(R)
z = tf.reduce_sum(R * tf.gather(V, ys), axis=1) + b2
output.append(tf.reshape(z, shape=[-1, 1]))
return output
def build(self, graph, name_scopes, training):
"""Constructs the graph architecture of model: n_tasks * sigmoid nodes.
This method creates the following Placeholders:
mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape
batch_size x n_features.
"""
placeholder_scope = TensorflowGraph.get_placeholder_scope(
graph, name_scopes)
n_features = self.n_features
with graph.as_default():
with placeholder_scope:
self.mol_features = tf.placeholder(
tf.float32,
shape=[None, n_features],
name='mol_features')
weight_init_stddevs = self.weight_init_stddevs
bias_init_consts = self.bias_init_consts
lg_list = []
for task in range(self.n_tasks):
#setting up n_tasks nodes(output nodes)
lg = model_ops.fully_connected_layer(
tensor=self.mol_features,
size = 1,
weight_init=tf.truncated_normal(
shape=[self.n_features, 1],
stddev=weight_init_stddevs[0]),
bias_init=tf.constant(value=bias_init_consts[0],
shape=[1]))
lg_list.append(lg)
return lg_list
def build(self, graph, name_scopes, training):
"""Constructs the graph architecture of model: n_tasks * sigmoid nodes.
This method creates the following Placeholders:
mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape
batch_size x n_features.
"""
placeholder_scope = TensorflowGraph.get_placeholder_scope(
graph, name_scopes)
n_features = self.n_features
with graph.as_default():
with placeholder_scope:
self.mol_features = tf.placeholder(tf.float32,
shape=[None, n_features],
name='mol_features')
weight_init_stddevs = self.weight_init_stddevs
bias_init_consts = self.bias_init_consts
lg_list = []
for task in range(self.n_tasks):
#setting up n_tasks nodes(output nodes)
lg = model_ops.fully_connected_layer(
tensor=self.mol_features,
size=1,
weight_init=tf.truncated_normal(
shape=[self.n_features, 1],
stddev=weight_init_stddevs[0]),
bias_init=tf.constant(value=bias_init_consts[0],
shape=[1]))
lg_list.append(lg)
return lg_list
def construct_graph(self, training, seed):
"""Returns a TensorflowGraph object."""
graph = tf.Graph()
# Lazily created by _get_shared_session().
shared_session = None
# Cache of TensorFlow scopes, to prevent '_1' appended scope names
# when subclass-overridden methods use the same scopes.
name_scopes = {}
# Setup graph
with graph.as_default():
if seed is not None:
tf.set_random_seed(seed)
features, labels, weights = self.add_placeholders(
graph, name_scopes)
outputs = self.add_progressive_lattice(graph, name_scopes,
training)
if training:
loss = self.add_task_training_costs(graph, name_scopes,
outputs, labels, weights)
else:
loss = None
return TensorflowGraph(graph=graph,
session=shared_session,
name_scopes=name_scopes,
output=outputs,
labels=labels,
weights=weights,
loss=loss)
def construct_feed_dict(self, X_b, y_b=None, w_b=None, ids_b=None):
"""Construct a feed dictionary from minibatch data.
TODO(rbharath): ids_b is not used here. Can we remove it?
Args:
X_b: np.ndarray of shape (batch_size, n_features)
y_b: np.ndarray of shape (batch_size, n_tasks)
w_b: np.ndarray of shape (batch_size, n_tasks)
ids_b: List of length (batch_size) with datapoint identifiers.
"""
orig_dict = {}
orig_dict["mol_features"] = X_b
for task in range(self.n_tasks):
if y_b is not None:
orig_dict["labels_%d" % task] = to_one_hot(y_b[:, task])
else:
# Dummy placeholders
orig_dict["labels_%d" % task] = np.squeeze(to_one_hot(
np.zeros((self.batch_size,))))
if w_b is not None:
orig_dict["weights_%d" % task] = w_b[:, task]
else:
# Dummy placeholders
orig_dict["weights_%d" % task] = np.ones(
(self.batch_size,))
return TensorflowGraph.get_feed_dict(orig_dict)
def build(self, graph, name_scopes, training):
"""Constructs the graph architecture of IRV as described in:
https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2750043/
"""
placeholder_scope = TensorflowGraph.get_placeholder_scope(
graph, name_scopes)
K = self.K
with graph.as_default():
output = []
with placeholder_scope:
mol_features = tf.placeholder(tf.float32,
shape=[None, self.n_features],
name='mol_features')
with tf.name_scope('variable'):
V = tf.Variable(tf.constant([0.01, 1.]),
name="vote",
dtype=tf.float32)
W = tf.Variable(tf.constant([1., 1.]),
name="w",
dtype=tf.float32)
b = tf.Variable(tf.constant([0.01]),
name="b",
dtype=tf.float32)
b2 = tf.Variable(tf.constant([0.01]),
name="b2",
dtype=tf.float32)
label_placeholders = self.add_label_placeholders(
graph, name_scopes)
weight_placeholders = self.add_example_weight_placeholders(
graph, name_scopes)
if training:
graph.queue = tf.FIFOQueue(
capacity=5,
dtypes=[tf.float32] *
(len(label_placeholders) + len(weight_placeholders) + 1))
graph.enqueue = graph.queue.enqueue([mol_features] +
label_placeholders +
weight_placeholders)
queue_outputs = graph.queue.dequeue()
labels = queue_outputs[1:len(label_placeholders) + 1]
weights = queue_outputs[len(label_placeholders) + 1:]
features = queue_outputs[0]
else:
labels = label_placeholders
weights = weight_placeholders
features = mol_features
for count in range(self.n_tasks):
similarity = features[:, 2 * K * count:(2 * K * count + K)]
ys = tf.to_int32(features[:, (2 * K * count + K):2 * K *
(count + 1)])
R = b + W[0] * similarity + W[1] * tf.constant(
np.arange(K) + 1, dtype=tf.float32)
R = tf.sigmoid(R)
z = tf.reduce_sum(R * tf.gather(V, ys), axis=1) + b2
output.append(tf.reshape(z, shape=[-1, 1]))
return (output, labels, weights)
def build(self, graph, name_scopes, training):
"""Constructs the graph architecture as specified in its config.
This method creates the following Placeholders:
mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape
batch_size x n_features.
"""
n_features = self.n_features
placeholder_scope = TensorflowGraph.get_placeholder_scope(
graph, name_scopes)
with graph.as_default():
with placeholder_scope:
self.mol_features = tf.placeholder(
tf.float32,
shape=[None, n_features],
name='mol_features')
layer_sizes = self.layer_sizes
weight_init_stddevs = self.weight_init_stddevs
bias_init_consts = self.bias_init_consts
dropouts = self.dropouts
lengths_set = {
len(layer_sizes),
len(weight_init_stddevs),
len(bias_init_consts),
len(dropouts),
}
assert len(lengths_set) == 1, 'All layer params must have same length.'
n_layers = lengths_set.pop()
assert n_layers > 0, 'Must have some layers defined.'
prev_layer = self.mol_features
prev_layer_size = n_features
for i in range(n_layers):
layer = tf.nn.relu(model_ops.fully_connected_layer(
tensor=prev_layer,
size=layer_sizes[i],
weight_init=tf.truncated_normal(
shape=[prev_layer_size, layer_sizes[i]],
stddev=weight_init_stddevs[i]),
bias_init=tf.constant(value=bias_init_consts[i],
shape=[layer_sizes[i]])))
layer = model_ops.dropout(layer, dropouts[i])
prev_layer = layer
prev_layer_size = layer_sizes[i]
output = []
for task in range(self.n_tasks):
output.append(tf.squeeze(
model_ops.fully_connected_layer(
tensor=prev_layer,
size=layer_sizes[i],
weight_init=tf.truncated_normal(
shape=[prev_layer_size, 1],
stddev=weight_init_stddevs[i]),
bias_init=tf.constant(value=bias_init_consts[i],
shape=[1]))))
return output
def build(self, graph, name_scopes, training):
"""Constructs the graph architecture of model: n_tasks * sigmoid nodes.
This method creates the following Placeholders:
mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape
batch_size x n_features.
"""
warnings.warn(
"TensorflowLogisticRegression is deprecated. "
"Will be removed in DeepChem 1.4.", DeprecationWarning)
placeholder_scope = TensorflowGraph.get_placeholder_scope(
graph, name_scopes)
n_features = self.n_features
with graph.as_default():
with placeholder_scope:
mol_features = tf.placeholder(tf.float32,
shape=[None, n_features],
name='mol_features')
weight_init_stddevs = self.weight_init_stddevs
bias_init_consts = self.bias_init_consts
lg_list = []
label_placeholders = self.add_label_placeholders(
graph, name_scopes)
weight_placeholders = self.add_example_weight_placeholders(
graph, name_scopes)
if training:
graph.queue = tf.FIFOQueue(
capacity=5,
dtypes=[tf.float32] *
(len(label_placeholders) + len(weight_placeholders) + 1))
graph.enqueue = graph.queue.enqueue([mol_features] +
label_placeholders +
weight_placeholders)
queue_outputs = graph.queue.dequeue()
labels = queue_outputs[1:len(label_placeholders) + 1]
weights = queue_outputs[len(label_placeholders) + 1:]
prev_layer = queue_outputs[0]
else:
labels = label_placeholders
weights = weight_placeholders
prev_layer = mol_features
for task in range(self.n_tasks):
#setting up n_tasks nodes(output nodes)
lg = model_ops.fully_connected_layer(
tensor=prev_layer,
size=1,
weight_init=tf.truncated_normal(
shape=[self.n_features, 1],
stddev=weight_init_stddevs[0]),
bias_init=tf.constant(value=bias_init_consts[0],
shape=[1]))
lg_list.append(lg)
return (lg_list, labels, weights)
def add_label_placeholders(self, graph, name_scopes):
#label placeholders with size batch_size * 1
labels = []
placeholder_scope = TensorflowGraph.get_placeholder_scope(graph, name_scopes)
with placeholder_scope:
for task in range(self.n_tasks):
labels.append(tf.identity(
tf.placeholder(tf.float32, shape=[None,1],
name='labels_%d' % task)))
return labels
def add_label_placeholders(self, graph, name_scopes):
#label placeholders with size batch_size * 1
labels = []
placeholder_scope = TensorflowGraph.get_placeholder_scope(graph, name_scopes)
with placeholder_scope:
for task in range(self.n_tasks):
labels.append(tf.identity(
tf.placeholder(tf.float32, shape=[None,1],
name='labels_%d' % task)))
return labels
def add_training_costs(self, graph, name_scopes, output, labels, weights):
with graph.as_default():
epsilon = 1e-3 # small float to avoid dividing by zero
weighted_costs = [] # weighted costs for each example
gradient_costs = [] # costs used for gradient calculation
with TensorflowGraph.shared_name_scope('costs', graph,
name_scopes):
for task in range(self.n_tasks):
task_str = str(task).zfill(len(str(self.n_tasks)))
with TensorflowGraph.shared_name_scope(
'cost_{}'.format(task_str), graph, name_scopes):
with tf.name_scope('weighted'):
weighted_cost = self.cost(output[task],
labels[task],
weights[task])
weighted_costs.append(weighted_cost)
with tf.name_scope('gradient'):
# Note that we divide by the batch size and not the number of
# non-zero weight examples in the batch. Also, instead of using
# tf.reduce_mean (which can put ops on the CPU) we explicitly
# calculate with div/sum so it stays on the GPU.
gradient_cost = tf.div(
tf.reduce_sum(weighted_cost), self.batch_size)
gradient_costs.append(gradient_cost)
# aggregated costs
with TensorflowGraph.shared_name_scope('aggregated', graph,
name_scopes):
with tf.name_scope('gradient'):
loss = tf.add_n(gradient_costs)
# weight decay
if self.penalty != 0.0:
penalty = model_ops.weight_decay(
self.penalty_type, self.penalty)
loss += penalty
return loss
def add_task_training_costs(self, graph, name_scopes, outputs, labels,
weights):
"""Adds the training costs for each task.
Since each task is trained separately, each task is optimized w.r.t a separate
task.
TODO(rbharath): Figure out how to support weight decay for this model.
Since each task is trained separately, weight decay should only be used
on weights in column for that task.
Parameters
----------
graph: tf.Graph
Graph for the model.
name_scopes: dict
Contains all the scopes for model
outputs: list
List of output tensors from model.
weights: list
List of weight placeholders for model.
"""
task_costs = {}
with TensorflowGraph.shared_name_scope('costs', graph, name_scopes):
for task in range(self.n_tasks):
with TensorflowGraph.shared_name_scope('cost_%d' % task, graph,
name_scopes):
weighted_cost = self.cost(outputs[task], labels[task],
weights[task])
# Note that we divide by the batch size and not the number of
# non-zero weight examples in the batch. Also, instead of using
# tf.reduce_mean (which can put ops on the CPU) we explicitly
# calculate with div/sum so it stays on the GPU.
task_cost = tf.div(tf.reduce_sum(weighted_cost),
self.batch_size)
task_costs[task] = task_cost
return task_costs
def add_training_cost(self, graph, name_scopes, output, labels, weights):
with graph.as_default():
epsilon = 1e-3 # small float to avoid dividing by zero
weighted_costs = [] # weighted costs for each example
gradient_costs = [] # costs used for gradient calculation
with TensorflowGraph.shared_name_scope('costs', graph, name_scopes):
for task in range(self.n_tasks):
task_str = str(task).zfill(len(str(self.n_tasks)))
with TensorflowGraph.shared_name_scope('cost_{}'.format(task_str),
graph, name_scopes):
with tf.name_scope('weighted'):
weighted_cost = self.cost(output[task], labels[task],
weights[task])
weighted_costs.append(weighted_cost)
with tf.name_scope('gradient'):
# Note that we divide by the batch size and not the number of
# non-zero weight examples in the batch. Also, instead of using
# tf.reduce_mean (which can put ops on the CPU) we explicitly
# calculate with div/sum so it stays on the GPU.
gradient_cost = tf.div(
tf.reduce_sum(weighted_cost), self.batch_size)
gradient_costs.append(gradient_cost)
# aggregated costs
with TensorflowGraph.shared_name_scope('aggregated', graph,
name_scopes):
with tf.name_scope('gradient'):
loss = tf.add_n(gradient_costs)
# weight decay
if self.penalty != 0.0:
# using self-defined regularization
penalty = weight_decay(self.penalty_type, self.penalty)
loss += penalty
return loss
def add_task_training_costs(self, graph, name_scopes, outputs, labels,
weights):
"""Adds the training costs for each task.
Since each task is trained separately, each task is optimized w.r.t a separate
task.
TODO(rbharath): Figure out how to support weight decay for this model.
Since each task is trained separately, weight decay should only be used
on weights in column for that task.
Parameters
----------
graph: tf.Graph
Graph for the model.
name_scopes: dict
Contains all the scopes for model
outputs: list
List of output tensors from model.
weights: list
List of weight placeholders for model.
"""
task_costs = {}
with TensorflowGraph.shared_name_scope('costs', graph, name_scopes):
for task in range(self.n_tasks):
with TensorflowGraph.shared_name_scope('cost_%d' % task, graph,
name_scopes):
weighted_cost = self.cost(outputs[task], labels[task], weights[task])
# Note that we divide by the batch size and not the number of
# non-zero weight examples in the batch. Also, instead of using
# tf.reduce_mean (which can put ops on the CPU) we explicitly
# calculate with div/sum so it stays on the GPU.
task_cost = tf.div(tf.reduce_sum(weighted_cost), self.batch_size)
task_costs[task] = task_cost
return task_costs
def build(self, graph, name_scopes, training):
"""Constructs the graph architecture of model: n_tasks * sigmoid nodes.
This method creates the following Placeholders:
mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape
batch_size x n_features.
"""
placeholder_scope = TensorflowGraph.get_placeholder_scope(graph,
name_scopes)
n_features = self.n_features
with graph.as_default():
with placeholder_scope:
mol_features = tf.placeholder(
tf.float32, shape=[None, n_features], name='mol_features')
weight_init_stddevs = self.weight_init_stddevs
bias_init_consts = self.bias_init_consts
lg_list = []
label_placeholders = self.add_label_placeholders(graph, name_scopes)
weight_placeholders = self.add_example_weight_placeholders(graph,
name_scopes)
if training:
graph.queue = tf.FIFOQueue(
capacity=5,
dtypes=[tf.float32] *
(len(label_placeholders) + len(weight_placeholders) + 1))
graph.enqueue = graph.queue.enqueue([mol_features] + label_placeholders
+ weight_placeholders)
queue_outputs = graph.queue.dequeue()
labels = queue_outputs[1:len(label_placeholders) + 1]
weights = queue_outputs[len(label_placeholders) + 1:]
prev_layer = queue_outputs[0]
else:
labels = label_placeholders
weights = weight_placeholders
prev_layer = mol_features
for task in range(self.n_tasks):
#setting up n_tasks nodes(output nodes)
lg = model_ops.fully_connected_layer(
tensor=prev_layer,
size=1,
weight_init=tf.truncated_normal(
shape=[self.n_features, 1], stddev=weight_init_stddevs[0]),
bias_init=tf.constant(value=bias_init_consts[0], shape=[1]))
lg_list.append(lg)
return (lg_list, labels, weights)
def build(self, graph, name_scopes, training):
"""Constructs the graph architecture of IRV as described in:
https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2750043/
"""
placeholder_scope = TensorflowGraph.get_placeholder_scope(graph,
name_scopes)
K = self.K
with graph.as_default():
output = []
with placeholder_scope:
mol_features = tf.placeholder(
tf.float32, shape=[None, self.n_features], name='mol_features')
with tf.name_scope('variable'):
V = tf.Variable(tf.constant([0.01, 1.]), name="vote", dtype=tf.float32)
W = tf.Variable(tf.constant([1., 1.]), name="w", dtype=tf.float32)
b = tf.Variable(tf.constant([0.01]), name="b", dtype=tf.float32)
b2 = tf.Variable(tf.constant([0.01]), name="b2", dtype=tf.float32)
label_placeholders = self.add_label_placeholders(graph, name_scopes)
weight_placeholders = self.add_example_weight_placeholders(graph,
name_scopes)
if training:
graph.queue = tf.FIFOQueue(
capacity=5,
dtypes=[tf.float32] *
(len(label_placeholders) + len(weight_placeholders) + 1))
graph.enqueue = graph.queue.enqueue([mol_features] + label_placeholders
+ weight_placeholders)
queue_outputs = graph.queue.dequeue()
labels = queue_outputs[1:len(label_placeholders) + 1]
weights = queue_outputs[len(label_placeholders) + 1:]
features = queue_outputs[0]
else:
labels = label_placeholders
weights = weight_placeholders
features = mol_features
for count in range(self.n_tasks):
similarity = features[:, 2 * K * count:(2 * K * count + K)]
ys = tf.to_int32(features[:, (2 * K * count + K):2 * K * (count + 1)])
R = b + W[0] * similarity + W[1] * tf.constant(
np.arange(K) + 1, dtype=tf.float32)
R = tf.sigmoid(R)
z = tf.reduce_sum(R * tf.gather(V, ys), axis=1) + b2
output.append(tf.reshape(z, shape=[-1, 1]))
return (output, labels, weights)
def construct_feed_dict(self, X_b, y_b=None, w_b=None, ids_b=None):
orig_dict = {}
orig_dict["mol_features"] = X_b
for task in range(self.n_tasks):
if y_b is not None:
y_2column = to_one_hot(y_b[:, task])
# fix the size to be [?,1]
orig_dict["labels_%d" % task] = y_2column[:, 1:2]
else:
# Dummy placeholders
orig_dict["labels_%d" % task] = np.zeros((self.batch_size, 1))
if w_b is not None:
orig_dict["weights_%d" % task] = w_b[:, task]
else:
# Dummy placeholders
orig_dict["weights_%d" % task] = np.ones((self.batch_size, ))
return TensorflowGraph.get_feed_dict(orig_dict)
def construct_feed_dict(self, X_b, y_b=None, w_b=None, ids_b=None):
orig_dict = {}
orig_dict["mol_features"] = X_b
for task in range(self.n_tasks):
if y_b is not None:
y_2column = to_one_hot(y_b[:, task])
# fix the size to be [?,1]
orig_dict["labels_%d" % task] = y_2column[:, 1:2]
else:
# Dummy placeholders
orig_dict["labels_%d" % task] = np.zeros((self.batch_size, 1))
if w_b is not None:
orig_dict["weights_%d" % task] = w_b[:, task]
else:
# Dummy placeholders
orig_dict["weights_%d" % task] = np.ones((self.batch_size,))
return TensorflowGraph.get_feed_dict(orig_dict)
def add_placeholders(self, graph, name_scopes):
"""Adds all placeholders for this model."""
# Create placeholders
placeholder_scope = TensorflowGraph.get_placeholder_scope(
graph, name_scopes)
labels, weights = [], []
n_features = self.n_features
with placeholder_scope:
self.mol_features = tf.placeholder(
tf.float32,
shape=[None, n_features],
name='mol_features')
for task in range(self.n_tasks):
weights.append(tf.identity(
tf.placeholder(tf.float32, shape=[None,],
name='weights_%d' % task)))
labels.append(tf.identity(
tf.placeholder(tf.float32, shape=[None,],
name='labels_%d' % task)))
return self.mol_features, labels, weights
def build(self, graph, name_scopes, training):
"""Constructs the graph architecture as specified in its config.
This method creates the following Placeholders:
mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape
batch_size x n_features.
"""
placeholder_scope = TensorflowGraph.get_placeholder_scope(graph,
name_scopes)
n_features = self.n_features
with graph.as_default():
with placeholder_scope:
mol_features = tf.placeholder(
tf.float32, shape=[None, n_features], name='mol_features')
layer_sizes = self.layer_sizes
weight_init_stddevs = self.weight_init_stddevs
bias_init_consts = self.bias_init_consts
dropouts = self.dropouts
lengths_set = {
len(layer_sizes),
len(weight_init_stddevs),
len(bias_init_consts),
len(dropouts),
}
assert len(lengths_set) == 1, 'All layer params must have same length.'
n_layers = lengths_set.pop()
assert n_layers > 0, 'Must have some layers defined.'
label_placeholders = self.add_label_placeholders(graph, name_scopes)
weight_placeholders = self.add_example_weight_placeholders(graph,
name_scopes)
if training:
graph.queue = tf.FIFOQueue(
capacity=5,
dtypes=[tf.float32] *
(len(label_placeholders) + len(weight_placeholders) + 1))
graph.enqueue = graph.queue.enqueue([mol_features] + label_placeholders
+ weight_placeholders)
queue_outputs = graph.queue.dequeue()
labels = queue_outputs[1:len(label_placeholders) + 1]
weights = queue_outputs[len(label_placeholders) + 1:]
prev_layer = queue_outputs[0]
else:
labels = label_placeholders
weights = weight_placeholders
prev_layer = mol_features
prev_layer_size = n_features
for i in range(n_layers):
layer = tf.nn.relu(
model_ops.fully_connected_layer(
tensor=prev_layer,
size=layer_sizes[i],
weight_init=tf.truncated_normal(
shape=[prev_layer_size, layer_sizes[i]],
stddev=weight_init_stddevs[i]),
bias_init=tf.constant(
value=bias_init_consts[i], shape=[layer_sizes[i]])))
layer = model_ops.dropout(layer, dropouts[i], training)
prev_layer = layer
prev_layer_size = layer_sizes[i]
output = model_ops.multitask_logits(layer, self.n_tasks)
return (output, labels, weights)
def build(self, graph, name_scopes, training):
"""Constructs the graph architecture as specified in its config.
This method creates the following Placeholders:
mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape
batch_size x num_features.
"""
num_features = self.n_features
placeholder_scope = TensorflowGraph.get_placeholder_scope(
graph, name_scopes)
with graph.as_default():
with placeholder_scope:
self.mol_features = tf.placeholder(tf.float32,
shape=[None, num_features],
name='mol_features')
layer_sizes = self.layer_sizes
weight_init_stddevs = self.weight_init_stddevs
bias_init_consts = self.bias_init_consts
dropouts = self.dropouts
bypass_layer_sizes = self.bypass_layer_sizes
bypass_weight_init_stddevs = self.bypass_weight_init_stddevs
bypass_bias_init_consts = self.bypass_bias_init_consts
bypass_dropouts = self.bypass_dropouts
lengths_set = {
len(layer_sizes),
len(weight_init_stddevs),
len(bias_init_consts),
len(dropouts),
}
assert len(
lengths_set) == 1, "All layer params must have same length."
num_layers = lengths_set.pop()
assert num_layers > 0, "Must have some layers defined."
bypass_lengths_set = {
len(bypass_layer_sizes),
len(bypass_weight_init_stddevs),
len(bypass_bias_init_consts),
len(bypass_dropouts),
}
assert len(bypass_lengths_set) == 1, ("All bypass_layer params" +
" must have same length.")
num_bypass_layers = bypass_lengths_set.pop()
prev_layer = self.mol_features
prev_layer_size = num_features
for i in range(num_layers):
# layer has shape [None, layer_sizes[i]]
########################################################## DEBUG
print("Adding weights of shape %s" %
str([prev_layer_size, layer_sizes[i]]))
########################################################## DEBUG
layer = tf.nn.relu(
model_ops.fully_connected_layer(
tensor=prev_layer,
size=layer_sizes[i],
weight_init=tf.truncated_normal(
shape=[prev_layer_size, layer_sizes[i]],
stddev=weight_init_stddevs[i]),
bias_init=tf.constant(value=bias_init_consts[i],
shape=[layer_sizes[i]])))
layer = model_ops.dropout(layer, dropouts[i], training)
prev_layer = layer
prev_layer_size = layer_sizes[i]
output = []
# top_multitask_layer has shape [None, layer_sizes[-1]]
top_multitask_layer = prev_layer
for task in range(self.n_tasks):
# TODO(rbharath): Might want to make it feasible to have multiple
# bypass layers.
# Construct task bypass layer
prev_bypass_layer = self.mol_features
prev_bypass_layer_size = num_features
for i in range(num_bypass_layers):
# bypass_layer has shape [None, bypass_layer_sizes[i]]
########################################################## DEBUG
print("Adding bypass weights of shape %s" %
str([prev_bypass_layer_size, bypass_layer_sizes[i]]))
########################################################## DEBUG
bypass_layer = tf.nn.relu(
model_ops.fully_connected_layer(
tensor=prev_bypass_layer,
size=bypass_layer_sizes[i],
weight_init=tf.truncated_normal(
shape=[
prev_bypass_layer_size,
bypass_layer_sizes[i]
],
stddev=bypass_weight_init_stddevs[i]),
bias_init=tf.constant(
value=bypass_bias_init_consts[i],
shape=[bypass_layer_sizes[i]])))
bypass_layer = model_ops.dropout(bypass_layer,
bypass_dropouts[i])
prev_bypass_layer = bypass_layer
prev_bypass_layer_size = bypass_layer_sizes[i]
top_bypass_layer = prev_bypass_layer
if num_bypass_layers > 0:
# task_layer has shape [None, layer_sizes[-1] + bypass_layer_sizes[-1]]
task_layer = tf.concat(
1, [top_multitask_layer, top_bypass_layer])
task_layer_size = layer_sizes[-1] + bypass_layer_sizes[-1]
else:
task_layer = top_multitask_layer
task_layer_size = layer_sizes[-1]
########################################################## DEBUG
print("Adding output weights of shape %s" %
str([task_layer_size, 1]))
########################################################## DEBUG
#################################################### DEBUG
print("task_layer_size")
print(task_layer_size)
#################################################### DEBUG
output.append(
tf.squeeze(
model_ops.logits(task_layer,
num_classes=2,
weight_init=tf.truncated_normal(
shape=[task_layer_size, 2],
stddev=weight_init_stddevs[-1]),
bias_init=tf.constant(
value=bias_init_consts[-1],
shape=[2]))))
return output
def build(self, graph, name_scopes, training):
"""Constructs the graph architecture as specified in its config.
This method creates the following Placeholders:
mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape
batch_size x n_features.
"""
n_features = self.n_features
placeholder_scope = TensorflowGraph.get_placeholder_scope(
graph, name_scopes)
with graph.as_default():
with placeholder_scope:
self.mol_features = tf.placeholder(tf.float32,
shape=[None, n_features],
name='mol_features')
layer_sizes = self.layer_sizes
weight_init_stddevs = self.weight_init_stddevs
bias_init_consts = self.bias_init_consts
dropouts = self.dropouts
lengths_set = {
len(layer_sizes),
len(weight_init_stddevs),
len(bias_init_consts),
len(dropouts),
}
assert len(
lengths_set) == 1, 'All layer params must have same length.'
n_layers = lengths_set.pop()
assert n_layers > 0, 'Must have some layers defined.'
prev_layer = self.mol_features
prev_layer_size = n_features
all_layers = {}
for i in range(n_layers):
for task in range(self.n_tasks):
task_scope = TensorflowGraph.shared_name_scope(
"task%d" % task, graph, name_scopes)
print("Adding weights for task %d, layer %d" % (task, i))
with task_scope as scope:
if i == 0:
prev_layer = self.mol_features
prev_layer_size = self.n_features
else:
prev_layer = all_layers[(i - 1, task)]
prev_layer_size = layer_sizes[i - 1]
if task > 0:
lateral_contrib = self.add_adapter(
all_layers, task, i)
print(
"Creating W_layer_%d_task%d of shape %s" %
(i, task, str([prev_layer_size, layer_sizes[i]])))
W = tf.Variable(tf.truncated_normal(
shape=[prev_layer_size, layer_sizes[i]],
stddev=self.weight_init_stddevs[i]),
name='W_layer_%d_task%d' % (i, task),
dtype=tf.float32)
print("Creating b_layer_%d_task%d of shape %s" %
(i, task, str([layer_sizes[i]])))
b = tf.Variable(tf.constant(
value=self.bias_init_consts[i],
shape=[layer_sizes[i]]),
name='b_layer_%d_task%d' % (i, task),
dtype=tf.float32)
layer = tf.matmul(prev_layer, W) + b
if i > 0 and task > 0:
layer = layer + lateral_contrib
layer = tf.nn.relu(layer)
layer = model_ops.dropout(layer, dropouts[i], training)
all_layers[(i, task)] = layer
output = []
for task in range(self.n_tasks):
prev_layer = all_layers[(i, task)]
prev_layer_size = layer_sizes[i]
task_scope = TensorflowGraph.shared_name_scope(
"task%d" % task, graph, name_scopes)
with task_scope as scope:
if task > 0:
lateral_contrib = tf.squeeze(
self.add_adapter(all_layers, task, i + 1))
weight_init = tf.truncated_normal(
shape=[prev_layer_size, 1],
stddev=weight_init_stddevs[i])
bias_init = tf.constant(value=bias_init_consts[i],
shape=[1])
print("Creating W_output_task%d of shape %s" %
(task, str([prev_layer_size, 1])))
w = tf.Variable(weight_init,
name='W_output_task%d' % task,
dtype=tf.float32)
print("Creating b_output_task%d of shape %s" %
(task, str([1])))
b = tf.Variable(bias_init,
name='b_output_task%d' % task,
dtype=tf.float32)
layer = tf.squeeze(tf.matmul(prev_layer, w) + b)
if i > 0 and task > 0:
layer = layer + lateral_contrib
output.append(layer)
return output
def build(self, graph, name_scopes, training):
"""Constructs the graph architecture as specified in its config.
This method creates the following Placeholders:
mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape
batch_size x num_features.
"""
num_features = self.n_features
placeholder_scope = TensorflowGraph.get_placeholder_scope(
graph, name_scopes)
with graph.as_default():
with placeholder_scope:
self.mol_features = tf.placeholder(
tf.float32,
shape=[None, num_features],
name='mol_features')
layer_sizes = self.layer_sizes
weight_init_stddevs = self.weight_init_stddevs
bias_init_consts = self.bias_init_consts
dropouts = self.dropouts
bypass_layer_sizes = self.bypass_layer_sizes
bypass_weight_init_stddevs = self.bypass_weight_init_stddevs
bypass_bias_init_consts = self.bypass_bias_init_consts
bypass_dropouts = self.bypass_dropouts
lengths_set = {
len(layer_sizes),
len(weight_init_stddevs),
len(bias_init_consts),
len(dropouts),
}
assert len(lengths_set) == 1, "All layer params must have same length."
num_layers = lengths_set.pop()
assert num_layers > 0, "Must have some layers defined."
bypass_lengths_set = {
len(bypass_layer_sizes),
len(bypass_weight_init_stddevs),
len(bypass_bias_init_consts),
len(bypass_dropouts),
}
assert (len(bypass_lengths_set) == 1,
"All bypass_layer params must have same length.")
num_bypass_layers = bypass_lengths_set.pop()
prev_layer = self.mol_features
prev_layer_size = num_features
for i in range(num_layers):
# layer has shape [None, layer_sizes[i]]
########################################################## DEBUG
print("Adding weights of shape %s" % str([prev_layer_size, layer_sizes[i]]))
########################################################## DEBUG
layer = tf.nn.relu(model_ops.fully_connected_layer(
tensor=prev_layer,
size=layer_sizes[i],
weight_init=tf.truncated_normal(
shape=[prev_layer_size, layer_sizes[i]],
stddev=weight_init_stddevs[i]),
bias_init=tf.constant(value=bias_init_consts[i],
shape=[layer_sizes[i]])))
layer = model_ops.dropout(layer, dropouts[i], training)
prev_layer = layer
prev_layer_size = layer_sizes[i]
output = []
# top_multitask_layer has shape [None, layer_sizes[-1]]
top_multitask_layer = prev_layer
for task in range(self.n_tasks):
# TODO(rbharath): Might want to make it feasible to have multiple
# bypass layers.
# Construct task bypass layer
prev_bypass_layer = self.mol_features
prev_bypass_layer_size = num_features
for i in range(num_bypass_layers):
# bypass_layer has shape [None, bypass_layer_sizes[i]]
########################################################## DEBUG
print("Adding bypass weights of shape %s"
% str([prev_bypass_layer_size, bypass_layer_sizes[i]]))
########################################################## DEBUG
bypass_layer = tf.nn.relu(model_ops.fully_connected_layer(
tensor = prev_bypass_layer,
size = bypass_layer_sizes[i],
weight_init=tf.truncated_normal(
shape=[prev_bypass_layer_size, bypass_layer_sizes[i]],
stddev=bypass_weight_init_stddevs[i]),
bias_init=tf.constant(value=bypass_bias_init_consts[i],
shape=[bypass_layer_sizes[i]])))
bypass_layer = model_ops.dropout(bypass_layer, bypass_dropouts[i])
prev_bypass_layer = bypass_layer
prev_bypass_layer_size = bypass_layer_sizes[i]
top_bypass_layer = prev_bypass_layer
if num_bypass_layers > 0:
# task_layer has shape [None, layer_sizes[-1] + bypass_layer_sizes[-1]]
task_layer = tf.concat(1, [top_multitask_layer, top_bypass_layer])
task_layer_size = layer_sizes[-1] + bypass_layer_sizes[-1]
else:
task_layer = top_multitask_layer
task_layer_size = layer_sizes[-1]
########################################################## DEBUG
print("Adding output weights of shape %s"
% str([task_layer_size, 1]))
########################################################## DEBUG
#################################################### DEBUG
print("task_layer_size")
print(task_layer_size)
#################################################### DEBUG
output.append(tf.squeeze(
model_ops.fully_connected_layer(
tensor=task_layer,
size=task_layer_size,
weight_init=tf.truncated_normal(
shape=[task_layer_size, 1],
stddev=weight_init_stddevs[-1]),
bias_init=tf.constant(value=bias_init_consts[-1],
shape=[1]))))
return output
def build(self, graph, name_scopes, training):
"""Constructs the graph architecture as specified in its config.
This method creates the following Placeholders:
mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape
batch_size x n_features.
"""
placeholder_scope = TensorflowGraph.get_placeholder_scope(
graph, name_scopes)
n_features = self.n_features
with graph.as_default():
with placeholder_scope:
mol_features = tf.placeholder(tf.float32,
shape=[None, n_features],
name='mol_features')
layer_sizes = self.layer_sizes
weight_init_stddevs = self.weight_init_stddevs
bias_init_consts = self.bias_init_consts
dropouts = self.dropouts
lengths_set = {
len(layer_sizes),
len(weight_init_stddevs),
len(bias_init_consts),
len(dropouts),
}
assert len(
lengths_set) == 1, 'All layer params must have same length.'
n_layers = lengths_set.pop()
assert n_layers > 0, 'Must have some layers defined.'
label_placeholders = self.add_label_placeholders(
graph, name_scopes)
weight_placeholders = self.add_example_weight_placeholders(
graph, name_scopes)
if training:
graph.queue = tf.FIFOQueue(
capacity=5,
dtypes=[tf.float32] *
(len(label_placeholders) + len(weight_placeholders) + 1))
graph.enqueue = graph.queue.enqueue([mol_features] +
label_placeholders +
weight_placeholders)
queue_outputs = graph.queue.dequeue()
labels = queue_outputs[1:len(label_placeholders) + 1]
weights = queue_outputs[len(label_placeholders) + 1:]
prev_layer = queue_outputs[0]
else:
labels = label_placeholders
weights = weight_placeholders
prev_layer = mol_features
prev_layer_size = n_features
for i in range(n_layers):
layer = tf.nn.relu(
model_ops.fully_connected_layer(
tensor=prev_layer,
size=layer_sizes[i],
weight_init=tf.truncated_normal(
shape=[prev_layer_size, layer_sizes[i]],
stddev=weight_init_stddevs[i]),
bias_init=tf.constant(value=bias_init_consts[i],
shape=[layer_sizes[i]])))
layer = model_ops.dropout(layer, dropouts[i], training)
prev_layer = layer
prev_layer_size = layer_sizes[i]
output = model_ops.multitask_logits(layer, self.n_tasks)
return (output, labels, weights)
def build(self, graph, name_scopes, training):
"""Constructs the graph architecture as specified in its config.
This method creates the following Placeholders:
mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape
batch_size x num_features.
"""
num_features = self.n_features
placeholder_scope = TensorflowGraph.get_placeholder_scope(
graph, name_scopes)
with graph.as_default():
with placeholder_scope:
mol_features = tf.placeholder(
tf.float32, shape=[None, num_features], name='mol_features')
layer_sizes = self.layer_sizes
weight_init_stddevs = self.weight_init_stddevs
bias_init_consts = self.bias_init_consts
dropouts = self.dropouts
bypass_layer_sizes = self.bypass_layer_sizes
bypass_weight_init_stddevs = self.bypass_weight_init_stddevs
bypass_bias_init_consts = self.bypass_bias_init_consts
bypass_dropouts = self.bypass_dropouts
lengths_set = {
len(layer_sizes),
len(weight_init_stddevs),
len(bias_init_consts),
len(dropouts),
}
assert len(lengths_set) == 1, "All layer params must have same length."
num_layers = lengths_set.pop()
assert num_layers > 0, "Must have some layers defined."
bypass_lengths_set = {
len(bypass_layer_sizes),
len(bypass_weight_init_stddevs),
len(bypass_bias_init_consts),
len(bypass_dropouts),
}
assert len(bypass_lengths_set) == 1, (
"All bypass_layer params" + " must have same length.")
num_bypass_layers = bypass_lengths_set.pop()
label_placeholders = self.add_label_placeholders(graph, name_scopes)
weight_placeholders = self.add_example_weight_placeholders(
graph, name_scopes)
if training:
graph.queue = tf.FIFOQueue(
capacity=5,
dtypes=[tf.float32] *
(len(label_placeholders) + len(weight_placeholders) + 1))
graph.enqueue = graph.queue.enqueue([mol_features] + label_placeholders
+ weight_placeholders)
queue_outputs = graph.queue.dequeue()
labels = queue_outputs[1:len(label_placeholders) + 1]
weights = queue_outputs[len(label_placeholders) + 1:]
prev_layer = queue_outputs[0]
else:
labels = label_placeholders
weights = weight_placeholders
prev_layer = mol_features
top_layer = prev_layer
prev_layer_size = num_features
for i in range(num_layers):
# layer has shape [None, layer_sizes[i]]
print("Adding weights of shape %s" % str(
[prev_layer_size, layer_sizes[i]]))
layer = tf.nn.relu(
model_ops.fully_connected_layer(
tensor=prev_layer,
size=layer_sizes[i],
weight_init=tf.truncated_normal(
shape=[prev_layer_size, layer_sizes[i]],
stddev=weight_init_stddevs[i]),
bias_init=tf.constant(
value=bias_init_consts[i], shape=[layer_sizes[i]])))
layer = model_ops.dropout(layer, dropouts[i], training)
prev_layer = layer
prev_layer_size = layer_sizes[i]
output = []
# top_multitask_layer has shape [None, layer_sizes[-1]]
top_multitask_layer = prev_layer
for task in range(self.n_tasks):
# TODO(rbharath): Might want to make it feasible to have multiple
# bypass layers.
# Construct task bypass layer
prev_bypass_layer = top_layer
prev_bypass_layer_size = num_features
for i in range(num_bypass_layers):
# bypass_layer has shape [None, bypass_layer_sizes[i]]
print("Adding bypass weights of shape %s" % str(
[prev_bypass_layer_size, bypass_layer_sizes[i]]))
bypass_layer = tf.nn.relu(
model_ops.fully_connected_layer(
tensor=prev_bypass_layer,
size=bypass_layer_sizes[i],
weight_init=tf.truncated_normal(
shape=[prev_bypass_layer_size, bypass_layer_sizes[i]],
stddev=bypass_weight_init_stddevs[i]),
bias_init=tf.constant(
value=bypass_bias_init_consts[i],
shape=[bypass_layer_sizes[i]])))
bypass_layer = model_ops.dropout(bypass_layer, bypass_dropouts[i],
training)
prev_bypass_layer = bypass_layer
prev_bypass_layer_size = bypass_layer_sizes[i]
top_bypass_layer = prev_bypass_layer
if num_bypass_layers > 0:
# task_layer has shape [None, layer_sizes[-1] + bypass_layer_sizes[-1]]
task_layer = tf.concat(
axis=1, values=[top_multitask_layer, top_bypass_layer])
task_layer_size = layer_sizes[-1] + bypass_layer_sizes[-1]
else:
task_layer = top_multitask_layer
task_layer_size = layer_sizes[-1]
print("Adding output weights of shape %s" % str([task_layer_size, 1]))
output.append(
tf.squeeze(
model_ops.fully_connected_layer(
tensor=task_layer,
size=1,
weight_init=tf.truncated_normal(
shape=[task_layer_size, 1],
stddev=weight_init_stddevs[-1]),
bias_init=tf.constant(
value=bias_init_consts[-1], shape=[1])),
axis=1))
return (output, labels, weights)
def add_progressive_lattice(self, graph, name_scopes, training):
"""Constructs the graph architecture as specified in its config.
This method creates the following Placeholders:
mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape
batch_size x n_features.
"""
n_features = self.n_features
placeholder_scope = TensorflowGraph.get_placeholder_scope(graph,
name_scopes)
with graph.as_default():
layer_sizes = self.layer_sizes
weight_init_stddevs = self.weight_init_stddevs
bias_init_consts = self.bias_init_consts
dropouts = self.dropouts
lengths_set = {
len(layer_sizes),
len(weight_init_stddevs),
len(bias_init_consts),
len(dropouts),
}
assert len(lengths_set) == 1, 'All layer params must have same length.'
n_layers = lengths_set.pop()
assert n_layers > 0, 'Must have some layers defined.'
prev_layer = self.mol_features
prev_layer_size = n_features
all_layers = {}
for i in range(n_layers):
for task in range(self.n_tasks):
task_scope = TensorflowGraph.shared_name_scope("task%d_ops" % task,
graph, name_scopes)
print("Adding weights for task %d, layer %d" % (task, i))
with task_scope as scope:
if i == 0:
prev_layer = self.mol_features
prev_layer_size = self.n_features
else:
prev_layer = all_layers[(i - 1, task)]
prev_layer_size = layer_sizes[i - 1]
if task > 0:
lateral_contrib = self.add_adapter(all_layers, task, i)
print("Creating W_layer_%d_task%d of shape %s" %
(i, task, str([prev_layer_size, layer_sizes[i]])))
W = tf.Variable(
tf.truncated_normal(
shape=[prev_layer_size, layer_sizes[i]],
stddev=self.weight_init_stddevs[i]),
name='W_layer_%d_task%d' % (i, task),
dtype=tf.float32)
print("Creating b_layer_%d_task%d of shape %s" %
(i, task, str([layer_sizes[i]])))
b = tf.Variable(
tf.constant(
value=self.bias_init_consts[i], shape=[layer_sizes[i]]),
name='b_layer_%d_task%d' % (i, task),
dtype=tf.float32)
layer = tf.matmul(prev_layer, W) + b
if i > 0 and task > 0:
layer = layer + lateral_contrib
layer = tf.nn.relu(layer)
layer = model_ops.dropout(layer, dropouts[i], training)
all_layers[(i, task)] = layer
output = []
for task in range(self.n_tasks):
prev_layer = all_layers[(i, task)]
prev_layer_size = layer_sizes[i]
task_scope = TensorflowGraph.shared_name_scope("task%d" % task, graph,
name_scopes)
with task_scope as scope:
if task > 0:
lateral_contrib = tf.squeeze(
self.add_adapter(all_layers, task, i + 1))
weight_init = tf.truncated_normal(
shape=[prev_layer_size, 1], stddev=weight_init_stddevs[i])
bias_init = tf.constant(value=bias_init_consts[i], shape=[1])
print("Creating W_output_task%d of shape %s" %
(task, str([prev_layer_size, 1])))
w = tf.Variable(
weight_init, name='W_output_task%d' % task, dtype=tf.float32)
print("Creating b_output_task%d of shape %s" % (task, str([1])))
b = tf.Variable(
bias_init, name='b_output_task%d' % task, dtype=tf.float32)
layer = tf.squeeze(tf.matmul(prev_layer, w) + b)
if i > 0 and task > 0:
layer = layer + lateral_contrib
output.append(layer)
return output
def build(self, graph, name_scopes, training):
"""Constructs the graph architecture as specified in its config.
This method creates the following Placeholders:
mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape
batch_size x num_features.
"""
num_features = self.n_features
placeholder_scope = TensorflowGraph.get_placeholder_scope(graph,
name_scopes)
with graph.as_default():
with placeholder_scope:
mol_features = tf.placeholder(
tf.float32, shape=[None, num_features], name='mol_features')
layer_sizes = self.layer_sizes
weight_init_stddevs = self.weight_init_stddevs
bias_init_consts = self.bias_init_consts
dropouts = self.dropouts
bypass_layer_sizes = self.bypass_layer_sizes
bypass_weight_init_stddevs = self.bypass_weight_init_stddevs
bypass_bias_init_consts = self.bypass_bias_init_consts
bypass_dropouts = self.bypass_dropouts
lengths_set = {
len(layer_sizes),
len(weight_init_stddevs),
len(bias_init_consts),
len(dropouts),
}
assert len(lengths_set) == 1, "All layer params must have same length."
num_layers = lengths_set.pop()
assert num_layers > 0, "Must have some layers defined."
bypass_lengths_set = {
len(bypass_layer_sizes),
len(bypass_weight_init_stddevs),
len(bypass_bias_init_consts),
len(bypass_dropouts),
}
assert len(bypass_lengths_set) == 1, (
"All bypass_layer params" + " must have same length.")
num_bypass_layers = bypass_lengths_set.pop()
label_placeholders = self.add_label_placeholders(graph, name_scopes)
weight_placeholders = self.add_example_weight_placeholders(graph,
name_scopes)
if training:
graph.queue = tf.FIFOQueue(
capacity=5,
dtypes=[tf.float32] *
(len(label_placeholders) + len(weight_placeholders) + 1))
graph.enqueue = graph.queue.enqueue([mol_features] + label_placeholders
+ weight_placeholders)
queue_outputs = graph.queue.dequeue()
labels = queue_outputs[1:len(label_placeholders) + 1]
weights = queue_outputs[len(label_placeholders) + 1:]
prev_layer = queue_outputs[0]
else:
labels = label_placeholders
weights = weight_placeholders
prev_layer = mol_features
top_layer = prev_layer
prev_layer_size = num_features
for i in range(num_layers):
# layer has shape [None, layer_sizes[i]]
print("Adding weights of shape %s" %
str([prev_layer_size, layer_sizes[i]]))
layer = tf.nn.relu(
model_ops.fully_connected_layer(
tensor=prev_layer,
size=layer_sizes[i],
weight_init=tf.truncated_normal(
shape=[prev_layer_size, layer_sizes[i]],
stddev=weight_init_stddevs[i]),
bias_init=tf.constant(
value=bias_init_consts[i], shape=[layer_sizes[i]])))
layer = model_ops.dropout(layer, dropouts[i], training)
prev_layer = layer
prev_layer_size = layer_sizes[i]
output = []
# top_multitask_layer has shape [None, layer_sizes[-1]]
top_multitask_layer = prev_layer
for task in range(self.n_tasks):
# TODO(rbharath): Might want to make it feasible to have multiple
# bypass layers.
# Construct task bypass layer
prev_bypass_layer = top_layer
prev_bypass_layer_size = num_features
for i in range(num_bypass_layers):
# bypass_layer has shape [None, bypass_layer_sizes[i]]
print("Adding bypass weights of shape %s" %
str([prev_bypass_layer_size, bypass_layer_sizes[i]]))
bypass_layer = tf.nn.relu(
model_ops.fully_connected_layer(
tensor=prev_bypass_layer,
size=bypass_layer_sizes[i],
weight_init=tf.truncated_normal(
shape=[prev_bypass_layer_size, bypass_layer_sizes[i]],
stddev=bypass_weight_init_stddevs[i]),
bias_init=tf.constant(
value=bypass_bias_init_consts[i],
shape=[bypass_layer_sizes[i]])))
bypass_layer = model_ops.dropout(bypass_layer, bypass_dropouts[i],
training)
prev_bypass_layer = bypass_layer
prev_bypass_layer_size = bypass_layer_sizes[i]
top_bypass_layer = prev_bypass_layer
if num_bypass_layers > 0:
# task_layer has shape [None, layer_sizes[-1] + bypass_layer_sizes[-1]]
task_layer = tf.concat(
axis=1, values=[top_multitask_layer, top_bypass_layer])
task_layer_size = layer_sizes[-1] + bypass_layer_sizes[-1]
else:
task_layer = top_multitask_layer
task_layer_size = layer_sizes[-1]
print("Adding output weights of shape %s" % str([task_layer_size, 1]))
output.append(
tf.squeeze(
model_ops.logits(
task_layer,
num_classes=2,
weight_init=tf.truncated_normal(
shape=[task_layer_size, 2],
stddev=weight_init_stddevs[-1]),
bias_init=tf.constant(
value=bias_init_consts[-1], shape=[2]))))
return (output, labels, weights)
