def add_placeholders(self, graph, name_scopes): """Adds all placeholders for this model.""" # Create placeholders placeholder_scope = TensorflowGraph.get_placeholder_scope( graph, name_scopes) labels, weights = [], [] n_features = self.n_features with placeholder_scope: self.mol_features = tf.placeholder(tf.float32, shape=[None, n_features], name='mol_features') for task in range(self.n_tasks): weights.append( tf.identity( tf.placeholder(tf.float32, shape=[ None, ], name='weights_%d' % task))) labels.append( tf.identity( tf.placeholder(tf.float32, shape=[ None, ], name='labels_%d' % task))) return self.mol_features, labels, weights
def construct_task_feed_dict(self, this_task, X_b, y_b=None, w_b=None, ids_b=None): """Construct a feed dictionary from minibatch data. TODO(rbharath): ids_b is not used here. Can we remove it? Args: X_b: np.ndarray of shape (batch_size, n_features) y_b: np.ndarray of shape (batch_size, n_tasks) w_b: np.ndarray of shape (batch_size, n_tasks) ids_b: List of length (batch_size) with datapoint identifiers. """ orig_dict = {} orig_dict["mol_features"] = X_b n_samples = len(X_b) for task in range(self.n_tasks): if (this_task == task) and y_b is not None: #orig_dict["labels_%d" % task] = np.reshape(y_b[:, task], (n_samples, 1)) orig_dict["labels_%d" % task] = np.reshape(y_b[:, task], (n_samples,)) else: # Dummy placeholders #orig_dict["labels_%d" % task] = np.zeros((n_samples, 1)) orig_dict["labels_%d" % task] = np.zeros((n_samples,)) if (this_task == task) and w_b is not None: #orig_dict["weights_%d" % task] = np.reshape(w_b[:, task], (n_samples, 1)) orig_dict["weights_%d" % task] = np.reshape(w_b[:, task], (n_samples,)) else: # Dummy placeholders #orig_dict["weights_%d" % task] = np.zeros((n_samples, 1)) orig_dict["weights_%d" % task] = np.zeros((n_samples,)) return TensorflowGraph.get_feed_dict(orig_dict)
def construct_feed_dict(self, X_b, y_b=None, w_b=None, ids_b=None): """Construct a feed dictionary from minibatch data. TODO(rbharath): ids_b is not used here. Can we remove it? Args: X_b: np.ndarray of shape (batch_size, n_features) y_b: np.ndarray of shape (batch_size, n_tasks) w_b: np.ndarray of shape (batch_size, n_tasks) ids_b: List of length (batch_size) with datapoint identifiers. """ orig_dict = {} orig_dict["mol_features"] = X_b for task in range(self.n_tasks): if y_b is not None: orig_dict["labels_%d" % task] = to_one_hot(y_b[:, task]) else: # Dummy placeholders orig_dict["labels_%d" % task] = np.squeeze( to_one_hot(np.zeros((self.batch_size, )))) if w_b is not None: orig_dict["weights_%d" % task] = w_b[:, task] else: # Dummy placeholders orig_dict["weights_%d" % task] = np.ones((self.batch_size, )) return TensorflowGraph.get_feed_dict(orig_dict)
def build(self, graph, name_scopes, training): """Constructs the graph architecture as specified in its config. This method creates the following Placeholders: mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape batch_size x n_features. """ n_features = self.n_features placeholder_scope = TensorflowGraph.get_placeholder_scope( graph, name_scopes) with graph.as_default(): with placeholder_scope: self.mol_features = tf.placeholder(tf.float32, shape=[None, n_features], name='mol_features') layer_sizes = self.layer_sizes weight_init_stddevs = self.weight_init_stddevs bias_init_consts = self.bias_init_consts dropouts = self.dropouts lengths_set = { len(layer_sizes), len(weight_init_stddevs), len(bias_init_consts), len(dropouts), } assert len( lengths_set) == 1, 'All layer params must have same length.' n_layers = lengths_set.pop() assert n_layers > 0, 'Must have some layers defined.' prev_layer = self.mol_features prev_layer_size = n_features for i in range(n_layers): layer = tf.nn.relu( model_ops.fully_connected_layer( tensor=prev_layer, size=layer_sizes[i], weight_init=tf.truncated_normal( shape=[prev_layer_size, layer_sizes[i]], stddev=weight_init_stddevs[i]), bias_init=tf.constant(value=bias_init_consts[i], shape=[layer_sizes[i]]))) layer = model_ops.dropout(layer, dropouts[i]) prev_layer = layer prev_layer_size = layer_sizes[i] output = [] for task in range(self.n_tasks): output.append( tf.squeeze( model_ops.fully_connected_layer( tensor=prev_layer, size=layer_sizes[i], weight_init=tf.truncated_normal( shape=[prev_layer_size, 1], stddev=weight_init_stddevs[i]), bias_init=tf.constant(value=bias_init_consts[i], shape=[1])))) return output
def build(self, graph, name_scopes, training): """Constructs the graph architecture of IRV as described in: https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2750043/ """ placeholder_scope = TensorflowGraph.get_placeholder_scope(graph, name_scopes) K = self.K with graph.as_default(): output = [] with placeholder_scope: self.features = tf.placeholder( tf.float32, shape=[None, self.n_features], name='mol_features') with tf.name_scope('variable'): V = tf.Variable(tf.constant([0.01, 1.]), name="vote", dtype=tf.float32) W = tf.Variable(tf.constant([1., 1.]), name="w", dtype=tf.float32) b = tf.Variable(tf.constant([0.01]), name="b", dtype=tf.float32) b2 = tf.Variable(tf.constant([0.01]), name="b2", dtype=tf.float32) for count in range(self.n_tasks): similarity = self.features[:, 2 * K * count:(2 * K * count + K)] ys = tf.to_int32( self.features[:, (2 * K * count + K):2 * K * (count + 1)]) R = b + W[0] * similarity + W[1] * tf.constant( np.arange(K) + 1, dtype=tf.float32) R = tf.sigmoid(R) z = tf.reduce_sum(R * tf.gather(V, ys), axis=1) + b2 output.append(tf.reshape(z, shape=[-1, 1])) return output
def build(self, graph, name_scopes, training): """Constructs the graph architecture of model: n_tasks * sigmoid nodes. This method creates the following Placeholders: mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape batch_size x n_features. """ placeholder_scope = TensorflowGraph.get_placeholder_scope( graph, name_scopes) n_features = self.n_features with graph.as_default(): with placeholder_scope: self.mol_features = tf.placeholder( tf.float32, shape=[None, n_features], name='mol_features') weight_init_stddevs = self.weight_init_stddevs bias_init_consts = self.bias_init_consts lg_list = [] for task in range(self.n_tasks): #setting up n_tasks nodes(output nodes) lg = model_ops.fully_connected_layer( tensor=self.mol_features, size = 1, weight_init=tf.truncated_normal( shape=[self.n_features, 1], stddev=weight_init_stddevs[0]), bias_init=tf.constant(value=bias_init_consts[0], shape=[1])) lg_list.append(lg) return lg_list
def build(self, graph, name_scopes, training): """Constructs the graph architecture of model: n_tasks * sigmoid nodes. This method creates the following Placeholders: mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape batch_size x n_features. """ placeholder_scope = TensorflowGraph.get_placeholder_scope( graph, name_scopes) n_features = self.n_features with graph.as_default(): with placeholder_scope: self.mol_features = tf.placeholder(tf.float32, shape=[None, n_features], name='mol_features') weight_init_stddevs = self.weight_init_stddevs bias_init_consts = self.bias_init_consts lg_list = [] for task in range(self.n_tasks): #setting up n_tasks nodes(output nodes) lg = model_ops.fully_connected_layer( tensor=self.mol_features, size=1, weight_init=tf.truncated_normal( shape=[self.n_features, 1], stddev=weight_init_stddevs[0]), bias_init=tf.constant(value=bias_init_consts[0], shape=[1])) lg_list.append(lg) return lg_list
def construct_graph(self, training, seed): """Returns a TensorflowGraph object.""" graph = tf.Graph() # Lazily created by _get_shared_session(). shared_session = None # Cache of TensorFlow scopes, to prevent '_1' appended scope names # when subclass-overridden methods use the same scopes. name_scopes = {} # Setup graph with graph.as_default(): if seed is not None: tf.set_random_seed(seed) features, labels, weights = self.add_placeholders( graph, name_scopes) outputs = self.add_progressive_lattice(graph, name_scopes, training) if training: loss = self.add_task_training_costs(graph, name_scopes, outputs, labels, weights) else: loss = None return TensorflowGraph(graph=graph, session=shared_session, name_scopes=name_scopes, output=outputs, labels=labels, weights=weights, loss=loss)
def construct_feed_dict(self, X_b, y_b=None, w_b=None, ids_b=None): """Construct a feed dictionary from minibatch data. TODO(rbharath): ids_b is not used here. Can we remove it? Args: X_b: np.ndarray of shape (batch_size, n_features) y_b: np.ndarray of shape (batch_size, n_tasks) w_b: np.ndarray of shape (batch_size, n_tasks) ids_b: List of length (batch_size) with datapoint identifiers. """ orig_dict = {} orig_dict["mol_features"] = X_b for task in range(self.n_tasks): if y_b is not None: orig_dict["labels_%d" % task] = to_one_hot(y_b[:, task]) else: # Dummy placeholders orig_dict["labels_%d" % task] = np.squeeze(to_one_hot( np.zeros((self.batch_size,)))) if w_b is not None: orig_dict["weights_%d" % task] = w_b[:, task] else: # Dummy placeholders orig_dict["weights_%d" % task] = np.ones( (self.batch_size,)) return TensorflowGraph.get_feed_dict(orig_dict)
def build(self, graph, name_scopes, training): """Constructs the graph architecture of IRV as described in: https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2750043/ """ placeholder_scope = TensorflowGraph.get_placeholder_scope( graph, name_scopes) K = self.K with graph.as_default(): output = [] with placeholder_scope: mol_features = tf.placeholder(tf.float32, shape=[None, self.n_features], name='mol_features') with tf.name_scope('variable'): V = tf.Variable(tf.constant([0.01, 1.]), name="vote", dtype=tf.float32) W = tf.Variable(tf.constant([1., 1.]), name="w", dtype=tf.float32) b = tf.Variable(tf.constant([0.01]), name="b", dtype=tf.float32) b2 = tf.Variable(tf.constant([0.01]), name="b2", dtype=tf.float32) label_placeholders = self.add_label_placeholders( graph, name_scopes) weight_placeholders = self.add_example_weight_placeholders( graph, name_scopes) if training: graph.queue = tf.FIFOQueue( capacity=5, dtypes=[tf.float32] * (len(label_placeholders) + len(weight_placeholders) + 1)) graph.enqueue = graph.queue.enqueue([mol_features] + label_placeholders + weight_placeholders) queue_outputs = graph.queue.dequeue() labels = queue_outputs[1:len(label_placeholders) + 1] weights = queue_outputs[len(label_placeholders) + 1:] features = queue_outputs[0] else: labels = label_placeholders weights = weight_placeholders features = mol_features for count in range(self.n_tasks): similarity = features[:, 2 * K * count:(2 * K * count + K)] ys = tf.to_int32(features[:, (2 * K * count + K):2 * K * (count + 1)]) R = b + W[0] * similarity + W[1] * tf.constant( np.arange(K) + 1, dtype=tf.float32) R = tf.sigmoid(R) z = tf.reduce_sum(R * tf.gather(V, ys), axis=1) + b2 output.append(tf.reshape(z, shape=[-1, 1])) return (output, labels, weights)
def build(self, graph, name_scopes, training): """Constructs the graph architecture as specified in its config. This method creates the following Placeholders: mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape batch_size x n_features. """ n_features = self.n_features placeholder_scope = TensorflowGraph.get_placeholder_scope( graph, name_scopes) with graph.as_default(): with placeholder_scope: self.mol_features = tf.placeholder( tf.float32, shape=[None, n_features], name='mol_features') layer_sizes = self.layer_sizes weight_init_stddevs = self.weight_init_stddevs bias_init_consts = self.bias_init_consts dropouts = self.dropouts lengths_set = { len(layer_sizes), len(weight_init_stddevs), len(bias_init_consts), len(dropouts), } assert len(lengths_set) == 1, 'All layer params must have same length.' n_layers = lengths_set.pop() assert n_layers > 0, 'Must have some layers defined.' prev_layer = self.mol_features prev_layer_size = n_features for i in range(n_layers): layer = tf.nn.relu(model_ops.fully_connected_layer( tensor=prev_layer, size=layer_sizes[i], weight_init=tf.truncated_normal( shape=[prev_layer_size, layer_sizes[i]], stddev=weight_init_stddevs[i]), bias_init=tf.constant(value=bias_init_consts[i], shape=[layer_sizes[i]]))) layer = model_ops.dropout(layer, dropouts[i]) prev_layer = layer prev_layer_size = layer_sizes[i] output = [] for task in range(self.n_tasks): output.append(tf.squeeze( model_ops.fully_connected_layer( tensor=prev_layer, size=layer_sizes[i], weight_init=tf.truncated_normal( shape=[prev_layer_size, 1], stddev=weight_init_stddevs[i]), bias_init=tf.constant(value=bias_init_consts[i], shape=[1])))) return output
def build(self, graph, name_scopes, training): """Constructs the graph architecture of model: n_tasks * sigmoid nodes. This method creates the following Placeholders: mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape batch_size x n_features. """ warnings.warn( "TensorflowLogisticRegression is deprecated. " "Will be removed in DeepChem 1.4.", DeprecationWarning) placeholder_scope = TensorflowGraph.get_placeholder_scope( graph, name_scopes) n_features = self.n_features with graph.as_default(): with placeholder_scope: mol_features = tf.placeholder(tf.float32, shape=[None, n_features], name='mol_features') weight_init_stddevs = self.weight_init_stddevs bias_init_consts = self.bias_init_consts lg_list = [] label_placeholders = self.add_label_placeholders( graph, name_scopes) weight_placeholders = self.add_example_weight_placeholders( graph, name_scopes) if training: graph.queue = tf.FIFOQueue( capacity=5, dtypes=[tf.float32] * (len(label_placeholders) + len(weight_placeholders) + 1)) graph.enqueue = graph.queue.enqueue([mol_features] + label_placeholders + weight_placeholders) queue_outputs = graph.queue.dequeue() labels = queue_outputs[1:len(label_placeholders) + 1] weights = queue_outputs[len(label_placeholders) + 1:] prev_layer = queue_outputs[0] else: labels = label_placeholders weights = weight_placeholders prev_layer = mol_features for task in range(self.n_tasks): #setting up n_tasks nodes(output nodes) lg = model_ops.fully_connected_layer( tensor=prev_layer, size=1, weight_init=tf.truncated_normal( shape=[self.n_features, 1], stddev=weight_init_stddevs[0]), bias_init=tf.constant(value=bias_init_consts[0], shape=[1])) lg_list.append(lg) return (lg_list, labels, weights)
def add_label_placeholders(self, graph, name_scopes): #label placeholders with size batch_size * 1 labels = [] placeholder_scope = TensorflowGraph.get_placeholder_scope(graph, name_scopes) with placeholder_scope: for task in range(self.n_tasks): labels.append(tf.identity( tf.placeholder(tf.float32, shape=[None,1], name='labels_%d' % task))) return labels
def add_training_costs(self, graph, name_scopes, output, labels, weights): with graph.as_default(): epsilon = 1e-3 # small float to avoid dividing by zero weighted_costs = [] # weighted costs for each example gradient_costs = [] # costs used for gradient calculation with TensorflowGraph.shared_name_scope('costs', graph, name_scopes): for task in range(self.n_tasks): task_str = str(task).zfill(len(str(self.n_tasks))) with TensorflowGraph.shared_name_scope( 'cost_{}'.format(task_str), graph, name_scopes): with tf.name_scope('weighted'): weighted_cost = self.cost(output[task], labels[task], weights[task]) weighted_costs.append(weighted_cost) with tf.name_scope('gradient'): # Note that we divide by the batch size and not the number of # non-zero weight examples in the batch. Also, instead of using # tf.reduce_mean (which can put ops on the CPU) we explicitly # calculate with div/sum so it stays on the GPU. gradient_cost = tf.div( tf.reduce_sum(weighted_cost), self.batch_size) gradient_costs.append(gradient_cost) # aggregated costs with TensorflowGraph.shared_name_scope('aggregated', graph, name_scopes): with tf.name_scope('gradient'): loss = tf.add_n(gradient_costs) # weight decay if self.penalty != 0.0: penalty = model_ops.weight_decay( self.penalty_type, self.penalty) loss += penalty return loss
def add_task_training_costs(self, graph, name_scopes, outputs, labels, weights): """Adds the training costs for each task. Since each task is trained separately, each task is optimized w.r.t a separate task. TODO(rbharath): Figure out how to support weight decay for this model. Since each task is trained separately, weight decay should only be used on weights in column for that task. Parameters ---------- graph: tf.Graph Graph for the model. name_scopes: dict Contains all the scopes for model outputs: list List of output tensors from model. weights: list List of weight placeholders for model. """ task_costs = {} with TensorflowGraph.shared_name_scope('costs', graph, name_scopes): for task in range(self.n_tasks): with TensorflowGraph.shared_name_scope('cost_%d' % task, graph, name_scopes): weighted_cost = self.cost(outputs[task], labels[task], weights[task]) # Note that we divide by the batch size and not the number of # non-zero weight examples in the batch. Also, instead of using # tf.reduce_mean (which can put ops on the CPU) we explicitly # calculate with div/sum so it stays on the GPU. task_cost = tf.div(tf.reduce_sum(weighted_cost), self.batch_size) task_costs[task] = task_cost return task_costs
def add_training_cost(self, graph, name_scopes, output, labels, weights): with graph.as_default(): epsilon = 1e-3 # small float to avoid dividing by zero weighted_costs = [] # weighted costs for each example gradient_costs = [] # costs used for gradient calculation with TensorflowGraph.shared_name_scope('costs', graph, name_scopes): for task in range(self.n_tasks): task_str = str(task).zfill(len(str(self.n_tasks))) with TensorflowGraph.shared_name_scope('cost_{}'.format(task_str), graph, name_scopes): with tf.name_scope('weighted'): weighted_cost = self.cost(output[task], labels[task], weights[task]) weighted_costs.append(weighted_cost) with tf.name_scope('gradient'): # Note that we divide by the batch size and not the number of # non-zero weight examples in the batch. Also, instead of using # tf.reduce_mean (which can put ops on the CPU) we explicitly # calculate with div/sum so it stays on the GPU. gradient_cost = tf.div( tf.reduce_sum(weighted_cost), self.batch_size) gradient_costs.append(gradient_cost) # aggregated costs with TensorflowGraph.shared_name_scope('aggregated', graph, name_scopes): with tf.name_scope('gradient'): loss = tf.add_n(gradient_costs) # weight decay if self.penalty != 0.0: # using self-defined regularization penalty = weight_decay(self.penalty_type, self.penalty) loss += penalty return loss
def build(self, graph, name_scopes, training): """Constructs the graph architecture of model: n_tasks * sigmoid nodes. This method creates the following Placeholders: mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape batch_size x n_features. """ placeholder_scope = TensorflowGraph.get_placeholder_scope(graph, name_scopes) n_features = self.n_features with graph.as_default(): with placeholder_scope: mol_features = tf.placeholder( tf.float32, shape=[None, n_features], name='mol_features') weight_init_stddevs = self.weight_init_stddevs bias_init_consts = self.bias_init_consts lg_list = [] label_placeholders = self.add_label_placeholders(graph, name_scopes) weight_placeholders = self.add_example_weight_placeholders(graph, name_scopes) if training: graph.queue = tf.FIFOQueue( capacity=5, dtypes=[tf.float32] * (len(label_placeholders) + len(weight_placeholders) + 1)) graph.enqueue = graph.queue.enqueue([mol_features] + label_placeholders + weight_placeholders) queue_outputs = graph.queue.dequeue() labels = queue_outputs[1:len(label_placeholders) + 1] weights = queue_outputs[len(label_placeholders) + 1:] prev_layer = queue_outputs[0] else: labels = label_placeholders weights = weight_placeholders prev_layer = mol_features for task in range(self.n_tasks): #setting up n_tasks nodes(output nodes) lg = model_ops.fully_connected_layer( tensor=prev_layer, size=1, weight_init=tf.truncated_normal( shape=[self.n_features, 1], stddev=weight_init_stddevs[0]), bias_init=tf.constant(value=bias_init_consts[0], shape=[1])) lg_list.append(lg) return (lg_list, labels, weights)
def build(self, graph, name_scopes, training): """Constructs the graph architecture of IRV as described in: https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2750043/ """ placeholder_scope = TensorflowGraph.get_placeholder_scope(graph, name_scopes) K = self.K with graph.as_default(): output = [] with placeholder_scope: mol_features = tf.placeholder( tf.float32, shape=[None, self.n_features], name='mol_features') with tf.name_scope('variable'): V = tf.Variable(tf.constant([0.01, 1.]), name="vote", dtype=tf.float32) W = tf.Variable(tf.constant([1., 1.]), name="w", dtype=tf.float32) b = tf.Variable(tf.constant([0.01]), name="b", dtype=tf.float32) b2 = tf.Variable(tf.constant([0.01]), name="b2", dtype=tf.float32) label_placeholders = self.add_label_placeholders(graph, name_scopes) weight_placeholders = self.add_example_weight_placeholders(graph, name_scopes) if training: graph.queue = tf.FIFOQueue( capacity=5, dtypes=[tf.float32] * (len(label_placeholders) + len(weight_placeholders) + 1)) graph.enqueue = graph.queue.enqueue([mol_features] + label_placeholders + weight_placeholders) queue_outputs = graph.queue.dequeue() labels = queue_outputs[1:len(label_placeholders) + 1] weights = queue_outputs[len(label_placeholders) + 1:] features = queue_outputs[0] else: labels = label_placeholders weights = weight_placeholders features = mol_features for count in range(self.n_tasks): similarity = features[:, 2 * K * count:(2 * K * count + K)] ys = tf.to_int32(features[:, (2 * K * count + K):2 * K * (count + 1)]) R = b + W[0] * similarity + W[1] * tf.constant( np.arange(K) + 1, dtype=tf.float32) R = tf.sigmoid(R) z = tf.reduce_sum(R * tf.gather(V, ys), axis=1) + b2 output.append(tf.reshape(z, shape=[-1, 1])) return (output, labels, weights)
def construct_feed_dict(self, X_b, y_b=None, w_b=None, ids_b=None): orig_dict = {} orig_dict["mol_features"] = X_b for task in range(self.n_tasks): if y_b is not None: y_2column = to_one_hot(y_b[:, task]) # fix the size to be [?,1] orig_dict["labels_%d" % task] = y_2column[:, 1:2] else: # Dummy placeholders orig_dict["labels_%d" % task] = np.zeros((self.batch_size, 1)) if w_b is not None: orig_dict["weights_%d" % task] = w_b[:, task] else: # Dummy placeholders orig_dict["weights_%d" % task] = np.ones((self.batch_size, )) return TensorflowGraph.get_feed_dict(orig_dict)
def construct_feed_dict(self, X_b, y_b=None, w_b=None, ids_b=None): orig_dict = {} orig_dict["mol_features"] = X_b for task in range(self.n_tasks): if y_b is not None: y_2column = to_one_hot(y_b[:, task]) # fix the size to be [?,1] orig_dict["labels_%d" % task] = y_2column[:, 1:2] else: # Dummy placeholders orig_dict["labels_%d" % task] = np.zeros((self.batch_size, 1)) if w_b is not None: orig_dict["weights_%d" % task] = w_b[:, task] else: # Dummy placeholders orig_dict["weights_%d" % task] = np.ones((self.batch_size,)) return TensorflowGraph.get_feed_dict(orig_dict)
def add_placeholders(self, graph, name_scopes): """Adds all placeholders for this model.""" # Create placeholders placeholder_scope = TensorflowGraph.get_placeholder_scope( graph, name_scopes) labels, weights = [], [] n_features = self.n_features with placeholder_scope: self.mol_features = tf.placeholder( tf.float32, shape=[None, n_features], name='mol_features') for task in range(self.n_tasks): weights.append(tf.identity( tf.placeholder(tf.float32, shape=[None,], name='weights_%d' % task))) labels.append(tf.identity( tf.placeholder(tf.float32, shape=[None,], name='labels_%d' % task))) return self.mol_features, labels, weights
def build(self, graph, name_scopes, training): """Constructs the graph architecture as specified in its config. This method creates the following Placeholders: mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape batch_size x n_features. """ placeholder_scope = TensorflowGraph.get_placeholder_scope(graph, name_scopes) n_features = self.n_features with graph.as_default(): with placeholder_scope: mol_features = tf.placeholder( tf.float32, shape=[None, n_features], name='mol_features') layer_sizes = self.layer_sizes weight_init_stddevs = self.weight_init_stddevs bias_init_consts = self.bias_init_consts dropouts = self.dropouts lengths_set = { len(layer_sizes), len(weight_init_stddevs), len(bias_init_consts), len(dropouts), } assert len(lengths_set) == 1, 'All layer params must have same length.' n_layers = lengths_set.pop() assert n_layers > 0, 'Must have some layers defined.' label_placeholders = self.add_label_placeholders(graph, name_scopes) weight_placeholders = self.add_example_weight_placeholders(graph, name_scopes) if training: graph.queue = tf.FIFOQueue( capacity=5, dtypes=[tf.float32] * (len(label_placeholders) + len(weight_placeholders) + 1)) graph.enqueue = graph.queue.enqueue([mol_features] + label_placeholders + weight_placeholders) queue_outputs = graph.queue.dequeue() labels = queue_outputs[1:len(label_placeholders) + 1] weights = queue_outputs[len(label_placeholders) + 1:] prev_layer = queue_outputs[0] else: labels = label_placeholders weights = weight_placeholders prev_layer = mol_features prev_layer_size = n_features for i in range(n_layers): layer = tf.nn.relu( model_ops.fully_connected_layer( tensor=prev_layer, size=layer_sizes[i], weight_init=tf.truncated_normal( shape=[prev_layer_size, layer_sizes[i]], stddev=weight_init_stddevs[i]), bias_init=tf.constant( value=bias_init_consts[i], shape=[layer_sizes[i]]))) layer = model_ops.dropout(layer, dropouts[i], training) prev_layer = layer prev_layer_size = layer_sizes[i] output = model_ops.multitask_logits(layer, self.n_tasks) return (output, labels, weights)
def build(self, graph, name_scopes, training): """Constructs the graph architecture as specified in its config. This method creates the following Placeholders: mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape batch_size x num_features. """ num_features = self.n_features placeholder_scope = TensorflowGraph.get_placeholder_scope( graph, name_scopes) with graph.as_default(): with placeholder_scope: self.mol_features = tf.placeholder(tf.float32, shape=[None, num_features], name='mol_features') layer_sizes = self.layer_sizes weight_init_stddevs = self.weight_init_stddevs bias_init_consts = self.bias_init_consts dropouts = self.dropouts bypass_layer_sizes = self.bypass_layer_sizes bypass_weight_init_stddevs = self.bypass_weight_init_stddevs bypass_bias_init_consts = self.bypass_bias_init_consts bypass_dropouts = self.bypass_dropouts lengths_set = { len(layer_sizes), len(weight_init_stddevs), len(bias_init_consts), len(dropouts), } assert len( lengths_set) == 1, "All layer params must have same length." num_layers = lengths_set.pop() assert num_layers > 0, "Must have some layers defined." bypass_lengths_set = { len(bypass_layer_sizes), len(bypass_weight_init_stddevs), len(bypass_bias_init_consts), len(bypass_dropouts), } assert len(bypass_lengths_set) == 1, ("All bypass_layer params" + " must have same length.") num_bypass_layers = bypass_lengths_set.pop() prev_layer = self.mol_features prev_layer_size = num_features for i in range(num_layers): # layer has shape [None, layer_sizes[i]] ########################################################## DEBUG print("Adding weights of shape %s" % str([prev_layer_size, layer_sizes[i]])) ########################################################## DEBUG layer = tf.nn.relu( model_ops.fully_connected_layer( tensor=prev_layer, size=layer_sizes[i], weight_init=tf.truncated_normal( shape=[prev_layer_size, layer_sizes[i]], stddev=weight_init_stddevs[i]), bias_init=tf.constant(value=bias_init_consts[i], shape=[layer_sizes[i]]))) layer = model_ops.dropout(layer, dropouts[i], training) prev_layer = layer prev_layer_size = layer_sizes[i] output = [] # top_multitask_layer has shape [None, layer_sizes[-1]] top_multitask_layer = prev_layer for task in range(self.n_tasks): # TODO(rbharath): Might want to make it feasible to have multiple # bypass layers. # Construct task bypass layer prev_bypass_layer = self.mol_features prev_bypass_layer_size = num_features for i in range(num_bypass_layers): # bypass_layer has shape [None, bypass_layer_sizes[i]] ########################################################## DEBUG print("Adding bypass weights of shape %s" % str([prev_bypass_layer_size, bypass_layer_sizes[i]])) ########################################################## DEBUG bypass_layer = tf.nn.relu( model_ops.fully_connected_layer( tensor=prev_bypass_layer, size=bypass_layer_sizes[i], weight_init=tf.truncated_normal( shape=[ prev_bypass_layer_size, bypass_layer_sizes[i] ], stddev=bypass_weight_init_stddevs[i]), bias_init=tf.constant( value=bypass_bias_init_consts[i], shape=[bypass_layer_sizes[i]]))) bypass_layer = model_ops.dropout(bypass_layer, bypass_dropouts[i]) prev_bypass_layer = bypass_layer prev_bypass_layer_size = bypass_layer_sizes[i] top_bypass_layer = prev_bypass_layer if num_bypass_layers > 0: # task_layer has shape [None, layer_sizes[-1] + bypass_layer_sizes[-1]] task_layer = tf.concat( 1, [top_multitask_layer, top_bypass_layer]) task_layer_size = layer_sizes[-1] + bypass_layer_sizes[-1] else: task_layer = top_multitask_layer task_layer_size = layer_sizes[-1] ########################################################## DEBUG print("Adding output weights of shape %s" % str([task_layer_size, 1])) ########################################################## DEBUG #################################################### DEBUG print("task_layer_size") print(task_layer_size) #################################################### DEBUG output.append( tf.squeeze( model_ops.logits(task_layer, num_classes=2, weight_init=tf.truncated_normal( shape=[task_layer_size, 2], stddev=weight_init_stddevs[-1]), bias_init=tf.constant( value=bias_init_consts[-1], shape=[2])))) return output
def build(self, graph, name_scopes, training): """Constructs the graph architecture as specified in its config. This method creates the following Placeholders: mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape batch_size x n_features. """ n_features = self.n_features placeholder_scope = TensorflowGraph.get_placeholder_scope( graph, name_scopes) with graph.as_default(): with placeholder_scope: self.mol_features = tf.placeholder(tf.float32, shape=[None, n_features], name='mol_features') layer_sizes = self.layer_sizes weight_init_stddevs = self.weight_init_stddevs bias_init_consts = self.bias_init_consts dropouts = self.dropouts lengths_set = { len(layer_sizes), len(weight_init_stddevs), len(bias_init_consts), len(dropouts), } assert len( lengths_set) == 1, 'All layer params must have same length.' n_layers = lengths_set.pop() assert n_layers > 0, 'Must have some layers defined.' prev_layer = self.mol_features prev_layer_size = n_features all_layers = {} for i in range(n_layers): for task in range(self.n_tasks): task_scope = TensorflowGraph.shared_name_scope( "task%d" % task, graph, name_scopes) print("Adding weights for task %d, layer %d" % (task, i)) with task_scope as scope: if i == 0: prev_layer = self.mol_features prev_layer_size = self.n_features else: prev_layer = all_layers[(i - 1, task)] prev_layer_size = layer_sizes[i - 1] if task > 0: lateral_contrib = self.add_adapter( all_layers, task, i) print( "Creating W_layer_%d_task%d of shape %s" % (i, task, str([prev_layer_size, layer_sizes[i]]))) W = tf.Variable(tf.truncated_normal( shape=[prev_layer_size, layer_sizes[i]], stddev=self.weight_init_stddevs[i]), name='W_layer_%d_task%d' % (i, task), dtype=tf.float32) print("Creating b_layer_%d_task%d of shape %s" % (i, task, str([layer_sizes[i]]))) b = tf.Variable(tf.constant( value=self.bias_init_consts[i], shape=[layer_sizes[i]]), name='b_layer_%d_task%d' % (i, task), dtype=tf.float32) layer = tf.matmul(prev_layer, W) + b if i > 0 and task > 0: layer = layer + lateral_contrib layer = tf.nn.relu(layer) layer = model_ops.dropout(layer, dropouts[i], training) all_layers[(i, task)] = layer output = [] for task in range(self.n_tasks): prev_layer = all_layers[(i, task)] prev_layer_size = layer_sizes[i] task_scope = TensorflowGraph.shared_name_scope( "task%d" % task, graph, name_scopes) with task_scope as scope: if task > 0: lateral_contrib = tf.squeeze( self.add_adapter(all_layers, task, i + 1)) weight_init = tf.truncated_normal( shape=[prev_layer_size, 1], stddev=weight_init_stddevs[i]) bias_init = tf.constant(value=bias_init_consts[i], shape=[1]) print("Creating W_output_task%d of shape %s" % (task, str([prev_layer_size, 1]))) w = tf.Variable(weight_init, name='W_output_task%d' % task, dtype=tf.float32) print("Creating b_output_task%d of shape %s" % (task, str([1]))) b = tf.Variable(bias_init, name='b_output_task%d' % task, dtype=tf.float32) layer = tf.squeeze(tf.matmul(prev_layer, w) + b) if i > 0 and task > 0: layer = layer + lateral_contrib output.append(layer) return output
def build(self, graph, name_scopes, training): """Constructs the graph architecture as specified in its config. This method creates the following Placeholders: mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape batch_size x num_features. """ num_features = self.n_features placeholder_scope = TensorflowGraph.get_placeholder_scope( graph, name_scopes) with graph.as_default(): with placeholder_scope: self.mol_features = tf.placeholder( tf.float32, shape=[None, num_features], name='mol_features') layer_sizes = self.layer_sizes weight_init_stddevs = self.weight_init_stddevs bias_init_consts = self.bias_init_consts dropouts = self.dropouts bypass_layer_sizes = self.bypass_layer_sizes bypass_weight_init_stddevs = self.bypass_weight_init_stddevs bypass_bias_init_consts = self.bypass_bias_init_consts bypass_dropouts = self.bypass_dropouts lengths_set = { len(layer_sizes), len(weight_init_stddevs), len(bias_init_consts), len(dropouts), } assert len(lengths_set) == 1, "All layer params must have same length." num_layers = lengths_set.pop() assert num_layers > 0, "Must have some layers defined." bypass_lengths_set = { len(bypass_layer_sizes), len(bypass_weight_init_stddevs), len(bypass_bias_init_consts), len(bypass_dropouts), } assert (len(bypass_lengths_set) == 1, "All bypass_layer params must have same length.") num_bypass_layers = bypass_lengths_set.pop() prev_layer = self.mol_features prev_layer_size = num_features for i in range(num_layers): # layer has shape [None, layer_sizes[i]] ########################################################## DEBUG print("Adding weights of shape %s" % str([prev_layer_size, layer_sizes[i]])) ########################################################## DEBUG layer = tf.nn.relu(model_ops.fully_connected_layer( tensor=prev_layer, size=layer_sizes[i], weight_init=tf.truncated_normal( shape=[prev_layer_size, layer_sizes[i]], stddev=weight_init_stddevs[i]), bias_init=tf.constant(value=bias_init_consts[i], shape=[layer_sizes[i]]))) layer = model_ops.dropout(layer, dropouts[i], training) prev_layer = layer prev_layer_size = layer_sizes[i] output = [] # top_multitask_layer has shape [None, layer_sizes[-1]] top_multitask_layer = prev_layer for task in range(self.n_tasks): # TODO(rbharath): Might want to make it feasible to have multiple # bypass layers. # Construct task bypass layer prev_bypass_layer = self.mol_features prev_bypass_layer_size = num_features for i in range(num_bypass_layers): # bypass_layer has shape [None, bypass_layer_sizes[i]] ########################################################## DEBUG print("Adding bypass weights of shape %s" % str([prev_bypass_layer_size, bypass_layer_sizes[i]])) ########################################################## DEBUG bypass_layer = tf.nn.relu(model_ops.fully_connected_layer( tensor = prev_bypass_layer, size = bypass_layer_sizes[i], weight_init=tf.truncated_normal( shape=[prev_bypass_layer_size, bypass_layer_sizes[i]], stddev=bypass_weight_init_stddevs[i]), bias_init=tf.constant(value=bypass_bias_init_consts[i], shape=[bypass_layer_sizes[i]]))) bypass_layer = model_ops.dropout(bypass_layer, bypass_dropouts[i]) prev_bypass_layer = bypass_layer prev_bypass_layer_size = bypass_layer_sizes[i] top_bypass_layer = prev_bypass_layer if num_bypass_layers > 0: # task_layer has shape [None, layer_sizes[-1] + bypass_layer_sizes[-1]] task_layer = tf.concat(1, [top_multitask_layer, top_bypass_layer]) task_layer_size = layer_sizes[-1] + bypass_layer_sizes[-1] else: task_layer = top_multitask_layer task_layer_size = layer_sizes[-1] ########################################################## DEBUG print("Adding output weights of shape %s" % str([task_layer_size, 1])) ########################################################## DEBUG #################################################### DEBUG print("task_layer_size") print(task_layer_size) #################################################### DEBUG output.append(tf.squeeze( model_ops.fully_connected_layer( tensor=task_layer, size=task_layer_size, weight_init=tf.truncated_normal( shape=[task_layer_size, 1], stddev=weight_init_stddevs[-1]), bias_init=tf.constant(value=bias_init_consts[-1], shape=[1])))) return output
def build(self, graph, name_scopes, training): """Constructs the graph architecture as specified in its config. This method creates the following Placeholders: mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape batch_size x n_features. """ placeholder_scope = TensorflowGraph.get_placeholder_scope( graph, name_scopes) n_features = self.n_features with graph.as_default(): with placeholder_scope: mol_features = tf.placeholder(tf.float32, shape=[None, n_features], name='mol_features') layer_sizes = self.layer_sizes weight_init_stddevs = self.weight_init_stddevs bias_init_consts = self.bias_init_consts dropouts = self.dropouts lengths_set = { len(layer_sizes), len(weight_init_stddevs), len(bias_init_consts), len(dropouts), } assert len( lengths_set) == 1, 'All layer params must have same length.' n_layers = lengths_set.pop() assert n_layers > 0, 'Must have some layers defined.' label_placeholders = self.add_label_placeholders( graph, name_scopes) weight_placeholders = self.add_example_weight_placeholders( graph, name_scopes) if training: graph.queue = tf.FIFOQueue( capacity=5, dtypes=[tf.float32] * (len(label_placeholders) + len(weight_placeholders) + 1)) graph.enqueue = graph.queue.enqueue([mol_features] + label_placeholders + weight_placeholders) queue_outputs = graph.queue.dequeue() labels = queue_outputs[1:len(label_placeholders) + 1] weights = queue_outputs[len(label_placeholders) + 1:] prev_layer = queue_outputs[0] else: labels = label_placeholders weights = weight_placeholders prev_layer = mol_features prev_layer_size = n_features for i in range(n_layers): layer = tf.nn.relu( model_ops.fully_connected_layer( tensor=prev_layer, size=layer_sizes[i], weight_init=tf.truncated_normal( shape=[prev_layer_size, layer_sizes[i]], stddev=weight_init_stddevs[i]), bias_init=tf.constant(value=bias_init_consts[i], shape=[layer_sizes[i]]))) layer = model_ops.dropout(layer, dropouts[i], training) prev_layer = layer prev_layer_size = layer_sizes[i] output = model_ops.multitask_logits(layer, self.n_tasks) return (output, labels, weights)
def build(self, graph, name_scopes, training): """Constructs the graph architecture as specified in its config. This method creates the following Placeholders: mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape batch_size x num_features. """ num_features = self.n_features placeholder_scope = TensorflowGraph.get_placeholder_scope( graph, name_scopes) with graph.as_default(): with placeholder_scope: mol_features = tf.placeholder( tf.float32, shape=[None, num_features], name='mol_features') layer_sizes = self.layer_sizes weight_init_stddevs = self.weight_init_stddevs bias_init_consts = self.bias_init_consts dropouts = self.dropouts bypass_layer_sizes = self.bypass_layer_sizes bypass_weight_init_stddevs = self.bypass_weight_init_stddevs bypass_bias_init_consts = self.bypass_bias_init_consts bypass_dropouts = self.bypass_dropouts lengths_set = { len(layer_sizes), len(weight_init_stddevs), len(bias_init_consts), len(dropouts), } assert len(lengths_set) == 1, "All layer params must have same length." num_layers = lengths_set.pop() assert num_layers > 0, "Must have some layers defined." bypass_lengths_set = { len(bypass_layer_sizes), len(bypass_weight_init_stddevs), len(bypass_bias_init_consts), len(bypass_dropouts), } assert len(bypass_lengths_set) == 1, ( "All bypass_layer params" + " must have same length.") num_bypass_layers = bypass_lengths_set.pop() label_placeholders = self.add_label_placeholders(graph, name_scopes) weight_placeholders = self.add_example_weight_placeholders( graph, name_scopes) if training: graph.queue = tf.FIFOQueue( capacity=5, dtypes=[tf.float32] * (len(label_placeholders) + len(weight_placeholders) + 1)) graph.enqueue = graph.queue.enqueue([mol_features] + label_placeholders + weight_placeholders) queue_outputs = graph.queue.dequeue() labels = queue_outputs[1:len(label_placeholders) + 1] weights = queue_outputs[len(label_placeholders) + 1:] prev_layer = queue_outputs[0] else: labels = label_placeholders weights = weight_placeholders prev_layer = mol_features top_layer = prev_layer prev_layer_size = num_features for i in range(num_layers): # layer has shape [None, layer_sizes[i]] print("Adding weights of shape %s" % str( [prev_layer_size, layer_sizes[i]])) layer = tf.nn.relu( model_ops.fully_connected_layer( tensor=prev_layer, size=layer_sizes[i], weight_init=tf.truncated_normal( shape=[prev_layer_size, layer_sizes[i]], stddev=weight_init_stddevs[i]), bias_init=tf.constant( value=bias_init_consts[i], shape=[layer_sizes[i]]))) layer = model_ops.dropout(layer, dropouts[i], training) prev_layer = layer prev_layer_size = layer_sizes[i] output = [] # top_multitask_layer has shape [None, layer_sizes[-1]] top_multitask_layer = prev_layer for task in range(self.n_tasks): # TODO(rbharath): Might want to make it feasible to have multiple # bypass layers. # Construct task bypass layer prev_bypass_layer = top_layer prev_bypass_layer_size = num_features for i in range(num_bypass_layers): # bypass_layer has shape [None, bypass_layer_sizes[i]] print("Adding bypass weights of shape %s" % str( [prev_bypass_layer_size, bypass_layer_sizes[i]])) bypass_layer = tf.nn.relu( model_ops.fully_connected_layer( tensor=prev_bypass_layer, size=bypass_layer_sizes[i], weight_init=tf.truncated_normal( shape=[prev_bypass_layer_size, bypass_layer_sizes[i]], stddev=bypass_weight_init_stddevs[i]), bias_init=tf.constant( value=bypass_bias_init_consts[i], shape=[bypass_layer_sizes[i]]))) bypass_layer = model_ops.dropout(bypass_layer, bypass_dropouts[i], training) prev_bypass_layer = bypass_layer prev_bypass_layer_size = bypass_layer_sizes[i] top_bypass_layer = prev_bypass_layer if num_bypass_layers > 0: # task_layer has shape [None, layer_sizes[-1] + bypass_layer_sizes[-1]] task_layer = tf.concat( axis=1, values=[top_multitask_layer, top_bypass_layer]) task_layer_size = layer_sizes[-1] + bypass_layer_sizes[-1] else: task_layer = top_multitask_layer task_layer_size = layer_sizes[-1] print("Adding output weights of shape %s" % str([task_layer_size, 1])) output.append( tf.squeeze( model_ops.fully_connected_layer( tensor=task_layer, size=1, weight_init=tf.truncated_normal( shape=[task_layer_size, 1], stddev=weight_init_stddevs[-1]), bias_init=tf.constant( value=bias_init_consts[-1], shape=[1])), axis=1)) return (output, labels, weights)
def add_progressive_lattice(self, graph, name_scopes, training): """Constructs the graph architecture as specified in its config. This method creates the following Placeholders: mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape batch_size x n_features. """ n_features = self.n_features placeholder_scope = TensorflowGraph.get_placeholder_scope(graph, name_scopes) with graph.as_default(): layer_sizes = self.layer_sizes weight_init_stddevs = self.weight_init_stddevs bias_init_consts = self.bias_init_consts dropouts = self.dropouts lengths_set = { len(layer_sizes), len(weight_init_stddevs), len(bias_init_consts), len(dropouts), } assert len(lengths_set) == 1, 'All layer params must have same length.' n_layers = lengths_set.pop() assert n_layers > 0, 'Must have some layers defined.' prev_layer = self.mol_features prev_layer_size = n_features all_layers = {} for i in range(n_layers): for task in range(self.n_tasks): task_scope = TensorflowGraph.shared_name_scope("task%d_ops" % task, graph, name_scopes) print("Adding weights for task %d, layer %d" % (task, i)) with task_scope as scope: if i == 0: prev_layer = self.mol_features prev_layer_size = self.n_features else: prev_layer = all_layers[(i - 1, task)] prev_layer_size = layer_sizes[i - 1] if task > 0: lateral_contrib = self.add_adapter(all_layers, task, i) print("Creating W_layer_%d_task%d of shape %s" % (i, task, str([prev_layer_size, layer_sizes[i]]))) W = tf.Variable( tf.truncated_normal( shape=[prev_layer_size, layer_sizes[i]], stddev=self.weight_init_stddevs[i]), name='W_layer_%d_task%d' % (i, task), dtype=tf.float32) print("Creating b_layer_%d_task%d of shape %s" % (i, task, str([layer_sizes[i]]))) b = tf.Variable( tf.constant( value=self.bias_init_consts[i], shape=[layer_sizes[i]]), name='b_layer_%d_task%d' % (i, task), dtype=tf.float32) layer = tf.matmul(prev_layer, W) + b if i > 0 and task > 0: layer = layer + lateral_contrib layer = tf.nn.relu(layer) layer = model_ops.dropout(layer, dropouts[i], training) all_layers[(i, task)] = layer output = [] for task in range(self.n_tasks): prev_layer = all_layers[(i, task)] prev_layer_size = layer_sizes[i] task_scope = TensorflowGraph.shared_name_scope("task%d" % task, graph, name_scopes) with task_scope as scope: if task > 0: lateral_contrib = tf.squeeze( self.add_adapter(all_layers, task, i + 1)) weight_init = tf.truncated_normal( shape=[prev_layer_size, 1], stddev=weight_init_stddevs[i]) bias_init = tf.constant(value=bias_init_consts[i], shape=[1]) print("Creating W_output_task%d of shape %s" % (task, str([prev_layer_size, 1]))) w = tf.Variable( weight_init, name='W_output_task%d' % task, dtype=tf.float32) print("Creating b_output_task%d of shape %s" % (task, str([1]))) b = tf.Variable( bias_init, name='b_output_task%d' % task, dtype=tf.float32) layer = tf.squeeze(tf.matmul(prev_layer, w) + b) if i > 0 and task > 0: layer = layer + lateral_contrib output.append(layer) return output
def build(self, graph, name_scopes, training): """Constructs the graph architecture as specified in its config. This method creates the following Placeholders: mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape batch_size x num_features. """ num_features = self.n_features placeholder_scope = TensorflowGraph.get_placeholder_scope(graph, name_scopes) with graph.as_default(): with placeholder_scope: mol_features = tf.placeholder( tf.float32, shape=[None, num_features], name='mol_features') layer_sizes = self.layer_sizes weight_init_stddevs = self.weight_init_stddevs bias_init_consts = self.bias_init_consts dropouts = self.dropouts bypass_layer_sizes = self.bypass_layer_sizes bypass_weight_init_stddevs = self.bypass_weight_init_stddevs bypass_bias_init_consts = self.bypass_bias_init_consts bypass_dropouts = self.bypass_dropouts lengths_set = { len(layer_sizes), len(weight_init_stddevs), len(bias_init_consts), len(dropouts), } assert len(lengths_set) == 1, "All layer params must have same length." num_layers = lengths_set.pop() assert num_layers > 0, "Must have some layers defined." bypass_lengths_set = { len(bypass_layer_sizes), len(bypass_weight_init_stddevs), len(bypass_bias_init_consts), len(bypass_dropouts), } assert len(bypass_lengths_set) == 1, ( "All bypass_layer params" + " must have same length.") num_bypass_layers = bypass_lengths_set.pop() label_placeholders = self.add_label_placeholders(graph, name_scopes) weight_placeholders = self.add_example_weight_placeholders(graph, name_scopes) if training: graph.queue = tf.FIFOQueue( capacity=5, dtypes=[tf.float32] * (len(label_placeholders) + len(weight_placeholders) + 1)) graph.enqueue = graph.queue.enqueue([mol_features] + label_placeholders + weight_placeholders) queue_outputs = graph.queue.dequeue() labels = queue_outputs[1:len(label_placeholders) + 1] weights = queue_outputs[len(label_placeholders) + 1:] prev_layer = queue_outputs[0] else: labels = label_placeholders weights = weight_placeholders prev_layer = mol_features top_layer = prev_layer prev_layer_size = num_features for i in range(num_layers): # layer has shape [None, layer_sizes[i]] print("Adding weights of shape %s" % str([prev_layer_size, layer_sizes[i]])) layer = tf.nn.relu( model_ops.fully_connected_layer( tensor=prev_layer, size=layer_sizes[i], weight_init=tf.truncated_normal( shape=[prev_layer_size, layer_sizes[i]], stddev=weight_init_stddevs[i]), bias_init=tf.constant( value=bias_init_consts[i], shape=[layer_sizes[i]]))) layer = model_ops.dropout(layer, dropouts[i], training) prev_layer = layer prev_layer_size = layer_sizes[i] output = [] # top_multitask_layer has shape [None, layer_sizes[-1]] top_multitask_layer = prev_layer for task in range(self.n_tasks): # TODO(rbharath): Might want to make it feasible to have multiple # bypass layers. # Construct task bypass layer prev_bypass_layer = top_layer prev_bypass_layer_size = num_features for i in range(num_bypass_layers): # bypass_layer has shape [None, bypass_layer_sizes[i]] print("Adding bypass weights of shape %s" % str([prev_bypass_layer_size, bypass_layer_sizes[i]])) bypass_layer = tf.nn.relu( model_ops.fully_connected_layer( tensor=prev_bypass_layer, size=bypass_layer_sizes[i], weight_init=tf.truncated_normal( shape=[prev_bypass_layer_size, bypass_layer_sizes[i]], stddev=bypass_weight_init_stddevs[i]), bias_init=tf.constant( value=bypass_bias_init_consts[i], shape=[bypass_layer_sizes[i]]))) bypass_layer = model_ops.dropout(bypass_layer, bypass_dropouts[i], training) prev_bypass_layer = bypass_layer prev_bypass_layer_size = bypass_layer_sizes[i] top_bypass_layer = prev_bypass_layer if num_bypass_layers > 0: # task_layer has shape [None, layer_sizes[-1] + bypass_layer_sizes[-1]] task_layer = tf.concat( axis=1, values=[top_multitask_layer, top_bypass_layer]) task_layer_size = layer_sizes[-1] + bypass_layer_sizes[-1] else: task_layer = top_multitask_layer task_layer_size = layer_sizes[-1] print("Adding output weights of shape %s" % str([task_layer_size, 1])) output.append( tf.squeeze( model_ops.logits( task_layer, num_classes=2, weight_init=tf.truncated_normal( shape=[task_layer_size, 2], stddev=weight_init_stddevs[-1]), bias_init=tf.constant( value=bias_init_consts[-1], shape=[2])))) return (output, labels, weights)