def test_single_task_classifier(self): n_data_points = 20 n_features = 2 X = np.random.rand(n_data_points, n_features) y = [[0, 1] for x in range(n_data_points)] dataset = NumpyDataset(X, y) features = Feature(shape=(None, n_features)) dense = Dense(out_channels=2, in_layers=[features]) output = SoftMax(in_layers=[dense]) label = Label(shape=(None, 2)) smce = SoftMaxCrossEntropy(in_layers=[label, dense]) loss = ReduceMean(in_layers=[smce]) tg = dc.models.TensorGraph(learning_rate=0.01) tg.add_output(output) tg.set_loss(loss) tg.fit(dataset, nb_epoch=1000) prediction = np.squeeze(tg.predict_on_batch(X)) assert_true(np.all(np.isclose(prediction, y, atol=0.4)))
def test_invoke_model_eager(self): """Test invoking the model with __call__() in eager mode.""" with context.eager_mode(): with tfe.IsolateTest(): batch_size = 5 tg = dc.models.TensorGraph(batch_size=batch_size) features = Feature(shape=(None, 10)) dense = Dense(10, in_layers=features) loss = ReduceMean(in_layers=dense) tg.add_output(dense) tg.set_loss(loss) input = np.random.rand(batch_size, 10).astype(np.float32) # We should get the same result with either predict_on_batch() or __call__(). output1 = tg.predict_on_batch(input) output2 = tg(input) assert np.allclose(output1, output2.numpy())
def test_initialize_variable(self): """Test methods for initializing a variable.""" # Set by variable constructor. tg = dc.models.TensorGraph(use_queue=False) features = Feature(shape=(None, 1)) tg.set_loss(Dense(1, in_layers=features)) var = Variable([10.0]) tg.add_output(var) assert tg.predict_on_batch(np.zeros((1, 1))) == [10.0] # Set by set_variable_initial_values(). tg = dc.models.TensorGraph(use_queue=False) tg.set_loss(Dense(1, in_layers=features)) var.set_variable_initial_values([[15.0]]) tg.add_output(var) assert tg.predict_on_batch(np.zeros((1, 1))) == [15.0]
def test_multi_task_regressor(self): n_data_points = 20 n_features = 2 X = np.random.rand(n_data_points, n_features) y1 = np.expand_dims(np.array([0.5 for x in range(n_data_points)]), axis=-1) y2 = np.expand_dims(np.array([-0.5 for x in range(n_data_points)]), axis=-1) X = NumpyDataset(X) ys = [NumpyDataset(y1), NumpyDataset(y2)] databag = Databag() features = Feature(shape=(None, n_features)) databag.add_dataset(features, X) outputs = [] losses = [] for i in range(2): label = Label(shape=(None, 1)) dense = Dense(out_channels=1, in_layers=[features]) loss = ReduceSquareDifference(in_layers=[dense, label]) outputs.append(dense) losses.append(loss) databag.add_dataset(label, ys[i]) total_loss = ReduceMean(in_layers=losses) tg = dc.models.TensorGraph(learning_rate=0.01) for output in outputs: tg.add_output(output) tg.set_loss(total_loss) tg.fit_generator( databag.iterbatches(epochs=1000, batch_size=tg.batch_size, pad_batches=True)) predictions = tg.predict_on_generator(databag.iterbatches()) for i in range(2): y_real = ys[i].X y_pred = predictions[i] assert_true(np.all(np.isclose(y_pred, y_real, atol=1.5)))
def test_multi_task_classifier(self): n_data_points = 20 n_features = 2 X = np.random.rand(n_data_points, n_features) y1 = np.array([[0, 1] for x in range(n_data_points)]) y2 = np.array([[1, 0] for x in range(n_data_points)]) X = NumpyDataset(X) ys = [NumpyDataset(y1), NumpyDataset(y2)] databag = Databag() features = Feature(shape=(None, n_features)) databag.add_dataset(features, X) outputs = [] entropies = [] for i in range(2): label = Label(shape=(None, 2)) dense = Dense(out_channels=2, in_layers=[features]) output = SoftMax(in_layers=[dense]) smce = SoftMaxCrossEntropy(in_layers=[label, dense]) entropies.append(smce) outputs.append(output) databag.add_dataset(label, ys[i]) total_loss = ReduceMean(in_layers=entropies) tg = dc.models.TensorGraph(learning_rate=0.01) for output in outputs: tg.add_output(output) tg.set_loss(total_loss) tg.fit_generator( databag.iterbatches(epochs=1000, batch_size=tg.batch_size, pad_batches=True)) predictions = tg.predict_on_generator(databag.iterbatches()) for i in range(2): y_real = ys[i].X y_pred = predictions[i] assert_true(np.all(np.isclose(y_pred, y_real, atol=0.6)))
def test_mnist(self): from tensorflow.examples.tutorials.mnist import input_data mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) train = dc.data.NumpyDataset(mnist.train.images, mnist.train.labels) valid = dc.data.NumpyDataset(mnist.validation.images, mnist.validation.labels) # Images are square 28x28 (batch, height, width, channel) feature = Feature(shape=(None, 784), name="Feature") make_image = Reshape(shape=(-1, 28, 28, 1), in_layers=[feature]) conv2d_1 = Conv2d(num_outputs=32, in_layers=[make_image]) maxpool_1 = MaxPool(in_layers=[conv2d_1]) conv2d_2 = Conv2d(num_outputs=64, in_layers=[maxpool_1]) maxpool_2 = MaxPool(in_layers=[conv2d_2]) flatten = Flatten(in_layers=[maxpool_2]) dense1 = Dense(out_channels=1024, activation_fn=tf.nn.relu, in_layers=[flatten]) dense2 = Dense(out_channels=10, in_layers=[dense1]) label = Label(shape=(None, 10), name="Label") smce = SoftMaxCrossEntropy(in_layers=[label, dense2]) loss = ReduceMean(in_layers=[smce]) output = SoftMax(in_layers=[dense2]) tg = dc.models.TensorGraph(model_dir='/tmp/mnist', batch_size=1000, use_queue=True) tg.add_output(output) tg.set_loss(loss) tg.fit(train, nb_epoch=2) prediction = np.squeeze(tg.predict_proba_on_batch(valid.X)) fpr = dict() tpr = dict() roc_auc = dict() for i in range(10): fpr[i], tpr[i], thresh = roc_curve(valid.y[:, i], prediction[:, i]) roc_auc[i] = auc(fpr[i], tpr[i]) assert_true(roc_auc[i] > 0.99)
def test_shared_layer(self): n_data_points = 20 n_features = 2 X = np.random.rand(n_data_points, n_features) y1 = np.array([[0, 1] for x in range(n_data_points)]) X = NumpyDataset(X) ys = [NumpyDataset(y1)] databag = Databag() features = Feature(shape=(None, n_features)) databag.add_dataset(features, X) outputs = [] label = Label(shape=(None, 2)) dense1 = Dense(out_channels=2, in_layers=[features]) dense2 = dense1.shared(in_layers=[features]) output1 = SoftMax(in_layers=[dense1]) output2 = SoftMax(in_layers=[dense2]) smce = SoftMaxCrossEntropy(in_layers=[label, dense1]) outputs.append(output1) outputs.append(output2) databag.add_dataset(label, ys[0]) total_loss = ReduceMean(in_layers=[smce]) tg = dc.models.TensorGraph(learning_rate=0.01) for output in outputs: tg.add_output(output) tg.set_loss(total_loss) tg.fit_generator( databag.iterbatches(epochs=1, batch_size=tg.batch_size, pad_batches=True)) prediction = tg.predict_on_generator(databag.iterbatches()) assert_true(np.all(np.isclose(prediction[0], prediction[1], atol=0.01)))
def test_submodels(self): """Test optimizing submodels.""" tg = dc.models.TensorGraph(learning_rate=0.1, batch_size=1, use_queue=False) features = Feature(shape=(None, 1)) var1 = Variable([2.0]) var2 = Variable([2.0]) tg.add_output(var1) tg.add_output(var2) loss = (var1 - 1) * (var1 - 1) + (var2 - 1) * (var2 - 1) + features tg.set_loss(loss) subloss1 = var1 * var1 + features subloss2 = var1 * var1 + var2 * var2 + features submodel1 = tg.create_submodel(loss=subloss1) submodel2 = tg.create_submodel(layers=[var2], loss=subloss2) data = np.zeros((1, 1)) generator = [{features: data}] * 500 # Optimize submodel 1. This should send var1 to 0 while leaving var2 unchanged. tg.fit_generator(generator, submodel=submodel1) self.assertAlmostEqual( 0.0, tg.predict_on_batch(data, outputs=var1)[0], places=4) self.assertAlmostEqual( 2.0, tg.predict_on_batch(data, outputs=var2)[0], places=4) # Optimize the main loss. This should send both variables toward 1. tg.fit_generator(generator) self.assertAlmostEqual( 1.0, tg.predict_on_batch(data, outputs=var1)[0], places=4) self.assertAlmostEqual( 1.0, tg.predict_on_batch(data, outputs=var2)[0], places=4) # Optimize submodel 2. This should send var2 to 0 while leaving var1 unchanged. tg.fit_generator(generator, submodel=submodel2) self.assertAlmostEqual( 1.0, tg.predict_on_batch(data, outputs=var1)[0], places=4) self.assertAlmostEqual( 0.0, tg.predict_on_batch(data, outputs=var2)[0], places=4)
def test_compute_model_performance_multitask_classifier(self): n_data_points = 20 n_features = 1 n_tasks = 2 n_classes = 2 X = np.ones(shape=(n_data_points // 2, n_features)) * -1 X1 = np.ones(shape=(n_data_points // 2, n_features)) X = np.concatenate((X, X1)) class_1 = np.array([[0.0, 1.0] for x in range(int(n_data_points / 2))]) class_0 = np.array([[1.0, 0.0] for x in range(int(n_data_points / 2))]) y1 = np.concatenate((class_0, class_1)) y2 = np.concatenate((class_1, class_0)) y = np.stack([y1, y2], axis=1) dataset = NumpyDataset(X, y) features = Feature(shape=(None, n_features)) label = Label(shape=(None, n_tasks, n_classes)) dense = Dense(out_channels=n_tasks * n_classes, in_layers=[features]) logits = Reshape(shape=(None, n_tasks, n_classes), in_layers=dense) output = SoftMax(in_layers=[logits]) smce = SoftMaxCrossEntropy(in_layers=[label, logits]) total_loss = ReduceMean(in_layers=smce) tg = dc.models.TensorGraph(learning_rate=0.01, batch_size=n_data_points) tg.add_output(output) tg.set_loss(total_loss) tg.fit(dataset, nb_epoch=1000) metric = dc.metrics.Metric(dc.metrics.roc_auc_score, np.mean, mode="classification") scores = tg.evaluate_generator(tg.default_generator(dataset), [metric], labels=[label], per_task_metrics=True) scores = list(scores[1].values()) # Loosening atol to see if tests stop failing sporadically assert_true(np.all(np.isclose(scores, [1.0, 1.0], atol=0.50)))
def build_graph(self): print("building") features = Feature(shape=(None, self.n_features)) last_layer = features for layer_size in self.encoder_layers: last_layer = Dense(in_layers=last_layer, activation_fn=tf.nn.elu, out_channels=layer_size) self.mean = Dense(in_layers=last_layer, activation_fn=None, out_channels=1) self.std = Dense(in_layers=last_layer, activation_fn=None, out_channels=1) readout = CombineMeanStd([self.mean, self.std], training_only=True) last_layer = readout for layer_size in self.decoder_layers: last_layer = Dense(in_layers=readout, activation_fn=tf.nn.elu, out_channels=layer_size) self.reconstruction = Dense(in_layers=last_layer, activation_fn=None, out_channels=self.n_features) weights = Weights(shape=(None, self.n_features)) reproduction_loss = L2Loss( in_layers=[features, self.reconstruction, weights]) reproduction_loss = ReduceSum(in_layers=reproduction_loss, axis=0) global_step = TensorWrapper(self._get_tf("GlobalStep")) kl_loss = KLDivergenceLoss( in_layers=[self.mean, self.std, global_step], annealing_start_step=self.kl_annealing_start_step, annealing_stop_step=self.kl_annealing_stop_step) loss = Add(in_layers=[kl_loss, reproduction_loss], weights=[0.5, 1]) self.add_output(self.mean) self.add_output(self.reconstruction) self.set_loss(loss)
def _build_graph(self, tf_graph, scope, model_dir): """Construct a TensorGraph containing the policy and loss calculations.""" state_shape = self._env.state_shape state_dtype = self._env.state_dtype if not self._state_is_list: state_shape = [state_shape] state_dtype = [state_dtype] features = [] for s, d in zip(state_shape, state_dtype): features.append(Feature(shape=[None] + list(s), dtype=tf.as_dtype(d))) policy_layers = self._policy.create_layers(features) action_prob = policy_layers['action_prob'] value = policy_layers['value'] rewards = Weights(shape=(None,)) advantages = Weights(shape=(None,)) old_action_prob = Weights(shape=(None,)) actions = Label(shape=(None, self._env.n_actions)) loss = PPOLoss( self.value_weight, self.entropy_weight, self.clipping_width, in_layers=[ rewards, actions, action_prob, value, advantages, old_action_prob ]) graph = TensorGraph( batch_size=self.max_rollout_length, use_queue=False, graph=tf_graph, model_dir=model_dir) for f in features: graph._add_layer(f) graph.add_output(action_prob) graph.add_output(value) graph.set_loss(loss) graph.set_optimizer(self._optimizer) with graph._get_tf("Graph").as_default(): with tf.variable_scope(scope): graph.build() assert len(loss.components) > 0 return graph, features, rewards, actions, action_prob, value, advantages, old_action_prob, loss.components
def test_copy_layers(self): """Test copying layers.""" tg = dc.models.TensorGraph() features = Feature(shape=(None, 10)) dense = Dense( 10, in_layers=features, biases_initializer=tf.random_normal_initializer) constant = Constant(10.0) output = dense + constant tg.add_output(output) tg.set_loss(output) tg.fit_generator([]) replacements = {constant: Constant(20.0)} copy = output.copy(replacements, tg) assert isinstance(copy, Add) assert isinstance(copy.in_layers[0], Dense) assert isinstance(copy.in_layers[0].in_layers[0], Feature) assert copy.in_layers[1] == replacements[constant] variables = tg.get_layer_variables(dense) with tg._get_tf("Graph").as_default(): values = tg.session.run(variables) for v1, v2 in zip(values, copy.in_layers[0].variable_values): assert np.array_equal(v1, v2)
def build_graph(self): """Constructs the graph architecture of IRV as described in: https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2750043/ """ self.mol_features = Feature(shape=(None, self.n_features)) predictions = IRVLayer(self.n_tasks, self.K, in_layers=[self.mol_features]) costs = [] self.labels_fd = [] for task in range(self.n_tasks): task_output = Slice(task, 1, in_layers=[predictions]) sigmoid = Sigmoid(in_layers=[task_output]) self.add_output(sigmoid) label = Label(shape=(None, 1)) self.labels_fd.append(label) cost = SigmoidCrossEntropy(in_layers=[label, task_output]) costs.append(cost) all_cost = Concat(in_layers=costs, axis=1) self.weights = Weights(shape=(None, self.n_tasks)) loss = WeightedError(in_layers=[all_cost, self.weights]) + \ IRVRegularize(predictions, self.penalty, in_layers=[predictions]) self.set_loss(loss)
def test_copy_layers_shared(self): """Test copying layers with shared variables.""" tg = dc.models.TensorGraph() features = Feature(shape=(None, 10)) dense = Dense( 10, in_layers=features, biases_initializer=tf.random_normal_initializer) constant = Constant(10.0) output = dense + constant tg.add_output(output) tg.set_loss(output) replacements = {features: features, constant: Constant(20.0)} copy = output.copy(replacements, shared=True) tg.add_output(copy) assert isinstance(copy, Add) assert isinstance(copy.in_layers[0], Dense) assert isinstance(copy.in_layers[0].in_layers[0], Feature) assert copy.in_layers[1] == replacements[constant] variables1 = tg.get_layer_variables(dense) variables2 = tg.get_layer_variables(copy.in_layers[0]) for v1, v2, in zip(variables1, variables2): assert v1 == v2 feed_dict = {features: np.random.random((5, 10))} v1, v2 = tg.predict_on_generator([feed_dict], outputs=[output, copy]) assert_true(np.all(np.isclose(v1 + 10, v2)))
def test_neighbor_list_simple(self): """Test that neighbor lists can be constructed.""" N_atoms = 10 start = 0 stop = 12 nbr_cutoff = 3 ndim = 3 M = 6 X = np.random.rand(N_atoms, ndim) y = np.random.rand(N_atoms, 1) dataset = NumpyDataset(X, y) features = Feature(shape=(N_atoms, ndim)) labels = Label(shape=(N_atoms,)) nbr_list = NeighborList( N_atoms, M, ndim, nbr_cutoff, start, stop, in_layers=[features]) nbr_list = ToFloat(in_layers=[nbr_list]) # This isn't a meaningful loss, but just for test loss = ReduceSum(in_layers=[nbr_list]) tg = dc.models.TensorGraph(use_queue=False) tg.add_output(nbr_list) tg.set_loss(loss) tg.build()
def _build_graph(self, tf_graph, scope, model_dir): """Construct a TensorGraph containing the policy and loss calculations.""" state_shape = self._env.state_shape state_dtype = self._env.state_dtype if not self._state_is_list: state_shape = [state_shape] state_dtype = [state_dtype] features = [] for s, d in zip(state_shape, state_dtype): features.append( Feature(shape=[None] + list(s), dtype=tf.as_dtype(d))) policy_layers = self._policy.create_layers(features) action_prob = policy_layers['action_prob'] value = policy_layers['value'] search_prob = Label(shape=(None, self._env.n_actions)) search_value = Label(shape=(None, )) loss = MCTSLoss( self.value_weight, in_layers=[action_prob, value, search_prob, search_value]) graph = TensorGraph(batch_size=self.max_search_depth, use_queue=False, graph=tf_graph, model_dir=model_dir) for f in features: graph._add_layer(f) graph.add_output(action_prob) graph.add_output(value) graph.set_loss(loss) graph.set_optimizer(self._optimizer) with graph._get_tf("Graph").as_default(): with tf.variable_scope(scope): graph.build() if len(graph.rnn_initial_states) > 0: raise ValueError( 'MCTS does not support policies with recurrent layers') return graph, features, action_prob, value, search_prob, search_value
def build_graph(self): self.vertex_features = Feature(shape=(None, self.max_atoms, 75)) self.adj_matrix = Feature(shape=(None, self.max_atoms, 1, self.max_atoms)) self.mask = Feature(shape=(None, self.max_atoms, 1)) gcnn1 = BatchNorm( GraphCNN( num_filters=64, in_layers=[self.vertex_features, self.adj_matrix, self.mask])) gcnn1 = Dropout(self.dropout, in_layers=gcnn1) gcnn2 = BatchNorm( GraphCNN(num_filters=64, in_layers=[gcnn1, self.adj_matrix, self.mask])) gcnn2 = Dropout(self.dropout, in_layers=gcnn2) gc_pool, adj_matrix = GraphCNNPool( num_vertices=32, in_layers=[gcnn2, self.adj_matrix, self.mask]) gc_pool = BatchNorm(gc_pool) gc_pool = Dropout(self.dropout, in_layers=gc_pool) gcnn3 = BatchNorm( GraphCNN(num_filters=32, in_layers=[gc_pool, adj_matrix])) gcnn3 = Dropout(self.dropout, in_layers=gcnn3) gc_pool2, adj_matrix2 = GraphCNNPool(num_vertices=8, in_layers=[gcnn3, adj_matrix]) gc_pool2 = BatchNorm(gc_pool2) gc_pool2 = Dropout(self.dropout, in_layers=gc_pool2) flattened = Flatten(in_layers=gc_pool2) readout = Dense(out_channels=256, activation_fn=tf.nn.relu, in_layers=flattened) costs = [] self.my_labels = [] for task in range(self.n_tasks): if self.mode == 'classification': classification = Dense(out_channels=2, activation_fn=None, in_layers=[readout]) softmax = SoftMax(in_layers=[classification]) self.add_output(softmax) label = Label(shape=(None, 2)) self.my_labels.append(label) cost = SoftMaxCrossEntropy(in_layers=[label, classification]) costs.append(cost) if self.mode == 'regression': regression = Dense(out_channels=1, activation_fn=None, in_layers=[readout]) self.add_output(regression) label = Label(shape=(None, 1)) self.my_labels.append(label) cost = L2Loss(in_layers=[label, regression]) costs.append(cost) if self.mode == "classification": entropy = Stack(in_layers=costs, axis=-1) elif self.mode == "regression": entropy = Stack(in_layers=costs, axis=1) self.my_task_weights = Weights(shape=(None, self.n_tasks)) loss = WeightedError(in_layers=[entropy, self.my_task_weights]) self.set_loss(loss)
def build_graph(self): """Building graph structures: Features => WeaveLayer => WeaveLayer => Dense => WeaveGather => Classification or Regression """ self.atom_features = Feature(shape=(None, self.n_atom_feat)) self.pair_features = Feature(shape=(None, self.n_pair_feat)) combined = Combine_AP(in_layers=[self.atom_features, self.pair_features]) self.pair_split = Feature(shape=(None,), dtype=tf.int32) self.atom_split = Feature(shape=(None,), dtype=tf.int32) self.atom_to_pair = Feature(shape=(None, 2), dtype=tf.int32) weave_layer1 = WeaveLayer( n_atom_input_feat=self.n_atom_feat, n_pair_input_feat=self.n_pair_feat, n_atom_output_feat=self.n_hidden, n_pair_output_feat=self.n_hidden, in_layers=[combined, self.pair_split, self.atom_to_pair]) weave_layer2 = WeaveLayer( n_atom_input_feat=self.n_hidden, n_pair_input_feat=self.n_hidden, n_atom_output_feat=self.n_hidden, n_pair_output_feat=self.n_hidden, update_pair=False, in_layers=[weave_layer1, self.pair_split, self.atom_to_pair]) separated = Separate_AP(in_layers=[weave_layer2]) dense1 = Dense( out_channels=self.n_graph_feat, activation_fn=tf.nn.tanh, in_layers=[separated]) batch_norm1 = BatchNormalization(epsilon=1e-5, mode=1, in_layers=[dense1]) weave_gather = WeaveGather( self.batch_size, n_input=self.n_graph_feat, gaussian_expand=True, in_layers=[batch_norm1, self.atom_split]) costs = [] self.labels_fd = [] for task in range(self.n_tasks): if self.mode == "classification": classification = Dense( out_channels=2, activation_fn=None, in_layers=[weave_gather]) softmax = SoftMax(in_layers=[classification]) self.add_output(softmax) label = Label(shape=(None, 2)) self.labels_fd.append(label) cost = SoftMaxCrossEntropy(in_layers=[label, classification]) costs.append(cost) if self.mode == "regression": regression = Dense( out_channels=1, activation_fn=None, in_layers=[weave_gather]) self.add_output(regression) label = Label(shape=(None, 1)) self.labels_fd.append(label) cost = L2Loss(in_layers=[label, regression]) costs.append(cost) if self.mode == "classification": all_cost = Concat(in_layers=costs, axis=1) elif self.mode == "regression": all_cost = Stack(in_layers=costs, axis=1) self.weights = Weights(shape=(None, self.n_tasks)) loss = WeightedError(in_layers=[all_cost, self.weights]) self.set_loss(loss)
def __init__(self, n_tasks, n_features, layer_sizes=[1000], weight_init_stddevs=0.02, bias_init_consts=1.0, weight_decay_penalty=0.0, weight_decay_penalty_type="l2", dropouts=0.5, activation_fns=tf.nn.relu, n_classes=2, **kwargs): """Create a MultitaskClassifier. In addition to the following arguments, this class also accepts all the keyword arguments from TensorGraph. Parameters ---------- n_tasks: int number of tasks n_features: int number of features layer_sizes: list the size of each dense layer in the network. The length of this list determines the number of layers. weight_init_stddevs: list or float the standard deviation of the distribution to use for weight initialization of each layer. The length of this list should equal len(layer_sizes). Alternatively this may be a single value instead of a list, in which case the same value is used for every layer. bias_init_consts: list or loat the value to initialize the biases in each layer to. The length of this list should equal len(layer_sizes). Alternatively this may be a single value instead of a list, in which case the same value is used for every layer. weight_decay_penalty: float the magnitude of the weight decay penalty to use weight_decay_penalty_type: str the type of penalty to use for weight decay, either 'l1' or 'l2' dropouts: list or float the dropout probablity to use for each layer. The length of this list should equal len(layer_sizes). Alternatively this may be a single value instead of a list, in which case the same value is used for every layer. activation_fns: list or object the Tensorflow activation function to apply to each layer. The length of this list should equal len(layer_sizes). Alternatively this may be a single value instead of a list, in which case the same value is used for every layer. n_classes: int the number of classes """ super(MultitaskClassifier, self).__init__(**kwargs) self.n_tasks = n_tasks self.n_features = n_features self.n_classes = n_classes n_layers = len(layer_sizes) if not isinstance(weight_init_stddevs, collections.Sequence): weight_init_stddevs = [weight_init_stddevs] * n_layers if not isinstance(bias_init_consts, collections.Sequence): bias_init_consts = [bias_init_consts] * n_layers if not isinstance(dropouts, collections.Sequence): dropouts = [dropouts] * n_layers if not isinstance(activation_fns, collections.Sequence): activation_fns = [activation_fns] * n_layers # Add the input features. mol_features = Feature(shape=(None, n_features)) prev_layer = mol_features # Add the dense layers for size, weight_stddev, bias_const, dropout, activation_fn in zip( layer_sizes, weight_init_stddevs, bias_init_consts, dropouts, activation_fns): layer = Dense(in_layers=[prev_layer], out_channels=size, activation_fn=activation_fn, weights_initializer=TFWrapper( tf.truncated_normal_initializer, stddev=weight_stddev), biases_initializer=TFWrapper(tf.constant_initializer, value=bias_const)) if dropout > 0.0: layer = Dropout(dropout, in_layers=[layer]) prev_layer = layer # Compute the loss function for each label. self.neural_fingerprint = prev_layer logits = Reshape(shape=(-1, n_tasks, n_classes), in_layers=[ Dense(in_layers=[prev_layer], out_channels=n_tasks * n_classes) ]) output = SoftMax(logits) self.add_output(output) labels = Label(shape=(None, n_tasks, n_classes)) weights = Weights(shape=(None, n_tasks)) loss = SoftMaxCrossEntropy(in_layers=[labels, logits]) weighted_loss = WeightedError(in_layers=[loss, weights]) if weight_decay_penalty != 0.0: weighted_loss = WeightDecay(weight_decay_penalty, weight_decay_penalty_type, in_layers=[weighted_loss]) self.set_loss(weighted_loss)
def graph_conv_net(batch_size, prior, num_task): """ Build a tensorgraph for multilabel classification task Return: features and labels layers """ tg = TensorGraph(use_queue=False) if prior == True: add_on = num_task else: add_on = 0 atom_features = Feature(shape=(None, 75 + 2 * add_on)) circular_features = Feature(shape=(batch_size, 256), dtype=tf.float32) degree_slice = Feature(shape=(None, 2), dtype=tf.int32) membership = Feature(shape=(None, ), dtype=tf.int32) deg_adjs = [] for i in range(0, 10 + 1): deg_adj = Feature(shape=(None, i + 1), dtype=tf.int32) deg_adjs.append(deg_adj) gc1 = GraphConv(64 + add_on, activation_fn=tf.nn.elu, in_layers=[atom_features, degree_slice, membership] + deg_adjs) batch_norm1 = BatchNorm(in_layers=[gc1]) gp1 = GraphPool(in_layers=[batch_norm1, degree_slice, membership] + deg_adjs) gc2 = GraphConv(64 + add_on, activation_fn=tf.nn.elu, in_layers=[gc1, degree_slice, membership] + deg_adjs) batch_norm2 = BatchNorm(in_layers=[gc2]) gp2 = GraphPool(in_layers=[batch_norm2, degree_slice, membership] + deg_adjs) add = Concat(in_layers=[gp1, gp2]) add = Dropout(0.5, in_layers=[add]) dense = Dense(out_channels=128, activation_fn=tf.nn.elu, in_layers=[add]) batch_norm3 = BatchNorm(in_layers=[dense]) readout = GraphGather(batch_size=batch_size, activation_fn=tf.nn.tanh, in_layers=[batch_norm3, degree_slice, membership] + deg_adjs) batch_norm4 = BatchNorm(in_layers=[readout]) dense1 = Dense(out_channels=128, activation_fn=tf.nn.elu, in_layers=[circular_features]) dense1 = BatchNorm(in_layers=[dense1]) dense1 = Dropout(0.5, in_layers=[dense1]) dense1 = Dense(out_channels=128, activation_fn=tf.nn.elu, in_layers=[circular_features]) dense1 = BatchNorm(in_layers=[dense1]) dense1 = Dropout(0.5, in_layers=[dense1]) merge_feat = Concat(in_layers=[dense1, batch_norm4]) merge = Dense(out_channels=256, activation_fn=tf.nn.elu, in_layers=[merge_feat]) costs = [] labels = [] for task in range(num_task): classification = Dense(out_channels=2, activation_fn=None, in_layers=[merge]) softmax = SoftMax(in_layers=[classification]) tg.add_output(softmax) label = Label(shape=(None, 2)) labels.append(label) cost = SoftMaxCrossEntropy(in_layers=[label, classification]) costs.append(cost) all_cost = Stack(in_layers=costs, axis=1) weights = Weights(shape=(None, num_task)) loss = WeightedError(in_layers=[all_cost, weights]) tg.set_loss(loss) #if prior == True: # return tg, atom_features,circular_features, degree_slice, membership, deg_adjs, labels, weights#, prior_layer return tg, atom_features, circular_features, degree_slice, membership, deg_adjs, labels, weights
def build_graph(self): """ Building graph structures: """ self.atom_features = Feature(shape=(None, 75)) self.degree_slice = Feature(shape=(None, 2), dtype=tf.int32) self.membership = Feature(shape=(None,), dtype=tf.int32) self.deg_adjs = [] for i in range(0, 10 + 1): deg_adj = Feature(shape=(None, i + 1), dtype=tf.int32) self.deg_adjs.append(deg_adj) gc1 = GraphConv( 64, activation_fn=tf.nn.relu, in_layers=[self.atom_features, self.degree_slice, self.membership] + self.deg_adjs) batch_norm1 = BatchNorm(in_layers=[gc1]) gp1 = GraphPool(in_layers=[batch_norm1, self.degree_slice, self.membership] + self.deg_adjs) gc2 = GraphConv( 64, activation_fn=tf.nn.relu, in_layers=[gp1, self.degree_slice, self.membership] + self.deg_adjs) batch_norm2 = BatchNorm(in_layers=[gc2]) gp2 = GraphPool(in_layers=[batch_norm2, self.degree_slice, self.membership] + self.deg_adjs) dense = Dense(out_channels=128, activation_fn=tf.nn.relu, in_layers=[gp2]) batch_norm3 = BatchNorm(in_layers=[dense]) readout = GraphGather( batch_size=self.batch_size, activation_fn=tf.nn.tanh, in_layers=[batch_norm3, self.degree_slice, self.membership] + self.deg_adjs) if self.error_bars == True: readout = Dropout(in_layers=[readout], dropout_prob=0.2) costs = [] self.my_labels = [] for task in range(self.n_tasks): if self.mode == 'classification': classification = Dense( out_channels=2, activation_fn=None, in_layers=[readout]) softmax = SoftMax(in_layers=[classification]) self.add_output(softmax) label = Label(shape=(None, 2)) self.my_labels.append(label) cost = SoftMaxCrossEntropy(in_layers=[label, classification]) costs.append(cost) if self.mode == 'regression': regression = Dense( out_channels=1, activation_fn=None, in_layers=[readout]) self.add_output(regression) label = Label(shape=(None, 1)) self.my_labels.append(label) cost = L2Loss(in_layers=[label, regression]) costs.append(cost) if self.mode == "classification": entropy = Concat(in_layers=costs, axis=-1) elif self.mode == "regression": entropy = Stack(in_layers=costs, axis=1) self.my_task_weights = Weights(shape=(None, self.n_tasks)) loss = WeightedError(in_layers=[entropy, self.my_task_weights]) self.set_loss(loss)
def graph_conv_model(batch_size, tasks): model = TensorGraph(model_dir=model_dir, batch_size=batch_size, use_queue=False) atom_features = Feature(shape=(None, 75)) degree_slice = Feature(shape=(None, 2), dtype=tf.int32) membership = Feature(shape=(None, ), dtype=tf.int32) deg_adjs = [] for i in range(0, 10 + 1): deg_adj = Feature(shape=(None, i + 1), dtype=tf.int32) deg_adjs.append(deg_adj) gc1 = GraphConv(64, activation_fn=tf.nn.relu, in_layers=[atom_features, degree_slice, membership] + deg_adjs) batch_norm1 = BatchNorm(in_layers=[gc1]) gp1 = GraphPool(in_layers=[batch_norm1, degree_slice, membership] + deg_adjs) gc2 = GraphConv(64, activation_fn=tf.nn.relu, in_layers=[gp1, degree_slice, membership] + deg_adjs) batch_norm2 = BatchNorm(in_layers=[gc2]) gp2 = GraphPool(in_layers=[batch_norm2, degree_slice, membership] + deg_adjs) dense = Dense(out_channels=128, activation_fn=None, in_layers=[gp2]) batch_norm3 = BatchNorm(in_layers=[dense]) gg1 = GraphGather(batch_size=batch_size, activation_fn=tf.nn.tanh, in_layers=[batch_norm3, degree_slice, membership] + deg_adjs) costs = [] labels = [] for task in tasks: classification = Dense(out_channels=2, activation_fn=None, in_layers=[gg1]) softmax = SoftMax(in_layers=[classification]) model.add_output(softmax) label = Label(shape=(None, 2)) labels.append(label) cost = SoftMaxCrossEntropy(in_layers=[label, classification]) costs.append(cost) entropy = Concat(in_layers=costs) task_weights = Weights(shape=(None, len(tasks))) loss = WeightedError(in_layers=[entropy, task_weights]) model.set_loss(loss) def feed_dict_generator(dataset, batch_size, epochs=1): for epoch in range(epochs): for ind, (X_b, y_b, w_b, ids_b) in enumerate( dataset.iterbatches(batch_size, pad_batches=True)): d = {} for index, label in enumerate(labels): d[label] = to_one_hot(y_b[:, index]) d[task_weights] = w_b multiConvMol = ConvMol.agglomerate_mols(X_b) d[atom_features] = multiConvMol.get_atom_features() d[degree_slice] = multiConvMol.deg_slice d[membership] = multiConvMol.membership for i in range(1, len(multiConvMol.get_deg_adjacency_lists())): d[deg_adjs[i - 1]] = multiConvMol.get_deg_adjacency_lists()[i] yield d return model, feed_dict_generator, labels, task_weights
def atomic_conv_model(frag1_num_atoms=70, frag2_num_atoms=634, complex_num_atoms=701, max_num_neighbors=12, batch_size=24, at=[ 6, 7., 8., 9., 11., 12., 15., 16., 17., 20., 25., 30., 35., 53., -1. ], radial=[[ 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0, 5.5, 6.0, 6.5, 7.0, 7.5, 8.0, 8.5, 9.0, 9.5, 10.0, 10.5, 11.0, 11.5, 12.0 ], [0.0, 4.0, 8.0], [0.4]], layer_sizes=[32, 32, 16], learning_rate=0.001): rp = [x for x in itertools.product(*radial)] frag1_X = Feature(shape=(batch_size, frag1_num_atoms, 3)) frag1_nbrs = Feature(shape=(batch_size, frag1_num_atoms, max_num_neighbors)) frag1_nbrs_z = Feature(shape=(batch_size, frag1_num_atoms, max_num_neighbors)) frag1_z = Feature(shape=(batch_size, frag1_num_atoms)) frag2_X = Feature(shape=(batch_size, frag2_num_atoms, 3)) frag2_nbrs = Feature(shape=(batch_size, frag2_num_atoms, max_num_neighbors)) frag2_nbrs_z = Feature(shape=(batch_size, frag2_num_atoms, max_num_neighbors)) frag2_z = Feature(shape=(batch_size, frag2_num_atoms)) complex_X = Feature(shape=(batch_size, complex_num_atoms, 3)) complex_nbrs = Feature(shape=(batch_size, complex_num_atoms, max_num_neighbors)) complex_nbrs_z = Feature(shape=(batch_size, complex_num_atoms, max_num_neighbors)) complex_z = Feature(shape=(batch_size, complex_num_atoms)) frag1_conv = AtomicConvolution( atom_types=at, radial_params=rp, boxsize=None, in_layers=[frag1_X, frag1_nbrs, frag1_nbrs_z]) frag2_conv = AtomicConvolution( atom_types=at, radial_params=rp, boxsize=None, in_layers=[frag2_X, frag2_nbrs, frag2_nbrs_z]) complex_conv = AtomicConvolution( atom_types=at, radial_params=rp, boxsize=None, in_layers=[complex_X, complex_nbrs, complex_nbrs_z]) score = AtomicConvScore(at, layer_sizes, in_layers=[ frag1_conv, frag2_conv, complex_conv, frag1_z, frag2_z, complex_z ]) label = Label(shape=(None, 1)) loss = ReduceMean(in_layers=L2Loss(in_layers=[score, label])) def feed_dict_generator(dataset, batch_size, epochs=1, pad_batches=True): def replace_atom_types(z): def place_holder(i): if i in at: return i return -1 return np.array([place_holder(x) for x in z]) for epoch in range(epochs): for ind, (F_b, y_b, w_b, ids_b) in enumerate( dataset.iterbatches(batch_size, deterministic=True, pad_batches=pad_batches)): N = complex_num_atoms N_1 = frag1_num_atoms N_2 = frag2_num_atoms M = max_num_neighbors orig_dict = {} batch_size = F_b.shape[0] num_features = F_b[0][0].shape[1] frag1_X_b = np.zeros((batch_size, N_1, num_features)) for i in range(batch_size): frag1_X_b[i] = F_b[i][0] orig_dict[frag1_X] = frag1_X_b frag2_X_b = np.zeros((batch_size, N_2, num_features)) for i in range(batch_size): frag2_X_b[i] = F_b[i][3] orig_dict[frag2_X] = frag2_X_b complex_X_b = np.zeros((batch_size, N, num_features)) for i in range(batch_size): complex_X_b[i] = F_b[i][6] orig_dict[complex_X] = complex_X_b frag1_Nbrs = np.zeros((batch_size, N_1, M)) frag1_Z_b = np.zeros((batch_size, N_1)) for i in range(batch_size): z = replace_atom_types(F_b[i][2]) frag1_Z_b[i] = z frag1_Nbrs_Z = np.zeros((batch_size, N_1, M)) for atom in range(N_1): for i in range(batch_size): atom_nbrs = F_b[i][1].get(atom, "") frag1_Nbrs[i, atom, :len(atom_nbrs)] = np.array(atom_nbrs) for j, atom_j in enumerate(atom_nbrs): frag1_Nbrs_Z[i, atom, j] = frag1_Z_b[i, atom_j] orig_dict[frag1_nbrs] = frag1_Nbrs orig_dict[frag1_nbrs_z] = frag1_Nbrs_Z orig_dict[frag1_z] = frag1_Z_b frag2_Nbrs = np.zeros((batch_size, N_2, M)) frag2_Z_b = np.zeros((batch_size, N_2)) for i in range(batch_size): z = replace_atom_types(F_b[i][5]) frag2_Z_b[i] = z frag2_Nbrs_Z = np.zeros((batch_size, N_2, M)) for atom in range(N_2): for i in range(batch_size): atom_nbrs = F_b[i][4].get(atom, "") frag2_Nbrs[i, atom, :len(atom_nbrs)] = np.array(atom_nbrs) for j, atom_j in enumerate(atom_nbrs): frag2_Nbrs_Z[i, atom, j] = frag2_Z_b[i, atom_j] orig_dict[frag2_nbrs] = frag2_Nbrs orig_dict[frag2_nbrs_z] = frag2_Nbrs_Z orig_dict[frag2_z] = frag2_Z_b complex_Nbrs = np.zeros((batch_size, N, M)) complex_Z_b = np.zeros((batch_size, N)) for i in range(batch_size): z = replace_atom_types(F_b[i][8]) complex_Z_b[i] = z complex_Nbrs_Z = np.zeros((batch_size, N, M)) for atom in range(N): for i in range(batch_size): atom_nbrs = F_b[i][7].get(atom, "") complex_Nbrs[i, atom, :len(atom_nbrs)] = np.array( atom_nbrs) for j, atom_j in enumerate(atom_nbrs): complex_Nbrs_Z[i, atom, j] = complex_Z_b[i, atom_j] orig_dict[complex_nbrs] = complex_Nbrs orig_dict[complex_nbrs_z] = complex_Nbrs_Z orig_dict[complex_z] = complex_Z_b orig_dict[label] = np.reshape(y_b, newshape=(batch_size, 1)) yield orig_dict tg = TensorGraph(batch_size=batch_size, mode=str("regression"), model_dir=str("/tmp/atom_conv"), learning_rate=learning_rate) tg.add_output(score) tg.set_loss(loss) return tg, feed_dict_generator, label
def __init__(self, n_tasks, n_features, layer_sizes=[1000], weight_init_stddevs=[0.02], bias_init_consts=[1.0], weight_decay_penalty=0.0, weight_decay_penalty_type="l2", dropouts=[0.5], n_classes=2, **kwargs): """Create a TensorGraphMultiTaskClassifier. In addition to the following arguments, this class also accepts all the keywork arguments from TensorGraph. Parameters ---------- n_tasks: int number of tasks n_features: int number of features layer_sizes: list the size of each dense layer in the network. The length of this list determines the number of layers. weight_init_stddevs: list the standard deviation of the distribution to use for weight initialization of each layer. The length of this list should equal len(layer_sizes). bias_init_consts: list the value to initialize the biases in each layer to. The length of this list should equal len(layer_sizes). weight_decay_penalty: float the magnitude of the weight decay penalty to use weight_decay_penalty_type: str the type of penalty to use for weight decay, either 'l1' or 'l2' dropouts: list the dropout probablity to use for each layer. The length of this list should equal len(layer_sizes). n_classes: int the number of classes """ super(TensorGraphMultiTaskClassifier, self).__init__(mode='classification', **kwargs) self.n_tasks = n_tasks self.n_features = n_features self.n_classes = n_classes # Add the input features. mol_features = Feature(shape=(None, n_features)) prev_layer = mol_features # Add the dense layers for size, weight_stddev, bias_const, dropout in zip( layer_sizes, weight_init_stddevs, bias_init_consts, dropouts): layer = Dense(in_layers=[prev_layer], out_channels=size, activation_fn=tf.nn.relu, weights_initializer=TFWrapper( tf.truncated_normal_initializer, stddev=weight_stddev), biases_initializer=TFWrapper(tf.constant_initializer, value=bias_const)) if dropout > 0.0: layer = Dropout(dropout, in_layers=[layer]) prev_layer = layer # Compute the loss function for each label. output = Reshape(shape=(-1, n_tasks, n_classes), in_layers=[ Dense(in_layers=[prev_layer], out_channels=n_tasks * n_classes) ]) self.add_output(output) labels = Label(shape=(None, n_tasks, n_classes)) weights = Weights(shape=(None, n_tasks)) loss = SoftMaxCrossEntropy(in_layers=[labels, output]) weighted_loss = WeightedError(in_layers=[loss, weights]) if weight_decay_penalty != 0.0: weighted_loss = WeightDecay(weight_decay_penalty, weight_decay_penalty_type, in_layers=[weighted_loss]) self.set_loss(weighted_loss)
def __init__(self, n_tasks, n_features, layer_sizes=[1000], weight_init_stddevs=0.02, bias_init_consts=1.0, weight_decay_penalty=0.0, weight_decay_penalty_type="l2", dropouts=0.5, activation_fns=tf.nn.relu, uncertainty=False, **kwargs): """Create a MultitaskRegressor. In addition to the following arguments, this class also accepts all the keywork arguments from TensorGraph. Parameters ---------- n_tasks: int number of tasks n_features: int number of features layer_sizes: list the size of each dense layer in the network. The length of this list determines the number of layers. weight_init_stddevs: list or float the standard deviation of the distribution to use for weight initialization of each layer. The length of this list should equal len(layer_sizes)+1. The final element corresponds to the output layer. Alternatively this may be a single value instead of a list, in which case the same value is used for every layer. bias_init_consts: list or float the value to initialize the biases in each layer to. The length of this list should equal len(layer_sizes)+1. The final element corresponds to the output layer. Alternatively this may be a single value instead of a list, in which case the same value is used for every layer. weight_decay_penalty: float the magnitude of the weight decay penalty to use weight_decay_penalty_type: str the type of penalty to use for weight decay, either 'l1' or 'l2' dropouts: list or float the dropout probablity to use for each layer. The length of this list should equal len(layer_sizes). Alternatively this may be a single value instead of a list, in which case the same value is used for every layer. activation_fns: list or object the Tensorflow activation function to apply to each layer. The length of this list should equal len(layer_sizes). Alternatively this may be a single value instead of a list, in which case the same value is used for every layer. uncertainty: bool if True, include extra outputs and loss terms to enable the uncertainty in outputs to be predicted """ super(MultitaskRegressor, self).__init__(**kwargs) self.n_tasks = n_tasks self.n_features = n_features n_layers = len(layer_sizes) if not isinstance(weight_init_stddevs, collections.Sequence): weight_init_stddevs = [weight_init_stddevs] * (n_layers + 1) if not isinstance(bias_init_consts, collections.Sequence): bias_init_consts = [bias_init_consts] * (n_layers + 1) if not isinstance(dropouts, collections.Sequence): dropouts = [dropouts] * n_layers if not isinstance(activation_fns, collections.Sequence): activation_fns = [activation_fns] * n_layers if uncertainty: if any(d == 0.0 for d in dropouts): raise ValueError( 'Dropout must be included in every layer to predict uncertainty' ) # Add the input features. mol_features = Feature(shape=(None, n_features)) prev_layer = mol_features # Add the dense layers for size, weight_stddev, bias_const, dropout, activation_fn in zip( layer_sizes, weight_init_stddevs, bias_init_consts, dropouts, activation_fns): layer = Dense(in_layers=[prev_layer], out_channels=size, activation_fn=activation_fn, weights_initializer=TFWrapper( tf.truncated_normal_initializer, stddev=weight_stddev), biases_initializer=TFWrapper(tf.constant_initializer, value=bias_const)) if dropout > 0.0: layer = Dropout(dropout, in_layers=[layer]) prev_layer = layer self.neural_fingerprint = prev_layer # Compute the loss function for each label. output = Reshape(shape=(-1, n_tasks, 1), in_layers=[ Dense(in_layers=[prev_layer], out_channels=n_tasks, weights_initializer=TFWrapper( tf.truncated_normal_initializer, stddev=weight_init_stddevs[-1]), biases_initializer=TFWrapper( tf.constant_initializer, value=bias_init_consts[-1])) ]) self.add_output(output) labels = Label(shape=(None, n_tasks, 1)) weights = Weights(shape=(None, n_tasks, 1)) if uncertainty: log_var = Reshape( shape=(-1, n_tasks, 1), in_layers=[ Dense(in_layers=[prev_layer], out_channels=n_tasks, weights_initializer=TFWrapper( tf.truncated_normal_initializer, stddev=weight_init_stddevs[-1]), biases_initializer=TFWrapper(tf.constant_initializer, value=0.0)) ]) var = Exp(log_var) self.add_variance(var) diff = labels - output weighted_loss = weights * (diff * diff / var + log_var) weighted_loss = ReduceSum(ReduceMean(weighted_loss, axis=[1, 2])) else: weighted_loss = ReduceSum( L2Loss(in_layers=[labels, output, weights])) if weight_decay_penalty != 0.0: weighted_loss = WeightDecay(weight_decay_penalty, weight_decay_penalty_type, in_layers=[weighted_loss]) self.set_loss(weighted_loss)
def build_graph(self): """Building graph structures: Features => WeaveLayer => WeaveLayer => Dense => WeaveGather => Classification or Regression """ self.atom_features = Feature(shape=(None, self.n_atom_feat)) self.pair_features = Feature(shape=(None, self.n_pair_feat)) self.pair_split = Feature(shape=(None, ), dtype=tf.int32) self.atom_split = Feature(shape=(None, ), dtype=tf.int32) self.atom_to_pair = Feature(shape=(None, 2), dtype=tf.int32) weave_layer1A, weave_layer1P = WeaveLayerFactory( n_atom_input_feat=self.n_atom_feat, n_pair_input_feat=self.n_pair_feat, n_atom_output_feat=self.n_hidden, n_pair_output_feat=self.n_hidden, in_layers=[ self.atom_features, self.pair_features, self.pair_split, self.atom_to_pair ]) weave_layer2A, weave_layer2P = WeaveLayerFactory( n_atom_input_feat=self.n_hidden, n_pair_input_feat=self.n_hidden, n_atom_output_feat=self.n_hidden, n_pair_output_feat=self.n_hidden, update_pair=False, in_layers=[ weave_layer1A, weave_layer1P, self.pair_split, self.atom_to_pair ]) dense1 = Dense(out_channels=self.n_graph_feat, activation_fn=tf.nn.tanh, in_layers=weave_layer2A) batch_norm1 = BatchNorm(epsilon=1e-5, in_layers=[dense1]) weave_gather = WeaveGather(self.batch_size, n_input=self.n_graph_feat, gaussian_expand=True, in_layers=[batch_norm1, self.atom_split]) n_tasks = self.n_tasks weights = Weights(shape=(None, n_tasks)) if self.mode == 'classification': n_classes = self.n_classes labels = Label(shape=(None, n_tasks, n_classes)) logits = Reshape(shape=(None, n_tasks, n_classes), in_layers=[ Dense(in_layers=weave_gather, out_channels=n_tasks * n_classes) ]) output = SoftMax(logits) self.add_output(output) loss = SoftMaxCrossEntropy(in_layers=[labels, logits]) weighted_loss = WeightedError(in_layers=[loss, weights]) self.set_loss(weighted_loss) else: labels = Label(shape=(None, n_tasks)) output = Reshape(shape=(None, n_tasks), in_layers=[ Dense(in_layers=weave_gather, out_channels=n_tasks) ]) self.add_output(output) weighted_loss = ReduceSum( L2Loss(in_layers=[labels, output, weights])) self.set_loss(weighted_loss)
def build_graph(self): # Build placeholders self.atom_features = Feature(shape=(None, self.n_atom_feat)) self.pair_features = Feature(shape=(None, self.n_pair_feat)) self.atom_split = Feature(shape=(None, ), dtype=tf.int32) self.atom_to_pair = Feature(shape=(None, 2), dtype=tf.int32) message_passing = MessagePassing(self.T, message_fn='enn', update_fn='gru', n_hidden=self.n_hidden, in_layers=[ self.atom_features, self.pair_features, self.atom_to_pair ]) atom_embeddings = Dense(self.n_hidden, in_layers=[message_passing]) mol_embeddings = SetGather( self.M, self.batch_size, n_hidden=self.n_hidden, in_layers=[atom_embeddings, self.atom_split]) dense1 = Dense(out_channels=2 * self.n_hidden, activation_fn=tf.nn.relu, in_layers=[mol_embeddings]) n_tasks = self.n_tasks weights = Weights(shape=(None, n_tasks)) if self.mode == 'classification': n_classes = self.n_classes labels = Label(shape=(None, n_tasks, n_classes)) logits = Reshape(shape=(None, n_tasks, n_classes), in_layers=[ Dense(in_layers=dense1, out_channels=n_tasks * n_classes) ]) logits = TrimGraphOutput([logits, weights]) output = SoftMax(logits) self.add_output(output) loss = SoftMaxCrossEntropy(in_layers=[labels, logits]) weighted_loss = WeightedError(in_layers=[loss, weights]) self.set_loss(weighted_loss) else: labels = Label(shape=(None, n_tasks)) output = Reshape( shape=(None, n_tasks), in_layers=[Dense(in_layers=dense1, out_channels=n_tasks)]) output = TrimGraphOutput([output, weights]) self.add_output(output) if self.uncertainty: log_var = Reshape( shape=(None, n_tasks), in_layers=[Dense(in_layers=dense1, out_channels=n_tasks)]) log_var = TrimGraphOutput([log_var, weights]) var = Exp(log_var) self.add_variance(var) diff = labels - output weighted_loss = weights * (diff * diff / var + log_var) weighted_loss = ReduceSum(ReduceMean(weighted_loss, axis=[1])) else: weighted_loss = ReduceSum( L2Loss(in_layers=[labels, output, weights])) self.set_loss(weighted_loss)
def build_graph(self): """ Building graph structures: """ self.atom_features = Feature(shape=(None, self.number_atom_features)) self.degree_slice = Feature(shape=(None, 2), dtype=tf.int32) self.membership = Feature(shape=(None, ), dtype=tf.int32) self.deg_adjs = [] for i in range(0, 10 + 1): deg_adj = Feature(shape=(None, i + 1), dtype=tf.int32) self.deg_adjs.append(deg_adj) in_layer = self.atom_features for layer_size, dropout in zip(self.graph_conv_layers, self.dropout): gc1_in = [in_layer, self.degree_slice, self.membership ] + self.deg_adjs gc1 = GraphConv(layer_size, activation_fn=tf.nn.relu, in_layers=gc1_in) batch_norm1 = BatchNorm(in_layers=[gc1]) if dropout > 0.0: batch_norm1 = Dropout(dropout, in_layers=batch_norm1) gp_in = [batch_norm1, self.degree_slice, self.membership ] + self.deg_adjs in_layer = GraphPool(in_layers=gp_in) dense = Dense(out_channels=self.dense_layer_size, activation_fn=tf.nn.relu, in_layers=[in_layer]) batch_norm3 = BatchNorm(in_layers=[dense]) if self.dropout[-1] > 0.0: batch_norm3 = Dropout(self.dropout[-1], in_layers=batch_norm3) readout = GraphGather( batch_size=self.batch_size, activation_fn=tf.nn.tanh, in_layers=[batch_norm3, self.degree_slice, self.membership] + self.deg_adjs) n_tasks = self.n_tasks weights = Weights(shape=(None, n_tasks)) if self.mode == 'classification': n_classes = self.n_classes labels = Label(shape=(None, n_tasks, n_classes)) logits = Reshape(shape=(None, n_tasks, n_classes), in_layers=[ Dense(in_layers=readout, out_channels=n_tasks * n_classes) ]) logits = TrimGraphOutput([logits, weights]) output = SoftMax(logits) self.add_output(output) loss = SoftMaxCrossEntropy(in_layers=[labels, logits]) weighted_loss = WeightedError(in_layers=[loss, weights]) self.set_loss(weighted_loss) else: labels = Label(shape=(None, n_tasks)) output = Reshape( shape=(None, n_tasks), in_layers=[Dense(in_layers=readout, out_channels=n_tasks)]) output = TrimGraphOutput([output, weights]) self.add_output(output) if self.uncertainty: log_var = Reshape( shape=(None, n_tasks), in_layers=[Dense(in_layers=readout, out_channels=n_tasks)]) log_var = TrimGraphOutput([log_var, weights]) var = Exp(log_var) self.add_variance(var) diff = labels - output weighted_loss = weights * (diff * diff / var + log_var) weighted_loss = ReduceSum(ReduceMean(weighted_loss, axis=[1])) else: weighted_loss = ReduceSum( L2Loss(in_layers=[labels, output, weights])) self.set_loss(weighted_loss)
def sluice_model(batch_size, tasks): model = TensorGraph(model_dir=model_dir, batch_size=batch_size, use_queue=False, tensorboard=True) atom_features = Feature(shape=(None, 75)) degree_slice = Feature(shape=(None, 2), dtype=tf.int32) membership = Feature(shape=(None, ), dtype=tf.int32) sluice_loss = [] deg_adjs = [] for i in range(0, 10 + 1): deg_adj = Feature(shape=(None, i + 1), dtype=tf.int32) deg_adjs.append(deg_adj) gc1 = GraphConv(64, activation_fn=tf.nn.relu, in_layers=[atom_features, degree_slice, membership] + deg_adjs) as1 = AlphaShare(in_layers=[gc1, gc1]) sluice_loss.append(gc1) batch_norm1a = BatchNorm(in_layers=[as1[0]]) batch_norm1b = BatchNorm(in_layers=[as1[1]]) gp1a = GraphPool(in_layers=[batch_norm1a, degree_slice, membership] + deg_adjs) gp1b = GraphPool(in_layers=[batch_norm1b, degree_slice, membership] + deg_adjs) gc2a = GraphConv(64, activation_fn=tf.nn.relu, in_layers=[gp1a, degree_slice, membership] + deg_adjs) gc2b = GraphConv(64, activation_fn=tf.nn.relu, in_layers=[gp1b, degree_slice, membership] + deg_adjs) as2 = AlphaShare(in_layers=[gc2a, gc2b]) sluice_loss.append(gc2a) sluice_loss.append(gc2b) batch_norm2a = BatchNorm(in_layers=[as2[0]]) batch_norm2b = BatchNorm(in_layers=[as2[1]]) gp2a = GraphPool(in_layers=[batch_norm2a, degree_slice, membership] + deg_adjs) gp2b = GraphPool(in_layers=[batch_norm2b, degree_slice, membership] + deg_adjs) densea = Dense(out_channels=128, activation_fn=None, in_layers=[gp2a]) denseb = Dense(out_channels=128, activation_fn=None, in_layers=[gp2b]) batch_norm3a = BatchNorm(in_layers=[densea]) batch_norm3b = BatchNorm(in_layers=[denseb]) as3 = AlphaShare(in_layers=[batch_norm3a, batch_norm3b]) sluice_loss.append(batch_norm3a) sluice_loss.append(batch_norm3b) gg1a = GraphGather(batch_size=batch_size, activation_fn=tf.nn.tanh, in_layers=[as3[0], degree_slice, membership] + deg_adjs) gg1b = GraphGather(batch_size=batch_size, activation_fn=tf.nn.tanh, in_layers=[as3[1], degree_slice, membership] + deg_adjs) costs = [] labels = [] count = 0 for task in tasks: if count < len(tasks) / 2: classification = Dense(out_channels=2, activation_fn=None, in_layers=[gg1a]) print("first half:") print(task) else: classification = Dense(out_channels=2, activation_fn=None, in_layers=[gg1b]) print('second half') print(task) count += 1 softmax = SoftMax(in_layers=[classification]) model.add_output(softmax) label = Label(shape=(None, 2)) labels.append(label) cost = SoftMaxCrossEntropy(in_layers=[label, classification]) costs.append(cost) entropy = Concat(in_layers=costs) task_weights = Weights(shape=(None, len(tasks))) task_loss = WeightedError(in_layers=[entropy, task_weights]) s_cost = SluiceLoss(in_layers=sluice_loss) total_loss = Add(in_layers=[task_loss, s_cost]) model.set_loss(total_loss) def feed_dict_generator(dataset, batch_size, epochs=1): for epoch in range(epochs): for ind, (X_b, y_b, w_b, ids_b) in enumerate( dataset.iterbatches(batch_size, pad_batches=True)): d = {} for index, label in enumerate(labels): d[label] = to_one_hot(y_b[:, index]) d[task_weights] = w_b multiConvMol = ConvMol.agglomerate_mols(X_b) d[atom_features] = multiConvMol.get_atom_features() d[degree_slice] = multiConvMol.deg_slice d[membership] = multiConvMol.membership for i in range(1, len(multiConvMol.get_deg_adjacency_lists())): d[deg_adjs[i - 1]] = multiConvMol.get_deg_adjacency_lists()[i] yield d return model, feed_dict_generator, labels, task_weights
def build_graph(self): """Building graph structures: Features => DAGLayer => DAGGather => Classification or Regression """ self.atom_features = Feature(shape=(None, self.n_atom_feat)) self.parents = Feature(shape=(None, self.max_atoms, self.max_atoms), dtype=tf.int32) self.calculation_orders = Feature(shape=(None, self.max_atoms), dtype=tf.int32) self.calculation_masks = Feature(shape=(None, self.max_atoms), dtype=tf.bool) self.membership = Feature(shape=(None, ), dtype=tf.int32) self.n_atoms = Feature(shape=(), dtype=tf.int32) dag_layer1 = DAGLayer(n_graph_feat=self.n_graph_feat, n_atom_feat=self.n_atom_feat, max_atoms=self.max_atoms, layer_sizes=self.layer_sizes, dropout=self.dropout, batch_size=self.batch_size, in_layers=[ self.atom_features, self.parents, self.calculation_orders, self.calculation_masks, self.n_atoms ]) dag_gather = DAGGather(n_graph_feat=self.n_graph_feat, n_outputs=self.n_outputs, max_atoms=self.max_atoms, layer_sizes=self.layer_sizes_gather, dropout=self.dropout, in_layers=[dag_layer1, self.membership]) n_tasks = self.n_tasks weights = Weights(shape=(None, n_tasks)) if self.mode == 'classification': n_classes = self.n_classes labels = Label(shape=(None, n_tasks, n_classes)) logits = Reshape(shape=(None, n_tasks, n_classes), in_layers=[ Dense(in_layers=dag_gather, out_channels=n_tasks * n_classes) ]) output = SoftMax(logits) self.add_output(output) loss = SoftMaxCrossEntropy(in_layers=[labels, logits]) weighted_loss = WeightedError(in_layers=[loss, weights]) self.set_loss(weighted_loss) else: labels = Label(shape=(None, n_tasks)) output = Reshape( shape=(None, n_tasks), in_layers=[Dense(in_layers=dag_gather, out_channels=n_tasks)]) self.add_output(output) if self.uncertainty: log_var = Reshape(shape=(None, n_tasks), in_layers=[ Dense(in_layers=dag_gather, out_channels=n_tasks) ]) var = Exp(log_var) self.add_variance(var) diff = labels - output weighted_loss = weights * (diff * diff / var + log_var) weighted_loss = ReduceSum(ReduceMean(weighted_loss, axis=[1])) else: weighted_loss = ReduceSum( L2Loss(in_layers=[labels, output, weights])) self.set_loss(weighted_loss)