def test_softmax_cross_entropy(self): """Test that SoftMaxCrossEntropy can be invoked.""" batch_size = 10 n_features = 5 logit_tensor = np.random.rand(batch_size, n_features) label_tensor = np.random.rand(batch_size, n_features) with self.session() as sess: logit_tensor = tf.convert_to_tensor(logit_tensor, dtype=tf.float32) label_tensor = tf.convert_to_tensor(label_tensor, dtype=tf.float32) out_tensor = SoftMaxCrossEntropy()(logit_tensor, label_tensor) out_tensor = out_tensor.eval() assert out_tensor.shape == (batch_size,)
def test_softmax_cross_entropy(self): """Test that SoftMaxCrossEntropy can be invoked.""" batch_size = 10 n_features = 5 logit_tensor = np.random.rand(batch_size, n_features) label_tensor = np.random.rand(batch_size, n_features) with self.session() as sess: logit_tensor = tf.convert_to_tensor(logit_tensor, dtype=tf.float32) label_tensor = tf.convert_to_tensor(label_tensor, dtype=tf.float32) out_tensor = SoftMaxCrossEntropy()(logit_tensor, label_tensor) out_tensor = out_tensor.eval() assert out_tensor.shape == (batch_size, )
def test_save_load(self): n_data_points = 20 n_features = 2 X = np.random.rand(n_data_points, n_features) y = [[0, 1] for x in range(n_data_points)] dataset = NumpyDataset(X, y) features = Feature(shape=(None, n_features)) dense = Dense(out_channels=2, in_layers=[features]) output = SoftMax(in_layers=[dense]) label = Label(shape=(None, 2)) smce = SoftMaxCrossEntropy(in_layers=[label, dense]) loss = ReduceMean(in_layers=[smce]) tg = dc.models.TensorGraph(learning_rate=0.01) tg.add_output(output) tg.set_loss(loss) submodel_loss = ReduceSum(in_layers=smce) submodel_opt = Adam(learning_rate=0.002) submodel = tg.create_submodel(layers=[dense], loss=submodel_loss, optimizer=submodel_opt) tg.fit(dataset, nb_epoch=1) prediction = np.squeeze(tg.predict_on_batch(X)) tg.save() dirpath = tempfile.mkdtemp() shutil.rmtree(dirpath) shutil.move(tg.model_dir, dirpath) tg1 = TensorGraph.load_from_dir(dirpath) prediction2 = np.squeeze(tg1.predict_on_batch(X)) assert_true(np.all(np.isclose(prediction, prediction2, atol=0.01)))
def test_set_optimizer(self): n_data_points = 20 n_features = 2 X = np.random.rand(n_data_points, n_features) y = [[0, 1] for x in range(n_data_points)] dataset = NumpyDataset(X, y) features = Feature(shape=(None, n_features)) dense = Dense(out_channels=2, in_layers=[features]) output = SoftMax(in_layers=[dense]) label = Label(shape=(None, 2)) smce = SoftMaxCrossEntropy(in_layers=[label, dense]) loss = ReduceMean(in_layers=[smce]) tg = dc.models.TensorGraph(learning_rate=0.01, use_queue=False) tg.add_output(output) tg.set_loss(loss) global_step = tg.get_global_step() learning_rate = ExponentialDecay(initial_rate=0.1, decay_rate=0.96, decay_steps=100000) tg.set_optimizer(GradientDescent(learning_rate=learning_rate)) tg.fit(dataset, nb_epoch=1000) prediction = np.squeeze(tg.predict_on_batch(X)) tg.save() tg1 = TensorGraph.load_from_dir(tg.model_dir) prediction2 = np.squeeze(tg1.predict_on_batch(X)) assert_true(np.all(np.isclose(prediction, prediction2, atol=0.01)))
def test_compute_model_performance_singletask_classifier(self): n_data_points = 20 n_features = 10 X = np.ones(shape=(int(n_data_points / 2), n_features)) * -1 X1 = np.ones(shape=(int(n_data_points / 2), n_features)) X = np.concatenate((X, X1)) class_1 = np.array([[0.0, 1.0] for x in range(int(n_data_points / 2))]) class_0 = np.array([[1.0, 0.0] for x in range(int(n_data_points / 2))]) y = np.concatenate((class_0, class_1)) dataset = NumpyDataset(X, y) features = Feature(shape=(None, n_features)) label = Label(shape=(None, 2)) dense = Dense(out_channels=2, in_layers=[features]) output = SoftMax(in_layers=[dense]) smce = SoftMaxCrossEntropy(in_layers=[label, dense]) total_loss = ReduceMean(in_layers=smce) tg = dc.models.TensorGraph(learning_rate=0.1) tg.add_output(output) tg.set_loss(total_loss) tg.fit(dataset, nb_epoch=1000) metric = dc.metrics.Metric(dc.metrics.roc_auc_score, np.mean, mode="classification") scores = tg.evaluate_generator(tg.default_generator(dataset), [metric], labels=[label], per_task_metrics=True) scores = list(scores[1].values()) assert_true(np.isclose(scores, [1.0], atol=0.05))
def test_tensorboard(self): n_data_points = 20 n_features = 2 X = np.random.rand(n_data_points, n_features) y = [[0, 1] for x in range(n_data_points)] dataset = NumpyDataset(X, y) features = Feature(shape=(None, n_features)) dense = Dense(out_channels=2, in_layers=[features]) output = SoftMax(in_layers=[dense]) label = Label(shape=(None, 2)) smce = SoftMaxCrossEntropy(in_layers=[label, dense]) loss = ReduceMean(in_layers=[smce]) tg = dc.models.TensorGraph(tensorboard=True, tensorboard_log_frequency=1, learning_rate=0.01, model_dir='/tmp/tensorgraph') tg.add_output(output) tg.set_loss(loss) tg.fit(dataset, nb_epoch=1000) files_in_dir = os.listdir(tg.model_dir) event_file = list( filter(lambda x: x.startswith("events"), files_in_dir)) assert_true(len(event_file) > 0) event_file = os.path.join(tg.model_dir, event_file[0]) file_size = os.stat(event_file).st_size assert_true(file_size > 0)
def _create_graph(self, feature_shape, label_shape): """This is called to create the full TensorGraph from the added layers.""" if self.built: return # The graph has already been created. # Add in features features = Feature(shape=feature_shape) # Add in labels labels = Label(shape=label_shape) # Add in all layers prev_layer = features if len(self._layer_list) == 0: raise ValueError("No layers have been added to model.") for ind, layer in enumerate(self._layer_list): if len(layer.in_layers) > 1: raise ValueError("Cannot specify more than one " "in_layer for Sequential.") layer.in_layers += [prev_layer] prev_layer = layer # The last layer is the output of the model self.outputs.append(prev_layer) if self._loss_function == "binary_crossentropy": smce = SoftMaxCrossEntropy(in_layers=[labels, prev_layer]) self.set_loss(ReduceMean(in_layers=[smce])) elif self._loss_function == "mse": mse = ReduceSquareDifference(in_layers=[prev_layer, labels]) self.set_loss(mse) else: # TODO(rbharath): Add in support for additional # losses. raise ValueError("Unsupported loss.") self.build()
def build_graph(self): # Layer 1 gc1_input = [self.atom_features, self.indexing, self.membership] + self.deg_adj_list gc1 = GraphConv(64, activation_fn=tf.nn.relu, in_layers=gc1_input) bn1 = BatchNorm(in_layers=[gc1]) gp1_input = [bn1, self.indexing, self.membership] + self.deg_adj_list gp1 = GraphPool(in_layers=gp1_input) # Layer 2 gc2_input = [gp1, self.indexing, self.membership] + self.deg_adj_list gc2 = GraphConv(64, activation_fn=tf.nn.relu, in_layers=gc2_input) bn2 = BatchNorm(in_layers=[gc2]) gp2_input = [bn2, self.indexing, self.membership] + self.deg_adj_list gp2 = GraphPool(in_layers=gp2_input) # Dense layer 1 d1 = Dense(out_channels=128, activation_fn=tf.nn.relu, in_layers=[gp2]) bn3 = BatchNorm(in_layers=[d1]) # Graph gather layer gg1_input = [bn3, self.indexing, self.membership] + self.deg_adj_list gg1 = GraphGather(batch_size=self.batch_size, activation=tf.nn.tanh, in_layers=gg1_input) # Output dense layer d2 = Dense(out_channels=2, activation_fn=None, in_layers=[gg1]) softmax = SoftMax(in_layers=[d2]) self.tg.add_output(softmax) # Set loss function self.label = Label(shape=(None, 2)) cost = SoftMaxCrossEntropy(in_layers=[self.label, d2]) self.weight = Weights(shape=(None, 1)) loss = WeightedError(in_layers=[cost, self.weight]) self.tg.set_loss(loss)
def build_graph(self): # Build placeholders self.atom_features = Feature(shape=(None, self.n_atom_feat)) self.pair_features = Feature(shape=(None, self.n_pair_feat)) self.atom_split = Feature(shape=(None, ), dtype=tf.int32) self.atom_to_pair = Feature(shape=(None, 2), dtype=tf.int32) message_passing = MessagePassing(self.T, message_fn='enn', update_fn='gru', n_hidden=self.n_hidden, in_layers=[ self.atom_features, self.pair_features, self.atom_to_pair ]) atom_embeddings = Dense(self.n_hidden, in_layers=[message_passing]) mol_embeddings = SetGather( self.M, self.batch_size, n_hidden=self.n_hidden, in_layers=[atom_embeddings, self.atom_split]) dense1 = Dense(out_channels=2 * self.n_hidden, activation_fn=tf.nn.relu, in_layers=[mol_embeddings]) costs = [] self.labels_fd = [] for task in range(self.n_tasks): if self.mode == "classification": classification = Dense(out_channels=2, activation_fn=None, in_layers=[dense1]) softmax = SoftMax(in_layers=[classification]) self.add_output(softmax) label = Label(shape=(None, 2)) self.labels_fd.append(label) cost = SoftMaxCrossEntropy(in_layers=[label, classification]) costs.append(cost) if self.mode == "regression": regression = Dense(out_channels=1, activation_fn=None, in_layers=[dense1]) self.add_output(regression) label = Label(shape=(None, 1)) self.labels_fd.append(label) cost = L2Loss(in_layers=[label, regression]) costs.append(cost) if self.mode == "classification": all_cost = Concat(in_layers=costs, axis=1) elif self.mode == "regression": all_cost = Stack(in_layers=costs, axis=1) self.weights = Weights(shape=(None, self.n_tasks)) loss = WeightedError(in_layers=[all_cost, self.weights]) self.set_loss(loss)
def test_SoftmaxCrossEntropy_pickle(): tg = TensorGraph() feature = Feature(shape=(tg.batch_size, 1)) layer = SoftMaxCrossEntropy(in_layers=[feature, feature]) tg.add_output(layer) tg.set_loss(layer) tg.build() tg.save()
def build_graph(self): """Building graph structures: Features => DAGLayer => DAGGather => Classification or Regression """ self.atom_features = Feature(shape=(None, self.n_atom_feat)) self.parents = Feature(shape=(None, self.max_atoms, self.max_atoms), dtype=tf.int32) self.calculation_orders = Feature(shape=(None, self.max_atoms), dtype=tf.int32) self.calculation_masks = Feature(shape=(None, self.max_atoms), dtype=tf.bool) self.membership = Feature(shape=(None, ), dtype=tf.int32) self.n_atoms = Feature(shape=(), dtype=tf.int32) dag_layer1 = DAGLayer(n_graph_feat=self.n_graph_feat, n_atom_feat=self.n_atom_feat, max_atoms=self.max_atoms, batch_size=self.batch_size, in_layers=[ self.atom_features, self.parents, self.calculation_orders, self.calculation_masks, self.n_atoms ]) dag_gather = DAGGather(n_graph_feat=self.n_graph_feat, n_outputs=self.n_outputs, max_atoms=self.max_atoms, in_layers=[dag_layer1, self.membership]) costs = [] self.labels_fd = [] for task in range(self.n_tasks): if self.mode == "classification": classification = Dense(out_channels=2, activation_fn=None, in_layers=[dag_gather]) softmax = SoftMax(in_layers=[classification]) self.add_output(softmax) label = Label(shape=(None, 2)) self.labels_fd.append(label) cost = SoftMaxCrossEntropy(in_layers=[label, classification]) costs.append(cost) if self.mode == "regression": regression = Dense(out_channels=1, activation_fn=None, in_layers=[dag_gather]) self.add_output(regression) label = Label(shape=(None, 1)) self.labels_fd.append(label) cost = L2Loss(in_layers=[label, regression]) costs.append(cost) if self.mode == "classification": all_cost = Concat(in_layers=costs, axis=1) elif self.mode == "regression": all_cost = Stack(in_layers=costs, axis=1) self.weights = Weights(shape=(None, self.n_tasks)) loss = WeightedError(in_layers=[all_cost, self.weights]) self.set_loss(loss)
def build_graph(self): self.smiles_seqs = Feature(shape=(None, self.seq_length), dtype=tf.int32) # Character embedding self.Embedding = DTNNEmbedding( n_embedding=self.n_embedding, periodic_table_length=len(self.char_dict.keys()) + 1, in_layers=[self.smiles_seqs]) self.pooled_outputs = [] self.conv_layers = [] for filter_size, num_filter in zip(self.kernel_sizes, self.num_filters): # Multiple convolutional layers with different filter widths self.conv_layers.append( Conv1D( kernel_size=filter_size, filters=num_filter, padding='valid', in_layers=[self.Embedding])) # Max-over-time pooling self.pooled_outputs.append( ReduceMax(axis=1, in_layers=[self.conv_layers[-1]])) # Concat features from all filters(one feature per filter) concat_outputs = Concat(axis=1, in_layers=self.pooled_outputs) dropout = Dropout(dropout_prob=self.dropout, in_layers=[concat_outputs]) dense = Dense( out_channels=200, activation_fn=tf.nn.relu, in_layers=[dropout]) # Highway layer from https://arxiv.org/pdf/1505.00387.pdf self.gather = Highway(in_layers=[dense]) costs = [] self.labels_fd = [] for task in range(self.n_tasks): if self.mode == "classification": classification = Dense( out_channels=2, activation_fn=None, in_layers=[self.gather]) softmax = SoftMax(in_layers=[classification]) self.add_output(softmax) label = Label(shape=(None, 2)) self.labels_fd.append(label) cost = SoftMaxCrossEntropy(in_layers=[label, classification]) costs.append(cost) if self.mode == "regression": regression = Dense( out_channels=1, activation_fn=None, in_layers=[self.gather]) self.add_output(regression) label = Label(shape=(None, 1)) self.labels_fd.append(label) cost = L2Loss(in_layers=[label, regression]) costs.append(cost) if self.mode == "classification": all_cost = Stack(in_layers=costs, axis=1) elif self.mode == "regression": all_cost = Stack(in_layers=costs, axis=1) self.weights = Weights(shape=(None, self.n_tasks)) loss = WeightedError(in_layers=[all_cost, self.weights]) self.set_loss(loss)
def build_graph(self): self.vertex_features = Feature(shape=(None, self.max_atoms, 75)) self.adj_matrix = Feature(shape=(None, self.max_atoms, 1, self.max_atoms)) self.mask = Feature(shape=(None, self.max_atoms, 1)) gcnn1 = BatchNorm( GraphCNN( num_filters=64, in_layers=[self.vertex_features, self.adj_matrix, self.mask])) gcnn1 = Dropout(self.dropout, in_layers=gcnn1) gcnn2 = BatchNorm( GraphCNN(num_filters=64, in_layers=[gcnn1, self.adj_matrix, self.mask])) gcnn2 = Dropout(self.dropout, in_layers=gcnn2) gc_pool, adj_matrix = GraphCNNPool( num_vertices=32, in_layers=[gcnn2, self.adj_matrix, self.mask]) gc_pool = BatchNorm(gc_pool) gc_pool = Dropout(self.dropout, in_layers=gc_pool) gcnn3 = BatchNorm(GraphCNN(num_filters=32, in_layers=[gc_pool, adj_matrix])) gcnn3 = Dropout(self.dropout, in_layers=gcnn3) gc_pool2, adj_matrix2 = GraphCNNPool( num_vertices=8, in_layers=[gcnn3, adj_matrix]) gc_pool2 = BatchNorm(gc_pool2) gc_pool2 = Dropout(self.dropout, in_layers=gc_pool2) flattened = Flatten(in_layers=gc_pool2) readout = Dense( out_channels=256, activation_fn=tf.nn.relu, in_layers=flattened) costs = [] self.my_labels = [] for task in range(self.n_tasks): if self.mode == 'classification': classification = Dense( out_channels=2, activation_fn=None, in_layers=[readout]) softmax = SoftMax(in_layers=[classification]) self.add_output(softmax) label = Label(shape=(None, 2)) self.my_labels.append(label) cost = SoftMaxCrossEntropy(in_layers=[label, classification]) costs.append(cost) if self.mode == 'regression': regression = Dense( out_channels=1, activation_fn=None, in_layers=[readout]) self.add_output(regression) label = Label(shape=(None, 1)) self.my_labels.append(label) cost = L2Loss(in_layers=[label, regression]) costs.append(cost) if self.mode == "classification": entropy = Stack(in_layers=costs, axis=-1) elif self.mode == "regression": entropy = Stack(in_layers=costs, axis=1) self.my_task_weights = Weights(shape=(None, self.n_tasks)) loss = WeightedError(in_layers=[entropy, self.my_task_weights]) self.set_loss(loss)
def __init__(self, img_rows=224, img_cols=224, weights="imagenet", classes=1000, **kwargs): super(ResNet50, self).__init__(use_queue=False, **kwargs) self.img_cols = img_cols self.img_rows = img_rows self.weights = weights self.classes = classes input = Feature(shape=(None, self.img_rows, self.img_cols, 3)) labels = Label(shape=(None, self.classes)) conv1 = Conv2D(num_outputs=64, kernel_size=7, stride=2, activation='linear', padding='same', in_layers=[input]) bn1 = BatchNorm(in_layers=[conv1]) ac1 = ReLU(bn1) pool1 = MaxPool2D(ksize=[1, 3, 3, 1], in_layers=[bn1]) cb1 = self.conv_block(pool1, 3, [64, 64, 256], 1) id1 = self.identity_block(cb1, 3, [64, 64, 256]) id1 = self.identity_block(id1, 3, [64, 64, 256]) cb2 = self.conv_block(id1, 3, [128, 128, 512]) id2 = self.identity_block(cb2, 3, [128, 128, 512]) id2 = self.identity_block(id2, 3, [128, 128, 512]) id2 = self.identity_block(id2, 3, [128, 128, 512]) cb3 = self.conv_block(id2, 3, [256, 256, 1024]) id3 = self.identity_block(cb3, 3, [256, 256, 1024]) id3 = self.identity_block(id3, 3, [256, 256, 1024]) id3 = self.identity_block(id3, 3, [256, 256, 1024]) id3 = self.identity_block(cb3, 3, [256, 256, 1024]) id3 = self.identity_block(id3, 3, [256, 256, 1024]) cb4 = self.conv_block(id3, 3, [512, 512, 2048]) id4 = self.identity_block(cb4, 3, [512, 512, 2048]) id4 = self.identity_block(id4, 3, [512, 512, 2048]) pool2 = AvgPool2D(ksize=[1, 7, 7, 1], in_layers=[id4]) flatten = Flatten(in_layers=[pool2]) dense = Dense(classes, in_layers=[flatten]) loss = SoftMaxCrossEntropy(in_layers=[labels, dense]) loss = ReduceMean(in_layers=[loss]) self.set_loss(loss) self.add_output(dense)
def test_compute_model_performance_multitask_classifier(self): n_data_points = 20 n_features = 2 X = np.ones(shape=(n_data_points // 2, n_features)) * -1 X1 = np.ones(shape=(n_data_points // 2, n_features)) X = np.concatenate((X, X1)) class_1 = np.array([[0.0, 1.0] for x in range(int(n_data_points / 2))]) class_0 = np.array([[1.0, 0.0] for x in range(int(n_data_points / 2))]) y1 = np.concatenate((class_0, class_1)) y2 = np.concatenate((class_1, class_0)) X = NumpyDataset(X) ys = [NumpyDataset(y1), NumpyDataset(y2)] databag = Databag() features = Feature(shape=(None, n_features)) databag.add_dataset(features, X) outputs = [] entropies = [] labels = [] for i in range(2): label = Label(shape=(None, 2)) labels.append(label) dense = Dense(out_channels=2, in_layers=[features]) output = SoftMax(in_layers=[dense]) smce = SoftMaxCrossEntropy(in_layers=[label, dense]) entropies.append(smce) outputs.append(output) databag.add_dataset(label, ys[i]) total_loss = ReduceMean(in_layers=entropies) tg = dc.models.TensorGraph(learning_rate=0.1) for output in outputs: tg.add_output(output) tg.set_loss(total_loss) tg.fit_generator( databag.iterbatches(epochs=1000, batch_size=tg.batch_size, pad_batches=True)) metric = dc.metrics.Metric(dc.metrics.roc_auc_score, np.mean, mode="classification") scores = tg.evaluate_generator(databag.iterbatches(), [metric], labels=labels, per_task_metrics=True) scores = list(scores[1].values()) # Loosening atol to see if tests stop failing sporadically assert_true(np.all(np.isclose(scores, [1.0, 1.0], atol=0.20)))
def _build_graph(self): self.smiles_seqs = Feature(shape=(None, self.seq_length), dtype=tf.int32) # Character embedding Embedding = DTNNEmbedding( n_embedding=self.n_embedding, periodic_table_length=len(self.char_dict.keys()) + 1, in_layers=[self.smiles_seqs]) pooled_outputs = [] conv_layers = [] for filter_size, num_filter in zip(self.kernel_sizes, self.num_filters): # Multiple convolutional layers with different filter widths conv_layers.append( Conv1D(kernel_size=filter_size, filters=num_filter, padding='valid', in_layers=[Embedding])) # Max-over-time pooling pooled_outputs.append( ReduceMax(axis=1, in_layers=[conv_layers[-1]])) # Concat features from all filters(one feature per filter) concat_outputs = Concat(axis=1, in_layers=pooled_outputs) dropout = Dropout(dropout_prob=self.dropout, in_layers=[concat_outputs]) dense = Dense(out_channels=200, activation_fn=tf.nn.relu, in_layers=[dropout]) # Highway layer from https://arxiv.org/pdf/1505.00387.pdf gather = Highway(in_layers=[dense]) if self.mode == "classification": logits = Dense(out_channels=self.n_tasks * 2, activation_fn=None, in_layers=[gather]) logits = Reshape(shape=(-1, self.n_tasks, 2), in_layers=[logits]) output = SoftMax(in_layers=[logits]) self.add_output(output) labels = Label(shape=(None, self.n_tasks, 2)) loss = SoftMaxCrossEntropy(in_layers=[labels, logits]) else: vals = Dense(out_channels=self.n_tasks * 1, activation_fn=None, in_layers=[gather]) vals = Reshape(shape=(-1, self.n_tasks, 1), in_layers=[vals]) self.add_output(vals) labels = Label(shape=(None, self.n_tasks, 1)) loss = ReduceSum(L2Loss(in_layers=[labels, vals])) weights = Weights(shape=(None, self.n_tasks)) weighted_loss = WeightedError(in_layers=[loss, weights]) self.set_loss(weighted_loss)
def build_graph(self): d1 = Dense(out_channels=256, activation_fn=tf.nn.relu, in_layers=[self.feature]) d2 = Dense(out_channels=64, activation_fn=tf.nn.relu, in_layers=[d1]) d3 = Dense(out_channels=16, activation=None, in_layers=[d2]) d4 = Dense(out_channels=2, activation=None, in_layers=[d3]) softmax = SoftMax(in_layers=[d4]) self.tg.add_output(softmax) self.label = Label(shape=(None, 2)) cost = SoftMaxCrossEntropy(in_layers=[self.label, d4]) loss = ReduceMean(in_layers=[cost]) self.tg.set_loss(loss)
def fit(self, dataset, loss, **kwargs): """Fits on the specified dataset. If called for the first time, constructs the TensorFlow graph for this model. Fits this graph on the specified dataset according to the specified loss. Parameters ---------- dataset: dc.data.Dataset Dataset with data loss: string Only "binary_crossentropy" or "mse" for now. """ X_shape, y_shape, _, _ = dataset.get_shape() # Calling fit() for first time if not self.built: feature_shape = X_shape[1:] label_shape = y_shape[1:] # Add in features features = Feature(shape=(None, ) + feature_shape) # Add in labels labels = Label(shape=(None, ) + label_shape) # Add in all layers prev_layer = features if len(self._layer_list) == 0: raise ValueError("No layers have been added to model.") for ind, layer in enumerate(self._layer_list): if len(layer.in_layers) > 1: raise ValueError("Cannot specify more than one " "in_layer for Sequential.") layer.in_layers += [prev_layer] prev_layer = layer # The last layer is the output of the model self.outputs.append(prev_layer) if loss == "binary_crossentropy": smce = SoftMaxCrossEntropy(in_layers=[labels, prev_layer]) self.set_loss(ReduceMean(in_layers=[smce])) elif loss == "mse": mse = ReduceSquareDifference(in_layers=[prev_layer, labels]) self.set_loss(mse) else: # TODO(rbharath): Add in support for additional # losses. raise ValueError("Unsupported loss.") super(Sequential, self).fit(dataset, **kwargs)
def test_mnist(self): from tensorflow.examples.tutorials.mnist import input_data mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) train = dc.data.NumpyDataset(mnist.train.images, mnist.train.labels) valid = dc.data.NumpyDataset(mnist.validation.images, mnist.validation.labels) # Images are square 28x28 (batch, height, width, channel) feature = Feature(shape=(None, 784), name="Feature") make_image = Reshape(shape=(-1, 28, 28, 1), in_layers=[feature]) conv2d_1 = Conv2D(num_outputs=32, normalizer_fn=tf.contrib.layers.batch_norm, in_layers=[make_image]) maxpool_1 = MaxPool(in_layers=[conv2d_1]) conv2d_2 = Conv2D(num_outputs=64, normalizer_fn=tf.contrib.layers.batch_norm, in_layers=[maxpool_1]) maxpool_2 = MaxPool(in_layers=[conv2d_2]) flatten = Flatten(in_layers=[maxpool_2]) dense1 = Dense(out_channels=1024, activation_fn=tf.nn.relu, in_layers=[flatten]) dense2 = Dense(out_channels=10, in_layers=[dense1]) label = Label(shape=(None, 10), name="Label") smce = SoftMaxCrossEntropy(in_layers=[label, dense2]) loss = ReduceMean(in_layers=[smce]) output = SoftMax(in_layers=[dense2]) tg = dc.models.TensorGraph(model_dir='/tmp/mnist', batch_size=1000, use_queue=True) tg.add_output(output) tg.set_loss(loss) tg.fit(train, nb_epoch=2) prediction = np.squeeze(tg.predict_proba_on_batch(valid.X)) fpr = dict() tpr = dict() roc_auc = dict() for i in range(10): fpr[i], tpr[i], thresh = roc_curve(valid.y[:, i], prediction[:, i]) roc_auc[i] = auc(fpr[i], tpr[i]) assert_true(roc_auc[i] > 0.99)
def test_single_task_classifier(self): n_data_points = 20 n_features = 2 X = np.random.rand(n_data_points, n_features) y = [[0, 1] for x in range(n_data_points)] dataset = NumpyDataset(X, y) features = Feature(shape=(None, n_features)) dense = Dense(out_channels=2, in_layers=[features]) output = SoftMax(in_layers=[dense]) label = Label(shape=(None, 2)) smce = SoftMaxCrossEntropy(in_layers=[label, dense]) loss = ReduceMean(in_layers=[smce]) tg = dc.models.TensorGraph(learning_rate=0.01) tg.add_output(output) tg.set_loss(loss) tg.fit(dataset, nb_epoch=1000) prediction = np.squeeze(tg.predict_on_batch(X)) assert_true(np.all(np.isclose(prediction, y, atol=0.4)))
def test_multi_task_classifier(self): n_data_points = 20 n_features = 2 X = np.random.rand(n_data_points, n_features) y1 = np.array([[0, 1] for x in range(n_data_points)]) y2 = np.array([[1, 0] for x in range(n_data_points)]) X = NumpyDataset(X) ys = [NumpyDataset(y1), NumpyDataset(y2)] databag = Databag() features = Feature(shape=(None, n_features)) databag.add_dataset(features, X) outputs = [] entropies = [] for i in range(2): label = Label(shape=(None, 2)) dense = Dense(out_channels=2, in_layers=[features]) output = SoftMax(in_layers=[dense]) smce = SoftMaxCrossEntropy(in_layers=[label, dense]) entropies.append(smce) outputs.append(output) databag.add_dataset(label, ys[i]) total_loss = ReduceMean(in_layers=entropies) tg = dc.models.TensorGraph(learning_rate=0.01) for output in outputs: tg.add_output(output) tg.set_loss(total_loss) tg.fit_generator( databag.iterbatches(epochs=1000, batch_size=tg.batch_size, pad_batches=True)) predictions = tg.predict_on_generator(databag.iterbatches()) for i in range(2): y_real = ys[i].X y_pred = predictions[i] assert_true(np.all(np.isclose(y_pred, y_real, atol=0.6)))
def test_shared_layer(self): n_data_points = 20 n_features = 2 X = np.random.rand(n_data_points, n_features) y1 = np.array([[0, 1] for x in range(n_data_points)]) X = NumpyDataset(X) ys = [NumpyDataset(y1)] databag = Databag() features = Feature(shape=(None, n_features)) databag.add_dataset(features, X) outputs = [] label = Label(shape=(None, 2)) dense1 = Dense(out_channels=2, in_layers=[features]) dense2 = dense1.shared(in_layers=[features]) output1 = SoftMax(in_layers=[dense1]) output2 = SoftMax(in_layers=[dense2]) smce = SoftMaxCrossEntropy(in_layers=[label, dense1]) outputs.append(output1) outputs.append(output2) databag.add_dataset(label, ys[0]) total_loss = ReduceMean(in_layers=[smce]) tg = dc.models.TensorGraph(learning_rate=0.01) for output in outputs: tg.add_output(output) tg.set_loss(total_loss) tg.fit_generator( databag.iterbatches(epochs=1, batch_size=tg.batch_size, pad_batches=True)) prediction = tg.predict_on_generator(databag.iterbatches()) assert_true(np.all(np.isclose(prediction[0], prediction[1], atol=0.01)))
def test_compute_model_performance_multitask_classifier(self): n_data_points = 20 n_features = 1 n_tasks = 2 n_classes = 2 X = np.ones(shape=(n_data_points // 2, n_features)) * -1 X1 = np.ones(shape=(n_data_points // 2, n_features)) X = np.concatenate((X, X1)) class_1 = np.array([[0.0, 1.0] for x in range(int(n_data_points / 2))]) class_0 = np.array([[1.0, 0.0] for x in range(int(n_data_points / 2))]) y1 = np.concatenate((class_0, class_1)) y2 = np.concatenate((class_1, class_0)) y = np.stack([y1, y2], axis=1) dataset = NumpyDataset(X, y) features = Feature(shape=(None, n_features)) label = Label(shape=(None, n_tasks, n_classes)) dense = Dense(out_channels=n_tasks * n_classes, in_layers=[features]) logits = Reshape(shape=(None, n_tasks, n_classes), in_layers=dense) output = SoftMax(in_layers=[logits]) smce = SoftMaxCrossEntropy(in_layers=[label, logits]) total_loss = ReduceMean(in_layers=smce) tg = dc.models.TensorGraph(learning_rate=0.01, batch_size=n_data_points) tg.add_output(output) tg.set_loss(total_loss) tg.fit(dataset, nb_epoch=1000) metric = dc.metrics.Metric(dc.metrics.roc_auc_score, np.mean, mode="classification") scores = tg.evaluate_generator(tg.default_generator(dataset), [metric], labels=[label], per_task_metrics=True) scores = list(scores[1].values()) # Loosening atol to see if tests stop failing sporadically assert_true(np.all(np.isclose(scores, [1.0, 1.0], atol=0.50)))
def build_graph(self): """Building graph structures: Features => WeaveLayer => WeaveLayer => Dense => WeaveGather => Classification or Regression """ self.atom_features = Feature(shape=(None, self.n_atom_feat)) self.pair_features = Feature(shape=(None, self.n_pair_feat)) combined = Combine_AP(in_layers=[self.atom_features, self.pair_features]) self.pair_split = Feature(shape=(None,), dtype=tf.int32) self.atom_split = Feature(shape=(None,), dtype=tf.int32) self.atom_to_pair = Feature(shape=(None, 2), dtype=tf.int32) weave_layer1 = WeaveLayer( n_atom_input_feat=self.n_atom_feat, n_pair_input_feat=self.n_pair_feat, n_atom_output_feat=self.n_hidden, n_pair_output_feat=self.n_hidden, in_layers=[combined, self.pair_split, self.atom_to_pair]) weave_layer2 = WeaveLayer( n_atom_input_feat=self.n_hidden, n_pair_input_feat=self.n_hidden, n_atom_output_feat=self.n_hidden, n_pair_output_feat=self.n_hidden, update_pair=False, in_layers=[weave_layer1, self.pair_split, self.atom_to_pair]) separated = Separate_AP(in_layers=[weave_layer2]) dense1 = Dense( out_channels=self.n_graph_feat, activation_fn=tf.nn.tanh, in_layers=[separated]) batch_norm1 = BatchNormalization(epsilon=1e-5, mode=1, in_layers=[dense1]) weave_gather = WeaveGather( self.batch_size, n_input=self.n_graph_feat, gaussian_expand=True, in_layers=[batch_norm1, self.atom_split]) costs = [] self.labels_fd = [] for task in range(self.n_tasks): if self.mode == "classification": classification = Dense( out_channels=2, activation_fn=None, in_layers=[weave_gather]) softmax = SoftMax(in_layers=[classification]) self.add_output(softmax) label = Label(shape=(None, 2)) self.labels_fd.append(label) cost = SoftMaxCrossEntropy(in_layers=[label, classification]) costs.append(cost) if self.mode == "regression": regression = Dense( out_channels=1, activation_fn=None, in_layers=[weave_gather]) self.add_output(regression) label = Label(shape=(None, 1)) self.labels_fd.append(label) cost = L2Loss(in_layers=[label, regression]) costs.append(cost) if self.mode == "classification": all_cost = Concat(in_layers=costs, axis=1) elif self.mode == "regression": all_cost = Stack(in_layers=costs, axis=1) self.weights = Weights(shape=(None, self.n_tasks)) loss = WeightedError(in_layers=[all_cost, self.weights]) self.set_loss(loss)
def build_graph(self): """ Building graph structures: """ self.atom_features = Feature(shape=(None, 75)) self.degree_slice = Feature(shape=(None, 2), dtype=tf.int32) self.membership = Feature(shape=(None,), dtype=tf.int32) self.deg_adjs = [] for i in range(0, 10 + 1): deg_adj = Feature(shape=(None, i + 1), dtype=tf.int32) self.deg_adjs.append(deg_adj) gc1 = GraphConv( 64, activation_fn=tf.nn.relu, in_layers=[self.atom_features, self.degree_slice, self.membership] + self.deg_adjs) batch_norm1 = BatchNorm(in_layers=[gc1]) gp1 = GraphPool(in_layers=[batch_norm1, self.degree_slice, self.membership] + self.deg_adjs) gc2 = GraphConv( 64, activation_fn=tf.nn.relu, in_layers=[gp1, self.degree_slice, self.membership] + self.deg_adjs) batch_norm2 = BatchNorm(in_layers=[gc2]) gp2 = GraphPool(in_layers=[batch_norm2, self.degree_slice, self.membership] + self.deg_adjs) dense = Dense(out_channels=128, activation_fn=tf.nn.relu, in_layers=[gp2]) batch_norm3 = BatchNorm(in_layers=[dense]) readout = GraphGather( batch_size=self.batch_size, activation_fn=tf.nn.tanh, in_layers=[batch_norm3, self.degree_slice, self.membership] + self.deg_adjs) if self.error_bars == True: readout = Dropout(in_layers=[readout], dropout_prob=0.2) costs = [] self.my_labels = [] for task in range(self.n_tasks): if self.mode == 'classification': classification = Dense( out_channels=2, activation_fn=None, in_layers=[readout]) softmax = SoftMax(in_layers=[classification]) self.add_output(softmax) label = Label(shape=(None, 2)) self.my_labels.append(label) cost = SoftMaxCrossEntropy(in_layers=[label, classification]) costs.append(cost) if self.mode == 'regression': regression = Dense( out_channels=1, activation_fn=None, in_layers=[readout]) self.add_output(regression) label = Label(shape=(None, 1)) self.my_labels.append(label) cost = L2Loss(in_layers=[label, regression]) costs.append(cost) if self.mode == "classification": entropy = Concat(in_layers=costs, axis=-1) elif self.mode == "regression": entropy = Stack(in_layers=costs, axis=1) self.my_task_weights = Weights(shape=(None, self.n_tasks)) loss = WeightedError(in_layers=[entropy, self.my_task_weights]) self.set_loss(loss)
def graph_conv_net(batch_size, prior, num_task): """ Build a tensorgraph for multilabel classification task Return: features and labels layers """ tg = TensorGraph(use_queue=False) if prior == True: add_on = num_task else: add_on = 0 atom_features = Feature(shape=(None, 75 + 2 * add_on)) circular_features = Feature(shape=(batch_size, 256), dtype=tf.float32) degree_slice = Feature(shape=(None, 2), dtype=tf.int32) membership = Feature(shape=(None, ), dtype=tf.int32) deg_adjs = [] for i in range(0, 10 + 1): deg_adj = Feature(shape=(None, i + 1), dtype=tf.int32) deg_adjs.append(deg_adj) gc1 = GraphConv(64 + add_on, activation_fn=tf.nn.elu, in_layers=[atom_features, degree_slice, membership] + deg_adjs) batch_norm1 = BatchNorm(in_layers=[gc1]) gp1 = GraphPool(in_layers=[batch_norm1, degree_slice, membership] + deg_adjs) gc2 = GraphConv(64 + add_on, activation_fn=tf.nn.elu, in_layers=[gc1, degree_slice, membership] + deg_adjs) batch_norm2 = BatchNorm(in_layers=[gc2]) gp2 = GraphPool(in_layers=[batch_norm2, degree_slice, membership] + deg_adjs) add = Concat(in_layers=[gp1, gp2]) add = Dropout(0.5, in_layers=[add]) dense = Dense(out_channels=128, activation_fn=tf.nn.elu, in_layers=[add]) batch_norm3 = BatchNorm(in_layers=[dense]) readout = GraphGather(batch_size=batch_size, activation_fn=tf.nn.tanh, in_layers=[batch_norm3, degree_slice, membership] + deg_adjs) batch_norm4 = BatchNorm(in_layers=[readout]) dense1 = Dense(out_channels=128, activation_fn=tf.nn.elu, in_layers=[circular_features]) dense1 = BatchNorm(in_layers=[dense1]) dense1 = Dropout(0.5, in_layers=[dense1]) dense1 = Dense(out_channels=128, activation_fn=tf.nn.elu, in_layers=[circular_features]) dense1 = BatchNorm(in_layers=[dense1]) dense1 = Dropout(0.5, in_layers=[dense1]) merge_feat = Concat(in_layers=[dense1, batch_norm4]) merge = Dense(out_channels=256, activation_fn=tf.nn.elu, in_layers=[merge_feat]) costs = [] labels = [] for task in range(num_task): classification = Dense(out_channels=2, activation_fn=None, in_layers=[merge]) softmax = SoftMax(in_layers=[classification]) tg.add_output(softmax) label = Label(shape=(None, 2)) labels.append(label) cost = SoftMaxCrossEntropy(in_layers=[label, classification]) costs.append(cost) all_cost = Stack(in_layers=costs, axis=1) weights = Weights(shape=(None, num_task)) loss = WeightedError(in_layers=[all_cost, weights]) tg.set_loss(loss) #if prior == True: # return tg, atom_features,circular_features, degree_slice, membership, deg_adjs, labels, weights#, prior_layer return tg, atom_features, circular_features, degree_slice, membership, deg_adjs, labels, weights
def graph_conv_model(batch_size, tasks): model = TensorGraph(model_dir=model_dir, batch_size=batch_size, use_queue=False) atom_features = Feature(shape=(None, 75)) degree_slice = Feature(shape=(None, 2), dtype=tf.int32) membership = Feature(shape=(None, ), dtype=tf.int32) deg_adjs = [] for i in range(0, 10 + 1): deg_adj = Feature(shape=(None, i + 1), dtype=tf.int32) deg_adjs.append(deg_adj) gc1 = GraphConv(64, activation_fn=tf.nn.relu, in_layers=[atom_features, degree_slice, membership] + deg_adjs) batch_norm1 = BatchNorm(in_layers=[gc1]) gp1 = GraphPool(in_layers=[batch_norm1, degree_slice, membership] + deg_adjs) gc2 = GraphConv(64, activation_fn=tf.nn.relu, in_layers=[gp1, degree_slice, membership] + deg_adjs) batch_norm2 = BatchNorm(in_layers=[gc2]) gp2 = GraphPool(in_layers=[batch_norm2, degree_slice, membership] + deg_adjs) dense = Dense(out_channels=128, activation_fn=None, in_layers=[gp2]) batch_norm3 = BatchNorm(in_layers=[dense]) gg1 = GraphGather(batch_size=batch_size, activation_fn=tf.nn.tanh, in_layers=[batch_norm3, degree_slice, membership] + deg_adjs) costs = [] labels = [] for task in tasks: classification = Dense(out_channels=2, activation_fn=None, in_layers=[gg1]) softmax = SoftMax(in_layers=[classification]) model.add_output(softmax) label = Label(shape=(None, 2)) labels.append(label) cost = SoftMaxCrossEntropy(in_layers=[label, classification]) costs.append(cost) entropy = Concat(in_layers=costs) task_weights = Weights(shape=(None, len(tasks))) loss = WeightedError(in_layers=[entropy, task_weights]) model.set_loss(loss) def feed_dict_generator(dataset, batch_size, epochs=1): for epoch in range(epochs): for ind, (X_b, y_b, w_b, ids_b) in enumerate( dataset.iterbatches(batch_size, pad_batches=True)): d = {} for index, label in enumerate(labels): d[label] = to_one_hot(y_b[:, index]) d[task_weights] = w_b multiConvMol = ConvMol.agglomerate_mols(X_b) d[atom_features] = multiConvMol.get_atom_features() d[degree_slice] = multiConvMol.deg_slice d[membership] = multiConvMol.membership for i in range(1, len(multiConvMol.get_deg_adjacency_lists())): d[deg_adjs[i - 1]] = multiConvMol.get_deg_adjacency_lists()[i] yield d return model, feed_dict_generator, labels, task_weights
def __init__(self, n_tasks, n_features, layer_sizes=[1000], weight_init_stddevs=[0.02], bias_init_consts=[1.0], weight_decay_penalty=0.0, weight_decay_penalty_type="l2", dropouts=[0.5], n_classes=2, **kwargs): """Create a TensorGraphMultiTaskClassifier. In addition to the following arguments, this class also accepts all the keywork arguments from TensorGraph. Parameters ---------- n_tasks: int number of tasks n_features: int number of features layer_sizes: list the size of each dense layer in the network. The length of this list determines the number of layers. weight_init_stddevs: list the standard deviation of the distribution to use for weight initialization of each layer. The length of this list should equal len(layer_sizes). bias_init_consts: list the value to initialize the biases in each layer to. The length of this list should equal len(layer_sizes). weight_decay_penalty: float the magnitude of the weight decay penalty to use weight_decay_penalty_type: str the type of penalty to use for weight decay, either 'l1' or 'l2' dropouts: list the dropout probablity to use for each layer. The length of this list should equal len(layer_sizes). n_classes: int the number of classes """ super(TensorGraphMultiTaskClassifier, self).__init__(mode='classification', **kwargs) self.n_tasks = n_tasks self.n_features = n_features self.n_classes = n_classes # Add the input features. mol_features = Feature(shape=(None, n_features)) prev_layer = mol_features # Add the dense layers for size, weight_stddev, bias_const, dropout in zip( layer_sizes, weight_init_stddevs, bias_init_consts, dropouts): layer = Dense(in_layers=[prev_layer], out_channels=size, activation_fn=tf.nn.relu, weights_initializer=TFWrapper( tf.truncated_normal_initializer, stddev=weight_stddev), biases_initializer=TFWrapper(tf.constant_initializer, value=bias_const)) if dropout > 0.0: layer = Dropout(dropout, in_layers=[layer]) prev_layer = layer # Compute the loss function for each label. output = Reshape(shape=(-1, n_tasks, n_classes), in_layers=[ Dense(in_layers=[prev_layer], out_channels=n_tasks * n_classes) ]) self.add_output(output) labels = Label(shape=(None, n_tasks, n_classes)) weights = Weights(shape=(None, n_tasks)) loss = SoftMaxCrossEntropy(in_layers=[labels, output]) weighted_loss = WeightedError(in_layers=[loss, weights]) if weight_decay_penalty != 0.0: weighted_loss = WeightDecay(weight_decay_penalty, weight_decay_penalty_type, in_layers=[weighted_loss]) self.set_loss(weighted_loss)
def build_graph(self): """Building graph structures: Features => WeaveLayer => WeaveLayer => Dense => WeaveGather => Classification or Regression """ self.atom_features = Feature(shape=(None, self.n_atom_feat)) self.pair_features = Feature(shape=(None, self.n_pair_feat)) self.pair_split = Feature(shape=(None, ), dtype=tf.int32) self.atom_split = Feature(shape=(None, ), dtype=tf.int32) self.atom_to_pair = Feature(shape=(None, 2), dtype=tf.int32) weave_layer1A, weave_layer1P = WeaveLayerFactory( n_atom_input_feat=self.n_atom_feat, n_pair_input_feat=self.n_pair_feat, n_atom_output_feat=self.n_hidden, n_pair_output_feat=self.n_hidden, in_layers=[ self.atom_features, self.pair_features, self.pair_split, self.atom_to_pair ]) weave_layer2A, weave_layer2P = WeaveLayerFactory( n_atom_input_feat=self.n_hidden, n_pair_input_feat=self.n_hidden, n_atom_output_feat=self.n_hidden, n_pair_output_feat=self.n_hidden, update_pair=False, in_layers=[ weave_layer1A, weave_layer1P, self.pair_split, self.atom_to_pair ]) dense1 = Dense(out_channels=self.n_graph_feat, activation_fn=tf.nn.tanh, in_layers=weave_layer2A) batch_norm1 = BatchNorm(epsilon=1e-5, in_layers=[dense1]) weave_gather = WeaveGather(self.batch_size, n_input=self.n_graph_feat, gaussian_expand=True, in_layers=[batch_norm1, self.atom_split]) n_tasks = self.n_tasks weights = Weights(shape=(None, n_tasks)) if self.mode == 'classification': n_classes = self.n_classes labels = Label(shape=(None, n_tasks, n_classes)) logits = Reshape(shape=(None, n_tasks, n_classes), in_layers=[ Dense(in_layers=weave_gather, out_channels=n_tasks * n_classes) ]) output = SoftMax(logits) self.add_output(output) loss = SoftMaxCrossEntropy(in_layers=[labels, logits]) weighted_loss = WeightedError(in_layers=[loss, weights]) self.set_loss(weighted_loss) else: labels = Label(shape=(None, n_tasks)) output = Reshape(shape=(None, n_tasks), in_layers=[ Dense(in_layers=weave_gather, out_channels=n_tasks) ]) self.add_output(output) weighted_loss = ReduceSum( L2Loss(in_layers=[labels, output, weights])) self.set_loss(weighted_loss)
def sluice_model(batch_size, tasks): model = TensorGraph(model_dir=model_dir, batch_size=batch_size, use_queue=False, tensorboard=True) atom_features = Feature(shape=(None, 75)) degree_slice = Feature(shape=(None, 2), dtype=tf.int32) membership = Feature(shape=(None, ), dtype=tf.int32) sluice_loss = [] deg_adjs = [] for i in range(0, 10 + 1): deg_adj = Feature(shape=(None, i + 1), dtype=tf.int32) deg_adjs.append(deg_adj) gc1 = GraphConv(64, activation_fn=tf.nn.relu, in_layers=[atom_features, degree_slice, membership] + deg_adjs) as1 = AlphaShare(in_layers=[gc1, gc1]) sluice_loss.append(gc1) batch_norm1a = BatchNorm(in_layers=[as1[0]]) batch_norm1b = BatchNorm(in_layers=[as1[1]]) gp1a = GraphPool(in_layers=[batch_norm1a, degree_slice, membership] + deg_adjs) gp1b = GraphPool(in_layers=[batch_norm1b, degree_slice, membership] + deg_adjs) gc2a = GraphConv(64, activation_fn=tf.nn.relu, in_layers=[gp1a, degree_slice, membership] + deg_adjs) gc2b = GraphConv(64, activation_fn=tf.nn.relu, in_layers=[gp1b, degree_slice, membership] + deg_adjs) as2 = AlphaShare(in_layers=[gc2a, gc2b]) sluice_loss.append(gc2a) sluice_loss.append(gc2b) batch_norm2a = BatchNorm(in_layers=[as2[0]]) batch_norm2b = BatchNorm(in_layers=[as2[1]]) gp2a = GraphPool(in_layers=[batch_norm2a, degree_slice, membership] + deg_adjs) gp2b = GraphPool(in_layers=[batch_norm2b, degree_slice, membership] + deg_adjs) densea = Dense(out_channels=128, activation_fn=None, in_layers=[gp2a]) denseb = Dense(out_channels=128, activation_fn=None, in_layers=[gp2b]) batch_norm3a = BatchNorm(in_layers=[densea]) batch_norm3b = BatchNorm(in_layers=[denseb]) as3 = AlphaShare(in_layers=[batch_norm3a, batch_norm3b]) sluice_loss.append(batch_norm3a) sluice_loss.append(batch_norm3b) gg1a = GraphGather(batch_size=batch_size, activation_fn=tf.nn.tanh, in_layers=[as3[0], degree_slice, membership] + deg_adjs) gg1b = GraphGather(batch_size=batch_size, activation_fn=tf.nn.tanh, in_layers=[as3[1], degree_slice, membership] + deg_adjs) costs = [] labels = [] count = 0 for task in tasks: if count < len(tasks) / 2: classification = Dense(out_channels=2, activation_fn=None, in_layers=[gg1a]) print("first half:") print(task) else: classification = Dense(out_channels=2, activation_fn=None, in_layers=[gg1b]) print('second half') print(task) count += 1 softmax = SoftMax(in_layers=[classification]) model.add_output(softmax) label = Label(shape=(None, 2)) labels.append(label) cost = SoftMaxCrossEntropy(in_layers=[label, classification]) costs.append(cost) entropy = Concat(in_layers=costs) task_weights = Weights(shape=(None, len(tasks))) task_loss = WeightedError(in_layers=[entropy, task_weights]) s_cost = SluiceLoss(in_layers=sluice_loss) total_loss = Add(in_layers=[task_loss, s_cost]) model.set_loss(total_loss) def feed_dict_generator(dataset, batch_size, epochs=1): for epoch in range(epochs): for ind, (X_b, y_b, w_b, ids_b) in enumerate( dataset.iterbatches(batch_size, pad_batches=True)): d = {} for index, label in enumerate(labels): d[label] = to_one_hot(y_b[:, index]) d[task_weights] = w_b multiConvMol = ConvMol.agglomerate_mols(X_b) d[atom_features] = multiConvMol.get_atom_features() d[degree_slice] = multiConvMol.deg_slice d[membership] = multiConvMol.membership for i in range(1, len(multiConvMol.get_deg_adjacency_lists())): d[deg_adjs[i - 1]] = multiConvMol.get_deg_adjacency_lists()[i] yield d return model, feed_dict_generator, labels, task_weights
def build_graph(self): # Build placeholders self.atom_features = Feature(shape=(None, self.n_atom_feat)) self.pair_features = Feature(shape=(None, self.n_pair_feat)) self.atom_split = Feature(shape=(None, ), dtype=tf.int32) self.atom_to_pair = Feature(shape=(None, 2), dtype=tf.int32) message_passing = MessagePassing(self.T, message_fn='enn', update_fn='gru', n_hidden=self.n_hidden, in_layers=[ self.atom_features, self.pair_features, self.atom_to_pair ]) atom_embeddings = Dense(self.n_hidden, in_layers=[message_passing]) mol_embeddings = SetGather( self.M, self.batch_size, n_hidden=self.n_hidden, in_layers=[atom_embeddings, self.atom_split]) dense1 = Dense(out_channels=2 * self.n_hidden, activation_fn=tf.nn.relu, in_layers=[mol_embeddings]) n_tasks = self.n_tasks weights = Weights(shape=(None, n_tasks)) if self.mode == 'classification': n_classes = self.n_classes labels = Label(shape=(None, n_tasks, n_classes)) logits = Reshape(shape=(None, n_tasks, n_classes), in_layers=[ Dense(in_layers=dense1, out_channels=n_tasks * n_classes) ]) logits = TrimGraphOutput([logits, weights]) output = SoftMax(logits) self.add_output(output) loss = SoftMaxCrossEntropy(in_layers=[labels, logits]) weighted_loss = WeightedError(in_layers=[loss, weights]) self.set_loss(weighted_loss) else: labels = Label(shape=(None, n_tasks)) output = Reshape( shape=(None, n_tasks), in_layers=[Dense(in_layers=dense1, out_channels=n_tasks)]) output = TrimGraphOutput([output, weights]) self.add_output(output) if self.uncertainty: log_var = Reshape( shape=(None, n_tasks), in_layers=[Dense(in_layers=dense1, out_channels=n_tasks)]) log_var = TrimGraphOutput([log_var, weights]) var = Exp(log_var) self.add_variance(var) diff = labels - output weighted_loss = weights * (diff * diff / var + log_var) weighted_loss = ReduceSum(ReduceMean(weighted_loss, axis=[1])) else: weighted_loss = ReduceSum( L2Loss(in_layers=[labels, output, weights])) self.set_loss(weighted_loss)