def build(self, hidden_layers=[16], activations=['relu'], dropout=0.5, learning_rate=0.01, l2_norm=5e-4, p1=1., p2=1., n_power_iterations=1, epsilon=0.03, xi=1e-6): with self.device: x = Input(batch_shape=[self.n_nodes, self.n_features], dtype=tf.float32, name='features') adj = Input(batch_shape=[self.n_nodes, self.n_nodes], dtype=tf.float32, sparse=True, name='adj_matrix') index = Input(batch_shape=[None], dtype=tf.int32, name='index') self.GCN_layers = [GraphConvolution(hidden_layers[0], activation=activations[0], kernel_regularizer=regularizers.l2(l2_norm)), GraphConvolution(self.n_classes)] self.dropout_layer = Dropout(dropout) logit = self.propagation(x, adj) output = tf.gather(logit, index) output = Softmax()(output) model = Model(inputs=[x, adj, index], outputs=output) self.model = model self.train_metric = SparseCategoricalAccuracy() self.test_metric = SparseCategoricalAccuracy() self.optimizer = Adam(lr=learning_rate) self.built = True self.p1 = p1 # Alpha self.p2 = p2 # Beta self.xi = xi # Small constant for finite difference self.epsilon = epsilon # Norm length for (virtual) adversarial training self.n_power_iterations = n_power_iterations # Number of power iterations
def length_accuracy(dataset): """ generates the lengthwise accuracy of a dataset :param dataset: dataset to consider :return: """ model = load_model(constants.text_recognition) for x, y in dataset: # create the accuracy evaluation object accuracy = SparseCategoricalAccuracy() # make a prediction and update the state of the accuracy using it prediction = model.predict(x) accuracy.update_state(y, prediction) print(y[0].numpy()) print(np.argmax(prediction, axis=-1)[0]) print("sequences of length", x[1].shape[1] - 1, "had an accuracy of", accuracy.result().numpy())
def evaluate(self): results = [] for x_test, y_test in self.gen.evaluate(): m = SparseCategoricalAccuracy() predictions = [] predictions.append(self.models[0].predict(np.array(x_test))) predictions.append(self.models[0].predict(np.array(x_test))) # calculate average outcomes = average(predictions).numpy() print(outcomes) print(y_test) if len(results) > 0: curr_avg = sum(results) / len(results) print("Accuracy:", curr_avg) m.update_state( # We have changed y_true = [[2], [1], [3]] to the following y_true=y_test, y_pred=outcomes, sample_weight=[1, 1, 1]) results.append(m.result().numpy()) avg = sum(results) / len(results) return avg
def init_loss(self): self.loss_function = SparseCategoricalCrossentropy() self.train_loss = Mean(name="train_loss") self.train_accuracy = SparseCategoricalAccuracy(name="train_accuracy") self.test_loss = Mean(name="test_loss") self.test_accuracy = SparseCategoricalAccuracy(name="test_accuracy")
def evaluate(self, dataset): accuracy_metric = SparseCategoricalAccuracy() for x, y in dataset: y_hat = self.predict(x) accuracy_metric.update_state(y, y_hat) return accuracy_metric.result().numpy()
def build(self, hiddens=[16], activations=['relu'], dropout=0.5, lr=0.01, l2_norm=5e-4, use_bias=False, p1=1., p2=1., n_power_iterations=1, epsilon=0.03, xi=1e-6): with tf.device(self.device): x = Input(batch_shape=[None, self.graph.n_attrs], dtype=self.floatx, name='attr_matrix') adj = Input(batch_shape=[None, None], dtype=self.floatx, sparse=True, name='adj_matrix') index = Input(batch_shape=[None], dtype=self.intx, name='node_index') GCN_layers = [] dropout_layers = [] for hidden, activation in zip(hiddens, activations): GCN_layers.append( GraphConvolution( hidden, activation=activation, use_bias=use_bias, kernel_regularizer=regularizers.l2(l2_norm))) dropout_layers.append(Dropout(rate=dropout)) GCN_layers.append( GraphConvolution(self.graph.n_classes, use_bias=use_bias)) self.GCN_layers = GCN_layers self.dropout_layers = dropout_layers logit = self.forward(x, adj) output = Gather()([logit, index]) model = Model(inputs=[x, adj, index], outputs=output) self.model = model self.train_metric = SparseCategoricalAccuracy() self.test_metric = SparseCategoricalAccuracy() self.loss_fn = SparseCategoricalCrossentropy(from_logits=True) self.optimizer = Adam(lr=lr) self.p1 = p1 # Alpha self.p2 = p2 # Beta self.xi = xi # Small constant for finite difference # Norm length for (virtual) adversarial training self.epsilon = epsilon self.n_power_iterations = n_power_iterations # Number of power iterations
def load_metrics(): global train_loss, train_acc global valid_loss, valid_acc global test_loss, test_acc train_loss = Mean() valid_loss = Mean() test_loss = Mean() train_acc = SparseCategoricalAccuracy() valid_acc = SparseCategoricalAccuracy() test_acc = SparseCategoricalAccuracy()
def on_epoch_end(self, epoch, logs={}): y_pred = self.model.predict(self.X_val, verbose=0) scce = SparseCategoricalCrossentropy(from_logits=True) score = scce(self.y_val, y_pred).numpy() acc = SparseCategoricalAccuracy() acc.update_state(self.y_val, y_pred) print("\n") print("The loss is : {}, the accuracy is: {}".format( score, acc.result().numpy())) gc.collect() K.clear_session()
def __init__(self, fbnet, input_shape, initial_temperature=5, temperature_decay_rate=0.956, temperature_decay_steps=1, latency_alpha=0.2, latency_beta=0.6, weight_lr=0.01, weight_momentum=0.9, weight_decay=1e-4, theta_lr=1e-3, theta_beta1=0.9, theta_beta2=0.999, theta_decay=5e-4): self._epoch = 0 self.initial_temperature = initial_temperature self.temperature = initial_temperature self.latency_alpha = latency_alpha self.latency_beta = latency_beta self.exponential_decay = lambda step: exponential_decay( initial_temperature, temperature_decay_rate, temperature_decay_steps, step) fbnet.build(input_shape) self.fbnet = fbnet self.weights = [] self.thetas = [] for trainable_weight in fbnet.trainable_weights: if 'theta' in trainable_weight.name: self.thetas.append(trainable_weight) else: self.weights.append(trainable_weight) self.weight_opt = SGD(learning_rate=weight_lr, momentum=weight_momentum, decay=weight_decay) self.theta_opt = Adam(learning_rate=theta_lr, beta_1=theta_beta1, beta_2=theta_beta2, decay=theta_decay) self.loss_fn = SparseCategoricalCrossentropy(from_logits=True) self.accuracy_metric = SparseCategoricalAccuracy() self.loss_metric = Mean()
def compile_model(model, learning_rate, decay): logger.info('Compiling the model...') # Instantiate the optimizer optimizer = Adam(learning_rate=learning_rate, decay=decay) # Instantiate the loss function loss_fn = SparseCategoricalCrossentropy(from_logits=True) # Prepare the metrics train_acc_metric = SparseCategoricalAccuracy() val_acc_metric = SparseCategoricalAccuracy() # Compile the model model.compile(optimizer=optimizer, loss=loss_fn, metrics=[train_acc_metric]) logger.info('Completed compiling the model.') return optimizer, loss_fn, train_acc_metric, val_acc_metric
def define_loss(self): """Store attributes containing loss and accuracy functions. Training loss is defined by Cross Categorical Cross Entropy and by Dice loss, an intersection-over-union loss function. """ cce = SparseCategoricalCrossentropy(from_logits=True, reduction=Reduction.NONE, name='sparse_crossentropy') self.cce_loss = cce self.dice_loss = jaccard self.tst_loss = tf.keras.metrics.Mean(name='test_loss') self.trn_acc = SparseCategoricalAccuracy(name='trn_acc') self.tst_acc = SparseCategoricalAccuracy(name='tst_acc')
def get_classification_metrics(): train_loss = Mean() train_acc = SparseCategoricalAccuracy() valid_loss = Mean() valid_acc = SparseCategoricalAccuracy() test_loss = Mean() test_acc = SparseCategoricalAccuracy() metric_objects = dict() metric_objects['train_loss'] = train_loss metric_objects['train_acc'] = train_acc metric_objects['valid_loss'] = valid_loss metric_objects['valid_acc'] = valid_acc metric_objects['test_loss'] = test_loss metric_objects['test_acc'] = test_acc return metric_objects
def get_classification_metrics(losses=None, accuracy=None): train_loss = Mean() train_acc = SparseCategoricalAccuracy() validation_loss = Mean() validation_acc = SparseCategoricalAccuracy() test_lost = Mean() test_acc = SparseCategoricalAccuracy() metrics_objects = dict() metrics_objects['train_loss'] = train_loss metrics_objects['train_accuracy'] = train_acc metrics_objects['validation_loss'] = validation_loss metrics_objects['validation_accuracy'] = validation_acc metrics_objects['test_loss'] = test_lost metrics_objects['test_accuracy'] = test_acc return metrics_objects
def build_estimator(self): model = TFRobertaForSequenceClassification.from_pretrained( ROBERTA_BASE) optimizer = AdamWeightDecay( learning_rate=LEARNING_RATE, epsilon=EPSILON, weight_decay_rate=DECAY, beta_1=BETA) # we do not have one-hot vectors, we can use sparce categorical cross entropy and accuracy loss = SparseCategoricalCrossentropy(from_logits=True) metric = SparseCategoricalAccuracy('accuracy') model.compile(optimizer=optimizer, loss=loss, metrics=[metric]) self.model = model
def get_model(): model = Sequential([Conv2D(filters=16, input_shape=(32, 32, 3), kernel_size=(3,3), activation='relu'), Conv2D(filters=8, kernel_size=(3,3), activation='relu'), MaxPooling2D((4, 4)), Flatten(), BatchNormalization(), Dense(32, activation='relu'), Dense(10, activation='softmax')]) model.compile(optimizer=Adam(), loss=SparseCategoricalCrossentropy(), metrics=[SparseCategoricalAccuracy()]) return model
def __init__(self, model, train_log_dir, test_log_dir, manager): self._model = model self._loss_fn = tf.nn.sparse_softmax_cross_entropy_with_logits self._manager = manager self._train_loss = Mean(name='train_loss') self._test_loss = Mean(name='test_loss') self._train_acc = SparseCategoricalAccuracy(name='train_acc') self._test_acc = SparseCategoricalAccuracy(name='test_acc') self._train_loss.reset_states() self._test_loss.reset_states() self._train_acc.reset_states() self._test_acc.reset_states() os.makedirs(train_log_dir, exist_ok=True) os.makedirs(test_log_dir, exist_ok=True) self._train_summary_writer = create_file_writer(train_log_dir) self._test_summary_writer = create_file_writer(test_log_dir)
def build_model(self): input_layer = Input(shape=784) dense_layer_0 = Dense(512, activation='relu')(input_layer) dense_layer_1 = Dense(256, activation='relu')(dense_layer_0) dense_layer_2 = Dense(128, activation='relu')(dense_layer_1) dense_layer_3 = Dense(64, activation='relu')(dense_layer_2) dense_layer_4 = Dense(32, activation='relu')(dense_layer_3) output_layer = Dense(10, activation='softmax')(dense_layer_4) self.model = Model(input_layer, output_layer) print(self.model.summary()) loss = SparseCategoricalCrossentropy() optimizer = Adam(learning_rate=1e-3) self.model.compile(loss=loss, optimizer=optimizer, metrics=[SparseCategoricalAccuracy()])
def __init__(self, params): self.lr = params.lr self.epochs = params.epochs # Define loss: self.loss_object = SparseCategoricalCrossentropy() # Define optimizer: self.optimizer = Adam() # Define metrics for loss: self.train_loss = Mean() self.train_accuracy = SparseCategoricalAccuracy() self.test_loss = Mean() self.test_accuracy = SparseCategoricalAccuracy() # Define pre processor (params): preprocessor = Process(32, 1) self.train_ds, self.test_ds, encoder_stats = preprocessor.get_datasets( ) # Define model dims d_model = 512 ff_dim = 2048 heads = 8 encoder_dim = 6 decoder_dim = 6 dk = d_model / heads dv = d_model / heads vocab_size = encoder_stats['vocab_size'] max_pos = 10000 # Define model: self.model = Transformer(d_model, ff_dim, dk, dv, heads, encoder_dim, decoder_dim, vocab_size, max_pos) # Define Checkpoints: self.ckpt = tf.train.Checkpoint(step=tf.Variable(1), optimizer=self.optimizer, net=self.model) # Define Checkpoint manager: self.ckpt_manager = tf.train.CheckpointManager( self.ckpt, f'checkpoints{params.ckpt_dir}', max_to_keep=3)
def print_bin_predictions_match(y_test, yhat, binary=True): accuracy = BinaryAccuracy() if binary else SparseCategoricalAccuracy() for i in range(y_test.shape[0]): ix = colored(str(f'{i:02d} |'), 'grey') sep = colored('|', 'grey') y = int(y_test[i][0]) pred = f"{yhat[i][0]:.02f}" if binary else f"{np.argmax(yhat[i])}" accuracy.reset_states() accuracy.update_state(y, yhat[i]) true_pred = accuracy.result().numpy() == 1.0 color = "green" if true_pred else "red" pred = colored(pred, color) if i % 9 == 0: print(f"\n{ix} ", end='') print(f"{y} {sep} {pred} {sep} ", end="") print(colored("\n", "white"))
def fit(self, dataset, augment=True): """ Dataset name, intent class """ self._load_fit_parameters() (x_train, y_train), (x_valid, y_valid) = self._load_format_dataset(dataset, augment) self.model = self._load_base_model(self.fit_params['output_length']) opt = Adam(learning_rate=self.fit_params['learning_rate'], epsilon=self.fit_params['epsilon']) if self.fit_params['from_logits']: loss = SparseCategoricalCrossentropy(from_logits=True) else: loss = SparseCategoricalCrossentropy(from_logits=False) metrics = SparseCategoricalAccuracy('accuracy') self.model.compile(optimizer=opt, loss=loss, metrics=metrics) time = datetime.datetime.now() checkpoint_path = os.path.join(self.model_dir, f"classifier") if not os.path.exists(checkpoint_path): os.makedirs(checkpoint_path) cp_callback = ModelCheckpoint(filepath=os.path.join(checkpoint_path, f"classifier_model"), \ save_weights_only=True, verbose=1, save_best_only=True) if self.fit_params['validate']: history = self.model.fit(x_train, y_train, epochs=self.fit_params['epochs'], batch_size=self.fit_params['batch_size'], validation_data=(x_valid, y_valid), callbacks=cp_callback) else: history = self.model.fit(x_train, y_train, epochs=self.fit_params['epochs'], batch_size=self.fit_params['batch_size'], callbacks=cp_callback) dump(self.fit_params, os.path.join(checkpoint_path, 'params.json')) self.fitted = True # LOGGING print(f"Classifier model fitted in {elapsed_time(time)}s")
def __init__(self, model_name, max_length, curdir): df_train, df_valid, df_test, intent_names, \ self.intent_map, self.slot_map = load_prepare_dataset(curdir) # Y's: self.intent_train = df_train["intent_label"].map( self.intent_map).values self.intent_valid = df_valid["intent_label"].map( self.intent_map).values self.intent_test = df_test["intent_label"].map(self.intent_map).values tokenizer = BertTokenizer.from_pretrained(model_name) self.curdir = curdir # X's: print('Encoding data...') self.encoded_train = encode_dataset(tokenizer, df_train["words"], max_length) self.encoded_valid = encode_dataset(tokenizer, df_valid["words"], max_length) self.encoded_test = encode_dataset(tokenizer, df_test["words"], max_length) self.slot_train = encode_token_labels(df_train["words"], df_train["word_labels"], tokenizer, self.slot_map, max_length) self.slot_valid = encode_token_labels(df_valid["words"], df_valid["word_labels"], tokenizer, self.slot_map, max_length) self.slot_test = encode_token_labels(df_test["words"], df_test["word_labels"], tokenizer, self.slot_map, max_length) self.intent_model = SlotIntentDetectorModelBase( intent_num_labels=len(self.intent_map), slot_num_labels=len(self.slot_map)) opt = Adam(learning_rate=3e-5, epsilon=1e-08) losses = [ SparseCategoricalCrossentropy(from_logits=True), SparseCategoricalCrossentropy(from_logits=True) ] metrics = [SparseCategoricalAccuracy('accuracy')] self.intent_model.compile(optimizer=opt, loss=losses, metrics=metrics)
def fit(self, dataset, intent, merge_intents): """ Dataset name, intent class """ self.intent = intent self._load_fit_parameters() (x_train, y_train), (x_valid, y_valid) = self._load_format_dataset(dataset, merge_intents) self.model = self._load_base_model(self.fit_params['output_length']) opt = Adam(learning_rate=self.fit_params['learning_rate'], epsilon=self.fit_params['epsilon']) if self.fit_params['from_logits']: loss = SparseCategoricalCrossentropy(from_logits=True) else: loss = SparseCategoricalCrossentropy(from_logits=False) metrics = SparseCategoricalAccuracy('accuracy') # metrics = [Precision(), Recall()] self.model.compile(optimizer=opt, loss=loss, metrics=metrics) time = datetime.datetime.now() # if isinstance(self.intent, list): # self.intent = '@'.join(self.intent) checkpoint_path = os.path.join(self.models_dir, f"{self.intent}") if not os.path.exists(checkpoint_path): os.makedirs(checkpoint_path) cp_callback = ModelCheckpoint(filepath=os.path.join(checkpoint_path, f"{self.intent}_model"), \ save_weights_only=True, verbose=1, save_best_only=True) if self.fit_params['validate']: history = self.model.fit(x_train, y_train, epochs=self.fit_params['epochs'], batch_size=self.fit_params['batch_size'], validation_data=(x_valid, y_valid), callbacks=[cp_callback]) else: history = self.model.fit(x_train, y_train, epochs=self.fit_params['epochs'], batch_size=self.fit_params['batch_size'], callbacks=[cp_callback]) # LOGGING print(f"{self.intent} tagger fitted") dump(self.fit_params, os.path.join(checkpoint_path, 'params.json'))
def __init__(self, in_channels, out_channels, hiddens=[16], activations=['relu'], dropout=0.5, weight_decay=5e-4, lr=0.01, use_bias=True): super().__init__() self.convs = [] inc = in_channels for hidden, activation in zip(hiddens, activations): layer = GraphConv(inc, hidden, bias=use_bias, activation=get(activation)) self.convs.append(layer) inc = hidden layer = GraphConv(inc, out_channels, bias=use_bias) self.convs.append(layer) self.dropout = layers.Dropout(dropout) self.compile(loss=SparseCategoricalCrossentropy(from_logits=True), optimizer=Adam(lr=lr), metrics=['accuracy']) self.weight_decay = weight_decay self.metric = SparseCategoricalAccuracy()
def main(cfg: DictConfig): train_ds, val_ds = create_dataset(cfg=cfg, output_dir=os.getenv( "OUTPUT_DIR", "../data/processed")) model = create_model(cfg=cfg) optimizer = SGD(learning_rate=cfg.TRAINING.LEARNING_RATE, momentum=cfg.TRAINING.MOMENTUM) loss_object = SparseCategoricalCrossentropy(from_logits=True) tb_callback = tf.keras.callbacks.TensorBoard(log_dir=os.getcwd()) model.compile( optimizer=optimizer, loss=loss_object, metrics=[ SparseCategoricalAccuracy(), MeanIouWithLogits(num_classes=cfg.DATASET.NUM_CLASSES) ], ) model.fit(train_ds, epochs=cfg.TRAINING.EPOCHS, validation_data=val_ds, callbacks=[tb_callback])
def create_model(args, num_classes=5): def create_regularizer(): if args.l1 == None: return None if args.l2 == None else regularizers.l2(args.l2) else: return regularizers.l1( argsl1) if args.l2 == None else regularizers.l1_l2(l1=args.l1, l2=args.l2) def purgeNones( layers ): # Used to remove layers that have been set to None (e.g. unused Dropout) return [layer for layer in layers if layer is not None] product = Sequential( purgeNones([ Rescaling(1. / 255), Conv2D(32, 3, activation='relu'), MaxPooling2D(), Conv2D(32, 3, activation='relu'), MaxPooling2D(), Conv2D(32, 3, activation='relu'), MaxPooling2D(), Flatten(), Dropout(args.dropout) if args.dropout else None, Dense(128, activation='relu', kernel_regularizer=create_regularizer()), Dropout(args.dropout) if args.dropout else None, Dense(num_classes) ])) product.compile(optimizer='adam', loss=SparseCategoricalCrossentropy(from_logits=True), metrics=[SparseCategoricalAccuracy()]) return product
attack = cgf['ATTACK']['name'] original_data = cgf['DATASET']['arguments']['original_images'] adv_data = cgf['DATASET']['arguments']["adv_images"] adv_labels = cgf['DATASET']['arguments']['adv_labels'] adv_diffs = cgf['DATASET']['arguments']['adv_diffs'] weights_path = join(model_path, filepath) optimizer = cgf['TRAIN']['optim']['type'] loss_type = cgf['TRAIN']['loss']['type'] metric_list = list(cgf['TRAIN']['metrics'].values()) if loss_type == 'SparseCategoricalCrossentropy': loss_type = SparseCategoricalCrossentropy(from_logits=False) metric_list = [SparseCategoricalAccuracy()] output_shape = 2 labels = np.reshape(labels, (-1)) model_name = cgf['MODEL']['name'] model_arguments = cgf['MODEL']['arguments'] #model = mb.model_selector(model_name, input_shape, output_shape, model_arguments) model = tf.keras.models.load_model(weights_path) # Preprocessing if model_name =='resnet': preprocessing = res_prep data = 255*data data = data - 122
from tensorflow.keras.losses import SparseCategoricalCrossentropy from tensorflow.keras.metrics import SparseCategoricalAccuracy from tensorflow.keras.datasets import mnist (train_x, train_y), (test_x, test_y) = mnist.load_data() train_x, test_x = train_x / 255.0, test_x / 255.0 print('train data:', train_x.shape, train_y.shape) print('test data: ', test_x.shape, test_y.shape) model = Sequential() model.add(Flatten()) model.add( Dense(128, activation='relu', kernel_regularizer=tf.keras.regularizers.L2())) model.add(Dense(10, activation='softmax')) model.compile(optimizer=Adam(), loss=SparseCategoricalCrossentropy(from_logits=False), metrics=[SparseCategoricalAccuracy()]) model.fit(train_x, train_y, batch_size=32, epochs=10, validation_data=(test_x, test_y), validation_freq=5) model.summary()
def call(self, inputs): x, a = inputs x = self.conv1([x, a]) x = self.conv2([x, a]) output = self.flatten(x) output = self.fc1(output) output = self.fc2(output) return output # Create model model = Net() optimizer = Adam() loss_fn = SparseCategoricalCrossentropy() acc_fn = SparseCategoricalAccuracy() # Training function @tf.function def train_on_batch(inputs, target): with tf.GradientTape() as tape: predictions = model(inputs, training=True) loss = loss_fn(target, predictions) + sum(model.losses) acc = acc_fn(target, predictions) gradients = tape.gradient(loss, model.trainable_variables) optimizer.apply_gradients(zip(gradients, model.trainable_variables)) return loss, acc
# Add classification layer self.fc2 = Dense(18, activation='softmax') def call(self, inputs): x = self.base_model(inputs) x = self.pool(x) x = self.fc2(x) return x optimizer = SGD(learning_rate=params['learning_rate'], momentum=params['momentum']) loss_object = SparseCategoricalCrossentropy() # Define our metrics train_loss = Mean('train_loss', dtype=tf.float32) train_accuracy = SparseCategoricalAccuracy('train_accuracy') eval_loss = Mean('eval_loss', dtype=tf.float32) eval_accuracy = SparseCategoricalAccuracy('eval_accuracy') current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") train_log_dir = 'Exc_2/logs/gradient_tape/' + current_time + '/train' test_log_dir = 'Exc_2/logs/gradient_tape/' + current_time + '/test' img_train_log_dir = 'Exc_2/logs/gradient_tape/' + current_time + '/img_train' img_eval_log_dir = 'Exc_2/logs/gradient_tape/' + current_time + '/img_eval' @tf.function def train_step(sample, model): with tf.GradientTape() as tape: features, labels = sample logits = model(features, training=True)
class SBVAT(SemiSupervisedModel): """ Implementation of sample-based Batch Virtual Adversarial Training Graph Convolutional Networks (SBVAT). `Batch Virtual Adversarial Training for Graph Convolutional Networks <https://arxiv.org/abs/1902.09192>` Tensorflow 1.x implementation: <https://github.com/thudzj/BVAT> """ def __init__(self, *graph, n_samples=50, adj_transform="normalize_adj", attr_transform=None, device='cpu:0', seed=None, name=None, **kwargs): """Create a sample-based Batch Virtual Adversarial Training Graph Convolutional Networks (SBVAT) model. This can be instantiated in several ways: model = SBVAT(graph) with a `graphgallery.data.Graph` instance representing A sparse, attributed, labeled graph. model = SBVAT(adj_matrix, attr_matrix, labels) where `adj_matrix` is a 2D Scipy sparse matrix denoting the graph, `attr_matrix` is a 2D Numpy array-like matrix denoting the node attributes, `labels` is a 1D Numpy array denoting the node labels. Parameters: ---------- graph: An instance of `graphgallery.data.Graph` or a tuple (list) of inputs. A sparse, attributed, labeled graph. n_samples (Positive integer, optional): The number of sampled subset nodes in the graph where the length of the shortest path between them is at least `4`. (default :obj: `50`) adj_transform: string, `transform`, or None. optional How to transform the adjacency matrix. See `graphgallery.transforms` (default: :obj:`'normalize_adj'` with normalize rate `-0.5`. i.e., math:: \hat{A} = D^{-\frac{1}{2}} A D^{-\frac{1}{2}}) attr_transform: string, `transform`, or None. optional How to transform the node attribute matrix. See `graphgallery.transforms` (default :obj: `None`) device: string. optional The device where the model is running on. You can specified `CPU` or `GPU` for the model. (default: :str: `CPU:0`, i.e., running on the 0-th `CPU`) seed: interger scalar. optional Used in combination with `tf.random.set_seed` & `np.random.seed` & `random.seed` to create a reproducible sequence of tensors across multiple calls. (default :obj: `None`, i.e., using random seed) name: string. optional Specified name for the model. (default: :str: `class.__name__`) kwargs: other customized keyword Parameters. """ super().__init__(*graph, device=device, seed=seed, name=name, **kwargs) self.adj_transform = T.get(adj_transform) self.attr_transform = T.get(attr_transform) self.n_samples = n_samples self.process() def process_step(self): graph = self.graph adj_matrix = self.adj_transform(graph.adj_matrix) attr_matrix = self.attr_transform(graph.attr_matrix) self.neighbors = find_4o_nbrs(adj_matrix) self.feature_inputs, self.structure_inputs = T.astensors( attr_matrix, adj_matrix, device=self.device) # use decorator to make sure all list arguments have the same length @EqualVarLength() def build(self, hiddens=[16], activations=['relu'], dropout=0.5, lr=0.01, l2_norm=5e-4, use_bias=False, p1=1., p2=1., n_power_iterations=1, epsilon=0.03, xi=1e-6): with tf.device(self.device): x = Input(batch_shape=[None, self.graph.n_attrs], dtype=self.floatx, name='attr_matrix') adj = Input(batch_shape=[None, None], dtype=self.floatx, sparse=True, name='adj_matrix') index = Input(batch_shape=[None], dtype=self.intx, name='node_index') GCN_layers = [] dropout_layers = [] for hidden, activation in zip(hiddens, activations): GCN_layers.append( GraphConvolution( hidden, activation=activation, use_bias=use_bias, kernel_regularizer=regularizers.l2(l2_norm))) dropout_layers.append(Dropout(rate=dropout)) GCN_layers.append( GraphConvolution(self.graph.n_classes, use_bias=use_bias)) self.GCN_layers = GCN_layers self.dropout_layers = dropout_layers logit = self.forward(x, adj) output = Gather()([logit, index]) model = Model(inputs=[x, adj, index], outputs=output) self.model = model self.train_metric = SparseCategoricalAccuracy() self.test_metric = SparseCategoricalAccuracy() self.loss_fn = SparseCategoricalCrossentropy(from_logits=True) self.optimizer = Adam(lr=lr) self.p1 = p1 # Alpha self.p2 = p2 # Beta self.xi = xi # Small constant for finite difference # Norm length for (virtual) adversarial training self.epsilon = epsilon self.n_power_iterations = n_power_iterations # Number of power iterations def forward(self, x, adj, training=True): h = x for dropout_layer, GCN_layer in zip(self.dropout_layers, self.GCN_layers[:-1]): h = GCN_layer([h, adj]) h = dropout_layer(h, training=training) h = self.GCN_layers[-1]([h, adj]) return h @tf.function def train_step(self, sequence): with tf.device(self.device): self.train_metric.reset_states() for inputs, labels in sequence: x, adj, index, adv_mask = inputs with tf.GradientTape() as tape: logit = self.forward(x, adj) output = tf.gather(logit, index) loss = self.loss_fn(labels, output) entropy_loss = entropy_y_x(logit) vat_loss = self.virtual_adversarial_loss(x, adj, logit=logit, adv_mask=adv_mask) loss += self.p1 * vat_loss + self.p2 * entropy_loss self.train_metric.update_state(labels, output) trainable_variables = self.model.trainable_variables gradients = tape.gradient(loss, trainable_variables) self.optimizer.apply_gradients( zip(gradients, trainable_variables)) return loss, self.train_metric.result() @tf.function def test_step(self, sequence): with tf.device(self.device): self.test_metric.reset_states() for inputs, labels in sequence: x, adj, index, _ = inputs logit = self.forward(x, adj, training=False) output = tf.gather(logit, index) loss = self.loss_fn(labels, output) self.test_metric.update_state(labels, output) return loss, self.test_metric.result() def virtual_adversarial_loss(self, x, adj, logit, adv_mask): d = tf.random.normal(shape=tf.shape(x), dtype=self.floatx) for _ in range(self.n_power_iterations): d = get_normalized_vector(d) * self.xi logit_p = logit with tf.GradientTape() as tape: tape.watch(d) logit_m = self.forward(x + d, adj) dist = kl_divergence_with_logit(logit_p, logit_m, adv_mask) grad = tape.gradient(dist, d) d = tf.stop_gradient(grad) r_vadv = get_normalized_vector(d) * self.epsilon logit_p = tf.stop_gradient(logit) logit_m = self.forward(x + r_vadv, adj) loss = kl_divergence_with_logit(logit_p, logit_m, adv_mask) return tf.identity(loss) def train_sequence(self, index): index = T.asintarr(index) labels = self.graph.labels[index] sequence = SBVATSampleSequence( [self.feature_inputs, self.structure_inputs, index], labels, neighbors=self.neighbors, n_samples=self.n_samples, device=self.device) return sequence def test_sequence(self, index): index = T.asintarr(index) labels = self.graph.labels[index] sequence = SBVATSampleSequence( [self.feature_inputs, self.structure_inputs, index], labels, neighbors=self.neighbors, n_samples=self.n_samples, resample=False, device=self.device) return sequence def predict_step(self, sequence): with tf.device(self.device): for inputs, _ in sequence: x, adj, index, adv_mask = inputs output = self.forward(x, adj, training=False) logit = tf.gather(output, index) if tf.is_tensor(logit): logit = logit.numpy() return logit