def build(self, hidden_layers=[16], activations=['relu'], dropout=0.5,
learning_rate=0.01, l2_norm=5e-4, p1=1., p2=1.,
n_power_iterations=1, epsilon=0.03, xi=1e-6):
with self.device:
x = Input(batch_shape=[self.n_nodes, self.n_features], dtype=tf.float32, name='features')
adj = Input(batch_shape=[self.n_nodes, self.n_nodes], dtype=tf.float32, sparse=True, name='adj_matrix')
index = Input(batch_shape=[None], dtype=tf.int32, name='index')
self.GCN_layers = [GraphConvolution(hidden_layers[0],
activation=activations[0],
kernel_regularizer=regularizers.l2(l2_norm)),
GraphConvolution(self.n_classes)]
self.dropout_layer = Dropout(dropout)
logit = self.propagation(x, adj)
output = tf.gather(logit, index)
output = Softmax()(output)
model = Model(inputs=[x, adj, index], outputs=output)
self.model = model
self.train_metric = SparseCategoricalAccuracy()
self.test_metric = SparseCategoricalAccuracy()
self.optimizer = Adam(lr=learning_rate)
self.built = True
self.p1 = p1 # Alpha
self.p2 = p2 # Beta
self.xi = xi # Small constant for finite difference
self.epsilon = epsilon # Norm length for (virtual) adversarial training
self.n_power_iterations = n_power_iterations # Number of power iterations
def length_accuracy(dataset):
"""
generates the lengthwise accuracy of a dataset
:param dataset: dataset to consider
:return:
"""
model = load_model(constants.text_recognition)
for x, y in dataset:
# create the accuracy evaluation object
accuracy = SparseCategoricalAccuracy()
# make a prediction and update the state of the accuracy using it
prediction = model.predict(x)
accuracy.update_state(y, prediction)
print(y[0].numpy())
print(np.argmax(prediction, axis=-1)[0])
print("sequences of length", x[1].shape[1] - 1,
"had an accuracy of", accuracy.result().numpy())
def evaluate(self):
results = []
for x_test, y_test in self.gen.evaluate():
m = SparseCategoricalAccuracy()
predictions = []
predictions.append(self.models[0].predict(np.array(x_test)))
predictions.append(self.models[0].predict(np.array(x_test)))
# calculate average
outcomes = average(predictions).numpy()
print(outcomes)
print(y_test)
if len(results) > 0:
curr_avg = sum(results) / len(results)
print("Accuracy:", curr_avg)
m.update_state(
# We have changed y_true = [[2], [1], [3]] to the following
y_true=y_test,
y_pred=outcomes,
sample_weight=[1, 1, 1])
results.append(m.result().numpy())
avg = sum(results) / len(results)
return avg
def init_loss(self):
self.loss_function = SparseCategoricalCrossentropy()
self.train_loss = Mean(name="train_loss")
self.train_accuracy = SparseCategoricalAccuracy(name="train_accuracy")
self.test_loss = Mean(name="test_loss")
self.test_accuracy = SparseCategoricalAccuracy(name="test_accuracy")
def evaluate(self, dataset):
accuracy_metric = SparseCategoricalAccuracy()
for x, y in dataset:
y_hat = self.predict(x)
accuracy_metric.update_state(y, y_hat)
return accuracy_metric.result().numpy()
def build(self,
hiddens=[16],
activations=['relu'],
dropout=0.5,
lr=0.01,
l2_norm=5e-4,
use_bias=False,
p1=1.,
p2=1.,
n_power_iterations=1,
epsilon=0.03,
xi=1e-6):
with tf.device(self.device):
x = Input(batch_shape=[None, self.graph.n_attrs],
dtype=self.floatx,
name='attr_matrix')
adj = Input(batch_shape=[None, None],
dtype=self.floatx,
sparse=True,
name='adj_matrix')
index = Input(batch_shape=[None],
dtype=self.intx,
name='node_index')
GCN_layers = []
dropout_layers = []
for hidden, activation in zip(hiddens, activations):
GCN_layers.append(
GraphConvolution(
hidden,
activation=activation,
use_bias=use_bias,
kernel_regularizer=regularizers.l2(l2_norm)))
dropout_layers.append(Dropout(rate=dropout))
GCN_layers.append(
GraphConvolution(self.graph.n_classes, use_bias=use_bias))
self.GCN_layers = GCN_layers
self.dropout_layers = dropout_layers
logit = self.forward(x, adj)
output = Gather()([logit, index])
model = Model(inputs=[x, adj, index], outputs=output)
self.model = model
self.train_metric = SparseCategoricalAccuracy()
self.test_metric = SparseCategoricalAccuracy()
self.loss_fn = SparseCategoricalCrossentropy(from_logits=True)
self.optimizer = Adam(lr=lr)
self.p1 = p1 # Alpha
self.p2 = p2 # Beta
self.xi = xi # Small constant for finite difference
# Norm length for (virtual) adversarial training
self.epsilon = epsilon
self.n_power_iterations = n_power_iterations # Number of power iterations
def load_metrics():
global train_loss, train_acc
global valid_loss, valid_acc
global test_loss, test_acc
train_loss = Mean()
valid_loss = Mean()
test_loss = Mean()
train_acc = SparseCategoricalAccuracy()
valid_acc = SparseCategoricalAccuracy()
test_acc = SparseCategoricalAccuracy()
def on_epoch_end(self, epoch, logs={}):
y_pred = self.model.predict(self.X_val, verbose=0)
scce = SparseCategoricalCrossentropy(from_logits=True)
score = scce(self.y_val, y_pred).numpy()
acc = SparseCategoricalAccuracy()
acc.update_state(self.y_val, y_pred)
print("\n")
print("The loss is : {}, the accuracy is: {}".format(
score,
acc.result().numpy()))
gc.collect()
K.clear_session()
def __init__(self,
fbnet,
input_shape,
initial_temperature=5,
temperature_decay_rate=0.956,
temperature_decay_steps=1,
latency_alpha=0.2,
latency_beta=0.6,
weight_lr=0.01,
weight_momentum=0.9,
weight_decay=1e-4,
theta_lr=1e-3,
theta_beta1=0.9,
theta_beta2=0.999,
theta_decay=5e-4):
self._epoch = 0
self.initial_temperature = initial_temperature
self.temperature = initial_temperature
self.latency_alpha = latency_alpha
self.latency_beta = latency_beta
self.exponential_decay = lambda step: exponential_decay(
initial_temperature, temperature_decay_rate,
temperature_decay_steps, step)
fbnet.build(input_shape)
self.fbnet = fbnet
self.weights = []
self.thetas = []
for trainable_weight in fbnet.trainable_weights:
if 'theta' in trainable_weight.name:
self.thetas.append(trainable_weight)
else:
self.weights.append(trainable_weight)
self.weight_opt = SGD(learning_rate=weight_lr,
momentum=weight_momentum,
decay=weight_decay)
self.theta_opt = Adam(learning_rate=theta_lr,
beta_1=theta_beta1,
beta_2=theta_beta2,
decay=theta_decay)
self.loss_fn = SparseCategoricalCrossentropy(from_logits=True)
self.accuracy_metric = SparseCategoricalAccuracy()
self.loss_metric = Mean()
def compile_model(model, learning_rate, decay):
logger.info('Compiling the model...')
# Instantiate the optimizer
optimizer = Adam(learning_rate=learning_rate, decay=decay)
# Instantiate the loss function
loss_fn = SparseCategoricalCrossentropy(from_logits=True)
# Prepare the metrics
train_acc_metric = SparseCategoricalAccuracy()
val_acc_metric = SparseCategoricalAccuracy()
# Compile the model
model.compile(optimizer=optimizer,
loss=loss_fn,
metrics=[train_acc_metric])
logger.info('Completed compiling the model.')
return optimizer, loss_fn, train_acc_metric, val_acc_metric
def define_loss(self):
"""Store attributes containing loss and accuracy functions.
Training loss is defined by Cross Categorical Cross Entropy and by
Dice loss, an intersection-over-union loss function.
"""
cce = SparseCategoricalCrossentropy(from_logits=True,
reduction=Reduction.NONE,
name='sparse_crossentropy')
self.cce_loss = cce
self.dice_loss = jaccard
self.tst_loss = tf.keras.metrics.Mean(name='test_loss')
self.trn_acc = SparseCategoricalAccuracy(name='trn_acc')
self.tst_acc = SparseCategoricalAccuracy(name='tst_acc')
def get_classification_metrics():
train_loss = Mean()
train_acc = SparseCategoricalAccuracy()
valid_loss = Mean()
valid_acc = SparseCategoricalAccuracy()
test_loss = Mean()
test_acc = SparseCategoricalAccuracy()
metric_objects = dict()
metric_objects['train_loss'] = train_loss
metric_objects['train_acc'] = train_acc
metric_objects['valid_loss'] = valid_loss
metric_objects['valid_acc'] = valid_acc
metric_objects['test_loss'] = test_loss
metric_objects['test_acc'] = test_acc
return metric_objects
def get_classification_metrics(losses=None, accuracy=None):
train_loss = Mean()
train_acc = SparseCategoricalAccuracy()
validation_loss = Mean()
validation_acc = SparseCategoricalAccuracy()
test_lost = Mean()
test_acc = SparseCategoricalAccuracy()
metrics_objects = dict()
metrics_objects['train_loss'] = train_loss
metrics_objects['train_accuracy'] = train_acc
metrics_objects['validation_loss'] = validation_loss
metrics_objects['validation_accuracy'] = validation_acc
metrics_objects['test_loss'] = test_lost
metrics_objects['test_accuracy'] = test_acc
return metrics_objects
def build_estimator(self):
model = TFRobertaForSequenceClassification.from_pretrained(
ROBERTA_BASE)
optimizer = AdamWeightDecay(
learning_rate=LEARNING_RATE, epsilon=EPSILON, weight_decay_rate=DECAY, beta_1=BETA)
# we do not have one-hot vectors, we can use sparce categorical cross entropy and accuracy
loss = SparseCategoricalCrossentropy(from_logits=True)
metric = SparseCategoricalAccuracy('accuracy')
model.compile(optimizer=optimizer, loss=loss, metrics=[metric])
self.model = model
def get_model():
model = Sequential([Conv2D(filters=16, input_shape=(32, 32, 3), kernel_size=(3,3),
activation='relu'),
Conv2D(filters=8, kernel_size=(3,3), activation='relu'),
MaxPooling2D((4, 4)),
Flatten(),
BatchNormalization(),
Dense(32, activation='relu'),
Dense(10, activation='softmax')])
model.compile(optimizer=Adam(), loss=SparseCategoricalCrossentropy(),
metrics=[SparseCategoricalAccuracy()])
return model
def __init__(self, model, train_log_dir, test_log_dir, manager):
self._model = model
self._loss_fn = tf.nn.sparse_softmax_cross_entropy_with_logits
self._manager = manager
self._train_loss = Mean(name='train_loss')
self._test_loss = Mean(name='test_loss')
self._train_acc = SparseCategoricalAccuracy(name='train_acc')
self._test_acc = SparseCategoricalAccuracy(name='test_acc')
self._train_loss.reset_states()
self._test_loss.reset_states()
self._train_acc.reset_states()
self._test_acc.reset_states()
os.makedirs(train_log_dir, exist_ok=True)
os.makedirs(test_log_dir, exist_ok=True)
self._train_summary_writer = create_file_writer(train_log_dir)
self._test_summary_writer = create_file_writer(test_log_dir)
def build_model(self):
input_layer = Input(shape=784)
dense_layer_0 = Dense(512, activation='relu')(input_layer)
dense_layer_1 = Dense(256, activation='relu')(dense_layer_0)
dense_layer_2 = Dense(128, activation='relu')(dense_layer_1)
dense_layer_3 = Dense(64, activation='relu')(dense_layer_2)
dense_layer_4 = Dense(32, activation='relu')(dense_layer_3)
output_layer = Dense(10, activation='softmax')(dense_layer_4)
self.model = Model(input_layer, output_layer)
print(self.model.summary())
loss = SparseCategoricalCrossentropy()
optimizer = Adam(learning_rate=1e-3)
self.model.compile(loss=loss, optimizer=optimizer, metrics=[SparseCategoricalAccuracy()])
def __init__(self, params):
self.lr = params.lr
self.epochs = params.epochs
# Define loss:
self.loss_object = SparseCategoricalCrossentropy()
# Define optimizer:
self.optimizer = Adam()
# Define metrics for loss:
self.train_loss = Mean()
self.train_accuracy = SparseCategoricalAccuracy()
self.test_loss = Mean()
self.test_accuracy = SparseCategoricalAccuracy()
# Define pre processor (params):
preprocessor = Process(32, 1)
self.train_ds, self.test_ds, encoder_stats = preprocessor.get_datasets(
)
# Define model dims
d_model = 512
ff_dim = 2048
heads = 8
encoder_dim = 6
decoder_dim = 6
dk = d_model / heads
dv = d_model / heads
vocab_size = encoder_stats['vocab_size']
max_pos = 10000
# Define model:
self.model = Transformer(d_model, ff_dim, dk, dv, heads, encoder_dim,
decoder_dim, vocab_size, max_pos)
# Define Checkpoints:
self.ckpt = tf.train.Checkpoint(step=tf.Variable(1),
optimizer=self.optimizer,
net=self.model)
# Define Checkpoint manager:
self.ckpt_manager = tf.train.CheckpointManager(
self.ckpt, f'checkpoints{params.ckpt_dir}', max_to_keep=3)
def print_bin_predictions_match(y_test, yhat, binary=True):
accuracy = BinaryAccuracy() if binary else SparseCategoricalAccuracy()
for i in range(y_test.shape[0]):
ix = colored(str(f'{i:02d} |'), 'grey')
sep = colored('|', 'grey')
y = int(y_test[i][0])
pred = f"{yhat[i][0]:.02f}" if binary else f"{np.argmax(yhat[i])}"
accuracy.reset_states()
accuracy.update_state(y, yhat[i])
true_pred = accuracy.result().numpy() == 1.0
color = "green" if true_pred else "red"
pred = colored(pred, color)
if i % 9 == 0:
print(f"\n{ix} ", end='')
print(f"{y} {sep} {pred} {sep} ", end="")
print(colored("\n", "white"))
def fit(self, dataset, augment=True):
"""
Dataset name, intent class
"""
self._load_fit_parameters()
(x_train,
y_train), (x_valid,
y_valid) = self._load_format_dataset(dataset, augment)
self.model = self._load_base_model(self.fit_params['output_length'])
opt = Adam(learning_rate=self.fit_params['learning_rate'],
epsilon=self.fit_params['epsilon'])
if self.fit_params['from_logits']:
loss = SparseCategoricalCrossentropy(from_logits=True)
else:
loss = SparseCategoricalCrossentropy(from_logits=False)
metrics = SparseCategoricalAccuracy('accuracy')
self.model.compile(optimizer=opt, loss=loss, metrics=metrics)
time = datetime.datetime.now()
checkpoint_path = os.path.join(self.model_dir, f"classifier")
if not os.path.exists(checkpoint_path):
os.makedirs(checkpoint_path)
cp_callback = ModelCheckpoint(filepath=os.path.join(checkpoint_path, f"classifier_model"), \
save_weights_only=True, verbose=1, save_best_only=True)
if self.fit_params['validate']:
history = self.model.fit(x_train,
y_train,
epochs=self.fit_params['epochs'],
batch_size=self.fit_params['batch_size'],
validation_data=(x_valid, y_valid),
callbacks=cp_callback)
else:
history = self.model.fit(x_train,
y_train,
epochs=self.fit_params['epochs'],
batch_size=self.fit_params['batch_size'],
callbacks=cp_callback)
dump(self.fit_params, os.path.join(checkpoint_path, 'params.json'))
self.fitted = True
# LOGGING
print(f"Classifier model fitted in {elapsed_time(time)}s")
def __init__(self, model_name, max_length, curdir):
df_train, df_valid, df_test, intent_names, \
self.intent_map, self.slot_map = load_prepare_dataset(curdir)
# Y's:
self.intent_train = df_train["intent_label"].map(
self.intent_map).values
self.intent_valid = df_valid["intent_label"].map(
self.intent_map).values
self.intent_test = df_test["intent_label"].map(self.intent_map).values
tokenizer = BertTokenizer.from_pretrained(model_name)
self.curdir = curdir
# X's:
print('Encoding data...')
self.encoded_train = encode_dataset(tokenizer, df_train["words"],
max_length)
self.encoded_valid = encode_dataset(tokenizer, df_valid["words"],
max_length)
self.encoded_test = encode_dataset(tokenizer, df_test["words"],
max_length)
self.slot_train = encode_token_labels(df_train["words"],
df_train["word_labels"],
tokenizer, self.slot_map,
max_length)
self.slot_valid = encode_token_labels(df_valid["words"],
df_valid["word_labels"],
tokenizer, self.slot_map,
max_length)
self.slot_test = encode_token_labels(df_test["words"],
df_test["word_labels"], tokenizer,
self.slot_map, max_length)
self.intent_model = SlotIntentDetectorModelBase(
intent_num_labels=len(self.intent_map),
slot_num_labels=len(self.slot_map))
opt = Adam(learning_rate=3e-5, epsilon=1e-08)
losses = [
SparseCategoricalCrossentropy(from_logits=True),
SparseCategoricalCrossentropy(from_logits=True)
]
metrics = [SparseCategoricalAccuracy('accuracy')]
self.intent_model.compile(optimizer=opt, loss=losses, metrics=metrics)
def fit(self, dataset, intent, merge_intents):
"""
Dataset name, intent class
"""
self.intent = intent
self._load_fit_parameters()
(x_train, y_train), (x_valid, y_valid) = self._load_format_dataset(dataset, merge_intents)
self.model = self._load_base_model(self.fit_params['output_length'])
opt = Adam(learning_rate=self.fit_params['learning_rate'], epsilon=self.fit_params['epsilon'])
if self.fit_params['from_logits']:
loss = SparseCategoricalCrossentropy(from_logits=True)
else:
loss = SparseCategoricalCrossentropy(from_logits=False)
metrics = SparseCategoricalAccuracy('accuracy')
# metrics = [Precision(), Recall()]
self.model.compile(optimizer=opt, loss=loss, metrics=metrics)
time = datetime.datetime.now()
# if isinstance(self.intent, list):
# self.intent = '@'.join(self.intent)
checkpoint_path = os.path.join(self.models_dir, f"{self.intent}")
if not os.path.exists(checkpoint_path):
os.makedirs(checkpoint_path)
cp_callback = ModelCheckpoint(filepath=os.path.join(checkpoint_path, f"{self.intent}_model"), \
save_weights_only=True, verbose=1, save_best_only=True)
if self.fit_params['validate']:
history = self.model.fit(x_train, y_train,
epochs=self.fit_params['epochs'], batch_size=self.fit_params['batch_size'],
validation_data=(x_valid, y_valid), callbacks=[cp_callback])
else:
history = self.model.fit(x_train, y_train,
epochs=self.fit_params['epochs'], batch_size=self.fit_params['batch_size'],
callbacks=[cp_callback])
# LOGGING
print(f"{self.intent} tagger fitted")
dump(self.fit_params, os.path.join(checkpoint_path, 'params.json'))
def __init__(self, in_channels, out_channels,
hiddens=[16],
activations=['relu'],
dropout=0.5,
weight_decay=5e-4,
lr=0.01, use_bias=True):
super().__init__()
self.convs = []
inc = in_channels
for hidden, activation in zip(hiddens, activations):
layer = GraphConv(inc, hidden, bias=use_bias,
activation=get(activation))
self.convs.append(layer)
inc = hidden
layer = GraphConv(inc, out_channels, bias=use_bias)
self.convs.append(layer)
self.dropout = layers.Dropout(dropout)
self.compile(loss=SparseCategoricalCrossentropy(from_logits=True),
optimizer=Adam(lr=lr), metrics=['accuracy'])
self.weight_decay = weight_decay
self.metric = SparseCategoricalAccuracy()
def main(cfg: DictConfig):
train_ds, val_ds = create_dataset(cfg=cfg,
output_dir=os.getenv(
"OUTPUT_DIR", "../data/processed"))
model = create_model(cfg=cfg)
optimizer = SGD(learning_rate=cfg.TRAINING.LEARNING_RATE,
momentum=cfg.TRAINING.MOMENTUM)
loss_object = SparseCategoricalCrossentropy(from_logits=True)
tb_callback = tf.keras.callbacks.TensorBoard(log_dir=os.getcwd())
model.compile(
optimizer=optimizer,
loss=loss_object,
metrics=[
SparseCategoricalAccuracy(),
MeanIouWithLogits(num_classes=cfg.DATASET.NUM_CLASSES)
],
)
model.fit(train_ds,
epochs=cfg.TRAINING.EPOCHS,
validation_data=val_ds,
callbacks=[tb_callback])
def create_model(args, num_classes=5):
def create_regularizer():
if args.l1 == None:
return None if args.l2 == None else regularizers.l2(args.l2)
else:
return regularizers.l1(
argsl1) if args.l2 == None else regularizers.l1_l2(l1=args.l1,
l2=args.l2)
def purgeNones(
layers
): # Used to remove layers that have been set to None (e.g. unused Dropout)
return [layer for layer in layers if layer is not None]
product = Sequential(
purgeNones([
Rescaling(1. / 255),
Conv2D(32, 3, activation='relu'),
MaxPooling2D(),
Conv2D(32, 3, activation='relu'),
MaxPooling2D(),
Conv2D(32, 3, activation='relu'),
MaxPooling2D(),
Flatten(),
Dropout(args.dropout) if args.dropout else None,
Dense(128,
activation='relu',
kernel_regularizer=create_regularizer()),
Dropout(args.dropout) if args.dropout else None,
Dense(num_classes)
]))
product.compile(optimizer='adam',
loss=SparseCategoricalCrossentropy(from_logits=True),
metrics=[SparseCategoricalAccuracy()])
return product
attack = cgf['ATTACK']['name']
original_data = cgf['DATASET']['arguments']['original_images']
adv_data = cgf['DATASET']['arguments']["adv_images"]
adv_labels = cgf['DATASET']['arguments']['adv_labels']
adv_diffs = cgf['DATASET']['arguments']['adv_diffs']
weights_path = join(model_path, filepath)
optimizer = cgf['TRAIN']['optim']['type']
loss_type = cgf['TRAIN']['loss']['type']
metric_list = list(cgf['TRAIN']['metrics'].values())
if loss_type == 'SparseCategoricalCrossentropy':
loss_type = SparseCategoricalCrossentropy(from_logits=False)
metric_list = [SparseCategoricalAccuracy()]
output_shape = 2
labels = np.reshape(labels, (-1))
model_name = cgf['MODEL']['name']
model_arguments = cgf['MODEL']['arguments']
#model = mb.model_selector(model_name, input_shape, output_shape, model_arguments)
model = tf.keras.models.load_model(weights_path)
# Preprocessing
if model_name =='resnet':
preprocessing = res_prep
data = 255*data
data = data - 122
from tensorflow.keras.losses import SparseCategoricalCrossentropy
from tensorflow.keras.metrics import SparseCategoricalAccuracy
from tensorflow.keras.datasets import mnist
(train_x, train_y), (test_x, test_y) = mnist.load_data()
train_x, test_x = train_x / 255.0, test_x / 255.0
print('train data:', train_x.shape, train_y.shape)
print('test data: ', test_x.shape, test_y.shape)
model = Sequential()
model.add(Flatten())
model.add(
Dense(128,
activation='relu',
kernel_regularizer=tf.keras.regularizers.L2()))
model.add(Dense(10, activation='softmax'))
model.compile(optimizer=Adam(),
loss=SparseCategoricalCrossentropy(from_logits=False),
metrics=[SparseCategoricalAccuracy()])
model.fit(train_x,
train_y,
batch_size=32,
epochs=10,
validation_data=(test_x, test_y),
validation_freq=5)
model.summary()
def call(self, inputs):
x, a = inputs
x = self.conv1([x, a])
x = self.conv2([x, a])
output = self.flatten(x)
output = self.fc1(output)
output = self.fc2(output)
return output
# Create model
model = Net()
optimizer = Adam()
loss_fn = SparseCategoricalCrossentropy()
acc_fn = SparseCategoricalAccuracy()
# Training function
@tf.function
def train_on_batch(inputs, target):
with tf.GradientTape() as tape:
predictions = model(inputs, training=True)
loss = loss_fn(target, predictions) + sum(model.losses)
acc = acc_fn(target, predictions)
gradients = tape.gradient(loss, model.trainable_variables)
optimizer.apply_gradients(zip(gradients, model.trainable_variables))
return loss, acc
# Add classification layer
self.fc2 = Dense(18, activation='softmax')
def call(self, inputs):
x = self.base_model(inputs)
x = self.pool(x)
x = self.fc2(x)
return x
optimizer = SGD(learning_rate=params['learning_rate'],
momentum=params['momentum'])
loss_object = SparseCategoricalCrossentropy()
# Define our metrics
train_loss = Mean('train_loss', dtype=tf.float32)
train_accuracy = SparseCategoricalAccuracy('train_accuracy')
eval_loss = Mean('eval_loss', dtype=tf.float32)
eval_accuracy = SparseCategoricalAccuracy('eval_accuracy')
current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
train_log_dir = 'Exc_2/logs/gradient_tape/' + current_time + '/train'
test_log_dir = 'Exc_2/logs/gradient_tape/' + current_time + '/test'
img_train_log_dir = 'Exc_2/logs/gradient_tape/' + current_time + '/img_train'
img_eval_log_dir = 'Exc_2/logs/gradient_tape/' + current_time + '/img_eval'
@tf.function
def train_step(sample, model):
with tf.GradientTape() as tape:
features, labels = sample
logits = model(features, training=True)
class SBVAT(SemiSupervisedModel):
"""
Implementation of sample-based Batch Virtual Adversarial Training
Graph Convolutional Networks (SBVAT).
`Batch Virtual Adversarial Training for Graph Convolutional Networks
<https://arxiv.org/abs/1902.09192>`
Tensorflow 1.x implementation: <https://github.com/thudzj/BVAT>
"""
def __init__(self,
*graph,
n_samples=50,
adj_transform="normalize_adj",
attr_transform=None,
device='cpu:0',
seed=None,
name=None,
**kwargs):
"""Create a sample-based Batch Virtual Adversarial Training
Graph Convolutional Networks (SBVAT) model.
This can be instantiated in several ways:
model = SBVAT(graph)
with a `graphgallery.data.Graph` instance representing
A sparse, attributed, labeled graph.
model = SBVAT(adj_matrix, attr_matrix, labels)
where `adj_matrix` is a 2D Scipy sparse matrix denoting the graph,
`attr_matrix` is a 2D Numpy array-like matrix denoting the node
attributes, `labels` is a 1D Numpy array denoting the node labels.
Parameters:
----------
graph: An instance of `graphgallery.data.Graph` or a tuple (list) of inputs.
A sparse, attributed, labeled graph.
n_samples (Positive integer, optional):
The number of sampled subset nodes in the graph where the length of the
shortest path between them is at least `4`. (default :obj: `50`)
adj_transform: string, `transform`, or None. optional
How to transform the adjacency matrix. See `graphgallery.transforms`
(default: :obj:`'normalize_adj'` with normalize rate `-0.5`.
i.e., math:: \hat{A} = D^{-\frac{1}{2}} A D^{-\frac{1}{2}})
attr_transform: string, `transform`, or None. optional
How to transform the node attribute matrix. See `graphgallery.transforms`
(default :obj: `None`)
device: string. optional
The device where the model is running on. You can specified `CPU` or `GPU`
for the model. (default: :str: `CPU:0`, i.e., running on the 0-th `CPU`)
seed: interger scalar. optional
Used in combination with `tf.random.set_seed` & `np.random.seed`
& `random.seed` to create a reproducible sequence of tensors across
multiple calls. (default :obj: `None`, i.e., using random seed)
name: string. optional
Specified name for the model. (default: :str: `class.__name__`)
kwargs: other customized keyword Parameters.
"""
super().__init__(*graph, device=device, seed=seed, name=name, **kwargs)
self.adj_transform = T.get(adj_transform)
self.attr_transform = T.get(attr_transform)
self.n_samples = n_samples
self.process()
def process_step(self):
graph = self.graph
adj_matrix = self.adj_transform(graph.adj_matrix)
attr_matrix = self.attr_transform(graph.attr_matrix)
self.neighbors = find_4o_nbrs(adj_matrix)
self.feature_inputs, self.structure_inputs = T.astensors(
attr_matrix, adj_matrix, device=self.device)
# use decorator to make sure all list arguments have the same length
@EqualVarLength()
def build(self,
hiddens=[16],
activations=['relu'],
dropout=0.5,
lr=0.01,
l2_norm=5e-4,
use_bias=False,
p1=1.,
p2=1.,
n_power_iterations=1,
epsilon=0.03,
xi=1e-6):
with tf.device(self.device):
x = Input(batch_shape=[None, self.graph.n_attrs],
dtype=self.floatx,
name='attr_matrix')
adj = Input(batch_shape=[None, None],
dtype=self.floatx,
sparse=True,
name='adj_matrix')
index = Input(batch_shape=[None],
dtype=self.intx,
name='node_index')
GCN_layers = []
dropout_layers = []
for hidden, activation in zip(hiddens, activations):
GCN_layers.append(
GraphConvolution(
hidden,
activation=activation,
use_bias=use_bias,
kernel_regularizer=regularizers.l2(l2_norm)))
dropout_layers.append(Dropout(rate=dropout))
GCN_layers.append(
GraphConvolution(self.graph.n_classes, use_bias=use_bias))
self.GCN_layers = GCN_layers
self.dropout_layers = dropout_layers
logit = self.forward(x, adj)
output = Gather()([logit, index])
model = Model(inputs=[x, adj, index], outputs=output)
self.model = model
self.train_metric = SparseCategoricalAccuracy()
self.test_metric = SparseCategoricalAccuracy()
self.loss_fn = SparseCategoricalCrossentropy(from_logits=True)
self.optimizer = Adam(lr=lr)
self.p1 = p1 # Alpha
self.p2 = p2 # Beta
self.xi = xi # Small constant for finite difference
# Norm length for (virtual) adversarial training
self.epsilon = epsilon
self.n_power_iterations = n_power_iterations # Number of power iterations
def forward(self, x, adj, training=True):
h = x
for dropout_layer, GCN_layer in zip(self.dropout_layers,
self.GCN_layers[:-1]):
h = GCN_layer([h, adj])
h = dropout_layer(h, training=training)
h = self.GCN_layers[-1]([h, adj])
return h
@tf.function
def train_step(self, sequence):
with tf.device(self.device):
self.train_metric.reset_states()
for inputs, labels in sequence:
x, adj, index, adv_mask = inputs
with tf.GradientTape() as tape:
logit = self.forward(x, adj)
output = tf.gather(logit, index)
loss = self.loss_fn(labels, output)
entropy_loss = entropy_y_x(logit)
vat_loss = self.virtual_adversarial_loss(x,
adj,
logit=logit,
adv_mask=adv_mask)
loss += self.p1 * vat_loss + self.p2 * entropy_loss
self.train_metric.update_state(labels, output)
trainable_variables = self.model.trainable_variables
gradients = tape.gradient(loss, trainable_variables)
self.optimizer.apply_gradients(
zip(gradients, trainable_variables))
return loss, self.train_metric.result()
@tf.function
def test_step(self, sequence):
with tf.device(self.device):
self.test_metric.reset_states()
for inputs, labels in sequence:
x, adj, index, _ = inputs
logit = self.forward(x, adj, training=False)
output = tf.gather(logit, index)
loss = self.loss_fn(labels, output)
self.test_metric.update_state(labels, output)
return loss, self.test_metric.result()
def virtual_adversarial_loss(self, x, adj, logit, adv_mask):
d = tf.random.normal(shape=tf.shape(x), dtype=self.floatx)
for _ in range(self.n_power_iterations):
d = get_normalized_vector(d) * self.xi
logit_p = logit
with tf.GradientTape() as tape:
tape.watch(d)
logit_m = self.forward(x + d, adj)
dist = kl_divergence_with_logit(logit_p, logit_m, adv_mask)
grad = tape.gradient(dist, d)
d = tf.stop_gradient(grad)
r_vadv = get_normalized_vector(d) * self.epsilon
logit_p = tf.stop_gradient(logit)
logit_m = self.forward(x + r_vadv, adj)
loss = kl_divergence_with_logit(logit_p, logit_m, adv_mask)
return tf.identity(loss)
def train_sequence(self, index):
index = T.asintarr(index)
labels = self.graph.labels[index]
sequence = SBVATSampleSequence(
[self.feature_inputs, self.structure_inputs, index],
labels,
neighbors=self.neighbors,
n_samples=self.n_samples,
device=self.device)
return sequence
def test_sequence(self, index):
index = T.asintarr(index)
labels = self.graph.labels[index]
sequence = SBVATSampleSequence(
[self.feature_inputs, self.structure_inputs, index],
labels,
neighbors=self.neighbors,
n_samples=self.n_samples,
resample=False,
device=self.device)
return sequence
def predict_step(self, sequence):
with tf.device(self.device):
for inputs, _ in sequence:
x, adj, index, adv_mask = inputs
output = self.forward(x, adj, training=False)
logit = tf.gather(output, index)
if tf.is_tensor(logit):
logit = logit.numpy()
return logit