def Encoder(img_size, in_channel, conv_channel, filter_size, latent_dim,
dense_size, bn):
inner_conv_channel = conv_channel // 2
if img_size % 4 != 0:
print("WARNING: image size mod 4 != 0, may produce bug.")
# total input number of the input of the last conv layer, new image size = old / 2 / 2
flatten_img_size = inner_conv_channel * img_size / 4 * img_size / 4
# explain: first two layer's padding = 2, because we set W/S = W/S + floor((-F+2P)/S+1), S=2,F=5,so P=2
if VERBOSE:
print(img_size, in_channel, conv_channel, filter_size, latent_dim, bn)
model = nn.Sequential(
layers.ConvLayer(in_channel,
conv_channel,
filter_size,
stride=2,
padding=2,
bn=bn),
layers.ConvLayer(conv_channel,
inner_conv_channel,
filter_size,
stride=2,
padding=2,
bn=bn),
layers.ConvLayer(inner_conv_channel,
inner_conv_channel,
filter_size,
stride=1,
padding=2,
bn=bn), layers.Flatten(),
layers.Dense(flatten_img_size, dense_size),
layers.Dense(dense_size, latent_dim))
model = model.to(device=device, dtype=dtype)
model = torch.nn.DataParallel(model, device_ids=GPU_IDs)
return model
def __init__(self,
input_size,
output_size,
hidden_layers_sizes,
loss=mtr.CrossEntropyLoss(),
learn_rate=0.01,
problem='classification',
scorer=mtr.AccuracyScore()):
"""
Create multi-layer perceptron
Args:
input_size: neurons on input layer
output_size: neurons on output layer
hidden_layers_sizes: list of neurons on each hidden layer
problem: problem to solve ('classification" or 'regression')
"""
layer_sizes = [input_size] + hidden_layers_sizes + [output_size]
layers = []
for i in range(len(layer_sizes) - 2):
layers.append(
lrs.Dense(layer_sizes[i], layer_sizes[i + 1], learn_rate))
layers.append(lrs.LeakyReLU())
layers.append(lrs.Dense(layer_sizes[-2], layer_sizes[-1], learn_rate))
if problem == 'classification':
layers.append(lrs.Softmax())
super().__init__(layers, loss, scorer)
def Classifier(input_dim, dense_size, s_classes, bn):
model = nn.Sequential(layers.Dense(input_dim, dense_size, bn=bn),
layers.Dense(dense_size, dense_size, bn=bn),
layers.Dense(dense_size, s_classes, bn=bn))
model = model.to(device=device, dtype=dtype)
model = torch.nn.DataParallel(model, device_ids=GPU_IDs)
return model
def __init__(self):
# Layers
self.conv1 = l.Convolution("conv1",3,6,5,1)
self.conv2 = l.Convolution("conv2",6,16,5,1)
self.relu = l.ReLU("relu")
self.pool = l.Maxpooling("pooling",2,2)
self.dense1 = l.Dense("dense1",16*5*5,120)
self.dense2 = l.Dense("dense1",120,84)
self.dense3 = l.Dense("dense1",84,10)
def _build_encoder(self, x):
dim_mult = 2 if self.concat else 1
h = layers.Dense(dim_mult* self.dims[-1], dim_mult* self.dims[-1],
dropout=self.placeholders['dropout'], act=tf.nn.relu)(x)
mu = layers.Dense(dim_mult* self.dims[-1], dim_mult* self.dims[-1],
dropout=self.placeholders['dropout'])(h)
log_sigma_squared = layers.Dense(dim_mult*self.dims[-1], dim_mult* self.dims[-1],
dropout=self.placeholders['dropout'])(h)
sigma_squared = tf.exp(log_sigma_squared)
sigma = tf.sqrt(sigma_squared)
return mu, log_sigma_squared, sigma_squared, sigma
def _build(self):
self.layers.append(layers.Dense(input_dim=self.input_dim,
output_dim=self.dims[1],
act=tf.nn.relu,
dropout=self.placeholders['dropout'],
sparse_inputs=False,
logging=self.logging))
self.layers.append(layers.Dense(input_dim=self.dims[1],
output_dim=self.output_dim,
act=lambda x: x,
dropout=self.placeholders['dropout'],
logging=self.logging))
def build(self):
#sample Sample neighbors to be the supportive fields for placeholders["batch1"]
samples1, support_sizes1 = self.sample(self.inputs1, self.layer_infos)
num_samples = [layer_info.num_samples for layer_info in self.layer_infos]
#Return The hidden representatio at the final layer for all nodes in batch placeholders["batch1"]
self.outputs1, self.aggregators = self.aggregate(samples1, [self.features], self.dims, num_samples,
support_sizes1, concat=self.concat, model_size=self.model_size)
dim_mult = 2 if self.concat else 1
self.outputs1 = tf.nn.l2_normalize(self.outputs1, 1)
dim_mult = 2 if self.concat else 1
self.node_pred = layers.Dense(dim_mult*self.dims[-1], self.num_classes,
dropout=self.placeholders['dropout'],
act=lambda x : x)
# TF graph management
self.node_preds = self.node_pred(self.outputs1)
self._loss()
grads_and_vars = self.optimizer.compute_gradients(self.loss)
clipped_grads_and_vars = [(tf.clip_by_value(grad, -5.0, 5.0) if grad is not None else None, var)
for grad, var in grads_and_vars]
self.grad, _ = clipped_grads_and_vars[0]
self.opt_op = self.optimizer.apply_gradients(clipped_grads_and_vars)
self.preds = self.predict()
def build(self):
'''start SAMPLE'''
samples1, support_sizes1 = self.sample(self.inputs1, self.layer_infos)
num_samples = [
layer_info.num_samples for layer_info in self.layer_infos
]
'''start AGGREGATE'''
self.outputs1, self.aggregators = self.aggregate(
samples1, [self.features],
self.dims,
num_samples,
support_sizes1,
concat=self.concat,
model_size=self.model_size)
dim_mult = 2 if self.concat else 1
self.outputs1 = tf.nn.l2_normalize(self.outputs1, 1)
dim_mult = 2 if self.concat else 1
self.node_pred = layers.Dense(dim_mult * self.dims[-1],
self.num_classes,
dropout=self.placeholders['dropout'],
act=lambda x: x)
# TF graph management
self.node_preds = self.node_pred(self.outputs1)
self._loss()
grads_and_vars = self.optimizer.compute_gradients(self.loss)
clipped_grads_and_vars = [
(tf.clip_by_value(grad, -5.0, 5.0) if grad is not None else None,
var) for grad, var in grads_and_vars
]
self.grad, _ = clipped_grads_and_vars[0]
self.opt_op = self.optimizer.apply_gradients(clipped_grads_and_vars)
self.preds = self.predict()
def run_classification():
model = Model.Model()
model.add_layer(layers.Input(784))
model.add_layer(layers.Dense(100, activation=af.relu))
model.add_layer(layers.Dense(10, activation=af.softmax))
model.compile(losses.crossentropy, optimizers.Adam())
with gzip.open('data/mnist.pkl.gz', 'rb') as f:
train_set, validation_set, test_set = pickle.load(f, encoding='latin1')
n_train = train_set[0].shape[0]
n_test = test_set[0].shape[0]
train_set_onehots = helpers.make_onehot_2d(train_set[1], 10)
test_set_onehots = helpers.make_onehot_2d(test_set[1], 10)
model.fit(train_set[0], train_set_onehots, 50, 50, metric_dataset_x=test_set[0], metric_dataset_y=test_set_onehots,
metric_callback=classification_metric_accuracy)
def load(self, file_path="./saved_dlmb_model.json"):
"""
Goes to the file where the model has been saved and retrieves the data.
Arguments:
file_path : str : A file path where the .json file has been saved.
"""
layers = {
"Dense":la.Dense(0, 0),
"Batchnorm":la.Batchnorm(0),
"Dropout":la.Dropout()
}
# Try to open the file at file_path.
try:
with open(file_path, "r") as json_file:
model_layers = json.loads(json_file.read())
for i in range(len(model_layers)):
layer_data = model_layers["layer: %s" % i]
new_layer = copy.copy(layers[layer_data["name"]])
new_layer.load(layer_data)
self.model.append(new_layer)
# Gets called if the program can't find the file_path.
except Exception as e:
raise FileNotFoundError("Can't find file path %s. Try saving the model or enter a correct file path." % file_path)
def _build_encoder(self, x):
dim_mult = 2 if self.concat else 1
fc1 = layers.Dense(dim_mult * self.dims[-1],
dim_mult * self.dims[-1],
dropout=self.placeholders['dropout'],
act=tf.nn.relu)
fc2 = layers.Dense(dim_mult * self.dims[-1],
dim_mult * self.dims[-1],
dropout=self.placeholders['dropout'])
fc3 = layers.Dense(dim_mult * self.dims[-1],
dim_mult * self.dims[-1],
dropout=self.placeholders['dropout'])
h = fc1(x)
mu = fc2(h)
log_sigma_squared = fc3(h)
sigma_squared = tf.exp(log_sigma_squared)
sigma = tf.sqrt(sigma_squared)
return mu, log_sigma_squared, sigma_squared, sigma, [fc1, fc2, fc3]
def Decoder(s_dim, z_dim, img_size, img_channel, conv_channel, filter_size,
dense_size, bn):
# TODO
# essentially the mirror version of Encoder
inner_conv_channel = conv_channel // 2
back_img_size = img_size // 4
flatten_img_size = inner_conv_channel * back_img_size * back_img_size
input_dim = s_dim + z_dim
pad = int(
np.floor(filter_size /
2)) # chose pad this way to fullfill floor((-F+2P)/1+1)==0
model = nn.Sequential(
layers.Dense(input_dim, dense_size),
layers.Dense(dense_size,
inner_conv_channel * back_img_size * back_img_size),
layers.Reshape((-1, inner_conv_channel, back_img_size, back_img_size)),
layers.ConvLayer(inner_conv_channel,
inner_conv_channel,
filter_size,
stride=1,
padding=pad,
bn=bn,
upsampling=True),
layers.ConvLayer(inner_conv_channel,
conv_channel,
filter_size,
stride=1,
padding=pad,
bn=bn,
upsampling=True),
layers.ConvLayer(conv_channel,
img_channel,
filter_size,
stride=1,
padding=pad,
bn=bn,
upsampling=False),
)
model = model.to(device=device, dtype=dtype)
model = torch.nn.DataParallel(model, device_ids=GPU_IDs)
return model
def run_regression():
df = np.array(pd.read_csv('data/Dataset/Training/Features_Variant_1.csv'))
model = Model.Model()
model.add_layer(layers.Input(53))
model.add_layer(layers.Dense(20, activation=af.relu))
model.add_layer(layers.Dense(1, activation=af.sigmoid))
model.compile(losses.mse, optimizers.SGD())
input_set = np.array([x[:-1] for x in df])
output_set = np.array([x[-1] for x in df]).reshape(len(input_set), 1)
# Model.save_model(model, "test")
# tmp = Model.load_model("test")
# tmp.fit(input_set, output_set, 50, 50, metric_callback=regression_metric_mse)
input_set = helpers.standard_scaler(input_set)
output_set = helpers.standard_scaler(output_set)
np.seterr(all="raise")
model.fit(input_set, output_set, 50, 100, metric_callback=regression_metric_mse)
Model.save_model(model,"SGD")
def _build_encoder(self, inputs, hiddens, fc1=None, fc2=None, fc3=None):
dim_mult = 2 if self.concat else 1
if fc1 is None:
fc1 = layers.Dense(2 * dim_mult * self.dims[-1],
dim_mult * self.dims[-1],
dropout=self.placeholders['dropout'],
act=tf.nn.relu)
if fc2 is None:
fc2 = layers.Dense(dim_mult * self.dims[-1],
dim_mult * self.dims[-1],
dropout=self.placeholders['dropout'])
if fc3 is None:
fc3 = layers.Dense(dim_mult * self.dims[-1],
dim_mult * self.dims[-1],
dropout=self.placeholders['dropout'])
x = tf.concat([inputs, hiddens], axis=-1)
h = fc1(x)
mu = fc2(h)
log_sigma_squared = fc3(h)
sigma_squared = tf.exp(log_sigma_squared)
sigma = tf.sqrt(sigma_squared)
return mu, log_sigma_squared, sigma_squared, sigma, fc1, fc2, fc3
def build(self):
samples1_list, support_sizes1_list = [], []
for r_idx in range(self.num_relations):
samples1, support_sizes1 = self.sample(self.inputs1,
self.layer_infos[r_idx])
samples1_list.append(samples1)
support_sizes1_list.append(support_sizes1)
num_samples = [
layer_info.num_samples for layer_info in self.layer_infos[0]
]
self.outputs1_list = []
dim_mult = 2 if self.concat else 1 # multiplication to get the correct output dimension
dim_mult = dim_mult * 2
for r_idx in range(self.num_relations):
outputs1, self.aggregators = self.aggregate(
samples1_list[r_idx], [self.features],
self.dims,
num_samples,
support_sizes1,
concat=self.concat,
model_size=self.model_size)
self.relation_batch = tf.tile(
[tf.nn.embedding_lookup(self.relation_vectors, r_idx)],
[self.batch_size, 1])
outputs1 = tf.concat([outputs1, self.relation_batch], 1)
self.attention_weights = tf.matmul(outputs1, self.attention_vec)
self.attention_weights = tf.tile(self.attention_weights,
[1, dim_mult * self.dims[-1]])
outputs1 = tf.multiply(self.attention_weights, outputs1)
self.outputs1_list += [outputs1]
# self.outputs1 = tf.reduce_mean(self.outputs1_list, 0)
self.outputs1 = tf.stack(self.outputs1_list, 1)
self.outputs1 = tf.reduce_sum(self.outputs1, axis=1, keepdims=False)
self.outputs1 = tf.nn.l2_normalize(self.outputs1, 1)
self.node_pred = layers.Dense(dim_mult * self.dims[-1],
self.num_classes,
dropout=self.placeholders['dropout'],
act=lambda x: x)
# TF graph management
self.node_preds = self.node_pred(self.outputs1)
self._loss()
grads_and_vars = self.optimizer.compute_gradients(self.loss)
clipped_grads_and_vars = [
(tf.clip_by_value(grad, -5.0, 5.0) if grad is not None else None,
var) for grad, var in grads_and_vars
]
self.grad, _ = clipped_grads_and_vars[0]
self.opt_op = self.optimizer.apply_gradients(clipped_grads_and_vars)
self.preds = self.predict()
def build(self):
samples, support_size = self.sample(self.inputs, self.layer_infos)
num_samples = [
layer_info.num_samples for layer_info in self.layer_infos
]
self.hiddens, self.aggregators = self.aggregate(
samples, [self.features],
self.dims,
num_samples,
support_size,
concat=self.concat,
model_size=self.model_size)
dim_mult = 2 if self.concat else 1
self.hiddens = tf.nn.l2_normalize(self.hiddens, 1)
# VAE
self.mu, log_sigma_squared, sigma_squared, sigma = self._build_encoder(
self.hiddens)
self.z = tf.random_normal([dim_mult * self.dims[-1]],
mean=self.mu,
stddev=sigma)
self.regular_vae = -0.5 * tf.reduce_sum(
1 + log_sigma_squared - tf.square(self.mu) - sigma_squared, 1)
self.node_pred = layers.Dense(dim_mult * self.dims[-1],
self.num_classes,
dropout=self.placeholders['dropout'],
act=tf.nn.sigmoid)
self.outputs = self.node_pred(self.z)
self._loss()
grads_and_vars = self.optimizer.compute_gradients(self.loss)
clipped_grads_and_vars = [
(tf.clip_by_value(grad, -5.0, 5.0) if grad is not None else None,
var) for grad, var in grads_and_vars
]
self.grad, _ = clipped_grads_and_vars[0]
self.opt_op = self.optimizer.apply_gradients(clipped_grads_and_vars)
def build(self):
samples1, support_sizes1, attentions1, num_nz, out_mean = self.sample(
self.inputs1, self.layer_infos)
self.att = attentions1
self.out_mean = out_mean
num_samples = [
layer_info.num_samples for layer_info in self.layer_infos
]
self.outputs1, self.aggregators = self.aggregate(
samples1, [self.features],
self.dims,
num_samples,
support_sizes1,
attentions1,
num_nz,
concat=self.concat,
model_size=self.model_size)
self.outputs1 = tf.nn.l2_normalize(self.outputs1, 1)
dim_mult = 2 if self.concat else 1
self.node_pred = layers.Dense(dim_mult * self.dims[-1],
self.num_classes,
self.weight_decay,
dropout=self.placeholders['dropout'],
act=lambda x: x)
# TF graph management
self.node_preds = self.node_pred(self.outputs1)
self._loss()
# added
self.sparse_loss_to_node(samples1, support_sizes1, num_samples)
grads_and_vars = self.optimizer.compute_gradients(self.loss)
clipped_grads_and_vars = [
(tf.clip_by_value(grad, -5.0, 5.0) if grad is not None else None,
var) for grad, var in grads_and_vars
]
self.grad, _ = clipped_grads_and_vars[0]
self.opt_op = self.optimizer.apply_gradients(clipped_grads_and_vars)
self.preds = self.predict()
def define_weights(self, hidden_dims):
W = {}
for i in range(self.num_layers):
W[i] = tf.get_variable("W%s" % i,
shape=(hidden_dims[i], hidden_dims[i + 1]))
Ws_att = {} # attention weight matrix
for i in range(len(hidden_dims) - 1):
v = {}
v[0] = tf.get_variable("v%s_0" % i, shape=(hidden_dims[i + 1], 1))
v[1] = tf.get_variable("v%s_1" % i, shape=(hidden_dims[i + 1], 1))
Ws_att[i] = v
self.W, self.v = W, Ws_att
self.C = {}
self.project_layer = layers.Dense(input_dim=self.hidden_dims[-1] +
FLAGS.fm_dims,
output_dim=self.output_dim,
placeholders=self.placeholders,
act=lambda x: x,
dropout=True,
logging=self.logging,
norm=False)
def _build(self):
self.layers.append(
layers.GraphConvolution(input_dim=self.input_dim if
not self.residual else self.input_dim * 2,
output_dim=FLAGS.hidden1,
placeholders=self.placeholders,
act=tf.nn.relu,
dropout=True,
sparse_inputs=self.sparse_inputs,
logging=self.logging,
norm=self.norm,
precalc=self.precalc,
residual=self.residual))
for _ in range(self.num_layers - 1):
self.layers.append(
layers.GraphConvolution(
input_dim=FLAGS.hidden1
if not self.residual else FLAGS.hidden1 * 2,
output_dim=FLAGS.hidden1,
placeholders=self.placeholders,
act=tf.nn.relu,
dropout=True,
sparse_inputs=False,
logging=self.logging,
norm=self.norm,
precalc=False,
residual=self.residual))
self.project_layer = layers.Dense(input_dim=FLAGS.hidden1 * 2,
output_dim=self.output_dim,
placeholders=self.placeholders,
act=lambda x: x,
dropout=True,
logging=self.logging,
norm=False)
def build(self):
samples, support_size = self.sample(self.inputs, self.layer_infos)
num_samples = [
layer_info.num_samples for layer_info in self.layer_infos
]
self.hiddens, self.aggregators = self.aggregate(
samples, [self.features],
self.dims,
num_samples,
support_size,
concat=self.concat,
model_size=self.model_size)
dim_mult = 2 if self.concat else 1
self.hiddens = tf.nn.l2_normalize(self.hiddens, 1)
# Two Channel VAE
self.mu_rec, log_sigma_squared, sigma_squared, sigma_rec, self.vars_vae_rec = self._build_encoder(
self.hiddens)
self.z_rec = tf.random_normal([dim_mult * self.dims[-1]],
mean=self.mu_rec,
stddev=sigma_rec)
self.normal_rec = -0.5 * tf.reduce_sum(
1 + log_sigma_squared - tf.square(self.mu_rec) - sigma_squared, 1)
self.node_pred_rec = layers.Dense(dim_mult * self.dims[-1],
self.num_classes,
dropout=self.placeholders['dropout'],
act=None)
self.outputs_rec = self.node_pred_rec(self.z_rec)
self.mu_abn, log_sigma_squared, sigma_squared, sigma_abn, self.vars_vae_abn = self._build_encoder(
self.hiddens)
self.z_abn = tf.random_normal([dim_mult * self.dims[-1]],
mean=self.mu_abn,
stddev=sigma_abn)
self.normal_abn = -0.5 * tf.reduce_sum(
1 + log_sigma_squared - tf.square(self.mu_abn) - sigma_squared, 1)
u = tf.random_uniform(shape=(self.batch_size,
dim_mult * self.dims[-1]),
dtype=tf.float32)
self.hidden_trans = layers.Dense(dim_mult * self.dims[-1],
dim_mult * self.dims[-1],
dropout=self.placeholders['dropout'],
act=tf.nn.sigmoid)
self.s = self.hidden_trans(self.hiddens)
self.s_abn = tf.sigmoid((tf.log(self.s + 1e-20) + tf.log(u + 1e-20) -
tf.log(1 - u + 1e-20)) / self.temperature)
self.bernoulli_abn = tf.reduce_sum(
tf.log(self.s + 1e-20) + tf.log(1 - self.s + 1e-20) -
2 * tf.log(0.5), 1)
self.r_abn = tf.multiply(self.z_abn, self.s_abn)
self.node_pred_abn = layers.Dense(dim_mult * self.dims[-1],
self.num_classes,
dropout=self.placeholders['dropout'],
act=None)
self.outputs_abn = self.node_pred_abn(self.r_abn)
print(self.outputs_rec.get_shape())
self.outputs = tf.reduce_max(tf.concat([
tf.expand_dims(tf.nn.sigmoid(self.outputs_rec), -1),
tf.expand_dims(tf.nn.sigmoid(self.outputs_abn), -1)
],
axis=-1),
axis=-1)
print(self.outputs.get_shape())
self._loss()
self.output_rec = tf.nn.sigmoid(self.outputs_rec)
self.output_abn = tf.nn.sigmoid(self.outputs_abn)
grads_and_vars = self.optimizer.compute_gradients(self.loss)
clipped_grads_and_vars = [
(tf.clip_by_value(grad, -5.0, 5.0) if grad is not None else None,
var) for grad, var in grads_and_vars
]
self.grad, _ = clipped_grads_and_vars[0]
self.opt_op = self.optimizer.apply_gradients(clipped_grads_and_vars)
import numpy as np import layers import functions x = np.asarray([[1, 1, 1], [2, 2, 2], [3, 3, 3]]) a = x.shape // 2 w = x[0, :] x = np.sqrt(1e-8) / np.sqrt(1e-8) net = layers.Network(layers.Adam(0.01)) net.add(layers.Dense(2, 5, functions.LeakyReLu(), '1st hidden')) net.add(layers.Dense(5, 20, functions.LeakyReLu(), '1st hidden')) net.add(layers.Dense(20, 5, functions.LeakyReLu(), '2nd hidden')) net.add(layers.Flatten()) net.add(layers.Dense(5, 3, functions.LeakyReLu(), '3nd hidden')) net.add(layers.Dense(3, 1, functions.Sigmoid(), 'Out')) inp = np.asarray([[0, 0], [1, 0], [0, 1], [1, 1]]) shape = inp.shape x = np.reshape(inp, (8)) y = np.reshape(x, shape) print(inp) target = np.asarray([0, 1, 1, 0]) print(target)
# number of attention heads
num_heads = 2
# hidden layer size in feed forward network
ff_dim = 32
# maximum length of each sentence
max_len = splitting.max_len
# implementing layers
inputs = keras.Input(shape=(max_len, ), )
embedding_layer = TokenAndPositionEmbedding(max_len, len(preprocess.vocab)+1, embed_dim)
x = embedding_layer(inputs)
transformer_block = TransformerBlock(embed_dim, num_heads, ff_dim)
x = transformer_block(x)
x = layers.GlobalAveragePooling1D()(x)
x = layers.Dropout(0.1)(x)
x = layers.Dense(20, activation='relu')(x)
x = layers.Dropout(0.1)(x)
outputs = layers.Dense(7, activation='softmax')(x)
# transformer model
model = keras.Model(inputs=inputs, outputs=outputs)
# summary of the model
model.summary()
# training and fitting the model to the data
early_stopping = keras.callbacks.EarlyStopping()
model.compile('adam', 'sparse_categorical_crossentropy', metrics=['accuracy'])
history = model.fit(
x_train, y_train, batch_size=32, epochs=10, validation_data=(x_val, y_val), callbacks=[early_stopping]
)
# print("Predictions: ", Yhat)
matches = (Yhat == Y.reshape(self.m))
# print(matches)
return np.sum(matches) / len(matches)
if __name__ == "__main__":
nx = 10
m = 12
X = np.array(np.random.randn(nx, 12))
Y = np.array([[0, 1, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1]])
nn = NeuralNetwork()
nn.add_layer(layers.Flatten(nx))
nn.add_layer(
layers.Dense(20,
activation=ActivationFunctions.relu,
initialization="He"))
nn.add_layer(
layers.Dense(30,
activation=ActivationFunctions.relu,
initialization="He"))
nn.add_layer(
layers.Dense(1,
activation=ActivationFunctions.sigmoid,
initialization="He"))
costs = nn.train(X, Y, alpha=0.01, num_epochs=1500, print_cost=True)
accuracy_on_train = nn.get_accuracy(X, Y)
print(accuracy_on_train)
# Normalizing the data
train_image = train_image / SCALER
test_image = test_image / SCALER
X_train, X_valid, y_train, y_valid = train_valid_split(train_image,
train_label,
test_size=test_size)
N_train = X_train.shape[0]
dimensions = X_train.shape[1]
num_L1 = dimensions
# Model
model = dict()
model['L1'] = layers.Dense(input_D=num_L1, output_D=num_L2)
model['relu1'] = layers.Relu()
model['L2'] = layers.Dense(input_D=num_L2, output_D=num_L3)
model['loss'] = layers.SoftmaxCrossEntropy()
for t in range(num_epoch):
# Minibatch generate
idx_permute = np.random.permutation(N_train)
num_batches = int(N_train // batch_size)
for i in range(num_batches):
X_batch, y_batch = get_minibatch(
X_train, y_train,
idx_permute[i * batch_size:(i + 1) * batch_size])
a1 = model['L1'].forward(X_batch)
def build(self):
dim_mult = 2 if self.concat else 1
self.outputs_rec, self.outputs_abn = [], []
#print(self.placeholders["labels_abn"].get_shape().as_list())
self.hidden_states = []
for timestamp in range(self.num_steps):
print("timestamp:", timestamp)
samples, support_size = self.sample(self.inputs,
self.layer_infos,
timestamp=timestamp)
num_samples = [
layer_info.num_samples for layer_info in self.layer_infos
]
if timestamp == 0:
self.hiddens, self.aggregators = self.aggregate(
samples,
self.features[timestamp],
self.dims,
num_samples,
support_size,
concat=self.concat,
model_size=self.model_size)
else:
self.hiddens, _ = self.aggregate(samples,
self.features[timestamp],
self.dims,
num_samples,
support_size,
aggregators=self.aggregators,
concat=self.concat,
model_size=self.model_size)
self.hiddens = tf.nn.l2_normalize(self.hiddens, 1)
# Two Channel VAE
if timestamp == 0:
self.hidden_states.append(
tf.ones_like(self.hiddens, dtype=tf.float32))
self.prior_noli_rec = layers.Dense(
dim_mult * self.dims[-1],
dim_mult * self.dims[-1],
dropout=self.placeholders['dropout'],
act=tf.nn.relu)
self.prior_mu_rec = layers.Dense(
dim_mult * self.dims[-1],
dim_mult * self.dims[-1],
dropout=self.placeholders['dropout'],
act=None)
self.prior_sigma_rec = layers.Dense(
dim_mult * self.dims[-1],
dim_mult * self.dims[-1],
dropout=self.placeholders['dropout'],
act=None)
if timestamp == 0:
self.mu_rec, log_sigma_squared_rec, sigma_squared_rec, sigma_rec, self.fc1_vae_rec, self.fc2_vae_rec, self.fc3_vae_rec = self._build_encoder(
self.hiddens, self.hidden_states[-1])
else:
self.mu_rec, log_sigma_squared_rec, sigma_squared_rec, sigma_rec, fc1, fc2, fc3 = self._build_encoder(
self.hiddens,
self.hidden_states[-1],
fc1=self.fc1_vae_rec,
fc2=self.fc2_vae_rec,
fc3=self.fc3_vae_rec)
self.z = tf.random_normal([dim_mult * self.dims[-1]],
mean=self.mu_rec,
stddev=sigma_rec)
self.sigma_rec = sigma_rec
# KL divergence with prior distribution and approximate postier distribution
if timestamp == 0:
self.normal_rec = -0.5 * tf.reduce_sum(
1 + log_sigma_squared_rec - tf.square(self.mu_rec) -
sigma_squared_rec, 1)
else:
mu_pri, log_sigma_squared_pri, sigma_squared_pri, sigma_pri = self._build_prior(
self.hidden_states[-1],
fc1=self.prior_noli_rec,
fc2=self.prior_mu_rec,
fc3=self.prior_sigma_rec)
sigma_pri = tf.math.reciprocal(sigma_pri + 1e-10)
sigma_trace = tf.multiply(sigma_pri, sigma_rec)
mu_sigma = tf.multiply(mu_pri - self.mu_rec, sigma_pri)
mu_sigma = tf.multiply(sigma_pri, mu_pri - self.mu_rec)
self.normal_rec += -0.5 * tf.reduce_sum(
1 + log_sigma_squared_rec - log_sigma_squared_pri -
sigma_trace - mu_sigma, 1)
if timestamp == 0:
self.node_pred_rec = layers.Dense(
dim_mult * self.dims[-1],
self.num_classes,
dropout=self.placeholders['dropout'],
act=None)
outputs_rec = self.node_pred_rec(self.z)
u = tf.random_uniform(shape=(self.batch_size, FLAGS.num_classes),
dtype=tf.float32)
if timestamp == 0:
self.bernoulli_trans = layers.Dense(
2 * dim_mult * self.dims[-1],
FLAGS.num_classes,
dropout=self.placeholders['dropout'],
act=tf.nn.sigmoid)
self.node_pred_abn = layers.Dense(
2 * dim_mult * self.dims[-1],
FLAGS.num_classes,
dropout=self.placeholders['dropout'],
act=None)
self.s = self.bernoulli_trans(
tf.concat([self.z, self.hidden_states[-1]], axis=-1))
self.s_abn = tf.sigmoid(
(tf.log(self.s + 1e-20) + tf.log(u + 1e-20) -
tf.log(1 - u + 1e-20)) / self.temperature)
self.z_abn = self.node_pred_abn(
tf.concat([self.z, self.hidden_states[-1]], axis=-1))
self.r_abn = tf.multiply(self.z_abn, self.s_abn)
outputs_abn = self.r_abn
print(outputs_rec.get_shape())
self.outputs_rec.append(outputs_rec)
self.outputs_abn.append(outputs_abn)
#self.outputs = tf.reduce_max(tf.concat([tf.expand_dims(tf.nn.sigmoid(self.outputs_rec),-1) , tf.expand_dims(tf.nn.sigmoid(self.outputs_abn),-1) ], axis=-1), axis=-1)
# output next hidden states
if timestamp == 0:
self.hidden_trans = layers.Dense(
3 * dim_mult * self.dims[-1] + FLAGS.num_classes,
dim_mult * self.dims[-1],
dropout=self.placeholders['dropout'],
act=tf.nn.relu)
next_hidden_state = self.hidden_trans(
tf.concat([
self.hidden_states[-1], self.mu_rec, self.sigma_rec, self.s
],
axis=-1))
self.hidden_states.append(tf.nn.l2_normalize(next_hidden_state, 1))
self._loss()
self.output_rec = tf.nn.sigmoid(self.outputs_rec[-1])
self.output_abn = tf.nn.sigmoid(self.outputs_abn[-1])
self.outputs = tf.clip_by_value(self.output_abn, 1e-8, 1.0 - 1e-8)
grads_and_vars = self.optimizer.compute_gradients(self.loss)
clipped_grads_and_vars = [
(tf.clip_by_value(grad, -5.0, 5.0) if grad is not None else None,
var) for grad, var in grads_and_vars
]
self.grad, _ = clipped_grads_and_vars[0]
self.opt_op = self.optimizer.apply_gradients(clipped_grads_and_vars)
import numpy as np
import layers
import functions
net = layers.Network(layers.Adam(0.01))
net.add(layers.Dense(2, 5, functions.LeakyReLu(), '1st hidden'))
net.add(layers.Dense(5, 20, functions.LeakyReLu(), '1st hidden'))
net.add(layers.Dense(20, 5, functions.LeakyReLu(), '2nd hidden'))
net.add(layers.Dense(5, 3, functions.LeakyReLu(), '3nd hidden'))
net.add(layers.Dense(3, 2, functions.SoftMax(), 'Out'))
inp = np.asarray([[0 , 0], [1, 0], [0, 1], [1, 1]])
shape = inp.shape
x = np.reshape(inp, (8))
y = np.reshape(x, shape)
print(inp)
target = np.asarray([[1, 0],
[0, 1],
[0, 1],
[1, 0]])
print(target)
result = net.propagate(inp[0])
print(result)
print("START")
for _ in range(30):
for i, inputX in enumerate(inp):
result = net.propagate(inputX)
plt.plot(X_test, y_test)
plt.plot(X_test, predictions)
plt.show()
''' ''
EPOCHS = 10001
LEARNING_RATE = 0.05
X_train, y_train = spiral_data(samples=100, classes=3)
X_val, y_val = spiral_data(samples=100, classes=3)
model = network.NeuralNetwork()
model.add_layer(
layers.Dense(2,
64,
weight_regularizer_l2=0.000005,
bias_regularizer_l2=0.000005))
model.add_layer(activations.ReLU())
model.add_layer(layers.Dropout(rate=0.2))
model.add_layer(layers.Dense(64, 3))
model.add_layer(activations.Softmax())
model.set(loss=losses.CategoricalCrossentropy(),
optimizier=optimizers.Adam(learning_rate=LEARNING_RATE),
accuracy=metrics.CategoricalAccuracy())
model.fit(X_train, y_train, epochs=EPOCHS, validation_data=(X_val, y_val))
def autoencoder(bioma_shape=717,
domain_shape=36,
output_shape=717,
latent_space=10,
bioma_layers=[128, 64],
domain_layers=[32, 16],
input_transform=CenterLogRatio(),
output_transform=None,
activation_function_encoder=layers.ReLU(),
activation_function_decoder=layers.ReLU(),
activation_function_latent='tanh',
):
has_domain = domain_shape is not None
has_bioma = bioma_shape is not None
if not has_bioma and not has_domain:
raise Exception('Either bioma or domain has to be expecified.')
# encoder bioma
if has_bioma:
in_bioma = layers.Input(shape=(bioma_shape,), name='bioma_input_{}'.format(bioma_shape))
net = in_bioma
if input_transform is not None:
net = input_transform(net)
for s in bioma_layers:
net = layers.Dense(s, activation=activation_function_encoder,
name="encoder_bioma_dense_{}".format(s))(net)
encoded_bioma = layers.Dense(latent_space, activation=activation_function_latent,
name='encoded_bioma_{}'.format(latent_space))(net)
encoder_bioma = keras.Model(inputs=in_bioma, outputs=encoded_bioma, name='EncoderBioma')
else:
encoded_bioma = None
encoder_bioma = None
# encoder domain
if has_domain:
in_domain = layers.Input(shape=(domain_shape,), name='domain_input_{}'.format(domain_shape))
net = in_domain
for s in domain_layers:
net = layers.Dense(s, activation=activation_function_encoder,
name="encoder_domain_dense_{}".format(s))(net)
encoded_domain = layers.Dense(latent_space, activation=activation_function_latent,
name='encoded_domain_{}'.format(latent_space))(net)
encoder_domain = keras.Model(inputs=in_domain, outputs=encoded_domain, name='EncoderDomain')
else:
encoded_domain = None
encoder_domain = None
# decoder bioma for both autoencoders
in_latent_space = layers.Input(shape=(latent_space,), name='latent_space_input')
net = in_latent_space
net_bioma = encoded_bioma
net_domain = encoded_domain
for s in reversed(bioma_layers):
layer = layers.Dense(s, activation=activation_function_decoder,
name="decoder_dense_{}".format(s))
net = layer(net)
if has_bioma:
net_bioma = layer(net_bioma)
if has_domain:
net_domain = layer(net_domain)
layer = layers.Dense(output_shape, activation=None, name='decoded_bioma')
decoded_bioma = layer(net)
if has_bioma:
net_bioma = layer(net_bioma)
if has_domain:
net_domain = layer(net_domain)
if output_transform is not None:
decoded_bioma = output_transform(decoded_bioma)
if has_bioma:
net_bioma = output_transform(net_bioma)
if has_domain:
net_domain = output_transform(net_domain)
decoder_bioma = keras.Model(inputs=in_latent_space, outputs=decoded_bioma, name='DecoderBioma')
# combined model for training
if has_domain and has_bioma:
diff_encoders = tf.math.abs(encoded_domain - encoded_bioma, name='diff_encoded')
diff_encoders = Identity(name='latent')(diff_encoders)
net_bioma = Identity(name='bioma')(net_bioma)
net_domain = Identity(name='domain')(net_domain)
model = keras.Model(inputs=[in_bioma, in_domain],
outputs=[net_bioma, net_domain, diff_encoders],
name='model')
else:
if has_bioma:
net_bioma = Identity(name='bioma')(net_bioma)
model = keras.Model(inputs=[in_bioma],
outputs=[net_bioma],
name='model')
if has_domain:
net_domain = Identity(name='domain')(net_domain)
model = keras.Model(inputs=[in_domain],
outputs=[net_domain],
name='model')
return model, encoder_bioma, encoder_domain, decoder_bioma
def __init__(self, nb_feature, nb_hidden, nb_class):
self.dense1 = layers.Dense(N=nb_feature, H=nb_hidden)
self.sigmoid = layers.Sigmoid()
self.dense2 = layers.Dense(N=nb_hidden, H=nb_class)
self.softmaxce = layers.SoftmaxCE()
model = get_model()
model.train(training_data)
validation_score = model.evaluate(validation_data)
validation_scores.append(validation_score)
validation_score = np.average(validation_scores)
model = get_model()
model.train(data)
test_score = model.evaluate(test_data)
#4.4.1 ネットワークのサイズを削減する
from keras import models
form keras import layers
model = models.Sequential()
model.add(layers.Dense(16, activation='relu', input_shape=(10000,)))
model.add(layers.Dense(16, activation='relu'))
model.add(layers.Dense(1, activation='sigmoid'))
#小さくする
model = models.Sequential()
model.add(layers.Dense(4, activation='relu', input_shape=(10000,)))
model.add(layers.Dense(4, activation='relu'))
model.add(layers.Dense(1, activation='sigmoid'))
#大きくする
model =models.Sequential()
model.add(layers.Dense(512, activaion='relu', input_shape=(10000,)))
model.add(layers.Dense(512, activaion='relu'))
model.add(layers.Dense(1, activaion='sigmoid'))
