def test_update_gradient(self):
layer = Dense(10, sigmoid)
dW = np.random.randn(5, 10)
db = np.random.randn(5, 1)
layer.update_gradients(dW, db)
self.assertEqual(layer.dW.all(), dW.all())
self.assertEqual(layer.db.all(), db.all())
def generate_attention(self, input_z, input_dim, graph_dim, reuse=False):
input_dim = int(input_dim)
with tf.variable_scope('generate') as scope:
if reuse == True:
scope.reuse_variables()
self.dense1 = Dense(input_dim=FLAGS.latent_dim,
output_dim=2 * FLAGS.latent_dim,
act=tf.nn.tanh,
bias=False,
name="gene_dense_1")
self.dense2 = Dense(input_dim=2 * FLAGS.latent_dim,
output_dim=1,
act=tf.nn.sigmoid,
bias=False,
name="gene_dense_2")
self.dense3 = Dense(input_dim=FLAGS.latent_dim,
output_dim=1,
act=tf.nn.sigmoid,
bias=False,
name="gene_dense_3")
final_update = input_z[0, :] * input_z
## the element wise product to replace the current inner product with size n^2*d
for i in range(1, self.n_samples):
update_temp = input_z[i, :] * input_z
final_update = tf.concat([final_update, update_temp], axis=0)
#final_update_d1 = tf.tanh(self.dense1(final_update))
final_update_d1 = self.dense1(final_update)
reconstructions_weights = tf.nn.softmax(
self.dense2(final_update_d1))
reconstructions = reconstructions_weights * final_update
#reconstructions =tf.sigmoid(self.dense3(reconstructions))
reconstructions = tf.nn.softmax(self.dense3(reconstructions))
reconstructions = tf.reshape(reconstructions,
[self.n_samples, self.n_samples])
return reconstructions
def create(cls, n_hidden_inputs=24, n_outputs=4, n_hidden_layers=2, activation=None, use_bias=True, **kwargs):
""" Sets up a basic brain class"""
layers = [Dense("input_layer", cls.N_INPUTS, n_hidden_inputs, activation=activation, use_bias=use_bias)]
for i in range(n_hidden_layers):
layers.append(Dense("hidden_{:02d}".format(i), n_hidden_inputs, n_hidden_inputs, activation=activation, use_bias=use_bias))
layers.append(Dense("output_layer", n_hidden_inputs, n_outputs, activation=activation, use_bias=use_bias))
return cls(SequentialModel(layers), **kwargs)
def __init__(self, hidden1, hidden2, num_features, num_nodes,
features_nonzero, dropout):
super(CAN, self).__init__()
self.input_dim = num_features
self.features_nonzero = features_nonzero
self.n_samples = num_nodes
self.dropout = dropout
'''init里定义的这些layer的参数都是传到对应layer的init处'''
self.hidden1 = GraphConvolutionSparse(
input_dim=self.input_dim,
output_dim=hidden1,
dropout=self.dropout,
features_nonzero=self.features_nonzero)
self.hidden2 = Dense(input_dim=self.n_samples,
output_dim=hidden1,
sparse_inputs=True)
self.z_u_mean = GraphConvolution(input_dim=hidden1,
output_dim=hidden2,
dropout=self.dropout)
self.z_u_log_std = GraphConvolution(input_dim=hidden1,
output_dim=hidden2,
dropout=self.dropout)
self.z_a_mean = Dense(input_dim=hidden1,
output_dim=hidden2,
dropout=self.dropout)
self.z_a_log_std = Dense(input_dim=hidden1,
output_dim=hidden2,
dropout=self.dropout)
self.reconstructions = InnerDecoder(input_dim=hidden2)
def vgg_bn():
return [
Conv2D([3, 3], 32, [1, 1, 1, 1], padding='SAME'),
Conv2DBatchNorm(32),
Activation(tf.nn.relu),
Conv2D([3, 3], 32, [1, 1, 1, 1], padding='SAME'),
Conv2DBatchNorm(32),
Activation(tf.nn.relu),
Conv2D([3, 3], 64, [1, 2, 2, 1]),
Conv2DBatchNorm(64),
Activation(tf.nn.relu),
Conv2D([3, 3], 64, [1, 1, 1, 1], padding='SAME'),
Conv2DBatchNorm(64),
Activation(tf.nn.relu),
Conv2D([3, 3], 128, [1, 2, 2, 1]),
Conv2DBatchNorm(128),
Activation(tf.nn.relu),
Conv2D([3, 3], 128, [1, 1, 1, 1], padding='SAME'),
Conv2DBatchNorm(128),
Activation(tf.nn.relu),
Flatten(),
Dense(128),
Activation(tf.sigmoid),
Dropout(0.5),
Dense(10),
Activation(tf.nn.softmax),
]
def _build_high_layers(self,args):
#RS
use_inner_product = True
if use_inner_product:
self.scores = tf.reduce_sum(self.user_embeddings*self.item_embeddings,axis=1)
else:
self.user_item_concat = tf.concat([self.user_embeddings,self.item_embeddings],axis=1)
for _ in range(args.H - 1):
rs_mlp = Dense(input_dim = args.dim * 2 , output_dim = args.dim * 2)
self.user_item_concat = rs_mlp(self.user_item_concat)
self.vars_rs.extend(rs_mlp.vars)
rs_pred_mlp = Dense(input_dim=args.dim * 2,output_dim=1)
self.scores = tf.squeeze(rs_pred_mlp(self.user_item_concat))
self.vars_rs.extend(rs_pred_mlp)
self.scores_normalized = tf.nn.sigmoid(self.scores)
#KGE
self.head_relation_concat = tf.concat([self.head_embeddings,self.relation_embeddings],axis=1)
for _ in range(args.H - 1):
kge_mlp = Dense(input_dim=args.dim * 2,output_dim = args.dim * 2)
self.head_relation_concat = kge_mlp(self.head_relation_concat)
self.vars_kge.extend(kge_mlp.vars)
kge_pred_mlp = Dense(input_dim=args.dim * 2,output_dim = args.dim)
self.tail_pred = kge_pred_mlp(self.head_relation_concat)
self.vars_kge.extend(kge_pred_mlp.vars)
self.tail_pred = tf.nn.sigmoid(self.tail_pred)
self.scores_kge = tf.nn.sigmoid(tf.reduce_sum(self.tail_embeddings * self.tail_pred,axis=1))
self.rmse = tf.reduce_mean(tf.sqrt(tf.reduce_sum(tf.square(self.tail_embeddings - self.tail_pred),axis=1) / args.dim))
def __init__(self, args, n_users, n_items, n_entities, n_relations):
super(MKR, self).__init__()
self.n_user = n_users
self.n_item = n_items
self.n_entity = n_entities
self.n_relation = n_relations
self.L = args.L
self.H = args.H
self.dim = args.dim
# 定义embedding矩阵
self.user_emb_matrix = nn.Embedding(n_users, args.dim)
self.item_emb_matrix = nn.Embedding(n_items, args.dim)
self.entity_emb_matrix = nn.Embedding(n_entities, args.dim)
self.relation_emb_matrix = nn.Embedding(n_relations, args.dim)
# 定义网络
self.user_mlps, self.tail_mlps, self.cc_units = [], [], []
self.kge_mlps = []
for _ in range(args.L):
self.user_mlps.append(Dense(args.dim, args.dim))
self.tail_mlps.append(Dense(args.dim, args.dim))
self.cc_units.append(CrossCompressUnit(args.dim))
for _ in range(args.H):
self.kge_mlps.append(Dense(args.dim * 2, args.dim * 2))
self.kge_pred_mlp = Dense(args.dim * 2, args.dim)
self.sigmoid = nn.Sigmoid()
def _build_low_layers(self, args):
self.user_emb_matrix = tf.get_variable('user_emb_matrix',
[self.n_user, args.dim])
self.item_emb_matrix = tf.get_variable('item_emb_matrix',
[self.n_item, args.dim])
self.entity_emb_matrix = tf.get_variable('entity_emb_matrix',
[self.n_entity, args.dim])
self.relation_emb_matrix = tf.get_variable('relation_emb_matrix',
[self.n_relation, args.dim])
# [batch_size, dim]
self.user_embeddings = tf.nn.embedding_lookup(self.user_emb_matrix,
self.user_indices)
self.item_embeddings = tf.nn.embedding_lookup(self.item_emb_matrix,
self.item_indices)
self.head_embeddings = tf.nn.embedding_lookup(self.entity_emb_matrix,
self.head_indices)
self.relation_embeddings = tf.nn.embedding_lookup(
self.relation_emb_matrix, self.relation_indices)
self.tail_embeddings = tf.nn.embedding_lookup(self.entity_emb_matrix,
self.tail_indices)
for _ in range(args.L):
user_mlp = Dense(input_dim=args.dim, output_dim=args.dim)
tail_mlp = Dense(input_dim=args.dim, output_dim=args.dim)
cc_unit = CrossCompressUnit(args.dim)
self.user_embeddings = user_mlp(self.user_embeddings)
self.item_embeddings, self.head_embeddings = cc_unit(
[self.item_embeddings, self.head_embeddings])
self.tail_embeddings = tail_mlp(self.tail_embeddings)
self.vars_rs.extend(user_mlp.vars)
self.vars_rs.extend(cc_unit.vars)
self.vars_kge.extend(tail_mlp.vars)
self.vars_kge.extend(cc_unit.vars)
def main():
(x_train, y_train), (x_test, y_test) = mnist.load_data()
print 'Imported MNIST data: training input %s and training labels %s.' % (
x_train.shape, y_train.shape)
print 'Imported MNIST data: test input %s and test labels %s.' % (
x_test.shape, y_test.shape)
N, H, W = x_train.shape
x = x_train.reshape((N, H * W)).astype('float') / 255
y = to_categorical(y_train, num_classes=10)
model = Sequential()
model.add(Dense(), ReLU(), layer_dim=(28 * 28, 300), weight_scale=1e-2)
model.add(Dense(), ReLU(), layer_dim=(300, 100), weight_scale=1e-2)
model.add(Dense(), Softmax(), layer_dim=(100, 10), weight_scale=1e-2)
model.compile(optimizer=GradientDescent(learning_rate=1e-2),
loss_func=categorical_cross_entropy)
model.fit(x, y, epochs=10, batch_size=50, verbose=False)
N, H, W = x_test.shape
x = x_test.reshape((N, H * W)).astype('float') / 255
y = to_categorical(y_test, num_classes=10)
model.evaluate(x, y)
def test_dense_layer_NUMERICAL_GRADIENT_CHECK(self):
x = np.linspace(-1, 1, 10 * 32).reshape([10, 32])
l = Dense(32, 64)
numeric_grads = eval_numerical_gradient(lambda x: l.forward(x).sum(),
x)
grads = l.backward(x, np.ones([10, 64]), optim='gd', lr=0)
self.assertTrue(np.allclose(grads, numeric_grads, rtol=1e-5, atol=0),
msg="input gradient does not match numeric grad")
def __init__(self,
input_dim,
output_dim,
neigh_input_dim=None,
dropout=0,
bias=True,
act=tf.nn.relu,
name=None,
concat=False,
mode="train",
**kwargs):
super(AttentionAggregator, self).__init__(**kwargs)
self.dropout = dropout
self.bias = bias
self.act = act
self.concat = concat
self.mode = mode
if name is not None:
name = '/' + name
else:
name = ''
if neigh_input_dim == None:
neigh_input_dim = input_dim
self.input_dim = input_dim
self.output_dim = output_dim
with tf.variable_scope(self.name + name + '_vars'):
if self.bias:
self.vars['bias'] = zeros([self.output_dim], name='bias')
self.q_dense_layer = Dense(input_dim=input_dim,
output_dim=input_dim,
bias=False,
sparse_inputs=False,
name="q")
self.k_dense_layer = Dense(input_dim=input_dim,
output_dim=input_dim,
bias=False,
sparse_inputs=False,
name="k")
self.v_dense_layer = Dense(input_dim=input_dim,
output_dim=input_dim,
bias=False,
sparse_inputs=False,
name="v")
self.output_dense_layer = Dense(input_dim=input_dim,
output_dim=output_dim,
bias=False,
sparse_inputs=False,
name="output_transform")
def __init__(self, inputDim=1, outputDim=1, optimizer=Adam()):
self.inputDim = inputDim
self.outputDim = outputDim
self.mean = Dense(self.inputDim,
self.outputDim,
activation=Identity(),
optimizer=copy.copy(optimizer))
self.logVar = Dense(self.inputDim,
self.outputDim,
activation=Identity(),
optimizer=copy.copy(optimizer))
def __init__(self, dmemory, daddress, nstates, dinput, doutput):
self.layers = {}
self.layers['INPUT'] = Dense(dinput, dmemory)
self.layers['PREVIOUS_READ'] = Dense(dmemory, dmemory)
self.layers['CONTROL_KEY'] = LSTM(dmemory + dmemory, nstates)
self.layers['OUTPUT'] = Dense(doutput, doutput)
self.daddress = daddress
self.dmemory = dmemory
self.doutput = doutput
def test_dense_layer_FORWARD(self):
layer = Dense(3, 4)
x = np.linspace(-1, 1, 2 * 3).reshape([2, 3])
layer.weights = np.linspace(-1, 1, 3 * 4).reshape([3, 4])
layer.biases = np.linspace(-1, 1, 4)
self.assertTrue(
np.allclose(
layer.forward(x),
np.array([[0.07272727, 0.41212121, 0.75151515, 1.09090909],
[-0.90909091, 0.08484848, 1.07878788, 2.07272727]])))
def __init__(self, input_dim, output_dim, model_size="small", neigh_input_dim=None,
dropout=0., bias=False, act=tf.nn.relu, name=None, concat=False, **kwargs):
super(TwoMaxLayerPoolingAggregator, self).__init__(**kwargs)
self.dropout = dropout
self.bias = bias
self.act = act
self.concat = concat
if neigh_input_dim is None:
neigh_input_dim = input_dim
if name is not None:
name = '/' + name
else:
name = ''
if model_size == "small":
hidden_dim_1 = self.hidden_dim_1 = 512
hidden_dim_2 = self.hidden_dim_2 = 256
elif model_size == "big":
hidden_dim_1 = self.hidden_dim_1 = 1024
hidden_dim_2 = self.hidden_dim_2 = 512
self.mlp_layers = []
self.mlp_layers.append(Dense(input_dim=neigh_input_dim,
output_dim=hidden_dim_1,
act=tf.nn.relu,
dropout=dropout,
sparse_inputs=False,
logging=self.logging))
self.mlp_layers.append(Dense(input_dim=hidden_dim_1,
output_dim=hidden_dim_2,
act=tf.nn.relu,
dropout=dropout,
sparse_inputs=False,
logging=self.logging))
with tf.variable_scope(self.name + name + '_vars'):
self.vars['neigh_weights'] = glorot([hidden_dim_2, output_dim],
name='neigh_weights')
self.vars['self_weights'] = glorot([input_dim, output_dim],
name='self_weights')
if self.bias:
self.vars['bias'] = zeros([self.output_dim], name='bias')
if self.logging:
self._log_vars()
self.input_dim = input_dim
self.output_dim = output_dim
self.neigh_input_dim = neigh_input_dim
def build_encoder(self, optimizer, loss_function):
encoder = NeuralNetwork(optimizer=optimizer, loss=loss_function)
encoder.add(Dense(512, input_shape=(self.img_dim, ), first_layer=True))
encoder.add(Activation('leaky_relu'))
#encoder.add(BatchNormalization(momentum=0.8))
encoder.add(Dense(256))
encoder.add(Activation('leaky_relu'))
#encoder.add(BatchNormalization(momentum=0.8))
encoder.add(Dense(self.latent_dim, latent_layer=True))
return encoder
def _create_model(self, input_nodes, output_nodes, random_seed):
self.model = []
np.random.seed(random_seed)
self.model.append(Dense(input_nodes, self.hidden_layer_sizes[0]))
self.model.append(self.activation())
for i in range(len(self.hidden_layer_sizes) - 1):
np.random.seed(random_seed + (i + 1) * random_seed)
self.model.append(
Dense(self.hidden_layer_sizes[i],
self.hidden_layer_sizes[i + 1]))
self.model.append(self.activation())
self.model.append(Dense(self.hidden_layer_sizes[-1], output_nodes))
def build_decoder(self, optimizer, loss_function):
decoder = NeuralNetwork(optimizer=optimizer, loss=loss_function)
decoder.add(Dense(256, input_shape=(self.latent_dim, )))
decoder.add(Activation('leaky_relu'))
decoder.add(BatchNormalization(momentum=0.8))
decoder.add(Dense(512))
decoder.add(Activation('leaky_relu'))
decoder.add(BatchNormalization(momentum=0.8))
decoder.add(Dense(self.img_dim))
decoder.add(Activation('tanh'))
return decoder
def _build_high_layers(self, args):
# RS
use_inner_product = True
if use_inner_product:
# [batch_size]
self.scores = tf.reduce_sum(self.user_embeddings *
self.item_embeddings,
axis=1)
else:
# [batch_size, dim * 2]
self.user_item_concat = tf.concat(
[self.user_embeddings, self.item_embeddings], axis=1)
for _ in range(args.H - 1):
rs_mlp = Dense(input_dim=args.dim * 2,
output_dim=args.dim * 2,
dropout=self.dropout_param)
# [batch_size, dim * 2]
self.user_item_concat = rs_mlp(self.user_item_concat)
self.vars_rs.extend(rs_mlp.vars)
rs_pred_mlp = Dense(input_dim=args.dim * 2,
output_dim=1,
dropout=self.dropout_param)
# [batch_size]
self.scores = tf.squeeze(rs_pred_mlp(self.user_item_concat))
self.vars_rs.extend(rs_pred_mlp.vars)
self.scores_normalized = tf.nn.sigmoid(self.scores)
# KGE
# [batch_size, dim * 2]
self.head_relation_concat = tf.concat(
[self.head_embeddings, self.relation_embeddings], axis=1)
for _ in range(args.H - 1):
kge_mlp = Dense(input_dim=args.dim * 2,
output_dim=args.dim * 2,
dropout=self.dropout_param)
# [batch_size, dim]
self.head_relation_concat = kge_mlp(self.head_relation_concat)
self.vars_kge.extend(kge_mlp.vars)
kge_pred_mlp = Dense(input_dim=args.dim * 2,
output_dim=args.dim,
dropout=self.dropout_param)
# [batch_size, 1]
self.tail_pred = kge_pred_mlp(self.head_relation_concat)
self.vars_kge.extend(kge_pred_mlp.vars)
self.tail_pred = tf.nn.sigmoid(self.tail_pred)
self.scores_kge = tf.nn.sigmoid(
tf.reduce_sum(self.tail_embeddings * self.tail_pred, axis=1))
def model_builder(n_inputs, n_outputs):
model = NeuralNetwork(optimizer=Adam(), loss=CrossEntropy)
model.add(Dense(64, input_shape=(n_inputs, )))
model.add(Activation('relu'))
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dense(64))
model.add(Activation('relu'))
model.add(Dense(1))
model.add(Activation('linear'))
return model
def load(self, name):
modelDir = f"./models/{name}"
layerDir = [
dir for dir in os.listdir(modelDir)
if os.path.isdir(os.path.join(modelDir, dir))
]
layerDir.sort(key=lambda x: int(x.strip("layer")))
for dir in layerDir:
layerFolder = os.path.join(modelDir, dir)
if "dense.json" in os.listdir(layerFolder):
# this is a dense layer
newLayer = Dense()
newLayer.load(layerFolder)
self.layers.append(newLayer)
class Sampler():
def __init__(self, inputDim=1, outputDim=1, optimizer=Adam()):
self.inputDim = inputDim
self.outputDim = outputDim
self.mean = Dense(self.inputDim,
self.outputDim,
activation=Identity(),
optimizer=copy.copy(optimizer))
self.logVar = Dense(self.inputDim,
self.outputDim,
activation=Identity(),
optimizer=copy.copy(optimizer))
def feedforward(self, input):
self.latentMean = self.mean.feedforward(input)
self.latentLogVar = self.logVar.feedforward(input)
self.epsilon = np.random.standard_normal(size=(self.outputDim,
input.shape[1]))
self.sample = self.latentMean + np.exp(
self.latentLogVar / 2.) * self.epsilon
return self.sample
def backpropagate(self, lastGradient):
gradLogVar = {}
gradMean = {}
tmp = self.outputDim * lastGradient.shape[1]
# KL divergence gradients
gradLogVar["KL"] = (np.exp(self.latentLogVar) - 1) / (2 * tmp)
gradMean["KL"] = self.latentMean / tmp
# MSE gradients
gradLogVar["MSE"] = 0.5 * lastGradient * self.epsilon * np.exp(
self.latentLogVar / 2.)
gradMean["MSE"] = lastGradient
# backpropagate gradients thorugh self.mean and self.logVar
return self.mean.backward(gradMean["KL"] +
gradMean["MSE"]) + self.logVar.backward(
gradLogVar["KL"] + gradLogVar["MSE"])
def getKLDivergence(self, output):
# output.shape[1] == batchSize
return -np.sum(1 + self.latentLogVar - np.square(self.latentMean) -
np.exp(self.latentLogVar)) / (2 * self.outputDim *
output.shape[1])
def linear_regression(a=1.0, b=0.0):
X = np.linspace(-100, 100, 200)
X = X.reshape((-1, 1))
[train_x, test_x] = split_data(X, ratio=0.8, random=True)
train_y = a * train_x + b
test_y = a * test_x + b
i = Input(1)
x = Dense(1)(i)
# define trainer
trainer = Trainer(loss='mse',
optimizer=Adam(learning_rate=0.2),
batch_size=50,
epochs=50)
# create model
model = Sequential(i, x, trainer)
model.summary()
# training process
model.fit(train_x, train_y)
# predict
y_hat = model.predict(test_x)
plt.plot(test_x, test_y, 'b')
plt.plot(test_x, y_hat, 'r')
plt.show()
def create_Dense_layer(layer_info):
if not len(layer_info) == 5:
raise RuntimeError('Dense layer must have 5 specs')
return Dense(input_dim=int(layer_info['input_dim']),
output_dim=int(layer_info['output_dim']),
dropout=parse_as_bool(layer_info['dropout']),
act=create_activation(layer_info['act']),
bias=parse_as_bool(layer_info['bias']))
def run():
file_path = os.path.dirname(
os.path.realpath(__file__)) + "/dlmb_mnist_example.json"
# If a file of the neural-net model's architexture already exists,
# then there is no need to build a new model.
if os.path.isfile(file_path):
# load the model and get its predictions based on x_test
nn_model = Sequential()
nn_model.load(file_path)
predictions = nn_model.predict(x_test)
# compare the predictions to the correct labels
print(
f"This model got a {validate_model(predictions, y_test)/100}% accuracy"
)
# If the file doesn't exist then we need to build a neural-net model and train it.
else:
# Build the neural-net model
nn_model = Sequential([
Dense(
128, 784, activation="ReLU"
), # for the layer_dim we want 128 outputs and 784 inputs (each pixel on the image)
Batchnorm(128),
Dense(128, 128, activation="ReLU"),
Batchnorm(128),
Dense(32, 128, activation="ReLU"),
Batchnorm(32),
Dense(10, 32, activation="Softmax"
) # We have 10 nodes in the layer for each number from 0 - 9
])
nn_model.build(loss="crossentropy", optimizer="adam")
# Crossentropy is a good loss function when you are doing logistic regression (classification)
# Adam is one of the most popular optimizers
nn_model.train(x_train, y_train, epochs=10, batch_size=1000)
# Train the model
# We go through the data 10 times and split the data of 60000 samples into 1000 sized batches leaving 60 samples
# Now we save the model so we can use it again without re-training
nn_model.save(file_path) # When saving, files must end in .json
def _build(self):
self.layers.append(
Dense(input_dim=self.input_dim,
output_dim=FLAGS.hidden1,
placeholders=self.placeholders,
act=tf.nn.relu,
dropout=True,
sparse_inputs=True,
logging=self.logging))
self.layers.append(
Dense(input_dim=FLAGS.hidden1,
output_dim=self.output_dim,
placeholders=self.placeholders,
act=lambda x: x,
dropout=True,
logging=self.logging))
def vgg_bn():
return [
#1
Conv2D([7, 7], 64, [1, 3, 3, 1]),
Conv2DBatchNorm(64),
Activation(tf.nn.relu),
MaxPool([1,4,4,1],[1,1,1,1]),
#2
Convolutional_block(f = 3, filters = [64,64,256],s = 1),
MaxPool([1,5,5,1],[1,1,1,1]),
Dropout(0.5),
Identity_block(f = 3, filters=[64,64,256]),
Dropout(0.5),
Identity_block(f = 3, filters=[64,64,256]),
Dropout(0.5),
MaxPool([1,2,2,1],[1,1,1,1]),
#3
Convolutional_block(f = 3, filters = [128,128,512],s = 2),
Dropout(0.5),
Identity_block(f = 3, filters=[128,128,512]),
Dropout(0.5),
Identity_block(f = 3, filters=[128,128,512]),
Dropout(0.5),
MaxPool([1,2,2,1],[1,1,1,1]),
#4
Convolutional_block(f = 3, filters = [256,256,1024],s = 2),
Identity_block(f = 3, filters=[256,256,1024]),
Identity_block(f = 3, filters=[256,256,1024]),
Identity_block(f = 3, filters=[256,256,1024]),
Identity_block(f = 3, filters=[256,256,1024]),
Identity_block(f = 3, filters=[256,256,1024]),
Flatten(),
Dense(128),
Activation(tf.sigmoid),
Dropout(0.5),
Dense(10),
#Fully_connected(),
Activation(tf.nn.softmax),
]
def get_shapes(self):
'''Draw layer shapes.'''
shapes = []
for i, layer in enumerate(self.layers):
if 'input' in layer.name:
shapes = np.append(
shapes,
Input(name=layer.name,
position=layer.position,
output_dim=layer.output_dim,
depth=layer.depth,
flatten=layer.flatten,
output_dim_label=layer.output_dim_label).draw())
if 'dense' in layer.name:
shapes = np.append(
shapes,
Dense(name=layer.name,
position=layer.position,
output_dim=layer.output_dim,
depth=layer.depth,
activation=layer.activation,
maxpool=layer.maxpool,
flatten=layer.flatten,
output_dim_label=layer.output_dim_label).draw())
if 'conv' in layer.name:
shapes = np.append(
shapes,
Convolution(
name=layer.name,
position=layer.position,
output_dim=layer.output_dim,
depth=layer.depth,
activation=layer.activation,
maxpool=layer.maxpool,
flatten=layer.flatten,
output_dim_label=layer.output_dim_label).draw())
if 'output' in layer.name:
shapes = np.append(
shapes,
Output(name=layer.name,
position=layer.position,
output_dim=layer.output_dim,
depth=layer.depth,
output_dim_label=layer.output_dim_label).draw())
if i:
shapes = np.append(
shapes,
Funnel(prev_position=self.layers[i - 1].position,
prev_depth=self.layers[i - 1].depth,
prev_output_dim=self.layers[i - 1].output_dim,
curr_position=layer.position,
curr_depth=layer.depth,
curr_output_dim=layer.output_dim,
color=(178.0 / 255, 178.0 / 255,
178.0 / 255)).draw())
self.shapes = shapes
def test_update_parameters(self):
layer = Dense(10, sigmoid)
layer.build(20)
learning_rate = 0.1
dW = np.random.randn(10, 20)
db = np.random.randn(10, 1)
layer.update_gradients(dW, db)
new_dW = layer.dW - learning_rate * dW
new_db = layer.db - learning_rate * db
layer.update_parameters(learning_rate)
self.assertEqual(layer.dW.all(), new_dW.all())
self.assertEqual(layer.db.all(), new_db.all())
def main():
c = color_codes()
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
try:
net = load_model('/home/mariano/Desktop/test.tf')
except IOError:
x = Input([784])
x_image = Reshape([28, 28, 1])(x)
x_conv1 = Conv(filters=32,
kernel_size=(5, 5),
activation='relu',
padding='same')(x_image)
h_pool1 = MaxPool((2, 2), padding='same')(x_conv1)
h_conv2 = Conv(filters=64,
kernel_size=(5, 5),
activation='relu',
padding='same')(h_pool1)
h_pool2 = MaxPool((2, 2), padding='same')(h_conv2)
h_fc1 = Dense(1024, activation='relu')(h_pool2)
h_drop = Dropout(0.5)(h_fc1)
y_conv = Dense(10)(h_drop)
net = Model(x,
y_conv,
optimizer='adam',
loss='categorical_cross_entropy',
metrics='accuracy')
print(c['c'] + '[' + strftime("%H:%M:%S") + '] ' + c['g'] + c['b'] +
'Original (MNIST)' + c['nc'] + c['g'] + ' net ' + c['nc'] + c['b'] +
'(%d parameters)' % net.count_trainable_parameters() + c['nc'])
net.fit(mnist.train.images,
mnist.train.labels,
val_data=mnist.test.images,
val_labels=mnist.test.labels,
patience=10,
epochs=200,
batch_size=1024)
save_model(net, '/home/mariano/Desktop/test.tf')
