def load_pathnet(filename):
log = None
with open(filename, 'rb') as f:
log = pickle.load(f)
layers = []
for layer_log in log['layer_logs']:
if layer_log['layer_type'] == 'dense':
layers.append(DenseLayer.build_from_log(layer_log))
if layer_log['layer_type'] == 'conv':
layers.append(ConvLayer.build_from_log(layer_log))
Layer.initialize_whole_network(layers, log['in_shape'])
for layer, layer_log in zip(layers, log['layer_logs']):
layer.load_layer_log(layer_log)
pathnet = PathNet(input_shape=log['in_shape'],
width=log['width'],
depth=log['depth'])
pathnet._layers = layers
pathnet.training_counter = log['training_counter']
pathnet.max_modules_pr_layer = log['max_modules_pr_layer']
pathnet.min_modules_pr_layer = log['min_modules_pr_layer']
tasks = []
for task_log in log['task_logs']:
task = TaskContainer.build_from_log(task_log)
pathnet.path2model(pathnet.random_path(), task)
task.layer.set_weights(task_log['layer_weights'])
tasks.append(task)
pathnet._tasks = tasks
return pathnet
def __init__(self, inplanes, planes, stride=1, downsample=None):
super(BasicBlock, self).__init__()
self.conv1 = conv3x3(inplanes, planes, stride)
self.bn1 = Layer(planes)
self.relu = nn.ReLU(inplace=True)
self.conv2 = conv3x3(planes, planes)
self.bn2 = Layer(planes)
self.downsample = downsample
self.stride = stride
def __init__(self, config=None, sess=None):
self.sess = sess
self.config = config
self.num_class = self.config.num_class
self.input_height = self.config.input_height
self.input_width = self.config.input_width
self.input_channel = self.config.input_channel
self.batchsize = self.config.batchsize
self.layer = Layer()
def _build_layers(self, inputs):
self.input_layer = InputLayer(inputs)
self.layers = []
for idx, layer_params in enumerate(self.layers_params):
neurons_num, act_func = layer_params
if len(self.layers) == 0:
layer = Layer(neurons_num, self.input_layer.neurons_number,
act_func, inputs.shape[0])
else:
layer = Layer(neurons_num, self.layers[idx - 1].neurons_number,
act_func, inputs.shape[0])
self.layers.append(layer)
def add(self, layer: Layer):
if layer._built:
assert layer.input_dim == self.shape[-1], \
"cannot match the input dimensionality of %s" % layer
else:
layer.setup(self.shape[-1])
self.layers.append(layer)
self.shape.append(layer.size)
self.parameters.extend(layer.parameters.values())
if layer.activation is False:
layer.activation = self.activation
def __init__(self, embeddings, pad_mask, placeholders, net_kwargs, dropout, loss_f): out_kwargs = dict(net_kwargs) out_kwargs['in_size'], out_kwargs['out_size'] = out_kwargs['out_size']*2, \ placeholders['y'].get_shape().as_list()[1] out_kwargs['activation'] = 'sigmoid' self.output_layer = Layer(name='Encoder_Output', **out_kwargs) self.net_kwargs = net_kwargs self.embeddings = embeddings self.placeholders = placeholders self.pad_mask = pad_mask self.dropout = dropout self.loss_f = loss_f
def mnist(output_size=10):
conv_config = [{
'channels': 1,
'kernel': (3, 3),
'stride': (1, 1),
'activation': 'relu'
}]
config = [{'out': 20, 'activation': 'relu'}]
input_shape = [28, 28, 1]
output_size = output_size
depth = 3
width = 10
max_modules_pr_layer = 3
min_modules_pr_layer = 1
learning_rate = 0.0001
optimizer_type = Adam
loss = 'categorical_crossentropy'
flatten_in_unique = True
layers = []
#layers.append(DenseLayer(width, 'L0', config, flatten=not flatten_in_unique))
#layers.append(DenseLayer(width, 'L1', config))
#layers.append(DenseLayer(width, 'L2', config))
layers.append(ConvLayer(width, 'L0', conv_config))
layers.append(ConvLayer(width, 'L1', conv_config))
layers.append(ConvLayer(width, 'L2', conv_config, maxpool=True))
Layer.initialize_whole_network(layers, input_shape)
task = TaskContainer(input_shape,
output_size,
flatten_in_unique,
name='unique_mnist',
optimizer=optimizer_type,
loss=loss,
lr=learning_rate)
pathnet = PathNet(input_shape=input_shape,
width=width,
depth=depth,
max_active_modules=20)
pathnet._layers = layers
pathnet._tasks = [task]
pathnet.max_modules_pr_layer = max_modules_pr_layer
pathnet.min_modules_pr_layer = min_modules_pr_layer
for layer in pathnet._layers:
layer.save_initialized_weights()
return pathnet, task
def __init__(self,
layer_dims,
momentum=0,
beta=0,
lRate=0.45,
n_iters=5000,
activation='ReLu',
initializer='He',
GD_type='StochasticGD',
batch_size=None,
optimizer=None,
regularizer=None,
regularizer_const=0):
self.lRate = lRate
self.n_iters = n_iters
self.loss = CrossEntropy()
self.optimizer = optimizer
self.regularizer = regularizer
if regularizer != None:
self.regularizer = regularizers[regularizer](regularizer_const)
self.GD_type = None
if GD_type == 'BatchGD':
self.GD_type = GD_variants[GD_type](self.lRate)
elif GD_type == 'StochasticGD':
self.GD_type = GD_variants[GD_type](self.lRate, momentum, beta,
self.optimizer)
elif GD_type == 'MiniBatchGD':
self.GD_type = GD_variants[GD_type](self.lRate, momentum, beta,
batch_size, self.optimizer)
self.layers = []
self.n_layers = len(layer_dims) - 1
#Initializing all layers with ReLu except last
for l in range(1, self.n_layers):
layer_shape = (layer_dims[l], layer_dims[l - 1])
self.layers.append(
Layer(l, layer_shape, activation, initializer,
self.regularizer))
print(self.layers[l - 1].__str__())
layer_shape = (layer_dims[self.n_layers],
layer_dims[self.n_layers - 1])
self.layers.append(
Layer(self.n_layers, layer_shape, 'Sigmoid', 'Random',
self.regularizer))
print(self.layers[self.n_layers - 1].__str__())
def plot_accuracy():
import numpy as np
from matplotlib import pyplot as plt
train, test, vadilation = load_mnist_simple()
dnn = DNN(input=28 * 28, layers=[Layer(100, LQ), Layer(10, LCE)], eta=0.05, lmbda=1)
for l in dnn.layers:
l.w = np.random.random(l.w.shape) - 0.5
acc1 = list(dnn.learn_iter(train, epochs=20, test=vadilation))
dnn.initialize_rand()
acc2 = list(dnn.learn_iter(train, epochs=20, test=vadilation))
print(acc1)
print(acc2)
plt.plot(acc1)
plt.plot(acc2)
plt.show()
def create_layer_item(self, stroke_id, layer_name):
"""
Creates layer item in layer panel using stroke data
Args:
stroke_id (int): unique index of stroke
layer_name (str): name of stroke layer
"""
stroke_info = ['', layer_name]
layer = Layer(stroke_info, stroke_index=stroke_id)
highest_group = None
if self.layers_tree.selectedItems():
iterator = QtGui.QTreeWidgetItemIterator(self.layers_tree)
while iterator.value():
item = iterator.value()
if isinstance(item, Folder) and item in self.layers_tree.selectedItems():
highest_group = item
break
iterator += 1
if highest_group:
highest_group.insertChild(0, layer)
else:
self.layers_tree.insertTopLevelItem(0, layer)
self.update_layer_index()
def __init__(self, inplanes, planes, stride=1, downsample=None):
super(Bottleneck, self).__init__()
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
self.bn1 = Layer(planes)
self.conv2 = nn.Conv2d(planes,
planes,
kernel_size=3,
stride=stride,
padding=1,
bias=False)
self.bn2 = Layer(planes)
self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
self.bn3 = Layer(planes * 4)
self.relu = nn.ReLU(inplace=True)
self.downsample = downsample
self.stride = stride
def __init__(self, block, layers, num_classes=1000):
super(ResNet, self).__init__()
self.inplanes = 64
self.conv1 = nn.Conv2d(3,
64,
kernel_size=7,
stride=2,
padding=3,
bias=False)
self.bn1 = Layer(64)
self.relu = nn.ReLU(inplace=True)
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
self.layer1 = self._make_layer(block, 64, layers[0])
self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
self.avgpool = nn.AvgPool2d(7, stride=1)
self.fc = nn.Linear(512 * block.expansion, num_classes)
self.lastbn = nn.BatchNorm1d(num_classes, affine=False)
for m in self.modules():
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(1. / n))
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
def main2():
dnn = DNN(input=28 * 28,
layers=[DropoutLayer(160, LQ),
Layer(10, LCE)],
eta=0.05,
lmbda=1) # 98%
dnn.initialize_rand()
train, test, vadilation = load_mnist_simple()
f_names = [f'mnist_expaned_k0{i}.pkl.gz' for i in range(50)]
shuffle(f_names)
for f_name in f_names:
print(f_name)
with timing("load"):
raw_data = load_data(f_name)
with timing("shuffle"):
shuffle(raw_data)
with timing("reshape"):
data = [(x.reshape((784, 1)), y)
for x, y in islice(raw_data, 100000)]
del raw_data
with timing("learn"):
dnn.learn(data)
del data
print('TEST:', dnn.test(test))
def __init__(self, dims, n_layers, n_mixtures=10, hook=None, cond=False, cond_dims=1):
super().__init__()
# Input layers
self.freq_input = nn.Linear(1, dims)
self.time_input = nn.Linear(1, dims)
if cond:
# Paper states that there are two condition networks: W^t_z, W^f_z
self.cond_freq = nn.Linear(cond_dims, dims)
self.cond_time = nn.Linear(cond_dims, dims)
self.c_freq = None
self.c_time = None
self.cond = cond
# Main layers
self.layers = nn.Sequential(
*[Layer(dims, hook) for _ in range(n_layers)]
)
# Output layer
self.fc_out = nn.Linear(2 * dims, 3 * n_mixtures)
self.n_mixtures = n_mixtures
# Print model size
self.num_params()
def initialize(self,
input_dimension,
output_dimension,
output_activation='sigmoid'):
# Insert the output layer as the last layer
self.add_layer(Layer(output_dimension, activation=output_activation))
# Initialize the weights of all layers
self.initialize_layers(input_dimension)
def __init__(self, in_channels, growth_rate):
super().__init__()
#"""In our experiments, we let each 1×1 convolution
#produce 4k feature-maps."""
inner_channel = 4 * growth_rate
#"""We find this design especially effective for DenseNet and
#we refer to our network with such a bottleneck layer, i.e.,
#to the BN-ReLU-Conv(1×1)-BN-ReLU-Conv(3×3) version of H ` ,
#as DenseNet-B."""
self.bottle_neck = nn.Sequential(
Layer(in_channels),
nn.ReLU(inplace=True),
nn.Conv2d(in_channels, inner_channel, kernel_size=1, bias=False),
Layer(inner_channel),
nn.ReLU(inplace=True),
nn.Conv2d(inner_channel, growth_rate, kernel_size=3, padding=1, bias=False)
)
def test(): #np.random.seed(42) #X = np.matrix(data[X_names]) #y = np.matrix(data[y_names]) nn = NeuralNetwork(learning_rate=0.4) nn.add_layer(Layer(30)) nn.fit(X, matrix, epochs=10000, batch_size=32)
def cifar10():
conv_config = [{
'channels': 3,
'kernel': (3, 3),
'stride': (1, 1),
'activation': 'relu'
}]
dense_config = [{'out': 20, 'activation': 'relu'}]
input_shape = [32, 32, 3]
output_size = 10
depth = 3
width = 10
max_modules_pr_layer = 3
learning_rate = 0.001
optimizer_type = Adam
loss = 'categorical_crossentropy'
layers = []
layers.append(ConvLayer(width, 'L0', conv_config))
layers.append(ConvLayer(width, 'L1', conv_config))
layers.append(ConvLayer(width, 'L2', conv_config, maxpool=True))
#layers.append(DenseLayer(width, 'L2', dense_config, flatten=True))
Layer.initialize_whole_network(layers, input_shape)
task = TaskContainer(input_shape,
output_size,
True,
name='unique_cifar10',
optimizer=optimizer_type,
loss=loss,
lr=learning_rate)
pathnet = PathNet(input_shape=input_shape, width=width, depth=depth)
pathnet._layers = layers
pathnet._tasks = [task]
pathnet.max_modules_pr_layer = max_modules_pr_layer
for layer in pathnet._layers:
layer.save_initialized_weights()
return pathnet, task
def __init__(self, in_channels, out_channels):
super().__init__()
#"""The transition layers used in our experiments
#consist of a batch normalization layer and an 1×1
#convolutional layer followed by a 2×2 average pooling
#layer""".
self.down_sample = nn.Sequential(
Layer(in_channels),
nn.Conv2d(in_channels, out_channels, 1, bias=False),
nn.AvgPool2d(2, stride=2)
)
def fit(self,
training_values,
expected_output,
output_activation='sigmoid',
epochs=100,
batch_size=10):
x = np.matrix(training_values)
y = np.matrix(expected_output)
print("Network input (un-scaled)")
print(x)
# Compute the maximum values of the input and output for normalizing
x_max = np.max(x)
y_max = np.max(y)
# Insert the output layer as the last layer
self.add_layer(Layer(y.shape[1], activation=output_activation))
# Initialize the weights of all layers
self.initialize_layers(x.shape[1])
x_normalized = x / x_max
y_normalized = y / y_max
examples = x.shape[0]
batch_size = examples
for k in range(epochs):
# Shuffle both X and y in unison
x_normalized, y_normalized = shuffle(x_normalized, y_normalized)
print(k)
errors = []
accuracies = []
for start in range(0, examples, batch_size):
end = min(start + batch_size, examples)
x_batch = x_normalized[start:end]
y_batch = y_normalized[start:end]
output = self.execute_batch(x_batch, y_batch)
errors.append(self.compute_loss(output, y_batch))
accuracies.append(self.compute_accuracy(
output, y_batch, y_max))
print(np.mean(np.array(errors)))
print("acc")
print(np.mean(np.array(accuracies)))
# Denormalize the values
#output = previous_a * y_max
if k < epochs - 1:
pass
def __init__(self, structure, activationFunction, derivative, bias=False):
self.structure = structure[:]
self.activationFunction = activationFunction
self.derivate = derivative
self.bias = bias
self.noLayers = len(self.structure)
self.layers = [FirstLayer(self.structure[0], bias)]
for i in range(1, self.noLayers):
self.layers = self.layers + [
Layer(self.structure[i - 1], activationFunction,
self.structure[i])
]
class Encoder(object, metaclass=VarScopeClass):
def __init__(self, embeddings, pad_mask, placeholders, net_kwargs,
dropout, loss_f):
out_kwargs = dict(net_kwargs)
out_kwargs['in_size'], out_kwargs['out_size'] = out_kwargs['out_size']*2, \
placeholders['y'].get_shape().as_list()[1]
out_kwargs['activation'] = 'sigmoid'
self.output_layer = Layer(name='Encoder_Output', **out_kwargs)
self.net_kwargs = net_kwargs
self.embeddings = embeddings
self.placeholders = placeholders
self.pad_mask = pad_mask
self.dropout = dropout
self.loss_f = loss_f
def create_minimization(self, samples):
# Create networks and initial states
layers = [RCNN(name=str(_)+'ERCNN', **self.net_kwargs) for _ in range(self.net_kwargs['depth'])]
h_prev = self.embeddings
h_prev.set_shape([None, FLAGS.batch, self.net_kwargs['in_size']])
z = tf.expand_dims(samples, -1)
z.set_shape([None, FLAGS.batch, 1])
# Pass samples through all networks
states = []
for l in layers:
h_next = l.forward2(h_prev, z)
states+= [h_next[-1]]
h_prev = self.dropout(h_next)
# Use values to get a final prediction on the text
preds = self.preds = self.output_layer.forward(self.dropout(tf.concat_v2(states, 1)))
loss_mat = self.loss_f(preds, self.placeholders['y'])
if FLAGS.aspect == -1: self.loss_vec = tf.reduce_mean(loss_mat, 1)
else: self.loss_vec = loss_mat[:, FLAGS.aspect]
self.loss = tf.reduce_mean(self.loss_vec)
variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, 'Encoder')
l2_cost = tf.add_n([tf.nn.l2_loss(v) for v in variables])
cost_enc = self.loss + l2_cost * self.placeholders['lambda']
self.train_e = create_gradients(cost_enc, variables, FLAGS.learning_rate)
tf.summary.histogram('Loss_Vec',self.loss_vec)
tf.summary.histogram('Predictions', preds[:,FLAGS.aspect])
tf.summary.histogram('Y', self.placeholders['y'][:,FLAGS.aspect])
tf.summary.scalar('Encoder Cost', cost_enc)
def reset_backend_session(self):
#print('==> Reseting backend session')
for layer in self._layers:
layer.save_layer_weights()
for task in self._tasks:
task.save_layer_weights()
K.clear_session()
for layer in self._layers:
layer._init_layer()
Layer.initialize_whole_network(self._layers, self.input_shape)
for layer in self._layers:
layer.load_layer_weights()
for task in self._tasks:
task.load_layer_weights()
self._models_created_in_current_session = 1 + len(self._tasks)
def load_model(filepath):
model_dict = dill.load(open(filepath, 'rb'))
params = model_dict['params']
layers = model_dict['layers']
for name, layer_dict in layers.items():
W_name = layer_dict.pop('W') if 'W' in layer_dict else None
b_name = layer_dict.pop('b') if 'b' in layer_dict else None
layer = layer_from_dicts(layer_dict)
if W_name is not None:
layer.W = Layer._weight_variable(params[W_name][0], layer.name)
Model.session.run(layer.W.assign(params[W_name][1]))
if b_name is not None:
layer.b = Layer._bias_variable(params[b_name][0], layer.name)
Model.session.run(layer.b.assign(params[b_name][1]))
layers.update({name: layer})
tensor_dict = dict()
for tensor in model_dict['tensors']:
tensor_name = tensor[0]
layer_name = tensor[2]
if 'Input.T.' in layer_name:
tensor_dict.update({
tensor_name:
Input(ast.literal_eval(layer_name.replace('Input.T.', '')))
})
else:
layer = layers[layer_name]
input_tensor_name = tensor[1]
input_tensor = tensor_dict[input_tensor_name] if not isinstance(input_tensor_name, list) else\
[tensor_dict[i_name] for i_name in input_tensor_name]
tensor_dict.update({tensor_name: layer(input_tensor)})
inputs = model_dict['inputs']
inputs = [tensor_dict[i_name] for i_name in inputs] if isinstance(
inputs, list) else tensor_dict[inputs]
outputs = model_dict['outputs']
outputs = [tensor_dict[i_name] for i_name in outputs] if isinstance(
outputs, list) else tensor_dict[outputs]
return Model(inputs, outputs, model_dict['optimizer'], model_dict['loss'],
model_dict['metrics'])
def binary_mnist():
config = [{'out': 20, 'activation': 'relu'}]
input_shape = [28, 28, 1]
output_size = 2
depth = 3
width = 10
max_modules_pr_layer = 3
learning_rate = 0.0001
optimizer_type = SGD
loss = 'binary_crossentropy'
layers = []
for l in range(depth):
if len(layers) == 0:
layers.append(DenseLayer(width, 'L0', config, flatten=True))
else:
layers.append(DenseLayer(width, 'L' + str(l), config))
Layer.initialize_whole_network(layers, input_shape)
task = TaskContainer(input_shape,
output_size,
name='unique_binary_mnist',
optimizer=optimizer_type,
loss=loss,
lr=learning_rate)
pathnet = PathNet(input_shape=input_shape, width=width, depth=depth)
pathnet._layers = layers
pathnet._tasks = [task]
pathnet.max_modules_pr_layer = max_modules_pr_layer
for layer in pathnet._layers:
layer.save_initialized_weights()
return pathnet, task
def input(self, dim):
"""Input Layer
Parameters
----------
dim: tuple
Dimension of images
activation: str
Type of activation
"""
input_n = dim[1] * dim[0]
self.input_layer = Layer.input(input_n)
self.layers.append(self.input_layer)
self.num_layers += 1
def main():
train, test, vadilation = load_mnist_simple()
# x, y = train[0]
# print("x: ", x.shape)
# print("y: ", y)
with timing(f""):
# dnn = DNN(input=28 * 28, layers=[Layer(30, LQ), Layer(10, LCE)], eta=0.05) # 96%
# dnn = DNN(input=28 * 28, layers=[Layer(30, LQ), Layer(10, SM)], eta=0.001) # 68%
# dnn = DNN(input=28 * 28, layers=[Layer(100, LQ), Layer(10, LCE)], eta=0.05, lmbda=5) # 98%
# dnn = DNN(input=28 * 28, layers=[DropoutLayer(100, LQ), Layer(10, LCE)], eta=0.05) # 97.5%
dnn = DNN(input=28 * 28, layers=[DropoutLayer(160, LQ), Layer(10, LCE)], eta=0.05, lmbda=3)
dnn.initialize_rand()
dnn.learn(train, epochs=30, test=vadilation, batch_size=29)
print('test:', dnn.test(test))
print(dnn.stats())
def split_by_layers(self, singularity_bounds, singularity_step,
density_values):
layers = list()
sin = singularity_bounds.begin
while sin <= singularity_bounds.end:
layerSingularity = Interval(sin, sin + singularity_step)
points = list()
for i in range(0, self.height):
for j in range(0, self.width):
if sin <= density_values[i, j] < sin + singularity_step:
points.append(Point(i, j))
layers.append(Layer(points, layerSingularity))
sin += singularity_step
return layers
def _make_layer(self, block, planes, blocks, stride=1):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
nn.Conv2d(self.inplanes,
planes * block.expansion,
kernel_size=1,
stride=stride,
bias=False),
Layer(planes * block.expansion),
)
layers = []
layers.append(block(self.inplanes, planes, stride, downsample))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes))
return nn.Sequential(*layers)
def make_layers(cfg, batch_norm=False):
layers = []
input_channel = 3
bias=True
for l in cfg:
if l == 'M':
layers += [nn.MaxPool2d(kernel_size=2, stride=2)]
continue
layers += [nn.Conv2d(input_channel, l, kernel_size=3, padding=1,bias=bias)]
if batch_norm:
layers += [Layer(l)]
layers += [nn.ReLU(inplace=True)]
input_channel = l
return nn.Sequential(*layers)
