def load_pathnet(filename): log = None with open(filename, 'rb') as f: log = pickle.load(f) layers = [] for layer_log in log['layer_logs']: if layer_log['layer_type'] == 'dense': layers.append(DenseLayer.build_from_log(layer_log)) if layer_log['layer_type'] == 'conv': layers.append(ConvLayer.build_from_log(layer_log)) Layer.initialize_whole_network(layers, log['in_shape']) for layer, layer_log in zip(layers, log['layer_logs']): layer.load_layer_log(layer_log) pathnet = PathNet(input_shape=log['in_shape'], width=log['width'], depth=log['depth']) pathnet._layers = layers pathnet.training_counter = log['training_counter'] pathnet.max_modules_pr_layer = log['max_modules_pr_layer'] pathnet.min_modules_pr_layer = log['min_modules_pr_layer'] tasks = [] for task_log in log['task_logs']: task = TaskContainer.build_from_log(task_log) pathnet.path2model(pathnet.random_path(), task) task.layer.set_weights(task_log['layer_weights']) tasks.append(task) pathnet._tasks = tasks return pathnet
def __init__(self, inplanes, planes, stride=1, downsample=None): super(BasicBlock, self).__init__() self.conv1 = conv3x3(inplanes, planes, stride) self.bn1 = Layer(planes) self.relu = nn.ReLU(inplace=True) self.conv2 = conv3x3(planes, planes) self.bn2 = Layer(planes) self.downsample = downsample self.stride = stride
def __init__(self, config=None, sess=None): self.sess = sess self.config = config self.num_class = self.config.num_class self.input_height = self.config.input_height self.input_width = self.config.input_width self.input_channel = self.config.input_channel self.batchsize = self.config.batchsize self.layer = Layer()
def _build_layers(self, inputs): self.input_layer = InputLayer(inputs) self.layers = [] for idx, layer_params in enumerate(self.layers_params): neurons_num, act_func = layer_params if len(self.layers) == 0: layer = Layer(neurons_num, self.input_layer.neurons_number, act_func, inputs.shape[0]) else: layer = Layer(neurons_num, self.layers[idx - 1].neurons_number, act_func, inputs.shape[0]) self.layers.append(layer)
def add(self, layer: Layer): if layer._built: assert layer.input_dim == self.shape[-1], \ "cannot match the input dimensionality of %s" % layer else: layer.setup(self.shape[-1]) self.layers.append(layer) self.shape.append(layer.size) self.parameters.extend(layer.parameters.values()) if layer.activation is False: layer.activation = self.activation
def __init__(self, embeddings, pad_mask, placeholders, net_kwargs, dropout, loss_f): out_kwargs = dict(net_kwargs) out_kwargs['in_size'], out_kwargs['out_size'] = out_kwargs['out_size']*2, \ placeholders['y'].get_shape().as_list()[1] out_kwargs['activation'] = 'sigmoid' self.output_layer = Layer(name='Encoder_Output', **out_kwargs) self.net_kwargs = net_kwargs self.embeddings = embeddings self.placeholders = placeholders self.pad_mask = pad_mask self.dropout = dropout self.loss_f = loss_f
def mnist(output_size=10): conv_config = [{ 'channels': 1, 'kernel': (3, 3), 'stride': (1, 1), 'activation': 'relu' }] config = [{'out': 20, 'activation': 'relu'}] input_shape = [28, 28, 1] output_size = output_size depth = 3 width = 10 max_modules_pr_layer = 3 min_modules_pr_layer = 1 learning_rate = 0.0001 optimizer_type = Adam loss = 'categorical_crossentropy' flatten_in_unique = True layers = [] #layers.append(DenseLayer(width, 'L0', config, flatten=not flatten_in_unique)) #layers.append(DenseLayer(width, 'L1', config)) #layers.append(DenseLayer(width, 'L2', config)) layers.append(ConvLayer(width, 'L0', conv_config)) layers.append(ConvLayer(width, 'L1', conv_config)) layers.append(ConvLayer(width, 'L2', conv_config, maxpool=True)) Layer.initialize_whole_network(layers, input_shape) task = TaskContainer(input_shape, output_size, flatten_in_unique, name='unique_mnist', optimizer=optimizer_type, loss=loss, lr=learning_rate) pathnet = PathNet(input_shape=input_shape, width=width, depth=depth, max_active_modules=20) pathnet._layers = layers pathnet._tasks = [task] pathnet.max_modules_pr_layer = max_modules_pr_layer pathnet.min_modules_pr_layer = min_modules_pr_layer for layer in pathnet._layers: layer.save_initialized_weights() return pathnet, task
def __init__(self, layer_dims, momentum=0, beta=0, lRate=0.45, n_iters=5000, activation='ReLu', initializer='He', GD_type='StochasticGD', batch_size=None, optimizer=None, regularizer=None, regularizer_const=0): self.lRate = lRate self.n_iters = n_iters self.loss = CrossEntropy() self.optimizer = optimizer self.regularizer = regularizer if regularizer != None: self.regularizer = regularizers[regularizer](regularizer_const) self.GD_type = None if GD_type == 'BatchGD': self.GD_type = GD_variants[GD_type](self.lRate) elif GD_type == 'StochasticGD': self.GD_type = GD_variants[GD_type](self.lRate, momentum, beta, self.optimizer) elif GD_type == 'MiniBatchGD': self.GD_type = GD_variants[GD_type](self.lRate, momentum, beta, batch_size, self.optimizer) self.layers = [] self.n_layers = len(layer_dims) - 1 #Initializing all layers with ReLu except last for l in range(1, self.n_layers): layer_shape = (layer_dims[l], layer_dims[l - 1]) self.layers.append( Layer(l, layer_shape, activation, initializer, self.regularizer)) print(self.layers[l - 1].__str__()) layer_shape = (layer_dims[self.n_layers], layer_dims[self.n_layers - 1]) self.layers.append( Layer(self.n_layers, layer_shape, 'Sigmoid', 'Random', self.regularizer)) print(self.layers[self.n_layers - 1].__str__())
def plot_accuracy(): import numpy as np from matplotlib import pyplot as plt train, test, vadilation = load_mnist_simple() dnn = DNN(input=28 * 28, layers=[Layer(100, LQ), Layer(10, LCE)], eta=0.05, lmbda=1) for l in dnn.layers: l.w = np.random.random(l.w.shape) - 0.5 acc1 = list(dnn.learn_iter(train, epochs=20, test=vadilation)) dnn.initialize_rand() acc2 = list(dnn.learn_iter(train, epochs=20, test=vadilation)) print(acc1) print(acc2) plt.plot(acc1) plt.plot(acc2) plt.show()
def create_layer_item(self, stroke_id, layer_name): """ Creates layer item in layer panel using stroke data Args: stroke_id (int): unique index of stroke layer_name (str): name of stroke layer """ stroke_info = ['', layer_name] layer = Layer(stroke_info, stroke_index=stroke_id) highest_group = None if self.layers_tree.selectedItems(): iterator = QtGui.QTreeWidgetItemIterator(self.layers_tree) while iterator.value(): item = iterator.value() if isinstance(item, Folder) and item in self.layers_tree.selectedItems(): highest_group = item break iterator += 1 if highest_group: highest_group.insertChild(0, layer) else: self.layers_tree.insertTopLevelItem(0, layer) self.update_layer_index()
def __init__(self, inplanes, planes, stride=1, downsample=None): super(Bottleneck, self).__init__() self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) self.bn1 = Layer(planes) self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) self.bn2 = Layer(planes) self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False) self.bn3 = Layer(planes * 4) self.relu = nn.ReLU(inplace=True) self.downsample = downsample self.stride = stride
def __init__(self, block, layers, num_classes=1000): super(ResNet, self).__init__() self.inplanes = 64 self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False) self.bn1 = Layer(64) self.relu = nn.ReLU(inplace=True) self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) self.layer1 = self._make_layer(block, 64, layers[0]) self.layer2 = self._make_layer(block, 128, layers[1], stride=2) self.layer3 = self._make_layer(block, 256, layers[2], stride=2) self.layer4 = self._make_layer(block, 512, layers[3], stride=2) self.avgpool = nn.AvgPool2d(7, stride=1) self.fc = nn.Linear(512 * block.expansion, num_classes) self.lastbn = nn.BatchNorm1d(num_classes, affine=False) for m in self.modules(): if isinstance(m, nn.Conv2d): n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels m.weight.data.normal_(0, math.sqrt(1. / n)) elif isinstance(m, nn.BatchNorm2d): m.weight.data.fill_(1) m.bias.data.zero_()
def main2(): dnn = DNN(input=28 * 28, layers=[DropoutLayer(160, LQ), Layer(10, LCE)], eta=0.05, lmbda=1) # 98% dnn.initialize_rand() train, test, vadilation = load_mnist_simple() f_names = [f'mnist_expaned_k0{i}.pkl.gz' for i in range(50)] shuffle(f_names) for f_name in f_names: print(f_name) with timing("load"): raw_data = load_data(f_name) with timing("shuffle"): shuffle(raw_data) with timing("reshape"): data = [(x.reshape((784, 1)), y) for x, y in islice(raw_data, 100000)] del raw_data with timing("learn"): dnn.learn(data) del data print('TEST:', dnn.test(test))
def __init__(self, dims, n_layers, n_mixtures=10, hook=None, cond=False, cond_dims=1): super().__init__() # Input layers self.freq_input = nn.Linear(1, dims) self.time_input = nn.Linear(1, dims) if cond: # Paper states that there are two condition networks: W^t_z, W^f_z self.cond_freq = nn.Linear(cond_dims, dims) self.cond_time = nn.Linear(cond_dims, dims) self.c_freq = None self.c_time = None self.cond = cond # Main layers self.layers = nn.Sequential( *[Layer(dims, hook) for _ in range(n_layers)] ) # Output layer self.fc_out = nn.Linear(2 * dims, 3 * n_mixtures) self.n_mixtures = n_mixtures # Print model size self.num_params()
def initialize(self, input_dimension, output_dimension, output_activation='sigmoid'): # Insert the output layer as the last layer self.add_layer(Layer(output_dimension, activation=output_activation)) # Initialize the weights of all layers self.initialize_layers(input_dimension)
def __init__(self, in_channels, growth_rate): super().__init__() #"""In our experiments, we let each 1×1 convolution #produce 4k feature-maps.""" inner_channel = 4 * growth_rate #"""We find this design especially effective for DenseNet and #we refer to our network with such a bottleneck layer, i.e., #to the BN-ReLU-Conv(1×1)-BN-ReLU-Conv(3×3) version of H ` , #as DenseNet-B.""" self.bottle_neck = nn.Sequential( Layer(in_channels), nn.ReLU(inplace=True), nn.Conv2d(in_channels, inner_channel, kernel_size=1, bias=False), Layer(inner_channel), nn.ReLU(inplace=True), nn.Conv2d(inner_channel, growth_rate, kernel_size=3, padding=1, bias=False) )
def test(): #np.random.seed(42) #X = np.matrix(data[X_names]) #y = np.matrix(data[y_names]) nn = NeuralNetwork(learning_rate=0.4) nn.add_layer(Layer(30)) nn.fit(X, matrix, epochs=10000, batch_size=32)
def cifar10(): conv_config = [{ 'channels': 3, 'kernel': (3, 3), 'stride': (1, 1), 'activation': 'relu' }] dense_config = [{'out': 20, 'activation': 'relu'}] input_shape = [32, 32, 3] output_size = 10 depth = 3 width = 10 max_modules_pr_layer = 3 learning_rate = 0.001 optimizer_type = Adam loss = 'categorical_crossentropy' layers = [] layers.append(ConvLayer(width, 'L0', conv_config)) layers.append(ConvLayer(width, 'L1', conv_config)) layers.append(ConvLayer(width, 'L2', conv_config, maxpool=True)) #layers.append(DenseLayer(width, 'L2', dense_config, flatten=True)) Layer.initialize_whole_network(layers, input_shape) task = TaskContainer(input_shape, output_size, True, name='unique_cifar10', optimizer=optimizer_type, loss=loss, lr=learning_rate) pathnet = PathNet(input_shape=input_shape, width=width, depth=depth) pathnet._layers = layers pathnet._tasks = [task] pathnet.max_modules_pr_layer = max_modules_pr_layer for layer in pathnet._layers: layer.save_initialized_weights() return pathnet, task
def __init__(self, in_channels, out_channels): super().__init__() #"""The transition layers used in our experiments #consist of a batch normalization layer and an 1×1 #convolutional layer followed by a 2×2 average pooling #layer""". self.down_sample = nn.Sequential( Layer(in_channels), nn.Conv2d(in_channels, out_channels, 1, bias=False), nn.AvgPool2d(2, stride=2) )
def fit(self, training_values, expected_output, output_activation='sigmoid', epochs=100, batch_size=10): x = np.matrix(training_values) y = np.matrix(expected_output) print("Network input (un-scaled)") print(x) # Compute the maximum values of the input and output for normalizing x_max = np.max(x) y_max = np.max(y) # Insert the output layer as the last layer self.add_layer(Layer(y.shape[1], activation=output_activation)) # Initialize the weights of all layers self.initialize_layers(x.shape[1]) x_normalized = x / x_max y_normalized = y / y_max examples = x.shape[0] batch_size = examples for k in range(epochs): # Shuffle both X and y in unison x_normalized, y_normalized = shuffle(x_normalized, y_normalized) print(k) errors = [] accuracies = [] for start in range(0, examples, batch_size): end = min(start + batch_size, examples) x_batch = x_normalized[start:end] y_batch = y_normalized[start:end] output = self.execute_batch(x_batch, y_batch) errors.append(self.compute_loss(output, y_batch)) accuracies.append(self.compute_accuracy( output, y_batch, y_max)) print(np.mean(np.array(errors))) print("acc") print(np.mean(np.array(accuracies))) # Denormalize the values #output = previous_a * y_max if k < epochs - 1: pass
def __init__(self, structure, activationFunction, derivative, bias=False): self.structure = structure[:] self.activationFunction = activationFunction self.derivate = derivative self.bias = bias self.noLayers = len(self.structure) self.layers = [FirstLayer(self.structure[0], bias)] for i in range(1, self.noLayers): self.layers = self.layers + [ Layer(self.structure[i - 1], activationFunction, self.structure[i]) ]
class Encoder(object, metaclass=VarScopeClass): def __init__(self, embeddings, pad_mask, placeholders, net_kwargs, dropout, loss_f): out_kwargs = dict(net_kwargs) out_kwargs['in_size'], out_kwargs['out_size'] = out_kwargs['out_size']*2, \ placeholders['y'].get_shape().as_list()[1] out_kwargs['activation'] = 'sigmoid' self.output_layer = Layer(name='Encoder_Output', **out_kwargs) self.net_kwargs = net_kwargs self.embeddings = embeddings self.placeholders = placeholders self.pad_mask = pad_mask self.dropout = dropout self.loss_f = loss_f def create_minimization(self, samples): # Create networks and initial states layers = [RCNN(name=str(_)+'ERCNN', **self.net_kwargs) for _ in range(self.net_kwargs['depth'])] h_prev = self.embeddings h_prev.set_shape([None, FLAGS.batch, self.net_kwargs['in_size']]) z = tf.expand_dims(samples, -1) z.set_shape([None, FLAGS.batch, 1]) # Pass samples through all networks states = [] for l in layers: h_next = l.forward2(h_prev, z) states+= [h_next[-1]] h_prev = self.dropout(h_next) # Use values to get a final prediction on the text preds = self.preds = self.output_layer.forward(self.dropout(tf.concat_v2(states, 1))) loss_mat = self.loss_f(preds, self.placeholders['y']) if FLAGS.aspect == -1: self.loss_vec = tf.reduce_mean(loss_mat, 1) else: self.loss_vec = loss_mat[:, FLAGS.aspect] self.loss = tf.reduce_mean(self.loss_vec) variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, 'Encoder') l2_cost = tf.add_n([tf.nn.l2_loss(v) for v in variables]) cost_enc = self.loss + l2_cost * self.placeholders['lambda'] self.train_e = create_gradients(cost_enc, variables, FLAGS.learning_rate) tf.summary.histogram('Loss_Vec',self.loss_vec) tf.summary.histogram('Predictions', preds[:,FLAGS.aspect]) tf.summary.histogram('Y', self.placeholders['y'][:,FLAGS.aspect]) tf.summary.scalar('Encoder Cost', cost_enc)
def reset_backend_session(self): #print('==> Reseting backend session') for layer in self._layers: layer.save_layer_weights() for task in self._tasks: task.save_layer_weights() K.clear_session() for layer in self._layers: layer._init_layer() Layer.initialize_whole_network(self._layers, self.input_shape) for layer in self._layers: layer.load_layer_weights() for task in self._tasks: task.load_layer_weights() self._models_created_in_current_session = 1 + len(self._tasks)
def load_model(filepath): model_dict = dill.load(open(filepath, 'rb')) params = model_dict['params'] layers = model_dict['layers'] for name, layer_dict in layers.items(): W_name = layer_dict.pop('W') if 'W' in layer_dict else None b_name = layer_dict.pop('b') if 'b' in layer_dict else None layer = layer_from_dicts(layer_dict) if W_name is not None: layer.W = Layer._weight_variable(params[W_name][0], layer.name) Model.session.run(layer.W.assign(params[W_name][1])) if b_name is not None: layer.b = Layer._bias_variable(params[b_name][0], layer.name) Model.session.run(layer.b.assign(params[b_name][1])) layers.update({name: layer}) tensor_dict = dict() for tensor in model_dict['tensors']: tensor_name = tensor[0] layer_name = tensor[2] if 'Input.T.' in layer_name: tensor_dict.update({ tensor_name: Input(ast.literal_eval(layer_name.replace('Input.T.', ''))) }) else: layer = layers[layer_name] input_tensor_name = tensor[1] input_tensor = tensor_dict[input_tensor_name] if not isinstance(input_tensor_name, list) else\ [tensor_dict[i_name] for i_name in input_tensor_name] tensor_dict.update({tensor_name: layer(input_tensor)}) inputs = model_dict['inputs'] inputs = [tensor_dict[i_name] for i_name in inputs] if isinstance( inputs, list) else tensor_dict[inputs] outputs = model_dict['outputs'] outputs = [tensor_dict[i_name] for i_name in outputs] if isinstance( outputs, list) else tensor_dict[outputs] return Model(inputs, outputs, model_dict['optimizer'], model_dict['loss'], model_dict['metrics'])
def binary_mnist(): config = [{'out': 20, 'activation': 'relu'}] input_shape = [28, 28, 1] output_size = 2 depth = 3 width = 10 max_modules_pr_layer = 3 learning_rate = 0.0001 optimizer_type = SGD loss = 'binary_crossentropy' layers = [] for l in range(depth): if len(layers) == 0: layers.append(DenseLayer(width, 'L0', config, flatten=True)) else: layers.append(DenseLayer(width, 'L' + str(l), config)) Layer.initialize_whole_network(layers, input_shape) task = TaskContainer(input_shape, output_size, name='unique_binary_mnist', optimizer=optimizer_type, loss=loss, lr=learning_rate) pathnet = PathNet(input_shape=input_shape, width=width, depth=depth) pathnet._layers = layers pathnet._tasks = [task] pathnet.max_modules_pr_layer = max_modules_pr_layer for layer in pathnet._layers: layer.save_initialized_weights() return pathnet, task
def input(self, dim): """Input Layer Parameters ---------- dim: tuple Dimension of images activation: str Type of activation """ input_n = dim[1] * dim[0] self.input_layer = Layer.input(input_n) self.layers.append(self.input_layer) self.num_layers += 1
def main(): train, test, vadilation = load_mnist_simple() # x, y = train[0] # print("x: ", x.shape) # print("y: ", y) with timing(f""): # dnn = DNN(input=28 * 28, layers=[Layer(30, LQ), Layer(10, LCE)], eta=0.05) # 96% # dnn = DNN(input=28 * 28, layers=[Layer(30, LQ), Layer(10, SM)], eta=0.001) # 68% # dnn = DNN(input=28 * 28, layers=[Layer(100, LQ), Layer(10, LCE)], eta=0.05, lmbda=5) # 98% # dnn = DNN(input=28 * 28, layers=[DropoutLayer(100, LQ), Layer(10, LCE)], eta=0.05) # 97.5% dnn = DNN(input=28 * 28, layers=[DropoutLayer(160, LQ), Layer(10, LCE)], eta=0.05, lmbda=3) dnn.initialize_rand() dnn.learn(train, epochs=30, test=vadilation, batch_size=29) print('test:', dnn.test(test)) print(dnn.stats())
def split_by_layers(self, singularity_bounds, singularity_step, density_values): layers = list() sin = singularity_bounds.begin while sin <= singularity_bounds.end: layerSingularity = Interval(sin, sin + singularity_step) points = list() for i in range(0, self.height): for j in range(0, self.width): if sin <= density_values[i, j] < sin + singularity_step: points.append(Point(i, j)) layers.append(Layer(points, layerSingularity)) sin += singularity_step return layers
def _make_layer(self, block, planes, blocks, stride=1): downsample = None if stride != 1 or self.inplanes != planes * block.expansion: downsample = nn.Sequential( nn.Conv2d(self.inplanes, planes * block.expansion, kernel_size=1, stride=stride, bias=False), Layer(planes * block.expansion), ) layers = [] layers.append(block(self.inplanes, planes, stride, downsample)) self.inplanes = planes * block.expansion for i in range(1, blocks): layers.append(block(self.inplanes, planes)) return nn.Sequential(*layers)
def make_layers(cfg, batch_norm=False): layers = [] input_channel = 3 bias=True for l in cfg: if l == 'M': layers += [nn.MaxPool2d(kernel_size=2, stride=2)] continue layers += [nn.Conv2d(input_channel, l, kernel_size=3, padding=1,bias=bias)] if batch_norm: layers += [Layer(l)] layers += [nn.ReLU(inplace=True)] input_channel = l return nn.Sequential(*layers)