def __init__(self, device=torch.device('cuda:0')): super().__init__(device) self.fc1 = Linear(28*28, 50, noise_std=1e-0, device=self.device) self.act1 = Activation('TanH') self.fc2 = Linear(50, 10, noise_std=1e-0, device=self.device) self.softmax = Activation('Softmax') self.layers = [self.fc1, self.fc2]
def __init__(self, device=torch.device('cuda:0')): super().__init__(device) # self.fc1 = Linear(28*28, 25, noise_std=1e-0, device=self.device) self.fc1 = Conv(1, 25, kernel_size=25, noise_std=1e-0, act='TanH', device=self.device) self.act1 = Activation('TanH') self.fc2 = Linear(16*25, 10, noise_std=1e-0, device=self.device) self.softmax = Activation('Softmax') self.layers = [self.fc1, self.fc2]
def build_encoder(self, optimizer, loss_function): encoder = NeuralNetwork(optimizer=optimizer, loss=loss_function) encoder.add(Dense(512, input_shape=(self.img_dim, ), first_layer=True)) encoder.add(Activation('leaky_relu')) #encoder.add(BatchNormalization(momentum=0.8)) encoder.add(Dense(256)) encoder.add(Activation('leaky_relu')) #encoder.add(BatchNormalization(momentum=0.8)) encoder.add(Dense(self.latent_dim, latent_layer=True)) return encoder
def __init__(self, device=torch.device('cuda:0')): super().__init__(device) self.conv1 = Conv(1, 6, kernel_size=5, noise_std=1e-0, act='TanH', device=self.device) self.act1 = Activation('TanH') self.pool1 = Pool(2, device=self.device) self.fc1 = Linear(6*12*12, 100, noise_std=1e-0, act='TanH', device=self.device) self.act2 = Activation('TanH') self.fc2 = Linear(100, 10, noise_std=1e-0, act='TanH', device=self.device) self.softmax = Activation('Softmax') self.layers = [self.conv1, self.fc1, self.fc2]
def build_decoder(self, optimizer, loss_function): decoder = NeuralNetwork(optimizer=optimizer, loss=loss_function) decoder.add(Dense(256, input_shape=(self.latent_dim, ))) decoder.add(Activation('leaky_relu')) decoder.add(BatchNormalization(momentum=0.8)) decoder.add(Dense(512)) decoder.add(Activation('leaky_relu')) decoder.add(BatchNormalization(momentum=0.8)) decoder.add(Dense(self.img_dim)) decoder.add(Activation('tanh')) return decoder
def vgg_bn(): return [ Conv2D([3, 3], 32, [1, 1, 1, 1], padding='SAME'), Conv2DBatchNorm(32), Activation(tf.nn.relu), Conv2D([3, 3], 32, [1, 1, 1, 1], padding='SAME'), Conv2DBatchNorm(32), Activation(tf.nn.relu), Conv2D([3, 3], 64, [1, 2, 2, 1]), Conv2DBatchNorm(64), Activation(tf.nn.relu), Conv2D([3, 3], 64, [1, 1, 1, 1], padding='SAME'), Conv2DBatchNorm(64), Activation(tf.nn.relu), Conv2D([3, 3], 128, [1, 2, 2, 1]), Conv2DBatchNorm(128), Activation(tf.nn.relu), Conv2D([3, 3], 128, [1, 1, 1, 1], padding='SAME'), Conv2DBatchNorm(128), Activation(tf.nn.relu), Flatten(), Dense(128), Activation(tf.sigmoid), Dropout(0.5), Dense(10), Activation(tf.nn.softmax), ]
def model_builder(n_inputs, n_outputs): model = NeuralNetwork(optimizer=Adam(), loss=CrossEntropy) model.add(Dense(64, input_shape=(n_inputs, ))) model.add(Activation('relu')) model.add(Dense(128)) model.add(Activation('relu')) model.add(Dense(128)) model.add(Activation('relu')) model.add(Dense(64)) model.add(Activation('relu')) model.add(Dense(1)) model.add(Activation('linear')) return model
def vgg_bn(): return [ #1 Conv2D([7, 7], 64, [1, 3, 3, 1]), Conv2DBatchNorm(64), Activation(tf.nn.relu), MaxPool([1,4,4,1],[1,1,1,1]), #2 Convolutional_block(f = 3, filters = [64,64,256],s = 1), MaxPool([1,5,5,1],[1,1,1,1]), Dropout(0.5), Identity_block(f = 3, filters=[64,64,256]), Dropout(0.5), Identity_block(f = 3, filters=[64,64,256]), Dropout(0.5), MaxPool([1,2,2,1],[1,1,1,1]), #3 Convolutional_block(f = 3, filters = [128,128,512],s = 2), Dropout(0.5), Identity_block(f = 3, filters=[128,128,512]), Dropout(0.5), Identity_block(f = 3, filters=[128,128,512]), Dropout(0.5), MaxPool([1,2,2,1],[1,1,1,1]), #4 Convolutional_block(f = 3, filters = [256,256,1024],s = 2), Identity_block(f = 3, filters=[256,256,1024]), Identity_block(f = 3, filters=[256,256,1024]), Identity_block(f = 3, filters=[256,256,1024]), Identity_block(f = 3, filters=[256,256,1024]), Identity_block(f = 3, filters=[256,256,1024]), Flatten(), Dense(128), Activation(tf.sigmoid), Dropout(0.5), Dense(10), #Fully_connected(), Activation(tf.nn.softmax), ]
def __init__(self, device=torch.device('cuda:0')): super().__init__(device) self.conv1 = Conv(1, 6, kernel_size=5, noise_std=1e-0, act='ReLU', device=self.device) self.act1 = Activation('ReLU') self.pool1 = Pool(2, device=self.device) self.conv2 = Conv(6, 16, kernel_size=5, noise_std=1e-0, act='ReLU', device=self.device) self.act2 = Activation('ReLU') self.pool2 = Pool(2, device=self.device) self.fc1 = Linear(256, 120, noise_std=1e-0, act='ReLU', device=self.device) self.act3 = Activation('ReLU') self.fc2 = Linear(120, 84, noise_std=1e-0, act='ReLU', device=self.device) self.act4 = Activation('ReLU') self.fc3 = Linear(84, 10, noise_std=1e-0, act='ReLU', device=self.device) self.softmax = Activation('Softmax') self.layers = [self.conv1, self.conv2, self.fc1, self.fc2, self.fc3]
def convModule(flow, filters, dropout, strides=(1, 1), s=(3, 3)): # {{{ if mirroring: flow = CReLU()(flow) flow = Convolution(filters, s=s, initialisation=init, initKWArgs=initKWArgs, strides=strides, regFunction=regF, reg=regP)(flow) if observing and not resNet: flow = Observation()(flow) if not mirroring: flow = Activation('relu')(flow) if doDropout: flow = Dropout(dropout)(flow) return flow # }}}
def fcModule(flow, w, dropout, regf=regF, reg=regP): # {{{ if mirroring: flow = CReLU()(flow) flow = FC(w, initialisation=init, initKWArgs=initKWArgs, reg=regP, regFunction=regF)(flow) if observing and not resNet: flow = Observation()(flow) if not mirroring: flow = Activation('relu')(flow) if doDropout: flow = Dropout(dropout)(flow) return flow # }}}
def _deserialize(self, params): layers_attrs = ['layers'] if params['best_layers_']: layers_attrs.append('best_layers_') for layers_attr in layers_attrs: for i, layer_dict in enumerate(params[layers_attr]): if layer_dict['layer'] == 'activation': params[layers_attr][i] = Activation(**layer_dict) if layer_dict['layer'] == 'dropout': params[layers_attr][i] = Dropout(**layer_dict) if layer_dict['layer'] == 'fully_connected': fc = FullyConnected(**layer_dict) fc.W = np.asarray(layer_dict['W']) fc.b = np.asarray(layer_dict['b']) fc.dW = np.asarray(layer_dict['dW']) fc.db = np.asarray(layer_dict['db']) params[layers_attr][i] = fc return params
def main(): test_id = 1 if test_id == 1: #---------- # Conv Net #---------- optimizer = Adam() data = datasets.load_digits() X = data.data # (1797, 64) y = data.target # Convert to one-hot encoding y = to_categorical(y.astype("int")) # (n_sample, n_class) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, seed=1) # Reshape X to (n_samples, channels, height, width) X_train = X_train.reshape((-1, 1, 8, 8)) X_test = X_test.reshape((-1, 1, 8, 8)) clf = NeuralNetwork(optimizer=optimizer, loss=CrossEntropy, validation_data=(X_test, y_test)) clf.add( Conv2D(n_filters=16, filter_shape=(3, 3), stride=1, input_shape=(1, 8, 8), padding='same')) clf.add(Activation('relu')) clf.add(Dropout(0.25)) clf.add(BatchNormalization()) clf.add( Conv2D(n_filters=32, filter_shape=(3, 3), stride=1, padding='same')) clf.add(Activation('relu')) clf.add(Dropout(0.25)) clf.add(BatchNormalization()) clf.add(Flatten()) # 展平层 clf.add(Dense(256)) # 全连接层 clf.add(Activation('relu')) clf.add(Dropout(0.4)) clf.add(BatchNormalization()) clf.add(Dense(10)) clf.add(Activation('softmax')) print() clf.summary(name="ConvNet") train_err, val_err = clf.fit(X_train, y_train, n_epochs=50, batch_size=64) # Training and validation error plot n = len(train_err) training, = plt.plot(range(n), train_err, label="Training Error") validation, = plt.plot(range(n), val_err, label="Validation Error") plt.legend(handles=[training, validation]) plt.title("Error Plot") plt.ylabel('Error') plt.xlabel('Iterations') plt.show() _, accuracy = clf.test_on_batch(X_test, y_test) print("Accuracy:", accuracy) y_pred = np.argmax(clf.predict(X_test), axis=1) X_test = X_test.reshape(-1, 8 * 8) # Reduce dimension to 2D using PCA and plot the results Plot().plot_in_2d(X_test, y_pred, title="Convolutional Neural Network", accuracy=accuracy, legend_labels=range(10)) if test_id == 2: dataset = MultiClassDataset(n_samples=300, centers=3, n_features=2, center_box=(-10.0, 10.0), cluster_std=1.0, norm=True, one_hot=True) X = dataset.datas y = dataset.labels X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, seed=1) clf = NeuralNetwork(optimizer=optimizer, loss=CrossEntropy, validation_data=(X_test, y_test)) clf.add(Dense(3)) clf.add(Activation('softmax')) clf.summary(name="SoftmaxReg") train_err, val_err = clf.fit(X_train, y_train, n_epochs=50, batch_size=256)
W1 = np.array([[.15, .20], [.25, .30]]) W2 = np.array([[.40, .45], [.50, .55]]) b1 = .35 b2 = 0.60 y_true = np.array([[.01, .99]]) #Layers Generation dense = Dense(2, W1, b1) dense2 = Dense(2, W2, b2) activation1 = Sigmoid() # activation2=Sigmoid() activation2 = Activation("sigmoid") loss_func = MSE() #Forward Pass # Dense -> Activation -> Dense -> Activation -> y_pred z1 = dense.forward(x) a1 = activation1.forward(z1) print("Activation Value:", a1) z2 = dense2.forward(a1) a2 = activation2.forward(z2) y_pred = a2 loss = loss_func.loss(y_true, y_pred)
y = np.zeros([nums, 10, 20], dtype=float) for i in range(nums): start = np.random.randint(0, 10) num_seq = np.arange(start, start + 10) X[i] = to_categorical(num_seq, n_col=20) y[i] = np.roll(X[i], -1, axis=0) y[:, -1, 1] = 1 # Mark endpoint as 1 return X, y if __name__ == '__main__': X, y = gen_mult_ser(3000) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4) clf = NeuralNetwork(optimizer=optimizer, loss=CrossEntropy) clf.add(RNN(10, activation="tanh", bptt_trunc=5, input_shape=(10, 61))) clf.add(Activation('softmax')) tmp_X = np.argmax(X_train[0], axis=1) tmp_y = np.argmax(y_train[0], axis=1) print("Number Series Problem:") print("X = [" + " ".join(tmp_X.astype("str")) + "]") print("y = [" + " ".join(tmp_y.astype("str")) + "]") print() train_err, _ = clf.fit(X_train, y_train, n_epochs=500, batch_size=512) y_pred = np.argmax(clf.predict(X_test), axis=2) y_test = np.argmax(y_test, axis=2) accuracy = np.mean(accuracy_score(y_test, y_pred)) print(accuracy) print() print("Results:")
def convert(keras_model, class_map, description="Neural Network Model"): """ Convert a keras model to PMML @model. The keras model object @class_map. A map in the form {class_id: class_name} @description. A short description of the model Returns a DeepNeuralNetwork object which can be exported to PMML """ pmml = DeepNetwork(description=description, class_map=class_map) pmml.keras_model = keras_model pmml.model_name = keras_model.name config = keras_model.get_config() for layer in config['layers']: layer_class = layer['class_name'] layer_config = layer['config'] layer_inbound_nodes = layer['inbound_nodes'] # Input if layer_class is "InputLayer": pmml._append_layer(InputLayer( name=layer_config['name'], input_size=layer_config['batch_input_shape'][1:] )) # Conv2D elif layer_class is "Conv2D": pmml._append_layer(Conv2D( name=layer_config['name'], channels=layer_config['filters'], kernel_size=layer_config['kernel_size'], dilation_rate=layer_config['dilation_rate'], use_bias=layer_config['use_bias'], activation=layer_config['activation'], strides=layer_config['strides'], padding=layer_config['padding'], inbound_nodes=get_inbound_nodes(layer_inbound_nodes), )) # DepthwiseConv2D elif layer_class is "DepthwiseConv2D": pmml._append_layer(DepthwiseConv2D( name=layer_config['name'], kernel_size=layer_config['kernel_size'], depth_multiplier=layer_config['depth_multiplier'], use_bias=layer_config['use_bias'], activation=layer_config['activation'], strides=layer_config['strides'], padding=layer_config['padding'], inbound_nodes=get_inbound_nodes(layer_inbound_nodes), )) # MaxPooling elif layer_class is "MaxPooling2D": pmml._append_layer(MaxPooling2D( name=layer_config['name'], pool_size=layer_config['pool_size'], strides=layer_config['strides'], inbound_nodes=get_inbound_nodes(layer_inbound_nodes), )) elif layer_class is "AveragePooling2D": pmml._append_layer(AveragePooling2D( name=layer_config['name'], pool_size=layer_config['pool_size'], strides=layer_config['strides'], inbound_nodes=get_inbound_nodes(layer_inbound_nodes), )) elif layer_class is "GlobalAveragePooling2D": pmml._append_layer(GlobalAveragePooling2D( name=layer_config['name'], inbound_nodes=get_inbound_nodes(layer_inbound_nodes), )) # Flatten elif layer_class is "Flatten": pmml._append_layer(Flatten( name=layer_config['name'], inbound_nodes=get_inbound_nodes(layer_inbound_nodes), )) # Dense elif layer_class is "Dense": pmml._append_layer(Dense( name=layer_config['name'], channels=layer_config['units'], use_bias=layer_config['use_bias'], activation=layer_config['activation'], inbound_nodes=get_inbound_nodes(layer_inbound_nodes), )) # Zero padding layer elif layer_class is "ZeroPadding2D": pmml._append_layer(ZeroPadding2D( name=layer_config['name'], padding=layer_config['padding'], inbound_nodes=get_inbound_nodes(layer_inbound_nodes), )) # Reshape layer elif layer_class is "Reshape": pmml._append_layer(Reshape( name=layer_config['name'], target_shape=layer_config['target_shape'], inbound_nodes=get_inbound_nodes(layer_inbound_nodes), )) elif layer_class is "Dropout": pmml._append_layer(Dropout( name=layer_config['name'], inbound_nodes=get_inbound_nodes(layer_inbound_nodes), )) # Batch Normalization elif layer_class is "BatchNormalization": pmml._append_layer(BatchNormalization( name=layer_config['name'], axis=layer_config['axis'], momentum=layer_config['momentum'], epsilon=layer_config['epsilon'], center=layer_config['center'], inbound_nodes=get_inbound_nodes(layer_inbound_nodes), )) elif layer_class is "Add": pmml._append_layer(Merge( name=layer_config['name'], inbound_nodes=get_inbound_nodes(layer_inbound_nodes) )) elif layer_class is "Subtract": pmml._append_layer(Merge( name=layer_config['name'], operator='subtract', inbound_nodes=get_inbound_nodes(layer_inbound_nodes) )) elif layer_class is "Dot": pmml._append_layer(Merge( name=layer_config['name'], operator='dot', inbound_nodes=get_inbound_nodes(layer_inbound_nodes) )) elif layer_class is "Concatenate": pmml._append_layer(Merge( name=layer_config['name'], axis=layer_config['axis'], operator='concatenate', inbound_nodes=get_inbound_nodes(layer_inbound_nodes) )) elif layer_class is "Activation": pmml._append_layer(Activation( name=layer_config['name'], activation=layer_config['activation'], inbound_nodes=get_inbound_nodes(layer_inbound_nodes), )) elif layer_class is "ReLU": pmml._append_layer(Activation( name=layer_config['name'], activation='relu', threshold = layer_config['threshold'], max_value = layer_config['max_value'], negative_slope = layer_config['negative_slope'], inbound_nodes=get_inbound_nodes(layer_inbound_nodes), )) # Unknown layer else: raise ValueError("Unknown layer type:",layer_class) return pmml
def get_brats_nets(input_shape, filters_list, kernel_size_list, dense_size, nlabels): inputs = Input(shape=input_shape) conv = inputs for filters, kernel_size in zip(filters_list, kernel_size_list): conv = Conv(filters, kernel_size=(kernel_size, ) * 3, activation='relu', data_format='channels_first')(conv) full = Conv(dense_size, kernel_size=(1, 1, 1), data_format='channels_first', name='fc_dense', activation='relu')(conv) full_roi = Conv(nlabels[0], kernel_size=(1, 1, 1), data_format='channels_first', name='fc_roi')(full) full_sub = Conv(nlabels[1], kernel_size=(1, 1, 1), data_format='channels_first', name='fc_sub')(full) rf_roi = Concatenate(axis=1)([conv, full_roi]) rf_sub = Concatenate(axis=1)([conv, full_sub]) rf_num = 1 while np.product(rf_roi.shape[2:]) > 1: rf_roi = Conv(dense_size, kernel_size=(3, 3, 3), data_format='channels_first', name='rf_roi%d' % rf_num)(rf_roi) rf_sub = Conv(dense_size, kernel_size=(3, 3, 3), data_format='channels_first', name='rf_sub%d' % rf_num)(rf_sub) rf_num += 1 full_roi = Reshape((nlabels[0], -1))(full_roi) full_sub = Reshape((nlabels[1], -1))(full_sub) full_roi = Permute((2, 1))(full_roi) full_sub = Permute((2, 1))(full_sub) full_roi_out = Activation('softmax', name='fc_roi_out')(full_roi) full_sub_out = Activation('softmax', name='fc_sub_out')(full_sub) combo_roi = Concatenate(axis=1)([Flatten()(conv), Flatten()(rf_roi)]) combo_sub = Concatenate(axis=1)([Flatten()(conv), Flatten()(rf_sub)]) tumor_roi = Dense(nlabels[0], activation='softmax', name='tumor_roi')(combo_roi) tumor_sub = Dense(nlabels[1], activation='softmax', name='tumor_sub')(combo_sub) outputs_roi = [tumor_roi, full_roi_out] net_roi = Model(inputs=inputs, outputs=outputs_roi, optimizer='adadelta', loss='categorical_cross_entropy', metrics='accuracy') outputs_sub = [tumor_sub, full_sub_out] net_sub = Model(inputs=inputs, outputs=outputs_sub, optimizer='adadelta', loss='categorical_cross_entropy', metrics='accuracy') return net_roi, net_sub
def vgg_bn(): return [ #1 Conv2D([3, 3], 64, [1, 1, 1, 1], padding='SAME'), Conv2DBatchNorm(64), Activation(tf.nn.relu), #2 Conv2D([3, 3], 64, [1, 1, 1, 1], padding='SAME'), Conv2DBatchNorm(64), Activation(tf.nn.relu), #3 MaxPool([1,2,2,1],[1,2,2,1],padding='SAME'), #4 Conv2D([3, 3], 128, [1, 1, 1, 1],padding='SAME'), Conv2DBatchNorm(128), Activation(tf.nn.relu), #5 Conv2D([3, 3], 128, [1, 1, 1, 1],padding='SAME'), Conv2DBatchNorm(128), Activation(tf.nn.relu), #6 MaxPool([1,2,2,1],[1,2,2,1],padding='SAME'), #7 Conv2D([3, 3], 256, [1, 1, 1, 1],padding='SAME'), Conv2DBatchNorm(256), Activation(tf.nn.relu), #8 Conv2D([3, 3], 256, [1, 1, 1, 1],padding='SAME'), Conv2DBatchNorm(256), Activation(tf.nn.relu), #9 Conv2D([3, 3], 256, [1, 1, 1, 1],padding='SAME'), Conv2DBatchNorm(256), Activation(tf.nn.relu), #10 MaxPool([1,2,2,1],[1,2,2,1],padding='SAME'), #11 Conv2D([3, 3], 512, [1, 1, 1, 1],padding='SAME'), Conv2DBatchNorm(512), Activation(tf.nn.relu), #12 Conv2D([3, 3], 512, [1, 1, 1, 1],padding='SAME'), Conv2DBatchNorm(512), Activation(tf.nn.relu), #13 Conv2D([3, 3], 512, [1, 1, 1, 1],padding='SAME'), Conv2DBatchNorm(512), Activation(tf.nn.relu), #14 MaxPool([1,2,2,1],[1,2,2,1],padding='SAME'), #15 Conv2D([3, 3], 512, [1, 1, 1, 1],padding='SAME'), Conv2DBatchNorm(512), Activation(tf.nn.relu), #16 Conv2D([3, 3], 512, [1, 1, 1, 1],padding='SAME'), Conv2DBatchNorm(512), Activation(tf.nn.relu), #17 Conv2D([3, 3], 512, [1, 1, 1, 1],padding='SAME'), Conv2DBatchNorm(512), Activation(tf.nn.relu), #18 MaxPool([1,2,2,1],[1,2,2,1],padding='SAME'), Flatten(), Dense(4096), Activation(tf.nn.relu), Dropout(0.5), Dense(4096), Activation(tf.nn.relu), Dense(1000), Activation(tf.nn.relu), Dense(10), Activation(tf.nn.softmax), ]
# Output padding sys.stdout.write(' ' * 20) # Allow progress bar to persist if it's complete if current == total: sys.stdout.write('\n') # Flush to standard out sys.stdout.flush() # Return the time of the progress update return time.time() model = Sequential() model.add(Input(2)) model.add(Dense(25)) model.add(Activation("relu")) model.add(Dense(50)) model.add(Activation("relu")) model.add(Dense(50)) model.add(Activation("relu")) model.add(Dense(25)) model.add(Activation("relu")) model.add(Dense(1)) model.add(Activation("sigmoid")) def initialise_layer_parameters(seed=2): # random seed initiation np.random.seed(seed) # Iterate over the layers of the neural network
def main(): train_x, train_y, valid_x, valid_y, test_x, test_y = get_cifar10('./cifar-10-batches-py/') labels = unpickle('./cifar-10-batches-py/batches.meta')['label_names'] train_x = train_x.astype(np.float32) / 255.0 valid_x = valid_x.astype(np.float32) / 255.0 test_x = test_x.astype(np.float32) / 255.0 num_epochs = args.epochs eta = args.lr batch_size = args.batch_size # input x = T.tensor4("x") y = T.ivector("y") # test values # x.tag.test_value = np.random.randn(6, 3, 32, 32).astype(np.float32) # y.tag.test_value = np.array([1,2,1,4,5]).astype(np.int32) # x.tag.test_value = x.tag.test_value / x.tag.test_value.max() # import ipdb; ipdb.set_trace() # network definition conv1 = BinaryConv2D(input=x, num_filters=50, input_channels=3, size=3, strides=(1,1), padding=1, name="conv1") act1 = Activation(input=conv1.output, activation="relu", name="act1") pool1 = Pool2D(input=act1.output, stride=(2,2), name="pool1") conv2 = BinaryConv2D(input=pool1.output, num_filters=100, input_channels=50, size=3, strides=(1,1), padding=1, name="conv2") act2 = Activation(input=conv2.output, activation="relu", name="act2") pool2 = Pool2D(input=act2.output, stride=(2,2), name="pool2") conv3 = BinaryConv2D(input=pool2.output, num_filters=200, input_channels=100, size=3, strides=(1,1), padding=1, name="conv3") act3 = Activation(input=conv3.output, activation="relu", name="act3") pool3 = Pool2D(input=act3.output, stride=(2,2), name="pool3") flat = Flatten(input=pool3.output) fc1 = BinaryDense(input=flat.output, n_in=200*4*4, n_out=500, name="fc1") act4 = Activation(input=fc1.output, activation="relu", name="act4") fc2 = BinaryDense(input=act4.output, n_in=500, n_out=10, name="fc2") softmax = Activation(input=fc2.output, activation="softmax", name="softmax") # loss xent = T.nnet.nnet.categorical_crossentropy(softmax.output, y) cost = xent.mean() # errors y_pred = T.argmax(softmax.output, axis=1) errors = T.mean(T.neq(y, y_pred)) # updates + clipping (+-1) params = conv1.params + conv2.params + conv3.params + fc1.params + fc2.params params_bin = conv1.params_bin + conv2.params_bin + conv3.params_bin + fc1.params_bin + fc2.params_bin grads = [T.grad(cost, param) for param in params_bin] # calculate grad w.r.t binary parameters updates = [] for p,g in zip(params, grads): updates.append( (p, clip_weights(p - eta*g)) #sgd + clipping update ) # compiling train, predict and test fxns train = theano.function( inputs = [x,y], outputs = cost, updates = updates ) predict = theano.function( inputs = [x], outputs = y_pred ) test = theano.function( inputs = [x,y], outputs = errors ) # train checkpoint = ModelCheckpoint(folder="snapshots") logger = Logger("logs/{}".format(time())) for epoch in range(num_epochs): print "Epoch: ", epoch print "LR: ", eta epoch_hist = {"loss": []} t = tqdm(range(0, len(train_x), batch_size)) for lower in t: upper = min(len(train_x), lower + batch_size) loss = train(train_x[lower:upper], train_y[lower:upper].astype(np.int32)) t.set_postfix(loss="{:.2f}".format(float(loss))) epoch_hist["loss"].append(loss.astype(np.float32)) # epoch loss average_loss = sum(epoch_hist["loss"])/len(epoch_hist["loss"]) t.set_postfix(loss="{:.2f}".format(float(average_loss))) logger.log_scalar( tag="Training Loss", value= average_loss, step=epoch ) # validation accuracy val_acc = 1.0 - test(valid_x, valid_y.astype(np.int32)) print "Validation Accuracy: ", val_acc logger.log_scalar( tag="Validation Accuracy", value= val_acc, step=epoch ) checkpoint.check(val_acc, params) # Report Results on test set (w/ best val acc file) best_val_acc_filename = checkpoint.best_val_acc_filename print "Using ", best_val_acc_filename, " to calculate best test acc." load_model(path=best_val_acc_filename, params=params) test_acc = 1.0 - test(test_x, test_y.astype(np.int32)) print "Test accuracy: ",test_acc
def main(): train_x, train_y, valid_x, valid_y, test_x, test_y = get_mnist() num_epochs = args.epochs eta = args.lr batch_size = args.batch_size # input x = T.matrix("x") y = T.ivector("y") #x.tag.test_value = np.random.randn(3, 784).astype("float32") #y.tag.test_value = np.array([1,2,3]) #drop_switch.tag.test_value = 0 #import ipdb; ipdb.set_trace() hidden_1 = BinaryDense(input=x, n_in=784, n_out=2048, name="hidden_1") act_1 = Activation(input=hidden_1.output, activation="relu", name="act_1") hidden_2 = BinaryDense(input=act_1.output, n_in=2048, n_out=2048, name="hidden_2") act_2 = Activation(input=hidden_2.output, activation="relu", name="act_2") hidden_3 = BinaryDense(input=act_2.output, n_in=2048, n_out=2048, name="hidden_3") act_3 = Activation(input=hidden_3.output, activation="relu", name="act_3") output = BinaryDense(input=act_3.output, n_in=2048, n_out=10, name="output") softmax = Activation(input=output.output, activation="softmax", name="softmax") # loss xent = T.nnet.nnet.categorical_crossentropy(softmax.output, y) cost = xent.mean() # errors y_pred = T.argmax(softmax.output, axis=1) errors = T.mean(T.neq(y, y_pred)) # updates + clipping (+-1) params_bin = hidden_1.params_bin + hidden_2.params_bin + hidden_3.params_bin params = hidden_1.params + hidden_2.params + hidden_3.params grads = [T.grad(cost, param) for param in params_bin] # calculate grad w.r.t binary parameters updates = [] for p, g in zip( params, grads ): # gradient update on full precision weights (NOT binarized wts) updates.append((p, clip_weights(p - eta * g)) #sgd + clipping update ) # compiling train, predict and test fxns train = theano.function(inputs=[x, y], outputs=cost, updates=updates) predict = theano.function(inputs=[x], outputs=y_pred) test = theano.function(inputs=[x, y], outputs=errors) # train checkpoint = ModelCheckpoint(folder="snapshots") logger = Logger("logs/{}".format(time())) for epoch in range(num_epochs): print "Epoch: ", epoch print "LR: ", eta epoch_hist = {"loss": []} t = tqdm(range(0, len(train_x), batch_size)) for lower in t: upper = min(len(train_x), lower + batch_size) loss = train(train_x[lower:upper], train_y[lower:upper].astype(np.int32)) t.set_postfix(loss="{:.2f}".format(float(loss))) epoch_hist["loss"].append(loss.astype(np.float32)) # epoch loss average_loss = sum(epoch_hist["loss"]) / len(epoch_hist["loss"]) t.set_postfix(loss="{:.2f}".format(float(average_loss))) logger.log_scalar(tag="Training Loss", value=average_loss, step=epoch) # validation accuracy val_acc = 1.0 - test(valid_x, valid_y.astype(np.int32)) print "Validation Accuracy: ", val_acc logger.log_scalar(tag="Validation Accuracy", value=val_acc, step=epoch) checkpoint.check(val_acc, params) # Report Results on test set best_val_acc_filename = checkpoint.best_val_acc_filename print "Using ", best_val_acc_filename, " to calculate best test acc." load_model(path=best_val_acc_filename, params=params) test_acc = 1.0 - test(test_x, test_y.astype(np.int32)) print "Test accuracy: ", test_acc
X_train = X_train[:256] y_train = y_train[:256] X_train = X_train.reshape((-1, 1, 8, 8)) X_test = X_test.reshape((-1, 1, 8, 8)) X_test = X_train[:256] y_test = y_train[:256] # Model model = NeuralNetwork(SquareLoss(), (X_test, y_test)) model.add( Conv2D(16, filter_shape=(3, 3), stride=1, input_shape=(1, 8, 8), padding='same')) model.add(Activation('relu')) model.add(Dropout(p=0.2)) model.add(Conv2D(n_filters=32, filter_shape=(3, 3), stride=1, padding='same')) model.add(Activation('relu')) model.add(Dropout(p=0.2)) model.add(Flatten()) model.add(Dense(256)) model.add(Activation('relu')) model.add(Dropout(0.4)) model.add(Dense(10)) model.add(Activation('softmax')) train_err = model.fit(X_train, y_train, n_epochs=5, batch_size=256) # print(model.layers[-1])
def call(self, inputs, *args, **kwargs): x = inputs x = super().call(x) if self.activation: x = Activation(self.activation)(x) return tf.nn.depth_to_space(x, self.rate)