def __init__(self, input_shape, filter_sizes, init, strides=[1,1,1,1], padding='SAME', activation=None, bias=0., use_bias=True, name=None, load=None, train=True): self.input_shape = input_shape self.filter_sizes = filter_sizes self.batch_size, self.h, self.w, self.fin = self.input_shape self.fh, self.fw, self.fin, self.fout = self.filter_sizes self.init = init self.strides = strides _, self.sh, self.sw, _ = self.strides self.padding = padding self.activation = Linear() if activation == None else activation self.use_bias = use_bias self.name = name self.train_flag = train if load: print ("Loading Weights: " + self.name) weight_dict = np.load(load, encoding='latin1', allow_pickle=True).item() filters = weight_dict[self.name] bias = weight_dict[self.name + '_bias'] else: filters = init_filters(size=self.filter_sizes, init=self.init) bias = np.ones(shape=self.fout) * bias self.filters = tf.Variable(filters, dtype=tf.float32) self.bias = tf.Variable(bias, dtype=tf.float32)
def __init__(self, input_shape, num_classes, name=None): self.num_classes = num_classes self.input_shape = input_shape self.name = name ''' if load: weight_dict = np.load(load).item() self.B = tf.cast(tf.Variable(weight_dict[self.name]), tf.float32) elif std is not None: b = np.random.normal(loc=0., scale=std, size=(self.num_classes, self.output_size)) self.B = tf.cast(tf.Variable(b), tf.float32) else: # var = 1. / self.output_size # std = np.sqrt(var) # b = np.random.normal(loc=0., scale=std, size=(self.num_classes, self.output_size)) b = FeedbackMatrix(size=(self.num_classes, self.output_size), sparse=self.sparse, rank=self.rank) self.B = tf.cast(tf.Variable(b), tf.float32) ''' # THE PROBLEM WAS NEVER THE BIAS ... IT WAS THE FACT WE WERNT DIVIDING BY N # l0 = FullyConnected(input_shape=input_shape, size=self.input_shape, init='alexnet', activation=Relu(), bias=1., name=self.name) self.l0 = FullyConnected(input_shape=input_shape, size=self.num_classes, init='alexnet', activation=Linear(), bias=0., name=self.name)
def __init__(self, input_sizes, filter_sizes, strides, padding, init=None, alpha=0., activation=None, bias=0., name=None, load=None, train=True): self.input_sizes = input_sizes self.filter_sizes = filter_sizes self.batch_size, self.h, self.w, self.fin = self.input_sizes self.fh, self.fw, self.fin, self.fout = self.filter_sizes bias = np.ones(shape=self.fout) * bias self.strides = strides self.padding = padding self.alpha = alpha self.activation = Linear() if activation == None else activation self.name = name self._train = train if load: print("Loading Weights: " + self.name) weight_dict = np.load(load, encoding='latin1').item() filters = weight_dict[self.name] bias = weight_dict[self.name + '_bias'] else: if init == "zero": filters = np.zeros(shape=self.filter_sizes) elif init == "sqrt_fan_in": sqrt_fan_in = math.sqrt(self.h * self.w * self.fin) filters = np.random.uniform(low=-1.0 / sqrt_fan_in, high=1.0 / sqrt_fan_in, size=self.filter_sizes) elif init == "alexnet": filters = np.random.normal(loc=0.0, scale=0.01, size=self.filter_sizes) else: # https://www.tensorflow.org/api_docs/python/tf/glorot_uniform_initializer # can verify we did this right ... fan_in = self.fh * self.fw * self.fin fan_out = self.fout lim = np.sqrt(6. / (fan_in + fan_out)) filters = np.random.uniform(low=-lim, high=lim, size=self.filter_sizes) self.filters = tf.Variable(filters, dtype=tf.float32) self.bias = tf.Variable(bias, dtype=tf.float32)
def __init__(self, input_shape, size, init=None, activation=None, bias=0., alpha=0., name=None, load=None, train=True): self.input_size = input_shape self.output_size = size self.size = [self.input_size, self.output_size] bias = np.ones(shape=self.output_size) * bias self.alpha = alpha self.activation = Linear() if activation == None else activation self.name = name self._train = train if load: print("Loading Weights: " + self.name) weight_dict = np.load(load).item() weights = weight_dict[self.name] bias = weight_dict[self.name + '_bias'] else: if init == "zero": weights = np.zeros(shape=self.size) elif init == "sqrt_fan_in": sqrt_fan_in = math.sqrt(self.input_size) weights = np.random.uniform(low=-1.0 / sqrt_fan_in, high=1.0 / sqrt_fan_in, size=self.size) elif init == "alexnet": weights = np.random.normal(loc=0.0, scale=0.01, size=self.size) else: # https://www.tensorflow.org/api_docs/python/tf/glorot_uniform_initializer # can verify we did this right ... fan_in = self.input_size fan_out = self.output_size lim = np.sqrt(6. / (fan_in + fan_out)) weights = np.random.uniform(low=-lim, high=lim, size=self.size) self.weights = tf.Variable(weights, dtype=tf.float32) self.bias = tf.Variable(bias, dtype=tf.float32)
def __init__(self, input_shape, pool_shape, num_classes, name=None): self.input_shape = input_shape self.batch_size, self.h, self.w, self.fin = self.input_shape self.pool_shape = pool_shape self.num_classes = num_classes self.name = name l1 = AvgPool(size=self.input_shape, ksize=self.pool_shape, strides=self.pool_shape, padding='SAME') l2_input_shape = l1.output_shape() l2 = ConvToFullyConnected(input_shape=l2_input_shape) l3_input_shape = l2.output_shape() l3 = FullyConnected(input_shape=l3_input_shape, size=self.num_classes, init='alexnet', activation=Linear(), bias=0., name=self.name) self.B = Model(layers=[l1, l2, l3])
def main(): parser = argparse.ArgumentParser() parser.add_argument('--epochs', type=int, default=200) parser.add_argument('--batch_size', type=int, default=128) parser.add_argument('--alpha', type=float, default=1e-4) parser.add_argument('--beta', type=float, default=1e-4) #feedback weights, B, learning rate parser.add_argument('--sigma', type=float, default=0.1) #node pert standard deviation parser.add_argument('--l2', type=float, default=0.) parser.add_argument('--decay', type=float, default=1.) parser.add_argument('--eps', type=float, default=1e-5) parser.add_argument('--dropout', type=float, default=0.5) parser.add_argument('--act', type=str, default='tanh') parser.add_argument('--bias', type=float, default=0.1) parser.add_argument('--gpu', type=int, default=1) parser.add_argument('--dfa', type=int, default=1) parser.add_argument('--feedbacklearning', type=int, default=1) #Whether or not to learn feedback weights parser.add_argument('--sparse', type=int, default=0) parser.add_argument('--rank', type=int, default=0) parser.add_argument('--init', type=str, default="sqrt_fan_in") parser.add_argument('--opt', type=str, default="adam") parser.add_argument('--N', type=int, default=50) parser.add_argument('--save', type=int, default=0) parser.add_argument('--name', type=str, default="cifar10_conv_np") parser.add_argument('--load', type=str, default=None) args = parser.parse_args() if args.gpu >= 0: os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu) cifar10 = tf.keras.datasets.cifar10.load_data() ############################################## EPOCHS = args.epochs TRAIN_EXAMPLES = 50000 TEST_EXAMPLES = 10000 BATCH_SIZE = args.batch_size if args.act == 'tanh': act = Tanh() elif args.act == 'relu': act = Relu() else: assert (False) train_fc = True if args.load: train_conv = False else: train_conv = True weights_fc = None weights_conv = args.load #Setup the parameters attrs = ['sigma', 'alpha', 'beta'] log_scale = [True, True, True] ranges = [[-4, -1], [-6, -3], [-6, -3]] params = [] isnan = [] train_accs = [] test_accs = [] #Here we run a bunch of times for different parameters... for idx in range(args.N): #Choose some random parameters... param = set_random_hyperparameters(args, attrs, ranges, log_scale) params.append(param) if args.feedbacklearning == 0: args.beta = 0 #Tell me the params.... print('Alpha, beta, sigma are: ', args.alpha, args.beta, args.sigma) tf.set_random_seed(0) tf.reset_default_graph() batch_size = tf.placeholder(tf.int32, shape=()) dropout_rate = tf.placeholder(tf.float32, shape=()) learning_rate = tf.placeholder(tf.float32, shape=()) sigma = tf.placeholder(tf.float32, shape=(), name="Sigma") X = tf.placeholder(tf.float32, [None, 32, 32, 3]) X = tf.map_fn(lambda frame: tf.image.per_image_standardization(frame), X) Y = tf.placeholder(tf.float32, [None, 10]) l0 = Convolution(input_sizes=[batch_size, 32, 32, 3], filter_sizes=[5, 5, 3, 96], num_classes=10, init_filters=args.init, strides=[1, 1, 1, 1], padding="SAME", alpha=learning_rate, activation=act, bias=args.bias, last_layer=False, name='conv1', load=weights_conv, train=train_conv) l1 = MaxPool(size=[batch_size, 32, 32, 96], ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding="SAME") #Add perturbation to activity to get output to train feedback weights with l2p = NodePert(size=[batch_size, 16, 16, 96], sigma=sigma) l2 = FeedbackConv(size=[batch_size, 16, 16, 96], num_classes=10, sparse=args.sparse, rank=args.rank, name='conv1_fb') l3 = Convolution(input_sizes=[batch_size, 16, 16, 96], filter_sizes=[5, 5, 96, 128], num_classes=10, init_filters=args.init, strides=[1, 1, 1, 1], padding="SAME", alpha=learning_rate, activation=act, bias=args.bias, last_layer=False, name='conv2', load=weights_conv, train=train_conv) l4 = MaxPool(size=[batch_size, 16, 16, 128], ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding="SAME") l5p = NodePert(size=[batch_size, 8, 8, 128], sigma=sigma) l5 = FeedbackConv(size=[batch_size, 8, 8, 128], num_classes=10, sparse=args.sparse, rank=args.rank, name='conv2_fb') l6 = Convolution(input_sizes=[batch_size, 8, 8, 128], filter_sizes=[5, 5, 128, 256], num_classes=10, init_filters=args.init, strides=[1, 1, 1, 1], padding="SAME", alpha=learning_rate, activation=act, bias=args.bias, last_layer=False, name='conv3', load=weights_conv, train=train_conv) l7 = MaxPool(size=[batch_size, 8, 8, 256], ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding="SAME") l8p = NodePert(size=[batch_size, 4, 4, 256], sigma=sigma) l8 = FeedbackConv(size=[batch_size, 4, 4, 256], num_classes=10, sparse=args.sparse, rank=args.rank, name='conv3_fb') l9 = ConvToFullyConnected(shape=[4, 4, 256]) l10p = NodePert(size=[batch_size, 4 * 4 * 256], sigma=sigma) l10 = FullyConnected(size=[4 * 4 * 256, 2048], num_classes=10, init_weights=args.init, alpha=learning_rate, activation=act, bias=args.bias, last_layer=False, name='fc1', load=weights_fc, train=train_fc) l11 = Dropout(rate=dropout_rate) l12 = FeedbackFC(size=[4 * 4 * 256, 2048], num_classes=10, sparse=args.sparse, rank=args.rank, name='fc1_fb') l13p = NodePert(size=[batch_size, 2048], sigma=sigma) l13 = FullyConnected(size=[2048, 2048], num_classes=10, init_weights=args.init, alpha=learning_rate, activation=act, bias=args.bias, last_layer=False, name='fc2', load=weights_fc, train=train_fc) l14 = Dropout(rate=dropout_rate) l15 = FeedbackFC(size=[2048, 2048], num_classes=10, sparse=args.sparse, rank=args.rank, name='fc2_fb') l16 = FullyConnected(size=[2048, 10], num_classes=10, init_weights=args.init, alpha=learning_rate, activation=Linear(), bias=args.bias, last_layer=True, name='fc3', load=weights_fc, train=train_fc) ############################################## model = Model(layers=[ l0, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11, l12, l13, l14, l15, l16 ]) model_perturbed = Model(layers=[ l0, l1, l2p, l2, l3, l4, l5p, l5, l6, l7, l8p, l8, l9, l10p, l10, l11, l12, l13p, l13, l14, l15, l16 ]) predict = model.predict(X=X) predict_perturbed = model_perturbed.predict(X=X) ####### #Pairs of perturbations and feedback weights #feedbackpairs = [[l2p, l2], [l5p, l5], [l8p, l8], [l10p, l12], [l13p, l15]] #Test one at a time... this works, so it must be l10p, 12 pair that fails feedbackpairs = [[l2p, l2], [l5p, l5], [l8p, l8], [l13p, l15]] #Get noise, feedback matrices, and loss function and unperturbed loss function, to make update rule for feedback weights loss = tf.reduce_sum(tf.pow(tf.nn.softmax(predict) - Y, 2), 1) / 2 loss_perturbed = tf.reduce_sum( tf.pow(tf.nn.softmax(predict_perturbed) - Y, 2), 1) / 2 train_B = [] E = tf.nn.softmax(predict) - Y for idx, (noise, feedback) in enumerate(feedbackpairs): print(idx, batch_size, feedback.output_size) xi = tf.reshape(noise.get_noise(), (batch_size, feedback.output_size)) B = feedback.B lambd = tf.matmul( tf.diag(loss_perturbed - loss) / args.sigma / args.sigma, xi) np_error = tf.matmul(E, B) - lambd grad_B = tf.matmul(tf.transpose(E), np_error) new_B = B.assign(B - args.beta * grad_B) train_B.append(new_B) ####### weights = model.get_weights() if args.opt == "adam" or args.opt == "rms" or args.opt == "decay": if args.dfa: grads_and_vars = model.dfa_gvs(X=X, Y=Y) else: grads_and_vars = model.gvs(X=X, Y=Y) if args.opt == "adam": train = tf.train.AdamOptimizer( learning_rate=learning_rate, beta1=0.9, beta2=0.999, epsilon=args.eps).apply_gradients( grads_and_vars=grads_and_vars) elif args.opt == "rms": train = tf.train.RMSPropOptimizer( learning_rate=learning_rate, decay=0.99, epsilon=args.eps).apply_gradients( grads_and_vars=grads_and_vars) elif args.opt == "decay": train = tf.train.GradientDescentOptimizer( learning_rate=learning_rate).apply_gradients( grads_and_vars=grads_and_vars) else: assert (False) else: if args.dfa: train = model.dfa(X=X, Y=Y) else: train = model.train(X=X, Y=Y) correct = tf.equal(tf.argmax(predict, 1), tf.argmax(Y, 1)) total_correct = tf.reduce_sum(tf.cast(correct, tf.float32)) ############################################## sess = tf.InteractiveSession() tf.global_variables_initializer().run() tf.local_variables_initializer().run() (x_train, y_train), (x_test, y_test) = cifar10 x_train = x_train.reshape(TRAIN_EXAMPLES, 32, 32, 3) y_train = keras.utils.to_categorical(y_train, 10) x_test = x_test.reshape(TEST_EXAMPLES, 32, 32, 3) y_test = keras.utils.to_categorical(y_test, 10) ############################################## filename = args.name + '.results' f = open(filename, "w") f.write(filename + "\n") f.write("total params: " + str(model.num_params()) + "\n") f.close() ############################################## for ii in range(EPOCHS): if args.opt == 'decay' or args.opt == 'gd': decay = np.power(args.decay, ii) lr = args.alpha * decay else: lr = args.alpha print(ii) ############################# _count = 0 _total_correct = 0 #The training loop... here we add something to also update the feedback weights with the node pert for jj in range(int(TRAIN_EXAMPLES / BATCH_SIZE)): xs = x_train[jj * BATCH_SIZE:(jj + 1) * BATCH_SIZE] ys = y_train[jj * BATCH_SIZE:(jj + 1) * BATCH_SIZE] _correct, _ = sess.run( [total_correct, train], feed_dict={ sigma: 0.0, batch_size: BATCH_SIZE, dropout_rate: args.dropout, learning_rate: lr, X: xs, Y: ys }) #Add step to update B...... _ = sess.run( [train_B], feed_dict={ sigma: args.sigma, batch_size: BATCH_SIZE, dropout_rate: args.dropout, learning_rate: lr, X: xs, Y: ys }) _total_correct += _correct _count += BATCH_SIZE train_acc = 1.0 * _total_correct / _count train_accs.append(train_acc) ############################# _count = 0 _total_correct = 0 for jj in range(int(TEST_EXAMPLES / BATCH_SIZE)): xs = x_test[jj * BATCH_SIZE:(jj + 1) * BATCH_SIZE] ys = y_test[jj * BATCH_SIZE:(jj + 1) * BATCH_SIZE] _correct = sess.run(total_correct, feed_dict={ sigma: 0.0, batch_size: BATCH_SIZE, dropout_rate: 0.0, learning_rate: 0.0, X: xs, Y: ys }) _total_correct += _correct _count += BATCH_SIZE test_acc = 1.0 * _total_correct / _count test_accs.append(test_acc) isnan.append(None) #try: # trainer.train() #except ValueError: # print("Method fails to converge for these parameters") # isnan[n,m] = 1 #Save results... ############################# print("train acc: %f test acc: %f" % (train_acc, test_acc)) f = open(filename, "a") f.write("train acc: %f test acc: %f\n" % (train_acc, test_acc)) f.close() #Save params after each run fn = "./cifar10_conv_np_hyperparam_search_varalpha_septsearch_2_dfa_%d_fblearning_%d.npz" % ( args.dfa, args.feedbacklearning) to_save = { 'attr': attrs, 'params': params, 'train_accs': train_accs, 'test_accs': test_accs, 'isnan': isnan } pickle.dump(to_save, open(fn, "wb"))
l8p = NodePert(size=[batch_size, 4, 4, 256], sigma = sigma) l8 = FeedbackConv(size=[batch_size, 4, 4, 256], num_classes=10, sparse=args.sparse, rank=args.rank, name='conv3_fb') l9 = ConvToFullyConnected(shape=[4, 4, 256]) l10p = NodePert(size=[batch_size, 4*4*256], sigma = sigma) l10 = FullyConnected(size=[4*4*256, 2048], num_classes=10, init_weights=args.init, alpha=learning_rate, activation=act, bias=args.bias, last_layer=False, name='fc1', load=weights_fc, train=train_fc) l11 = Dropout(rate=dropout_rate) l12 = FeedbackFC(size=[4*4*256, 2048], num_classes=10, sparse=args.sparse, rank=args.rank, name='fc1_fb') l13p = NodePert(size=[batch_size, 2048], sigma = sigma) l13 = FullyConnected(size=[2048, 2048], num_classes=10, init_weights=args.init, alpha=learning_rate, activation=act, bias=args.bias, last_layer=False, name='fc2', load=weights_fc, train=train_fc) l14 = Dropout(rate=dropout_rate) l15 = FeedbackFC(size=[2048, 2048], num_classes=10, sparse=args.sparse, rank=args.rank, name='fc2_fb') l16 = FullyConnected(size=[2048, 10], num_classes=10, init_weights=args.init, alpha=learning_rate, activation=Linear(), bias=args.bias, last_layer=True, name='fc3', load=weights_fc, train=train_fc) ############################################## model = Model(layers=[l0, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11, l12, l13, l14, l15, l16]) model_perturbed = Model(layers=[l0, l1, l2p, l2, l3, l4, l5p, l5, l6, l7, l8p, l8, l9, l10p, l10, l11, l12, l13p, l13, l14, l15, l16]) predict = model.predict(X=X) predict_perturbed = model_perturbed.predict(X=X) ####### #Pairs of perturbations and feedback weights #feedbackpairs = [[l2p, l2], [l5p, l5], [l8p, l8], [l10p, l12], [l13p, l15]] #Test one at a time... this works, so it must be l10p, 12 pair that fails feedbackpairs = [[l2p, l2], [l5p, l5], [l8p, l8], [l13p, l15]]
activation=act, bias=args.bias, last_layer=False, name="fc3") l8 = Dropout(rate=dropout_rate) l9 = FeedbackFC(size=[1000, 1000], num_classes=10, sparse=args.sparse, rank=args.rank, name="fc3_fb") l10 = FullyConnected(size=[1000, 10], num_classes=10, init_weights=args.init, alpha=learning_rate, activation=Linear(), bias=args.bias, last_layer=True, name="fc4") model = Model(layers=[l0, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10]) ############################################## predict = model.predict(X=X) weights = model.get_weights() if args.opt == "adam" or args.opt == "rms" or args.opt == "decay": if args.dfa: grads_and_vars = model.dfa_gvs(X=X, Y=Y)
tf.set_random_seed(0) tf.reset_default_graph() batch_size = tf.placeholder(tf.int32, shape=()) dropout_rate = tf.placeholder(tf.float32, shape=()) learning_rate = tf.placeholder(tf.float32, shape=()) X = tf.placeholder(tf.float32, [None, 784]) Y = tf.placeholder(tf.float32, [None, 10]) l0 = FullyConnected(size=[784, 400], num_classes=10, init_weights=args.init, alpha=learning_rate, activation=Tanh(), bias=args.bias, l2=args.l2, last_layer=False, name="fc1") l1 = Dropout(rate=dropout_rate) l2 = FeedbackFC(size=[784, 400], num_classes=10, sparse=args.sparse, rank=args.rank, name="fc1_fb") l3 = FullyConnected(size=[400, 10], num_classes=10, init_weights=args.init, alpha=learning_rate, activation=Linear(), bias=args.bias, l2=args.l2, last_layer=True, name="fc2") model = Model(layers=[l0, l1, l2, l3]) ############################################## predict = model.predict(X=X) weights = model.get_weights() if args.opt == "adam" or args.opt == "rms" or args.opt == "decay": if args.dfa: grads_and_vars = model.dfa_gvs(X=X, Y=Y) else: grads_and_vars = model.gvs(X=X, Y=Y)