def deepnet(Ws, xTr, yTr, wst, transname='sigmoid'): entry = np.cumsum(wst[0:-1] * wst[1:] + wst[0:-1]) # entry points into weights if Ws.size == 0: Ws = np.random.randn(entry[-1], 1) / 2 W = [] e = 0 for i in range(len(entry)): W.append(np.reshape(Ws[e:entry[i]], [wst[i], wst[i + 1] + 1])) e = entry[i] #??? # print(Ws.shape) trans_func, trans_func_der = get_transition_func(transname) aas, zzs = forward_pass(W, xTr, trans_func) if len(yTr) == 0: loss = zzs[0] return loss loss = compute_loss(zzs, yTr) gradientList = backprop(W, aas, zzs, yTr, trans_func_der) #print(gradientList[0]) #%% reformat the gradient from a cell-array of matrices to one vector gradient = np.zeros((entry[-1], 1)) e = 0 for i in range(len(entry)): gradient[e:entry[i]] = np.reshape(gradientList[i], (entry[i] - e, 1)) e = entry[i] return loss, gradient
def train_to_dataset(): tree_dataframes = [] for filename in os.listdir('../training_data'): if filename.endswith(".dis"): # try: print(filename) # Build RST tree T = tree_builder.buildtree_from_train(filename) # Binarize the RST tree T = tree_builder.binarizetree(T) # Back-propagating information from # leaf node to root node T = backprop.backprop(T) if filename.split('.')[0] in [ 'file1', 'file2', 'file3', 'file4', 'file5' ]: edu_list = open( '/Users/tal/Desktop/rst_parser_toolkit/training_data/' + filename.split('.')[0] + '.edus', 'r').read().replace(' ', '').split('\n') else: edu_list = open( '/Users/tal/Desktop/rst_parser_toolkit/training_data/' + filename.split('.')[0] + '.out.edus', 'r').read().replace(' ', '').split('\n') decision_set = tree_to_classification_decisions(T, edu_list) tree_dataset = class_decisions_to_dataset(decision_set, edu_list) tree_dataframes.append(tree_dataset) # except: # continue total_dataset = pd.concat(tree_dataframes) total_dataset.to_csv('shift_reduce_dataset7.csv', index=False)
def deriv(x, network, data, targets, error_func=error, dE_func=dE_cross_entropy): '''Network error derivatives using backprop. x is a single vector with all of a network's parameters.''' network.set_parameters(x) a,b = backprop(network, data, targets, tau=0, dE_func= dE_func) flat_grad = [] for matrix in a: flat_grad = concatenate((flat_grad, matrix.flatten()),1) for matrix in b: flat_grad = concatenate((flat_grad, matrix.flatten()),1) return flat_grad
def train_to_gold(): for filename in os.listdir('training_data'): if filename.endswith(".dis"): print(filename) # text = open('training_data/{}'.format(filename), 'r').read() # Build RST tree T = tree_builder.buildtree_from_train( 'training_data/{}'.format(filename)) # Binarize the RST tree T = tree_builder.binarizetree(T) # Back-propagating information from # leaf node to root node T = backprop.backprop(T) file_prefix = filename.partition(".")[0] tree_to_dev(T, 'GOLD', file_prefix)
def deriv(x, network, data, targets, error_func=error, dE_func=dE_cross_entropy): '''Network error derivatives using backprop. x is a single vector with all of a network's parameters.''' network.set_parameters(x) a, b = backprop(network, data, targets, tau=0, dE_func=dE_func) flat_grad = [] for matrix in a: flat_grad = concatenate((flat_grad, matrix.flatten()), 1) for matrix in b: flat_grad = concatenate((flat_grad, matrix.flatten()), 1) return flat_grad
# stocahstic gradient descent y_hat = forwardPass(nn, X) # Perform forward pass on whole data set L.append( 0.5 * np.mean((Y - y_hat)**2) ) # Compute the MSE loss at this iteration for the entire training data set (not just the samples chosen) randInd = np.random.randint( 0, high=n, size=K) # Randomly sample a minibatch of size K Y_temp = Y[:, randInd] # Get the label X_temp = X[:, randInd] # Get the input features Y_hat_temp = forwardPass(nn, X_temp) # Get predicted label g = backprop(nn, X_temp, Y_temp) # Compute the gradient for the neural network for j in range(len(nn)): nn[j][0] = nn[j][0] - alpha * g[j][0] nn[j][ 1] = nn[j][1] - alpha * g[j][1] # Update the neural network biases # Plot the loss after every 5 iteration vs. the iteration number. print('Iteration: ', i) if i % 1000 == 999: fig, axes = plt.subplots(1, 1) axes.semilogx(range(1, i + 1), L[:i], '.-g') axes.set_xlabel('Iterations') axes.set_ylabel('Loss') axes.set_yscale('log') fig.canvas.draw()
# ============================================================================= train=open( "train_input_part3.p", "rb" ) test=open( "test_input_part3.p", "rb" ) target=open( "target_part3.p", "rb" ) train_input_sec5=pickle.load( train ) test_input_sec5=pickle.load( test ) target_sec5=pickle.load( target ) train.close() test.close() target.close() etta=0.5 train_itt_num=1000 a=backprop(14**2,10,16) # ============================================================================= ##uncomment the following section if the the network has already been trained ##and weights of the trained network, pre saved at the parent directory a.train(train_input_sec5,target_sec5,train_itt_num,etta,'sigmoid','online',mu=0) #a.save('part4') # ============================================================================= a.load('part4') errors = (np.argmax(target_sec5,axis=1)!=np.argmax(a.test(test_input_sec5),axis=1))*1 print('Total Misses: %d/%d (%.2f%%)'%(sum(errors),len(errors),(100*sum(errors)/len(errors)))) #############confusion matrix########## conf_mat=np.zeros((10,10)) ideal_mat=np.zeros((10,10)) for num in range(10): for i in range(250):
if __name__ == '__main__': # edu = 'cat dog worm' # vec = get_mean_word_vec(edu) # print(vec) filename = '0600.out.dis' edu_list = open('training_data/' + filename.split('.')[0] + '.out.edus', 'r').read().replace(' ', '').split('\n') filename = 'training_data/0600.out.dis' T = tree_builder.buildtree_from_train(filename) # Binarize the RST tree T = tree_builder.binarizetree(T) # Back-propagating information from # leaf node to root node T = backprop.backprop(T) list_of_nodes = backprop._BFTbin(T) # result = get_dist_from_start_for_queue(edu_list[1], edu_list) # print(result) # result = get_dist_from_start_for_stack(list_of_nodes[1]) # print(list_of_nodes[1].eduspan) # print(result) # result = get_dist_from_end_for_stack(list_of_nodes[1], len(edu_list)) # print(result) # result = get_dist_from_end_for_queue(edu_list[1], edu_list) # print(result) # print(list_of_nodes[1].eduspan) # print(list_of_nodes[3].eduspan) # result = get_dist_between_edus_for_stack(list_of_nodes[2], list_of_nodes[3]) # print(result)
from backprop import backprop import numpy as np #####################XOR gate############################ i=np.array([[1, 1], [0, 1], [1, 0], [0, 0]]) targets=np.array([[0], [1], [1], [0]]) etta=0.5 a=backprop(2,1,3) a.train(i,targets,10000,etta,'sigmoid') print('XOR gate classification:') print('input: [1 1] output: {}'.format(a.test(i,targets)[0])) print('input: [0 1] output: {}'.format(a.test(i,targets)[1])) print('input: [1 0] output: {}'.format(a.test(i,targets)[2])) print('input: [0 0] output: {}'.format(a.test(i,targets)[3]))
target = np.zeros((grayscale.shape[0], grayscale.shape[1], 3)) target[:, :, 0] = r target[:, :, 1] = g target[:, :, 2] = b loss = bp.compute_loss(layer_output_wo_activation[layers], target) # loss = bp.compute_loss(conv.sigmoid_activation(layer_output_wo_activation[layers]), target) # print sum(loss[:, :, 0]) curr_loss = loss is_output_layer = True for l in range(0, layers): print "Backward pass layer number: ", l delta_x_matrix = bp.backprop(curr_loss, layer_output[layers - 1 - l], layer_output_wo_activation[layers - l], W[layers - 1 - l], is_output_layer) curr_loss = delta_x_matrix is_output_layer = False pyplot.imshow(layer_output[layers]) pyplot.show() # temp = layer_output[layers] * 255 # print "temp ", temp # newtemp = np.array(temp, dtype=np.uint8) # outt = Image.fromarray(newtemp) # outt.show() # filename = str("sigmoid_try3_iter_") + str(i) + str(".png") # outt.save(filename)
HIDGENBIASES = VISBIASES print('Pretraining Layer 3 with RBM: {}-{}'.format(NUMPEN, NUMPEN2)) BATCH_DATA = BATCHPOSHIDPROBS NUM_HID = NUMPEN2 RESTART = 1 BATCHPOSHIDPROBS, VISHID, HIDBIASES, VISBIASES = rbm(BATCH_DATA, RESTART, NUM_HID, NUM_DIMS, MAX_EPOCH, RANDOMIN, VISHID) HIDPEN2 = VISHID PENRECBIASES2 = HIDBIASES HIDGENBIASES2 = VISBIASES print('Pretraining Layer 4 with RBM: {}-{}'.format(NUMPEN2, NUMOPEN)) BATCH_DATA = BATCHPOSHIDPROBS NUM_HID = NUMOPEN RESTART = 1 VISHID, HIDBIASES, VISBIASES = rbmhidlinear(BATCH_DATA, RESTART, NUM_HID, NUM_DIMS, MAX_EPOCH, RANDOMIN, VISHID) HIDTOP = VISHID TOPRECBIASES = HIDBIASES TOPGENBIASES = VISBIASES print('END of Pretraining Layer 4 with RBM: {}-{}'.format(NUMPEN2, NUMOPEN)) ERR = backprop(VISHID__, VISBIASES__, PENRECBIASES, PENRECBIASES2, HIDRECBIASES, HIDPEN, HIDPEN2, HIDGENBIASES, HIDGENBIASES2, HIDTOP, TOPRECBIASES, TOPGENBIASES)
train = np.genfromtxt("train_x.csv", delimiter=",") test = np.genfromtxt("test_x.csv", delimiter=",") train_label = np.genfromtxt("train_y.csv", delimiter=',') train_label = np.expand_dims(train_label, axis=1) test_label = np.genfromtxt("test_y.csv", delimiter=',') test_label = np.expand_dims(test_label, axis=1) # print(train_label.shape) # print("Loading finished") model = bp.backprop(train, train_label, 2, 50, max_iter=250, eta=0.0002, f_type="R", b_type="R", b=16, disp_step=10) # model.fit() # model.baseline(test, test_label) err = model.score(test, test_label) # x = [np.arange(-2, 2, 0.001)] * y_hat_train = model.predict(train) plt.scatter(np.sum(train[:, [4, 21, 38, 55, 72]], axis=1), y_hat_train, label='Predicted') plt.scatter(np.sum(train[:, [4, 21, 38, 55, 72]], axis=1),
onehot[i, int(data[i])] = 1 return onehot os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' tf.logging.set_verbosity(tf.logging.ERROR) train = np.genfromtxt("mnist_train.csv", delimiter = ",") test = np.genfromtxt("mnist_test.csv", delimiter = ",") print("Loading finished") train_label = train[:, 0] train_label = OneHotEncoder(train_label) test_label = test[:, 0] test_label = OneHotEncoder(test_label) print('converted labels') model = bp.backprop(train[:, 1:]/255, train_label, 2, 256, max_iter = 100, eta = 0.002, f_type = "C", b_type = "R", b = 100, disp_step = 10) model.fit() model.baseline(test[:, 1:], test_label) err = model.score(test[:, 1:]/255, test_label) target_num = np.asarray([0, 0, 0, 1, 0, 0, 0, 0, 0, 0]) target_num = np.reshape(target_num, (1, 10)) image = model.predict(target_num, direction = "B") disp = np.reshape(image, (28, 28)) disp = disp*255 seven = disp.astype(int) seven[seven > 255] = 255 seven[seven < 0] = 0 print(seven) plt.imshow(seven) plt.imsave(os.getcwd() + "/result3.png", seven, cmap = plt.get_cmap('gray')) print(err)
for example in range(250): if num == 0 and example == 0: target_sec2 = 0 elif num != 2: target_sec2 = np.vstack((target_sec2, np.array([0]))) else: target_sec2 = np.vstack((target_sec2, np.array([1]))) # lets see the false rates: Twos = test_input[500:750, :] notTwos = test_input[0:500, :] notTwos = np.vstack((notTwos, test_input[750:2501, :])) etta = 0.5 train_itt_num = 200 a = backprop(14**2, 1, 5) # ============================================================================= ##uncomment the following section if the the network has already been trained ##and weights of the trained network, pre saved at the parent directory #a.train(train_input,target_sec2,train_itt_num,etta,'sigmoid') #a.save('part2') # ============================================================================= a.load('part2') print( 'for {} perceptron itterations, and Batch algorithm (better than Online):'. format(train_itt_num)) errors = (target_sec2 != a.test(test_input)) * 1 false_pos = ((target_sec2 != a.test(test_input)) * 1) * (target_sec2 == 1) print('Total Misses: %d/%d (%.2f%%)' % (sum(errors), len(errors), (100 * sum(errors) / len(errors)))) print(
# Write to the log: print("Training set has {0[0]} rows and {0[1]} columns".format(train.shape)) print("Test set has {0[0]} rows and {0[1]} columns".format(test.shape)) # Any files you write to the current directory get shown as outputs results_raw = train.loc[:, 'label'] results = np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0][9-r:19-r] for r in results_raw.values]) inputs = train.loc[:, 'pixel0':'pixel783'] / 256 print("Input set has {0[0]} rows and {0[1]} columns".format(inputs.shape)) print("Result set has {0[0]} rows and 1 columns".format(results.shape)) print("Index : {0}".format(train.index)) print("Columns : {0}".format(train.columns)) print("Range of values : inputs [{0}:{1}], results [{2}:{3}]".format(np.min(np.min(inputs)), np.max(np.max(inputs)), np.min(results), np.max(results))) # Need to set max_training_examples to limit memory usage # Train on each example twice bp = backprop(inputs.shape[1], 10, hidden_nodes_1 = 100, hidden_nodes_2 = 10, max_training_examples = 42, training_cycles = 100) bp.train(inputs.values, results) test_inputs = test.values / 256 print("Range of test values : {0} - {1}".format(np.min(np.min(test_inputs)), np.max(np.max(test_inputs)))) test_outputs_raw = bp.classify(test_inputs) test_outputs = [np.argmax(outs) for outs in test_outputs_raw] print("Range of {0} results = [{1}:{2}]".format(len(test_outputs), np.min(test_outputs), np.max(test_outputs))) print(test_outputs_raw) print("Weights : 1 = {0}, 2 = {1}, 3 = {2}".format(bp.weights1, bp.weights2, bp.weights3))
from backprop import backprop from forward import forward import numpy as np from numpy import genfromtxt # fetch data data = genfromtxt('winequality-red.csv', delimiter=';') # Standardize data for i in range(11): data[:, i] = (data[:, i] - np.mean(data[1:800, i])) / np.std(data[1:800, i]) + 1 # split test / train X_train = data[1:801, 0:11] y_train = data[1:801, [11]] X_test = data[800:1600, 0:11] y_test = data[800:1600, [11]] # train model using backpropogation w1, w2, error_over_time = backprop(X=X_train, y=y_train, M=30, iters=1000, eta=0.025) # test model Z, y_test_pred = forward(X_test, w1, w2) # evaluate total accuracy print(np.sqrt(np.mean(np.square(y_test_pred - y_test))))