def translate(): processed_test = [] input_var = T.ftensor3('inputs') network,train_fn,val_fn,output,h_out = Models.Create_simple_GRU(input_var=input_var,N_HIDDEN=N_HIDDEN,layer=4,vocab=vocab) processed = [] if(os.path.isfile(weights+'.params')): print("loading Weights") params.read_model_data(network, weights) english_set = pd.read_csv('../data/test_english.txt',header=None,delimiter=',',names=['english','en_length']) french_set = pd.read_csv('../data/test_french.txt',header=None,delimiter=',',names=['french','fr_length']) print('Combining the files') combined_set = pd.concat([english_set,french_set],axis=1) print('Removing Duplicates') print(len(combined_set['french'])) combined_set = combined_set.drop_duplicates() print(len(combined_set['french'])) processed = create_training_set(combined_set,1,1) print('Grouping sentences together by input and output sentence length') #params.write_model_data(network, weights) # And a full pass over the validation data: val_err = 0 val_acc = 0 start_time = time.time() val_batches = 0 for batch in processed: curr_batch = helpers.convert_to_vector(batch,french_vocab,char_to_ix) fr,eng = helpers.shift_to_input(curr_batch,0,ix_to_vector) np.savetxt(open('hid','ab'),h_out(fr),delimiter=',') error,acc = val_fn(fr,eng[:,0]) val_err += error val_acc += acc val_batches += 1 length = 0 for word in range(1,curr_batch[0].shape[1]+2): length += 1 eng[:,0] = T.argmax(lasagne.layers.get_output(network,fr,allow_input_downcast=True),axis=1).eval().transpose() print ix_to_char[eng[0,0]], fr,eng = helpers.special_shift([fr,eng],0,ix_to_vector) error,acc = val_fn(fr,eng[:,0]) val_err += error val_acc += acc val_batches += 1 print batch['english'] # Then we print the results for this epoch: print("Epoch {} of {} took {:.3f}s".format( i, epoch, time.time() - start_time)) print(" validation loss:\t\t{:.6f}".format(val_err / val_batches)) print(" validation accuracy:\t\t{:.2f} %".format( val_acc / val_batches * 100)) params.write_model_data(network, weights)
def main(): input_var = T.ftensor3('inputs') network, train_fn, val_fn, output = Models.Create_simple_LSTM( input_var=input_var, N_HIDDEN=1000, layer=4) processed_batch = [] if (os.path.isfile('stored_weights.params')): print("loading Weights") params.read_model_data(network, weights) if (os.path.isfile('stored_batch.p') != True): if (os.path.isfile('stored_processed.p') != True): print('Creating processed sentences file') print('Loading english and french data files') english_set = pd.read_csv('../data/processed_en', header=None, delimiter=',', names=['english', 'en_length']) french_set = pd.read_csv('../data/processed_fr', header=None, delimiter=',', names=['french', 'fr_length']) print('Combining the files') combined_set = pd.concat([english_set, french_set], axis=1) print('Removing Duplicates') print(len(combined_set['french'])) combined_set = combined_set.drop_duplicates() print(len(combined_set['french'])) print( 'Grouping sentences together by input and output sentence length' ) processed = create_training_set(combined_set, 3, 100) print('Store batches in a pickle file') pickle.dump(processed, open('stored_processed.p', 'wb')) gc.collect() else: print('Loading grouped sentences') processed = pickle.load(open('stored_processed.p', 'rb')) #print('Creating matrix file for grouped sentences') gc.collect() #pool = mp.Pool(processes=2) #processed_batch = [pool.apply_async(helpers.convert_to_vector,args=(batch,french_vocab,char_to_ix)) for batch in processed] #processed_batch = [p.get() for p in processed_batch] #for batch in processed: # processed_batch.append(helpers.convert_to_vector(batch,french_vocab,char_to_ix)) #print(len(processed_batch)) #print('Dumping matrix data to file') #pickle.dump(processed_batch,open('stored_batch.p','wb')) else: print('Loading input and output matrix file') processed_batch = pickle.load(open('stored_batch.p', 'rb')) #print(ix_to_char) shuffle(processed) #Shuffle Batches processed_test = processed[:len(processed) - 500] processed_val = processed[len(processed) - 500:] for i in range(epoch): train_main_b = 0 train_err = 0 train_batches = 0 start_time = time.time() for batch in processed_test: curr_batch = helpers.convert_to_vector(batch, french_vocab, char_to_ix) fr, eng = helpers.shift_to_input(curr_batch, 0, ix_to_vector) train_err += train_fn(fr, eng[:, 0]) train_batches += 1 train_main_b += 1 print("new batch ", train_main_b, len(processed_test)) if (train_main_b % 2000 == 0): print("saving model", train_main_b) params.write_model_data(network, weights) for word in range(1, curr_batch[1].shape[1] - 1): #print(word) #print(T.argmax(lasagne.layers.get_output(network,fr,allow_input_downcast=True),axis=1).eval()) fr, eng = helpers.shift_to_input([fr, eng], word, ix_to_vector) train_err += train_fn(fr, eng[:, 0]) train_batches += 1 params.write_model_data(network, weights) # And a full pass over the validation data: val_err = 0 val_acc = 0 val_batches = 0 for batch in processed_val: curr_batch = helpers.convert_to_vector(batch, french_vocab, char_to_ix) fr, eng = helpers.shift_to_input(curr_batch, 0, ix_to_vector) error, acc = val_fn(fr, eng[:, 0]) val_err += error val_acc += acc val_batches += 1 for word in range(1, curr_batch[1].shape[1] - 1): fr, eng = helpers.shift_to_input([fr, eng], word, ix_to_vector) error, acc = val_fn(fr, eng[:, 0]) val_err += error val_acc += acc val_batches += 1 # Then we print the results for this epoch: print("Epoch {} of {} took {:.3f}s".format(epoch + 1, num_epochs, time.time() - start_time)) print(" training loss:\t\t{:.6f}".format(train_err / train_batches)) print(" validation loss:\t\t{:.6f}".format(val_err / val_batches)) print(" validation accuracy:\t\t{:.2f} %".format(val_acc / val_batches * 100)) network.save_weights_to("weights")
def main(): input_var = T.ftensor3('inputs') network,train_fn,val_fn,output = Models.Create_simple_LSTM(input_var=input_var,N_HIDDEN=1000,layer=4) processed_batch = [] if(os.path.isfile('stored_weights.params')): print("loading Weights") params.read_model_data(network, weights) if(os.path.isfile('stored_batch.p')!=True): if(os.path.isfile('stored_processed.p')!=True): print('Creating processed sentences file') print('Loading english and french data files') english_set = pd.read_csv('../data/processed_en',header=None,delimiter=',',names=['english','en_length']) french_set = pd.read_csv('../data/processed_fr',header=None,delimiter=',',names=['french','fr_length']) print('Combining the files') combined_set = pd.concat([english_set,french_set],axis=1) print('Removing Duplicates') print(len(combined_set['french'])) combined_set = combined_set.drop_duplicates() print(len(combined_set['french'])) print('Grouping sentences together by input and output sentence length') processed = create_training_set(combined_set,3,100) print('Store batches in a pickle file') pickle.dump(processed,open('stored_processed.p','wb')) gc.collect() else: print('Loading grouped sentences') processed = pickle.load(open('stored_processed.p','rb')) #print('Creating matrix file for grouped sentences') gc.collect() #pool = mp.Pool(processes=2) #processed_batch = [pool.apply_async(helpers.convert_to_vector,args=(batch,french_vocab,char_to_ix)) for batch in processed] #processed_batch = [p.get() for p in processed_batch] #for batch in processed: # processed_batch.append(helpers.convert_to_vector(batch,french_vocab,char_to_ix)) #print(len(processed_batch)) #print('Dumping matrix data to file') #pickle.dump(processed_batch,open('stored_batch.p','wb')) else: print('Loading input and output matrix file') processed_batch = pickle.load(open('stored_batch.p','rb')) #print(ix_to_char) shuffle(processed) #Shuffle Batches processed_test = processed[:len(processed)-500] processed_val = processed[len(processed)-500:] for i in range(epoch): train_main_b = 0 train_err = 0 train_batches = 0 start_time = time.time() for batch in processed_test: curr_batch = helpers.convert_to_vector(batch,french_vocab,char_to_ix) fr,eng = helpers.shift_to_input(curr_batch,0,ix_to_vector) train_err += train_fn(fr,eng[:,0]) train_batches += 1 train_main_b += 1 print("new batch ",train_main_b,len(processed_test)) if(train_main_b % 2000 == 0): print("saving model",train_main_b) params.write_model_data(network, weights) for word in range(1,curr_batch[1].shape[1]-1): #print(word) #print(T.argmax(lasagne.layers.get_output(network,fr,allow_input_downcast=True),axis=1).eval()) fr,eng = helpers.shift_to_input([fr,eng],word,ix_to_vector) train_err += train_fn(fr,eng[:,0]) train_batches += 1 params.write_model_data(network, weights) # And a full pass over the validation data: val_err = 0 val_acc = 0 val_batches = 0 for batch in processed_val: curr_batch = helpers.convert_to_vector(batch,french_vocab,char_to_ix) fr,eng = helpers.shift_to_input(curr_batch,0,ix_to_vector) error,acc = val_fn(fr,eng[:,0]) val_err += error val_acc += acc val_batches += 1 for word in range(1,curr_batch[1].shape[1]-1): fr,eng = helpers.shift_to_input([fr,eng],word,ix_to_vector) error,acc = val_fn(fr,eng[:,0]) val_err += error val_acc += acc val_batches += 1 # Then we print the results for this epoch: print("Epoch {} of {} took {:.3f}s".format( epoch + 1, num_epochs, time.time() - start_time)) print(" training loss:\t\t{:.6f}".format(train_err / train_batches)) print(" validation loss:\t\t{:.6f}".format(val_err / val_batches)) print(" validation accuracy:\t\t{:.2f} %".format( val_acc / val_batches * 100)) network.save_weights_to("weights")