def translate():
processed_test = []
input_var = T.ftensor3('inputs')
network,train_fn,val_fn,output,h_out = Models.Create_simple_GRU(input_var=input_var,N_HIDDEN=N_HIDDEN,layer=4,vocab=vocab)
processed = []
if(os.path.isfile(weights+'.params')):
print("loading Weights")
params.read_model_data(network, weights)
english_set = pd.read_csv('../data/test_english.txt',header=None,delimiter=',',names=['english','en_length'])
french_set = pd.read_csv('../data/test_french.txt',header=None,delimiter=',',names=['french','fr_length'])
print('Combining the files')
combined_set = pd.concat([english_set,french_set],axis=1)
print('Removing Duplicates')
print(len(combined_set['french']))
combined_set = combined_set.drop_duplicates()
print(len(combined_set['french']))
processed = create_training_set(combined_set,1,1)
print('Grouping sentences together by input and output sentence length')
#params.write_model_data(network, weights)
# And a full pass over the validation data:
val_err = 0
val_acc = 0
start_time = time.time()
val_batches = 0
for batch in processed:
curr_batch = helpers.convert_to_vector(batch,french_vocab,char_to_ix)
fr,eng = helpers.shift_to_input(curr_batch,0,ix_to_vector)
np.savetxt(open('hid','ab'),h_out(fr),delimiter=',')
error,acc = val_fn(fr,eng[:,0])
val_err += error
val_acc += acc
val_batches += 1
length = 0
for word in range(1,curr_batch[0].shape[1]+2):
length += 1
eng[:,0] = T.argmax(lasagne.layers.get_output(network,fr,allow_input_downcast=True),axis=1).eval().transpose()
print ix_to_char[eng[0,0]],
fr,eng = helpers.special_shift([fr,eng],0,ix_to_vector)
error,acc = val_fn(fr,eng[:,0])
val_err += error
val_acc += acc
val_batches += 1
print batch['english']
# Then we print the results for this epoch:
print("Epoch {} of {} took {:.3f}s".format(
i, epoch, time.time() - start_time))
print(" validation loss:\t\t{:.6f}".format(val_err / val_batches))
print(" validation accuracy:\t\t{:.2f} %".format(
val_acc / val_batches * 100))
params.write_model_data(network, weights)
def main():
input_var = T.ftensor3('inputs')
network, train_fn, val_fn, output = Models.Create_simple_LSTM(
input_var=input_var, N_HIDDEN=1000, layer=4)
processed_batch = []
if (os.path.isfile('stored_weights.params')):
print("loading Weights")
params.read_model_data(network, weights)
if (os.path.isfile('stored_batch.p') != True):
if (os.path.isfile('stored_processed.p') != True):
print('Creating processed sentences file')
print('Loading english and french data files')
english_set = pd.read_csv('../data/processed_en',
header=None,
delimiter=',',
names=['english', 'en_length'])
french_set = pd.read_csv('../data/processed_fr',
header=None,
delimiter=',',
names=['french', 'fr_length'])
print('Combining the files')
combined_set = pd.concat([english_set, french_set], axis=1)
print('Removing Duplicates')
print(len(combined_set['french']))
combined_set = combined_set.drop_duplicates()
print(len(combined_set['french']))
print(
'Grouping sentences together by input and output sentence length'
)
processed = create_training_set(combined_set, 3, 100)
print('Store batches in a pickle file')
pickle.dump(processed, open('stored_processed.p', 'wb'))
gc.collect()
else:
print('Loading grouped sentences')
processed = pickle.load(open('stored_processed.p', 'rb'))
#print('Creating matrix file for grouped sentences')
gc.collect()
#pool = mp.Pool(processes=2)
#processed_batch = [pool.apply_async(helpers.convert_to_vector,args=(batch,french_vocab,char_to_ix)) for batch in processed]
#processed_batch = [p.get() for p in processed_batch]
#for batch in processed:
# processed_batch.append(helpers.convert_to_vector(batch,french_vocab,char_to_ix))
#print(len(processed_batch))
#print('Dumping matrix data to file')
#pickle.dump(processed_batch,open('stored_batch.p','wb'))
else:
print('Loading input and output matrix file')
processed_batch = pickle.load(open('stored_batch.p', 'rb'))
#print(ix_to_char)
shuffle(processed) #Shuffle Batches
processed_test = processed[:len(processed) - 500]
processed_val = processed[len(processed) - 500:]
for i in range(epoch):
train_main_b = 0
train_err = 0
train_batches = 0
start_time = time.time()
for batch in processed_test:
curr_batch = helpers.convert_to_vector(batch, french_vocab,
char_to_ix)
fr, eng = helpers.shift_to_input(curr_batch, 0, ix_to_vector)
train_err += train_fn(fr, eng[:, 0])
train_batches += 1
train_main_b += 1
print("new batch ", train_main_b, len(processed_test))
if (train_main_b % 2000 == 0):
print("saving model", train_main_b)
params.write_model_data(network, weights)
for word in range(1, curr_batch[1].shape[1] - 1):
#print(word)
#print(T.argmax(lasagne.layers.get_output(network,fr,allow_input_downcast=True),axis=1).eval())
fr, eng = helpers.shift_to_input([fr, eng], word, ix_to_vector)
train_err += train_fn(fr, eng[:, 0])
train_batches += 1
params.write_model_data(network, weights)
# And a full pass over the validation data:
val_err = 0
val_acc = 0
val_batches = 0
for batch in processed_val:
curr_batch = helpers.convert_to_vector(batch, french_vocab,
char_to_ix)
fr, eng = helpers.shift_to_input(curr_batch, 0, ix_to_vector)
error, acc = val_fn(fr, eng[:, 0])
val_err += error
val_acc += acc
val_batches += 1
for word in range(1, curr_batch[1].shape[1] - 1):
fr, eng = helpers.shift_to_input([fr, eng], word, ix_to_vector)
error, acc = val_fn(fr, eng[:, 0])
val_err += error
val_acc += acc
val_batches += 1
# Then we print the results for this epoch:
print("Epoch {} of {} took {:.3f}s".format(epoch + 1, num_epochs,
time.time() - start_time))
print(" training loss:\t\t{:.6f}".format(train_err / train_batches))
print(" validation loss:\t\t{:.6f}".format(val_err / val_batches))
print(" validation accuracy:\t\t{:.2f} %".format(val_acc /
val_batches * 100))
network.save_weights_to("weights")
