buck_width = BUCKET_WIDTH

buckets = [[] for i in range(NUM_BUCKETS)]

print("Splitting data into {0:d} buckets, each of width={1:d}".format(NUM_BUCKETS, buck_width))

with open(text_fname["fr"], "rb") as fr_file, open(text_fname["en"], "rb") as en_file:

for i, (line_fr, line_en) in enumerate(zip(fr_file, en_file), start=1):

if i > NUM_TRAINING_SENTENCES:

break

else:

if not CHAR_LEVEL:

fr_sent = line_fr.strip().split()

en_sent = line_en.strip().split()

else:

fr_sent = [c.encode() for c in list(line_fr.strip().decode())]

en_sent = [c.encode() for c in list(line_en.strip().decode())]

if len(fr_sent) > 0 and len(en_sent) > 0:

max_len = min(max(len(fr_sent), len(en_sent)),

MAX_PREDICT_LEN)

buck_indx = ((max_len-1) // buck_width)

fr_ids = [w2i["fr"].get(w, UNK_ID) for w in fr_sent[:max_len]]

en_ids = [w2i["en"].get(w, UNK_ID) for w in en_sent[:max_len]]

buckets[buck_indx].append((fr_ids, en_ids))

# Saving bucket data

print("Saving bucket data")

for i, bucket in enumerate(buckets):

print("Bucket {0:d}, # items={1:d}".format((i+1)*BUCKET_WIDTH, len(bucket)))

pickle.dump(bucket, open(bucket_data_fname.format(i+1), "wb"))

metrics = predict(s=NUM_TRAINING_SENTENCES,

num=NUM_DEV_SENTENCES, display=False, plot=False)

prec = np.sum(metrics["cp"]) / np.sum(metrics["tp"])

rec = np.sum(metrics["cp"]) / np.sum(metrics["t"])

f_score = 2 * (prec * rec) / (prec + rec)

print("{0:s}".format("-"*50))

print("{0:s} | {1:0.4f}".format("precision", prec))

print("{0:s} | {1:0.4f}".format("recall", rec))

loss = 0

num_words = 0

# with open(test_fname["fr"], "rb") as fr_file, open(test_fname["en"], "rb") as en_file:

with open(src_fname, "rb") as fr_file, open(tar_fname, "rb") as en_file:

with tqdm(total=num_sent) as pbar:

sys.stderr.flush()

out_str = "loss={0:.6f}".format(0)

pbar.set_description(out_str)

for i, (line_fr, line_en) in enumerate(zip(fr_file, en_file), start=1):

if i > num_sent:

break

if not CHAR_LEVEL:

fr_sent = line_fr.strip().split()

en_sent = line_en.strip().split()

else:

fr_sent = [c.encode() for c in list(line_fr.strip().decode())]

en_sent = [c.encode() for c in list(line_en.strip().decode())]

fr_ids = [w2i["fr"].get(w, UNK_ID) for w in fr_sent]

en_ids = [w2i["en"].get(w, UNK_ID) for w in en_sent]

if len(fr_ids) > 0 and len(en_ids) > 0:

# compute loss

curr_loss = float(model.encode_decode_train(fr_ids, en_ids, train=False).data)

loss += curr_loss

num_words += len(en_ids)

out_str = "loss={0:.6f}".format(curr_loss)

pbar.set_description(out_str)

pbar.update(1)

# end of for

# end of pbar

# end of with open file

loss_per_word = loss / num_words

pplx = 2 ** loss_per_word

random_pplx = vocab_size_en

print("{0:s}".format("-"*50))

print("{0:s} | {1:0.6f}".format("dev perplexity", pplx))

print("{0:s}".format("-"*50))

yield len(hypothesis)

yield len(reference)

for n in range(1,5):

s_ngrams = Counter([tuple(hypothesis[i:i+n]) for i in range(len(hypothesis)+1-n)])

r_ngrams = Counter([tuple(reference[i:i+n]) for i in range(len(reference)+1-n)])

yield max([sum((s_ngrams & r_ngrams).values()), 0])

if len(list(filter(lambda x: x==0, stats))) > 0:

return 0

(c, r) = stats[:2]

log_bleu_prec = sum([math.log(float(x)/y) for x,y in zip(stats[2::2],stats[3::2])]) / 4.

list_of_references = []

list_of_hypotheses = []

with open(src_fname, "rb") as fr_file, open(tar_fname, "rb") as en_file:

with tqdm(total=num_sent) as pbar:

sys.stderr.flush()

for i, (line_fr, line_en) in enumerate(zip(fr_file, en_file), start=1):

if i > num_sent:

break

out_str = "predicting sentence={0:d}".format(i)

pbar.update(1)

if not CHAR_LEVEL:

fr_sent = line_fr.strip().split()

else:

fr_sent = [c.encode() for c in list(line_fr.strip().decode())]

en_sent = line_en.strip().split()

fr_ids = [w2i["fr"].get(w, UNK_ID) for w in fr_sent]

reference_words = [w.decode() for w in line_en.strip().split()]

list_of_references.append(reference_words)

if len(fr_ids) > 0 and len(en_sent) > 0:

pred_sent, _ = model.encode_decode_predict(fr_ids)

pred_words = [i2w["en"][w].decode() for w in pred_sent if w != EOS_ID]

if CHAR_LEVEL:

pred_words = "".join(pred_words)

pred_words = pred_words.split()

else:

pred_words = []

list_of_hypotheses.append(pred_words)

stats = [0 for i in range(10)]

for (r,h) in zip(list_of_references, list_of_hypotheses):

stats = [sum(scores) for scores in zip(stats, bleu_stats(h,r))]

print("BLEU: %0.2f" % (100 * bleu(stats)))

batch_size=10, num_buckets=NUM_BUCKETS,

num_training=2,

bucket_width=BUCKET_WIDTH, log_mode="a", last_epoch_id=0):

# Set up log file for loss

log_train_fil = open(log_train_fil_name, mode=log_mode)

log_train_csv = csv.writer(log_train_fil, lineterminator="\n")

log_dev_fil = open(log_dev_fil_name, mode=log_mode)

log_dev_csv = csv.writer(log_dev_fil, lineterminator="\n")

# initialize perplexity on dev set

# save model when new epoch value is lower than previous

pplx = float("inf")

bleu_score = 0

sys.stderr.flush()

for epoch in range(num_epochs):

train_count = 0

with tqdm(total=num_training) as pbar:

sys.stderr.flush()

loss_per_epoch = 0

out_str = "epoch={0:d}, iter={1:d}, loss={2:.4f}, mean loss={3:.4f}, bucket={4:d}".format(

epoch+1, 0, 0, 0,0)

pbar.set_description(out_str)

for buck_indx in range(num_buckets):

bucket_data = pickle.load(open(bucket_data_fname.format(buck_indx+1), "rb"))

buck_pad_lim = (buck_indx+1) * bucket_width

for i in range(0, len(bucket_data), batch_size):

if train_count >= num_training:

break

next_batch_end = min(batch_size, (num_training-train_count))

# print("current batch")

# print(bucket_data[i:i+next_batch_end])

# print("bucket limit", buck_pad_lim)

curr_len = len(bucket_data[i:i+next_batch_end])

loss = model.encode_decode_train_batch(bucket_data[i:i+next_batch_end], buck_pad_lim, buck_pad_lim)

train_count += curr_len

# set up for backprop

model.cleargrads()

loss.backward()

# update parameters

optimizer.update()

# store loss value for display

loss_val = float(loss.data)

loss_per_epoch += loss_val

it = (epoch * NUM_TRAINING_SENTENCES) + curr_len

out_str = "epoch={0:d}, iter={1:d}, loss={2:.4f}, mean loss={3:.4f}, bucket={4:d}".format(

epoch+1, it, loss_val, (loss_per_epoch / (i+1)), (buck_indx+1))

pbar.set_description(out_str)

pbar.update(curr_len)

# log every 10 batches

if i % 10 == 0:

log_train_csv.writerow([it, loss_val])

if train_count >= num_training:

break

print("finished training on {0:d} sentences".format(num_training))

print("{0:s}".format("-"*50))

print("computing perplexity")

# pplx_new = compute_dev_pplx()

pplx_new = compute_pplx(dev_fname["fr"], dev_fname["en"], NUM_MINI_DEV_SENTENCES)

if pplx_new > pplx:

print("perplexity went up during training, breaking out of loop")

break

if (epoch+1) % ITERS_TO_SAVE == 0:

print("Saving model")

serializers.save_npz(model_fil.replace(".model", "_{0:d}.model".format(last_epoch_id+epoch+1)), model)

print("Finished saving model")

pplx = pplx_new

print(log_train_fil_name)

print(log_dev_fil_name)

print(model_fil.replace(".model", "_{0:d}.model".format(epoch+1)))

if (epoch+1) % ITERS_TO_SAVE == 0:

bleu_score = compute_bleu(dev_fname["fr"], dev_fname["en"], NUM_MINI_DEV_SENTENCES)

# log pplx and bleu score

log_dev_csv.writerow([(last_epoch_id+epoch+1), pplx_new, bleu_score])

log_train_fil.flush()

log_dev_fil.flush()

print("Simple predictions (╯°□°）╯︵ ┻━┻")

print("training set predictions")

_ = predict(s=0, num=2, plot=False)

print("Simple predictions (╯°□°）╯︵ ┻━┻")

print("dev set predictions")

_ = predict(s=NUM_TRAINING_SENTENCES, num=3, plot=False)

# print("{0:s}".format("-"*50))

# compute_bleu(dev_fname["fr"], dev_fname["en"], NUM_MINI_DEV_SENTENCES)

# print("{0:s}".format("-"*50))

print("Final saving model")

serializers.save_npz(model_fil, model)

print("Finished saving model")

# close log file

log_train_fil.close()

log_dev_fil.close()

print(log_train_fil_name)

print(log_dev_fil_name)

if os.path.exists(model_fname):

print("Loading model file: {0:s}".format(model_fname))

serializers.load_npz(model_fname, model)

else:

print("model file: {0:s} not found".format(model_fname))

if gpuid >= 0:

alpha_arr = cuda.to_cpu(alpha_arr).astype(np.float32)

#alpha_arr /= np.max(np.abs(alpha_arr),axis=0)

fig = plt.figure()

fig.set_size_inches(8, 8)

gs = gridspec.GridSpec(2, 2, width_ratios=[12,1],height_ratios=[12,1])

ax = plt.subplot(gs[0])

ax_c = plt.subplot(gs[1])

cmap = sns.light_palette((200, 75, 60), input="husl", as_cmap=True)

#prop = FontProperties(fname='fonts/IPAfont00303/ipam.ttf', size=12)

ax = sns.heatmap(alpha_arr, xticklabels=fr, yticklabels=en, ax=ax, cmap=cmap, cbar_ax=ax_c)

ax.xaxis.tick_top()

ax.yaxis.tick_right()

ax.set_xticklabels(en, minor=True, rotation=60, size=12)

for label in ax.get_xticklabels(minor=False):

label.set_fontsize(12)

#label.set_font_properties(prop)

for label in ax.get_yticklabels(minor=False):

label.set_fontsize(12)

label.set_rotation(-90)

label.set_horizontalalignment('left')

ax.set_xlabel("Source", size=20)

ax.set_ylabel("Hypothesis", size=20)

if plot_name:

if not CHAR_LEVEL:

fr_sent = line_fr.strip().split()

else:

fr_sent = [c.encode() for c in list(line_fr.strip().decode())]

fr_ids = [w2i["fr"].get(w, UNK_ID) for w in fr_sent]

# english reference is optional. If provided, compute precision/recall

if line_en:

en_sent = line_en.strip().split()

en_ids = [w2i["en"].get(w, UNK_ID) for w in en_sent]

pred_ids, alpha_arr = model.encode_decode_predict(fr_ids)

pred_words = [i2w["en"][w].decode() if w != EOS_ID else " _EOS" for w in pred_ids]

# print(pred_ids)

# print(pred_words)

prec = 0

rec = 0

filter_match = False

matches = count_match(en_ids, pred_ids)

if EOS_ID in pred_ids:

pred_len = len(pred_ids)-1

else:

pred_len = len(pred_ids)

# subtract 1 from length for EOS id

prec = (matches/pred_len) if pred_len > 0 else 0

rec = matches/len(en_ids)

if display and (prec >= p_filt and rec >= r_filt):

filter_match = True

# convert raw binary into string

# fr_words = [w.decode() for w in fr_sent]

print("{0:s}".format("-"*50))

print("sentence: {0:d}".format(line_num))

print("{0:s} | {1:80s}".format("Src", line_fr.strip().decode()))

print("{0:s} | {1:80s}".format("Ref", line_en.strip().decode()))

if not CHAR_LEVEL:

print("{0:s} | {1:80s}".format("Hyp", " ".join(pred_words)))

else:

print("{0:s} | {1:80s}".format("Hyp", "".join(pred_words)))

print("{0:s}".format("-"*50))

print("{0:s} | {1:0.4f}".format("precision", prec))

print("{0:s} | {1:0.4f}".format("recall", rec))

# if plot_name and use_attn:

# plot_attention(alpha_arr, fr_words, pred_words, plot_name)

print("English predictions, s={0:d}, num={1:d}:".format(s, num))

metrics = {"cp":[], "tp":[], "t":[]}

filter_count = 0

with open(fil_name["fr"], "rb") as fr_file, open(fil_name["en"], "rb") as en_file:

for i, (line_fr, line_en) in enumerate(zip(fr_file, en_file), start=0):

if i >= s and i < (s+num):

if plot:

plot_name = os.path.join(model_dir, "sample_{0:d}_plot.png".format(i+1))

else:

plot_name=None

# make prediction

cp, tp, t, f = predict_sentence(i, line_fr,

line_en,

display=display,

plot_name=plot_name,

p_filt=p_filt, r_filt=r_filt)

metrics["cp"].append(cp)

metrics["tp"].append(tp)

metrics["t"].append(t)

filter_count += (1 if f else 0)

print("sentences matching filter = {0:d}".format(filter_count))

# each list can have repeated elements. The count should account for this.

count1 = Counter(list1)

count2 = Counter(list2)

count2_keys = count2.keys()-set([UNK_ID, EOS_ID])

common_w = set(count1.keys()) & set(count2_keys)

#all_w = set(count1.keys()) + set(count2.keys())

matches = sum([min(count1[w], count2[w]) for w in common_w])

#matches = sum([max(0, count2[v]-count1[v]) for v in (count2-count1).values()])

#matches = sum([max(0, count2[v]-count1[v]) for v in common_w])

print("here", os.path.exists(model_fil))

if create_buckets_flag:

create_buckets()

else:

print("not creating buckets as requested. will crash if buckets not present")

max_epoch_id = 0

if os.path.exists(model_fil):

# check last saved epoch model:

for fname in [f for f in os.listdir(model_dir) if f.endswith("")]:

if model_fil != os.path.join(model_dir, fname) and model_fil.replace(".model", "") in os.path.join(model_dir, fname):

try:

epoch_id = int(fname.split("_")[-1].replace(".model", ""))

if epoch_id > max_epoch_id:

max_epoch_id = epoch_id

except:

print("{0:s} not a valid model file".format(fname))

print("last saved epoch model={0:d}".format(max_epoch_id))

if load_existing_model:

print("loading model ...")

serializers.load_npz(model_fil, model)

print("finished loading: {0:s}".format(model_fil))

else:

print("""model file already exists!!

Delete before continuing, or enable load_existing flag""".format(model_fil))

return

if epochs > 0:

#train_loop(text_fname, NUM_TRAINING_SENTENCES, NUM_EPOCHS)

batch_train_loop(bucket_data_fname,

num_epochs=epochs,

batch_size=BATCH_SIZE,

num_buckets=NUM_BUCKETS,

num_training=NUM_TRAINING_SENTENCES,

bucket_width=BUCKET_WIDTH, last_epoch_id=max_epoch_id)

# compute_pplx(dev_fname["fr"], dev_fname["en"], NUM_DEV_SENTENCES)

# compute_bleu(dev_fname["fr"], dev_fname["en"], NUM_DEV_SENTENCES)

# compute_pplx(dev_fname["fr"], dev_fname["en"], NUM_MINI_DEV_SENTENCES)