def roll_vocab_update_dicts(self, new_large2small_src, new_large2small_trgt):
# Update dictionaries
logger.debug("Called roll_vocab_update_dicts()")
self.model.large2small_src = new_large2small_src
self.model.large2small_trgt = new_large2small_trgt
self.model.small2large_src = invert_dict(self.model.large2small_src)
self.model.small2large_trgt = invert_dict(self.model.large2small_trgt)
self.model.word_indxs_src = {} # small index to word
self.model.word_indxs = {}
for small in self.model.small2large_src:
large = self.model.small2large_src[small]
self.model.word_indxs_src[small] = self.model.large2word_src[large]
for small in self.model.small2large_trgt:
large = self.model.small2large_trgt[small]
self.model.word_indxs[small] = self.model.large2word_trgt[large]
def roll_vocab_update_dicts(self, new_large2small_src, new_large2small_trgt):
# Update dictionaries
logger.debug("Called roll_vocab_update_dicts()")
self.model.large2small_src = new_large2small_src
self.model.large2small_trgt = new_large2small_trgt
self.model.small2large_src = invert_dict(self.model.large2small_src)
self.model.small2large_trgt = invert_dict(self.model.large2small_trgt)
self.model.word_indxs_src = {} # small index to word
self.model.word_indxs = {}
for small in self.model.small2large_src:
large = self.model.small2large_src[small]
self.model.word_indxs_src[small] = self.model.large2word_src[large]
for small in self.model.small2large_trgt:
large = self.model.small2large_trgt[small]
self.model.word_indxs[small] = self.model.large2word_trgt[large]
rolling_vocab_dict = cPickle.load(f)
total_num_batches = max(rolling_vocab_dict)
Dx_shelve = shelve.open(state['Dx_file'])
Dy_shelve = shelve.open(state['Dy_file'])
step_modulo = step % total_num_batches
if step_modulo in rolling_vocab_dict: # 0 always in.
cur_key = step_modulo
else:
cur_key = 0
for key in rolling_vocab_dict:
if (key < step_modulo) and (key > cur_key): # Find largest key smaller than step_modulo
cur_key = key
large2small_src = Dx_shelve[str(cur_key)]
large2small_trgt = Dy_shelve[str(cur_key)]
small2large_src = invert_dict(large2small_src)
small2large_trgt = invert_dict(large2small_trgt)
restricted_list = ['W_0_enc_approx_embdr', 'W_0_dec_approx_embdr', 'W2_dec_deep_softmax', 'b_dec_deep_softmax']
d = {}
for elt in restricted_list:
temp = cur_model[elt]
arr = cur_large['large_' + elt]
if '_enc_' in elt:
for large in large2small_src:
small = large2small_src[large]
arr[large] = temp[small]
else:
for large in large2small_trgt:
total_num_batches = max(rolling_vocab_dict)
Dx_shelve = shelve.open(state['Dx_file'])
Dy_shelve = shelve.open(state['Dy_file'])
step_modulo = step % total_num_batches
if step_modulo in rolling_vocab_dict: # 0 always in.
cur_key = step_modulo
else:
cur_key = 0
for key in rolling_vocab_dict:
if (key < step_modulo) and (
key > cur_key): # Find largest key smaller than step_modulo
cur_key = key
large2small_src = Dx_shelve[str(cur_key)]
large2small_trgt = Dy_shelve[str(cur_key)]
small2large_src = invert_dict(large2small_src)
small2large_trgt = invert_dict(large2small_trgt)
restricted_list = [
'W_0_enc_approx_embdr', 'W_0_dec_approx_embdr', 'W2_dec_deep_softmax',
'b_dec_deep_softmax'
]
d = {}
for elt in restricted_list:
temp = cur_model[elt]
arr = cur_large['large_' + elt]
if '_enc_' in elt:
for large in large2small_src:
small = large2small_src[large]
