def roll_vocab_update_dicts(self, new_large2small_src, new_large2small_trgt): # Update dictionaries logger.debug("Called roll_vocab_update_dicts()") self.model.large2small_src = new_large2small_src self.model.large2small_trgt = new_large2small_trgt self.model.small2large_src = invert_dict(self.model.large2small_src) self.model.small2large_trgt = invert_dict(self.model.large2small_trgt) self.model.word_indxs_src = {} # small index to word self.model.word_indxs = {} for small in self.model.small2large_src: large = self.model.small2large_src[small] self.model.word_indxs_src[small] = self.model.large2word_src[large] for small in self.model.small2large_trgt: large = self.model.small2large_trgt[small] self.model.word_indxs[small] = self.model.large2word_trgt[large]
rolling_vocab_dict = cPickle.load(f) total_num_batches = max(rolling_vocab_dict) Dx_shelve = shelve.open(state['Dx_file']) Dy_shelve = shelve.open(state['Dy_file']) step_modulo = step % total_num_batches if step_modulo in rolling_vocab_dict: # 0 always in. cur_key = step_modulo else: cur_key = 0 for key in rolling_vocab_dict: if (key < step_modulo) and (key > cur_key): # Find largest key smaller than step_modulo cur_key = key large2small_src = Dx_shelve[str(cur_key)] large2small_trgt = Dy_shelve[str(cur_key)] small2large_src = invert_dict(large2small_src) small2large_trgt = invert_dict(large2small_trgt) restricted_list = ['W_0_enc_approx_embdr', 'W_0_dec_approx_embdr', 'W2_dec_deep_softmax', 'b_dec_deep_softmax'] d = {} for elt in restricted_list: temp = cur_model[elt] arr = cur_large['large_' + elt] if '_enc_' in elt: for large in large2small_src: small = large2small_src[large] arr[large] = temp[small] else: for large in large2small_trgt:
total_num_batches = max(rolling_vocab_dict) Dx_shelve = shelve.open(state['Dx_file']) Dy_shelve = shelve.open(state['Dy_file']) step_modulo = step % total_num_batches if step_modulo in rolling_vocab_dict: # 0 always in. cur_key = step_modulo else: cur_key = 0 for key in rolling_vocab_dict: if (key < step_modulo) and ( key > cur_key): # Find largest key smaller than step_modulo cur_key = key large2small_src = Dx_shelve[str(cur_key)] large2small_trgt = Dy_shelve[str(cur_key)] small2large_src = invert_dict(large2small_src) small2large_trgt = invert_dict(large2small_trgt) restricted_list = [ 'W_0_enc_approx_embdr', 'W_0_dec_approx_embdr', 'W2_dec_deep_softmax', 'b_dec_deep_softmax' ] d = {} for elt in restricted_list: temp = cur_model[elt] arr = cur_large['large_' + elt] if '_enc_' in elt: for large in large2small_src: small = large2small_src[large]