def init(path):
##------------------------------------------------------------------------------##
# Compile and load the model. #
# Compile the encoder. #
##------------------------------------------------------------------------------##
state = prototype_state()
state_path = path + "_state.pkl"
model_path = path + "_model.npz"
with open(state_path) as src:
state.update(pickle.load(src))
logging.basicConfig(
level=getattr(logging, state['level']),
format="%(asctime)s: %(name)s: %(levelname)s: %(message)s")
model = DialogEncoderDecoder(state)
if os.path.isfile(model_path):
logger.debug("Loading previous model")
model.load(model_path)
else:
raise Exception("Must specify a valid model path")
encoding_function = model.build_encoder_function()
return model, encoding_function
def init(path):
##------------------------------------------------------------------------------##
# Compile and load the model. #
# Compile the encoder. #
##------------------------------------------------------------------------------##
state = prototype_state()
state_path = path + "_state.pkl"
model_path = path + "_model.npz"
with open(state_path) as src:
state.update(cPickle.load(src))
logging.basicConfig(level=getattr(logging, state['level']), format="%(asctime)s: %(name)s: %(levelname)s: %(message)s")
model = DialogEncoderDecoder(state)
if os.path.isfile(model_path):
logger.debug("Loading previous model")
model.load(model_path)
else:
raise Exception("Must specify a valid model path")
encoding_function = model.build_encoder_function()
return model, encoding_function
def _build_vhred_model(self):
# Update the state dictionary.
state = VHRED_prototype_state()
model_prefix = self.config['vhred_prefix']
state_path = model_prefix + "_state.pkl"
model_path = model_prefix + "_model.npz"
with open(state_path, 'rb') as handle:
state.update(cPickle.load(handle))
# Update the bs for the current data.
state['bs'] = 100
state['dictionary'] = self.f_dict
# Create the model:
## load trained parameters
model = VHRED_DialogEncoderDecoder(state)
model.load(model_path)
enc_fn = model.build_encoder_function()
dec_fn = model.build_decoder_encoding()
return model, enc_fn, dec_fn
def main():
args = parse_args()
state = prototype_state()
state_path = args.model_prefix + "_state.pkl"
model_path = args.model_prefix + "_model.npz"
with open(state_path) as src:
state.update(cPickle.load(src))
logging.basicConfig(level=getattr(logging, state['level']), format="%(asctime)s: %(name)s: %(levelname)s: %(message)s")
model = DialogEncoderDecoder(state)
if os.path.isfile(model_path):
logger.debug("Loading previous model")
model.load(model_path)
else:
raise Exception("Must specify a valid model path")
contexts = [[]]
lines = open(args.dialogues, "r").readlines()
if len(lines):
contexts = [x.strip().split('\t') for x in lines]
model_compute_encoding = model.build_encoder_function()
dialogue_encodings = []
# Start loop
joined_contexts = []
batch_index = 0
batch_total = int(math.ceil(float(len(contexts)) / float(model.bs)))
for context_id, context_sentences in enumerate(contexts):
# Convert contextes into list of ids
joined_context = []
if len(context_sentences) == 0:
joined_context = [model.eos_sym]
else:
joined_context += [model.eos_sym]
for sentence in context_sentences:
sentence_ids = model.words_to_indices(sentence.split())
# Add eos tokens
joined_context += sentence_ids + [model.eos_sym]
# HACK
#for i in range(0, 50):
# joined_context += [0] + [model.eos_sym]
joined_contexts.append(joined_context)
if len(joined_contexts) == model.bs:
batch_index = batch_index + 1
logger.debug("[COMPUTE] - Got batch %d / %d" % (batch_index, batch_total))
encs = compute_encodings(joined_contexts, model, model_compute_encoding, args.use_second_last_state)
for i in range(len(encs)):
dialogue_encodings.append(encs[i])
joined_contexts = []
if len(joined_contexts) > 0:
logger.debug("[COMPUTE] - Got batch %d / %d" % (batch_total, batch_total))
encs = compute_encodings(joined_contexts, model, model_compute_encoding, args.use_second_last_state)
for i in range(len(encs)):
dialogue_encodings.append(encs[i])
# Save encodings to disc
cPickle.dump(dialogue_encodings, open(args.output + '.pkl', 'w'))
def main():
args = parse_args()
state = prototype_state()
state_path = args.model_prefix + "_state.pkl"
model_path = args.model_prefix + "_model.npz"
with open(state_path) as src:
state.update(cPickle.load(src))
logging.basicConfig(
level=getattr(logging, state['level']),
format="%(asctime)s: %(name)s: %(levelname)s: %(message)s")
model = DialogEncoderDecoder(state)
if os.path.isfile(model_path):
logger.debug("Loading previous model")
model.load(model_path)
else:
raise Exception("Must specify a valid model path")
contexts = [[]]
lines = open(args.dialogues, "r").readlines()
if len(lines):
contexts = [x.strip().split('\t') for x in lines]
model_compute_encoding = model.build_encoder_function()
dialogue_encodings = []
# Start loop
joined_contexts = []
batch_index = 0
batch_total = int(math.ceil(float(len(contexts)) / float(model.bs)))
for context_id, context_sentences in enumerate(contexts):
# Convert contextes into list of ids
joined_context = []
if len(context_sentences) == 0:
joined_context = [model.eos_sym]
else:
for sentence in context_sentences:
sentence_ids = model.words_to_indices(sentence.split())
# Add sos and eos tokens
joined_context += [model.sos_sym
] + sentence_ids + [model.eos_sym]
# HACK
for i in range(0, 50):
joined_context += [model.sos_sym] + [0] + [model.eos_sym]
joined_contexts.append(joined_context)
if len(joined_contexts) == model.bs:
batch_index = batch_index + 1
logger.debug("[COMPUTE] - Got batch %d / %d" %
(batch_index, batch_total))
encs = compute_encodings(joined_contexts, model,
model_compute_encoding,
args.use_second_last_state)
for i in range(len(encs)):
dialogue_encodings.append(encs[i])
joined_contexts = []
if len(joined_contexts) > 0:
logger.debug("[COMPUTE] - Got batch %d / %d" %
(batch_total, batch_total))
encs = compute_encodings(joined_contexts, model,
model_compute_encoding,
args.use_second_last_state)
for i in range(len(encs)):
dialogue_encodings.append(encs[i])
# Save encodings to disc
cPickle.dump(dialogue_encodings, open(args.output + '.pkl', 'w'))
def main(model_prefix, dialogue_file, use_second_last_state):
state = prototype_state()
state_path = model_prefix + "_state.pkl"
model_path = model_prefix + "_model.npz"
with open(state_path) as src:
state.update(cPickle.load(src))
logging.basicConfig(
level=getattr(logging, state['level']),
format="%(asctime)s: %(name)s: %(levelname)s: %(message)s")
state['bs'] = 10
model = DialogEncoderDecoder(state)
if os.path.isfile(model_path):
logger.debug("Loading previous model")
model.load(model_path)
else:
raise Exception("Must specify a valid model path")
contexts = [[]]
lines = open(dialogue_file, "r").readlines()
if len(lines):
contexts = [x.strip() for x in lines]
model_compute_encoding = model.build_encoder_function()
dialogue_encodings = []
# Start loop
joined_contexts = []
batch_index = 0
batch_total = int(math.ceil(float(len(contexts)) / float(model.bs)))
for context_id, context_sentences in enumerate(contexts):
# Convert contexts into list of ids
joined_context = []
if len(context_sentences) == 0:
joined_context = [model.eos_sym]
else:
joined_context = model.words_to_indices(context_sentences.split())
if joined_context[0] != model.eos_sym:
joined_context = [model.eos_sym] + joined_context
if joined_context[-1] != model.eos_sym:
joined_context += [model.eos_sym]
joined_contexts.append(joined_context)
if len(joined_contexts) == model.bs:
batch_index = batch_index + 1
logger.debug("[COMPUTE] - Got batch %d / %d" %
(batch_index, batch_total))
encs = compute_encodings(joined_contexts, model,
model_compute_encoding,
use_second_last_state)
for i in range(len(encs)):
dialogue_encodings.append(encs[i])
joined_contexts = []
if len(joined_contexts) > 0:
logger.debug("[COMPUTE] - Got batch %d / %d" %
(batch_total, batch_total))
encs = compute_encodings(joined_contexts, model,
model_compute_encoding, use_second_last_state)
for i in range(len(encs)):
dialogue_encodings.append(encs[i])
return dialogue_encodings
