def test_context_memory(config_filename, search_engine, src_sentence): config_training = train_config.load_config_train(config_filename) (encdec, eos_idx, src_indexer, tgt_indexer ), model_infos = train.create_encdec_and_indexers_from_config_dict( config_training, load_config_model="yes", return_model_infos=True) ctxt_mem = create_context_memory(encdec, (src_indexer, tgt_indexer), search_engine, src_sentence)
def create_and_load_encdec_from_files(config_training_fn, trained_model): log.info("loading model config from %s" % config_training_fn) config_training = train_config.load_config_train(config_training_fn) encdec, eos_idx, src_indexer, tgt_indexer = train.create_encdec_and_indexers_from_config_dict(config_training) log.info("loading model from %s" % trained_model) serializers.load_npz(trained_model, encdec) return encdec, eos_idx, src_indexer, tgt_indexer
def get_src_tgt_dev_from_config_eval(config_eval): if config_eval.training_config is not None: training_config_file = config_eval.training_config elif 'load_model_config' in config_eval.process and config_eval.process.load_model_config is not None: training_config_file = config_eval.process.load_model_config[0] log.info("attempting to retrieve dev/test files from %s", training_config_file) training_config = train_config.load_config_train(training_config_file) data_prefix = training_config.training_management.data_prefix data_config_filename = data_prefix + ".data.config" data_config = make_data_conf.load_config(data_config_filename) return (data_config["data"]["dev_src"], data_config["data"]["dev_tgt"], data_config["data"]["test_src"], data_config["data"]["test_tgt"])
def create_encdec(config_eval): encdec_list = [] eos_idx, src_indexer, tgt_indexer = None, None, None model_infos_list = [] if config_eval.training_config is not None: assert config_eval.trained_model is not None encdec, eos_idx, src_indexer, tgt_indexer = create_and_load_encdec_from_files( config_eval.training_config, config_eval.trained_model) model_infos_list.append( create_filename_infos(config_eval.trained_model)) encdec_list.append(encdec) if 'load_model_config' in config_eval.process and config_eval.process.load_model_config is not None: for config_filename_and_others in config_eval.process.load_model_config: other_models_for_averaging = None if "," in config_filename_and_others: config_filename_and_others_splitted = config_filename_and_others.split( ",") config_filename = config_filename_and_others_splitted[0] other_models_for_averaging = config_filename_and_others_splitted[ 1:] else: config_filename = config_filename_and_others log.info("loading model and parameters from config %s" % config_filename) config_training = train_config.load_config_train(config_filename) ( encdec, this_eos_idx, this_src_indexer, this_tgt_indexer ), model_infos = train.create_encdec_and_indexers_from_config_dict( config_training, load_config_model="yes", return_model_infos=True, additional_models_parameters_for_averaging= other_models_for_averaging) model_infos_list.append(model_infos) if eos_idx is None: assert len(encdec_list) == 0 assert src_indexer is None assert tgt_indexer is None eos_idx, src_indexer, tgt_indexer = this_eos_idx, this_src_indexer, this_tgt_indexer else: check_if_vocabulary_info_compatible(this_eos_idx, this_src_indexer, this_tgt_indexer, eos_idx, src_indexer, tgt_indexer) encdec_list.append(encdec) assert len(encdec_list) > 0 if 'additional_training_config' in config_eval.process and config_eval.process.additional_training_config is not None: assert len(config_eval.process.additional_training_config) == len( config_eval.process.additional_trained_model) for (config_training_fn, trained_model_fn) in six.moves.zip( config_eval.process.additional_training_config, config_eval.process.additional_trained_model): this_encdec, this_eos_idx, this_src_indexer, this_tgt_indexer = create_and_load_encdec_from_files( config_training_fn, trained_model_fn) check_if_vocabulary_info_compatible(this_eos_idx, this_src_indexer, this_tgt_indexer, eos_idx, src_indexer, tgt_indexer) model_infos_list.append(create_filename_infos(trained_model_fn)) encdec_list.append(this_encdec) if config_eval.process.use_chainerx: if 'gpu' in config_eval.process and config_eval.process.gpu is not None: encdec_list = [ encdec.to_device("cuda:%i" % config_eval.process.gpu) for encdec in encdec_list ] else: encdec_list = [ encdec.to_device("native:0") for encdec in encdec_list ] else: if 'gpu' in config_eval.process and config_eval.process.gpu is not None: encdec_list = [ encdec.to_gpu(config_eval.process.gpu) for encdec in encdec_list ] if 'reverse_training_config' in config_eval.process and config_eval.process.reverse_training_config is not None: reverse_encdec, reverse_eos_idx, reverse_src_indexer, reverse_tgt_indexer = create_and_load_encdec_from_files( config_eval.process.reverse_training_config, config_eval.process.reverse_trained_model) if eos_idx != reverse_eos_idx: raise Exception("incompatible models") if len(src_indexer) != len(reverse_src_indexer): raise Exception("incompatible models") if len(tgt_indexer) != len(reverse_tgt_indexer): raise Exception("incompatible models") if config_eval.process.gpu is not None: reverse_encdec = reverse_encdec.to_gpu(config_eval.process.gpu) else: reverse_encdec = None return encdec_list, eos_idx, src_indexer, tgt_indexer, reverse_encdec, model_infos_list
def process_eval_config(config_fn, dest_dir): log.info("processing eval config %s " % config_fn) assert config_fn.endswith(eval_config_suffix) eval_prefix = config_fn[:-len(eval_config_suffix)] log.info("eval prefix: %s " % eval_prefix) urlname = filenamize(eval_prefix) + ".html" log.info("urlname: %s " % eval_prefix) time_config_created = os.path.getmtime(config_fn) dest_filename = os.path.join(dest_dir, urlname) log.info("create file: %s" % dest_filename) f = open(dest_filename, "w") config = load_config_eval(config_fn) # config = json.load(open(config_fn)) # ref_fn = config["ref"] ref_fn = config.output.ref if ref_fn is None and "translation_infos" in config: ref_fn = config["translation_infos"].ref src_fn = config.process.src_fn if src_fn is None and "translation_infos" in config: src_fn = config["translation_infos"].src training_config_fn = config.training_config if training_config_fn is None and config.process.load_model_config is not None and len( config.process.load_model_config) > 0: training_config_fn = config.process.load_model_config[0] if "," in training_config_fn: training_config_fn = training_config_fn.split(",")[0] config_training = load_config_train(training_config_fn, no_error=True) description_training = config_training.get("training_management", {}).get("description", "") nb_models_used = 0 if config.training_config is not None: nb_models_used += 1 if "load_model_config" in config.process and config.process.load_model_config is not None: nb_models_used += len(config.process.load_model_config) if config.process.additional_training_config is not None: nb_models_used += len(config.process.additional_training_config) if ref_fn is not None: bc_with_unk = bleu_computer.get_bc_from_files(ref_fn, eval_prefix) bleu_with_unk = bc_with_unk.bleu() if not os.path.isfile(eval_prefix + ".unk_replaced"): bc_unk_replaced = "#No unk replaced file#" bleu_unk_replaced = -1 else: bc_unk_replaced = bleu_computer.get_bc_from_files( ref_fn, eval_prefix + ".unk_replaced") bleu_unk_replaced = bc_unk_replaced.bleu() else: bc_with_unk = "#No Ref#" bc_unk_replaced = "#No Ref#" bleu_unk_replaced = -1 bleu_with_unk = -1 desc = { "description_training": description_training, "nb_models_used": nb_models_used, "bleu": bleu_unk_replaced if bleu_unk_replaced >= 0 else bleu_with_unk } f.write("""<html> <meta charset="UTF-8"> <style> table, th, td { border: 1px solid black; } </style><body>""") f.write("Config Filename: %s <p>" % config_fn) f.write("eval prefix: %s <p>" % eval_prefix) f.write("config file created/modified:%s <p>" % time.ctime(time_config_created)) f.write("BLEU: %s <p>" % bc_with_unk) f.write("BLEU with unk replaced: %s <p>" % bc_unk_replaced) f.write("<pre><code>\n") for line in open(config_fn): f.write(line) f.write("</code></pre>\n") lst_outputs = [] if src_fn is not None: lst_outputs.append(("src", codecs.open(src_fn, encoding="utf8"))) if ref_fn is not None: lst_outputs.append(("ref", codecs.open(ref_fn, encoding="utf8"))) if os.path.isfile(eval_prefix + ".unprocessed"): lst_outputs.append(("unprocessed", codecs.open(eval_prefix + ".unprocessed", encoding="utf8"))) lst_outputs.append(("out", codecs.open(eval_prefix, encoding="utf8"))) if os.path.isfile(eval_prefix + ".unk_replaced"): lst_outputs.append(("unk_repl", codecs.open(eval_prefix + ".unk_replaced", encoding="utf8"))) tags_list, files_list = zip(*lst_outputs) for num_line, line_list in enumerate(itertools.izip(*files_list)): f.write("""<table style="width:100%">""") f.write("<tr><td>%s</td><td>%i</td></tr>\n" % ("NUM", num_line)) for tag, line in zip(tags_list, line_list): f.write("<tr><td>%s</td><td>%s</td></tr>\n" % (tag, cgi.escape(line.encode("utf8")))) f.write("</table>") f.write("</body></html>") return urlname, desc
def process_train_config(config_fn, dest_dir): log.info("processing train config %s " % config_fn) assert config_fn.endswith(train_config_suffix) train_prefix = config_fn[:-len(train_config_suffix)] log.info("train prefix: %s " % train_prefix) urlname = filenamize(train_prefix) + ".html" log.info("urlname: %s " % train_prefix) dest_filename = os.path.join(dest_dir, urlname) if os.path.exists(dest_filename): time_file_modif = os.path.getmtime(dest_filename) else: time_file_modif = None try: time_last_exp = os.path.getmtime(train_prefix + ".result.sqlite") except OSError: time_last_exp = 1 time_config_created = os.path.getmtime(config_fn) if time_file_modif is not None and (time_last_exp + 20) < time_file_modif: log.info("skipping creation for %s" % train_prefix) log.info("create file: %s" % dest_filename) f = open(dest_filename, "w") try: db = sqlite3.connect(train_prefix + ".result.sqlite") c = db.cursor() c.execute( "SELECT date, bleu_info, iteration, loss, bleu, dev_loss, dev_bleu, avg_time, avg_training_loss FROM exp_data" ) date, bleu_info, iteration, test_loss, test_bleu, dev_loss, dev_bleu, avg_time, avg_training_loss = zip( *list(c.fetchall())) infos = dict(last_nb_iterations=iteration[-1], nb_records=len(test_loss), date_last_exp=date[-1], best_test_loss=min(test_loss), best_dev_loss=min(dev_loss), max_test_bleu=max(test_bleu * 100), max_dev_bleu=max(dev_bleu * 100), best_avg_training_loss=min(avg_training_loss[1:]) if len(avg_training_loss) > 1 else 0, avg_time_all=sum(t for t in avg_time if t is not None) / len(avg_time)) except sqlite3.OperationalError: infos = None config = load_config_train(config_fn, no_error=True) # config = json.load(open(config_fn)) # data_prefix = config["command_line"]["data_prefix"] data_prefix = config.training_management.data_prefix try: graph_training.generate_graph( sqldb=train_prefix + ".result.sqlite", # lib = "plotly", dest=os.path.join(dest_dir, filenamize(train_prefix) + ".graph.html"), title=train_prefix) has_graph = True except sqlite3.OperationalError as e: log.warn("could not create graph for %s" % train_prefix) print e has_graph = False f.write("<html><body>") f.write("Config Filename: %s <p>" % config_fn) f.write("train prefix: %s <p>" % train_prefix) f.write("data: <a href = '../data/%s'>%s</a><p>" % (filenamize(data_prefix) + ".html", filenamize(data_prefix))) if has_graph: f.write("<a href = '%s'>graph</a><p>" % (filenamize(train_prefix) + ".graph.html")) f.write("config file created/modified:%s <p>" % time.ctime(time_config_created)) f.write("last exp:%s <p>" % time.ctime(time_last_exp)) time_file_modif_new = os.path.getmtime(dest_filename) f.write("this file created/modified:%s <p>" % time.ctime(time_file_modif_new)) if infos is not None: for key in sorted(infos.keys()): f.write("%s : %r <p>" % (key, infos[key])) f.write("<pre><code>") for line in open(config_fn): f.write(line) f.write("</code></pre>") f.write("</body></html>") description = config.training_management.get("description", "") return urlname, data_prefix, time_last_exp, infos, description
def create_encdec(config_eval): encdec_list = [] eos_idx, src_indexer, tgt_indexer = None, None, None model_infos_list = [] if config_eval.training_config is not None: assert config_eval.trained_model is not None encdec, eos_idx, src_indexer, tgt_indexer = create_and_load_encdec_from_files( config_eval.training_config, config_eval.trained_model) model_infos_list.append(create_filename_infos(config_eval.trained_model)) encdec_list.append(encdec) if 'load_model_config' in config_eval.process and config_eval.process.load_model_config is not None: for config_filename in config_eval.process.load_model_config: log.info( "loading model and parameters from config %s" % config_filename) config_training = train_config.load_config_train(config_filename) (encdec, this_eos_idx, this_src_indexer, this_tgt_indexer), model_infos = train.create_encdec_and_indexers_from_config_dict(config_training, load_config_model="yes", return_model_infos=True) model_infos_list.append(model_infos) if eos_idx is None: assert len(encdec_list) == 0 assert src_indexer is None assert tgt_indexer is None eos_idx, src_indexer, tgt_indexer = this_eos_idx, this_src_indexer, this_tgt_indexer else: check_if_vocabulary_info_compatible(this_eos_idx, this_src_indexer, this_tgt_indexer, eos_idx, src_indexer, tgt_indexer) encdec_list.append(encdec) assert len(encdec_list) > 0 if 'additional_training_config' in config_eval.process and config_eval.process.additional_training_config is not None: assert len(config_eval.process.additional_training_config) == len(config_eval.process.additional_trained_model) for (config_training_fn, trained_model_fn) in zip(config_eval.process.additional_training_config, config_eval.process.additional_trained_model): this_encdec, this_eos_idx, this_src_indexer, this_tgt_indexer = create_and_load_encdec_from_files( config_training_fn, trained_model_fn) check_if_vocabulary_info_compatible(this_eos_idx, this_src_indexer, this_tgt_indexer, eos_idx, src_indexer, tgt_indexer) model_infos_list.append(create_filename_infos(trained_model_fn)) # if args.gpu is not None: # this_encdec = this_encdec.to_gpu(args.gpu) encdec_list.append(this_encdec) if 'gpu' in config_eval.process and config_eval.process.gpu is not None: encdec_list = [encdec.to_gpu(config_eval.process.gpu) for encdec in encdec_list] if 'reverse_training_config' in config_eval.process and config_eval.process.reverse_training_config is not None: reverse_encdec, reverse_eos_idx, reverse_src_indexer, reverse_tgt_indexer = create_and_load_encdec_from_files( config_eval.process.reverse_training_config, config_eval.process.reverse_trained_model) if eos_idx != reverse_eos_idx: raise Exception("incompatible models") if len(src_indexer) != len(reverse_src_indexer): raise Exception("incompatible models") if len(tgt_indexer) != len(reverse_tgt_indexer): raise Exception("incompatible models") if config_eval.process.gpu is not None: reverse_encdec = reverse_encdec.to_gpu(config_eval.process.gpu) else: reverse_encdec = None return encdec_list, eos_idx, src_indexer, tgt_indexer, reverse_encdec, model_infos_list