def do_extract(args): datapath, destination_fn = args.datapath, args.dest_fn voc_fn = datapath + ".voc" data_fn = datapath + ".data.json.gz" log.info("extracting data from %s using processor in %s", data_fn, voc_fn) data = json.load(gzip.open(data_fn, "rb")) bi_pp = processors.load_pp_pair_from_file(voc_fn) tgt_processor = bi_pp.tgt_processor() for key in data: src_fn = destination_fn + ".%s.src.txt" % key tgt_fn = destination_fn + ".%s.tgt.txt" % key tgt_swallow_fn = destination_fn + ".%s.tgt.swallow.txt" % key log.info("extracting key %s into %s and %s and %s", key, src_fn, tgt_fn, tgt_swallow_fn) src_f = codecs.open(src_fn, "w", encoding="utf8") tgt_f = codecs.open(tgt_fn, "w", encoding="utf8") tgt_swallow_f = codecs.open(tgt_swallow_fn, "w", encoding="utf8") for src, tgt in data[key]: src_dec, tgt_dec = bi_pp.deconvert(src, tgt) src_f.write(src_dec + "\n") tgt_f.write(tgt_dec + "\n") tgt_swallow = tgt_processor.deconvert_swallow(tgt) tgt_swallow_string = " ".join( ("[@%i]" % w if isinstance(w, int) else w) for w in tgt_swallow) tgt_swallow_f.write(tgt_swallow_string + "\n")
def load_voc_and_update_training_config(config_training): data_prefix = config_training["training_management"]["data_prefix"] voc_fn = data_prefix + ".voc" data_fn = data_prefix + ".data.json.gz" log.info("loading voc from %s" % voc_fn) # src_voc, tgt_voc = json.load(open(voc_fn)) bi_idx = processors.load_pp_pair_from_file(voc_fn) src_indexer, tgt_indexer = bi_idx.src_processor(), bi_idx.tgt_processor() # src_indexer = processors.PreProcessor.make_from_serializable(src_voc) # tgt_indexer = processors.PreProcessor.make_from_serializable(tgt_voc) # tgt_voc = None # src_voc = None # Vi = len(src_voc) + 1 # + UNK # Vo = len(tgt_voc) + 1 # + UNK Vi = len(src_indexer) # + UNK Vo = len(tgt_indexer) # + UNK config_training.add_section("data", keep_at_bottom="metadata", overwrite=False) config_training["data"]["data_fn"] = data_fn config_training["data"]["Vi"] = Vi config_training["data"]["Vo"] = Vo config_training["data"]["voc"] = voc_fn config_training.set_readonly() return src_indexer, tgt_indexer
def create_encdec_and_indexers_from_config_dict(config_dict, src_indexer=None, tgt_indexer=None, load_config_model="no", return_model_infos=False): assert load_config_model in "yes no if_exists".split() if src_indexer is None or tgt_indexer is None: voc_fn = config_dict.data["voc"] log.info("loading voc from %s" % voc_fn) # src_voc, tgt_voc = json.load(open(voc_fn)) bi_idx = processors.load_pp_pair_from_file(voc_fn) if src_indexer is None: src_indexer = bi_idx.src_processor() if tgt_indexer is None: tgt_indexer = bi_idx.tgt_processor() # tgt_voc = None # src_voc = None encdec = create_encdec_from_config_dict(config_dict["model"], src_indexer, tgt_indexer) eos_idx = len(tgt_indexer) model_infos = None if load_config_model != "no": if "model_parameters" not in config_dict: if load_config_model == "yes": log.error("cannot find model parameters in config file") raise ValueError( "Config file do not contain model_parameters section") else: if config_dict.model_parameters.type == "model": model_filename = config_dict.model_parameters.filename log.info( "loading model parameters from file specified by config file:%s" % model_filename) serializers.load_npz(model_filename, encdec) if return_model_infos: model_infos = create_filename_infos(model_filename) else: if load_config_model == "yes": log.error( "model parameters in config file is of type snapshot, not model") raise ValueError("Config file model is not of type model") result = encdec, eos_idx, src_indexer, tgt_indexer if return_model_infos: return result, model_infos else: return result
def do_make_data(config): # raw_input("Press Enter to Continue 222") save_prefix_dir, save_prefix_fn = os.path.split(config.data.save_prefix) ensure_path(save_prefix_dir) config_fn = config.data.save_prefix + ".data.config" voc_fn = config.data.save_prefix + ".voc" data_fn = config.data.save_prefix + ".data.json.gz" # valid_data_fn = config.save_prefix + "." + config.model + ".valid.data.npz" # voc_fn_src = config.save_prefix + ".src.voc" # voc_fn_tgt = config.save_prefix + ".tgt.voc" files_that_will_be_created = [config_fn, voc_fn, data_fn] if config.processing.bpe_src is not None: bpe_data_file_src = config.data.save_prefix + ".src.bpe" files_that_will_be_created.append(bpe_data_file_src) if config.processing.bpe_tgt is not None: bpe_data_file_tgt = config.data.save_prefix + ".tgt.bpe" files_that_will_be_created.append(bpe_data_file_tgt) if config.processing.joint_bpe is not None: bpe_data_file_joint = config.data.save_prefix + ".joint.bpe" files_that_will_be_created.append(bpe_data_file_joint) already_existing_files = [] for filename in files_that_will_be_created: # , valid_data_fn]: if os.path.exists(filename): already_existing_files.append(filename) if len(already_existing_files ) > 0 and not config.processing.force_overwrite: print "Warning: existing files are going to be replaced: ", already_existing_files raw_input("Press Enter to Continue") if config.processing.use_voc is not None: log.info("loading voc from %s" % config.processing.use_voc) # src_voc, tgt_voc = json.load(open(config.use_voc)) # src_pp = processors.load_pp_from_data(json.load(open(src_voc))) # tgt_pp = IndexingPrePostProcessor.make_from_serializable(tgt_voc) bi_idx = processors.load_pp_pair_from_file(config.processing.use_voc) else: bi_idx = processors.BiIndexingPrePostProcessor( voc_limit1=config.processing.src_voc_size, voc_limit2=config.processing.tgt_voc_size) pp = processors.BiProcessorChain() if config.processing.source_char_conversion is not None: log.info("using source char conversion %s", config.processing.source_char_conversion) char_conv_dic = json.load( open(config.processing.source_char_conversion)) pp.add_src_processor( processors.SourceCharacterConverter(char_conv_dic)) if config.processing.latin_tgt: pp.add_tgt_processor( processors.LatinScriptProcess(config.processing.latin_type)) if config.processing.latin_src: pp.add_src_processor( processors.LatinScriptProcess(config.processing.latin_type)) pp.add_src_processor( processors.SimpleSegmenter( config.processing.src_segmentation_type)) if config.processing.bpe_src is not None: pp.add_src_processor( processors.BPEProcessing(bpe_data_file=bpe_data_file_src, symbols=config.processing.bpe_src, separator="._@@@")) pp.add_tgt_processor( processors.SimpleSegmenter( config.processing.tgt_segmentation_type)) if config.processing.bpe_tgt is not None: pp.add_tgt_processor( processors.BPEProcessing(bpe_data_file=bpe_data_file_tgt, symbols=config.processing.bpe_tgt, separator="._@@@")) if config.processing.joint_bpe is not None: pp.add_biprocessor( processors.JointBPEBiProcessor( bpe_data_file=bpe_data_file_joint, symbols=config.processing.joint_bpe, separator="._@@@")) bi_idx.add_preprocessor(pp) def load_data(src_fn, tgt_fn, max_nb_ex=None, infos_dict=None): training_data, stats_src, stats_tgt = processors.build_dataset_pp( src_fn, tgt_fn, bi_idx, max_nb_ex=max_nb_ex) log.info("src data stats:\n%s", stats_src.make_report()) log.info("tgt data stats:\n%s", stats_tgt.make_report()) if infos_dict is not None: infos_dict["src"] = stats_src.report_as_obj() infos_dict["tgt"] = stats_tgt.report_as_obj() return training_data infos = collections.OrderedDict() infos["train"] = collections.OrderedDict() log.info("loading training data from %s and %s" % (config.data.src_fn, config.data.tgt_fn)) training_data = load_data(config.data.src_fn, config.data.tgt_fn, max_nb_ex=config.data.max_nb_ex, infos_dict=infos["train"]) dev_data = None if config.data.dev_src is not None: log.info("loading dev data from %s and %s" % (config.data.dev_src, config.data.dev_tgt)) infos["dev"] = collections.OrderedDict() dev_data = load_data(config.data.dev_src, config.data.dev_tgt, infos_dict=infos["dev"]) test_data = None if config.data.test_src is not None: log.info("loading test data from %s and %s" % (config.data.test_src, config.data.test_tgt)) infos["test"] = collections.OrderedDict() test_data = load_data(config.data.test_src, config.data.test_tgt, infos_dict=infos["test"]) config.insert_section("infos", infos, even_if_readonly=True, keep_at_bottom="metadata", overwrite=False) # if config.shuffle: # log.info("shuffling data") # if config.enable_fast_shuffle: # shuffle_in_unison_faster(data_input, data_target) # else: # data_input, data_target = shuffle_in_unison(data_input, data_target) log.info("saving config to %s" % config_fn) config.save_to(config_fn) # json.dump(config.__dict__, open(config_fn, "w"), # indent=2, separators=(',', ': ')) log.info("saving voc to %s" % voc_fn) processors.save_pp_pair_to_file(bi_idx, voc_fn) # json.dump([src_pp.to_serializable(), tgt_pp.to_serializable()], # open(voc_fn, "w"), indent=2, separators=(',', ': ')) log.info("saving train_data to %s" % data_fn) data_all = {"train": training_data} if test_data is not None: data_all["test"] = test_data if dev_data is not None: data_all["dev"] = dev_data json.dump(data_all, gzip.open(data_fn, "wb"), indent=2, separators=(',', ': '))
def create_encdec_and_indexers_from_config_dict( config_dict, src_indexer=None, tgt_indexer=None, load_config_model="no", return_model_infos=False, additional_models_parameters_for_averaging=None): assert load_config_model in "yes no if_exists".split() if src_indexer is None or tgt_indexer is None: voc_fn = config_dict.data["voc"] log.info("loading voc from %s" % voc_fn) # src_voc, tgt_voc = json.load(open(voc_fn)) bi_idx = processors.load_pp_pair_from_file(voc_fn) if src_indexer is None: src_indexer = bi_idx.src_processor() if tgt_indexer is None: tgt_indexer = bi_idx.tgt_processor() # tgt_voc = None # src_voc = None encdec = create_encdec_from_config_dict(config_dict["model"], src_indexer, tgt_indexer) eos_idx = len(tgt_indexer) model_infos = None if load_config_model != "no": if "model_parameters" not in config_dict: assert additional_models_parameters_for_averaging is None if load_config_model == "yes": log.error("cannot find model parameters in config file") raise ValueError( "Config file do not contain model_parameters section") else: model_filename = config_dict.model_parameters.filename if additional_models_parameters_for_averaging is not None: load_model_flexible([model_filename] + additional_models_parameters_for_averaging, encdec) else: load_model_flexible(model_filename, encdec) # if config_dict.model_parameters.type == "model": # log.info( # "loading model parameters from file specified by config file:%s" % # model_filename) # serializers.load_npz(model_filename, encdec) # if return_model_infos: # model_infos = create_filename_infos(model_filename) # else: # log.info("loading model parameters from snapshot file specified by config file:%s" %model_filename) # with np.load(model_filename) as fs: # dics = serializers.NpzDeserializer(fs, path="updater/model:main/") # dics.load(encdec) if return_model_infos: model_infos = create_filename_infos(model_filename) else: assert additional_models_parameters_for_averaging is None result = encdec, eos_idx, src_indexer, tgt_indexer if return_model_infos: return result, model_infos else: return result