def preprocess(s, r):
parsed = apply_preprocessors(from_string(s), pp_params["preprocessors"],
{'interesting_context_words': []})
params = PrepConfig.from_encoded_string(r)
init_splitting_config(DEFAULT_DATASET, params, DEFAULT_BPE_BASE_REPR,
DEFAULT_BPE_N_MERGES, None)
return to_repr(params, parsed)
def calc_stats_for_prepconfig(prepconfig,
lang_checker,
token_list,
include_sample=False):
repr = to_token_list(
to_repr(PrepConfig.from_encoded_string(prepconfig), token_list,
NgramSplitConfig())).split(' ')
return lang_checker.calc_lang_stats(repr, include_sample=include_sample)
def test(self):
for input, output_tuple in test_cases.items():
parsed = apply_preprocessors(from_string(input), pp_params["preprocessors"], {})
self.assertEqual(output_tuple[0], parsed)
repred = to_repr(PrepConfig.from_encoded_string('104111'), parsed, ngram_split_config)
self.assertEqual(output_tuple[1], repred)
def run(dataset: str, preprocessing_params: str, bpe_base_repr: Optional[str],
bpe_n_merges: Optional[int], splitting_file: Optional[str]):
path_to_dataset = os.path.join(DEFAULT_PARSED_DATASETS_DIR, args.dataset)
full_src_dir = os.path.join(path_to_dataset, PARSED_DIR)
if not os.path.exists(full_src_dir):
logger.error(f"Dir does not exist: {full_src_dir}")
exit(3)
logger.info(f"Reading parsed files from: {os.path.abspath(full_src_dir)}")
preprocessing_params = PrepConfig.from_encoded_string(preprocessing_params)
init_splitting_config(dataset, preprocessing_params, bpe_base_repr,
bpe_n_merges, splitting_file)
repr = str(preprocessing_params)
full_dest_dir = os.path.join(path_to_dataset, REPR_EXTENSION, repr)
full_metadata_dir = os.path.join(path_to_dataset, METADATA_DIR, repr)
logger.info(
f"Writing preprocessed files to {os.path.abspath(full_dest_dir)}")
if not os.path.exists(full_dest_dir):
os.makedirs(full_dest_dir)
if not os.path.exists(full_metadata_dir):
os.makedirs(full_metadata_dir)
with open(os.path.join(full_dest_dir, 'preprocessing_types.json'),
"w") as f:
json_str = jsons.dumps(preprocessing_params)
f.write(json_str)
params = []
for root, dirs, files in os.walk(full_src_dir):
for file in files:
if file.endswith(f".{PARSED_FILE_EXTENSION}"):
full_dest_dir_with_sub_dir = os.path.join(
full_dest_dir, os.path.relpath(root, full_src_dir))
if not os.path.exists(full_dest_dir_with_sub_dir):
os.makedirs(full_dest_dir_with_sub_dir)
params.append((os.path.join(root, file),
os.path.join(full_dest_dir_with_sub_dir,
file), preprocessing_params))
files_total = len(params)
current_file = 0
start_time = time.time()
with Pool() as pool:
it = pool.imap_unordered(preprocess_and_write, params)
for _ in it:
current_file += 1
logger.info(f"Processed {current_file} out of {files_total}")
time_elapsed = time.time() - start_time
logger.info(
f"Time elapsed: {time_elapsed:.2f} s, estimated time until completion: "
f"{time_elapsed / current_file * files_total - time_elapsed:.2f} s"
)
def run(dataset: str, preprocessing_params: str, bpe_base_repr: Optional[str],
bpe_n_merges: Optional[int], splitting_file: Optional[str], merges_file):
path_to_dataset = os.path.join(DEFAULT_PARSED_DATASETS_DIR, args.dataset)
full_src_dir = os.path.join(path_to_dataset, PARSED_DIR)
if not os.path.exists(full_src_dir):
logger.error(f"Dir does not exist: {full_src_dir}")
exit(3)
logger.info(f"Reading parsed files from: {os.path.abspath(full_src_dir)}")
preprocessing_params = PrepConfig.from_encoded_string(preprocessing_params)
init_splitting_config(dataset, preprocessing_params, bpe_base_repr, bpe_n_merges, splitting_file, merges_file)
repr = str(preprocessing_params)
full_dest_dir = os.path.join(path_to_dataset, REPR_EXTENSION, f'{repr}_{bpe_n_merges if bpe_n_merges else ""}_{os.path.basename(merges_file)}')
full_metadata_dir = os.path.join(path_to_dataset, METADATA_DIR, repr)
logger.info(f"Writing preprocessed files to {os.path.abspath(full_dest_dir)}")
if not os.path.exists(full_dest_dir):
os.makedirs(full_dest_dir)
if not os.path.exists(full_metadata_dir):
os.makedirs(full_metadata_dir)
with open(os.path.join(full_dest_dir, 'preprocessing_types.json'), "w") as f:
json_str = jsons.dumps(preprocessing_params)
f.write(json_str)
params = []
for root, dirs, files in os.walk(full_src_dir):
for file in files:
if file.endswith(f".{PARSED_FILE_EXTENSION}"):
full_dest_dir_with_sub_dir = os.path.join(full_dest_dir, os.path.relpath(root, full_src_dir))
if not os.path.exists(full_dest_dir_with_sub_dir):
os.makedirs(full_dest_dir_with_sub_dir)
params.append((os.path.join(root, file),
os.path.join(full_dest_dir_with_sub_dir, file),
preprocessing_params))
files_total = len(params)
with Pool() as pool:
it = pool.imap_unordered(preprocess_and_write, params)
for _ in tqdm(it, total=files_total):
pass
def gen():
with open(path_to_file, 'r') as f:
identifiers = [line.rstrip('\n') for line in f]
csv_lines = [
DELIMITER.join(["config"] + [p for p in PrepParam] + identifiers)
]
for prep in prep_configs:
csv_line = [prep]
for p in PrepParam:
csv_line.append(PrepConfig.human_readable_values[p][
PrepConfig.from_encoded_string(prep).get_param_value(p)])
for identifier in identifiers:
tokens = preprocess(identifier, prep)
csv_line.append(' '.join(tokens))
csv_lines.append(DELIMITER.join(csv_line))
with open(path_to_file_out, 'w') as f:
for line in csv_lines:
f.write(f'{line}\n')
def to_repr_l(lst):
return to_repr(PrepConfig.from_encoded_string('000010'), lst,
NgramSplitConfig())
