ef = ExperimentFiles(lang_map.keys())
# -------------------------------------------
# 1) Filter the data
# -------------------------------------------
filtration_done = False
for lang in ef.langs:
orig_f = ef.get_original_file(lang)
filtered_f = ef.get_filtered_file(lang)
if not os.path.exists(filtered_f):
filtration_done = True
if USE_CONDOR:
model_prefix, name = ef.get_condor_filter(lang)
run_cmd([p3path, intent_script, 'filter',
'--require-aln', '--require-gloss', '--require-trans', '--require-lang',
orig_f, filtered_f], model_prefix, name, False)
else:
filter_corpus([orig_f], filtered_f, require_lang=True, require_gloss=True, require_trans=True, require_aln=True)
if USE_CONDOR and filtration_done:
condor_wait_notify("Data has been filtered.", email_address, "CONDOR: Filtration complete.")
# -------------------------------------------
# 2) Enriched data
# -------------------------------------------
enrichment_done = False
for lang in ef.langs:
filtered_f = ef.get_filtered_file(lang)
enriched_f = ef.get_enriched_file(lang)
def condorify(args, prefix, name):
p3path = which('python3')
condor.run_cmd([p3path, intent_script] + args,
prefix=prefix, name=name, email=False)
