ef = ExperimentFiles(lang_map.keys()) # ------------------------------------------- # 1) Filter the data # ------------------------------------------- filtration_done = False for lang in ef.langs: orig_f = ef.get_original_file(lang) filtered_f = ef.get_filtered_file(lang) if not os.path.exists(filtered_f): filtration_done = True if USE_CONDOR: model_prefix, name = ef.get_condor_filter(lang) run_cmd([p3path, intent_script, 'filter', '--require-aln', '--require-gloss', '--require-trans', '--require-lang', orig_f, filtered_f], model_prefix, name, False) else: filter_corpus([orig_f], filtered_f, require_lang=True, require_gloss=True, require_trans=True, require_aln=True) if USE_CONDOR and filtration_done: condor_wait_notify("Data has been filtered.", email_address, "CONDOR: Filtration complete.") # ------------------------------------------- # 2) Enriched data # ------------------------------------------- enrichment_done = False for lang in ef.langs: filtered_f = ef.get_filtered_file(lang) enriched_f = ef.get_enriched_file(lang)
def condorify(args, prefix, name): p3path = which('python3') condor.run_cmd([p3path, intent_script] + args, prefix=prefix, name=name, email=False)