def build_translation_model(args): conf = fetch_config(args) if args.t_translate_config is None: tconf = conf.system.files.data.translate elif os.path.isfile(args.t_translate_config): tconf = TranslateConfig(args.t_translate_config, conf) else: logger.error(args.t_translate_config + " doesn't exist") return if os.path.exists(tconf.paths.project) is False: os.makedirs(tconf.paths.project) elif os.path.isfile(tconf.paths.project): logger.error(tconf.paths.project + " is a file") sys.exit(1) elif os.listdir(tconf.paths.project) != []: logger.error(tconf.paths.project + " must be empty") sys.exit(1) with open(os.path.join(tconf.paths.project, "translate.yaml"), 'w') as f: yaml.dump(tconf.dict(), f, default_flow_style=False) tconf.conf.runstate.pool_size = tconf.settings.pool_size run_args = get_run_args(tconf) app = BuildApp.new(pool_type=conf.runstate.runner, pool_size=conf.runstate.pool_size, force=conf.runstate.force) os.environ['IRSTLM'] = tconf.paths.irstlm setup_train(tconf) setup_tune(tconf) setup_test(tconf) for idx, parameter_set in enumerate(run_args): parameter_set = list(parameter_set) parameter_set.append(idx) parameter_set.append(tconf) t = app.add() t.job = build_model t.args = parameter_set t.description = "model_" + str(parameter_set[9]) app.run() aggregate_model_data(tconf.paths.project) from_addr = "*****@*****.**" to_addr = [tconf.settings.email] with open(tconf.paths.project + "/data.csv") as data: msg = MIMEText(data.read()) msg['Subject'] = "Model Complete" msg['From'] = from_addr msg['To'] = ", ".join(to_addr) server = smtplib.SMTP("localhost") server.sendmail(from_addr, to_addr, msg.as_string()) server.quit()
def build_translation_model(args): conf = fetch_config(args) if args.t_translate_config is None: tconf = conf.system.files.data.translate elif os.path.isfile(args.t_translate_config): tconf = TranslateConfig(args.t_translate_config, conf) else: logger.error(args.t_translate_config + " doesn't exist") return if os.path.exists(tconf.paths.project) is False: os.makedirs(tconf.paths.project) elif os.path.isfile(tconf.paths.project): logger.error(tconf.paths.project + " is a file") sys.exit(1) elif os.listdir(tconf.paths.project) != []: logger.error(tconf.paths.project + " must be empty") sys.exit(1) with open(os.path.join(tconf.paths.project, "translate.yaml"), 'w') as f: yaml.dump(tconf.dict(), f, default_flow_style=False) tconf.conf.runstate.pool_size = tconf.settings.pool_size run_args = get_run_args(tconf) app = BuildApp.new(pool_type=conf.runstate.runner, pool_size=conf.runstate.pool_size, force=conf.runstate.force) os.environ['IRSTLM'] = tconf.paths.irstlm setup_train(tconf) setup_tune(tconf) setup_test(tconf) for idx, parameter_set in enumerate(run_args): parameter_set = list(parameter_set) parameter_set.append(idx) parameter_set.append(tconf) t = app.add() t.job = build_model t.args = parameter_set t.description = "model_" + str(parameter_set[9]) app.run() aggregate_model_data(tconf.paths.project) from_addr = "*****@*****.**" to_addr = [tconf.settings.email] with open(tconf.paths.project + "/data.csv") as data: msg = MIMEText(data.read()) msg['Subject'] = "Model Complete" msg['From'] = from_addr msg['To'] = ", ".join(to_addr) server = smtplib.SMTP("localhost") server.sendmail(from_addr, to_addr, msg.as_string()) server.quit()
def model_results(args): conf = fetch_config(args) if args.t_translate_config is None: tconf = conf.system.files.data.translate elif os.path.isfile(args.t_translate_config): tconf = TranslateConfig(ingest_yaml_doc(args.t_translate_config), conf) else: logger.error(args.t_translate_config + " doesn't exist") return aggregate_model_data(tconf.paths.project)
def model_results(args): conf = fetch_config(args) if args.t_translate_config is None: tconf = conf.system.files.data.translate elif os.path.isfile(args.t_translate_config): tconf = TranslateConfig(ingest_yaml_doc(args.t_translate_config), conf) else: logger.error(args.t_translate_config + " doesn't exist") return aggregate_model_data(tconf.paths.project)