def setUp(self): self.cconf = ingest_yaml_doc(self.conf_file) for source in self.cconf['sources']: source['source_file_path'] = os.path.join(TEST_PATH, source['source_file_path']) source['target_file_path'] = os.path.join(TEST_PATH, source['target_file_path']) self.cconf['container_path'] = os.path.join(TEST_PATH, self.cconf['container_path']) self.cconf = CorporaConfig(self.cconf)
def build_translation_model(args): conf = fetch_config(args) if args.t_translate_config is None: tconf = conf.system.files.data.translate elif os.path.isfile(args.t_translate_config): tconf = TranslateConfig(ingest_yaml_doc(args.t_translate_config), conf) else: logger.error(args.t_translate_config + " doesn't exist") return if os.path.exists(tconf.paths.project) is False: os.makedirs(tconf.paths.project) elif os.path.isfile(tconf.paths.project): logger.error(tconf.paths.project + " is a file") sys.exit(1) elif os.listdir(tconf.paths.project) != []: logger.error(tconf.paths.project + " must be empty") sys.exit(1) with open(os.path.join(tconf.paths.project, "translate.yaml"), 'w') as f: yaml.dump(tconf.dict(), f, default_flow_style=False) tconf.conf.runstate.pool_size = tconf.settings.pool_size run_args = get_run_args(tconf) app = BuildApp(conf) os.environ['IRSTLM'] = tconf.paths.irstlm setup_train(tconf) setup_tune(tconf) setup_test(tconf) for idx, parameter_set in enumerate(run_args): parameter_set = list(parameter_set) parameter_set.append(idx) parameter_set.append(tconf) t = app.add() t.job = build_model t.args = parameter_set t.description = "model_" + str(parameter_set[9]) app.run() aggregate_model_data(tconf.paths.project) from_addr = "*****@*****.**" to_addr = [tconf.settings.email] with open(tconf.paths.project+"/data.csv") as data: msg = MIMEText(data.read()) msg['Subject'] = "Model Complete" msg['From'] = from_addr msg['To'] = ", ".join(to_addr) server = smtplib.SMTP("localhost") server.sendmail(from_addr, to_addr, msg.as_string()) server.quit()
def test_cconf_creation(self): self.cconf = ingest_yaml_doc(self.conf_file) for source in self.cconf['sources']: source['source_file_path'] = os.path.join(TEST_PATH, source['source_file_path']) source['target_file_path'] = os.path.join(TEST_PATH, source['target_file_path']) self.cconf['container_path'] = os.path.join(TEST_PATH, self.cconf['container_path']) with self.assertRaises(Exception): self.cconf = CorporaConfig(self.cconf)
def translate_text_doc(args): conf = fetch_config(args) if args.t_translate_config is None: tconf = conf.system.files.data.translate elif os.path.isfile(args.t_translate_config): tconf = TranslateConfig(ingest_yaml_doc(args.t_translate_config), conf) else: logger.error(args.t_translate_config + " doesn't exist") return translate_file(args.t_input_file, args.t_output_file, tconf, args.t_protected_regex)
def model_results(args): conf = fetch_config(args) if args.t_translate_config is None: tconf = conf.system.files.data.translate elif os.path.isfile(args.t_translate_config): tconf = TranslateConfig(ingest_yaml_doc(args.t_translate_config), conf) else: logger.error(args.t_translate_config + " doesn't exist") return aggregate_model_data(tconf.paths.project)
def ingest(self, input_obj=None): if input_obj is None: return elif isinstance(input_obj, dict): pass elif not isinstance(input_obj, ConfigurationBase) and os.path.isfile(input_obj): input_obj = ingest_yaml_doc(input_obj) else: msg = 'cannot ingest Configuration obj from object with type {0}'.format(type(input_obj)) logger.critical(msg) raise TypeError(msg) for key, value in input_obj.items(): setattr(self, key, value) logger.debug('setting {0} using default setter in {1} object'.format(key, type(self)))
def create_corpora(args): conf = fetch_config(args) if args.t_corpora_config is None: cconf = conf.system.files.data.corpora elif os.path.isfile(args.t_corpora_config): cconf = CorporaConfig(ingest_yaml_doc(args.t_corpora_config)) else: logger.error(args.t_corpora_config + " doesn't exist") return if os.path.exists(cconf.container_path): logger.error(cconf.container_path + " already exists. Please delete it or change the container and try again") return create_hybrid_corpora(cconf)
def create_manual_symlink(conf): fpath = os.path.join(conf.paths.projectroot, conf.paths.builddata, 'integration.yaml') if os.path.exists(fpath): iconf = ingest_yaml_doc(fpath) else: return False if 'base' not in iconf: return True else: if 'links' not in iconf['base']: return True else: links = get_top_level_links(iconf['base']['links'], conf) if links: for name, target in links: create_link(target, name)
def finalize_dirhtml_build(builder, conf): pjoin = os.path.join single_html_dir = get_single_html_dir(conf) search_page = pjoin(conf.paths.branch_output, builder, 'index.html') if os.path.exists(search_page): copy_if_needed(source_file=search_page, target_file=pjoin(single_html_dir, 'search.html')) dest = pjoin(conf.paths.projectroot, conf.paths.public_site_output) command('rsync -a {source}/ {destination}'.format(source=pjoin(conf.paths.projectroot, conf.paths.branch_output, builder), destination=dest)) logger.info('{0}: migrated build to {1}'.format(builder, dest)) if conf.git.branches.current in conf.git.branches.published: sitemap_exists = sitemap(config_path=None, conf=conf) if sitemap_exists is True: copy_if_needed(source_file=pjoin(conf.paths.projectroot, conf.paths.branch_output, 'sitemap.xml.gz'), target_file=pjoin(conf.paths.projectroot, conf.paths.public_site_output, 'sitemap.xml.gz')) sconf_path = pjoin(conf.paths.projectroot, conf.paths.builddata, 'sphinx.yaml') sconf = ingest_yaml_doc(sconf_path) if 'dirhtml' in sconf and 'excluded_files' in sconf['dirhtml']: fns = [ pjoin(conf.paths.projectroot, conf.paths.public_site_output, fn) for fn in sconf['dirhtml']['excluded_files'] ] cleaner(fns) logging.info('removed excluded files from dirhtml output directory')
def generated_includes(conf): toc_spec_files = [] step_files = [] for fn in expand_tree(os.path.join(conf.paths.includes), input_extension='yaml'): base = os.path.basename(fn) if base.startswith('toc-spec'): toc_spec_files.append(fn) elif base.startswith('ref-spec'): toc_spec_files.append(fn) elif base.startswith('steps'): step_files.append(fn) maskl = len(conf.paths.source) path_prefix = conf.paths.includes[len(conf.paths.source):] mapping = {} for spec_file in toc_spec_files: if os.path.exists(spec_file): data = ingest_yaml_doc(spec_file) else: continue deps = [ os.path.join(path_prefix, i ) for i in data['sources']] mapping[spec_file[maskl:]] = deps for step_def in step_files: data = ingest_yaml_list(step_def) deps = [] for step in data: if 'source' in step: deps.append(step['source']['file']) if len(deps) != 0: deps = [ os.path.join(path_prefix, i ) for i in deps ] mapping[step_def[maskl:]] = deps return mapping
def get_sconf_base(conf): sconf_path = os.path.join(conf.paths.projectroot, conf.paths.builddata, "sphinx.yaml") return ingest_yaml_doc(sconf_path)
def main(): m = MakefileCloth() conf = new_config() output_file = os.path.join(conf.paths.projectroot, conf.paths.output, 'makefile.giza_build') m.section_break('giza build integration') m.newline() m.section_break('content generation targets') for gen_target in [ 'api', 'assets', 'images', 'intersphinx', 'options', 'primer', 'steps', 'tables', 'toc']: m.target(hyph_concat('giza', gen_target)) m.job('giza generate ' + gen_target) m.target(hyph_concat('giza', 'force', gen_target)) m.job('giza --force generate ' + gen_target) m.newline() m.section_break('sphinx targets') sconf = ingest_yaml_doc(os.path.join(conf.paths.projectroot, conf.paths.builddata, 'sphinx.yaml')) builders = [b for b in sconf if not b.endswith('base') and b not in ('prerequisites', 'generated-source', 'languages', 'editions', 'sphinx_builders')] if 'editions' in sconf: editions = sconf['editions'] else: editions = [] if 'root-base' in sconf and 'languages' in sconf['root-base']: languages = sconf['root-base']['languages'] else: languages = [] complete = [] for builder in builders: if '-' in builder: builder = builder.split('-')[0] if builder in complete: continue m.comment(builder + ' targets') for edition in editions: m.target(hyph_concat('giza', builder, edition)) m.job('giza sphinx --builder {0} --edition {1}'.format(builder, edition)) for language in languages: m.target(hyph_concat('giza', builder, edition, language)) m.job('giza sphinx --builder {0} --edition {1} --language {2}'.format(builder, edition, language)) if len(editions) == 0: m.target(hyph_concat('giza', builder)) m.job('giza sphinx --builder ' + builder) for language in languages: m.target(hyph_concat('giza', builder, language)) m.job('giza sphinx --builder {0} --language {1}'.format(builder, language)) else: m.target(hyph_concat('giza', builder)) m.job('giza sphinx --builder {0} --edition {1}'.format(builder, ' '.join(editions))) m.newline() complete.append(builder) m.section_break('deploy targets') if 'push' in conf.system.files.data: for ptarget in conf.system.files.data.push: name = ptarget['target'] m.target(hyph_concat('giza-deploy', name)) m.job('giza deploy --target ' + name) m.newline() integration_path = os.path.join(conf.paths.projectroot, conf.paths.builddata, 'integration.yaml') if os.path.exists(integration_path): m.section_break('integration and publish targets') iconf = ingest_yaml_doc(integration_path) if 'base' in iconf: languages = [ k for k in iconf.keys() if not k.endswith('base') ] iconf = iconf['base'] else: languages = [] targets = set([ target.split('-')[0] for target in iconf['targets'] if '/' not in target and not target.startswith('htaccess') ]) base_job = 'giza sphinx --builder {0}'.format(' '.join(targets)) if len(editions) > 0: base_job += " --serial_sphinx --edition " + ' '.join(editions) m.target('giza-publish') m.job(base_job) m.newline() for lang in languages: m.target('giza-publish-' + lang) m.job(base_job + ' --language ' + lang) m.newline() # following targets build a group of sphinx targets followed by running # one or more deploy actions. m.section_break('push targets') if 'push' in conf.system.files.data: for ptarget in conf.system.files.data.push: push_base_job = 'giza push --deploy {0} --builder {1}'.format(ptarget['target'], ' '.join(targets)) if len(editions) > 0: push_base_job += " --serial_sphinx --edition " + ' '.join(editions) m.target('giza-' + ptarget['target']) m.job(push_base_job) m.newline() for lang in languages: m.target('giza-{0}-{1}'.format(ptarget['target'], lang)) m.job(push_base_job + ' --language ' + lang) m.newline() m.write(output_file) print('[meta-build]: built "build/makefile.giza_build" to integrate giza')