def cheatsheet_tex(): print(f"Using docs directory: {DOCS_DIR}") print(f"Using ops docs directory: {OP_DOCS_DIR}") print() output = "" for (section, title, new_page) in OPS_SECTIONS: toml_file = Path(OP_DOCS_DIR, section + ".toml") if toml_file.exists() and toml_file.is_file(): output += f"\group{{{ title }}}\n\n" print(f"Reading {toml_file}") # n.b. Python 3.6 dicts maintain insertion order ops = toml.loads(toml_file.read_text()) validate_toml(ops) ops_array = pool.map(inject_latex, ops.values()) for op in ops_array: prototype = op["prototype"] if "prototype_set" in op: prototype += " / " + op["prototype_set"] output += "\\begin{op}" if "aliases" in op and len(op["aliases"]) > 0: output += "[" + latex_safe(" ".join(op["aliases"])) + "]" output += "{" + prototype + "}" output += "\n" output += op["short_latex"] output += "\\end{op}" output += "\n\n" if new_page: output += "\pagebreak\n\n" return output
def uuid_info(cfg, uuid): """Return information about the composition :param cfg: Configuration settings :type cfg: ComposerConfig :param uuid: The UUID of the build :type uuid: str :returns: dictionary of information about the composition :rtype: dict :raises: RuntimeError if there was a problem This will return a dict with the following fields populated: * id - The uuid of the comoposition * config - containing the configuration settings used to run Anaconda * blueprint - The depsolved blueprint used to generate the kickstart * commit - The (local) git commit hash for the blueprint used * deps - The NEVRA of all of the dependencies used in the composition * compose_type - The type of output generated (tar, iso, etc.) * queue_status - The final status of the composition (FINISHED or FAILED) """ uuid_dir = joinpaths(cfg.get("composer", "lib_dir"), "results", uuid) if not os.path.exists(uuid_dir): raise RuntimeError("%s is not a valid build_id" % uuid) # Load the compose configuration cfg_path = joinpaths(uuid_dir, "config.toml") if not os.path.exists(cfg_path): raise RuntimeError("Missing config.toml for %s" % uuid) cfg_dict = toml.loads(open(cfg_path, "r").read()) frozen_path = joinpaths(uuid_dir, "frozen.toml") if not os.path.exists(frozen_path): raise RuntimeError("Missing frozen.toml for %s" % uuid) frozen_dict = toml.loads(open(frozen_path, "r").read()) deps_path = joinpaths(uuid_dir, "deps.toml") if not os.path.exists(deps_path): raise RuntimeError("Missing deps.toml for %s" % uuid) deps_dict = toml.loads(open(deps_path, "r").read()) details = compose_detail(uuid_dir) commit_path = joinpaths(uuid_dir, "COMMIT") if not os.path.exists(commit_path): raise RuntimeError("Missing commit hash for %s" % uuid) commit_id = open(commit_path, "r").read().strip() return { "id": uuid, "config": cfg_dict, "blueprint": frozen_dict, "commit": commit_id, "deps": deps_dict, "compose_type": details["compose_type"], "queue_status": details["queue_status"], "image_size": details["image_size"] }
def test_default_config(tmpdir, tljh_dir): state_dir = tmpdir.mkdir("state") traefik.ensure_traefik_config(str(state_dir)) assert state_dir.join("traefik.toml").exists() traefik_toml = os.path.join(state_dir, "traefik.toml") with open(traefik_toml) as f: toml_cfg = f.read() # print config for debugging on failure print(config.CONFIG_FILE) print(toml_cfg) cfg = toml.loads(toml_cfg) assert cfg["defaultEntryPoints"] == ["http"] assert len(cfg["entryPoints"]["auth_api"]["auth"]["basic"]["users"]) == 1 # runtime generated entry, value not testable cfg["entryPoints"]["auth_api"]["auth"]["basic"]["users"] = [""] assert cfg["entryPoints"] == { "http": { "address": ":80" }, "auth_api": { "address": "127.0.0.1:8099", "auth": { "basic": { "users": [""] } }, "whiteList": { "sourceRange": ["127.0.0.1"] } }, }
def get_sphinx_configuration(project_dir): """Read the Sphinx configuration from ``pyproject.toml```.""" try: project = toml.loads((Path(project_dir) / 'pyproject.toml').read_text()) return reduce(lambda a, b: a[b], 'tool.flinx.configuration'.split('.'), project) except (FileNotFoundError, KeyError): return {}
def main(filesystem, locator, link_dir, handle_error): with filesystem.open(locator.root.descendant("virtualenvs.toml")) as venvs: contents = pytoml.load(venvs) for name, config in contents["virtualenv"].items(): config.setdefault("sys.version", sys.version) virtualenv = locator.for_name(name=name) existing_config_path = virtualenv.path.descendant("installed.toml") try: with filesystem.open(existing_config_path) as existing_config: if pytoml.loads(existing_config.read()) == config: continue except FileNotFound: virtualenv.create() else: virtualenv.recreate_on(filesystem=filesystem) packages, requirements = _to_install(config=config) try: virtualenv.install(packages=packages, requirements=requirements) except Exception: handle_error(virtualenv) continue for link in config.get("link", []): _link( source=virtualenv.binary(name=link), to=link_dir.descendant(link), filesystem=filesystem, ) with filesystem.open(existing_config_path, "w") as existing_config: existing_config.write(pytoml.dumps(config).encode("utf-8"))
def __init__(self, project_path='pyproject.toml'): project = toml.loads(Path(project_path).read_text()) try: dotpath = self._toml_path.split('.') self._metadata = reduce(lambda a, b: a[b], dotpath, project) except KeyError: pass
def insert_wp_paths(): """A one-time operation: insert \"path =\" definitions in the front matter for the blog posts, to preserve the old Wordpress URLs. """ import pytoml for path in all_blog_posts(): fm_text, content = read_markdown(path) fm = pytoml.loads(fm_text) web_path = fm['extra']['wp_rel_permalink'] if web_path[0] == '/': web_path = web_path[1:] if web_path[-1] == '/': web_path = web_path[:-1] fm_lines = fm_text.splitlines() for i in range(len(fm_lines)): if fm_lines[i].startswith('title ='): break else: raise Exception(f'no title in {path}?') fm_lines = fm_lines[:i + 1] + [f'path = "{web_path}"' ] + fm_lines[i + 1:] fm_text = '\n'.join(fm_lines) rewrite_markdown(path, fm_text, content)
def load_rule_contents(rule_file: Path, single_only=False) -> list: """Load a rule file from multiple formats.""" _, extension = os.path.splitext(rule_file) raw_text = rule_file.read_text() if extension in ('.ndjson', '.jsonl'): # kibana exported rule object is ndjson with the export metadata on the last line contents = [json.loads(line) for line in raw_text.splitlines()] if len(contents) > 1 and 'exported_count' in contents[-1]: contents.pop(-1) if single_only and len(contents) > 1: raise ValueError('Multiple rules not allowed') return contents or [{}] elif extension == '.toml': rule = pytoml.loads(raw_text) else: rule = load_dump(rule_file) if isinstance(rule, dict): return [rule] elif isinstance(rule, list): return rule else: raise ValueError(f"Expected a list or dictionary in {rule_file}")
def test_manual_ssl_config(tljh_dir): state_dir = config.STATE_DIR config.set_config_value(config.CONFIG_FILE, "https.enabled", True) config.set_config_value(config.CONFIG_FILE, "https.tls.key", "/path/to/ssl.key") config.set_config_value(config.CONFIG_FILE, "https.tls.cert", "/path/to/ssl.cert") traefik.ensure_traefik_config(str(state_dir)) traefik_toml = os.path.join(state_dir, "traefik.toml") with open(traefik_toml) as f: toml_cfg = f.read() # print config for debugging on failure print(config.CONFIG_FILE) print(toml_cfg) cfg = toml.loads(toml_cfg) assert cfg["defaultEntryPoints"] == ["http", "https"] assert "acme" not in cfg assert cfg["entryPoints"] == { "http": { "address": ":80", "redirect": { "entryPoint": "https" } }, "https": { "address": ":443", "backend": "jupyterhub", "tls": { "certificates": [{ "certFile": "/path/to/ssl.cert", "keyFile": "/path/to/ssl.key" }] }, }, }
def load_config(): filename = sys.argv[1] if len(sys.argv) > 1 else 'benchmark.toml' with open(filename) as configfile: config = pytoml.loads(configfile.read()) items = config['item'] default = items['default'] if 'options' in config: options = config['options'] else: options = dict(ref_result="C", time_field="elapsed", show_diff_below=0.9, verbose=True) ret_items = {} for name, item in zip(items, items.values()): if name == 'default': continue if name.startswith('"') and name.endswith('"'): import ast name = ast.literal_eval(name) profile = dict(default) profile.update(item) profile['name'] = name for k, v in zip(profile, profile.values()): if type(v) is str: profile[k] = Template(v).safe_substitute(**profile) ret_items[name] = profile return ret_items, options
def main(filename, timestamp=None, with_group_by=with_group_by, execute=False): if filename == 'batch': batch_execute('examples', True) return source = open(filename).read() if not timestamp: timestamp = datetime.now() else: timestamp = datetime.strptime(timestamp, timestamp_format) sql_query = compile_source(source, timestamp, int(with_group_by)) if not int(execute): print(sql_query) else: import pytoml import pyodbc config = pytoml.loads(open('config.toml').read()) odbc_connection_template = Template(config['odbc']['connect_template']) odbc_connection_string = odbc_connection_template.substitute(**config['odbc']) connection = pyodbc.connect(odbc_connection_string) cur = connection.cursor() cur.execute(sql_query) print('-'*79) for row in cur: print(*row, sep='\t| ')
def parse_metadata (self): ''' Open and parse the included metadata file in the TAB. ''' metadata_tarinfo = self.tab.getmember('metadata.toml') metadata_str = self.tab.extractfile(metadata_tarinfo).read().decode('utf-8') return pytoml.loads(metadata_str)
def parse(self, parser): """ Parse a column or row. :param parser: Template parser :type parser: jinja2.parser.Parser :return: A null output node. :rtype: jinja2.nodes.Node """ lineno = next(parser.stream).lineno name = parse_constantlike(self.environment, parser) # Parse the plugin name. body = parser.parse_statements(['name:endplugin'], drop_needle=True) layout = self._get_layout(parser) config = None if body: try: config = flatten_const_node_list(self.environment, body) except Unflattenable as uf: raise NonConstant( "A `plugin` block may only contain static layout (found: %r)" % uf.args[0]) config = toml.loads( config, "<%s.%s in %s>" % (layout.placeholder_name, name, parser.name)) layout.add_plugin(name, config) return noop_node(lineno)
def unmarshall_config_reader(file_, d, config_type): config_type = config_type.lower() if config_type in ["yaml", "yml"]: try: f = yaml.load(file_) try: d.update(yaml.load(f)) except AttributeError: # to read files d.update(f) except Exception as e: raise ConfigParserError(e) elif config_type == "json": try: f = json.loads(file_) d.update(f) except Exception as e: raise ConfigParserError(e) elif config_type == "toml": try: try: f = toml.loads(file_) d.update(f) except AttributeError: # to read streams d.update(file_) except Exception as e: raise ConfigParserError(e) return d
def test_add_unit_to_not_empty_course_at_end(self): self._create_testing_unit( "Python Intro", 'unit-1-python-intro', 'f4ed574a-a11b-4119-bb64-c1feaa05ea55') unit_1_path = self.course_directory_path / 'unit-1-python-intro' self.assertDirectoryExists(unit_1_path) io.add_unit_to_course( course_directory_path=self.course_directory_path, name='Data Types') unit_path = self.course_directory_path / 'unit-2-data-types' dot_rmotr_path = unit_path / '.rmotr' readme_path = unit_path / 'README.md' self.assertDirectoryExists(unit_path) self.assertFileExists(dot_rmotr_path) self.assertFileExists(readme_path) with dot_rmotr_path.open() as fp: dot_rmotr_content = toml.loads(fp.read()) self.assertTrue('uuid' in dot_rmotr_content) self.assertEqual(dot_rmotr_content['name'], 'Data Types') with readme_path.open('r') as fp: content = fp.read() self.assertEqual(content, """# Data Types """)
def prepare(): toml = s.get(config_url).text toml = pytoml.loads(toml) version_date = toml['date'] version = toml['pkg']['rust']['version'].split('-', 1)[0] cargo_version = toml['pkg']['cargo']['version'].split('-', 1)[0] rustfmt_version = toml['pkg']['rustfmt-preview']['version'].split('-', 1)[0] clippy_version = toml['pkg']['clippy-preview']['version'].split('-', 1)[0] try: clippy_url = toml['pkg']['clippy-preview']['target'] \ ['x86_64-unknown-linux-gnu']['xz_url'] except KeyError: return 'no clippy available?' stds = [ Std(target, toml['pkg']['rust-std']['target'][target]) for target in STDS ] loader = tornado.template.Loader('.') global PKGBUILD PKGBUILD = loader.load('PKGBUILD.tmpl').generate( stds=stds, version=version, version_date=version_date.replace('-', ''), version_date_raw=version_date, cargo_version=cargo_version, rustfmt_version=rustfmt_version, clippy_version=clippy_version, clippy_url=clippy_url, )
def pre_build(): toml = s.get(config_url).text toml = pytoml.loads(toml) version_date = toml['date'] version = toml['pkg']['rust']['version'].split('-', 1)[0] cargo_version = toml['pkg']['cargo']['version'].split('-', 1)[0] rustfmt_version = toml['pkg']['rustfmt-preview']['version'].split('-', 1)[0] clippy_version = toml['pkg']['clippy-preview']['version'].split('-', 1)[0] clippy_url = toml['pkg']['clippy-preview']['target'] \ ['x86_64-unknown-linux-gnu']['xz_url'] if not debug: oldfiles = (glob.glob('*.xz') + glob.glob('*.xz.asc') + glob.glob('*.part')) for f in oldfiles: os.unlink(f) stds = [Std(target, toml['pkg']['rust-std']['target'][target]) for target in STDS] loader = tornado.template.Loader('.') content = loader.load('PKGBUILD.tmpl').generate( stds = stds, version = version, version_date = version_date.replace('-', ''), version_date_raw = version_date, cargo_version = cargo_version, rustfmt_version = rustfmt_version, clippy_version = clippy_version, clippy_url = clippy_url, ) with open('PKGBUILD', 'wb') as output: output.write(content)
def main(filename, timestamp=None, with_group_by=with_group_by, execute=False): if filename == 'batch': batch_execute('examples', True) return source = open(filename).read() if not timestamp: timestamp = datetime.now() else: timestamp = datetime.strptime(timestamp, timestamp_format) sql_query = compile_source(source, timestamp, int(with_group_by)) if not int(execute): print(sql_query) else: import pytoml import pyodbc config = pytoml.loads(open('config.toml').read()) odbc_connection_template = Template(config['odbc']['connect_template']) odbc_connection_string = odbc_connection_template.substitute( **config['odbc']) connection = pyodbc.connect(odbc_connection_string) cur = connection.cursor() cur.execute(sql_query) print('-' * 79) for row in cur: print(*row, sep='\t| ')
def test_default_config(tmpdir, tljh_dir): state_dir = tmpdir.mkdir("state") traefik.ensure_traefik_config(str(state_dir)) assert state_dir.join("traefik.toml").exists() traefik_toml = os.path.join(state_dir, "traefik.toml") with open(traefik_toml) as f: toml_cfg = f.read() # print config for debugging on failure print(config.CONFIG_FILE) print(toml_cfg) cfg = toml.loads(toml_cfg) assert cfg["defaultEntryPoints"] == ["http"] assert cfg["entryPoints"] == {"http": {"address": ":80"}} assert cfg["frontends"] == { "jupyterhub": { "backend": "jupyterhub", "passHostHeader": True } } assert cfg["backends"] == { "jupyterhub": { "servers": { "chp": { "url": "http://127.0.0.1:15003" } } } }
def load(filename=None): """ Load application configuration from a file and merge it with the default configuration. If the ``FEDORA_MESSAGING_CONF`` environment variable is set to a filesystem path, the configuration will be loaded from that location. Otherwise, the path defaults to ``/etc/fedora-messaging/config.toml``. """ config = DEFAULTS.copy() if filename: config_path = filename elif 'FEDORA_MESSAGING_CONF' in os.environ: config_path = os.environ['FEDORA_MESSAGING_CONF'] else: config_path = '/etc/fedora-messaging/config.toml' if os.path.exists(config_path): _log.info('Loading configuration from {}'.format(config_path)) with open(config_path) as fd: try: file_config = pytoml.loads(fd.read()) for key in file_config: config[key.lower()] = file_config[key] except pytoml.core.TomlError as e: _log.error('Failed to parse {}: {}'.format(config_path, str(e))) sys.exit(1) else: _log.info('The configuration file, {}, does not exist.'.format(config_path)) return config
def prepare(): toml = s.get(config_url).text toml = pytoml.loads(toml) version_date = toml['date'] version = toml['pkg']['rust']['version'].split('-', 1)[0] cargo_version = toml['pkg']['cargo']['version'].split('-', 1)[0] rustfmt_version = toml['pkg']['rustfmt-preview']['version'].split('-', 1)[0] clippy_version = toml['pkg']['clippy-preview']['version'].split('-', 1)[0] try: clippy_url = toml['pkg']['clippy-preview']['target'] \ ['x86_64-unknown-linux-gnu']['xz_url'] except KeyError: return 'no clippy available?' stds = [Std(target, toml['pkg']['rust-std']['target'][target]) for target in STDS] loader = tornado.template.Loader('.') global PKGBUILD PKGBUILD = loader.load('PKGBUILD.tmpl').generate( stds = stds, version = version, version_date = version_date.replace('-', ''), version_date_raw = version_date, cargo_version = cargo_version, rustfmt_version = rustfmt_version, clippy_version = clippy_version, clippy_url = clippy_url, )
def get_config(self): try: with open(self.config_path, 'r') as f: content = pytoml.loads(f.read().strip()) except FileNotFoundError: content = {} return content
def parse(cls, contents): """ Return parsed python data-structure from the input content. """ return pytoml.loads( contents )
def createWorkdir(self, contents): workdir = self.graph.target.git.workdir locales = set() includes = [] for tpath, content_list in contents.items(): try: b_content = merge_channels(tpath, content_list) except MergeNotSupportedError: b_content = content_list[0] if tpath.endswith("l10n.toml"): try: data = toml.loads(b_content) if 'locales' in data: locales.update(data['locales']) includes.append(tpath) except Exception as e: print(e) pass tpath = mozpath.join(workdir, tpath) tdir = mozpath.dirname(tpath) if not os.path.isdir(tdir): os.makedirs(tdir) with open(tpath, "wb") as fh: fh.write(b_content) self.ensureL10nToml(workdir, locales, includes)
def gitrpm_repo_test(self): """Test creating a dnf repo of the git rpms""" recipe = toml.loads(""" [[repos.git]] rpmname="repo-test-alpha" rpmversion="1.1.0" rpmrelease="1" summary="Testing the git rpm code" repo="file://%s" ref="v1.1.0" destination="/srv/testing-alpha/" [[repos.git]] rpmname="repo-test-beta" rpmversion="1.0.0" rpmrelease="1" summary="Testing the git rpm code" repo="file://%s" ref="v1.0.0" destination="/srv/testing-beta/" """ % (self.repodir, self.repodir)) try: temp_dir = tempfile.mkdtemp(prefix="git-rpm-test.") repo_dir = create_gitrpm_repo(temp_dir, recipe) self.assertTrue(len(repo_dir) > 0) self.assertTrue(os.path.exists(joinpaths(repo_dir, "repo-test-alpha-1.1.0-1.noarch.rpm"))) self.assertTrue(os.path.exists(joinpaths(repo_dir, "repo-test-beta-1.0.0-1.noarch.rpm"))) finally: shutil.rmtree(temp_dir)
def __init__(self, path_data, env=None, ignore_missing_includes=False): # mock, use the path as data. Yeah, not nice super(MockTOMLParser, self).__init__( '/tmp/base.toml', env=env, ignore_missing_includes=ignore_missing_includes ) self.data = toml.loads(path_data)
def deprecate_rule(ctx: click.Context, rule_file: str): """Deprecate a rule.""" import pytoml from .packaging import load_versions version_info = load_versions() rule_file = Path(rule_file) contents = pytoml.loads(rule_file.read_text()) rule = TOMLRule(path=rule_file, contents=contents) if rule.id not in version_info: click.echo( 'Rule has not been version locked and so does not need to be deprecated. ' 'Delete the file or update the maturity to `development` instead') ctx.exit() today = time.strftime('%Y/%m/%d') new_meta = dataclasses.replace(rule.contents.metadata, updated_date=today, deprecation_date=today, maturity='deprecated') contents = dataclasses.replace(rule.contents, metadata=new_meta) deprecated_path = get_path('rules', '_deprecated', rule_file.name) # create the new rule and save it new_rule = TOMLRule(contents=contents, path=Path(deprecated_path)) new_rule.save_toml() # remove the old rule rule_file.unlink() click.echo( f'Rule moved to {deprecated_path} - remember to git add this file')
def load(s, config_file): log.info("loading configuration file %s", config_file) try: with open(config_file, "r") as f: s.config = toml.loads(f.read()) except FileNotFoundError as e: log.error("configuration file %s not found!" % config_file) raise e
def branches(self): rev = self.lookup_branch(self.branch).target tree = self[rev].tree if 'l10n.toml' not in tree: return [self.branch] toml_data = self[tree['l10n.toml'].id].data config = toml.loads(toml_data) return config.get("branches", [self.branch])
def load_toml(path, keys_to_convert_to_set=None): _check_file_exists(path) with open(path, 'r') as the_file: contents = the_file.read() loaded = pytoml.loads(contents) if keys_to_convert_to_set: convert_dict_keys(loaded, keys=keys_to_convert_to_set, func=set) return loaded
def _main(): ap = argparse.ArgumentParser() ap.add_argument('-d', '--dir', action='append') ap.add_argument('testcase', nargs='*') args = ap.parse_args() if not args.dir: args.dir = [os.path.join(os.path.split(__file__)[0], 'toml-test/tests')] succeeded = [] failed = [] for path in args.dir: if not os.path.isdir(path): print('error: not a dir: {}'.format(path)) return 2 for top, dirnames, fnames in os.walk(path): for fname in fnames: if not fname.endswith('.toml'): continue if args.testcase and not any(arg in fname for arg in args.testcase): continue parse_error = None try: with open(os.path.join(top, fname), 'rb') as fin: parsed = toml.load(fin) except toml.TomlError: parsed = None parse_error = sys.exc_info() else: dumped = toml.dumps(parsed) parsed2 = toml.loads(dumped) if parsed != parsed2: failed.append((fname, None)) continue with open(os.path.join(top, fname), 'rb') as fin: parsed = toml.load(fin, translate=_testbench_literal) try: with io.open(os.path.join(top, fname[:-5] + '.json'), 'rt', encoding='utf-8') as fin: bench = json.load(fin) except IOError: bench = None if parsed != adjust_bench(bench): failed.append((fname, parsed, bench, parse_error)) else: succeeded.append(fname) for f, parsed, bench, e in failed: print('failed: {}\n{}\n{}'.format(f, json.dumps(parsed, indent=4), json.dumps(bench, indent=4))) if e: traceback.print_exception(*e) print('succeeded: {}'.format(len(succeeded))) return 1 if failed or not succeeded else 0
def _main(): ap = argparse.ArgumentParser() ap.add_argument('-d', '--dir', action='append') ap.add_argument('testcase', nargs='*') args = ap.parse_args() if not args.dir: args.dir = [os.path.join(os.path.split(__file__)[0], 'toml-test/tests')] succeeded = [] failed = [] for path in args.dir: if not os.path.isdir(path): print('error: not a dir: {0}'.format(path)) return 2 for top, dirnames, fnames in os.walk(path): for fname in fnames: if not fname.endswith('.toml'): continue if args.testcase and not any(arg in fname for arg in args.testcase): continue parse_error = None try: with open(os.path.join(top, fname), 'rb') as fin: parsed = toml.load(fin) except toml.TomlError: parsed = None parse_error = sys.exc_info() else: dumped = toml.dumps(parsed) parsed2 = toml.loads(dumped) if parsed != parsed2: failed.append((fname, None)) continue with open(os.path.join(top, fname), 'rb') as fin: parsed = toml.load(fin, translate=_testbench_literal) try: with io.open(os.path.join(top, fname[:-5] + '.json'), 'rt', encoding='utf-8') as fin: bench = json.load(fin) except IOError: bench = None if parsed != adjust_bench(bench): failed.append((fname, parsed, bench, parse_error)) else: succeeded.append(fname) for f, parsed, bench, e in failed: print('failed: {}\n{}\n{}'.format(f, json.dumps(parsed, indent=4), json.dumps(bench, indent=4))) if e: traceback.print_exception(*e) print('succeeded: {0}'.format(len(succeeded))) return 1 if failed or not succeeded else 0
def test_splitting_array(): toml_text = """ somethingweird = false [section] k = [4, 8, 15, 16, 23, 42, 4, 8, 15, 16, 23, 42, 4, 8, 15, 16, 23, 42, 4, 8, 15, 16, 23, 42, 4, 8, 15, 16, 23, 42, 4, 8, 15, 16, 23, 42, 4, 8, 15, 16, 23, 42] [data] id = 12 """ prettified = elements_to_text(line_length_limiter(text_to_elements(toml_text))) assert pytoml.loads(prettified) == pytoml.loads(toml_text) assert all(len(line) < 120 for line in prettified.split('\n'))
def fetch_manifest(channel='stable'): url = 'https://static.rust-lang.org/dist/channel-rust-' + channel + '.toml' req = requests.get(url) req.raise_for_status() manifest = toml.loads(req.content) if manifest['manifest-version'] != '2': raise NotImplementedError('Unrecognized manifest version %s.' % manifest['manifest-version']) return manifest
def __init__(self, cfg): """ Load MeTA's filter chain, tagger, and parser. """ toml = pytoml.loads(open(cfg).read()) self.tgr = metapy.sequence.PerceptronTagger(toml['sequence']['prefix']) self.psr = metapy.parser.Parser(toml['parser']['prefix']) self.stream = metapy.analyzers.ICUTokenizer() self.stream = metapy.analyzers.PennTreebankNormalizer(self.stream)
def add(index_path, key, value): index = pytoml.loads(subprocess.check_output(decrypt + [index_path])) if key not in index: index[key] = value cmd = encrypt + ['--yes', '--output', index_path] with subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) as gpg: err = gpg.communicate(pytoml.dumps(index))[1].encode() if gpg.wait() != 0: raise subprocess.CalledProcessError(gpg.poll(), cmd, err)
def _init_configs(self): self.v.set_config_type("yaml") r = yaml.dump(text(yaml_example)) self.v._unmarshall_reader(r, self.v._config) self.v.set_config_type("json") r = json.dumps(json_example) self.v._unmarshall_reader(r, self.v._config) self.v.set_config_type("toml") r = toml.loads(toml_example) self.v._unmarshall_reader(r, self.v._config)
def _load_remote_files_data(get_new_remote_data = False) -> dict: if get_new_remote_data: response = urlopen(config.remote_data_file) data = response.read() data_as_dict = toml.loads(data) global cached_remote_data cached_remote_data = data_as_dict return data_as_dict else: return cached_remote_data
def _setup_logging(): try: with open('logging.toml') as f: raw = f.read() except FileNotFoundError: try: with open(os.path.expanduser('~/.config/ion/logging.toml')) as f: raw = f.read() except FileNotFoundError: raw = pkg_resources.resource_string(__name__, 'data/logging.toml') conf = toml.loads(raw) logging.config.dictConfig(conf)
def fetch_manifest(channel='stable'): if '-' in channel: channel, date = channel.split('-', 1) prefix = '/' + date else: prefix = '' url = 'https://static.rust-lang.org/dist%s/channel-rust-%s.toml' % ( prefix, channel) req = requests.get(url) req.raise_for_status() manifest = toml.loads(req.content) if manifest['manifest-version'] != '2': raise NotImplementedError('Unrecognized manifest version %s.' % manifest['manifest-version']) return manifest
def config(args): config = default_config() if args.config: try: with open(args.config) as config_file: data = toml.loads(config_file.read()) if "os" in data: config["os"].update(data["os"]) if "core" in data: config["core"].update(data["core"]) except toml.core.TomlError as err: parser.error("Errors found in the config file:\n\n", err) return merge(config, args)
def config(args): config = default_config() if args.config: with open(args.config) as config_file: data = toml.loads(config_file.read()) if "os" in data: config["os"].update(data["os"]) if "core" in data: config["core"].update(data["core"]) if config["os"]["proxy-settings"] is True and (sys.platform != "darwin"): raise StandardError("Proxy settings is only available on Mac OX") return merge(config, args)
def _main(): succeeded = [] failed = [] for top, dirnames, fnames in os.walk('.'): for fname in fnames: if not fname.endswith('.toml'): continue if sys.argv[1:] and not any(arg in fname for arg in sys.argv[1:]): continue parse_error = None try: with open(os.path.join(top, fname), 'rb') as fin: parsed = toml.load(fin) except toml.TomlError: parsed = None parse_error = sys.exc_info() else: dumped = toml.dumps(parsed) parsed2 = toml.loads(dumped) if parsed != parsed2: failed.append((fname, None)) continue with open(os.path.join(top, fname), 'rb') as fin: parsed = toml.load(fin, translate=_testbench_literal) try: with io.open(os.path.join(top, fname[:-5] + '.json'), 'rt', encoding='utf-8') as fin: bench = json.load(fin) except IOError: bench = None if parsed != bench: failed.append((fname, parsed, bench, parse_error)) else: succeeded.append(fname) for f, parsed, bench, e in failed: print('failed: {}\n{}\n{}'.format(f, json.dumps(parsed, indent=4), json.dumps(bench, indent=4))) if e: traceback.print_exception(*e) print('succeeded: {}'.format(len(succeeded))) return 1 if failed else 0
def loads(self, raw): conf = toml.loads(raw) self.USERNAME = _conf_get(conf, 'auth', 'username', default=self.USERNAME) self.PASSWORD = _conf_get(conf, 'auth', 'password', default=self.PASSWORD) self.CLIENT_ID = _conf_get(conf, 'auth', 'client-id', default=self.CLIENT_ID) self.CLIENT_SECRET = _conf_get(conf, 'auth', 'client-secret', default=self.CLIENT_SECRET) self.REDIRECT_URI = _conf_get(conf, 'auth', 'redirect-uri', default=self.REDIRECT_URI) self.REFRESH_TOKEN = _conf_get(conf, 'auth', 'refresh-token', default=self.REFRESH_TOKEN) self.NETWORK_NAME = _conf_get(conf, 'network', 'name', default=self.NETWORK_NAME) self.MULTIREDDIT = _conf_get(conf, 'network', 'multireddit', default=self.MULTIREDDIT) self.MULTI_OWNER = _conf_get(conf, 'network', 'multi-owner', default=self.USERNAME) self.DEFAULT_MODS = _conf_get(conf, 'network', 'mods', default=self.DEFAULT_MODS) self.WIKI_PAGES = _conf_get(conf, 'network', 'wiki-pages', default=self.WIKI_PAGES) self.NSFW_OK = _conf_get(conf, 'network', 'nsfw', default=self.NSFW_OK) self.NSFW_WHITELIST_OK = _conf_get(conf, 'network', 'nsfw-whitelist', default=self.NSFW_WHITELIST_OK) self.COMMENT_FOOTER = _conf_get(conf, 'network', 'comment-footer', default=self.COMMENT_FOOTER) self.DOMAINS = _conf_get(conf, 'posts', 'domains', default=self.DOMAINS) self.EXTENSIONS = _conf_get(conf, 'posts', 'extensions', default=self.EXTENSIONS) self.PARENT_SUB = _conf_get(conf, 'parent', 'name', default=self.PARENT_SUB) update_children = _conf_get(conf, 'update_children', default=False) self.CHILD_SUBS = self._load_group(conf, 'child', self.CHILD_SUBS, update_children) update_cousins = _conf_get(conf, 'update_cousins', default=False) self.COUSIN_SUBS = self._load_group(conf, 'cousin', self.COUSIN_SUBS, update_cousins) self.DISCORD_CLIENTID = _conf_get(conf, 'discord', 'client_id', default=self.DISCORD_CLIENTID) self.DISCORD_TOKEN = _conf_get(conf, 'discord', 'token', default=self.DISCORD_TOKEN) self.DISCORD_INBOX_CHAN_ID = _conf_get(conf, 'discord', 'inbox_channel', default=None) self.DISCORD_FALSEPOS_CHAN_ID = _conf_get(conf, 'discord', 'falsepos_channel', default=None) self.DISCORD_OC_CHAN_ID = _conf_get(conf, 'discord', 'oc_channel', default=None) self.DISCORD_GITHUB_CHAN_ID = _conf_get(conf, 'discord', 'github_channel', default=None) self.DISCORD_MOD_CHAN_ID = _conf_get(conf, 'discord', 'mod_channel', default=None) self.DISCORD_KEEPALIVE_CHAN_ID = _conf_get(conf, 'discord', 'keepalive_channel', default=None) self.GITHUB_OAUTH_TOKEN = _conf_get(conf, 'github', 'token', default=None) self.GITHUB_REPO_USER = _conf_get(conf, 'github', 'user', default=None) self.GITHUB_REPO_NAME = _conf_get(conf, 'github', 'repo_name', default=None)
def test_add_unit_to_course_in_between_other_units(self): self._create_testing_unit( "Python Intro", 'unit-1-python-intro', 'f4ed574a-a11b-4119-bb64-c1feaa05ea55') self._create_testing_unit( "Data Types", 'unit-2-data-types', '8a22574a-a11b-4119-a964-c1feaa05c833') self._create_testing_unit( "Collections", 'unit-3-collections', 'c822574a-a81b-4aa9-a964-c1feaa05a7b2') unit_1_path = self.course_directory_path / 'unit-1-python-intro' self.assertDirectoryExists(unit_1_path) unit_2_path = self.course_directory_path / 'unit-2-data-types' self.assertDirectoryExists(unit_2_path) unit_3_path = self.course_directory_path / 'unit-3-collections' self.assertDirectoryExists(unit_3_path) io.add_unit_to_course( course_directory_path=self.course_directory_path, name='Interpreters', order=2) unit_path = self.course_directory_path / 'unit-2-interpreters' dot_rmotr_path = unit_path / '.rmotr' self.assertDirectoryExists(unit_path) self.assertFileExists(dot_rmotr_path) self.assertFileExists(unit_path / 'README.md') with dot_rmotr_path.open() as fp: dot_rmotr_content = toml.loads(fp.read()) self.assertTrue('uuid' in dot_rmotr_content) self.assertEqual(dot_rmotr_content['name'], 'Interpreters') # Postconditions unit_1_path = self.course_directory_path / 'unit-1-python-intro' self.assertDirectoryExists(unit_1_path) unit_3_path = self.course_directory_path / 'unit-3-data-types' self.assertDirectoryExists(unit_3_path) unit_4_path = self.course_directory_path / 'unit-4-collections' self.assertDirectoryExists(unit_4_path)
def test_add_lesson_at_the_end_of_unit(self): io.add_lesson_to_unit(self.unit_2_path, 'Python Lists', 'reading') # Postconditions self.assertDirectoryIsNotEmpty(self.unit_2_path) lesson_path = self.unit_2_path / 'lesson-2-python-lists' dot_romtr_path = lesson_path / '.rmotr' readme_path = lesson_path / 'README.md' self.assertDirectoryExists(lesson_path) self.assertDirectoryIsNotEmpty(lesson_path) with dot_romtr_path.open('r') as fp: dot_rmotr_content = toml.loads(fp.read()) self.assertTrue('uuid' in dot_rmotr_content) self.assertEqual(dot_rmotr_content['name'], 'Python Lists') self.assertFileExists(readme_path)
def load(): """ Load application configuration from a file and merge it with the default configuration. If the ``ANITYA_WEB_CONFIG`` environment variable is set to a filesystem path, the configuration will be loaded from that location. Otherwise, the path defaults to ``/etc/anitya/anitya.toml``. """ config = DEFAULTS.copy() if "ANITYA_WEB_CONFIG" in os.environ: config_path = os.environ["ANITYA_WEB_CONFIG"] else: config_path = "/etc/anitya/anitya.toml" if os.path.exists(config_path): _log.info("Loading Anitya configuration from {}".format(config_path)) with open(config_path) as fd: try: file_config = pytoml.loads(fd.read()) for key in file_config: config[key.upper()] = file_config[key] except pytoml.core.TomlError as e: _log.error("Failed to parse {}: {}".format(config_path, str(e))) else: _log.info( "The Anitya configuration file, {}, does not exist.".format(config_path) ) if not isinstance(config["PERMANENT_SESSION_LIFETIME"], timedelta): config["PERMANENT_SESSION_LIFETIME"] = timedelta( seconds=config["PERMANENT_SESSION_LIFETIME"] ) if config["SECRET_KEY"] == DEFAULTS["SECRET_KEY"]: _log.warning( "SECRET_KEY is not configured, falling back to the default. " "This is NOT safe for production deployments!" ) return config
def crate_info_from_toml(target, cdir): if 'url-0.5.7' in cdir: url_057_toml = '''[package] name = "url" version = "0.5.7" authors = [ "Simon Sapin <*****@*****.**>" ] description = "URL library for Rust, based on the WHATWG URL Standard" documentation = "http://servo.github.io/rust-url/url/index.html" repository = "https://github.com/servo/rust-url" readme = "README.md" keywords = ["url", "parser"] license = "MIT/Apache-2.0" [features] query_encoding = ["encoding"] serde_serialization = ["serde"] [dependencies.encoding] version = "0.2" optional = true [dependencies.serde] version = ">=0.6.1, <0.8" optional = true [dependencies] uuid = "0.1.17" rustc-serialize = "0.3" unicode-bidi = "0.2.3" unicode-normalization = "0.1.2" matches = "0.1" ''' cfg = toml.loads(url_057_toml) else: ctoml = open(os.path.join(cdir, 'Cargo.toml'), 'rb') cfg = toml.load(ctoml) return CrateInfo(target, cdir, cfg)
def test_add_assignment_lesson_to_empty_unit(self): io.add_lesson_to_unit(self.unit_3_path, 'Python Lists', 'assignment') # Postconditions self.assertDirectoryIsNotEmpty(self.unit_3_path) lesson_path = self.unit_3_path / 'lesson-1-python-lists' dot_romtr_path = lesson_path / '.rmotr' readme_path = lesson_path / 'README.md' self.assertDirectoryExists(lesson_path) self.assertDirectoryIsNotEmpty(lesson_path) with dot_romtr_path.open('r') as fp: dot_rmotr_content = toml.loads(fp.read()) self.assertTrue('uuid' in dot_rmotr_content) self.assertEqual(dot_rmotr_content['name'], 'Python Lists') self.assertFileExists(readme_path) with readme_path.open('r') as fp: self.assertEqual(fp.read(), "# Python Lists\n") # Dirs to create tests_path = lesson_path / 'tests' solutions_path = lesson_path / 'solutions' # Check dirs are created self.assertDirectoryExists(tests_path) self.assertDirectoryExists(solutions_path) # Files to create main_py_path = lesson_path / 'main.py' empty_test_path = tests_path / 'test_.py' empty_solution_path = solutions_path / 'solution_.py' # Check files self.assertFileExists(main_py_path) self.assertFileExists(empty_test_path) self.assertFileExists(empty_solution_path)
def _main(): succeeded = [] failed = [] for top, dirnames, fnames in os.walk('.'): for fname in fnames: if not fname.endswith('.toml'): continue try: with open(os.path.join(top, fname), 'rb') as fin: parsed = toml.load(fin) except toml.TomlError: parsed = None else: dumped = toml.dumps(parsed) parsed2 = toml.loads(dumped) if parsed != parsed2: failed.append(fname) continue with open(os.path.join(top, fname), 'rb') as fin: parsed = toml.load(fin, _testbench_literal, _testbench_array) try: with io.open(os.path.join(top, fname[:-5] + '.json'), 'rt', encoding='utf-8') as fin: bench = json.load(fin) except IOError: bench = None if parsed != bench: failed.append(fname) else: succeeded.append(fname) for f in failed: print('failed: {}'.format(f)) print('succeeded: {}'.format(len(succeeded))) return 1 if failed else 0
def build(): j_env = jinja2.Environment(loader=jinja2.PackageLoader("main", "template")) j_env.filters["format_time"] = jinja2_format_datetime j_template = j_env.get_template("article.html") articles = [] file_names = os.listdir("article") for file_name in file_names: file = open("article/%s" % file_name, "rt") article_head_text, article_content_markdown = file.read().split("+++++++") file.close() article_head = pytoml.loads(article_head_text) article_title = article_head["title"] article_create_time = datetime.datetime.strptime(article_head["create_time"], '%Y-%m-%dT%H:%M:%S.%f') article_content_html = mistune.markdown(article_content_markdown) article_html = j_template.render( title = article_title, create_time = article_create_time, content = article_content_html ) html_file_name = "%s-%s.html" % (article_create_time.isoformat("T").split(".")[0], article_title) file = open("site/%s" % html_file_name, "wt") file.write(article_html) file.close() articles.append({"title": article_title, "file_name": html_file_name, "create_time": article_create_time}) articles = sorted(articles, key=lambda item: item["create_time"], reverse=True) j_template = j_env.get_template("index.html") index_html = j_template.render(articles=articles) file = open("site/index.html", "wt") file.write(index_html) file.close()
def parse(self, parser): """ Parse a column or row. :param parser: Template parser :type parser: jinja2.parser.Parser :return: A null output node. :rtype: jinja2.nodes.Node """ lineno = next(parser.stream).lineno name = parse_constantlike(self.environment, parser) # Parse the plugin name. body = parser.parse_statements(['name:endplugin'], drop_needle=True) layout = self._get_layout(parser) config = None if body: try: config = flatten_const_node_list(self.environment, body) except Unflattenable as uf: raise NonConstant("A `plugin` block may only contain static layout (found: %r)" % uf.args[0]) config = toml.loads(config, "<%s.%s in %s>" % (layout.placeholder_name, name, parser.name)) layout.add_plugin(name, config) return noop_node(lineno)
def test_add_unit_to_empty_course(self): generated_unit_path = io.add_unit_to_course( course_directory_path=self.course_directory_path, name='Python Intro') unit_path = self.course_directory_path / 'unit-1-python-intro' dot_rmotr_path = unit_path / '.rmotr' readme_path = unit_path / 'README.md' self.assertEqual(generated_unit_path, unit_path) self.assertDirectoryExists(unit_path) self.assertFileExists(dot_rmotr_path) self.assertFileExists(readme_path) with dot_rmotr_path.open() as fp: dot_rmotr_content = toml.loads(fp.read()) self.assertTrue('uuid' in dot_rmotr_content) self.assertEqual(dot_rmotr_content['name'], 'Python Intro') with readme_path.open('r') as fp: content = fp.read() self.assertEqual(content, """# Python Intro """)
def test_prettifying_against_humanly_verified_sample(): toml_source = open('sample.toml').read() expected = open('sample-prettified.toml').read() assert_prettifier_works(toml_source, expected, prettify) assert pytoml.loads(toml_source) == pytoml.loads(expected)
def convert_file(file_path,out_dir,file_temp=None): """ file_path 需要转换的文件 out_dir 文件输出目录 file_temp 输出模板 """ filename = os.path.basename(file_path)#.replace('.','_') # 似乎标题有.会有些问题 temp_content ='' m_head={} if file_temp: with open(file_temp,'r') as f: temp_content = f.read() m_temp = temp_regex.match(temp_content) m_head = toml.loads(m_temp.group(1)) content = '' with open(file_path,'r') as f : content = f.read() m = content_regex.match(content) if not m: print 'Error match content: %s' % file_path return False front_data = yaml.load(m.group(1)) if not front_data: print 'Error load yaml: %s' % file_path return False m_head['title']= front_data.get('title','') if front_data.get("description",None): m_head['description']=front_data.get("description",None) if front_data.get('date',None): #time.strftime("%Y-%m-%dT%H:%M:%S+08:00",b) m_head['date']=front_data.get('date',None).strftime('%Y-%m-%dT%H:%M:%S+08:00') m_tags = front_data.get('tags',None) if m_tags: print type(m_tags),m_tags m_head['tags']=[] if isinstance(m_tags,list): m_head['tags'].extend(m_tags) else: m_head['tags'].append(m_tags) print m_head['tags'] m_categories = front_data.get('categories',None) if m_categories: m_head['categories']=[] if isinstance(m_categories,list): m_head['categories'].extend(m_categories) else: m_head['categories'].append(m_categories) print m_head['categories'] write_out_file(m_head,m.group(2),out_dir+'/'+filename) return True