def cheatsheet_tex():
print(f"Using docs directory: {DOCS_DIR}")
print(f"Using ops docs directory: {OP_DOCS_DIR}")
print()
output = ""
for (section, title, new_page) in OPS_SECTIONS:
toml_file = Path(OP_DOCS_DIR, section + ".toml")
if toml_file.exists() and toml_file.is_file():
output += f"\group{{{ title }}}\n\n"
print(f"Reading {toml_file}")
# n.b. Python 3.6 dicts maintain insertion order
ops = toml.loads(toml_file.read_text())
validate_toml(ops)
ops_array = pool.map(inject_latex, ops.values())
for op in ops_array:
prototype = op["prototype"]
if "prototype_set" in op:
prototype += " / " + op["prototype_set"]
output += "\\begin{op}"
if "aliases" in op and len(op["aliases"]) > 0:
output += "[" + latex_safe(" ".join(op["aliases"])) + "]"
output += "{" + prototype + "}"
output += "\n"
output += op["short_latex"]
output += "\\end{op}"
output += "\n\n"
if new_page:
output += "\pagebreak\n\n"
return output
def uuid_info(cfg, uuid):
"""Return information about the composition
:param cfg: Configuration settings
:type cfg: ComposerConfig
:param uuid: The UUID of the build
:type uuid: str
:returns: dictionary of information about the composition
:rtype: dict
:raises: RuntimeError if there was a problem
This will return a dict with the following fields populated:
* id - The uuid of the comoposition
* config - containing the configuration settings used to run Anaconda
* blueprint - The depsolved blueprint used to generate the kickstart
* commit - The (local) git commit hash for the blueprint used
* deps - The NEVRA of all of the dependencies used in the composition
* compose_type - The type of output generated (tar, iso, etc.)
* queue_status - The final status of the composition (FINISHED or FAILED)
"""
uuid_dir = joinpaths(cfg.get("composer", "lib_dir"), "results", uuid)
if not os.path.exists(uuid_dir):
raise RuntimeError("%s is not a valid build_id" % uuid)
# Load the compose configuration
cfg_path = joinpaths(uuid_dir, "config.toml")
if not os.path.exists(cfg_path):
raise RuntimeError("Missing config.toml for %s" % uuid)
cfg_dict = toml.loads(open(cfg_path, "r").read())
frozen_path = joinpaths(uuid_dir, "frozen.toml")
if not os.path.exists(frozen_path):
raise RuntimeError("Missing frozen.toml for %s" % uuid)
frozen_dict = toml.loads(open(frozen_path, "r").read())
deps_path = joinpaths(uuid_dir, "deps.toml")
if not os.path.exists(deps_path):
raise RuntimeError("Missing deps.toml for %s" % uuid)
deps_dict = toml.loads(open(deps_path, "r").read())
details = compose_detail(uuid_dir)
commit_path = joinpaths(uuid_dir, "COMMIT")
if not os.path.exists(commit_path):
raise RuntimeError("Missing commit hash for %s" % uuid)
commit_id = open(commit_path, "r").read().strip()
return {
"id": uuid,
"config": cfg_dict,
"blueprint": frozen_dict,
"commit": commit_id,
"deps": deps_dict,
"compose_type": details["compose_type"],
"queue_status": details["queue_status"],
"image_size": details["image_size"]
}
def test_default_config(tmpdir, tljh_dir):
state_dir = tmpdir.mkdir("state")
traefik.ensure_traefik_config(str(state_dir))
assert state_dir.join("traefik.toml").exists()
traefik_toml = os.path.join(state_dir, "traefik.toml")
with open(traefik_toml) as f:
toml_cfg = f.read()
# print config for debugging on failure
print(config.CONFIG_FILE)
print(toml_cfg)
cfg = toml.loads(toml_cfg)
assert cfg["defaultEntryPoints"] == ["http"]
assert len(cfg["entryPoints"]["auth_api"]["auth"]["basic"]["users"]) == 1
# runtime generated entry, value not testable
cfg["entryPoints"]["auth_api"]["auth"]["basic"]["users"] = [""]
assert cfg["entryPoints"] == {
"http": {
"address": ":80"
},
"auth_api": {
"address": "127.0.0.1:8099",
"auth": {
"basic": {
"users": [""]
}
},
"whiteList": {
"sourceRange": ["127.0.0.1"]
}
},
}
def get_sphinx_configuration(project_dir):
"""Read the Sphinx configuration from ``pyproject.toml```."""
try:
project = toml.loads((Path(project_dir) / 'pyproject.toml').read_text())
return reduce(lambda a, b: a[b], 'tool.flinx.configuration'.split('.'), project)
except (FileNotFoundError, KeyError):
return {}
def main(filesystem, locator, link_dir, handle_error):
with filesystem.open(locator.root.descendant("virtualenvs.toml")) as venvs:
contents = pytoml.load(venvs)
for name, config in contents["virtualenv"].items():
config.setdefault("sys.version", sys.version)
virtualenv = locator.for_name(name=name)
existing_config_path = virtualenv.path.descendant("installed.toml")
try:
with filesystem.open(existing_config_path) as existing_config:
if pytoml.loads(existing_config.read()) == config:
continue
except FileNotFound:
virtualenv.create()
else:
virtualenv.recreate_on(filesystem=filesystem)
packages, requirements = _to_install(config=config)
try:
virtualenv.install(packages=packages, requirements=requirements)
except Exception:
handle_error(virtualenv)
continue
for link in config.get("link", []):
_link(
source=virtualenv.binary(name=link),
to=link_dir.descendant(link),
filesystem=filesystem,
)
with filesystem.open(existing_config_path, "w") as existing_config:
existing_config.write(pytoml.dumps(config).encode("utf-8"))
def __init__(self, project_path='pyproject.toml'):
project = toml.loads(Path(project_path).read_text())
try:
dotpath = self._toml_path.split('.')
self._metadata = reduce(lambda a, b: a[b], dotpath, project)
except KeyError:
pass
def insert_wp_paths():
"""A one-time operation: insert \"path =\" definitions in the front matter for
the blog posts, to preserve the old Wordpress URLs.
"""
import pytoml
for path in all_blog_posts():
fm_text, content = read_markdown(path)
fm = pytoml.loads(fm_text)
web_path = fm['extra']['wp_rel_permalink']
if web_path[0] == '/':
web_path = web_path[1:]
if web_path[-1] == '/':
web_path = web_path[:-1]
fm_lines = fm_text.splitlines()
for i in range(len(fm_lines)):
if fm_lines[i].startswith('title ='):
break
else:
raise Exception(f'no title in {path}?')
fm_lines = fm_lines[:i + 1] + [f'path = "{web_path}"'
] + fm_lines[i + 1:]
fm_text = '\n'.join(fm_lines)
rewrite_markdown(path, fm_text, content)
def load_rule_contents(rule_file: Path, single_only=False) -> list:
"""Load a rule file from multiple formats."""
_, extension = os.path.splitext(rule_file)
raw_text = rule_file.read_text()
if extension in ('.ndjson', '.jsonl'):
# kibana exported rule object is ndjson with the export metadata on the last line
contents = [json.loads(line) for line in raw_text.splitlines()]
if len(contents) > 1 and 'exported_count' in contents[-1]:
contents.pop(-1)
if single_only and len(contents) > 1:
raise ValueError('Multiple rules not allowed')
return contents or [{}]
elif extension == '.toml':
rule = pytoml.loads(raw_text)
else:
rule = load_dump(rule_file)
if isinstance(rule, dict):
return [rule]
elif isinstance(rule, list):
return rule
else:
raise ValueError(f"Expected a list or dictionary in {rule_file}")
def test_manual_ssl_config(tljh_dir):
state_dir = config.STATE_DIR
config.set_config_value(config.CONFIG_FILE, "https.enabled", True)
config.set_config_value(config.CONFIG_FILE, "https.tls.key",
"/path/to/ssl.key")
config.set_config_value(config.CONFIG_FILE, "https.tls.cert",
"/path/to/ssl.cert")
traefik.ensure_traefik_config(str(state_dir))
traefik_toml = os.path.join(state_dir, "traefik.toml")
with open(traefik_toml) as f:
toml_cfg = f.read()
# print config for debugging on failure
print(config.CONFIG_FILE)
print(toml_cfg)
cfg = toml.loads(toml_cfg)
assert cfg["defaultEntryPoints"] == ["http", "https"]
assert "acme" not in cfg
assert cfg["entryPoints"] == {
"http": {
"address": ":80",
"redirect": {
"entryPoint": "https"
}
},
"https": {
"address": ":443",
"backend": "jupyterhub",
"tls": {
"certificates": [{
"certFile": "/path/to/ssl.cert",
"keyFile": "/path/to/ssl.key"
}]
},
},
}
def load_config():
filename = sys.argv[1] if len(sys.argv) > 1 else 'benchmark.toml'
with open(filename) as configfile:
config = pytoml.loads(configfile.read())
items = config['item']
default = items['default']
if 'options' in config:
options = config['options']
else:
options = dict(ref_result="C",
time_field="elapsed",
show_diff_below=0.9,
verbose=True)
ret_items = {}
for name, item in zip(items, items.values()):
if name == 'default':
continue
if name.startswith('"') and name.endswith('"'):
import ast
name = ast.literal_eval(name)
profile = dict(default)
profile.update(item)
profile['name'] = name
for k, v in zip(profile, profile.values()):
if type(v) is str:
profile[k] = Template(v).safe_substitute(**profile)
ret_items[name] = profile
return ret_items, options
def main(filename, timestamp=None, with_group_by=with_group_by, execute=False):
if filename == 'batch':
batch_execute('examples', True)
return
source = open(filename).read()
if not timestamp:
timestamp = datetime.now()
else:
timestamp = datetime.strptime(timestamp, timestamp_format)
sql_query = compile_source(source, timestamp, int(with_group_by))
if not int(execute):
print(sql_query)
else:
import pytoml
import pyodbc
config = pytoml.loads(open('config.toml').read())
odbc_connection_template = Template(config['odbc']['connect_template'])
odbc_connection_string = odbc_connection_template.substitute(**config['odbc'])
connection = pyodbc.connect(odbc_connection_string)
cur = connection.cursor()
cur.execute(sql_query)
print('-'*79)
for row in cur:
print(*row, sep='\t| ')
def parse_metadata (self):
'''
Open and parse the included metadata file in the TAB.
'''
metadata_tarinfo = self.tab.getmember('metadata.toml')
metadata_str = self.tab.extractfile(metadata_tarinfo).read().decode('utf-8')
return pytoml.loads(metadata_str)
def parse(self, parser):
"""
Parse a column or row.
:param parser: Template parser
:type parser: jinja2.parser.Parser
:return: A null output node.
:rtype: jinja2.nodes.Node
"""
lineno = next(parser.stream).lineno
name = parse_constantlike(self.environment,
parser) # Parse the plugin name.
body = parser.parse_statements(['name:endplugin'], drop_needle=True)
layout = self._get_layout(parser)
config = None
if body:
try:
config = flatten_const_node_list(self.environment, body)
except Unflattenable as uf:
raise NonConstant(
"A `plugin` block may only contain static layout (found: %r)"
% uf.args[0])
config = toml.loads(
config,
"<%s.%s in %s>" % (layout.placeholder_name, name, parser.name))
layout.add_plugin(name, config)
return noop_node(lineno)
def unmarshall_config_reader(file_, d, config_type):
config_type = config_type.lower()
if config_type in ["yaml", "yml"]:
try:
f = yaml.load(file_)
try:
d.update(yaml.load(f))
except AttributeError: # to read files
d.update(f)
except Exception as e:
raise ConfigParserError(e)
elif config_type == "json":
try:
f = json.loads(file_)
d.update(f)
except Exception as e:
raise ConfigParserError(e)
elif config_type == "toml":
try:
try:
f = toml.loads(file_)
d.update(f)
except AttributeError: # to read streams
d.update(file_)
except Exception as e:
raise ConfigParserError(e)
return d
def cheatsheet_tex():
print(f"Using docs directory: {DOCS_DIR}")
print(f"Using ops docs directory: {OP_DOCS_DIR}")
print()
output = ""
for (section, title, new_page) in OPS_SECTIONS:
toml_file = Path(OP_DOCS_DIR, section + ".toml")
if toml_file.exists() and toml_file.is_file():
output += f"\group{{{ title }}}\n\n"
print(f"Reading {toml_file}")
# n.b. Python 3.6 dicts maintain insertion order
ops = toml.loads(toml_file.read_text())
validate_toml(ops)
ops_array = pool.map(inject_latex, ops.values())
for op in ops_array:
prototype = op["prototype"]
if "prototype_set" in op:
prototype += " / " + op["prototype_set"]
output += "\\begin{op}"
if "aliases" in op and len(op["aliases"]) > 0:
output += "[" + latex_safe(" ".join(op["aliases"])) + "]"
output += "{" + prototype + "}"
output += "\n"
output += op["short_latex"]
output += "\\end{op}"
output += "\n\n"
if new_page:
output += "\pagebreak\n\n"
return output
def test_add_unit_to_not_empty_course_at_end(self):
self._create_testing_unit(
"Python Intro", 'unit-1-python-intro',
'f4ed574a-a11b-4119-bb64-c1feaa05ea55')
unit_1_path = self.course_directory_path / 'unit-1-python-intro'
self.assertDirectoryExists(unit_1_path)
io.add_unit_to_course(
course_directory_path=self.course_directory_path,
name='Data Types')
unit_path = self.course_directory_path / 'unit-2-data-types'
dot_rmotr_path = unit_path / '.rmotr'
readme_path = unit_path / 'README.md'
self.assertDirectoryExists(unit_path)
self.assertFileExists(dot_rmotr_path)
self.assertFileExists(readme_path)
with dot_rmotr_path.open() as fp:
dot_rmotr_content = toml.loads(fp.read())
self.assertTrue('uuid' in dot_rmotr_content)
self.assertEqual(dot_rmotr_content['name'], 'Data Types')
with readme_path.open('r') as fp:
content = fp.read()
self.assertEqual(content, """# Data Types
""")
def prepare():
toml = s.get(config_url).text
toml = pytoml.loads(toml)
version_date = toml['date']
version = toml['pkg']['rust']['version'].split('-', 1)[0]
cargo_version = toml['pkg']['cargo']['version'].split('-', 1)[0]
rustfmt_version = toml['pkg']['rustfmt-preview']['version'].split('-',
1)[0]
clippy_version = toml['pkg']['clippy-preview']['version'].split('-', 1)[0]
try:
clippy_url = toml['pkg']['clippy-preview']['target'] \
['x86_64-unknown-linux-gnu']['xz_url']
except KeyError:
return 'no clippy available?'
stds = [
Std(target, toml['pkg']['rust-std']['target'][target])
for target in STDS
]
loader = tornado.template.Loader('.')
global PKGBUILD
PKGBUILD = loader.load('PKGBUILD.tmpl').generate(
stds=stds,
version=version,
version_date=version_date.replace('-', ''),
version_date_raw=version_date,
cargo_version=cargo_version,
rustfmt_version=rustfmt_version,
clippy_version=clippy_version,
clippy_url=clippy_url,
)
def pre_build():
toml = s.get(config_url).text
toml = pytoml.loads(toml)
version_date = toml['date']
version = toml['pkg']['rust']['version'].split('-', 1)[0]
cargo_version = toml['pkg']['cargo']['version'].split('-', 1)[0]
rustfmt_version = toml['pkg']['rustfmt-preview']['version'].split('-', 1)[0]
clippy_version = toml['pkg']['clippy-preview']['version'].split('-', 1)[0]
clippy_url = toml['pkg']['clippy-preview']['target'] \
['x86_64-unknown-linux-gnu']['xz_url']
if not debug:
oldfiles = (glob.glob('*.xz') + glob.glob('*.xz.asc') +
glob.glob('*.part'))
for f in oldfiles:
os.unlink(f)
stds = [Std(target, toml['pkg']['rust-std']['target'][target])
for target in STDS]
loader = tornado.template.Loader('.')
content = loader.load('PKGBUILD.tmpl').generate(
stds = stds,
version = version,
version_date = version_date.replace('-', ''),
version_date_raw = version_date,
cargo_version = cargo_version,
rustfmt_version = rustfmt_version,
clippy_version = clippy_version,
clippy_url = clippy_url,
)
with open('PKGBUILD', 'wb') as output:
output.write(content)
def main(filename, timestamp=None, with_group_by=with_group_by, execute=False):
if filename == 'batch':
batch_execute('examples', True)
return
source = open(filename).read()
if not timestamp:
timestamp = datetime.now()
else:
timestamp = datetime.strptime(timestamp, timestamp_format)
sql_query = compile_source(source, timestamp, int(with_group_by))
if not int(execute):
print(sql_query)
else:
import pytoml
import pyodbc
config = pytoml.loads(open('config.toml').read())
odbc_connection_template = Template(config['odbc']['connect_template'])
odbc_connection_string = odbc_connection_template.substitute(
**config['odbc'])
connection = pyodbc.connect(odbc_connection_string)
cur = connection.cursor()
cur.execute(sql_query)
print('-' * 79)
for row in cur:
print(*row, sep='\t| ')
def load_config():
filename = sys.argv[1] if len(sys.argv) > 1 else 'benchmark.toml'
with open(filename) as configfile:
config = pytoml.loads(configfile.read())
items = config['item']
default = items['default']
if 'options' in config:
options = config['options']
else:
options = dict(ref_result="C",
time_field="elapsed",
show_diff_below=0.9,
verbose=True)
ret_items = {}
for name, item in zip(items, items.values()):
if name == 'default':
continue
if name.startswith('"') and name.endswith('"'):
import ast
name = ast.literal_eval(name)
profile = dict(default)
profile.update(item)
profile['name'] = name
for k, v in zip(profile, profile.values()):
if type(v) is str:
profile[k] = Template(v).safe_substitute(**profile)
ret_items[name] = profile
return ret_items, options
def test_default_config(tmpdir, tljh_dir):
state_dir = tmpdir.mkdir("state")
traefik.ensure_traefik_config(str(state_dir))
assert state_dir.join("traefik.toml").exists()
traefik_toml = os.path.join(state_dir, "traefik.toml")
with open(traefik_toml) as f:
toml_cfg = f.read()
# print config for debugging on failure
print(config.CONFIG_FILE)
print(toml_cfg)
cfg = toml.loads(toml_cfg)
assert cfg["defaultEntryPoints"] == ["http"]
assert cfg["entryPoints"] == {"http": {"address": ":80"}}
assert cfg["frontends"] == {
"jupyterhub": {
"backend": "jupyterhub",
"passHostHeader": True
}
}
assert cfg["backends"] == {
"jupyterhub": {
"servers": {
"chp": {
"url": "http://127.0.0.1:15003"
}
}
}
}
def load(filename=None):
"""
Load application configuration from a file and merge it with the default
configuration.
If the ``FEDORA_MESSAGING_CONF`` environment variable is set to a
filesystem path, the configuration will be loaded from that location.
Otherwise, the path defaults to ``/etc/fedora-messaging/config.toml``.
"""
config = DEFAULTS.copy()
if filename:
config_path = filename
elif 'FEDORA_MESSAGING_CONF' in os.environ:
config_path = os.environ['FEDORA_MESSAGING_CONF']
else:
config_path = '/etc/fedora-messaging/config.toml'
if os.path.exists(config_path):
_log.info('Loading configuration from {}'.format(config_path))
with open(config_path) as fd:
try:
file_config = pytoml.loads(fd.read())
for key in file_config:
config[key.lower()] = file_config[key]
except pytoml.core.TomlError as e:
_log.error('Failed to parse {}: {}'.format(config_path, str(e)))
sys.exit(1)
else:
_log.info('The configuration file, {}, does not exist.'.format(config_path))
return config
def prepare():
toml = s.get(config_url).text
toml = pytoml.loads(toml)
version_date = toml['date']
version = toml['pkg']['rust']['version'].split('-', 1)[0]
cargo_version = toml['pkg']['cargo']['version'].split('-', 1)[0]
rustfmt_version = toml['pkg']['rustfmt-preview']['version'].split('-', 1)[0]
clippy_version = toml['pkg']['clippy-preview']['version'].split('-', 1)[0]
try:
clippy_url = toml['pkg']['clippy-preview']['target'] \
['x86_64-unknown-linux-gnu']['xz_url']
except KeyError:
return 'no clippy available?'
stds = [Std(target, toml['pkg']['rust-std']['target'][target])
for target in STDS]
loader = tornado.template.Loader('.')
global PKGBUILD
PKGBUILD = loader.load('PKGBUILD.tmpl').generate(
stds = stds,
version = version,
version_date = version_date.replace('-', ''),
version_date_raw = version_date,
cargo_version = cargo_version,
rustfmt_version = rustfmt_version,
clippy_version = clippy_version,
clippy_url = clippy_url,
)
def get_config(self):
try:
with open(self.config_path, 'r') as f:
content = pytoml.loads(f.read().strip())
except FileNotFoundError:
content = {}
return content
def parse(cls, contents):
"""
Return parsed python data-structure from the input content.
"""
return pytoml.loads(
contents
)
def createWorkdir(self, contents):
workdir = self.graph.target.git.workdir
locales = set()
includes = []
for tpath, content_list in contents.items():
try:
b_content = merge_channels(tpath, content_list)
except MergeNotSupportedError:
b_content = content_list[0]
if tpath.endswith("l10n.toml"):
try:
data = toml.loads(b_content)
if 'locales' in data:
locales.update(data['locales'])
includes.append(tpath)
except Exception as e:
print(e)
pass
tpath = mozpath.join(workdir, tpath)
tdir = mozpath.dirname(tpath)
if not os.path.isdir(tdir):
os.makedirs(tdir)
with open(tpath, "wb") as fh:
fh.write(b_content)
self.ensureL10nToml(workdir, locales, includes)
def gitrpm_repo_test(self):
"""Test creating a dnf repo of the git rpms"""
recipe = toml.loads("""
[[repos.git]]
rpmname="repo-test-alpha"
rpmversion="1.1.0"
rpmrelease="1"
summary="Testing the git rpm code"
repo="file://%s"
ref="v1.1.0"
destination="/srv/testing-alpha/"
[[repos.git]]
rpmname="repo-test-beta"
rpmversion="1.0.0"
rpmrelease="1"
summary="Testing the git rpm code"
repo="file://%s"
ref="v1.0.0"
destination="/srv/testing-beta/"
""" % (self.repodir, self.repodir))
try:
temp_dir = tempfile.mkdtemp(prefix="git-rpm-test.")
repo_dir = create_gitrpm_repo(temp_dir, recipe)
self.assertTrue(len(repo_dir) > 0)
self.assertTrue(os.path.exists(joinpaths(repo_dir, "repo-test-alpha-1.1.0-1.noarch.rpm")))
self.assertTrue(os.path.exists(joinpaths(repo_dir, "repo-test-beta-1.0.0-1.noarch.rpm")))
finally:
shutil.rmtree(temp_dir)
def __init__(self, path_data, env=None, ignore_missing_includes=False):
# mock, use the path as data. Yeah, not nice
super(MockTOMLParser, self).__init__(
'/tmp/base.toml',
env=env, ignore_missing_includes=ignore_missing_includes
)
self.data = toml.loads(path_data)
def deprecate_rule(ctx: click.Context, rule_file: str):
"""Deprecate a rule."""
import pytoml
from .packaging import load_versions
version_info = load_versions()
rule_file = Path(rule_file)
contents = pytoml.loads(rule_file.read_text())
rule = TOMLRule(path=rule_file, contents=contents)
if rule.id not in version_info:
click.echo(
'Rule has not been version locked and so does not need to be deprecated. '
'Delete the file or update the maturity to `development` instead')
ctx.exit()
today = time.strftime('%Y/%m/%d')
new_meta = dataclasses.replace(rule.contents.metadata,
updated_date=today,
deprecation_date=today,
maturity='deprecated')
contents = dataclasses.replace(rule.contents, metadata=new_meta)
deprecated_path = get_path('rules', '_deprecated', rule_file.name)
# create the new rule and save it
new_rule = TOMLRule(contents=contents, path=Path(deprecated_path))
new_rule.save_toml()
# remove the old rule
rule_file.unlink()
click.echo(
f'Rule moved to {deprecated_path} - remember to git add this file')
def load(s, config_file):
log.info("loading configuration file %s", config_file)
try:
with open(config_file, "r") as f:
s.config = toml.loads(f.read())
except FileNotFoundError as e:
log.error("configuration file %s not found!" % config_file)
raise e
def branches(self):
rev = self.lookup_branch(self.branch).target
tree = self[rev].tree
if 'l10n.toml' not in tree:
return [self.branch]
toml_data = self[tree['l10n.toml'].id].data
config = toml.loads(toml_data)
return config.get("branches", [self.branch])
def load_toml(path, keys_to_convert_to_set=None):
_check_file_exists(path)
with open(path, 'r') as the_file:
contents = the_file.read()
loaded = pytoml.loads(contents)
if keys_to_convert_to_set:
convert_dict_keys(loaded, keys=keys_to_convert_to_set, func=set)
return loaded
def load(s, config_file):
log.info("loading configuration file %s", config_file)
try:
with open(config_file, "r") as f:
s.config = toml.loads(f.read())
except FileNotFoundError as e:
log.error("configuration file %s not found!" % config_file)
raise e
def _main():
ap = argparse.ArgumentParser()
ap.add_argument('-d', '--dir', action='append')
ap.add_argument('testcase', nargs='*')
args = ap.parse_args()
if not args.dir:
args.dir = [os.path.join(os.path.split(__file__)[0], 'toml-test/tests')]
succeeded = []
failed = []
for path in args.dir:
if not os.path.isdir(path):
print('error: not a dir: {}'.format(path))
return 2
for top, dirnames, fnames in os.walk(path):
for fname in fnames:
if not fname.endswith('.toml'):
continue
if args.testcase and not any(arg in fname for arg in args.testcase):
continue
parse_error = None
try:
with open(os.path.join(top, fname), 'rb') as fin:
parsed = toml.load(fin)
except toml.TomlError:
parsed = None
parse_error = sys.exc_info()
else:
dumped = toml.dumps(parsed)
parsed2 = toml.loads(dumped)
if parsed != parsed2:
failed.append((fname, None))
continue
with open(os.path.join(top, fname), 'rb') as fin:
parsed = toml.load(fin, translate=_testbench_literal)
try:
with io.open(os.path.join(top, fname[:-5] + '.json'), 'rt', encoding='utf-8') as fin:
bench = json.load(fin)
except IOError:
bench = None
if parsed != adjust_bench(bench):
failed.append((fname, parsed, bench, parse_error))
else:
succeeded.append(fname)
for f, parsed, bench, e in failed:
print('failed: {}\n{}\n{}'.format(f, json.dumps(parsed, indent=4), json.dumps(bench, indent=4)))
if e:
traceback.print_exception(*e)
print('succeeded: {}'.format(len(succeeded)))
return 1 if failed or not succeeded else 0
def _main():
ap = argparse.ArgumentParser()
ap.add_argument('-d', '--dir', action='append')
ap.add_argument('testcase', nargs='*')
args = ap.parse_args()
if not args.dir:
args.dir = [os.path.join(os.path.split(__file__)[0], 'toml-test/tests')]
succeeded = []
failed = []
for path in args.dir:
if not os.path.isdir(path):
print('error: not a dir: {0}'.format(path))
return 2
for top, dirnames, fnames in os.walk(path):
for fname in fnames:
if not fname.endswith('.toml'):
continue
if args.testcase and not any(arg in fname for arg in args.testcase):
continue
parse_error = None
try:
with open(os.path.join(top, fname), 'rb') as fin:
parsed = toml.load(fin)
except toml.TomlError:
parsed = None
parse_error = sys.exc_info()
else:
dumped = toml.dumps(parsed)
parsed2 = toml.loads(dumped)
if parsed != parsed2:
failed.append((fname, None))
continue
with open(os.path.join(top, fname), 'rb') as fin:
parsed = toml.load(fin, translate=_testbench_literal)
try:
with io.open(os.path.join(top, fname[:-5] + '.json'), 'rt', encoding='utf-8') as fin:
bench = json.load(fin)
except IOError:
bench = None
if parsed != adjust_bench(bench):
failed.append((fname, parsed, bench, parse_error))
else:
succeeded.append(fname)
for f, parsed, bench, e in failed:
print('failed: {}\n{}\n{}'.format(f, json.dumps(parsed, indent=4), json.dumps(bench, indent=4)))
if e:
traceback.print_exception(*e)
print('succeeded: {0}'.format(len(succeeded)))
return 1 if failed or not succeeded else 0
def test_splitting_array():
toml_text = """
somethingweird = false
[section]
k = [4, 8, 15, 16, 23, 42, 4, 8, 15, 16, 23, 42, 4, 8, 15, 16, 23, 42, 4, 8, 15, 16, 23, 42, 4, 8, 15, 16, 23, 42, 4, 8, 15, 16, 23, 42, 4, 8, 15, 16, 23, 42]
[data]
id = 12
"""
prettified = elements_to_text(line_length_limiter(text_to_elements(toml_text)))
assert pytoml.loads(prettified) == pytoml.loads(toml_text)
assert all(len(line) < 120 for line in prettified.split('\n'))
def fetch_manifest(channel='stable'):
url = 'https://static.rust-lang.org/dist/channel-rust-' + channel + '.toml'
req = requests.get(url)
req.raise_for_status()
manifest = toml.loads(req.content)
if manifest['manifest-version'] != '2':
raise NotImplementedError('Unrecognized manifest version %s.' %
manifest['manifest-version'])
return manifest
def __init__(self, cfg):
"""
Load MeTA's filter chain, tagger, and parser.
"""
toml = pytoml.loads(open(cfg).read())
self.tgr = metapy.sequence.PerceptronTagger(toml['sequence']['prefix'])
self.psr = metapy.parser.Parser(toml['parser']['prefix'])
self.stream = metapy.analyzers.ICUTokenizer()
self.stream = metapy.analyzers.PennTreebankNormalizer(self.stream)
def add(index_path, key, value):
index = pytoml.loads(subprocess.check_output(decrypt + [index_path]))
if key not in index:
index[key] = value
cmd = encrypt + ['--yes', '--output', index_path]
with subprocess.Popen(cmd,
stdin=subprocess.PIPE, stderr=subprocess.PIPE,
universal_newlines=True) as gpg:
err = gpg.communicate(pytoml.dumps(index))[1].encode()
if gpg.wait() != 0:
raise subprocess.CalledProcessError(gpg.poll(), cmd, err)
def _init_configs(self):
self.v.set_config_type("yaml")
r = yaml.dump(text(yaml_example))
self.v._unmarshall_reader(r, self.v._config)
self.v.set_config_type("json")
r = json.dumps(json_example)
self.v._unmarshall_reader(r, self.v._config)
self.v.set_config_type("toml")
r = toml.loads(toml_example)
self.v._unmarshall_reader(r, self.v._config)
def _load_remote_files_data(get_new_remote_data = False) -> dict:
if get_new_remote_data:
response = urlopen(config.remote_data_file)
data = response.read()
data_as_dict = toml.loads(data)
global cached_remote_data
cached_remote_data = data_as_dict
return data_as_dict
else:
return cached_remote_data
def _setup_logging():
try:
with open('logging.toml') as f:
raw = f.read()
except FileNotFoundError:
try:
with open(os.path.expanduser('~/.config/ion/logging.toml')) as f:
raw = f.read()
except FileNotFoundError:
raw = pkg_resources.resource_string(__name__, 'data/logging.toml')
conf = toml.loads(raw)
logging.config.dictConfig(conf)
def fetch_manifest(channel='stable'):
if '-' in channel:
channel, date = channel.split('-', 1)
prefix = '/' + date
else:
prefix = ''
url = 'https://static.rust-lang.org/dist%s/channel-rust-%s.toml' % (
prefix, channel)
req = requests.get(url)
req.raise_for_status()
manifest = toml.loads(req.content)
if manifest['manifest-version'] != '2':
raise NotImplementedError('Unrecognized manifest version %s.' %
manifest['manifest-version'])
return manifest
def config(args):
config = default_config()
if args.config:
try:
with open(args.config) as config_file:
data = toml.loads(config_file.read())
if "os" in data:
config["os"].update(data["os"])
if "core" in data:
config["core"].update(data["core"])
except toml.core.TomlError as err:
parser.error("Errors found in the config file:\n\n", err)
return merge(config, args)
def config(args):
config = default_config()
if args.config:
with open(args.config) as config_file:
data = toml.loads(config_file.read())
if "os" in data:
config["os"].update(data["os"])
if "core" in data:
config["core"].update(data["core"])
if config["os"]["proxy-settings"] is True and (sys.platform != "darwin"):
raise StandardError("Proxy settings is only available on Mac OX")
return merge(config, args)
def _main():
succeeded = []
failed = []
for top, dirnames, fnames in os.walk('.'):
for fname in fnames:
if not fname.endswith('.toml'):
continue
if sys.argv[1:] and not any(arg in fname for arg in sys.argv[1:]):
continue
parse_error = None
try:
with open(os.path.join(top, fname), 'rb') as fin:
parsed = toml.load(fin)
except toml.TomlError:
parsed = None
parse_error = sys.exc_info()
else:
dumped = toml.dumps(parsed)
parsed2 = toml.loads(dumped)
if parsed != parsed2:
failed.append((fname, None))
continue
with open(os.path.join(top, fname), 'rb') as fin:
parsed = toml.load(fin, translate=_testbench_literal)
try:
with io.open(os.path.join(top, fname[:-5] + '.json'), 'rt', encoding='utf-8') as fin:
bench = json.load(fin)
except IOError:
bench = None
if parsed != bench:
failed.append((fname, parsed, bench, parse_error))
else:
succeeded.append(fname)
for f, parsed, bench, e in failed:
print('failed: {}\n{}\n{}'.format(f, json.dumps(parsed, indent=4), json.dumps(bench, indent=4)))
if e:
traceback.print_exception(*e)
print('succeeded: {}'.format(len(succeeded)))
return 1 if failed else 0
def loads(self, raw):
conf = toml.loads(raw)
self.USERNAME = _conf_get(conf, 'auth', 'username', default=self.USERNAME)
self.PASSWORD = _conf_get(conf, 'auth', 'password', default=self.PASSWORD)
self.CLIENT_ID = _conf_get(conf, 'auth', 'client-id', default=self.CLIENT_ID)
self.CLIENT_SECRET = _conf_get(conf, 'auth', 'client-secret', default=self.CLIENT_SECRET)
self.REDIRECT_URI = _conf_get(conf, 'auth', 'redirect-uri', default=self.REDIRECT_URI)
self.REFRESH_TOKEN = _conf_get(conf, 'auth', 'refresh-token', default=self.REFRESH_TOKEN)
self.NETWORK_NAME = _conf_get(conf, 'network', 'name', default=self.NETWORK_NAME)
self.MULTIREDDIT = _conf_get(conf, 'network', 'multireddit', default=self.MULTIREDDIT)
self.MULTI_OWNER = _conf_get(conf, 'network', 'multi-owner', default=self.USERNAME)
self.DEFAULT_MODS = _conf_get(conf, 'network', 'mods', default=self.DEFAULT_MODS)
self.WIKI_PAGES = _conf_get(conf, 'network', 'wiki-pages', default=self.WIKI_PAGES)
self.NSFW_OK = _conf_get(conf, 'network', 'nsfw', default=self.NSFW_OK)
self.NSFW_WHITELIST_OK = _conf_get(conf, 'network', 'nsfw-whitelist',
default=self.NSFW_WHITELIST_OK)
self.COMMENT_FOOTER = _conf_get(conf, 'network', 'comment-footer', default=self.COMMENT_FOOTER)
self.DOMAINS = _conf_get(conf, 'posts', 'domains', default=self.DOMAINS)
self.EXTENSIONS = _conf_get(conf, 'posts', 'extensions', default=self.EXTENSIONS)
self.PARENT_SUB = _conf_get(conf, 'parent', 'name', default=self.PARENT_SUB)
update_children = _conf_get(conf, 'update_children', default=False)
self.CHILD_SUBS = self._load_group(conf, 'child', self.CHILD_SUBS, update_children)
update_cousins = _conf_get(conf, 'update_cousins', default=False)
self.COUSIN_SUBS = self._load_group(conf, 'cousin', self.COUSIN_SUBS, update_cousins)
self.DISCORD_CLIENTID = _conf_get(conf, 'discord', 'client_id', default=self.DISCORD_CLIENTID)
self.DISCORD_TOKEN = _conf_get(conf, 'discord', 'token', default=self.DISCORD_TOKEN)
self.DISCORD_INBOX_CHAN_ID = _conf_get(conf, 'discord', 'inbox_channel', default=None)
self.DISCORD_FALSEPOS_CHAN_ID = _conf_get(conf, 'discord', 'falsepos_channel', default=None)
self.DISCORD_OC_CHAN_ID = _conf_get(conf, 'discord', 'oc_channel', default=None)
self.DISCORD_GITHUB_CHAN_ID = _conf_get(conf, 'discord', 'github_channel', default=None)
self.DISCORD_MOD_CHAN_ID = _conf_get(conf, 'discord', 'mod_channel', default=None)
self.DISCORD_KEEPALIVE_CHAN_ID = _conf_get(conf, 'discord', 'keepalive_channel', default=None)
self.GITHUB_OAUTH_TOKEN = _conf_get(conf, 'github', 'token', default=None)
self.GITHUB_REPO_USER = _conf_get(conf, 'github', 'user', default=None)
self.GITHUB_REPO_NAME = _conf_get(conf, 'github', 'repo_name', default=None)
def test_add_unit_to_course_in_between_other_units(self):
self._create_testing_unit(
"Python Intro", 'unit-1-python-intro',
'f4ed574a-a11b-4119-bb64-c1feaa05ea55')
self._create_testing_unit(
"Data Types", 'unit-2-data-types',
'8a22574a-a11b-4119-a964-c1feaa05c833')
self._create_testing_unit(
"Collections", 'unit-3-collections',
'c822574a-a81b-4aa9-a964-c1feaa05a7b2')
unit_1_path = self.course_directory_path / 'unit-1-python-intro'
self.assertDirectoryExists(unit_1_path)
unit_2_path = self.course_directory_path / 'unit-2-data-types'
self.assertDirectoryExists(unit_2_path)
unit_3_path = self.course_directory_path / 'unit-3-collections'
self.assertDirectoryExists(unit_3_path)
io.add_unit_to_course(
course_directory_path=self.course_directory_path,
name='Interpreters', order=2)
unit_path = self.course_directory_path / 'unit-2-interpreters'
dot_rmotr_path = unit_path / '.rmotr'
self.assertDirectoryExists(unit_path)
self.assertFileExists(dot_rmotr_path)
self.assertFileExists(unit_path / 'README.md')
with dot_rmotr_path.open() as fp:
dot_rmotr_content = toml.loads(fp.read())
self.assertTrue('uuid' in dot_rmotr_content)
self.assertEqual(dot_rmotr_content['name'], 'Interpreters')
# Postconditions
unit_1_path = self.course_directory_path / 'unit-1-python-intro'
self.assertDirectoryExists(unit_1_path)
unit_3_path = self.course_directory_path / 'unit-3-data-types'
self.assertDirectoryExists(unit_3_path)
unit_4_path = self.course_directory_path / 'unit-4-collections'
self.assertDirectoryExists(unit_4_path)
def test_add_lesson_at_the_end_of_unit(self):
io.add_lesson_to_unit(self.unit_2_path, 'Python Lists', 'reading')
# Postconditions
self.assertDirectoryIsNotEmpty(self.unit_2_path)
lesson_path = self.unit_2_path / 'lesson-2-python-lists'
dot_romtr_path = lesson_path / '.rmotr'
readme_path = lesson_path / 'README.md'
self.assertDirectoryExists(lesson_path)
self.assertDirectoryIsNotEmpty(lesson_path)
with dot_romtr_path.open('r') as fp:
dot_rmotr_content = toml.loads(fp.read())
self.assertTrue('uuid' in dot_rmotr_content)
self.assertEqual(dot_rmotr_content['name'], 'Python Lists')
self.assertFileExists(readme_path)
def load():
"""
Load application configuration from a file and merge it with the default
configuration.
If the ``ANITYA_WEB_CONFIG`` environment variable is set to a filesystem
path, the configuration will be loaded from that location. Otherwise, the
path defaults to ``/etc/anitya/anitya.toml``.
"""
config = DEFAULTS.copy()
if "ANITYA_WEB_CONFIG" in os.environ:
config_path = os.environ["ANITYA_WEB_CONFIG"]
else:
config_path = "/etc/anitya/anitya.toml"
if os.path.exists(config_path):
_log.info("Loading Anitya configuration from {}".format(config_path))
with open(config_path) as fd:
try:
file_config = pytoml.loads(fd.read())
for key in file_config:
config[key.upper()] = file_config[key]
except pytoml.core.TomlError as e:
_log.error("Failed to parse {}: {}".format(config_path, str(e)))
else:
_log.info(
"The Anitya configuration file, {}, does not exist.".format(config_path)
)
if not isinstance(config["PERMANENT_SESSION_LIFETIME"], timedelta):
config["PERMANENT_SESSION_LIFETIME"] = timedelta(
seconds=config["PERMANENT_SESSION_LIFETIME"]
)
if config["SECRET_KEY"] == DEFAULTS["SECRET_KEY"]:
_log.warning(
"SECRET_KEY is not configured, falling back to the default. "
"This is NOT safe for production deployments!"
)
return config
def crate_info_from_toml(target, cdir):
if 'url-0.5.7' in cdir:
url_057_toml = '''[package]
name = "url"
version = "0.5.7"
authors = [ "Simon Sapin <*****@*****.**>" ]
description = "URL library for Rust, based on the WHATWG URL Standard"
documentation = "http://servo.github.io/rust-url/url/index.html"
repository = "https://github.com/servo/rust-url"
readme = "README.md"
keywords = ["url", "parser"]
license = "MIT/Apache-2.0"
[features]
query_encoding = ["encoding"]
serde_serialization = ["serde"]
[dependencies.encoding]
version = "0.2"
optional = true
[dependencies.serde]
version = ">=0.6.1, <0.8"
optional = true
[dependencies]
uuid = "0.1.17"
rustc-serialize = "0.3"
unicode-bidi = "0.2.3"
unicode-normalization = "0.1.2"
matches = "0.1"
'''
cfg = toml.loads(url_057_toml)
else:
ctoml = open(os.path.join(cdir, 'Cargo.toml'), 'rb')
cfg = toml.load(ctoml)
return CrateInfo(target, cdir, cfg)
def test_add_assignment_lesson_to_empty_unit(self):
io.add_lesson_to_unit(self.unit_3_path, 'Python Lists', 'assignment')
# Postconditions
self.assertDirectoryIsNotEmpty(self.unit_3_path)
lesson_path = self.unit_3_path / 'lesson-1-python-lists'
dot_romtr_path = lesson_path / '.rmotr'
readme_path = lesson_path / 'README.md'
self.assertDirectoryExists(lesson_path)
self.assertDirectoryIsNotEmpty(lesson_path)
with dot_romtr_path.open('r') as fp:
dot_rmotr_content = toml.loads(fp.read())
self.assertTrue('uuid' in dot_rmotr_content)
self.assertEqual(dot_rmotr_content['name'], 'Python Lists')
self.assertFileExists(readme_path)
with readme_path.open('r') as fp:
self.assertEqual(fp.read(), "# Python Lists\n")
# Dirs to create
tests_path = lesson_path / 'tests'
solutions_path = lesson_path / 'solutions'
# Check dirs are created
self.assertDirectoryExists(tests_path)
self.assertDirectoryExists(solutions_path)
# Files to create
main_py_path = lesson_path / 'main.py'
empty_test_path = tests_path / 'test_.py'
empty_solution_path = solutions_path / 'solution_.py'
# Check files
self.assertFileExists(main_py_path)
self.assertFileExists(empty_test_path)
self.assertFileExists(empty_solution_path)
def _main():
succeeded = []
failed = []
for top, dirnames, fnames in os.walk('.'):
for fname in fnames:
if not fname.endswith('.toml'):
continue
try:
with open(os.path.join(top, fname), 'rb') as fin:
parsed = toml.load(fin)
except toml.TomlError:
parsed = None
else:
dumped = toml.dumps(parsed)
parsed2 = toml.loads(dumped)
if parsed != parsed2:
failed.append(fname)
continue
with open(os.path.join(top, fname), 'rb') as fin:
parsed = toml.load(fin, _testbench_literal, _testbench_array)
try:
with io.open(os.path.join(top, fname[:-5] + '.json'), 'rt', encoding='utf-8') as fin:
bench = json.load(fin)
except IOError:
bench = None
if parsed != bench:
failed.append(fname)
else:
succeeded.append(fname)
for f in failed:
print('failed: {}'.format(f))
print('succeeded: {}'.format(len(succeeded)))
return 1 if failed else 0
def build():
j_env = jinja2.Environment(loader=jinja2.PackageLoader("main", "template"))
j_env.filters["format_time"] = jinja2_format_datetime
j_template = j_env.get_template("article.html")
articles = []
file_names = os.listdir("article")
for file_name in file_names:
file = open("article/%s" % file_name, "rt")
article_head_text, article_content_markdown = file.read().split("+++++++")
file.close()
article_head = pytoml.loads(article_head_text)
article_title = article_head["title"]
article_create_time = datetime.datetime.strptime(article_head["create_time"], '%Y-%m-%dT%H:%M:%S.%f')
article_content_html = mistune.markdown(article_content_markdown)
article_html = j_template.render(
title = article_title,
create_time = article_create_time,
content = article_content_html
)
html_file_name = "%s-%s.html" % (article_create_time.isoformat("T").split(".")[0], article_title)
file = open("site/%s" % html_file_name, "wt")
file.write(article_html)
file.close()
articles.append({"title": article_title, "file_name": html_file_name, "create_time": article_create_time})
articles = sorted(articles, key=lambda item: item["create_time"], reverse=True)
j_template = j_env.get_template("index.html")
index_html = j_template.render(articles=articles)
file = open("site/index.html", "wt")
file.write(index_html)
file.close()
def parse(self, parser):
"""
Parse a column or row.
:param parser: Template parser
:type parser: jinja2.parser.Parser
:return: A null output node.
:rtype: jinja2.nodes.Node
"""
lineno = next(parser.stream).lineno
name = parse_constantlike(self.environment, parser) # Parse the plugin name.
body = parser.parse_statements(['name:endplugin'], drop_needle=True)
layout = self._get_layout(parser)
config = None
if body:
try:
config = flatten_const_node_list(self.environment, body)
except Unflattenable as uf:
raise NonConstant("A `plugin` block may only contain static layout (found: %r)" % uf.args[0])
config = toml.loads(config, "<%s.%s in %s>" % (layout.placeholder_name, name, parser.name))
layout.add_plugin(name, config)
return noop_node(lineno)
def test_add_unit_to_empty_course(self):
generated_unit_path = io.add_unit_to_course(
course_directory_path=self.course_directory_path,
name='Python Intro')
unit_path = self.course_directory_path / 'unit-1-python-intro'
dot_rmotr_path = unit_path / '.rmotr'
readme_path = unit_path / 'README.md'
self.assertEqual(generated_unit_path, unit_path)
self.assertDirectoryExists(unit_path)
self.assertFileExists(dot_rmotr_path)
self.assertFileExists(readme_path)
with dot_rmotr_path.open() as fp:
dot_rmotr_content = toml.loads(fp.read())
self.assertTrue('uuid' in dot_rmotr_content)
self.assertEqual(dot_rmotr_content['name'], 'Python Intro')
with readme_path.open('r') as fp:
content = fp.read()
self.assertEqual(content, """# Python Intro
""")
def test_prettifying_against_humanly_verified_sample():
toml_source = open('sample.toml').read()
expected = open('sample-prettified.toml').read()
assert_prettifier_works(toml_source, expected, prettify)
assert pytoml.loads(toml_source) == pytoml.loads(expected)
def convert_file(file_path,out_dir,file_temp=None):
"""
file_path 需要转换的文件
out_dir 文件输出目录
file_temp 输出模板
"""
filename = os.path.basename(file_path)#.replace('.','_') # 似乎标题有.会有些问题
temp_content =''
m_head={}
if file_temp:
with open(file_temp,'r') as f:
temp_content = f.read()
m_temp = temp_regex.match(temp_content)
m_head = toml.loads(m_temp.group(1))
content = ''
with open(file_path,'r') as f :
content = f.read()
m = content_regex.match(content)
if not m:
print 'Error match content: %s' % file_path
return False
front_data = yaml.load(m.group(1))
if not front_data:
print 'Error load yaml: %s' % file_path
return False
m_head['title']= front_data.get('title','')
if front_data.get("description",None):
m_head['description']=front_data.get("description",None)
if front_data.get('date',None):
#time.strftime("%Y-%m-%dT%H:%M:%S+08:00",b)
m_head['date']=front_data.get('date',None).strftime('%Y-%m-%dT%H:%M:%S+08:00')
m_tags = front_data.get('tags',None)
if m_tags:
print type(m_tags),m_tags
m_head['tags']=[]
if isinstance(m_tags,list):
m_head['tags'].extend(m_tags)
else:
m_head['tags'].append(m_tags)
print m_head['tags']
m_categories = front_data.get('categories',None)
if m_categories:
m_head['categories']=[]
if isinstance(m_categories,list):
m_head['categories'].extend(m_categories)
else:
m_head['categories'].append(m_categories)
print m_head['categories']
write_out_file(m_head,m.group(2),out_dir+'/'+filename)
return True
