def main():
opts = getArgs()
for o, a in opts:
if o in ("-h", "--help"):
usage()
sys.exit()
elif o in ("-k", "--key"):
d_key = a
elif o in ("-v", "--value"):
d_value = a
else:
assert False, "unhandled option"
yaml = YAML()
yaml.explicit_start = True
yaml.allow_unicode = True
yaml.width = 300
result = []
for data in list(yaml.load_all(sys.stdin)):
if data is not None:
if (data['kind'] == "ConfigMap") or \
(data['kind'] == "Secret"):
# update data: key=value
data['data'][d_key] = d_value
result.append(data)
elif 'kind' in data.keys():
result.append(data)
yaml.dump_all(result, sys.stdout)
def dict_to_yaml(data, width=None, sort=False):
""" Convert dictionary into yaml """
output = StringIO()
# Set formatting options
yaml = YAML()
yaml.indent(mapping=4, sequence=4, offset=2)
yaml.default_flow_style = False
yaml.allow_unicode = True
yaml.encoding = 'utf-8'
yaml.width = width
# Make sure that multiline strings keep the formatting
data = copy.deepcopy(data)
scalarstring.walk_tree(data)
# Sort the data https://stackoverflow.com/a/40227545
if sort:
sorted_data = CommentedMap()
for key in sorted(data):
sorted_data[key] = data[key]
data = sorted_data
yaml.dump(data, output)
return output.getvalue()
def main():
opts = getArgs()
for o, a in opts:
if o in ("-h", "--help"):
usage()
sys.exit()
elif o in ("-k", "--key"):
key_to_update = a
elif o in ("-v", "--value"):
value_to_update = a
else:
assert False, "unhandled option"
yaml = YAML()
yaml.explicit_start = True
yaml.allow_unicode = True
yaml.width = 300
data_list = key_to_update.split(".")
data_to_refer = value_to_update
for k in data_list[::-1]:
data_to_refer = {k: data_to_refer}
result = []
for data in list(yaml.load_all(sys.stdin)):
if data is not None:
data = update_data(data, data_to_refer)
result.append(data)
yaml.dump_all(result, sys.stdout)
def _read_file(file_path):
ryaml = YAML(pure=True)
ryaml.allow_unicode = True
with open(file_path, 'r', encoding="utf-8") as r_f:
file_data = r_f.read()
file_data = ryaml.load(file_data)
return file_data
def write_to_yaml(fpath: Path, yaml_obj: YAML,
dictionary: RawDictionary) -> None:
"""Write an ordered dictionary as YAML to a file"""
if not dictionary:
return
yaml_obj.allow_unicode = True
yaml_obj.width = 1000
with fpath.open("w") as fp:
yaml_obj.dump(dictionary, stream=fp)
def write_ybe_string(ybe_exam, minimal=True):
"""Dump the provided YbeExam as a .ybe formatted string.
Args:
ybe_exam (ybe.lib.ybe_contents.YbeExam): the ybe file contents to dump
minimal (boolean): if set to False we print all available fields.
If set to True (the default) we print only fields with non-default values.
Returns:
str: an .ybe (Yaml) formatted string
"""
visitor = YbeConversionVisitor(minimal=minimal)
content = {'ybe_version': __version__}
content.update(visitor.convert(ybe_exam))
yaml = YAML(typ='rt')
yaml.register_class(TextHTML)
yaml.register_class(TextMarkdown)
yaml.default_flow_style = False
yaml.allow_unicode = True
yaml.width = float('inf')
yaml.indent(mapping=4, offset=4, sequence=4)
def beautify_line_spacings(s):
ret_val = ''
previous_new_line = ''
in_questions_block = False
for line in s.splitlines(True):
new_line = line
if in_questions_block:
if line.startswith(' '):
new_line = line[4:]
elif line.startswith('\n'):
pass
else:
in_questions_block = False
else:
if line.startswith('questions:'):
in_questions_block = True
if any(new_line.startswith(el) for el in ['info', 'questions:', '- multiple_choice:', '- open:',
'- multiple_response:', '- text_only:'])\
and not previous_new_line.startswith('\nquestions:'):
new_line = '\n' + new_line
previous_new_line = new_line
ret_val += new_line
return ret_val
yaml.dump(content, result := StringIO(), transform=beautify_line_spacings)
return result.getvalue()
def update_environment_yml():
"""Update conda_dev_env.yml file for conda."""
import re
from ruamel.yaml import YAML
from ruamel.yaml.comments import CommentedMap, CommentedSeq
environment_filename = "conda_dev_env.yml"
cmap = CommentedMap()
cmap.yaml_set_start_comment(
("Usage: conda env create -n myenvname -f {} python=3.6\n"
" conda activate myenvname\n"
" pip install --no-deps -e .".format(environment_filename)))
cmap["name"] = "aiida_crystal17"
cmap["channels"] = CommentedSeq(["conda-forge", "cjs14"])
cmap["channels"].yaml_add_eol_comment("for sqlalchemy-diff and pgtest", 1)
cmap["dependencies"] = dmap = CommentedSeq()
# additional packages
dmap.append("pip")
dmap.append("aiida-core.services")
# fix incompatibilities between conda and pypi
replacements = {"pre-commit": "pre_commit"}
setup_json = get_setup_json()
for base, key in [
(None, "install_requires"),
("extras_require", "testing"),
("extras_require", "code_style"),
("extras_require", "docs"),
]:
requirements = setup_json.get(base, setup_json)[key]
count = 0
for req in sorted(requirements, key=lambda x: x.lower()):
# skip packages required for specific python versions < 3
if re.findall("python_version\\s*\\<\\s*\\'?3", req):
continue
req = req.split(";")[0]
for (regex, replacement) in iter(replacements.items()):
req = re.sub(regex, replacement, req)
count += 1
dmap.append(req.lower())
dmap.yaml_set_comment_before_after_key(len(dmap) - count, before=key)
yaml = YAML(typ="rt")
yaml.default_flow_style = False
yaml.encoding = "utf-8"
yaml.allow_unicode = True
file_path = os.path.join(ROOT_DIR, environment_filename)
with open(file_path, "w") as env_file:
yaml.dump(cmap, env_file)
def read_yaml_file(filename):
yaml = YAML()
yaml.explicit_start = True
yaml.indent(mapping=2, sequence=4, offset=2)
yaml.allow_unicode = True
data = None
with open(filename, 'r') as stream:
data = yaml.load(stream)
with open(filename, 'w') as f:
yaml.dump(data, f, transform=sequence_indent_four)
def dump_yaml(foo, outfile, no_anchors=False):
if no_anchors:
pyyaml.add_representer(int, hexint_presenter)
pyyaml.dump(foo, outfile, Dumper=NoAliasDumper)
else:
yaml = YAML(typ="rt")
yaml.default_flow_style = False
yaml.allow_unicode = True
yaml.compact(seq_seq=False, seq_map=False)
yaml.indent = 4
yaml.block_seq_indent = 2
yaml.dump(foo, outfile)
def dump_yaml(emoji, pretty=False):
""" Dump list of emoji from key:val pairs to dict to yaml """
from collections import OrderedDict
from ruamel.yaml import YAML
import sys
emoji = OrderedDict(emoji)
yaml = YAML()
yaml.allow_unicode = True
if pretty:
yaml.indent(sequence=4, offset=2)
yaml.dump(dict(emoji), sys.stdout)
def write_ybe_string(ybe_exam, minimal=False):
"""Dump the provided YbeExam as a .ybe formatted string.
Args:
ybe_exam (ybe.lib.ybe_contents.YbeExam): the ybe file contents to dump
minimal (boolean): if set to True we only print the configured options.
By default this flag is False, meaning we print all the available options, if needed with null placeholders.
Returns:
str: an .ybe (Yaml) formatted string
"""
visitor = YbeConversionVisitor(minimal=minimal)
content = visitor.visit(ybe_exam)
yaml = YAML(typ='rt')
yaml.default_flow_style = False
yaml.allow_unicode = True
yaml.width = float('inf')
yaml.indent(mapping=4, offset=4, sequence=4)
def beautify_line_spacings(s):
ret_val = ''
previous_new_line = ''
in_questions_block = False
for line in s.splitlines(True):
new_line = line
if in_questions_block:
if line.startswith(' '):
new_line = line[4:]
elif line.startswith('\n'):
pass
else:
in_questions_block = False
else:
if line.startswith('questions:'):
in_questions_block = True
if any(new_line.startswith(el) for el in ['info', 'questions:', '- multiple_choice:', '- open:',
'- multiple_response:', '- text_only:'])\
and not previous_new_line.startswith('\nquestions:'):
new_line = '\n' + new_line
previous_new_line = new_line
ret_val += new_line
return ret_val
yaml.dump(content, result := StringIO(), transform=beautify_line_spacings)
return result.getvalue()
def write_yaml(data, file_path):
"""Write plain data to a file as yaml
Parameters
----------
data
Data to write (should be lists, dicts and simple values)
file_path : str
The path of the configuration file to write
"""
with open(file_path, 'w+') as file_handle:
yaml = YAML(typ='unsafe')
yaml.default_flow_style = False
yaml.allow_unicode = True
return yaml.dump(data, file_handle)
def yaml_dump(data, file):
"""
Dump a data structure to a file in YAML format.
Strings containing line breaks are emitted in literal block scalar style.
"""
yaml = YAML(typ='safe', pure=True)
yaml.default_flow_style = False
yaml.indent(mapping=2, sequence=4, offset=2)
yaml.allow_unicode = True
yaml.representer.add_representer(
str, lambda dumper, data: dumper.represent_scalar(
u'tag:yaml.org,2002:str',
data,
style='|' if '\n' in data else None))
yaml.dump(data, file)
def main():
yaml = YAML()
yaml.explicit_start = True
yaml.allow_unicode = True
yaml.width = 300
result = []
for data in list(yaml.load_all(sys.stdin)):
if data is not None:
if data['kind'] == 'Secret':
for k, v in data['data'].items():
data['data'][k] = base64.b64encode(
v.encode('utf-8')).decode('utf-8')
result.append(data)
elif 'kind' in data.keys():
result.append(data)
yaml.dump_all(result, sys.stdout)
def main():
opts = getArgs()
for o, a in opts:
if o in ("-h", "--help"):
usage()
sys.exit()
elif o in ("-t", "--type"):
c_type = a
elif o in ("-n", "--name"):
c_name = a
elif o in ("-i", "--image"):
c_image = a
else:
assert False, "unhandled option"
yaml = YAML()
yaml.explicit_start = True
yaml.allow_unicode = True
yaml.width = 300
result = []
for data in list(yaml.load_all(sys.stdin)):
if data is not None:
if (data['kind'] == "Deployment") or \
(data['kind'] == "DaemonSet") or \
(data['kind'] == "StatefulSet"):
# update image
container_ls = [
c['name'] for c in data['spec']['template']['spec'][c_type]
]
for n, c in enumerate(container_ls):
if c == c_name:
data['spec']['template']['spec'][c_type][n][
'image'] = c_image
result.append(data)
elif data['kind'] == "CronJob":
# update image
container_ls = [
c['name'] for c in data['spec']['jobTemplate']['spec']
['template']['spec'][c_type]
]
for n, c in enumerate(container_ls):
if c == c_name:
data['spec']['jobTemplate']['spec']['template'][
'spec'][c_type][n]['image'] = c_image
result.append(data)
elif 'kind' in data.keys():
result.append(data)
yaml.dump_all(result, sys.stdout)
def update_environment_yml():
"""
Updates conda_dev_env.yml file for conda.
"""
import re
from ruamel.yaml.comments import CommentedMap, CommentedSeq
from ruamel.yaml import YAML
environment_filename = 'conda_dev_env.yml'
cmap = CommentedMap()
cmap.yaml_set_start_comment(
'Usage: conda env create -n myenvname -f {} python=3.6'.format(
environment_filename))
cmap['name'] = 'aiida_icl'
cmap['channels'] = CommentedSeq(['conda-forge', 'cjs'])
cmap['channels'].yaml_add_eol_comment('for sqlalchemy-diff and pgtest', 1)
cmap['dependencies'] = dmap = CommentedSeq()
# fix incompatibilities between conda and pypi
replacements = {}
setup_json = get_setup_json()
for base, key in [(None, 'install_requires'),
('extras_require', 'testing'),
('extras_require', 'code_style')]:
requirements = setup_json.get(base, setup_json)[key]
count = 0
for req in sorted(requirements, key=lambda x: x.lower()):
# skip packages required for specific python versions < 3
if re.findall("python_version\\s*\\<\\s*\\'?3", req):
continue
req = req.split(';')[0]
for (regex, replacement) in iter(replacements.items()):
req = re.sub(regex, replacement, req)
count += 1
dmap.append(req.lower())
dmap.yaml_set_comment_before_after_key(len(dmap) - count, before=key)
yaml = YAML(typ='rt')
yaml.default_flow_style = False
yaml.encoding = 'utf-8'
yaml.allow_unicode = True
file_path = os.path.join(ROOT_DIR, environment_filename)
with open(file_path, 'w') as env_file:
yaml.dump(cmap, env_file)
def _write_yaml_file(directory, name, data):
"""Write plain data to a file as yaml
Parameters
----------
directory : str
name : str
file basename (without yml extension)
data
Data to write (should be lists, dicts and simple values)
"""
path = os.path.join(directory, "{}.yml".format(name))
with open(path, 'w') as file_handle:
yaml = YAML()
yaml.default_flow_style = False
yaml.allow_unicode = True
return yaml.dump(data, file_handle)
def main():
opts = getArgs()
for o, a in opts:
if o in ("-h", "--help"):
usage()
sys.exit()
elif o in ("-t", "--tag"):
tag = a
else:
assert False, "unhandled option"
yaml = YAML()
yaml.explicit_start = True
yaml.allow_unicode = True
yaml.width = 300
result = []
for data in list(yaml.load_all(sys.stdin)):
if data is not None:
if (data['kind'] == "Deployment") or \
(data['kind'] == "DaemonSet") or \
(data['kind'] == "StatefulSet"):
# add label: tag
data['spec']['template']['metadata']['labels']['tag'] = tag
# add label: imageName.n, imageVersion.n
image_ls = [c['image'] for c in data['spec']['template']['spec']['containers']]
for n,i in enumerate(image_ls):
i_name = i.split(':')[0][i.split(':')[0].rfind('/')+1:]
i_version = i.split(':')[1]
data['spec']['template']['metadata']['labels']['imageName.' + str(n)] = i_name
data['spec']['template']['metadata']['labels']['imageVersion.' + str(n)] = i_version
result.append(data)
elif data['kind'] == "CronJob":
# add label: tag
data['spec']['jobTemplate']['spec']['template']['metadata']['labels']['tag'] = tag
# add label: imageName.n, imageVersion.n
image_ls = [c['image'] for c in data['spec']['jobTemplate']['spec']['template']['spec']['containers']]
for n,i in enumerate(image_ls):
i_name = i.split(':')[0][i.split(':')[0].rfind('/')+1:]
i_version = i.split(':')[1]
data['spec']['jobTemplate']['spec']['template']['metadata']['labels']['imageName.' + str(n)] = i_name
data['spec']['jobTemplate']['spec']['template']['metadata']['labels']['imageVersion.' + str(n)] = i_version
result.append(data)
elif 'kind' in data.keys():
result.append(data)
yaml.dump_all(result, sys.stdout)
def _get_yaml(cls, typ):
yaml = YAML(typ=typ)
# If allow_unicode=True, true unicode characters will be written to the
# file instead of being replaced by escape sequence.
yaml.allow_unicode = ('utf' in cls.YAML_ENCODING)
yaml.default_flow_style = False
yaml.indent = 4
yaml.constructor.add_constructor('!include', functools.partial(cls._yaml_include_constructor, typ))
yaml.constructor.add_constructor('!var', cls._yaml_var_constructor)
yaml.constructor.add_multi_constructor('!env:', cls._yaml_env_var_constructor)
yaml.constructor.add_multi_constructor('!call:', cls._yaml_call_constructor)
# Replace unknown tags by a placeholder object containing the data.
# This happens when the class was not imported at the time the object
# was deserialized
yaml.constructor.add_constructor(None, cls._yaml_unknown_tag_constructor)
return yaml
def write_yaml(filename:str, dictionary:dict):
""" Function to convert a dictionary into a YAML file
"""
yml = YAML()
yml.explicit_start = True
yml.default_flow_style = False
yml.encoding = "utf-8" # default when using YAML() or YAML(typ="rt")
yml.allow_unicode = True # always default in the new API
yml.errors = "strict"
yml.indent(sequence=4, offset=2)
yml.explicit_end = True
if isinstance(dictionary,dict):
with open(filename, 'w') as outfile:
print(filename)
yml.dump(dictionary, outfile)
else:
raise Exception('its not a dictionary')
def load_yaml(foo, no_anchors=False):
if no_anchors:
yaml = YAML(typ="safe")
else:
yaml = YAML(typ="rt")
yaml.default_flow_style = False
yaml.allow_unicode = True
yaml.compact(seq_seq=False, seq_map=False)
yaml.indent = 4
yaml.block_seq_indent = 2
try:
with open(foo, "r") as file:
return yaml.load(file)
except ruamel.yaml.constructor.DuplicateKeyError as msg:
logger = logging.getLogger(__name__)
error = "\n".join(str(msg).split("\n")[2:-7])
logger.error(error)
raise SystemExit
"""Common Utils """
import sys
import os
import subprocess
import shlex
from riscv_isac.log import logger
import ruamel
from ruamel.yaml import YAML
from ruamel.yaml.representer import RoundTripRepresenter, SafeRepresenter
import yaml as pyyaml
from elftools.elf.elffile import ELFFile
yaml = YAML(typ="rt")
yaml.default_flow_style = False
yaml.explicit_start = True
yaml.allow_unicode = True
yaml.allow_duplicate_keys = False
safe_yaml = YAML(typ="safe")
safe_yaml.default_flow_style = False
safe_yaml.explicit_start = True
safe_yaml.allow_unicode = True
safe_yaml.allow_duplicate_keys = False
def collect_label_address(elf, label):
with open(elf, 'rb') as f:
elffile = ELFFile(f)
# elfclass is a public attribute of ELFFile, read from its header
symtab = elffile.get_section_by_name('.symtab')
size = symtab.num_symbols()
import logging
import argparse
import operator
import ruamel
from ruamel.yaml import YAML
yaml = YAML(typ="rt")
yaml.default_flow_style = False
yaml.allow_unicode = True
def load_yaml(foo):
try:
with open(foo, "r") as file:
return yaml.load(file)
except ruamel.yaml.constructor.DuplicateKeyError as msg:
logger = logging.getLogger(__name__)
error = "\n".join(str(msg).split("\n")[2:-7])
logger.error(error)
raise SystemExit
class ColoredFormatter(logging.Formatter):
"""
Class to create a log output which is colored based on level.
"""
def __init__(self, *args, **kwargs):
super(ColoredFormatter, self).__init__(*args, **kwargs)
self.colors = {
class RequestHandler(http.server.SimpleHTTPRequestHandler):
def do_GET(self):
if self.path == '/':
self.path = GEN_FILE
return http.server.SimpleHTTPRequestHandler.do_GET(self)
PORT = 8080
HANDLER = RequestHandler
#Yaml configuration
YML = YAML()
YML.indent(mapping=2, sequence=4, offset=2)
YML.encoding = "utf-8"
YML.allow_unicode = True
def get_config():
with open(MAIN_DIR + '/config/cloud-config.yml', 'r') as sys_config_file:
sys_config = sys_config_file.read()
return YML.load(sys_config)
def write_keys(sys_yaml):
if not 'ssh_authorized_keys' in sys_yaml:
sys_yaml['ssh_authorized_keys'] = list()
try:
with open(MAIN_DIR + '/config/pub_keys', 'r') as pub_key_file:
for key in pub_key_file:
sys_yaml['ssh_authorized_keys'].append(key.replace('\n', ''))
def _write_file(file_path, data):
ryaml = YAML(pure=True)
ryaml.allow_unicode = True
with open(file_path, 'w+', encoding="utf-8") as w_f:
ryaml.dump(data, w_f)
from importlib import import_module
from importlib.util import spec_from_file_location
from importlib.util import module_from_spec
import os
import sys
import json
import string
from io import StringIO, IOBase
from ruamel.yaml import YAML
# A shiny global ruamel.yaml obj with sane options (dumps should pass yamllint)
YM = YAML()
YM.indent(mapping=2, sequence=4, offset=2)
YM.explicit_start = True
YM.explicit_end = True
YM.allow_unicode = True
YM.preserve_quotes = True
# global formatting (any changes *should* propagate to later directives)
EOL = '\n' # line separator
RENDER_JS = False
##
# substitution
##
def sub_stream(strm, meta, method):
"""
substitute strings in strm
return (the same? or a new) stream with substituted values
"""
