def test_constructor_types(data_filename, code_filename, verbose=False):
_make_objects()
native1 = None
native2 = None
yaml = ruamel.yaml.YAML(typ='safe', pure=True)
yaml.loader = MyLoader
try:
with open(data_filename, 'rb') as fp0:
native1 = list(yaml.load_all(fp0))
if len(native1) == 1:
native1 = native1[0]
with open(code_filename, 'rb') as fp0:
native2 = _load_code(fp0.read())
try:
if native1 == native2:
return
except TypeError:
pass
# print('native1', native1)
if verbose:
print('SERIALIZED NATIVE1:')
print(_serialize_value(native1))
print('SERIALIZED NATIVE2:')
print(_serialize_value(native2))
assert _serialize_value(native1) == _serialize_value(native2), (
native1, native2)
finally:
if verbose:
print('NATIVE1:')
pprint.pprint(native1)
print('NATIVE2:')
pprint.pprint(native2)
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
parser.add_argument(
"-m",
"--multi",
"--allow-multiple-documents",
action="store_true",
)
parser.add_argument(
"-k",
"--kinds",
"--banned-kinds",
default="Secret",
)
parser.add_argument("filenames", nargs="*", help="Filenames to check.")
args = parser.parse_args(argv)
kinds = [k.lower() for k in args.kinds.split(",")]
retval = 0
for filename in args.filenames:
try:
with open(filename, encoding="UTF-8") as f:
if args.multi:
docs = yaml.load_all(f)
else:
docs = [yaml.load(f)]
for doc in docs:
if isinstance(doc, dict) and doc.get("kind", "").lower() in kinds:
print(f"Found {doc['kind']} in {filename}")
retval = 1
except ruamel.yaml.YAMLError as exc:
print(exc)
retval = 1
return retval
def test_roundtrip_data(code_filename, roundtrip_filename, verbose=False):
_make_objects()
with open(code_filename, 'rb') as fp0:
value1 = fp0.read()
yaml = YAML(typ='safe', pure=True)
yaml.Loader = MyLoader
native2 = list(yaml.load_all(value1))
if len(native2) == 1:
native2 = native2[0]
try:
value2 = ruamel.yaml.dump(
native2,
Dumper=MyDumper,
default_flow_style=False,
allow_unicode=True,
encoding='utf-8',
)
# value2 += x
if verbose:
print('SERIALIZED NATIVE1:')
print(value1)
print('SERIALIZED NATIVE2:')
print(value2)
assert value1 == value2, (value1, value2)
finally:
if verbose:
print('NATIVE2:')
pprint.pprint(native2)
def loadReachability():
filename = "%s/reach/dart.yaml" % results_dir
if (len(sys.argv) > 1):
filename = sys.argv[1]
with open(filename, 'r') as f:
docs = yaml.load_all(f)
return next(docs), next(docs)
def _create_config_file():
with open(config_template, 'r', encoding='utf-8') as f:
loader = yaml.load_all(f)
next(loader) # discard first document (used for comment)
ydoc = next(loader)
with open(config_file, 'w', encoding='utf-8') as f:
yaml.dump(ydoc, f)
def main(argv):
yaml_file = argv[1]
yaml_output = argv[2]
image_record_files = argv[3:]
document_out = collections.OrderedDict()
new_image_dict = {}
image_records = []
# Read all lines from all files in image_records list
for image_record_file in image_record_files:
with open(image_record_file) as ir_file:
new_records = [line.rstrip() for line in ir_file.readlines()]
image_records.extend(new_records)
# Create a dictionary to map image name to image location/tag
for image in image_records:
name = get_image_name(image)
if name != '':
new_image_dict[name] = image
# Load chart into dictionary(s) and then modify any image locations/tags if required
for document in yaml.load_all(open(yaml_file),
Loader=yaml.RoundTripLoader,
preserve_quotes=True,
version=(1, 1)):
document_name = (document['schema'], document['metadata']['schema'],
document['metadata']['name'])
modify_yaml(document, '', '', new_image_dict)
document_out[document_name] = document
# Save modified yaml to file
yaml.dump_all(document_out.values(),
open(yaml_output, 'w'),
Dumper=yaml.RoundTripDumper,
default_flow_style=False)
def update_utterances(intent, query, file_name):
input_filename = "yaml/" + str(file_name) + ".yaml"
output_filename = "yaml/" + str(file_name) + "_temp.yaml"
inp = open(input_filename, 'r+')
op = open(output_filename, 'w+')
input = yaml.load_all(inp, yaml.RoundTripLoader)
go = False
utterances = []
expand = synonymous(query)
for s in expand:
utterances.append(s[0])
bk = False
for i in input:
for k, v in i.items():
if(v == intent):
go = True
if(k == "utterances" and go):
v.extend(utterances)
bk = True
if(bk):
break
yaml.dump(i, op, Dumper=yaml.RoundTripDumper, explicit_start=True)
if(bk):
break
inp.close()
op.close()
clean_up(output_filename, input_filename)
def move_annotations(f):
"""Modifies a YAML ProwJob file in-place by moving name and annotations
to the top of the spec elements.
:param f:
:return:
"""
files = list(yaml.load_all(open(f)))
# pylint: disable=R1702
for lvl1 in files:
for lvl2 in lvl1.values():
if isinstance(lvl2, ruamel.yaml.comments.CommentedSeq):
for job in lvl2:
if not 'annotations' in job:
continue
job.move_to_end('annotations', last=False)
job.move_to_end('name', last=False)
elif isinstance(lvl2, ruamel.yaml.comments.CommentedMap):
for lvl3 in lvl2.values():
if isinstance(lvl3, bool):
continue
for job in lvl3:
if not 'annotations' in job:
continue
job.move_to_end('annotations', last=False)
job.move_to_end('name', last=False)
else:
print('skipping', lvl2)
yaml.dump_all(files, open(f, 'w'))
def __init__(self, contents: str):
"""
:param contents: The parsed YAML contents of a full Kubernetes object, as a Dict
"""
docs = yaml.load_all(contents, Loader=yaml.RoundTripLoader)
self.config_maps = []
self.role_bindings = []
self.services = []
# TODO: Improve generalization here
for d in docs:
kind = d["kind"]
if kind == "ConfigMap":
self.config_maps.append(addict.Dict(d))
elif kind in ["CronJob", "Job", "MPIJob"]:
self._root = addict.Dict(d)
elif kind == "RoleBinding":
self.role_bindings.append(addict.Dict(d))
elif kind == "Service":
self.services.append(addict.Dict(d))
else:
raise ValueError(
"Kubernetes yaml object is of an unsupported kind type: {}"
.format(d["kind"]))
if kind == "MPIJob":
self._validate_mpi_spec()
for replica_type in [MPI_JOB_LAUNCHER, MPI_JOB_WORKER]:
self.create_empty_fields(mpiReplicaType=replica_type)
else:
self._validate()
self.create_empty_fields()
def round_trip_load_all(inp, preserve_quotes=None, version=None):
import ruamel.yaml # NOQA
dinp = dedent(inp)
yaml = ruamel.yaml.YAML()
yaml.preserve_quotes = preserve_quotes
yaml.version = version
return yaml.load_all(dinp)
def load_yaml(yaml_file):
# Load the YAML file
yaml = ruamel.yaml.YAML()
yaml.preserve_quotes = True
with open(yaml_file) as filepath:
# Use yaml.load_all to be able to load multi-yaml files
data = list(yaml.load_all(filepath))
return data
def load_all_gen(stream):
"""
Load all documents within the given YAML string.
:param stream: A valid YAML stream.
:return: Generator that yields each document found in the YAML stream.
"""
return yaml.load_all(stream, OrderedRoundTripLoader)
def test_loader_error_string(error_filename, verbose=False):
try:
with open(error_filename, 'rb') as fp0:
list(yaml.load_all(fp0.read()))
except yaml.YAMLError as exc:
if verbose:
print('%s:' % exc.__class__.__name__, exc)
else:
raise AssertionError('expected an exception')
def _process_datasets(output_dir: Path, datasets: Iterable[Path],
do_checksum: bool, newer_than: datetime):
logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s",
level=logging.INFO)
for dataset_path in datasets:
(mode, ino, dev, nlink, uid, gid, size, atime, mtime,
ctime) = os.stat(str(dataset_path))
create_date = datetime.utcfromtimestamp(ctime)
if create_date <= newer_than:
logging.info(
"Dataset creation time %s is older than start date %s...SKIPPING",
create_date,
newer_than,
)
else:
if dataset_path.is_dir():
dataset_path = dataset_path.joinpath(
dataset_path.stem.replace("PRD_MSIL1C", "MTD_SAFL1C") +
".xml")
if dataset_path.suffix not in [".xml", ".zip"]:
raise RuntimeError("want xml or zipped archive")
logging.info("Processing %s", dataset_path)
output_path = Path(output_dir)
yaml_path = output_path.joinpath(dataset_path.name + ".yaml")
logging.info("Output %s", yaml_path)
if os.path.exists(str(yaml_path)):
logging.info("Output already exists %s", yaml_path)
with open(str(yaml_path)) as f:
if do_checksum:
logging.info("Running checksum comparison")
datamap = yaml.load_all(f)
for data in datamap:
yaml_sha1 = data["checksum_sha1"]
checksum_sha1 = hashlib.sha1(
dataset_path.open("rb").read()).hexdigest()
if checksum_sha1 == yaml_sha1:
logging.info(
"Dataset preparation already done...SKIPPING")
continue
else:
logging.info(
"Dataset has changed...ARCHIVING out of date yaml"
)
archive_yaml(yaml_path, output_dir)
else:
logging.info(
"Dataset preparation already done...SKIPPING")
continue
documents = prepare_dataset(dataset_path)
if documents:
logging.info("Writing %s dataset(s) into %s", len(documents),
yaml_path)
with open(str(yaml_path), "w") as stream:
yaml.dump_all(documents, stream)
else:
logging.info("No datasets discovered. Bye!")
def test_loader_error(error_filename, verbose=False):
yaml = YAML(typ='safe', pure=True)
try:
with open(error_filename, 'rb') as fp0:
list(yaml.load_all(fp0))
except yaml.YAMLError as exc:
if verbose:
print('%s:' % exc.__class__.__name__, exc)
else:
raise AssertionError('expected an exception')
def load(self, content_type='yaml'):
''' return yaml file '''
contents = self.read()
if not contents and not self.content:
return None
if self.content:
if isinstance(self.content, dict):
self.yaml_dict = self.content
return self.yaml_dict
elif isinstance(self.content, str):
contents = self.content
# check if it is yaml
try:
if content_type == 'yaml' and contents:
# Try to set format attributes if supported
try:
self.yaml_dict.fa.set_block_style()
except AttributeError:
pass
# Try to use RoundTripLoader if supported.
try:
index = 0
self.yaml_dict = {}
items = yaml.load_all(contents)
for data in items:
object_key = 'object_%s' % index
index = index + 1
self.yaml_dict[object_key] = data
# self.yaml_dict = yaml.load(contents, yaml.RoundTripLoader)
# self.yaml_dict = {'data': data}
except AttributeError:
self.yaml_dict = yaml.safe_load(contents)
# Try to set format attributes if supported
try:
self.yaml_dict.fa.set_block_style()
except AttributeError:
pass
elif content_type == 'json' and contents:
self.yaml_dict = json.loads(contents)
except yaml.YAMLError as err:
# Error loading yaml or json
raise YeditException(
'Problem with loading yaml file. {0}'.format(err))
return self.yaml_dict
def read_input_file(input_file, yaml):
"""
:param input_file: string, path to valid yaml file
:param yaml: ruamel.yaml instance
:return: list of dicts, each dict a valid subset of input yaml file
"""
print(f"Reading input file: {input_file}")
with open(os.path.expanduser(input_file), "r") as stream:
try:
return [doc for doc in yaml.load_all(stream)]
except Exception as e:
print(e)
def read_yaml_file(self, key=None):
"""
to read yaml data
:return:
"""
warnings.simplefilter('ignore', ruamel.yaml.error.UnsafeLoaderWarning)
with open(self.file, 'r', encoding='utf-8') as f:
rsl = yaml.load_all(f.read())
for i in rsl:
if key is not None and key in i.keys():
return i[key]
else:
return i
def readDefaultsYaml(self, fname):
fnames = [fname]
fnames.extend([os.path.join(p, fname) for p in self.incPath])
for name in fnames:
try:
f = IncParser(name)
docs = yaml.load_all(f)
self.defaults = self.mergeDocs(docs)
self.combine()
return
except Exception as ex:
pass
raise Exception("Could not find defaults file in %s" % str(fnames))
def __init__(self, fname, i_options={}, dptype=None, debug=False):
txt = read_file_or_url(fname)
try:
model_data, hmodel_data = ry.load_all(txt, ry.RoundTripLoader)
except Exception as ex:
print(
"Error while parsing YAML file. Probable YAML syntax error in file : ",
fname)
raise ex
model_data, hmodel_data = ry.load_all(txt, Loader=ry.RoundTripLoader)
self.__model__ = Model(model_data)
self.data = hmodel_data
self.discretization_options = i_options
# cache for functions
self.__equilibrium__ = None
self.__projection__ = None
self.__features__ = None
self.debug = debug
self.check()
self.__set_changed__()
from dolo.numeric.processes import IIDProcess, ProductProcess
if dptype is None and isinstance(
self.model.exogenous,
ProductProcess) and (self.model.exogenous.processes[1],
IIDProcess):
dptype = 'iid'
else:
dptype = 'mc'
self.dptype = dptype
def operation_yaml(self, mode='r', *data):
# to operation the yaml file
if mode == "w":
with open(self.__tgt_file, 'w', encoding='utf-8') as f:
yaml.dump_all(data, f, Dumper=yaml.RoundTripDumper)
logger.info("yaml文件写入成功。")
if mode == "r":
warnings.simplefilter('ignore',
ruamel.yaml.error.UnsafeLoaderWarning)
with open(self.__tgt_file, 'r', encoding='utf-8') as f:
result = yaml.load_all(f.read())
logger.info("yaml文件读取成功。")
return result
def dir_entries():
"""
Iterator over all data files in the cloned green directory
"""
path = os.path.join(green_directory_local_path, green_direcory_data_path)
for root, dirs, files in os.walk(path):
for fname in files:
filepath = os.path.join(root, fname)
if not filepath.endswith(".yaml"):
continue
with open(filepath, 'r') as yamlfile:
for doc in yaml.load_all(yamlfile, Loader=yaml.Loader):
yield doc
def test_write_header(tmpdir):
"""Check you can write just the yaml"""
test_header = ruamel.yaml.comments.CommentedMap({"a": 1})
test_md = ["\n", "the first thing\n"]
test_yamlmd = [test_header, test_md]
tmp_file = tmpdir.mkdir("md").join("test.yml").strpath
write_yaml(test_yamlmd, tmp_file)
yaml = YAML(typ='rt')
with open(tmp_file, encoding="UTF-8") as f:
meta = next(yaml.load_all(f))
assert 'a' in meta
assert meta['a'] == 1
def load_yaml_file(file: str, multiple: bool = False) -> Union[dict, list]:
"""
读取YAML文件
@param file: 文件路径
@param multiple: 同文件内是否包含多个文档
@return: 字典或列表
"""
with open(file, 'r', encoding="utf-8") as f:
yaml_data = f.read()
if multiple:
yaml_load = yaml.load_all(yaml_data, Loader=yaml.Loader)
else:
yaml_load = yaml.load(yaml_data, Loader=yaml.Loader)
data_list = []
for item in yaml_load:
data_list.append(item)
return data_list
def load_summary(self, path):
""" Load the list of generated overrides files
Generate a list of override files that were written for the manifest.
This is used to generate Armada --values overrides for the manifest.
:param path: location of the overrides summary file
:return: a list of override files written
"""
files_written = []
summary_fqpn = os.path.join(path, SUMMARY_FILE)
if os.path.exists(summary_fqpn):
self.manifest_path = os.path.dirname(summary_fqpn)
with open(summary_fqpn, 'r') as f:
files_written = list(
yaml.load_all(f, Loader=yaml.RoundTripLoader))[0]
return files_written
def main(templated_helm_file, require_file):
yaml = ruamel.yaml.YAML()
yaml.indent(mapping=2)
with open(templated_helm_file, 'r') as f:
templated_helm = yaml.load_all(f.read())
with open(require_file, 'r') as f:
requirements = yaml.load(f.read())
checker = RequirementChecker(requirements.get('resources'))
new_doc = {}
for yaml_doc in templated_helm:
if yaml_doc is None:
continue
if checker.is_required(yaml_doc):
new_doc[get_resource_key(yaml_doc)] = yaml_doc
print('# GENERATED FILE: edits made by hand will not be preserved.')
print('---')
yaml.dump_all(same_sort(requirements, new_doc), sys.stdout)
def read_remote_url_file(remote_url, yaml):
"""
:param remote_url: string, url containing remote yaml file
:param yaml: ruamel.yaml instance
:return: list of dicts, each dict a valid subset of remote yaml file
"""
remote_url_path = urllib.parse.urlparse(remote_url)
remote_filename = remote_url_path[2].split("/")[-1]
print(f"Retrieving url: {remote_url}")
print(f"Reading remote file: {remote_filename}")
# TODO: verify HTTPResponse is `Content-Type: text/plain;` otherwise err
resp = urllib.request.urlopen(remote_url)
decoded_data = resp.read().decode("utf-8")
try:
return [doc for doc in yaml.load_all(decoded_data)]
except Exception as e:
print(e)
def load_all(stream, file_hint=None):
"""Loads multiple YAML documents from the given steam.
Args:
stream: A file like object or string that can be read from.
file_hint: str, The name of a file or url that the stream data is coming
from. See load() for more information.
Raises:
YAMLParseError: If the data could not be parsed.
Yields:
The parsed YAML data.
"""
try:
for x in yaml.load_all(stream, yaml.SafeLoader, version='1.1'):
yield x
except yaml.YAMLError as e:
raise YAMLParseError(e, f=file_hint)
def load(self, manifest_fqpn):
""" Load the application manifest for processing
:param manifest_fqpn: fully qualified path name of the application manifest
"""
if os.path.exists(manifest_fqpn):
# Save the path for writing overrides files
self.manifest_path = os.path.dirname(manifest_fqpn)
# Save the name for a delete manifest
self.delete_manifest = "%s-del%s" % os.path.splitext(manifest_fqpn)
with open(manifest_fqpn, 'r') as f:
# The RoundTripLoader removes the superfluous quotes by default,
# resulting the dumped out charts not readable in Armada.
# Set preserve_quotes=True to preserve all the quotes.
self.content = list(
yaml.load_all(f,
Loader=yaml.RoundTripLoader,
preserve_quotes=True))
# Generate the lookup tables
# For the individual chart docs
self.docs[KEY_DATA_CHART_NAME] = {
i[KEY_METADATA][KEY_METADATA_NAME]: i
for i in self.content if i[KEY_SCHEMA] == VAL_SCHEMA_CHART
}
# For the chart group docs
self.docs[KEY_DATA_CHART_GROUP] = {
i[KEY_METADATA][KEY_METADATA_NAME]: i
for i in self.content
if i[KEY_SCHEMA] == VAL_SCHEMA_CHART_GROUP
}
# For the single manifest doc
self.docs[KEY_DATA_CHART_GROUPS] = {
i[KEY_METADATA][KEY_METADATA_NAME]: i
for i in self.content if i[KEY_SCHEMA] == VAL_SCHEMA_MANIFEST
}
else:
LOG.error("Manifest file %s does not exist" % manifest_fqpn)
def load_all(stream, file_hint=None):
# type: (Union[str, IO[AnyStr]], Optional[str]) -> Generator[Any]
"""Loads multiple YAML documents from the given steam.
Args:
stream: A file like object or string that can be read from.
file_hint: str, The name of a file or url that the stream data is coming
from. See load() for more information.
Raises:
YAMLParseError: If the data could not be parsed.
Yields:
The parsed YAML data.
"""
try:
for x in yaml.load_all(stream, yaml.SafeLoader, version='1.1'):
yield x
except yaml.YAMLError as e:
raise YAMLParseError(e, f=file_hint)
def load_all(stream, file_hint=None):
# type: (typing.Union[str, typing.IO[typing.AnyStr]], typing.Optional[str]) -> typing.Generator[typing.Any] # pylint: disable=line-too-long
"""Loads multiple YAML documents from the given steam.
Args:
stream: A file like object or string that can be read from.
file_hint: str, The name of a file or url that the stream data is coming
from. See load() for more information.
Raises:
YAMLParseError: If the data could not be parsed.
Yields:
The parsed YAML data.
"""
try:
for x in yaml.load_all(stream, yaml.SafeLoader, version='1.1'):
yield x
except yaml.YAMLError as e:
raise YAMLParseError(e, f=file_hint)
def _load_all(*args, **kwargs): # type: (*Any, **Any) -> None
_exhaust(yaml.load_all(*args, **kwargs))
from os.path import isfile, join
from icalendar import Calendar, Event
baseURL = "https://prowadzacy.eka.pwr.edu.pl/pobierz.php?pole=%s_%s_%i"
print 'Hello!'
peoplePath = '_people'
onlyfiles = [f for f in listdir(peoplePath) if isfile(join(peoplePath, f))]
people = []
for file in onlyfiles:
filePath = '%s/%s' % (peoplePath, file)
stream = open(filePath, "r")
docs = yaml.load_all(stream)
for index, doc in enumerate(docs):
if index == 1:
break
if isinstance(doc, dict):
name = None
calendarCode = None
for k, v in doc.items():
if k == 'calendarcode':
calendarCode = v
elif k == 'name':
name = v
if calendarCode and name:
person = (name, calendarCode)
people.append(person)