def __init__(self, stream):
Reader.__init__(self, stream)
Scanner.__init__(self)
Parser.__init__(self)
Composer.__init__(self)
SafeConstructor.__init__(self)
Resolver.__init__(self)
def __init__(self, stream):
Reader.__init__(self, stream)
Scanner12.__init__(self)
Parser.__init__(self)
Composer.__init__(self)
# PKM2014 - uses SafeConstructor instead of Constructor. This should make no
# difference, because "Is it YAML?" only gets as far as the composition stage,
# but - well: it reduces the risk of something bad happening.
# Constructor.__init__(self)
SafeConstructor.__init__(self)
Resolver.__init__(self)
def __init__(self):
self.add_multi_constructor("!vol/", self.__class__.vol_constructor)
self.add_multi_constructor("!type/", self.__class__.type_constructor)
self.add_constructor(
"!include", self.__class__.include_file_constructor)
self.add_constructor(
"!include_merge", self.__class__.include_merge_constructor)
self.add_constructor(
"!include_dir_file_mapped",
self.__class__.include_dir_file_mapped_constructor)
self.add_constructor("!env_var", self.__class__.env_var_constructor)
self.add_constructor("!path", self.__class__.path_constructor)
SafeConstructor.__init__(self)
def construct_yaml_map(self, node):
"""
Return a node class representing a line-marked dictionary.
"""
obj, = SafeConstructor.construct_yaml_map(self, node)
return DictNode(obj, node.start_mark, node.end_mark)
def construct_yaml_seq(self, node):
"""
Return a node class representing a line-marked sequence.
"""
obj, = SafeConstructor.construct_yaml_seq(self, node)
return ListNode(obj, node.start_mark, node.end_mark)
def construct_yaml_str(self, node):
"""
Return a node class representing a line-marked string.
"""
obj = SafeConstructor.construct_scalar(self, node)
assert da.util.is_string(obj)
return StringNode(obj, node.start_mark, node.end_mark)
def construct_yaml_str(self, node):
obj = SafeConstructor.construct_yaml_str(self, node)
try:
obj = str(obj)
except UnicodeEncodeError:
raise DSLParsingInputTypeException(
ERROR_INVALID_CHARS,
'illegal characters in line: {0}, column: {1}. '
'Only valid ascii chars are supported.'.format(
node.start_mark.line, node.start_mark.column))
return self._holder(obj, node)
def construct_mapping(self, node, deep=False):
if not isinstance(node, yaml.nodes.MappingNode):
raise ConstructorError(
None, None,
"expected a mapping node, but found %s" % node.id,
node.start_mark)
keys = set([])
for key_node, value_node in node.value:
key = self.construct_object(key_node, deep=deep)
if key in keys:
raise ConstructorError(
"while constructing a mapping", node.start_mark,
"found duplicate key (%s)" % key, key_node.start_mark)
keys.add(key)
return SafeConstructor.construct_mapping(self, node, deep)
@author: usingh
"""
import yaml
from pyrpipe import pyrpipe_utils as pu
#ignore boolean coversion of yes/no values when loading yaml
from yaml.constructor import SafeConstructor
def add_bool(self, node):
return self.construct_scalar(node)
SafeConstructor.add_constructor(u'tag:yaml.org,2002:bool', add_bool)
class YAML_loader():
"""
Load parameters from a yaml file
"""
def __init__(self, file):
self.__params = None
self.__kwargs = None
if not pu.check_files_exist(file):
return
#read yaml
with open(file) as f:
#self.__params=yaml.full_load(f)
def construct_yaml_binary(self, node):
obj = SafeConstructor.construct_yaml_binary(self, node)
return self._holder(obj, node)
def construct_yaml_bool(self, node):
obj = SafeConstructor.construct_yaml_bool(self, node)
return int_node(obj, node.start_mark, node.end_mark)
def construct_yaml_timestamp(self, node):
obj = SafeConstructor.construct_yaml_timestamp(self, node)
return self._holder(obj, node)
def construct_yaml_map(self, node):
(obj, ) = SafeConstructor.construct_yaml_map(self, node)
return dict_node(obj, node.start_mark, node.end_mark)
def construct_yaml_seq(self, node):
obj, = SafeConstructor.construct_yaml_seq(self, node)
assert isinstance(obj, list) # nosec
return list_node(obj, node.start_mark, node.end_mark) # nosec
def construct_yaml_str(self, node):
obj = SafeConstructor.construct_yaml_str(self, node)
assert isinstance(obj, (six.string_types)) # nosec
return str_node(obj, node.start_mark, node.end_mark)
def __init__(self, filename=None):
"""
"""
SafeConstructor.__init__(self)
self.__filename = filename
def construct_mapping(node, deep=False):
mapping = SafeConstructor.construct_mapping(loader, node, deep=deep)
mapping["__line__"] = LineTracker(filename, node.__line__)
return mapping
def construct_yaml_str(self, node):
obj = SafeConstructor.construct_yaml_str(self, node)
assert isinstance(obj, str) # nosec
return StrNode(obj, node.start_mark, node.end_mark)
def construct_yaml_map(self, node):
obj, = SafeConstructor.construct_yaml_map(self, node)
return self._holder(obj, node)
def parse(self, content, **kwargs):
""" Parses the given YAML content to create stringset and template
Steps are:
1. Load yaml content using our custom loader TxYamlLoader that
in addition to the value for each key notes the `start` and
`end` index of each node in the file and some metadata.
2. Flattens the output of the loader to be a list of the form:
```
[{
'key': 'string_key1',
'value': 'string1',
'end': <end_index_of_node>,
'start': <start_index_value>,
'style': '|, >, ...'
},
...
]
```
3. Iterates over the flattened list and for each entry creates an
OpenString object, appends it to stringset and replace its value
with the template_replacement in the template.
4. Returns the (template, stringset) tuple.
"""
template = []
stringset = []
# The first argument of the add_constructor method is the tag you want
# to handle. If you provide None as an argument, all the unknown tags
# will be handled by the constructor specified in the second argument.
# We need this in order parse all unknown custom-tagged values as
# strings.
SafeConstructor.add_constructor(None,
SafeConstructor.construct_yaml_str)
yaml_data = self._load_yaml(content, loader=TxYamlLoader)
yaml_data = self._get_yaml_data_to_parse(yaml_data)
# Helper to store the processed data while parsing the file
self._parsed_data = []
self._parse_yaml_data(yaml_data, '', '')
self._parsed_data = sorted(self._parsed_data,
key=lambda node: node.get('start'))
end = 0
order = 0
for node in self._parsed_data:
start = node.get('start')
end_ = node.get('end')
key = node.get('key')
tag = node.get('tag')
value = node.get('value')
style = node.get('style')
if not value:
continue
if isinstance(value, dict) and not all(six.itervalues(value)):
continue
string_object = OpenString(
key, value, context=tag or '', flags=style, order=order,
)
stringset.append(string_object)
order += 1
template.append(u"{}{}".format(content[end:start],
string_object.template_replacement))
comment = self._find_comment(content, end, start)
string_object.developer_comment = comment
end = end_
template.append(content[end:])
template = u''.join(template)
return template, stringset
def __init__(self, filename):
SafeConstructor.__init__(self)
self.filename = filename
def construct_yaml_timestamp(self, node):
obj = SafeConstructor.construct_yaml_timestamp(self, node)
return self._holder(obj, node)
value = self.construct_object(value_node, deep=deep)
mapping[key] = value
return mapping
BaseConstructor.construct_mapping = construct_ordered_mapping
def construct_yaml_map_with_ordered_dict(self, node):
data = OrderedDict()
yield data
value = self.construct_mapping(node)
data.update(value)
for t in [u'tag:yaml.org,2002:map', u'tag:yaml.org,2002:omap']:
SafeConstructor.add_constructor( t, construct_yaml_map_with_ordered_dict )
Constructor.add_constructor( t, construct_yaml_map_with_ordered_dict )
yaml.add_constructor( t, construct_yaml_map_with_ordered_dict )
def represent_ordered_mapping(self, tag, mapping, flow_style=None):
value = []
node = yaml.MappingNode(tag, value, flow_style=flow_style)
best_style = True
if self.alias_key is not None:
self.represented_objects[self.alias_key] = node
if hasattr(mapping, 'items'):
mapping = list(mapping.items())
def construct_yaml_seq(self, node):
obj, = SafeConstructor.construct_yaml_seq(self, node)
return list_node(obj, node.start_mark, node.end_mark)
def construct_yaml_str(self, node):
obj = SafeConstructor.construct_yaml_str(self, node)
assert isinstance(obj, (str))
return str_node(obj, node.start_mark, node.end_mark)
def parse(self, content, **kwargs):
""" Parses the given YAML content to create stringset and template
Steps are:
1. Load yaml content using our custom loader TxYamlLoader that
in addition to the value for each key notes the `start` and
`end` index of each node in the file and some metadata.
2. Flattens the output of the loader to be a list of the form:
```
[{
'key': 'string_key1',
'value': 'string1',
'end': <end_index_of_node>,
'start': <start_index_value>,
'style': '|, >, ...'
},
...
]
```
3. Iterates over the flattened list and for each entry creates an
OpenString object, appends it to stringset and replace its value
with the template_replacement in the template.
4. Returns the (template, stringset) tuple.
"""
template = []
stringset = []
# The first argument of the add_constructor method is the tag you want
# to handle. If you provide None as an argument, all the unknown tags
# will be handled by the constructor specified in the second argument.
# We need this in order parse all unknown custom-tagged values as
# strings.
SafeConstructor.add_constructor(None,
SafeConstructor.construct_yaml_str)
yaml_data = self._load_yaml(content, loader=TxYamlLoader)
yaml_data = self._get_yaml_data_to_parse(yaml_data)
# Helper to store the processed data while parsing the file
self._parsed_data = []
self._parse_yaml_data(yaml_data, '', '')
self._parsed_data = sorted(self._parsed_data,
key=lambda node: node.get('start'))
end = 0
order = 0
for node in self._parsed_data:
start = node.get('start')
end_ = node.get('end')
key = node.get('key')
tag = node.get('tag')
value = node.get('value')
style = node.get('style')
if not value:
continue
if isinstance(value, dict) and not all(six.itervalues(value)):
continue
string_object = OpenString(
key,
value,
context=tag or '',
flags=style,
order=order,
)
stringset.append(string_object)
order += 1
template.append(u"{}{}".format(content[end:start],
string_object.template_replacement))
comment = self._find_comment(content, end, start)
string_object.developer_comment = comment
end = end_
template.append(content[end:])
template = u''.join(template)
return template, stringset
def construct_yaml_binary(self, node):
obj = SafeConstructor.construct_yaml_binary(self, node)
return self._holder(obj, node)
REQUIRED_KEYS = [
"cli_version",
"globals",
"rules",
"providers",
"backend",
"engine_version",
]
def add_bool(self, node):
"""Allows bools to be set as strings."""
return self.construct_scalar(node)
SafeConstructor.add_constructor("tag:yaml.org,2002:bool", add_bool)
class ConfigParser:
"""Parses and validates reflex yaml config file."""
def __init__(self, config_file):
self.config_file = config_file
self.raw_coniguration = {}
self.rule_list = []
def parse_valid_config(self):
"""Entrypoint to generate and validate config"""
self.raw_configuration = ( # pylint: disable=attribute-defined-outside-init
self.parse_yaml_config())
self.rule_list = self.create_rule_list()
valid_config = self.validate_config(self.raw_configuration)
def construct_yaml_map(self, node):
obj, = SafeConstructor.construct_yaml_map(self, node)
return self._holder(obj, node)
def construct_yaml_seq(self, node):
obj, = SafeConstructor.construct_yaml_seq(self, node)
return list_node(obj, node.start_mark, node.end_mark)
def construct_yaml_str(self, node):
obj = SafeConstructor.construct_scalar(self, node)
assert isinstance(obj, unicode)
return unicode_node(obj, node.start_mark, node.end_mark)
return mapping
BaseConstructor.construct_mapping = construct_ordered_mapping
def construct_yaml_map_with_ordered_dict(self, node):
data = OrderedDict()
yield data
value = self.construct_mapping(node)
data.update(value)
for t in [u'tag:yaml.org,2002:map', u'tag:yaml.org,2002:omap']:
SafeConstructor.add_constructor(t, construct_yaml_map_with_ordered_dict)
Constructor.add_constructor(t, construct_yaml_map_with_ordered_dict)
yaml.add_constructor(t, construct_yaml_map_with_ordered_dict)
def represent_ordered_mapping(self, tag, mapping, flow_style=None):
value = []
node = yaml.MappingNode(tag, value, flow_style=flow_style)
best_style = True
if self.alias_key is not None:
self.represented_objects[self.alias_key] = node
if hasattr(mapping, 'items'):
mapping = list(mapping.items())
def construct_yaml_float(self, node):
obj = SafeConstructor.construct_scalar(self, node)
return float_node(obj, node.start_mark, node.end_mark)
def __init__(self, filename):
SafeConstructor.__init__(self)
self.filename = filename