def build_schema_for_scores():
"""
Returns
-------
strictyaml.Map
schema used for validation and type-coercion
"""
schemas = {}
for elem in artm.scores.__all__:
if "Score" in elem:
class_of_object = getattr(artm.scores, elem)
# TODO: check if every key is Optional. If it is, then "| EmptyDict()"
# otherwise, just Map()
res = wrap_in_map(build_schema_from_signature(class_of_object))
specific_schema = Map({class_of_object.__name__: res})
schemas[class_of_object.__name__] = specific_schema
for elem in tnscores.__all__:
if "Score" in elem:
class_of_object = getattr(tnscores, elem)
res = build_schema_from_signature(class_of_object)
# res["name"] = Str() # TODO: support custom names
res = wrap_in_map(res)
specific_schema = Map({class_of_object.__name__: res})
schemas[class_of_object.__name__] = specific_schema
return schemas
def __init__(self, filename):
"""Load config from YAML file."""
filename = path.abspath(filename)
if filename is None:
self._config = []
else:
try:
with open(filename, 'r') as handle:
self._yaml = handle.read()
self._config = load(
self._yaml,
Seq(
Map({
Optional("name"):
"name",
"request":
Map({
Optional("path"):
Str(),
Optional("method"):
Enum([
"get",
"post",
"put",
"delete",
"GET",
"POST",
"PUT",
"DELETE",
]),
Optional("headers"):
MapPattern(Str(), Str()),
Optional("data"):
Str(),
}),
"response":
Map({
"content": Str() | Map({"file": Str()}),
Optional("code"): Int(),
Optional("headers"): MapPattern(Str(), Str()),
}),
})))
except Exception as e:
sys.stderr.write(
"Error reading YAML config file: {0}\n".format(str(e)))
sys.exit(1)
# Read and store all references to external content files
for pair in self._config:
content = pair.get('response', {}).get('content')
if type(content) != str and "file" in content:
with open(
path.join(path.dirname(filename), content['file']),
'r') as content_file_handle:
pair['response']['content'] = \
content_file_handle.read()
def _get_info(cls):
# https://hitchdev.com/strictyaml
schema = Map({
Optional("knobs"):
Map({
Optional("log_to_console"): Bool(),
Optional("log_level_debug"): Bool(),
})
| EmptyDict(),
"globals":
Map({
"topic_prefix":
Str(),
Optional(
"reconnect_interval",
default=const.MQTT_DEFAULT_RECONNECT_INTERVAL,
):
Float(),
Optional("poll_interval",
default=const.MYQ_DEFAULT_POLL_INTERVAL):
Float(),
Optional(
"periodic_mqtt_report",
default=const.DEFAULT_PERIODIC_MQTT_REPORT,
):
Float(),
Optional("user_agent"):
Str(),
}),
"mqtt":
Map({
"host":
Str(),
Optional("client_id", default=const.MQTT_DEFAULT_CLIENT_ID):
Str(),
Optional("username"):
Str(),
Optional("password"):
Str(),
}),
"myq":
Map({
"email": Email(),
"password": Str(),
}),
Optional("alias"):
MapPattern(Str(), Str()) | EmptyDict(),
})
if not cls._info:
config_filename = cls._get_config_filename()
logger.info("loading yaml config file %s", config_filename)
with open(config_filename, "r") as ymlfile:
raw_cfg = load(ymlfile.read(), schema).data
cls._parse_raw_cfg(raw_cfg)
return cls._info
def _get_schema(cls):
return Map({
PHASE_DEFINITIONS_TAG:
MapPattern(Str(), Map(cls._get_phase_mapping())),
ROUTE_DEFINITIONS_TAG:
MapPattern(Str(), Map(cls._get_route_mapping())),
MISSION_DEFINITION_TAG:
Map(cls._get_mission_mapping()),
})
def load(path: Path, schema_pointer):
"""Load and validate .yaml file."""
schema = copy.deepcopy(schema_pointer)
with path.open() as f:
yaml = f.read()
data = yaml_load(yaml, Any())
is_template = path.name == "template.yaml"
# Replace real Country and Timezone values with fakes
if is_template:
schema["woo/woocommerce_default_country"] = Enum(["LL"])
schema["wp/timezone_string"] = Enum(["Region/Country"])
schema["wp/DEFAULT_WPLANG"] = Enum(["ll_LL"])
schema["woo/woocommerce_currency"] = Enum(["LLL"])
if "woo/woocommerce_tax_classes" in data:
# Inspect that tax classes and taxes match
# create enum for taxes from defined tax_classes
tax_classes = [
str(tax).lower().replace(" ", "-")
for tax in data["woo/woocommerce_tax_classes"]
]
# +1 is for standard schema which is never defined in tax class
for x in range(len(tax_classes) + 1):
# start counting with 1
schema[f"wootax/{x+1}"] = Map({
"country":
Enum(["LL"]) if is_template else Enum(COUNTRIES),
"state":
Str(),
"rate":
Decimal(),
"name":
Str(),
"priority":
Int(),
"compound":
Int(),
"shipping":
Int(),
"order":
Int(),
"class":
Enum([""]) if x == 0 else Enum(tax_classes),
"locations":
Map({}),
})
try:
return yaml_load(yaml, Map(schema), path)
except YAMLError:
raise
return as_document(schema)
def _get_target_schema(cls) -> Map:
target_schema_map = {}
for key in [f.name for f in fields(FlightPoint)]:
target_schema_map[Optional(
key,
default=None)] = (Float()
| Str()
| Map({
"value": Float() | Str(),
Optional("unit", default=None): Str()
}))
return Map(target_schema_map)
def _get_route_schema(cls) -> Map:
"""Schema of the route section."""
return Map({
Optional("range", default=None):
cls._get_value_schema(),
Optional(CLIMB_PARTS_TAG, default=None):
Seq(Map({PHASE_TAG: Str()})),
CRUISE_PART_TAG:
cls._get_segment_schema(),
Optional(DESCENT_PARTS_TAG, default=None):
Seq(Map({PHASE_TAG: Str()})),
})
def __init__(self, config_file):
# yaml schema
schema = Map({"api": Map({"url": Str(), "query": Str()}),
"dir": Map({"anime": Str(), "exclude": Seq(Str())})})
# open yaml file and load
self.config_file = config_file
with open(config_file, "r", encoding="utf8") as f:
self.data = load(f.read(), schema)
# adding atribute to config
for key1 in self.data.data:
for key2 in self.data.data[key1]:
setattr(self, key1+"_"+key2, self.data.data[key1][key2])
def get_type_schema_yaml_validator() -> Map:
seq_validator = Seq(
Map({
"field": Enum([str(el) for el in Fields]),
"condition": Str(),
"value": Str() | Seq(Str()),
}))
return Map({
Optional(str(RequirementTypes.INPUT_REQUIREMENTS)):
seq_validator,
Optional(str(RequirementTypes.OUTPUT_REQUIREMENTS)):
seq_validator,
})
def FileMapping(glob_allowed: bool):
"""Validator that matches a file mapping: a string or a mapping with
source and target.
"""
if glob_allowed:
return Str() | Map({
'source': Str(),
Optional('target'): Str(),
Optional('mode'): Choice('simple', 'glob')
})
return Str() | Map({
'source': Str(),
Optional('target'): Str(),
})
def get_world_schema():
"""
Getter for world schema
:return: schema that is used to verify the world yaml
"""
return Map({
'robots':
Seq(
Map({
'name':
Str(),
'center_x':
Int(),
'center_y':
Int(),
'orientation':
Int(),
Optional('type'):
Str(),
Optional('parts'):
Seq(ConfigChecker.get_robot_part_schema())
})),
'board_height':
Int(),
'board_width':
Int(),
'board_color':
CommaSeparated(Int()),
'obstacles':
Seq(
Map({
'name': Str(),
'type': Str(),
Optional('outer_spacing'): Int(),
Optional('depth'): Int(),
Optional('color'): CommaSeparated(Int()),
Optional('border_width'): Int(),
Optional('inner_radius'): Int(),
Optional('x'): Int(),
Optional('y'): Int(),
Optional('width'): Int(),
Optional('height'): Int(),
Optional('angle'): Int(),
Optional('movable'): Bool(),
Optional('hole'): Bool(),
Optional('radius'): Int(),
})),
})
def _get_phase_mapping(cls) -> dict:
phase_map = {
Optional(STEPS_TAG, default=None):
Seq(Map(cls._get_segment_mapping()))
}
phase_map.update(cls._get_base_step_mapping())
return phase_map
def __init__(self, config_file):
# yaml schema
schema = Map({
"api": Map({
"url": Str(),
"query": Str()
}),
"dir": Map({
"anime": Str(),
"exclude": Seq(Str())
})
})
# open yaml file and load
self.config_file = config_file
with open(config_file, "r", encoding="utf8") as f:
self.data = load(f.read(), schema)
def schema(cls):
return Map({
**super().schema()._validator,
Optional("default"): Int(),
Optional("min"): Int(),
Optional("max"): Int(),
})
def preprocess_parameters_for_cube_creator(elem_args):
"""
This function does two things:
1) convert class_ids from
name: class_ids@text, values: [0, 1, 2, 3]
to
name: class_ids, values: {"@text": [0, 1, 2, 3]}
2) type conversion for "values" field.
Parameters
----------
elem_args: strictyaml.YAML object
(contains dict inside)
Returns
-------
new_elem_args: dict
"""
for param_portion in elem_args["parameters"]:
name = str(param_portion["name"])
if name.startswith("class_ids"):
validator = Float() | Seq(Float())
else:
validator = Seq(ARTM_TYPES[name])
param_schema = Map({
"name": Str(),
"values": validator
})
param_portion.revalidate(param_schema)
def _get_target_schema(cls) -> Map:
"""Schema for segment target."""
target_schema_map = {}
for key in [f.name for f in fields(FlightPoint)]:
target_schema_map[Optional(
key, default=None)] = cls._get_value_schema()
return Map(target_schema_map)
def _get_base_part_mapping(cls) -> dict:
"""Base mapping for segment/phase schemas."""
polar_coeff_schema = CommaSeparated(Float()) | Str()
polar_schema = Map({
"CL": polar_coeff_schema,
"CD": polar_coeff_schema
}) | Str()
return {
# TODO: this mapping covers all possible segments, but some options are relevant
# only for some segments. A better check could be done in second-pass validation.
Optional("target", default=None):
cls._get_target_schema(),
Optional("engine_setting", default=None):
cls._get_value_schema(Str(), False),
Optional(POLAR_TAG, default=None):
polar_schema,
Optional("thrust_rate", default=None):
cls._get_value_schema(has_unit=False),
Optional("climb_thrust_rate", default=None):
cls._get_value_schema(has_unit=False),
Optional("time_step", default=None):
cls._get_value_schema(),
Optional("maximum_flight_level", default=None):
cls._get_value_schema(has_unit=False),
Optional("mass_ratio", default=None):
cls._get_value_schema(has_unit=False),
Optional("reserve_mass_ratio", default=None):
cls._get_value_schema(has_unit=False),
Optional("use_max_lift_drag_ratio", default=None):
cls._get_value_schema(Bool(), False),
}
def _get_phase_schema(cls) -> Map:
"""Schema of the phase section."""
phase_map = {
Optional(PARTS_TAG, default=None): Seq(cls._get_segment_schema())
}
phase_map.update(cls._get_base_part_mapping())
return Map(phase_map)
def load_string(data: bytes, schema, path: str):
"""Load and validate yaml data."""
try:
return yaml_load(data, Map(schema), path)
except YAMLError:
raise
return as_document(schema)
def FileMapping():
"""Validator that matches a file mapping: a string or a mapping with
source and target.
"""
return Str() | Map({
'source': Str(),
'target': Str(),
})
def build_schema_for_regs():
"""
Returns
-------
strictyaml.Map
schema used for validation and type-coercion
"""
schemas = {}
for elem in artm.regularizers.__all__:
if "Regularizer" in elem:
class_of_object = getattr(artm.regularizers, elem)
res = Map(build_schema_from_signature(class_of_object))
specific_schema = Map({class_of_object.__name__: res})
schemas[class_of_object.__name__] = specific_schema
return schemas
def _get_mission_schema(cls) -> Map:
"""Schema of the mission section."""
return Map({
PARTS_TAG:
Seq(
Map({
Optional(ROUTE_TAG, default=None):
Str(),
Optional(PHASE_TAG, default=None):
Str(),
Optional(RESERVE_TAG, default=None):
Map({
"ref": Str(),
"multiplier": Float() | Str()
}),
})),
})
def Regex():
"""Validator that matches a regex: a mapping with pattern and optional
mode.
"""
return Map({
'pattern': Str(),
Optional('mode'): Choice('first', 'last', 'all'),
})
def NumberCapture():
"""Validator that matches a predefined integer or float capture."""
return Map({
'type': Choice('integer', 'float'),
'name': Str(),
'prefix': Str(),
Optional('mode'): Choice('first', 'last', 'all'),
})
def _validate(cls, content: YAML):
"""
Does a second pass validation of file content.
Also applies this feature:
- polar: foo:bar
is translated to:
- polar:
CL: foo:bar:CL
CD: foo:bar:CD
Errors are raised if file content is incorrect.
:param content:
"""
step_names = set(content[PHASE_DEFINITIONS_TAG].keys())
for phase_definition in content[PHASE_DEFINITIONS_TAG].values():
cls._process_polar_definition(phase_definition)
for segment_definition in phase_definition[STEPS_TAG]:
cls._process_polar_definition(segment_definition)
for route_definition in content[ROUTE_DEFINITIONS_TAG].values():
# Routes are expected to contain some phases and ONE cruise phase
cruise_step_count = 0
for step in route_definition[STEPS_TAG]:
cls._process_polar_definition(step)
Ensure(step.keys()).contains_one_of(
[PHASE_TAG, CRUISE_TYPE_TAG])
if PHASE_TAG in step:
Ensure(step[PHASE_TAG]).is_in(step_names)
YAML(step).revalidate(
Map(cls._get_phase_in_route_mapping()))
else: # CRUISE_TYPE_TAG in step
cruise_step_count += 1
YAML(step).revalidate(
Map(cls._get_segment_mapping(CRUISE_TYPE_TAG)))
Ensure(cruise_step_count).is_less_than_or_equal_to(1)
for step in content[MISSION_DEFINITION_TAG][STEPS_TAG]:
step_type, step_name = tuple(*step.items())
if step_type == PHASE_TAG:
Ensure(step_name).is_in(content[PHASE_DEFINITIONS_TAG].keys())
elif step_type == ROUTE_TAG:
Ensure(step_name).is_in(content[ROUTE_DEFINITIONS_TAG].keys())
def get_robot_schema():
"""
Getter for robot schema
:return: schema that is used to verify the robot yaml
"""
return Map({
'parts': Seq(ConfigChecker.get_robot_part_schema())
})
def _get_value_schema(cls,
value_type: ScalarValidator = Float(),
has_unit=True) -> Validator:
"""Schema for parameter value."""
map_dict = {"value": Float() | Str()}
if has_unit:
map_dict[Optional("unit", default=None)] = Str()
return value_type | Str() | Map(map_dict)
def parse_yaml(yaml_text):
selectors = load(
yaml_text,
MapPattern(Str(), Map({"appears when": Str(), "elements": ELEMENTS_SCHEMA})),
)
for page, page_properties in selectors.items():
revalidate_subelements(page_properties["elements"])
return selectors
def _personal_settings():
settings_file = DIR.key.joinpath("personalsettings.yml")
if not settings_file.exists():
settings_file.write_text(("engine:\n"
" rewrite: no\n"
" cprofile: no\n"
"params:\n"
" python version: 3.5.0\n"))
return load(
settings_file.bytes().decode("utf8"),
Map({
"engine": Map({
"rewrite": Bool(),
"cprofile": Bool()
}),
"params": Map({"python version": Str()}),
}),
)
def _get_schema(cls):
"""Schema of the whole mission file."""
return Map({
PHASE_DEFINITIONS_TAG:
MapPattern(Str(), cls._get_phase_schema()),
ROUTE_DEFINITIONS_TAG:
MapPattern(Str(), cls._get_route_schema()),
MISSION_DEFINITION_TAG:
MapPattern(Str(), cls._get_mission_schema()),
})
