def create_instance(self, family_name: str,
instance_name_or_spec: Union[dict, str]):
dtu.check_isinstance(instance_name_or_spec, (dict, str))
family = self.get_family(family_name)
if not family.valid:
msg = f"Cannot instantiate {instance_name_or_spec!r} because its family {family_name!r} is " \
f"invalid."
msg += "\n\n" + dtu.indent(family.error_if_invalid, " > ")
raise dtu.DTConfigException(msg)
if isinstance(instance_name_or_spec, str):
instance_name = instance_name_or_spec
dtu.check_is_in("instance", instance_name, family.instances)
instance = family.instances[instance_name]
if not instance.valid:
msg = f"Cannot instantiate {instance_name!r} because it is invalid:\n" \
f"{dtu.indent(instance.error_if_invalid, '> ')}"
raise dtu.DTConfigException(msg)
res = dtu.instantiate(instance.constructor, instance.parameters)
elif isinstance(instance_name_or_spec, dict):
_name, spec = _parse_inline_spec(instance_name_or_spec)
res = dtu.instantiate(spec.constructor, spec.parameters)
else:
assert False
interface = dtu.import_name(family.interface)
if not isinstance(res, interface):
msg = f"I expected that {instance_name_or_spec!r} would be a {interface.__name__} but it is a " \
f"{type(res).__name__}."
raise dtu.DTConfigException(msg)
return res
def load_configuration_publisher(name, data):
try:
desc = data.pop("desc", None)
desc = preprocess_desc(desc)
latch = data.pop("latch", None)
topic = data.pop("topic")
type_ = data.pop("type")
queue_size = data.pop("queue_size", None)
except KeyError as e:
msg = f"Could not find field {e!r}."
raise dtu.DTConfigException(msg)
if not isinstance(desc, (str, NoneType)):
msg = f"Description should be a string, not {type(desc).__name__}."
raise dtu.DTConfigException(msg)
if data:
msg = f"Extra keys: {data!r}"
raise dtu.DTConfigException(msg)
T = message_class_from_string(type_)
return EasyNodePublisher(name=name,
desc=desc,
topic=topic,
type=T,
queue_size=queue_size,
latch=latch)
def resolve(self,
package_name: str,
node_name: str,
config_sequence: Union[list, tuple],
date=None):
"""Returns a QueryResult"""
if len(config_sequence) == 0:
msg = f"Invalid empty config_sequence while querying for {package_name}/{node_name}"
raise ValueError(msg)
values = {}
origin = {}
origin_filename = {}
if not package_name in self.package2nodes:
msg = f'Could not find package "{package_name}"; I know {sorted(self.package2nodes)}.'
raise dtu.DTConfigException(msg)
nodes = self.package2nodes[package_name]
if not node_name in nodes:
msg = f'Could not find node "{node_name}" in package "{package_name}"; I know {sorted(nodes)}.'
raise dtu.DTConfigException(msg)
node_config = nodes[node_name]
all_keys = list(node_config.parameters)
overridden = defaultdict(lambda: [])
using = []
for config_name in config_sequence:
if config_name == "defaults":
using.append(config_name)
for p in list(node_config.parameters.values()):
if p.has_default:
values[p.name] = p.default
origin_filename[p.name] = node_config.filename
origin[p.name] = config_name
else:
c = self.find(package_name, node_name, config_name, date=date)
if c is not None:
using.append(config_name)
for k, v in list(c.values.items()):
if k in values:
overridden[k].append(origin[k])
values[k] = v
origin_filename[k] = c.filename
origin[k] = config_name
if not using:
msg = (
f"Cannot find any configuration for {package_name}/{node_name} with config sequence "
f"{':'.join(config_sequence)}")
raise dtu.DTConfigException(msg)
return QueryResult(package_name, node_name, config_sequence, all_keys,
values, origin, origin_filename, overridden)
def load_configuration_parameter(name, data):
# verbose:
# desc: Whether the node is verbose or not.
# type: bool
# default: true
#
try:
desc = data.pop("desc", None)
desc = preprocess_desc(desc)
type_ = data.pop("type")
if "default" in data:
default = data.pop("default")
has_default = True
else:
default = DEFAULT_NOT_GIVEN
has_default = False
except KeyError as e:
msg = f"Could not find field {e.args[0]!r}."
raise dtu.DTConfigException(msg)
if data:
msg = f"Extra keys: {data!r}"
raise dtu.DTConfigException(msg)
if not isinstance(desc, (str, NoneType)):
msg = f"Description should be a string, not {type(desc).__name__}."
raise dtu.DTConfigException(msg)
type2T = {
"bool": bool,
"str": str,
"int": int,
"float": float,
"any": None,
None: None,
"dict": dict,
}
if not type_ in type2T:
raise NotImplementedError(type_)
T = type2T[type_]
if has_default and default is not None and T is not None:
default = T(default)
return EasyNodeParameter(name=name,
desc=desc,
type=T,
has_default=has_default,
default=default)
def _init_parameters(self):
parameters = self._configuration.parameters
class Config:
def __getattr__(self, name):
if not name in parameters:
msg = f"The user is trying to use {name!r}, which is not a parameter "
msg += "for this node.\n"
s = "\n- ".join(list(parameters))
msg += f"The declared parameters that can be used are:\n- {s}"
raise AttributeError(msg)
return object.__getattr__(self, name)
self.config = Config()
values = {}
# load the configuration
self.info("Loading parameters...")
with dtu.rospy_timeit_wall("getting configuration files"):
qr = get_user_configuration(self.package_name, self.node_type_name)
if not qr.is_complete():
msg = "\nThe configuration that I could load is not complete:\n"
msg += dtu.indent(str(qr), " | ")
msg = dtu.indent(
msg,
f"{self.package_name} / {self.node_type_name} fatal error > ")
raise dtu.DTConfigException(msg)
self.info(f"Loaded configuration:\n{qr}")
for p in list(parameters.values()):
try:
val = qr.values.get(p.name)
except KeyError:
msg = f"Could not load required parameter {p.name!r}."
raise dtu.DTConfigException(msg)
# write to parameter server, for transparency
if val is not None: # we cannot set None parameters
rospy.set_param("~" + p.name, val)
setattr(self.config, p.name, val)
values[p.name] = val
self._on_parameters_changed(first_time=True, values=values)
duration = self.config.en_update_params_interval
duration = rospy.Duration.from_sec(duration)
rospy.Timer(duration, self._update_parameters)
def load_configuration(realpath, contents) -> EasyNodeConfig:
# TODO: load "version" string
try:
try:
data = dtu.yaml_load(contents)
except Exception as e:
msg = "Could not parse YAML file properly:"
dtu.raise_wrapped(dtu.DTConfigException, e, msg, compact=True)
raise # ide not smart
if not isinstance(data, dict):
msg = f"Expected a dict, got {type(data).__name__}."
raise dtu.DTConfigException(msg)
try:
parameters = data.pop("parameters")
subscriptions = data.pop("subscriptions")
publishers = data.pop("publishers")
contracts = data.pop("contracts")
description = data.pop("description")
except KeyError as e:
key = e.args[0]
msg = f"Invalid configuration: missing field {key!r}."
raise dtu.DTConfigException(msg)
if not isinstance(description, (str, NoneType)):
msg = f"Description should be a string, not {type(description).__name__}."
raise dtu.DTConfigException(msg)
if data:
msg = f"Spurious fields found: {sorted(data)}"
raise dtu.DTConfigException(msg)
parameters = load_configuration_parameters(parameters)
subscriptions = load_configuration_subscriptions(subscriptions)
contracts = load_configuration_contracts(contracts)
publishers = load_configuration_publishers(publishers)
return EasyNodeConfig(
filename=realpath,
parameters=parameters,
contracts=contracts,
subscriptions=subscriptions,
publishers=publishers,
package_name=None,
description=description,
node_type_name=None,
)
except dtu.DTConfigException as e:
msg = f"Invalid configuration at {realpath}: "
dtu.raise_wrapped(dtu.DTConfigException, e, msg, compact=True)
def get_config_sequence() -> Tuple[str, ...]:
""" Reads the variable EasyNode.ENV
it is taken as a colon-separated list of names.
"""
from easy_node.easy_node import EasyNode
if not EasyNode.ENV in os.environ:
msg = f'Could not find the environment variable "{EasyNode.ENV}".'
raise dtu.DTConfigException(msg)
s = os.environ[EasyNode.ENV]
tokens = [_ for _ in s.split(":") if _.strip()]
if not tokens:
msg = f"The variable {EasyNode.ENV} is empty."
raise dtu.DTConfigException(msg)
return tuple(tokens)
def load_configuration_subscription(name, data):
# image:
# desc: Image to read
# topic: ~image
# type: CompressedImage
# queue_size: 1
try:
desc = data.pop("desc", None)
desc = preprocess_desc(desc)
latch = data.pop("latch", None)
timeout = data.pop("timeout", None)
topic = data.pop("topic")
type_ = data.pop("type")
queue_size = data.pop("queue_size", None)
process = data.pop("process", PROCESS_SYNCHRONOUS)
if not process in PROCESS_VALUES:
msg = f"Invalid value of process {process!r} not in {PROCESS_VALUES!r}."
raise dtu.DTConfigException(msg)
except KeyError as e:
msg = f"Could not find field {e!r}."
raise dtu.DTConfigException(msg)
if not isinstance(desc, (str, NoneType)):
msg = f"Description should be a string, not {type(desc).__name__}."
raise dtu.DTConfigException(msg)
if data:
msg = f"Extra keys: {data!r}"
raise dtu.DTConfigException(msg)
T = message_class_from_string(type_)
return EasyNodeSubscription(
name=name,
desc=desc,
topic=topic,
timeout=timeout,
type=T,
queue_size=queue_size,
latch=latch,
process=process,
)
def _parse_regression_test_check(line: str) -> Wrapper:
line = line.strip()
delim = " "
tokens = line.split(delim)
if len(tokens) != 3:
msg = f'I expect exactly 3 tokens with delimiter {delim}.\nLine: "{line}"\nTokens: {tokens}'
raise dtu.DTConfigException(msg)
try:
ref1 = parse_reference(tokens[0])
binary = parse_binary(tokens[1])
ref2 = parse_reference(tokens[2])
evaluable = BinaryEval(ref1, binary, ref2)
except RTParseError as e:
msg = f'Cannot parse string "{line}".'
dtu.raise_wrapped(RTParseError, e, msg, compact=True)
raise
return Wrapper(evaluable)
def message_class_from_string(s):
if not "/" in s:
msg = ""
msg += f'Invalid message name "{s}".\n'
msg += 'I expected that the name of the message is in the format "PACKAGE/MSG".\n '
msg += 'E.g. "sensor_msgs/Joy" or "duckietown_msgs/BoolStamped".'
raise dtu.DTConfigException(msg)
# e.g. "std_msgs/Header"
i = s.index("/")
package = s[:i]
name = s[i + 1:]
symbol = f"{package}.msg.{name}"
try:
msgclass = dtu.import_name(symbol)
return msgclass
except ValueError as e:
msg = f'Cannot import type for message "{s}" ({symbol}).'
dtu.raise_wrapped(dtu.DTConfigException, e, msg, compact=True)
def load_configuration_package_node(package_name: str,
node_type_name: str) -> EasyNodeConfig:
path = dru.get_ros_package_path(package_name)
look_for = f"{node_type_name}.easy_node.yaml"
found = dtu.locate_files(path, look_for)
if not found:
msg = f"Could not find EasyNode configuration {look_for!r}."
raise dtu.DTConfigException(msg) # XXX
fn = found[0]
contents = open(fn).read()
c = load_configuration(fn, contents)
c = c._replace(package_name=package_name)
c = c._replace(node_type_name=node_type_name)
# Add the common parameters
if node_type_name != "easy_node":
c0 = load_configuration_baseline()
c = merge_configuration(c0, c)
return c
def interpret_easy_algo_config(filename: str, data: dict) -> EasyAlgoFamily:
""" Interprets the family config """
basename = os.path.basename(filename)
family_name = dtu.id_from_basename_pattern(basename, EasyAlgoDB.pattern)
instances_pattern = f"*.{family_name}.yaml"
# tests_pattern = '*.%s_test.yaml' % family_name
data0 = dict(data)
try:
dtu.dt_check_isinstance("contents", data, dict)
description = data.pop("description")
dtu.dt_check_isinstance("description", description, str)
interface = data.pop("interface")
dtu.dt_check_isinstance("interface", interface, str)
locations = data.pop("locations", None)
default_constructor = data.pop("default_constructor", None)
except Exception as e:
msg = f"Cannot interpret data: \n\n{data0}"
raise Exception(msg) from e
if data:
msg = f'Extra keys in configuration: {list(data)}'
raise dtu.DTConfigException(msg)
return EasyAlgoFamily(
interface=interface,
family_name=family_name,
filename=filename,
instances=None,
instances_pattern=instances_pattern,
description=description,
valid=True,
locations=locations,
default_constructor=default_constructor,
error_if_invalid=False,
)
def interpret_config_file(filename: str) -> ConfigInfo:
"""
Returns a ConfigInfo.
"""
try:
basename = os.path.basename(filename)
base = basename.replace(SUFFIX, "")
# now we have something like
# package-node.config_name.date
# or
# package-node.config_name
if not "." in base:
msg = f"Invalid filename {filename!r}."
raise dtu.DTConfigException(msg)
tokens = base.split(".")
if len(tokens) > 3:
msg = f"Too many periods/tokens (tokens={tokens})"
raise dtu.DTConfigException(msg)
if len(tokens) <= 2:
# package-node.config_name
package_node = tokens[0]
if not "-" in package_node:
msg = f'Expected a "-" in "{package_node}".'
raise dtu.DTConfigException(msg)
i = package_node.index("-")
package_name = package_node[:i]
node_name = package_node[i + 1 :]
else:
package_name = node_name = None # FIXME: should we bail?
config_name = tokens[1]
if len(tokens) == 3:
# package-node.config_name.date
date_effective = tokens[2]
else:
date_effective = "20170101"
try:
date_effective = parse(date_effective)
except:
msg = f'Cannot interpret "{date_effective}" as a date.'
raise dtu.DTConfigException(msg)
# now read file
with open(filename) as f:
contents = f.read()
try:
try:
data = yaml.load(contents, Loader=yaml.Loader)
except YAMLError as e:
dtu.raise_wrapped(dtu.DTConfigException, e, "Invalid YAML", compact=True)
raise
if not isinstance(data, dict):
msg = "Expected a dictionary inside."
raise dtu.DTConfigException(msg)
for field in ["description", "values"]:
if not field in data:
msg = f'Missing field "{field}".'
raise dtu.DTConfigException(msg)
description = data.pop("description")
if not isinstance(description, str):
msg = f'I expected that "description" is a string, obtained {description!r}.'
raise dtu.DTConfigException(msg)
extends = data.pop("extends", [])
if not isinstance(extends, list):
msg = f'I expected that "extends" is a list, obtained {extends!r}.'
raise dtu.DTConfigException(msg)
values = data.pop("values")
if not isinstance(values, dict):
msg = f'I expected that "values" is a dictionary, obtained {type(values)}.'
raise dtu.DTConfigException(msg)
# Freeze the data
extends = tuple(extends)
values = frozendict(values)
except dtu.DTConfigException as e:
msg = "Could not interpret the contents of the file\n"
msg += f" {dtu.friendly_path(filename)}\n"
msg += "Contents:\n" + dtu.indent(contents, " > ")
dtu.raise_wrapped(dtu.DTConfigException, e, msg, compact=True)
raise
return ConfigInfo(
filename=filename,
package_name=package_name,
node_name=node_name,
config_name=config_name,
date_effective=date_effective,
extends=extends,
description=description,
values=values,
# not decided
valid=None,
error_if_invalid=None,
)
except dtu.DTConfigException as e:
msg = f"Invalid file {dtu.friendly_path(filename)}"
dtu.raise_wrapped(dtu.DTConfigException, e, msg, compact=True)
def load_family_config(all_yaml: dict) -> Dict[str, EasyAlgoFamily]:
"""
# now, for each family, we look for configuration files, which are:
#
# ![ID].![family_name].yaml
#
#
"""
if not isinstance(all_yaml, dict):
msg = f"Expected a dict, got {type(all_yaml).__name__}"
raise ValueError(msg)
family_name2config = {}
configs = dtu.look_everywhere_for_config_files2(EasyAlgoDB.pattern,
all_yaml)
configs.update(
dtu.look_everywhere_for_config_files2("*.family.yaml", all_yaml))
for filename, contents in list(configs.items()):
c: EasyAlgoFamily = dtu.interpret_yaml_file(
filename, contents, interpret_easy_algo_config)
if c.family_name in family_name2config:
one = family_name2config[c.family_name].filename
two = c.filename
msg = f"Repeated filename:\n{one}\n{two}"
raise dtu.DTConfigException(msg)
def interpret_instance_spec(filename, data):
dtu.dt_check_isinstance("data", data, dict)
basename = os.path.basename(filename)
instance_name = dtu.id_from_basename_pattern(
basename, c.instances_pattern)
if c.default_constructor is not None and not "constructor" in data:
description = "(not given)"
constructor = c.default_constructor
parameters = dict(data)
else:
description = data.pop("description")
dtu.dt_check_isinstance("description", description, str)
constructor = data.pop("constructor")
dtu.dt_check_isinstance("constructor", constructor, str)
parameters = data.pop("parameters")
dtu.dt_check_isinstance("parameters", parameters,
(dict, NoneType))
if parameters is None:
parameters = {}
return EasyAlgoInstance(
family_name=c.family_name,
instance_name=instance_name,
description=description,
filename=filename,
constructor=constructor,
parameters=parameters,
valid=True,
error_if_invalid=None,
)
c = check_validity_family(c)
instances = {}
_ = dtu.look_everywhere_for_config_files2(c.instances_pattern,
all_yaml)
# noinspection PyAssignmentToLoopOrWithParameter
for filename, contents in list(_.items()):
i = dtu.interpret_yaml_file(filename,
contents,
interpret_instance_spec,
plain_yaml=True)
if i.instance_name in instances:
one = instances[i.instance_name].filename
two = i.filename
msg = f"Repeated filename:\n{one}\n{two}"
raise dtu.DTConfigException(msg)
# i = check_validity_instance(c, i)
instances[i.instance_name] = i
c = c._replace(instances=instances)
family_name2config[c.family_name] = c
return family_name2config
