def create_instance(self, family_name: str, instance_name_or_spec: Union[dict, str]): dtu.check_isinstance(instance_name_or_spec, (dict, str)) family = self.get_family(family_name) if not family.valid: msg = f"Cannot instantiate {instance_name_or_spec!r} because its family {family_name!r} is " \ f"invalid." msg += "\n\n" + dtu.indent(family.error_if_invalid, " > ") raise dtu.DTConfigException(msg) if isinstance(instance_name_or_spec, str): instance_name = instance_name_or_spec dtu.check_is_in("instance", instance_name, family.instances) instance = family.instances[instance_name] if not instance.valid: msg = f"Cannot instantiate {instance_name!r} because it is invalid:\n" \ f"{dtu.indent(instance.error_if_invalid, '> ')}" raise dtu.DTConfigException(msg) res = dtu.instantiate(instance.constructor, instance.parameters) elif isinstance(instance_name_or_spec, dict): _name, spec = _parse_inline_spec(instance_name_or_spec) res = dtu.instantiate(spec.constructor, spec.parameters) else: assert False interface = dtu.import_name(family.interface) if not isinstance(res, interface): msg = f"I expected that {instance_name_or_spec!r} would be a {interface.__name__} but it is a " \ f"{type(res).__name__}." raise dtu.DTConfigException(msg) return res
def load_configuration_publisher(name, data): try: desc = data.pop("desc", None) desc = preprocess_desc(desc) latch = data.pop("latch", None) topic = data.pop("topic") type_ = data.pop("type") queue_size = data.pop("queue_size", None) except KeyError as e: msg = f"Could not find field {e!r}." raise dtu.DTConfigException(msg) if not isinstance(desc, (str, NoneType)): msg = f"Description should be a string, not {type(desc).__name__}." raise dtu.DTConfigException(msg) if data: msg = f"Extra keys: {data!r}" raise dtu.DTConfigException(msg) T = message_class_from_string(type_) return EasyNodePublisher(name=name, desc=desc, topic=topic, type=T, queue_size=queue_size, latch=latch)
def resolve(self, package_name: str, node_name: str, config_sequence: Union[list, tuple], date=None): """Returns a QueryResult""" if len(config_sequence) == 0: msg = f"Invalid empty config_sequence while querying for {package_name}/{node_name}" raise ValueError(msg) values = {} origin = {} origin_filename = {} if not package_name in self.package2nodes: msg = f'Could not find package "{package_name}"; I know {sorted(self.package2nodes)}.' raise dtu.DTConfigException(msg) nodes = self.package2nodes[package_name] if not node_name in nodes: msg = f'Could not find node "{node_name}" in package "{package_name}"; I know {sorted(nodes)}.' raise dtu.DTConfigException(msg) node_config = nodes[node_name] all_keys = list(node_config.parameters) overridden = defaultdict(lambda: []) using = [] for config_name in config_sequence: if config_name == "defaults": using.append(config_name) for p in list(node_config.parameters.values()): if p.has_default: values[p.name] = p.default origin_filename[p.name] = node_config.filename origin[p.name] = config_name else: c = self.find(package_name, node_name, config_name, date=date) if c is not None: using.append(config_name) for k, v in list(c.values.items()): if k in values: overridden[k].append(origin[k]) values[k] = v origin_filename[k] = c.filename origin[k] = config_name if not using: msg = ( f"Cannot find any configuration for {package_name}/{node_name} with config sequence " f"{':'.join(config_sequence)}") raise dtu.DTConfigException(msg) return QueryResult(package_name, node_name, config_sequence, all_keys, values, origin, origin_filename, overridden)
def load_configuration_parameter(name, data): # verbose: # desc: Whether the node is verbose or not. # type: bool # default: true # try: desc = data.pop("desc", None) desc = preprocess_desc(desc) type_ = data.pop("type") if "default" in data: default = data.pop("default") has_default = True else: default = DEFAULT_NOT_GIVEN has_default = False except KeyError as e: msg = f"Could not find field {e.args[0]!r}." raise dtu.DTConfigException(msg) if data: msg = f"Extra keys: {data!r}" raise dtu.DTConfigException(msg) if not isinstance(desc, (str, NoneType)): msg = f"Description should be a string, not {type(desc).__name__}." raise dtu.DTConfigException(msg) type2T = { "bool": bool, "str": str, "int": int, "float": float, "any": None, None: None, "dict": dict, } if not type_ in type2T: raise NotImplementedError(type_) T = type2T[type_] if has_default and default is not None and T is not None: default = T(default) return EasyNodeParameter(name=name, desc=desc, type=T, has_default=has_default, default=default)
def _init_parameters(self): parameters = self._configuration.parameters class Config: def __getattr__(self, name): if not name in parameters: msg = f"The user is trying to use {name!r}, which is not a parameter " msg += "for this node.\n" s = "\n- ".join(list(parameters)) msg += f"The declared parameters that can be used are:\n- {s}" raise AttributeError(msg) return object.__getattr__(self, name) self.config = Config() values = {} # load the configuration self.info("Loading parameters...") with dtu.rospy_timeit_wall("getting configuration files"): qr = get_user_configuration(self.package_name, self.node_type_name) if not qr.is_complete(): msg = "\nThe configuration that I could load is not complete:\n" msg += dtu.indent(str(qr), " | ") msg = dtu.indent( msg, f"{self.package_name} / {self.node_type_name} fatal error > ") raise dtu.DTConfigException(msg) self.info(f"Loaded configuration:\n{qr}") for p in list(parameters.values()): try: val = qr.values.get(p.name) except KeyError: msg = f"Could not load required parameter {p.name!r}." raise dtu.DTConfigException(msg) # write to parameter server, for transparency if val is not None: # we cannot set None parameters rospy.set_param("~" + p.name, val) setattr(self.config, p.name, val) values[p.name] = val self._on_parameters_changed(first_time=True, values=values) duration = self.config.en_update_params_interval duration = rospy.Duration.from_sec(duration) rospy.Timer(duration, self._update_parameters)
def load_configuration(realpath, contents) -> EasyNodeConfig: # TODO: load "version" string try: try: data = dtu.yaml_load(contents) except Exception as e: msg = "Could not parse YAML file properly:" dtu.raise_wrapped(dtu.DTConfigException, e, msg, compact=True) raise # ide not smart if not isinstance(data, dict): msg = f"Expected a dict, got {type(data).__name__}." raise dtu.DTConfigException(msg) try: parameters = data.pop("parameters") subscriptions = data.pop("subscriptions") publishers = data.pop("publishers") contracts = data.pop("contracts") description = data.pop("description") except KeyError as e: key = e.args[0] msg = f"Invalid configuration: missing field {key!r}." raise dtu.DTConfigException(msg) if not isinstance(description, (str, NoneType)): msg = f"Description should be a string, not {type(description).__name__}." raise dtu.DTConfigException(msg) if data: msg = f"Spurious fields found: {sorted(data)}" raise dtu.DTConfigException(msg) parameters = load_configuration_parameters(parameters) subscriptions = load_configuration_subscriptions(subscriptions) contracts = load_configuration_contracts(contracts) publishers = load_configuration_publishers(publishers) return EasyNodeConfig( filename=realpath, parameters=parameters, contracts=contracts, subscriptions=subscriptions, publishers=publishers, package_name=None, description=description, node_type_name=None, ) except dtu.DTConfigException as e: msg = f"Invalid configuration at {realpath}: " dtu.raise_wrapped(dtu.DTConfigException, e, msg, compact=True)
def get_config_sequence() -> Tuple[str, ...]: """ Reads the variable EasyNode.ENV it is taken as a colon-separated list of names. """ from easy_node.easy_node import EasyNode if not EasyNode.ENV in os.environ: msg = f'Could not find the environment variable "{EasyNode.ENV}".' raise dtu.DTConfigException(msg) s = os.environ[EasyNode.ENV] tokens = [_ for _ in s.split(":") if _.strip()] if not tokens: msg = f"The variable {EasyNode.ENV} is empty." raise dtu.DTConfigException(msg) return tuple(tokens)
def load_configuration_subscription(name, data): # image: # desc: Image to read # topic: ~image # type: CompressedImage # queue_size: 1 try: desc = data.pop("desc", None) desc = preprocess_desc(desc) latch = data.pop("latch", None) timeout = data.pop("timeout", None) topic = data.pop("topic") type_ = data.pop("type") queue_size = data.pop("queue_size", None) process = data.pop("process", PROCESS_SYNCHRONOUS) if not process in PROCESS_VALUES: msg = f"Invalid value of process {process!r} not in {PROCESS_VALUES!r}." raise dtu.DTConfigException(msg) except KeyError as e: msg = f"Could not find field {e!r}." raise dtu.DTConfigException(msg) if not isinstance(desc, (str, NoneType)): msg = f"Description should be a string, not {type(desc).__name__}." raise dtu.DTConfigException(msg) if data: msg = f"Extra keys: {data!r}" raise dtu.DTConfigException(msg) T = message_class_from_string(type_) return EasyNodeSubscription( name=name, desc=desc, topic=topic, timeout=timeout, type=T, queue_size=queue_size, latch=latch, process=process, )
def _parse_regression_test_check(line: str) -> Wrapper: line = line.strip() delim = " " tokens = line.split(delim) if len(tokens) != 3: msg = f'I expect exactly 3 tokens with delimiter {delim}.\nLine: "{line}"\nTokens: {tokens}' raise dtu.DTConfigException(msg) try: ref1 = parse_reference(tokens[0]) binary = parse_binary(tokens[1]) ref2 = parse_reference(tokens[2]) evaluable = BinaryEval(ref1, binary, ref2) except RTParseError as e: msg = f'Cannot parse string "{line}".' dtu.raise_wrapped(RTParseError, e, msg, compact=True) raise return Wrapper(evaluable)
def message_class_from_string(s): if not "/" in s: msg = "" msg += f'Invalid message name "{s}".\n' msg += 'I expected that the name of the message is in the format "PACKAGE/MSG".\n ' msg += 'E.g. "sensor_msgs/Joy" or "duckietown_msgs/BoolStamped".' raise dtu.DTConfigException(msg) # e.g. "std_msgs/Header" i = s.index("/") package = s[:i] name = s[i + 1:] symbol = f"{package}.msg.{name}" try: msgclass = dtu.import_name(symbol) return msgclass except ValueError as e: msg = f'Cannot import type for message "{s}" ({symbol}).' dtu.raise_wrapped(dtu.DTConfigException, e, msg, compact=True)
def load_configuration_package_node(package_name: str, node_type_name: str) -> EasyNodeConfig: path = dru.get_ros_package_path(package_name) look_for = f"{node_type_name}.easy_node.yaml" found = dtu.locate_files(path, look_for) if not found: msg = f"Could not find EasyNode configuration {look_for!r}." raise dtu.DTConfigException(msg) # XXX fn = found[0] contents = open(fn).read() c = load_configuration(fn, contents) c = c._replace(package_name=package_name) c = c._replace(node_type_name=node_type_name) # Add the common parameters if node_type_name != "easy_node": c0 = load_configuration_baseline() c = merge_configuration(c0, c) return c
def interpret_easy_algo_config(filename: str, data: dict) -> EasyAlgoFamily: """ Interprets the family config """ basename = os.path.basename(filename) family_name = dtu.id_from_basename_pattern(basename, EasyAlgoDB.pattern) instances_pattern = f"*.{family_name}.yaml" # tests_pattern = '*.%s_test.yaml' % family_name data0 = dict(data) try: dtu.dt_check_isinstance("contents", data, dict) description = data.pop("description") dtu.dt_check_isinstance("description", description, str) interface = data.pop("interface") dtu.dt_check_isinstance("interface", interface, str) locations = data.pop("locations", None) default_constructor = data.pop("default_constructor", None) except Exception as e: msg = f"Cannot interpret data: \n\n{data0}" raise Exception(msg) from e if data: msg = f'Extra keys in configuration: {list(data)}' raise dtu.DTConfigException(msg) return EasyAlgoFamily( interface=interface, family_name=family_name, filename=filename, instances=None, instances_pattern=instances_pattern, description=description, valid=True, locations=locations, default_constructor=default_constructor, error_if_invalid=False, )
def interpret_config_file(filename: str) -> ConfigInfo: """ Returns a ConfigInfo. """ try: basename = os.path.basename(filename) base = basename.replace(SUFFIX, "") # now we have something like # package-node.config_name.date # or # package-node.config_name if not "." in base: msg = f"Invalid filename {filename!r}." raise dtu.DTConfigException(msg) tokens = base.split(".") if len(tokens) > 3: msg = f"Too many periods/tokens (tokens={tokens})" raise dtu.DTConfigException(msg) if len(tokens) <= 2: # package-node.config_name package_node = tokens[0] if not "-" in package_node: msg = f'Expected a "-" in "{package_node}".' raise dtu.DTConfigException(msg) i = package_node.index("-") package_name = package_node[:i] node_name = package_node[i + 1 :] else: package_name = node_name = None # FIXME: should we bail? config_name = tokens[1] if len(tokens) == 3: # package-node.config_name.date date_effective = tokens[2] else: date_effective = "20170101" try: date_effective = parse(date_effective) except: msg = f'Cannot interpret "{date_effective}" as a date.' raise dtu.DTConfigException(msg) # now read file with open(filename) as f: contents = f.read() try: try: data = yaml.load(contents, Loader=yaml.Loader) except YAMLError as e: dtu.raise_wrapped(dtu.DTConfigException, e, "Invalid YAML", compact=True) raise if not isinstance(data, dict): msg = "Expected a dictionary inside." raise dtu.DTConfigException(msg) for field in ["description", "values"]: if not field in data: msg = f'Missing field "{field}".' raise dtu.DTConfigException(msg) description = data.pop("description") if not isinstance(description, str): msg = f'I expected that "description" is a string, obtained {description!r}.' raise dtu.DTConfigException(msg) extends = data.pop("extends", []) if not isinstance(extends, list): msg = f'I expected that "extends" is a list, obtained {extends!r}.' raise dtu.DTConfigException(msg) values = data.pop("values") if not isinstance(values, dict): msg = f'I expected that "values" is a dictionary, obtained {type(values)}.' raise dtu.DTConfigException(msg) # Freeze the data extends = tuple(extends) values = frozendict(values) except dtu.DTConfigException as e: msg = "Could not interpret the contents of the file\n" msg += f" {dtu.friendly_path(filename)}\n" msg += "Contents:\n" + dtu.indent(contents, " > ") dtu.raise_wrapped(dtu.DTConfigException, e, msg, compact=True) raise return ConfigInfo( filename=filename, package_name=package_name, node_name=node_name, config_name=config_name, date_effective=date_effective, extends=extends, description=description, values=values, # not decided valid=None, error_if_invalid=None, ) except dtu.DTConfigException as e: msg = f"Invalid file {dtu.friendly_path(filename)}" dtu.raise_wrapped(dtu.DTConfigException, e, msg, compact=True)
def load_family_config(all_yaml: dict) -> Dict[str, EasyAlgoFamily]: """ # now, for each family, we look for configuration files, which are: # # ![ID].![family_name].yaml # # """ if not isinstance(all_yaml, dict): msg = f"Expected a dict, got {type(all_yaml).__name__}" raise ValueError(msg) family_name2config = {} configs = dtu.look_everywhere_for_config_files2(EasyAlgoDB.pattern, all_yaml) configs.update( dtu.look_everywhere_for_config_files2("*.family.yaml", all_yaml)) for filename, contents in list(configs.items()): c: EasyAlgoFamily = dtu.interpret_yaml_file( filename, contents, interpret_easy_algo_config) if c.family_name in family_name2config: one = family_name2config[c.family_name].filename two = c.filename msg = f"Repeated filename:\n{one}\n{two}" raise dtu.DTConfigException(msg) def interpret_instance_spec(filename, data): dtu.dt_check_isinstance("data", data, dict) basename = os.path.basename(filename) instance_name = dtu.id_from_basename_pattern( basename, c.instances_pattern) if c.default_constructor is not None and not "constructor" in data: description = "(not given)" constructor = c.default_constructor parameters = dict(data) else: description = data.pop("description") dtu.dt_check_isinstance("description", description, str) constructor = data.pop("constructor") dtu.dt_check_isinstance("constructor", constructor, str) parameters = data.pop("parameters") dtu.dt_check_isinstance("parameters", parameters, (dict, NoneType)) if parameters is None: parameters = {} return EasyAlgoInstance( family_name=c.family_name, instance_name=instance_name, description=description, filename=filename, constructor=constructor, parameters=parameters, valid=True, error_if_invalid=None, ) c = check_validity_family(c) instances = {} _ = dtu.look_everywhere_for_config_files2(c.instances_pattern, all_yaml) # noinspection PyAssignmentToLoopOrWithParameter for filename, contents in list(_.items()): i = dtu.interpret_yaml_file(filename, contents, interpret_instance_spec, plain_yaml=True) if i.instance_name in instances: one = instances[i.instance_name].filename two = i.filename msg = f"Repeated filename:\n{one}\n{two}" raise dtu.DTConfigException(msg) # i = check_validity_instance(c, i) instances[i.instance_name] = i c = c._replace(instances=instances) family_name2config[c.family_name] = c return family_name2config