def create_instance(self, family_name, instance_name_or_spec):
dtu.check_isinstance(instance_name_or_spec, (dict, str))
family = self.get_family(family_name)
if not family.valid:
msg = ('Cannot instantiate %r because its family %r is invalid.' %
(instance_name_or_spec, family_name))
msg += '\n\n' + dtu.indent(family.error_if_invalid, " > ")
raise dtu.DTConfigException(msg)
if isinstance(instance_name_or_spec, str):
instance_name = instance_name_or_spec
dtu.check_is_in('instance', instance_name, family.instances)
instance = family.instances[instance_name]
if not instance.valid:
msg = ('Cannot instantiate %r because it is invalid:\n%s' %
(instance_name,
dtu.indent(instance.error_if_invalid, '> ')))
raise dtu.DTConfigException(msg)
res = dtu.instantiate(instance.constructor, instance.parameters)
elif isinstance(instance_name_or_spec, dict):
_name, spec = _parse_inline_spec(instance_name_or_spec)
res = dtu.instantiate(spec.constructor, spec.parameters)
else:
assert False
interface = dtu.import_name(family.interface)
if not isinstance(res, interface):
msg = ('I expected that %r would be a %s but it is a %s.' %
(instance_name, interface.__name__, type(res).__name__))
raise dtu.DTConfigException(msg)
return res
def load_configuration_publisher(name, data):
try:
desc = data.pop('desc', None)
desc = preprocess_desc(desc)
latch = data.pop('latch', None)
topic = data.pop('topic')
type_ = data.pop('type')
queue_size = data.pop('queue_size', None)
except KeyError as e:
msg = 'Could not find field %r.' % e
raise dtu.DTConfigException(msg)
if not isinstance(desc, (str, NoneType)):
msg = 'Description should be a string, not %s.' % type(desc).__name__
raise dtu.DTConfigException(msg)
if data:
msg = 'Extra keys: %r' % data
raise dtu.DTConfigException(msg)
T = message_class_from_string(type_)
return EasyNodePublisher(name=name,
desc=desc,
topic=topic,
type=T,
queue_size=queue_size,
latch=latch)
def resolve(self, package_name, node_name, config_sequence, date=None):
""" Returns a QueryResult """
if len(config_sequence) == 0:
msg = 'Invalid empty config_sequence while querying for %s/%s' % (
package_name, node_name)
raise ValueError(msg)
values = {}
origin = {}
origin_filename = {}
if not package_name in self.package2nodes:
msg = ('Could not find package "%s"; I know %s.' %
(package_name, sorted(self.package2nodes)))
raise dtu.DTConfigException(msg)
nodes = self.package2nodes[package_name]
if not node_name in nodes:
msg = ('Could not find node "%s" in package "%s"; I know %s.' %
(node_name, package_name, sorted(nodes)))
raise dtu.DTConfigException(msg)
node_config = nodes[node_name]
all_keys = list(node_config.parameters)
overridden = defaultdict(lambda: [])
using = []
for config_name in config_sequence:
if config_name == 'defaults':
using.append(config_name)
for p in node_config.parameters.values():
if p.has_default:
values[p.name] = p.default
origin_filename[p.name] = node_config.filename
origin[p.name] = config_name
else:
c = self.find(package_name, node_name, config_name, date=date)
if c is not None:
using.append(config_name)
for k, v in c.values.items():
if k in values:
overridden[k].append(origin[k])
values[k] = v
origin_filename[k] = c.filename
origin[k] = config_name
if not using:
msg = (
'Cannot find any configuration for %s/%s with config sequence %s'
% (package_name, node_name, ":".join(config_sequence)))
raise dtu.DTConfigException(msg)
return QueryResult(package_name, node_name, config_sequence, all_keys,
values, origin, origin_filename, overridden)
def load_configuration_parameter(name, data):
# verbose:
# desc: Whether the node is verbose or not.
# type: bool
# default: true
#
try:
desc = data.pop('desc', None)
desc = preprocess_desc(desc)
type_ = data.pop('type')
if 'default' in data:
default = data.pop('default')
has_default = True
else:
default = DEFAULT_NOT_GIVEN
has_default = False
except KeyError as e:
msg = 'Could not find field %r.' % e.args[0]
raise dtu.DTConfigException(msg)
if data:
msg = 'Extra keys: %r' % data
raise dtu.DTConfigException(msg)
if not isinstance(desc, (str, NoneType)):
msg = 'Description should be a string, not %s.' % type(desc).__name__
raise dtu.DTConfigException(msg)
type2T = {
'bool': bool,
'str': str,
'int': int,
'float': float,
'any': None,
None: None,
'dict': dict,
}
if not type_ in type2T:
raise NotImplementedError(type_)
T = type2T[type_]
if has_default and default is not None and T is not None:
default = T(default)
return EasyNodeParameter(name=name,
desc=desc,
type=T,
has_default=has_default,
default=default)
def _init_parameters(self):
parameters = self._configuration.parameters
class Config():
def __getattr__(self, name):
if not name in parameters:
msg = 'The user is trying to use %r, which is not a parameter ' % name
msg += 'for this node.\n'
msg += 'The declared parameters that can be used are:\n- %s' % "\n- ".join(
list(parameters))
raise AttributeError(msg)
return object.__getattr__(self, name)
self.config = Config()
values = {}
# load the configuration
self.info('Loading parameters...')
with dtu.rospy_timeit_wall('getting configuration files'):
qr = get_user_configuration(self.package_name, self.node_type_name)
if not qr.is_complete():
msg = '\nThe configuration that I could load is not complete:\n'
msg += dtu.indent(str(qr), ' | ')
msg = dtu.indent(
msg, '%s / %s fatal error > ' %
(self.package_name, self.node_type_name))
raise dtu.DTConfigException(msg)
self.info('Loaded configuration:\n%s' % qr)
for p in parameters.values():
try:
val = qr.values.get(p.name) # @UndefinedVariable
except KeyError:
msg = 'Could not load required parameter %r.' % p.name
raise dtu.DTConfigException(msg)
# write to parameter server, for transparency
if val is not None: # we cannot set None parameters
rospy.set_param('~' + p.name, val) # @UndefinedVariable
setattr(self.config, p.name, val)
values[p.name] = val
self._on_parameters_changed(first_time=True, values=values)
duration = self.config.en_update_params_interval
duration = rospy.Duration.from_sec(duration) # @UndefinedVariable
rospy.Timer(duration, self._update_parameters) # @UndefinedVariable
def interpret_easy_algo_config(filename, data):
""" Interprets the family config """
basename = os.path.basename(filename)
family_name = dtu.id_from_basename_pattern(basename, EasyAlgoDB.pattern)
instances_pattern = '*.%s.yaml' % family_name
# tests_pattern = '*.%s_test.yaml' % family_name
dtu.dt_check_isinstance('contents', data, dict)
description = data.pop('description')
dtu.dt_check_isinstance('description', description, str)
interface = data.pop('interface')
dtu.dt_check_isinstance('interface', interface, str)
locations = data.pop('locations', None)
default_constructor = data.pop('default_constructor', None)
if data:
msg = 'Extra keys in configuration: %s' % list(data)
raise dtu.DTConfigException(msg)
return EasyAlgoFamily(interface=interface,
family_name=family_name,
filename=filename,
instances=None,
instances_pattern=instances_pattern,
description=description,
valid=True,
locations=locations,
default_constructor=default_constructor,
error_if_invalid=False)
def get_images():
found = dtu.locate_files(dtu.get_duckiefleet_root(), '*.jpg')
basename2filename = dict((os.path.basename(_), _) for _ in found)
if not nopic in basename2filename:
msg = 'I expect a file %r to represent missing pictures.' % nopic
raise dtu.DTConfigException(msg)
return basename2filename
def load_configuration(realpath, contents):
# TODO: load "version" string
try:
try:
data = dtu.yaml_load(contents)
except Exception as e:
msg = 'Could not parse YAML file properly:'
dtu.raise_wrapped(dtu.DTConfigException, e, msg, compact=True)
if not isinstance(data, dict):
msg = 'Expected a dict, got %s.' % type(data).__name__
raise dtu.DTConfigException(msg)
try:
parameters = data.pop('parameters')
subscriptions = data.pop('subscriptions')
publishers = data.pop('publishers')
contracts = data.pop('contracts')
description = data.pop('description')
except KeyError as e:
key = e.args[0]
msg = 'Invalid configuration: missing field %r.' % (key)
raise dtu.DTConfigException(msg)
if not isinstance(description, (str, NoneType)):
msg = 'Description should be a string, not %s.' % type(
description).__name__
raise dtu.DTConfigException(msg)
if data:
msg = 'Spurious fields found: %s' % sorted(data)
raise dtu.DTConfigException(msg)
parameters = load_configuration_parameters(parameters)
subscriptions = load_configuration_subscriptions(subscriptions)
contracts = load_configuration_contracts(contracts)
publishers = load_configuration_publishers(publishers)
return EasyNodeConfig(filename=realpath,
parameters=parameters,
contracts=contracts,
subscriptions=subscriptions,
publishers=publishers,
package_name=None,
description=description,
node_type_name=None)
except dtu.DTConfigException as e:
msg = 'Invalid configuration at %s: ' % realpath
dtu.raise_wrapped(dtu.DTConfigException, e, msg, compact=True)
def get_config_sequence():
""" Reads the variable EasyNode.ENV
it is taken as a colon-separated list of names.
"""
from easy_node.easy_node import EasyNode
if not EasyNode.ENV in os.environ:
msg = 'Could not find the environment variable "%s".' % EasyNode.ENV
raise dtu.DTConfigException(msg)
s = os.environ[EasyNode.ENV]
tokens = [_ for _ in s.split(':') if _.strip()]
if not tokens:
msg = 'The variable %s is empty.' % EasyNode.ENV
raise dtu.DTConfigException(msg)
return tuple(tokens)
def load_configuration_subscription(name, data):
# image:
# desc: Image to read
# topic: ~image
# type: CompressedImage
# queue_size: 1
try:
desc = data.pop('desc', None)
desc = preprocess_desc(desc)
latch = data.pop('latch', None)
timeout = data.pop('timeout', None)
topic = data.pop('topic')
type_ = data.pop('type')
queue_size = data.pop('queue_size', None)
process = data.pop('process', PROCESS_SYNCHRONOUS)
if not process in PROCESS_VALUES:
msg = 'Invalid value of process %r not in %r.' % (process,
PROCESS_VALUES)
raise dtu.DTConfigException(msg)
except KeyError as e:
msg = 'Could not find field %r.' % e
raise dtu.DTConfigException(msg)
if not isinstance(desc, (str, NoneType)):
msg = 'Description should be a string, not %s.' % type(desc).__name__
raise dtu.DTConfigException(msg)
if data:
msg = 'Extra keys: %r' % data
raise dtu.DTConfigException(msg)
T = message_class_from_string(type_)
return EasyNodeSubscription(name=name,
desc=desc,
topic=topic,
timeout=timeout,
type=T,
queue_size=queue_size,
latch=latch,
process=process)
def message_class_from_string(s):
if not '/' in s:
msg = ''
msg += 'Invalid message name "%s".\n' % s
msg += 'I expected that the name of the message is in the format "PACKAGE/MSG".\n '
msg += 'E.g. "sensor_msgs/Joy" or "duckietown_msgs/BoolStamped".'
raise dtu.DTConfigException(msg)
# e.g. "std_msgs/Header"
i = s.index('/')
package = s[:i]
name = s[i + 1:]
symbol = '%s.msg.%s' % (package, name)
try:
msgclass = dtu.import_name(symbol)
return msgclass
except ValueError as e:
msg = 'Cannot import type for message "%s" (%s).' % (s, symbol)
dtu.raise_wrapped(dtu.DTConfigException, e, msg, compact=True)
def _parse_regression_test_check(line):
line = line.strip()
delim = ' '
tokens = line.split(delim)
if len(tokens) != 3:
msg = 'I expect exactly 3 tokens with delimiter %s.\nLine: "%s"\nTokens: %s' % (
delim, line, tokens)
raise dtu.DTConfigException(msg)
try:
ref1 = parse_reference(tokens[0])
binary = parse_binary(tokens[1])
ref2 = parse_reference(tokens[2])
evaluable = BinaryEval(ref1, binary, ref2)
except RTParseError as e:
msg = 'Cannot parse string "%s".' % line
dtu.raise_wrapped(RTParseError, e, msg, compact=True)
return Wrapper(evaluable)
def load_configuration_package_node(package_name, node_type_name):
path = dtu.get_ros_package_path(package_name)
look_for = '%s.easy_node.yaml' % node_type_name
found = dtu.locate_files(path, look_for)
if not found:
msg = 'Could not find EasyNode configuration %r.' % look_for
raise dtu.DTConfigException(msg) # XXX
fn = found[0]
contents = open(fn).read()
c = load_configuration(fn, contents)
c = c._replace(package_name=package_name)
c = c._replace(node_type_name=node_type_name)
# Add the common parameters
if node_type_name != 'easy_node':
c0 = load_configuration_baseline()
c = merge_configuration(c0, c)
return c
def load_family_config(all_yaml):
"""
# now, for each family, we look for configuration files, which are:
#
# ![ID].![family_name].yaml
#
#
"""
if not isinstance(all_yaml, dict):
msg = 'Expected a dict, got %s' % type(all_yaml).__name__
raise ValueError(msg)
family_name2config = {}
configs = dtu.look_everywhere_for_config_files2(EasyAlgoDB.pattern,
all_yaml)
configs.update(
dtu.look_everywhere_for_config_files2("*.family.yaml", all_yaml))
for filename, contents in configs.items():
c = dtu.interpret_yaml_file(filename, contents,
interpret_easy_algo_config)
if c.family_name in family_name2config:
one = family_name2config[c.family_name].filename
two = c.filename
msg = 'Repeated filename:\n%s\n%s' % (one, two)
raise dtu.DTConfigException(msg)
def interpret_instance_spec(filename, data):
dtu.dt_check_isinstance('data', data, dict)
basename = os.path.basename(filename)
instance_name = dtu.id_from_basename_pattern(
basename, c.instances_pattern)
if c.default_constructor is not None and not 'constructor' in data:
description = '(not given)'
constructor = c.default_constructor
parameters = OrderedDict(data)
else:
description = data.pop('description')
dtu.dt_check_isinstance('description', description, str)
constructor = data.pop('constructor')
dtu.dt_check_isinstance('constructor', constructor, str)
parameters = data.pop('parameters')
dtu.dt_check_isinstance('parameters', parameters,
(dict, NoneType))
if parameters is None: parameters = {}
return EasyAlgoInstance(family_name=c.family_name,
instance_name=instance_name,
description=description,
filename=filename,
constructor=constructor,
parameters=parameters,
valid=True,
error_if_invalid=None)
c = check_validity_family(c)
instances = {}
_ = dtu.look_everywhere_for_config_files2(c.instances_pattern,
all_yaml)
for filename, contents in _.items():
i = dtu.interpret_yaml_file(filename,
contents,
interpret_instance_spec,
plain_yaml=True)
if i.instance_name in instances:
one = instances[i.instance_name].filename
two = i.filename
msg = 'Repeated filename:\n%s\n%s' % (one, two)
raise dtu.DTConfigException(msg)
# i = check_validity_instance(c, i)
instances[i.instance_name] = i
c = c._replace(instances=instances)
family_name2config[c.family_name] = c
return family_name2config
def interpret_config_file(filename):
"""
Returns a ConfigInfo.
"""
try:
basename = os.path.basename(filename)
base = basename.replace(SUFFIX, '')
# now we have something like
# package-node.config_name.date
# or
# package-node.config_name
if not '.' in base:
msg = 'Invalid filename %r.' % filename
raise dtu.DTConfigException(msg)
tokens = base.split('.')
if len(tokens) > 3:
msg = 'Too many periods/tokens (tokens=%s)' % tokens
raise dtu.DTConfigException(msg)
if len(tokens) <= 2:
# package-node.config_name
package_node = tokens[0]
if not '-' in package_node:
msg = 'Expected a "-" in "%s".' % package_node
raise dtu.DTConfigException(msg)
i = package_node.index('-')
package_name = package_node[:i]
node_name = package_node[i + 1:]
config_name = tokens[1]
if len(tokens) == 3:
# package-node.config_name.date
date_effective = tokens[2]
else:
date_effective = '20170101'
from dateutil.parser import parse
try:
date_effective = parse(date_effective)
except:
msg = 'Cannot interpret "%s" as a date.' % date_effective
raise dtu.DTConfigException(msg)
# now read file
contents = open(filename).read()
try:
try:
data = yaml.load(contents)
except YAMLError as e:
dtu.raise_wrapped(dtu.DTConfigException,
e,
'Invalid YAML',
compact=True)
if not isinstance(data, dict):
msg = 'Expected a dictionary inside.'
raise dtu.DTConfigException(msg)
for field in ['description', 'values']:
if not field in data:
msg = 'Missing field "%s".' % field
raise dtu.DTConfigException(msg)
description = data.pop('description')
if not isinstance(description, str):
msg = 'I expected that "description" is a string, obtained %r.' % description
raise dtu.DTConfigException(msg)
extends = data.pop('extends', [])
if not isinstance(extends, list):
msg = 'I expected that "extends" is a list, obtained %r.' % extends
raise dtu.DTConfigException(msg)
values = data.pop('values')
if not isinstance(values, dict):
msg = 'I expected that "values" is a dictionary, obtained %s.' % type(
values)
raise dtu.DTConfigException(msg)
# Freeze the data
extends = tuple(extends)
values = frozendict(values)
except dtu.DTConfigException as e:
msg = 'Could not interpret the contents of the file\n'
msg += ' %s\n' % dtu.friendly_path(filename)
msg += 'Contents:\n' + dtu.indent(contents, ' > ')
dtu.raise_wrapped(dtu.DTConfigException, e, msg, compact=True)
return ConfigInfo(
filename=filename,
package_name=package_name,
node_name=node_name,
config_name=config_name,
date_effective=date_effective,
extends=extends,
description=description,
values=values,
# not decided
valid=None,
error_if_invalid=None)
except dtu.DTConfigException as e:
msg = 'Invalid file %s' % dtu.friendly_path(filename)
dtu.raise_wrapped(dtu.DTConfigException, e, msg, compact=True)
