def _load_next(self):
try:
try:
res = self.iterator.next()
if not isinstance(res, tuple):
msg = "Expected tuple (signal, timestamp, value), obtained %s" % describe_type(res)
raise ValueError(msg)
if not len(res) == 3:
raise ValueError("Required len 3 tuple; obtained %d." % len(res))
signal, timestamp, value = res
except StopIteration:
raise
except Exception as e:
msg = "Could not call next() on user-given iterator.\n"
msg += " iterator: %s\n" % str(self.iterator)
msg += " of type: %s\n" % describe_type(self.iterator)
msg += "because of this error:\n"
msg += indent(string.strip("%s\n%s" % (e, traceback.format_exc(e))), "| ")
raise ModelExecutionError(msg, self)
if not isinstance(signal, (str, int)):
msg = "Expected a string or number for the signal, got %s" % describe_value(signal)
raise ValueError(msg)
if not isinstance(timestamp, float):
msg = "Expected a number for the timestamp, got %s" % describe_value(timestamp)
raise ValueError(msg)
self.next_signal = signal
self.next_timestamp = timestamp
self.next_value = value
self.has_next = True
except StopIteration:
self.has_next = False
def __init__(self, nid=None, children=None, caption=None):
check_isinstance(nid, (type(None),) + six.string_types)
check_isinstance(caption, (type(None), six.text_type))
if children is not None and not isinstance(children, list):
raise ValueError(
"Received a %s object as children list, should"
" be None or list." % describe_type(children)
)
self.nid = nid
if children is None:
children = []
self.childid2node = {}
self.children = []
for c in children:
if not isinstance(c, Node):
msg = "Expected Node, got %s." % describe_type(c)
raise ValueError(msg)
self.add_child(c)
self.parent = None
self.caption = caption
# used by subsection(), set_subsections_needed()
self._subsections_needed = None
# Reference to a figure that was created automatically
# (see code in add_to_autofigure)
self._auto_figure = None
def __init__(self, nid=None, children=None, caption=None):
if children is not None and not isinstance(children, list):
raise ValueError('Received a %s object as children list, should'
' be None or list.' % describe_type(children))
self.nid = nid
if children is None:
children = []
self.childid2node = {}
self.children = []
for c in children:
if not isinstance(c, Node):
msg = 'Expected Node, got %s.' % describe_type(c)
raise ValueError(msg)
self.add_child(c)
self.parent = None
self.caption = caption
# used by subsection(), set_subsections_needed()
self._subsections_needed = None
# Reference to a figure that was created automatically
# (see code in add_to_autofigure)
self._auto_figure = None
def __init__(self, obs_spec, cmd_spec, id_robot=None,
desc=None, extra=None):
self.id_robot = id_robot
self.desc = desc
self.extra = extra
if not isinstance(obs_spec, StreamSpec):
msg = ('Expected StreamSpec for observations, not %s.' %
describe_type(obs_spec))
raise ValueError(msg)
if not isinstance(cmd_spec, StreamSpec):
msg = ('Expected StreamSpec for commands, not %s.' %
describe_type(cmd_spec))
raise ValueError(msg)
self._observations = obs_spec
self._commands = cmd_spec
def save(self, obj):
desc = 'object of type %s' % (describe_type(obj))
# , describe_value(obj, 100))
# self.stack.append(describe_value(obj, 120))
self.stack.append(desc)
Pickler.save(self, obj)
self.stack.pop()
def find_pickling_error(obj, protocol=pickle.HIGHEST_PROTOCOL):
sio = StringIO()
try:
pickle.dumps(obj)
except Exception:
msg_old = '\n --- Old exception----\n%s' % traceback.format_exc()
else:
msg_old = ''
msg = ('Strange! I could not reproduce the pickling error '
'for the object of class %s' % describe_type(obj))
logger.info(msg)
pickler = MyPickler(sio, protocol)
try:
pickler.dump(obj)
except Exception as e1:
msg = pickler.get_stack_description()
msg += '\n --- Current exception----\n%s' % traceback.format_exc()
msg += msg_old
return msg
else:
msg = 'I could not find the exact pickling error.'
raise Exception(msg)
def do_convert_job2(id_robot,
id_robot_res,
id_explog, id_stream, id_episode,
filename):
rs2b_config = get_rs2b_config()
boot_config = get_boot_config()
"""
id_robot: original robot must be a ROSRobot
"""
if os.path.exists(filename):
msg = 'Output file exists: %s\n' % friendly_path(filename)
msg += 'Delete to force recreating the log.'
logger.info(msg)
return
explog = rs2b_config.explogs.instance(id_explog)
robot = boot_config.robots.instance(id_robot)
orig_robot = robot.get_inner_components()[-1]
if not isinstance(orig_robot, ROSRobot):
msg = 'Expected ROSRobot, got %s' % describe_type(robot)
raise ValueError(msg)
orig_robot.read_from_log(explog)
id_environment = explog.get_id_environment()
write_robot_observations(id_stream, filename, id_robot_res,
robot, id_episode, id_environment)
def formatm(*args, **kwargs):
#name_len = 10
assert len(args) > 0
assert len(args) % 2 == 0
cols = []
for i in range(len(args) / 2):
name = args[i * 2]
matrix = args[i * 2 + 1]
if not isinstance(name, str):
raise ValueError('I expect a string for label, not %s.'
% describe_type(name))
# varname = ' %s:' % rjust(name, name_len)
varname = ' %s:' % name
if isinstance(matrix, np.ndarray):
# raise ValueError('I expect a numpy array for value, not %s.' %
# describe_type(matrix))
value = format_matrix(matrix, **kwargs)
if matrix.ndim > 1:
varname = '\n' + varname
else:
value = describe_value(matrix)
cols.append(varname)
cols.append(value)
cols = add_spacer(cols)
return join_columns(cols)
def format_obs(d):
""" Shows objects values and typed for the given dictionary """
lines = []
for name, value in d.items():
lines.append('%15s: %s' % (name, describe_value(value)))
lines.append('%15s %s' % ('of type', describe_type(value)))
return '\n'.join(lines)
def resolve_url(self, url, already_visited=None):
if not isinstance(url, six.string_types):
raise ValueError(describe_type(url))
if already_visited is None:
already_visited = [self]
else:
if self in already_visited:
raise NotExistent(url)
already_visited += [self]
if url.find("..") >= 0:
# if it starts with .. we assume it's a precise url
# and we don't do anything smart to find it
return self.resolve_url_dumb(url)
try:
return self.resolve_url_dumb(url)
except NotExistent:
pass
for child in self.children:
try:
return child.resolve_url(url, already_visited)
except NotExistent:
pass
if self.parent:
return self.parent.resolve_url(url, already_visited)
else:
# in the end, we get here
# raise again the error
return self.resolve_url_dumb(url)
def init_iterator(self):
id_robot = self.config.id_robot
bagfile = self.config.file
boot_config = get_boot_config()
robot = boot_config.robots.instance(id_robot)
warnings.warn('Make this more general')
if isinstance(robot, EquivRobot):
orig_robot = robot.get_original_robot()
else:
orig_robot = robot
if not isinstance(orig_robot, ROSRobot):
msg = 'Expected ROSRobot, got %s' % describe_type(robot)
raise ValueError(msg)
orig_robot.read_from_bag(bagfile)
bd_seq = bd_sequence_from_robot(id_robot, robot, sleep_wait=0,
id_episode='xxx', id_environment='xxx',
check_valid_values=False)
def make_boot_iterator():
for bd in bd_seq:
yield 0, bd['timestamp'].item(), bd
return make_boot_iterator()
def instance_dds(config, id_symdds):
dds = config.symdds.instance(id_symdds) # @UndefinedVariable
if not isinstance(dds, DiffeoSystem):
msg = 'I expect to find a DiffeoSystem, not %r' % describe_type(dds)
raise ValueError(msg)
dds.label = id_symdds
return dds
def formatm(*args, **kwargs):
# name_len = 10
assert len(args) > 0
assert len(args) % 2 == 0
cols = []
for i in range(int(len(args) / 2)):
name = args[i * 2]
matrix = args[i * 2 + 1]
if not isinstance(name, str):
raise ValueError('I expect a string for label, not %s.'
% describe_type(name))
# varname = ' %s:' % rjust(name, name_len)
varname = ' %s:' % name
if isinstance(matrix, np.ndarray):
# raise ValueError('I expect a numpy array for value, not %s.' %
# describe_type(matrix))
value = format_matrix(matrix, **kwargs)
if matrix.ndim > 1:
varname = '\n' + varname
else:
value = describe_value(matrix)
cols.append(varname)
cols.append(value)
cols = add_spacer(cols)
return join_columns(cols)
def enumerate_entries_from_file(filename):
''' Yields (filename, num_entry), entry '''
with open(filename) as f:
try:
parsed = yaml.load(f)
except YAMLError as e:
msg = 'Cannot parse YAML file %s:\n%s' % (friendly_path(filename), e)
raise SyntaxMistake(msg) # TODO: make UserError
if parsed is None:
logger.warning('Found an empty file %r.' % friendly_path(filename))
else:
if not isinstance(parsed, list):
msg = ('Expect the file %r to contain a list of dicts,'
' found %s.' % (friendly_path(filename),
describe_type(parsed)))
raise SyntaxMistake(msg)
if not all([isinstance(x, dict) for x in parsed]):
msg = ('Expect the file %r to contain a list of dicts.' %
filename)
raise SyntaxMistake(msg)
if not parsed:
logger.warning('Found an empty file %r.' % friendly_path(filename))
for num_entry, entry in enumerate(parsed):
yield (filename, num_entry), entry
def instance_dds(config, id_symdds):
dds = config.symdds.instance(id_symdds) #@UndefinedVariable
if not isinstance(dds, DiffeoSystem):
msg = 'I expect to find a DiffeoSystem, not %r' % describe_type(dds)
raise ValueError(msg)
dds.label = id_symdds
return dds
def write_report_and_update(report, report_nid, report_html, all_reports, index_filename,
this_report,
other_reports_same_type,
most_similar_other_type,
static_dir,
write_pickle=False):
if not isinstance(report, Report):
msg = 'Expected Report, got %s.' % describe_type(report)
raise ValueError(msg)
links = create_links_html(this_report, other_reports_same_type, index_filename,
most_similar_other_type=most_similar_other_type)
tree_html = '<pre style="display:none">%s</pre>' % report.format_tree()
extras = dict(extra_html_body_start=links,
extra_html_body_end=tree_html)
report.nid = report_nid
html = write_report(report=report,
report_html=report_html,
static_dir=static_dir,
write_pickle=write_pickle, **extras)
index_reports(reports=all_reports, index=index_filename, update=html)
def __repr__(self):
return "CacheResult(size=%s,clock=%s,wall=%s,value=%s)" % (
memstring(self.size),
timestring(self.clock),
timestring(self.wall),
describe_type(self.value),
)
def resolve_url(self, url, already_visited=None):
if not isinstance(url, str):
raise ValueError(describe_type(url))
if already_visited is None:
already_visited = [self]
else:
if self in already_visited:
raise NotExistent(url)
already_visited += [self]
if url.find('..') >= 0:
# if it starts with .. we assume it's a precise url
# and we don't do anything smart to find it
return self.resolve_url_dumb(url)
try:
return self.resolve_url_dumb(url)
except NotExistent:
pass
for child in self.children:
try:
return child.resolve_url(url, already_visited)
except NotExistent:
pass
if self.parent:
return self.parent.resolve_url(url, already_visited)
else:
# in the end, we get here
# raise again the error
return self.resolve_url_dumb(url)
def save(self, obj):
desc = 'object of type %s' % (describe_type(obj))
# , describe_value(obj, 100))
# self.stack.append(describe_value(obj, 120))
self.stack.append(desc)
Pickler.save(self, obj)
self.stack.pop()
def predict_all_streams(streams, predictor, skip_initial=5):
''' yields dict with fields "data", "predict_y" '''
for stream in streams:
last_observations = None
for observations in stream.read(read_extra=False):
if observations['counter'] > skip_initial:
predict_y = predictor.predict_y(dt=observations['dt'])
if not isinstance(predict_y, np.ndarray):
msg = 'Want array, got %s' % describe_type(predict_y)
raise Exception(msg)
expected = observations['observations'].shape
found = predict_y.shape
if expected != found:
msg = 'Want shape %s, got %s.' % (expected, found)
raise Exception(msg)
est_u = predictor.estimate_u()
yield dict(prev=last_observations,
data=observations,
predict_y=predict_y,
est_u=est_u)
predictor.process_observations(observations)
last_observations = observations
def find_pickling_error(obj, protocol=pickle.HIGHEST_PROTOCOL):
sio = StringIO()
try:
pickle.dumps(obj)
except Exception:
msg_old = '\n --- Old exception----\n%s' % traceback.format_exc()
else:
msg_old = ''
msg = ('Strange! I could not reproduce the pickling error '
'for the object of class %s' % describe_type(obj))
logger.info(msg)
pickler = MyPickler(sio, protocol)
try:
pickler.dump(obj)
except Exception as e1:
msg = pickler.get_stack_description()
msg += '\n --- Current exception----\n%s' % traceback.format_exc()
msg += msg_old
return msg
else:
msg = 'I could not find the exact pickling error.'
raise Exception(msg)
def make_relative(val, dirname):
if not isinstance(val, str):
msg = ('Only strings can be used as keys, not %s.'
% describe_type(val))
raise SyntaxMistake(msg)
joined = os.path.join(dirname, val)
nval = os.path.realpath(joined)
return nval
def check_necessary(x, necessary):
for f, types in necessary:
if not f in x:
raise BadConfig(x, 'Field %r missing.' % f)
if not isinstance(x[f], types):
msg = ('Field %r must be one of %s, instead of %s.' %
(f, types, describe_type(x[f])))
raise BadConfig(x, msg)
def set_resource(self, goal, rtype, **params):
key = dict(rtype=rtype, **params)
if not isinstance(goal, Promise):
msg = 'Warning, resource %s' % key
msg += ' has type %s' % describe_type(goal)
# logger.error(msg)
raise ValueError(msg)
self.allresources[key] = goal
def load_extra(extra_table, index):
extra_string = str(extra_table[index])
extra_string = zlib.decompress(extra_string)
extra = yaml_load(extra_string)
if not isinstance(extra, dict):
msg = ('Expected to deserialize a dict, obtained %r'
% describe_type(extra))
raise Exception(msg)
return extra
def check_type(self, x):
expected = self.ptype
if self.ptype == float:
expected = (float, int)
if not isinstance(x, expected):
msg = ("For param %r, expected %s, got %s.\n%s" %
(self.name, self.ptype, describe_type(x),
describe_value(x)))
raise DecentParamsSemanticError(self.params, self, msg)
def set_resource(self, goal, rtype, **params):
key = dict(rtype=rtype, **params)
if not isinstance(goal, Promise):
msg = 'Warning, resource did not return a Compmake Promise.'
msg += '\n key: %s' % key
msg += '\n type: %s' % describe_type(goal)
# logger.error(msg)
raise ValueError(msg)
self.allresources[key] = goal
def safe_pickle_dump(value, filename, protocol=pickle.HIGHEST_PROTOCOL, **safe_write_options):
with safe_write(filename, **safe_write_options) as f:
try:
pickle.dump(value, f, protocol)
except Exception:
msg = 'Cannot pickle object of class %s' % describe_type(value)
logger.error(msg)
msg = find_pickling_error(value, protocol)
logger.error(msg)
raise
def __init__(self, robot, obs_nuisance=[], cmd_nuisance=[]):
self.inner_robot_name = robot
boot_config = get_boot_config()
id_robot, self.robot = boot_config.robots.instance_smarter(robot)
if not isinstance(self.robot, RobotInterface):
msg = 'Expected RobotInterface, got %s' % describe_type(self.robot)
raise ValueError(msg)
warnings.warn('handle the case better')
self.desc = ('EquivRobot(%s,obs:%s,cmd:%s)'
% (id_robot, obs_nuisance, cmd_nuisance))
# convert to (possibly empty) list of strings
if isinstance(obs_nuisance, str):
obs_nuisance = [obs_nuisance]
if isinstance(cmd_nuisance, str):
cmd_nuisance = [cmd_nuisance]
instance = lambda y: boot_config.nuisances.instance_smarter(y)[1]
self.obs_nuisances = [instance(x) for x in obs_nuisance]
self.cmd_nuisances = [instance(x) for x in cmd_nuisance]
# No - we should not call inverse() before transform_spec()
obs_spec = self.robot.get_spec().get_observations()
for n in self.obs_nuisances:
obs_spec = n.transform_spec(obs_spec)
cmd_spec = self.robot.get_spec().get_commands()
for n in self.cmd_nuisances:
cmd_spec = n.transform_spec(cmd_spec)
# We don't really need to compute this...
try:
self.cmd_nuisances_inv = [x.inverse() for x in self.cmd_nuisances]
# now initialize in reverse
cmd_spec_i = cmd_spec
for n in reversed(self.cmd_nuisances_inv):
cmd_spec_i = n.transform_spec(cmd_spec_i)
StreamSpec.check_same_spec(cmd_spec_i,
self.robot.get_spec().get_commands())
# TODO: why do we do this for commands, not for osbservations?
except Exception as e:
logger.warning('It seems that this chain of nuisances is not '
'exact, but it could be OK to continue. '
' The chain is %s; the error is:\n%s' %
(cmd_nuisance, indent(str(e).strip(), '> ')))
self.spec = BootSpec(obs_spec=obs_spec, cmd_spec=cmd_spec)
self.obs_nuisances_id = obs_nuisance
self.cmd_nuisances_id = cmd_nuisance
def write_report_single(report,
report_nid, report_html,
static_dir,
write_pickle=False):
from quickapp.report_manager import write_report
if not isinstance(report, Report):
msg = 'Expected Report, got %s.' % describe_type(report)
raise ValueError(msg)
report.nid = report_nid
write_report(report, report_html, static_dir=static_dir, write_pickle=write_pickle)
def check_valid_value(self, x):
'''
Checks if the value x is valid according to this spec
and raises BootInvalidValue (derived from ValueError) if it is not.
'''
def bail(msg):
msg += '\n stream: %s' % self
msg += '\n value: %s' % describe_value(x)
raise BootInvalidValue(msg)
if not isinstance(x, np.ndarray):
msg = 'Type is %s instead of numpy array.' % describe_type(x)
bail(msg)
if x.shape != self.streamels.shape:
msg = ('Expected shape %s instead of %s.' %
(self.streamels.shape, x.shape))
bail(msg)
invalids = self.kind == ValueFormats.Invalid
valid = np.logical_not(invalids)
# continuous = self.kind == ValueFormats.Continuous
discrete = self.kind == ValueFormats.Discrete
# First check all invalids are NaN
if not np.all(np.isnan(x[invalids])): # XXX: slow
bail('Not all invalids are set to NaN.')
# Check that all valid are not NaN
xv = x[valid]
xv_nan = np.isnan(xv)
if np.any(xv_nan):
msg = ('Found NaNs in the valid values. %s' %
display_some(xv, xv_nan))
bail(msg)
# Check all discrete values are integers
d = x[discrete]
not_discrete = np.round(d) != d
if np.any(not_discrete):
msg = ('Not all discrete are integers. %s' %
display_some(d, not_discrete))
bail(msg)
lv = self.lower[valid] # XXX: is this only 1D?
uv = self.upper[valid]
out_of_range = np.logical_or(xv < lv, xv > uv)
if np.any(out_of_range):
# print 'bounds dtype', lv.dtype, uv.dtype
msg = ('Found elements out of range:\n %s' %
display_some_extended(xv, self.streamels[valid], out_of_range))
bail(msg)
def check_vehicle_sensor_entry(s):
if not isinstance(s, dict):
raise BadConfig(s, 'I expect this to be a dictionary, not %s' %
describe_type(s))
if 'pose' in s:
check_valid_pose_spec(s['pose'])
alternatives = [('sensor', dict), ('id_sensor', str)]
check_has_exactly_one(s, alternatives)
if 'sensor' in s:
wrap_check(s, 'checking "sensor" entry',
check_valid_sensor_config, s['sensor'])
def __setitem__(self, attrs, value):
if not isinstance(value, dict):
msg = ('Values to this dictionary must be dicts; found %s' %
describe_type(value))
raise ValueError(msg)
for k in attrs:
if k in value:
msg = ('The same field %r is found in both key and value. \n'
' key: %s \n'
'value: %s' % (k, attrs, value))
raise ValueError(msg)
super(StoreResultsDict, self).__setitem__(attrs, value)
def belongs(self, x):
# TODO: more checks
if not isinstance(x, np.ndarray):
msg = 'Expected a numpy array (%s)' % describe_type(x)
raise ValueError(msg)
if not x.shape == (self.n, self.n):
msg = ('Expected shape %dx%d instead of (%s)' %
(self.n, self.n, x.shape))
raise ValueError(msg)
R, t, zero, one = extract_pieces(x) # @UnusedVariable
self.SOn.belongs(R)
assert_allclose(zero, 0, err_msg='I expect the lower row to be 0.')
assert_allclose(one, 1)
def belongs(self, x):
# TODO: more checks
if not isinstance(x, np.ndarray):
msg = 'Expected a numpy array (%s)' % describe_type(x)
raise ValueError(msg)
if not x.shape == (self.n, self.n):
msg = ('Expected shape %dx%d instead of (%s)' %
(self.n, self.n, x.shape))
raise ValueError(msg)
R, t, zero, one = extract_pieces(x) # @UnusedVariable
self.SOn.belongs(R)
assert_allclose(zero, 0, err_msg='I expect the lower row to be 0.')
assert_allclose(one, 1)
def safe_pickle_dump(value, filename, protocol=pickle.HIGHEST_PROTOCOL,
**safe_write_options):
with safe_write(filename, **safe_write_options) as f:
try:
pickle.dump(value, f, protocol)
except KeyboardInterrupt:
raise
except Exception:
msg = 'Cannot pickle object of class %s' % describe_type(value)
logger.error(msg)
msg = find_pickling_error(value, protocol)
logger.error(msg)
raise
def check_yaml_friendly_fast(s):
if isinstance(s, (str, int, float, NoneType)):
return
elif isinstance(s, list):
for x in s:
check_yaml_friendly(x)
elif isinstance(s, dict):
for k, v in s.items():
check_yaml_friendly(k)
check_yaml_friendly(v)
else:
msg = 'Invalid type for YAML serialization %s' % describe_type(s)
raise ValueError(msg)
def get_vsim_from_robot(robot):
""" Checks that the robot is a VehicleSimulation
or an EquivRobot. If not, it raises an exception.
Returns an instance to the VehicleSimulation. """
if isinstance(robot, EquivRobot):
robot = robot.get_original_robot()
return get_vsim_from_robot(robot)
from vehicles_boot import BOVehicleSimulation
if isinstance(robot, BOVehicleSimulation):
return robot
msg = ('I require that the robot is a VehiclesSimulation, '
'but obtained %s' % describe_type(robot))
raise ValueError(msg)
def __setitem__(self, attrs, value):
if not isinstance(value, dict):
msg = "Values to this dictionary must be dicts; found %s" % describe_type(
value
)
raise ValueError(msg)
for k in attrs:
if k in value:
msg = (
"The same field %r is found in both key and value. \n"
" key: %s \n"
"value: %s" % (k, attrs, value)
)
raise ValueError(msg)
super(StoreResultsDict, self).__setitem__(attrs, value)
def interpret_commands(commands_str, context, cq, separator=';'):
"""
Interprets what could possibly be a list of commands (separated by ";")
Returns None
"""
if not isinstance(commands_str, six.string_types):
msg = 'I expected a string, got %s.' % describe_type(commands_str)
raise ValueError(msg)
# split with separator
commands = commands_str.split(separator)
# remove extra spaces
commands = [x.strip() for x in commands]
# filter dummy commands
commands = [x for x in commands if x]
if not commands:
# nothing to do
return None
for cmd in commands:
try:
publish(context, 'command-starting', command=cmd)
retcode = interpret_single_command(cmd, context=context, cq=cq)
except KeyboardInterrupt:
publish(context,
'command-interrupted',
command=cmd,
reason='KeyboardInterrupt')
raise
except UserError as e:
publish(context, 'command-failed', command=cmd, reason=e)
raise
# TODO: all the rest is unexpected
if retcode == 0 or retcode is None:
continue
else:
if isinstance(retcode, int):
publish(context,
'command-failed',
command=cmd,
reason='Return code %r' % retcode)
raise CommandFailed('ret code %s' % retcode)
else:
publish(context, 'command-failed', command=cmd, reason=retcode)
raise CommandFailed('ret code %s' % retcode)
def from_yaml(x):
if not isinstance(x, list):
raise ValueError('I expect a list with two elements.')
form = x[0]
if not isinstance(form, str):
raise ValueError('I expect a string describing the format,'
' not %s, while decoding %s' %
(describe_type(form), describe_value(x)))
value = x[1]
space, representation = form.split(':')
key = (space, representation)
if not key in converters:
raise ValueError('Unknown format %s; I know %s.' %
(key, converters.keys()))
conv = converters[key]
return conv.from_yaml(value)
def write_python_data(parent, name, mime, data):
tables = get_tables()
from tables.flavor import flavor_of
hf = parent._v_file
group = hf.createGroup(parent, name)
hf.createArray(group, "mime", mime)
try:
flavor_of(data)
ok_pytables = True
except:
ok_pytables = False
# 2014-01-02 XXX this is a hack
if data == []:
ok_pytables = False
if ok_pytables:
try:
hf.createArray(group, "value", data)
except:
msg = "Error while writing python data"
msg += "\n parent: %s" % parent
msg += "\n name: %s" % name
msg += "\n mime: %s" % mime
msg += "\n data: %s" % describe_type(data)
msg += "\n %s" % describe_value(data)
msg += "\n flavor: %s" % flavor_of(data)
msg += "\nraw:\n%s" % data.__repr__()
logger.error(msg)
raise
serialized = "pytables"
else:
serialized = "pickle"
s = StringIO()
cPickle.dump(data, s, protocol=2)
hf.createVLArray(group, "pickle", tables.VLStringAtom(), filters=None)
group.pickle.append(s.getvalue())
group._v_attrs["reprep_data_format"] = serialized
def instance(self, block_type, name, config, library=None):
from procgraph.core.exceptions import ModelInstantionError, SemanticError
if library is None:
library = self
generator = self.get_generator_for_block_type(block_type)
try:
block = generator(name=name, config=config, library=library)
except Exception as e:
msg = 'Could not instance block from generator.\n'
msg += ' name: %s \n' % name
msg += ' config: %s \n' % config
msg += ' generator: %s \n' % generator
msg += ' of type: %s \n' % describe_type(generator)
msg += 'Because of this exception:\n'
if isinstance(e, (SemanticError, ModelInstantionError)):
msg += indent('%s' % e, '| ')
else:
msg += indent('%s\n%s' % (e, traceback.format_exc(e)), '| ')
raise ModelInstantionError(
msg) # TODO: use Procgraph own exception
block.__dict__['generator'] = generator
return block
def check_yaml_friendly_detailed(s, context=None):
if context is None:
context = []
if isinstance(s, (str, int, float, NoneType)):
return
elif isinstance(s, list):
for i, x in enumerate(s):
context.append('- %dth element of array' % i)
check_yaml_friendly_detailed(x, context)
context.pop()
elif isinstance(s, dict):
for k, v in s.items():
context.append('- key of dictionary')
check_yaml_friendly_detailed(k, context)
context.pop()
context.append('- value of %r' % k)
check_yaml_friendly_detailed(v, context)
context.pop()
else:
msg = ('Invalid type %s for YAML serialization.\n%s' %
(describe_type(s), "\n".join(context)))
raise ValueError(msg)
def progress(taskname, iterations, iteration_desc=None):
"""
Function used by the user to describe the state of the computation.
Parameters
---------
- ``name``: must be a string, describing the current task.
- ``iterations``: must be a tuple of two integers (k,N), meaning
that the current iteration is the k-th out of N.
- ``iteration_desc``: an optional string describing the current i
teration
Example: ::
for i in range(n):
progress('Reading files', (i,n),
'processing file %s' % file[i])
"""
if not isinstance(taskname, str):
raise ValueError('The first argument to progress() is the task name ' +
'and must be a string; you passed a %s.' %
describe_type(taskname))
if not isinstance(iterations, tuple):
raise ValueError('The second argument to progress() must be a tuple,' +
' you passed a %s.' % describe_type(iterations))
if not len(iterations) == 2:
raise ValueError('The second argument to progress() must be a tuple ' +
' of length 2, not of length %s.' % len(iterations))
if not isinstance(iterations[0], (int, float)):
raise ValueError('The first element of the tuple passed to progress ' +
'must be integer or float, not a %s.' %
describe_type(iterations[0]))
if not iterations[1] is None and not isinstance(iterations[1],
(int, float)):
raise ValueError('The second element of the tuple passed to progress '
'must be either None or an integer, not a %s.' %
describe_type(iterations[1]))
if iterations[1] < iterations[0]:
raise ValueError('Invalid iteration tuple: %s' % str(iterations))
BROADCAST_INTERVAL = 0.5
is_last = iterations[0] == iterations[1] - 1
stack = Globals.stack
for i, stage in enumerate(stack):
if stage.name == taskname:
# remove children
has_children = i < len(stack) - 1
stack[i + 1:] = []
stage.iterations = iterations
stage.iteration_desc = iteration_desc
# TODO: only send every once in a while
if ((is_last or has_children) or (stage.last_broadcast is None) or
(time.time() - stage.last_broadcast > BROADCAST_INTERVAL)):
progress_stack_updated()
stage.last_broadcast = time.time()
if stage.last_broadcast is None:
stage.last_broadcast = time.time()
break
else:
# If we are here, we haven't found taskname in the stack.
# This means that it is either a child or a brother (next task)
# We check that the last stage was over
while stack and stack[-1].was_finished():
stack.pop()
# treat it as a brother
stack.append(ProgressStage(taskname, iterations, iteration_desc))
progress_stack_updated()
def __repr__(self):
return ('CacheResult(size=%s,clock=%s,wall=%s,value=%s)' %
(memstring(self.size), timestring(self.clock),
timestring(self.wall), describe_type(self.value)))
def needs(self, rtype, **params):
rm = self.get_resource_manager()
res = rm.get_resource_job(self, rtype, **params)
assert isinstance(res, Promise), describe_type(res)
self._extra_dep.append(res)
def save_pair(self, k, v):
self.stack.append('key %r = object of type %s' % (k, describe_type(v)))
self.save(k)
self.save(v)
self.stack.pop()
def sub(self, resource, caption=None, display=None, **kwargs):
""" Adds a subfigure displaying the given resource.
resource can either be a string or a data node.
"""
if isinstance(resource, six.text_type):
data = self.resolve_url(resource)
if not isinstance(data, DataNode):
msg = "I expected a DataNode for %r, got %s" % (resource, data)
raise ValueError(msg)
elif isinstance(resource, DataNode):
data = resource
else:
raise ValueError("The first parameter to sub() must be either"
" a string (url) or a reference to a DataNode, "
" not a %s." % describe_type(resource))
if caption is None:
caption = data.caption
if caption is None:
caption = data.nid
# TODO: check if automatically generated
# if data.nid in ['scale', 'posneg']:
# caption = data.parent
# if data.get_complete_id() in self.automatically_added:
if data in self.automatically_added:
warnings.warn(
"Node %r was automatically added to figure (new "
"behavior in 1.0)." % self.get_relative_url(data),
stacklevel=2,
)
return
#
# if not isinstance(data, DataNode):
# msg = ('I expect a DataNode as an argument to sub(), not a %s.'
# % describe_type(resource))
# raise ValueError(msg)
if display is not None:
image = data.display(display, **kwargs)
else:
image = data.get_first_child_with_mime(MIME_IMAGES)
if image is None:
self.parent.print_tree() # XXX
raise ValueError("Could not find candidate image for resource "
"%r; image node is %r." %
(resource, data.get_complete_id()))
# Get an image that can be shown in a browser
web_image = image.get_first_child_with_mime(MIME_WEB_IMAGES)
if web_image is None:
# logger.error('No image with mime %r found in:\n%s' %
# (MIME_WEB_IMAGES, indent(image.format_tree(), '>')))
# convert the image to web image
# TODO: to write
web_image = image # XXX
# logger.error('I need to convert %s into a web image.' %
# (image))
if web_image is not None:
web_image = self.get_relative_url(web_image)
sub = SubFigure(
resource=self.get_relative_url(data),
image=self.get_relative_url(image),
web_image=web_image,
caption=caption,
)
self.subfigures.append(sub)