def _load_next(self):
try:
try:
res = self.iterator.next()
if not isinstance(res, tuple):
msg = "Expected tuple (signal, timestamp, value), obtained %s" % describe_type(res)
raise ValueError(msg)
if not len(res) == 3:
raise ValueError("Required len 3 tuple; obtained %d." % len(res))
signal, timestamp, value = res
except StopIteration:
raise
except Exception as e:
msg = "Could not call next() on user-given iterator.\n"
msg += " iterator: %s\n" % str(self.iterator)
msg += " of type: %s\n" % describe_type(self.iterator)
msg += "because of this error:\n"
msg += indent(string.strip("%s\n%s" % (e, traceback.format_exc(e))), "| ")
raise ModelExecutionError(msg, self)
if not isinstance(signal, (str, int)):
msg = "Expected a string or number for the signal, got %s" % describe_value(signal)
raise ValueError(msg)
if not isinstance(timestamp, float):
msg = "Expected a number for the timestamp, got %s" % describe_value(timestamp)
raise ValueError(msg)
self.next_signal = signal
self.next_timestamp = timestamp
self.next_value = value
self.has_next = True
except StopIteration:
self.has_next = False
def torgb(rgb):
""" Converts all image formats to RGB uint8. """
try:
check_convertible_to_rgb(rgb)
except ValueError as e:
print describe_value(rgb)
raise BadInput(str(e), None, 0)
if rgb.ndim == 2:
if rgb.dtype == 'float32':
rgb = (rgb * 255).astype('uint8')
return gray2rgb(rgb)
nc = rgb.shape[2]
if nc == 4:
return rgb[:, :, :3]
if nc == 3:
if rgb.dtype == 'float32':
return (rgb * 255).astype('uint8')
elif rgb.dtype == 'uint8':
return rgb
else:
msg = 'Expected image format: %s' % describe_value(rgb)
raise ValueError(msg)
def inverse(self, g):
try:
self.belongs(g)
return np.linalg.inv(g)
except:
logger.error('Tried to invert %s' % describe_value(g))
raise
def formatm(*args, **kwargs):
#name_len = 10
assert len(args) > 0
assert len(args) % 2 == 0
cols = []
for i in range(len(args) / 2):
name = args[i * 2]
matrix = args[i * 2 + 1]
if not isinstance(name, str):
raise ValueError('I expect a string for label, not %s.'
% describe_type(name))
# varname = ' %s:' % rjust(name, name_len)
varname = ' %s:' % name
if isinstance(matrix, np.ndarray):
# raise ValueError('I expect a numpy array for value, not %s.' %
# describe_type(matrix))
value = format_matrix(matrix, **kwargs)
if matrix.ndim > 1:
varname = '\n' + varname
else:
value = describe_value(matrix)
cols.append(varname)
cols.append(value)
cols = add_spacer(cols)
return join_columns(cols)
def cmd_list_states(data_central, argv):
'''Shows a summary of the states present in DB. '''
parser = OptionParser(prog='list-states',
usage=cmd_list_states.short_usage)
parser.disable_interspersed_args()
parser.add_option("-v", dest='verbose', default=False, action='store_true',
help="Show more verbose output.")
(options, args) = parser.parse_args(argv)
check_no_spurious(args)
db = data_central.get_agent_state_db()
combinations = list(db.list_states())
if not combinations:
logger.info('No learning states saved in DB.')
else:
logger.info('Found %d combinations in DB.' % len(combinations))
for id_agent, id_robot in combinations:
logger.info('- Found state a: %-35s r: %-25s' % (id_agent, id_robot))
if options.verbose:
try:
state = db.get_state(id_robot=id_robot, id_agent=id_agent)
logger.info(' # episodes: %s' % len(state.id_episodes))
logger.info(' object: %s' %
describe_value(state.agent_state))
except Exception as e:
logger.error(' (could not load state: %s) ' % e)
if not options.verbose:
logger.debug('Use -v for more information.')
def torgb(rgb):
""" Converts all image formats to RGB uint8. """
try:
# check_convertible_to_rgb(rgb)
pass
except ValueError as e:
raise BadInput(str(e), None, 0)
if rgb.ndim == 2:
if rgb.dtype == 'float32':
m = np.amin(rgb)
M = np.amax(rgb)
if m < 0 or M > 1:
msg = 'If float, I assume range [0,1]; found [%s, %s].' % (m, M)
raise ValueError(msg)
rgb = (rgb * 255).astype('uint8')
return gray2rgb(rgb)
nc = rgb.shape[2]
if nc == 4:
return rgb[:, :, :3]
if nc == 3:
if rgb.dtype == 'float32':
return (rgb * 255).astype('uint8')
elif rgb.dtype == 'uint8':
return rgb
else:
msg = 'Expected image format: %s' % describe_value(rgb)
raise ValueError(msg)
def format_obs(d):
""" Shows objects values and typed for the given dictionary """
lines = []
for name, value in d.items():
lines.append('%15s: %s' % (name, describe_value(value)))
lines.append('%15s %s' % ('of type', describe_type(value)))
return '\n'.join(lines)
def formatm(*args, **kwargs):
# name_len = 10
assert len(args) > 0
assert len(args) % 2 == 0
cols = []
for i in range(int(len(args) / 2)):
name = args[i * 2]
matrix = args[i * 2 + 1]
if not isinstance(name, str):
raise ValueError('I expect a string for label, not %s.'
% describe_type(name))
# varname = ' %s:' % rjust(name, name_len)
varname = ' %s:' % name
if isinstance(matrix, np.ndarray):
# raise ValueError('I expect a numpy array for value, not %s.' %
# describe_type(matrix))
value = format_matrix(matrix, **kwargs)
if matrix.ndim > 1:
varname = '\n' + varname
else:
value = describe_value(matrix)
cols.append(varname)
cols.append(value)
cols = add_spacer(cols)
return join_columns(cols)
def _load_next(self):
try:
signal, timestamp, value = self.iterator.next()
if not isinstance(signal, (str, int)):
msg = ('Expected a string or number for the signal, got %s' %
describe_value(signal))
raise ValueError(msg)
if not isinstance(timestamp, float):
msg = ('Expected a number for the timestamp, got %s' %
describe_value(timestamp))
raise ValueError(msg)
self.next_signal = signal
self.next_timestamp = timestamp
self.next_value = value
self.has_next = True
except StopIteration:
self.has_next = False
def same_computation(jobargs1, jobargs2):
""" Returns boolean, string tuple """
cmd1, args1, kwargs1 = jobargs1
cmd2, args2, kwargs2 = jobargs2
equal_command = cmd1 == cmd2
equal_args = args1 == args2
equal_kwargs = kwargs1 == kwargs2
equal = equal_args and equal_kwargs and equal_command
if not equal:
reason = ""
if not equal_command:
reason += '* function changed \n'
reason += ' - old: %s \n' % cmd1
reason += ' - new: %s \n' % cmd2
# TODO: can we check the actual code?
warn = ' (or you did not implement proper __eq__)'
if len(args1) != len(args2):
reason += '* different number of arguments (%d -> %d)\n' % \
(len(args1), len(args2))
else:
for i, ob in enumerate(args1):
if ob != args2[i]:
reason += '* arg #%d changed %s \n' % (i, warn)
reason += ' - old: %s\n' % describe_value(ob)
reason += ' - old: %s\n' % describe_value(args2[i])
for key, value in kwargs1.items():
if key not in kwargs2:
reason += '* kwarg "%s" not found\n' % key
elif value != kwargs2[key]:
reason += '* argument "%s" changed %s \n' % (key, warn)
reason += ' - old: %s \n' % describe_value(value)
reason += ' - new: %s \n' % describe_value(kwargs2[key])
# TODO: different lengths
return False, reason
else:
return True, None
def same_computation(jobargs1, jobargs2):
""" Returns boolean, string tuple """
cmd1, args1, kwargs1 = jobargs1
cmd2, args2, kwargs2 = jobargs2
equal_command = cmd1 == cmd2
equal_args = args1 == args2
equal_kwargs = kwargs1 == kwargs2
equal = equal_args and equal_kwargs and equal_command
if not equal:
reason = ""
if not equal_command:
reason += '* function changed \n'
reason += ' - old: %s \n' % cmd1
reason += ' - new: %s \n' % cmd2
# TODO: can we check the actual code?
warn = ' (or you did not implement proper __eq__)'
if len(args1) != len(args2):
reason += '* different number of arguments (%d -> %d)\n' % \
(len(args1), len(args2))
else:
for i, ob in enumerate(args1):
if ob != args2[i]:
reason += '* arg #%d changed %s \n' % (i, warn)
reason += ' - old: %s\n' % describe_value(ob)
reason += ' - old: %s\n' % describe_value(args2[i])
for key, value in kwargs1.items():
if key not in kwargs2:
reason += '* kwarg "%s" not found\n' % key
elif value != kwargs2[key]:
reason += '* argument "%s" changed %s \n' % (key, warn)
reason += ' - old: %s \n' % describe_value(value)
reason += ' - new: %s \n' % describe_value(kwargs2[key])
# TODO: different lengths
return False, reason
else:
return True, None
def check_type(self, x):
expected = self.ptype
if self.ptype == float:
expected = (float, int)
if not isinstance(x, expected):
msg = ("For param %r, expected %s, got %s.\n%s" %
(self.name, self.ptype, describe_type(x),
describe_value(x)))
raise DecentParamsSemanticError(self.params, self, msg)
def __setattr__(self, leaf, value):
if leaf.startswith('_') or leaf in [
'mount_points', 'children_already_provided'
]:
return object.__setattr__(self, leaf, value)
v = self.child(leaf)
v.check_can_write()
try:
v._schema.validate(value)
except NotValid as e:
msg = 'Cannot set %s = %s:' % (leaf, describe_value(value))
raise_wrapped(NotValid, e, msg, compact=True)
from .memdata_events import event_leaf_set
from .memdata_events import event_struct_set
from .memdata_events import event_hash_set
from .memdata_events import event_list_set
if self._data[leaf] == value:
pass
else:
# case of a simple data
if is_simple_data(v._schema):
event = event_leaf_set(name=self._prefix,
leaf=leaf,
value=value,
**self._get_event_kwargs())
# struct
elif isinstance(v._schema, SchemaContext):
name = self._prefix + (leaf, )
event = event_struct_set(name=name,
value=value,
**self._get_event_kwargs())
elif isinstance(v._schema, SchemaHash):
name = self._prefix + (leaf, )
event = event_hash_set(name=name,
value=value,
**self._get_event_kwargs())
elif isinstance(v._schema, SchemaList):
name = self._prefix + (leaf, )
event = event_list_set(name=name,
value=value,
**self._get_event_kwargs())
else:
raise NotImplementedError(v._schema)
self._notify(event)
# logger.debug('setting leaf %s = %s' % (leaf, value))
# logger.debug('setting leaf schema = %s ' % (v._schema))
self._data[leaf] = value
def to_yaml(manifold, value, representation=None):
if representation is None:
representation = get_default_representation(manifold)
key = (manifold, representation)
if not key in converters:
raise ValueError("Unknown format %s; I know %s." % (key, converters.keys()))
conv = converters[key]
try:
x = conv.to_yaml(value)
except:
msg = "Error while trying to convert %s" % describe_value(value)
logger.error(msg)
raise
return ["%s:%s" % (manifold, representation), x]
def __str__(self):
if self.block is not None:
name = self.block.name
else:
name = "(unknown)"
# TODO: add short bad_value
s = "Bad input %r for block %r: %s" % (self.input_signal, name, self.error)
if self.block is not None:
s += "\n value: %s" % describe_value(self.bad_value)
s += format_where(self.block)
return s
def get_hint(self, s):
check_isinstance(s, SchemaBase)
if s in self.hints:
return self.hints[s]
if isinstance(s, (SchemaString, SchemaBytes, SchemaDate)):
return HintFile()
if isinstance(s, SchemaHash):
return HintDir()
if isinstance(s, SchemaContext):
return HintDir()
if isinstance(s, SchemaList):
return HintDir()
msg = 'Cannot find hint for %s' % describe_value(s)
raise ValueError(msg)
def __str__(self):
if self.block is not None:
name = self.block.name
else:
name = '(unknown)'
# TODO: add short bad_value
s = ("Bad input %r for block %r: %s" %
(self.input_signal, name, self.error))
if self.block is not None:
s += '\n value: %s' % describe_value(self.bad_value)
s += format_where(self.block)
return s
def check_type(self, x):
expected = self.ptype
if self.ptype == float:
expected = (float, int)
if six.PY2:
if self.ptype is unicode:
if isinstance(x, six.string_types):
return
if not isinstance(x, expected):
msg = ("For param '%s', expected %s, got %s.\n%s" %
(self.name, self.ptype, describe_type(x),
describe_value(x)))
raise DecentParamsSemanticError(self.params, self, msg)
def to_yaml(manifold, value, representation=None):
if representation is None:
representation = get_default_representation(manifold)
key = (manifold, representation)
if not key in converters:
raise ValueError('Unknown format %s; I know %s.' %
(key, converters.keys()))
conv = converters[key]
try:
x = conv.to_yaml(value)
except:
msg = 'Error while trying to convert %s' % describe_value(value)
logger.error(msg)
raise
return ['%s:%s' % (manifold, representation), x]
def from_yaml(x):
if not isinstance(x, list):
raise ValueError('I expect a list with two elements.')
form = x[0]
if not isinstance(form, str):
raise ValueError('I expect a string describing the format,'
' not %s, while decoding %s' %
(describe_type(form), describe_value(x)))
value = x[1]
space, representation = form.split(':')
key = (space, representation)
if not key in converters:
raise ValueError('Unknown format %s; I know %s.' %
(key, converters.keys()))
conv = converters[key]
return conv.from_yaml(value)
def from_yaml(x):
if not isinstance(x, list):
raise ValueError("I expect a list with two elements.")
form = x[0]
if not isinstance(form, str):
raise ValueError(
"I expect a string describing the format,"
" not %s, while decoding %s" % (describe_type(form), describe_value(x))
)
value = x[1]
space, representation = form.split(":")
key = (space, representation)
if not key in converters:
raise ValueError("Unknown format %s; I know %s." % (key, converters.keys()))
conv = converters[key]
return conv.from_yaml(value)
def add(self, report, report_type, **kwargs):
"""
Adds a report to the collection.
:param report: Promise of a Report object
:param report_type: A string that describes the "type" of the report
:param kwargs: str->str,int,float parameters used for grouping
"""
if not isinstance(report_type, str):
msg = 'Need a string for report_type, got %r.' % describe_value(report_type)
raise ValueError(msg)
from compmake import Promise
if not isinstance(report, Promise):
msg = ('ReportManager is mean to be given Promise objects, '
'which are the output of comp(). Obtained: %s'
% describe_type(report))
raise ValueError(msg)
# check the format is ok
self._check_report_format(report_type, **kwargs)
key = frozendict2(report=report_type, **kwargs)
if key in self.allreports:
msg = 'Already added report for %s' % key
msg += '\n its values is %s' % self.allreports[key]
msg += '\n new value would be %s' % report
raise ValueError(msg)
self.allreports[key] = report
report_type_sane = report_type.replace('_', '')
key_no_report = dict(**key)
del key_no_report['report']
basename = self.html_resources_prefix + report_type_sane
if key_no_report:
basename += '-' + basename_from_key(key_no_report)
dirname = os.path.join(self.outdir, report_type_sane)
filename = os.path.join(dirname, basename)
self.allreports_filename[key] = filename + '.html'
def main(self, args=None, parent=None):
""" Main entry point. Returns an integer as an error code. """
if "short" in type(self).__dict__:
msg = 'Class %s uses deprecated attribute "short".' % type(self)
msg += ' Use "description" instead.'
self.error(msg)
# Create the parameters and set them using args
self.parent = parent
self.set_options_from_args(args)
ret = self.go()
if ret is None:
ret = 0
if isinstance(ret, int):
return ret
else:
msg = 'Expected None or an integer fomr self.go(), got %s' % describe_value(ret)
raise ValueError(msg)
def caller():
try:
args = arguments(x)
except Exception as e:
msg = 'Error while preparing test case: %s.\n' % e
msg += 'Error while calling %s with argument %r' % (arguments, x)
logger.error(msg)
raise
try:
f(*args)
except:
msg = 'Error while executing test %r.\n' % name
msg += ' f = %s\n' % f
msg += ' f.__module__ = %s\n' % f.__module__
msg += ' x = %s\n' % str(x)
msg += ' arguments() = %s\n' % str(arguments)
msg += ' arguments(x) has size %d\n' % len(args)
for i, a in enumerate(args):
msg += ' arg %d = %s\n' % (i, describe_value(a))
logger.error(msg)
raise
def get_typsy_type(x):
"""
Returns the type of the object x.
There are multiple options that are checked in sequence:
1) The object has a '__typsy_type__' attribute.
2) The object has a callable attribute 'get_typsy_type()'.
3) TODO: the object is a function and it has been decorated
using contracts.
"""
if hasattr(x, attrname):
return getattr(x, attrname)
elif hasattr(x, funcname):
func = getattr(x, funcname)
return func() # it is already bound
else:
msg = 'Could not get typsy type for object.'
msg += '\n - object %r' % describe_value(x)
raise TypsyException(msg)
def write_python_data(parent, name, mime, data):
tables = get_tables()
from tables.flavor import flavor_of
hf = parent._v_file
group = hf.createGroup(parent, name)
hf.createArray(group, "mime", mime)
try:
flavor_of(data)
ok_pytables = True
except:
ok_pytables = False
# 2014-01-02 XXX this is a hack
if data == []:
ok_pytables = False
if ok_pytables:
try:
hf.createArray(group, "value", data)
except:
msg = "Error while writing python data"
msg += "\n parent: %s" % parent
msg += "\n name: %s" % name
msg += "\n mime: %s" % mime
msg += "\n data: %s" % describe_type(data)
msg += "\n %s" % describe_value(data)
msg += "\n flavor: %s" % flavor_of(data)
msg += "\nraw:\n%s" % data.__repr__()
logger.error(msg)
raise
serialized = "pytables"
else:
serialized = "pickle"
s = StringIO()
cPickle.dump(data, s, protocol=2)
hf.createVLArray(group, "pickle", tables.VLStringAtom(), filters=None)
group.pickle.append(s.getvalue())
group._v_attrs["reprep_data_format"] = serialized
def main(self, args=None, parent=None):
""" Main entry point. Returns an integer as an error code. """
if "short" in type(self).__dict__:
msg = 'Class %s uses deprecated attribute "short".' % type(self)
msg += ' Use "description" instead.'
self.error(msg)
# Create the parameters and set them using args
self.parent = parent
self.set_options_from_args(args)
profile = os.environ.get('QUICKAPP_PROFILE', False)
if not profile:
ret = self.go()
else:
import cProfile
out = profile
print('writing to %r' % out)
ret = cProfile.runctx('self.go()', globals(), locals(), out)
import pstats
p = pstats.Stats(out)
n = 30
p.sort_stats('cumulative').print_stats(n)
p.sort_stats('time').print_stats(n)
if ret is None:
ret = 0
if isinstance(ret, int):
return ret
else:
msg = 'Expected None or an integer fomr self.go(), got %s' % describe_value(ret)
raise ValueError(msg)
def write_python_data(parent, name, mime, data):
tables = get_tables()
from tables.flavor import flavor_of
hf = parent._v_file
group = hf.createGroup(parent, name)
hf.createArray(group, 'mime', mime)
try:
flavor_of(data)
ok_pytables = True
except:
ok_pytables = False
# 2014-01-02 XXX this is a hack
if data == []:
ok_pytables = False
if ok_pytables:
try:
hf.createArray(group, 'value', data)
except:
msg = 'Error while writing python data'
msg += '\n parent: %s' % parent
msg += '\n name: %s' % name
msg += '\n mime: %s' % mime
msg += '\n data: %s' % describe_type(data)
msg += '\n %s' % describe_value(data)
msg += '\n flavor: %s' % flavor_of(data)
msg += '\nraw:\n%s' % data.__repr__()
logger.error(msg)
raise
serialized = 'pytables'
else:
serialized = 'pickle'
s = StringIO()
cPickle.dump(data, s, protocol=2)
hf.createVLArray(group, 'pickle', tables.VLStringAtom(), filters=None)
group.pickle.append(s.getvalue())
group._v_attrs['reprep_data_format'] = serialized
def __init__(self, timestamps, values):
values = list(values)
timestamps = list(timestamps)
if not timestamps:
msg = 'Empty sequence.'
raise ValueError(msg)
if len(timestamps) != len(values):
msg = 'Length mismatch'
raise ValueError(msg)
for t in timestamps:
if not isinstance(t, (float, int)):
msg = 'I expected a number, got %s' % describe_value(t)
raise ValueError(msg)
for i in range(len(timestamps) - 1):
dt = timestamps[i + 1] - timestamps[i]
if dt <= 0:
msg = 'Invalid dt = %s at i = %s; ts= %s' % (dt, i, timestamps)
raise ValueError(msg)
timestamps = list(map(float, timestamps))
self.timestamps = timestamps
self.values = values
def add(self, context, report, report_type, **kwargs):
"""
Adds a report to the collection.
:param report: Promise of a Report object
:param report_type: A string that describes the "type" of the report
:param kwargs: str->str,int,float parameters used for grouping
"""
from quickapp.compmake_context import CompmakeContext
assert isinstance(context, CompmakeContext)
if not isinstance(report_type, six.string_types):
msg = 'Need a string for report_type, got %r.' % describe_value(report_type)
raise ValueError(msg)
if not isinstance(report, Promise):
msg = ('ReportManager is mean to be given Promise objects, '
'which are the output of comp(). Obtained: %s'
% describe_type(report))
raise ValueError(msg)
# check the format is ok
self._check_report_format(report_type, **kwargs)
key = frozendict2(report=report_type, **kwargs)
if key in self.allreports:
msg = 'Already added report for %s' % key
msg += '\n its values is %s' % self.allreports[key]
msg += '\n new value would be %s' % report
raise ValueError(msg)
self.allreports[key] = report
report_type_sane = report_type.replace('_', '')
key_no_report = dict(**key)
del key_no_report['report']
basename = self.html_resources_prefix + report_type_sane
if key_no_report:
basename += '-' + basename_from_key(key_no_report)
dirname = os.path.join(self.outdir, report_type_sane)
filename = os.path.join(dirname, basename)
self.allreports_filename[key] = filename + '.html'
write_singles = False
if write_singles:
is_root = context.currently_executing == ['root']
if not is_root:
# Add also a single report independent of a global index
# don't create the single report for the ones that are
# defined in the root session
filename_single = os.path.join(dirname, basename) + '_s.html'
# filename_index_dyn = os.path.join(dirname, basename) + '_dyn.html'
report_nid = self.html_resources_prefix + report_type_sane
if key:
report_nid += '-' + basename_from_key(key)
write_job_id = jobid_minus_prefix(context, report.job_id + '-writes')
# write_report_yaml(report_nid, report_job_id=report.job_id,
# key=key, html_filename=filename_single,
# report_html_indexed=filename_index_dyn)
context.comp(write_report_single,
report=report, report_nid=report_nid,
report_html=filename_single,
write_pickle=False,
static_dir=self.static_dir,
job_id=write_job_id)
def __init__(self,
nid: str,
data,
cols=None,
rows=None,
fmt=None,
caption=None):
"""
:type data: (array[R](fields[C]) | array[RxC] | list[R](list[C])
), R>0, C>0
:type cols: None|list[C](str)
:type rows: None|list[R](str)
:type caption: None|str
"""
if fmt is None:
fmt = "%s"
self.fmt = fmt
Node.__init__(self, nid)
check_isinstance(data, (list, np.ndarray))
if isinstance(data, list):
# check minimum length
if len(data) == 0:
raise ValueError("Expected at least one row")
# check that all of them are lists with same type
for row in data:
check_isinstance(row, list)
if not len(row) == len(data[0]):
msg = "I want all rows to be the same length" " Got %s != %s." % (
len(row),
len(data[0]),
)
raise ValueError(msg)
# create numpy array
nrows = len(data)
ncols = len(data[0])
if ncols == 0:
raise ValueError("At least one column expected")
if cols is None:
cols = [""] * ncols
if rows is None:
rows = [""] * nrows
elif isinstance(data, np.ndarray):
if not data.ndim in [1, 2]:
msg = "Expected array of 1D or 2D shape, got %s." % describe_value(
data)
raise ValueError(msg)
if data.ndim == 1:
# use fields name if desc not provided
if cols is None: # and data.dtype.fields is not None:
cols = list(data.dtype.fields)
nrows = len(data)
if rows is None:
rows = [""] * nrows
lol = []
for row in data:
lol.append(list(row))
data = lol
elif data.ndim == 2:
if data.dtype.fields is not None:
msg = ("Cannot convert ndarray to table using "
"the heuristics that I know (received: %s). " %
describe_value(data))
raise ValueError(msg)
nrows = data.shape[0]
ncols = data.shape[1]
if rows is None:
rows = [""] * nrows
if cols is None:
cols = [""] * ncols
data = data.tolist()
else:
assert False
#
# check_multiple([ (cols, 'list[C](str|None),C>0'),
# (rows, 'list[R](str|None),R>0'),
# (data, 'list[R](list[C])'),
# (caption, 'str|None') ])
# print('cols', cols)
# print('rows', rows)
# print('data', data)
# print('cols', cols)
self.data = data
self.cols = cols
self.rows = rows
self.caption = caption
def bail(msg):
msg += '\n stream: %s' % self
msg += '\n value: %s' % describe_value(x)
raise BootInvalidValue(msg)
def streamels_from_spec(shape, format, range, default): # @ReservedAssignment
streamels = np.zeros(shape=shape, dtype=streamel_dtype)
kind = streamels['kind']
lower = streamels['lower']
upper = streamels['upper']
# If the "format" is a string it is valid for all of them
if isinstance(format, str):
expect_one_of(format, ValueFormats.valid)
kind.flat[:] = format
# at this point the range must be unique
check_valid_bounds(range)
lower.flat[:] = range[0]
upper.flat[:] = range[1]
else:
# If the format is not a string, then it must be a list
if not isinstance(format, list):
msg = ('Expected list for "format", got %s.'
% describe_value(format))
raise ValueError(msg)
# And it must have the same number of elements
formats = np.array(format)
if formats.shape != streamels.shape:
msg = ('Expected format shape to be %s instead of %s.' %
(formats.shape, streamels.shape))
raise ValueError(msg)
for i in xrange(formats.size):
expect_one_of(formats.flat[i], ValueFormats.valid)
kind.flat[i] = formats.flat[i]
# also 'default' must
# Also the range must be a list
if not isinstance(range, list):
msg = 'Expected list for "range", got %s.' % describe_value(range)
raise ValueError(msg)
if len(streamels.shape) == 1:
if len(range) != streamels.shape[0]:
raise ValueError('Expected %s, got %s.' %
(streamels.shape[0], len(range)))
for i in xrange(streamels.shape[0]):
set_streamel_range(streamels[i], range[i])
elif len(streamels.shape) == 2:
if len(range) != streamels.shape[0]:
raise ValueError('Expected %s, got %s.' %
(streamels.shape[0], len(range)))
for i in xrange(streamels.shape[0]):
if len(range[i]) != streamels.shape[1]:
raise ValueError('Expected %s, got %s.' %
(streamels.shape[1], len(range[i])))
for j in xrange(streamels.shape[1]):
set_streamel_range(streamels[i, j], range[i][j])
else:
raise ValueError('Not implemented for shape %s.' %
str(streamels.shape))
# and also the default value must be a number
if default is None:
# Use half of the range
half = (streamels['upper'] + streamels['lower']) / 2.0
streamels['default'][:] = half # XXX
# round down, discretize if discrete
isdiscrete = kind == ValueFormats.Discrete
rounded = np.floor(half)
streamels['default'][isdiscrete] = rounded[isdiscrete]
elif isinstance(default, Number):
#
streamels['default'].flat[:] = default
elif isinstance(default, list): # explicit list
defaults = np.array(default, dtype=BOOT_OLYMPICS_SENSEL_RESOLUTION)
if defaults.shape != streamels.shape:
msg = ('Expected defaults shape to be %s instead of %s.' %
(defaults.shape, streamels.shape))
raise ValueError(msg)
if not (np.issubdtype(defaults.dtype, int) or
np.issubdtype(defaults.dtype, float)):
msg = ('Expect an array of numbers, not %s.'
% describe_value(defaults))
raise ValueError(msg)
streamels['default'].flat[:] = defaults.flat
else:
msg = 'Could not interpret default value %s.' % describe_value(default)
raise ValueError(msg)
return streamels
def comp_(context, command_, *args, **kwargs):
"""
Main method to define a computation step.
Extra arguments:
:arg:job_id: sets the job id (respects job_prefix)
:arg:extra_dep: extra dependencies (not passed as arguments)
:arg:command_name: used to define job name if job_id not provided.
If not given, command_.__name__ is used.
:arg:needs_context: if this is a dynamic job
Raises UserError if command is not pickable.
"""
db = context.get_compmake_db()
command = command_
if hasattr(command, '__module__') and command.__module__ == '__main__':
if not command in WarningStorage.warned:
if WarningStorage.warned:
# already warned for another function
msg = ('(Same warning for function %r.)' % command.__name__)
else:
msg = ("A warning about the function %r: " % command.__name__)
msg += (
"This function is defined directly in the __main__ "
"module, "
"which means that it cannot be pickled correctly due to "
"a limitation of Python and 'make new_process=1' will "
"fail. "
"For best results, please define functions in external "
"modules. "
'For more info, read '
'http://stefaanlippens.net/pickleproblem '
'and the bug report http://bugs.python.org/issue5509.')
warning(msg)
WarningStorage.warned.add(command)
if get_compmake_status() == CompmakeConstants.compmake_status_slave:
return None
# Check that this is a pickable function
try:
try_pickling(command)
except Exception as e:
msg = ('Cannot pickle function. Make sure it is not a lambda '
'function or a nested function. (This is a limitation of '
'Python)')
raise_wrapped(UserError, e, msg, command=command)
if CompmakeConstants.command_name_key in kwargs:
command_desc = kwargs.pop(CompmakeConstants.command_name_key)
elif hasattr(command, '__name__'):
command_desc = command.__name__
else:
command_desc = type(command).__name__
args = list(args) # args is a non iterable tuple
# Get job id from arguments
if CompmakeConstants.job_id_key in kwargs:
# make sure that command does not have itself a job_id key
try:
argspec = inspect.getargspec(command)
except TypeError:
# Assume Cython function
# XXX: write test
pass
else:
if CompmakeConstants.job_id_key in argspec.args:
msg = ("You cannot define the job id in this way because %r "
"is already a parameter of this function." %
CompmakeConstants.job_id_key)
raise UserError(msg)
job_id = kwargs[CompmakeConstants.job_id_key]
check_isinstance(job_id, six.string_types)
if ' ' in job_id:
msg = 'Invalid job id: %r' % job_id
raise UserError(msg)
job_prefix = context.get_comp_prefix()
if job_prefix:
job_id = '%s-%s' % (job_prefix, job_id)
del kwargs[CompmakeConstants.job_id_key]
if context.was_job_defined_in_this_session(job_id):
# unless it is dynamically geneterated
if not job_exists(job_id, db=db):
msg = 'The job %r was defined but not found in DB. I will let it slide.' % job_id
print(msg)
else:
msg = 'The job %r was already defined in this session.' % job_id
old_job = get_job(job_id, db=db)
msg += '\n old_job.defined_by: %s ' % old_job.defined_by
msg += '\n context.currently_executing: %s ' % context.currently_executing
msg += ' others defined in session: %s' % context.get_jobs_defined_in_this_session()
print(msg)
# warnings.warn('I know something is more complicated here')
# if old_job.defined_by is not None and
# old_job.defined_by == context.currently_executing:
# # exception, it's ok
# pass
# else:
msg = 'Job %r already defined.' % job_id
raise UserError(msg)
else:
if job_exists(job_id, db=db):
# ok, you gave us a job_id, but we still need to check whether
# it is the same job
stack = context.currently_executing
defined_by = get_job(job_id, db=db).defined_by
if defined_by == stack:
# this is the same job-redefining
pass
else:
for i in range(1000): # XXX
n = '%s-%d' % (job_id, i)
if not job_exists(n, db=db):
job_id = n
break
if False:
print(
'The job_id %r was given explicitly but already '
'defined.' % job_id)
print('current stack: %s' % stack)
print(' its stack: %s' % defined_by)
print('New job_id is %s' % job_id)
else:
job_id = generate_job_id(command_desc, context=context)
context.add_job_defined_in_this_session(job_id)
# could be done better
if 'needs_context' in kwargs:
needs_context = True
del kwargs['needs_context']
else:
needs_context = False
if CompmakeConstants.extra_dep_key in kwargs:
extra_dep = kwargs[CompmakeConstants.extra_dep_key]
del kwargs[CompmakeConstants.extra_dep_key]
if not isinstance(extra_dep, (list, Promise)):
msg = ('The "extra_dep" argument must be a list of promises; '
'got: %s' % describe_value(extra_dep))
raise ValueError(msg)
if isinstance(extra_dep, Promise):
extra_dep = [extra_dep]
assert isinstance(extra_dep, list)
for ed in extra_dep:
if not isinstance(ed, Promise):
msg = ('The "extra_dep" argument must be a list of promises; '
'got: %s' % describe_value(extra_dep))
raise ValueError(msg)
extra_dep = collect_dependencies(extra_dep)
else:
extra_dep = set()
children = collect_dependencies([args, kwargs])
children.update(extra_dep)
for c in children:
if not job_exists(c, db):
msg = "Job %r references a job %r that doesnt exist." % (job_id, c)
raise ValueError(msg)
all_args = (command, args, kwargs)
assert len(context.currently_executing) >= 1
assert context.currently_executing[0] == 'root'
c = Job(job_id=job_id,
children=children,
command_desc=command_desc,
needs_context=needs_context,
defined_by=context.currently_executing)
# Need to inherit the pickle
if context.currently_executing[-1] != 'root':
parent_job = get_job(context.currently_executing[-1], db)
c.pickle_main_context = parent_job.pickle_main_context
if job_exists(job_id, db):
old_job = get_job(job_id, db)
if old_job.defined_by != c.defined_by:
warning('Redefinition of %s: ' % job_id)
warning(' cur defined_by: %s' % c.defined_by)
warning(' old defined_by: %s' % old_job.defined_by)
if old_job.children != c.children:
#warning('Redefinition problem:')
#warning(' old children: %s' % (old_job.children))
#warning(' old dyn children: %s' % old_job.dynamic_children)
#warning(' new children: %s' % (c.children))
# fixing this
for x, deps in old_job.dynamic_children.items():
if not x in c.children:
# not a child any more
# FIXME: ok but note it might be a dependence of a child
# continue
pass
c.dynamic_children[x] = deps
for j in deps:
if not j in c.children:
c.children.add(j)
if old_job.parents != c.parents:
# warning('Redefinition of %s: ' % job_id)
# warning(' cur parents: %s' % (c.parents))
# warning(' old parents: %s' % old_job.parents)
for p in old_job.parents:
c.parents.add(p)
# TODO: preserve defines
# from compmake.ui.visualization import info
# info('defining job %r with children %r' % (job_id,
# c.children))
# if True or c.defined_by == ['root']:
for child in children:
db_job_add_parent_relation(child=child, parent=job_id, db=db)
if get_compmake_config('check_params') and job_exists(job_id, db):
# OK, this is going to be black magic.
# We want to load the previous job definition,
# however, by unpickling(), it will start
# __import__()ing the modules, perhaps
# even the one that is calling us.
# What happens, then is that it will try to
# add another time this computation recursively.
# What we do, is that we temporarely switch to
# slave mode, so that recursive calls to comp()
# are disabled.
# old_status = get_compmake_status()
# set_compmake_status(
# CompmakeConstants.compmake_status_slave)
all_args_old = get_job_args(job_id, db=db)
# set_compmake_status(old_status)
same, reason = same_computation(all_args, all_args_old)
if not same:
#print('different job, cleaning cache:\n%s ' % reason)
from compmake.jobs.actions import clean_targets
clean_targets([job_id], db)
# if job_cache_exists(job_id, db):
# delete_job_cache(job_id, db)
publish(context, 'job-redefined', job_id=job_id, reason=reason)
else:
# print('ok, same job')
pass
# XXX TODO clean the cache
# else:
# publish(context, 'job-already-defined',
# job_id=job_id)
set_job_args(job_id, all_args, db=db)
set_job(job_id, c, db=db)
publish(context, 'job-defined', job_id=job_id)
return Promise(job_id)
def init(self):
self.iterator = self.init_iterator()
if self.iterator is None:
msg = 'must return an iterator, got %s' % describe_value(self.iterator)
raise ValueError(msg)
self._load_next()
def comp_(context, command_, *args, **kwargs):
"""
Main method to define a computation step.
Extra arguments:
:arg:job_id: sets the job id (respects job_prefix)
:arg:extra_dep: extra dependencies (not passed as arguments)
:arg:command_name: used to define job name if job_id not provided.
If not given, command_.__name__ is used.
:arg:needs_context: if this is a dynamic job
Raises UserError if command is not pickable.
"""
db = context.get_compmake_db()
command = command_
if hasattr(command, '__module__') and command.__module__ == '__main__':
if not command in WarningStorage.warned:
if WarningStorage.warned:
# already warned for another function
msg = ('(Same warning for function %r.)' % command.__name__)
else:
msg = ("A warning about the function %r: " % command.__name__)
msg += (
"This function is defined directly in the __main__ "
"module, "
"which means that it cannot be pickled correctly due to "
"a limitation of Python and 'make new_process=1' will "
"fail. "
"For best results, please define functions in external "
"modules. "
'For more info, read '
'http://stefaanlippens.net/pickleproblem '
'and the bug report http://bugs.python.org/issue5509.')
warning(msg)
WarningStorage.warned.add(command)
if get_compmake_status() == CompmakeConstants.compmake_status_slave:
return None
# Check that this is a pickable function
try:
try_pickling(command)
except Exception as e:
msg = ('Cannot pickle function. Make sure it is not a lambda '
'function or a nested function. (This is a limitation of '
'Python)')
raise_wrapped(UserError, e, msg, command=command)
if CompmakeConstants.command_name_key in kwargs:
command_desc = kwargs.pop(CompmakeConstants.command_name_key)
elif hasattr(command, '__name__'):
command_desc = command.__name__
else:
command_desc = type(command).__name__
args = list(args) # args is a non iterable tuple
# Get job id from arguments
if CompmakeConstants.job_id_key in kwargs:
# make sure that command does not have itself a job_id key
try:
argspec = inspect.getargspec(command)
except TypeError:
# Assume Cython function
# XXX: write test
pass
else:
if CompmakeConstants.job_id_key in argspec.args:
msg = ("You cannot define the job id in this way because %r "
"is already a parameter of this function." %
CompmakeConstants.job_id_key)
raise UserError(msg)
job_id = kwargs[CompmakeConstants.job_id_key]
check_isinstance(job_id, six.string_types)
if ' ' in job_id:
msg = 'Invalid job id: %r' % job_id
raise UserError(msg)
job_prefix = context.get_comp_prefix()
if job_prefix:
job_id = '%s-%s' % (job_prefix, job_id)
del kwargs[CompmakeConstants.job_id_key]
if context.was_job_defined_in_this_session(job_id):
# unless it is dynamically geneterated
if not job_exists(job_id, db=db):
msg = 'The job %r was defined but not found in DB. I will let it slide.' % job_id
print(msg)
else:
msg = 'The job %r was already defined in this session.' % job_id
old_job = get_job(job_id, db=db)
msg += '\n old_job.defined_by: %s ' % old_job.defined_by
msg += '\n context.currently_executing: %s ' % context.currently_executing
msg += ' others defined in session: %s' % context.get_jobs_defined_in_this_session(
)
print(msg)
# warnings.warn('I know something is more complicated here')
# if old_job.defined_by is not None and
# old_job.defined_by == context.currently_executing:
# # exception, it's ok
# pass
# else:
msg = 'Job %r already defined.' % job_id
raise UserError(msg)
else:
if job_exists(job_id, db=db):
# ok, you gave us a job_id, but we still need to check whether
# it is the same job
stack = context.currently_executing
defined_by = get_job(job_id, db=db).defined_by
if defined_by == stack:
# this is the same job-redefining
pass
else:
for i in range(1000): # XXX
n = '%s-%d' % (job_id, i)
if not job_exists(n, db=db):
job_id = n
break
if False:
print('The job_id %r was given explicitly but already '
'defined.' % job_id)
print('current stack: %s' % stack)
print(' its stack: %s' % defined_by)
print('New job_id is %s' % job_id)
else:
job_id = generate_job_id(command_desc, context=context)
context.add_job_defined_in_this_session(job_id)
# could be done better
if 'needs_context' in kwargs:
needs_context = True
del kwargs['needs_context']
else:
needs_context = False
if CompmakeConstants.extra_dep_key in kwargs:
extra_dep = kwargs[CompmakeConstants.extra_dep_key]
del kwargs[CompmakeConstants.extra_dep_key]
if not isinstance(extra_dep, (list, Promise)):
msg = ('The "extra_dep" argument must be a list of promises; '
'got: %s' % describe_value(extra_dep))
raise ValueError(msg)
if isinstance(extra_dep, Promise):
extra_dep = [extra_dep]
assert isinstance(extra_dep, list)
for ed in extra_dep:
if not isinstance(ed, Promise):
msg = ('The "extra_dep" argument must be a list of promises; '
'got: %s' % describe_value(extra_dep))
raise ValueError(msg)
extra_dep = collect_dependencies(extra_dep)
else:
extra_dep = set()
children = collect_dependencies([args, kwargs])
children.update(extra_dep)
for c in children:
if not job_exists(c, db):
msg = "Job %r references a job %r that doesnt exist." % (job_id, c)
raise ValueError(msg)
all_args = (command, args, kwargs)
assert len(context.currently_executing) >= 1
assert context.currently_executing[0] == 'root'
c = Job(job_id=job_id,
children=children,
command_desc=command_desc,
needs_context=needs_context,
defined_by=context.currently_executing)
# Need to inherit the pickle
if context.currently_executing[-1] != 'root':
parent_job = get_job(context.currently_executing[-1], db)
c.pickle_main_context = parent_job.pickle_main_context
if job_exists(job_id, db):
old_job = get_job(job_id, db)
if old_job.defined_by != c.defined_by:
warning('Redefinition of %s: ' % job_id)
warning(' cur defined_by: %s' % c.defined_by)
warning(' old defined_by: %s' % old_job.defined_by)
if old_job.children != c.children:
#warning('Redefinition problem:')
#warning(' old children: %s' % (old_job.children))
#warning(' old dyn children: %s' % old_job.dynamic_children)
#warning(' new children: %s' % (c.children))
# fixing this
for x, deps in old_job.dynamic_children.items():
if not x in c.children:
# not a child any more
# FIXME: ok but note it might be a dependence of a child
# continue
pass
c.dynamic_children[x] = deps
for j in deps:
if not j in c.children:
c.children.add(j)
if old_job.parents != c.parents:
# warning('Redefinition of %s: ' % job_id)
# warning(' cur parents: %s' % (c.parents))
# warning(' old parents: %s' % old_job.parents)
for p in old_job.parents:
c.parents.add(p)
# TODO: preserve defines
# from compmake.ui.visualization import info
# info('defining job %r with children %r' % (job_id,
# c.children))
# if True or c.defined_by == ['root']:
for child in children:
db_job_add_parent_relation(child=child, parent=job_id, db=db)
if get_compmake_config('check_params') and job_exists(job_id, db):
# OK, this is going to be black magic.
# We want to load the previous job definition,
# however, by unpickling(), it will start
# __import__()ing the modules, perhaps
# even the one that is calling us.
# What happens, then is that it will try to
# add another time this computation recursively.
# What we do, is that we temporarely switch to
# slave mode, so that recursive calls to comp()
# are disabled.
# old_status = get_compmake_status()
# set_compmake_status(
# CompmakeConstants.compmake_status_slave)
all_args_old = get_job_args(job_id, db=db)
# set_compmake_status(old_status)
same, reason = same_computation(all_args, all_args_old)
if not same:
#print('different job, cleaning cache:\n%s ' % reason)
from compmake.jobs.actions import clean_targets
clean_targets([job_id], db)
# if job_cache_exists(job_id, db):
# delete_job_cache(job_id, db)
publish(context, 'job-redefined', job_id=job_id, reason=reason)
else:
# print('ok, same job')
pass
# XXX TODO clean the cache
# else:
# publish(context, 'job-already-defined',
# job_id=job_id)
set_job_args(job_id, all_args, db=db)
set_job(job_id, c, db=db)
publish(context, 'job-defined', job_id=job_id)
return Promise(job_id)
def __init__(self, nid, data, cols=None, rows=None, fmt=None, caption=None):
"""
:type data: (array[R](fields[C]) | array[RxC] | list[R](list[C])
), R>0, C>0
:type cols: None|list[C](str)
:type rows: None|list[R](str)
:type caption: None|str
"""
if fmt is None:
fmt = "%s"
self.fmt = fmt
Node.__init__(self, nid)
check_isinstance(data, (list, np.ndarray))
if isinstance(data, list):
# check minimum length
if len(data) == 0:
raise ValueError("Expected at least one row")
# check that all of them are lists with same type
for row in data:
check_isinstance(row, list)
if not len(row) == len(data[0]):
msg = "I want all rows to be the same length" " Got %s != %s." % (len(row), len(data[0]))
raise ValueError(msg)
# create numpy array
nrows = len(data)
ncols = len(data[0])
if ncols == 0:
raise ValueError("At least one column expected")
if cols is None:
cols = [""] * ncols
if rows is None:
rows = [""] * nrows
elif isinstance(data, np.ndarray):
if not data.ndim in [1, 2]:
msg = "Expected array of 1D or 2D shape, got %s." % describe_value(data)
raise ValueError(msg)
if data.ndim == 1:
# use fields name if desc not provided
if cols is None: # and data.dtype.fields is not None:
cols = list(data.dtype.fields)
nrows = len(data)
if rows is None:
rows = [""] * nrows
lol = []
for row in data:
lol.append(list(row))
data = lol
elif data.ndim == 2:
if data.dtype.fields is not None:
msg = (
"Cannot convert ndarray to table using "
"the heuristics that I know (received: %s). " % describe_value(data)
)
raise ValueError(msg)
nrows = data.shape[0]
ncols = data.shape[1]
if rows is None:
rows = [""] * nrows
if cols is None:
cols = [""] * ncols
data = data.tolist()
else:
assert False
#
# check_multiple([ (cols, 'list[C](str|None),C>0'),
# (rows, 'list[R](str|None),R>0'),
# (data, 'list[R](list[C])'),
# (caption, 'str|None') ])
# print('cols', cols)
# print('rows', rows)
# print('data', data)
# print('cols', cols)
self.data = data
self.cols = cols
self.rows = rows
self.caption = caption
def check_type(entry, etype, obtained):
if not isinstance(obtained, etype):
msg = 'Error in instantiating code spec:\n\t%s' % str(entry['code']).strip()
msg += '\nI expected: %s\nbut I got %s' % (etype, describe_value(obtained))
from conf_tools.exceptions import ConfToolsException
raise ConfToolsException(msg)