def test_flatten_struct():
a = {
'a': [1, 2, 3],
'b': {
'c': 4,
'd': {'e': 5, 6: [7, 8]},
},
'c': 1
}
b = dict(flatten_struct(a))
assert b["['a'][1]"]==2
assert b["['b']['d'][6][1]"]==8
class A(object):
def __init__(self, other_object, b):
self._a = other_object
self._b = b
thing = A(A(3, 'arr'), {'x': np.random.randn(3, 2), 'y': (3, 4, 5)})
flat_thing = dict(flatten_struct(thing))
assert flat_thing['._a._b']=='arr'
assert flat_thing["._b['y'][0]"]==3
def test_flatten_struct():
a = {
'a': [1, 2, 3],
'b': {
'c': 4,
'd': {'e': 5, 6: [7, 8]},
},
'c': 1
}
b = dict(flatten_struct(a))
assert b["['a'][1]"]==2
assert b["['b']['d'][6][1]"]==8
class A(object):
def __init__(self, other_object, b):
self._a = other_object
self._b = b
thing = A(A(3, 'arr'), {'x': np.random.randn(3, 2), 'y': (3, 4, 5)})
flat_thing = dict(flatten_struct(thing))
assert flat_thing['._a._b']=='arr'
assert flat_thing["._b['y'][0]"]==3
def get_record_invalid_arg_string(record, recursive=True, ignore_valid_keys=(), note_version = 'full'):
"""
Return a string identifying ig the arguments for this experiment are still valid.
:return:
"""
assert note_version in ('full', 'short')
experiment_id = record.get_experiment_id()
if is_experiment_loadable(experiment_id):
if record.info.has_field(ExpInfoFields.ARGS):
last_run_args = OrderedDict([(k,v) for k,v in record.get_args().items() if k not in ignore_valid_keys])
current_args = OrderedDict([(k,v) for k,v in record.get_experiment().get_args().items() if k not in ignore_valid_keys])
if recursive:
last_run_args = OrderedDict(flatten_struct(last_run_args, first_dict_is_namespace=True, primatives = PRIMATIVE_TYPES + (UnPicklableArg, )))
last_run_args = OrderedDict([(k, v) for k, v in last_run_args.items() if k not in ignore_valid_keys])
current_args = OrderedDict(flatten_struct(current_args, first_dict_is_namespace=True))
current_args = OrderedDict([(k, v) for k, v in current_args.items() if k not in ignore_valid_keys])
validity = record.args_valid(last_run_args=last_run_args, current_args=current_args)
if validity is False:
common, (old_args, new_args) = separate_common_items([list(last_run_args.items()), list(current_args.items())])
if len(old_args)+len(new_args)==0:
raise Exception('Error displaying different args. Bug Peter.')
all_changed_arg_names = remove_duplicates(list(name for name, _ in old_args)+list(name for name, _ in new_args))
changestr = ', '.join("{}:{}->{}".format(k, last_run_args[k] if k in last_run_args else '<N/A>', current_args[k] if k in current_args else '<N/A>') for k in all_changed_arg_names)
notes = ("Change: " if note_version=='full' else "") + changestr
elif validity is None:
notes = "Cannot Determine: Unhashable Args" if note_version=='full' else '<Unhashable Args>'
else:
notes = "<No Change>"
else:
notes = "Cannot Determine: Inconsistent Experiment Record" if note_version == 'full' else '<Inconsistent Record>'
else:
notes = "Cannot Determine: Experiment Not Imported" if note_version=='full' else '<Not Imported>'
return notes
def get_record_invalid_arg_string(record, recursive=True):
"""
Return a string identifying ig the arguments for this experiment are still valid.
:return:
"""
experiment_id = record.get_experiment_id()
if is_experiment_loadable(experiment_id):
if record.info.has_field(ExpInfoFields.ARGS):
last_run_args = record.get_args()
current_args = record.get_experiment().get_args()
validity = record.args_valid(last_run_args=last_run_args, current_args=current_args)
if validity is False:
if recursive:
last_run_args = OrderedDict(flatten_struct(last_run_args, first_dict_is_namespace=True))
current_args = OrderedDict(flatten_struct(current_args, first_dict_is_namespace=True))
last_arg_str, this_arg_str = [['{}:{}'.format(k, v) for k, v in argdict.iteritems()] for argdict in (last_run_args, current_args)]
common, (old_args, new_args) = separate_common_items([last_arg_str, this_arg_str])
notes = "No: Args changed!: {{{}}}->{{{}}}".format(','.join(old_args), ','.join(new_args))
elif validity is None:
notes = "Cannot Determine: Unhashable Args"
else:
notes = "Yes"
else:
notes = "Cannot Determine: Inconsistent Experiment Record"
else:
notes = "Cannot Determine: Experiment Not Imported"
return notes
def get_record_invalid_arg_string(record,
recursive=True,
ignore_valid_keys=(),
note_version='full'):
"""
Return a string identifying ig the arguments for this experiment are still valid.
:return:
"""
assert note_version in ('full', 'short')
experiment_id = record.get_experiment_id()
if is_experiment_loadable(experiment_id):
if record.info.has_field(ExpInfoFields.ARGS):
last_run_args = OrderedDict([(k, v)
for k, v in record.get_args().items()
if k not in ignore_valid_keys])
current_args = OrderedDict([
(k, v) for k, v in record.get_experiment().get_args().items()
if k not in ignore_valid_keys
])
if recursive:
last_run_args = OrderedDict(
flatten_struct(last_run_args,
first_dict_is_namespace=True))
last_run_args = OrderedDict([(k, v)
for k, v in last_run_args.items()
if k not in ignore_valid_keys])
current_args = OrderedDict(
flatten_struct(current_args, first_dict_is_namespace=True))
current_args = OrderedDict([(k, v)
for k, v in current_args.items()
if k not in ignore_valid_keys])
validity = record.args_valid(last_run_args=last_run_args,
current_args=current_args)
if validity is False:
last_arg_str, this_arg_str = [[
'{}:{}'.format(k, v) for k, v in argdict.items()
] for argdict in (last_run_args, current_args)]
common, (old_args, new_args) = separate_common_items(
[last_arg_str, this_arg_str])
if len(old_args) + len(new_args) == 0:
raise Exception(
'Error displaying different args. Bug Peter.')
changestr = "{{{}}}->{{{}}}".format(','.join(old_args),
','.join(new_args))
notes = ("Change: "
if note_version == 'full' else "") + changestr
elif validity is None:
notes = "Cannot Determine: Unhashable Args" if note_version == 'full' else '<Unhashable Args>'
else:
notes = "<No Change>"
else:
notes = "Cannot Determine: Inconsistent Experiment Record" if note_version == 'full' else '<Inconsistent Record>'
else:
notes = "Cannot Determine: Experiment Not Imported" if note_version == 'full' else '<Not Imported>'
return notes
def __call__(self, *args, **kwargs):
self.input_format.check((args, kwargs), self.fcn)
with StateCatcher(swallow_updates=False) as sc:
if ENABLE_OMNISCENCE:
with CaptureLocals() as c:
if self.attached_instance is None:
symbolic_return = self.fcn(*args, **kwargs)
else:
symbolic_return = self.fcn(self.attached_instance,
*args, **kwargs)
captured_anything = c.get_captured_locals()
captured_variables = flatten_struct(
captured_anything,
primatives=(Variable, SharedVariable),
break_into_objects=False)
captured_locals = {
k: v
for k, v in captured_variables if isinstance(v, Variable)
}
self._captured_locals = captured_locals
else:
if self.attached_instance is None:
symbolic_return = self.fcn(*args, **kwargs)
else:
symbolic_return = self.fcn(self.attached_instance, *args,
**kwargs)
self.update_format.check(sc.get_updates(), self.fcn)
self.output_format.check(symbolic_return, self.fcn)
return symbolic_return
def ezplot(anything, plots = None, hang = True, **plot_preference_kwargs):
"""
Make a plot of anything. Anything at all.
:param anything: Anything.
"""
data_dict = flatten_struct(anything)
figure, plots = plot_data_dict(data_dict, plots, mode = 'static', hang = hang, **plot_preference_kwargs)
return figure, plots
def ezplot(anything, plots = None, hang = True, **plot_preference_kwargs):
"""
Make a plot of anything. Anything at all.
:param anything: Anything.
"""
data_dict = flatten_struct(anything)
figure, plots = plot_data_dict(data_dict, plots, mode = 'static', hang = hang, **plot_preference_kwargs)
return figure, plots
def test_none_bug():
a = {'a': 1, 'b': None, 'c': [1, 2, None]}
fa = flatten_struct(a, first_dict_is_namespace=True)
assert dict(fa) == {'a': 1, 'b': None, 'c[0]': 1, 'c[1]': 2, 'c[2]': None}
def _get_data_structure(self):
struct = self._callback()
assert struct is not None, 'Your plotting-data callback returned None. Probably you forgot to include a return statememnt.'
flat_struct = flatten_struct(struct, custom_handlers=self._custom_handlers, detect_duplicates=False) # list<*tuple<str, data>>
return OrderedDict(flat_struct)
def test_none_bug():
a = {'a': 1, 'b': None, 'c': [1, 2, None]}
fa = flatten_struct(a, first_dict_is_namespace=True)
assert dict(fa) == {'a': 1, 'b': None, 'c[0]': 1, 'c[1]': 2, 'c[2]': None}
def get_record_invalid_arg_string(record,
recursive=True,
ignore_valid_keys=(),
note_version='full'):
"""
Return a string identifying ig the arguments for this experiment are still valid.
:return:
"""
assert note_version in ('full', 'short')
experiment_id = record.get_experiment_id()
if is_experiment_loadable(experiment_id):
if record.info.has_field(ExpInfoFields.ARGS):
last_run_args = OrderedDict([(k, v)
for k, v in record.get_args().items()
if k not in ignore_valid_keys])
current_args = OrderedDict([
(k, v) for k, v in record.get_experiment().get_args().items()
if k not in ignore_valid_keys
])
if recursive:
last_run_args = OrderedDict(
flatten_struct(last_run_args,
first_dict_is_namespace=True,
primatives=PRIMATIVE_TYPES +
(UnPicklableArg, )))
last_run_args = OrderedDict([(k, v)
for k, v in last_run_args.items()
if k not in ignore_valid_keys])
current_args = OrderedDict(
flatten_struct(current_args, first_dict_is_namespace=True))
current_args = OrderedDict([(k, v)
for k, v in current_args.items()
if k not in ignore_valid_keys])
validity = record.args_valid(last_run_args=last_run_args,
current_args=current_args)
if validity is False:
common, (old_args, new_args) = separate_common_items(
[list(last_run_args.items()),
list(current_args.items())])
if len(old_args) + len(new_args) == 0:
raise Exception(
'Error displaying different args. Bug Peter.')
all_changed_arg_names = remove_duplicates(
list(name for name, _ in old_args) +
list(name for name, _ in new_args))
changestr = ', '.join("{}:{}->{}".format(
k, last_run_args[k] if k in last_run_args else '<N/A>',
current_args[k] if k in current_args else '<N/A>')
for k in all_changed_arg_names)
notes = ("Change: "
if note_version == 'full' else "") + changestr
elif validity is None:
notes = "Cannot Determine: Unhashable Args" if note_version == 'full' else '<Unhashable Args>'
else:
notes = "<No Change>"
else:
notes = "Cannot Determine: Inconsistent Experiment Record" if note_version == 'full' else '<Inconsistent Record>'
else:
notes = "Cannot Determine: Experiment Not Imported" if note_version == 'full' else '<Not Imported>'
return notes
def _get_data_structure(self):
struct = self._callback()
assert struct is not None, 'Your plotting-data callback returned None. Probably you forgot to include a return statememnt.'
flat_struct = flatten_struct(struct, custom_handlers=self._custom_handlers, detect_duplicates=False) # list<*tuple<str, data>>
return OrderedDict(flat_struct)