def __init__(self, var, parameters=None, architectures=None):
super(ParametersLink, self).__init__(var)
parameters = [] if parameters is None else parameters
architectures = [] if architectures is None else architectures
self.parameters = assert_iterable_return_iterable(parameters, 'list')
self.architectures = assert_iterable_return_iterable(
architectures, 'list')
def __init__(self, ext, required_arguments=[], optional_arguments=[]):
self.serving_extension = ext
self.required_arguments = assert_iterable_return_iterable(
required_arguments, 'list')
self.optional_arguments = assert_iterable_return_iterable(
optional_arguments, 'list')
self.processed_arguments = []
def load_stream_external_func(self, s, v):
"""Load a data stream with an imported function not defined in this class.
It uses external_stream_loader_info dictionnary to pipe the correct
arguments to the external func and returns what the func returns.
The external func should return fuel.datastreams.
"""
dataset = self.config['dataset']
func = self.external_stream_loader_info['func']
options = self.external_stream_loader_info['options']
v = self.merge_config_with_priority_args(v, options)
v.pop('split',
None) # it should not exist anyway but just to make sure
split_redirection = v.pop('split_name_redirect', None)
if split_redirection is not None:
v_update = {
'split': assert_iterable_return_iterable(split_redirection)
}
else:
v_update = {'split': assert_iterable_return_iterable(s)}
v.update(v_update)
try:
batch_size = v.pop('batch_size')
except KeyError:
raise KeyError("No batch size could be found for stream {} and \
split {}".format(dataset, s))
return func(dataset, batch_size, **v)
def link_var_to_graph(self, var, var_name, graph):
var = assert_iterable_return_iterable(var)
var_name = assert_iterable_return_iterable(var_name)
assert len(var) == len(var_name), "Unsafe linking, lengths "+\
"of vars to link and their corresponding string names are not the same"
var_name_dict = dict(zip(var_name, var))
# first check if that GraphLink was already made
for link in self.linksholder:
if isinstance(link, GraphLink) and link.graph == graph:
link.update(var_name_dict)
return
# else make a new one
return GraphLink(graph, var_name_dict)
def define_action_on(cls, extension_args):
"""Any method defining an action to do on some extension arguments
have to be decorated with this decorator.
"""
extension_args = assert_iterable_return_iterable(extension_args)
def action_on_decorator(func):
cls.arg_action_map.update({func: extension_args})
return func
return action_on_decorator
def stream_loader(self, split, split_args):
# This is technically not respecting in the interface as it also uses imported functions.
# However, the final call to fuel.datastreams is defined here and the piping to the imported
# functions is intricate, so it is not too farfetched to use this method.
if not split_args.has_key('ssl'):
print "INFO: No ssl specified for split {}, defaulting to regular loading way".format(
split)
return self.load_stream_external_func(split, split_args)
dataset = self.config['dataset']
split_args = dictify_type(split_args, dict)
ssl_args = split_args.pop('ssl')
print_epoch_done = ssl_args.pop('print_epoch_done', False)
def popargs(args):
args.pop('split',
None) # it should not exist anyway but just to make sure
try:
batch_size = args.pop('batch_size')
except KeyError:
raise KeyError(
"No batch size could be found for stream {} and \
split {}".format(dataset, split))
return batch_size
ssl_kwargs = [
'normalization', 'load_in_memory', 'test', 'nb_class',
'examples_per_class', 'examples_start'
]
kwargs = [
'batch_size', 'sources', 'normalization', 'load_in_memory', 'test'
]
ssl_args = self.merge_config_with_priority_args(ssl_args, ssl_kwargs)
ssl_args.update({'normalization': '01'})
split_args = self.merge_config_with_priority_args(split_args, kwargs)
batch_size = popargs(split_args)
ssl_batch_size = popargs(ssl_args)
split = assert_iterable_return_iterable(split)
split_args.update({'split': split})
ssl_args.update({'split': split})
stream = create_stream(dataset, batch_size, **split_args)
ssl_stream = create_ssl_stream(dataset, ssl_batch_size, **ssl_args)
name_dict = {'features': 'ssl_features', 'targets': 'ssl_targets'}
ssl_stream = Rename(ssl_stream, name_dict)
streams = [stream, ssl_stream]
sources = flatten([s.sources for s in streams])
rval = MultipleStreams(streams,
sources,
print_other_streams_epoch_done=print_epoch_done)
return rval
def fetch_vars(self, vars_name, graph=None):
vars_name = assert_iterable_return_iterable(vars_name, 'list')
"""Fetch multiple vars and return them in the
_same_ order of vars_names. A typical usage would then
be at the beginning of a graph defining method:
x, y, z = self.fetch_vars(['x','y','z'])
...
"""
varz = [self.fetch_var(v, graph) for v in vars_name]
return tuple(varz)
def recursive_parse(yml, D):
format_parser.check(yml)
if isinstance(yml, dict):
if yml.has_key('path'):
for p in assert_iterable_return_iterable(yml['path']):
recursive_parse(p, D)
update_dict(yml, D)
return
next_yml = YmlLoader.load_from_path(yml)
recursive_parse(next_yml, D)
def assert_for_keys(self, keys):
"""Check if config has the keys and throw an error if not.
It is possible to look in a hiarachy fasion if the user know that a
key map to a dict with other keys the user would like to check.
Ex.: self.check_for_keys(['foo'/bar']) will in effect result in a
call to self.config['foo'].has_key('bar')
"""
keys = assert_iterable_return_iterable(keys, 'list')
has_keys = map(self.config.has_key, keys)
assert sum(has_keys) == len(keys), "Missing config keys {}".format(
[x for x, y in zip(keys, has_keys) if not y])
def __init__(self,
path_to_yml,
mandatory_fields=None,
preprocess=True,
key_for_ymlpaths='ymlpath',
key_for_python='pypath'):
self.path_to_yml = path_to_yml
self.mandatory_fields = [] if mandatory_fields is None else \
assert_iterable_return_iterable(mandatory_fields)
self.preprocessing = preprocess
self.key_for_ymlpaths = key_for_ymlpaths
self.key_for_python = key_for_python
def store_links(self, newlinks):
newlinks = assert_iterable_return_iterable(newlinks, 'list')
# execute checks for all new links among themselves
LinksHelper.sanity_check(newlinks)
# execute checks for all new links compared to existing ones
for newlink in newlinks:
for link in self.links:
# can we compare a link with a link of a diff class?
if link.__class__ == newlink.__class__:
link.sanity_check(newlink)
self._links.extend(newlinks)
def recursive_parse(yml, D):
if isinstance(yml, dict):
if yml.has_key(key_for_ymlpaths):
if not isinstance(yml[key_for_ymlpaths], (str, list)):
raise YmlFormatError(key_for_ymlpaths)
for p in assert_iterable_return_iterable(
yml[key_for_ymlpaths]):
if not os.path.isfile(p):
raise IOError("Cannot find {}".format(p))
recursive_parse(p, D)
update_dict(yml, D)
return
next_yml = YmlLoader.load_from_path(yml)
recursive_parse(next_yml, D)
def write_on_config(self, newitem, from_key=None):
"""See above why this method exists compared on writing directly on self.config
from_key specifies the root key on which to update since the config
is a dict of dicts / non dicts. It can be a list.
If unspecified, we write from the base of the dict.
"""
config = dictify_type(self.config, dict) # unfreeze the config
if from_key is not None:
D = config
from_key = assert_iterable_return_iterable(from_key, 'list')
for k in from_key:
D = D[k]
D.update(newitem)
else:
config.update(newitem)
self.config = dictify_type(config, configdict)
def __init__(self, links=None):
links = [] if links is None else \
assert_iterable_return_iterable(links, 'list')
LinksHelper.sanity_check(links)
self._links = links
def __init__(self, var):
var = assert_iterable_return_iterable(var, 'list')
super(VariableLink, self).__init__(var)
def __init__(self, var, which_set='all'):
super(TrackingLink, self).__init__(var)
self.which_set = assert_iterable_return_iterable(which_set)
def __init__(self, parameters, **kwargs):
self.parameters = assert_iterable_return_iterable(parameters)
super(ModelParametersLink, self).__init__(parameters, **kwargs)
def __init__(self, inputs=[], outputs=[], **kwargs):
inputs = assert_iterable_return_iterable(inputs)
outputs = assert_iterable_return_iterable(outputs)
obj = (inputs, outputs)
super(FunctionLink, self).__init__(obj, **kwargs)
