def modify(self, mods, inputs=None, outputs=None):
"""
Allows to make modifications to the model. It will create a modified configuration file, build the corresponding model and
set the weights of the current model when possible.
mods: a list of dictionaries. Each dictionary can have as key: delete (in this case the value, which is a path,
is deleted from config), or a config path (in this case, the path value is replaced by the value of the dictionary).
inputs: a list with the names of the inputs
outputs: a list with the names of the outputs
"""
model_weights = self.get_weights()
yaml_loader = YAML()
m_conf = Config(self.core_model.processed_config)
original_keys = list(m_conf.keys())
deep_conf = Config(shallow_to_deep(m_conf))
for mod in mods:
mod_key = list(mod.keys())[0]
mod_value = mod[mod_key]
if mod_key == 'delete':
deep_conf.pop(mod_value)
if mod_value in original_keys:
original_keys.remove(mod_value)
elif '*' in mod_key:
mod_key = mod_key.lstrip('/')
found_paths = [
k for k in deep_conf.to_shallow().keys()
if fnmatch.fnmatch(k, mod_key)
]
for k in found_paths:
k = k.replace('.', '/')
if isinstance(mod_value, str):
deep_conf[k] = yaml_loader.load(mod_value)
else:
deep_conf[k] = mod_value
else:
mod_key = mod_key.replace('.', '/')
if mod_key.split('/')[0] not in deep_conf.keys(
): #This means we are adding a new layer
layer_name = mod_key.split('/')[0]
original_keys.append(layer_name)
deep_conf['{}/name'.format(layer_name)] = layer_name
if isinstance(mod_value, str):
deep_conf[mod_key] = yaml_loader.load(mod_value)
else:
deep_conf[mod_key] = mod_value
new_model_architecture = shallow_to_original_keys(
deep_conf.to_shallow(), original_keys)
model = self.build(processed_config=new_model_architecture,
input_names=inputs,
output_names=outputs)
layer_names = [l.name for l in model.layers]
for k, v in model_weights.items():
if k in layer_names:
layer = model.get_layer(k)
layer.set_weights(v)
self.core_model.model = model
def insert_yaml_value(config, special_tags, global_config, default_config,
missing_paths):
found_paths = config.find_path(symbols['insert_config'], mode='startswith')
#,action=lambda x: process_config(Config(x.split(symbols['insert_config'])[-1],special_tags=special_tags),special_tags=special_tags,global_config=global_config)
for path in found_paths:
tag_data = config[path]
insert_yaml_path = tag_data.split(symbols['insert_config'])[-1]
insert_config = Config(insert_yaml_path, yaml_tags=special_tags)
global_config.update(insert_config.get('global', {}))
if 'defaults' in insert_config:
default_config.update(insert_config.pop('defaults'))
insert_config = process_config(insert_config, special_tags,
global_config, default_config,
missing_paths)
config[path] = insert_config
def replace_yamls(main_config, special_tags):
main_config.find_path(
symbols['insert_config'],
mode='startswith',
action=lambda x: Config(x.split(symbols['insert_config'])[-1],
special_tags=special_tags))
return main_config
def _serialize_model(self, save_optimizer=False, extra_data=None):
model_output = {}
model_output['weights'] = self.get_weights()
original_config = Config(self.original_config)
for p in original_config.all_paths():
if type(original_config[p]).__name__ == 'BatchGenerator':
original_config[p] = original_config[p].data_processor_config
model_output['original_config'] = original_config
if self.architecture_config:
model_output['hierarchy'] = self.architecture_config.hierarchy
if self.core_model:
model_output['unfolded_config'] = self.core_model.processed_config
if save_optimizer:
model_output['optimizer_state'] = self.get_optimizer()
if extra_data:
model_output.update(extra_data)
model_output['input_shapes'] = self.input_shapes
model_output['output_shapes'] = self.output_shapes
return model_output
def __make_hash_dict(self):
"""
Creates a dictionary to hash the task. Parameters that are TaskIOs get replaced by their hash
"""
self._hash_dict = copy.deepcopy(self.parameters)
#Remove not cacheable parameters
if not isinstance(self._hash_dict, Config):
self._hash_dict = Config(self._hash_dict)
if not isinstance(self.parameters, Config):
self.parameters = Config(self.parameters)
_ = self._hash_dict.find_path(symbols['nocache'],
mode='startswith',
action='remove_value')
_ = self.parameters.find_path(symbols['nocache'],
mode='startswith',
action='remove_substring')
for k, v in self._hash_dict.to_shallow().items():
if isinstance(v, TaskIO):
self._hash_dict[k] = self._hash_dict[k].get_hash()
def include_config(config, special_tags, global_config, default_config,
missing_paths):
found_paths = config.find_keys('include')
for p in found_paths:
includes = config[p]
switch = None
if isinstance(includes, dict):
switch = includes.get('switch', None)
if switch is not None:
if not isinstance(switch, list):
switch = [switch]
filtered_includes = []
for include_config in includes['configs']:
if include_config.get('name', None) in switch:
filtered_includes.append(include_config)
includes = filtered_includes
for include_config in includes:
if include_config.get('enable', True) and include_config.get(
'config', None):
path_yaml_to_include = Path(config.yaml_path.parent,
include_config.pop('config'))
imported_config = Config(path_yaml_to_include,
yaml_tags=special_tags)
if 'defaults' in imported_config:
default_config.update(imported_config.pop('defaults'))
mods = include_config.get('mods', None)
for r, v in include_config.items():
r = '({})'.format(r)
imported_config = replace_in_config(imported_config, r, v)
if '/' in p:
p_parent = '/'.join(p.split('/')[:-1])
else:
p_parent = None
imported_config = process_config(imported_config, special_tags,
global_config, default_config,
missing_paths)
if mods:
apply_mods(mods, imported_config)
if p_parent:
p_config = Config(config[p_parent])
p_config.yaml_path = config.yaml_path
new_config = merge_configs([p_config, imported_config])
config[p_parent] = new_config
else:
original_yaml_path = config.yaml_path
config = merge_configs([Config(config), imported_config])
config.yaml_path = original_yaml_path
config.pop(p)
return config
def apply_mods(modstr, config):
yaml = YAML()
if modstr is not None:
if isinstance(modstr, str):
mods = modstr.split('&')
for mod in mods:
if '=' in mod:
mod_parts = mod.split('=')
mod_k = '='.join(mod_parts[:-1])
mod_v = mod_parts[-1]
#if mod_parts[1].startswith('['):
if '!' in mod_v:
config[mod_k] = mod_v
#elif mod_parts[1].lower() == 'null':
# config[mod_parts[0]] = None
else:
config[mod_k] = yaml.load(mod_v)
elif isinstance(modstr, list):
for mod in modstr:
config.update(Config(mod).to_shallow())
def get_config(filename):
config = Config(filename, safe=False)
return config
def external_unfold(name, config, metadata=None, logger=None):
external_models = metadata['externals']['Models']
external_model_name = config['model']
external_layer_name = config.get('layer', None)
external_last_layer = config.get('up_to', None)
external_first_layer = config.get('from', None)
external_exclude_inputs = config.get('exclude_input', True)
external_reset_weights = config.get('reset_weights', False)
external_mods = config.get('mods', None)
external_time_distributed = config.get('time_distributed', False)
trainable_from = config.get('trainable_from', None)
trainable_layers = config.get('trainable_layers', None)
trainable = config.get('trainable', True)
training_flag = config.get('training', False)
import dienen
if isinstance(external_models[external_model_name], str):
external_model = joblib.load(external_models[external_model_name])
if isinstance(external_model, dict):
external_model_architecture = external_model['unfolded_config']
external_hierarchy = external_model['hierarchy']
elif isinstance(external_model, dienen.core.model.Model):
external_model_architecture = external_model.core_model.processed_config
external_hierarchy = external_model.architecture_config.hierarchy
elif isinstance(external_models[external_model_name],
dienen.core.model.Model):
external_model = external_models[external_model_name]
external_model_architecture = external_model.core_model.processed_config
external_hierarchy = external_model.architecture_config.hierarchy
if external_mods:
import fnmatch
yaml_loader = YAML()
m_conf = Config(external_model_architecture)
original_keys = list(m_conf.keys())
deep_conf = Config(shallow_to_deep(m_conf))
for mod in external_mods:
mod_key = list(mod.keys())[0]
mod_value = mod[mod_key]
if mod_key == 'delete':
deep_conf.pop(mod_value)
if mod_value in original_keys:
original_keys.remove(mod_value)
elif '*' in mod_key:
mod_key = mod_key.lstrip('/')
found_paths = [
k for k in deep_conf.to_shallow().keys()
if fnmatch.fnmatch(k, mod_key)
]
for k in found_paths:
k = k.replace('.', '/')
if isinstance(mod_value, str):
deep_conf[k] = yaml_loader.load(mod_value)
else:
deep_conf[k] = mod_value
else:
mod_key = mod_key.replace('.', '/')
if mod_key.split('/')[0] not in deep_conf.keys(
): #This means we are adding a new layer
layer_name = mod_key.split('/')[0]
original_keys.append(layer_name)
deep_conf['{}/name'.format(layer_name)] = layer_name
if isinstance(mod_value, str):
deep_conf[mod_key] = yaml_loader.load(mod_value)
else:
deep_conf[mod_key] = mod_value
external_model_architecture = shallow_to_original_keys(
deep_conf.to_shallow(), original_keys)
unfolded_layers = []
g = nx.DiGraph()
for layer_name, layer_config in external_model_architecture.items():
if layer_config['class'] != 'Input':
if isinstance(layer_config['input'], list):
for k in layer_config['input']:
g.add_edge(k, layer_name)
else:
g.add_edge(layer_config['input'], layer_name)
if external_layer_name and external_layer_name not in g.nodes(
) and external_layer_name in external_hierarchy:
external_layer_name = external_hierarchy[external_layer_name][
'output'][0]
if external_last_layer and external_last_layer not in g.nodes(
) and external_last_layer in external_hierarchy:
external_last_layer = external_hierarchy[external_last_layer][
'output'][0]
if external_first_layer and external_first_layer not in g.nodes(
) and external_first_layer in external_hierarchy:
external_first_layer = external_hierarchy[external_first_layer][
'input'][0]
if external_last_layer and not external_first_layer and not external_layer_name:
layers_subset = list(nx.ancestors(
g, external_last_layer)) + [external_last_layer]
elif external_first_layer and not external_last_layer and not external_layer_name:
layers_subset = nx.dfs_successors(
g, external_first_layer)[external_first_layer] + [
external_first_layer
]
elif external_first_layer and external_last_layer and not external_layer_name:
after_from = set(nx.dfs_successors(g, external_first_layer).keys())
before_to = set(nx.ancestors(g, external_last_layer))
layers_subset = list(
after_from.intersection(before_to)) + [external_last_layer]
elif external_layer_name:
layers_subset = [external_layer_name]
else:
layers_subset = list(external_model_architecture.keys())
if external_exclude_inputs:
layers_subset = [
layer for layer in layers_subset
if external_model_architecture[layer]['class'] != 'Input'
]
unfolded_layers = [external_model_architecture[l] for l in layers_subset]
#if len(unfolded_layers) == 1:
# in_layers = [name]
#else:
in_layers = []
for l in unfolded_layers:
ins = l['input']
if not isinstance(ins, list):
ins = [ins]
for x in ins:
if x not in layers_subset:
in_layers.append(l['name'])
#in_layers = [l['name'] for l in unfolded_layers if l['input'] not in layers_subset]
new_nodes = [{
layer['name']: pop_dictreturn(layer, 'name')
} for layer in unfolded_layers]
new_config, hierarchy = new_nodes_to_config(new_nodes, name)
if 'input' in config:
for layer in in_layers:
new_config[layer]['input'] = config['input']
if trainable and not trainable_from and not trainable_layers:
trainable_layers = [
layer_name for layer_name, layer in new_config.items()
if layer['class'] != 'Input'
]
elif trainable_from:
if trainable_from not in external_model_architecture and trainable_from in external_hierarchy:
trainable_from = external_hierarchy[trainable_from]['inputs'][0]
trainable_layers = nx.dfs_successors(g, trainable_from)
elif not trainable:
trainable_layers = []
#Set trainable false in non-trainable layers
for layer_name, layer_config in new_config.items():
if layer_name not in trainable_layers:
layer_config['trainable'] = False
layer_config[
'training'] = training_flag #This is to avoid problems with BN accumulated statistics
else:
layer_config['trainable'] = True
if external_time_distributed:
for layer_name, layer_config in new_config.items():
layer_config['time_distributed'] = True
#Make each layer search for the weights from external model
if isinstance(external_reset_weights, bool) and not external_reset_weights:
external_weight_layers = [
layer_name for layer_name, layer in new_config.items()
if layer['class'] != 'Input'
]
elif isinstance(external_reset_weights, list):
external_weight_layers = external_reset_weights
if not config.get('reset_weights', False):
for layer in external_weight_layers:
new_config[layer]['from_model'] = external_model_name
new_config[layer]['from_layer'] = layer
return new_config, hierarchy
def load_experiment(configs, mods=None, global_config=None, logger=None):
#Get main config
#By default, yaml uses custom tags marked as !, however, we want to use it in a more general way even in dictionaries.
#To avoid raising exceptions, an ignorable tag is created which will return the string unchanged for later processing
ignorable_tags = [
v.strip() for k, v in symbols.items() if v.startswith('!')
]
special_tags = [IgnorableTag(tag) for tag in ignorable_tags]
configs = [Config(path_i, yaml_tags=special_tags) for path_i in configs]
#main_config = merge_configs(configs)
main_config = configs[0]
apply_mods(mods, main_config)
if global_config is None:
global_config = {}
global_config.update(main_config.get('global', {}))
default_config = main_config.get('defaults', {})
if 'global' in main_config:
main_config['global'].update(global_config)
else:
main_config['global'] = global_config
#Config processing/merging/expanding
missing_paths = []
main_config = process_config(main_config, special_tags, global_config,
default_config, missing_paths)
n_tries = 20
while n_tries > 0 and len(missing_paths) > 0:
n_tries -= 1
global_config.update(main_config['global'])
default_config.update(main_config.get('default', {}))
missing_paths = []
main_config = process_config(main_config, special_tags, global_config,
default_config, missing_paths)
if len(missing_paths) > 0:
print('Warning: Cannot resolve tags {}'.format(missing_paths))
for k in missing_paths:
global_config[k] = None
missing_paths = []
main_config = process_config(main_config, special_tags, global_config,
default_config, missing_paths)
default_cluster_config = {'manager': None, 'n_cores': 1, 'niceness': 20}
cluster_config = main_config.get('cluster_config', default_cluster_config)
main_config['cluster_config'] = cluster_config
main_config['global_config'] = global_config
#For every task with a variable that we want to loop,
#we find the tag and create a parameter 'parallel' which holds the names
#of the loopable params, and adds a '!nocache' so that it is not cached
parallel_paths = main_config.find_path(symbols['distributed-pool'],
mode='startswith',
action='remove_substring')
parallel_paths = [p for p in parallel_paths if not p.startswith('global')]
parallel_paths = [(task_parameters_level_from_path(p),
p.split(task_parameters_level_from_path(p) + '/')[-1])
for p in parallel_paths]
parallel_paths_async = [
k for k in list(main_config.all_paths())
if k.endswith('async') and main_config[k] == True
]
parallel_paths_async = [
p for p in parallel_paths_async if not p.startswith('global')
]
parallel_paths_async = [
(task_parameters_level_from_path(p),
p.split(task_parameters_level_from_path(p) + '/')[-1])
for p in parallel_paths_async
]
parallel_paths_ = {}
for p in parallel_paths_async:
main_config[p[0] + '/niceness'] = cluster_config.get('niceness', 20)
for p in parallel_paths:
path = p[0] + '/parallel'
if path not in parallel_paths_:
parallel_paths_[path] = [p[1]]
else:
parallel_paths_[path].append(p[1])
if 'n_cores' not in main_config[p[0]]:
main_config[p[0] + '/n_cores'] = cluster_config['n_cores']
if 'niceness' not in main_config[p[0]]:
main_config[p[0] + '/niceness'] = cluster_config.get(
'niceness', 20)
map_paths = main_config.find_path(symbols['serial-map'],
mode='startswith',
action='remove_substring')
map_paths = [p for p in map_paths if not p.startswith('global')]
map_paths = [(task_parameters_level_from_path(p),
p.split(task_parameters_level_from_path(p) + '/')[-1])
for p in map_paths]
map_paths_ = {}
for p in map_paths:
path = p[0] + '/map_vars'
if path not in map_paths_:
map_paths_[path] = [p[1]]
else:
map_paths_[path].append(p[1])
yaml = YAML()
for k, v in parallel_paths_.items():
v_yaml_stream = StringIO()
yaml.dump(v, v_yaml_stream)
parallel_paths_[k] = symbols['nocache'] + ' ' + str(v)
v_yaml_stream.close()
for k, v in map_paths_.items():
v_yaml_stream = StringIO()
yaml.dump(v, v_yaml_stream)
map_paths_[k] = symbols['nocache'] + ' ' + str(v)
v_yaml_stream.close()
main_config.update(parallel_paths_)
main_config.update(map_paths_)
#main_task = TaskGraph(main_config,global_config,name='MainTask',logger=paips_logger)
return main_config
def __init__(self,
parameters,
global_parameters=None,
name=None,
logger=None,
simulate=False):
"""
parameters: dictionary with all parameters given to a task
global_parameters: dictionary with parameters common to all tasks
name: task name
logger: task logger
simulate: if True, the task won't get executed
"""
self.global_parameters = {
'cache': True,
'cache_path': 'cache',
'cache_compression': 0,
'output_path': 'experiments',
'overwrite_export': True
}
if global_parameters:
self.global_parameters.update(global_parameters)
if not GenericFile(self.global_parameters['output_path']).exists():
GenericFile(
self.global_parameters['output_path']).mkdir(parents=True)
self.name = name
self.valid_args = []
self.default_no_cache: []
self.parameters = parameters
self.simulate = simulate
self.output_names = self.parameters.pop('output_names', ['out'])
self.cache = get_delete_param(self.parameters, 'cache',
self.global_parameters['cache'])
self.in_memory = get_delete_param(self.parameters, 'in_memory',
self.global_parameters['in_memory'])
self.dependencies = []
self.logger = logger
if 'mods' in self.parameters:
apply_mods(self.parameters['mods'], Config(self.parameters))
self.parameters.pop('mods')
self.__make_hash_dict()
self.initial_parameters = copy.deepcopy(self.parameters)
self.export_path = Path(self.global_parameters.get('output_path'),
self.name)
self.export = self.parameters.get('export', False)
self.symlinkdb_path = Path(self.global_parameters.get('output_path'),
'links.txt')
fname = GenericFile(self.global_parameters['output_path'], 'configs',
'{}.yaml'.format(self.name))
self.parameters.save(Path(fname.local_filename), mode='unsafe')
if fname.filesystem == 's3':
fname.upload_from(fname.local_filename)
class Task():
def __init__(self,
parameters,
global_parameters=None,
name=None,
logger=None,
simulate=False):
"""
parameters: dictionary with all parameters given to a task
global_parameters: dictionary with parameters common to all tasks
name: task name
logger: task logger
simulate: if True, the task won't get executed
"""
self.global_parameters = {
'cache': True,
'cache_path': 'cache',
'cache_compression': 0,
'output_path': 'experiments',
'overwrite_export': True
}
if global_parameters:
self.global_parameters.update(global_parameters)
if not GenericFile(self.global_parameters['output_path']).exists():
GenericFile(
self.global_parameters['output_path']).mkdir(parents=True)
self.name = name
self.valid_args = []
self.default_no_cache: []
self.parameters = parameters
self.simulate = simulate
self.output_names = self.parameters.pop('output_names', ['out'])
self.cache = get_delete_param(self.parameters, 'cache',
self.global_parameters['cache'])
self.in_memory = get_delete_param(self.parameters, 'in_memory',
self.global_parameters['in_memory'])
self.dependencies = []
self.logger = logger
if 'mods' in self.parameters:
apply_mods(self.parameters['mods'], Config(self.parameters))
self.parameters.pop('mods')
self.__make_hash_dict()
self.initial_parameters = copy.deepcopy(self.parameters)
self.export_path = Path(self.global_parameters.get('output_path'),
self.name)
self.export = self.parameters.get('export', False)
self.symlinkdb_path = Path(self.global_parameters.get('output_path'),
'links.txt')
fname = GenericFile(self.global_parameters['output_path'], 'configs',
'{}.yaml'.format(self.name))
self.parameters.save(Path(fname.local_filename), mode='unsafe')
if fname.filesystem == 's3':
fname.upload_from(fname.local_filename)
def __make_hash_dict(self):
"""
Creates a dictionary to hash the task. Parameters that are TaskIOs get replaced by their hash
"""
self._hash_dict = copy.deepcopy(self.parameters)
#Remove not cacheable parameters
if not isinstance(self._hash_dict, Config):
self._hash_dict = Config(self._hash_dict)
if not isinstance(self.parameters, Config):
self.parameters = Config(self.parameters)
_ = self._hash_dict.find_path(symbols['nocache'],
mode='startswith',
action='remove_value')
_ = self.parameters.find_path(symbols['nocache'],
mode='startswith',
action='remove_substring')
for k, v in self._hash_dict.to_shallow().items():
if isinstance(v, TaskIO):
self._hash_dict[k] = self._hash_dict[k].get_hash()
def search_dependencies(self):
"""
Finds all the tasks needed to run this task. It does it by searching for the -> symbol in its config
"""
stop_propagate_dot = self.parameters.get('stop_propagate_dot', None)
dependency_paths = self.parameters.find_path(symbols['dot'],
mode='contains')
#dependency_paths = [p for p in dependency_paths if 'Tasks' not in ]
if self.__class__.__name__ == 'TaskGraph':
dependency_paths = [
p for p in dependency_paths
if ('Tasks' not in p) and (not p.startswith('outputs'))
]
#Esto es porque dienen tambien usa el simbolo -> entonces debo decir que si encuentra ahi no lo tenga en cuenta.
if stop_propagate_dot:
dependency_paths = [
p for p in dependency_paths
if not p.startswith(stop_propagate_dot)
]
self._dependencies = [
self.parameters[path].split(symbols['dot'])[0]
for path in dependency_paths
]
self._dependencies = [d for d in self._dependencies if d != 'self']
return self._dependencies
def reset_task_state(self):
"""
Returns the task to its initial parameters and hash_dict
"""
self.parameters = copy.deepcopy(self.initial_parameters)
self.__make_hash_dict()
def __check_valid_args(self):
"""
Each task can have a valid_args list which lists the allowed parameters
"""
for k in self.parameters.keys():
if k not in self.valid_args:
raise Exception(
'{} not recognized as a valid parameter'.format(k))
def send_dependency_data(self, data):
"""
Replace TaskIOs in parameters with the corresponding data. Also adds its associated hashes to the hash dictionary
"""
glob_keys = self.parameters.find_path(
'*',
mode='contains',
action=lambda x: fnmatch.filter(list(data.keys()), x)
if '->' in x else x)
glob_keys = self._hash_dict.find_path(
'*',
mode='contains',
action=lambda x: fnmatch.filter(list(data.keys()), x)
if '->' in x else x)
for k, v in data.items():
paths = self._hash_dict.find_path(k, action=lambda x: v.get_hash())
if len(paths) > 0:
self.parameters.find_path(k, action=lambda x: v.load())
else:
if self.simulate and not k.startswith('self'):
k_ = k.split('->')[0] + '->'
paths = self._hash_dict.find_path(
k_, action=lambda x: v.get_hash(), mode='startswith')
def get_hash(self):
"""
Returns an unique identifier for the task
"""
task_hash = self.parameters.get('task_hash', None)
if task_hash is None:
return self._hash_dict.hash()
else:
return task_hash
def process(self):
pass
def find_cache(self):
"""
Finds all the associated files in cache
"""
cache_paths = find_cache(self.task_hash,
self.global_parameters['cache_path'])
return cache_paths
def __process_outputs(self, outs):
"""
Task outputs are turned into TaskIOs and saved if in_memory = False. All outputs TaskIO are returned in a dictionary.
"""
if type(outs).__name__ == 'ObjectRef':
filter_outputs = self.parameters.get('outputs', None)
if filter_outputs:
self.output_names = []
for k, v in filter_outputs.items():
self.output_names.append(k)
if not isinstance(outs, tuple):
outs = (outs, )
out_dict = {
'{}{}{}'.format(self.name, symbols['dot'], out_name):
TaskIO(out_val,
self.get_hash(),
iotype='data',
name=out_name,
position=str(i))
for i, (out_name,
out_val) in enumerate(zip(self.output_names, outs))
}
if not self.in_memory:
self.logger.info('{}: Saving outputs'.format(self.name))
for k, v in out_dict.items():
if v.iotype == 'data':
out_dict[k] = v.save(
cache_path=self.global_parameters['cache_path'],
export_path=self.export_path,
compression_level=self.
global_parameters['cache_compression'],
export=self.export,
symlink_db=self.symlinkdb_path,
overwrite_export=self.
global_parameters['overwrite_export'])
return out_dict
def __parallel_run_ray(self, run_async=False):
"""
Initializes a ray pool. Asynchronous pools are still not implemented.
"""
from ray.util.multiprocessing.pool import Pool
def set_niceness(niceness): # pool initializer
os.nice(niceness)
def worker_wrapper(x):
os.nice(self.parameters.get('niceness', 20))
for k, v in zip(self.parameters['parallel'], x):
self.parameters[k] = v
out = self.process()
return out
iterable_vars = list(
zip(*[self.parameters[k] for k in self.parameters['parallel']]))
n_cores = self.parameters.get('n_cores', 4)
pool = Pool(processes=n_cores,
initializer=set_niceness,
initargs=(self.parameters.get('niceness', 20), ),
ray_address='auto') #(Run in same host it was called)
outs = pool.map(worker_wrapper, iterable_vars)
return self.__process_outputs(outs)
def __serial_run(self, run_async=False):
"""
Run the task. Can be ran asynchronously using ray, and custom resources can be assigned through 'resources' parameter.
"""
if run_async:
import ray
import os
import sys
def run_process_async(self):
os.nice(self.parameters.get('niceness', 20))
self.logger.info('{}: Setting niceness {}'.format(
self.name, self.parameters.get('niceness', 20)))
return self.process()
resource_settings = self.parameters.get('resources', None)
if resource_settings and 'gpus' in resource_settings:
num_gpus = resource_settings['gpus']
resource_settings.pop('gpus')
else:
num_gpus = 0
if resource_settings:
outs = ray.remote(run_process_async)._remote(
args=[self],
resources=resource_settings,
num_gpus=num_gpus)
else:
outs = ray.remote(run_process_async).remote(self)
else:
outs = self.process()
return self.__process_outputs(outs)
def __serial_map(self, iteration=None, run_async=False):
"""
Run the task over each input element. Can be ran asynchronously.
"""
self.initial_parameters = copy.deepcopy(self.parameters)
self.original_name = copy.deepcopy(self.name)
self.original_export_path = copy.deepcopy(self.export_path)
map_var_names = self.parameters['map_vars']
map_vars = zip(*[self.parameters[k].load() for k in map_var_names])
if iteration is not None:
map_vars = list(map_vars)
map_vars = [map_vars[iteration]]
initial_iter = iteration
else:
initial_iter = 0
outs = []
for i, iteration in enumerate(map_vars):
self.parameters = copy.deepcopy(self.initial_parameters)
self.parameters['iter'] = i + initial_iter
self.cache_dir = Path(self.global_parameters['cache_path'],
self.task_hash)
self.export_path = Path(self.original_export_path, str(i))
self.__make_hash_dict()
self.task_hash = self.get_hash()
for k, var in zip(map_var_names, iteration):
self.parameters[k] = TaskIO(var,
self.task_hash,
iotype='data',
name=k)
if self.cache:
cache_paths = self.find_cache()
else:
cache_paths = False
if cache_paths:
self.logger.info('Caching task {}'.format(self.name))
out_dict = {
'{}{}{}'.format(self.name, symbols['dot'],
Path(cache_i).stem):
TaskIO(cache_i,
self.task_hash,
iotype='path',
name=Path(cache_i).stem,
position=Path(cache_i).parts[-2].split('_')[-1])
for cache_i in cache_paths
}
for task_name, task in out_dict.items():
task.create_link(
Path(task.data).parent, Path(self.export_path))
elif run_async:
outs.append({
'{}->ray_reference'.format(self.name):
list(self.__serial_run(run_async=run_async).values())[0],
'{}->output_names'.format(self.name):
TaskIO(self.output_names,
self.get_hash(),
iotype='data',
name='output_names',
position='1')
})
self.output_names = ['ray_reference', 'output_names']
else:
outs.append(self.__serial_run(run_async=run_async))
print('serial map')
#Restore original parameters
self.parameters = copy.deepcopy(self.initial_parameters)
self.name = copy.deepcopy(self.original_name)
self.export_path = Path(self.original_export_path, 'merged')
self.__make_hash_dict()
self.task_hash = self.get_hash()
merge_map = {}
for iter in outs:
for k, v in iter.items():
if k not in merge_map:
merge_map[k] = [v]
else:
merge_map[k].extend([v])
outs = tuple(
[[r.load() for r in merge_map['{}->{}'.format(self.name, name)]]
for name in self.output_names])
return self.__process_outputs(outs)
def run(self, iteration=None):
"""
The task is ran. If it is doing a map over the inputs, then iteration is given.
Handles the different behaviours like return_as_class/function, map and parallel.
"""
self.task_hash = self.get_hash()
self.cache_dir = Path(self.global_parameters['cache_path'],
self.task_hash)
self.export_dir = Path(self.global_parameters['output_path'],
self.name)
self.return_as_function = self.parameters.get('return_as_function',
False)
self.return_as_class = self.parameters.get('return_as_class', False)
if self.logger is not None:
self.logger.info('{}: Hash {}'.format(self.name, self.task_hash))
if self.cache:
cache_paths = self.find_cache()
else:
cache_paths = False
if cache_paths:
if self.logger is not None:
self.logger.info('{}: Caching'.format(self.name))
out_dict = {
'{}{}{}'.format(self.name, symbols['dot'],
Path(cache_i).stem):
TaskIO(cache_i,
self.task_hash,
iotype='path',
name=Path(cache_i).stem,
position=Path(cache_i).parts[-2].split('_')[-1])
for cache_i in cache_paths
}
for task_name, task in out_dict.items():
export = self.parameters.get('export', False)
task.create_link(Path(task.data).parent,
Path(self.export_path),
copy_files=export)
else:
run_async = self.parameters.get('async', False)
if self.return_as_function:
if self.logger is not None:
self.logger.info('{}: Lazy run'.format(self.name))
self.parameters['return_as_function'] = False
out_dict = self.__process_outputs(self.process)
elif self.return_as_class:
if self.logger is not None:
self.logger.info('{}: Lazy run'.format(self.name))
self.parameters['return_as_class'] = False
out_dict = self.__process_outputs(self)
elif (('parallel' not in self.parameters)
and ('map_vars' not in self.parameters)):
if self.logger is not None:
self.logger.info('{}: Running'.format(self.name))
out_dict = self.__serial_run(run_async=run_async)
elif 'parallel' in self.parameters and not 'map_vars' in self.parameters:
if self.logger is not None:
self.logger.info(
'{}: Running with pool of {} workers'.format(
self.name, self.parameters['n_cores']))
out_dict = self.__parallel_run_ray(run_async=run_async)
elif 'map_vars' in self.parameters and not 'parallel' in self.parameters:
if iteration is not None:
if self.logger is not None:
self.logger.info('{}: Running iteration {}'.format(
self.name, iteration))
else:
if self.logger is not None:
self.logger.info(
'{}: Running multiple iterations'.format(
self.name))
out_dict = self.__serial_map(iteration=iteration,
run_async=run_async)
else:
raise Exception(
'Mixing !parallel-map and !map in a task is not allowed')
return out_dict
def __init__(self, config, logger=None):
"""
Main class of the dienen library. It represents the model, which is built from a configuration file.
config: can be a string or pathlib.Path pointing to a .yaml file, a dictionary or a kahnfigh Config.
logger: optionally, a logger can be supplied to log all information related to dienen model.
"""
config = Config(config, safe=False)
self.original_config = config
self.config = copy.deepcopy(config)
self.core_model = None
self.architecture_config = None
self.model_path = None
self.name = self.config['Model'].get(
'name',
datetime.now().strftime("%d/%m/%Y %H:%M:%S"))
if not self.model_path:
self.model_path = self.config['Model'].get('path',
'{}'.format(self.name))
self.weights = None
self.optimizer_weights = None
self.extra_data = None
self.modules = self.config.get('Module', [])
self.gpu_config = self.config.get('gpu_config', {
'device': 'auto',
'allow_growth': True
})
self.cache = True
self.logger = logger
self.input_shapes = None
self.output_shapes = None
training_strategy = self.config['Model'].get('DistributedStrategy',
None)
if training_strategy is None:
self.training_strategy = tf.distribute.get_strategy()
elif training_strategy == 'Mirrored':
self.training_strategy = tf.distribute.MirroredStrategy()
if training_strategy is None:
self.gpu_device = self.gpu_config.get('device', 'auto')
if self.gpu_device == 'auto':
gpu, mem = get_available_gpus()
self.gpu_device = int(gpu)
if self.logger:
self.logger.info(
"Automatically selected device {} with {} available memory"
.format(self.gpu_device, mem))
gpu_growth = self.gpu_config.get('allow_growth', True)
gpus = tf.config.experimental.list_physical_devices('GPU')
if gpus:
try:
if len(gpus) < self.gpu_device:
raise Exception(
'There are only {} available GPUs and the {} was requested'
.format(len(gpus), self.gpu_device))
tf.config.experimental.set_visible_devices(
gpus[self.gpu_device], 'GPU')
except RuntimeError as e:
warnings.warn('Failed setting GPUs. {}'.format(e))
if gpu_growth:
for gpu in gpus:
try:
tf.config.experimental.set_memory_growth(
gpu, gpu_growth)
except RuntimeError as e:
warnings.warn(
'Failed setting GPU dynamic memory allocation. {}'.
format(e))
logical_gpus = tf.config.experimental.list_logical_devices('GPU')
if self.logger:
self.logger.debug("Physical GPUs: {}. Logical GPUs: {}".format(
len(gpus), len(logical_gpus)))
self.externals = self.config['Model'].get('External', None)
self.validation_data = None