def _filter_experiment_record_list(user_range, experiment_record_ids):
if user_range=='all':
return [True]*len(experiment_record_ids)
elif user_range=='new':
return detect_duplicates(experiment_record_ids, key=record_id_to_experiment_id, keep_last=True)
# return [n for n, is_old in izip_equal(get_record_ids(), old) if not old]
elif user_range=='old':
return [not x for x in _filter_records(user_range, 'new')]
elif user_range=='orphans':
orphans = []
global_lib = get_global_experiment_library()
for i, record_id in enumerate(experiment_record_ids):
info = load_experiment_record(record_id).info
if 'Module' in info:
try:
import_module(info['Module'])
if not record_id_to_experiment_id(record_id) in global_lib:
orphans.append(True)
else:
orphans.append(False)
except ImportError:
orphans.append(True)
else: # They must be old... lets kill them!
orphans.append(True)
return orphans
# elif user_range
else:
which_ones = interpret_numbers(user_range)
if which_ones is None:
raise Exception('Could not interpret user range: "{}"'.format(user_range))
filters = [False]*len(experiment_record_ids)
for i in which_ones:
filters[i] = True
return filters
def _filter_experiment_record_list(user_range, experiment_record_ids):
if user_range=='all':
return [True]*len(experiment_record_ids)
elif user_range=='new':
return detect_duplicates(experiment_record_ids, key=record_id_to_experiment_id, keep_last=True)
# return [n for n, is_old in izip_equal(get_record_ids(), old) if not old]
elif user_range=='old':
return [not x for x in _filter_records(user_range, 'new')]
elif user_range=='orphans':
orphans = []
global_lib = get_global_experiment_library()
for i, record_id in enumerate(experiment_record_ids):
info = load_experiment_record(record_id).info
if 'Module' in info:
try:
import_module(info['Module'])
if not record_id_to_experiment_id(record_id) in global_lib:
orphans.append(True)
else:
orphans.append(False)
except ImportError:
orphans.append(True)
else: # They must be old... lets kill them!
orphans.append(True)
return orphans
else:
which_ones = interpret_numbers(user_range)
if which_ones is None:
raise Exception('Could not interpret user range: "{}"'.format(user_range))
filters = [False]*len(experiment_record_ids)
for i in which_ones:
filters[i] = True
return filters
def test_detect_duplicates():
assert detect_duplicates(['a', 'b', 'a', 'c',
'c']) == [False, False, True, False, True]
assert detect_duplicates(['a', 'b', 'a', 'c', 'c'], keep_last=True) == [
True, False, False, True, False
]
def test_detect_duplicates():
assert detect_duplicates(['a', 'b', 'a', 'c', 'c'])==[False, False, True, False, True]
assert detect_duplicates(['a', 'b', 'a', 'c', 'c'], keep_last=True)==[True, False, False, True, False]
def copy_modules_to_dir(object,
destination_dir,
root_package,
code_subpackage=None,
helper_subpackage='helpers',
scope='project',
clear_old_package=False):
modules, names = get_module_import_dict(object, scope=scope)
if isinstance(names, basestring):
names = [names]
root_dir = os.path.join(destination_dir, root_package)
if clear_old_package and os.path.exists(root_dir):
rmtree(root_dir)
code_dir = root_dir if code_subpackage is None else os.path.join(
root_dir, code_subpackage)
code_module_name = root_package if code_subpackage is None else root_package + '.' + code_subpackage
helper_module_name = root_package if helper_subpackage is None else root_package + '.' + helper_subpackage
helper_dir = root_dir if helper_subpackage is None else os.path.join(
root_dir, helper_subpackage)
for direct in (root_dir, code_dir, helper_dir):
try:
os.makedirs(direct)
except OSError:
pass
with open(os.path.join(direct, '__init__.py'), 'w') as f:
f.write('')
old_name_to_new_name = {
module_name:
code_module_name + '.' + module_name.split('.')[-1] if module_name
in names else helper_module_name + '.' + module_name.split('.')[-1]
for module_name in modules.keys()
}
duplicates = {
k: v
for (k, v), isdup in izip_equal(
old_name_to_new_name.iteritems(),
detect_duplicates(old_name_to_new_name.values())) if isdup
}
if len(duplicates) > 0:
raise Exception(
'There is a collision between two or more modules names: {}\n. You need to rename these modules to have unique names.'
.format(duplicates))
old_name_to_new_path = {
module_name: os.path.join(code_dir,
os.path.split(module_file)[1]) if module_name
in names else os.path.join(helper_dir,
os.path.split(module_file)[1])
for module_name, module_file in modules.iteritems()
}
for module_name, module_path in modules.iteritems():
_, file_name = os.path.split(module_path)
with open(module_path) as f:
txt = f.read()
for dep_module_name, new_module_name in old_name_to_new_name.iteritems(
):
txt = txt.replace('from {} import '.format(dep_module_name),
'from {} import '.format(new_module_name))
txt = txt.replace('import {}'.format(dep_module_name),
'import {}'.format(new_module_name))
with open(old_name_to_new_path[module_name], 'w') as f:
f.write(txt)
LOGGER.info('Copied {} -> {}'.format(
module_path, old_name_to_new_path[module_name]))