def convert_emodel_input(emodels_in_repo, conf_dict, continu):
"""Convert e-model input to BluePyMM file structure and return path to that
structure.
Args:
emodels_in_repo: True if the input e-models are organized in separate
branches of a git repository, false if the e-models are organized
into separate subdirectories.
conf_dict: A dict with e-model input configuration.
continu: True if this BluePyMM run builds on a previous run, False
otherwise
Returns:
Path to BluePyMM file structure.
"""
tmp_emodels_dir = os.path.abspath(
os.path.join(conf_dict['tmp_dir'], 'emodels_repo'))
if not continu:
if emodels_in_repo:
print('Cloning input e-models repository in %s' % tmp_emodels_dir)
sh.git('clone', conf_dict['emodels_repo'], tmp_emodels_dir)
with tools.cd(tmp_emodels_dir):
sh.git('checkout', conf_dict['emodels_githash'])
else:
shutil.copytree(conf_dict['emodels_dir'], tmp_emodels_dir)
return tmp_emodels_dir
def test_create_mm_sqlite():
"""prepare_combos.create_mm_sqlite: test create_mm_sqlite
based on test example 'simple1'.
"""
output_filename = 'scores.sqlite'
recipe_filename = 'data/simple1_recipe.xml'
morph_dir = 'data/morphs/'
rep_morph_dir = 'data/rep_morphs/'
emodel_dir = os.path.join(TEST_DIR, 'data/emodels_dir/subdir/')
emodel_etype_map = tools.load_json(
os.path.join(emodel_dir, 'emodel_etype_map.json'))
final_dict = tools.load_json(os.path.join(emodel_dir, 'final.json'))
emodel_dirs = {m: emodel_dir for m in ['emodel1', 'emodel2']}
skip_repaired_exemplar = True
with tools.cd(TEST_DIR):
create_mm_sqlite.create_mm_sqlite(output_filename, recipe_filename,
morph_dir, rep_morph_dir,
rep_morph_dir, emodel_etype_map,
final_dict, emodel_dirs,
skip_repaired_exemplar)
nt.assert_true(os.path.isfile(output_filename))
# TODO: test database contents
# clear output
os.remove(output_filename)
def test_calculate_scores():
"""run_combos.calculate_scores: test calculate_scores"""
# write database
test_db_filename = os.path.join(TMP_DIR, 'test4.sqlite')
morph_name = 'morph1'
morph_dir = './data/morphs'
mtype = 'mtype1'
etype = 'etype1'
layer = 1
emodel = 'emodel1'
exception = None
row = {
'morph_name': morph_name,
'morph_ext': None,
'morph_dir': morph_dir,
'mtype': mtype,
'etype': etype,
'layer': layer,
'emodel': emodel,
'original_emodel': emodel,
'to_run': 1,
'scores': None,
'extra_values': None,
'exception': exception
}
_write_test_scores_database(row, test_db_filename)
# calculate scores
emodel_dir = os.path.join(TEST_DIR, 'data/emodels_dir/subdir/')
emodel_dirs = {emodel: emodel_dir}
final_dict_path = os.path.join(emodel_dir, 'final.json')
final_dict = tools.load_json(final_dict_path)
with tools.cd(TEST_DIR):
run_combos.calculate_scores.calculate_scores(final_dict, emodel_dirs,
test_db_filename)
# verify database
scores = {'Step1.SpikeCount': 20.0}
extra_values = {'holding_current': None, 'threshold_current': None}
expected_db_row = {
'index': 0,
'morph_name': morph_name,
'morph_ext': None,
'morph_dir': morph_dir,
'mtype': mtype,
'etype': etype,
'layer': layer,
'emodel': emodel,
'original_emodel': emodel,
'to_run': 0,
'scores': json.dumps(scores),
'extra_values': json.dumps(extra_values),
'exception': exception
}
with sqlite3.connect(test_db_filename) as scores_db:
scores_db.row_factory = _dict_factory
scores_cursor = scores_db.execute('SELECT * FROM scores')
db_row = scores_cursor.fetchall()[0]
nt.assert_dict_equal(db_row, expected_db_row)
def _test_convert_emodel_input(test_dir, emodels_in_repo, conf_dict, continu):
"""Helper function to test convert_emodel_input"""
with tools.cd(test_dir):
ret_dir = prepare_emodel_dirs.convert_emodel_input(emodels_in_repo,
conf_dict, continu)
nt.assert_true(os.path.isdir(ret_dir))
for k in ['emodel_etype_map_path', 'final_json_path']:
nt.assert_true(os.path.isfile(os.path.join(ret_dir, conf_dict[k])))
def test_cd():
"""bluepymm.tools: test cd"""
old_cwd = os.getcwd()
with tools.cd(EXAMPLES):
nt.assert_equal(os.getcwd(), EXAMPLES)
nt.assert_equal(old_cwd, os.getcwd())
def test_prepare_emodel_dirs_multi_process():
"""prepare_combos.prepare_emodel_dirs_multi_process:
test prepare_emodel_dirs based on test example 'simple1'
using multiprocessing.
"""
# create test directory, where output of this test will be created
test_dir = os.path.join(TMP_DIR, 'test_prepare_emodel_dirs')
tools.makedirs(test_dir)
# input parameters
final_dict = {'emodel1': {'main_path': '.',
'seed': 2,
'rank': 0,
'notes': '',
'branch': 'emodel1',
'params': {'cm': 1.0},
'fitness': {'Step1.SpikeCount': 20.0},
'score': 104.72906197480131,
'morph_path': 'morphologies/morph1.asc'},
'emodel2': {'main_path': '.',
'seed': 2,
'rank': 0,
'notes': '',
'branch': 'emodel2',
'params': {'cm': 0.5},
'fitness': {'Step1.SpikeCount': 20.0},
'score': 104.72906197480131,
'morph_path': 'morphologies/morph2.asc'}}
emodel_etype_map = {'emodel1': {'mm_recipe': 'emodel1',
'etype': 'etype1',
'layer': ['1', 'str1']},
'emodel2': {'mm_recipe': 'emodel2',
'etype': 'etype2',
'layer': ['1', '2']}}
emodels_dir = os.path.join(test_dir, 'tmp/emodels/')
opt_dir = os.path.join(TEST_DATA_DIR, 'data/emodels_dir/subdir/')
emodels_hoc_dir = os.path.join(test_dir, './output/emodels_hoc/')
hoc_template = os.path.join(
TEMPLATE_DIR, "cell_template_neurodamus.jinja2"
)
hoc_template = os.path.abspath(hoc_template)
emodels_in_repo = False
continu = False
# run function
with tools.cd(TEST_DATA_DIR):
ret = prepare_emodel_dirs.prepare_emodel_dirs(
final_dict, emodel_etype_map, emodels_dir, opt_dir,
emodels_hoc_dir, emodels_in_repo, hoc_template, continu,
n_processes=None)
# verify output
expected_ret = {emodel: os.path.join(
emodels_dir, emodel) for emodel in final_dict}
nt.assert_dict_equal(ret, expected_ret)
nt.assert_true(os.path.isdir(emodels_dir))
nt.assert_true(os.path.isdir(emodels_hoc_dir))
def _test_select_combos(test_data_dir, tmp_dir, config_template_path,
benchmark_dir, n_processes=None):
"""Helper function to perform functional test of select_combos"""
with tools.cd(test_data_dir):
# prepare input data
config = _config_select_combos(config_template_path, tmp_dir)
# run combination selection
select_combos.main.select_combos_from_conf(config, n_processes)
# verify output
_verify_output(benchmark_dir, config['output_dir'])
def test_prepare_emodel_dir():
"""prepare_combos.prepare_emodel_dirs: test prepare_emodel_dir
based on test example 'simple1'.
"""
# create test directory, where output of this test will be created
test_dir = os.path.join(TMP_DIR, 'test_prepare_emodel_dir')
tools.makedirs(test_dir)
# input parameters
original_emodel = 'emodel1'
emodel = 'emodel1'
emodel_dict = {
'main_path': '.',
'seed': 2,
'rank': 0,
'notes': '',
'branch': 'emodel1',
'params': {
'cm': 1.0
},
'fitness': {
'Step1.SpikeCount': 20.0
},
'score': 104.72906197480131,
'morph_path': 'morphologies/morph1.asc'
}
emodels_dir = os.path.join(test_dir, 'tmp/emodels/')
opt_dir = os.path.join(TEST_DATA_DIR, 'data/emodels_dir/subdir/')
hoc_dir = os.path.join(test_dir, 'output/emodels_hoc/')
emodels_in_repo = False
continu = False
with tools.cd(TEST_DATA_DIR):
# create output directories and run function
for path in [emodels_dir, hoc_dir]:
tools.makedirs(path)
arg_list = (original_emodel, emodel, emodel_dict, emodels_dir, opt_dir,
os.path.abspath(hoc_dir), emodels_in_repo, continu)
ret = prepare_emodel_dirs.prepare_emodel_dir(arg_list)
# test side effects: creation of .hoc-file
nt.assert_true(os.path.isdir(os.path.join(emodels_dir, emodel)))
hoc_path = os.path.join(hoc_dir, '{}.hoc'.format(emodel))
nt.assert_true(os.path.isfile(hoc_path))
# test returned dict
expected_emodel_dir = os.path.join(emodels_dir, emodel)
expected_ret = {
emodel: expected_emodel_dir,
original_emodel: expected_emodel_dir
}
nt.assert_dict_equal(ret, expected_ret)
def test_get_emodel_dicts():
"""prepare_combos.prepare_emodel_dirs: test get_emodel_dicts
based on test example 'simple1'.
"""
emodels_dir = './data/emodels_dir'
final_json_path = 'subdir/final.json'
emodel_etype_map_path = 'subdir/emodel_etype_map.json'
expected_final_dict = {
'emodel1': {
'main_path': '.',
'seed': 2,
'rank': 0,
'notes': '',
'branch': 'emodel1',
'params': {
'cm': 1.0},
'fitness': {
'Step1.SpikeCount': 20.0},
'score': 104.72906197480131,
'morph_path': 'morphologies/morph1.asc'},
'emodel2': {
'main_path': '.',
'seed': 2,
'rank': 0,
'notes': '',
'branch': 'emodel2',
'params': {
'cm': 0.5},
'fitness': {
'Step1.SpikeCount': 20.0},
'score': 104.72906197480131,
'morph_path': 'morphologies/morph2.asc'}}
expected_emodel_etype_map = {'emodel1': {'mm_recipe': 'emodel1',
'etype': 'etype1',
'layer': ['1', 'str1']},
'emodel2': {'mm_recipe': 'emodel2',
'etype': 'etype2',
'layer': ['1', '2']}}
expected_dict_dir = os.path.dirname(os.path.join(emodels_dir,
final_json_path))
with tools.cd(TEST_DATA_DIR):
final, ee_map, dict_dir = prepare_emodel_dirs.get_emodel_dicts(
emodels_dir, final_json_path, emodel_etype_map_path)
nt.assert_dict_equal(expected_final_dict, final)
nt.assert_dict_equal(expected_emodel_etype_map, ee_map)
nt.assert_equal(expected_dict_dir, dict_dir)
def test_run_combos():
"""bluepymm.run_combos: test run_combos based on example simple1"""
config_template_path = 'simple1_conf_run.json'
with tools.cd(TEST_DATA_DIR):
# prepare input data
shutil.copytree('output_expected', TMP_DIR)
config = tools.load_json(config_template_path)
config['scores_db'] = os.path.join(TMP_DIR, 'scores.sqlite')
config['output_dir'] = TMP_DIR
# run combination preparation
run_combos.main.run_combos_from_conf(config)
# verify output
_verify_run_combos_output(config['scores_db'])
def _test_prepare_combos(test_data_dir, config_template_path, nb_emodels,
test_dir):
"""Helper function to perform functional test prepare_combos"""
with tools.cd(test_data_dir):
# prepare new configuration file based on test_dir
config_path = _prepare_config_json(config_template_path, test_dir)
# run combination preparation
prepare_combos.main.prepare_combos(conf_filename=config_path,
continu=False)
# test output
config = tools.load_json(config_path)
_verify_prepare_combos_output(config['scores_db'],
config['emodels_hoc_dir'],
config['output_dir'], nb_emodels)
def _test_create_and_write_hoc_file(test_dir, emodel, emodel_dir, hoc_dir,
emodel_parameters, template, morph_path,
model_name):
"""Test create_and_write_hoc_files"""
with tools.cd(test_dir):
tools.makedirs(hoc_dir)
prepare_emodel_dirs.create_and_write_hoc_file(emodel,
emodel_dir,
hoc_dir,
emodel_parameters,
template,
morph_path=morph_path,
model_name=model_name)
# TODO: test hoc file contents
template_name = model_name or emodel
hoc_filename = '{}.hoc'.format(template_name)
hoc_path = os.path.join(hoc_dir, hoc_filename)
nt.assert_true(os.path.isfile(hoc_path))
def prepare_emodel_dir(input_args):
"""Clone e-model input and prepare the e-model directory.
Args:
input_args: 9-tuple
- original_emodel(str): e-model name
- emodel(str): e-model name
- emodel_dict: dict with all e-model parameters
- emodels_dir: directory with all e-models
- opt_dir: directory with all opt e-models (TODO: clarify)
- hoc_dir: absolute path to the directory to which the .hoc
files will be written out
- emodels_in_repo: True if the input e-models are organized
in separate branches of
a git repository, false if the e-models are organized into
separate subdirectories.
- continu: True if this BluePyMM run builds on a previous run,
False otherwise
Returns:
A dict mapping the e-model and the original e-model to the e-model dir
"""
original_emodel, emodel, emodel_dict, emodels_dir, \
opt_dir, hoc_dir, emodels_in_repo, continu = input_args
try:
print('Preparing: %s' % emodel)
emodel_dir = os.path.join(emodels_dir, emodel)
if not continu:
tar_filename = os.path.abspath(
os.path.join(emodels_dir, '%s.tar' % emodel))
main_path = emodel_dict.get('main_path', '.')
with tools.cd(os.path.join(opt_dir, main_path)):
if emodels_in_repo:
sh.git('archive',
'--format=tar',
'--prefix=%s/' % emodel,
'origin/%s' % emodel_dict['branch'],
_out=tar_filename)
else:
with tarfile.open(tar_filename, 'w') as tar_file:
tar_file.add('.', arcname=emodel)
# extract in e-model directory, compile mechanisms and create .hoc
# TODO: clean up .tar file?
with tools.cd(emodels_dir):
sh.tar('xf', tar_filename)
with tools.cd(emodel):
print('Compiling mechanisms ...')
if os.path.exists('mechanisms'):
sh.nrnivmodl('mechanisms')
create_and_write_hoc_file(
emodel, emodel_dir, hoc_dir, emodel_dict['params'],
'cell_template_neurodamus.jinja2')
os.remove(tar_filename)
except Exception:
raise Exception(''.join(traceback.format_exception(*sys.exc_info())))
return {emodel: emodel_dir, original_emodel: emodel_dir}
def test_calculate_scores():
"""run_combos.calculate_scores: test calculate_scores"""
# write database
test_db_filename = os.path.join(TMP_DIR, 'test4.sqlite')
morph_name = 'morph1'
morph_dir = os.path.join(TEST_DIR, 'data/morphs')
mtype = 'mtype1'
etype = 'etype1'
layer = 1
emodel = 'emodel1'
exception = None
row = {
'morph_name': morph_name,
'morph_ext': None,
'morph_dir': morph_dir,
'mtype': mtype,
'etype': etype,
'layer': layer,
'emodel': emodel,
'original_emodel': emodel,
'to_run': 1,
'scores': None,
'extra_values': None,
'exception': exception
}
_write_test_scores_database(row, test_db_filename)
# calculate scores
emodel_dir = os.path.join(TEST_DIR, 'data/emodels_dir/subdir/')
emodel_dirs = {emodel: emodel_dir}
final_dict_path = os.path.join(emodel_dir, 'final.json')
final_dict = tools.load_json(final_dict_path)
for use_ipyp in [False, True]:
if use_ipyp:
ip_proc = subprocess.Popen(["ipcluster", "start", "-n=2"])
# ensure that ipcluster has enough time to start
time.sleep(10)
with tools.cd(TEST_DIR):
run_combos.calculate_scores.calculate_scores(final_dict,
emodel_dirs,
test_db_filename,
use_ipyp=use_ipyp)
if use_ipyp:
ip_proc.terminate()
# verify database
scores = {'Step1.SpikeCount': 20.0}
extra_values = {'holding_current': None, 'threshold_current': None}
expected_db_row = {
'index': 0,
'morph_name': morph_name,
'morph_ext': None,
'morph_dir': morph_dir,
'mtype': mtype,
'etype': etype,
'layer': layer,
'emodel': emodel,
'original_emodel': emodel,
'to_run': 0,
'scores': json.dumps(scores),
'extra_values': json.dumps(extra_values),
'exception': exception
}
with sqlite3.connect(test_db_filename) as scores_db:
scores_db.row_factory = _dict_factory
scores_cursor = scores_db.execute('SELECT * FROM scores')
db_row = scores_cursor.fetchall()[0]
nt.assert_dict_equal(db_row, expected_db_row)
def test_create_exemplar_rows_skip_repaired_exemplar():
"""prepare_combos.create_mm_sqlite: test create_exemplar_rows
based on test example 'simple1'.
"""
emodel = 'emodel1'
final_dict = {
emodel: {
'main_path': '.',
'seed': 2,
'rank': 0,
'notes': '',
'branch': 'emodel1',
'params': {
'cm': 1.0
},
'fitness': {
'Step1.SpikeCount': 20.0
},
'score': 104.72906197480131,
'morph_path': 'morphologies/morph1.asc'
}
}
fullmtype_morph_map = {} # not used in case of skip_repaired_exemplar
emodel_etype_map = {
emodel: {
'mm_recipe': 'emodel1',
'etype': 'etype1',
'layer': ['1', 'str1']
}
}
emodels = ['emodel1']
emodel_dir = os.path.join(TEST_DIR, 'data/emodels_dir/subdir/')
emodel_dirs = {emodel: emodel_dir}
rep_morph_dir = os.path.join(TEST_DIR, 'data/morphs')
skip_repaired_exemplar = True
# construct expected output
unrep_morph_dir = os.path.dirname(
os.path.join(emodel_dirs[emodel], final_dict[emodel]['morph_path']))
with tools.cd(TEST_DIR):
ret = create_mm_sqlite.create_exemplar_rows(final_dict,
fullmtype_morph_map,
emodel_etype_map, emodels,
emodel_dirs, rep_morph_dir,
unrep_morph_dir,
skip_repaired_exemplar)
data = [(None, None, None, None, emodel_etype_map[emodel]['etype'],
'morph1', '.asc', emodel, emodel, unrep_morph_dir, None,
json.dumps(final_dict[emodel]['fitness']), None, True, True,
False, False),
(None, None, None, None, emodel_etype_map[emodel]['etype'],
'morph1', '.asc', emodel, emodel, unrep_morph_dir, None,
json.dumps(final_dict[emodel]['fitness']), None, True, True,
False, True)]
columns = [
'layer', 'fullmtype', 'mtype', 'msubtype', 'etype', 'morph_name',
'morph_ext', 'emodel', 'original_emodel', 'morph_dir', 'scores',
'opt_scores', 'exception', 'to_run', 'is_exemplar', 'is_repaired',
'is_original'
]
expected_ret = pandas.DataFrame(data, columns=columns)
expected_ret.sort_index(axis=1, inplace=True)
ret.sort_index(axis=1, inplace=True)
pandas.testing.assert_frame_equal(ret, expected_ret)
def run_emodel_morph(
emodel,
emodel_dir,
emodel_params,
morph_path,
apical_point_isec,
extra_values_error=True):
"""Run e-model morphology combination.
Args:
emodel: e-model name
emodel_dir: directory containing e-model files
emodel_params: dict that maps e-model parameters to their values
morph_path: path to morphology
apical_point_isec: integer value of the apical point isection
extra_values_error: boolean to raise an exception upon a missing key
Returns:
tuple:
- dict that maps features to scores
- dict with extra values: 'holding_current' and 'threshold_current'
"""
try:
sys.stdout = open('/dev/null', 'w')
print('Running e-model %s on morphology %s in %s' %
(emodel, morph_path, emodel_dir))
setup = tools.load_module('setup', emodel_dir)
print("Changing path to %s" % emodel_dir)
with tools.cd(emodel_dir):
if hasattr(setup, 'multieval'):
prefix = 'mm'
altmorph = [[prefix, morph_path, apical_point_isec]]
evaluator = setup.evaluator.create(etype='%s' % emodel,
altmorph=altmorph)
evaluator = evaluator.evaluators[0] # only one evaluator
responses = evaluator.run_protocols(
evaluator.fitness_protocols.values(),
emodel_params)
scores = evaluator.fitness_calculator.calculate_scores(
responses)
extra_values = {}
for response_key, extra_values_key in [
('%s.bpo_holding_current' % prefix,
'holding_current'),
('%s.bpo_threshold_current' % prefix,
'threshold_current')]:
if response_key in responses:
extra_values[extra_values_key] = responses[
response_key]
else:
if extra_values_error:
raise ValueError(
"Key %s not found in responses: %s" %
(response_key, str(responses)))
else:
extra_values[extra_values_key] = None
else:
evaluator = setup.evaluator.create(etype='%s' % emodel)
evaluator.cell_model.morphology.morphology_path = morph_path
responses = evaluator.run_protocols(
evaluator.fitness_protocols.values(),
emodel_params)
scores = evaluator.fitness_calculator.calculate_scores(
responses)
extra_values = {}
extra_values['holding_current'] = \
responses.get('bpo_holding_current', None)
extra_values['threshold_current'] = \
responses.get('bpo_threshold_current', None)
return scores, extra_values
except Exception:
# Make sure exception and backtrace are thrown back to parent process
raise Exception(
"".join(traceback.format_exception(*sys.exc_info())))
