def test_create_mm_sqlite():
"""prepare_combos.create_mm_sqlite: test create_mm_sqlite
based on test example 'simple1'.
"""
output_filename = 'scores.sqlite'
recipe_filename = 'data/simple1_recipe.xml'
morph_dir = 'data/morphs/'
rep_morph_dir = 'data/rep_morphs/'
emodel_dir = os.path.join(TEST_DIR, 'data/emodels_dir/subdir/')
emodel_etype_map = tools.load_json(
os.path.join(emodel_dir, 'emodel_etype_map.json'))
final_dict = tools.load_json(os.path.join(emodel_dir, 'final.json'))
emodel_dirs = {m: emodel_dir for m in ['emodel1', 'emodel2']}
skip_repaired_exemplar = True
with tools.cd(TEST_DIR):
create_mm_sqlite.create_mm_sqlite(output_filename, recipe_filename,
morph_dir, rep_morph_dir,
rep_morph_dir, emodel_etype_map,
final_dict, emodel_dirs,
skip_repaired_exemplar)
nt.assert_true(os.path.isfile(output_filename))
# TODO: test database contents
# clear output
os.remove(output_filename)
def run_combos_from_conf(conf_dict,
ipyp=None,
ipyp_profile=None,
timeout=10,
n_processes=None):
"""Run combos from conf dictionary"""
output_dir = conf_dict['output_dir']
final_dict = tools.load_json(os.path.join(output_dir, 'final.json'))
emodel_dirs = tools.load_json(os.path.join(output_dir, 'emodel_dirs.json'))
scores_db_path = os.path.abspath(conf_dict['scores_db'])
if 'use_apical_points' in conf_dict:
use_apical_points = conf_dict['use_apical_points']
else:
use_apical_points = True
print('Calculating scores')
calculate_scores.calculate_scores(final_dict,
emodel_dirs,
scores_db_path,
use_ipyp=ipyp,
ipyp_profile=ipyp_profile,
timeout=timeout,
use_apical_points=use_apical_points,
n_processes=n_processes)
def run_combos_from_conf(conf_dict, ipyp=None, ipyp_profile=None):
"""Run combos from conf dictionary"""
output_dir = conf_dict['output_dir']
final_dict = tools.load_json(os.path.join(output_dir, 'final.json'))
emodel_dirs = tools.load_json(os.path.join(output_dir, 'emodel_dirs.json'))
scores_db_path = os.path.abspath(conf_dict['scores_db'])
print('Calculating scores')
calculate_scores.calculate_scores(final_dict,
emodel_dirs,
scores_db_path,
use_ipyp=ipyp,
ipyp_profile=ipyp_profile)
def _get_emodel_infos(conf_dict_path):
"""Return information from each emodel of a run
Args:
conf_dict_path (str): path to select.json for the run
Returns:
tuple with structure from table_processing.process_emodels
"""
conf_dict = tools.load_json(conf_dict_path)
scores, score_values = sqlite_io.read_and_process_sqlite_score_tables(
conf_dict["scores_db"]
)
emodels = sorted(scores[scores.is_original == 0].emodel.unique())
to_skip_patterns, to_skip_features = proc_config.read_to_skip_features(
conf_dict
)
megate_patterns, megate_thresholds = proc_config.read_megate_thresholds(
conf_dict
)
return table_processing.process_emodels(
emodels,
scores,
score_values,
to_skip_patterns,
megate_patterns,
conf_dict.get("skip_repaired_exemplar", False),
conf_dict.get("check_opt_scores", True),
conf_dict.get("select_perc_best", None),
)
def _verify_emodel_json(filename, output_dir, nb_emodels):
"""Helper function to verify emodel json output"""
data_json = os.path.join(output_dir, filename)
nt.assert_true(os.path.isfile(data_json))
data = tools.load_json(data_json)
nt.assert_equal(len(data), nb_emodels)
return data
def run_combos(conf_filename, ipyp=None, ipyp_profile=None):
"""Run combos"""
print('Reading configuration at %s' % conf_filename)
conf_dict = tools.load_json(conf_filename)
run_combos_from_conf(conf_dict, ipyp, ipyp_profile)
def test_calculate_scores():
"""run_combos.calculate_scores: test calculate_scores"""
# write database
test_db_filename = os.path.join(TMP_DIR, 'test4.sqlite')
morph_name = 'morph1'
morph_dir = './data/morphs'
mtype = 'mtype1'
etype = 'etype1'
layer = 1
emodel = 'emodel1'
exception = None
row = {
'morph_name': morph_name,
'morph_ext': None,
'morph_dir': morph_dir,
'mtype': mtype,
'etype': etype,
'layer': layer,
'emodel': emodel,
'original_emodel': emodel,
'to_run': 1,
'scores': None,
'extra_values': None,
'exception': exception
}
_write_test_scores_database(row, test_db_filename)
# calculate scores
emodel_dir = os.path.join(TEST_DIR, 'data/emodels_dir/subdir/')
emodel_dirs = {emodel: emodel_dir}
final_dict_path = os.path.join(emodel_dir, 'final.json')
final_dict = tools.load_json(final_dict_path)
with tools.cd(TEST_DIR):
run_combos.calculate_scores.calculate_scores(final_dict, emodel_dirs,
test_db_filename)
# verify database
scores = {'Step1.SpikeCount': 20.0}
extra_values = {'holding_current': None, 'threshold_current': None}
expected_db_row = {
'index': 0,
'morph_name': morph_name,
'morph_ext': None,
'morph_dir': morph_dir,
'mtype': mtype,
'etype': etype,
'layer': layer,
'emodel': emodel,
'original_emodel': emodel,
'to_run': 0,
'scores': json.dumps(scores),
'extra_values': json.dumps(extra_values),
'exception': exception
}
with sqlite3.connect(test_db_filename) as scores_db:
scores_db.row_factory = _dict_factory
scores_cursor = scores_db.execute('SELECT * FROM scores')
db_row = scores_cursor.fetchall()[0]
nt.assert_dict_equal(db_row, expected_db_row)
def _prepare_config_json(original_filename, test_dir):
"""Helper function to prepare new configuration file."""
config = tools.load_json(original_filename)
config['tmp_dir'] = os.path.join(test_dir, 'tmp')
config['output_dir'] = os.path.join(test_dir, 'output')
config['scores_db'] = os.path.join(config['output_dir'], 'scores.sqlite')
config['emodels_hoc_dir'] = os.path.join(config['output_dir'],
'emodels_hoc')
tools.makedirs(test_dir)
return tools.write_json(test_dir, 'config.json', config)
def test_json():
"""bluepymm.tools: test load_json and write_json"""
output_dir = TMP_DIR
output_name = 'tmp.json'
config = {'test': ['1', 'two']}
tools.makedirs(output_dir)
ret_path = tools.write_json(output_dir, output_name, config)
nt.assert_equal(os.path.join(output_dir, output_name), ret_path)
ret = tools.load_json(ret_path)
nt.assert_dict_equal(config, ret)
def _config_select_combos(config_template_path, tmp_dir):
"""Helper function to prepare input data for select_combos"""
# copy input data
shutil.copytree('output_expected', tmp_dir)
# set configuration dict
config = tools.load_json(config_template_path)
config['scores_db'] = os.path.join(tmp_dir, 'scores.sqlite')
config['pdf_filename'] = os.path.join(tmp_dir, 'megating.pdf')
config['output_dir'] = os.path.join(tmp_dir, 'output')
config['emodels_hoc_dir'] = os.path.join(tmp_dir, 'output/emodels_hoc')
return config
def main(arg_list):
"""Main"""
# parse and process arguments
args = get_parser().parse_args(arg_list)
config = tools.load_json(args.conf_filename)
config_dir = os.path.abspath(os.path.dirname(args.conf_filename))
config = add_full_paths(config, config_dir)
# process configuration
mecombo_emodel_filename = config['mecombo_emodel_filename']
combinations_dict = load_combinations_dict(mecombo_emodel_filename)
final_dict = tools.load_json(config['final_json_path'])
emodels_dir = config['emodels_tmp_dir']
# create output directory for .hoc files
tools.makedirs(config['hoc_output_dir'])
# create hoc files
create_hoc_files(combinations_dict, emodels_dir, final_dict,
config['template'], config['hoc_output_dir'])
def get_emodel_dicts(emodels_dir, final_json_path, emodel_etype_map_path):
"""Read and return detailed e-model information.
Args:
emodels_dir: Path to BluePyMM file structure.
final_json_path: Path to final e-model map, relative to
`emodels_dir`.
emodel_etype_map_path: Path to e-model e-type map, relative to
`emodels_dir`.
Returns:
(string, dict, dict)-tuple with:
- final e-model map,
- e-model e-type map,
- name of directory containing final e-model map.
"""
final_dict_path = os.path.join(emodels_dir, final_json_path)
final_dict = tools.load_json(final_dict_path)
e_map_path = os.path.join(emodels_dir, emodel_etype_map_path)
emodel_etype_map = tools.load_json(e_map_path)
dict_dir = os.path.dirname(final_dict_path)
return final_dict, emodel_etype_map, dict_dir
def test_run_combos():
"""bluepymm.run_combos: test run_combos based on example simple1"""
config_template_path = 'simple1_conf_run.json'
with tools.cd(TEST_DATA_DIR):
# prepare input data
shutil.copytree('output_expected', TMP_DIR)
config = tools.load_json(config_template_path)
config['scores_db'] = os.path.join(TMP_DIR, 'scores.sqlite')
config['output_dir'] = TMP_DIR
# run combination preparation
run_combos.main.run_combos_from_conf(config)
# verify output
_verify_run_combos_output(config['scores_db'])
def _test_prepare_combos(test_data_dir, config_template_path, nb_emodels,
test_dir):
"""Helper function to perform functional test prepare_combos"""
with tools.cd(test_data_dir):
# prepare new configuration file based on test_dir
config_path = _prepare_config_json(config_template_path, test_dir)
# run combination preparation
prepare_combos.main.prepare_combos(conf_filename=config_path,
continu=False)
# test output
config = tools.load_json(config_path)
_verify_prepare_combos_output(config['scores_db'],
config['emodels_hoc_dir'],
config['output_dir'], nb_emodels)
def prepare_combos(conf_filename, continu, n_processes=None):
"""Prepare combos"""
print('Reading configuration at %s' % conf_filename)
conf_dict = tools.load_json(conf_filename)
scores_db_path = os.path.abspath(conf_dict['scores_db'])
final_dict, emodel_dirs = prepare_emodels(conf_dict, continu,
scores_db_path, n_processes)
# Save output
# TODO: gather all output business here?
output_dir = conf_dict['output_dir']
tools.makedirs(output_dir)
tools.write_json(output_dir, 'final.json', final_dict)
tools.write_json(output_dir, 'emodel_dirs.json', emodel_dirs)
def test_calculate_scores():
"""run_combos.calculate_scores: test calculate_scores"""
# write database
test_db_filename = os.path.join(TMP_DIR, 'test4.sqlite')
morph_name = 'morph1'
morph_dir = os.path.join(TEST_DIR, 'data/morphs')
mtype = 'mtype1'
etype = 'etype1'
layer = 1
emodel = 'emodel1'
exception = None
row = {
'morph_name': morph_name,
'morph_ext': None,
'morph_dir': morph_dir,
'mtype': mtype,
'etype': etype,
'layer': layer,
'emodel': emodel,
'original_emodel': emodel,
'to_run': 1,
'scores': None,
'extra_values': None,
'exception': exception
}
_write_test_scores_database(row, test_db_filename)
# calculate scores
emodel_dir = os.path.join(TEST_DIR, 'data/emodels_dir/subdir/')
emodel_dirs = {emodel: emodel_dir}
final_dict_path = os.path.join(emodel_dir, 'final.json')
final_dict = tools.load_json(final_dict_path)
for use_ipyp in [False, True]:
if use_ipyp:
ip_proc = subprocess.Popen(["ipcluster", "start", "-n=2"])
# ensure that ipcluster has enough time to start
time.sleep(10)
with tools.cd(TEST_DIR):
run_combos.calculate_scores.calculate_scores(final_dict,
emodel_dirs,
test_db_filename,
use_ipyp=use_ipyp)
if use_ipyp:
ip_proc.terminate()
# verify database
scores = {'Step1.SpikeCount': 20.0}
extra_values = {'holding_current': None, 'threshold_current': None}
expected_db_row = {
'index': 0,
'morph_name': morph_name,
'morph_ext': None,
'morph_dir': morph_dir,
'mtype': mtype,
'etype': etype,
'layer': layer,
'emodel': emodel,
'original_emodel': emodel,
'to_run': 0,
'scores': json.dumps(scores),
'extra_values': json.dumps(extra_values),
'exception': exception
}
with sqlite3.connect(test_db_filename) as scores_db:
scores_db.row_factory = _dict_factory
scores_cursor = scores_db.execute('SELECT * FROM scores')
db_row = scores_cursor.fetchall()[0]
nt.assert_dict_equal(db_row, expected_db_row)
def create_arg_list(scores_db_filename, emodel_dirs, final_dict,
extra_values_error=False, use_apical_points=True):
"""Create list of argument tuples to be used as an input for
run_emodel_morph.
Args:
scores_db_filename: path to .sqlite database
emodel_dirs: a dict mapping e-models to the directories with e-model
input files
final_dict: a dict mapping e-models to dicts with e-model parameters
extra_values_error: boolean to raise an exception upon a missing key
use_apical_points: boolean to use apical points or not
Raises:
ValueError, if one of the database entries contains has value None for
the key 'emodel'.
"""
arg_list = []
with sqlite3.connect(scores_db_filename) as scores_db:
scores_db.row_factory = sqlite3.Row
one_row = scores_db.execute('SELECT * FROM scores LIMIT 1').fetchone()
apical_points_isec = {}
setup = tools.load_module('setup', emodel_dirs[one_row['emodel']])
if hasattr(setup, 'multieval') and use_apical_points:
apical_points_isec = tools.load_json(
os.path.join(one_row['morph_dir'], "apical_points_isec.json")
)
scores_cursor = scores_db.execute('SELECT * FROM scores')
for row in scores_cursor.fetchall():
index = row['index']
morph_name = row['morph_name']
morph_ext = row['morph_ext']
if morph_ext is None:
morph_ext = '.asc'
apical_point_isec = None
if morph_name in apical_points_isec:
apical_point_isec = int(apical_points_isec[morph_name])
morph_filename = morph_name + morph_ext
morph_path = os.path.abspath(os.path.join(row['morph_dir'],
morph_filename))
if row['to_run'] == 1:
emodel = row['emodel']
original_emodel = row['original_emodel']
if emodel is None:
raise ValueError(
"scores db row %s for morph %s, etype %s, mtype %s, "
"layer %s doesn't have an e-model assigned to it" %
(index, morph_name, row['etype'], row['mtype'],
row['layer']))
args = (index, emodel,
os.path.abspath(emodel_dirs[emodel]),
final_dict[original_emodel]['params'],
morph_path, apical_point_isec, extra_values_error)
arg_list.append(args)
print('Found %d rows in score database to run' % len(arg_list))
return arg_list
def select_combos(conf_filename):
"""Parse conf file and run select combos"""
# Parse configuration file
conf_dict = tools.load_json(conf_filename)
select_combos_from_conf(conf_dict)
