def convert_emodel_input(emodels_in_repo, conf_dict, continu): """Convert e-model input to BluePyMM file structure and return path to that structure. Args: emodels_in_repo: True if the input e-models are organized in separate branches of a git repository, false if the e-models are organized into separate subdirectories. conf_dict: A dict with e-model input configuration. continu: True if this BluePyMM run builds on a previous run, False otherwise Returns: Path to BluePyMM file structure. """ tmp_emodels_dir = os.path.abspath( os.path.join(conf_dict['tmp_dir'], 'emodels_repo')) if not continu: if emodels_in_repo: print('Cloning input e-models repository in %s' % tmp_emodels_dir) sh.git('clone', conf_dict['emodels_repo'], tmp_emodels_dir) with tools.cd(tmp_emodels_dir): sh.git('checkout', conf_dict['emodels_githash']) else: shutil.copytree(conf_dict['emodels_dir'], tmp_emodels_dir) return tmp_emodels_dir
def test_create_mm_sqlite(): """prepare_combos.create_mm_sqlite: test create_mm_sqlite based on test example 'simple1'. """ output_filename = 'scores.sqlite' recipe_filename = 'data/simple1_recipe.xml' morph_dir = 'data/morphs/' rep_morph_dir = 'data/rep_morphs/' emodel_dir = os.path.join(TEST_DIR, 'data/emodels_dir/subdir/') emodel_etype_map = tools.load_json( os.path.join(emodel_dir, 'emodel_etype_map.json')) final_dict = tools.load_json(os.path.join(emodel_dir, 'final.json')) emodel_dirs = {m: emodel_dir for m in ['emodel1', 'emodel2']} skip_repaired_exemplar = True with tools.cd(TEST_DIR): create_mm_sqlite.create_mm_sqlite(output_filename, recipe_filename, morph_dir, rep_morph_dir, rep_morph_dir, emodel_etype_map, final_dict, emodel_dirs, skip_repaired_exemplar) nt.assert_true(os.path.isfile(output_filename)) # TODO: test database contents # clear output os.remove(output_filename)
def test_calculate_scores(): """run_combos.calculate_scores: test calculate_scores""" # write database test_db_filename = os.path.join(TMP_DIR, 'test4.sqlite') morph_name = 'morph1' morph_dir = './data/morphs' mtype = 'mtype1' etype = 'etype1' layer = 1 emodel = 'emodel1' exception = None row = { 'morph_name': morph_name, 'morph_ext': None, 'morph_dir': morph_dir, 'mtype': mtype, 'etype': etype, 'layer': layer, 'emodel': emodel, 'original_emodel': emodel, 'to_run': 1, 'scores': None, 'extra_values': None, 'exception': exception } _write_test_scores_database(row, test_db_filename) # calculate scores emodel_dir = os.path.join(TEST_DIR, 'data/emodels_dir/subdir/') emodel_dirs = {emodel: emodel_dir} final_dict_path = os.path.join(emodel_dir, 'final.json') final_dict = tools.load_json(final_dict_path) with tools.cd(TEST_DIR): run_combos.calculate_scores.calculate_scores(final_dict, emodel_dirs, test_db_filename) # verify database scores = {'Step1.SpikeCount': 20.0} extra_values = {'holding_current': None, 'threshold_current': None} expected_db_row = { 'index': 0, 'morph_name': morph_name, 'morph_ext': None, 'morph_dir': morph_dir, 'mtype': mtype, 'etype': etype, 'layer': layer, 'emodel': emodel, 'original_emodel': emodel, 'to_run': 0, 'scores': json.dumps(scores), 'extra_values': json.dumps(extra_values), 'exception': exception } with sqlite3.connect(test_db_filename) as scores_db: scores_db.row_factory = _dict_factory scores_cursor = scores_db.execute('SELECT * FROM scores') db_row = scores_cursor.fetchall()[0] nt.assert_dict_equal(db_row, expected_db_row)
def _test_convert_emodel_input(test_dir, emodels_in_repo, conf_dict, continu): """Helper function to test convert_emodel_input""" with tools.cd(test_dir): ret_dir = prepare_emodel_dirs.convert_emodel_input(emodels_in_repo, conf_dict, continu) nt.assert_true(os.path.isdir(ret_dir)) for k in ['emodel_etype_map_path', 'final_json_path']: nt.assert_true(os.path.isfile(os.path.join(ret_dir, conf_dict[k])))
def test_cd(): """bluepymm.tools: test cd""" old_cwd = os.getcwd() with tools.cd(EXAMPLES): nt.assert_equal(os.getcwd(), EXAMPLES) nt.assert_equal(old_cwd, os.getcwd())
def test_prepare_emodel_dirs_multi_process(): """prepare_combos.prepare_emodel_dirs_multi_process: test prepare_emodel_dirs based on test example 'simple1' using multiprocessing. """ # create test directory, where output of this test will be created test_dir = os.path.join(TMP_DIR, 'test_prepare_emodel_dirs') tools.makedirs(test_dir) # input parameters final_dict = {'emodel1': {'main_path': '.', 'seed': 2, 'rank': 0, 'notes': '', 'branch': 'emodel1', 'params': {'cm': 1.0}, 'fitness': {'Step1.SpikeCount': 20.0}, 'score': 104.72906197480131, 'morph_path': 'morphologies/morph1.asc'}, 'emodel2': {'main_path': '.', 'seed': 2, 'rank': 0, 'notes': '', 'branch': 'emodel2', 'params': {'cm': 0.5}, 'fitness': {'Step1.SpikeCount': 20.0}, 'score': 104.72906197480131, 'morph_path': 'morphologies/morph2.asc'}} emodel_etype_map = {'emodel1': {'mm_recipe': 'emodel1', 'etype': 'etype1', 'layer': ['1', 'str1']}, 'emodel2': {'mm_recipe': 'emodel2', 'etype': 'etype2', 'layer': ['1', '2']}} emodels_dir = os.path.join(test_dir, 'tmp/emodels/') opt_dir = os.path.join(TEST_DATA_DIR, 'data/emodels_dir/subdir/') emodels_hoc_dir = os.path.join(test_dir, './output/emodels_hoc/') hoc_template = os.path.join( TEMPLATE_DIR, "cell_template_neurodamus.jinja2" ) hoc_template = os.path.abspath(hoc_template) emodels_in_repo = False continu = False # run function with tools.cd(TEST_DATA_DIR): ret = prepare_emodel_dirs.prepare_emodel_dirs( final_dict, emodel_etype_map, emodels_dir, opt_dir, emodels_hoc_dir, emodels_in_repo, hoc_template, continu, n_processes=None) # verify output expected_ret = {emodel: os.path.join( emodels_dir, emodel) for emodel in final_dict} nt.assert_dict_equal(ret, expected_ret) nt.assert_true(os.path.isdir(emodels_dir)) nt.assert_true(os.path.isdir(emodels_hoc_dir))
def _test_select_combos(test_data_dir, tmp_dir, config_template_path, benchmark_dir, n_processes=None): """Helper function to perform functional test of select_combos""" with tools.cd(test_data_dir): # prepare input data config = _config_select_combos(config_template_path, tmp_dir) # run combination selection select_combos.main.select_combos_from_conf(config, n_processes) # verify output _verify_output(benchmark_dir, config['output_dir'])
def test_prepare_emodel_dir(): """prepare_combos.prepare_emodel_dirs: test prepare_emodel_dir based on test example 'simple1'. """ # create test directory, where output of this test will be created test_dir = os.path.join(TMP_DIR, 'test_prepare_emodel_dir') tools.makedirs(test_dir) # input parameters original_emodel = 'emodel1' emodel = 'emodel1' emodel_dict = { 'main_path': '.', 'seed': 2, 'rank': 0, 'notes': '', 'branch': 'emodel1', 'params': { 'cm': 1.0 }, 'fitness': { 'Step1.SpikeCount': 20.0 }, 'score': 104.72906197480131, 'morph_path': 'morphologies/morph1.asc' } emodels_dir = os.path.join(test_dir, 'tmp/emodels/') opt_dir = os.path.join(TEST_DATA_DIR, 'data/emodels_dir/subdir/') hoc_dir = os.path.join(test_dir, 'output/emodels_hoc/') emodels_in_repo = False continu = False with tools.cd(TEST_DATA_DIR): # create output directories and run function for path in [emodels_dir, hoc_dir]: tools.makedirs(path) arg_list = (original_emodel, emodel, emodel_dict, emodels_dir, opt_dir, os.path.abspath(hoc_dir), emodels_in_repo, continu) ret = prepare_emodel_dirs.prepare_emodel_dir(arg_list) # test side effects: creation of .hoc-file nt.assert_true(os.path.isdir(os.path.join(emodels_dir, emodel))) hoc_path = os.path.join(hoc_dir, '{}.hoc'.format(emodel)) nt.assert_true(os.path.isfile(hoc_path)) # test returned dict expected_emodel_dir = os.path.join(emodels_dir, emodel) expected_ret = { emodel: expected_emodel_dir, original_emodel: expected_emodel_dir } nt.assert_dict_equal(ret, expected_ret)
def test_get_emodel_dicts(): """prepare_combos.prepare_emodel_dirs: test get_emodel_dicts based on test example 'simple1'. """ emodels_dir = './data/emodels_dir' final_json_path = 'subdir/final.json' emodel_etype_map_path = 'subdir/emodel_etype_map.json' expected_final_dict = { 'emodel1': { 'main_path': '.', 'seed': 2, 'rank': 0, 'notes': '', 'branch': 'emodel1', 'params': { 'cm': 1.0}, 'fitness': { 'Step1.SpikeCount': 20.0}, 'score': 104.72906197480131, 'morph_path': 'morphologies/morph1.asc'}, 'emodel2': { 'main_path': '.', 'seed': 2, 'rank': 0, 'notes': '', 'branch': 'emodel2', 'params': { 'cm': 0.5}, 'fitness': { 'Step1.SpikeCount': 20.0}, 'score': 104.72906197480131, 'morph_path': 'morphologies/morph2.asc'}} expected_emodel_etype_map = {'emodel1': {'mm_recipe': 'emodel1', 'etype': 'etype1', 'layer': ['1', 'str1']}, 'emodel2': {'mm_recipe': 'emodel2', 'etype': 'etype2', 'layer': ['1', '2']}} expected_dict_dir = os.path.dirname(os.path.join(emodels_dir, final_json_path)) with tools.cd(TEST_DATA_DIR): final, ee_map, dict_dir = prepare_emodel_dirs.get_emodel_dicts( emodels_dir, final_json_path, emodel_etype_map_path) nt.assert_dict_equal(expected_final_dict, final) nt.assert_dict_equal(expected_emodel_etype_map, ee_map) nt.assert_equal(expected_dict_dir, dict_dir)
def test_run_combos(): """bluepymm.run_combos: test run_combos based on example simple1""" config_template_path = 'simple1_conf_run.json' with tools.cd(TEST_DATA_DIR): # prepare input data shutil.copytree('output_expected', TMP_DIR) config = tools.load_json(config_template_path) config['scores_db'] = os.path.join(TMP_DIR, 'scores.sqlite') config['output_dir'] = TMP_DIR # run combination preparation run_combos.main.run_combos_from_conf(config) # verify output _verify_run_combos_output(config['scores_db'])
def _test_prepare_combos(test_data_dir, config_template_path, nb_emodels, test_dir): """Helper function to perform functional test prepare_combos""" with tools.cd(test_data_dir): # prepare new configuration file based on test_dir config_path = _prepare_config_json(config_template_path, test_dir) # run combination preparation prepare_combos.main.prepare_combos(conf_filename=config_path, continu=False) # test output config = tools.load_json(config_path) _verify_prepare_combos_output(config['scores_db'], config['emodels_hoc_dir'], config['output_dir'], nb_emodels)
def _test_create_and_write_hoc_file(test_dir, emodel, emodel_dir, hoc_dir, emodel_parameters, template, morph_path, model_name): """Test create_and_write_hoc_files""" with tools.cd(test_dir): tools.makedirs(hoc_dir) prepare_emodel_dirs.create_and_write_hoc_file(emodel, emodel_dir, hoc_dir, emodel_parameters, template, morph_path=morph_path, model_name=model_name) # TODO: test hoc file contents template_name = model_name or emodel hoc_filename = '{}.hoc'.format(template_name) hoc_path = os.path.join(hoc_dir, hoc_filename) nt.assert_true(os.path.isfile(hoc_path))
def prepare_emodel_dir(input_args): """Clone e-model input and prepare the e-model directory. Args: input_args: 9-tuple - original_emodel(str): e-model name - emodel(str): e-model name - emodel_dict: dict with all e-model parameters - emodels_dir: directory with all e-models - opt_dir: directory with all opt e-models (TODO: clarify) - hoc_dir: absolute path to the directory to which the .hoc files will be written out - emodels_in_repo: True if the input e-models are organized in separate branches of a git repository, false if the e-models are organized into separate subdirectories. - continu: True if this BluePyMM run builds on a previous run, False otherwise Returns: A dict mapping the e-model and the original e-model to the e-model dir """ original_emodel, emodel, emodel_dict, emodels_dir, \ opt_dir, hoc_dir, emodels_in_repo, continu = input_args try: print('Preparing: %s' % emodel) emodel_dir = os.path.join(emodels_dir, emodel) if not continu: tar_filename = os.path.abspath( os.path.join(emodels_dir, '%s.tar' % emodel)) main_path = emodel_dict.get('main_path', '.') with tools.cd(os.path.join(opt_dir, main_path)): if emodels_in_repo: sh.git('archive', '--format=tar', '--prefix=%s/' % emodel, 'origin/%s' % emodel_dict['branch'], _out=tar_filename) else: with tarfile.open(tar_filename, 'w') as tar_file: tar_file.add('.', arcname=emodel) # extract in e-model directory, compile mechanisms and create .hoc # TODO: clean up .tar file? with tools.cd(emodels_dir): sh.tar('xf', tar_filename) with tools.cd(emodel): print('Compiling mechanisms ...') if os.path.exists('mechanisms'): sh.nrnivmodl('mechanisms') create_and_write_hoc_file( emodel, emodel_dir, hoc_dir, emodel_dict['params'], 'cell_template_neurodamus.jinja2') os.remove(tar_filename) except Exception: raise Exception(''.join(traceback.format_exception(*sys.exc_info()))) return {emodel: emodel_dir, original_emodel: emodel_dir}
def test_calculate_scores(): """run_combos.calculate_scores: test calculate_scores""" # write database test_db_filename = os.path.join(TMP_DIR, 'test4.sqlite') morph_name = 'morph1' morph_dir = os.path.join(TEST_DIR, 'data/morphs') mtype = 'mtype1' etype = 'etype1' layer = 1 emodel = 'emodel1' exception = None row = { 'morph_name': morph_name, 'morph_ext': None, 'morph_dir': morph_dir, 'mtype': mtype, 'etype': etype, 'layer': layer, 'emodel': emodel, 'original_emodel': emodel, 'to_run': 1, 'scores': None, 'extra_values': None, 'exception': exception } _write_test_scores_database(row, test_db_filename) # calculate scores emodel_dir = os.path.join(TEST_DIR, 'data/emodels_dir/subdir/') emodel_dirs = {emodel: emodel_dir} final_dict_path = os.path.join(emodel_dir, 'final.json') final_dict = tools.load_json(final_dict_path) for use_ipyp in [False, True]: if use_ipyp: ip_proc = subprocess.Popen(["ipcluster", "start", "-n=2"]) # ensure that ipcluster has enough time to start time.sleep(10) with tools.cd(TEST_DIR): run_combos.calculate_scores.calculate_scores(final_dict, emodel_dirs, test_db_filename, use_ipyp=use_ipyp) if use_ipyp: ip_proc.terminate() # verify database scores = {'Step1.SpikeCount': 20.0} extra_values = {'holding_current': None, 'threshold_current': None} expected_db_row = { 'index': 0, 'morph_name': morph_name, 'morph_ext': None, 'morph_dir': morph_dir, 'mtype': mtype, 'etype': etype, 'layer': layer, 'emodel': emodel, 'original_emodel': emodel, 'to_run': 0, 'scores': json.dumps(scores), 'extra_values': json.dumps(extra_values), 'exception': exception } with sqlite3.connect(test_db_filename) as scores_db: scores_db.row_factory = _dict_factory scores_cursor = scores_db.execute('SELECT * FROM scores') db_row = scores_cursor.fetchall()[0] nt.assert_dict_equal(db_row, expected_db_row)
def test_create_exemplar_rows_skip_repaired_exemplar(): """prepare_combos.create_mm_sqlite: test create_exemplar_rows based on test example 'simple1'. """ emodel = 'emodel1' final_dict = { emodel: { 'main_path': '.', 'seed': 2, 'rank': 0, 'notes': '', 'branch': 'emodel1', 'params': { 'cm': 1.0 }, 'fitness': { 'Step1.SpikeCount': 20.0 }, 'score': 104.72906197480131, 'morph_path': 'morphologies/morph1.asc' } } fullmtype_morph_map = {} # not used in case of skip_repaired_exemplar emodel_etype_map = { emodel: { 'mm_recipe': 'emodel1', 'etype': 'etype1', 'layer': ['1', 'str1'] } } emodels = ['emodel1'] emodel_dir = os.path.join(TEST_DIR, 'data/emodels_dir/subdir/') emodel_dirs = {emodel: emodel_dir} rep_morph_dir = os.path.join(TEST_DIR, 'data/morphs') skip_repaired_exemplar = True # construct expected output unrep_morph_dir = os.path.dirname( os.path.join(emodel_dirs[emodel], final_dict[emodel]['morph_path'])) with tools.cd(TEST_DIR): ret = create_mm_sqlite.create_exemplar_rows(final_dict, fullmtype_morph_map, emodel_etype_map, emodels, emodel_dirs, rep_morph_dir, unrep_morph_dir, skip_repaired_exemplar) data = [(None, None, None, None, emodel_etype_map[emodel]['etype'], 'morph1', '.asc', emodel, emodel, unrep_morph_dir, None, json.dumps(final_dict[emodel]['fitness']), None, True, True, False, False), (None, None, None, None, emodel_etype_map[emodel]['etype'], 'morph1', '.asc', emodel, emodel, unrep_morph_dir, None, json.dumps(final_dict[emodel]['fitness']), None, True, True, False, True)] columns = [ 'layer', 'fullmtype', 'mtype', 'msubtype', 'etype', 'morph_name', 'morph_ext', 'emodel', 'original_emodel', 'morph_dir', 'scores', 'opt_scores', 'exception', 'to_run', 'is_exemplar', 'is_repaired', 'is_original' ] expected_ret = pandas.DataFrame(data, columns=columns) expected_ret.sort_index(axis=1, inplace=True) ret.sort_index(axis=1, inplace=True) pandas.testing.assert_frame_equal(ret, expected_ret)
def run_emodel_morph( emodel, emodel_dir, emodel_params, morph_path, apical_point_isec, extra_values_error=True): """Run e-model morphology combination. Args: emodel: e-model name emodel_dir: directory containing e-model files emodel_params: dict that maps e-model parameters to their values morph_path: path to morphology apical_point_isec: integer value of the apical point isection extra_values_error: boolean to raise an exception upon a missing key Returns: tuple: - dict that maps features to scores - dict with extra values: 'holding_current' and 'threshold_current' """ try: sys.stdout = open('/dev/null', 'w') print('Running e-model %s on morphology %s in %s' % (emodel, morph_path, emodel_dir)) setup = tools.load_module('setup', emodel_dir) print("Changing path to %s" % emodel_dir) with tools.cd(emodel_dir): if hasattr(setup, 'multieval'): prefix = 'mm' altmorph = [[prefix, morph_path, apical_point_isec]] evaluator = setup.evaluator.create(etype='%s' % emodel, altmorph=altmorph) evaluator = evaluator.evaluators[0] # only one evaluator responses = evaluator.run_protocols( evaluator.fitness_protocols.values(), emodel_params) scores = evaluator.fitness_calculator.calculate_scores( responses) extra_values = {} for response_key, extra_values_key in [ ('%s.bpo_holding_current' % prefix, 'holding_current'), ('%s.bpo_threshold_current' % prefix, 'threshold_current')]: if response_key in responses: extra_values[extra_values_key] = responses[ response_key] else: if extra_values_error: raise ValueError( "Key %s not found in responses: %s" % (response_key, str(responses))) else: extra_values[extra_values_key] = None else: evaluator = setup.evaluator.create(etype='%s' % emodel) evaluator.cell_model.morphology.morphology_path = morph_path responses = evaluator.run_protocols( evaluator.fitness_protocols.values(), emodel_params) scores = evaluator.fitness_calculator.calculate_scores( responses) extra_values = {} extra_values['holding_current'] = \ responses.get('bpo_holding_current', None) extra_values['threshold_current'] = \ responses.get('bpo_threshold_current', None) return scores, extra_values except Exception: # Make sure exception and backtrace are thrown back to parent process raise Exception( "".join(traceback.format_exception(*sys.exc_info())))