def test_create_mm_sqlite(): """prepare_combos.create_mm_sqlite: test create_mm_sqlite based on test example 'simple1'. """ output_filename = 'scores.sqlite' recipe_filename = 'data/simple1_recipe.xml' morph_dir = 'data/morphs/' rep_morph_dir = 'data/rep_morphs/' emodel_dir = os.path.join(TEST_DIR, 'data/emodels_dir/subdir/') emodel_etype_map = tools.load_json( os.path.join(emodel_dir, 'emodel_etype_map.json')) final_dict = tools.load_json(os.path.join(emodel_dir, 'final.json')) emodel_dirs = {m: emodel_dir for m in ['emodel1', 'emodel2']} skip_repaired_exemplar = True with tools.cd(TEST_DIR): create_mm_sqlite.create_mm_sqlite(output_filename, recipe_filename, morph_dir, rep_morph_dir, rep_morph_dir, emodel_etype_map, final_dict, emodel_dirs, skip_repaired_exemplar) nt.assert_true(os.path.isfile(output_filename)) # TODO: test database contents # clear output os.remove(output_filename)
def run_combos_from_conf(conf_dict, ipyp=None, ipyp_profile=None, timeout=10, n_processes=None): """Run combos from conf dictionary""" output_dir = conf_dict['output_dir'] final_dict = tools.load_json(os.path.join(output_dir, 'final.json')) emodel_dirs = tools.load_json(os.path.join(output_dir, 'emodel_dirs.json')) scores_db_path = os.path.abspath(conf_dict['scores_db']) if 'use_apical_points' in conf_dict: use_apical_points = conf_dict['use_apical_points'] else: use_apical_points = True print('Calculating scores') calculate_scores.calculate_scores(final_dict, emodel_dirs, scores_db_path, use_ipyp=ipyp, ipyp_profile=ipyp_profile, timeout=timeout, use_apical_points=use_apical_points, n_processes=n_processes)
def run_combos_from_conf(conf_dict, ipyp=None, ipyp_profile=None): """Run combos from conf dictionary""" output_dir = conf_dict['output_dir'] final_dict = tools.load_json(os.path.join(output_dir, 'final.json')) emodel_dirs = tools.load_json(os.path.join(output_dir, 'emodel_dirs.json')) scores_db_path = os.path.abspath(conf_dict['scores_db']) print('Calculating scores') calculate_scores.calculate_scores(final_dict, emodel_dirs, scores_db_path, use_ipyp=ipyp, ipyp_profile=ipyp_profile)
def _get_emodel_infos(conf_dict_path): """Return information from each emodel of a run Args: conf_dict_path (str): path to select.json for the run Returns: tuple with structure from table_processing.process_emodels """ conf_dict = tools.load_json(conf_dict_path) scores, score_values = sqlite_io.read_and_process_sqlite_score_tables( conf_dict["scores_db"] ) emodels = sorted(scores[scores.is_original == 0].emodel.unique()) to_skip_patterns, to_skip_features = proc_config.read_to_skip_features( conf_dict ) megate_patterns, megate_thresholds = proc_config.read_megate_thresholds( conf_dict ) return table_processing.process_emodels( emodels, scores, score_values, to_skip_patterns, megate_patterns, conf_dict.get("skip_repaired_exemplar", False), conf_dict.get("check_opt_scores", True), conf_dict.get("select_perc_best", None), )
def _verify_emodel_json(filename, output_dir, nb_emodels): """Helper function to verify emodel json output""" data_json = os.path.join(output_dir, filename) nt.assert_true(os.path.isfile(data_json)) data = tools.load_json(data_json) nt.assert_equal(len(data), nb_emodels) return data
def run_combos(conf_filename, ipyp=None, ipyp_profile=None): """Run combos""" print('Reading configuration at %s' % conf_filename) conf_dict = tools.load_json(conf_filename) run_combos_from_conf(conf_dict, ipyp, ipyp_profile)
def test_calculate_scores(): """run_combos.calculate_scores: test calculate_scores""" # write database test_db_filename = os.path.join(TMP_DIR, 'test4.sqlite') morph_name = 'morph1' morph_dir = './data/morphs' mtype = 'mtype1' etype = 'etype1' layer = 1 emodel = 'emodel1' exception = None row = { 'morph_name': morph_name, 'morph_ext': None, 'morph_dir': morph_dir, 'mtype': mtype, 'etype': etype, 'layer': layer, 'emodel': emodel, 'original_emodel': emodel, 'to_run': 1, 'scores': None, 'extra_values': None, 'exception': exception } _write_test_scores_database(row, test_db_filename) # calculate scores emodel_dir = os.path.join(TEST_DIR, 'data/emodels_dir/subdir/') emodel_dirs = {emodel: emodel_dir} final_dict_path = os.path.join(emodel_dir, 'final.json') final_dict = tools.load_json(final_dict_path) with tools.cd(TEST_DIR): run_combos.calculate_scores.calculate_scores(final_dict, emodel_dirs, test_db_filename) # verify database scores = {'Step1.SpikeCount': 20.0} extra_values = {'holding_current': None, 'threshold_current': None} expected_db_row = { 'index': 0, 'morph_name': morph_name, 'morph_ext': None, 'morph_dir': morph_dir, 'mtype': mtype, 'etype': etype, 'layer': layer, 'emodel': emodel, 'original_emodel': emodel, 'to_run': 0, 'scores': json.dumps(scores), 'extra_values': json.dumps(extra_values), 'exception': exception } with sqlite3.connect(test_db_filename) as scores_db: scores_db.row_factory = _dict_factory scores_cursor = scores_db.execute('SELECT * FROM scores') db_row = scores_cursor.fetchall()[0] nt.assert_dict_equal(db_row, expected_db_row)
def _prepare_config_json(original_filename, test_dir): """Helper function to prepare new configuration file.""" config = tools.load_json(original_filename) config['tmp_dir'] = os.path.join(test_dir, 'tmp') config['output_dir'] = os.path.join(test_dir, 'output') config['scores_db'] = os.path.join(config['output_dir'], 'scores.sqlite') config['emodels_hoc_dir'] = os.path.join(config['output_dir'], 'emodels_hoc') tools.makedirs(test_dir) return tools.write_json(test_dir, 'config.json', config)
def test_json(): """bluepymm.tools: test load_json and write_json""" output_dir = TMP_DIR output_name = 'tmp.json' config = {'test': ['1', 'two']} tools.makedirs(output_dir) ret_path = tools.write_json(output_dir, output_name, config) nt.assert_equal(os.path.join(output_dir, output_name), ret_path) ret = tools.load_json(ret_path) nt.assert_dict_equal(config, ret)
def _config_select_combos(config_template_path, tmp_dir): """Helper function to prepare input data for select_combos""" # copy input data shutil.copytree('output_expected', tmp_dir) # set configuration dict config = tools.load_json(config_template_path) config['scores_db'] = os.path.join(tmp_dir, 'scores.sqlite') config['pdf_filename'] = os.path.join(tmp_dir, 'megating.pdf') config['output_dir'] = os.path.join(tmp_dir, 'output') config['emodels_hoc_dir'] = os.path.join(tmp_dir, 'output/emodels_hoc') return config
def main(arg_list): """Main""" # parse and process arguments args = get_parser().parse_args(arg_list) config = tools.load_json(args.conf_filename) config_dir = os.path.abspath(os.path.dirname(args.conf_filename)) config = add_full_paths(config, config_dir) # process configuration mecombo_emodel_filename = config['mecombo_emodel_filename'] combinations_dict = load_combinations_dict(mecombo_emodel_filename) final_dict = tools.load_json(config['final_json_path']) emodels_dir = config['emodels_tmp_dir'] # create output directory for .hoc files tools.makedirs(config['hoc_output_dir']) # create hoc files create_hoc_files(combinations_dict, emodels_dir, final_dict, config['template'], config['hoc_output_dir'])
def get_emodel_dicts(emodels_dir, final_json_path, emodel_etype_map_path): """Read and return detailed e-model information. Args: emodels_dir: Path to BluePyMM file structure. final_json_path: Path to final e-model map, relative to `emodels_dir`. emodel_etype_map_path: Path to e-model e-type map, relative to `emodels_dir`. Returns: (string, dict, dict)-tuple with: - final e-model map, - e-model e-type map, - name of directory containing final e-model map. """ final_dict_path = os.path.join(emodels_dir, final_json_path) final_dict = tools.load_json(final_dict_path) e_map_path = os.path.join(emodels_dir, emodel_etype_map_path) emodel_etype_map = tools.load_json(e_map_path) dict_dir = os.path.dirname(final_dict_path) return final_dict, emodel_etype_map, dict_dir
def test_run_combos(): """bluepymm.run_combos: test run_combos based on example simple1""" config_template_path = 'simple1_conf_run.json' with tools.cd(TEST_DATA_DIR): # prepare input data shutil.copytree('output_expected', TMP_DIR) config = tools.load_json(config_template_path) config['scores_db'] = os.path.join(TMP_DIR, 'scores.sqlite') config['output_dir'] = TMP_DIR # run combination preparation run_combos.main.run_combos_from_conf(config) # verify output _verify_run_combos_output(config['scores_db'])
def _test_prepare_combos(test_data_dir, config_template_path, nb_emodels, test_dir): """Helper function to perform functional test prepare_combos""" with tools.cd(test_data_dir): # prepare new configuration file based on test_dir config_path = _prepare_config_json(config_template_path, test_dir) # run combination preparation prepare_combos.main.prepare_combos(conf_filename=config_path, continu=False) # test output config = tools.load_json(config_path) _verify_prepare_combos_output(config['scores_db'], config['emodels_hoc_dir'], config['output_dir'], nb_emodels)
def prepare_combos(conf_filename, continu, n_processes=None): """Prepare combos""" print('Reading configuration at %s' % conf_filename) conf_dict = tools.load_json(conf_filename) scores_db_path = os.path.abspath(conf_dict['scores_db']) final_dict, emodel_dirs = prepare_emodels(conf_dict, continu, scores_db_path, n_processes) # Save output # TODO: gather all output business here? output_dir = conf_dict['output_dir'] tools.makedirs(output_dir) tools.write_json(output_dir, 'final.json', final_dict) tools.write_json(output_dir, 'emodel_dirs.json', emodel_dirs)
def test_calculate_scores(): """run_combos.calculate_scores: test calculate_scores""" # write database test_db_filename = os.path.join(TMP_DIR, 'test4.sqlite') morph_name = 'morph1' morph_dir = os.path.join(TEST_DIR, 'data/morphs') mtype = 'mtype1' etype = 'etype1' layer = 1 emodel = 'emodel1' exception = None row = { 'morph_name': morph_name, 'morph_ext': None, 'morph_dir': morph_dir, 'mtype': mtype, 'etype': etype, 'layer': layer, 'emodel': emodel, 'original_emodel': emodel, 'to_run': 1, 'scores': None, 'extra_values': None, 'exception': exception } _write_test_scores_database(row, test_db_filename) # calculate scores emodel_dir = os.path.join(TEST_DIR, 'data/emodels_dir/subdir/') emodel_dirs = {emodel: emodel_dir} final_dict_path = os.path.join(emodel_dir, 'final.json') final_dict = tools.load_json(final_dict_path) for use_ipyp in [False, True]: if use_ipyp: ip_proc = subprocess.Popen(["ipcluster", "start", "-n=2"]) # ensure that ipcluster has enough time to start time.sleep(10) with tools.cd(TEST_DIR): run_combos.calculate_scores.calculate_scores(final_dict, emodel_dirs, test_db_filename, use_ipyp=use_ipyp) if use_ipyp: ip_proc.terminate() # verify database scores = {'Step1.SpikeCount': 20.0} extra_values = {'holding_current': None, 'threshold_current': None} expected_db_row = { 'index': 0, 'morph_name': morph_name, 'morph_ext': None, 'morph_dir': morph_dir, 'mtype': mtype, 'etype': etype, 'layer': layer, 'emodel': emodel, 'original_emodel': emodel, 'to_run': 0, 'scores': json.dumps(scores), 'extra_values': json.dumps(extra_values), 'exception': exception } with sqlite3.connect(test_db_filename) as scores_db: scores_db.row_factory = _dict_factory scores_cursor = scores_db.execute('SELECT * FROM scores') db_row = scores_cursor.fetchall()[0] nt.assert_dict_equal(db_row, expected_db_row)
def create_arg_list(scores_db_filename, emodel_dirs, final_dict, extra_values_error=False, use_apical_points=True): """Create list of argument tuples to be used as an input for run_emodel_morph. Args: scores_db_filename: path to .sqlite database emodel_dirs: a dict mapping e-models to the directories with e-model input files final_dict: a dict mapping e-models to dicts with e-model parameters extra_values_error: boolean to raise an exception upon a missing key use_apical_points: boolean to use apical points or not Raises: ValueError, if one of the database entries contains has value None for the key 'emodel'. """ arg_list = [] with sqlite3.connect(scores_db_filename) as scores_db: scores_db.row_factory = sqlite3.Row one_row = scores_db.execute('SELECT * FROM scores LIMIT 1').fetchone() apical_points_isec = {} setup = tools.load_module('setup', emodel_dirs[one_row['emodel']]) if hasattr(setup, 'multieval') and use_apical_points: apical_points_isec = tools.load_json( os.path.join(one_row['morph_dir'], "apical_points_isec.json") ) scores_cursor = scores_db.execute('SELECT * FROM scores') for row in scores_cursor.fetchall(): index = row['index'] morph_name = row['morph_name'] morph_ext = row['morph_ext'] if morph_ext is None: morph_ext = '.asc' apical_point_isec = None if morph_name in apical_points_isec: apical_point_isec = int(apical_points_isec[morph_name]) morph_filename = morph_name + morph_ext morph_path = os.path.abspath(os.path.join(row['morph_dir'], morph_filename)) if row['to_run'] == 1: emodel = row['emodel'] original_emodel = row['original_emodel'] if emodel is None: raise ValueError( "scores db row %s for morph %s, etype %s, mtype %s, " "layer %s doesn't have an e-model assigned to it" % (index, morph_name, row['etype'], row['mtype'], row['layer'])) args = (index, emodel, os.path.abspath(emodel_dirs[emodel]), final_dict[original_emodel]['params'], morph_path, apical_point_isec, extra_values_error) arg_list.append(args) print('Found %d rows in score database to run' % len(arg_list)) return arg_list
def select_combos(conf_filename): """Parse conf file and run select combos""" # Parse configuration file conf_dict = tools.load_json(conf_filename) select_combos_from_conf(conf_dict)