def load_celltypes(self): """ :return: """ rank = self.comm.Get_rank() size = self.comm.Get_size() celltypes = self.celltypes typenames = sorted(celltypes.keys()) if rank == 0: self.logger.info('env.data_file_path = %s' % str(self.data_file_path)) (population_ranges, _) = read_population_ranges(self.data_file_path, self.comm) if rank == 0: self.logger.info('population_ranges = %s' % str(population_ranges)) for k in typenames: population_range = population_ranges.get(k, None) if population_range is not None: celltypes[k]['start'] = population_ranges[k][0] celltypes[k]['num'] = population_ranges[k][1] if 'mechanism file' in celltypes[k]: celltypes[k]['mech_file_path'] = '%s/%s' % ( self.config_prefix, celltypes[k]['mechanism file']) mech_dict = read_from_yaml(celltypes[k]['mech_file_path']) celltypes[k]['mech_dict'] = mech_dict if 'synapses' in celltypes[k]: synapses_dict = celltypes[k]['synapses'] if 'weights' in synapses_dict: weights_config = synapses_dict['weights'] if isinstance(weights_config, list): weights_dicts = weights_config else: weights_dicts = [weights_config] for weights_dict in weights_dicts: if 'expr' in weights_dict: expr = weights_dict['expr'] parameter = weights_dict['parameter'] const = weights_dict.get('const', {}) clos = ExprClosure(parameter, expr, const) weights_dict['closure'] = clos synapses_dict['weights'] = weights_dicts population_names = read_population_names(self.data_file_path, self.comm) if rank == 0: self.logger.info('population_names = %s' % str(population_names)) self.cell_attribute_info = read_cell_attribute_info( self.data_file_path, population_names, comm=self.comm) if rank == 0: self.logger.info('attribute info: %s' % str(self.cell_attribute_info))
def main(config_path, params_id, output_file_name, verbose): config_logging(verbose) logger = utils.get_script_logger(os.path.basename(__file__)) eval_config = read_from_yaml(config_path, include_loader=utils.IncludeLoader) network_param_spec_src = eval_config['param_spec'] network_param_values = eval_config['param_values'] target_populations = eval_config['target_populations'] network_param_spec = make_param_spec(target_populations, network_param_spec_src) def from_param_list(x): result = [] for i, (param_name, param_tuple) in enumerate( zip(network_param_spec.param_names, network_param_spec.param_tuples)): param_range = param_tuple.param_range # assert((x[i] >= param_range[0]) and (x[i] <= param_range[1])) result.append((param_tuple, x[i])) return result x = network_param_values[params_id] param_tuple_values = from_param_list(x) def rec_dd(): return defaultdict(rec_dd) def dd2dict(d): for k, v in d.items(): if isinstance(v, dict): d[k] = dd2dict(v) return dict(d) param_output_ddict = rec_dd() for param_tuple, param_value in param_tuple_values: if isinstance(param_tuple.param_path, tuple): param_output_ddict[param_tuple.population][param_tuple.source][ param_tuple.sec_type][param_tuple.syn_name][ param_tuple.param_path[0]][ param_tuple.param_path[1]] = param_value else: param_output_ddict[param_tuple.population][param_tuple.source][ param_tuple.sec_type][param_tuple.syn_name][ param_tuple.param_path] = param_value param_output_dict = dd2dict(param_output_ddict) pprint.pprint(param_output_dict) write_to_yaml(output_file_name, param_output_dict)
def main(config_path, params_id, output_file_name, verbose): config_logging(verbose) logger = utils.get_script_logger(os.path.basename(__file__)) eval_config = read_from_yaml(config_path, include_loader=utils.IncludeLoader) network_param_spec_src = eval_config['param_spec'] network_param_values = eval_config['param_values'] target_populations = eval_config['target_populations'] network_param_spec = make_param_spec(target_populations, network_param_spec_src) def from_param_list(x): result = [] for i, (param_name, param_tuple) in enumerate( zip(network_param_spec.param_names, network_param_spec.param_tuples)): param_range = param_tuple.param_range # assert((x[i] >= param_range[0]) and (x[i] <= param_range[1])) result.append((param_tuple, x[i])) return result params_id_list = [] if params_id is None: params_id_list = list(network_param_values.keys()) else: params_id_list = [params_id] param_output_dict = dict() for this_params_id in params_id_list: x = network_param_values[this_params_id] param_tuple_values = from_param_list(x) this_param_list = [] for param_tuple, param_value in param_tuple_values: this_param_list.append((param_tuple.population, param_tuple.source, param_tuple.sec_type, param_tuple.syn_name, param_tuple.param_path, param_value)) param_output_dict[this_params_id] = this_param_list pprint.pprint(param_output_dict) if output_file_name is not None: write_to_yaml(output_file_name, param_output_dict)
def generate_param_lattice(config_path, n_samples, output_file_dir, output_file_name, maxiter=5, verbose=False): from dmosopt import sampling logger = utils.get_script_logger(os.path.basename(__file__)) output_path = None if output_file_name is not None: output_path = f'{output_file_dir}/{output_file_name}' eval_config = read_from_yaml(config_path, include_loader=utils.IncludeLoader) network_param_spec_src = eval_config['param_spec'] target_populations = eval_config['target_populations'] network_param_spec = make_param_spec(target_populations, network_param_spec_src) param_tuples = network_param_spec.param_tuples param_names = network_param_spec.param_names n_params = len(param_tuples) n_init = n_params * n_samples Xinit = sampling.glp(n_init, n_params, maxiter=maxiter) ub = [] lb = [] for param_name, param_tuple in zip(param_names, param_tuples): param_range = param_tuple.param_range ub.append(param_range[1]) lb.append(param_range[0]) ub = np.asarray(ub) lb = np.asarray(lb) for i in range(n_init): Xinit[i,:] = Xinit[i,:] * (ub - lb) + lb output_dict = {} for i in range(Xinit.shape[0]): output_dict[i] = list([float(x) for x in Xinit[i, :]]) if output_path is not None: write_to_yaml(output_path, output_dict) else: pprint.pprint(output_dict)
def main(config_path, target_features_path, target_features_namespace, optimize_file_dir, optimize_file_name, nprocs_per_worker, n_epochs, n_initial, initial_maxiter, initial_method, optimizer_method, population_size, num_generations, resample_fraction, mutation_rate, collective_mode, spawn_startup_wait, verbose): network_args = click.get_current_context().args network_config = {} for arg in network_args: kv = arg.split("=") if len(kv) > 1: k,v = kv network_config[k.replace('--', '').replace('-', '_')] = v else: k = kv[0] network_config[k.replace('--', '').replace('-', '_')] = True run_ts = datetime.datetime.today().strftime('%Y%m%d_%H%M') if optimize_file_name is None: optimize_file_name=f"dmosopt.optimize_network_{run_ts}.h5" operational_config = read_from_yaml(config_path) operational_config['run_ts'] = run_ts if target_features_path is not None: operational_config['target_features_path'] = target_features_path if target_features_namespace is not None: operational_config['target_features_namespace'] = target_features_namespace network_config.update(operational_config.get('kwargs', {})) env = Env(**network_config) objective_names = operational_config['objective_names'] param_config_name = operational_config['param_config_name'] target_populations = operational_config['target_populations'] opt_param_config = optimization_params(env.netclamp_config.optimize_parameters, target_populations, param_config_name) opt_targets = opt_param_config.opt_targets param_names = opt_param_config.param_names param_tuples = opt_param_config.param_tuples hyperprm_space = { param_pattern: [param_tuple.param_range[0], param_tuple.param_range[1]] for param_pattern, param_tuple in zip(param_names, param_tuples) } init_objfun = 'init_network_objfun' init_params = { 'operational_config': operational_config, 'opt_targets': opt_targets, 'param_tuples': [ param_tuple._asdict() for param_tuple in param_tuples ], 'param_names': param_names } init_params.update(network_config.items()) nworkers = env.comm.size-1 if resample_fraction is None: resample_fraction = float(nworkers) / float(population_size) if resample_fraction > 1.0: resample_fraction = 1.0 if resample_fraction < 0.1: resample_fraction = 0.1 # Create an optimizer feature_dtypes = [(feature_name, np.float32) for feature_name in objective_names] constraint_names = [f'{target_pop_name} positive rate' for target_pop_name in target_populations ] dmosopt_params = {'opt_id': 'dentate.optimize_network', 'obj_fun_init_name': init_objfun, 'obj_fun_init_module': 'dentate.optimize_network', 'obj_fun_init_args': init_params, 'reduce_fun_name': 'compute_objectives', 'reduce_fun_module': 'dentate.optimize_network', 'reduce_fun_args': (operational_config, opt_targets), 'problem_parameters': {}, 'space': hyperprm_space, 'objective_names': objective_names, 'feature_dtypes': feature_dtypes, 'constraint_names': constraint_names, 'n_initial': n_initial, 'initial_maxiter': initial_maxiter, 'initial_method': initial_method, 'optimizer': optimizer_method, 'n_epochs': n_epochs, 'population_size': population_size, 'num_generations': num_generations, 'resample_fraction': resample_fraction, 'mutation_rate': mutation_rate, 'file_path': f'{optimize_file_dir}/{optimize_file_name}', 'termination_conditions': True, 'save_surrogate_eval': True, 'save': True, 'save_eval': 5 } #dmosopt_params['broker_fun_name'] = 'dmosopt_broker_init' #dmosopt_params['broker_module_name'] = 'dentate.optimize_network' best = dmosopt.run(dmosopt_params, spawn_workers=True, sequential_spawn=False, spawn_startup_wait=spawn_startup_wait, nprocs_per_worker=nprocs_per_worker, collective_mode=collective_mode, verbose=True, worker_debug=True) if best is not None: if optimize_file_dir is not None: results_file_id = 'DG_optimize_network_%s' % run_ts yaml_file_path = '%s/optimize_network.%s.yaml' % (optimize_file_dir, str(results_file_id)) prms = best[0] prms_dict = dict(prms) n_res = prms[0][1].shape[0] results_config_dict = {} for i in range(n_res): result_param_list = [] for param_pattern, param_tuple in zip(param_names, param_tuples): result_param_list.append([param_pattern, float(prms_dict[param_pattern][i])]) results_config_dict[i] = result_param_list write_to_yaml(yaml_file_path, results_config_dict)
def load_celltypes(self): """ :return: """ rank = self.comm.Get_rank() size = self.comm.Get_size() celltypes = self.celltypes if rank == 0: color = 1 else: color = 0 ## comm0 includes only rank 0 comm0 = self.comm.Split(color, 0) if rank == 0: self.logger.info('env.data_file_path = %s' % str(self.data_file_path)) self.cell_attribute_info = None population_ranges = None population_names = None type_names = None if rank == 0: population_names = read_population_names(self.data_file_path, comm0) (population_ranges, _) = read_population_ranges(self.data_file_path, comm0) type_names = sorted(population_ranges.keys()) self.cell_attribute_info = read_cell_attribute_info( self.data_file_path, population_names, comm=comm0) self.logger.info('population_names = %s' % str(population_names)) self.logger.info('population_ranges = %s' % str(population_ranges)) self.logger.info('attribute info: %s' % str(self.cell_attribute_info)) population_ranges = self.comm.bcast(population_ranges, root=0) population_names = self.comm.bcast(population_names, root=0) type_names = self.comm.bcast(type_names, root=0) self.cell_attribute_info = self.comm.bcast(self.cell_attribute_info, root=0) comm0.Free() for k in type_names: population_range = population_ranges.get(k, None) if population_range is not None: if k not in celltypes: celltypes[k] = {} celltypes[k]['start'] = population_ranges[k][0] celltypes[k]['num'] = population_ranges[k][1] if 'mechanism file' in celltypes[k]: celltypes[k]['mech_file_path'] = '%s/%s' % ( self.config_prefix, celltypes[k]['mechanism file']) mech_dict = None if rank == 0: mech_dict = read_from_yaml( celltypes[k]['mech_file_path']) mech_dict = self.comm.bcast(mech_dict, root=0) celltypes[k]['mech_dict'] = mech_dict if 'synapses' in celltypes[k]: synapses_dict = celltypes[k]['synapses'] if 'weights' in synapses_dict: weights_config = synapses_dict['weights'] if isinstance(weights_config, list): weights_dicts = weights_config else: weights_dicts = [weights_config] for weights_dict in weights_dicts: if 'expr' in weights_dict: expr = weights_dict['expr'] parameter = weights_dict['parameter'] const = weights_dict.get('const', None) clos = ExprClosure(parameters=parameter, expr=expr, consts=const) weights_dict['closure'] = clos synapses_dict['weights'] = weights_dicts
def main(config_path, params_id, n_samples, target_features_path, target_features_namespace, output_file_dir, output_file_name, verbose): config_logging(verbose) logger = utils.get_script_logger(os.path.basename(__file__)) if params_id is None: if n_samples is not None: logger.info("Generating parameter lattice ...") generate_param_lattice(config_path, n_samples, output_file_dir, output_file_name, verbose) return network_args = click.get_current_context().args network_config = {} for arg in network_args: kv = arg.split("=") if len(kv) > 1: k,v = kv network_config[k.replace('--', '').replace('-', '_')] = v else: k = kv[0] network_config[k.replace('--', '').replace('-', '_')] = True run_ts = datetime.datetime.today().strftime('%Y%m%d_%H%M') if output_file_name is None: output_file_name=f"network_features_{run_ts}.h5" output_path = f'{output_file_dir}/{output_file_name}' eval_config = read_from_yaml(config_path, include_loader=utils.IncludeLoader) eval_config['run_ts'] = run_ts if target_features_path is not None: eval_config['target_features_path'] = target_features_path if target_features_namespace is not None: eval_config['target_features_namespace'] = target_features_namespace network_param_spec_src = eval_config['param_spec'] network_param_values = eval_config['param_values'] feature_names = eval_config['feature_names'] target_populations = eval_config['target_populations'] network_config.update(eval_config.get('kwargs', {})) if params_id is None: network_config['results_file_id'] = f"DG_eval_network_{eval_config['run_ts']}" else: network_config['results_file_id'] = f"DG_eval_network_{params_id}_{eval_config['run_ts']}" env = init_network(comm=MPI.COMM_WORLD, kwargs=network_config) gc.collect() t_start = 50. t_stop = env.tstop time_range = (t_start, t_stop) target_trj_rate_map_dict = {} target_features_arena = env.arena_id target_features_trajectory = env.trajectory_id for pop_name in target_populations: if ('%s target rate dist residual' % pop_name) not in feature_names: continue my_cell_index_set = set(env.biophys_cells[pop_name].keys()) trj_rate_maps = {} trj_rate_maps = rate_maps_from_features(env, pop_name, cell_index_set=list(my_cell_index_set), input_features_path=target_features_path, input_features_namespace=target_features_namespace, time_range=time_range) target_trj_rate_map_dict[pop_name] = trj_rate_maps network_param_spec = make_param_spec(target_populations, network_param_spec_src) def from_param_list(x): result = [] for pop_param in x: this_population, source, sec_type, syn_name, param_path, param_val = pop_param param_tuple = SynParam(this_population, source, sec_type, syn_name, param_path, None) result.append((param_tuple, param_val)) return result def from_param_dict(x): result = [] for pop_name, param_specs in viewitems(x): keyfun = lambda kv: str(kv[0]) for source, source_dict in sorted(viewitems(param_specs), key=keyfun): for sec_type, sec_type_dict in sorted(viewitems(source_dict), key=keyfun): for syn_name, syn_mech_dict in sorted(viewitems(sec_type_dict), key=keyfun): for param_fst, param_rst in sorted(viewitems(syn_mech_dict), key=keyfun): if isinstance(param_rst, dict): for const_name, const_value in sorted(viewitems(param_rst)): param_path = (param_fst, const_name) param_tuple = SynParam(pop_name, source, sec_type, syn_name, param_path, const_value) result.append(param_tuple, const_value) else: param_name = param_fst param_value = param_rst param_tuple = SynParam(pop_name, source, sec_type, syn_name, param_name, param_value) result.append(param_tuple, param_value) return result eval_network(env, network_config, from_param_list, from_param_dict, network_param_spec, network_param_values, params_id, target_trj_rate_map_dict, t_start, t_stop, target_populations, output_path)