def simulate_experiment_trials(self, experiment: model.Experiment): try: working_directory = self._prepare_batch_files(experiment) except OSError: experiment.state = model.ExperimentState.ERROR return utils.getJSONError("The specified folder already exists", "") try: simulations.run(platform="local", parallel=self.run_config.parallel, cores=self.run_config.cores, method=self.run_config.type, batch=True, asynchronous=self.run_config.asynchronous, working_directory=working_directory) except InvalidConfigError as e: experiment.state = model.ExperimentState.ERROR return utils.getJSONError(str(e), "") if self.run_config.asynchronous: message = f"Experiment {experiment.name} started. " \ f"You can view the Experiment status in the Experiment Manager." else: message = f"Experiment {experiment.name} finished, you can view the results in the Experiment Manager." return utils.getJSONError(message, "")
def simulateNetPyNEModelInGeppetto(self, args): try: with redirect_stdout(sys.__stdout__): # TODO mpi is not finding libmpi.dylib.. set LD_LIBRARY_PATH to openmpi bin folder, but nothing if args['parallelSimulation']: logging.debug('Running parallel simulation') if not 'usePrevInst' in args or not args['usePrevInst']: self.netParams.save("netParams.json") self.simConfig.saveJson = True self.simConfig.save("simParams.json") template = os.path.join(os.path.dirname(__file__), 'template.py') else: sim.cfg.saveJson = True oldName = sim.cfg.filename sim.cfg.filename = 'model_output' sim.saveData() sim.cfg.filename = oldName template = os.path.join(os.path.dirname(__file__), 'template2.py') copyfile(template, './init.py') cp = subprocess.run([ "mpiexec", "-n", args['cores'], "nrniv", "-mpi", "-python", "init.py" ], capture_output=True) print(cp.stdout.decode() + cp.stderr.decode()) if cp.returncode != 0: return utils.getJSONError( "Error while simulating the NetPyNE model", cp.stderr.decode()) sim.load('model_output.json') self.geppetto_model = self.model_interpreter.getGeppettoModel( sim) netpyne_model = sim else: logging.info("Starting simulation") if not args.get('usePrevInst', False): logging.debug('Instantiating single thread simulation') netpyne_model = self.instantiateNetPyNEModel() self.geppetto_model = self.model_interpreter.getGeppettoModel( netpyne_model) logging.debug('Running single thread simulation') netpyne_model = self.simulateNetPyNEModel() return json.loads( GeppettoModelSerializer.serialize(self.geppetto_model)) except: return utils.getJSONError( "Error while simulating the NetPyNE model", sys.exc_info())
def importCellTemplate(self, modelParameters): try: with redirect_stdout(sys.__stdout__): rule = modelParameters["label"] # Get Current dir owd = os.getcwd() conds = {} if rule not in self.netParams.cellParams else self.netParams.cellParams[ rule]['conds'] compileModMechFiles(modelParameters["compileMod"], modelParameters["modFolder"]) del modelParameters["modFolder"] del modelParameters["compileMod"] # import cell template self.netParams.importCellParams(**modelParameters, conds=conds) # convert fron netpyne.specs.dict to dict self.netParams.cellParams[rule] = self.netParams.cellParams[ rule].todict() return utils.getJSONReply() except: return utils.getJSONError( "Error while importing the NetPyNE cell template", sys.exc_info()) finally: os.chdir(owd)
def instantiateModelInGeppetto(self): """ Instantiates model in geppetto Runs simulation Returns ------- JSON serialized geppetto model """ try: netpyne_model = self.instantiateModel() self.geppetto_model = self.model_interpreter.getGeppettoModel( netpyne_model) logging.debug('Running single thread simulation') self.simulateModel() return json.loads( GeppettoModelSerializer.serialize(self.geppetto_model)) except Exception as e: print(e) return utils.getJSONError( "Error while instantiating the NetPyNE model", e)
def exportNeuroML(self, modelParams): try: with redirect_stdout(sys.__stdout__): sim.exportNeuroML2(modelParams['fileName'], specs.SimConfig()) return utils.getJSONReply() except: return utils.getJSONError( "Error while exporting the NetPyNE model", sys.exc_info())
def importModel(self, modelParameters): """ Imports a model stored in form of Python files. :param modelParameters: :return: """ if self.doIhaveInstOrSimData()['haveInstance']: # TODO: this must be integrated into the general lifecycle of "model change -> simulate" # Shouldn't be specific to Import sim.clearAll() try: # Get Current dir owd = os.getcwd() compileModMechFiles(modelParameters['compileMod'], modelParameters['modFolder']) with redirect_stdout(sys.__stdout__): # NetParams net_params_path = str(modelParameters["netParamsPath"]) sys.path.append(net_params_path) os.chdir(net_params_path) # Import Module net_params_module_name = importlib.import_module( str(modelParameters["netParamsModuleName"])) # Import Model attributes self.netParams = getattr( net_params_module_name, str(modelParameters["netParamsVariable"])) for key, value in self.netParams.cellParams.items(): if hasattr(value, 'todict'): self.netParams.cellParams[key] = value.todict() # SimConfig sim_config_path = str(modelParameters["simConfigPath"]) sys.path.append(sim_config_path) os.chdir(sim_config_path) # Import Module sim_config_module_name = importlib.import_module( str(modelParameters["simConfigModuleName"])) # Import Model attributes self.simConfig = getattr( sim_config_module_name, str(modelParameters["simConfigVariable"])) # TODO: when should sim.initialize be called? # Only on import or better before every simulation or network instantiation? sim.initialize() return utils.getJSONReply() except: return utils.getJSONError( "Error while importing the NetPyNE model", sys.exc_info()) finally: os.chdir(owd)
def create_celltype_from_template(self, label="CellType", conds={}, cell_template_name="Blank"): try: with redirect_stdout(sys.__stdout__): self.netParams.addCellParamsTemplate( label=label, template=cell_template_name) return True except: return utils.getJSONError( f"Error while creating cellType from template {cell_template_name}", sys.exc_info())
def instantiateNetPyNEModelInGeppetto(self, args): try: with redirect_stdout(sys.__stdout__): if not 'usePrevInst' in args or not args['usePrevInst']: netpyne_model = self.instantiateNetPyNEModel() self.geppetto_model = self.model_interpreter.getGeppettoModel( netpyne_model) return json.loads( GeppettoModelSerializer.serialize(self.geppetto_model)) except: return utils.getJSONError( "Error while instantiating the NetPyNE model", sys.exc_info())
def instantiateNetPyNEModelInGeppetto(self, args): try: with redirect_stdout(sys.__stdout__): if not args.get("usePrevInst", False): netpyne_model = self.instantiateNetPyNEModel() self.geppetto_model = self.model_interpreter.getGeppettoModel( netpyne_model) return json.loads( GeppettoModelSerializer.serialize(self.geppetto_model)) except Exception as e: message = "Error while instantiating the NetPyNE model" logging.exception(message) return utils.getJSONError(message, sys.exc_info())
def deleteModel(self, modelParams): try: with redirect_stdout(sys.__stdout__): self.netParams = specs.NetParams() self.simConfig = specs.SimConfig() self.netParams.todict() self.netParams.todict() if self.doIhaveInstOrSimData()['haveInstance']: sim.clearAll() self.geppetto_model = None return utils.getJSONReply() except: return utils.getJSONError( "Error while exporting the NetPyNE model", sys.exc_info())
def importNeuroML(self, modelParams): try: with redirect_stdout(sys.__stdout__): sim.initialize() sim.importNeuroML2(modelParams['neuroMLFolder'], simConfig=specs.SimConfig(), simulate=False, analyze=False) self.geppetto_model = self.model_interpreter.getGeppettoModel( sim) return json.loads( GeppettoModelSerializer.serialize(self.geppetto_model)) except: return utils.getJSONError( "Error while exporting the NetPyNE model", sys.exc_info())
def deleteModel(self, modelParams): try: with redirect_stdout(sys.__stdout__): self.netParams = specs.NetParams() self.simConfig = specs.SimConfig() sim.initialize(specs.NetParams(), specs.SimConfig()) self.geppetto_model = None except Exception: message = "Error while exporting the NetPyNE model" logging.exception(message) return utils.getJSONError(message, sys.exc_info()) try: sim.clearAll() except: logging.exception("Failed to clear simulation") return utils.getJSONReply()
def exportModel(self, args): try: with redirect_stdout(sys.__stdout__): if not args['netCells']: sim.initialize(netParams=self.netParams, simConfig=self.simConfig) sim.cfg.filename = args['fileName'] include = [ el for el in specs.SimConfig().saveDataInclude if el in args.keys() and args[el] ] if args['netCells']: include += ['netPops'] sim.cfg.saveJson = True sim.saveData(include) sim.cfg.saveJson = False return utils.getJSONReply() except: return utils.getJSONError( "Error while exporting the NetPyNE model", sys.exc_info())
def importModel(self, modelParameters): try: # Get Current dir owd = os.getcwd() self.compileModMechFiles(modelParameters['compileMod'], modelParameters['modFolder']) with redirect_stdout(sys.__stdout__): # NetParams netParamsPath = str(modelParameters["netParamsPath"]) sys.path.append(netParamsPath) os.chdir(netParamsPath) # Import Module netParamsModuleName = importlib.import_module( str(modelParameters["netParamsModuleName"])) # Import Model attributes self.netParams = getattr( netParamsModuleName, str(modelParameters["netParamsVariable"])) for key, value in self.netParams.cellParams.items(): if hasattr(value, 'todict'): self.netParams.cellParams[key] = value.todict() # SimConfig simConfigPath = str(modelParameters["simConfigPath"]) sys.path.append(simConfigPath) os.chdir(simConfigPath) # Import Module simConfigModuleName = importlib.import_module( str(modelParameters["simConfigModuleName"])) # Import Model attributes self.simConfig = getattr( simConfigModuleName, str(modelParameters["simConfigVariable"])) return utils.getJSONReply() except: return utils.getJSONError( "Error while importing the NetPyNE model", sys.exc_info()) finally: os.chdir(owd)
def simulate_single_model(self, experiment: model.Experiment = None, use_prev_inst: bool = False): if experiment: working_directory = self._prepare_simulation_files( experiment, use_prev_inst) simulations.run( parallel=self.run_config.parallel, cores=self.run_config.cores, asynchronous=self.run_config.asynchronous, method=simulations.MPI_BULLETIN, working_directory=working_directory, ) if self.run_config.asynchronous: message = "Experiment is pending! " \ f"Results will be stored in your workspace at ./{os.path.join(constants.EXPERIMENTS_FOLDER, experiment.name)}" return utils.getJSONError(message, "") else: sim.load(f'{constants.MODEL_OUTPUT_FILENAME}.json') self.geppetto_model = self.model_interpreter.getGeppettoModel( sim) response = json.loads( GeppettoModelSerializer.serialize(self.geppetto_model)) return response else: # Run in same process if not use_prev_inst: logging.debug('Instantiating single thread simulation') netpyne_model = self.instantiateNetPyNEModel() self.geppetto_model = self.model_interpreter.getGeppettoModel( netpyne_model) simulations.run() if self.geppetto_model: response = json.loads( GeppettoModelSerializer.serialize(self.geppetto_model)) return response
def deleteModel(self, modelParams): try: with redirect_stdout(sys.__stdout__): self.netParams = specs.NetParams() self.simConfig = specs.SimConfig() sim.initialize(specs.NetParams(), specs.SimConfig()) self.geppetto_model = None except: return utils.getJSONError( "Error while exporting the NetPyNE model", sys.exc_info()) try: # This function fails is some keys don't exists # sim.clearAll() # TODO: as part of #264 we should remove the method and use clearAll intstead self.clearSim() except: pass return utils.getJSONReply()
def viewExperimentResult(self, payload: dict): """ Loads the output file of a simulated experiment trial. :param payload: {name: str, trial: str, onlyModelSpecification: bool} :return: geppetto model """ name = payload.get("name", None) trial = payload.get("trial", None) only_model_spec = payload.get("onlyModelSpecification", False) file = experiments.get_trial_output_path(name, trial) if file is None or not os.path.exists(file): return utils.getJSONError( f"Couldn't find output file of condition. Please take a look at the simulation log.", "") if self.doIhaveInstOrSimData()['haveInstance']: sim.clearAll() sim.initialize() if only_model_spec: # Load only model specification sim.loadNetParams(file) sim.loadSimCfg(file) self.netParams = sim.net.params self.simConfig = sim.cfg netpyne_ui_utils.remove(self.simConfig.todict()) netpyne_ui_utils.remove(self.netParams.todict()) return else: # Load the complete simulation sim.loadAll(file) self._create3D_shapes(file) self.geppetto_model = self.model_interpreter.getGeppettoModel(sim) return json.loads( GeppettoModelSerializer.serialize(self.geppetto_model))
def exportModel(self, args): try: with redirect_stdout(sys.__stdout__): if not args['netCells']: sim.initialize(netParams=self.netParams, simConfig=self.simConfig) sim.cfg.filename = args['fileName'] include = [ el for el in specs.SimConfig().saveDataInclude if el in args.keys() and args[el] ] if args['netCells']: include += ['netPops'] sim.cfg.saveJson = True sim.saveData(include) sim.cfg.saveJson = False with open(f"{sim.cfg.filename}_data.json") as json_file: data = json.load(json_file) return data except Exception: message = "Error while exporting the NetPyNE model" logging.exception(message) return utils.getJSONError(message, sys.exc_info())
def exportHLS(self, args): def convert2bool(string): return string.replace('true', 'True').replace('false', 'False').replace( 'null', 'False') def header(title, spacer='-'): return '\n# ' + title.upper() + ' ' + spacer * (77 - len(title)) + '\n' try: params = ['popParams', 'cellParams', 'synMechParams'] params += ['connParams', 'stimSourceParams', 'stimTargetParams'] fname = args['fileName'] if args['fileName'][ -3:] == '.py' else args['fileName'] + '.py' with open(fname, 'w') as script: script.write('from netpyne import specs, sim\n') script.write(header('documentation')) script.write( "''' Script generated with NetPyNE-UI. Please visit:\n") script.write( " - https://www.netpyne.org\n - https://github.com/MetaCell/NetPyNE-UI\n'''\n" ) script.write(header('script', spacer='=')) script.write('netParams = specs.NetParams()\n') script.write('simConfig = specs.SimConfig()\n') script.write(header('single value attributes')) for attr, value in list(self.netParams.__dict__.items()): if attr not in params: if value != getattr(specs.NetParams(), attr): script.write('netParams.' + attr + ' = ') script.write( convert2bool(json.dumps(value, indent=4)) + '\n') script.write(header('network attributes')) for param in params: for key, value in list( getattr(self.netParams, param).items()): script.write("netParams." + param + "['" + key + "'] = ") script.write( convert2bool(json.dumps(value, indent=4)) + '\n') script.write(header('network configuration')) for attr, value in list(self.simConfig.__dict__.items()): if value != getattr(specs.SimConfig(), attr): script.write('simConfig.' + attr + ' = ') script.write( convert2bool(json.dumps(value, indent=4)) + '\n') script.write(header('create simulate analyze network')) script.write( '# sim.createSimulateAnalyze(netParams=netParams, simConfig=simConfig)\n' ) script.write(header('end script', spacer='=')) with open(fname) as f: return f.read() except: return utils.getJSONError( "Error while importing the NetPyNE model", sys.exc_info())
def simulateNetPyNEModelInGeppetto(self, args): """ Starts simulation of the currently loaded NetPyNe model. * runConfiguration is used to determine asynch/synch & other parameters. * complete flag in args decides if we simulate single model as Experiment or complete Experiment. * if Experiment in design does not exist, we create a new one & start single sim. * All Simulations run in different process. :param args: { allTrials: bool, usePrevInst: bool } :return: geppetto model. """ allTrials = args.get('allTrials', True) use_prev_inst = args.get('usePrevInst', False) sim_id = args.get('simId', 0) try: experiment = experiments.get_current() if experiment: if self.experiments.any_in_state([ model.ExperimentState.PENDING, model.ExperimentState.SIMULATING ]): return utils.getJSONError( "Experiment is already simulating or pending", "") if simulations.local.is_running(): simulations.local.stop() experiment.state = model.ExperimentState.PENDING try: if allTrials: if len(experiment.trials) == 1 and experiment.trials[ 0].id == experiments.BASE_TRIAL_ID: # special case where we don't want to run a batch simulation return self.simulate_single_model( experiment, use_prev_inst) else: return self.simulate_experiment_trials(experiment) else: return self.simulate_single_model( experiment, use_prev_inst) except Exception: experiment.state = model.ExperimentState.ERROR message = ( "Unknown error during simulation of Experiment. SimulationId %i" % sim_id) logging.exception(message) return utils.getJSONError( "Unknown error during simulation of Experiment", sys.exc_info(), {"sim_id": sim_id}) else: return self.simulate_single_model(use_prev_inst=use_prev_inst) except Exception as e: message = ( "Error while simulating the NetPyNE model: %s. SimulationId %f" % (e, sim_id)) logging.exception(message) return utils.getJSONError(message, sys.exc_info(), {"sim_id": sim_id})
def loadModel(self, args): """ Imports a model stored as file in json format. :param args: :return: """ if not any([ args[option] for option in ['loadNetParams', 'loadSimCfg', 'loadSimData', 'loadNet'] ]): return utils.getJSONError( "Error while loading data", 'You have to select at least one option') try: owd = os.getcwd() compileModMechFiles(args['compileMod'], args['modFolder']) except Exception: message = "Error while importing/compiling mods" logging.exception(message) return utils.getJSONError(message, sys.exc_info()) finally: os.chdir(owd) try: with redirect_stdout(sys.__stdout__): sim.initialize() wake_up_geppetto = False if all([ args[option] for option in ['loadNetParams', 'loadSimCfg', 'loadSimData', 'loadNet'] ]): wake_up_geppetto = True if self.doIhaveInstOrSimData()['haveInstance']: sim.clearAll() sim.initialize() sim.loadAll(args['jsonModelFolder']) self.netParams = sim.net.params self.simConfig = sim.cfg netpyne_ui_utils.remove(self.netParams.todict()) netpyne_ui_utils.remove(self.simConfig.todict()) else: if args['loadNet']: wake_up_geppetto = True if self.doIhaveInstOrSimData()['haveInstance']: sim.clearAll() sim.initialize() sim.loadNet(args['jsonModelFolder']) # TODO (https://github.com/Neurosim-lab/netpyne/issues/360) if args['loadSimData']: wake_up_geppetto = True if not self.doIhaveInstOrSimData()['haveInstance']: sim.create(specs.NetParams(), specs.SimConfig()) sim.net.defineCellShapes() sim.gatherData(gatherLFP=False) sim.loadSimData(args['jsonModelFolder']) if args['loadSimCfg']: sim.loadSimCfg(args['jsonModelFolder']) self.simConfig = sim.cfg netpyne_ui_utils.remove(self.simConfig.todict()) if args['loadNetParams']: if self.doIhaveInstOrSimData()['haveInstance']: sim.clearAll() sim.loadNetParams(args['jsonModelFolder']) self.netParams = sim.net.params netpyne_ui_utils.remove(self.netParams.todict()) if wake_up_geppetto: self._create3D_shapes(args['jsonModelFolder']) # TODO: Fix me - gatherData will remove allSimData! sim.gatherData() self.geppetto_model = self.model_interpreter.getGeppettoModel( sim) return json.loads( GeppettoModelSerializer.serialize(self.geppetto_model)) else: return utils.getJSONReply() except Exception: message = "Error while loading the NetPyNE model" logging.exception(message) return utils.getJSONError(message, sys.exc_info())
def loadModel(self, args): """ Imports a model stored as file in json format. :param args: :return: """ def remove(dictionary): # remove reserved keys such as __dict__, __Method__, etc # they appear when we do sim.loadAll(json_file) if isinstance(dictionary, dict): for key, value in list(dictionary.items()): if key.startswith('__'): dictionary.pop(key) else: remove(value) if not any([ args[option] for option in ['loadNetParams', 'loadSimCfg', 'loadSimData', 'loadNet'] ]): return utils.getJSONError( "Error while loading data", 'You have to select at least one option') try: owd = os.getcwd() compileModMechFiles(args['compileMod'], args['modFolder']) except: return utils.getJSONError("Error while importing/compiling mods", sys.exc_info()) finally: os.chdir(owd) try: with redirect_stdout(sys.__stdout__): sim.initialize() wake_up_geppetto = False if all([ args[option] for option in ['loadNetParams', 'loadSimCfg', 'loadSimData', 'loadNet'] ]): wake_up_geppetto = True if self.doIhaveInstOrSimData()['haveInstance']: sim.clearAll() sim.initialize() sim.loadAll(args['jsonModelFolder']) self.netParams = sim.net.params self.simConfig = sim.cfg remove(self.netParams.todict()) remove(self.simConfig.todict()) else: if args['loadNet']: wake_up_geppetto = True if self.doIhaveInstOrSimData()['haveInstance']: sim.clearAll() sim.initialize() sim.loadNet(args['jsonModelFolder']) if args['loadSimData']: # TODO (https://github.com/Neurosim-lab/netpyne/issues/360) wake_up_geppetto = True if not self.doIhaveInstOrSimData()['haveInstance']: sim.create(specs.NetParams(), specs.SimConfig()) sim.net.defineCellShapes() sim.gatherData(gatherLFP=False) sim.loadSimData(args['jsonModelFolder']) if args['loadSimCfg']: sim.loadSimCfg(args['jsonModelFolder']) self.simConfig = sim.cfg remove(self.simConfig.todict()) if args['loadNetParams']: if self.doIhaveInstOrSimData()['haveInstance']: sim.clearAll() sim.loadNetParams(args['jsonModelFolder']) self.netParams = sim.net.params remove(self.netParams.todict()) if wake_up_geppetto: if len(sim.net.cells) > 0: section = list(sim.net.cells[0].secs.keys())[0] if 'pt3d' not in list( sim.net.cells[0].secs[section].geom.keys()): sim.net.defineCellShapes() sim.gatherData() sim.loadSimData(args['jsonModelFolder']) sim.gatherData() self.geppetto_model = self.model_interpreter.getGeppettoModel( sim) return json.loads( GeppettoModelSerializer.serialize(self.geppetto_model)) else: return utils.getJSONReply() except: return utils.getJSONError("Error while loading the NetPyNE model", sys.exc_info())
def exportHLS(self, args): def convert2bool(string): return string.replace('true', 'True').replace('false', 'False').replace( 'null', 'False') def header(title, spacer='-'): return '\n# ' + title.upper() + ' ' + spacer * (77 - len(title)) + '\n' try: params = ['popParams', 'cellParams', 'synMechParams'] params += ['connParams', 'stimSourceParams', 'stimTargetParams'] fname = args['fileName'] if not fname: # default option fname = 'output.py' if not fname[-3:] == '.py': fname = f"{fname}.py" # TODO: use methods offered by netpyne to create this script! with open(fname, 'w') as script: script.write("from netpyne import specs, sim\n") script.write(header("documentation")) script.write( "Script generated with NetPyNE-UI. Please visit:\n") script.write( " - https://www.netpyne.org\n - https://github.com/MetaCell/NetPyNE-UI\n\n" ) script.write(header("script", spacer="=")) script.write("netParams = specs.NetParams()\n") script.write("simConfig = specs.SimConfig()\n") script.write(header("single value attributes")) for attr, value in list(self.netParams.__dict__.items()): if attr not in params: if value != getattr(specs.NetParams(), attr): script.write("netParams." + attr + " = ") script.write( convert2bool(json.dumps(value, indent=4)) + "\n") script.write(header("network attributes")) for param in params: for key, value in list( getattr(self.netParams, param).items()): script.write("netParams." + param + "[" + key + "] = ") script.write( convert2bool(json.dumps(value, indent=4)) + "\n") script.write(header("network configuration")) for attr, value in list(self.simConfig.__dict__.items()): if value != getattr(specs.SimConfig(), attr): script.write("simConfig." + attr + " = ") script.write( convert2bool(json.dumps(value, indent=4)) + "\n") script.write(header("create simulate analyze network")) script.write( "# sim.createSimulateAnalyze(netParams=netParams, simConfig=simConfig)\n" ) script.write(header("end script", spacer="=")) with open(fname) as f: file_b64 = base64.b64encode(bytes(f.read(), 'utf-8')).decode() export_info = {"fileContent": file_b64, "fileName": fname} return export_info except Exception: message = "Error while exporting NetPyNE model to python" logging.exception(message) return utils.getJSONError(message, sys.exc_info())