def parameter_search_run_script( simulation_name, master_results_dir, run_script, core_number ):
"""
Executes *run_script*, one per each parameter combination of an existing parameter search run.
Each execution receives as the first commandline argument the directory in which the results for the given
parameter combination were stored.
Parameters
----------
simulation_name : str
The name of the simulation.
master_results_dir : str
The directory where the parameter search results are stored.
run_script : str
The name of the script to be run. The directory name of the given parameter combination datastore will be passed to it as the first command line argument.
core_number : int
How many cores to reserve per process.
"""
f = open(master_results_dir+'/parameter_combinations','rb')
combinations = pickle.load(f)
f.close()
# first check whether all parameter combinations contain the same parameter names
assert len(set([tuple(set(comb.keys())) for comb in combinations])) == 1 , "The parameter search didn't occur over a fixed set of parameters"
from subprocess import Popen, PIPE, STDOUT
for i,combination in enumerate(combinations):
rdn = master_results_dir+'/'+result_directory_name('ParameterSearch',simulation_name,combination)
#print rdn
subprocess.call(' '.join(["python", run_script, "'"+rdn+"'"] +['>'] + ["'"+rdn +'/OUTFILE_analysis'+str(time.time()) + "'"]), shell=True)
def parameter_search_run_script_distributed_slurm(simulation_name,
master_results_dir,
run_script, core_number):
"""
Scheadules the execution of *run_script*, one per each parameter combination of an existing parameter search run.
Each execution receives as the first commandline argument the directory in which the results for the given
parameter combination were stored.
Parameters
----------
simulation_name : str
The name of the simulation.
master_results_dir : str
The directory where the parameter search results are stored.
run_script : str
The name of the script to be run. The directory name of the given parameter combination datastore will be passed to it as the first command line argument.
core_number : int
How many cores to reserve per process.
"""
f = open(master_results_dir + '/parameter_combinations', 'rb')
combinations = pickle.load(f)
f.close()
# first check whether all parameter combinations contain the same parameter names
assert len(
set([tuple(set(comb.keys())) for comb in combinations])
) == 1, "The parameter search didn't occur over a fixed set of parameters"
from subprocess import Popen, PIPE, STDOUT
for i, combination in enumerate(combinations):
rdn = master_results_dir + '/' + result_directory_name(
'ParameterSearch', simulation_name, combination)
p = Popen(['sbatch'] +
['-o', master_results_dir + "/slurm_analysis-%j.out"],
stdin=PIPE,
stdout=PIPE,
stderr=PIPE,
text=True)
# THIS IS A BIT OF A HACK, have to add customization for other people ...
data = '\n'.join([
'#!/bin/bash',
'#SBATCH -J MozaikParamSearchAnalysis',
'#SBATCH -c ' + str(core_number),
'source /opt/software/mpi/openmpi-1.6.3-gcc/env',
'source /home/antolikjan/env/mozaiknew/bin/activate',
'cd ' + os.getcwd(),
'echo "DSADSA"',
' '.
join([
"mpirun", " --mca mtl ^psm python", run_script, "'" + rdn + "'"
] + ['>'] +
["'" + rdn + '/OUTFILE_analysis' + str(time.time()) + "'"]),
])
print(p.communicate(input=data)[0])
print(data)
p.stdin.close()
def load_fixed_parameter_set_parameter_search(simulation_name,
master_results_dir,
filter=None):
"""
Loads all datastores of parameter search over a fixed set of parameters.
Parameters
----------
simulation_name : str
The name of the simulation.
master_results_dir : str
The directory where the parameter search results are stored.
Returns
-------
A tuple (parameters,datastores), where `parameters` is a list of parameters over which the parameter search was performed.
The dsvs is a list of tuples (values,datastore) where `values` is a list of values (in the order as im `parameters`) of the
parameters, and dsv is a DataStore with results recorded to the combination of parameter values.
"""
f = open(master_results_dir + '/parameter_combinations', 'rb')
combinations = pickle.load(f)
f.close()
# first check whether all parameter combinations contain the same parameter names
assert len(
set([tuple(set(comb.keys())) for comb in combinations])
) == 1, "The parameter search didn't occur over a fixed set of parameters"
parameters = combinations[0].keys()
datastore = []
number_of_unloadable_datastores = 0
for i, combination in enumerate(combinations):
print i
rdn = result_directory_name('ParameterSearch', simulation_name,
combination)
try:
data_store = PickledDataStore(load=True,
parameters=ParameterSet({
'root_directory':
master_results_dir + '/' + rdn,
'store_stimuli':
False
}),
replace=False)
if filter != None:
filter.query(data_store).remove_ads_outside_of_dsv()
datastore.append(
([combination[k] for k in parameters], data_store))
except IOError:
number_of_unloadable_datastores = number_of_unloadable_datastores + 1
print "Error loading datastore: " + rdn
return (parameters, datastore, number_of_unloadable_datastores)
def load_fixed_parameter_set_parameter_search(simulation_name,master_results_dir,filter=None):
"""
Loads all datastores of parameter search over a fixed set of parameters.
Parameters
----------
simulation_name : str
The name of the simulation.
master_results_dir : str
The directory where the parameter search results are stored.
Returns
-------
A tuple (parameters,datastores), where `parameters` is a list of parameters over which the parameter search was performed.
The dsvs is a list of tuples (values,datastore) where `values` is a list of values (in the order as im `parameters`) of the
parameters, and dsv is a DataStore with results recorded to the combination of parameter values.
"""
f = open(master_results_dir+'/parameter_combinations','rb')
combinations = pickle.load(f)
f.close()
# first check whether all parameter combinations contain the same parameter names
assert len(set([tuple(set(comb.keys())) for comb in combinations])) == 1 , "The parameter search didn't occur over a fixed set of parameters"
parameters = combinations[0].keys()
datastore = []
number_of_unloadable_datastores = 0
for i,combination in enumerate(combinations):
print i
rdn = result_directory_name('ParameterSearch',simulation_name,combination)
try:
data_store = PickledDataStore(load=True,parameters=ParameterSet({'root_directory': master_results_dir + '/' + rdn,'store_stimuli' : False}),replace=False)
if filter != None:
filter.query(data_store).remove_ads_outside_of_dsv()
datastore.append(([combination[k] for k in parameters],data_store))
except IOError:
number_of_unloadable_datastores = number_of_unloadable_datastores + 1
print "Error loading datastore: " + rdn
except ValueError:
number_of_unloadable_datastores = number_of_unloadable_datastores + 1
print "Error loading datastore: " + rdn
except EOFError:
number_of_unloadable_datastores = number_of_unloadable_datastores + 1
print "Error loading datastore: " + rdn
return (parameters,datastore,number_of_unloadable_datastores)
def parameter_search_run_script_distributed_slurm(simulation_name,master_results_dir,run_script,core_number):
"""
Scheadules the execution of *run_script*, one per each parameter combination of an existing parameter search run.
Each execution receives as the first commandline argument the directory in which the results for the given
parameter combination were stored.
Parameters
----------
simulation_name : str
The name of the simulation.
master_results_dir : str
The directory where the parameter search results are stored.
run_script : str
The name of the script to be run. The directory name of the given parameter combination datastore will be passed to it as the first command line argument.
core_number : int
How many cores to reserve per process.
"""
f = open(master_results_dir+'/parameter_combinations','rb')
combinations = pickle.load(f)
f.close()
# first check whether all parameter combinations contain the same parameter names
assert len(set([tuple(set(comb.keys())) for comb in combinations])) == 1 , "The parameter search didn't occur over a fixed set of parameters"
from subprocess import Popen, PIPE, STDOUT
for i,combination in enumerate(combinations):
rdn = master_results_dir+'/'+result_directory_name('ParameterSearch',simulation_name,combination)
p = Popen(['sbatch'] + ['-o',master_results_dir+"/slurm_analysis-%j.out" ],stdin=PIPE,stdout=PIPE,stderr=PIPE)
# THIS IS A BIT OF A HACK, have to add customization for other people ...
data = '\n'.join([
'#!/bin/bash',
'#SBATCH -J MozaikParamSearchAnalysis',
'#SBATCH -c ' + str(core_number),
'source /opt/software/mpi/openmpi-1.6.3-gcc/env',
'source /home/antolikjan/env/mozaik/bin/activate',
'cd ' + os.getcwd(),
'echo "DSADSA"',
' '.join(["mpirun"," --mca mtl ^psm python",run_script,"'"+rdn+"'"] +['>'] + ["'"+rdn +'/OUTFILE_analysis'+str(time.time()) + "'"]),
])
print p.communicate(input=data)[0]
print data
p.stdin.close()
def run_workflow(simulation_name, model_class, create_experiments):
"""
This is the main function that executes a workflow.
It expects it gets the simulation, class of the model, and a function that will create_experiments.
The create experiments function get a instance of a model as the only parameter and it is expected to return
a list of Experiment instances that should be executed over the model.
The run workflow will automatically parse the command line to determine the simulator to be used and the path to the root parameter file.
It will also accept . (point) delimited path to parameteres in the configuration tree, and corresponding values. It will replace each such provided
parameter's value with the provided one on the command line.
Parameters
----------
simulation_name : str
The name of the simulation.
model_class : class
The class from which the model instance will be created from.
create_experiments : func
The function that returns the list of experiments that will be executed on the model.
Examples
--------
The intended syntax of the commandline is as follows (note that the simulation run name is the last argument):
>>> python userscript simulator_name num_threads parameter_file_path modified_parameter_path_1 modified_parameter_value_1 ... modified_parameter_path_n modified_parameter_value_n simulation_run_name
"""
(
simulation_run_name,
simulator_name,
num_threads,
parameters_url,
modified_parameters,
) = parse_workflow_args()
print "Loading parameters"
parameters = load_parameters(parameters_url, modified_parameters)
print "Finished loading parameters"
p = {}
if parameters.has_key('mozaik_seed'):
p['mozaik_seed'] = parameters['mozaik_seed']
if parameters.has_key('pynn_seed'):
p['pynn_seed'] = parameters['pynn_seed']
print "START MPI"
mozaik.setup_mpi(**p)
# Read parameters
exec "import pyNN.nest as sim" in globals(), locals()
# Create results directory
timestamp = datetime.now().strftime('%Y%m%d-%H%M%S')
ddir = result_directory_name(simulation_run_name, simulation_name,
modified_parameters)
if mozaik.mpi_comm and mozaik.mpi_comm.rank != 0:
Global.root_directory = parameters.results_dir + ddir + '/' + str(
mozaik.mpi_comm.rank) + '/'
mozaik.mpi_comm.barrier()
else:
Global.root_directory = parameters.results_dir + ddir + '/'
os.makedirs(Global.root_directory)
if mozaik.mpi_comm and mozaik.mpi_comm.rank == 0:
mozaik.mpi_comm.barrier()
if mozaik.mpi_comm.rank == 0:
#let's store the full and modified parameters, if we are the 0 rank process
parameters.save(Global.root_directory + "parameters", expand_urls=True)
import pickle
f = open(Global.root_directory + "modified_parameters", "w")
pickle.dump(modified_parameters, f)
f.close()
setup_logging()
model = model_class(sim, num_threads, parameters)
if mozaik.mpi_comm.rank == 0:
#let's store some basic info about the simulation run
f = open(Global.root_directory + "info", "w")
f.write(
str({
'model_class': str(model_class),
'model_docstring': model_class.__doc__,
'simulation_run_name': simulation_run_name,
'model_name': simulation_name,
'creation_data': datetime.now().strftime('%d/%m/%Y-%H:%M:%S')
}))
f.close()
#import cProfile
#cProfile.run('run_experiments(model,create_experiments(model),parameters)','stats_new')
data_store = run_experiments(model, create_experiments(model), parameters)
if mozaik.mpi_comm.rank == 0:
data_store.save()
import resource
print "Final memory usage: %iMB" % (
resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / (1024))
return (data_store, model)
from parameters import ParameterSet
#mpi_comm = MPI.COMM_WORLD
logger = mozaik.getMozaikLogger()
simulation_name = "VogelsAbbott2005"
simulation_run_name, _, _, _, modified_parameters = parse_workflow_args()
if True:
data_store,model = run_workflow(simulation_name,VogelsAbbott,create_experiments)
model.connectors['ExcExcConnection'].store_connections(data_store)
else:
setup_logging()
data_store = PickledDataStore(
load=True,
parameters=ParameterSet(
{
"root_directory": result_directory_name(
simulation_run_name, simulation_name, modified_parameters
),
"store_stimuli": False,
}
),
replace=True,
)
logger.info('Loaded data store')
#if mpi_comm.rank == 0:
print("Starting visualization")
perform_analysis_and_visualization(data_store)
data_store.save()
# first check whether all parameter combinations contain the same parameter names
assert len(
set([tuple(set(comb.keys())) for comb in combinations])
) == 1, "The parameter search didn't occur over a fixed set of parameters"
simulation_runs = []
working_combinations = []
for i, combination in enumerate(combinations):
combination = dict([
(x, y.decode('string_escape').decode('string_escape').replace(
"'", '') if type(y) == str else y)
for x, y in combination.items()
])
rdn = result_directory_name('ParameterSearch', sys.argv[2],
combination)
print(rdn)
try:
simulation_runs.append(
createSimulationRunDocumentAndUploadImages(
os.path.join(master_results_dir, rdn), gfs))
working_combinations.append(combination)
except Exception as e:
print("WARRNING, error in: >> " + rdn + ".\n Error:" + str(e))
document = {
'submission_date':
datetime.datetime.now().strftime('%d/%m/%Y-%H:%M:%S'),
'name': simulation_name,
#'name' : master_results_dir,
'simulation_runs': simulation_runs,
def run_workflow(simulation_name, model_class, create_experiments):
"""
This is the main function that executes a workflow.
It expects it gets the simulation, class of the model, and a function that will create_experiments.
The create experiments function get a instance of a model as the only parameter and it is expected to return
a list of Experiment instances that should be executed over the model.
The run workflow will automatically parse the command line to determine the simulator to be used and the path to the root parameter file.
It will also accept . (point) delimited path to parameteres in the configuration tree, and corresponding values. It will replace each such provided
parameter's value with the provided one on the command line.
Parameters
----------
simulation_name : str
The name of the simulation.
model_class : class
The class from which the model instance will be created from.
create_experiments : func
The function that returns the list of experiments that will be executed on the model.
Examples
--------
The intended syntax of the commandline is as follows (note that the simulation run name is the last argument):
>>> python userscript simulator_name num_threads parameter_file_path modified_parameter_path_1 modified_parameter_value_1 ... modified_parameter_path_n modified_parameter_value_n simulation_run_name
"""
mozaik.setup_mpi()
# Read parameters
exec "import pyNN.nest as sim" in globals(), locals()
if len(sys.argv) > 4 and len(sys.argv)%2 == 1:
simulation_run_name = sys.argv[-1]
simulator_name = sys.argv[1]
num_threads = sys.argv[2]
parameters_url = sys.argv[3]
modified_parameters = { sys.argv[i*2+4] : eval(sys.argv[i*2+5]) for i in xrange(0,(len(sys.argv)-5)/2)}
else:
raise ValueError("Usage: runscript simulator_name num_threads parameter_file_path modified_parameter_path_1 modified_parameter_value_1 ... modified_parameter_path_n modified_parameter_value_n simulation_run_name")
p
parameters = load_parameters(parameters_url,modified_parameters)
# Create results directory
timestamp = datetime.now().strftime('%Y%m%d-%H%M%S')
ddir = result_directory_name(simulation_run_name,simulation_name,modified_parameters)
if mozaik.mpi_comm and mozaik.mpi_comm.rank != 0:
Global.root_directory = parameters.results_dir + ddir + '/' + str(mozaik.mpi_comm.rank) + '/'
mozaik.mpi_comm.barrier()
else:
Global.root_directory = parameters.results_dir + ddir + '/'
os.makedirs(Global.root_directory)
if mozaik.mpi_comm and mozaik.mpi_comm.rank == 0:
mozaik.mpi_comm.barrier()
#let's store the full and modified parameters, if we are the 0 rank process
if mozaik.mpi_comm.rank == 0:
parameters.save(Global.root_directory + "parameters", expand_urls=True)
import pickle
f = open(Global.root_directory+"modified_parameters","w")
pickle.dump(modified_parameters,f)
f.close()
setup_logging()
model = model_class(sim,num_threads,parameters)
data_store = run_experiments(model,create_experiments(model),parameters)
if mozaik.mpi_comm.rank == 0:
data_store.save()
import resource
print "Final memory usage: %iMB" % (resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/(1024))
return (data_store,model)
def prepare_workflow(simulation_name, model_class):
"""
Executes the following preparatory steps for simulation workflow:
- Load simulation parameters
- Initialize random seeds
- Create directory for results
- Store loaded parameters
- Setup logging
- Store some initial info about the simulation
Returns
-------
sim : module
NEST module, to use for simulation
num_threads : int
Number of threads to use for the simulation
parameters : dict
Loaded parameters to initialize the simulation and model with
"""
(
simulation_run_name,
simulator_name,
num_threads,
parameters_url,
modified_parameters,
) = parse_workflow_args()
# First we load the parameters just to retrieve seeds. We will throw them away, because at this stage the PyNNDistribution values were not yet initialized correctly.
parameters = load_parameters(parameters_url, modified_parameters)
p = {}
if parameters.has_key('mozaik_seed'):
p['mozaik_seed'] = parameters['mozaik_seed']
if parameters.has_key('pynn_seed'):
p['pynn_seed'] = parameters['pynn_seed']
# Now initialize mpi with the seeds
print "START MPI"
mozaik.setup_mpi(**p)
# Now really load parameters
print "Loading parameters"
parameters = load_parameters(parameters_url, modified_parameters)
print "Finished loading parameters"
exec "import pyNN.nest as sim" in globals(), locals()
# Create results directory
timestamp = datetime.now().strftime('%Y%m%d-%H%M%S')
ddir = result_directory_name(simulation_run_name, simulation_name,
modified_parameters)
if mozaik.mpi_comm and mozaik.mpi_comm.rank != 0:
Global.root_directory = parameters.results_dir + ddir + '/' + str(
mozaik.mpi_comm.rank) + '/'
mozaik.mpi_comm.barrier()
else:
Global.root_directory = parameters.results_dir + ddir + '/'
os.makedirs(Global.root_directory)
if mozaik.mpi_comm and mozaik.mpi_comm.rank == 0:
mozaik.mpi_comm.barrier()
if mozaik.mpi_comm.rank == 0:
# Let's store the full and modified parameters, if we are the 0 rank process
parameters.save(Global.root_directory + "parameters", expand_urls=True)
import pickle
f = open(Global.root_directory + "modified_parameters", "w")
pickle.dump(modified_parameters, f)
f.close()
setup_logging()
if mozaik.mpi_comm.rank == 0:
# Let's store some basic info about the simulation run
f = open(Global.root_directory + "info", "w")
f.write(
str({
'model_class': str(model_class),
'model_docstring': model_class.__doc__,
'simulation_run_name': simulation_run_name,
'model_name': simulation_name,
'creation_data': datetime.now().strftime('%d/%m/%Y-%H:%M:%S')
}))
f.close()
return sim, num_threads, parameters
