def add_LPU(config, manager):
config_photor = config['Photoreceptor']
gexf_file = config_photor['gexf_file']
input_file = config_photor['input_file']
output_file = config_photor['output_file']
G = generate_gexf(config_photor, gexf_file)
LPU.graph_to_dicts(G)
comp_dict, conns = LPU.graph_to_dicts(G)
LPU_id = 'photoreceptor'
debug = config_photor['debug']
dt = config['General']['dt']
extra_comps = [PhotoreceptorModel]
input_processor = StepInputProcessor('photon', G.nodes(), 10000, 0.2, 1)
output_processor = FileOutputProcessor([('V', G.nodes())],
output_file,
sample_interval=1)
manager.add(LPU,
LPU_id,
dt,
comp_dict,
conns,
device=0,
input_processors=[input_processor],
output_processors=[output_processor],
debug=debug,
time_sync=False,
extra_comps=extra_comps)
def simulation(dt, N, output_n, nsteps = 10000, output = None):
start_time = time.time()
dur = nsteps * dt
steps = nsteps
man = core.Manager()
G = create_graph(N)
print("Creating graph completed in {} seconds.".format(time.time()-start_time))
start_time = time.time()
#comp_dict, conns = LPU.graph_to_dicts(G, remove_edge_id=False)
fl_input_processor = StepInputProcessor('I', ['neuron_{}'.format(i) for i in range(N)], 20.0, 0.0, dur)
fl_output_processor = []
if output == 'disk':
fl_output_processor.append(
FileOutputProcessor([('V', None), ('spike_state', None)],
'neurodriver_output_{}.h5'.format(output_n),
sample_interval=1, cache_length=2000)])
elif output == 'memory':
fl_output_processor.append(
OutputRecorder([('spike_state', None), ('V', None)],
sample_interval = 1)])
def add_retina_LPU(config, retina_index, retina, manager):
'''
This method adds Retina LPU and its parameters to the manager
so that it can be initialized later. Depending on configuration
input can either be created in advance and read from file or
generated during simulation by a generator object.
--
config: configuration dictionary like object
i: identifier of eye in case more than one is used
retina: retina array object required for the generation of
graph.
manager: manager object to which LPU will be added
generator: generator object or None
'''
dt = config['General']['dt']
debug = config['Retina']['debug']
time_sync = config['Retina']['time_sync']
input_filename = config['Retina']['input_file']
output_filename = config['Retina']['output_file']
gexf_filename = config['Retina']['gexf_file']
suffix = config['General']['file_suffix']
output_file = '{}{}{}.h5'.format(output_filename, retina_index, suffix)
gexf_file = '{}{}{}.gexf.gz'.format(gexf_filename, retina_index, suffix)
inputmethod = config['Retina']['inputmethod']
if inputmethod == 'read':
print('Generating input files')
with Timer('input generation'):
input_processor = RetinaFileInputProcessor(config, retina)
else:
print('Using input generating function')
input_processor = RetinaInputProcessor(config, retina)
output_processor = FileOutputProcessor([('V',None),('spike_state',None)], output_file, sample_interval=1)
# retina also allows a subset of its graph to be taken
# in case it is needed later to split the retina model to more
# GPUs
G = retina.get_worker_nomaster_graph()
nx.write_gexf(G, gexf_file)
(comp_dict, conns) = LPU.graph_to_dicts(G)
retina_id = get_retina_id(retina_index)
extra_comps = [LeakyIAF, BufferPhoton]
manager.add(LPU, retina_id, dt, comp_dict, conns,
device = retina_index, input_processors = [input_processor],
output_processors = [output_processor],
debug=debug, time_sync=time_sync, extra_comps = extra_comps)
def add_master_LPU(config, retina_index, retina, manager):
dt = config['General']['dt']
debug = config['Retina']['debug']
time_sync = config['Retina']['time_sync']
input_filename = config['Retina']['input_file']
output_filename = config['Retina']['output_file']
gexf_filename = config['Retina']['gexf_file']
suffix = config['General']['file_suffix']
output_file = '{}{}{}.h5'.format(output_filename, retina_index, suffix)
gexf_file = '{}{}{}.gexf.gz'.format(gexf_filename, retina_index, suffix)
inputmethod = config['Retina']['inputmethod']
if inputmethod == 'read':
print('Generating input files')
with Timer('input generation'):
input_processor = RetinaFileInputProcessor(config, retina)
else:
print('Using input generating function')
input_processor = RetinaInputProcessor(config, retina)
input_processor = get_input_gen(config, retina)
uids_to_record = [
'ret_{}_{}'.format(name, i) for i in range(retina.num_elements)
for name in ['R1', 'R2', 'R3', 'R4', 'R5', 'R6']
]
output_processor = FileOutputProcessor([('V', uids_to_record)],
output_file,
sample_interval=1)
G = retina.get_master_graph()
nx.write_gexf(G, gexf_file)
(comp_dict, conns) = LPU.lpu_parser(gexf_file)
master_id = get_master_id(retina_index)
extra_comps = [BufferPhoton, BufferVoltage]
manager.add(LPU,
master_id,
dt,
comp_dict,
conns,
device=retina_index,
input_processors=[input_processor],
output_processors=[output_processor],
debug=debug,
time_sync=time_sync,
extra_comps=extra_comps)
def loadOutputProcessors(filename, outputProcessor_dicts):
outList = []
record = {}
for a in outputProcessor_dicts:
outprocessor_class = a.get('class')
if outprocessor_class == 'Record':
to_record = [(k, v['uids']) for k, v in a['uid_dict'].items()]
processor = FileOutputProcessor(to_record,
filename,
sample_interval = a.get('sample_interval', 1))
outList.append(processor)
record = a['uid_dict']
else:
d = importlib.import_module(a.get('module'))
processor = getattr(d, outprocessor_class)
sig = inspect.signature(processor)
arg_dict = {param_name: a.get(param_name) if param.default is param.empty\
else a.get(param_name, param.default) \
for param_name, param in sig.parameters.items()}
outList.append(processor(**arg_dict))
return outList, record
def add_lamina_LPU(config, lamina_index, lamina, manager):
'''
This method adds Lamina LPU and its parameters to the manager
so that it can be initialized later.
--
config: configuration dictionary like object
i: identifier of eye in case more than one is used
lamina: lamina array object required for the generation of
graph.
manager: manager object to which LPU will be added
generator: generator object or None
'''
output_filename = config['Lamina']['output_file']
gexf_filename = config['Lamina']['gexf_file']
suffix = config['General']['file_suffix']
dt = config['General']['dt']
debug = config['Lamina']['debug']
time_sync = config['Lamina']['time_sync']
output_file = '{}{}{}.h5'.format(output_filename, lamina_index, suffix)
gexf_file = '{}{}{}.gexf.gz'.format(gexf_filename, lamina_index, suffix)
G = lamina.get_graph()
nx.write_gexf(G, gexf_file)
(comp_dict, conns) = LPU.graph_to_dicts(G)
lamina_id = get_lamina_id(lamina_index)
extra_comps = [BufferVoltage]
output_processor = FileOutputProcessor(
[('V', None)], output_file,
sample_interval=1)
manager.add(LPU, lamina_id, dt, comp_dict, conns,
output_processors = [output_processor],
device=lamina_index+1, debug=debug, time_sync=time_sync,
extra_comps = extra_comps)
'class': 'PowerGPotGPot',
'name': 'PowerGPotGPot',
'gmax': 0.4,
'threshold': -55.0,
'slope': 0.02,
'power': 1.0,
'saturation': 0.4,
'reverse': 0.0
})
comp_dict, conns = LPU.graph_to_dicts(G)
fl_input_processor = RampInputProcessor('V', ['synapse0'], 0.0, 1.0, -70.0,
-30.0)
fl_output_processor = FileOutputProcessor([('g', None)],
'new_output.h5',
sample_interval=1)
man.add(LPU,
'ge',
dt,
comp_dict,
conns,
device=args.gpu_dev,
input_processors=[fl_input_processor],
output_processors=[fl_output_processor],
debug=args.debug)
man.spawn()
man.start(steps=args.steps)
man.wait()
from neurokernel.LPU.LPU import LPU
(comp_dict, conns) = LPU.lpu_parser('neuroballad_temp_model.gexf.gz')
with open('run_parameters.pickle', 'rb') as f:
run_parameters = pickle.load(f)
with open('record_parameters.pickle', 'rb') as f:
record_parameters = pickle.load(f)
dur = 1.0
dt = 1e-4
dur = run_parameters[0]
dt = run_parameters[1]
fl_input_processor = FileInputProcessor('neuroballad_temp_model_input.h5')
from neurokernel.LPU.OutputProcessors.FileOutputProcessor import FileOutputProcessor
output_processor = FileOutputProcessor(record_parameters,
'neuroballad_temp_model_output.h5',
sample_interval=1)
#Parse extra arguments
parser = argparse.ArgumentParser()
parser.add_argument(
'--debug',
default=True,
dest='debug',
action='store_true',
help=
'Write connectivity structures and inter-LPU routed data in debug folder')
parser.add_argument(
'-l',
'--log',
default='both',
def run(connected):
"""
Set `connected` to True to connect the LPUs.
"""
import neurokernel.mpi_relaunch
out_name = 'un' if not connected else 'co'
man = core.Manager()
lpu_file_0 = './data/generic_lpu_0.gexf.gz'
lpu_file_1 = './data/generic_lpu_1.gexf.gz'
comp_dict_0, conns_0 = LPU.lpu_parser(lpu_file_0)
comp_dict_1, conns_1 = LPU.lpu_parser(lpu_file_1)
fl_input_processor_0 = FileInputProcessor(
'./data/generic_lpu_0_input.h5')
fl_output_processor_0 = FileOutputProcessor(
[('V', None), ('spike_state', None)],
'generic_lpu_0_%s_output.h5' % out_name,
sample_interval=1)
lpu_0_id = 'lpu_0'
man.add(LPU,
lpu_0_id,
dt,
comp_dict_0,
conns_0,
input_processors=[fl_input_processor_0],
output_processors=[fl_output_processor_0],
device=args.gpu_dev[0],
debug=args.debug,
time_sync=args.time_sync)
fl_input_processor_1 = FileInputProcessor(
'./data/generic_lpu_1_input.h5')
fl_output_processor_1 = FileOutputProcessor(
[('V', None), ('spike_state', None)],
'generic_lpu_1_%s_output.h5' % out_name,
sample_interval=1)
lpu_1_id = 'lpu_1'
man.add(LPU,
lpu_1_id,
dt,
comp_dict_1,
conns_1,
input_processors=[fl_input_processor_1],
output_processors=[fl_output_processor_1],
device=args.gpu_dev[1],
debug=args.debug,
time_sync=args.time_sync)
# Create random connections between the input and output ports if the LPUs
# are to be connected:
if connected:
# Find all output and input port selectors in each LPU:
out_ports_spk_0 = plsel.Selector(','.join(
LPU.extract_out_spk(comp_dict_0, 'id')[0]))
out_ports_gpot_0 = plsel.Selector(','.join(
LPU.extract_out_gpot(comp_dict_0, 'id')[0]))
out_ports_spk_1 = plsel.Selector(','.join(
LPU.extract_out_spk(comp_dict_1, 'id')[0]))
out_ports_gpot_1 = plsel.Selector(','.join(
LPU.extract_out_gpot(comp_dict_1, 'id')[0]))
in_ports_spk_0 = plsel.Selector(','.join(
LPU.extract_in_spk(comp_dict_0, 'id')[0]))
in_ports_gpot_0 = plsel.Selector(','.join(
LPU.extract_in_gpot(comp_dict_0, 'id')[0]))
in_ports_spk_1 = plsel.Selector(','.join(
LPU.extract_in_spk(comp_dict_1, 'id')[0]))
in_ports_gpot_1 = plsel.Selector(','.join(
LPU.extract_in_gpot(comp_dict_1, 'id')[0]))
out_ports_0 = plsel.Selector.union(out_ports_spk_0,
out_ports_gpot_0)
out_ports_1 = plsel.Selector.union(out_ports_spk_1,
out_ports_gpot_1)
in_ports_0 = plsel.Selector.union(in_ports_spk_0, in_ports_gpot_0)
in_ports_1 = plsel.Selector.union(in_ports_spk_1, in_ports_gpot_1)
# Initialize a connectivity pattern between the two sets of port
# selectors:
pat = pattern.Pattern(
plsel.Selector.union(out_ports_0, in_ports_0),
plsel.Selector.union(out_ports_1, in_ports_1))
# Create connections from the ports with identifiers matching the output
# ports of one LPU to the ports with identifiers matching the input
# ports of the other LPU:
N_conn_spk_0_1 = min(len(out_ports_spk_0), len(in_ports_spk_1))
N_conn_gpot_0_1 = min(len(out_ports_gpot_0), len(in_ports_gpot_1))
for src, dest in zip(
random.sample(out_ports_spk_0.identifiers, N_conn_spk_0_1),
random.sample(in_ports_spk_1.identifiers, N_conn_spk_0_1)):
pat[src, dest] = 1
pat.interface[src, 'type'] = 'spike'
pat.interface[dest, 'type'] = 'spike'
for src, dest in zip(
random.sample(out_ports_gpot_0.identifiers,
N_conn_gpot_0_1),
random.sample(in_ports_gpot_1.identifiers,
N_conn_gpot_0_1)):
pat[src, dest] = 1
pat.interface[src, 'type'] = 'gpot'
pat.interface[dest, 'type'] = 'gpot'
man.connect(lpu_0_id, lpu_1_id, pat, 0, 1)
man.spawn()
man.start(steps=args.steps)
man.wait()
dt = 1e-4
dur = 0.2
steps = int(dur / dt)
debug = False
for name in lpu_name_list:
if name in ['BU', 'bu', 'PB']:
#in_file_name = '%s_input.h5' % name
input_processor = [FileInputProcessor('{}_input.h5'.format(name))]
else:
#in_file_name = None
input_processor = []
#out_file_name = '%s_output.h5' % name
output_processor = [
FileOutputProcessor([('spike_state', None)],
'{}_output.h5'.format(name),
sample_interval=1)
]
# Since the LPUs are relatively small, they can all use the same GPU:
# man.add(LPU, name, dt, lpu_name_to_n_dict[name],
# lpu_name_to_s_dict[name],
# input_file=in_file_name,
# output_file=out_file_name,
# device=0,
# debug=debug, time_sync=False)
man.add(LPU,
name,
dt,
lpu_name_to_comp_dict[name],
lpu_name_to_conn_list[name],
def main():
import neurokernel.mpi_relaunch
import neurokernel.core_gpu as core
(comp_dict, conns) = LPU.lpu_parser('neuroballad_temp_model.gexf.gz')
with open('run_parameters.pickle', 'rb') as f:
run_parameters = pickle.load(f)
with open('record_parameters.pickle', 'rb') as f:
record_parameters = pickle.load(f)
dur = 1.0
dt = 1e-4
dur = run_parameters[0]
dt = run_parameters[1]
fl_input_processor = FileInputProcessor('neuroballad_temp_model_input.h5')
from neurokernel.LPU.OutputProcessors.FileOutputProcessor import FileOutputProcessor
output_processor = FileOutputProcessor(record_parameters,
'neuroballad_temp_model_output.h5',
sample_interval=1)
#Parse extra arguments
parser = argparse.ArgumentParser()
parser.add_argument(
'--debug',
default=False,
dest='debug',
action='store_true',
help=
'Write connectivity structures and inter-LPU routed data in debug folder'
)
parser.add_argument(
'-l',
'--log',
default='file',
type=str,
help='Log output to screen [file, screen, both, or none; default:none]'
)
parser.add_argument('-r',
'--time_sync',
default=False,
action='store_true',
help='Time data reception throughput [default: False]')
parser.add_argument('-g',
'--gpu_dev',
default=[0, 1],
type=int,
nargs='+',
help='GPU device numbers [default: 0 1]')
parser.add_argument('-d',
'--disconnect',
default=False,
action='store_true',
help='Run with disconnected LPUs [default: False]')
args = parser.parse_args()
file_name = None
screen = False
if args.log.lower() in ['file', 'both']:
file_name = 'neurokernel.log'
if args.log.lower() in ['screen', 'both']:
screen = True
logger = setup_logger(file_name=file_name, screen=screen)
man = core.Manager()
man.add(LPU,
'lpu',
dt,
comp_dict,
conns,
input_processors=[fl_input_processor],
output_processors=[output_processor],
device=args.gpu_dev[1],
debug=True)
steps = int(dur / dt)
man.spawn()
man.start(steps=steps)
man.wait()
def launch(self, user_id, task):
neuron_uid_list = [str(a) for a in task['neuron_list']]
conf_obj = get_config_obj()
config = conf_obj.conf
if config['Retina']['intype'] == 'Natural':
coord_file = config['InputType']['Natural']['coord_file']
tmp = os.path.splitext(coord_file)
config['InputType']['Natural']['coord_file'] = '{}_{}{}'.format(
tmp[0], user_id, tmp[1])
setup_logger(file_name='neurokernel_' + user_id + '.log', screen=True)
manager = core.Manager()
lpus = {}
patterns = {}
G = task['data']
with open('G.pickle', 'wb') as f:
pickle.dump(G, f, protocol=pickle.HIGHEST_PROTOCOL)
print(G)
print(G.keys())
print(G['LPU'])
print(G['LPU'].keys())
# get graph and output_uid_list for each LPU
for k, lpu in G['LPU'].items():
lpus[k] = {}
g_lpu_na = create_graph_from_database_returned(lpu)
lpu_nk_graph = nk.na_lpu_to_nk_new(g_lpu_na)
lpus[k]['graph'] = lpu_nk_graph
lpus[k]['output_uid_list'] = list(
set(lpu_nk_graph.nodes()).intersection(set(neuron_uid_list)))
lpus[k]['output_file'] = '{}_output_{}.h5'.format(k, user_id)
for kkey, lpu in lpus.items():
graph = lpu['graph']
for uid, comp in graph.node.items():
if 'attr_dict' in comp:
print('Found attr_dict; fixing...')
nx.set_node_attributes(graph, {uid: comp['attr_dict']})
# print('changed',uid)
graph.nodes[uid].pop('attr_dict')
for i, j, k, v in graph.edges(keys=True, data=True):
if 'attr_dict' in v:
for key in v['attr_dict']:
nx.set_edge_attributes(
graph, {(i, j, k): {
key: v['attr_dict'][key]
}})
graph.edges[(i, j, k)].pop('attr_dict')
lpus[kkey]['graph'] = graph
# get graph for each Pattern
for k, pat in G['Pattern'].items():
l1, l2 = k.split('-')
if l1 in lpus and l2 in lpus:
g_pattern_na = create_graph_from_database_returned(pat)
pattern_nk = nk.na_pat_to_nk(g_pattern_na)
print(lpus[l1]['graph'].nodes(data=True))
lpu_ports = [node[1]['selector'] \
for node in lpus[l1]['graph'].nodes(data=True) \
if node[1]['class']=='Port'] + \
[node[1]['selector'] \
for node in lpus[l2]['graph'].nodes(data=True) \
if node[1]['class']=='Port']
pattern_ports = pattern_nk.nodes()
patterns[k] = {}
patterns[k]['graph'] = pattern_nk.subgraph(
list(set(lpu_ports).intersection(set(pattern_ports))))
dt = config['General']['dt']
if 'dt' in task:
dt = task['dt']
print(dt)
# add LPUs to manager
for k, lpu in lpus.items():
lpu_name = k
graph = lpu['graph']
for uid, comp in graph.node.items():
if 'attr_dict' in comp:
nx.set_node_attributes(graph, {uid: comp['attr_dict']})
# print('changed',uid)
graph.nodes[uid].pop('attr_dict')
for i, j, ko, v in graph.edges(keys=True, data=True):
if 'attr_dict' in v:
for key in v['attr_dict']:
nx.set_edge_attributes(
graph, {(i, j, ko): {
key: v['attr_dict'][key]
}})
graph.edges[(i, j, ko)].pop('attr_dict')
nx.write_gexf(graph, 'name.gexf')
with open(lpu_name + '.pickle', 'wb') as f:
pickle.dump(graph, f, protocol=pickle.HIGHEST_PROTOCOL)
comps = graph.node.items()
#for uid, comp in comps:
# if 'attr_dict' in comp:
# nx.set_node_attributes(graph, {uid: comp['attr_dict']})
# print('changed',uid)
# if 'class' in comp:
if k == 'retina':
prs = [node for node in graph.nodes(data=True) \
if node[1]['class'] == 'PhotoreceptorModel']
for pr in prs:
graph.node[pr[0]]['num_microvilli'] = 3000
input_processors = [
RetinaInputIndividual(config, prs, user_id)
]
extra_comps = [PhotoreceptorModel]
retina_input_uids = [a[0] for a in prs]
elif k == 'EB':
input_processor = StepInputProcessor('I', [node[0] for node in graph.nodes(data=True) \
if node[1]['class'] == 'LeakyIAF'], 40.0, 0.0, 1.0)
input_processors = [input_processor]
extra_comps = [BufferVoltage]
else:
input_processors = []
extra_comps = [BufferVoltage]
if 'inputProcessors' in task:
input_processors = loadExperimentSettings(
task['inputProcessors'])
output_processor = FileOutputProcessor(
[('V', lpu['output_uid_list'])],
lpu['output_file'],
sample_interval=10)
(comp_dict, conns) = LPU.graph_to_dicts(graph)
# print(comp_dict)
# print(conns)
print(k)
manager.add(LPU,
k,
dt,
comp_dict,
conns,
device=0,
input_processors=input_processors,
output_processors=[output_processor],
extra_comps=extra_comps,
debug=True)
# connect LPUs by Patterns
for k, pattern in patterns.items():
l1, l2 = k.split('-')
if l1 in lpus and l2 in lpus:
print('Connecting {} and {}'.format(l1, l2))
pat, key_order = Pattern.from_graph(nx.DiGraph(
pattern['graph']),
return_key_order=True)
print(l1, l2)
print(key_order)
with Timer('update of connections in Manager'):
manager.connect(l1,
l2,
pat,
int_0=key_order.index(l1),
int_1=key_order.index(l2))
# start simulation
steps = config['General']['steps']
ignored_steps = config['General']['ignored_steps']
if 'steps' in task:
steps = task['steps']
if 'ignored_steps' in task:
ignored_steps = task['ignored_steps']
# ignored_steps = 0
# steps = 100
manager.spawn()
manager.start(steps=steps)
manager.wait()
time.sleep(5)
print(task)
# post-processing inputs (hard coded, can be better organized)
inputs = {
u'ydomain': 1.0,
u'xdomain': dt * (steps - ignored_steps),
u'dt': dt * 10,
u'data': {}
}
if 'retina' in lpus:
input_array = si.read_array('{}_{}.h5'.format(
config['Retina']['input_file'], user_id))
inputs[u'ydomain'] = input_array.max()
for i, item in enumerate(retina_input_uids):
inputs['data'][item] = np.hstack(
(np.arange(int((steps - ignored_steps) / 10)).reshape(
(-1, 1)) * dt * 10, input_array[ignored_steps::10,
i:i + 1])).tolist()
del input_array
# post-processing outputs from all LPUs and combine them into one dictionary
result = {
u'ydomain': 1,
u'xdomain': dt * (steps - ignored_steps),
u'dt': dt * 10,
u'data': {}
}
for k, lpu in lpus.items():
with h5py.File(lpu['output_file']) as output_file:
uids = output_file['V']['uids'][:]
output_array = output_file['V']['data'][:]
for i, item in enumerate(uids):
output = output_array[int(ignored_steps / 10):, i:i + 1]
# tmp = output.max()-output.min()
# if tmp <= 0.01: #mV
# output = (output - output.min()) + 0.5
# else:
# output = (output - output.min())/tmp*0.9+0.1
result['data'][item] = np.hstack(
(np.arange(int((steps - ignored_steps) / 10)).reshape(
(-1, 1)) * dt * 10, output)).tolist()
return inputs, result
def compile(self, duration, dt=None, steps=None, in_list=None,
record=('V', 'spike_state', 'I'), extra_comps=None,
input_filename='neuroballad_temp_model_input.h5',
output_filename='neuroballad_temp_model_output.h5',
graph_filename='neuroballand_temp_graph.gexf.gz',
device=0, sample_interval=1):
if dt is not None:
if steps is not None:
assert dt*steps == duration, 'dt*step != duration'
else:
steps = int(duration/dt)
t = np.linspace(0, duration, steps)
else:
if steps is not None:
t = np.linspace(0, duration, steps)
dt = t[1] - t[0]
else:
raise ValueError('dt and step cannot both be None')
self.config = self.config._replace(duration=duration,
steps=steps,
dt=dt,
t=t,
device=device)
# compile inputs
if in_list is None:
in_list = self._inputs
uids = []
for i in in_list:
uids.append(self.encode_name(str(i.node_id),
experiment_name=i.experiment_name))
input_vars = []
for i in in_list:
if isinstance(i.var, list):
for j in i.var:
input_vars.append(j)
else:
input_vars.append(i.var)
input_vars = list(set(input_vars))
uids = np.array(list(set(uids)), dtype='S')
Is = {}
Inodes = {}
for i in input_vars:
Inodes[i] = []
for i in in_list:
in_name = self.encode_name(str(i.node_id),
experiment_name=i.experiment_name)
if in_name in list(self.G.nodes(data=False)):
pass
else:
raise ValueError(
'Input node {} not found in Circuit.'.format(in_name))
if isinstance(i.var, list):
for j in i.var:
Inodes[j].append(
self.encode_name(str(i.node_id),
experiment_name=i.experiment_name))
else:
Inodes[i.var].append(
self.encode_name(str(i.node_id),
experiment_name=i.experiment_name))
for i in input_vars:
Inodes[i] = np.array(list(set(Inodes[i])), dtype='S')
for i in input_vars:
Is[i] = np.zeros((self.config.steps, len(Inodes[i])),
dtype=self.dtype)
for i in in_list:
if isinstance(i.var, list):
for j in i.var:
Is[j] = i.add(self, Inodes[j], Is[j], t, var=j)
else:
Is[i.var] = i.add(self, Inodes[i.var], Is[i.var], t, var=i.var)
with h5py.File(input_filename, 'w') as f:
for i in input_vars:
# print(i + '/uids')
i_nodes = Inodes[i]
"""
try:
i_nodes = [i.decode('ascii') for i in i_nodes]
except:
pass
i_nodes = [self.encode_name(i) for i in i_nodes]
"""
i_nodes = np.array(i_nodes, dtype='S')
f.create_dataset(i + '/uids', data=i_nodes)
f.create_dataset(i + '/data', (self.config.steps, len(Inodes[i])),
dtype=self.dtype,
data=Is[i])
if graph_filename is not None:
nx.write_gexf(self.G, graph_filename)
from neurokernel.core_gpu import Manager
from neurokernel.LPU.LPU import LPU
import neurokernel.mpi_relaunch
from neurokernel.LPU.InputProcessors.FileInputProcessor import \
FileInputProcessor
from neurokernel.LPU.OutputProcessors.FileOutputProcessor import \
FileOutputProcessor
input_processor = FileInputProcessor(input_filename)
(comp_dict, conns) = LPU.graph_to_dicts(self.G)
output_processor = FileOutputProcessor([(i, None) for i in list(record)],
output_filename,
sample_interval=sample_interval)
self.manager = Manager()
self.manager.add(LPU, self.experiment_name, self.config.dt,
comp_dict, conns,
device=self.config.device,
input_processors=[input_processor],
output_processors=[output_processor],
debug=False,
extra_comps=extra_comps if extra_comps is not None else [])
def compile(self,
duration,
dt=None,
steps=None,
in_list=None,
record=('V', 'spike_state', 'I'),
extra_comps=None,
input_filename='neuroballad_temp_model_input.h5',
output_filename='neuroballad_temp_model_output.h5',
graph_filename='neuroballad_temp_graph.gexf.gz',
device=0,
sample_interval=1,
execute_in_same_thread=True):
"""
Compiles a neuroballad circuit before execution.
# Arguments
duration (float): Simulation duration.
dt (float): Time step size.
steps (int): Number of steps to use in simulation. Optional; don't use dt if provided.
in_list (list): List of inputs to use during compilation.
record (tuple): Tuple of variables to record. Defaults to ('V', 'spike_state', 'I').
extra_comps (list): List of new, custom components to include for your simulation.
input_filename (str): The .h5 file name to use for the input.
output_filename (str): The .h5 file name to use for recording the output.
graph_filename (str): Name of the graph file to save the circuit to. Uses the .gexf format.
device (int): Device to use for execution.
sample_interval (int): Sampling interval for recording simulation output.
execute_in_same_thread (bool): Whether to execute the circuit in the current thread.
"""
if dt is not None:
if steps is not None:
warnings.warn(
"Both 'steps' and 'duration' arguments were specified. 'steps' argument is ignored."
)
steps = int(duration / dt)
else:
steps = int(duration / dt)
t = np.linspace(0, duration, steps)
else:
if steps is not None:
t = np.linspace(0, duration, steps)
dt = t[1] - t[0]
else:
raise ValueError('dt and step cannot both be None')
self.config = self.config._replace(duration=duration,
steps=steps,
dt=dt,
t=t,
device=device)
run_parameters = [duration, dt]
with open('run_parameters.pickle', 'wb') as f:
pickle.dump(run_parameters, f, protocol=pickle.HIGHEST_PROTOCOL)
# Compile inputs
if in_list is None:
in_list = self._inputs
uids = []
for i in in_list:
uids.append(
self.encode_name(str(i.node_id),
experiment_name=i.experiment_name))
input_vars = []
for i in in_list:
if isinstance(i.var, list):
for j in i.var:
input_vars.append(j)
else:
input_vars.append(i.var)
input_vars = list(set(input_vars))
uids = np.array(list(set(uids)), dtype='S')
Is = {}
Inodes = {}
for i in input_vars:
Inodes[i] = []
for i in in_list:
in_name = self.encode_name(str(i.node_id),
experiment_name=i.experiment_name)
if in_name in list(self.G.nodes(data=False)):
pass
else:
raise ValueError(
'Input node {} not found in Circuit.'.format(in_name))
if isinstance(i.var, list):
for j in i.var:
Inodes[j].append(
self.encode_name(str(i.node_id),
experiment_name=i.experiment_name))
else:
Inodes[i.var].append(
self.encode_name(str(i.node_id),
experiment_name=i.experiment_name))
for i in input_vars:
Inodes[i] = np.array(list(set(Inodes[i])), dtype='S')
for i in input_vars:
Is[i] = np.zeros((self.config.steps, len(Inodes[i])),
dtype=self.dtype)
for i in in_list:
if isinstance(i.var, list):
for j in i.var:
Is[j] = i.add(self, Inodes[j], Is[j], t, var=j)
else:
Is[i.var] = i.add(self, Inodes[i.var], Is[i.var], t, var=i.var)
with h5py.File(input_filename, 'w') as f:
for i in input_vars:
# print(i + '/uids')
i_nodes = Inodes[i]
"""
try:
i_nodes = [i.decode('ascii') for i in i_nodes]
except:
pass
i_nodes = [self.encode_name(i) for i in i_nodes]
"""
i_nodes = np.array(i_nodes, dtype='S')
f.create_dataset(i + '/uids', data=i_nodes)
f.create_dataset(i + '/data',
(self.config.steps, len(Inodes[i])),
dtype=self.dtype,
data=Is[i])
recorders = []
for i in record:
recorders.append((i, None))
with open('record_parameters.pickle', 'wb') as f:
pickle.dump(recorders, f, protocol=pickle.HIGHEST_PROTOCOL)
if graph_filename is not None:
nx.write_gexf(self.G, graph_filename)
if execute_in_same_thread:
from neurokernel.core_gpu import Manager
from neurokernel.LPU.LPU import LPU
# import neurokernel.mpi_relaunch
from neurokernel.LPU.InputProcessors.FileInputProcessor import \
FileInputProcessor
from neurokernel.LPU.OutputProcessors.FileOutputProcessor import \
FileOutputProcessor
input_processor = FileInputProcessor(input_filename)
(comp_dict, conns) = LPU.graph_to_dicts(self.G)
output_processor = FileOutputProcessor(
[(i, None) for i in list(record)],
output_filename,
sample_interval=sample_interval)
self.manager = Manager()
self.manager.add(
LPU,
self.experiment_name,
self.config.dt,
comp_dict,
conns,
device=self.config.device,
input_processors=[input_processor],
output_processors=[output_processor],
debug=False,
extra_comps=extra_comps if extra_comps is not None else [])
man = core.Manager()
for i, neu_num in neu_dict.iteritems():
lpu_entry = {}
if i == 0:
in_file_name = in_file_name_0
fl_input_processors = [FileInputProcessor(in_file_name)]
else:
in_file_name = None
fl_input_processors = []
lpu_file_name = 'generic_lpu_%s.gexf.gz' % i
out_file_name = 'generic_lpu_%s_output.h5' % i
fl_output_processors = [
FileOutputProcessor([('V', None), ('spike_state', None)],
out_file_name,
sample_interval=1)
]
id = 'lpu_%s' % i
g.create_lpu(lpu_file_name, id, *neu_num)
(comp_dict, conns) = LPU.lpu_parser(lpu_file_name)
man.add(LPU,
id,
dt,
comp_dict,
conns,
input_processors=fl_input_processors,
output_processors=fl_output_processors,
def launch(self, user_id, task):
neuron_uid_list = [str(a) for a in task['neuron_list']]
conf_obj = get_config_obj()
config = conf_obj.conf
if config['Retina']['intype'] == 'Natural':
coord_file = config['InputType']['Natural']['coord_file']
tmp = os.path.splitext(coord_file)
config['InputType']['Natural']['coord_file'] = '{}_{}{}'.format(
tmp[0], user_id, tmp[1])
setup_logger(file_name='neurokernel_' + user_id + '.log', screen=False)
manager = core.Manager()
lpus = {}
patterns = {}
G = task['success']['data']
# get graph and output_uid_list for each LPU
for k, lpu in G['LPU'].iteritems():
lpus[k] = {}
g_lpu_na = create_graph_from_database_returned(lpu)
lpu_nk_graph = nk.na_lpu_to_nk_new(g_lpu_na)
lpus[k]['graph'] = lpu_nk_graph
lpus[k]['output_uid_list'] = list(
set(lpu_nk_graph.nodes()).intersection(set(neuron_uid_list)))
lpus[k]['output_file'] = '{}_output_{}.h5'.format(k, user_id)
# get graph for each Pattern
for k, pat in G['Pattern'].iteritems():
l1, l2 = k.split('-')
if l1 in lpus and l2 in lpus:
g_pattern_na = create_graph_from_database_returned(pat)
pattern_nk = nk.na_pat_to_nk(g_pattern_na)
lpu_ports = [node[1]['selector'] \
for node in lpus[l1]['graph'].nodes(data=True) \
if node[1]['class']=='Port'] + \
[node[1]['selector'] \
for node in lpus[l2]['graph'].nodes(data=True) \
if node[1]['class']=='Port']
pattern_ports = pattern_nk.nodes()
patterns[k] = {}
patterns[k]['graph'] = pattern_nk.subgraph(
list(set(lpu_ports).intersection(set(pattern_ports))))
dt = config['General']['dt']
# add LPUs to manager
for k, lpu in lpus.iteritems():
graph = lpu['graph']
if k == 'retina':
prs = [node for node in graph.nodes(data=True) \
if node[1]['class'] == 'PhotoreceptorModel']
for pr in prs:
graph.node[pr[0]]['num_microvilli'] = 3000
input_processors = [
RetinaInputIndividual(config, prs, user_id)
]
extra_comps = [PhotoreceptorModel]
retina_input_uids = [a[0] for a in prs]
else:
input_processors = []
extra_comps = [BufferVoltage]
output_processor = FileOutputProcessor(
[('V', lpu['output_uid_list'])],
lpu['output_file'],
sample_interval=10)
(comp_dict, conns) = LPU.graph_to_dicts(graph)
manager.add(LPU,
k,
dt,
comp_dict,
conns,
device=0,
input_processors=input_processors,
output_processors=[output_processor],
extra_comps=extra_comps)
# connect LPUs by Patterns
for k, pattern in patterns.iteritems():
l1, l2 = k.split('-')
if l1 in lpus and l2 in lpus:
print('Connecting {} and {}'.format(l1, l2))
pat, key_order = Pattern.from_graph(
nx.DiGraph(pattern['graph']))
with Timer('update of connections in Manager'):
manager.connect(l1,
l2,
pat,
int_0=key_order.index(l1),
int_1=key_order.index(l2))
# start simulation
steps = config['General']['steps']
ignored_steps = config['General']['ignored_steps']
manager.spawn()
manager.start(steps=steps)
manager.wait()
time.sleep(5)
# post-processing inputs (hard coded, can be better organized)
inputs = {
u'ydomain': 1.0,
u'xdomain': dt * (steps - ignored_steps),
u'dt': dt * 10,
u'data': {}
}
if 'retina' in lpus:
input_array = si.read_array('{}_{}.h5'.format(
config['Retina']['input_file'], user_id))
inputs[u'ydomain'] = input_array.max()
for i, item in enumerate(retina_input_uids):
inputs['data'][item] = np.hstack(
(np.arange(int((steps - ignored_steps) / 10)).reshape(
(-1, 1)) * dt * 10, input_array[ignored_steps::10,
i:i + 1])).tolist()
del input_array
# post-processing outputs from all LPUs and combine them into one dictionary
result = {
u'ydomain': 1,
u'xdomain': dt * (steps - ignored_steps),
u'dt': dt * 10,
u'data': {}
}
for k, lpu in lpus.iteritems():
with h5py.File(lpu['output_file']) as output_file:
uids = output_file['V']['uids'][:]
output_array = output_file['V']['data'][:]
for i, item in enumerate(uids):
output = output_array[int(ignored_steps / 10):, i:i + 1]
tmp = output.max() - output.min()
if tmp <= 0.01: #mV
output = (output - output.min()) + 0.5
else:
output = (output - output.min()) / tmp * 0.9 + 0.1
result['data'][item] = np.hstack(
(np.arange(int((steps - ignored_steps) / 10)).reshape(
(-1, 1)) * dt * 10, output)).tolist()
return inputs, result
G.add_node(
'neuron0',
{
'class': 'LeakyIAF',
'name': 'LeakyIAF',
'resting_potential': -70.0,
'threshold': -45.0,
'capacitance': 0.07, # in mS
'resistance': 0.2, # in Ohm
})
comp_dict, conns = LPU.graph_to_dicts(G)
fl_input_processor = StepInputProcessor('I', ['neuron0'], 40, 0.2, 0.8)
fl_output_processor = FileOutputProcessor([('spike_state', None),
('V', None)],
'new_output.h5',
sample_interval=1)
man.add(LPU,
'ge',
dt,
comp_dict,
conns,
device=args.gpu_dev,
input_processors=[fl_input_processor],
output_processors=[fl_output_processor],
debug=args.debug)
man.spawn()
man.start(steps=args.steps)
man.wait()
def main():
logging.basicConfig(level=logging.DEBUG, stream=sys.stdout,
format='%(asctime)s %(name)s %(levelname)s %(message)s')
logger = logging.getLogger('cx')
sys.setrecursionlimit(10000)
### Graph ###
graph = Graph(Config.from_url(cx_db, 'admin', 'admin',
initial_drop=False))
graph.include(models.Node.registry)
graph.include(models.Relationship.registry)
### Retina ###
config=retlam_demo.ConfigReader('retlam_default.cfg','../template_spec.cfg').conf
retina = get_retina(config)
##### Configuration ######
logger = setup_logger(screen=True)
lpu_selectors, to_list = get_cx_selectors_name()
lpu_name_to_comp_dict, lpu_name_to_conn_list, pat_name_list, pat_name_to_pat = cx_component(graph)
man = core.Manager()
dt = 1e-4
dur = 0.2
steps = int(dur/dt)
debug = True
lpu_name_list = ['BU', 'bu', 'EB', 'FB', 'PB']
for name in lpu_name_list:
input_processor = []
output_processor = [FileOutputProcessor([('spike_state', None), ('V',None), ('g',None), ('I', None)], '{}_output.h5'.format(name), sample_interval = 1)]
man.add(LPU, name, dt, lpu_name_to_comp_dict[name],
lpu_name_to_conn_list[name],
input_processors = input_processor,
output_processors = output_processor,
device=0,
debug=debug, time_sync=False)
retlam_demo.add_retina_LPU(config, 0, retina, man)
logger.info('add retina lpu')
for name in pat_name_list:
id_0, id_1 = name.split('-')
man.connect(id_0, id_1, pat_name_to_pat[name][0], pat_name_to_pat[name][1].index('0'), pat_name_to_pat[name][1].index('1'))
logger.info('link lpus among cx lpus')
link_retina_pat_cx(retina, lpu_selectors, to_list, man)
logger.info('link retina and cx lpu')
man.spawn()
man.start(steps)
man.wait()