def main():
import neurokernel.mpi_relaunch
parser = argparse.ArgumentParser()
parser.add_argument('-c',
'--config',
default='default',
help='configuration file')
parser.add_argument('-v',
'--value',
type=int,
default=-1,
help='Value that can overwrite configuration'
'by changing this script accordingly.'
'It is useful when need to run this script'
'repeatedly for different configuration')
args = parser.parse_args()
with Timer('getting configuration'):
conf_obj = get_config_obj(args)
config = conf_obj.conf
change_config(config, args.value)
edit_files(config)
setup_logging(config)
# with Timer('input generation'):
# gi.gen_input_norfscreen(config)
manager = core.Manager()
with Timer('photoreceptor instantiation'):
add_LPU(config, manager)
start_simulation(config, manager)
def simulation(dt, N, output_n, nsteps = 10000, output = None):
start_time = time.time()
dur = nsteps * dt
steps = nsteps
man = core.Manager()
G = create_graph(N)
print("Creating graph completed in {} seconds.".format(time.time()-start_time))
start_time = time.time()
#comp_dict, conns = LPU.graph_to_dicts(G, remove_edge_id=False)
fl_input_processor = StepInputProcessor('I', ['neuron_{}'.format(i) for i in range(N)], 20.0, 0.0, dur)
fl_output_processor = []
if output == 'disk':
fl_output_processor.append(
FileOutputProcessor([('V', None), ('spike_state', None)],
'neurodriver_output_{}.h5'.format(output_n),
sample_interval=1, cache_length=2000)])
elif output == 'memory':
fl_output_processor.append(
OutputRecorder([('spike_state', None), ('V', None)],
sample_interval = 1)])
def main():
import neurokernel.mpi_relaunch
# default limit is low for pickling
# the data structures passed through mpi
sys.setrecursionlimit(RECURSION_LIMIT)
resource.setrlimit(resource.RLIMIT_STACK,
(resource.RLIM_INFINITY, resource.RLIM_INFINITY))
parser = argparse.ArgumentParser()
parser.add_argument('-c',
'--config',
default='default',
help='configuration file')
parser.add_argument('-v',
'--value',
type=int,
default=-1,
help='Value that can overwrite configuration '
'by changing this script accordingly. '
'It is useful when need to run this script '
'repeatedly for different configuration')
args = parser.parse_args()
with Timer('getting configuration'):
conf_obj = get_config_obj(args)
config = conf_obj.conf
change_config(config, args.value)
setup_logging(config)
worker_num = config['Retina']['worker_num']
num_rings = config['Retina']['rings']
radius = config['Retina']['radius']
eulerangles = config['Retina']['eulerangles']
manager = core.Manager()
with Timer('instantiation of retina'):
transform = AlbersProjectionMap(radius, eulerangles).invmap
hexagon = hx.HexagonArray(num_rings=num_rings,
radius=radius,
transform=transform)
retina = ret.RetinaArray(hexagon, config)
# sets retina attribute which is required for the generation of
# receptive fields
#if generator is not None:
# generator.generate_datafiles(0, retina)
add_master_LPU(config, 0, retina, manager)
for j in range(worker_num):
add_worker_LPU(config, j, retina, manager)
connect_master_worker(config, j, retina, manager)
start_simulation(config, manager)
def main():
import neurokernel.mpi_relaunch
# default limit is low for pickling
# the data structures passed through mpi
sys.setrecursionlimit(RECURSION_LIMIT)
resource.setrlimit(resource.RLIMIT_STACK,
(resource.RLIM_INFINITY, resource.RLIM_INFINITY))
parser = argparse.ArgumentParser()
parser.add_argument('-c', '--config', default='default',
help='configuration file')
parser.add_argument('-v', '--value', type=int, default=-1,
help='Value that can overwrite configuration '
'by changing this script accordingly. '
'It is useful when need to run this script '
'repeatedly for different configuration')
args = parser.parse_args()
with Timer('getting configuration'):
conf_obj = get_config_obj(args)
config = conf_obj.conf
change_config(config, args.value)
setup_logging(config)
num_rings = config['Retina']['rings']
eulerangles = config['Retina']['eulerangles']
radius = config['Retina']['radius']
manager = core.Manager()
with Timer('instantiation of retina and lamina'):
transform = AlbersProjectionMap(radius, eulerangles).invmap
r_hexagon = r_hx.HexagonArray(num_rings=num_rings, radius=radius,
transform=transform)
l_hexagon = l_hx.HexagonArray(num_rings=num_rings, radius=radius,
transform=transform)
retina = ret.RetinaArray(r_hexagon, config)
lamina = lam.LaminaArray(l_hexagon, config)
add_retina_LPU(config, 0, retina, manager)
add_lamina_LPU(config, 0, lamina, manager)
connect_retina_lamina(config, 0, retina, lamina, manager)
start_simulation(config, manager)
parser.add_argument('-g',
'--gpu_dev',
default=0,
type=int,
help='GPU device number [default: 0]')
args = parser.parse_args()
file_name = None
screen = False
if args.log.lower() in ['file', 'both']:
file_name = 'neurokernel.log'
if args.log.lower() in ['screen', 'both']:
screen = True
logger = setup_logger(file_name=file_name, screen=screen)
man = core.Manager()
G = nx.MultiDiGraph()
G.add_node(
'neuron0',
{
'class': 'LeakyIAF',
'name': 'LeakyIAF',
'resting_potential': -70.0,
'threshold': -45.0,
'capacitance': 0.07, # in mS
'resistance': 0.2, # in Ohm
})
comp_dict, conns = LPU.graph_to_dicts(G)
def run(connected):
"""
Set `connected` to True to connect the LPUs.
"""
import neurokernel.mpi_relaunch
out_name = 'un' if not connected else 'co'
man = core.Manager()
lpu_file_0 = './data/generic_lpu_0.gexf.gz'
lpu_file_1 = './data/generic_lpu_1.gexf.gz'
comp_dict_0, conns_0 = LPU.lpu_parser(lpu_file_0)
comp_dict_1, conns_1 = LPU.lpu_parser(lpu_file_1)
fl_input_processor_0 = FileInputProcessor(
'./data/generic_lpu_0_input.h5')
fl_output_processor_0 = FileOutputProcessor(
[('V', None), ('spike_state', None)],
'generic_lpu_0_%s_output.h5' % out_name,
sample_interval=1)
lpu_0_id = 'lpu_0'
man.add(LPU,
lpu_0_id,
dt,
comp_dict_0,
conns_0,
input_processors=[fl_input_processor_0],
output_processors=[fl_output_processor_0],
device=args.gpu_dev[0],
debug=args.debug,
time_sync=args.time_sync)
fl_input_processor_1 = FileInputProcessor(
'./data/generic_lpu_1_input.h5')
fl_output_processor_1 = FileOutputProcessor(
[('V', None), ('spike_state', None)],
'generic_lpu_1_%s_output.h5' % out_name,
sample_interval=1)
lpu_1_id = 'lpu_1'
man.add(LPU,
lpu_1_id,
dt,
comp_dict_1,
conns_1,
input_processors=[fl_input_processor_1],
output_processors=[fl_output_processor_1],
device=args.gpu_dev[1],
debug=args.debug,
time_sync=args.time_sync)
# Create random connections between the input and output ports if the LPUs
# are to be connected:
if connected:
# Find all output and input port selectors in each LPU:
out_ports_spk_0 = plsel.Selector(','.join(
LPU.extract_out_spk(comp_dict_0, 'id')[0]))
out_ports_gpot_0 = plsel.Selector(','.join(
LPU.extract_out_gpot(comp_dict_0, 'id')[0]))
out_ports_spk_1 = plsel.Selector(','.join(
LPU.extract_out_spk(comp_dict_1, 'id')[0]))
out_ports_gpot_1 = plsel.Selector(','.join(
LPU.extract_out_gpot(comp_dict_1, 'id')[0]))
in_ports_spk_0 = plsel.Selector(','.join(
LPU.extract_in_spk(comp_dict_0, 'id')[0]))
in_ports_gpot_0 = plsel.Selector(','.join(
LPU.extract_in_gpot(comp_dict_0, 'id')[0]))
in_ports_spk_1 = plsel.Selector(','.join(
LPU.extract_in_spk(comp_dict_1, 'id')[0]))
in_ports_gpot_1 = plsel.Selector(','.join(
LPU.extract_in_gpot(comp_dict_1, 'id')[0]))
out_ports_0 = plsel.Selector.union(out_ports_spk_0,
out_ports_gpot_0)
out_ports_1 = plsel.Selector.union(out_ports_spk_1,
out_ports_gpot_1)
in_ports_0 = plsel.Selector.union(in_ports_spk_0, in_ports_gpot_0)
in_ports_1 = plsel.Selector.union(in_ports_spk_1, in_ports_gpot_1)
# Initialize a connectivity pattern between the two sets of port
# selectors:
pat = pattern.Pattern(
plsel.Selector.union(out_ports_0, in_ports_0),
plsel.Selector.union(out_ports_1, in_ports_1))
# Create connections from the ports with identifiers matching the output
# ports of one LPU to the ports with identifiers matching the input
# ports of the other LPU:
N_conn_spk_0_1 = min(len(out_ports_spk_0), len(in_ports_spk_1))
N_conn_gpot_0_1 = min(len(out_ports_gpot_0), len(in_ports_gpot_1))
for src, dest in zip(
random.sample(out_ports_spk_0.identifiers, N_conn_spk_0_1),
random.sample(in_ports_spk_1.identifiers, N_conn_spk_0_1)):
pat[src, dest] = 1
pat.interface[src, 'type'] = 'spike'
pat.interface[dest, 'type'] = 'spike'
for src, dest in zip(
random.sample(out_ports_gpot_0.identifiers,
N_conn_gpot_0_1),
random.sample(in_ports_gpot_1.identifiers,
N_conn_gpot_0_1)):
pat[src, dest] = 1
pat.interface[src, 'type'] = 'gpot'
pat.interface[dest, 'type'] = 'gpot'
man.connect(lpu_0_id, lpu_1_id, pat, 0, 1)
man.spawn()
man.start(steps=args.steps)
man.wait()
def launch(self, user_id, task):
neuron_uid_list = [str(a) for a in task['neuron_list']]
conf_obj = get_config_obj()
config = conf_obj.conf
if config['Retina']['intype'] == 'Natural':
coord_file = config['InputType']['Natural']['coord_file']
tmp = os.path.splitext(coord_file)
config['InputType']['Natural']['coord_file'] = '{}_{}{}'.format(
tmp[0], user_id, tmp[1])
setup_logger(file_name='neurokernel_' + user_id + '.log', screen=False)
manager = core.Manager()
lpus = {}
patterns = {}
G = task['success']['data']
# get graph and output_uid_list for each LPU
for k, lpu in G['LPU'].iteritems():
lpus[k] = {}
g_lpu_na = create_graph_from_database_returned(lpu)
lpu_nk_graph = nk.na_lpu_to_nk_new(g_lpu_na)
lpus[k]['graph'] = lpu_nk_graph
lpus[k]['output_uid_list'] = list(
set(lpu_nk_graph.nodes()).intersection(set(neuron_uid_list)))
lpus[k]['output_file'] = '{}_output_{}.h5'.format(k, user_id)
# get graph for each Pattern
for k, pat in G['Pattern'].iteritems():
l1, l2 = k.split('-')
if l1 in lpus and l2 in lpus:
g_pattern_na = create_graph_from_database_returned(pat)
pattern_nk = nk.na_pat_to_nk(g_pattern_na)
lpu_ports = [node[1]['selector'] \
for node in lpus[l1]['graph'].nodes(data=True) \
if node[1]['class']=='Port'] + \
[node[1]['selector'] \
for node in lpus[l2]['graph'].nodes(data=True) \
if node[1]['class']=='Port']
pattern_ports = pattern_nk.nodes()
patterns[k] = {}
patterns[k]['graph'] = pattern_nk.subgraph(
list(set(lpu_ports).intersection(set(pattern_ports))))
dt = config['General']['dt']
# add LPUs to manager
for k, lpu in lpus.iteritems():
graph = lpu['graph']
if k == 'retina':
prs = [node for node in graph.nodes(data=True) \
if node[1]['class'] == 'PhotoreceptorModel']
for pr in prs:
graph.node[pr[0]]['num_microvilli'] = 3000
input_processors = [
RetinaInputIndividual(config, prs, user_id)
]
extra_comps = [PhotoreceptorModel]
retina_input_uids = [a[0] for a in prs]
else:
input_processors = []
extra_comps = [BufferVoltage]
output_processor = FileOutputProcessor(
[('V', lpu['output_uid_list'])],
lpu['output_file'],
sample_interval=10)
(comp_dict, conns) = LPU.graph_to_dicts(graph)
manager.add(LPU,
k,
dt,
comp_dict,
conns,
device=0,
input_processors=input_processors,
output_processors=[output_processor],
extra_comps=extra_comps)
# connect LPUs by Patterns
for k, pattern in patterns.iteritems():
l1, l2 = k.split('-')
if l1 in lpus and l2 in lpus:
print('Connecting {} and {}'.format(l1, l2))
pat, key_order = Pattern.from_graph(
nx.DiGraph(pattern['graph']))
with Timer('update of connections in Manager'):
manager.connect(l1,
l2,
pat,
int_0=key_order.index(l1),
int_1=key_order.index(l2))
# start simulation
steps = config['General']['steps']
ignored_steps = config['General']['ignored_steps']
manager.spawn()
manager.start(steps=steps)
manager.wait()
time.sleep(5)
# post-processing inputs (hard coded, can be better organized)
inputs = {
u'ydomain': 1.0,
u'xdomain': dt * (steps - ignored_steps),
u'dt': dt * 10,
u'data': {}
}
if 'retina' in lpus:
input_array = si.read_array('{}_{}.h5'.format(
config['Retina']['input_file'], user_id))
inputs[u'ydomain'] = input_array.max()
for i, item in enumerate(retina_input_uids):
inputs['data'][item] = np.hstack(
(np.arange(int((steps - ignored_steps) / 10)).reshape(
(-1, 1)) * dt * 10, input_array[ignored_steps::10,
i:i + 1])).tolist()
del input_array
# post-processing outputs from all LPUs and combine them into one dictionary
result = {
u'ydomain': 1,
u'xdomain': dt * (steps - ignored_steps),
u'dt': dt * 10,
u'data': {}
}
for k, lpu in lpus.iteritems():
with h5py.File(lpu['output_file']) as output_file:
uids = output_file['V']['uids'][:]
output_array = output_file['V']['data'][:]
for i, item in enumerate(uids):
output = output_array[int(ignored_steps / 10):, i:i + 1]
tmp = output.max() - output.min()
if tmp <= 0.01: #mV
output = (output - output.min()) + 0.5
else:
output = (output - output.min()) / tmp * 0.9 + 0.1
result['data'][item] = np.hstack(
(np.arange(int((steps - ignored_steps) / 10)).reshape(
(-1, 1)) * dt * 10, output)).tolist()
return inputs, result
def launch(self, user_id, task):
neuron_uid_list = [str(a) for a in task['neuron_list']]
conf_obj = get_config_obj()
config = conf_obj.conf
if config['Retina']['intype'] == 'Natural':
coord_file = config['InputType']['Natural']['coord_file']
tmp = os.path.splitext(coord_file)
config['InputType']['Natural']['coord_file'] = '{}_{}{}'.format(
tmp[0], user_id, tmp[1])
setup_logger(file_name='neurokernel_' + user_id + '.log', screen=True)
manager = core.Manager()
lpus = {}
patterns = {}
G = task['data']
with open('G.pickle', 'wb') as f:
pickle.dump(G, f, protocol=pickle.HIGHEST_PROTOCOL)
print(G)
print(G.keys())
print(G['LPU'])
print(G['LPU'].keys())
# get graph and output_uid_list for each LPU
for k, lpu in G['LPU'].items():
lpus[k] = {}
g_lpu_na = create_graph_from_database_returned(lpu)
lpu_nk_graph = nk.na_lpu_to_nk_new(g_lpu_na)
lpus[k]['graph'] = lpu_nk_graph
lpus[k]['output_uid_list'] = list(
set(lpu_nk_graph.nodes()).intersection(set(neuron_uid_list)))
lpus[k]['output_file'] = '{}_output_{}.h5'.format(k, user_id)
for kkey, lpu in lpus.items():
graph = lpu['graph']
for uid, comp in graph.node.items():
if 'attr_dict' in comp:
print('Found attr_dict; fixing...')
nx.set_node_attributes(graph, {uid: comp['attr_dict']})
# print('changed',uid)
graph.nodes[uid].pop('attr_dict')
for i, j, k, v in graph.edges(keys=True, data=True):
if 'attr_dict' in v:
for key in v['attr_dict']:
nx.set_edge_attributes(
graph, {(i, j, k): {
key: v['attr_dict'][key]
}})
graph.edges[(i, j, k)].pop('attr_dict')
lpus[kkey]['graph'] = graph
# get graph for each Pattern
for k, pat in G['Pattern'].items():
l1, l2 = k.split('-')
if l1 in lpus and l2 in lpus:
g_pattern_na = create_graph_from_database_returned(pat)
pattern_nk = nk.na_pat_to_nk(g_pattern_na)
print(lpus[l1]['graph'].nodes(data=True))
lpu_ports = [node[1]['selector'] \
for node in lpus[l1]['graph'].nodes(data=True) \
if node[1]['class']=='Port'] + \
[node[1]['selector'] \
for node in lpus[l2]['graph'].nodes(data=True) \
if node[1]['class']=='Port']
pattern_ports = pattern_nk.nodes()
patterns[k] = {}
patterns[k]['graph'] = pattern_nk.subgraph(
list(set(lpu_ports).intersection(set(pattern_ports))))
dt = config['General']['dt']
if 'dt' in task:
dt = task['dt']
print(dt)
# add LPUs to manager
for k, lpu in lpus.items():
lpu_name = k
graph = lpu['graph']
for uid, comp in graph.node.items():
if 'attr_dict' in comp:
nx.set_node_attributes(graph, {uid: comp['attr_dict']})
# print('changed',uid)
graph.nodes[uid].pop('attr_dict')
for i, j, ko, v in graph.edges(keys=True, data=True):
if 'attr_dict' in v:
for key in v['attr_dict']:
nx.set_edge_attributes(
graph, {(i, j, ko): {
key: v['attr_dict'][key]
}})
graph.edges[(i, j, ko)].pop('attr_dict')
nx.write_gexf(graph, 'name.gexf')
with open(lpu_name + '.pickle', 'wb') as f:
pickle.dump(graph, f, protocol=pickle.HIGHEST_PROTOCOL)
comps = graph.node.items()
#for uid, comp in comps:
# if 'attr_dict' in comp:
# nx.set_node_attributes(graph, {uid: comp['attr_dict']})
# print('changed',uid)
# if 'class' in comp:
if k == 'retina':
prs = [node for node in graph.nodes(data=True) \
if node[1]['class'] == 'PhotoreceptorModel']
for pr in prs:
graph.node[pr[0]]['num_microvilli'] = 3000
input_processors = [
RetinaInputIndividual(config, prs, user_id)
]
extra_comps = [PhotoreceptorModel]
retina_input_uids = [a[0] for a in prs]
elif k == 'EB':
input_processor = StepInputProcessor('I', [node[0] for node in graph.nodes(data=True) \
if node[1]['class'] == 'LeakyIAF'], 40.0, 0.0, 1.0)
input_processors = [input_processor]
extra_comps = [BufferVoltage]
else:
input_processors = []
extra_comps = [BufferVoltage]
if 'inputProcessors' in task:
input_processors = loadExperimentSettings(
task['inputProcessors'])
output_processor = FileOutputProcessor(
[('V', lpu['output_uid_list'])],
lpu['output_file'],
sample_interval=10)
(comp_dict, conns) = LPU.graph_to_dicts(graph)
# print(comp_dict)
# print(conns)
print(k)
manager.add(LPU,
k,
dt,
comp_dict,
conns,
device=0,
input_processors=input_processors,
output_processors=[output_processor],
extra_comps=extra_comps,
debug=True)
# connect LPUs by Patterns
for k, pattern in patterns.items():
l1, l2 = k.split('-')
if l1 in lpus and l2 in lpus:
print('Connecting {} and {}'.format(l1, l2))
pat, key_order = Pattern.from_graph(nx.DiGraph(
pattern['graph']),
return_key_order=True)
print(l1, l2)
print(key_order)
with Timer('update of connections in Manager'):
manager.connect(l1,
l2,
pat,
int_0=key_order.index(l1),
int_1=key_order.index(l2))
# start simulation
steps = config['General']['steps']
ignored_steps = config['General']['ignored_steps']
if 'steps' in task:
steps = task['steps']
if 'ignored_steps' in task:
ignored_steps = task['ignored_steps']
# ignored_steps = 0
# steps = 100
manager.spawn()
manager.start(steps=steps)
manager.wait()
time.sleep(5)
print(task)
# post-processing inputs (hard coded, can be better organized)
inputs = {
u'ydomain': 1.0,
u'xdomain': dt * (steps - ignored_steps),
u'dt': dt * 10,
u'data': {}
}
if 'retina' in lpus:
input_array = si.read_array('{}_{}.h5'.format(
config['Retina']['input_file'], user_id))
inputs[u'ydomain'] = input_array.max()
for i, item in enumerate(retina_input_uids):
inputs['data'][item] = np.hstack(
(np.arange(int((steps - ignored_steps) / 10)).reshape(
(-1, 1)) * dt * 10, input_array[ignored_steps::10,
i:i + 1])).tolist()
del input_array
# post-processing outputs from all LPUs and combine them into one dictionary
result = {
u'ydomain': 1,
u'xdomain': dt * (steps - ignored_steps),
u'dt': dt * 10,
u'data': {}
}
for k, lpu in lpus.items():
with h5py.File(lpu['output_file']) as output_file:
uids = output_file['V']['uids'][:]
output_array = output_file['V']['data'][:]
for i, item in enumerate(uids):
output = output_array[int(ignored_steps / 10):, i:i + 1]
# tmp = output.max()-output.min()
# if tmp <= 0.01: #mV
# output = (output - output.min()) + 0.5
# else:
# output = (output - output.min())/tmp*0.9+0.1
result['data'][item] = np.hstack(
(np.arange(int((steps - ignored_steps) / 10)).reshape(
(-1, 1)) * dt * 10, output)).tolist()
return inputs, result
def main():
import neurokernel.mpi_relaunch
logger = setup_logger(file_name=None, screen=True)
parser = argparse.ArgumentParser()
parser.add_argument('-l',
'--layers',
dest='num_layers',
type=int,
default=16,
help='number of layers of ommatidia on circle')
parser.add_argument('-i',
'--input',
action="store_true",
help='generates input if set')
parser.add_argument('-g',
'--gexf',
action="store_true",
help='generates gexf of LPU if set')
parser.add_argument('--steps',
default=100,
type=int,
help='simulation steps')
args = parser.parse_args()
dt = 1e-4
GEXF_FILE = 'retina.gexf.gz'
INPUT_FILE = 'vision_input.h5'
IMAGE_FILE = 'image1.mat'
OUTPUT_FILE = 'retina_output.h5'
if args.input:
print('Generating input of model from image file')
generate_input(INPUT_FILE, IMAGE_FILE, args.num_layers)
if args.gexf:
print('Writing retina lpu')
n = args.num_layers
photoreceptor_num = 6 * (3 * n * (n + 1) + 1)
generate_gexf(GEXF_FILE, photoreceptor_num)
man = core_gpu.Manager()
print('Parsing lpu data')
n_dict_ret, s_dict_ret = LPU.lpu_parser(GEXF_FILE)
print('Initializing LPU')
man.add(LPU,
'retina',
dt,
n_dict_ret,
s_dict_ret,
input_file=INPUT_FILE,
output_file=OUTPUT_FILE,
device=0,
debug=True,
time_sync=False)
man.spawn()
print('Starting simulation')
start_time = time.time()
man.start(steps=args.steps)
man.wait()
print('Simulation complete: Duration {} seconds'.format(time.time() -
start_time))
def launch(self, user_id, task):
# neuron_uid_list = [str(a) for a in task['neuron_list']]
try:
# conf_obj = get_config_obj()
# config = conf_obj.conf
setup_logger(file_name = 'neurokernel_'+user_id+'.log', screen = False)
manager = core.Manager()
lpus = {}
patterns = {}
G = task['data']
for i in list(G['Pattern'].keys()):
a = G['Pattern'][i]['nodes']
if len([k for k,v in a.items() if v['class'] == 'Port']) == 0:
del G['Pattern'][i]
for i in list(G['LPU'].keys()):
a = G['LPU'][i]['nodes']
if len(a) < 3:
del G['LPU'][i]
# with open('G.pickle', 'wb') as f:
# pickle.dump(G, f, protocol=pickle.HIGHEST_PROTOCOL)
# print(G)
# print(G.keys())
# print(G['LPU'])
# print(G['LPU'].keys())
# get graph and output_uid_list for each LPU
for k, lpu in G['LPU'].items():
lpus[k] = {}
g_lpu_na = create_graph_from_database_returned(lpu)
lpu_nk_graph = nk.na_lpu_to_nk_new(g_lpu_na)
lpus[k]['graph'] = lpu_nk_graph
# lpus[k]['output_uid_list'] = list(
# set(lpu_nk_graph.nodes()).intersection(
# set(neuron_uid_list)))
# lpus[k]['output_file'] = '{}_output_{}.h5'.format(k, user_id)
for kkey, lpu in lpus.items():
graph = lpu['graph']
for uid, comp in graph.nodes.items():
if 'attr_dict' in comp:
print('Found attr_dict; fixing...')
nx.set_node_attributes(graph, {uid: comp['attr_dict']})
# print('changed',uid)
graph.nodes[uid].pop('attr_dict')
if 'params' in comp:
params = graph.nodes[uid].pop('params')
nx.set_node_attributes(graph, {uid: {k: float(v) for k, v in params.items()}})
if 'states' in comp:
states = graph.nodes[uid].pop('states')
nx.set_node_attributes(graph, {uid: {'init{}'.format(k): float(v) for k, v in states.items()}})
for i,j,k,v in graph.edges(keys=True, data=True):
if 'attr_dict' in v:
for key in v['attr_dict']:
nx.set_edge_attributes(graph, {(i,j,k): {key: v['attr_dict'][key]}})
graph.edges[(i,j,k)].pop('attr_dict')
lpus[kkey]['graph'] = graph
# get graph for each Pattern
for k, pat in G['Pattern'].items():
l1,l2 = k.split('-')
if l1 in lpus and l2 in lpus:
g_pattern_na = create_graph_from_database_returned(pat)
pattern_nk = nk.na_pat_to_nk(g_pattern_na)
#print(lpus[l1]['graph'].nodes(data=True))
lpu_ports = [node[1]['selector'] \
for node in lpus[l1]['graph'].nodes(data=True) \
if node[1]['class']=='Port'] + \
[node[1]['selector'] \
for node in lpus[l2]['graph'].nodes(data=True) \
if node[1]['class']=='Port']
pattern_ports = pattern_nk.nodes()
patterns[k] = {}
patterns[k]['graph'] = pattern_nk.subgraph(
list(set(lpu_ports).intersection(set(pattern_ports))))
# dt = config['General']['dt']
# if 'dt' in task:
dt = task['dt']
print(dt)
device_count = 0
# add LPUs to manager
for k, lpu in lpus.items():
lpu_name = k
graph = lpu['graph']
for uid, comp in graph.nodes.items():
if 'attr_dict' in comp:
nx.set_node_attributes(graph, {uid: comp['attr_dict']})
# print('changed',uid)
graph.nodes[uid].pop('attr_dict')
for i,j,ko,v in graph.edges(keys=True, data=True):
if 'attr_dict' in v:
for key in v['attr_dict']:
nx.set_edge_attributes(graph, {(i,j,ko): {key: v['attr_dict'][key]}})
graph.edges[(i,j,ko)].pop('attr_dict')
# nx.write_gexf(graph,'name.gexf')
# with open(lpu_name + '.pickle', 'wb') as f:
# pickle.dump(graph, f, protocol=pickle.HIGHEST_PROTOCOL)
comps = graph.nodes.items()
#for uid, comp in comps:
# if 'attr_dict' in comp:
# nx.set_node_attributes(graph, {uid: comp['attr_dict']})
# print('changed',uid)
# if 'class' in comp:
# if k == 'retina':
# if config['Retina']['intype'] == 'Natural':
# coord_file = config['InputType']['Natural']['coord_file']
# tmp = os.path.splitext(coord_file)
# config['InputType']['Natural']['coord_file'] = '{}_{}{}'.format(
# tmp[0], user_id, tmp[1])
# prs = [node for node in graph.nodes(data=True) \
# if node[1]['class'] == 'PhotoreceptorModel']
# for pr in prs:
# graph.node[pr[0]]['num_microvilli'] = 3000
# input_processors = [RetinaInputIndividual(config, prs, user_id)]
# extra_comps = [PhotoreceptorModel]
# retina_input_uids = [a[0] for a in prs]
# # elif k == 'EB':
# # input_processor = StepInputProcessor('I', [node[0] for node in graph.nodes(data=True) \
# # if node[1]['class'] == 'LeakyIAF'], 40.0, 0.0, 1.0)
# # input_processors = [input_processor]
# # extra_comps = []#[BufferVoltage]
# else:
# input_processors = []
# extra_comps = [BufferVoltage]
if 'inputProcessors' in task:
if lpu_name in task['inputProcessors']:
input_processors, record = \
loadInputProcessors(task['inputProcessors'][lpu_name])
lpus[k]['input_record'] = record
else:
input_processors = []
else:
input_processors = []
# configure output processors
lpus[k]['output_file'] = '{}_output_{}.h5'.format(k, user_id)
output_processors = []
if 'outputProcessors' in task:
if lpu_name in task['outputProcessors']:
output_processors, record = loadOutputProcessors(
lpus[k]['output_file'],
task['outputProcessors'][lpu_name])
if len(record):
lpus[k]['output_uid_dict'] = record
# (comp_dict, conns) = LPU.graph_to_dicts(graph)
manager.add(LPU, k, dt, 'pickle', pickle.dumps(graph),#comp_dict, conns,
device = device_count, input_processors = input_processors,
output_processors = output_processors,
extra_comps = [], debug = False)
device_count = (device_count+1) % self.ngpus
# connect LPUs by Patterns
for k, pattern in patterns.items():
l1,l2 = k.split('-')
if l1 in lpus and l2 in lpus:
print('Connecting {} and {}'.format(l1, l2))
pat, key_order = Pattern.from_graph(nx.DiGraph(pattern['graph']),
return_key_order = True)
print(l1,l2)
print(key_order)
with Timer('update of connections in Manager'):
try:
manager.connect(l1, l2, pat,
int_0 = key_order.index('{}/{}'.format(k,l1)),
int_1 = key_order.index('{}/{}'.format(k,l2)))
except ValueError:
manager.connect(l1, l2, pat,
int_0 = key_order.index(l1),
int_1 = key_order.index(l2))
# start simulation
# steps = config['General']['steps']
# ignored_steps = config['General']['ignored_steps']
# if 'steps' in task:
steps = task['steps']
# if 'ignored_steps' in task:
# ignored_steps = task['ignored_steps']
# ignored_steps = 0
# steps = 100
manager.spawn()
manager.start(steps=steps)
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
tb = ''.join(traceback.format_exception(exc_type, exc_value, exc_traceback))
print('An error occured during the compilation\n' + tb)
return {'error':
{'exception': tb,
'message': 'An error occured during the compilation in execution'}}
try:
manager.wait()
except LPUExecutionError:
ErrorMessage = 'An error occured during execution of LPU {} at step {}:\n'.format(
manager._errors[0][0], manager._errors[0][1]) + \
''.join(manager._errors[0][2])
print(ErrorMessage)
return {'error':
{'exception': ''.join(manager._errors[0][2]),
'message': 'An error occured during execution of LPU {} at step {}:\n'.format(
manager._errors[0][0], manager._errors[0][1])}}
time.sleep(5)
# print(task)
ignored_steps = 0
try:
# post-processing inputs (hard coded, can be better organized)
result = {'sensory': {}, 'input': {}, 'output': {}}
for k, lpu in lpus.items():
records = lpu.get('input_record', [])
for record in records:
if record['sensory_file'] is not None:
if k not in result['sensory']:
result['sensory'][k] = []
with h5py.File(record['sensory_file']) as sensory_file:
result['sensory'][k].append({'dt': record['sensory_interval']*dt,
'data': sensory_file['sensory'][:]})
if record['input_file'] is not None:
with h5py.File(record['input_file']) as input_file:
sample_interval = input_file['metadata'].attrs['sample_interval']
for var in input_file.keys():
if var == 'metadata': continue
uids = [n.decode() for n in input_file[var]['uids'][:]]
input_array = input_file[var]['data'][:]
for i, item in enumerate(uids):
if var == 'spike_state':
input = np.nonzero(input_array[ignored_steps:, i:i+1].reshape(-1))[0]*dt
if item in result['input']:
if 'spike_time' in result['input'][item]:
result['input'][item]['spike_time']['data'].append(input)
result['input'][item]['spike_time']['data'] = \
np.sort(result['input'][item]['spike_time']['data'])
else:
result['input'][item]['spike_time'] = {
'data': input.copy(),
'dt': dt*sample_interval}
else:
result['input'][item] = {'spike_time': {
'data': input.copy(),
'dt': dt*sample_interval}}
else:
input = input_array[ignored_steps:, i:i+1]
if item in result['input']:
if var in result['input'][item]:
result['input'][item][var]['data'] += input
else:
result['input'][item][var] = {
'data': input.copy(),
'dt': dt*sample_interval}
else:
result['input'][item] = {var: {
'data': input.copy(),
'dt': dt*sample_interval}}
# if 'retina' in lpus:
# input_array = si.read_array(
# '{}_{}.h5'.format(config['Retina']['input_file'], user_id))
# inputs[u'ydomain'] = input_array.max()
# for i, item in enumerate(retina_input_uids):
# inputs['data'][item] = np.hstack(
# (np.arange(int((steps-ignored_steps)/10)).reshape((-1,1))*dt*10,
# input_array[ignored_steps::10,i:i+1])).tolist()
#
# del input_array
# post-processing outputs from all LPUs and combine them into one dictionary
# result = {u'data': {}}
for k, lpu in lpus.items():
uid_dict = lpu.get('output_uid_dict', None)
if uid_dict is not None:
with h5py.File(lpu['output_file']) as output_file:
sample_interval = output_file['metadata'].attrs['sample_interval']
for var in uid_dict:
if var == 'spike_state':
uids = [n.decode() for n in output_file[var]['uids'][:]]
spike_times = output_file[var]['data']['time'][:]
index = output_file[var]['data']['index'][:]
for i, item in enumerate(uids):
# output = np.nonzero(output_array[ignored_steps:, i:i+1].reshape(-1))[0]*dt
output = spike_times[index == i]
output = output[output>ignored_steps*dt]-ignored_steps*dt
if item in result['output']:
result['output'][item]['spike_time'] = {
'data': output,
'dt': dt*sample_interval}
else:
result['output'][item] = {'spike_time': {
'data': output,
'dt': dt*sample_interval}}
else:
uids = [n.decode() for n in output_file[var]['uids'][:]]
output_array = output_file[var]['data'][:]
for i, item in enumerate(uids):
# if var == 'spike_state':
# output = np.nonzero(output_array[ignored_steps:, i:i+1].reshape(-1))[0]*dt
# if item in result['output']:
# result['output'][item]['spike_time'] = {
# 'data': output.tolist(),
# 'dt': dt}
# else:
# result['output'][item] = {'spike_time': {
# 'data': output.tolist(),
# 'dt': dt}}
# else:
output = output_array[ignored_steps:, i:i+1]
if item in result['output']:
result['output'][item][var] = {
'data': output.copy(),
'dt': dt*sample_interval}
else:
result['output'][item] = {var: {
'data': output.copy(),
'dt': dt*sample_interval}}
result = {'success': {'result': result, 'meta': {'dur': steps*dt}}}
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
tb = ''.join(traceback.format_exception(exc_type, exc_value, exc_traceback))
print('An error occured postprocessing of results\n' + tb)
return {'error':
{'exception': tb,
'message': 'An error occured when postprocessing of results in execution'}}
return result
def main():
import neurokernel.mpi_relaunch
import neurokernel.core_gpu as core
(comp_dict, conns) = LPU.lpu_parser('neuroballad_temp_model.gexf.gz')
with open('run_parameters.pickle', 'rb') as f:
run_parameters = pickle.load(f)
with open('record_parameters.pickle', 'rb') as f:
record_parameters = pickle.load(f)
dur = 1.0
dt = 1e-4
dur = run_parameters[0]
dt = run_parameters[1]
fl_input_processor = FileInputProcessor('neuroballad_temp_model_input.h5')
from neurokernel.LPU.OutputProcessors.FileOutputProcessor import FileOutputProcessor
output_processor = FileOutputProcessor(record_parameters,
'neuroballad_temp_model_output.h5',
sample_interval=1)
#Parse extra arguments
parser = argparse.ArgumentParser()
parser.add_argument(
'--debug',
default=False,
dest='debug',
action='store_true',
help=
'Write connectivity structures and inter-LPU routed data in debug folder'
)
parser.add_argument(
'-l',
'--log',
default='file',
type=str,
help='Log output to screen [file, screen, both, or none; default:none]'
)
parser.add_argument('-r',
'--time_sync',
default=False,
action='store_true',
help='Time data reception throughput [default: False]')
parser.add_argument('-g',
'--gpu_dev',
default=[0, 1],
type=int,
nargs='+',
help='GPU device numbers [default: 0 1]')
parser.add_argument('-d',
'--disconnect',
default=False,
action='store_true',
help='Run with disconnected LPUs [default: False]')
args = parser.parse_args()
file_name = None
screen = False
if args.log.lower() in ['file', 'both']:
file_name = 'neurokernel.log'
if args.log.lower() in ['screen', 'both']:
screen = True
logger = setup_logger(file_name=file_name, screen=screen)
man = core.Manager()
man.add(LPU,
'lpu',
dt,
comp_dict,
conns,
input_processors=[fl_input_processor],
output_processors=[output_processor],
device=args.gpu_dev[1],
debug=True)
steps = int(dur / dt)
man.spawn()
man.start(steps=steps)
man.wait()
def main():
logging.basicConfig(level=logging.DEBUG, stream=sys.stdout,
format='%(asctime)s %(name)s %(levelname)s %(message)s')
logger = logging.getLogger('cx')
sys.setrecursionlimit(10000)
### Graph ###
graph = Graph(Config.from_url(cx_db, 'admin', 'admin',
initial_drop=False))
graph.include(models.Node.registry)
graph.include(models.Relationship.registry)
### Retina ###
config=retlam_demo.ConfigReader('retlam_default.cfg','../template_spec.cfg').conf
retina = get_retina(config)
##### Configuration ######
logger = setup_logger(screen=True)
lpu_selectors, to_list = get_cx_selectors_name()
lpu_name_to_comp_dict, lpu_name_to_conn_list, pat_name_list, pat_name_to_pat = cx_component(graph)
man = core.Manager()
dt = 1e-4
dur = 0.2
steps = int(dur/dt)
debug = True
lpu_name_list = ['BU', 'bu', 'EB', 'FB', 'PB']
for name in lpu_name_list:
input_processor = []
output_processor = [FileOutputProcessor([('spike_state', None), ('V',None), ('g',None), ('I', None)], '{}_output.h5'.format(name), sample_interval = 1)]
man.add(LPU, name, dt, lpu_name_to_comp_dict[name],
lpu_name_to_conn_list[name],
input_processors = input_processor,
output_processors = output_processor,
device=0,
debug=debug, time_sync=False)
retlam_demo.add_retina_LPU(config, 0, retina, man)
logger.info('add retina lpu')
for name in pat_name_list:
id_0, id_1 = name.split('-')
man.connect(id_0, id_1, pat_name_to_pat[name][0], pat_name_to_pat[name][1].index('0'), pat_name_to_pat[name][1].index('1'))
logger.info('link lpus among cx lpus')
link_retina_pat_cx(retina, lpu_selectors, to_list, man)
logger.info('link retina and cx lpu')
man.spawn()
man.start(steps)
man.wait()
