def main(): import neurokernel.mpi_relaunch parser = argparse.ArgumentParser() parser.add_argument('-c', '--config', default='default', help='configuration file') parser.add_argument('-v', '--value', type=int, default=-1, help='Value that can overwrite configuration' 'by changing this script accordingly.' 'It is useful when need to run this script' 'repeatedly for different configuration') args = parser.parse_args() with Timer('getting configuration'): conf_obj = get_config_obj(args) config = conf_obj.conf change_config(config, args.value) edit_files(config) setup_logging(config) # with Timer('input generation'): # gi.gen_input_norfscreen(config) manager = core.Manager() with Timer('photoreceptor instantiation'): add_LPU(config, manager) start_simulation(config, manager)
def simulation(dt, N, output_n, nsteps = 10000, output = None): start_time = time.time() dur = nsteps * dt steps = nsteps man = core.Manager() G = create_graph(N) print("Creating graph completed in {} seconds.".format(time.time()-start_time)) start_time = time.time() #comp_dict, conns = LPU.graph_to_dicts(G, remove_edge_id=False) fl_input_processor = StepInputProcessor('I', ['neuron_{}'.format(i) for i in range(N)], 20.0, 0.0, dur) fl_output_processor = [] if output == 'disk': fl_output_processor.append( FileOutputProcessor([('V', None), ('spike_state', None)], 'neurodriver_output_{}.h5'.format(output_n), sample_interval=1, cache_length=2000)]) elif output == 'memory': fl_output_processor.append( OutputRecorder([('spike_state', None), ('V', None)], sample_interval = 1)])
def main(): import neurokernel.mpi_relaunch # default limit is low for pickling # the data structures passed through mpi sys.setrecursionlimit(RECURSION_LIMIT) resource.setrlimit(resource.RLIMIT_STACK, (resource.RLIM_INFINITY, resource.RLIM_INFINITY)) parser = argparse.ArgumentParser() parser.add_argument('-c', '--config', default='default', help='configuration file') parser.add_argument('-v', '--value', type=int, default=-1, help='Value that can overwrite configuration ' 'by changing this script accordingly. ' 'It is useful when need to run this script ' 'repeatedly for different configuration') args = parser.parse_args() with Timer('getting configuration'): conf_obj = get_config_obj(args) config = conf_obj.conf change_config(config, args.value) setup_logging(config) worker_num = config['Retina']['worker_num'] num_rings = config['Retina']['rings'] radius = config['Retina']['radius'] eulerangles = config['Retina']['eulerangles'] manager = core.Manager() with Timer('instantiation of retina'): transform = AlbersProjectionMap(radius, eulerangles).invmap hexagon = hx.HexagonArray(num_rings=num_rings, radius=radius, transform=transform) retina = ret.RetinaArray(hexagon, config) # sets retina attribute which is required for the generation of # receptive fields #if generator is not None: # generator.generate_datafiles(0, retina) add_master_LPU(config, 0, retina, manager) for j in range(worker_num): add_worker_LPU(config, j, retina, manager) connect_master_worker(config, j, retina, manager) start_simulation(config, manager)
def main(): import neurokernel.mpi_relaunch # default limit is low for pickling # the data structures passed through mpi sys.setrecursionlimit(RECURSION_LIMIT) resource.setrlimit(resource.RLIMIT_STACK, (resource.RLIM_INFINITY, resource.RLIM_INFINITY)) parser = argparse.ArgumentParser() parser.add_argument('-c', '--config', default='default', help='configuration file') parser.add_argument('-v', '--value', type=int, default=-1, help='Value that can overwrite configuration ' 'by changing this script accordingly. ' 'It is useful when need to run this script ' 'repeatedly for different configuration') args = parser.parse_args() with Timer('getting configuration'): conf_obj = get_config_obj(args) config = conf_obj.conf change_config(config, args.value) setup_logging(config) num_rings = config['Retina']['rings'] eulerangles = config['Retina']['eulerangles'] radius = config['Retina']['radius'] manager = core.Manager() with Timer('instantiation of retina and lamina'): transform = AlbersProjectionMap(radius, eulerangles).invmap r_hexagon = r_hx.HexagonArray(num_rings=num_rings, radius=radius, transform=transform) l_hexagon = l_hx.HexagonArray(num_rings=num_rings, radius=radius, transform=transform) retina = ret.RetinaArray(r_hexagon, config) lamina = lam.LaminaArray(l_hexagon, config) add_retina_LPU(config, 0, retina, manager) add_lamina_LPU(config, 0, lamina, manager) connect_retina_lamina(config, 0, retina, lamina, manager) start_simulation(config, manager)
parser.add_argument('-g', '--gpu_dev', default=0, type=int, help='GPU device number [default: 0]') args = parser.parse_args() file_name = None screen = False if args.log.lower() in ['file', 'both']: file_name = 'neurokernel.log' if args.log.lower() in ['screen', 'both']: screen = True logger = setup_logger(file_name=file_name, screen=screen) man = core.Manager() G = nx.MultiDiGraph() G.add_node( 'neuron0', { 'class': 'LeakyIAF', 'name': 'LeakyIAF', 'resting_potential': -70.0, 'threshold': -45.0, 'capacitance': 0.07, # in mS 'resistance': 0.2, # in Ohm }) comp_dict, conns = LPU.graph_to_dicts(G)
def run(connected): """ Set `connected` to True to connect the LPUs. """ import neurokernel.mpi_relaunch out_name = 'un' if not connected else 'co' man = core.Manager() lpu_file_0 = './data/generic_lpu_0.gexf.gz' lpu_file_1 = './data/generic_lpu_1.gexf.gz' comp_dict_0, conns_0 = LPU.lpu_parser(lpu_file_0) comp_dict_1, conns_1 = LPU.lpu_parser(lpu_file_1) fl_input_processor_0 = FileInputProcessor( './data/generic_lpu_0_input.h5') fl_output_processor_0 = FileOutputProcessor( [('V', None), ('spike_state', None)], 'generic_lpu_0_%s_output.h5' % out_name, sample_interval=1) lpu_0_id = 'lpu_0' man.add(LPU, lpu_0_id, dt, comp_dict_0, conns_0, input_processors=[fl_input_processor_0], output_processors=[fl_output_processor_0], device=args.gpu_dev[0], debug=args.debug, time_sync=args.time_sync) fl_input_processor_1 = FileInputProcessor( './data/generic_lpu_1_input.h5') fl_output_processor_1 = FileOutputProcessor( [('V', None), ('spike_state', None)], 'generic_lpu_1_%s_output.h5' % out_name, sample_interval=1) lpu_1_id = 'lpu_1' man.add(LPU, lpu_1_id, dt, comp_dict_1, conns_1, input_processors=[fl_input_processor_1], output_processors=[fl_output_processor_1], device=args.gpu_dev[1], debug=args.debug, time_sync=args.time_sync) # Create random connections between the input and output ports if the LPUs # are to be connected: if connected: # Find all output and input port selectors in each LPU: out_ports_spk_0 = plsel.Selector(','.join( LPU.extract_out_spk(comp_dict_0, 'id')[0])) out_ports_gpot_0 = plsel.Selector(','.join( LPU.extract_out_gpot(comp_dict_0, 'id')[0])) out_ports_spk_1 = plsel.Selector(','.join( LPU.extract_out_spk(comp_dict_1, 'id')[0])) out_ports_gpot_1 = plsel.Selector(','.join( LPU.extract_out_gpot(comp_dict_1, 'id')[0])) in_ports_spk_0 = plsel.Selector(','.join( LPU.extract_in_spk(comp_dict_0, 'id')[0])) in_ports_gpot_0 = plsel.Selector(','.join( LPU.extract_in_gpot(comp_dict_0, 'id')[0])) in_ports_spk_1 = plsel.Selector(','.join( LPU.extract_in_spk(comp_dict_1, 'id')[0])) in_ports_gpot_1 = plsel.Selector(','.join( LPU.extract_in_gpot(comp_dict_1, 'id')[0])) out_ports_0 = plsel.Selector.union(out_ports_spk_0, out_ports_gpot_0) out_ports_1 = plsel.Selector.union(out_ports_spk_1, out_ports_gpot_1) in_ports_0 = plsel.Selector.union(in_ports_spk_0, in_ports_gpot_0) in_ports_1 = plsel.Selector.union(in_ports_spk_1, in_ports_gpot_1) # Initialize a connectivity pattern between the two sets of port # selectors: pat = pattern.Pattern( plsel.Selector.union(out_ports_0, in_ports_0), plsel.Selector.union(out_ports_1, in_ports_1)) # Create connections from the ports with identifiers matching the output # ports of one LPU to the ports with identifiers matching the input # ports of the other LPU: N_conn_spk_0_1 = min(len(out_ports_spk_0), len(in_ports_spk_1)) N_conn_gpot_0_1 = min(len(out_ports_gpot_0), len(in_ports_gpot_1)) for src, dest in zip( random.sample(out_ports_spk_0.identifiers, N_conn_spk_0_1), random.sample(in_ports_spk_1.identifiers, N_conn_spk_0_1)): pat[src, dest] = 1 pat.interface[src, 'type'] = 'spike' pat.interface[dest, 'type'] = 'spike' for src, dest in zip( random.sample(out_ports_gpot_0.identifiers, N_conn_gpot_0_1), random.sample(in_ports_gpot_1.identifiers, N_conn_gpot_0_1)): pat[src, dest] = 1 pat.interface[src, 'type'] = 'gpot' pat.interface[dest, 'type'] = 'gpot' man.connect(lpu_0_id, lpu_1_id, pat, 0, 1) man.spawn() man.start(steps=args.steps) man.wait()
def launch(self, user_id, task): neuron_uid_list = [str(a) for a in task['neuron_list']] conf_obj = get_config_obj() config = conf_obj.conf if config['Retina']['intype'] == 'Natural': coord_file = config['InputType']['Natural']['coord_file'] tmp = os.path.splitext(coord_file) config['InputType']['Natural']['coord_file'] = '{}_{}{}'.format( tmp[0], user_id, tmp[1]) setup_logger(file_name='neurokernel_' + user_id + '.log', screen=False) manager = core.Manager() lpus = {} patterns = {} G = task['success']['data'] # get graph and output_uid_list for each LPU for k, lpu in G['LPU'].iteritems(): lpus[k] = {} g_lpu_na = create_graph_from_database_returned(lpu) lpu_nk_graph = nk.na_lpu_to_nk_new(g_lpu_na) lpus[k]['graph'] = lpu_nk_graph lpus[k]['output_uid_list'] = list( set(lpu_nk_graph.nodes()).intersection(set(neuron_uid_list))) lpus[k]['output_file'] = '{}_output_{}.h5'.format(k, user_id) # get graph for each Pattern for k, pat in G['Pattern'].iteritems(): l1, l2 = k.split('-') if l1 in lpus and l2 in lpus: g_pattern_na = create_graph_from_database_returned(pat) pattern_nk = nk.na_pat_to_nk(g_pattern_na) lpu_ports = [node[1]['selector'] \ for node in lpus[l1]['graph'].nodes(data=True) \ if node[1]['class']=='Port'] + \ [node[1]['selector'] \ for node in lpus[l2]['graph'].nodes(data=True) \ if node[1]['class']=='Port'] pattern_ports = pattern_nk.nodes() patterns[k] = {} patterns[k]['graph'] = pattern_nk.subgraph( list(set(lpu_ports).intersection(set(pattern_ports)))) dt = config['General']['dt'] # add LPUs to manager for k, lpu in lpus.iteritems(): graph = lpu['graph'] if k == 'retina': prs = [node for node in graph.nodes(data=True) \ if node[1]['class'] == 'PhotoreceptorModel'] for pr in prs: graph.node[pr[0]]['num_microvilli'] = 3000 input_processors = [ RetinaInputIndividual(config, prs, user_id) ] extra_comps = [PhotoreceptorModel] retina_input_uids = [a[0] for a in prs] else: input_processors = [] extra_comps = [BufferVoltage] output_processor = FileOutputProcessor( [('V', lpu['output_uid_list'])], lpu['output_file'], sample_interval=10) (comp_dict, conns) = LPU.graph_to_dicts(graph) manager.add(LPU, k, dt, comp_dict, conns, device=0, input_processors=input_processors, output_processors=[output_processor], extra_comps=extra_comps) # connect LPUs by Patterns for k, pattern in patterns.iteritems(): l1, l2 = k.split('-') if l1 in lpus and l2 in lpus: print('Connecting {} and {}'.format(l1, l2)) pat, key_order = Pattern.from_graph( nx.DiGraph(pattern['graph'])) with Timer('update of connections in Manager'): manager.connect(l1, l2, pat, int_0=key_order.index(l1), int_1=key_order.index(l2)) # start simulation steps = config['General']['steps'] ignored_steps = config['General']['ignored_steps'] manager.spawn() manager.start(steps=steps) manager.wait() time.sleep(5) # post-processing inputs (hard coded, can be better organized) inputs = { u'ydomain': 1.0, u'xdomain': dt * (steps - ignored_steps), u'dt': dt * 10, u'data': {} } if 'retina' in lpus: input_array = si.read_array('{}_{}.h5'.format( config['Retina']['input_file'], user_id)) inputs[u'ydomain'] = input_array.max() for i, item in enumerate(retina_input_uids): inputs['data'][item] = np.hstack( (np.arange(int((steps - ignored_steps) / 10)).reshape( (-1, 1)) * dt * 10, input_array[ignored_steps::10, i:i + 1])).tolist() del input_array # post-processing outputs from all LPUs and combine them into one dictionary result = { u'ydomain': 1, u'xdomain': dt * (steps - ignored_steps), u'dt': dt * 10, u'data': {} } for k, lpu in lpus.iteritems(): with h5py.File(lpu['output_file']) as output_file: uids = output_file['V']['uids'][:] output_array = output_file['V']['data'][:] for i, item in enumerate(uids): output = output_array[int(ignored_steps / 10):, i:i + 1] tmp = output.max() - output.min() if tmp <= 0.01: #mV output = (output - output.min()) + 0.5 else: output = (output - output.min()) / tmp * 0.9 + 0.1 result['data'][item] = np.hstack( (np.arange(int((steps - ignored_steps) / 10)).reshape( (-1, 1)) * dt * 10, output)).tolist() return inputs, result
def launch(self, user_id, task): neuron_uid_list = [str(a) for a in task['neuron_list']] conf_obj = get_config_obj() config = conf_obj.conf if config['Retina']['intype'] == 'Natural': coord_file = config['InputType']['Natural']['coord_file'] tmp = os.path.splitext(coord_file) config['InputType']['Natural']['coord_file'] = '{}_{}{}'.format( tmp[0], user_id, tmp[1]) setup_logger(file_name='neurokernel_' + user_id + '.log', screen=True) manager = core.Manager() lpus = {} patterns = {} G = task['data'] with open('G.pickle', 'wb') as f: pickle.dump(G, f, protocol=pickle.HIGHEST_PROTOCOL) print(G) print(G.keys()) print(G['LPU']) print(G['LPU'].keys()) # get graph and output_uid_list for each LPU for k, lpu in G['LPU'].items(): lpus[k] = {} g_lpu_na = create_graph_from_database_returned(lpu) lpu_nk_graph = nk.na_lpu_to_nk_new(g_lpu_na) lpus[k]['graph'] = lpu_nk_graph lpus[k]['output_uid_list'] = list( set(lpu_nk_graph.nodes()).intersection(set(neuron_uid_list))) lpus[k]['output_file'] = '{}_output_{}.h5'.format(k, user_id) for kkey, lpu in lpus.items(): graph = lpu['graph'] for uid, comp in graph.node.items(): if 'attr_dict' in comp: print('Found attr_dict; fixing...') nx.set_node_attributes(graph, {uid: comp['attr_dict']}) # print('changed',uid) graph.nodes[uid].pop('attr_dict') for i, j, k, v in graph.edges(keys=True, data=True): if 'attr_dict' in v: for key in v['attr_dict']: nx.set_edge_attributes( graph, {(i, j, k): { key: v['attr_dict'][key] }}) graph.edges[(i, j, k)].pop('attr_dict') lpus[kkey]['graph'] = graph # get graph for each Pattern for k, pat in G['Pattern'].items(): l1, l2 = k.split('-') if l1 in lpus and l2 in lpus: g_pattern_na = create_graph_from_database_returned(pat) pattern_nk = nk.na_pat_to_nk(g_pattern_na) print(lpus[l1]['graph'].nodes(data=True)) lpu_ports = [node[1]['selector'] \ for node in lpus[l1]['graph'].nodes(data=True) \ if node[1]['class']=='Port'] + \ [node[1]['selector'] \ for node in lpus[l2]['graph'].nodes(data=True) \ if node[1]['class']=='Port'] pattern_ports = pattern_nk.nodes() patterns[k] = {} patterns[k]['graph'] = pattern_nk.subgraph( list(set(lpu_ports).intersection(set(pattern_ports)))) dt = config['General']['dt'] if 'dt' in task: dt = task['dt'] print(dt) # add LPUs to manager for k, lpu in lpus.items(): lpu_name = k graph = lpu['graph'] for uid, comp in graph.node.items(): if 'attr_dict' in comp: nx.set_node_attributes(graph, {uid: comp['attr_dict']}) # print('changed',uid) graph.nodes[uid].pop('attr_dict') for i, j, ko, v in graph.edges(keys=True, data=True): if 'attr_dict' in v: for key in v['attr_dict']: nx.set_edge_attributes( graph, {(i, j, ko): { key: v['attr_dict'][key] }}) graph.edges[(i, j, ko)].pop('attr_dict') nx.write_gexf(graph, 'name.gexf') with open(lpu_name + '.pickle', 'wb') as f: pickle.dump(graph, f, protocol=pickle.HIGHEST_PROTOCOL) comps = graph.node.items() #for uid, comp in comps: # if 'attr_dict' in comp: # nx.set_node_attributes(graph, {uid: comp['attr_dict']}) # print('changed',uid) # if 'class' in comp: if k == 'retina': prs = [node for node in graph.nodes(data=True) \ if node[1]['class'] == 'PhotoreceptorModel'] for pr in prs: graph.node[pr[0]]['num_microvilli'] = 3000 input_processors = [ RetinaInputIndividual(config, prs, user_id) ] extra_comps = [PhotoreceptorModel] retina_input_uids = [a[0] for a in prs] elif k == 'EB': input_processor = StepInputProcessor('I', [node[0] for node in graph.nodes(data=True) \ if node[1]['class'] == 'LeakyIAF'], 40.0, 0.0, 1.0) input_processors = [input_processor] extra_comps = [BufferVoltage] else: input_processors = [] extra_comps = [BufferVoltage] if 'inputProcessors' in task: input_processors = loadExperimentSettings( task['inputProcessors']) output_processor = FileOutputProcessor( [('V', lpu['output_uid_list'])], lpu['output_file'], sample_interval=10) (comp_dict, conns) = LPU.graph_to_dicts(graph) # print(comp_dict) # print(conns) print(k) manager.add(LPU, k, dt, comp_dict, conns, device=0, input_processors=input_processors, output_processors=[output_processor], extra_comps=extra_comps, debug=True) # connect LPUs by Patterns for k, pattern in patterns.items(): l1, l2 = k.split('-') if l1 in lpus and l2 in lpus: print('Connecting {} and {}'.format(l1, l2)) pat, key_order = Pattern.from_graph(nx.DiGraph( pattern['graph']), return_key_order=True) print(l1, l2) print(key_order) with Timer('update of connections in Manager'): manager.connect(l1, l2, pat, int_0=key_order.index(l1), int_1=key_order.index(l2)) # start simulation steps = config['General']['steps'] ignored_steps = config['General']['ignored_steps'] if 'steps' in task: steps = task['steps'] if 'ignored_steps' in task: ignored_steps = task['ignored_steps'] # ignored_steps = 0 # steps = 100 manager.spawn() manager.start(steps=steps) manager.wait() time.sleep(5) print(task) # post-processing inputs (hard coded, can be better organized) inputs = { u'ydomain': 1.0, u'xdomain': dt * (steps - ignored_steps), u'dt': dt * 10, u'data': {} } if 'retina' in lpus: input_array = si.read_array('{}_{}.h5'.format( config['Retina']['input_file'], user_id)) inputs[u'ydomain'] = input_array.max() for i, item in enumerate(retina_input_uids): inputs['data'][item] = np.hstack( (np.arange(int((steps - ignored_steps) / 10)).reshape( (-1, 1)) * dt * 10, input_array[ignored_steps::10, i:i + 1])).tolist() del input_array # post-processing outputs from all LPUs and combine them into one dictionary result = { u'ydomain': 1, u'xdomain': dt * (steps - ignored_steps), u'dt': dt * 10, u'data': {} } for k, lpu in lpus.items(): with h5py.File(lpu['output_file']) as output_file: uids = output_file['V']['uids'][:] output_array = output_file['V']['data'][:] for i, item in enumerate(uids): output = output_array[int(ignored_steps / 10):, i:i + 1] # tmp = output.max()-output.min() # if tmp <= 0.01: #mV # output = (output - output.min()) + 0.5 # else: # output = (output - output.min())/tmp*0.9+0.1 result['data'][item] = np.hstack( (np.arange(int((steps - ignored_steps) / 10)).reshape( (-1, 1)) * dt * 10, output)).tolist() return inputs, result
def main(): import neurokernel.mpi_relaunch logger = setup_logger(file_name=None, screen=True) parser = argparse.ArgumentParser() parser.add_argument('-l', '--layers', dest='num_layers', type=int, default=16, help='number of layers of ommatidia on circle') parser.add_argument('-i', '--input', action="store_true", help='generates input if set') parser.add_argument('-g', '--gexf', action="store_true", help='generates gexf of LPU if set') parser.add_argument('--steps', default=100, type=int, help='simulation steps') args = parser.parse_args() dt = 1e-4 GEXF_FILE = 'retina.gexf.gz' INPUT_FILE = 'vision_input.h5' IMAGE_FILE = 'image1.mat' OUTPUT_FILE = 'retina_output.h5' if args.input: print('Generating input of model from image file') generate_input(INPUT_FILE, IMAGE_FILE, args.num_layers) if args.gexf: print('Writing retina lpu') n = args.num_layers photoreceptor_num = 6 * (3 * n * (n + 1) + 1) generate_gexf(GEXF_FILE, photoreceptor_num) man = core_gpu.Manager() print('Parsing lpu data') n_dict_ret, s_dict_ret = LPU.lpu_parser(GEXF_FILE) print('Initializing LPU') man.add(LPU, 'retina', dt, n_dict_ret, s_dict_ret, input_file=INPUT_FILE, output_file=OUTPUT_FILE, device=0, debug=True, time_sync=False) man.spawn() print('Starting simulation') start_time = time.time() man.start(steps=args.steps) man.wait() print('Simulation complete: Duration {} seconds'.format(time.time() - start_time))
def launch(self, user_id, task): # neuron_uid_list = [str(a) for a in task['neuron_list']] try: # conf_obj = get_config_obj() # config = conf_obj.conf setup_logger(file_name = 'neurokernel_'+user_id+'.log', screen = False) manager = core.Manager() lpus = {} patterns = {} G = task['data'] for i in list(G['Pattern'].keys()): a = G['Pattern'][i]['nodes'] if len([k for k,v in a.items() if v['class'] == 'Port']) == 0: del G['Pattern'][i] for i in list(G['LPU'].keys()): a = G['LPU'][i]['nodes'] if len(a) < 3: del G['LPU'][i] # with open('G.pickle', 'wb') as f: # pickle.dump(G, f, protocol=pickle.HIGHEST_PROTOCOL) # print(G) # print(G.keys()) # print(G['LPU']) # print(G['LPU'].keys()) # get graph and output_uid_list for each LPU for k, lpu in G['LPU'].items(): lpus[k] = {} g_lpu_na = create_graph_from_database_returned(lpu) lpu_nk_graph = nk.na_lpu_to_nk_new(g_lpu_na) lpus[k]['graph'] = lpu_nk_graph # lpus[k]['output_uid_list'] = list( # set(lpu_nk_graph.nodes()).intersection( # set(neuron_uid_list))) # lpus[k]['output_file'] = '{}_output_{}.h5'.format(k, user_id) for kkey, lpu in lpus.items(): graph = lpu['graph'] for uid, comp in graph.nodes.items(): if 'attr_dict' in comp: print('Found attr_dict; fixing...') nx.set_node_attributes(graph, {uid: comp['attr_dict']}) # print('changed',uid) graph.nodes[uid].pop('attr_dict') if 'params' in comp: params = graph.nodes[uid].pop('params') nx.set_node_attributes(graph, {uid: {k: float(v) for k, v in params.items()}}) if 'states' in comp: states = graph.nodes[uid].pop('states') nx.set_node_attributes(graph, {uid: {'init{}'.format(k): float(v) for k, v in states.items()}}) for i,j,k,v in graph.edges(keys=True, data=True): if 'attr_dict' in v: for key in v['attr_dict']: nx.set_edge_attributes(graph, {(i,j,k): {key: v['attr_dict'][key]}}) graph.edges[(i,j,k)].pop('attr_dict') lpus[kkey]['graph'] = graph # get graph for each Pattern for k, pat in G['Pattern'].items(): l1,l2 = k.split('-') if l1 in lpus and l2 in lpus: g_pattern_na = create_graph_from_database_returned(pat) pattern_nk = nk.na_pat_to_nk(g_pattern_na) #print(lpus[l1]['graph'].nodes(data=True)) lpu_ports = [node[1]['selector'] \ for node in lpus[l1]['graph'].nodes(data=True) \ if node[1]['class']=='Port'] + \ [node[1]['selector'] \ for node in lpus[l2]['graph'].nodes(data=True) \ if node[1]['class']=='Port'] pattern_ports = pattern_nk.nodes() patterns[k] = {} patterns[k]['graph'] = pattern_nk.subgraph( list(set(lpu_ports).intersection(set(pattern_ports)))) # dt = config['General']['dt'] # if 'dt' in task: dt = task['dt'] print(dt) device_count = 0 # add LPUs to manager for k, lpu in lpus.items(): lpu_name = k graph = lpu['graph'] for uid, comp in graph.nodes.items(): if 'attr_dict' in comp: nx.set_node_attributes(graph, {uid: comp['attr_dict']}) # print('changed',uid) graph.nodes[uid].pop('attr_dict') for i,j,ko,v in graph.edges(keys=True, data=True): if 'attr_dict' in v: for key in v['attr_dict']: nx.set_edge_attributes(graph, {(i,j,ko): {key: v['attr_dict'][key]}}) graph.edges[(i,j,ko)].pop('attr_dict') # nx.write_gexf(graph,'name.gexf') # with open(lpu_name + '.pickle', 'wb') as f: # pickle.dump(graph, f, protocol=pickle.HIGHEST_PROTOCOL) comps = graph.nodes.items() #for uid, comp in comps: # if 'attr_dict' in comp: # nx.set_node_attributes(graph, {uid: comp['attr_dict']}) # print('changed',uid) # if 'class' in comp: # if k == 'retina': # if config['Retina']['intype'] == 'Natural': # coord_file = config['InputType']['Natural']['coord_file'] # tmp = os.path.splitext(coord_file) # config['InputType']['Natural']['coord_file'] = '{}_{}{}'.format( # tmp[0], user_id, tmp[1]) # prs = [node for node in graph.nodes(data=True) \ # if node[1]['class'] == 'PhotoreceptorModel'] # for pr in prs: # graph.node[pr[0]]['num_microvilli'] = 3000 # input_processors = [RetinaInputIndividual(config, prs, user_id)] # extra_comps = [PhotoreceptorModel] # retina_input_uids = [a[0] for a in prs] # # elif k == 'EB': # # input_processor = StepInputProcessor('I', [node[0] for node in graph.nodes(data=True) \ # # if node[1]['class'] == 'LeakyIAF'], 40.0, 0.0, 1.0) # # input_processors = [input_processor] # # extra_comps = []#[BufferVoltage] # else: # input_processors = [] # extra_comps = [BufferVoltage] if 'inputProcessors' in task: if lpu_name in task['inputProcessors']: input_processors, record = \ loadInputProcessors(task['inputProcessors'][lpu_name]) lpus[k]['input_record'] = record else: input_processors = [] else: input_processors = [] # configure output processors lpus[k]['output_file'] = '{}_output_{}.h5'.format(k, user_id) output_processors = [] if 'outputProcessors' in task: if lpu_name in task['outputProcessors']: output_processors, record = loadOutputProcessors( lpus[k]['output_file'], task['outputProcessors'][lpu_name]) if len(record): lpus[k]['output_uid_dict'] = record # (comp_dict, conns) = LPU.graph_to_dicts(graph) manager.add(LPU, k, dt, 'pickle', pickle.dumps(graph),#comp_dict, conns, device = device_count, input_processors = input_processors, output_processors = output_processors, extra_comps = [], debug = False) device_count = (device_count+1) % self.ngpus # connect LPUs by Patterns for k, pattern in patterns.items(): l1,l2 = k.split('-') if l1 in lpus and l2 in lpus: print('Connecting {} and {}'.format(l1, l2)) pat, key_order = Pattern.from_graph(nx.DiGraph(pattern['graph']), return_key_order = True) print(l1,l2) print(key_order) with Timer('update of connections in Manager'): try: manager.connect(l1, l2, pat, int_0 = key_order.index('{}/{}'.format(k,l1)), int_1 = key_order.index('{}/{}'.format(k,l2))) except ValueError: manager.connect(l1, l2, pat, int_0 = key_order.index(l1), int_1 = key_order.index(l2)) # start simulation # steps = config['General']['steps'] # ignored_steps = config['General']['ignored_steps'] # if 'steps' in task: steps = task['steps'] # if 'ignored_steps' in task: # ignored_steps = task['ignored_steps'] # ignored_steps = 0 # steps = 100 manager.spawn() manager.start(steps=steps) except: exc_type, exc_value, exc_traceback = sys.exc_info() tb = ''.join(traceback.format_exception(exc_type, exc_value, exc_traceback)) print('An error occured during the compilation\n' + tb) return {'error': {'exception': tb, 'message': 'An error occured during the compilation in execution'}} try: manager.wait() except LPUExecutionError: ErrorMessage = 'An error occured during execution of LPU {} at step {}:\n'.format( manager._errors[0][0], manager._errors[0][1]) + \ ''.join(manager._errors[0][2]) print(ErrorMessage) return {'error': {'exception': ''.join(manager._errors[0][2]), 'message': 'An error occured during execution of LPU {} at step {}:\n'.format( manager._errors[0][0], manager._errors[0][1])}} time.sleep(5) # print(task) ignored_steps = 0 try: # post-processing inputs (hard coded, can be better organized) result = {'sensory': {}, 'input': {}, 'output': {}} for k, lpu in lpus.items(): records = lpu.get('input_record', []) for record in records: if record['sensory_file'] is not None: if k not in result['sensory']: result['sensory'][k] = [] with h5py.File(record['sensory_file']) as sensory_file: result['sensory'][k].append({'dt': record['sensory_interval']*dt, 'data': sensory_file['sensory'][:]}) if record['input_file'] is not None: with h5py.File(record['input_file']) as input_file: sample_interval = input_file['metadata'].attrs['sample_interval'] for var in input_file.keys(): if var == 'metadata': continue uids = [n.decode() for n in input_file[var]['uids'][:]] input_array = input_file[var]['data'][:] for i, item in enumerate(uids): if var == 'spike_state': input = np.nonzero(input_array[ignored_steps:, i:i+1].reshape(-1))[0]*dt if item in result['input']: if 'spike_time' in result['input'][item]: result['input'][item]['spike_time']['data'].append(input) result['input'][item]['spike_time']['data'] = \ np.sort(result['input'][item]['spike_time']['data']) else: result['input'][item]['spike_time'] = { 'data': input.copy(), 'dt': dt*sample_interval} else: result['input'][item] = {'spike_time': { 'data': input.copy(), 'dt': dt*sample_interval}} else: input = input_array[ignored_steps:, i:i+1] if item in result['input']: if var in result['input'][item]: result['input'][item][var]['data'] += input else: result['input'][item][var] = { 'data': input.copy(), 'dt': dt*sample_interval} else: result['input'][item] = {var: { 'data': input.copy(), 'dt': dt*sample_interval}} # if 'retina' in lpus: # input_array = si.read_array( # '{}_{}.h5'.format(config['Retina']['input_file'], user_id)) # inputs[u'ydomain'] = input_array.max() # for i, item in enumerate(retina_input_uids): # inputs['data'][item] = np.hstack( # (np.arange(int((steps-ignored_steps)/10)).reshape((-1,1))*dt*10, # input_array[ignored_steps::10,i:i+1])).tolist() # # del input_array # post-processing outputs from all LPUs and combine them into one dictionary # result = {u'data': {}} for k, lpu in lpus.items(): uid_dict = lpu.get('output_uid_dict', None) if uid_dict is not None: with h5py.File(lpu['output_file']) as output_file: sample_interval = output_file['metadata'].attrs['sample_interval'] for var in uid_dict: if var == 'spike_state': uids = [n.decode() for n in output_file[var]['uids'][:]] spike_times = output_file[var]['data']['time'][:] index = output_file[var]['data']['index'][:] for i, item in enumerate(uids): # output = np.nonzero(output_array[ignored_steps:, i:i+1].reshape(-1))[0]*dt output = spike_times[index == i] output = output[output>ignored_steps*dt]-ignored_steps*dt if item in result['output']: result['output'][item]['spike_time'] = { 'data': output, 'dt': dt*sample_interval} else: result['output'][item] = {'spike_time': { 'data': output, 'dt': dt*sample_interval}} else: uids = [n.decode() for n in output_file[var]['uids'][:]] output_array = output_file[var]['data'][:] for i, item in enumerate(uids): # if var == 'spike_state': # output = np.nonzero(output_array[ignored_steps:, i:i+1].reshape(-1))[0]*dt # if item in result['output']: # result['output'][item]['spike_time'] = { # 'data': output.tolist(), # 'dt': dt} # else: # result['output'][item] = {'spike_time': { # 'data': output.tolist(), # 'dt': dt}} # else: output = output_array[ignored_steps:, i:i+1] if item in result['output']: result['output'][item][var] = { 'data': output.copy(), 'dt': dt*sample_interval} else: result['output'][item] = {var: { 'data': output.copy(), 'dt': dt*sample_interval}} result = {'success': {'result': result, 'meta': {'dur': steps*dt}}} except: exc_type, exc_value, exc_traceback = sys.exc_info() tb = ''.join(traceback.format_exception(exc_type, exc_value, exc_traceback)) print('An error occured postprocessing of results\n' + tb) return {'error': {'exception': tb, 'message': 'An error occured when postprocessing of results in execution'}} return result
def main(): import neurokernel.mpi_relaunch import neurokernel.core_gpu as core (comp_dict, conns) = LPU.lpu_parser('neuroballad_temp_model.gexf.gz') with open('run_parameters.pickle', 'rb') as f: run_parameters = pickle.load(f) with open('record_parameters.pickle', 'rb') as f: record_parameters = pickle.load(f) dur = 1.0 dt = 1e-4 dur = run_parameters[0] dt = run_parameters[1] fl_input_processor = FileInputProcessor('neuroballad_temp_model_input.h5') from neurokernel.LPU.OutputProcessors.FileOutputProcessor import FileOutputProcessor output_processor = FileOutputProcessor(record_parameters, 'neuroballad_temp_model_output.h5', sample_interval=1) #Parse extra arguments parser = argparse.ArgumentParser() parser.add_argument( '--debug', default=False, dest='debug', action='store_true', help= 'Write connectivity structures and inter-LPU routed data in debug folder' ) parser.add_argument( '-l', '--log', default='file', type=str, help='Log output to screen [file, screen, both, or none; default:none]' ) parser.add_argument('-r', '--time_sync', default=False, action='store_true', help='Time data reception throughput [default: False]') parser.add_argument('-g', '--gpu_dev', default=[0, 1], type=int, nargs='+', help='GPU device numbers [default: 0 1]') parser.add_argument('-d', '--disconnect', default=False, action='store_true', help='Run with disconnected LPUs [default: False]') args = parser.parse_args() file_name = None screen = False if args.log.lower() in ['file', 'both']: file_name = 'neurokernel.log' if args.log.lower() in ['screen', 'both']: screen = True logger = setup_logger(file_name=file_name, screen=screen) man = core.Manager() man.add(LPU, 'lpu', dt, comp_dict, conns, input_processors=[fl_input_processor], output_processors=[output_processor], device=args.gpu_dev[1], debug=True) steps = int(dur / dt) man.spawn() man.start(steps=steps) man.wait()
def main(): logging.basicConfig(level=logging.DEBUG, stream=sys.stdout, format='%(asctime)s %(name)s %(levelname)s %(message)s') logger = logging.getLogger('cx') sys.setrecursionlimit(10000) ### Graph ### graph = Graph(Config.from_url(cx_db, 'admin', 'admin', initial_drop=False)) graph.include(models.Node.registry) graph.include(models.Relationship.registry) ### Retina ### config=retlam_demo.ConfigReader('retlam_default.cfg','../template_spec.cfg').conf retina = get_retina(config) ##### Configuration ###### logger = setup_logger(screen=True) lpu_selectors, to_list = get_cx_selectors_name() lpu_name_to_comp_dict, lpu_name_to_conn_list, pat_name_list, pat_name_to_pat = cx_component(graph) man = core.Manager() dt = 1e-4 dur = 0.2 steps = int(dur/dt) debug = True lpu_name_list = ['BU', 'bu', 'EB', 'FB', 'PB'] for name in lpu_name_list: input_processor = [] output_processor = [FileOutputProcessor([('spike_state', None), ('V',None), ('g',None), ('I', None)], '{}_output.h5'.format(name), sample_interval = 1)] man.add(LPU, name, dt, lpu_name_to_comp_dict[name], lpu_name_to_conn_list[name], input_processors = input_processor, output_processors = output_processor, device=0, debug=debug, time_sync=False) retlam_demo.add_retina_LPU(config, 0, retina, man) logger.info('add retina lpu') for name in pat_name_list: id_0, id_1 = name.split('-') man.connect(id_0, id_1, pat_name_to_pat[name][0], pat_name_to_pat[name][1].index('0'), pat_name_to_pat[name][1].index('1')) logger.info('link lpus among cx lpus') link_retina_pat_cx(retina, lpu_selectors, to_list, man) logger.info('link retina and cx lpu') man.spawn() man.start(steps) man.wait()