] and subregion.name.split('/')[0] == 'EB': continue if post_family in ['BU-EB']: post_ind = [ i for i, n in enumerate(post_neuron_parse) if n['neuropil'] != 'EB' ][0] if list(post_neuron_parse[post_ind] ['regions'])[0][1] != '3': continue name = pre_name+'->'+ \ post_name+'_in_'+ \ subregion.name#.replace('/','-') syn = graph.Synapses.create(name=name) graph.SendsTo.create(graph.get_element(pre_rid), syn) graph.SendsTo.create(syn, graph.get_element(post_rid)) graph.Owns.create(subregion, syn) graph.Owns.create(post_neuropil, syn) if subregion_neuropil.name != post_neuropil.name: pre_neuropil = neuron_rid_to_neuropil[pre_rid] if pre_neuropil.name != post_neuropil.name: neuropil_pair = frozenset( [pre_neuropil.name, post_neuropil.name]) if neuropil_pair not in tract_dict: tract_name = '-'.join(sorted(list(neuropil_pair))) tract = graph.Tracts.create( name=tract_name, version=cx_version, neuropils=set(neuropil_pair)) tract_dict[neuropil_pair] = tract
logger.info('connection discovered: %s <- %s' % (rid0, rid1)) # Create a new synapse node for every connection and connect the input and # output nodes of the connection to the synapse node: for nc in nodes_to_connect: # Create a new synapse node for every connection: name = df_neurons.ix[nc[0]]['name'] + '->' + df_neurons.ix[nc[1]]['name'] syn = graph.Synapses.create(name=name) logger.info('created Synapse node: %s' % name) # Save family and neuropil of postsynaptic neuron in the synapse node: syn.update(family=df_neurons.ix[nc[1]]['family'], neuropil=df_neurons.ix[nc[1]]['neuropil']) from_neuropil = graph.get_element(nc[0]).in_(models.Owns)[0] to_neuropil = graph.get_element(nc[1]).in_(models.Owns)[0] if from_neuropil != to_neuropil: logger.info('elements in different neuropils: %s, %s' % \ (from_neuropil.name, to_neuropil.name)) else: logger.info('elements in the same neuropil: %s' % from_neuropil.name) # Connect the neurons associated with the synapse node (XXX this # relationship isn't specified in the NeuroArch data model): graph.SendsTo.create(graph.get_element(nc[0]), syn) logger.info('connected %s %s -[SendsTo]-> Synapse %s' % \ (graph.get_element(nc[0]).__class__.__name__, graph.get_element(nc[0]).name, syn.name)) graph.SendsTo.create(syn, graph.get_element(nc[1]))
class neuroarch_server(object): """ Methods to process neuroarch json tasks """ def __init__(self, database='/na_server', username='******', password='******', user=None): try: self.graph = Graph( Config.from_url(database, username, password, initial_drop=False, serialization_type=OrientSerialization.Binary)) except: #print "WARNING: Serialisation flag ignored" self.graph = Graph( Config.from_url(database, username, password, initial_drop=False)) self.graph.include(Node.registry) self.graph.include(Relationship.registry) self.user = user self.query_processor = query_processor(self.graph) self._busy = False def retrieve_neuron(self, nid): # WIP: Currently retrieves all information for the get_as method, this will be refined when we know what data we want to store and pull out here try: n = self.graph.get_element(nid) if n == None: return {} else: output = QueryWrapper.from_objs(self.graph, [n]) return output.get_as()[0].to_json() except Exception as e: raise e def process_query(self, task): """ configure a task processing, and format the results as desired """ # WIP: Expand type of information that can be retrieved assert 'query' in task.keys() try: self.query_processor.process(task['query'], self.user) return True except Exception as e: print e return False @staticmethod def process_verb(output, user, verb): if verb == 'add': assert (len(user.state) >= 2) user.state[-1] = output + user.state[-2] elif verb == 'keep': assert (len(user.state) >= 2) user.state[-1] = output & user.state[-2] output = user.state[-2] - user.state[-1] elif verb == 'remove': assert (len(user.state) >= 2) user.state[-1] = user.state[-2] - output else: assert (len(user.state) >= 2) cmd = {'undo': {'states': 1}} output = output & user.state[-2] user.process_command(cmd) return output def receive_task(self, task, threshold=None, query_results=True): """ process a task of form {'query':...} or {'command': ...} update the user states, and query neuroarch This is the default access route """ while (self._busy): time.sleep(1) try: self._busy = True if not type(task) == dict: task = json.loads(task) task = byteify(task) if 'format' not in task: task['format'] = 'morphology' assert 'query' in task or 'command' in task user = self.user if 'command' in task: output = user.process_command(task['command']) if 'verb' in task and not task['verb'] == 'show': try: output = self.process_verb(output, user, task['verb']) except Exception as e: print e if not task['verb'] == 'add': if task['format'] == 'morphology': output = output.get_data_rids(cls='MorphologyData') else: output = output._records_to_list(output.nodes) self._busy = False return (output, True) if isinstance(output, QueryWrapper): #print output._records_to_list(output.nodes) if task['format'] == 'morphology': #df = output.get_data(cls='MorphologyData')[0] try: #output= df[['sample','identifier','x','y','z','r','parent','name']].to_dict(orient='index') #output= df.to_dict(orient='index') output = output.get_data(cls='MorphologyData', as_type='nx').node except KeyError: output = {} elif task['format'] == 'no_result': output = {} elif task['format'] == 'get_data': if 'cls' in task: output = output.get_data( cls=task['cls'])[0].to_dict(orient='index') else: output = output.get_data()[0].to_dict( orient='index') elif task['format'] == 'nx': nx_graph = output.get_as('nx') output = { 'nodes': nx_graph.node, 'edges': nx_graph.edge } elif task['format'] == 'nk': output = output.traverse_owned_by_get_toplevel() for x in output['LPU']: g = output['LPU'][x].get_as('nx') output['LPU'][x] = { 'nodes': g.node, 'edges': g.edge } for x in output['Pattern']: g = output['Pattern'][x].get_as('nx') output['Pattern'][x] = { 'nodes': g.node, 'edges': g.edge } elif task['format'] == 'df': dfs = output.get_as() output = {} if 'node_cols' in task: output['nodes'] = dfs[0][ task['node_cols']].to_dict(orient='index') else: output['nodes'] = dfs[0].to_dict(orient='index') if 'edge_cols' in task: output['edges'] = dfs[1][ task['edge_cols']].to_dict(orient='index') else: output['edges'] = dfs[1].to_dict(orient='index') elif task['format'] == 'qw': pass # Default to nodes and edges df else: dfs = output.get_as() output = { 'nodes': dfs[0].to_dict(orient='index'), 'edges': dfs[1].to_dict(orient='index') } else: output = str(output) if threshold and isinstance(output, dict): chunked_output = [] for c in chunks(output, threshold): chunked_output.append(c) output = chunked_output self._busy = False return (output, True) elif 'query' in task: succ = self.process_query(task) if query_results: task['command'] = {"retrieve": {"state": 0}} output = (None, ) try: self._busy = False output = self.receive_task(task, threshold) if output[0] == None: succ = False except Exception as e: print e succ = False self._busy = False if 'temp' in task and task['temp'] and len( user.state) >= 2: user.process_command({'undo': {'states': 1}}) return (output[0], succ) self._busy = False return succ except Exception as e: print e self._busy = False
class neuroarch_server(object): """ Methods to process neuroarch json tasks """ def __init__(self, database='/na_server', username='******', password='******', user=None): try: self.graph = Graph( Config.from_url(database, username, password, initial_drop=False, serialization_type=OrientSerialization.Binary)) except: print "WARNING: Serialisation flag ignored" self.graph = Graph( Config.from_url(database, username, password, initial_drop=False)) self.graph.include(Node.registry) self.graph.include(Relationship.registry) self.user = user self.query_processor = query_processor(self.graph) def retrieve_neuron(self, nid): # WIP: Currently retrieves all information for the get_as method, this will be refined when we know what data we want to store and pull out here try: n = self.graph.get_element(nid) if n == None: return {} else: output = QueryWrapper.from_objs(self.graph, [n]) return output.get_as()[0].to_json() except Exception as e: raise e # Hackathon 2017 def retrieve_by_id(self, task, threshold): # Retrieve an object by ID, in order to allow direct addressing to na objects or vfb neurons. # WIP: Test thresholding and chunking with larger NA 'tags' key_types = ['na', 'vfb'] #,'fc'] # A list of valid ids # na is a direct neuroarch ID minus the # # vfb is virtual fly brain with tag vib_id # fc will be fly circuit, currently in name if not type(task) == dict: task = json.loads(task) task = byteify(task) user = self.user key_type = task["key_type"] key = task["key"] assert key_type in key_types if key_type == 'na': try: n = self.graph.get_element('#' + key) except Exception as e: raise e elif key_type == 'vfb': n = self.graph.Neurons.query(vfb_id=key).first() else: pass if n == None: return ({}, False) else: output = QueryWrapper.from_objs(self.graph, [n]) # Add hook into user system user.append(output) df = output.get_data(cls='MorphologyData')[0] output = df[[ 'sample', 'identifier', 'x', 'y', 'z', 'r', 'parent', 'name' ]].to_dict(orient='index') if threshold and isinstance(output, dict): chunked_output = [] for c in chunks(output, threshold): chunked_output.append(c) output = chunked_output return (output, True) def process_query(self, task): """ configure a task processing, and format the results as desired """ # WIP: Expand type of information that can be retrieved assert 'query' in task.keys() try: self.query_processor.process(task['query'], self.user) return True except Exception as e: print e return False @staticmethod def process_verb(output, user, verb): if verb == 'add': assert (len(user.state) >= 2) user.state[-1] = output + user.state[-2] elif verb == 'keep': assert (len(user.state) >= 2) user.state[-1] = output & user.state[-2] output = user.state[-2] - user.state[-1] elif verb == 'remove': assert (len(user.state) >= 2) user.state[-1] = user.state[-2] - output else: assert (len(user.state) >= 2) cmd = {'undo': {'states': 1}} output = output & user.state[-2] user.process_command(cmd) return output def receive_task(self, task, threshold=None, query_results=True): """ process a task of form {'query':...} or {'command': ...} update the user states, and query neuroarch This is the default access route """ if not type(task) == dict: task = json.loads(task) task = byteify(task) if 'format' not in task: task['format'] = 'morphology' assert 'query' in task or 'command' in task user = self.user if 'command' in task: output = user.process_command(task['command']) if 'verb' in task and not task['verb'] == 'show': try: output = self.process_verb(output, user, task['verb']) except Exception as e: print e if not task['verb'] == 'add': if task['format'] == 'morphology': output = output.get_data_rids(cls='MorphologyData') else: output = output._records_to_list(output.nodes) return (output, True) if isinstance(output, QueryWrapper): #print output._records_to_list(output.nodes) if task['format'] == 'morphology': df = output.get_data(cls='MorphologyData')[0] try: output = df[[ 'sample', 'identifier', 'x', 'y', 'z', 'r', 'parent', 'name' ]].to_dict(orient='index') except KeyError: output = {} elif task['format'] == 'no_result': output = {} elif task['format'] == 'get_data': if 'cls' in task: output = output.get_data(cls=task['cls'])[0].to_dict( orient='index') else: output = output.get_data()[0].to_dict(orient='index') elif task['format'] == 'nx': nx_graph = output.get_as('nx') output = {'nodes': nx_graph.node, 'edges': nx_graph.edge} elif task['format'] == 'nk': output = output.traverse_owned_by_get_toplevel() for x in output['LPU']: g = output['LPU'][x].get_as('nx') output['LPU'][x] = {'nodes': g.node, 'edges': g.edge} for x in output['Pattern']: g = output['Pattern'][x].get_as('nx') output['Pattern'][x] = { 'nodes': g.node, 'edges': g.edge } elif task['format'] == 'df': dfs = output.get_as() output = {} if 'node_cols' in task: output['nodes'] = dfs[0][task['node_cols']].to_dict( orient='index') else: output['nodes'] = dfs[0].to_dict(orient='index') if 'edge_cols' in task: output['edges'] = dfs[1][task['edge_cols']].to_dict( orient='index') else: output['edges'] = dfs[1].to_dict(orient='index') # Default to nodes and edges df else: dfs = output.get_as() output = { 'nodes': dfs[0].to_dict(orient='index'), 'edges': dfs[1].to_dict(orient='index') } else: output = str(output) if threshold and isinstance(output, dict): chunked_output = [] for c in chunks(output, threshold): chunked_output.append(c) output = chunked_output return (output, True) elif 'query' in task: succ = self.process_query(task) if query_results: task['command'] = {"retrieve": {"state": 0}} output = (None, ) try: output = self.receive_task(task, threshold) if output[0] == None: succ = False except Exception as e: print e succ = False return (output[0], succ) return succ