] and subregion.name.split('/')[0] == 'EB':
continue
if post_family in ['BU-EB']:
post_ind = [
i for i, n in enumerate(post_neuron_parse)
if n['neuropil'] != 'EB'
][0]
if list(post_neuron_parse[post_ind]
['regions'])[0][1] != '3':
continue
name = pre_name+'->'+ \
post_name+'_in_'+ \
subregion.name#.replace('/','-')
syn = graph.Synapses.create(name=name)
graph.SendsTo.create(graph.get_element(pre_rid), syn)
graph.SendsTo.create(syn, graph.get_element(post_rid))
graph.Owns.create(subregion, syn)
graph.Owns.create(post_neuropil, syn)
if subregion_neuropil.name != post_neuropil.name:
pre_neuropil = neuron_rid_to_neuropil[pre_rid]
if pre_neuropil.name != post_neuropil.name:
neuropil_pair = frozenset(
[pre_neuropil.name, post_neuropil.name])
if neuropil_pair not in tract_dict:
tract_name = '-'.join(sorted(list(neuropil_pair)))
tract = graph.Tracts.create(
name=tract_name,
version=cx_version,
neuropils=set(neuropil_pair))
tract_dict[neuropil_pair] = tract
logger.info('connection discovered: %s <- %s' % (rid0, rid1))
# Create a new synapse node for every connection and connect the input and
# output nodes of the connection to the synapse node:
for nc in nodes_to_connect:
# Create a new synapse node for every connection:
name = df_neurons.ix[nc[0]]['name'] + '->' + df_neurons.ix[nc[1]]['name']
syn = graph.Synapses.create(name=name)
logger.info('created Synapse node: %s' % name)
# Save family and neuropil of postsynaptic neuron in the synapse node:
syn.update(family=df_neurons.ix[nc[1]]['family'],
neuropil=df_neurons.ix[nc[1]]['neuropil'])
from_neuropil = graph.get_element(nc[0]).in_(models.Owns)[0]
to_neuropil = graph.get_element(nc[1]).in_(models.Owns)[0]
if from_neuropil != to_neuropil:
logger.info('elements in different neuropils: %s, %s' % \
(from_neuropil.name, to_neuropil.name))
else:
logger.info('elements in the same neuropil: %s' % from_neuropil.name)
# Connect the neurons associated with the synapse node (XXX this
# relationship isn't specified in the NeuroArch data model):
graph.SendsTo.create(graph.get_element(nc[0]), syn)
logger.info('connected %s %s -[SendsTo]-> Synapse %s' % \
(graph.get_element(nc[0]).__class__.__name__,
graph.get_element(nc[0]).name,
syn.name))
graph.SendsTo.create(syn, graph.get_element(nc[1]))
class neuroarch_server(object):
""" Methods to process neuroarch json tasks """
def __init__(self,
database='/na_server',
username='******',
password='******',
user=None):
try:
self.graph = Graph(
Config.from_url(database,
username,
password,
initial_drop=False,
serialization_type=OrientSerialization.Binary))
except:
#print "WARNING: Serialisation flag ignored"
self.graph = Graph(
Config.from_url(database,
username,
password,
initial_drop=False))
self.graph.include(Node.registry)
self.graph.include(Relationship.registry)
self.user = user
self.query_processor = query_processor(self.graph)
self._busy = False
def retrieve_neuron(self, nid):
# WIP: Currently retrieves all information for the get_as method, this will be refined when we know what data we want to store and pull out here
try:
n = self.graph.get_element(nid)
if n == None:
return {}
else:
output = QueryWrapper.from_objs(self.graph, [n])
return output.get_as()[0].to_json()
except Exception as e:
raise e
def process_query(self, task):
""" configure a task processing, and format the results as desired """
# WIP: Expand type of information that can be retrieved
assert 'query' in task.keys()
try:
self.query_processor.process(task['query'], self.user)
return True
except Exception as e:
print e
return False
@staticmethod
def process_verb(output, user, verb):
if verb == 'add':
assert (len(user.state) >= 2)
user.state[-1] = output + user.state[-2]
elif verb == 'keep':
assert (len(user.state) >= 2)
user.state[-1] = output & user.state[-2]
output = user.state[-2] - user.state[-1]
elif verb == 'remove':
assert (len(user.state) >= 2)
user.state[-1] = user.state[-2] - output
else:
assert (len(user.state) >= 2)
cmd = {'undo': {'states': 1}}
output = output & user.state[-2]
user.process_command(cmd)
return output
def receive_task(self, task, threshold=None, query_results=True):
""" process a task of form
{'query':...} or {'command': ...}
update the user states, and query neuroarch
This is the default access route
"""
while (self._busy):
time.sleep(1)
try:
self._busy = True
if not type(task) == dict:
task = json.loads(task)
task = byteify(task)
if 'format' not in task:
task['format'] = 'morphology'
assert 'query' in task or 'command' in task
user = self.user
if 'command' in task:
output = user.process_command(task['command'])
if 'verb' in task and not task['verb'] == 'show':
try:
output = self.process_verb(output, user, task['verb'])
except Exception as e:
print e
if not task['verb'] == 'add':
if task['format'] == 'morphology':
output = output.get_data_rids(cls='MorphologyData')
else:
output = output._records_to_list(output.nodes)
self._busy = False
return (output, True)
if isinstance(output, QueryWrapper):
#print output._records_to_list(output.nodes)
if task['format'] == 'morphology':
#df = output.get_data(cls='MorphologyData')[0]
try:
#output= df[['sample','identifier','x','y','z','r','parent','name']].to_dict(orient='index')
#output= df.to_dict(orient='index')
output = output.get_data(cls='MorphologyData',
as_type='nx').node
except KeyError:
output = {}
elif task['format'] == 'no_result':
output = {}
elif task['format'] == 'get_data':
if 'cls' in task:
output = output.get_data(
cls=task['cls'])[0].to_dict(orient='index')
else:
output = output.get_data()[0].to_dict(
orient='index')
elif task['format'] == 'nx':
nx_graph = output.get_as('nx')
output = {
'nodes': nx_graph.node,
'edges': nx_graph.edge
}
elif task['format'] == 'nk':
output = output.traverse_owned_by_get_toplevel()
for x in output['LPU']:
g = output['LPU'][x].get_as('nx')
output['LPU'][x] = {
'nodes': g.node,
'edges': g.edge
}
for x in output['Pattern']:
g = output['Pattern'][x].get_as('nx')
output['Pattern'][x] = {
'nodes': g.node,
'edges': g.edge
}
elif task['format'] == 'df':
dfs = output.get_as()
output = {}
if 'node_cols' in task:
output['nodes'] = dfs[0][
task['node_cols']].to_dict(orient='index')
else:
output['nodes'] = dfs[0].to_dict(orient='index')
if 'edge_cols' in task:
output['edges'] = dfs[1][
task['edge_cols']].to_dict(orient='index')
else:
output['edges'] = dfs[1].to_dict(orient='index')
elif task['format'] == 'qw':
pass
# Default to nodes and edges df
else:
dfs = output.get_as()
output = {
'nodes': dfs[0].to_dict(orient='index'),
'edges': dfs[1].to_dict(orient='index')
}
else:
output = str(output)
if threshold and isinstance(output, dict):
chunked_output = []
for c in chunks(output, threshold):
chunked_output.append(c)
output = chunked_output
self._busy = False
return (output, True)
elif 'query' in task:
succ = self.process_query(task)
if query_results:
task['command'] = {"retrieve": {"state": 0}}
output = (None, )
try:
self._busy = False
output = self.receive_task(task, threshold)
if output[0] == None:
succ = False
except Exception as e:
print e
succ = False
self._busy = False
if 'temp' in task and task['temp'] and len(
user.state) >= 2:
user.process_command({'undo': {'states': 1}})
return (output[0], succ)
self._busy = False
return succ
except Exception as e:
print e
self._busy = False
class neuroarch_server(object):
""" Methods to process neuroarch json tasks """
def __init__(self,
database='/na_server',
username='******',
password='******',
user=None):
try:
self.graph = Graph(
Config.from_url(database,
username,
password,
initial_drop=False,
serialization_type=OrientSerialization.Binary))
except:
print "WARNING: Serialisation flag ignored"
self.graph = Graph(
Config.from_url(database,
username,
password,
initial_drop=False))
self.graph.include(Node.registry)
self.graph.include(Relationship.registry)
self.user = user
self.query_processor = query_processor(self.graph)
def retrieve_neuron(self, nid):
# WIP: Currently retrieves all information for the get_as method, this will be refined when we know what data we want to store and pull out here
try:
n = self.graph.get_element(nid)
if n == None:
return {}
else:
output = QueryWrapper.from_objs(self.graph, [n])
return output.get_as()[0].to_json()
except Exception as e:
raise e
# Hackathon 2017
def retrieve_by_id(self, task, threshold):
# Retrieve an object by ID, in order to allow direct addressing to na objects or vfb neurons.
# WIP: Test thresholding and chunking with larger NA 'tags'
key_types = ['na', 'vfb'] #,'fc'] # A list of valid ids
# na is a direct neuroarch ID minus the #
# vfb is virtual fly brain with tag vib_id
# fc will be fly circuit, currently in name
if not type(task) == dict:
task = json.loads(task)
task = byteify(task)
user = self.user
key_type = task["key_type"]
key = task["key"]
assert key_type in key_types
if key_type == 'na':
try:
n = self.graph.get_element('#' + key)
except Exception as e:
raise e
elif key_type == 'vfb':
n = self.graph.Neurons.query(vfb_id=key).first()
else:
pass
if n == None:
return ({}, False)
else:
output = QueryWrapper.from_objs(self.graph, [n])
# Add hook into user system
user.append(output)
df = output.get_data(cls='MorphologyData')[0]
output = df[[
'sample', 'identifier', 'x', 'y', 'z', 'r', 'parent', 'name'
]].to_dict(orient='index')
if threshold and isinstance(output, dict):
chunked_output = []
for c in chunks(output, threshold):
chunked_output.append(c)
output = chunked_output
return (output, True)
def process_query(self, task):
""" configure a task processing, and format the results as desired """
# WIP: Expand type of information that can be retrieved
assert 'query' in task.keys()
try:
self.query_processor.process(task['query'], self.user)
return True
except Exception as e:
print e
return False
@staticmethod
def process_verb(output, user, verb):
if verb == 'add':
assert (len(user.state) >= 2)
user.state[-1] = output + user.state[-2]
elif verb == 'keep':
assert (len(user.state) >= 2)
user.state[-1] = output & user.state[-2]
output = user.state[-2] - user.state[-1]
elif verb == 'remove':
assert (len(user.state) >= 2)
user.state[-1] = user.state[-2] - output
else:
assert (len(user.state) >= 2)
cmd = {'undo': {'states': 1}}
output = output & user.state[-2]
user.process_command(cmd)
return output
def receive_task(self, task, threshold=None, query_results=True):
""" process a task of form
{'query':...} or {'command': ...}
update the user states, and query neuroarch
This is the default access route
"""
if not type(task) == dict:
task = json.loads(task)
task = byteify(task)
if 'format' not in task:
task['format'] = 'morphology'
assert 'query' in task or 'command' in task
user = self.user
if 'command' in task:
output = user.process_command(task['command'])
if 'verb' in task and not task['verb'] == 'show':
try:
output = self.process_verb(output, user, task['verb'])
except Exception as e:
print e
if not task['verb'] == 'add':
if task['format'] == 'morphology':
output = output.get_data_rids(cls='MorphologyData')
else:
output = output._records_to_list(output.nodes)
return (output, True)
if isinstance(output, QueryWrapper):
#print output._records_to_list(output.nodes)
if task['format'] == 'morphology':
df = output.get_data(cls='MorphologyData')[0]
try:
output = df[[
'sample', 'identifier', 'x', 'y', 'z', 'r',
'parent', 'name'
]].to_dict(orient='index')
except KeyError:
output = {}
elif task['format'] == 'no_result':
output = {}
elif task['format'] == 'get_data':
if 'cls' in task:
output = output.get_data(cls=task['cls'])[0].to_dict(
orient='index')
else:
output = output.get_data()[0].to_dict(orient='index')
elif task['format'] == 'nx':
nx_graph = output.get_as('nx')
output = {'nodes': nx_graph.node, 'edges': nx_graph.edge}
elif task['format'] == 'nk':
output = output.traverse_owned_by_get_toplevel()
for x in output['LPU']:
g = output['LPU'][x].get_as('nx')
output['LPU'][x] = {'nodes': g.node, 'edges': g.edge}
for x in output['Pattern']:
g = output['Pattern'][x].get_as('nx')
output['Pattern'][x] = {
'nodes': g.node,
'edges': g.edge
}
elif task['format'] == 'df':
dfs = output.get_as()
output = {}
if 'node_cols' in task:
output['nodes'] = dfs[0][task['node_cols']].to_dict(
orient='index')
else:
output['nodes'] = dfs[0].to_dict(orient='index')
if 'edge_cols' in task:
output['edges'] = dfs[1][task['edge_cols']].to_dict(
orient='index')
else:
output['edges'] = dfs[1].to_dict(orient='index')
# Default to nodes and edges df
else:
dfs = output.get_as()
output = {
'nodes': dfs[0].to_dict(orient='index'),
'edges': dfs[1].to_dict(orient='index')
}
else:
output = str(output)
if threshold and isinstance(output, dict):
chunked_output = []
for c in chunks(output, threshold):
chunked_output.append(c)
output = chunked_output
return (output, True)
elif 'query' in task:
succ = self.process_query(task)
if query_results:
task['command'] = {"retrieve": {"state": 0}}
output = (None, )
try:
output = self.receive_task(task, threshold)
if output[0] == None:
succ = False
except Exception as e:
print e
succ = False
return (output[0], succ)
return succ