def na_get_data(task, details=None):
if not isinstance(task, dict):
task = json.loads(task)
task = byteify(task)
user_id = task['user'] if (details.caller_authrole == 'processor' and 'user' in task) \
else details.caller
threshold = None
self.log.info("na_get_data() called with task: {task}", task=task)
server = self.user_list.user(user_id)['server']
try:
if not is_rid(task['id']):
returnValue({})
elem = server.graph.get_element(task['id'])
q = QueryWrapper.from_objs(server.graph, [elem])
callback = get_data_sub if elem.element_type == 'Neuron' else get_syn_data_sub
if not (elem.element_type == 'Neuron'
or elem.element_type == 'Synapse'
or elem.element_type == 'InferredSynapse'):
qn = q.gen_traversal_in(['HasData', 'Neuron'], min_depth=1)
if not qn:
q = q.gen_traversal_in(
['HasData', ['Synapse', 'InferredSynapse']],
min_depth=1)
else:
q = qn
callback = get_data_sub
#res = yield threads.deferToThread(get_data_sub, q)
res = yield callback(q)
except Exception as e:
print e
self.log.failure("Error Retrieveing Data")
res = {}
returnValue(res)
def retrieve_tag(task, details=None):
if not "tag" in task:
return {"info": {"error": "Tag must be provided"}}
if not isinstance(task, dict):
task = json.loads(task)
task = byteify(task)
user_id = task['user'] if (details.caller_authrole == 'processor' and 'user' in task) \
else details.caller
self.log.info("retrieve_tag() called with task: {task} ",
task=task)
server = self.user_list.user(user_id)['server']
tagged_result = QueryWrapper.from_tag(graph=server.graph,
tag=task['tag'])
if tagged_result and tagged_result[
'metadata'] and tagged_result['metadata'] != '{}':
server.user.append(tagged_result['qw'])
return {
'data': tagged_result['metadata'],
'info': {
'success': 'Server Retrieved Tag Succesfully'
}
}
else:
return {
"info": {
"error": "No such tag exists in this database server"
}
}
def retrieve_neuron(self, nid):
# WIP: Currently retrieves all information for the get_as method, this will be refined when we know what data we want to store and pull out here
try:
n = self.graph.get_element(nid)
if n == None:
return {}
else:
output = QueryWrapper.from_objs(self.graph, [n])
return output.get_as()[0].to_json()
except Exception as e:
raise e
def retrieve_by_id(self, task, threshold):
# Retrieve an object by ID, in order to allow direct addressing to na objects or vfb neurons.
# WIP: Test thresholding and chunking with larger NA 'tags'
key_types = ['na', 'vfb'] #,'fc'] # A list of valid ids
# na is a direct neuroarch ID minus the #
# vfb is virtual fly brain with tag vib_id
# fc will be fly circuit, currently in name
if not type(task) == dict:
task = json.loads(task)
task = byteify(task)
user = self.user
key_type = task["key_type"]
key = task["key"]
assert key_type in key_types
if key_type == 'na':
try:
n = self.graph.get_element('#' + key)
except Exception as e:
raise e
elif key_type == 'vfb':
n = self.graph.Neurons.query(vfb_id=key).first()
else:
pass
if n == None:
return ({}, False)
else:
output = QueryWrapper.from_objs(self.graph, [n])
# Add hook into user system
user.append(output)
df = output.get_data(cls='MorphologyData')[0]
output = df[[
'sample', 'identifier', 'x', 'y', 'z', 'r', 'parent', 'name'
]].to_dict(orient='index')
if threshold and isinstance(output, dict):
chunked_output = []
for c in chunks(output, threshold):
chunked_output.append(c)
output = chunked_output
return (output, True)
def process_single(self, query, user, task_memory):
""" accetpt a single query object or form
[{'object':...:,'action...'}]
"""
assert 'object' in query and 'action' in query
assert 'class' in query['object'] or 'state' in query[
'object'] or 'memory' in query['object']
'''
if 'class' in query['object']:
# Retrieve Class
class_name = query['object']['class']
na_object = self.class_list[class_name]
# convert result to a query wrapper to save
'''
if 'state' in query['object']:
state_num = query['object']['state']
if type(state_num) is long:
state_num = int(state_num)
assert type(state_num) in [int, long]
na_object = user.retrieve(index=state_num)
elif 'memory' in query['object']:
assert task_memory is not []
memory_index = query['object']['memory']
if type(memory_index) is long:
memory_index = int(memory_index)
assert type(memory_index) is int
assert len(task_memory) > memory_index
na_object = task_memory[-1 - memory_index]
# Retrieve method
if 'method' in query[
'action']: # A class query can only take a method.
if 'class' in query['object']:
method_call = query['action']['method']
assert len(method_call.keys()) == 1
method_name = method_call.keys()[0]
method_args = method_call[method_name]
columns = ""
attrs = []
for k, v in method_args.iteritems():
if not (isinstance(v, list)):
if isinstance(v, (basestring, numbers.Number)):
v = [str(v)]
else:
# To prevent issues with unicode objects
if v and isinstance(v[0], basestring):
v = [str(val) for val in v]
if len(v) == 1 and isinstance(
v[0], (unicode, str)) and len(v[0]) >= 2 and str(
v[0][:2]) == '/r':
attrs.append("%s matches '%s'" %
(str(k), str(v[0][2:])))
else:
attrs.append("%s in %s" % (str(k), str(v)))
attrs = " and ".join(attrs)
if attrs: attrs = "where " + attrs
query['object']['class'] = _list_repr(query['object']['class'])
q = {}
for i, a in enumerate(query['object']['class']):
var = '$q' + str(i)
q[var] = "{var} = (select from {cls} {attrs})".format(
var=var, cls=str(a), attrs=str(attrs))
query_str = "select from (select expand($a) let %s, $a = unionall(%s))" % \
(", ".join(q.values()), ", ".join(q.keys()) )
query_str = QueryString(query_str, 'sql')
query_result = QueryWrapper(self.graph, query_str)
else:
method_call = query['action']['method']
assert len(method_call.keys()) == 1
method_name = method_call.keys()[0]
# check method is valid
assert method_name in dir(type(na_object))
# Retrieve arguments
method_args = byteify(method_call[method_name])
if 'pass_through' in method_args:
pass_through = method_args.pop('pass_through')
if isinstance(pass_through,
list) and pass_through and isinstance(
pass_through[0], list):
query_result = getattr(na_object,
method_name)(*pass_through,
**method_args)
else:
query_result = getattr(na_object,
method_name)(pass_through,
**method_args)
else:
query_result = getattr(na_object,
method_name)(**method_args)
elif 'op' in query['action']:
method_call = query['action']['op']
assert len(method_call.keys()) == 1
method_name = method_call.keys()[0]
# WIP: Check which operators are supported
# What if we want to be a op between two past states!
# retieve past external state or internal memory state
if 'state' in method_call[method_name]:
state_num = method_call[method_name]['state']
assert type(state_num) in [int, long]
past_object = user.retrieve(index=state_num)
elif 'memory' in method_call[method_name]:
assert task_memory is not []
memory_index = method_call[method_name]['memory']
if type(memory_index) is long:
memory_index = int(memory_index)
assert type(memory_index) is int
assert len(task_memory) > memory_index
past_object = task_memory[-1 - memory_index]
#query_result = getattr(na_object, method_name)(method_args)
## INVERSE THIS na and method argis (WHY?)
query_result = getattr(na_object, method_name)(past_object)
# convert result to a query wrapper to save
if type(query_result) is not QueryWrapper:
output = QueryWrapper.from_objs(self.graph, query_result.all())
else:
output = query_result
return output