def do_reindex(self, handler):
req = parseRequestEntity(handler.request.body)
if req is None:
return None
if '_id' not in req.keys():
return None
'''
we might have to replace parts of the request with
secret values.
'''
if 'task_path' in req.keys():
req_keys = list(
chain.from_iterable(
[k.keys() for k in MIME_TYPE_TASK_REQUIREMENTS]))
if req['task_path'] in req_keys:
try:
req_field = [
k for k in MIME_TYPE_TASK_REQUIREMENTS
if k.keys()[0] == req['task_path']
][0][req['task_path']]
print req_field
except Exception as e:
if DEBUG:
print "Could not resolve required task info. %s" % e
for repl in req_field.iteritems():
if repl[0] in req.keys():
handler.request.body = handler.request.body.replace(
repl[1], getSecrets(repl[1]))
return self.passToAnnex(handler)
def do_reindex(self, handler): req = parseRequestEntity(handler.request.body) if req is None: return None if '_id' not in req.keys(): return None ''' we might have to replace parts of the request with secret values. ''' if 'task_path' in req.keys(): req_keys = list(chain.from_iterable( [k.keys() for k in TASK_REQUIREMENTS])) if req['task_path'] in req_keys: try: req_field = [k for k in TASK_REQUIREMENTS if k.keys()[0] == req['task_path']][0][req['task_path']] print req_field except Exception as e: if DEBUG: print "Could not resolve required task info. %s" % e for repl in req_field.iteritems(): if repl[0] in req.keys(): handler.request.body = handler.request.body.replace( repl[1], getSecrets(repl[1])) return self.passToAnnex(handler)
def do_tasks(self, request): args = parseRequestEntity(request.query) if len(args.keys()) == 1 and '_id' in args.keys(): return self.get(_id=args['_id']) return self.do_list(request, query=deepcopy(QUERY_DEFAULTS['UV_TASK']))
def do_open_drive_file(self, handler):
if DEBUG: print "opening this drive file in unveillance annex"
status = self.do_get_status(handler)
if status not in [2,3]:
if DEBUG: print "NO-ACCESS TO THIS METHOD (\"do_open_drive_file\")"
return None
files = None
self.initDriveClient(handler)
for _id in parseRequestEntity(handler.request.query)['_ids']:
_id = urllib.unquote(_id).replace("'", "")[1:]
file_name = self.drive_client.getFileName(_id)
if file_name is None: return None
url = "%s/documents/?file_name=%s" % (buildServerURL(), file_name)
entry = None
handled_file = None
if DEBUG: print url
# look up the file in annex. (annex/documents/?file_name=file)
# if this file exists in annex, return its _id for opening in-app
try:
entry = json.loads(requests.get(
url, verify=False).content)['data']['documents'][0]
except Exception as e:
if DEBUG: print "COULD NOT GET ENTRY:\n%s" % e
if entry is not None:
if DEBUG: print type(entry['_id'])
handled_file = { '_id' : entry['_id'] }
else:
if status != 3:
if DEBUG:
print "** at this point, we would process file if you were admin"
print "** but you are not admin."
return None
entry = self.drive_client.download(_id,
save_as=file_name, save=False, return_content=True)
if entry is not None:
self.addToNetcatQueue({
'file' : entry[0],
'save_as' : entry[1],
'importer_source' : "google_drive"
})
handled_file = { 'file_name' : entry[1] }
if handled_file is not None:
if files is None: files = []
files.append(handled_file)
return files
def do_logout(self, handler): status = self.do_get_status(handler) if status not in [2, 3]: if DEBUG: print "CANNOT LOG IN USER, DON'T EVEN TRY (status %d)" % status return None credentials = parseRequestEntity(handler.request.body) if credentials is None: return None if DEBUG: print credentials return self.logoutUser(credentials, handler)
def do_logout(self, handler):
status = self.do_get_status(handler)
if status not in [2, 3]:
if DEBUG:
print "CANNOT LOG IN USER, DON'T EVEN TRY (status %d)" % status
return None
credentials = parseRequestEntity(handler.request.body)
if credentials is None: return None
if DEBUG: print credentials
return self.logoutUser(credentials, handler)
def do_create_new_user(self, handler): status = self.do_get_status(handler) if status != 3: return None new_user = parseRequestEntity(handler.request.body) if new_user is None: return None if DEBUG: print new_user n_set = sorted(['nu_username', 'nu_password']) if sorted(list(set(n_set) & set(new_user.keys()))) != n_set: return None from Utils.funcs import createNewUser return createNewUser(new_user['nu_username'], new_user['nu_password'])
def do_create_new_user(self, handler):
status = self.do_get_status(handler)
if status != 3: return None
new_user = parseRequestEntity(handler.request.body)
if new_user is None: return None
if DEBUG: print new_user
n_set = sorted(['nu_username', 'nu_password'])
if sorted(list(set(n_set) & set(new_user.keys()))) != n_set:
return None
from Utils.funcs import createNewUser
return createNewUser(new_user['nu_username'], new_user['nu_password'])
def do_login(self, handler): status = self.do_get_status(handler) if status != 1: if DEBUG: print "CANNOT LOG IN USER, DON'T EVEN TRY (status %d)" % status return None credentials = parseRequestEntity(handler.request.body) if credentials is None: return None if DEBUG: print credentials try: return self.loginUser(credentials['username'], credentials['password'], handler) except KeyError as e: if DEBUG: print "CANNOT LOG IN USER: %s missing" % e return None
def do_login(self, handler):
status = self.do_get_status(handler)
if status != 1:
if DEBUG:
print "CANNOT LOG IN USER, DON'T EVEN TRY (status %d)" % status
return None
credentials = parseRequestEntity(handler.request.body)
if credentials is None: return None
if DEBUG: print credentials
try:
return self.loginUser(credentials['username'],
credentials['password'], handler)
except KeyError as e:
if DEBUG: print "CANNOT LOG IN USER: %s missing" % e
return None
def do_reindex(self, request):
print "DOING REINDEX"
query = parseRequestEntity(request.query)
if query is None: return None
if '_id' not in query.keys(): return None
document = self.get(_id=query['_id'])
if document is None:
return None
document = UnveillanceDocument(_id=document['_id'])
inflate={
'doc_id' : document._id,
'queue' : UUID
}
del query['_id']
if 'task_path' not in query.keys() and 'task_queue' not in query.keys():
document.reset()
inflate.update({
'task_path' : "Documents.evaluate_document.evaluateDocument"
})
else:
inflate.update(query)
if 'task_queue' in inflate.keys():
inflate.update({
'task_path' : inflate['task_queue'][0],
'task_queue' : inflate['task_queue']
})
else:
inflate.update({
'no_continue' : True
})
uv_task = UnveillanceTask(inflate=inflate)
uv_task.run()
return uv_task.emit()
def do_cluster(self, request): """ request must be inflated with must !missing asset.tags.file_metadata/key_words/topics, etc. """ args = parseRequestEntity(request.query) if len(args.keys()) == 0: return None for required in ['task_path', 'documents']: if required not in args.keys(): return None cluster = UnveillanceCluster(inflate=args) try: return cluster.communicate() except Exception as e: if DEBUG: print e return None
def runTask(self, handler): try: args = parseRequestEntity(handler.request.body) except AttributeError as e: if DEBUG: print "No body?\n%s" % e return None uv_task = None if len(args.keys()) == 1 and '_id' in args.keys(): uv_task = UnveillanceTask(_id=args['_id']) else: # TODO: XXX: IF REFERER IS LOCALHOST ONLY (and other auth TBD)! if 'task_path' in args.keys(): args['queue'] = UUID uv_task = UnveillanceTask(inflate=args) if uv_task is None: return None uv_task.run() return uv_task.emit()
def post(self):
res = Result()
res.result = 412
query = parseRequestEntity(self.request.body)
if query is None or len(query.keys()) != 1 or '_id' not in query.keys():
self.set_status(res.result)
self.finish(res.emit())
return
r = requests.post("%stask/" % buildServerURL(),
data={ '_id' : query['_id'] }, verify=False)
try:
res.data = json.loads(r.content)['data']
res.result = 200
except Exception as e:
if DEBUG: print e
self.set_status(res.result)
self.finish(res.emit())
def do_documents(self, request):
args = parseRequestEntity(request.query)
if len(args.keys()) in [1, 2, 3]:
doc_type = None
try:
doc_type = args['doc_type']
except KeyError as e: pass
media_id = None
try:
media_id = args['media_id']
del args['media_id']
except KeyError as e: pass
if '_id' in args.keys():
return self.get(_id=args['_id'], els_doc_root=doc_type, parent=media_id)
elif '_ids' in args.keys():
return self.query({"ids" : {"values" : args['_ids']}}, doc_type=doc_type, exclude_fields=False)
return self.do_list(request)
def post(self):
res = Result()
res.result = 412
query = parseRequestEntity(self.request.body)
if query is None or len(
query.keys()) != 1 or '_id' not in query.keys():
self.set_status(res.result)
self.finish(res.emit())
return
r = requests.post("%stask/" % buildServerURL(),
data={'_id': query['_id']},
verify=False)
try:
res.data = json.loads(r.content)['data']
res.result = 200
except Exception as e:
if DEBUG: print e
self.set_status(res.result)
self.finish(res.emit())
def do_open_drive_file(self, handler):
if DEBUG: print "opening this drive file in unveillance annex"
status = self.do_get_status(handler)
if status not in [2, 3]:
if DEBUG: print "NO-ACCESS TO THIS METHOD (\"do_open_drive_file\")"
return None
files = None
self.initDriveClient(handler)
for _id in parseRequestEntity(handler.request.query)['_ids']:
_id = urllib.unquote(_id).replace("'", "")[1:]
file_name = self.drive_client.getFileName(_id)
if file_name is None: return None
url = "%s/documents/?file_name=%s" % (buildServerURL(), file_name)
entry = None
handled_file = None
if DEBUG: print url
# look up the file in annex. (annex/documents/?file_name=file)
# if this file exists in annex, return its _id for opening in-app
try:
entry = json.loads(requests.get(
url, verify=False).content)['data']['documents'][0]
except Exception as e:
if DEBUG: print "COULD NOT GET ENTRY:\n%s" % e
if entry is not None:
if DEBUG: print type(entry['_id'])
handled_file = {'_id': entry['_id']}
else:
if status != 3:
if DEBUG:
print "** at this point, we would process file if you were admin"
print "** but you are not admin."
return None
entry = self.drive_client.download(_id,
save_as=file_name,
save=False,
return_content=True)
if entry is not None:
self.addToNetcatQueue({
'file': entry[0],
'save_as': entry[1],
'importer_source': "google_drive"
})
handled_file = {'file_name': entry[1]}
if handled_file is not None:
if files is None: files = []
files.append(handled_file)
return files
def do_list(self, request, query=None):
count_only = False
limit = None
cast_as = None
sort = None
doc_type = "uv_document"
exclude_fields = True
args = parseRequestEntity(request.query)
if DEBUG: print "\n\nARGS:\n%s\n\n" % args
try:
doc_type = args['doc_type']
del args['doc_type']
except KeyError as e: pass
try:
exclude_fields = not args['get_all']
print exclude_fields
del args['get_all']
except KeyError as e: pass
try:
sort = args['sort']
del args['sort']
except KeyError as e: pass
if query is None:
try:
query = deepcopy(QUERY_DEFAULTS[doc_type.upper()])
except Exception as e:
if DEBUG: print "could not find default query for %s" % doc_type.upper()
query = deepcopy(QUERY_DEFAULTS['MATCH_ALL'])
if len(args.keys()) > 0:
# pull out variables that don't go in search
try:
count_only = args['count']
del args['count']
except KeyError as e: pass
try:
limit = args['limit']
del args['limit']
except KeyError as e: pass
try:
cast_as = args['cast_as']
del args['cast_as']
except KeyError as e: pass
if len(args.keys()) > 0:
# extend out top-level query
musts = []
for a in args.keys():
must = None
if a in QUERY_KEYS['match']:
must = {
"match" : { "%s.%s" % (doc_type, a) : args[a] }
}
elif a in QUERY_KEYS['filter_terms']:
must = {
"constant_score" : {
"filter" : {
"terms" : {
"%s.%s" % (doc_type, a) : args[a] if type(args[a]) is list else [args[a]]
}
}
}
}
elif a in QUERY_KEYS['filter_ids']:
must = {
"ids" : {
"type" : doc_type,
"values" : args[a] if type(args[a]) is list else [args[a]]
}
}
musts = [must]
del args[a]
break
if must is not None:
del args[a]
musts.append(must)
if len(musts) > 0:
if 'match_all' in query.keys():
del query['match_all']
if musts[0].keys()[0] == "ids":
query = musts[0]
else:
if 'bool' not in query.keys():
query['bool'] = { "must" : [] }
if 'must' not in query['bool'].keys():
query['bool']['must'] = []
for must in musts: query['bool']['must'].append(must)
if len(args.keys()) > 0:
# this becomes a filtered query
query = {
"filtered" : {
"query" : query,
"filter" : {}
}
}
filters = []
for a in args.keys():
filter = None
if 'term' in QUERY_KEYS and a in QUERY_KEYS['term']:
filter = { "term": { "%s.%s" % (doc_type, a) : args[a] }}
elif 'range' in QUERY_KEYS and a in QUERY_KEYS['range']:
try:
day = datetime.date.fromtimestamp(args[a]/1000)
except Exception as e:
print "TIME ERROR: %s" % e
continue
if "upper" in args.keys():
lte = datetime.date.fromtimestamp(args['upper']/1000)
gte = datetime.date.fromtimestamp(args[a]/1000)
else:
lte = datetime.date(day.year, day.month, day.day + 1)
gte = datetime.date(day.year, day.month, day.day)
filter = {
"range" : {
"%s.%s" % (doc_type, a) : {
"gte" : format(mktime(gte.timetuple()) * 1000, '0.0f'),
"lte" : format(mktime(lte.timetuple()) * 1000, '0.0f')
}
}
}
elif 'geo_distance' in QUERY_KEYS and a in QUERY_KEYS['geo_distance']:
if "radius" not in args.keys():
radius = 3
else:
radius = args['radius']
filter = {
"geo_distance" : {
"distance" : "%dmi" % radius,
"%s.%s" % (doc_type, a) : args[a]
}
}
if filter is not None:
filters.append(filter)
if len(filters) > 1:
query['filtered']['filter']['and'] = filters
else:
if len(filters) == 1:
try:
query['filtered']['filter'] = filters[0]
except Exception as e:
print "COULD NOT BUILD QUERY: %s" % e
return None
return self.query(query, doc_type=doc_type if doc_type != "uv_document" else None,
sort=sort, count_only=count_only, cast_as=cast_as, exclude_fields=exclude_fields)