def do_reindex(self, handler): req = parseRequestEntity(handler.request.body) if req is None: return None if '_id' not in req.keys(): return None ''' we might have to replace parts of the request with secret values. ''' if 'task_path' in req.keys(): req_keys = list( chain.from_iterable( [k.keys() for k in MIME_TYPE_TASK_REQUIREMENTS])) if req['task_path'] in req_keys: try: req_field = [ k for k in MIME_TYPE_TASK_REQUIREMENTS if k.keys()[0] == req['task_path'] ][0][req['task_path']] print req_field except Exception as e: if DEBUG: print "Could not resolve required task info. %s" % e for repl in req_field.iteritems(): if repl[0] in req.keys(): handler.request.body = handler.request.body.replace( repl[1], getSecrets(repl[1])) return self.passToAnnex(handler)
def do_reindex(self, handler): req = parseRequestEntity(handler.request.body) if req is None: return None if '_id' not in req.keys(): return None ''' we might have to replace parts of the request with secret values. ''' if 'task_path' in req.keys(): req_keys = list(chain.from_iterable( [k.keys() for k in TASK_REQUIREMENTS])) if req['task_path'] in req_keys: try: req_field = [k for k in TASK_REQUIREMENTS if k.keys()[0] == req['task_path']][0][req['task_path']] print req_field except Exception as e: if DEBUG: print "Could not resolve required task info. %s" % e for repl in req_field.iteritems(): if repl[0] in req.keys(): handler.request.body = handler.request.body.replace( repl[1], getSecrets(repl[1])) return self.passToAnnex(handler)
def do_tasks(self, request): args = parseRequestEntity(request.query) if len(args.keys()) == 1 and '_id' in args.keys(): return self.get(_id=args['_id']) return self.do_list(request, query=deepcopy(QUERY_DEFAULTS['UV_TASK']))
def do_open_drive_file(self, handler): if DEBUG: print "opening this drive file in unveillance annex" status = self.do_get_status(handler) if status not in [2,3]: if DEBUG: print "NO-ACCESS TO THIS METHOD (\"do_open_drive_file\")" return None files = None self.initDriveClient(handler) for _id in parseRequestEntity(handler.request.query)['_ids']: _id = urllib.unquote(_id).replace("'", "")[1:] file_name = self.drive_client.getFileName(_id) if file_name is None: return None url = "%s/documents/?file_name=%s" % (buildServerURL(), file_name) entry = None handled_file = None if DEBUG: print url # look up the file in annex. (annex/documents/?file_name=file) # if this file exists in annex, return its _id for opening in-app try: entry = json.loads(requests.get( url, verify=False).content)['data']['documents'][0] except Exception as e: if DEBUG: print "COULD NOT GET ENTRY:\n%s" % e if entry is not None: if DEBUG: print type(entry['_id']) handled_file = { '_id' : entry['_id'] } else: if status != 3: if DEBUG: print "** at this point, we would process file if you were admin" print "** but you are not admin." return None entry = self.drive_client.download(_id, save_as=file_name, save=False, return_content=True) if entry is not None: self.addToNetcatQueue({ 'file' : entry[0], 'save_as' : entry[1], 'importer_source' : "google_drive" }) handled_file = { 'file_name' : entry[1] } if handled_file is not None: if files is None: files = [] files.append(handled_file) return files
def do_logout(self, handler): status = self.do_get_status(handler) if status not in [2, 3]: if DEBUG: print "CANNOT LOG IN USER, DON'T EVEN TRY (status %d)" % status return None credentials = parseRequestEntity(handler.request.body) if credentials is None: return None if DEBUG: print credentials return self.logoutUser(credentials, handler)
def do_create_new_user(self, handler): status = self.do_get_status(handler) if status != 3: return None new_user = parseRequestEntity(handler.request.body) if new_user is None: return None if DEBUG: print new_user n_set = sorted(['nu_username', 'nu_password']) if sorted(list(set(n_set) & set(new_user.keys()))) != n_set: return None from Utils.funcs import createNewUser return createNewUser(new_user['nu_username'], new_user['nu_password'])
def do_login(self, handler): status = self.do_get_status(handler) if status != 1: if DEBUG: print "CANNOT LOG IN USER, DON'T EVEN TRY (status %d)" % status return None credentials = parseRequestEntity(handler.request.body) if credentials is None: return None if DEBUG: print credentials try: return self.loginUser(credentials['username'], credentials['password'], handler) except KeyError as e: if DEBUG: print "CANNOT LOG IN USER: %s missing" % e return None
def do_reindex(self, request): print "DOING REINDEX" query = parseRequestEntity(request.query) if query is None: return None if '_id' not in query.keys(): return None document = self.get(_id=query['_id']) if document is None: return None document = UnveillanceDocument(_id=document['_id']) inflate={ 'doc_id' : document._id, 'queue' : UUID } del query['_id'] if 'task_path' not in query.keys() and 'task_queue' not in query.keys(): document.reset() inflate.update({ 'task_path' : "Documents.evaluate_document.evaluateDocument" }) else: inflate.update(query) if 'task_queue' in inflate.keys(): inflate.update({ 'task_path' : inflate['task_queue'][0], 'task_queue' : inflate['task_queue'] }) else: inflate.update({ 'no_continue' : True }) uv_task = UnveillanceTask(inflate=inflate) uv_task.run() return uv_task.emit()
def do_cluster(self, request): """ request must be inflated with must !missing asset.tags.file_metadata/key_words/topics, etc. """ args = parseRequestEntity(request.query) if len(args.keys()) == 0: return None for required in ['task_path', 'documents']: if required not in args.keys(): return None cluster = UnveillanceCluster(inflate=args) try: return cluster.communicate() except Exception as e: if DEBUG: print e return None
def runTask(self, handler): try: args = parseRequestEntity(handler.request.body) except AttributeError as e: if DEBUG: print "No body?\n%s" % e return None uv_task = None if len(args.keys()) == 1 and '_id' in args.keys(): uv_task = UnveillanceTask(_id=args['_id']) else: # TODO: XXX: IF REFERER IS LOCALHOST ONLY (and other auth TBD)! if 'task_path' in args.keys(): args['queue'] = UUID uv_task = UnveillanceTask(inflate=args) if uv_task is None: return None uv_task.run() return uv_task.emit()
def post(self): res = Result() res.result = 412 query = parseRequestEntity(self.request.body) if query is None or len(query.keys()) != 1 or '_id' not in query.keys(): self.set_status(res.result) self.finish(res.emit()) return r = requests.post("%stask/" % buildServerURL(), data={ '_id' : query['_id'] }, verify=False) try: res.data = json.loads(r.content)['data'] res.result = 200 except Exception as e: if DEBUG: print e self.set_status(res.result) self.finish(res.emit())
def do_documents(self, request): args = parseRequestEntity(request.query) if len(args.keys()) in [1, 2, 3]: doc_type = None try: doc_type = args['doc_type'] except KeyError as e: pass media_id = None try: media_id = args['media_id'] del args['media_id'] except KeyError as e: pass if '_id' in args.keys(): return self.get(_id=args['_id'], els_doc_root=doc_type, parent=media_id) elif '_ids' in args.keys(): return self.query({"ids" : {"values" : args['_ids']}}, doc_type=doc_type, exclude_fields=False) return self.do_list(request)
def post(self): res = Result() res.result = 412 query = parseRequestEntity(self.request.body) if query is None or len( query.keys()) != 1 or '_id' not in query.keys(): self.set_status(res.result) self.finish(res.emit()) return r = requests.post("%stask/" % buildServerURL(), data={'_id': query['_id']}, verify=False) try: res.data = json.loads(r.content)['data'] res.result = 200 except Exception as e: if DEBUG: print e self.set_status(res.result) self.finish(res.emit())
def do_open_drive_file(self, handler): if DEBUG: print "opening this drive file in unveillance annex" status = self.do_get_status(handler) if status not in [2, 3]: if DEBUG: print "NO-ACCESS TO THIS METHOD (\"do_open_drive_file\")" return None files = None self.initDriveClient(handler) for _id in parseRequestEntity(handler.request.query)['_ids']: _id = urllib.unquote(_id).replace("'", "")[1:] file_name = self.drive_client.getFileName(_id) if file_name is None: return None url = "%s/documents/?file_name=%s" % (buildServerURL(), file_name) entry = None handled_file = None if DEBUG: print url # look up the file in annex. (annex/documents/?file_name=file) # if this file exists in annex, return its _id for opening in-app try: entry = json.loads(requests.get( url, verify=False).content)['data']['documents'][0] except Exception as e: if DEBUG: print "COULD NOT GET ENTRY:\n%s" % e if entry is not None: if DEBUG: print type(entry['_id']) handled_file = {'_id': entry['_id']} else: if status != 3: if DEBUG: print "** at this point, we would process file if you were admin" print "** but you are not admin." return None entry = self.drive_client.download(_id, save_as=file_name, save=False, return_content=True) if entry is not None: self.addToNetcatQueue({ 'file': entry[0], 'save_as': entry[1], 'importer_source': "google_drive" }) handled_file = {'file_name': entry[1]} if handled_file is not None: if files is None: files = [] files.append(handled_file) return files
def do_list(self, request, query=None): count_only = False limit = None cast_as = None sort = None doc_type = "uv_document" exclude_fields = True args = parseRequestEntity(request.query) if DEBUG: print "\n\nARGS:\n%s\n\n" % args try: doc_type = args['doc_type'] del args['doc_type'] except KeyError as e: pass try: exclude_fields = not args['get_all'] print exclude_fields del args['get_all'] except KeyError as e: pass try: sort = args['sort'] del args['sort'] except KeyError as e: pass if query is None: try: query = deepcopy(QUERY_DEFAULTS[doc_type.upper()]) except Exception as e: if DEBUG: print "could not find default query for %s" % doc_type.upper() query = deepcopy(QUERY_DEFAULTS['MATCH_ALL']) if len(args.keys()) > 0: # pull out variables that don't go in search try: count_only = args['count'] del args['count'] except KeyError as e: pass try: limit = args['limit'] del args['limit'] except KeyError as e: pass try: cast_as = args['cast_as'] del args['cast_as'] except KeyError as e: pass if len(args.keys()) > 0: # extend out top-level query musts = [] for a in args.keys(): must = None if a in QUERY_KEYS['match']: must = { "match" : { "%s.%s" % (doc_type, a) : args[a] } } elif a in QUERY_KEYS['filter_terms']: must = { "constant_score" : { "filter" : { "terms" : { "%s.%s" % (doc_type, a) : args[a] if type(args[a]) is list else [args[a]] } } } } elif a in QUERY_KEYS['filter_ids']: must = { "ids" : { "type" : doc_type, "values" : args[a] if type(args[a]) is list else [args[a]] } } musts = [must] del args[a] break if must is not None: del args[a] musts.append(must) if len(musts) > 0: if 'match_all' in query.keys(): del query['match_all'] if musts[0].keys()[0] == "ids": query = musts[0] else: if 'bool' not in query.keys(): query['bool'] = { "must" : [] } if 'must' not in query['bool'].keys(): query['bool']['must'] = [] for must in musts: query['bool']['must'].append(must) if len(args.keys()) > 0: # this becomes a filtered query query = { "filtered" : { "query" : query, "filter" : {} } } filters = [] for a in args.keys(): filter = None if 'term' in QUERY_KEYS and a in QUERY_KEYS['term']: filter = { "term": { "%s.%s" % (doc_type, a) : args[a] }} elif 'range' in QUERY_KEYS and a in QUERY_KEYS['range']: try: day = datetime.date.fromtimestamp(args[a]/1000) except Exception as e: print "TIME ERROR: %s" % e continue if "upper" in args.keys(): lte = datetime.date.fromtimestamp(args['upper']/1000) gte = datetime.date.fromtimestamp(args[a]/1000) else: lte = datetime.date(day.year, day.month, day.day + 1) gte = datetime.date(day.year, day.month, day.day) filter = { "range" : { "%s.%s" % (doc_type, a) : { "gte" : format(mktime(gte.timetuple()) * 1000, '0.0f'), "lte" : format(mktime(lte.timetuple()) * 1000, '0.0f') } } } elif 'geo_distance' in QUERY_KEYS and a in QUERY_KEYS['geo_distance']: if "radius" not in args.keys(): radius = 3 else: radius = args['radius'] filter = { "geo_distance" : { "distance" : "%dmi" % radius, "%s.%s" % (doc_type, a) : args[a] } } if filter is not None: filters.append(filter) if len(filters) > 1: query['filtered']['filter']['and'] = filters else: if len(filters) == 1: try: query['filtered']['filter'] = filters[0] except Exception as e: print "COULD NOT BUILD QUERY: %s" % e return None return self.query(query, doc_type=doc_type if doc_type != "uv_document" else None, sort=sort, count_only=count_only, cast_as=cast_as, exclude_fields=exclude_fields)