def get_file_score(srl, **kwargs): """ Get the score of the latest service run for a given file. Variables: srl => A resource locator for the file (SHA256) Arguments: None Data Block: None API call example: /api/v3/file/score/123456...654321/ Result example: {"file_info": {}, # File info Block "result_keys": [<keys>] # List of keys used to compute the score "score": 0} # Latest score for the file """ user = kwargs['user'] file_obj = STORAGE.get_file(srl) if not file_obj: return make_api_response([], "This file does not exists", 404) args = [ ("group", "on"), ("group.field", "response.service_name"), ("group.format", "simple"), ("fl", "result.score,_yz_rk"), ("sort", "created desc"), ("rows", "100") ] if user and Classification.is_accessible(user['classification'], file_obj['classification']): score = 0 keys = [] res = STORAGE.direct_search("result", "_yz_rk:%s*" % srl, args, __access_control__=user["access_control"]) docs = res['grouped']['response.service_name']['doclist']['docs'] for d in docs: score += d['result.score'] keys.append(d["_yz_rk"]) return make_api_response({"file_info": file_obj, "score": score, "result_keys": keys}) else: return make_api_response([], "You are not allowed to view this file", 403)
def get_file_results_for_service(srl, service, **kwargs): """ Get the all the file results of a specific file and a specific query. Variables: srl => A resource locator for the file (SHA256) Arguments: all => if all argument is present, it will return all versions NOTE: Max to 100 results... Data Block: None API call example: /api/v3/file/result/123456...654321/service_name/ Result example: {"file_info": {}, # File info Block "results": {}} # Full result list for the service """ user = kwargs['user'] file_obj = STORAGE.get_file(srl) args = [("fl", "_yz_rk"), ("sort", "created desc")] if "all" in request.args: args.append(("rows", "100")) else: args.append(("rows", "1")) if not file_obj: return make_api_response([], "This file does not exists", 404) if user and Classification.is_accessible(user['classification'], file_obj['classification']): res = STORAGE.direct_search("result", "_yz_rk:%s.%s*" % (srl, service), args, __access_control__=user["access_control"])['response']['docs'] keys = [k["_yz_rk"] for k in res] results = [] for r in STORAGE.get_results(keys): result = format_result(user['classification'], r, file_obj['classification']) if result: results.append(result) return make_api_response({"file_info": file_obj, "results": results}) else: return make_api_response([], "You are not allowed to view this file", 403)
def inspect_search(bucket, **kwargs): """ Inspect a search query to find out how much result items are going to be returned. * Uses Apache Solr search language. Variables: bucket => Buckets to be used to stream the search query from Arguments: q => Query to search for Optional Arguments: fq => Filter queries to be applied after the query Data Block: None Result example: { count: 0 } # number of items return by the query """ user = kwargs['user'] if bucket not in STORAGE.INDEXED_BUCKET_LIST and bucket not in STORAGE.ADMIN_INDEXED_BUCKET_LIST: return make_api_response({}, "Bucket '%s' does not exists." % bucket, 404) if bucket not in ACCEPTABLE_BUCKETS: return make_api_response("", "You're not allowed to query bucket %s." % bucket, 403) query = request.args.get('query', None) or request.args.get('q', None) if not query: return make_api_response({"success": False}, "Please specify a query...", 406) args = [('fq', x) for x in request.args.getlist('fq')] args.append(('rows', "0")) # noinspection PyProtectedMember result = STORAGE.direct_search(bucket, query, args, __access_control__=user['access_control']) return make_api_response({"count": result.get('response', {}).get("numFound", 0)})
def advanced_search(bucket, **kwargs): """ This is a search API that has not been simplified and can leverage the full power of SOLR searches. You should only use this API if you know what you are doing Variables: None Arguments: q => The query you are trying to make Optional Arguments: *Any arguments the SORL can take in* Data Block: None Result example: <<RAW SOLR API OUTPUT>> """ if bucket not in STORAGE.INDEXED_BUCKET_LIST and bucket not in STORAGE.ADMIN_INDEXED_BUCKET_LIST: return make_api_response({}, "Bucket '%s' does not exists." % bucket, 404) user = kwargs['user'] query = request.args.get('q', "*") df = request.args.get('df', "text") if bucket in ACCESS_CONTROL_BUCKETS: fq = user['access_control'] else: fq = None args = [] for k in request.args: args.extend([(k, v) for v in request.args.getlist(k)]) return make_api_response(STORAGE.direct_search(bucket, query, args, df=df, __access_control__=fq))
def run_query(bucket): return STORAGE.direct_search(bucket, "__expiry_ts__:[NOW/DAY TO NOW/DAY-2DAY]", args=[ ("rows", "0"), ("timeAllowed", "500") ])['response']['numFound'] == 0
def add_signature(**kwargs): """ Add a signature to the system and assigns it a new ID WARNING: If two person call this method at exactly the same time, they might get the same ID. Variables: None Arguments: None Data Block (REQUIRED): # Signature block {"name": "sig_name", # Signature name "tags": ["PECheck"], # Signature tags "comments": [""], # Signature comments lines "meta": { # Meta fields ( **kwargs ) "id": "SID", # Mandatory ID field "rule_version": 1 }, # Mandatory Revision field "type": "rule", # Rule type (rule, private rule ...) "strings": ['$ = "a"'], # Rule string section (LIST) "condition": ["1 of them"]} # Rule condition section (LIST) Result example: {"success": true, #If saving the rule was a success or not "sid": "0000000000", #SID that the rule was assigned "rev": 2 } #Revision number at which the rule was saved. """ user = kwargs['user'] new_id = STORAGE.get_last_signature_id(ORGANISATION) + 1 new_rev = 1 data = request.json if not Classification.is_accessible(user['classification'], data['meta'].get('classification', Classification.UNRESTRICTED)): return make_api_response("", "You are not allowed to add a signature with " "higher classification than yours", 403) if not user['is_admin'] and "global" in data['type']: return make_api_response("", "Only admins are allowed to add global signatures.", 403) sid = "%s_%06d" % (data['meta']['organisation'], new_id) data['meta']['id'] = sid data['meta']['rule_version'] = new_rev data['meta']['creation_date'] = datetime.date.today().isoformat() data['meta']['last_saved_by'] = user['uname'] key = "%sr.%s" % (data['meta']['id'], data['meta']['rule_version']) yara_version = data['meta'].get('yara_version', None) data['depends'], data['modules'] = \ YARA_PARSER.parse_dependencies(data['condition'], YARA_PARSER.YARA_MODULES.get(yara_version, None)) res = YARA_PARSER.validate_rule(data) if res['valid']: query = "name:{name} AND NOT _yz_rk:{sid}*" other = STORAGE.direct_search( 'signature', query.format(name=data['name'], sid=sid), args=[('fl', '_yz_rk'), ('rows', '0')], ) if other.get('response', {}).get('numFound', 0) > 0: return make_api_response( {"success": False}, "A signature with that name already exists", 403 ) data['warning'] = res.get('warning', None) STORAGE.save_signature(key, data) return make_api_response({"success": True, "sid": data['meta']['id'], "rev": data['meta']['rule_version']}) else: return make_api_response({"success": False}, res, 403)