def get_file_score(srl, **kwargs):
"""
Get the score of the latest service run for a given file.
Variables:
srl => A resource locator for the file (SHA256)
Arguments:
None
Data Block:
None
API call example:
/api/v3/file/score/123456...654321/
Result example:
{"file_info": {}, # File info Block
"result_keys": [<keys>] # List of keys used to compute the score
"score": 0} # Latest score for the file
"""
user = kwargs['user']
file_obj = STORAGE.get_file(srl)
if not file_obj:
return make_api_response([], "This file does not exists", 404)
args = [
("group", "on"),
("group.field", "response.service_name"),
("group.format", "simple"),
("fl", "result.score,_yz_rk"),
("sort", "created desc"),
("rows", "100")
]
if user and Classification.is_accessible(user['classification'], file_obj['classification']):
score = 0
keys = []
res = STORAGE.direct_search("result", "_yz_rk:%s*" % srl, args,
__access_control__=user["access_control"])
docs = res['grouped']['response.service_name']['doclist']['docs']
for d in docs:
score += d['result.score']
keys.append(d["_yz_rk"])
return make_api_response({"file_info": file_obj, "score": score, "result_keys": keys})
else:
return make_api_response([], "You are not allowed to view this file", 403)
def get_file_results_for_service(srl, service, **kwargs):
"""
Get the all the file results of a specific file and a specific query.
Variables:
srl => A resource locator for the file (SHA256)
Arguments:
all => if all argument is present, it will return all versions
NOTE: Max to 100 results...
Data Block:
None
API call example:
/api/v3/file/result/123456...654321/service_name/
Result example:
{"file_info": {}, # File info Block
"results": {}} # Full result list for the service
"""
user = kwargs['user']
file_obj = STORAGE.get_file(srl)
args = [("fl", "_yz_rk"),
("sort", "created desc")]
if "all" in request.args:
args.append(("rows", "100"))
else:
args.append(("rows", "1"))
if not file_obj:
return make_api_response([], "This file does not exists", 404)
if user and Classification.is_accessible(user['classification'], file_obj['classification']):
res = STORAGE.direct_search("result", "_yz_rk:%s.%s*" % (srl, service), args,
__access_control__=user["access_control"])['response']['docs']
keys = [k["_yz_rk"] for k in res]
results = []
for r in STORAGE.get_results(keys):
result = format_result(user['classification'], r, file_obj['classification'])
if result:
results.append(result)
return make_api_response({"file_info": file_obj, "results": results})
else:
return make_api_response([], "You are not allowed to view this file", 403)
def inspect_search(bucket, **kwargs):
"""
Inspect a search query to find out how much result items are
going to be returned.
* Uses Apache Solr search language.
Variables:
bucket => Buckets to be used to stream the search query from
Arguments:
q => Query to search for
Optional Arguments:
fq => Filter queries to be applied after the query
Data Block:
None
Result example:
{ count: 0 } # number of items return by the query
"""
user = kwargs['user']
if bucket not in STORAGE.INDEXED_BUCKET_LIST and bucket not in STORAGE.ADMIN_INDEXED_BUCKET_LIST:
return make_api_response({}, "Bucket '%s' does not exists." % bucket, 404)
if bucket not in ACCEPTABLE_BUCKETS:
return make_api_response("", "You're not allowed to query bucket %s." % bucket, 403)
query = request.args.get('query', None) or request.args.get('q', None)
if not query:
return make_api_response({"success": False}, "Please specify a query...", 406)
args = [('fq', x) for x in request.args.getlist('fq')]
args.append(('rows', "0"))
# noinspection PyProtectedMember
result = STORAGE.direct_search(bucket, query, args, __access_control__=user['access_control'])
return make_api_response({"count": result.get('response', {}).get("numFound", 0)})
def advanced_search(bucket, **kwargs):
"""
This is a search API that has not been simplified and can leverage the full
power of SOLR searches.
You should only use this API if you know what you are doing
Variables:
None
Arguments:
q => The query you are trying to make
Optional Arguments:
*Any arguments the SORL can take in*
Data Block:
None
Result example:
<<RAW SOLR API OUTPUT>>
"""
if bucket not in STORAGE.INDEXED_BUCKET_LIST and bucket not in STORAGE.ADMIN_INDEXED_BUCKET_LIST:
return make_api_response({}, "Bucket '%s' does not exists." % bucket, 404)
user = kwargs['user']
query = request.args.get('q', "*")
df = request.args.get('df', "text")
if bucket in ACCESS_CONTROL_BUCKETS:
fq = user['access_control']
else:
fq = None
args = []
for k in request.args:
args.extend([(k, v) for v in request.args.getlist(k)])
return make_api_response(STORAGE.direct_search(bucket, query, args, df=df, __access_control__=fq))
def run_query(bucket):
return STORAGE.direct_search(bucket,
"__expiry_ts__:[NOW/DAY TO NOW/DAY-2DAY]",
args=[
("rows", "0"), ("timeAllowed", "500")
])['response']['numFound'] == 0
def add_signature(**kwargs):
"""
Add a signature to the system and assigns it a new ID
WARNING: If two person call this method at exactly the
same time, they might get the same ID.
Variables:
None
Arguments:
None
Data Block (REQUIRED): # Signature block
{"name": "sig_name", # Signature name
"tags": ["PECheck"], # Signature tags
"comments": [""], # Signature comments lines
"meta": { # Meta fields ( **kwargs )
"id": "SID", # Mandatory ID field
"rule_version": 1 }, # Mandatory Revision field
"type": "rule", # Rule type (rule, private rule ...)
"strings": ['$ = "a"'], # Rule string section (LIST)
"condition": ["1 of them"]} # Rule condition section (LIST)
Result example:
{"success": true, #If saving the rule was a success or not
"sid": "0000000000", #SID that the rule was assigned
"rev": 2 } #Revision number at which the rule was saved.
"""
user = kwargs['user']
new_id = STORAGE.get_last_signature_id(ORGANISATION) + 1
new_rev = 1
data = request.json
if not Classification.is_accessible(user['classification'], data['meta'].get('classification',
Classification.UNRESTRICTED)):
return make_api_response("", "You are not allowed to add a signature with "
"higher classification than yours", 403)
if not user['is_admin'] and "global" in data['type']:
return make_api_response("", "Only admins are allowed to add global signatures.", 403)
sid = "%s_%06d" % (data['meta']['organisation'], new_id)
data['meta']['id'] = sid
data['meta']['rule_version'] = new_rev
data['meta']['creation_date'] = datetime.date.today().isoformat()
data['meta']['last_saved_by'] = user['uname']
key = "%sr.%s" % (data['meta']['id'], data['meta']['rule_version'])
yara_version = data['meta'].get('yara_version', None)
data['depends'], data['modules'] = \
YARA_PARSER.parse_dependencies(data['condition'], YARA_PARSER.YARA_MODULES.get(yara_version, None))
res = YARA_PARSER.validate_rule(data)
if res['valid']:
query = "name:{name} AND NOT _yz_rk:{sid}*"
other = STORAGE.direct_search(
'signature', query.format(name=data['name'], sid=sid),
args=[('fl', '_yz_rk'), ('rows', '0')],
)
if other.get('response', {}).get('numFound', 0) > 0:
return make_api_response(
{"success": False},
"A signature with that name already exists",
403
)
data['warning'] = res.get('warning', None)
STORAGE.save_signature(key, data)
return make_api_response({"success": True, "sid": data['meta']['id'], "rev": data['meta']['rule_version']})
else:
return make_api_response({"success": False}, res, 403)
