def get_results(keys): out = {} res = {} retry = 0 while keys and retry < max_retry: if retry: time.sleep(2**(retry - 7)) res.update( STORAGE.get_multiple_results(keys, Classification, as_obj=False)) keys = [x for x in keys if x not in res] retry += 1 results = {} for k, v in res.items(): file_info = data['file_infos'].get(k[:64], None) if file_info: v = format_result(user['classification'], v, file_info['classification']) if v: results[k] = v out["results"] = results out["missing_result_keys"] = keys return out
def get_file_submission_results(sid, sha256, **kwargs): """ Get the all the results and errors of a specific file for a specific Submission ID Variables: sid => Submission ID to get the result for sha256 => Resource locator to get the result for Arguments (POST only): extra_result_keys => List of extra result keys to get extra_error_keys => List of extra error keys to get Data Block: None Result example: {"errors": [], # List of error blocks "file_info": {}, # File information block (md5, ...) "results": [], # List of result blocks "tags": [] } # List of generated tags """ user = kwargs['user'] # Check if submission exist data = STORAGE.submission.get(sid, as_obj=False) if data is None: return make_api_response("", "Submission ID %s does not exists." % sid, 404) if data and user and Classification.is_accessible(user['classification'], data['classification']): # Prepare output output = { "file_info": {}, "results": [], "tags": {}, "errors": [], "attack_matrix": {}, 'heuristics': {}, "signatures": set() } # Extra keys - This is a live mode optimisation res_keys = data.get("results", []) err_keys = data.get("errors", []) if request.method == "POST" and request.json is not None and data[ 'state'] != "completed": extra_rkeys = request.json.get("extra_result_keys", []) extra_ekeys = request.json.get("extra_error_keys", []) # Load keys res_keys.extend(extra_rkeys) err_keys.extend(extra_ekeys) res_keys = list(set(res_keys)) err_keys = list(set(err_keys)) # Get File, results and errors temp_file = STORAGE.file.get(sha256, as_obj=False) if not temp_file: output['file_info']['sha256'] = sha256 output['signatures'] = list(output['signatures']) output['missing'] = True return make_api_response( output, "The file you are trying to view is missing from the system", 404) if not Classification.is_accessible(user['classification'], temp_file['classification']): return make_api_response( "", "You are not allowed to view the data of this file", 403) output['file_info'] = temp_file max_c12n = output['file_info']['classification'] temp_results = list( STORAGE.get_multiple_results( [x for x in res_keys if x.startswith(sha256)], cl_engine=Classification, as_obj=False).values()) results = [] for r in temp_results: r = format_result(user['classification'], r, temp_file['classification'], build_hierarchy=True) if r: max_c12n = Classification.max_classification( max_c12n, r['classification']) results.append(r) output['results'] = results try: output['errors'] = STORAGE.error.multiget( [x for x in err_keys if x.startswith(sha256)], as_obj=False, as_dictionary=False) except MultiKeyError as e: LOGGER.warning( f"Trying to get multiple errors but some are missing: {str(e.keys)}" ) output['errors'] = e.partial_output output['metadata'] = STORAGE.get_file_submission_meta( sha256, config.ui.statistics.submission, user["access_control"]) for res in output['results']: for sec in res['result']['sections']: h_type = "info" if sec.get('heuristic', False): # Get the heuristics data if sec['heuristic']['score'] < 100: h_type = "info" elif sec['heuristic']['score'] < 1000: h_type = "suspicious" else: h_type = "malicious" item = (sec['heuristic']['heur_id'], sec['heuristic']['name']) output['heuristics'].setdefault(h_type, []) if item not in output['heuristics'][h_type]: output['heuristics'][h_type].append(item) # Process Attack matrix for attack in sec['heuristic'].get('attack', []): attack_id = attack['attack_id'] for cat in attack['categories']: output['attack_matrix'].setdefault(cat, []) item = (attack_id, attack['pattern'], h_type) if item not in output['attack_matrix'][cat]: output['attack_matrix'][cat].append(item) # Process Signatures for signature in sec['heuristic'].get('signature', []): sig = (signature['name'], h_type) if sig not in output['signatures']: output['signatures'].add(sig) # Process tags for t in sec['tags']: output["tags"].setdefault(t['type'], {}) current_htype = output["tags"][t['type']].get( t['value'], None) if not current_htype: output["tags"][t['type']][t['value']] = h_type else: if current_htype == 'malicious' or h_type == 'malicious': output["tags"][t['type']][t['value']] = 'malicious' elif current_htype == 'suspicious' or h_type == 'suspicious': output["tags"][t['type']][ t['value']] = 'suspicious' else: output["tags"][t['type']][t['value']] = 'info' for t_type in output["tags"]: output["tags"][t_type] = [ (k, v) for k, v in output['tags'][t_type].items() ] output['signatures'] = list(output['signatures']) output['file_info']['classification'] = max_c12n return make_api_response(output) else: return make_api_response( "", "You are not allowed to view the data of this submission", 403)
def get_multiple_service_results(**kwargs): """ Get multiple result and error keys at the same time Variables: None Arguments: None Data Block: {"error": [], #List of error keys to lookup "result": [] #List of result keys to lookup } Result example: {"error": {}, #Dictionary of error object matching the keys "result": {} #Dictionary of result object matching the keys } """ user = kwargs['user'] data = request.json try: errors = STORAGE.error.multiget(data.get('error', []), as_dictionary=True, as_obj=False) except MultiKeyError as e: LOGGER.warning( f"Trying to get multiple errors but some are missing: {str(e.keys)}" ) errors = e.partial_output results = STORAGE.get_multiple_results(data.get('result', []), CLASSIFICATION, as_obj=False) try: file_infos = STORAGE.file.multiget(list( set([x[:64] for x in results.keys()])), as_dictionary=True, as_obj=False) except MultiKeyError as e: LOGGER.warning( f"Trying to get multiple files but some are missing: {str(e.keys)}" ) file_infos = e.partial_output for r_key in list(results.keys()): r_value = format_result(user['classification'], results[r_key], file_infos.get(r_key[:64], {}).get( 'classification', CLASSIFICATION.UNRESTRICTED), build_hierarchy=True) if not r_value: del results[r_key] else: results[r_key] = r_value out = {"error": errors, "result": results} return make_api_response(out)