def analyze_binary(self, md5sum, binary_file_stream):
if not self.submit_full_binaries:
raise AnalysisPermanentError(
message="NOT SUBMITTING FULL BINARIES")
response = self.tiscale_client.upload_file(
md5sum=md5sum, binary_file_stream=binary_file_stream)
if response.status_code == 200:
log.info(
"File {} uploaded successfully. Response status_code = {}".
format(md5sum, response.status_code))
return self.make_result(md5sum=md5sum, uploaded=True)
elif response.status_code == 403:
log.info("Quota reached. Response status_code: {}".format(
response.status_code))
raise AnalysisTemporaryError(
message="Quota reached, will retry in 1 hour",
retry_in=60 * 60)
else:
log.info("Unable to upload file. Response status_code: {}".format(
response.status_code))
raise AnalysisTemporaryError(
message="Uploading file failed, will try again in 30 min",
retry_in=30 * 60)
def analyze_binary(self, md5sum, binary_file_stream):
if not self.submit_full_binaries:
raise AnalysisPermanentError(
message="NOT SUBMITTING FULL BINARIES!")
log.info("Submitting FULL binary %s to VT for analysis" % md5sum)
try:
response = self.virustotal_analysis.submit_file(
resource_hash=md5sum, stream=binary_file_stream)
except VTAPIQUOTAREACHED as vte:
raise AnalysisTemporaryError(message="VTAPIQUOTAREACHED",
retry_in=15 * 60)
response_code = response.get("response_code", -1)
verbose_msg = response.get("verbose_msg", "")
# response_code == -2 or "scan request successfully queued" is the wait condition
if response_code == -2 or "Scan request successfully queued" in verbose_msg:
raise AnalysisTemporaryError(
message="VirusTotal report not yet ready -> %s" % verbose_msg,
retry_in=120)
elif response_code == 1:
scan_id = response.get("scan_id", None)
return self.make_result(scan_id=scan_id, result=response)
else:
raise AnalysisTemporaryError(message="Unknown error? % s" %
response,
retry_in=120)
def check_result_for(self, md5sum):
log.info("Submitting hash %s to RL for analysis" % md5sum)
try:
response = self.rl_analysis.get_report(resource_hash=md5sum)
except RLAPIQUOTAREACHED as rle:
log.info(traceback.format_exc())
raise AnalysisTemporaryError(message="Error: {}".format(str(rle)),
retry_in=15 * 60)
except Exception as err:
log.info(traceback.format_exc())
raise AnalysisTemporaryError(message="Error: {}".format(str(err)),
retry_in=15 * 60)
malware_presence = response["rl"]["malware_presence"]
status = malware_presence.get("status").upper()
if 'UNKNOWN' in status:
return AnalysisInProgress(retry_in=15 * 60)
# calculate if hash needs rescan
datetime_now = datetime.utcnow()
log.info("current time: %s" % datetime_now)
if self.days_rescan:
rescan_date = datetime_now - timedelta(days=int(self.days_rescan))
else:
rescan_date = None
log.info("rescan date: %s" % rescan_date)
last_seen_date_str = malware_presence.get('last_seen')
log.info("last seen str: %s" % last_seen_date_str)
last_seen_date = datetime.strptime(
last_seen_date_str,
"%Y-%m-%dT%H:%M:%S") if last_seen_date_str else None
log.info("last seen date: %s" % last_seen_date)
log.info(
"Binary %s has not been scanned since: %s - timenow: %s. Should do rescan if last seen is older than %s"
% (md5sum, last_seen_date, datetime_now, rescan_date))
if self.days_rescan and last_seen_date <= rescan_date:
log.info("HIT rescan date: Binary %s" % md5sum)
try:
self.rl_analysis.rescan_hash(md5sum)
except RLAPIQUOTAREACHED as rle:
log.info(traceback.format_exc())
raise AnalysisTemporaryError(message="Error: {}".format(
str(rle)),
retry_in=15 * 60)
except Exception as err:
log.info(traceback.format_exc())
raise AnalysisTemporaryError(
message="There was an error. Error: {}".format(str(err)),
retry_in=15 * 60)
return AnalysisInProgress(
message="Rescaning hash {}".format(md5sum), retry_in=60 * 60)
else:
return self.make_result(md5=md5sum, result=response)
def get_uuid(self, response):
try:
task_uuid = response.get('data', {}).get('task_uuid', None)
except AttributeError:
raise AnalysisTemporaryError(message="Invalid response from LastLine: %s" % response, retry_in=120)
else:
if not task_uuid:
raise AnalysisTemporaryError(message="No UUID for result: %s" % response, retry_in=120)
return task_uuid
def make_result(self, result=None, md5=None):
log.info("making result for md5 {0}".format(md5 if md5 else "None"))
try:
result = self.fortisandbox_analysis.get_report(
resource_hash=md5).json() if not result else result
except Exception as e:
raise AnalysisTemporaryError(message="API error: %s" % str(e),
retry_in=120)
else:
result = result.get('result', {})
data = result.get('data', {})
score = int(data.get('score'))
"""RISK_UNKNOWN = 0
RISK_CLEAN = 1
RISK_MALICIOUS = 2
RISK_LOW = 4
RISK_MEDIUM = 8
RISK_HIGH = 16"""
RISK_MATRIX = {0: 0, 1: 0, 4: 25, 8: 50, 16: 75, 2: 100}
score = RISK_MATRIX[score] if score in RISK_MATRIX else score
untrusted = int(data.get('untrusted', "0"))
if score == 0:
if (self.fortisandbox_trust_untrusted
and untrusted == 1) or untrusted == 0:
return AnalysisResult(message="Benign",
extended_message="",
link=str(data['rating']),
score=score)
else:
raise AnalysisTemporaryError(
message=
"Configured to not trust untrusted scans, retrying in 120 seconds",
retry_in=120)
else:
ratings = data.get("rating", [])
vids = data.get("vid", ['N/A'])
jids = data.get("jid", ['N/A'])
malware_names = data.get("malware_name", [])
report_string = "Fortisandbox Report for {0}:\n".format(md5)
link = "{0}/job-detail/?jid={1}".format(
self.host, str(jids[0]))
report_string += "Score: {0}\n".format(score)
report_string += "Malware Names: {0}\n".format(
",".join(malware_names))
report_string += "Malware Ratings: {0}\n".format(
",".join(ratings))
report_string += "Virus Ids: {0}\n".format(",".join(
[str(vid) for vid in vids]))
malware_result = "[{0}] FortiSandbox report for {1}".format(
score, md5)
return AnalysisResult(message=malware_result,
extended_message=report_string,
link=link,
score=score)
def submit_checkpoint(self, md5sum, file_stream):
"""
submit a file to the checkpoint api
returns a checkpoint submission status code
"""
headers = {
"Authorization": self.api_key,
}
payload = {
"request": {
"md5": md5sum,
"file_name": "CarbonBlack_{0}".format(md5sum)
}
}
json_input = StringIO(json.dumps(payload))
files = {
'file': ("CarbonBlack_{0}".format(md5sum), file_stream),
'request': ('request', json_input)
}
log.info(
"Submitting {0} to Checkpoint Threat Emulation".format(md5sum))
try:
status_code, content = self._call_checkpoint_api(
"POST",
"/tecloud/api/v1/file/upload",
headers=headers,
payload=None,
files=files)
except Exception as e:
log.exception(
"Exception while submitting MD5 %s to checkpoint: %s" %
(md5sum, e))
raise AnalysisTemporaryError(
"Exception while submitting to checkpoint: %s" % e)
else:
if status_code == 200:
return True
else:
time.sleep(5)
raise AnalysisTemporaryError(
"Received HTTP error code %d while submitting to checkpoint"
% status_code)
def create_result(self, sample_id, submission_id=None):
"""Create Carbon Black result for the given sample"""
LOGGER.debug(
"Creating result for sample with ID %u (submission_id=%s)",
sample_id, submission_id)
try:
# get sample
sample = self.rest_api.call("GET", "/rest/sample/%u" % (sample_id))
# get analyses
analyses = self.rest_api.call(
"GET", "/rest/analysis/sample/%u" % (sample_id))
except BaseException as exc:
LOGGER.debug(
"Error getting sample and analyses info for sample ID %u",
sample_id,
exc_info=True)
raise AnalysisTemporaryError(message="API error: %s" % (str(exc)))
if submission_id is not None:
# filter by submission ID
analyses = [
analysis for analysis in analyses
if analysis["analysis_submission_id"] == submission_id
]
# check if any error occurred
for analysis in analyses:
if analysis["analysis_result_code"] != 1:
LOGGER.warning(
"Analysis with ID %u of sample with ID %u failed with error code 0x%x: %s",
analysis["analysis_id"], sample_id,
analysis["analysis_result_code"],
analysis["analysis_result_str"])
# check sample VTI score
if (sample["sample_vti_score"] is None) or (
sample["sample_vti_score"] < self.vti_score_threshold):
message = "Benign"
else:
message = "Potential malware"
# Check for None in the score
if not sample["sample_vti_score"] and message == 'Benign':
sample["sample_vti_score"] = 0
if not sample["sample_vti_score"] and message == 'Potential malware':
sample["sample_vti_score"] = 100
LOGGER.debug(
"Analysis result of sample with ID %u created successfully (message=%s link=%s score=%u)",
sample_id, message, sample["sample_webif_url"],
sample["sample_vti_score"])
return AnalysisResult(
message=message,
extended_message="",
link=sample["sample_webif_url"],
score=sample["sample_vti_score"],
)
def make_result(self, scan_id, result=None, md5=None):
try:
result = self.virustotal_analysis.get_report(
scan_id) if not result else result
except Exception as e:
raise AnalysisTemporaryError(message="API error: %s" % str(e),
retry_in=120)
else:
total = int(result.get("total", 1))
positives = int(result.get("positives", 0))
score = int(float(positives) / float(total) * 100)
if score == 0:
return AnalysisResult(message="Benign",
extended_message="",
link=result['permalink'],
score=score)
else:
scans = result.get("scans", {})
detected_by = filter(
lambda (k, v): v.get('detected', False) is True,
scans.iteritems())
detected_by = map(
lambda (k, v): (k, v.get("result", "potential_malware")),
detected_by)
log.info("detected by = %s " % detected_by)
report_string = "VirusTotal Report:\n" + "\n".join(
[k + " :\t" + v for (k, v) in detected_by])
malware_result = "[%d / %d] VirusTotal report for %s" % (
positives, total, md5)
return AnalysisResult(message=malware_result,
extended_message=report_string,
link=result['permalink'],
score=score)
def analyze_binary(self, md5sum, binary_file_stream):
log.debug("%s: in analyze_binary" % md5sum)
d = binary_file_stream.read()
try:
start_analyze_time = time.time()
matches = self.yara_rules.match(data=d, timeout=60)
end_analyze_time = time.time()
log.debug("%s: Took %0.3f seconds to analyze the file" %
(md5sum, end_analyze_time - start_analyze_time))
except yara.TimeoutError:
raise AnalysisPermanentError(
message="Analysis timed out after 60 seconds")
except yara.Error:
raise AnalysisTemporaryError(message="Yara exception", retry_in=10)
else:
if matches:
score = self.getHighScore(matches)
return AnalysisResult(
message="Matched yara rules: %s" %
', '.join([match.rule for match in matches]),
extended_message="%s" %
', '.join([match.rule for match in matches]),
analysis_version=1,
score=score)
else:
return AnalysisResult(score=0)
def analyze_binary(self, md5sum, binary_file_stream):
log.info("Submitting {0} to FortiSandbox for analysis".format(
str(md5sum)))
try:
response = self.fortisandbox_analysis.submit_file(
resource_hash=md5sum, stream=binary_file_stream)
except BaseException as be:
log.error("EXCEPTION WHEN trying to submit binary: " + str(md5sum))
log.error(str(be))
log.error(traceback.format_exc())
raise AnalysisTemporaryError(message=str(be), retry_in=15 * 60)
result = response.json().get("result", {})
response_code = result.get("status", {}).get("message", None)
if response_code == "OK":
log.info("Sucessfully submitted {0} to FortiSandbox for scanning".
format(md5sum))
else:
if response_code == "INVALID_SESSION":
self.fortisandbox_analysis.invalidate_session()
raise AnalysisPermanentError(
message="FortiSandbox analysis failed -> %s" % response.json())
try:
response = self.fortisandbox_analysis.get_report(
resource_hash=md5sum)
log.debug("Fortinet report: " + str(response.json()))
result = response.json().get("result", {})
response_code = result.get("status", {}).get("message", None)
if response_code == "OK":
log.info("Got analysis report from Fortisandbox for %s" %
md5sum)
return self.make_result(md5=md5sum, result=response.json())
else:
log.info(
"No analysis report from Fortisandbox for %s, try again in 180 seconds"
% md5sum)
raise AnalysisTemporaryError(
message="FortiSandbox analysis failed -> %s" %
response_code,
retry_in=180)
except AnalysisTemporaryError as ate:
raise ate
except:
log.error("Fortisandbox Analysis failed , permanent!")
log.error(traceback.format_exc())
raise AnalysisPermanentError(
message="FortiSandbox Anlaysis failed -> %s" % response_code)
def submit_wildfire(self, md5sum, file_stream):
"""
submit a file to the wildfire api
returns a wildfire submission status code
"""
files = {'file': ('CarbonBlack_%s' % md5sum, file_stream)}
try:
status_code, content = self._call_wildfire_api("POST", "/publicapi/submit/file", files=files)
except Exception as e:
log.exception("Exception while submitting MD5 %s to WildFire: %s" % (md5sum, e))
raise AnalysisTemporaryError("Exception while submitting to WildFire: %s" % e)
else:
if status_code == 200:
return True
else:
raise AnalysisTemporaryError("Received HTTP error code %d while submitting to WildFire" % status_code)
def make_result(self, md5sum=None, result=None):
try:
result = self.rl_analysis.get_report(
md5sum) if not result else result
except Exception as err:
raise AnalysisTemporaryError(message="API error: %s" % str(err),
retry_in=360)
if result.get('code'):
raise AnalysisTemporaryError(
message=
'No results on A1000. Allow submit_full_binaries to get results on A1000',
retry_in=15 * 60)
log.info("Result for md5: %s" % md5sum)
result_link = urljoin(self.base_url, md5sum)
threat_score = int(result.get("threat_level"))
trust_factor = int(result.get("trust_factor"))
threat_name = result.get("treat_name")
score = SEVERITY[threat_score]
status = result.get("threat_status").upper()
if 'UNKNOWN' in status:
malware_result = """ReversingLabs report for md5: %s
Status: %s""" % (md5sum, status)
report_string = """Report string (test string)"""
return AnalysisResult(message=malware_result,
extended_message=report_string,
link=result_link,
score=0)
malware_result = """"ReversingLabs report for md5: %s.
Status: %s
Threat name: %s
Threat score: %s
Trust factor: %s""" % (md5sum, status, threat_name, threat_score,
trust_factor)
report_string = """Report string (test string)"""
return AnalysisResult(message=malware_result,
extended_message=report_string,
link=result_link,
score=score)
def make_result(self, task_uuid):
try:
result = self.lastline_analysis.get_result(task_uuid)
result = result.get('data', {})
except Exception as e:
raise AnalysisTemporaryError(message="API error: %s" % str(e), retry_in=120)
else:
if 'error' in result:
raise AnalysisTemporaryError(message=result['error'], retry_in=120)
score = int(result.get('score', 0))
if score == 0:
malware_result = "Benign"
else:
reasons = "; ".join(result.get('malicious_activity', []))
malware_result = "Potential malware: %s" % reasons
return AnalysisResult(message=malware_result, extended_message="",
link=re.sub("(?<!:)/{2,}", "/", "%s/%s" % (self.feed_link_prefix, task_uuid)),
score=score)
def analyze_binary(self, md5sum, binary_file_stream):
log.info("Submitting binary %s to LastLine" % md5sum)
try:
response = self.lastline_analysis.submit_file(binary_file_stream)
except AnalysisAPIError as e:
raise AnalysisTemporaryError(message="API error: %s" % str(e), retry_in=120)
task_uuid = self.get_uuid(response)
retries = 10
while retries:
sleep(10)
result = self.lastline_analysis.get_progress(task_uuid)
if result.get('data', {}).get('completed', 0) == 1:
return self.make_result(task_uuid)
retries -= 1
raise AnalysisTemporaryError(message="Maximum retries (10) exceeded submitting to LastLine", retry_in=120)
def check_result_for(self, md5sum):
try:
response = self.lastline_analysis.submit_file_hash(md5=md5sum)
except FileNotAvailableError as e:
# the file does not exist yet.
return None
except AnalysisAPIError as e:
raise AnalysisTemporaryError(message="API error: %s" % str(e), retry_in=120)
else:
task_uuid = self.get_uuid(response)
return self.make_result(task_uuid)
def make_result(self, md5sum, uploaded=False):
if not uploaded:
raise AnalysisTemporaryError(
message="Hash is not yet uploaded to TitaniumScale",
retry_in=30 * 60)
malware_result = "File {} uploaded to TitaniumScale, date: {}".format(
md5sum, datetime.utcnow())
return AnalysisResult(message=malware_result,
extended_message=malware_result,
link="Result should be stored in Splunk",
score=20)
def analyze_binary(self, md5sum, binary_file_stream):
self.submit_wildfire(md5sum, binary_file_stream)
retries = 15
while retries:
time.sleep(60)
result = self.check_result_for(md5sum)
if result:
return result
retries -= 1
raise AnalysisTemporaryError(message="Maximum retries (20) exceeded submitting to WildFire", retry_in=120)
def check_result_for(self, md5_hash):
LOGGER.debug("Checking result for md5 %s", md5_hash)
try:
result = self.rest_api.call("GET", "/rest/sample/md5/%s" % (md5_hash.lower()))
except BaseException as exc:
LOGGER.debug("Error while checking for md5 %s", md5_hash, exc_info=True)
raise AnalysisTemporaryError(message="API error: %s" % (str(exc)), retry_in=self.retry_wait_time)
LOGGER.debug("%u sample(s) found with md5 %s", len(result), md5_hash)
if len(result) == 0:
return None
else:
return self.create_result(result[0]["sample_id"])
def analyze_binary(self, md5sum, binary_file_stream):
if not self.submit_full_binaries:
raise AnalysisPermanentError(
message="NOT SUBMITTING FULL BINARIES")
log.info("Submitting FULL binary %s to ReversingLabs for analysis" %
md5sum)
try:
response = self.rl_analysis.submit_file(resource_hash=md5sum,
stream=binary_file_stream)
except RLAPIQUOTAREACHED:
raise AnalysisTemporaryError(message="RLAPIQUOTAREACHED",
retry_in=15 * 60)
if response.status_code == 200 or response.status_code == 201:
return self.check_result_for(md5sum=md5sum)
else:
raise AnalysisTemporaryError(message="Unknown error: %s" %
str(response),
retry_in=15 * 60)
def analyze_binary(self, md5sum, binary_file_stream):
self.submit_checkpoint(md5sum, binary_file_stream)
retries = 30
while retries:
time.sleep(10)
result = self.query_checkpoint(md5sum)
if result:
return result
retries -= 1
raise AnalysisTemporaryError(
message="Maximum retries (20) exceeded submitting to checkpoint",
retry_in=120)
def make_result(self, md5=None, result=None):
try:
result = self.rl_analysis.get_report(md5) if not result else result
except Exception as err:
log.info(traceback.format_exc())
raise AnalysisTemporaryError(message="API error: %s" % str(err),
retry_in=5 * 60)
log.info("Result for md5: %s" % md5)
result_link = "%s/uploads/?q=hash%%3A%s" % (
self.report_visualisation_url.rstrip("/"), md5)
log.info("Result link: %s" % result_link)
malware_presence = result['rl']['malware_presence']
status = malware_presence.get("status").upper()
threat_name = malware_presence.get("threat_name")
if threat_name is None:
threat_name = ""
threat_level = int(malware_presence.get("threat_level"))
trust_factor = int(malware_presence.get("trust_factor"))
score = SEVERITY[threat_level]
if "scanner_count" in malware_presence and "scanner_match" in malware_presence:
total_scanners = int(malware_presence.get("scanner_count"))
scanner_match = int(malware_presence.get("scanner_match"))
malware_result = """ReversingLabs report for md5: %s;
RL Status: %s %s;
RL Trust Factor: %s;
Threat Level: %s;
AV detection number: %s/%s;
""" % \
(md5, status, threat_name, trust_factor, threat_level, scanner_match, total_scanners)
else:
malware_result = """ReversingLabs report for md5: %s;
RL Status: %s %s;
RL Trust factor: %s;
Threat level: %s;
""" % \
(md5, status, threat_name, trust_factor, threat_level)
report_string = """Report string (test string)"""
return AnalysisResult(message=malware_result,
title=malware_result,
description=report_string,
link=result_link,
score=score)
def check_result_for(self, md5sum):
log.info("Submitting hash %s to RL for analysis" % md5sum)
try:
response = self.rl_analysis.get_report(resource_hash=md5sum)
except RLAPIQUOTAREACHED as rle:
log.info(rle)
raise AnalysisTemporaryError(
message="Quota reached. Will retry in 30 min",
retry_in=30 * 60)
except Exception as err:
log.info(err)
raise AnalysisTemporaryError(
message="There was an error. Error: {}".format(str(err)),
retry_in=60 * 60)
if response.get('code'):
raise AnalysisTemporaryError(
message=
'No results on A1000. Allow submit_full_binaries to get results on A1000',
retry_in=30 * 60)
return self.make_result(md5sum=md5sum, result=response)
def analyze_binary(self, md5sum, binary_file_stream):
if not self.submit_full_binaries:
raise AnalysisPermanentError(
"Submitting full binaries is not enabled")
log.info("Submitting binary {}".format(md5sum))
successfull_upload = self.rl_analysis.submit_file(
md5sum=md5sum, stream=binary_file_stream)
if successfull_upload:
return self.make_result(md5=md5sum)
else:
raise AnalysisTemporaryError(
"Unable to upload file. md5sum: {}".format(md5sum),
retry_in=30 * 60)
def query_wildfire(self, md5sum):
"""
query the wildfire api to get a report on an md5
"""
log.info("Querying wildfire for md5sum %s" % md5sum)
status_code, content = self._call_wildfire_api("POST", "/publicapi/get/verdict",
{'hash': md5sum.lower()})
if status_code == 404:
return None # can't find the binary
elif status_code != 200:
log.info("Received unknown HTTP status code %d from WildFire" % status_code)
log.info("-> response content: %s" % content)
raise AnalysisTemporaryError("Received unknown HTTP status code %d from WildFire" % status_code,
retry_in=120)
response = etree.fromstring(content)
# Return 0 Benign verdict
# 1 Malware verdict
# 2 Grayware verdict
# -100 Verdict is pending
# -101 Indicates a file error
# -102 The file could not be found
# -103 The hash submitted is invalid
if md5sum.lower() == response.findtext("./get-verdict-info/md5").lower():
verdict = response.findtext("./get-verdict-info/verdict").strip()
if verdict == "-100":
return None # waiting for WildFire verdict
elif verdict == "-102":
return None # file not in WildFire yet
elif verdict.startswith("-"):
raise AnalysisPermanentError("WildFire could not process file: error %s" % verdict)
elif verdict == "1":
return self.generate_malware_result(md5sum, 100)
elif verdict == "2":
return self.generate_malware_result(md5sum, 50)
else:
return AnalysisResult(score=0)
def analyze_binary(self, md5_hash, binary_file_stream):
LOGGER.info("Submitting binary with md5 %s to VMRay" % (md5_hash))
# submit file to VMRay
try:
result = self.rest_api.call("POST",
"/rest/sample/submit",
params={
"archive_action":
"ignore",
"sample_file":
binary_file_stream,
"sample_filename_b64enc":
base64.encodestring(md5_hash)
})
except VMRayRESTAPIError as exc:
LOGGER.debug("Error submitting sample with md5 %s",
md5_hash,
exc_info=True)
raise AnalysisTemporaryError(message="API error: %s" % str(exc),
retry_in=self.retry_wait_time)
sample_id = result["samples"][0]["sample_id"]
submission_id = result["submissions"][0]["submission_id"]
LOGGER.debug("Waiting for submission with ID %u to finish all jobs",
submission_id)
# wait until all analyses have finished
open_jobs = list(result["jobs"])
wait_start = time.time()
while len(open_jobs) > 0:
# check for timeout
if (time.time() - wait_start) > self.max_analysis_wait_time:
LOGGER.debug(
"Timed out waiting for result of submission with ID %u",
submission_id)
raise AnalysisTemporaryError(
message=
"Timed out waiting for analysis jobs to finish for submission %u"
% (submission_id),
retry_in=self.retry_wait_time)
check_jobs = list(open_jobs)
open_jobs = []
for job in check_jobs:
try:
self.rest_api.call("GET", "/rest/job/%u" % (job["job_id"]))
except VMRayRESTAPIError as exc:
if exc.status_code == 404:
# job has finished
continue
# job is still there or server is unreachable
open_jobs.append(job)
if len(open_jobs) == 0:
break
time.sleep(self.loop_wait_time)
LOGGER.debug("All jobs for submission with ID %u have finished",
submission_id)
return self.create_result(sample_id, submission_id=submission_id)
def create_result(self, sample_id, submission_id=None):
"""Create Carbon Black result for the given sample"""
try:
sample = self.rest_api.call("GET", "/rest/sample/%u" % (sample_id))
if submission_id is not None:
LOGGER.debug(
"Creating result for sample with ID %u (submission_id=%s)",
sample_id, submission_id)
submission = self.rest_api.call(
"GET", "/rest/submission/%u" % (submission_id))
if "submission_finished" in submission:
# new way
if submission["submission_finished"]:
analyses = self.rest_api.call(
"GET",
"/rest/analysis/submission/%u" % (submission_id))
else:
raise AnalysisTemporaryError(
message=
"API error: Submission ID %u not finished yet" %
(submission_id),
retry_in=self.retry_wait_time)
else:
# deprecated old way
# get analyses
analyses = self.rest_api.call(
"GET", "/rest/analysis/sample/%u" % (sample_id))
# filter by submission_id
analyses = [
analysis for analysis in analyses
if analysis["analysis_submission_id"] == submission_id
]
else:
LOGGER.debug("Creating result for sample with ID %u",
sample_id)
# get all analyses
analyses = self.rest_api.call(
"GET", "/rest/analysis/sample/%u" % (sample_id))
except BaseException as exc:
LOGGER.debug(
"Error getting sample and analyses info for sample ID %s",
sample_id,
exc_info=True)
raise AnalysisTemporaryError(message="API error: %s" % (str(exc)))
# check if any error occurred
for analysis in analyses:
if analysis["analysis_result_code"] != 1:
LOGGER.warning(
"Analysis with ID %u of sample with ID %u failed with error code 0x%x: %s",
analysis["analysis_id"], sample_id,
analysis["analysis_result_code"],
analysis["analysis_result_str"])
# check sample VTI score
if (sample["sample_vti_score"] is None) or (
sample["sample_vti_score"] < self.vti_score_threshold):
message = "Benign"
else:
message = "Potential malware"
# Check for None in the score
if not sample["sample_vti_score"] and message == 'Benign':
sample["sample_vti_score"] = 0
if not sample["sample_vti_score"] and message == 'Potential malware':
sample["sample_vti_score"] = 100
#
# Override if sample_severity is blacklisted or malicious
#
if sample['sample_severity'].lower() == "blacklisted":
sample["sample_vti_score"] = 100
message = "blacklisted"
elif sample['sample_severity'].lower() == "malicious":
sample["sample_vti_score"] = 100
message = "malicious"
LOGGER.debug(
"Analysis result of sample with ID %u created successfully (message=%s link=%s score=%u)",
sample_id, message, sample["sample_webif_url"],
sample["sample_vti_score"])
return AnalysisResult(
message=message,
extended_message="",
link=sample["sample_webif_url"],
score=sample["sample_vti_score"],
)
def query_checkpoint(self, md5sum):
"""
query the checkpoint api to get a report on an md5
"""
headers = {
"Content-Type": "application/json",
"Authorization": self.api_key,
}
payload = json.dumps({
"request": [{
"md5": md5sum,
"features": ["te", "av", "extraction"]
}]
})
log.info("Querying checkpoint for md5sum %s" % md5sum)
try:
status_code, content = self._call_checkpoint_api(
method="POST",
path="/tecloud/api/v1/file/query",
headers=headers,
payload=payload,
files=None)
if status_code != 200:
log.info(
"Received unknown HTTP status code %d from checkpoint" %
status_code)
log.info("-> response content: %s" % content)
raise AnalysisTemporaryError(
"Received unknown HTTP status code %d from checkpoint" %
status_code,
retry_in=120)
except Exception as e:
log.info(str(e))
raise AnalysisTemporaryError(
"There was an error connecting to Checkpoint %s" % str(e),
retry_in=120)
dict_response = json.loads(content)
try:
checkpoint_status_code = dict_response.get("response", [])[0].get(
"te", {}).get("status", {}).get("code", -1)
except Exception as e:
checkpoint_status_code = -1
log.error("Failed to parse checkpoint response JSON")
log.error(traceback.format_exc())
log.info("md5: {0} returned status_code: {1}".format(
md5sum, checkpoint_status_code))
if checkpoint_status_code == 1001:
severity = dict_response.get("response", [])[0].get("te", {}).get(
"combined_verdict", None)
if severity.lower() == "malicious":
score = 100
elif severity.lower() == "unknown":
return None
elif severity.lower() == "benign":
score = 0
else:
return None
log.info("{0} has score of {1}".format(md5sum, score))
return AnalysisResult(score=score)
elif checkpoint_status_code == 1003:
#
# Pending
#
return None
elif checkpoint_status_code == 1004:
#
# 1004 NOT_Found
# File is not in checkpoint yet
#
return None
elif checkpoint_status_code == 1005:
#
# Out of Quota
#
return AnalysisTemporaryError("Out of Quota")
elif checkpoint_status_code == 1006:
#
# Partially found
#
return None
elif checkpoint_status_code == 1007:
#
# FILE_TYPE_NOT_SUPPORTED
#
return AnalysisPermanentError("Filetype is not supported")
elif checkpoint_status_code == 1009:
#
# Internal Error
#
return AnalysisTemporaryError("Internal Error from Checkpoint")
elif checkpoint_status_code == 1011:
#
# Insufficient resources
#
return AnalysisTemporaryError(
"Checkpoint reports insufficient resources")
else:
return None
