def run(self):
# construct a valid filename into which thug will save the result
fname = str(self.filename).replace("/", "_").replace(" ", "_")
# get the file to send
binary = get_binary(self.job_id)
# construct arguments, For example this corresponds to,
# box-js sample.js --output-dir=/tmp/boxjs --no-kill ...
args = [
f"@{fname}",
"--output-dir=/tmp/boxjs",
"--no-kill",
"--no-shell-error",
"--no-echo",
]
# Box-js by default has a timeout of 10 seconds,
# but subprocess is not able to catch that
# that's why it's necessary to provide a custom timeout of
# 10 seconds only to the subprocess itself.
req_data = {
"args": args,
"timeout": 10,
"callback_context": {"read_result_from": fname},
}
req_files = {fname: binary}
return self._docker_run(req_data, req_files)
def run(self):
# get binary
fname = str(self.filename).replace("/", "_").replace(" ", "_")
self.args.extend(["-n", "/tmp/thug/" + fname, "-l", f"@{fname}"])
logger.debug(
f"Making request with arguments: {self.args}"
f" for analyzer: {self.analyzer_name}, job_id: #{self.job_id}."
)
# request new analysis
r = requests.post(
self.url, files={fname: get_binary(self.job_id)}, data={"args": self.args,},
)
# handle cases in case of error
if self._check_status_code(self.name, r):
# if no error, continue..
errors = []
# this is to check whether analysis completed or not..
resp = self._poll_for_result(r.json()["key"])
err = resp.get("error", None)
if err:
errors.append(err)
logger.info(
f"Fetching final report ({self.analyzer_name}, job_id: #{self.job_id})"
)
# if no error, we fetch the final report..
result_resp = requests.get(f"{self.base_url}/get-result?name={fname}")
if not result_resp.status_code == 200:
e = resp.json()["error"]
errors.append(e)
raise AnalyzerRunException(", ".join(errors))
return result_resp.json()
def run(self):
mwdb = mwdblib.MWDB(api_key=self.__api_key)
binary = get_binary(self.job_id)
query = str(hashlib.sha256(binary).hexdigest())
if self.upload_file:
logger.info(f"mwdb_scan uploading sample: {self.md5}")
file_object = mwdb.upload_file(query, binary)
file_object.flush()
for _try in range(self.max_tries):
logger.info(
f"mwdb_scan sample: {self.md5} polling for result try #{_try + 1}"
)
time.sleep(self.poll_distance)
file_info = mwdb.query_file(file_object.data["id"])
if self.file_analysis(file_info):
break
if not self.file_analysis(file_info):
raise AnalyzerRunException("max retry attempts exceeded")
else:
try:
file_info = mwdb.query_file(query)
except Exception:
raise AnalyzerRunException(
"File not found in the MWDB. Set 'upload_file=true' "
"if you want to upload and poll results. ")
result = {"data": file_info.data, "metakeys": file_info.metakeys}
result["permalink"] = f"https://mwdb.cert.pl/file/{query}"
return result
def run(self):
results = {}
results["magic"] = magic.from_file(self.filepath)
results["mimetype"] = magic.from_file(self.filepath, mime=True)
binary = get_binary(self.job_id)
results["md5"] = hashlib.md5(binary).hexdigest()
results["sha1"] = hashlib.sha1(binary).hexdigest()
results["sha256"] = hashlib.sha256(binary).hexdigest()
results["ssdeep"] = pydeep.hash_file(self.filepath).decode()
try:
with ExifTool(self.exiftool_path) as et:
exif_report = et.execute_json(self.filepath)
if exif_report:
exif_single_report = exif_report[0]
exif_report_cleaned = {
key: value
for key, value in exif_single_report.items()
if not (key.startswith("File") or key.startswith("SourceFile"))
}
# compatibility with the previous version of this analyzer
results["filetype"] = exif_single_report.get("File:FileType", "")
results["exiftool"] = exif_report_cleaned
except Exception as e:
logger.exception(e)
return results
def run(self):
# get binary
binary = get_binary(self.job_id)
# make request data
fname = str(self.filename).replace("/", "_").replace(" ", "_")
args = ["flarestrings", f"@{fname}"]
req_data = {
"args": args,
"timeout": self.timeout,
}
req_files = {fname: binary}
result = self._docker_run(req_data, req_files)
exceed_max_strings = len(result) > self.max_no_of_strings
if exceed_max_strings:
result = [s for s in result[:self.max_no_of_strings]]
if self.rank_strings:
args = [
"rank_strings",
"--limit",
str(self.max_no_of_strings),
"--strings",
json_dumps(result),
]
req_data = {"args": args, "timeout": self.timeout}
result = self._docker_run(req_data)
result = {
"data": [row[:self.max_chars_for_string] for row in result],
"exceeded_max_number_of_strings": exceed_max_strings,
}
return result
def __intezer_scan_file(self, intezer_token):
session = requests.session()
session.headers["Authorization"] = f"Bearer {intezer_token}"
name_to_send = self.filename if self.filename else self.md5
binary = get_binary(self.job_id)
files = {"file": (name_to_send, binary)}
logger.info(f"intezer md5 {self.md5} sending sample for analysis")
response = session.post(self.base_url + "/analyze", files=files)
if response.status_code != 201:
raise AnalyzerRunException(
f"failed analyze request, status code {response.status_code}")
for chance in range(self.max_tries):
if response.status_code != 200:
time.sleep(self.poll_distance)
logger.info(
f"intezer md5 {self.md5} polling for result try #{chance + 1}"
)
result_url = response.json().get("result_url", "")
response = session.get(self.base_url + result_url)
response.raise_for_status()
if response.status_code != 200 and not self.is_test:
raise AnalyzerRunException("received max tries attempts")
return response.json()
def run(self):
# get the file to send
fname = str(self.filename).replace("/", "_").replace(" ", "_")
binary = get_binary(self.job_id)
# make request parameters
req_data = {"args": [f"@{fname}"] + self.args}
req_files = {fname: binary}
return self._docker_run(req_data, req_files)
def run(self):
# get binary
binary = get_binary(self.job_id)
# make request data
fname = str(self.filename).replace("/", "_").replace(" ", "_")
args = [f"@{fname}", "-j"]
req_data = {"args": args, "timeout": self.timeout}
req_files = {fname: binary}
return self._docker_run(req_data, req_files)
def run(self):
final_report = {}
if not self.__api_key:
raise AnalyzerRunException(
f"No API key retrieved with name: {self.api_key_name}")
self.headers = {"Authorization": f"Bearer {self.__api_key}"}
name_to_send = self.filename if self.filename else self.md5
binary = get_binary(self.job_id)
files = {
"file": (name_to_send, binary),
"_json": (None, b'{"kind": "file", "interactive": false}'),
}
logger.info(f"triage md5 {self.md5} sending sample for analysis")
for _try in range(self.max_tries):
logger.info(
f"triage md5 {self.md5} polling for result try #{_try + 1}")
response = requests.post(self.base_url + "samples",
headers=self.headers,
files=files)
if response.status_code == 200:
break
time.sleep(self.poll_distance)
if response.status_code != 200:
raise AnalyzerRunException("max retry attempts exceeded")
sample_id = response.json().get("id", None)
if sample_id is None:
raise AnalyzerRunException("error sending sample")
requests.get(self.base_url + f"samples/{sample_id}/events",
headers=self.headers)
final_report["overview"] = self.get_overview_report(sample_id)
if self.report_type == "complete":
final_report["static_report"] = self.get_static_report(sample_id)
final_report["task_report"] = {}
for task in final_report["overview"]["tasks"].keys():
status_code, task_report_json = self.get_task_report(
sample_id, task)
if status_code == 200:
final_report["task_report"][f"{task}"] = task_report_json
analysis_id = final_report["overview"].get("sample", {}).get("id", "")
if analysis_id:
final_report["permalink"] = f"{self.report_url}{analysis_id}"
return final_report
def run(self):
if not self.cuckoo_url:
raise AnalyzerConfigurationException("cuckoo URL missing")
binary = get_binary(self.job_id)
if not binary:
raise AnalyzerRunException("is the binary empty?!")
self.__cuckoo_request_scan(binary)
self.__cuckoo_poll_result()
result = self.__cuckoo_retrieve_and_create_report()
return result
def _scan_binary(self):
"""scan a binary against all YARA rules in Malpedia"""
url = self.base_url + "scan/binary"
headers = {"Authorization": f"APIToken {self.__api_key}"}
binary = get_binary(self.job_id)
files = {"file": binary}
try:
response = requests.post(url, headers=headers, files=files)
response.raise_for_status()
except requests.RequestException as e:
raise AnalyzerRunException(e)
result = response.json()
return result
def run(self):
# get binary
binary = get_binary(self.job_id)
# make request data
fname = str(self.filename).replace("/", "_").replace(" ", "_")
req_data = {"args": ["-j", f"@{fname}"]}
req_files = {fname: binary}
result = self._docker_run(req_data, req_files)
if result:
# limit strings dump to first 100
if "strings" in result and "dump" in result["strings"]:
result["strings"]["dump"] = result["strings"]["dump"][:100]
return result
def run(self):
# get binary
binary = get_binary(self.job_id)
# request new analysis
req_data = {"args": ["-j", "@filetoscan"]}
req_files = {"filetoscan": binary}
r = requests.post(self.url, files=req_files, data=req_data)
# handle cases in case of error
if self._check_status_code(self.name, r):
# if no error, continue..
resp = self._poll_for_result(r.json()["key"])
result = resp.get("report", None)
if result:
result = json.loads(result)
if "strings" in result and "dump" in result["strings"]:
result["strings"]["dump"] = result["strings"]["dump"][:100]
return result
def __vt_request_scan(self, notify_url):
binary = get_binary(self.job_id)
params = {"apikey": self.__api_key}
if notify_url:
params["notify_url"] = notify_url
files = {"file": binary}
try:
resp = requests.post(self.base_url + "file/scan",
files=files,
params=params)
resp.raise_for_status()
except requests.RequestException as e:
raise AnalyzerRunException(e)
json_resp = resp.json()
response_code = json_resp.get("response_code", 1)
if response_code == -1:
raise AnalyzerRunException("response code -1")
return json_resp
def run(self):
# construct a valid filename into which thug will save the result
fname = str(self.filename).replace("/", "_").replace(" ", "_")
# get the file to send
binary = get_binary(self.job_id)
# construct arguments, For example this corresponds to,
# apkid -j file.apk
args = [
"-t",
"20",
"-j",
f"@{fname}",
]
req_data = {
"args": args,
}
req_files = {fname: binary}
return self._docker_run(req_data, req_files)
def run(self):
# construct a valid dir name into which thug will save the result
fname = str(self.filename).replace("/", "_").replace(" ", "_")
tmp_dir = f"{fname}_{secrets.token_hex(4)}"
# get the file to send
binary = get_binary(self.job_id)
# append final arguments,
# -n -> output directory
# -l -> the local file to analyze
self.args.extend(["-n", "/home/thug/" + tmp_dir, "-l", f"@{fname}"])
# make request parameters
req_data = {
"args": self.args,
"callback_context": {
"read_result_from": tmp_dir
},
}
req_files = {fname: binary}
return self._docker_run(req_data, req_files)
def run(self):
results = {}
results["magic"] = magic.from_file(self.filepath)
results["mimetype"] = magic.from_file(self.filepath, mime=True)
results["filetype"] = pyexifinfo.fileType(self.filepath)
exif_report = pyexifinfo.get_json(self.filepath)
if exif_report:
exif_report_cleaned = {
key: value
for key, value in exif_report[0].items()
if not (key.startswith("File") or key.startswith("SourceFile"))
}
results["exiftool"] = exif_report_cleaned
binary = get_binary(self.job_id)
results["md5"] = hashlib.md5(binary).hexdigest()
results["sha1"] = hashlib.sha1(binary).hexdigest()
results["sha256"] = hashlib.sha256(binary).hexdigest()
results["ssdeep"] = pydeep.hash_file(self.filepath).decode()
return results
def run(self):
results = {}
rtfobj_results = {}
binary = get_binary(self.job_id)
rtfp = RtfObjParser(binary)
rtfp.parse()
rtfobj_results["ole_objects"] = []
for rtfobj in rtfp.objects:
if rtfobj.is_ole:
class_name = rtfobj.class_name.decode()
ole_dict = {
"format_id": rtfobj.format_id,
"class_name": class_name,
"ole_datasize": rtfobj.oledata_size,
}
if rtfobj.is_package:
ole_dict["is_package"] = True
ole_dict["filename"] = rtfobj.filename
ole_dict["src_path"] = rtfobj.src_path
ole_dict["temp_path"] = rtfobj.temp_path
ole_dict["olepkgdata_md5"] = rtfobj.olepkgdata_md5
else:
ole_dict["ole_md5"] = rtfobj.oledata_md5
if rtfobj.clsid:
ole_dict["clsid_desc"] = rtfobj.clsid_desc
ole_dict["clsid_id"] = rtfobj.clsid
rtfobj_results["ole_objects"].append(ole_dict)
# http://www.kb.cert.org/vuls/id/921560
if class_name == "OLE2Link":
rtfobj_results["exploit_ole2link_vuln"] = True
# https://www.kb.cert.org/vuls/id/421280/
elif class_name.lower() == "equation.3":
rtfobj_results["exploit_equation_editor"] = True
results["rtfobj"] = rtfobj_results
return results
def run(self):
# get binary
binary = get_binary(self.job_id)
# make request data
fname = str(self.filename).replace("/", "_").replace(" ", "_")
args = [f"@{fname}", f"--output-json=/tmp/{fname}.json"]
req_data = {
"args": args,
"timeout": self.timeout,
"callback_context": {
"read_result_from": fname
},
}
req_files = {fname: binary}
result = self._docker_run(req_data, req_files)
result["exceeded_max_number_of_strings"] = {}
self.url = self.ranking_url
for key in self.max_no_of_strings.keys():
if self.rank_strings[key]:
args = [
"rank_strings",
"--limit",
str(self.max_no_of_strings[key]),
"--strings",
json_dumps(result["strings"][key]),
]
req_data = {"args": args, "timeout": self.timeout}
result["strings"][key] = self._docker_run(req_data)
else:
if (len(result.get("strings", {}).get(key, [])) >
self.max_no_of_strings[key]):
result["strings"][key] = [
s for s in result["strings"][key]
]
result["exceeded_max_number_of_strings"][key] = True
return result
def vt_scan_file(
api_key,
md5,
job_id,
rescan_instead=False,
max_tries=100,
poll_distance=5,
):
try:
binary = get_binary(job_id)
except Exception:
raise AnalyzerRunException("couldn't retrieve the binary to perform a scan")
headers = {"x-apikey": api_key}
if rescan_instead:
logger.info(f"md5 {md5} job {job_id} VT analyzer requested rescan")
files = {}
uri = f"files/{md5}/analyse"
else:
logger.info(f"md5 {md5} job {job_id} VT analyzer requested scan")
files = {"file": binary}
uri = "files"
try:
response = requests.post(vt_base + uri, files=files, headers=headers)
response.raise_for_status()
except requests.RequestException as e:
raise AnalyzerRunException(e)
result = response.json()
result_data = result.get("data", {})
scan_id = result_data.get("id", "")
if not scan_id:
raise AnalyzerRunException(
"no scan_id given by VirusTotal to retrieve the results"
)
# max 5 minutes waiting
got_result = False
uri = f"analyses/{scan_id}"
for chance in range(max_tries):
time.sleep(poll_distance)
logger.info(
f"vt polling, try n.{chance + 1}. job_id {job_id}. starting the query"
)
try:
response = requests.get(vt_base + uri, headers=headers)
response.raise_for_status()
except requests.RequestException as e:
raise AnalyzerRunException(e)
json_response = response.json()
# pprint.pprint(json_response)
analysis_status = (
json_response.get("data", {}).get("attributes", {}).get("status", "")
)
if analysis_status == "completed":
got_result = True
break
else:
logger.info(
f"vt polling: try #{chance + 1}. job_id: #{job_id}."
f" status:{analysis_status}"
)
if not got_result and not rescan_instead:
raise AnalyzerRunException(
f"max VT polls tried without getting any result. job_id {job_id}"
)
# retrieve the FULL report, not only scans results.
# If it's a new sample, it's free of charge.
return vt3_get.vt_get_report(api_key, md5, "hash", {}, job_id)
