def get(self, section):
"""Get option.
@param section: section to fetch.
@raise CuckooOperationalError: if section not found.
@return: option value.
"""
try:
return getattr(self, section)
except AttributeError as e:
raise CuckooOperationalError("Option %s is not found in "
"configuration, error: %s" %
(section, e))
def create_folder(root=".", folder=None):
"""Create directory.
@param root: root path.
@param folder: folder name to be created.
@raise CuckooOperationalError: if fails to create folder.
"""
if not os.path.exists(os.path.join(root, folder)) and folder:
try:
folder_path = os.path.join(root, folder)
os.makedirs(folder_path)
except OSError as e:
raise CuckooOperationalError("Unable to create folder: %s" % folder_path)
def delete(*folder):
"""Delete a folder and all its subdirectories.
@param folder: path or components to path to delete.
@raise CuckooOperationalError: if fails to delete folder.
"""
folder = os.path.join(*folder)
if os.path.exists(folder):
try:
shutil.rmtree(folder)
except OSError as e:
raise CuckooOperationalError(
f"Unable to delete folder: {folder}") from e
def get(self, section):
"""Get options for the given section.
@param section: section to fetch.
@raise CuckooOperationalError: if section not found.
@return: dict of option key/values.
"""
try:
return getattr(self, section)
except AttributeError as e:
raise CuckooOperationalError(
f"Option {section} is not found in configuration, error: {e}"
) from e
def netlog_sanitize_fname(path):
"""Validate agent-provided path for result files"""
path = path.replace(b"\\", b"/")
dir_part, name = os.path.split(path)
if dir_part not in RESULT_UPLOADABLE:
raise CuckooOperationalError(
"Netlog client requested banned path: %r" % path)
if any(c in BANNED_PATH_CHARS for c in name):
for c in BANNED_PATH_CHARS:
path.replace(bytes([c]), b"X")
return path
def read_next_message(self):
# Read until newline for file path, e.g.,
# shots/0001.jpg or files/9498687557/libcurl-4.dll.bin
buf = self.handler.read_newline().strip().replace("\\", "/")
log.debug("File upload request for {0}".format(buf))
dir_part, filename = os.path.split(buf)
if "./" in buf or not dir_part:
raise CuckooOperationalError("FileUpload failure, banned path.")
for restricted in self.RESTRICTED_DIRECTORIES:
if restricted in dir_part:
raise CuckooOperationalError(
"FileUpload failure, banned path.")
try:
create_folder(self.storagepath, dir_part)
except CuckooOperationalError:
log.error("Unable to create folder %s" % dir_part)
return False
file_path = os.path.join(self.storagepath, buf.strip())
self.fd = open(file_path, "wb")
chunk = self.handler.read_any()
while chunk:
self.fd.write(chunk)
if self.fd.tell() >= self.upload_max_size:
self.fd.write("... (truncated)")
break
try:
chunk = self.handler.read_any()
except:
break
log.debug("Uploaded file length: {0}".format(self.fd.tell()))
def negotiate_protocol(self):
# Read until newline.
buf = self.read_newline()
if "BSON" in buf:
self.protocol = BsonParser(self)
elif "FILE" in buf:
self.protocol = FileUpload(self)
elif "LOG" in buf:
self.protocol = LogHandler(self)
else:
raise CuckooOperationalError("Netlog failure, unknown "
"protocol requested.")
def _check_output(self, out, err):
if out:
raise CuckooOperationalError(
"Potential error while running tcpdump, did not expect "
"standard output, got: %r." % out)
err_whitelist = (
"packets captured",
"packets received by filter",
"packets dropped by kernel",
)
for line in err.split("\n"):
if not line or line.startswith("tcpdump: listening on "):
continue
if line.endswith(err_whitelist):
continue
raise CuckooOperationalError(
"Potential error while running tcpdump, did not expect "
"the following standard error output: %r." % line)
def run(self, results):
if not HAVE_REQUESTS:
raise CuckooOperationalError(
"The Mattermost processing module requires the requests "
"library (install with `pip install requests`)")
sigs, urls = [], []
for sig in results.get("signatures", {}):
sigs.append(sig.get("name"))
if sig.get("name") == "network_http":
for http in sig.get("marks"):
urls.append(http.get("ioc"))
post = "Finished analyze ::: [{0}]({1}{0}) ::: ".format(
results.get("info").get("id"), self.options.get("myurl"))
filename = results.get("target").get("file").get("name")
if self.options.get("hash-filename"):
filename = hashlib.sha256(filename).hexdigest()
post += "File : {0} ::: Score : **{1}** ::: ".format(
filename,
results.get("info").get("score"))
if self.options.get("show-virustotal"):
post += "**VT : {0} / {1}**\n".format(
results.get("virustotal").get("positives"),
results.get("virustotal").get("total"),
)
if self.options.get("show-signatures"):
post += "**Signatures** ::: {0} \n".format(" : ".join(sigs))
if self.options.get("show-urls"):
post += "**URLS**\n`{0}`".format("\n".join(urls).replace(
".", "[.]"))
data = {
"username": self.options.get("username"),
"text": post,
}
headers = {"Content-Type": "application/json"}
try:
requests.post(self.options.get("url"),
headers=headers,
data=json.dumps(data))
except Exception as e:
raise CuckooReportError(
"Failed posting message to Mattermost: %s" % e)
def create_folder(root=".", folder=None):
"""Create directory.
@param root: root path.
@param folder: folder name to be created.
@raise CuckooOperationalError: if fails to create folder.
"""
folder_path = os.path.join(root, folder)
if folder and not os.path.isdir(folder_path):
try:
os.makedirs(folder_path)
except OSError as e:
if e.errno != errno.EEXIST:
raise CuckooOperationalError("Unable to create folder: %s" %
folder_path)
def _post_text(self, url, **kwargs):
"""Wrapper around doing a post and parsing its text output."""
try:
flags = {"flags": kwargs.get("flags", "")}
# log.debug(kwargs)
# log.debug("FLAGS %s" % flags)
files = {"sample": kwargs.get("sample")}
r = requests.post(url,
timeout=self.timeout,
data=flags,
files=files)
return r.text if r.status_code == 200 else {}
except (requests.ConnectionError, ValueError) as e:
raise CuckooOperationalError(
"Unable to POST to the API server: %r" % e.message)
def read_newline(self):
"""Read until the next newline character, but never more than
`MAX_NETLOG_LINE`."""
while True:
pos = self.buf.find(b"\n")
if pos < 0:
if len(self.buf) >= MAX_NETLOG_LINE:
raise CuckooOperationalError("Received overly long line")
buf = self.read()
if buf == b"":
raise EOFError
self.buf += buf
continue
line, self.buf = self.buf[:pos], self.buf[pos + 1:]
return line
def handle(self):
# Read until newline for file path, e.g.,
# shots/0001.jpg or files/9498687557/libcurl-4.dll.bin
self.handler.sock.settimeout(30)
dump_path = netlog_sanitize_fname(self.handler.read_newline())
if self.version and self.version >= 2:
# NB: filepath is only used as metadata
filepath = self.handler.read_newline()
pids = list(map(int, self.handler.read_newline().split()))
metadata = self.handler.read_newline()
category = self.handler.read_newline()
else:
filepath, pids, metadata, category = None, [], b"", b""
log.debug("Task #%s: File upload for %r", self.task_id, dump_path)
file_path = os.path.join(self.storagepath, dump_path.decode("utf-8"))
try:
self.fd = open_exclusive(file_path)
except OSError as e:
if e.errno == errno.EEXIST:
raise CuckooOperationalError("Analyzer for task #%s tried to " "overwrite an existing file" % self.task_id)
raise
# ToDo we need Windows path
# filter screens/curtain/sysmon
if not dump_path.startswith((b"shots/", b"curtain/", b"aux/", b"sysmon/", b"debugger/")):
# Append-writes are atomic
with open(self.filelog, "a") as f:
print(
json.dumps(
{
"path": dump_path.decode("utf-8", "replace"),
"filepath": filepath.decode("utf-8", "replace") if filepath else "",
"pids": pids,
"metadata": metadata.decode("utf-8", "replace"),
"category": category.decode("utf-8") if category in (b"CAPE", b"files", b"memory", b"procdump") else "",
},
ensure_ascii=False,
),
file=f,
)
self.handler.sock.settimeout(None)
try:
return self.handler.copy_to_fd(self.fd, self.upload_max_size)
finally:
log.debug("Task #%s uploaded file length: %s", self.task_id, self.fd.tell())
def negotiate_protocol(self):
# Read until newline.
buf = self.read_newline()
if "BSON" in buf:
self.protocol = BsonParser(self)
elif "FILE" in buf:
self.protocol = FileUpload(self, is_binary=False, duplicate=False)
elif "DUPLICATEBINARY" in buf:
self.protocol = FileUpload(self, is_binary=True, duplicate=True)
elif "BINARY" in buf:
self.protocol = FileUpload(self, is_binary=True, duplicate=False)
elif "LOG" in buf:
self.protocol = LogHandler(self)
else:
raise CuckooOperationalError("Netlog failure, unknown "
"protocol requested.")
def run(self):
"""Runs IRMA processing
@return: full IRMA report.
"""
if not HAVE_REQUESTS:
raise CuckooOperationalError(
"The IRMA processing module requires the requests "
"library (install with `pip install requests`)"
)
self.key = "irma"
""" Fall off if we don't deal with files """
if self.results.get("info").get("category") != "file":
log.debug("IRMA supports only file scanning !")
return {}
self.url = self.options.get("url")
self.timeout = int(self.options.get("timeout", 60))
self.scan = int(self.options.get("scan", 0))
self.force = int(self.options.get("force", 0))
sha256 = sha256_file(self.file_path)
results = self._get_results(sha256)
if not self.force and not self.scan and not results:
return {}
elif self.force or (not results and self.scan):
log.info("File scan requested: %s", sha256)
self._scan_file(self.file_path, self.force)
results = self._get_results(sha256) or {}
""" FIXME! could use a proper fix here
that probably needs changes on IRMA side aswell
--
related to https://github.com/elastic/elasticsearch/issues/15377
entropy value is sometimes 0 and sometimes like 0.10191042566270775
other issue is that results type changes between string and object :/
"""
for idx, result in enumerate(results["probe_results"]):
if result["name"] == "PE Static Analyzer":
log.debug("Ignoring PE results at index {0}".format(idx))
results["probe_results"][idx]["results"] = "... scrapped ..."
return results
def run(self, results):
if not HAVE_REQUESTS:
raise CuckooOperationalError(
"The Notification reporting module requires the requests "
"library (install with `pip install requests`)")
post = {
"identifier":
self.options.get("identifier"),
"data":
json.dumps(results.get("info"), default=default, sort_keys=False)
}
try:
requests.post(self.options.get("url"), data=post)
except Exception as e:
raise CuckooReportError(
"Failed posting message via Notification: %s" % e)
def log_call(self, context, apiname, modulename, arguments):
if not self.rawlogfd:
raise CuckooOperationalError("Netlog failure, call "
"before process.")
apiindex, status, returnval, tid, timediff = context
#log.debug("log_call> tid:{0} apiname:{1}".format(tid, apiname))
current_time = self.connect_time + datetime.timedelta(0, 0,
timediff*1000)
timestring = logtime(current_time)
argumentstrings = ["{0}->{1}".format(argname, repr(str(r))[1:-1])
for argname, r in arguments]
if self.logfd:
print >>self.logfd, ",".join("\"{0}\"".format(i) for i in [
timestring, self.pid, self.procname, tid, self.ppid,
modulename, apiname, status, returnval] + argumentstrings)
def negotiate_protocol(self):
protocol = self.read_newline(strip=True)
# Command with version number.
if " " in protocol:
command, version = protocol.split()
version = int(version)
else:
command, version = protocol, None
if command == "BSON":
self.protocol = BsonParser(self, version)
elif command == "FILE":
self.protocol = FileUpload(self, version)
elif command == "LOG":
self.protocol = LogHandler(self, version)
else:
raise CuckooOperationalError(
"Netlog failure, unknown protocol requested.")
self.protocol.init()
def stop(self):
"""Stop sniffing.
@return: operation status.
"""
# The tcpdump process was never started in the first place.
if not self.proc:
return
# The tcpdump process has already quit, generally speaking this
# indicates an error such as "permission denied".
if self.proc.poll():
out, err = self.proc.communicate()
raise CuckooOperationalError(
"Error running tcpdump to sniff the network traffic during "
"the analysis; stdout = %r and stderr = %r. Did you enable "
"the extra capabilities to allow running tcpdump as non-root "
"user and disable AppArmor properly (the latter only applies "
"to Ubuntu-based distributions with AppArmor, see also %s)?" %
(out, err, "permission-denied-for-tcpdump"))
try:
self.proc.terminate()
except:
try:
if not self.proc.poll():
log.debug("Killing sniffer")
self.proc.kill()
except OSError as e:
log.debug("Error killing sniffer: %s. Continue", e)
except Exception as e:
log.exception("Unable to stop the sniffer with pid %d: %s",
self.proc.pid, e)
# Ensure expected output was received from tcpdump.
out, err = self.proc.communicate()
self._check_output(out, err)
del self.proc
def run(self):
self.key = "boxjs"
""" Fall off if we don't deal with files """
if self.results.get("info", {}).get("category") != "file":
log.debug("Box-js supports only file scanning!")
return {}
self.url = self.options.get("url")
self.timeout = int(self.options.get("timeout", 60))
self.ioc = self.options.get("IOC")
# Post file for scanning.
postUrl = urljoin(self.url, "/sample")
analysis_id = self._post_text(postUrl,
sample=open(self.file_path,
"rb")) # returns a UUID
base_url = "{}/sample/{}".format(self.url, str(analysis_id))
flags = ""
# Wait for the analysis to be completed.
done = False
while not done:
time.sleep(1)
result = self.request_json(base_url)
code = result["code"]
retry = False
# Read the status code, and retry with different flags if necessary
if code == 0: # Success
done = True
elif code == 1: # Generic error
# We don't know how to handle this, so continue
done = True
# Todo: show result["stderr"] to the user?
elif code == 2: # Timeout
# Todo: choose whether to use longer timeout
done = True
elif code == 3: # Rewrite error
flags += "--no-rewrite "
retry = True
elif code == 4: # Syntax error
# Todo: implement JSE decoding if necessary
done = True
elif code == 5: # Retry with --no-shell-error
flags += "--no-shell-error "
retry = True
else:
raise CuckooOperationalError("Unknown error code: %s" % code)
if retry:
postUrl = urljoin(self.url, "/sample")
analysis_id = self._post_text(postUrl,
sample=open(
self.file_path, "rb"),
flags=flags) # returns a UUID
base_url = "{}/sample/{}".format(self.url, str(analysis_id))
# Fetch the results.
results = {}
urls_url = "{}/urls".format(base_url)
resources_url = "{}/resources".format(base_url)
iocs_ioc = "{}/IOC".format(base_url)
results["urls"] = self.request_json(urls_url)
results["resources"] = self.request_json(resources_url)
results["IOC"] = self.request_json(iocs_ioc)
# Delete the results.
try:
requests.delete(base_url, timeout=self.timeout)
except (requests.ConnectionError, ValueError) as e:
raise CuckooOperationalError(
"Unable to send a DELETE request: %r" % e.message)
return results
def read_next_message(self):
# Read until newline for file path, e.g.,
# shots/0001.jpg or files/9498687557/libcurl-4.dll.bin
buf = self.handler.read_newline().strip().replace("\\", "/")
guest_path = ""
if self.is_binary:
guest_path = sanitize_pathname(self.handler.read_newline().strip()[:32768])
dir_part, filename = os.path.split(buf)
filename = sanitize_pathname(filename)
buf = os.path.join(dir_part, filename)
log.debug("File upload request for {0}".format(buf))
if "./" in buf or not dir_part or buf.startswith("/"):
raise CuckooOperationalError("FileUpload failure, banned path.")
for restricted in self.RESTRICTED_DIRECTORIES:
if restricted in dir_part:
raise CuckooOperationalError("FileUpload failure, banned path.")
try:
create_folder(self.storagepath, dir_part)
except CuckooOperationalError:
log.error("Unable to create folder %s" % dir_part)
return False
file_path = os.path.join(self.storagepath, buf)
if not file_path.startswith(self.storagepath):
raise CuckooOperationalError("FileUpload failure, path sanitization failed.")
if guest_path != "":
guest_paths = []
if os.path.exists(file_path + "_info.txt"):
guest_paths = [line.strip() for line in open(file_path + "_info.txt")]
if guest_path not in guest_paths:
infofd = open(file_path + "_info.txt", "a")
infofd.write(guest_path + "\n")
infofd.close()
if not self.duplicate:
if os.path.exists(file_path):
log.warning("Analyzer tried to overwrite an existing file, closing connection.")
return False
self.fd = open(file_path, "wb")
chunk = self.handler.read_any()
while chunk:
self.fd.write(chunk)
if self.fd.tell() >= self.upload_max_size:
log.warning("Uploaded file length ({0}) larger than upload_max_size ({1}), stopping upload.".format(self.fd.tell(), self.upload_max_size))
self.fd.write("... (truncated)")
break
try:
chunk = self.handler.read_any()
except:
break
log.debug("Uploaded file length: {0}".format(self.fd.tell()))
def run(self):
self.key = "boxjs"
""" Fall off if we don't deal with files """
if self.results.get("info", {}).get("category") not in ("file", "static") and (
self.results.get("info", {}).get("package", "") in ("js", "jse", "jsevbe", "js_antivm")
or self.results.get("target", {}).get("file", {}).get("name", "").endswith(".js", ".jse")
):
log.debug("Box-js supports only file scanning")
return {}
self.url = self.options.get("url")
self.timeout = int(self.options.get("timeout", 60))
self.ioc = self.options.get("IOC")
# Post file for scanning.
postUrl = urljoin(self.url, "/sample")
analysis_id = self._post_text(postUrl, sample=open(self.file_path, "rb")) # returns a UUID
base_url = f"{self.url}/sample/{analysis_id}"
flags = ""
# Wait for the analysis to be completed.
done = False
while not done:
time.sleep(2)
result = self.request_json(base_url)
code = result.get("code")
retry = False
# Read the status code, and retry with different flags if necessary
if code is None:
continue
elif code == 0: # Success
done = True
elif code == 1: # Generic error
# We don't know how to handle this, so continue
done = True
# Todo: show result["stderr"] to the user?
elif code == 2: # Timeout
# Todo: choose whether to use longer timeout
done = True
elif code == 3: # Rewrite error
flags += "--no-rewrite "
retry = True
elif code == 4: # Syntax error
# Todo: implement JSE decoding if necessary
done = True
elif code == 5: # Retry with --no-shell-error
flags += "--no-shell-error "
retry = True
else:
done = True
log.info("BOXJS: %s", result)
return {}
if retry:
postUrl = urljoin(self.url, "/sample")
analysis_id = self._post_text(postUrl, sample=open(self.file_path, "rb"), flags=flags) # returns a UUID
base_url = f"{self.url}/sample/{analysis_id}"
# Fetch the results.
urls_url = f"{base_url}/urls"
resources_url = f"{base_url}/resources"
iocs_ioc = f"{base_url}/IOC"
results = {
"urls": self.request_json(urls_url),
"resources": self.request_json(resources_url),
"IOC": self.request_json(iocs_ioc),
}
# Delete the results.
try:
requests.delete(base_url, timeout=self.timeout)
except (requests.ConnectionError, ValueError) as e:
raise CuckooOperationalError(f"Unable to send a DELETE request: {e.message}") from e
return results
def __iter__(self):
# Read until newline for file path, e.g.,
# shots/0001.jpg or files/9498687557/libcurl-4.dll.bin
dump_path = self.handler.read_newline(strip=True).replace("\\", "/")
if self.version >= 2:
filepath = self.handler.read_newline(strip=True)
pids = map(int, self.handler.read_newline(strip=True).split())
else:
filepath, pids = None, []
log.debug("File upload request for %s", dump_path)
dir_part, filename = os.path.split(dump_path)
if "./" in dump_path or not dir_part or dump_path.startswith("/"):
raise CuckooOperationalError(
"FileUpload failure, banned path: %s" % dump_path)
for restricted in self.RESTRICTED_DIRECTORIES:
if restricted in dir_part:
raise CuckooOperationalError(
"FileUpload failure, banned path.")
try:
create_folder(self.storagepath, dir_part)
except CuckooOperationalError:
log.error("Unable to create folder %s", dir_part)
return
file_path = os.path.join(self.storagepath, dump_path.strip())
if not file_path.startswith(self.storagepath):
raise CuckooOperationalError(
"FileUpload failure, path sanitization failed.")
if os.path.exists(file_path):
log.warning("Analyzer tried to overwrite an existing file, "
"closing connection.")
return
self.fd = open(file_path, "wb")
chunk = self.handler.read_any()
while chunk:
self.fd.write(chunk)
if self.fd.tell() >= self.upload_max_size:
log.warning(
"Uploaded file length larger than upload_max_size, "
"stopping upload.")
self.fd.write("... (truncated)")
break
try:
chunk = self.handler.read_any()
except:
break
self.lock.acquire()
with open(self.filelog, "a+b") as f:
f.write("%s\n" % json.dumps({
"path": dump_path,
"filepath": filepath,
"pids": pids,
}))
self.lock.release()
log.debug("Uploaded file length: %s", self.fd.tell())
return
yield