def init_once(cls):
"""Connect to Elasticsearch.
@raise CuckooReportError: if unable to connect.
"""
# Do not change these types without changing the elasticsearch
# template as well.
cls.report_type = "cuckoo"
cls.call_type = "call"
if not elastic.init():
return
cls.template_name = "%s_template" % elastic.index
try:
elastic.connect()
except CuckooOperationalError as e:
raise CuckooReportError(
"Error running ElasticSearch reporting module: %s" % e
)
# check to see if the template exists apply it if it does not
if not elastic.client.indices.exists_template(cls.template_name):
if not cls.apply_template():
raise CuckooReportError("Cannot apply Elasticsearch template")
def run(self, results):
sigs, urls = [], []
for sig in results.get("signatures", []):
sigs.append(sig.get("name"))
if sig.get("name") == "network_http":
for http in sig.get("marks"):
urls.append(http.get("ioc"))
post = "Finished analyze ::: [{0}]({1}{0}) ::: ".format(
results.get("info", {}).get("id"),
self.options.get("myurl")
)
if results.get("info", {}).get("category") == "file":
target = results.get("target", {}).get("file", {}).get("name", "")
if self.options.get("hash_filename"):
target = hashlib.sha256(target).hexdigest()
elif results.get("info", {}).get("category") == "url":
target = results.get("target", {}).get("url", "")
if self.options.get("hash_url"):
target = hashlib.sha256(target).hexdigest()
else:
target = "???"
post += "Target : {0} ::: Score : **{1}** ::: ".format(
target, results.get("info", {}).get("score")
)
if self.options.get("show_virustotal"):
post += "**VT : {0} / {1}**\n".format(
results.get("virustotal", {}).get("positives"),
results.get("virustotal", {}).get("total"),
)
if self.options.get("show_signatures"):
post += "**Signatures** ::: {0} \n".format(" : ".join(sigs))
if self.options.get("show_urls"):
post += "**URLS**\n`{0}`".format(
"\n".join(urls).replace(".", "[.]")
)
data = {
"username": self.options.get("username"),
"text": post,
}
try:
r = requests.post(self.options.get("url"), json=data)
# Note that POST max size is 4000 chars by default.
if r.status_code != 200:
raise CuckooReportError(
"Failed posting message due to : {0}".format(r.text)
)
except Exception as e:
raise CuckooReportError(
"Failed posting message to Mattermost: {0}".format(e)
)
def do_bulk_index(self, bulk_reqs):
try:
elasticsearch.helpers.bulk(elastic.client, bulk_reqs)
except Exception as e:
raise CuckooReportError(
"Failed to save results in ElasticSearch for "
"task #%d: %s" % (self.task["id"], e))
def init_once(cls):
if not mongo.init():
return
mongo.connect()
cls.db = mongo.db
cls.fs = mongo.grid
# Set MongoDB schema version.
if "cuckoo_schema" in mongo.collection_names:
version = mongo.db.cuckoo_schema.find_one()["version"]
if version != cls.SCHEMA_VERSION:
raise CuckooReportError(
"Unknown MongoDB version schema version found. Cuckoo "
"doesn't really know how to proceed now.."
)
else:
mongo.db.cuckoo_schema.save({"version": cls.SCHEMA_VERSION})
# Set an unique index on stored files to avoid duplicates. As per the
# pymongo documentation this is basically a no-op if the index already
# exists. So we don't have to do that check ourselves.
mongo.db.fs.files.ensure_index(
"sha256", unique=True, sparse=True, name="sha256_unique"
)
def run(self, results):
"""Writes report.
@param results: Cuckoo results dict.
@raise CuckooReportError: if fails to write report.
"""
# Determine whether we want to include the behavioral data in the
# JSON report.
if "json.calls" in self.task["options"]:
self.calls = int(self.task["options"]["json.calls"])
else:
self.calls = self.options.get("calls", True)
self.erase_calls(results)
try:
filepath = os.path.join(self.reports_path, "report.json")
with open(filepath, "wb", buffering=1024 * 1024) as report:
json.dump(results,
report,
default=default,
sort_keys=False,
indent=self.options.indent,
encoding="latin-1")
except (TypeError, IOError) as e:
raise CuckooReportError("Failed to generate JSON report: %s" % e)
finally:
self.restore_calls(results)
def run(self, results):
report = self.generate_jinja2_template(results)
if self.options.get("html"):
report_path = os.path.join(self.reports_path, "report.html")
try:
report_data = report
except:
report_data = report.encode('utf-8', 'replace')
codecs.open(report_path, "wb").write(report_data)
if self.options.get("pdf"):
if not HAVE_WEASYPRINT:
raise CuckooReportError(
"The weasyprint library hasn't been installed on your "
"Operating System and as such we can't generate a PDF "
"report for you. You can install 'weasyprint' manually "
"by running 'pip install weasyprint' or by compiling and "
"installing package yourself.")
report_path = os.path.join(self.reports_path, "report.pdf")
try:
f = weasyprint.HTML(io.BytesIO(report.encode("utf8")))
except:
encoded_report = report.encode("utf-8")
f = weasyprint.HTML(io.BytesIO(encoded_report))
f.write_pdf(report_path)
def run(self, results):
try:
dnspacket = getDNSData(self.analysis_path)
goodIPs = getMicrosoftDomains(self.analysis_path)
synConn = getSYNInfo(self.analysis_path, goodIPs)
synackconn = getSYNACKInfo(self.analysis_path, goodIPs)
ackConn = getACKInfo(self.analysis_path, goodIPs)
resolvedIPsArray = resolvedIPs(self.analysis_path, goodIPs)
fullHTTPArray = getFullHTTP(self.analysis_path, dnspacket)
udpconn = getUDPData(self.analysis_path, goodIPs)
icmpPacket = getICMPData(self.analysis_path)
ftpconn = getFTPConn(self.analysis_path)
sshconn = getSSHConn(self.analysis_path)
foundIPs = findStaticIPs(results["strings"])
if synConn!=[] or synackconn!=[] or ackConn!=[] or resolvedIPsArray!=[] or fullHTTPArray!=[] or udpconn!=[] or dnspacket!=[] or icmpPacket!=[] or ftpconn!=[] or sshconn!=[] or foundIPs!=[]:
gatherIOCs(self.analysis_path, synConn, synackconn, ackConn,
resolvedIPsArray, results, fullHTTPArray, udpconn,
dnspacket, icmpPacket, ftpconn, sshconn, foundIPs)
else:
print "No IOCs to create"
except (UnicodeError, TypeError, IOError) as e:
print "Error", e
raise CuckooReportError("Failed to make STIX IOCs :(")
return True
class VisualizeJsonReport(Report):
"""Visualize data in json report file."""
# Make file run last, after analysis has completed
order = 3
def run(self, results):
"""Writes report.
@param results: Cuckoo results dict.
@raise CuckooReportError: if fails to write report.
"""
try:
# Get home directory path
home_path = expanduser("~")
# Full path to json report file
report_path = os.path.join(self.reports_path, "report.json")
# Full path to application visualize_json_report
script_path = os.path.join(home_path, ".cuckoo/visualize_json_report/run.py")
# Full path to location to store html file created, same directory as report json
new_file_path = os.path.join(self.reports_path, "report.html")
# Get choice from config file
choice = self.options['report_type']
print("The visualisation option is set to {}".format(choice))
parameters = list()
"""
Option 1: Visualize processes only
Option 2: Visualize processes and network
Option 3: Visualize processes and files
Option 4: Visualize processes and registry
Option 5: Visualize all data
Default : Visualize processes only
"""
if choice == int(1):
print("Visualize processes only")
parameters = ["/usr/bin/python3", script_path, "-f", new_file_path, "-fa", "-ra", "-na", report_path]
elif choice == int(2):
print("Visualize processes and network data")
parameters = ["/usr/bin/python3", script_path, "-f", new_file_path, "-fa", "-ra", report_path]
elif choice == int(3):
print("Visualize processes and file data")
parameters = ["/usr/bin/python3", script_path, "-f", new_file_path, "-na", "-ra", report_path]
elif choice == int(4):
print("Visualize processes and registry data")
parameters = ["/usr/bin/python3", script_path, "-f", new_file_path, "-fa", "-na", report_path]
elif choice == int(5):
print("Visualize all data")
parameters = ["/usr/bin/python3", script_path, "-f", new_file_path, report_path]
else:
print("Visualize processes only, this is the default mode")
parameters = ["/usr/bin/python3", script_path, "-f", new_file_path, "-fa", "-ra", "-na", report_path]
subprocess.call(parameters)
except (UnicodeError, TypeError, IOError) as e:
raise CuckooReportError("Failed to generate visualisation graph: %s" % e)
def run(self, results):
try:
report = codecs.open(os.path.join(self.reports_path, "result.yar"), "w", "utf-8")
rule = "import \"cuckoo\"\n\n"
rule += self.httpRequest(results)
rule += self.dnsLookup(results)
report.write(rule)
report.close()
except (UnicodeError, TypeError, IOError) as e:
raise CuckooReportError("Failed to generate Yara rule: %s" % e)
def run(self, results):
post = {
"identifier":
self.options.get("identifier"),
"data":
json.dumps(results.get("info"), default=default, sort_keys=False)
}
try:
requests.post(self.options.get("url"), data=post)
except Exception as e:
raise CuckooReportError(
"Failed posting message via Notification: %s" % e)
def do_index(self, obj):
base_document = self.get_base_document()
# Append the base document to the object to index.
base_document.update(obj)
try:
elastic.client.index(index=self.dated_index,
doc_type=self.report_type,
body=base_document)
except Exception as e:
raise CuckooReportError(
"Failed to save results in ElasticSearch for "
"task #%d: %s" % (self.task["id"], e))
def set_path(self, analysis_path):
"""Set analysis folder path.
@param analysis_path: analysis folder path.
"""
self.analysis_path = analysis_path
self.file_path = os.path.realpath(self._get_analysis_path("binary"))
self.reports_path = self._get_analysis_path("reports")
self.shots_path = self._get_analysis_path("shots")
self.pcap_path = self._get_analysis_path("dump.pcap")
try:
Folders.create(self.reports_path)
except CuckooOperationalError as e:
raise CuckooReportError(e)
def run(self, results):
post = {
"task_id": self.task["id"],
"identifier": self.options.get("identifier"),
"data": json.dumps(
results.get("info"), default=default, sort_keys=False
)
}
try:
urls = self.options.get("url").split(',')
for url in urls:
requests.post(url.strip(), data=post)
except Exception as e:
raise CuckooReportError(
"Failed posting message via Notification: %s" % e
)
def init_once(cls):
"""Connect to Elasticsearch.
@raise CuckooReportError: if unable to connect.
"""
# Do not change these types without changing the elasticsearch
# template as well.
cls.report_type = "cuckoo"
cls.call_type = "call"
if not elastic.init():
return
cls.template_name = "%s_template" % elastic.index
try:
elastic.connect()
except CuckooOperationalError as e:
raise CuckooReportError(
"Error running ElasticSearch reporting module: %s" % e)
def apply_template(cls):
template_path = cwd("elasticsearch", "template.json")
if not os.path.exists(template_path):
return False
try:
template = json.loads(open(template_path, "rb").read())
except ValueError:
raise CuckooReportError(
"Unable to read valid JSON from the ElasticSearch "
"template JSON file located at: %s" % template_path)
# Create an index wildcard based off of the index name specified
# in the config file, this overwrites the settings in
# template.json.
template["template"] = elastic.index + "-*"
# if the template does not already exist then create it
if not elastic.client.indices.exists_template(cls.template_name):
elastic.client.indices.put_template(name=cls.template_name,
body=json.dumps(template))
return True
def run(self, results):
"""
Sends Cuckoo report as a DXL event on a DXL Fabric.
@param results: Cuckoo results dict.
@raise CuckooReportError: if fails to send Cuckoo report as a DXL event.
"""
try:
# Diction of data to send out as the report on DXL
report_dict = {}
if self.options.get("send_compressed_event", False):
# Convert results to JSON string
report_json_string = json.dumps(
results,
default=serialize_datetime_objects,
indent=self.options.indent,
encoding="UTF-8")
# Compress the Cuckoo results
zlib_obj = zlib.compressobj(-1, zlib.DEFLATED, 31)
compressed_report_data = zlib_obj.compress(
report_json_string) + zlib_obj.flush()
# Create the DXL Event for zipped data
zipped_event = Event(CUCKOO_ZIP_EVENT_TOPIC)
# Set the payload to be the compressed Cuckoo report analysis
zipped_event.payload = compressed_report_data
# Publish the full zipped reported if the payload size is smaller than the maximum configured size.
if sys.getsizeof(zipped_event.payload) <= self.options.get(
"compressed_event_max_size", 512000):
log.info(
"Publishing full zipped report to DXL on topic %s",
CUCKOO_ZIP_EVENT_TOPIC)
cuckoo_dxl_client.client.send_event(zipped_event)
else:
log.info(
"Report too large. Not publishing zipped report to DXL."
)
# Add the info and target entries from the Cuckoo results
report_dict[INFO_REPORT_KEY] = results.get(INFO_REPORT_KEY, {})
report_dict[TARGET_REPORT_KEY] = results.get(TARGET_REPORT_KEY, {})
# Add items listed from the "items_to_include_in_report" setting in the report.conf to the report
items_to_include_in_report = self.options.get(
"items_to_include_in_report")
if items_to_include_in_report is not None:
# Get rid of any white space characters in the items_to_include_in_report string
items_to_include_in_report = re.sub(
r"\s+", "", items_to_include_in_report)
# Loop over list of items to include
for report_include_item in items_to_include_in_report.split(
","):
if not report_include_item:
log.warn(
"items_to_include_in_report includes an emtpy item."
)
continue
# Separate report_include_item in to sub items
sub_sections_list = report_include_item.split(".")
# Find the value in the Cuckoo results dictionary
sub_section_value = reduce(sub_level_getter,
sub_sections_list, results)
if sub_section_value is NOT_FOUND_OBJ:
log.warn(report_include_item +
" is not found in the Cuckoo report.")
continue
# Create all of the sub item levels in the results reports dictionary
result_sub_section = reduce(create_and_get_sub_level,
sub_sections_list[0:-1],
report_dict)
# Add the value found in the Cuckoo results
result_sub_section.update(
{sub_sections_list[-1]: sub_section_value})
# Create the DXL Event
report_event = Event(CUCKOO_REPORT_EVENT_TOPIC)
# Set event payload to be the JSON of the results report dictionary
report_event.payload = json.dumps(
report_dict,
default=serialize_datetime_objects).encode("UTF-8")
# Publish the Event to the DXL Fabric
log.info("Publishing Cuckoo report to DXL on topic %s",
CUCKOO_REPORT_EVENT_TOPIC)
cuckoo_dxl_client.client.send_event(report_event)
except Exception as ex:
log.exception("Error sending Cuckoo report out as a DXL event.")
raise CuckooReportError(
"Failed to send Cuckoo report as a DXL event: %s" % ex)
def run(self, results):
"""Writes report.
@param results: Cuckoo results dict.
@raise CuckooReportError: if fails to write report.
"""
try:
currentMD5 = results["target"]["file"]["md5"]
currentSHA1 = results["target"]["file"]["sha1"]
currentSHA256 = results["target"]["file"]["sha256"]
currentFilename = results["target"]["file"]["name"]
currentTrustLevel = results["info"]["score"]
print currentMD5
print currentSHA1
print currentSHA256
print currentFilename
print currentTrustLevel
if float(currentTrustLevel) < 4.0:
print "Trust Level is " + str(
currentTrustLevel) + ". No update required."
return
print "Opening DXL connection"
with DxlClient(config) as client:
#Connect to DXL fabric
print "Connecting to DXL fabric."
client.connect()
#Create TIE Client
print "Connecting to TIE."
tie_client = TieClient(client)
print "Trust Level is " + str(
currentTrustLevel) + ". Updating TIE."
reputations_dict = \
tie_client.get_file_reputation({
HashType.MD5: currentMD5,
HashType.SHA1: currentSHA1,
HashType.SHA256: currentSHA256
})
print reputations_dict
#Check if there is an enterprise (custom set) reputation
if (reputations_dict[FileProvider.ENTERPRISE]["trustLevel"]==TrustLevel.NOT_SET or \
reputations_dict[FileProvider.ENTERPRISE]["trustLevel"]==TrustLevel.UNKNOWN or \
reputations_dict[FileProvider.ENTERPRISE]["trustLevel"]==TrustLevel.MIGHT_BE_TRUSTED or \
reputations_dict[FileProvider.ENTERPRISE]["trustLevel"]==TrustLevel.MOST_LIKELY_TRUSTED):
print "Current Trust Level is" + str(reputations_dict[
FileProvider.ENTERPRISE]["trustLevel"])
#also, let's make sure GTI trustLevels are either not being queried, or set to Unknown
#we are nesting for clarity
if (FileProvider.GTI not in reputations_dict.keys()
or reputations_dict[FileProvider.GTI]
== TrustLevel.UNKNOWN):
print "GTI either does not exist or set to UNKNOWN"
# If not set, go ahead and set it
tie_client.set_file_reputation(
TrustLevel.MOST_LIKELY_MALICIOUS, {
HashType.MD5: currentMD5,
HashType.SHA1: currentSHA1,
HashType.SHA256: currentSHA256
},
filename=currentFilename,
comment=
"Reputation set via OpenDXL Cuckoo Integration. Cuckoo scored this sample a "
+ str(currentTrustLevel) + " out of 10.")
print "Reputation set for: " + str(
currentFilename) + ": " + currentMD5
except (TypeError, IOError) as e:
raise CuckooReportError("Failed to update TIE with results: %s" %
e)
