def run(self):
"""Run extract of printable strings.
@return: list of printable strings.
"""
self.key = "strings"
strings = []
if self.task["category"] == "file":
if not os.path.exists(self.file_path):
raise CuckooProcessingError(
"Sample file doesn't exist: \"%s\"" % self.file_path)
try:
data = open(self.file_path, "rb").read(self.MAX_FILESIZE)
except (IOError, OSError) as e:
raise CuckooProcessingError("Error opening file %s" % e)
strings = []
for s in re.findall(b"[\x1f-\x7e]{6,}", data):
strings.append(s.decode("utf-8"))
for s in re.findall(b"(?:[\x1f-\x7e][\x00]){6,}", data):
strings.append(s.decode("utf-16le"))
# Now limit the amount & length of the strings.
strings = strings[:self.MAX_STRINGCNT]
for idx, s in enumerate(strings):
strings[idx] = s[:self.MAX_STRINGLEN]
return strings
def process_pcap_binary(self):
"""Process a PCAP file with Suricata by running Suricata.
Using the socket mode is preferred as the plain binary mode requires
Suricata to load all its rules for every PCAP file and thus takes a
couple of performance heavy seconds to set itself up.
"""
if not os.path.isfile(self.suricata):
raise CuckooProcessingError("Unable to locate Suricata binary")
if not os.path.isfile(self.config_path):
raise CuckooProcessingError(
"Unable to locate Suricata configuration")
args = [
self.suricata,
"-c",
self.config_path,
"-k",
"none",
"-l",
self.suricata_path,
"-r",
self.pcap_path,
]
try:
subprocess.check_call(args)
except subprocess.CalledProcessError as e:
raise CuckooProcessingError(
"Suricata returned an error processing this pcap: %s" % e)
def run(self):
"""Run VirusTotal processing
@return: full VirusTotal report.
"""
self.key = "virustotal"
if not config("processing:virustotal:key"):
raise CuckooProcessingError(
"VirusTotal API key not configured, skipping the VirusTotal "
"processing module.")
self.vt = VirusTotalAPI()
# Scan the original sample or URL.
if self.task["category"] == "file":
results = self.scan_file(self.file_path)
elif self.task["category"] == "url":
results = self.scan_url(self.task["target"])
elif self.task["category"] == "baseline":
return
elif self.task["category"] == "service":
return
else:
raise CuckooProcessingError("Unsupported task category: %s" %
self.task["category"])
# Scan any dropped files that have an interesting filetype.
for row in self.results.get("dropped", []):
if not self.should_scan_file(row["type"]):
continue
row["virustotal"] = self.scan_file(row["path"], summary=True)
return results
def run(self):
"""Run androguard to extract static android information
@return: list of static features
"""
self.key = "apkinfo"
apkinfo = {}
if "file" not in self.task["category"]:
return
f = File(self.task["target"])
if f.get_name().endswith((".zip", ".apk")) or "zip" in f.get_type():
if not os.path.exists(self.file_path):
raise CuckooProcessingError(
"Sample file doesn't exist: \"%s\"" % self.file_path)
try:
a = APK(self.file_path)
if a.is_valid_APK():
manifest = {}
apkinfo["files"] = self._apk_files(a)
manifest["package"] = a.get_package()
# manifest["permissions"]=a.get_details_permissions_new()
manifest["main_activity"] = a.get_main_activity()
manifest["activities"] = a.get_activities()
manifest["services"] = a.get_services()
manifest["receivers"] = a.get_receivers()
# manifest["receivers_actions"]=a.get__extended_receivers()
manifest["providers"] = a.get_providers()
manifest["libraries"] = a.get_libraries()
apkinfo["manifest"] = manifest
# apkinfo["certificate"] = a.get_certificate()
static_calls = {}
if self.check_size(apkinfo["files"]):
vm = DalvikVMFormat(a.get_dex())
vmx = uVMAnalysis(vm)
static_calls["all_methods"] = self.get_methods(vmx)
static_calls[
"is_native_code"] = analysis.is_native_code(vmx)
static_calls["is_dynamic_code"] = analysis.is_dyn_code(
vmx)
static_calls[
"is_reflection_code"] = analysis.is_reflection_code(
vmx)
# static_calls["dynamic_method_calls"]= analysis.get_show_DynCode(vmx)
# static_calls["reflection_method_calls"]= analysis.get_show_ReflectionCode(vmx)
# static_calls["permissions_method_calls"]= analysis.get_show_Permissions(vmx)
# static_calls["crypto_method_calls"]= analysis.get_show_CryptoCode(vmx)
# static_calls["native_method_calls"]= analysis.get_show_NativeMethods(vmx)
else:
log.warning("Dex size bigger than: %s",
self.options.decompilation_threshold)
apkinfo["static_method_calls"] = static_calls
except (IOError, OSError, zipfile.BadZipfile) as e:
raise CuckooProcessingError("Error opening file %s" % e)
return apkinfo
def run(self):
"""Run Google play unofficial python api the get the google play information
@return: list of google play features
"""
self.key = "googleplay"
googleplay = {}
if not HAVE_GOOGLEPLAY:
log.error("Unable to import the GooglePlay library, has it been "
"installed properly?")
return
if "file" not in self.task["category"]:
return
f = File(self.task["target"])
if f.get_name().endswith((".zip", ".apk")) or "zip" in f.get_type():
if not os.path.exists(self.file_path):
raise CuckooProcessingError("Sample file doesn't exist: \"%s\"" % self.file_path)
android_id = self.options.get("android_id")
google_login = self.options.get("google_login")
google_password = self.options.get("google_password")
# auth_token = self.options.get("auth_token", None)
if not android_id and not google_login and not google_password:
raise CuckooProcessingError("Google Play Credentials not configured, skip")
try:
a = APK(self.file_path)
if a.is_valid_APK():
package = a.get_package()
# Connect
api = GooglePlayAPI(android_id)
api.login(google_login, google_password, None)
# Get the version code and the offer type from the app details
app_data = api.details(package)
app_detail = app_data.docV2.details.appDetails
if not app_detail.installationSize:
return googleplay
googleplay["title"] = app_detail.title
googleplay["app_category"] = app_detail.appCategory._values
googleplay["version_code"] = app_detail.versionCode
googleplay["app_type"] = app_detail.appType
googleplay["content_rating"] = app_detail.contentRating
googleplay["developer_email"] = app_detail.developerEmail
googleplay["developer_name"] = app_detail.developerName
googleplay["developer_website"] = app_detail.developerWebsite
googleplay["installation_size"] = app_detail.installationSize
googleplay["num_downloads"] = app_detail.numDownloads
googleplay["upload_date"] = app_detail.uploadDate
googleplay["permissions"] = app_detail.permission._values
except (IOError, OSError, zipfile.BadZipfile) as e:
raise CuckooProcessingError("Error opening file %s" % e)
return googleplay
def run(self, results):
self.moloch_capture = \
self.options.get("moloch_capture", "/data/moloch/bin/moloch-capture")
self.config_path = self.options.get("conf",
"/data/moloch/etc/config.ini")
self.instance = self.options.get("instance", "cuckoo")
if not os.path.isfile(self.pcap_path):
log.warning("Unable to run Moloch as no pcap is available")
return
if not os.path.isfile(self.moloch_capture):
raise CuckooProcessingError("Unable to locate Moloch binary")
if not os.path.isfile(self.config_path):
raise CuckooProcessingError(
"Unable to locate Moloch configuration")
args = [
self.moloch_capture,
"-c",
self.config_path,
"-r",
self.pcap_path,
"-n",
self.instance,
"-q",
]
tags = {}
tags[self.instance] = self.task["id"]
if self.task["category"] == "file":
# Tag file hashes.
f = results.get("target", {}).get("file", {})
for field in ("md5", "sha1", "sha256", "sha512"):
if field in f:
tags[field] = f[field]
# Tag normalized VirusTotal results.
for variant in results.get("virustotal", {}).get("normalized", []):
tags["virustotal"] = variant
for key, value in tags.items():
args += [
"-t",
"%s:%s" % (key, value),
]
try:
subprocess.check_call(args)
except subprocess.CalledProcessError as e:
raise CuckooProcessingError("Error submitting PCAP to Moloch: %s" %
e)
def run(self):
self.key = "sysmon"
# Determine oldest sysmon log and remove the rest
lastlog = os.listdir("%s/sysmon/" % self.analysis_path)
lastlog.sort()
lastlog = lastlog[-1]
# Leave only the most recent file
for f in os.listdir("%s/sysmon/" % self.analysis_path):
if f != lastlog:
try:
os.remove("%s/sysmon/%s" % (self.analysis_path, f))
except:
log.error("Failed to remove sysmon file log %s" % f)
os.rename(
"%s/sysmon/%s" % (self.analysis_path, lastlog),
"%s/sysmon/sysmon.xml" % self.analysis_path
)
data = None
try:
xml = open("%s/sysmon/sysmon.xml" % self.analysis_path).read()
xml = xml.decode("latin1").encode("utf8")
data = xmltodict.parse(xml)["Events"]["Event"]
except Exception as e:
raise CuckooProcessingError("Failed parsing sysmon.xml: %s" % e.message)
return self.remove_noise(data)
def run(self):
self.key = "sysmon"
try:
with open('%s/sysmon/sysmon.xml' % self.analysis_path) as xmlfile:
data = xmltodict.parse(xmlfile)
except Exception as e:
raise CuckooProcessingError("Failed opening sysmon log: %s"
& e.message)
clean = {}
for event in data['root']['Event']:
clean[event['System']['EventRecordID']] = {}
clean[event['System']['EventRecordID']]['System'] = {}
clean[event['System']['EventRecordID']]['EventData'] = {}
for k, v in event['System'].items():
clean[event['System']['EventRecordID']]['System'][k] = v
for eventdata in event['EventData']['Data']:
clean[event['System']['EventRecordID']]['EventData'][
eventdata['@Name']] = eventdata.get('#text', None)
with open('%s/sysmon/sysmon.json' % self.analysis_path,
'w') as dump_file:
json.dump(clean, dump_file)
return clean
def process_pcap_socket(self):
"""Process a PCAP file with Suricata in socket mode."""
if not HAVE_SURICATASC:
raise CuckooProcessingError(
"Suricata has been configured to run in socket mode but "
"suricatasc has not been installed, please re-install "
"Suricata or SuricataSC"
)
if not os.path.exists(self.socket):
raise CuckooProcessingError(
"Suricata has been configured to run in socket mode "
"but the socket is unavailable"
)
suri = suricatasc.SuricataSC(self.socket)
try:
suri.connect()
except suricatasc.SuricataException as e:
raise CuckooProcessingError(
"Error connecting to Suricata in socket mode: %s" % e
)
# Submit the PCAP file.
ret = suri.send_command("pcap-file", {
"filename": self.pcap_path,
"output-dir": self.suricata_path,
})
if not ret or ret["return"] != "OK":
raise CuckooProcessingError(
"Error submitting PCAP file to Suricata in socket mode, "
"return value: %s" % ret
)
# TODO Should we add a timeout here? If we do so we should also add
# timeout logic to the binary mode.
while True:
ret = suri.send_command("pcap-current")
# When the pcap file has been processed the "current pcap" file
# will be none.
if ret and ret["message"] == "None":
break
time.sleep(1)
def run(self, results):
"""Submits results to MISP.
@param results: Cuckoo results dict.
"""
url = self.options.get("url")
apikey = self.options.get("apikey")
mode = shlex.split(self.options.get("mode") or "")
if not url or not apikey:
raise CuckooProcessingError(
"Please configure the URL and API key for your MISP instance."
)
self.misp = pymisp.PyMISP(url, apikey, False, "json")
sample_filename=os.path.basename(self.task["target"])
distribution = self.options.get("distribution") or 0
threat_level = self.options.get("threat_level") or 4
analysis = self.options.get("analysis") or 0
event = self.misp.new_event(
distribution = distribution,
threat_level_id = threat_level,
analysis = analysis,
info="Cuckoo Sandbox analysis for %s (#%d)" % (sample_filename, self.task["id"]),
)
if results.get("target", {}).get("category") == "file":
self.misp.upload_sample(
filename=sample_filename,
filepath_or_bytes=self.task["target"],
event_id=event["Event"]["id"],
distribution=distribution,
to_ids=False,
category="External analysis",
)
if "hashes" in mode:
self.sample_hashes(results, event)
if "maldoc" in mode:
self.maldoc_network(results, event)
if "url" in mode:
self.all_urls(results, event)
if "ipaddr" in mode:
self.domain_ipaddr(results, event)
if "dropped" in mode:
self.dropped_hashes(results, event)
tag = self.options.get("tag")
if tag:
results = self.misp.add_tag(event, tag)
if results.has_key('message'):
log.warning("Cannot tag event: %s" % results['message'])
def run(self):
self.key = "mlmd"
try:
if os.path.exists(self.file_path):
#extract PE data, send to MLMD server, set data=reply
features = feature_extractor.get_features(self.file_path)
res = requests.post("http://localhost:8080/ML",json=features)
data = res.json()
except SomethingFailed:
raise CuckooProcessingError("Failed")
return data
def iplayer_from_raw(raw, linktype=1):
"""Converts a raw packet to a dpkt packet regarding of link type.
@param raw: raw packet
@param linktype: integer describing link type as expected by dpkt
"""
if linktype == 1: # ethernet
try:
pkt = dpkt.ethernet.Ethernet(raw)
return pkt.data
except dpkt.NeedData:
pass
elif linktype == 101: # raw
return dpkt.ip.IP(raw)
else:
raise CuckooProcessingError("unknown PCAP linktype")
def run(self, results):
"""Submits results to MISP.
@param results: Cuckoo results dict.
"""
url = self.options.get("url")
apikey = self.options.get("apikey")
mode = shlex.split(self.options.get("mode") or "")
if not url or not apikey:
raise CuckooProcessingError(
"Please configure the URL and API key for your MISP instance.")
with warnings.catch_warnings():
warnings.simplefilter("ignore")
import pymisp
self.misp = pymisp.PyMISP(url, apikey, False, "json")
event = self.misp.new_event(
distribution=pymisp.Distribution.all_communities.value,
threat_level_id=pymisp.ThreatLevel.undefined.value,
analysis=pymisp.Analysis.completed.value,
info="Cuckoo Sandbox analysis #%d" % self.task["id"],
)
if results.get("target", {}).get("category") == "file":
self.misp.upload_sample(
filename=os.path.basename(self.task["target"]),
filepath_or_bytes=self.task["target"],
event_id=event["Event"]["id"],
category="External analysis",
)
self.signature(results, event)
if "hashes" in mode:
self.sample_hashes(results, event)
if "url" in mode:
self.all_urls(results, event)
if "ipaddr" in mode:
self.domain_ipaddr(results, event)
self.family(results, event)
def run(self):
"""Run analysis.
@return: MISP results dict.
"""
self.url = self.options.get("url", "")
self.apikey = self.options.get("apikey", "")
maxioc = int(self.options.get("maxioc", 100))
if not self.url or not self.apikey:
raise CuckooProcessingError(
"Please configure the URL and API key for your MISP instance."
)
self.key = "misp"
self.iocs = {}
with warnings.catch_warnings():
warnings.simplefilter("ignore")
import pymisp
self.misp = pymisp.PyMISP(self.url, self.apikey, False, "json")
iocs = set()
iocs.add(self.results.get("target", {}).get("file", {}).get("md5"))
for dropped in self.results.get("dropped", []):
iocs.add(dropped.get("md5"))
iocs.update(self.results.get("network", {}).get("hosts", []))
for block in self.results.get("network", {}).get("domains", []):
iocs.add(block.get("ip"))
iocs.add(block.get("domain"))
# Remove empty entries and turn the collection into a list.
iocs = list(iocs.difference((None, "")))
# Acquire all information related to IOCs.
for ioc in iocs[:maxioc]:
self.search_ioc(ioc)
# Sort IOC information by date and return all information.
return sorted(
self.iocs.values(), key=self._parse_date, reverse=True
)
def run(self, results):
"""Submits results to MISP.
@param results: Cuckoo results dict.
"""
url = self.options.get("url")
apikey = self.options.get("apikey")
mode = shlex.split(self.options.get("mode") or "")
if not url or not apikey:
raise CuckooProcessingError(
"Please configure the URL and API key for your MISP instance."
)
self.misp = pymisp.PyMISP(url, apikey, False, "json")
event = self.misp.new_event(
distribution=self.misp.distributions.all_communities,
threat_level_id=self.misp.threat_level.undefined,
analysis=self.misp.analysis.completed,
info="Cuckoo Sandbox analysis #%d" % self.task["id"],
)
if results.get("target", {}).get("category") == "file":
self.misp.upload_sample(
filename=os.path.basename(self.task["target"]),
filepath=self.task["target"],
event_id=event["Event"]["id"],
category="External analysis",
)
if "hashes" in mode:
self.sample_hashes(results, event)
if "maldoc" in mode:
self.maldoc_network(results, event)
if "url" in mode:
self.all_urls(results, event)
if "ipaddr" in mode:
self.domain_ipaddr(results, event)
def run(self):
self.key = "snort"
results = {
"alerts": [],
}
self.snort = self.options.get("snort", "/usr/local/bin/snort")
self.config_path = self.options.get("conf", "/etc/snort/snort.conf")
if not os.path.isfile(self.pcap_path):
log.warning("Unable to run Snort as no pcap is available")
return results
if not os.path.isfile(self.snort):
raise CuckooProcessingError("Unable to locate Snort binary")
if not os.path.isfile(self.config_path):
raise CuckooProcessingError("Unable to locate Snort configuration")
args = [
self.snort,
"-c",
self.config_path,
"-A",
"console",
"-r",
self.pcap_path,
"-q",
"-y",
]
try:
output = subprocess.check_output(args)
except subprocess.CalledProcessError as e:
raise CuckooProcessingError(
"Snort returned an error processing this pcap: %s" % e)
for line in output.split("\n"):
if not line:
continue
x = self.alert_re.match(line)
if not x:
log.warning("Error matching Snort line: %r", line)
continue
timestamp = datetime.datetime.strptime(x.group("timestamp"),
"%m/%d/%y-%H:%M:%S.%f")
if ":" in x.group("src"):
src_ip, src_port = x.group("src").rsplit(":", 1)
else:
src_ip = x.group("src")
src_port = None
if ":" in x.group("dest"):
dst_ip, dst_port = x.group("dest").rsplit(":", 1)
else:
dst_ip = x.group("dest")
dst_port = None
results["alerts"].append({
"timestamp":
timestamp,
"sid":
int(x.group("sid")),
"priority":
int(x.group("priority")),
"revision":
int(x.group("revision")),
"message":
x.group("message"),
"src_ip":
src_ip,
"src_port":
int(src_port) if src_port else None,
"dst_ip":
dst_ip,
"dst_port":
int(dst_port) if dst_port else None,
"protocol":
x.group("protocol"),
"classtype":
x.group("classtype"),
})
return results
def run(self):
self.key = "curtain"
# Remove some event entries which are commonly found in all samples (noise reduction)
noise = [
"$global:?", "# Compute file-hash using the crypto object",
"# Construct the strongly-typed crypto object",
"HelpInfoURI = 'http://go.microsoft.com/fwlink/?linkid=285758'",
"[System.Management.ManagementDateTimeConverter]::ToDmtfDateTime($args[0])",
"[System.Management.ManagementDateTimeConverter]::ToDateTime($args[0])",
"Set-Location Z:", "Set-Location Y:", "Set-Location X:",
"Set-Location W:", "Set-Location V:", "Set-Location U:",
"Set-Location T:", "Set-Location S:", "Set-Location R:",
"Set-Location Q:", "Set-Location P:", "Set-Location O:",
"Set-Location N:", "Set-Location M:", "Set-Location L:",
"Set-Location K:", "Set-Location J:", "Set-Location I:",
"Set-Location H:", "Set-Location G:", "Set-Location F:",
"Set-Location E:", "Set-Location D:", "Set-Location C:",
"Set-Location B:", "Set-Location A:", "Set-Location ..",
"Set-Location \\",
"$wrappedCmd = $ExecutionContext.InvokeCommand.GetCommand('Out-String',[System.Management.Automation.CommandTypes]::Cmdlet)",
"$str.Substring($str.LastIndexOf('Verbs') + 5)",
"[Parameter(ParameterSetName='nameSet', Position=0, ValueFromPipelineByPropertyName=$true)]",
"[ValidateSet('Alias','Cmdlet','Provider','General','FAQ','Glossary','HelpFile'",
"param([string[]]$paths)",
"$origin = New-Object System.Management.Automation.Host.Coordinates",
"Always resolve file paths using Resolve-Path -Relative.",
"PS $($executionContext.SessionState.Path.CurrentLocation)$('>' * ($nestedPromptLevel + 1))",
"$this.ServiceName",
"Read-Host 'Press Enter to continue...' | Out-Null",
"([System.Management.Automation.CommandTypes]::Script)",
"if ($myinv -and ($myinv.MyCommand -or ($_.CategoryInfo.Category -ne 'ParserError')))",
"CmdletsToExport=@(", "CmdletsToExport", "FormatsToProcess",
"AliasesToExport", "FunctionsToExport", "$_.PSParentPath.Replace",
"$ExecutionContext.SessionState.Path.Combine",
"get-help about_Command_Precedence"
]
# Determine oldest Curtain log and remove the rest
curtLog = os.listdir("%s/curtain/" % self.analysis_path)
curtLog.sort()
curtLog = curtLog[-1]
# Leave only the most recent file
for file in os.listdir("%s/curtain/" % self.analysis_path):
if file != curtLog:
try:
os.remove("%s/curtain/%s" % (self.analysis_path, file))
except:
pass
os.rename("%s/curtain/%s" % (self.analysis_path, curtLog),
"%s/curtain/curtain.log" % self.analysis_path)
try:
tree = ET.parse("%s/curtain/curtain.log" % self.analysis_path)
root = tree.getroot()
except Exception as e:
raise CuckooProcessingError("Failed opening curtain.log: %s" %
e.message)
pids = {}
COUNTER = 0
FILTERED = 0
for i in range(0, len(root)):
# Setup PID Dict
if root[i][0][1].text == "4104":
FILTERFLAG = 0
PID = root[i][0][10].attrib['ProcessID']
#TID = root[i][0][10].attrib['ThreadID']
MESSAGE = root[i][1][2].text
if PID not in pids:
pids[PID] = {"pid": PID, "events": [], "filter": []}
# Checks for unique strings in events to filter out
if MESSAGE != None:
for entry in noise:
if entry in MESSAGE:
FILTERFLAG = 1
FILTERED += 1
pids[PID]["filter"].append(
{str(FILTERED): MESSAGE.strip()})
# Save the record
if FILTERFLAG == 0 and MESSAGE != None:
COUNTER += 1
MODFLAG = 0
# Attempt to further decode token replacement/other common obfuscation
# Original and altered will be saved
ALTMSG = MESSAGE.strip()
if re.search("\x00", ALTMSG):
ALTMSG, MODFLAG = removeNull(ALTMSG, MODFLAG)
if re.search("(\\\"|\\\')", ALTMSG):
ALTMSG, MODFLAG = removeEscape(ALTMSG, MODFLAG)
if re.search("`", ALTMSG):
ALTMSG, MODFLAG = removeTick(ALTMSG, MODFLAG)
if re.search("\^", ALTMSG):
ALTMSG, MODFLAG = removeCaret(ALTMSG, MODFLAG)
while re.search("[\x20]{2,}", ALTMSG):
ALTMSG, MODFLAG = spaceReplace(ALTMSG, MODFLAG)
if re.search("\[[Cc][Hh][Aa][Rr]\][0-9]{1,3}", ALTMSG):
ALTMSG, MODFLAG = charReplace(ALTMSG, MODFLAG)
# One run pre formatReplace
if re.search("(\"\+\"|\'\+\')", ALTMSG):
ALTMSG, MODFLAG = joinStrings(ALTMSG, MODFLAG)
while re.search(
"(\"|\')(\{[0-9]{1,2}\})+(\"|\')[ -fF]+(\'.+?\'\))",
ALTMSG):
ALTMSG, MODFLAG = formatReplace(ALTMSG, MODFLAG)
# One run post formatReplace for new strings
if re.search("(\"\+\"|\'\+\')", ALTMSG):
ALTMSG, MODFLAG = joinStrings(ALTMSG, MODFLAG)
if "replace" in ALTMSG.lower():
try:
ALTMSG, MODFLAG = replaceDecoder(ALTMSG, MODFLAG)
except Exception as e:
log.error(
"Curtain processing error for entry - %s" % e)
# Remove camel case obfuscation as last step
ALTMSG, MODFLAG = adjustCase(ALTMSG, MODFLAG)
if MODFLAG == 0:
ALTMSG = "No alteration of event."
# Save the output
pids[PID]["events"].append({
str(COUNTER): {
"original": MESSAGE.strip(),
"altered": ALTMSG
}
})
remove = []
# Find Curtain PID if it was picked up in log
for pid in pids:
for event in pids[pid]["events"]:
for entry in event.values():
if "Process { [System.Diagnostics.Eventing.Reader.EventLogSession]::GlobalSession.ClearLog" in entry[
"original"]:
if pid not in remove:
remove.append(pid)
# Find empty PID
for pid in pids:
if len(pids[pid]["events"]) == 0:
if pid not in remove:
remove.append(pid)
# Remove PIDs
for pid in remove:
del pids[pid]
# Reorder event counts
for pid in pids:
tempEvents = []
eventCount = len(pids[pid]["events"])
for index, entry in enumerate(pids[pid]["events"]):
tempEvents.append(
{"%02d" % (eventCount - index): entry.values()[0]})
pids[pid]["events"] = tempEvents
tempEvents = []
eventCount = len(pids[pid]["filter"])
for index, entry in enumerate(pids[pid]["filter"]):
tempEvents.append(
{"%02d" % (eventCount - index): entry.values()[0]})
pids[pid]["filter"] = tempEvents
# Identify behaviors per PID
for pid in pids:
behaviorTags = []
for entry in pids[pid]["events"]:
behaviorTags = buildBehaviors(entry, behaviorTags)
pids[pid]["behaviors"] = behaviorTags
return pids
def run(self):
"""Runs TIE processing
@return: TIE results
"""
log.info("Processing TIE reputation analysis.")
self.key = "tie"
timeout = int(self.options.get("timeout", 60))
scan = int(self.options.get("scan", 0))
# Evaluate the original sample against TIE reputation
if self.task["category"] == "file":
# Create the client
with DxlClient(config) as client:
# Connect to the fabric
client.connect()
tie_client = TieClient(client)
#Generate relevant hash information
md5_hex = File(self.file_path).get_md5()
sha1_hex = File(self.file_path).get_sha1()
sha256_hex = File(self.file_path).get_sha256()
#Request raw json reputation results
reputations_dict = \
tie_client.get_file_reputation({
HashType.MD5: md5_hex,
HashType.SHA1: sha1_hex,
HashType.SHA256: sha256_hex
})
#debug
log.info("Raw TIE results: " +
json.dumps(reputations_dict,
sort_keys=True,
indent=4,
separators=(',', ': ')))
#initialize result array and tiekey counter for each result
proc_result = {}
tiekey = 0
strtiekey = str(tiekey)
# Display the Global Threat Intelligence
if FileProvider.GTI in reputations_dict:
gti_rep = reputations_dict[FileProvider.GTI]
proc_result[strtiekey] = {}
proc_result[strtiekey][
'title'] = "Global Threat Intelligence (GTI) Test Date:"
proc_result[strtiekey][
'value'] = EpochMixin.to_localtime_string(
gti_rep[ReputationProp.CREATE_DATE])
tiekey += 1
strtiekey = str(tiekey)
#Set GTI Trust Level
proc_result[strtiekey] = {}
proc_result[strtiekey][
'title'] = "Global Threat Intelligence (GTI) trust level:"
trustValue = gti_rep[ReputationProp.TRUST_LEVEL]
proc_result[strtiekey]['value'] = self.trustLevel(
trustValue)
tiekey += 1
strtiekey = str(tiekey)
# Display the Enterprise reputation information
if FileProvider.ENTERPRISE in reputations_dict:
ent_rep = reputations_dict[FileProvider.ENTERPRISE]
# Retrieve the enterprise reputation attributes
ent_rep_attribs = ent_rep[ReputationProp.ATTRIBUTES]
# Display prevalence (if it exists)
if FileEnterpriseAttrib.PREVALENCE in ent_rep_attribs:
proc_result[strtiekey] = {}
proc_result[strtiekey][
'title'] = "Enterprise prevalence:"
proc_result[strtiekey]['value'] = ent_rep_attribs[
FileEnterpriseAttrib.PREVALENCE]
tiekey += 1
strtiekey = str(tiekey)
# Display first contact date (if it exists)
if FileEnterpriseAttrib.FIRST_CONTACT in ent_rep_attribs:
proc_result[strtiekey] = {}
proc_result[strtiekey]['title'] = "First contact: "
proc_result[strtiekey][
'value'] = FileEnterpriseAttrib.to_localtime_string(
ent_rep_attribs[
FileEnterpriseAttrib.FIRST_CONTACT])
tiekey += 1
strtiekey = str(tiekey)
#These are lookup conversions for the ATD trust_score
valueDict = {}
valueDict['-1'] = "Known Trusted"
valueDict['0'] = "Most Likely Trusted"
valueDict['1'] = "Might Be Trusted"
valueDict['2'] = "Unknown"
valueDict['3'] = "Might Be Malicious"
valueDict['4'] = "Most Likely Malicious"
valueDict['5'] = "Known Malicious"
valueDict['-2'] = "Not Set"
# Display the ATD reputation information
if FileProvider.ATD in reputations_dict:
atd_rep = reputations_dict[FileProvider.ATD]
# Retrieve the ATD reputation attributes
atd_rep_attribs = atd_rep[ReputationProp.ATTRIBUTES]
proc_result[strtiekey] = {}
proc_result[strtiekey]['title'] = "ATD Test Date: "
proc_result[strtiekey][
'value'] = EpochMixin.to_localtime_string(
atd_rep[ReputationProp.CREATE_DATE])
tiekey += 1
strtiekey = str(tiekey)
# Display GAM Score (if it exists)
if AtdAttrib.GAM_SCORE in atd_rep_attribs:
proc_result[strtiekey] = {}
proc_result[strtiekey][
'title'] = "ATD Gateway AntiMalware Score: "
proc_result[strtiekey]['value'] = valueDict[
atd_rep_attribs[AtdAttrib.GAM_SCORE]]
tiekey += 1
strtiekey = str(tiekey)
# Display AV Engine Score (if it exists)
if AtdAttrib.AV_ENGINE_SCORE in atd_rep_attribs:
proc_result[strtiekey] = {}
proc_result[strtiekey][
'title'] = "ATD AV Engine Score: "
proc_result[strtiekey]['value'] = valueDict[
atd_rep_attribs[AtdAttrib.AV_ENGINE_SCORE]]
tiekey += 1
strtiekey = str(tiekey)
# Display Sandbox Score (if it exists)
if AtdAttrib.SANDBOX_SCORE in atd_rep_attribs:
proc_result[strtiekey] = {}
proc_result[strtiekey]['title'] = "ATD Sandbox Score: "
proc_result[strtiekey]['value'] = valueDict[
atd_rep_attribs[AtdAttrib.SANDBOX_SCORE]]
tiekey += 1
strtiekey = str(tiekey)
# Display Verdict (if it exists)
if AtdAttrib.VERDICT in atd_rep_attribs:
proc_result[strtiekey] = {}
proc_result[strtiekey]['title'] = "ATD Verdict: "
proc_result[strtiekey]['value'] = valueDict[
atd_rep_attribs[AtdAttrib.VERDICT]]
tiekey += 1
strtiekey = str(tiekey)
results = proc_result
elif self.task["category"] == "url":
return
elif self.task["category"] == "baseline":
return
elif self.task["category"] == "service":
return
else:
raise CuckooProcessingError("Unsupported task category: %s" %
self.task["category"])
log.info("Finished processing TIE reputation analysis.")
return results
import logging
import json
from cuckoo.common.abstracts import Processing
from cuckoo.common.exceptions import CuckooProcessingError
log = logging.getLogger(__name__)
__author__ = "haam3r"
__version__ = "1.0.0"
__date__ = "2018-09-05"
try:
import xmltodict
except ImportError:
raise CuckooProcessingError('Unable to import required xmltodict module')
class Sysmon(Processing):
"""Parse exported Sysmon XML file to a json file"""
def run(self):
self.key = "sysmon"
try:
with open('%s/sysmon/sysmon.xml' % self.analysis_path) as xmlfile:
data = xmltodict.parse(xmlfile)
except Exception as e:
raise CuckooProcessingError("Failed opening sysmon log: %s"
& e.message)
clean = {}
def run(self, results):
"""Submit results to MISP.
@param results: Cuckoo results dict.
"""
url = self.options.get("url")
apikey = self.options.get("apikey")
mode = shlex.split(self.options.get("mode") or "")
score = results.get("info", {}).get("score", 0)
upload_sample = self.options.get("upload_sample")
if results.get("target", {}).get("category") == "file":
f = results.get("target", {}).get("file", {})
hash_safelisted = is_safelisted_misphash(f["md5"]) or \
is_safelisted_misphash(f["sha1"]) or \
is_safelisted_misphash(f["sha256"])
if hash_safelisted:
return
if score < self.options.get("min_malscore", 0):
return
if not url or not apikey:
raise CuckooProcessingError(
"Please configure the URL and API key for your MISP "
"instance."
)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
import pymisp
self.misp = pymisp.PyMISP(url, apikey, False, "json")
# Get default settings for a new event
distribution = self.options.get("distribution") or 0
threat_level = self.options.get("threat_level") or 4
analysis = self.options.get("analysis") or 0
tag = self.options.get("tag") or "Cuckoo"
event = self.misp.new_event(
distribution=distribution,
threat_level_id=threat_level,
analysis=analysis,
info="Cuckoo Sandbox analysis #%d" % self.task["id"]
)
# Add a specific tag to flag Cuckoo's event
if tag:
mispresult = self.misp.tag(event["Event"]["uuid"], tag)
if mispresult.has_key("message"):
log.debug("tag event: %s" % mispresult["message"])
if upload_sample:
target = results.get("target", {})
if target.get("category") == "file" and target.get("file"):
self.misp.upload_sample(
filename=os.path.basename(self.task["target"]),
filepath_or_bytes=self.task["target"],
event_id=event["Event"]["id"],
category="External analysis",
)
self.signature(results, event)
if "hashes" in mode:
self.sample_hashes(results, event)
if "url" in mode:
self.all_urls(results, event)
if "ipaddr" in mode:
self.domain_ipaddr(results, event)
self.family(results, event)
def run(self):
"""Run analysis.
@return: MISP results dict.
"""
if not HAVE_MISP:
raise CuckooDependencyError(
"Unable to import PyMISP (install with `pip install pymisp`)"
)
self.url = self.options.get("url", "")
self.apikey = self.options.get("apikey", "")
maxioc = int(self.options.get("maxioc", 100))
if not self.url or not self.apikey:
raise CuckooProcessingError(
"Please configure the URL and API key for your MISP instance."
)
self.key = "misp"
self.iocs = {}
self.iocs_network = {}
self.iocs_hashes = {}
self.misp = PyMISP(self.url, self.apikey, False, "json")
iocs = set()
iocs_network = set()
iocs_hashes = set()
iocs.add(self.results.get("target", {}).get("file", {}).get("md5"))
iocs_hashes.add(self.results.get("target", {}).get("file", {}).get("md5"))
for dropped in self.results.get("dropped", []):
iocs.add(dropped.get("md5"))
iocs_hashes.add(dropped.get("md5"))
iocs.update(self.results.get("network", {}).get("hosts", []))
iocs_network.update(self.results.get("network", {}).get("hosts", []))
for block in self.results.get("network", {}).get("domains", []):
iocs.add(block.get("ip"))
iocs_network.add(block.get("ip"))
iocs.add(block.get("domain"))
iocs_network.add(block.get("domain"))
# Remove empty entries and turn the collection into a list.
iocs = list(iocs.difference((None, "")))
iocs_hashes = list(iocs_hashes.difference((None, "")))
iocs_network = list(iocs_network.difference((None, "")))
# Acquire all information related to IOCs.
# hash blacklist not used atm
#for ioc in iocs_hashes[:maxioc]:
#self.search_ioc(ioc, "hash")
for ioc in iocs_network[:maxioc]:
print ioc
self.search_ioc(ioc, "network")
# Sort IOC information by date and return all information.
return sorted(
self.iocs.values(), key=self._parse_date, reverse=True
)
def run(self):
"""Run debug analysis.
@return: debug information dict.
"""
self.key = "debug"
debug = {
"log": [],
"cuckoo": [],
"action": [],
"dbgview": [],
"errors": [],
}
if os.path.exists(self.log_path):
try:
f = codecs.open(self.log_path, "rb", "utf-8")
debug["log"] = f.readlines()
except ValueError as e:
raise CuckooProcessingError("Error decoding %s: %s" %
(self.log_path, e))
except (IOError, OSError) as e:
raise CuckooProcessingError("Error opening %s: %s" %
(self.log_path, e))
else:
log.error(
"Error processing task #%d: it appears that the Virtual "
"Machine (%s) hasn't been able to contact back to "
"the Cuckoo Host. There could be a few reasons for this, "
"please refer to our documentation on the matter: %s",
self.task.id,
self.machine.get("name"),
faq("troubleshooting-vm-network-configuration"),
extra={
"error_action": "vmrouting",
"action": "guest.communication",
"status": "error",
"task_id": self.task.id,
})
if os.path.exists(self.cuckoolog_path):
debug["cuckoo"] = Logfile(self.cuckoolog_path)
dbgview_log = os.path.join(self.analysis_path, "logs", "dbgview.log")
if os.path.exists(dbgview_log):
f = open(dbgview_log, "rb")
# Ignore the first line which identifies the machine.
f.readline()
for line in f:
idx, time, message = line.split("\t", 2)
debug["dbgview"].append(message.strip())
debug["errors"] = []
for error in Database().view_errors(self.task["id"]):
if error.message and error.message not in debug["errors"]:
debug["errors"].append(error.message)
if error.action and error.action not in debug["action"]:
debug["action"].append(error.action)
if os.path.exists(self.mitmerr_path):
mitmerr = open(self.mitmerr_path, "rb").read()
if mitmerr and mitmerr not in debug["errors"]:
debug["errors"].append(mitmerr)
return debug
def run(self):
"""Run Floss on analyzed file.
@return: Floss results dict.
"""
self.key = "strings"
self.floss = self.options.get("floss")
self.MIN_STRINGLEN = int(self.options.get("min_str_len"))
self.MAX_STRINGLEN = self.options.get("max_str_len")
self.MAX_STRINGCNT = self.options.get("max_str_cnt")
self.MAX_FILESIZE = 16*1024*1024
STRING_TYPES = [
"decoded",
"stack",
"static"
]
strings = {}
if self.task["category"] == "file":
if not os.path.exists(self.file_path):
raise CuckooProcessingError(
"Sample file doesn't exist: \"%s\"" % self.file_path
)
try:
f = File(self.file_path)
filename = os.path.basename(self.task["target"])
base_name = os.path.splitext(filename)[0]
ext = filename.split(os.path.extsep)[-1].lower()
data = open(self.file_path, "r").read(self.MAX_FILESIZE)
except (IOError, OSError) as e:
raise CuckooProcessingError("Error opening file %s" % e)
# Extract static strings
static_strings = re.findall("[\x1f-\x7e]{" + str(self.MIN_STRINGLEN) + ",}", data)
for s in re.findall("(?:[\x1f-\x7e][\x00]){" + str(self.MIN_STRINGLEN) + ",}", data):
static_strings.append(s.decode("utf-16le"))
if self.MAX_STRINGLEN != 0:
for i, s in enumerate(static_strings):
static_strings[i] = s[:self.MAX_STRINGLEN]
if self.MAX_STRINGCNT != 0 and len(static_strings) > self.MAX_STRINGCNT:
static_strings = static_strings[:self.MAX_STRINGCNT]
static_strings.append("[snip]")
package = self.task.get("package")
if self.floss and (package == "exe" or ext == "exe" or "PE32" in f.get_type()):
# Disable floss verbose logging
main.set_logging_levels()
try:
# Prepare Floss for extracting hidden & encoded strings
vw = vivisect.VivWorkspace()
vw.loadFromFile(self.file_path)
vw.analyze()
selected_functions = main.select_functions(vw, None)
decoding_functions_candidates = id_man.identify_decoding_functions(
vw, main.get_all_plugins(), selected_functions
)
except Exception as e:
raise CuckooProcessingError("Error analyzing file with vivisect: %s" % e)
try:
# Decode & extract hidden & encoded strings
decoded_strings = main.decode_strings(
vw, decoding_functions_candidates, self.MIN_STRINGLEN
)
decoded_strs = main.filter_unique_decoded(decoded_strings)
stack_strings = stackstrings.extract_stackstrings(
vw, selected_functions, self.MIN_STRINGLEN
)
stack_strings = list(stack_strings)
decoded_strings = [x for x in decoded_strs if not x in static_strings]
except Exception as e:
raise CuckooProcessingError("Error extracting strings with floss: %s" % e)
if len(decoded_strings) or len(stack_strings):
# Create annotated scripts
if self.options.get("idapro_str_sct"):
idapro_sct_name = base_name + ".idb"
strings["idapro_sct_name"] = idapro_sct_name
main.create_ida_script(
self.file_path, os.path.join(self.str_script_path, idapro_sct_name),
decoded_strings, stack_strings
)
if self.options.get("radare_str_sct"):
radare_sct_name = base_name + ".r2"
strings["radare_sct_name"] = radare_sct_name
main.create_r2_script(
self.file_path, os.path.join(self.str_script_path, radare_sct_name),
decoded_strings, stack_strings
)
if self.options.get("x64dbg_str_sct"):
x64dbg_sct_name = base_name + ".json"
strings["x64dbg_sct_name"] = x64dbg_sct_name
imagebase = vw.filemeta.values()[0]['imagebase']
main.create_x64dbg_database(
self.file_path, os.path.join(self.str_script_path, base_name + ".json"),
imagebase, decoded_strings
)
# convert Floss strings into regular, readable strings
for idx, s in enumerate(decoded_strings):
decoded_strings[idx] = main.sanitize_string_for_printing(s.s)
for idx, s in enumerate(stack_strings):
stack_strings[idx] = s.s
results = [decoded_strings, stack_strings, static_strings]
for idx, str_type in enumerate(STRING_TYPES):
strings[str_type] = results[idx]
else:
strings["static"] = static_strings
return strings
