def run(self, obj, config):
start_offset = config['start']
end_offset = config['end']
user = self.current_task.user
if not user.has_access_to(SampleACL.WRITE):
self._info("User does not have permission to add Samples to CRITs")
self._add_result(
"Service Canceled",
"User does not have permission to add Samples to CRITs")
return
# Start must be 0 or higher. If end is greater than zero it must
# also be greater than start_offset.
if start_offset < 0 or (end_offset > 0 and start_offset > end_offset):
self._error("Invalid offsets.")
return
data = obj.filedata.read()[start_offset:end_offset]
if not data:
self._error("No data.")
else:
filename = hashlib.md5(data).hexdigest()
handle_file(filename,
data,
obj.source,
related_id=str(obj.id),
related_type=str(obj._meta['crits_type']),
campaign=obj.campaign,
source_method=self.name,
relationship=RelationshipTypes.CONTAINS,
user=self.current_task.user)
# Filename is just the md5 of the data...
self._add_result("file_added", filename, {'md5': filename})
return
def run(self, obj, config):
try:
self._debug("Version: %s" % pefile.__version__ )
pe = pefile.PE(data=obj.filedata.read())
except pefile.PEFormatError as e:
self._error("A PEFormatError occurred: %s" % e)
return
self._get_sections(pe)
self._get_pehash(pe)
if hasattr(pe, 'DIRECTORY_ENTRY_RESOURCE'):
self._dump_resource_data("ROOT",
pe.DIRECTORY_ENTRY_RESOURCE,
pe,
config['resource'])
for f in self.added_files:
handle_file(f[0], f[1], obj.source,
related_id=str(obj.id),
related_type=str(obj._meta['crits_type']),
campaign=obj.campaign,
method=self.name,
relationship=RelationshipTypes.CONTAINED_WITHIN,
user=self.current_task.username)
rsrc_md5 = hashlib.md5(f[1]).hexdigest()
self._add_result("file_added", f[0], {'md5': rsrc_md5})
else:
self._debug("No resources")
if hasattr(pe, 'DIRECTORY_ENTRY_IMPORT'):
self._get_imports(pe)
else:
self._debug("No imports")
if hasattr(pe, 'DIRECTORY_ENTRY_EXPORT'):
self._get_exports(pe)
else:
self._debug("No exports")
if hasattr(pe, 'VS_VERSIONINFO'):
self._get_version_info(pe)
else:
self._debug("No Version information")
if hasattr(pe, 'DIRECTORY_ENTRY_DEBUG'):
self._get_debug_info(pe)
else:
self._debug("No debug info")
if hasattr(pe, 'DIRECTORY_ENTRY_TLS'):
self._get_tls_info(pe)
else:
self._debug("No TLS info")
if callable(getattr(pe, 'get_imphash', None)):
self._get_imphash(pe)
else:
self._debug("pefile does not support get_imphash, upgrade to 1.2.10-139")
self._get_timestamp(pe)
self._get_rich_header(pe)
def run(self, obj, config):
start_offset = config['start']
end_offset = config['end']
user = self.current_task.user
if not user.has_access_to(SampleACL.WRITE):
self._info("User does not have permission to add Samples to CRITs")
self._add_result("Service Canceled", "User does not have permission to add Samples to CRITs")
return
# Start must be 0 or higher. If end is greater than zero it must
# also be greater than start_offset.
if start_offset < 0 or (end_offset > 0 and start_offset > end_offset):
self._error("Invalid offsets.")
return
data = obj.filedata.read()[start_offset:end_offset]
if not data:
self._error("No data.")
else:
filename = hashlib.md5(data).hexdigest()
handle_file(filename, data, obj.source,
related_id=str(obj.id),
related_type=str(obj._meta['crits_type']),
campaign=obj.campaign,
source_method=self.name,
relationship=RelationshipTypes.CONTAINS,
user=self.current_task.user)
# Filename is just the md5 of the data...
self._add_result("file_added", filename, {'md5': filename})
return
def run(self, obj, config):
start_offset = config['start']
end_offset = config['end']
# Start must be 0 or higher. If end is greater than zero it must
# also be greater than start_offset.
if start_offset < 0 or (end_offset > 0 and start_offset > end_offset):
self._error("Invalid offsets.")
return
data = obj.filedata.read()[start_offset:end_offset]
if not data:
self._error("No data.")
else:
filename = hashlib.md5(data).hexdigest()
handle_file(filename,
data,
obj.source,
related_id=str(obj.id),
related_type=str(obj._meta['crits_type']),
campaign=obj.campaign,
method=self.name,
relationship=RelationshipTypes.CONTAINS,
user=self.current_task.username)
# Filename is just the md5 of the data...
self._add_result("file_added", filename, {'md5': filename})
return
def run(self, obj, config):
self.config = config
self.obj = obj
data = io.BytesIO(obj.filedata.read())
swf = bytearray()
try:
comp = data.read(3)
header = data.read(5)
if comp == 'CWS':
swf = 'FWS' + header + zlib.decompress(data.read())
if comp == 'ZWS':
data.seek(12) # seek to LZMA props
swf = 'FWS' + header + pylzma.decompress(data.read())
except Exception as exc:
self._error("unswf: (%s)." % exc)
return
if swf:
h = md5(str(swf)).hexdigest()
name = h
self._info("New file: %s (%d bytes, %s)" % (name, len(swf), h))
handle_file(name, swf, self.obj.source,
related_id=str(self.obj.id),
campaign=self.obj.campaign,
method=self.name,
relationship='Related_To',
user=self.current_task.username)
self._add_result("file_added", name, {'md5': h})
def _process_dropped(self, dropped):
# Dropped is a byte string of the .tar.bz2 file
self._debug("Processing dropped files.")
self._notify()
# TODO: Error handling
t = tarfile.open(mode='r:bz2', fileobj=BytesIO(dropped))
ignored = self.config.get('ignored_files', '').split('\r\n')
for f in t.getmembers():
if not f.isfile():
continue
data = t.extractfile(f).read()
name = os.path.basename(f.name)
if any([fnmatch.fnmatch(name, x) for x in ignored]):
self._debug("Ignoring file: %s" % name)
continue
h = md5(data).hexdigest()
self._info("New file: %s (%d bytes, %s)" % (name, len(data), h))
handle_file(name,
data,
self.obj.source,
related_id=str(self.obj.id),
related_type=str(self.obj._meta['crits_type']),
campaign=self.obj.campaign,
method=self.name,
relationship=RelationshipTypes.RELATED_TO,
user=self.current_task.username)
self._add_result("file_added", name, {'md5': h})
t.close()
def run(self, obj, config):
self.config = config
self.obj = obj
user = self.current_task.user
data = io.BytesIO(obj.filedata.read())
swf = bytearray()
try:
comp = data.read(3)
header = data.read(5)
if comp == 'CWS':
swf = 'FWS' + header + zlib.decompress(data.read())
if comp == 'ZWS':
data.seek(12) # seek to LZMA props
swf = 'FWS' + header + pylzma.decompress(data.read())
except Exception as exc:
self._error("unswf: (%s)." % exc)
return
if swf:
h = md5(str(swf)).hexdigest()
name = h
if not user.has_access_to(SampleACL.WRITE):
self._info("User does not have permission to add Samples to CRITs")
self._add_result("Extract Canceled", "User does not have permission to add Samples to CRITs")
return
self._info("New file: %s (%d bytes, %s)" % (name, len(swf), h))
handle_file(name, swf, self.obj.source,
related_id=str(self.obj.id),
related_type=str(self.obj._meta['crits_type']),
campaign=self.obj.campaign,
method=self.name,
relationship=RelationshipTypes.RELATED_TO,
user=self.current_task.user)
self._add_result("file_added", name, {'md5': h})
def run(self, obj, config):
self.config = config
self.obj = obj
data = io.BytesIO(obj.filedata.read())
swf = bytearray()
try:
comp = data.read(3)
header = data.read(5)
if comp == 'CWS':
swf = 'FWS' + header + zlib.decompress(data.read())
if comp == 'ZWS':
data.seek(12) # seek to LZMA props
swf = 'FWS' + header + pylzma.decompress(data.read())
except Exception as exc:
self._error("unswf: (%s)." % exc)
return
if swf:
h = md5(str(swf)).hexdigest()
name = h
self._info("New file: %s (%d bytes, %s)" % (name, len(swf), h))
handle_file(name, swf, self.obj.source,
related_id=str(self.obj.id),
related_type=str(self.obj._meta['crits_type']),
campaign=self.obj.campaign,
method=self.name,
relationship=RelationshipTypes.RELATED_TO,
user=self.current_task.username)
self._add_result("file_added", name, {'md5': h})
def _process_dropped(self, dropped):
# Dropped is a byte string of the .tar.bz2 file
self._debug("Processing dropped files.")
self._notify()
# TODO: Error handling
t = tarfile.open(mode='r:bz2', fileobj=StringIO(dropped))
ignored = self.config.get('ignored_files', '').split('\r\n')
for f in t.getmembers():
if not f.isfile():
continue
data = t.extractfile(f).read()
name = os.path.basename(f.name)
if any([fnmatch.fnmatch(name, x) for x in ignored]):
self._debug("Ignoring file: %s" % name)
continue
h = md5(data).hexdigest()
self._info("New file: %s (%d bytes, %s)" % (name, len(data), h))
handle_file(name, data, self.obj.source,
parent_id=str(self.obj.id),
campaign=self.obj.campaign,
method=self.name,
relationship='Related_To',
user=self.current_task.username)
self._add_result("file_added", name, {'md5': h})
t.close()
def run(self, obj, config):
try:
pe = pefile.PE(data=obj.filedata.read())
except pefile.PEFormatError as e:
self._error("A PEFormatError occurred: %s" % e)
return
self._get_sections(pe)
self._get_pehash(pe)
if hasattr(pe, 'DIRECTORY_ENTRY_RESOURCE'):
self._dump_resource_data("ROOT", pe.DIRECTORY_ENTRY_RESOURCE, pe,
config['resource'])
for f in self.added_files:
handle_file(f[0],
f[1],
obj.source,
related_id=str(obj.id),
campaign=obj.campaign,
method=self.name,
relationship='Extracted_From',
user=self.current_task.username)
rsrc_md5 = hashlib.md5(f[1]).hexdigest()
self._add_result("file_added", f[0], {'md5': rsrc_md5})
else:
self._debug("No resources")
if hasattr(pe, 'DIRECTORY_ENTRY_IMPORT'):
self._get_imports(pe)
else:
self._debug("No imports")
if hasattr(pe, 'DIRECTORY_ENTRY_EXPORT'):
self._get_exports(pe)
else:
self._debug("No exports")
if hasattr(pe, 'VS_VERSIONINFO'):
self._get_version_info(pe)
else:
self._debug("No Version information")
if hasattr(pe, 'DIRECTORY_ENTRY_DEBUG'):
self._get_debug_info(pe)
else:
self._debug("No debug info")
if hasattr(pe, 'DIRECTORY_ENTRY_TLS'):
self._get_tls_info(pe)
else:
self._debug("No TLS info")
if callable(getattr(pe, 'get_imphash', None)):
self._get_imphash(pe)
else:
self._debug(
"pefile does not support get_imphash, upgrade to 1.2.10-139")
self._get_timestamp(pe)
self._get_rich_header(pe)
def run(self, obj, config):
"""
Being plugin processing
"""
#Load data from file as libchm will only accept a filename
with self._write_to_file() as chm_file:
try:
self.chmparse.LoadCHM(chm_file)
except Exception as e:
raise e
#Conduct analysis
result = self.analyze()
#Handle output of results
if 'obj_items_summary' in result.keys():
obj_items_summary = result.pop('obj_items_summary')
#General CHM info
for key, value in result.items():
self._add_result('chm_overview', key, {'value': value})
if config['chm_items']:
#Data and details of each object/page in the CHM
user = self.current_task.user
if user.has_access_to(SampleACL.WRITE):
for f in self.added_files:
handle_file(f[0], f[3], obj.source,
related_id=str(obj.id),
related_type=str(obj._meta['crits_type']),
campaign=obj.campaign,
source_method=self.name,
relationship=RelationshipTypes.CONTAINED_WITHIN,
user=self.current_task.user)
self._add_result("chm_items_added", f[0], {'size': f[1],'md5': f[2]})
else:
self._info("User does not have permission to add samples to CRITs.")
self._add_result("chm_items_added","Items found but user does not have permission to add Samples to CRITs.")
else:
#Details of each object/page in the CHM
for object_item in obj_items_summary:
self._add_result('chm_items', object_item.get('name'),
{'size': object_item.get('size'),
'md5': object_item.get('md5')})
#Detection results from CHM analysis
for object_item in obj_items_summary:
if object_item.get('detection'):
for detection in object_item.get('detection'):
self._add_result('chm_detection', detection, {'chm_item': object_item.get('name')})
#URLs and IPs found in CHM
for object_item in obj_items_summary:
if object_item.get('urls'):
for url in object_item.get('urls'):
self._add_result('chm_urls', url, {'chm_item': object_item.get('name')})
#Close file in memory
self.chmparse.CloseCHM()
def run(self, obj, config):
self.config = config
self.obj = obj
# self._doit(obj.filedata.read()[start_offset:end_offset], filters, analyze )
# self._add_result('SEPLQ', "" % output, {'Value': output})
datacq = bytearray(obj.filedata.read())
(metaoutcsv, data) = ExtractPayloads(datacq)
h = md5(data).hexdigest()
name = h
metaout = metaoutcsv.split(",")
name = ntpath.basename(str(metaout[0]))
fields = (
"Filename",
"Num Failed Remediations",
"Num Legacy Infections",
"Num Remediations",
"Num Snapshots",
"Record Type",
"Remediations Type",
"Restore To Orig Locn Unavailable",
"Session Item Count",
"Session Item Index",
"Structure Version",
"Extra Info (QF Time)",
"Extra Info (SND Time)",
"Extra Info (Unique ID",
"Extra Info (VBin Time)",
"Flags",
"Full Path and LFN",
"Log Line",
"Record ID",
"Size",
"Storage Instance ID",
"Storage Key",
"Storage Name",
"WDescription",
"Timestamp (in local time)",
)
self._info("name: %s" % name)
n = 0
for i in metaout:
if i and i != 0 and i != "0" and i != "":
self._info("meta: %s" % str(i))
self._add_result("SEPLQ", str(i))
n += 1
self._info("New file: %s (%d bytes, %s)" % (name, len(data), h))
handle_file(
name,
io.BytesIO(data).read(),
self.obj.source,
related_id=str(obj.id),
campaign=obj.campaign,
method=self.name,
relationship=RelationshipTypes.RELATED_TO,
user=self.current_task.username,
)
self._add_result("file_added", name, {"md5": h})
def run(self, obj, config):
self.config = config
self.obj = obj
#self._doit(obj.filedata.read()[start_offset:end_offset], filters, analyze )
#self._add_result('SEPLQ', "" % output, {'Value': output})
datacq = bytearray(obj.filedata.read())
(metaoutcsv, data) = ExtractPayloads(datacq)
h = md5(data).hexdigest()
name = h
metaout = metaoutcsv.split(",")
name = ntpath.basename(str(metaout[0]))
fields = (
"Filename",
"Num Failed Remediations",
"Num Legacy Infections",
"Num Remediations",
"Num Snapshots",
"Record Type",
"Remediations Type",
"Restore To Orig Locn Unavailable",
"Session Item Count",
"Session Item Index",
"Structure Version",
"Extra Info (QF Time)",
"Extra Info (SND Time)",
"Extra Info (Unique ID",
"Extra Info (VBin Time)",
"Flags",
"Full Path and LFN",
"Log Line",
"Record ID",
"Size",
"Storage Instance ID",
"Storage Key",
"Storage Name",
"WDescription",
"Timestamp (in local time)"
)
self._info("name: %s" % name )
n = 0
for i in metaout:
if i and i != 0 and i != "0" and i != "":
self._info("meta: %s" % str(i))
self._add_result('SEPLQ', str(i))
n+=1
self._info("New file: %s (%d bytes, %s)" % (name, len(data), h))
handle_file(name, io.BytesIO(data).read(), self.obj.source,
related_id=str(obj.id),
related_type=str(obj._meta['crits_type']),
campaign=obj.campaign,
method=self.name,
relationship=RelationshipTypes.RELATED_TO,
user=self.current_task.username)
self._add_result("file_added", name, {'md5': h})
def run(self, obj, config):
upx_path = config.get("upx_path", "")
user = self.current_task.user
if not user.has_access_to(SampleACL.WRITE):
self._info("User does not have permission to add Samples to CRITs")
self._add_result(
"Unpacking Canceled",
"User does not have permission to add Samples to CRITs")
return
# _write_to_file() will delete this file at the end of the "with" block.
with self._write_to_file() as tmp_file:
(working_dir, filename) = os.path.split(tmp_file)
args = [upx_path, "-q", "-d", filename]
# UPX does not generate a lot of output, so we should not have to
# worry about this hanging because the buffer is full
proc = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
cwd=working_dir)
# Note that we are redirecting STDERR to STDOUT, so we can ignore
# the second element of the tuple returned by communicate().
output = proc.communicate()[0]
self._debug(output)
if proc.returncode:
# UPX return code of 1 indicates an error.
# UPX return code of 2 indicates a warning (usually, the
# file was not packed by UPX).
msg = ("UPX could not unpack the file.")
self._warning(msg)
return
with open(tmp_file, "rb") as newfile:
data = newfile.read()
#TODO: check to make sure file was modified (new MD5), indicating
# it was actually unpacked
md5 = hashlib.md5(data).hexdigest()
filename = md5 + ".upx"
handle_file(filename,
data,
obj.source,
related_id=str(obj.id),
related_type=str(obj._meta['crits_type']),
campaign=obj.campaign,
method=self.name,
relationship=RelationshipTypes.PACKED_FROM,
user=self.current_task.user)
# Filename is just the md5 of the data...
self._add_result("file_added", filename, {'md5': filename})
def run(self, obj, config):
oparser = OfficeParser(obj.filedata.read())
oparser.parse_office_doc()
added_files = []
user = self.current_task.user
if not oparser.office_header.get('maj_ver'):
self._error("Could not parse file as an office document")
return
self._add_result(
'office_header', '%d.%d' % (oparser.office_header.get('maj_ver'),
oparser.office_header.get('min_ver')))
for curr_dir in oparser.directory:
result = {
'md5':
curr_dir.get('md5', ''),
'size':
curr_dir.get('stream_size', 0),
'mod_time':
oparser.timestamp_string(curr_dir['modify_time'])[1],
'create_time':
oparser.timestamp_string(curr_dir['create_time'])[1],
}
name = curr_dir['norm_name'].decode('ascii', errors='ignore')
self._add_result('directory', name, result)
if user.has_access_to(SampleACL.WRITE) and config.get(
'save_streams', 0) == 1 and 'data' in curr_dir:
handle_file(name,
curr_dir['data'],
obj.source,
related_id=str(obj.id),
related_type=str(obj._meta['crits_type']),
campaign=obj.campaign,
source_method=self.name,
relationship=RelationshipTypes.CONTAINED_WITHIN,
user=self.current_task.user)
stream_md5 = hashlib.md5(curr_dir['data']).hexdigest()
added_files.append((name, stream_md5))
for prop_list in oparser.properties:
for prop in prop_list['property_list']:
prop_summary = oparser.summary_mapping.get(
binascii.unhexlify(prop['clsid']), None)
prop_name = prop_summary.get('name', 'Unknown')
for item in prop['properties']['properties']:
result = {
'name': item.get('name', 'Unknown'),
'value': item.get('date', item['value']),
'result': item.get('result', ''),
}
self._add_result('doc_meta', prop_name, result)
for f in added_files:
self._add_result("file_added", f[0], {'md5': f[1]})
def process_dropped_binaries(self, dropped_binaries, incident_report):
archive = zipfile.ZipFile(io.BytesIO(dropped_binaries))
archive.setpassword("infected")
names = archive.namelist()
for name in names:
binary = archive.read(name)
ret = handle_file(name, binary, self.obj.source,
related_id=str(self.obj.id),
related_type=str(self.obj._meta['crits_type']),
campaign=self.obj.campaign,
source_method=self.name,
relationship=RelationshipTypes.DROPPED_BY,
user=self.current_task.user,
is_return_only_md5=False)
if ret['success']:
md5 = hashlib.md5(binary).hexdigest()
files_in_report = [f for f in incident_report.findall("./dropped/file/md5/..")
if f.find('md5').text == md5.upper()]
for report_file in files_in_report:
data = {
'md5': md5,
'malicious': getattr(report_file.find('malicious'), 'text', 'unknown'),
}
filepath = getattr(report_file.find('name'), 'text', name)
self._add_result("Dropped Files", filepath, data)
else:
self._warning(ret["message"])
self._notify()
def import_object(request, type_, id_):
setup_access()
if type_ == "Threat Descriptors":
obj = ThreatDescriptor(id=id_)
obj.details(
fields=[f for f in ThreatDescriptor._default_fields if f not in
(td.PRIVACY_MEMBERS, td.SUBMITTER_COUNT, td.METADATA)]
)
itype = getattr(IndicatorTypes, obj.get(td.TYPE))
ithreat_type = getattr(IndicatorThreatTypes, obj.get(td.THREAT_TYPE))
results = handle_indicator_ind(
obj.get(td.RAW_INDICATOR),
"ThreatExchange",
itype,
ithreat_type,
None,
request.user.username,
method="ThreatExchange Service",
reference="id: %s, owner: %s, share_level: %s" % (obj.get(td.ID),
obj.get(td.OWNER)['name'],
obj.get(td.SHARE_LEVEL)),
add_domain=True,
add_relationship=True,
confidence=build_ci(obj.get(td.CONFIDENCE)),
description=obj.get(td.DESCRIPTION)
)
return results
elif type_ == "Malware Analyses":
obj = Malware(id=id_)
obj.details(
fields=[f for f in Malware._fields if f not in
(td.PRIVACY_MEMBERS, td.METADATA)]
)
filename = obj.get(m.MD5)
try:
data = obj.rf
except:
data = None
results = handle_file(
filename,
data,
"ThreatExchange",
method="ThreatExchange Service",
reference="id: %s, share_level: %s" % (obj.get(td.ID),
obj.get(td.SHARE_LEVEL)),
user=request.user.username,
md5_digest = obj.get(m.MD5),
sha1_digest = obj.get(m.SHA1),
sha256_digest = obj.get(m.SHA256),
size = obj.get(m.SAMPLE_SIZE),
mimetype = obj.get(m.SAMPLE_TYPE),
)
return {'success': True,
'md5': results}
else:
return {'success': False,
'message': "Invalid Type"}
return {'success': True}
def import_object(request, type_, id_):
setup_access()
if type_ == "Threat Descriptors":
obj = ThreatDescriptor(id=id_)
obj.details(fields=[
f for f in ThreatDescriptor._default_fields
if f not in (td.PRIVACY_MEMBERS, td.METADATA)
])
itype = get_mapped_itype(obj.get(td.TYPE))
if itype is None:
return {
'success': False,
'message': "Descriptor type is not supported by CRITs"
}
ithreat_type = getattr(IndicatorThreatTypes, obj.get(td.THREAT_TYPE))
results = handle_indicator_ind(
obj.get(td.RAW_INDICATOR),
"ThreatExchange",
itype,
ithreat_type,
None,
request.user.username,
method="ThreatExchange Service",
reference="id: %s, owner: %s, share_level: %s" % (obj.get(
td.ID), obj.get(td.OWNER)['name'], obj.get(td.SHARE_LEVEL)),
add_domain=True,
add_relationship=True,
confidence=build_ci(obj.get(td.CONFIDENCE)),
description=obj.get(td.DESCRIPTION))
return results
elif type_ == "Malware Analyses":
obj = Malware(id=id_)
obj.details(
fields=[f for f in Malware._fields if f not in (m.METADATA)])
filename = obj.get(m.MD5)
try:
data = obj.rf
except:
data = None
results = handle_file(
filename,
data,
"ThreatExchange",
method="ThreatExchange Service",
reference="id: %s, share_level: %s" %
(obj.get(td.ID), obj.get(td.SHARE_LEVEL)),
user=request.user.username,
md5_digest=obj.get(m.MD5),
sha1_digest=obj.get(m.SHA1),
sha256_digest=obj.get(m.SHA256),
size=obj.get(m.SAMPLE_SIZE),
mimetype=obj.get(m.SAMPLE_TYPE),
)
return {'success': True, 'md5': results}
else:
return {'success': False, 'message': "Invalid Type"}
return {'success': True}
def run(self, obj, config):
oparser = OfficeParser(obj.filedata.read())
oparser.parse_office_doc()
added_files = []
user = self.current_task.user
if not oparser.office_header.get('maj_ver'):
self._error("Could not parse file as an office document")
return
self._add_result('office_header', '%d.%d' %
(oparser.office_header.get('maj_ver'), oparser.office_header.get('min_ver')))
for curr_dir in oparser.directory:
result = {
'md5': curr_dir.get('md5', ''),
'size': curr_dir.get('stream_size', 0),
'mod_time': oparser.timestamp_string(curr_dir['modify_time'])[1],
'create_time': oparser.timestamp_string(curr_dir['create_time'])[1],
}
name = curr_dir['norm_name'].decode('ascii', errors='ignore')
self._add_result('directory', name, result)
if user.has_access_to(SampleACL.WRITE) and config.get('save_streams', 0) == 1 and 'data' in curr_dir:
handle_file(name, curr_dir['data'], obj.source,
related_id=str(obj.id),
related_type=str(obj._meta['crits_type']),
campaign=obj.campaign,
source_method=self.name,
relationship=RelationshipTypes.CONTAINED_WITHIN,
user=self.current_task.user)
stream_md5 = hashlib.md5(curr_dir['data']).hexdigest()
added_files.append((name, stream_md5))
for prop_list in oparser.properties:
for prop in prop_list['property_list']:
prop_summary = oparser.summary_mapping.get(binascii.unhexlify(prop['clsid']), None)
prop_name = prop_summary.get('name', 'Unknown')
for item in prop['properties']['properties']:
result = {
'name': item.get('name', 'Unknown'),
'value': item.get('date', item['value']),
'result': item.get('result', ''),
}
self._add_result('doc_meta', prop_name, result)
for f in added_files:
self._add_result("file_added", f[0], {'md5': f[1]})
def run(self, obj, config):
upx_path = config.get("upx_path", "")
# _write_to_file() will delete this file at the end of the "with" block.
with self._write_to_file() as tmp_file:
(working_dir, filename) = os.path.split(tmp_file)
args = [upx_path, "-q", "-d", filename]
# UPX does not generate a lot of output, so we should not have to
# worry about this hanging because the buffer is full
proc = subprocess.Popen(args, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, cwd=working_dir)
# Note that we are redirecting STDERR to STDOUT, so we can ignore
# the second element of the tuple returned by communicate().
output = proc.communicate()[0]
self._debug(output)
if proc.returncode:
# UPX return code of 1 indicates an error.
# UPX return code of 2 indicates a warning (usually, the
# file was not packed by UPX).
msg = ("UPX could not unpack the file.")
self._warning(msg)
return
with open(tmp_file, "rb") as newfile:
data = newfile.read()
#TODO: check to make sure file was modified (new MD5), indicating
# it was actually unpacked
md5 = hashlib.md5(data).hexdigest()
filename = md5 + ".upx"
handle_file(filename, data, obj.source,
related_id=str(obj.id),
related_type=str(obj._meta['crits_type']),
campaign=obj.campaign,
method=self.name,
relationship=RelationshipTypes.PACKED_FROM,
user=self.current_task.username)
# Filename is just the md5 of the data...
self._add_result("file_added", filename, {'md5': filename})
def _process_dropped(self, dropped):
# Dropped is a byte string of the .tar.bz2 file
self._debug("Processing dropped files.")
self._notify()
user = get_user_info(str(self.current_task.user))
if not user.has_access_to(SampleACL.WRITE):
self._info("User does not have permission to add samples to CRITs")
self._add_result("Processing Dropped Files Cancelled", "User does not have permission to add Samples to CRITs")
return
# TODO: Error handling
t = tarfile.open(mode='r:bz2', fileobj=BytesIO(dropped))
ignored = self.config.get('ignored_files', '').split('\r\n')
for f in t.getmembers():
if not f.isfile():
continue
data = t.extractfile(f).read()
name = os.path.basename(f.name)
if any([fnmatch.fnmatch(name, x) for x in ignored]):
self._debug("Ignoring file: %s" % name)
continue
h = md5(data).hexdigest()
self._info("New file: %s (%d bytes, %s)" % (name, len(data), h))
handle_file(name, data, self.obj.source,
related_id=str(self.obj.id),
related_type=str(self.obj._meta['crits_type']),
campaign=self.obj.campaign,
source_method=self.name,
relationship=RelationshipTypes.RELATED_TO,
user=self.current_task.user)
self._add_result("file_added", name, {'md5': h})
t.close()
def run(self, obj, config):
start_offset = config['start']
end_offset = config['end']
# Start must be 0 or higher. If end is greater than zero it must
# also be greater than start_offset.
if start_offset < 0 or (end_offset > 0 and start_offset > end_offset):
self._error("Invalid offsets.")
return
data = obj.filedata.read()[start_offset:end_offset]
if not data:
self._error("No data.")
else:
filename = hashlib.md5(data).hexdigest()
handle_file(filename, data, obj.source,
related_id=str(obj.id),
campaign=obj.campaign,
method=self.name,
relationship=RelationshipTypes.CONTAINS,
user=self.current_task.username)
# Filename is just the md5 of the data...
self._add_result("file_added", filename, {'md5': filename})
return
def parse_cybox_object(self, cbx_obj, description='', ind_id=None):
"""
Parse a CybOX object form a STIX doc. An object can contain
multiple related_objects, which in turn can have their own
related_objects, so this handles those recursively.
:param cbx_obj: The CybOX object to parse.
:type cbx_obj: A CybOX object.
:param description: Parent-level (e.g. Observable) description.
:type description: str
:param ind_id: The ID of a parent STIX Indicator.
:type ind_id: str
"""
# check for missing attributes
if not cbx_obj or not cbx_obj.properties:
if cbx_obj.idref: # just a reference, so nothing to parse
return
else:
cbx_id = getattr(cbx_obj, 'id_', 'None')
self.failed.append(("No valid object_properties was found!",
"Observable (%s)" % cbx_id,
cbx_id)) # note for display in UI
return
# Don't parse if already been parsed
# This is for artifacts that are related to CybOX File Objects
if cbx_obj.id_ in self.parsed:
return
try: # try to create CRITs object from Cybox Object
analyst = self.source_instance.analyst
item = cbx_obj.properties
val = cbx_obj.id_
if isinstance(item, Address) and not ind_id:
if item.category in ('cidr', 'ipv4-addr', 'ipv4-net',
'ipv4-netmask', 'ipv6-addr',
'ipv6-net', 'ipv6-netmask'):
imp_type = "IP"
for value in item.address_value.values:
val = str(value).strip()
if self.preview:
res = None
else:
iptype = get_crits_ip_type(item.category)
if iptype:
res = ip_add_update(val,
iptype,
[self.source],
analyst=analyst,
is_add_indicator=True)
else:
res = {'success': False, 'reason': 'No IP Type'}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
if (not ind_id and (isinstance(item, DomainName) or
(isinstance(item, URI) and item.type_ == 'Domain Name'))):
imp_type = "Domain"
for val in item.value.values:
if self.preview:
res = None
else:
res = upsert_domain(str(val),
[self.source],
username=analyst)
self.parse_res(imp_type, str(val), cbx_obj, res, ind_id)
elif isinstance(item, HTTPSession):
imp_type = "RawData"
val = cbx_obj.id_
try:
c_req = item.http_request_response[0].http_client_request
hdr = c_req.http_request_header
if hdr.raw_header:
data = hdr.raw_header.value
title = "HTTP Header from STIX: %s" % self.package.id_
method = self.source_instance.method
ref = self.source_instance.reference
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(data,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="HTTP Header",
tool_name="STIX",
tool_version=None,
method=method,
reference=ref)
else:
imp_type = "Indicator"
ind_type = "HTTP Request Header Fields - User-Agent"
val = hdr.parsed_header.user_agent.value
val = ','.join(val) if isinstance(val, list) else val
if self.preview:
res = None
else:
res = handle_indicator_ind(val,
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_relationship=True,
description=description)
except:
msg = "Unsupported use of 'HTTPSession' object."
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, WhoisEntry):
# No sure where else to put this
imp_type = "RawData"
val = cbx_obj.id_
if item.remarks:
data = item.remarks.value
title = "WHOIS Entry from STIX: %s" % self.package.id_
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(data,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="Text",
tool_name="WHOIS",
tool_version=None,
method=self.source_instance.method,
reference=self.source_instance.reference)
else:
msg = "Unsupported use of 'WhoisEntry' object."
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, Artifact):
# Not sure if this is right, and I believe these can be
# encoded in a couple different ways.
imp_type = "RawData"
val = cbx_obj.id_
rawdata = item.data.decode('utf-8')
# TODO: find out proper ways to determine title, datatype,
# tool_name, tool_version
title = "Artifact for Event: STIX Document %s" % self.package.id_
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(rawdata,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="Text",
tool_name="STIX",
tool_version=None,
method=self.source_instance.method,
reference=self.source_instance.reference)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif (isinstance(item, File) and
item.custom_properties and
item.custom_properties[0].name == "crits_type" and
item.custom_properties[0]._value == "Certificate"):
imp_type = "Certificate"
val = str(item.file_name)
data = None
if self.preview:
res = None
else:
for rel_obj in item.parent.related_objects:
if isinstance(rel_obj.properties, Artifact):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
res = handle_cert_file(val,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, File) and self.has_network_artifact(item):
imp_type = "PCAP"
val = str(item.file_name)
data = None
if self.preview:
res = None
else:
for rel_obj in item.parent.related_objects:
if (isinstance(rel_obj.properties, Artifact) and
rel_obj.properties.type_ == Artifact.TYPE_NETWORK):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
res = handle_pcap_file(val,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, File):
imp_type = "Sample"
md5 = item.md5
if md5:
md5 = md5.lower()
val = str(item.file_name or md5)
# add sha1/sha256/ssdeep once handle_file supports it
size = item.size_in_bytes
data = None
if item.file_path:
path = "File Path: " + str(item.file_path)
description += "\n" + path
for rel_obj in item.parent.related_objects:
if (isinstance(rel_obj.properties, Artifact) and
rel_obj.properties.type_ == Artifact.TYPE_FILE):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
if not md5 and not data and val and val != "None":
imp_type = "Indicator"
if self.preview:
res = None
else:
res = handle_indicator_ind(val,
self.source,
"Win File",
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
elif md5 or data:
if self.preview:
res = None
else:
res = handle_file(val,
data,
self.source,
user=analyst,
md5_digest=md5,
is_return_only_md5=False,
size=size,
description=description)
else:
val = cbx_obj.id_
msg = "CybOX 'File' object has no MD5, data, or filename"
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, EmailMessage):
imp_type = 'Email'
id_list = []
data = {}
val = cbx_obj.id_
get_attach = False
data['raw_body'] = str(item.raw_body)
data['raw_header'] = str(item.raw_header)
data['helo'] = str(item.email_server)
if item.header:
data['subject'] = str(item.header.subject)
if item.header.date:
data['date'] = item.header.date.value
val = "Date: %s, Subject: %s" % (data.get('date', 'None'),
data['subject'])
data['message_id'] = str(item.header.message_id)
data['sender'] = str(item.header.sender)
data['reply_to'] = str(item.header.reply_to)
data['x_originating_ip'] = str(item.header.x_originating_ip)
data['x_mailer'] = str(item.header.x_mailer)
data['boundary'] = str(item.header.boundary)
data['from_address'] = str(item.header.from_)
if item.header.to:
data['to'] = [str(r) for r in item.header.to.to_list()]
if data.get('date'): # Email TLOs must have a date
data['source'] = self.source.name
data['source_method'] = self.source_instance.method
data['source_reference'] = self.source_instance.reference
if self.preview:
res = None
else:
res = handle_email_fields(data,
analyst,
"STIX")
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
if not self.preview and res.get('status'):
id_list.append(cbx_obj.id_) # save ID for atchmnt rels
get_attach = True
else: # Can't be an Email TLO, so save fields
for x, key in enumerate(data):
if data[key] and data[key] != "None":
if key in ('raw_header', 'raw_body'):
if key == 'raw_header':
title = "Email Header from STIX Email: %s"
d_type = "Email Header"
else:
title = "Email Body from STIX Email: %s"
d_type = "Email Body"
imp_type = 'RawData'
title = title % cbx_obj.id_
if self.preview:
res = None
else:
res = handle_raw_data_file(data[key],
self.source,
analyst,
description,
title,
d_type,
"STIX",
self.stix_version)
self.parse_res(imp_type, title, cbx_obj,
res, ind_id)
elif key == 'to':
imp_type = 'Target'
for y, addr in enumerate(data[key]):
tgt_dict = {'email_address': addr}
if self.preview:
res = None
else:
res = upsert_target(tgt_dict, analyst)
if res['success']:
get_attach = True
tmp_obj = copy(cbx_obj)
tmp_obj.id_ = '%s-%s-%s' % (cbx_obj.id_,
x, y)
self.parse_res(imp_type, addr, tmp_obj,
res, ind_id)
self.ind2obj.setdefault(cbx_obj.id_,
[]).append(tmp_obj.id_)
id_list.append(tmp_obj.id_)
else:
imp_type = 'Indicator'
if key in ('sender', 'reply_to', 'from_address'):
ind_type = "Address - e-mail"
elif 'ip' in key:
ind_type = "Address - ipv4-addr"
elif key == 'raw_body':
ind_type = "Email Message"
else:
ind_type = "String"
if self.preview:
res = None
else:
res = handle_indicator_ind(data[key],
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
if res['success']:
get_attach = True
tmp_obj = copy(cbx_obj)
tmp_obj.id_ = '%s-%s' % (cbx_obj.id_, x)
self.parse_res(imp_type, data[key], tmp_obj,
res, ind_id)
self.ind2obj.setdefault(cbx_obj.id_,
[]).append(tmp_obj.id_)
id_list.append(tmp_obj.id_)
if not self.preview:
# Setup relationships between all Email attributes
for oid in id_list:
for oid2 in id_list:
if oid != oid2:
self.relationships.append((oid,
RelationshipTypes.RELATED_TO,
oid2, "High"))
# Should check for attachments and add them here.
if get_attach and item.attachments:
for attach in item.attachments:
rel_id = attach.to_dict()['object_reference']
for oid in id_list:
self.relationships.append((oid,
RelationshipTypes.CONTAINS,
rel_id, "High"))
else: # try to parse all other possibilities as Indicator
imp_type = "Indicator"
val = cbx_obj.id_
c_obj = make_crits_object(item)
# Ignore what was already caught above
if (ind_id or c_obj.object_type not in IPTypes.values()):
ind_type = c_obj.object_type
for val in [str(v).strip() for v in c_obj.value if v]:
if ind_type:
# handle domains mislabeled as URLs
if c_obj.object_type == 'URI' and '/' not in val:
ind_type = "Domain"
if self.preview:
res = None
else:
res = handle_indicator_ind(val,
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
except Exception, e: # probably caused by cybox object we don't handle
self.failed.append((e.message,
"%s (%s)" % (imp_type, val),
cbx_obj.id_)) # note for display in UI
def chopshop_carver(pcap_md5, options, analyst):
# Make sure we can find ChopShop
sc = get_config('ChopShop')
if not sc:
return {
'success': False,
'message': 'Could not find ChopShop service.'
}
shop_path = "%s/shop" % str(sc['basedir'])
if not os.path.exists(shop_path):
return {
'success': False,
'message': "ChopShop shop path does not exist."
}
sys.path.append(shop_path)
import ChopLib as CL
if StrictVersion(str(CL.VERSION)) < StrictVersion('4.0'):
return {'success': False, 'message': 'Need ChopShop 4.0 or newer'}
# Until we have an smtp_extractor in ChopShop we have to resort to
# to (ab)using payloads to dump the entire TCP stream and letting
# handle_eml() process everything. We also use the payloads module
# for handling raw carves. If a user wants to do SMTP and raw
# simultaneously it won't work because we can't distinguish one
# payloads module from another.
if options.get('raw', False) and options.get('smtp', False):
return {
'success': False,
'message': "Can not process SMTP and raw simultaneously."
}
# Make sure we have a PCAP to work with
pcap = PCAP.objects(md5=pcap_md5).first()
if not pcap:
return {'success': False, 'message': "No PCAP found."}
pcap_data = pcap.filedata.read()
if not pcap_data:
return {
'success': False,
'message': "Could not get PCAP from GridFS: %s" % pcap_md5
}
source = pcap['source'][0]['name'] # XXX: This kind of sucks...
# Create module string to pass to ChopShop
modules = []
if options.get('http_resp', False) or options.get('http_req', False):
modules.append("http | http_extractor")
if options.get('smtp', False) or options.get('raw', False):
# ChopShop really needs an smtp_extractor, but there's no good
# capability to do that yet. Maybe one day I'll build one. :)
# For now, just use payloads and let handle_eml() sort it out.
#
# Raw carving works exactly the same way, just post-processed
# differently.
modules.append("payloads -b")
if not modules:
return {'success': False, 'message': "No modules specified."}
mod_string = ';'.join(mod for mod in modules)
from ChopLib import ChopLib
from ChopUi import ChopUi
choplib = ChopLib()
chopui = ChopUi()
choplib.base_dir = str(sc['basedir'])
choplib.modules = mod_string
chopui.jsonout = jsonhandler
choplib.jsonout = True
# ChopShop (because of pynids) needs to read a file off disk.
# Write the pcap data to a temporary file.
temp_pcap = tempfile.NamedTemporaryFile(delete=False)
temp_pcap.write(pcap_data)
temp_pcap.close()
choplib.filename = temp_pcap.name
chopui.bind(choplib)
chopui.start()
if chopui.jsonclass == None:
os.unlink(temp_pcap.name)
chopui.join()
choplib.finish()
choplib.join()
return {
'success': False,
'message': 'Lost race condition in chopui. Try again.'
}
# ChopUI must be started before the jsonhandler class is insantiated.
# Tell the class what we are looking for now that it exists.
chopui.jsonclass.parse_options(options)
choplib.start()
while chopui.is_alive():
time.sleep(.1)
chopui.join()
choplib.finish()
choplib.join()
os.unlink(temp_pcap.name)
message = ''
# Grab any carved HTTP bodies.
for (md5_digest, (name, blob)) in chopui.jsonclass.http_files.items():
if handle_file(name,
blob,
source,
related_md5=pcap_md5,
user=analyst,
method='ChopShop Filecarver',
md5_digest=md5_digest,
related_type='PCAP'):
# Specifically not using name here as I don't want to deal
# with sanitizing it
message += "Saved HTTP body: <a href=\"%s\">%s</a><br />" % (
reverse('crits.samples.views.detail', args=[md5_digest
]), md5_digest)
else:
message += "Failed to save file %s." % md5_digest
# Grab any carved SMTP returns.
for blob in chopui.jsonclass.smtp_returns.values():
ret = handle_eml(blob,
source,
None,
analyst,
'ChopShop FileCarver',
related_id=pcap.id,
related_type='PCAP',
relationship_type=RelationshipTypes.RELATED_TO)
if not ret['status']:
message += ret['reason']
continue
message += "Saved email: <a href=\"%s\">%s</a><br />%i attachment(s)<br />" % (
reverse('crits.emails.views.email_detail', args=[
ret['object'].id
]), ret['object'].id, len(ret['attachments'].keys()))
for md5_digest in ret['attachments'].keys():
message += "<a href=\"%s\">%s</a><br />" % (reverse(
'crits.samples.views.detail', args=[md5_digest]), md5_digest)
# Handle raw returns.
for id_, blob in chopui.jsonclass.raw_returns.items():
md5_digest = handle_file(id_,
blob,
source,
related_md5=pcap_md5,
user=analyst,
method='ChopShop Filecarver',
related_type='PCAP')
if md5_digest:
message += "Saved raw %s: <a href=\"%s\">%s</a><br />" % (
id_, reverse('crits.samples.views.detail',
args=[md5_digest]), md5_digest)
else:
message += "Failed to save raw %s." % md5_digest
# It's possible to have no files here if nothing matched.
# Still return True as there were no problems.
if not message:
message = 'No files found.'
return {'success': True, 'message': message}
def run(self, obj, config):
"""
Being plugin processing
"""
#Load data from file as libchm will only accept a filename
with self._write_to_file() as chm_file:
try:
self.chmparse.LoadCHM(chm_file)
except Exception as e:
raise e
#Conduct analysis
result = self.analyze()
#Handle output of results
if 'obj_items_summary' in result.keys():
obj_items_summary = result.pop('obj_items_summary')
#General CHM info
for key, value in result.items():
self._add_result('chm_overview', key, {'value': value})
if config['chm_items']:
#Data and details of each object/page in the CHM
for f in self.added_files:
handle_file(f[0],
f[3],
obj.source,
related_id=str(obj.id),
campaign=obj.campaign,
method=self.name,
relationship='Extracted_From',
user=self.current_task.username)
self._add_result("chm_items_added", f[0], {
'size': f[1],
'md5': f[2]
})
else:
#Details of each object/page in the CHM
for object_item in obj_items_summary:
self._add_result('chm_items', object_item.get('name'), {
'size': object_item.get('size'),
'md5': object_item.get('md5')
})
#Detection results from CHM analysis
for object_item in obj_items_summary:
if object_item.get('detection'):
for detection in object_item.get('detection'):
self._add_result('chm_detection', detection,
{'chm_item': object_item.get('name')})
#URLs and IPs found in CHM
for object_item in obj_items_summary:
if object_item.get('urls'):
for url in object_item.get('urls'):
self._add_result('chm_urls', url,
{'chm_item': object_item.get('name')})
#Close file in memory
self.chmparse.CloseCHM()
def parse_observables(self, observables):
"""
Parse list of observables in STIX doc.
:param observables: List of STIX observables.
:type observables: List of STIX observables.
"""
analyst = self.source_instance.analyst
for obs in observables: # for each STIX observable
if not obs.object_ or not obs.object_.properties:
self.failed.append(("No valid object_properties was found!",
type(obs).__name__,
obs.id_)) # note for display in UI
continue
try: # try to create CRITs object from observable
item = obs.object_.properties
if isinstance(item, Address):
if item.category in ('cidr', 'ipv4-addr', 'ipv4-net',
'ipv4-netmask', 'ipv6-addr',
'ipv6-net', 'ipv6-netmask'):
imp_type = "IP"
for value in item.address_value.values:
ip = str(value).strip()
iptype = get_crits_ip_type(item.category)
if iptype:
res = ip_add_update(ip,
iptype,
[self.source],
analyst=analyst,
is_add_indicator=True)
self.parse_res(imp_type, obs, res)
if isinstance(item, DomainName):
imp_type = "Domain"
for value in item.value.values:
res = upsert_domain(str(value),
[self.source],
username=analyst)
self.parse_res(imp_type, obs, res)
elif isinstance(item, Artifact):
# Not sure if this is right, and I believe these can be
# encoded in a couple different ways.
imp_type = "RawData"
rawdata = item.data.decode('utf-8')
description = "None"
# TODO: find out proper ways to determine title, datatype,
# tool_name, tool_version
title = "Artifact for Event: STIX Document %s" % self.package.id_
res = handle_raw_data_file(rawdata,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="Text",
tool_name="STIX",
tool_version=None,
method=self.source_instance.method,
reference=self.source_instance.reference)
self.parse_res(imp_type, obs, res)
elif (isinstance(item, File) and
item.custom_properties and
item.custom_properties[0].name == "crits_type" and
item.custom_properties[0]._value == "Certificate"):
imp_type = "Certificate"
description = "None"
filename = str(item.file_name)
data = None
for obj in item.parent.related_objects:
if isinstance(obj.properties, Artifact):
data = obj.properties.data
res = handle_cert_file(filename,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, obs, res)
elif isinstance(item, File) and self.has_network_artifact(item):
imp_type = "PCAP"
description = "None"
filename = str(item.file_name)
data = None
for obj in item.parent.related_objects:
if (isinstance(obj.properties, Artifact) and
obj.properties.type_ == Artifact.TYPE_NETWORK):
data = obj.properties.data
res = handle_pcap_file(filename,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, obs, res)
elif isinstance(item, File):
imp_type = "Sample"
filename = str(item.file_name)
md5 = item.md5
data = None
for obj in item.parent.related_objects:
if (isinstance(obj.properties, Artifact) and
obj.properties.type_ == Artifact.TYPE_FILE):
data = obj.properties.data
res = handle_file(filename,
data,
self.source,
user=analyst,
md5_digest=md5,
is_return_only_md5=False)
self.parse_res(imp_type, obs, res)
elif isinstance(item, EmailMessage):
imp_type = "Email"
data = {}
data['source'] = self.source.name
data['source_method'] = self.source_instance.method
data['source_reference'] = self.source_instance.reference
data['raw_body'] = str(item.raw_body)
data['raw_header'] = str(item.raw_header)
data['helo'] = str(item.email_server)
if item.header:
data['message_id'] = str(item.header.message_id)
data['subject'] = str(item.header.subject)
data['sender'] = str(item.header.sender)
data['reply_to'] = str(item.header.reply_to)
data['x_originating_ip'] = str(item.header.x_originating_ip)
data['x_mailer'] = str(item.header.x_mailer)
data['boundary'] = str(item.header.boundary)
data['from_address'] = str(item.header.from_)
data['date'] = item.header.date.value
if item.header.to:
data['to'] = [str(r) for r in item.header.to.to_list()]
res = handle_email_fields(data,
analyst,
"STIX")
# Should check for attachments and add them here.
self.parse_res(imp_type, obs, res)
if res.get('status') and item.attachments:
for attach in item.attachments:
rel_id = attach.to_dict()['object_reference']
self.relationships.append((obs.id_,
"Contains",
rel_id, "High"))
else: # try to parse all other possibilities as Indicator
imp_type = "Indicator"
obj = make_crits_object(item)
if obj.object_type == 'Address':
# This was already caught above
continue
else:
ind_type = obj.object_type
for value in obj.value:
if value and ind_type:
res = handle_indicator_ind(value.strip(),
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True)
self.parse_res(imp_type, obs, res)
except Exception, e: # probably caused by cybox object we don't handle
self.failed.append((e.message,
type(item).__name__,
item.parent.id_)) # note for display in UI
def run(self, obj, config):
key = config.get('vt_api_key', '')
url = config.get('vt_download_url', '')
sizeLimit = config.get('size_limit', '')
replace = config.get('replace_sample', False)
do_triage = config.get('run_triage', False)
user = self.current_task.user
sample = Sample.objects(md5=obj.md5).first()
if not sample:
sample = Sample()
sample.md5 = md5_digest
self._info("Checking if binary already exists in CRITs.")
sample.discover_binary()
if sample.filedata and replace == False: #if we already have this binary and don't have permission to replace
self._info(
"CRITs already has this binary. Enable the 'Replace' option to overwrite with data from VirusTotal."
)
self._add_result("Download Canceled",
"Binary already exists in CRITs.")
return
if not user.has_access_to(SampleACL.WRITE):
self._info("User does not have permission to add Samples to CRITs")
self._add_result(
"Download Canceled",
"User does not have permission to add Samples to CRITs")
return
parameters = urllib.urlencode({"hash": obj.md5, "apikey": key})
if settings.HTTP_PROXY:
proxy = urllib2.ProxyHandler({
'http': settings.HTTP_PROXY,
'https': settings.HTTP_PROXY
})
opener = urllib2.build_opener(proxy)
urllib2.install_opener(opener)
try:
req = url + "?" + parameters
self._info(
"Requesting binary with md5 '{0}' from VirusTotal.".format(
obj.md5))
request = urllib2.Request(req)
response = urllib2.urlopen(request)
size = response.info().getheaders("Content-Length")[0]
self._info("Binary size: {0} bytes".format(size))
if int(size) > sizeLimit: # Check if within size limit
self._error(
"Binary size is {0} bytes, which is greater than maximum of {1} bytes. This limit can be changed in options."
.format(size, sizeLimit))
self._add_result(
"Download Aborted",
"Match found, but binary is larger than maximum size limit."
)
return
data = response.read()
except urllib2.HTTPError as e:
if e.code == 404:
self._info(
"No results were returned. Either VirusTotal does not have the requested binary, or the request URL is incorrect."
)
self._add_result(
"Not Found",
"Binary was not found in the VirusTotal database")
elif e.code == 403:
self._error("Download forbidden. {0}".format(e))
self._add_result(
"Download Canceled",
"CRITs was forbidden from downloading the binary.")
else:
self._error("An HTTP Error occurred: {0}".format(e))
return
except Exception as e:
logger.error("VirusTotal: Failed connection ({0})".format(e))
self._error("Failed to get data from VirusTotal: {0}".format(e))
return
if data: # Retrieved some data from VT
if replace == True:
try:
self._info(
"Replace = True. Deleting any previous binary with md5 {0}"
.format(obj.md5))
sample.filedata.delete()
except Exception as e:
logger.error(
"VirusTotal: Error deleting existing binary ({0})".
format(e))
self._error("Failed to delete existing binary")
self._info("Adding new binary to CRITs.")
try:
handle_file(
filename=obj.md5,
data=data,
source="VirusTotal",
source_reference="Binary downloaded from VT based on MD5",
user=self.current_task.user,
source_method="VirusTotal Download Service",
md5_digest=obj.md5)
except Exception as e:
logger.error(
"VirusTotal: Sample creation failed ({0})".format(e))
self._error("Failed to create new Sample: {0}".format(e))
return
if do_triage:
self._info("Running sample triage for data-reliant services.")
sample.reload()
run_triage(sample, user="******")
self._add_result(
"Download Successful",
"Binary was successfully downloaded from VirusTotal")
else:
self._error("No data returned by VirusTotal.")
def import_object(request, type_, id_):
setup_access()
user = request.user
if type_ == "Threat Descriptors":
obj = ThreatDescriptor(id=id_)
obj.details(
fields=[f for f in ThreatDescriptor._default_fields if f not in
(td.PRIVACY_MEMBERS, td.METADATA)]
)
itype = get_mapped_itype(obj.get(td.TYPE))
tags = obj.get(td.TAGS)
if itype is None:
return {'success': False,
'message': "Descriptor type is not supported by CRITs"}
if not user.has_access_to(IndicatorACL.WRITE):
return {'success': False,
'message': "User does not have permission to add Indicators to CRITs"}
ithreat_type = getattr(IndicatorThreatTypes, obj.get(td.THREAT_TYPE))
results = handle_indicator_ind(
obj.get(td.RAW_INDICATOR),
"ThreatExchange",
itype,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
request.user.username,
method="ThreatExchange Service",
reference="id: %s, owner: %s, share_level: %s" % (obj.get(td.ID),
obj.get(td.OWNER)['name'],
obj.get(td.SHARE_LEVEL)),
add_domain=True,
add_relationship=True,
confidence=build_ci(obj.get(td.CONFIDENCE)),
description=obj.get(td.DESCRIPTION),
bucket_list=tags
)
return results
elif type_ == "Malware Analyses":
if not user.has_access_to(SampleACL.WRITE):
return {'success': False,
'message': "User does not have permission to add Sample to CRITs"}
obj = Malware(id=id_)
obj.details(
fields=[f for f in Malware._fields if f not in
(m.METADATA)]
)
filename = obj.get(m.MD5)
tags = obj.get(m.TAGS)
try:
data = obj.rf
except:
data = None
results = handle_file(
filename,
data,
"ThreatExchange",
method="ThreatExchange Service",
reference="id: %s, share_level: %s" % (obj.get(td.ID),
obj.get(td.SHARE_LEVEL)),
user=request.user.username,
md5_digest = obj.get(m.MD5),
sha1_digest = obj.get(m.SHA1),
sha256_digest = obj.get(m.SHA256),
size = obj.get(m.SAMPLE_SIZE),
mimetype = obj.get(m.SAMPLE_TYPE),
bucket_list=tags,
)
return {'success': True,
'md5': results}
else:
return {'success': False,
'message': "Invalid Type"}
return {'success': True}
def run(self, obj, config):
key = config.get("malshare_api_key", "")
self.obj = obj
if obj.filedata.read():
logger.info("File already exists, no need to download")
self._info("File already exists, no need to download")
return
if not key:
logger.error("No valid MalShare API key found")
self._error("No valid MalShare API key found")
return
#Download URL: https://malshare.com/api.php?api_key=[API_KEY]&action=getfile&hash=[HASH]
parameters = {"api_key": key, "action": "getfile", "hash": obj.md5}
data = urllib.urlencode(parameters)
req = urllib2.Request("http://malshare.com/api.php", data)
logger.info("Connecting MalShare to download sample")
self._info("Connecting MalShare to download sample")
# Execute GET request
if settings.HTTP_PROXY:
proxy = urllib2.ProxyHandler({'http': settings.HTTP_PROXY})
opener = urllib2.build_opener(proxy)
urllib2.install_opener(opener)
try:
response = urllib2.urlopen(req)
sample_file = response.read()
except Exception as e:
logger.error("MalShare: network connection error (%s)" % e)
self._error("Network connection error checking MalShare (%s)" % e)
return
logger.info("Download completed")
self._info("Download completed")
if sample_file.startswith("Sample not found by hash"):
logger.error("Sample was not found on MalShare")
self._error("Sample was not found on MalShare")
return
else:
logger.info("Sample was found on MalShare!")
self._info("Sample was found on MalShare!")
#Verify file's MD5
if (hashlib.md5(sample_file).hexdigest() != obj.md5):
logger.error("Error while downloading sample from MalShare, MD5 missmatch")
self._error("Error while downloading sample from MalShare, MD5 missmatch")
return
else:
logger.info("MD5 verification successfull!")
self._info("MD5 verification successfull!")
#Write file
#Filename is just the md5
filename = obj.md5
handle_file(filename, sample_file, obj.source,
related_id=str(obj.id),
related_type=str(obj._meta['crits_type']),
campaign=obj.campaign,
method=self.name,
user=self.current_task.username)
self._add_result("file_downloaded", filename, {'md5': filename})
def parse_result(self, result_extract, acl_write, md5_parent):
stream_md5 = None
if type(result_extract) is dict:
#make reccursion extract each file embbed
if 'FileMD5' in result_extract and result_extract['FileMD5']:
tmp_dict = {}
b_yara = False
b_ioc = False
#extract info
no_info = [
'ExtractInfo', 'ContainedObjects', 'Yara', 'PathFile',
'FileMD5', 'RootFileType', 'TempDirExtract'
]
for key, value in result_extract.iteritems():
if not key in no_info:
self._add_result(
'File: ' + result_extract['FileMD5'] + ' - Info', key,
{'value': str(value)})
#extract yara match
if result_extract['Yara']:
for item_v in result_extract['Yara']:
for key, value in item_v.iteritems():
self._add_result(
'File: ' + result_extract['FileMD5'] +
' - Signatures yara matched', key, value)
b_yara = True
#extract IOC
if result_extract['ExtractInfo']:
for item_v in result_extract['ExtractInfo']:
for key, value in item_v.iteritems():
self._add_result(
'File: ' + result_extract['FileMD5'] +
' - Extract potential IOC', key,
{'value': str(value)})
b_ioc = True
#add_sample
if 'PathFile' in result_extract and type(
result_extract['PathFile']) is list and len(
result_extract['PathFile']) > 0:
if os.path.isfile(str(result_extract['PathFile'][0])):
with open(str(result_extract['PathFile'][0]),
'r') as content_file_tmp:
content_tmp = content_file_tmp.read()
stream_md5 = hashlib.md5(content_tmp).hexdigest()
name = str(stream_md5).decode('ascii', errors='ignore')
id_ = Sample.objects(md5=stream_md5).only('id').first()
if id_:
self._info(
'Add relationship with sample existed:' +
str(stream_md5))
#make relationship
id_.add_relationship(
rel_item=self.obj,
rel_type=RelationshipTypes.CONTAINED_WITHIN,
rel_date=datetime.now(),
analyst=self.current_task.user.username)
else:
if acl_write and (
self.config['import_file'] or
(self.config['import_file_yara'] and b_yara) or
(self.config['import_file_ioc'] and b_ioc)):
obj_parent = None
if md5_parent:
obj_parent = Sample.objects(
md5=md5_parent).only('id').first()
if not obj_parent:
sample = handle_file(
name,
content_tmp,
self.obj.source,
related_id=str(self.obj.id),
related_type=str(
self.obj._meta['crits_type']),
campaign=self.obj.campaign,
source_method=self.name,
relationship=RelationshipTypes.
CONTAINED_WITHIN,
user=self.current_task.user)
else:
sample = handle_file(
name,
content_tmp,
obj_parent.source,
related_id=str(obj_parent.id),
related_type=str(
obj_parent._meta['crits_type']),
campaign=obj_parent.campaign,
source_method=self.name,
relationship=RelationshipTypes.
CONTAINED_WITHIN,
user=self.current_task.user)
self._info('Add sample ' + str(stream_md5))
else:
#add IOC if not add sample
if self.current_task.user.has_access_to(
IndicatorACL.WRITE) and b_yara:
res = handle_indicator_ind(
stream_md5,
self.obj.source,
IndicatorTypes.MD5,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
self.current_task.user,
add_relationship=True,
source_method=self.name,
campaign=self.obj.campaign,
description='Extracted by service ' +
self.name)
self._info('Add indicator md5:' +
str(stream_md5) + ' -- id: ' +
str(res))
#contains file
if 'ContainedObjects' in result_extract and type(
result_extract['ContainedObjects']
) is list and result_extract['ContainedObjects']:
for item_v in result_extract['ContainedObjects']:
if item_v['FileMD5'] and item_v['FileType'] and item_v[
'FileSize']:
#search if file exist
id_ = Sample.objects(
md5=str(item_v['FileMD5'])).only('id').first()
sample_exist = False
ioc_exist = False
if id_:
sample_exist = True
id_ = Indicator.objects(
value=str(item_v['FileMD5'])).only('id').first()
if id_:
ioc_exist = True
self._add_result(
'File: ' + result_extract['FileMD5'] +
' - Contains md5 files', item_v['FileMD5'], {
'type': str(item_v['FileType']),
'size': str(item_v['FileSize']),
'Exists Sample': str(sample_exist),
'Exists IOC md5': str(ioc_exist)
})
for item_v in result_extract['ContainedObjects']:
#re do loop for best display result
parse_result(self, item_v, acl_write, stream_md5)
def finish_task(self, task):
"""
Finish a task.
"""
logger.debug("Finishing task %s" % task)
self.update_task(task)
obj = class_from_type(task.context.crits_type)
query = self.get_db_query(task.context)
sample = obj.objects(__raw__=query).first()
if task.files:
logger.debug("Adding samples")
for f in task.files:
logger.debug("Adding %s" % f['filename'])
#TODO: add in backdoor?, user
from crits.samples.handlers import handle_file
handle_file(f['filename'], f['data'], sample.source,
related_md5=task.context.identifier,
campaign=sample.campaign,
method=task.service.name,
relationship=f['relationship'],
user=task.context.username,
)
else:
logger.debug("No samples to add.")
if task.certificates:
logger.debug("Adding certificates")
for f in task.certificates:
logger.debug("Adding %s" % f['filename'])
from crits.certificates.handlers import handle_cert_file
# XXX: Add campaign from source?
handle_cert_file(f['filename'], f['data'], sample.source,
related_md5=task.context.identifier,
related_type=task.context.crits_type,
method=task.service.name,
relationship=f['relationship'],
user=task.context.username,
)
else:
logger.debug("No certificates to add.")
if task.pcaps:
logger.debug("Adding PCAPs")
for f in task.pcaps:
logger.debug("Adding %s" % f['filename'])
from crits.pcaps.handlers import handle_pcap_file
# XXX: Add campaign from source?
handle_pcap_file(f['filename'], f['data'], sample.source,
related_md5=task.context.identifier,
related_type=task.context.crits_type,
method=task.service.name,
relationship=f['relationship'],
user=task.context.username,
)
else:
logger.debug("No PCAPs to add.")
def parse_result(self, result_extract, response_dict, config, md5_parent):
stream_md5 = None
user = self.current_task.user
self.config = config
acl_write = user.has_access_to(SampleACL.WRITE)
if type(result_extract) is dict:
#make reccursion extract each file embbed
if 'FileMD5' in result_extract and result_extract['FileMD5']:
tmp_dict = {}
b_yara = False
b_ioc = False
#extract info
no_info = [
'ExtractInfo', 'ContainedObjects', 'Yara', 'PathFile',
'FileMD5', 'RootFileType', 'TempDirExtract', 'GlobalIOC'
]
for key, value in result_extract.iteritems():
if not key in no_info:
self._add_result(
'File: ' + result_extract['FileMD5'] + ' - Info', key,
{'value': str(value)})
#add download info
if result_extract['FileMD5'] in response_dict:
self._add_result(
'File: ' + result_extract['FileMD5'] + ' - Info',
'Download embed file', {'value': sfa_api + vx})
#GLOBAL IOC
if 'GlobalIOC' in result_extract and result_extract['GlobalIOC']:
for key, value in result_extract['GlobalIOC'].iteritems():
self._add_result('Global IOC by categories', key,
{'value': str(value)})
#extract yara match
if result_extract['Yara']:
for item_v in result_extract['Yara']:
#self._info("Dict:"+str(item_v))
for key, value in item_v.iteritems():
data = {
'description': '',
'ioc': '',
'tags': '',
'score': '0'
}
for kx, vx in value.iteritems():
data[kx] = str(vx)
self._add_result(
'File: ' + result_extract['FileMD5'] +
' - Signatures yara matched', key, data)
score_conf = re.sub("\D", "",
config['import_yara_score'])
if acl_write and config[
'import_file_yara'] and 'score' in value and int(
value['score']) >= int(score_conf):
id_ = Sample.objects(md5=result_extract['FileMD5']
).only('id').first()
if id_:
self._info(
'Add relationship with sample existed:' +
str(stream_md5))
#make relationship
id_.add_relationship(
rel_item=self.obj,
rel_type=RelationshipTypes.
CONTAINED_WITHIN,
rel_date=datetime.now(),
analyst=self.current_task.user.username)
elif result_extract['FileMD5'] in response_dict:
content_tmp = download(
self,
response_dict[result_extract['FileMD5']])
if content_tmp:
name = str(result_extract['FileMD5'])
if 'CDBNAME' in result_extract:
name = str(result_extract['CDBNAME'])
obj_parent = None
if md5_parent:
obj_parent = Sample.objects(
md5=md5_parent).only('id').first()
if not obj_parent:
sample = handle_file(
name,
content_tmp,
self.obj.source,
related_id=str(self.obj.id),
related_type=str(
self.obj._meta['crits_type']),
campaign=self.obj.campaign,
source_method=self.name,
relationship=RelationshipTypes.
CONTAINED_WITHIN,
user=self.current_task.user)
else:
sample = handle_file(
name,
content_tmp,
obj_parent.source,
related_id=str(obj_parent.id),
related_type=str(
obj_parent._meta['crits_type']
),
campaign=obj_parent.campaign,
source_method=self.name,
relationship=RelationshipTypes.
CONTAINED_WITHIN,
user=self.current_task.user)
self._info('Add sample ' + str(name) +
' - MD5:' +
str(result_extract['FileMD5']))
#extract IOC
if result_extract['ExtractInfo']:
for item_v in result_extract['ExtractInfo']:
for key, value in item_v.iteritems():
self._add_result(
'File: ' + result_extract['FileMD5'] +
' - Extract potential IOC', key,
{'value': str(value)})
if acl_write and config['import_file_ioc']:
id_ = Sample.objects(md5=result_extract['FileMD5']
).only('id').first()
if id_:
self._info(
'Add relationship with sample existed:' +
str(stream_md5))
#make relationship
id_.add_relationship(
rel_item=self.obj,
rel_type=RelationshipTypes.
CONTAINED_WITHIN,
rel_date=datetime.now(),
analyst=self.current_task.user.username)
elif result_extract['FileMD5'] in response_dict:
content_tmp = download(
self,
response_dict[result_extract['FileMD5']])
if content_tmp:
name = str(result_extract['FileMD5'])
if 'CDBNAME' in result_extract:
name = str(result_extract['CDBNAME'])
obj_parent = None
if md5_parent:
obj_parent = Sample.objects(
md5=md5_parent).only('id').first()
if not obj_parent:
sample = handle_file(
name,
content_tmp,
self.obj.source,
related_id=str(self.obj.id),
related_type=str(
self.obj._meta['crits_type']),
campaign=self.obj.campaign,
source_method=self.name,
relationship=RelationshipTypes.
CONTAINED_WITHIN,
user=self.current_task.user)
else:
sample = handle_file(
name,
content_tmp,
obj_parent.source,
related_id=str(obj_parent.id),
related_type=str(
obj_parent._meta['crits_type']
),
campaign=obj_parent.campaign,
source_method=self.name,
relationship=RelationshipTypes.
CONTAINED_WITHIN,
user=self.current_task.user)
self._info('Add sample ' + str(name) +
' - MD5:' +
str(result_extract['FileMD5']))
#contains file
if 'ContainedObjects' in result_extract and type(
result_extract['ContainedObjects']
) is list and result_extract['ContainedObjects']:
for item_v in result_extract['ContainedObjects']:
if item_v['FileMD5'] and item_v['FileType'] and item_v[
'FileSize']:
#search if file exist
id_ = Sample.objects(
md5=str(item_v['FileMD5'])).only('id').first()
sample_exist = False
ioc_exist = False
if id_:
sample_exist = True
id_.add_relationship(
rel_item=self.obj,
rel_type=RelationshipTypes.RELATED_TO,
#rel_date=datetime.now(),
analyst=self.current_task.user.username)
id_ = Indicator.objects(
value=str(item_v['FileMD5'])).only('id').first()
if id_:
ioc_exist = True
id_.add_relationship(
rel_item=self.obj,
rel_type=RelationshipTypes.RELATED_TO,
#rel_date=datetime.now(),
analyst=self.current_task.user.username)
self._add_result(
'File: ' + result_extract['FileMD5'] +
' - Contains md5 files', item_v['FileMD5'], {
'type': str(item_v['FileType']),
'size': str(item_v['FileSize']),
'Exists Sample': str(sample_exist),
'Exists IOC md5': str(ioc_exist)
})
for item_v in result_extract['ContainedObjects']:
#re do loop for best display result
parse_result(self, item_v, response_dict, config,
stream_md5)
def run(self, obj, config):
rparser = RtfParser(obj.filedata.read())
rparser.parse()
added_files = []
if not rparser.features.get('valid_rtf'):
self._error("Could not parse file as an RTF document")
return
props = [
'rtf_header_version',
'rtf_generator',
'ansi_code_page',
'ansi_code_page_name',
'deflang',
'binary_ratio',
]
for prop in props:
value = rparser.features.get(prop)
if value == None:
continue
result = {
'name': prop,
'value': value,
'result': value,
}
self._add_result('rtf_meta', prop, result)
for (k,v) in rparser.features.get('info', {}).items():
result = {
'name': k,
'value': v,
'result': v,
}
self._add_result('rtf_meta', prop, result)
hashes = [
'themedata',
'blipuid',
'colorschememapping',
'rsids',
]
for hash_type in hashes:
items = rparser.features.get(hash_type, [])
for item in items:
result = {
'name': hash_type,
'value': item,
'result': item,
}
self._add_result('Item Hashes', hash_type, result)
obj_num = 1
for obj_info in rparser.features.get('objects', []):
name = 'Object %d - %s' % (obj_num, obj_info.get('classname', 'Unknown'))
for (k,v) in obj_info.items():
val = hex(v) if type(v) == int else v
result = {
'name': k,
'value': val,
'result': v,
}
self._add_result(name, name, result)
obj_num += 1
obj_num = 1
for dstore in rparser.features.get('datastores', []):
name = 'Datastore %d - %s' % (obj_num, dstore.get('classname', 'Unknown'))
for (k,v) in dstore.items():
val = hex(v) if type(v) == int else v
result = {
'name': k,
'value': val,
'result': v,
}
self._add_result(name, name, result)
obj_num += 1
if config.get('save_streams', 0) == 1:
for i in range(len(rparser.objects)):
stream_md5 = hashlib.md5(rparser.objects[i]).hexdigest()
name = "Unknown object"
for obj_info in rparser.features.get('objects', []):
if stream_md5 == obj_info.get('content_md5'):
name = "Object - %s" % obj_info.get('classname', 'Unknown')
for obj_info in rparser.features.get('datastore', []):
if stream_md5 == obj_info.get('content_md5'):
name = "Object - %s" % obj_info.get('classname', 'Unknown')
handle_file(
name,
rparser.objects[i],
obj.source,
related_id=str(obj.id),
related_type=str(obj._meta['crits_type']),
campaign=obj.campaign,
method=self.name,
relationship=RelationshipTypes.CONTAINED_WITHIN,
user=self.current_task.username,
)
added_files.append((stream_md5, stream_md5))
for f in added_files:
self._add_result("file_added", f[0], {'md5': f[1]})
def _upload_md5(self):
print "[+] calling handle_file with md5_digest"
handle_file(self.test_filename, data='', source=self.sources[0],
md5_digest = self.test_md5)
def _upload_file(self):
print "[+] calling handle_file with file data"
handle_file(self.test_filename, self.test_data, self.sources[0])
def run(self, argv):
parser = OptionParser()
parser.add_option("-f", "--file", action="store", dest="filename",
type="string", help="scanned FILENAME")
parser.add_option("-s", "--source", action="store",
dest="source", type="string", help="source")
parser.add_option("-p", "--parent", action="store", dest="parent",
type="string", help="parent md5")
parser.add_option("-P", "--parent-type", action="store", dest="parent_type",
type="string", default="Sample", help="parent type (Sample, PCAP, ...)")
parser.add_option("-t", "--trojan", action="store", dest="trojan",
type="string", help="trojan")
parser.add_option("-r", "--reference", action="store", dest="reference",
type="string", help="reference field")
parser.add_option("-b", "--bucket", action="store", dest="bucket_list",
type="string", help="bucket list")
(opts, args) = parser.parse_args(argv)
md5hash = hashlib.md5()
if opts.source:
source = opts.source
else:
print "[-] Source required, none provided"
exit(1)
try:
fin = open(opts.filename, 'rb')
data = fin.read()
fin.close()
md5hash.update(data)
sourcemd5 = md5hash.hexdigest()
print "[+] Read %d from %s" % (len(data), opts.filename)
except:
print "[-] Cannot open %s for reading!" % opts.filename
return
if opts.parent:
parent = opts.parent
else:
parent = None
parent_type = opts.parent_type
if opts.trojan:
trojan = opts.trojan
else:
trojan = None
fname = opts.filename
(dirname, filename) = os.path.split(fname)
sample = handle_file(
filename,
data,
source,
opts.reference,
backdoor=trojan,
user=self.username,
parent_md5=parent,
parent_type=parent_type,
method="Command line add_file.py",
bucket_list=opts.bucket_list)
if sourcemd5 != sample:
print "[-] Source MD5: %s is not the same as the returned MD5: %s" % (sourcemd5, sample)
exit(1)
else:
print "[+] Added %s (MD5: %s)" % (filename, sample)
def chopshop_carver(pcap_md5, options, analyst):
# Make sure we can find ChopShop
sc = get_config('ChopShop')
user = get_user_info(analyst)
if not sc:
return {'success': False, 'message': 'Could not find ChopShop service.'}
shop_path = "%s/shop" % str(sc['basedir'])
if not os.path.exists(shop_path):
return {'success': False, 'message': "ChopShop shop path does not exist."}
sys.path.append(shop_path)
import ChopLib as CL
if StrictVersion(str(CL.VERSION)) < StrictVersion('4.0'):
return {'success': False, 'message': 'Need ChopShop 4.0 or newer'}
# Until we have an smtp_extractor in ChopShop we have to resort to
# to (ab)using payloads to dump the entire TCP stream and letting
# handle_eml() process everything. We also use the payloads module
# for handling raw carves. If a user wants to do SMTP and raw
# simultaneously it won't work because we can't distinguish one
# payloads module from another.
if options.get('raw', False) and options.get('smtp', False):
return {'success': False, 'message': "Can not process SMTP and raw simultaneously."}
# Make sure we have a PCAP to work with
pcap = PCAP.objects(md5=pcap_md5).first()
if not pcap:
return {'success': False, 'message': "No PCAP found."}
pcap_data = pcap.filedata.read()
if not pcap_data:
return {'success': False, 'message': "Could not get PCAP from GridFS: %s" % pcap_md5}
source = pcap['source'][0]['name'] # XXX: This kind of sucks...
# Create module string to pass to ChopShop
modules = []
if options.get('http_resp', False) or options.get('http_req', False):
modules.append("http | http_extractor")
if options.get('smtp', False) or options.get('raw', False):
# ChopShop really needs an smtp_extractor, but there's no good
# capability to do that yet. Maybe one day I'll build one. :)
# For now, just use payloads and let handle_eml() sort it out.
#
# Raw carving works exactly the same way, just post-processed
# differently.
modules.append("payloads -b")
if not modules:
return {'success': False, 'message': "No modules specified."}
mod_string = ';'.join(mod for mod in modules)
from ChopLib import ChopLib
from ChopUi import ChopUi
choplib = ChopLib()
chopui = ChopUi()
choplib.base_dir = str(sc['basedir'])
choplib.modules = mod_string
chopui.jsonout = jsonhandler
choplib.jsonout = True
# ChopShop (because of pynids) needs to read a file off disk.
# Write the pcap data to a temporary file.
temp_pcap = tempfile.NamedTemporaryFile(delete=False)
temp_pcap.write(pcap_data)
temp_pcap.close()
choplib.filename = temp_pcap.name
chopui.bind(choplib)
chopui.start()
if chopui.jsonclass == None:
os.unlink(temp_pcap.name)
chopui.join()
choplib.finish()
choplib.join()
return {'success': False,
'message': 'Lost race condition in chopui. Try again.'}
# ChopUI must be started before the jsonhandler class is insantiated.
# Tell the class what we are looking for now that it exists.
chopui.jsonclass.parse_options(options)
choplib.start()
while chopui.is_alive():
time.sleep(.1)
chopui.join()
choplib.finish()
choplib.join()
os.unlink(temp_pcap.name)
message = ''
# Grab any carved HTTP bodies.
for (md5_digest, (name, blob)) in chopui.jsonclass.http_files.items():
if user.has_access_to(SampleACL.WRITE) and handle_file(name, blob, source, related_md5=pcap_md5, user=user, source_method='ChopShop Filecarver', md5_digest=md5_digest, related_type='PCAP'):
# Specifically not using name here as I don't want to deal
# with sanitizing it
message += "Saved HTTP body: <a href=\"%s\">%s</a><br />" % (reverse('crits-samples-views-detail', args=[md5_digest]), md5_digest)
else:
message += "Failed to save file %s." % md5_digest
# Grab any carved SMTP returns.
for blob in chopui.jsonclass.smtp_returns.values():
ret = handle_eml(blob, source, None, analyst, 'ChopShop FileCarver',
related_id=pcap.id, related_type='PCAP',
relationship_type=RelationshipTypes.RELATED_TO)
if not ret['status']:
message += ret['reason']
continue
message += "Saved email: <a href=\"%s\">%s</a><br />%i attachment(s)<br />" % (reverse('crits-emails-views-email_detail', args=[ret['object'].id]), ret['object'].id, len(ret['attachments'].keys()))
for md5_digest in ret['attachments'].keys():
message += "<a href=\"%s\">%s</a><br />" % (reverse('crits-samples-views-detail', args=[md5_digest]), md5_digest)
# Handle raw returns.
for id_, blob in chopui.jsonclass.raw_returns.items():
if user.has_access_to(SampleACL.WRITE):
md5_digest = handle_file(id_, blob, source, related_md5=pcap_md5, user=user, source_method='ChopShop Filecarver', related_type='PCAP')
else:
md5_digest = None
if md5_digest:
message += "Saved raw %s: <a href=\"%s\">%s</a><br />" % (id_, reverse('crits-samples-views-detail', args=[md5_digest]), md5_digest)
else:
message += "Failed to save raw %s." % md5_digest
# It's possible to have no files here if nothing matched.
# Still return True as there were no problems.
if not message:
message = 'No files found.'
return {'success': True, 'message': message}
def parse_cybox_object(self, cbx_obj, description='', ind_id=None):
"""
Parse a CybOX object form a STIX doc. An object can contain
multiple related_objects, which in turn can have their own
related_objects, so this handles those recursively.
:param cbx_obj: The CybOX object to parse.
:type cbx_obj: A CybOX object.
:param description: Parent-level (e.g. Observable) description.
:type description: str
:param ind_id: The ID of a parent STIX Indicator.
:type ind_id: str
"""
# check for missing attributes
if not cbx_obj or not cbx_obj.properties:
if cbx_obj.idref: # just a reference, so nothing to parse
return
else:
cbx_id = getattr(cbx_obj, 'id_', 'None')
self.failed.append(("No valid object_properties was found!",
"Observable (%s)" % cbx_id,
cbx_id)) # note for display in UI
return
# Don't parse if already been parsed
# This is for artifacts that are related to CybOX File Objects
if cbx_obj.id_ in self.parsed:
return
try: # try to create CRITs object from Cybox Object
analyst = self.source_instance.analyst
item = cbx_obj.properties
val = cbx_obj.id_
if isinstance(item, Address) and not ind_id:
if item.category in ('cidr', 'ipv4-addr', 'ipv4-net',
'ipv4-netmask', 'ipv6-addr', 'ipv6-net',
'ipv6-netmask'):
imp_type = "IP"
for value in item.address_value.values:
val = str(value).strip()
if self.preview:
res = None
else:
iptype = get_crits_ip_type(item.category)
if iptype:
res = ip_add_update(val,
iptype, [self.source],
analyst=analyst,
is_add_indicator=True)
else:
res = {
'success': False,
'reason': 'No IP Type'
}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
if (not ind_id and
(isinstance(item, DomainName) or
(isinstance(item, URI) and item.type_ == 'Domain Name'))):
imp_type = "Domain"
for val in item.value.values:
if self.preview:
res = None
else:
res = upsert_domain(str(val), [self.source],
username=analyst)
self.parse_res(imp_type, str(val), cbx_obj, res, ind_id)
elif isinstance(item, HTTPSession):
imp_type = "RawData"
val = cbx_obj.id_
try:
c_req = item.http_request_response[0].http_client_request
hdr = c_req.http_request_header
if hdr.raw_header:
data = hdr.raw_header.value
title = "HTTP Header from STIX: %s" % self.package.id_
method = self.source_instance.method
ref = self.source_instance.reference
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(data,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="HTTP Header",
tool_name="STIX",
tool_version=None,
method=method,
reference=ref)
else:
imp_type = "Indicator"
ind_type = "HTTP Request Header Fields - User-Agent"
val = hdr.parsed_header.user_agent.value
val = ','.join(val) if isinstance(val, list) else val
if self.preview:
res = None
else:
res = handle_indicator_ind(
val,
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_relationship=True,
description=description)
except:
msg = "Unsupported use of 'HTTPSession' object."
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, WhoisEntry):
# No sure where else to put this
imp_type = "RawData"
val = cbx_obj.id_
if item.remarks:
data = item.remarks.value
title = "WHOIS Entry from STIX: %s" % self.package.id_
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(
data,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="Text",
tool_name="WHOIS",
tool_version=None,
method=self.source_instance.method,
reference=self.source_instance.reference)
else:
msg = "Unsupported use of 'WhoisEntry' object."
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, Artifact):
# Not sure if this is right, and I believe these can be
# encoded in a couple different ways.
imp_type = "RawData"
val = cbx_obj.id_
rawdata = item.data.decode('utf-8')
# TODO: find out proper ways to determine title, datatype,
# tool_name, tool_version
title = "Artifact for Event: STIX Document %s" % self.package.id_
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(
rawdata,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="Text",
tool_name="STIX",
tool_version=None,
method=self.source_instance.method,
reference=self.source_instance.reference)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif (isinstance(item, File) and item.custom_properties
and item.custom_properties[0].name == "crits_type"
and item.custom_properties[0]._value == "Certificate"):
imp_type = "Certificate"
val = str(item.file_name)
data = None
if self.preview:
res = None
else:
for rel_obj in item.parent.related_objects:
if isinstance(rel_obj.properties, Artifact):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
res = handle_cert_file(val,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, File) and self.has_network_artifact(item):
imp_type = "PCAP"
val = str(item.file_name)
data = None
if self.preview:
res = None
else:
for rel_obj in item.parent.related_objects:
if (isinstance(rel_obj.properties, Artifact)
and rel_obj.properties.type_
== Artifact.TYPE_NETWORK):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
res = handle_pcap_file(val,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, File):
imp_type = "Sample"
md5 = item.md5
if md5:
md5 = md5.lower()
val = str(item.file_name or md5)
# add sha1/sha256/ssdeep once handle_file supports it
size = item.size_in_bytes
data = None
if item.file_path:
path = "File Path: " + str(item.file_path)
description += "\n" + path
for rel_obj in item.parent.related_objects:
if (isinstance(rel_obj.properties, Artifact) and
rel_obj.properties.type_ == Artifact.TYPE_FILE):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
if not md5 and not data and val and val != "None":
imp_type = "Indicator"
if self.preview:
res = None
else:
res = handle_indicator_ind(
val,
self.source,
"Win File",
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
elif md5 or data:
if self.preview:
res = None
else:
res = handle_file(val,
data,
self.source,
user=analyst,
md5_digest=md5,
is_return_only_md5=False,
size=size,
description=description)
else:
val = cbx_obj.id_
msg = "CybOX 'File' object has no MD5, data, or filename"
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, EmailMessage):
imp_type = 'Email'
id_list = []
data = {}
val = cbx_obj.id_
get_attach = False
data['raw_body'] = str(item.raw_body)
data['raw_header'] = str(item.raw_header)
data['helo'] = str(item.email_server)
if item.header:
data['subject'] = str(item.header.subject)
if item.header.date:
data['date'] = item.header.date.value
val = "Date: %s, Subject: %s" % (data.get(
'date', 'None'), data['subject'])
data['message_id'] = str(item.header.message_id)
data['sender'] = str(item.header.sender)
data['reply_to'] = str(item.header.reply_to)
data['x_originating_ip'] = str(
item.header.x_originating_ip)
data['x_mailer'] = str(item.header.x_mailer)
data['boundary'] = str(item.header.boundary)
data['from_address'] = str(item.header.from_)
if item.header.to:
data['to'] = [str(r) for r in item.header.to.to_list()]
if data.get('date'): # Email TLOs must have a date
data['source'] = self.source.name
data['source_method'] = self.source_instance.method
data['source_reference'] = self.source_instance.reference
if self.preview:
res = None
else:
res = handle_email_fields(data, analyst, "STIX")
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
if not self.preview and res.get('status'):
id_list.append(cbx_obj.id_) # save ID for atchmnt rels
get_attach = True
else: # Can't be an Email TLO, so save fields
for x, key in enumerate(data):
if data[key] and data[key] != "None":
if key in ('raw_header', 'raw_body'):
if key == 'raw_header':
title = "Email Header from STIX Email: %s"
d_type = "Email Header"
else:
title = "Email Body from STIX Email: %s"
d_type = "Email Body"
imp_type = 'RawData'
title = title % cbx_obj.id_
if self.preview:
res = None
else:
res = handle_raw_data_file(
data[key], self.source, analyst,
description, title, d_type, "STIX",
self.stix_version)
self.parse_res(imp_type, title, cbx_obj, res,
ind_id)
elif key == 'to':
imp_type = 'Target'
for y, addr in enumerate(data[key]):
tgt_dict = {'email_address': addr}
if self.preview:
res = None
else:
res = upsert_target(tgt_dict, analyst)
if res['success']:
get_attach = True
tmp_obj = copy(cbx_obj)
tmp_obj.id_ = '%s-%s-%s' % (cbx_obj.id_, x,
y)
self.parse_res(imp_type, addr, tmp_obj,
res, ind_id)
self.ind2obj.setdefault(
cbx_obj.id_, []).append(tmp_obj.id_)
id_list.append(tmp_obj.id_)
else:
imp_type = 'Indicator'
if key in ('sender', 'reply_to',
'from_address'):
ind_type = "Address - e-mail"
elif 'ip' in key:
ind_type = "Address - ipv4-addr"
elif key == 'raw_body':
ind_type = "Email Message"
else:
ind_type = "String"
if self.preview:
res = None
else:
res = handle_indicator_ind(
data[key],
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
if res['success']:
get_attach = True
tmp_obj = copy(cbx_obj)
tmp_obj.id_ = '%s-%s' % (cbx_obj.id_, x)
self.parse_res(imp_type, data[key], tmp_obj,
res, ind_id)
self.ind2obj.setdefault(cbx_obj.id_,
[]).append(tmp_obj.id_)
id_list.append(tmp_obj.id_)
if not self.preview:
# Setup relationships between all Email attributes
for oid in id_list:
for oid2 in id_list:
if oid != oid2:
self.relationships.append(
(oid, RelationshipTypes.RELATED_TO, oid2,
"High"))
# Should check for attachments and add them here.
if get_attach and item.attachments:
for attach in item.attachments:
rel_id = attach.to_dict()['object_reference']
for oid in id_list:
self.relationships.append(
(oid, RelationshipTypes.CONTAINS, rel_id,
"High"))
else: # try to parse all other possibilities as Indicator
imp_type = "Indicator"
val = cbx_obj.id_
c_obj = make_crits_object(item)
# Ignore what was already caught above
if (ind_id or c_obj.object_type not in IPTypes.values()):
ind_type = c_obj.object_type
for val in [str(v).strip() for v in c_obj.value if v]:
if ind_type:
# handle domains mislabeled as URLs
if c_obj.object_type == 'URI' and '/' not in val:
ind_type = "Domain"
if self.preview:
res = None
else:
res = handle_indicator_ind(
val,
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
except Exception, e: # probably caused by cybox object we don't handle
self.failed.append((e.message, "%s (%s)" % (imp_type, val),
cbx_obj.id_)) # note for display in UI
def finish_task(self, task):
"""
Finish a task.
"""
logger.debug("Finishing task %s" % task)
self.update_task(task)
obj = class_from_type(task.context.crits_type)
query = self.get_db_query(task.context)
sample = obj.objects(__raw__=query).first()
if task.files:
logger.debug("Adding samples")
for f in task.files:
logger.debug("Adding %s" % f['filename'])
#TODO: add in backdoor?, user
from crits.samples.handlers import handle_file
handle_file(f['filename'], f['data'], sample.source,
parent_md5=task.context.identifier,
campaign=sample.campaign,
method=task.service.name,
relationship=f['relationship'],
user=task.context.username,
)
else:
logger.debug("No samples to add.")
if task.certificates:
logger.debug("Adding certificates")
for f in task.certificates:
logger.debug("Adding %s" % f['filename'])
from crits.certificates.handlers import handle_cert_file
# XXX: Add campaign from source?
handle_cert_file(f['filename'], f['data'], sample.source,
parent_md5=task.context.identifier,
parent_type=task.context.crits_type,
method=task.service.name,
relationship=f['relationship'],
user=task.context.username,
)
else:
logger.debug("No certificates to add.")
if task.pcaps:
logger.debug("Adding PCAPs")
for f in task.pcaps:
logger.debug("Adding %s" % f['filename'])
from crits.pcaps.handlers import handle_pcap_file
# XXX: Add campaign from source?
handle_pcap_file(f['filename'], f['data'], sample.source,
parent_md5=task.context.identifier,
parent_type=task.context.crits_type,
method=task.service.name,
relationship=f['relationship'],
user=task.context.username,
)
else:
logger.debug("No PCAPs to add.")
def run(self, obj, config):
key = config.get('vt_api_key', '')
url = config.get('vt_download_url', '')
sizeLimit = config.get('size_limit', '')
replace = config.get('replace_sample', False)
do_triage = config.get('run_triage', False)
user = self.current_task.user
sample = Sample.objects(md5=obj.md5).first()
if not sample:
sample = Sample()
sample.md5 = md5_digest
self._info("Checking if binary already exists in CRITs.")
sample.discover_binary()
if sample.filedata and replace == False: #if we already have this binary and don't have permission to replace
self._info("CRITs already has this binary. Enable the 'Replace' option to overwrite with data from VirusTotal.")
self._add_result("Download Canceled", "Binary already exists in CRITs.")
return
if not user.has_access_to(SampleACL.WRITE):
self._info("User does not have permission to add Samples to CRITs")
self._add_result("Download Canceled", "User does not have permission to add Samples to CRITs")
return
parameters = urllib.urlencode({"hash": obj.md5, "apikey": key})
if settings.HTTP_PROXY:
proxy = urllib2.ProxyHandler({'http': settings.HTTP_PROXY, 'https': settings.HTTP_PROXY})
opener = urllib2.build_opener(proxy)
urllib2.install_opener(opener)
try:
req = url + "?" + parameters
self._info("Requesting binary with md5 '{0}' from VirusTotal.".format(obj.md5))
request = urllib2.Request(req)
response = urllib2.urlopen(request)
size = response.info().getheaders("Content-Length")[0]
self._info("Binary size: {0} bytes".format(size))
if int(size) > sizeLimit: # Check if within size limit
self._error("Binary size is {0} bytes, which is greater than maximum of {1} bytes. This limit can be changed in options.".format(size, sizeLimit))
self._add_result("Download Aborted", "Match found, but binary is larger than maximum size limit.")
return
data = response.read()
except urllib2.HTTPError as e:
if e.code == 404:
self._info("No results were returned. Either VirusTotal does not have the requested binary, or the request URL is incorrect.")
self._add_result("Not Found", "Binary was not found in the VirusTotal database")
elif e.code == 403:
self._error("Download forbidden. {0}".format(e))
self._add_result("Download Canceled", "CRITs was forbidden from downloading the binary.")
else:
self._error("An HTTP Error occurred: {0}".format(e))
except Exception as e:
logger.error("VirusTotal: Failed connection ({0})".format(e))
self._error("Failed to get data from VirusTotal: {0}".format(e))
return
if data: # Retrieved some data from VT
if replace == True:
try:
self._info("Replace = True. Deleting any previous binary with md5 {0}".format(obj.md5))
sample.filedata.delete()
except Exception as e:
logger.error("VirusTotal: Error deleting existing binary ({0})".format(e))
self._error("Failed to delete existing binary")
self._info("Adding new binary to CRITs.")
try:
handle_file(filename = obj.md5,
data = data,
source = "VirusTotal",
reference = "Binary downloaded from VT based on MD5",
user = "******",
method = "VirusTotal Download Service",
md5_digest = obj.md5 )
except Exception as e:
logger.error("VirusTotal: Sample creation failed ({0})".format(e))
self._error("Failed to create new Sample: {0}".format(e))
return
if do_triage:
self._info("Running sample triage for data-reliant services.")
sample.reload()
run_triage(sample, user = "******")
self._add_result("Download Successful", "Binary was successfully downloaded from VirusTotal")
else:
self._error("No data returned by VirusTotal.")
def parse_observables(self, observables):
"""
Parse list of observables in STIX doc.
:param observables: List of STIX observables.
:type observables: List of STIX observables.
"""
analyst = self.source_instance.analyst
for obs in observables: # for each STIX observable
if obs.observable_composition:
object_list = obs.observable_composition.observables
else:
object_list = [obs]
for obs_comp in object_list:
if not obs_comp.object_ or not obs_comp.object_.properties:
self.failed.append(
("No valid object_properties was found!",
type(obs_comp).__name__,
obs_comp.id_)) # note for display in UI
continue
try: # try to create CRITs object from observable
item = obs_comp.object_.properties
if isinstance(item, Address):
if item.category in ('cidr', 'ipv4-addr', 'ipv4-net',
'ipv4-netmask', 'ipv6-addr',
'ipv6-net', 'ipv6-netmask',
'ipv6-subnet'):
imp_type = "IP"
for value in item.address_value.values:
ip = str(value).strip()
iptype = get_crits_ip_type(item.category)
if iptype:
res = ip_add_update(ip,
iptype, [self.source],
analyst=analyst,
id=self.package.id_)
self.parse_res(imp_type, obs, res)
if isinstance(item, DomainName):
imp_type = "Domain"
for value in item.value.values:
res = upsert_domain(str(value), [self.source],
username=analyst,
id=self.package.id_)
self.parse_res(imp_type, obs, res)
elif isinstance(item, Artifact):
# Not sure if this is right, and I believe these can be
# encoded in a couple different ways.
imp_type = "RawData"
rawdata = item.data.decode('utf-8')
description = "None"
# TODO: find out proper ways to determine title, datatype,
# tool_name, tool_version
title = "Artifact for Event: STIX Document %s" % self.package.id_
res = handle_raw_data_file(
rawdata,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="Text",
tool_name="STIX",
tool_version=None,
method=self.source_instance.method,
reference=self.source_instance.reference)
self.parse_res(imp_type, obs, res)
elif (isinstance(item, File) and item.custom_properties
and item.custom_properties[0].name == "crits_type"
and item.custom_properties[0]._value
== "Certificate"):
imp_type = "Certificate"
description = "None"
filename = str(item.file_name)
data = None
for obj in item.parent.related_objects:
if isinstance(obj.properties, Artifact):
data = obj.properties.data
res = handle_cert_file(filename,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, obs, res)
elif isinstance(item,
File) and self.has_network_artifact(item):
imp_type = "PCAP"
description = "None"
filename = str(item.file_name)
data = None
for obj in item.parent.related_objects:
if (isinstance(obj.properties, Artifact)
and obj.properties.type_
== Artifact.TYPE_NETWORK):
data = obj.properties.data
res = handle_pcap_file(filename,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, obs, res)
elif isinstance(item, File):
imp_type = "Sample"
filename = str(item.file_name)
md5 = item.md5
data = None
for obj in item.parent.related_objects:
if (isinstance(obj.properties, Artifact)
and obj.properties.type_
== Artifact.TYPE_FILE):
data = obj.properties.data
res = handle_file(filename,
data,
self.source,
user=analyst,
md5_digest=md5,
is_return_only_md5=False,
id=self.package.id_)
self.parse_res(imp_type, obs, res)
if item.extracted_features:
self.parse_filenames(item.extracted_features,
res['object'].id)
elif isinstance(item, EmailMessage):
imp_type = "Email"
data = {}
data['source'] = self.source.name
data['source_method'] = self.source_instance.method
data[
'source_reference'] = self.source_instance.reference
data['raw_body'] = str(item.raw_body)
data['raw_header'] = str(item.raw_header)
data['helo'] = str(item.email_server)
if item.header:
data['message_id'] = str(item.header.message_id)
data['subject'] = str(item.header.subject)
data['sender'] = str(item.header.sender)
data['reply_to'] = str(item.header.reply_to)
data['x_originating_ip'] = str(
item.header.x_originating_ip)
data['x_mailer'] = str(item.header.x_mailer)
data['boundary'] = str(item.header.boundary)
data['from_address'] = str(item.header.from_)
data['date'] = item.header.date.value
if item.header.to:
data['to'] = [str(r) for r in item.header.to]
if item.header.cc:
data['cc'] = [str(r) for r in item.header.cc]
res = handle_email_fields(data,
analyst,
"STIX",
id=self.package.id_)
# Should check for attachments and add them here.
self.parse_res(imp_type, obs, res)
if res.get('status') and item.attachments:
for attach in item.attachments:
rel_id = attach.to_dict()['object_reference']
self.relationships.append(
(obs.id_, "Contains", rel_id, "High"))
else: # try to parse all other possibilities as Indicator
imp_type = "Indicator"
obj = make_crits_object(item)
if obj.object_type == 'Address':
# This was already caught above
continue
else:
ind_type = obj.object_type
for value in obj.value:
if value and ind_type:
res = handle_indicator_ind(
value.strip(),
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True)
self.parse_res(imp_type, obs, res)
except Exception, e: # probably caused by cybox object we don't handle
self.failed.append(
(e.message, type(item).__name__,
item.parent.id_)) # note for display in UI
def run(self, argv):
parser = OptionParser()
parser.add_option("-f", "--file", action="store", dest="filename",
type="string", help="scanned FILENAME")
parser.add_option("-s", "--source", action="store",
dest="source", type="string", help="source")
parser.add_option("-p", "--parent", action="store", dest="parent_md5",
type="string", help="parent MD5")
parser.add_option("-i", "--parent-id", action="store", dest="parent_id",
type="string", help="parent ID")
parser.add_option("-P", "--parent-type", action="store", dest="parent_type",
type="string", default="Sample", help="parent type (Sample, PCAP, ...)")
parser.add_option("-t", "--trojan", action="store", dest="trojan",
type="string", help="trojan")
parser.add_option("-r", "--reference", action="store", dest="reference",
type="string", help="reference field")
parser.add_option("-b", "--bucket", action="store", dest="bucket_list",
type="string", help="bucket list")
parser.add_option("-T", "--tlp", action="store", dest="tlp",
type="string", default="red", help="TLP of data")
(opts, args) = parser.parse_args(argv)
md5hash = hashlib.md5()
if opts.source:
source = opts.source
else:
print "[-] Source required, none provided"
return
if opts.parent_md5 and opts.parent_id:
print "[-] Specify one of -p or -i!"
return
if not user.has_access_to(SampleACL.WRITE):
print "[-] User does not have permission to add Samples."
return
if not opts.tlp or opts.tlp not in ['red', 'amber', 'green', 'white']:
opts.tlp = 'red'
try:
fin = open(opts.filename, 'rb')
data = fin.read()
fin.close()
md5hash.update(data)
sourcemd5 = md5hash.hexdigest()
print "[+] Read %d from %s" % (len(data), opts.filename)
except:
print "[-] Cannot open %s for reading!" % opts.filename
return
if opts.parent_md5:
parent_md5 = opts.parent_md5
else:
parent_md5 = None
if opts.parent_id:
parent_id = opts.parent_id
else:
parent_id = None
parent_type = opts.parent_type
if opts.trojan:
trojan = opts.trojan
else:
trojan = None
fname = opts.filename
(dirname, filename) = os.path.split(fname)
sample = handle_file(
filename,
data,
source,
source_reference=opts.reference,
source_tlp=opts.tlp,
backdoor=trojan,
user=self.user,
related_md5=parent_md5,
related_id=parent_id,
related_type=parent_type,
source_method="Command line add_file.py",
bucket_list=opts.bucket_list)
if sourcemd5 != sample:
print "[-] Source MD5: %s is not the same as the returned MD5: %s" % (sourcemd5, sample)
else:
print "[+] Added %s (MD5: %s)" % (filename, sample)