def __enter__(self):
self.filesystems = (FileSystem(self.disks[0]),
FileSystem(self.disks[1]))
for filesystem in self.filesystems:
filesystem.mount()
return self
def parse_registry(hive, disk=None, sort=False):
if disk is not None:
with FileSystem(disk) as filesystem:
registry = extract_registry(filesystem, hive)
else:
registry = RegistryHive(hive)
registry.rootkey = registry_root(hive)
if sort:
keys = sorted((k for k in registry.keys()), key=lambda k: k.timestamp)
return OrderedDict((k.path, (k.timestamp, k.values)) for k in keys)
else:
return {k.path: (k.timestamp, k.values) for k in registry.keys()}
def list_files(disk, identify=False, size=False):
logger = logging.getLogger('filesystem')
with FileSystem(disk) as filesystem:
logger.debug("Listing files.")
files = hash_filesystem(filesystem)
if identify:
logger.debug("Gatering file types.")
for file_meta in files:
file_meta['type'] = filesystem.file(file_meta['path'])
if size:
logger.debug("Gatering file sizes.")
for file_meta in files:
file_meta['size'] = filesystem.stat(file_meta['path'])['size']
return files
class WinEventLog:
"""WinEventLog class.
Allows to retrieve the Events contained within Windows Event Log files.
"""
def __init__(self, disk):
self._disk = disk
self._filesystem = None
self.logger = logging.getLogger(
"%s.%s" % (self.__module__, self.__class__.__name__))
def __enter__(self):
self._filesystem = FileSystem(self._disk)
self._filesystem.mount()
return self
def __exit__(self, *_):
self._filesystem.umount()
def __getattr__(self, attr):
return getattr(self._filesystem, attr)
def eventlog(self, path):
"""Iterates over the Events contained within the log at the given path.
For each Event, yields a XML string.
"""
self.logger.debug("Parsing Event log file %s.", path)
with NamedTemporaryFile(buffering=0) as tempfile:
self._filesystem.download(path, tempfile.name)
file_header = FileHeader(tempfile.read(), 0)
for xml_string, _ in evtx_file_xml_view(file_header):
yield xml_string
def parse_usnjrnl(usnjrnl, disk=None):
if disk is not None:
with FileSystem(disk) as filesystem:
return extract_usnjrnl(filesystem, usnjrnl)
else:
return [e._asdict() for e in usn_journal(usnjrnl)]
def __enter__(self):
self._filesystem = FileSystem(self._disk)
self._filesystem.mount()
return self
class VTScanner:
"""VirusTotal scanner.
Allows to scan the given disk content and query VirusTotal.
disk must contain the path of a valid disk image.
apikey must be a valid VT API key.
The attribute batchsize controls the amount of object per VT query.
"""
def __init__(self, disk, apikey):
self._disk = disk
self._apikey = apikey
self._filesystem = None
self.batchsize = 1
self.logger = logging.getLogger(
"%s.%s" % (self.__module__, self.__class__.__name__))
def __enter__(self):
self._filesystem = FileSystem(self._disk)
self._filesystem.mount()
return self
def __exit__(self, *_):
self._filesystem.umount()
def __getattr__(self, attr):
return getattr(self._filesystem, attr)
@property
def apikey(self):
return self._apikey
def scan(self, filetypes=None):
"""Iterates over the content of the disk and queries VirusTotal
to determine whether it's malicious or not.
filetypes is a list containing regular expression patterns.
If given, only the files which type will match with one or more of
the given patterns will be queried against VirusTotal.
For each file which is unknown by VT or positive to any of its engines,
the method yields a namedtuple:
VTReport(path -> C:\\Windows\\System32\\infected.dll
hash -> ab231...
detections) -> dictionary engine -> detection
Files unknown by VirusTotal will contain the string 'unknown'
in the detection field.
"""
print("I'm here!1")
self.logger.debug("Scanning FS content.")
all_checksums = self._filesystem.checksums('/')
print("I'm here!2")
checksums = self.filetype_filter(all_checksums, filetypes=filetypes)
print(len(list(checksums)))
print("I'm here!3")
# self.logger.debug("Querying %d objects to VTotal.", checksums)
# print("checksum: " + checksums)
aggregated_files = {}
for files in chunks(checksums, size=self.batchsize):
files = dict((reversed(e) for e in files))
if len(aggregated_files) < 100:
aggregated_files.update(files)
else:
# print("@@@@@@@@@@@@@@ files @@@@@@@@@@@@@")
# for key in files.keys():
# print(key + ", " + files[key])
response = vtquery(self._apikey, aggregated_files.keys())
yield from self.parse_response(aggregated_files, response)
aggregated_files = {}
def filetype_filter(self, files, filetypes=None):
# print(len(list(files)))
#for f in files:
# print(f)
# print(type(filetypes))
# print(filetypes)
if filetypes is not None:
#return [f for f in files
# if any((re.match(t, self._filesystem.file(f[0]))
# for t in filetypes))]
matched_file_list = []
for f in files:
for t in filetypes:
if re.match(t, f[0]):
# print(self._filesystem.file(f[0]))
matched_file_list.append(f)
break
return matched_file_list
else:
return files
def parse_response(self, files, response):
response = isinstance(response, list) and response or [response]
for result in response:
yield from self.parse_result(result, files)
def parse_result(self, result, files):
sha1 = result['resource']
path = files[sha1]
if result['response_code'] > 0:
positives = result['positives']
self.logger.debug("%s - %d positives.", path, positives)
if positives > 0:
detections = {
engine: detection
for engine, detection in result['scans'].items()
if detection['detected']
}
yield VTReport(path, sha1, detections)
else:
self.logger.debug("%s - Unknown file.", path)
yield VTReport(path, sha1, 'UNKNOWN')
def __enter__(self):
self._filesystem = FileSystem(self._disk)
self._filesystem.mount()
return self
class VulnScanner:
"""Vulnerability scanner.
Allows to scan the given disk content and query
a CVE DB for vulnerabilities.
disk must contain the path of a valid disk image.
url must be a valid URL to a REST vulnerability service.
"""
def __init__(self, disk, url):
self._disk = disk
self._filesystem = None
self._url = url.rstrip('/')
self.logger = logging.getLogger(
"%s.%s" % (self.__module__, self.__class__.__name__))
def __enter__(self):
self._filesystem = FileSystem(self._disk)
self._filesystem.mount()
return self
def __exit__(self, *_):
self._filesystem.umount()
def __getattr__(self, attr):
return getattr(self._filesystem, attr)
def scan(self, concurrency=1):
"""Iterates over the applications installed within the disk
and queries the CVE DB to determine whether they are vulnerable.
Concurrency controls the amount of concurrent queries
against the CVE DB.
For each vulnerable application the method yields a namedtuple:
VulnApp(name -> application name
version -> application version
vulnerabilities) -> list of Vulnerabilities
Vulnerability(id -> CVE Id
summary) -> brief description of the vulnerability
"""
self.logger.debug("Scanning FS content.")
with ThreadPoolExecutor(max_workers=concurrency) as executor:
results = executor.map(self.query_vulnerabilities,
self.applications())
for report in results:
application, vulnerabilities = report
vulnerabilities = list(lookup_vulnerabilities(application.version,
vulnerabilities))
if vulnerabilities:
yield VulnApp(application.name,
application.version,
vulnerabilities)
def query_vulnerabilities(self, application):
self.logger.debug("Quering %s vulnerabilities.", application.name)
name = application.name.lower()
url = '/'.join((self._url, name, name))
response = requests.get(url)
response.raise_for_status()
return application, response.json()
def applications(self):
return (Application(a['app2_name'], a['app2_version'])
for a in self._filesystem.inspect_list_applications2(
self._filesystem._root))
class VulnScanner:
"""Vulnerability scanner.
Allows to scan the given disk content and query
a CVE DB for vulnerabilities.
disk must contain the path of a valid disk image.
"""
def __init__(self, disk, cvefeed):
self._disk = disk
self._filesystem = None
self._cvefeed = load_local(cvefeed)['CVE_Items']
self.logger = logging.getLogger(
"%s.%s" % (self.__module__, self.__class__.__name__))
self.logger.setLevel(50)
def __enter__(self):
self._filesystem = FileSystem(self._disk)
self._filesystem.mount()
return self
def __exit__(self, *_):
self._filesystem.umount()
def __getattr__(self, attr):
return getattr(self._filesystem, attr)
def scan(self, concurrency=1):
"""Iterates over the applications installed within the disk
and queries the CVE DB to determine whether they are vulnerable.
Concurrency controls the amount of concurrent queries
against the CVE DB.
For each vulnerable application the method yields a namedtuple:
VulnApp(name -> application name
version -> application version
vulnerabilities) -> list of Vulnerabilities
Vulnerability(id -> CVE Id
summary) -> brief description of the vulnerability
"""
self.logger.debug("Scanning FS content.")
applications = self.applications()
#print("#####application versions: ######")
#for application in applications:
#print(application.name + " : " + application.version + " : " + application.publisher)
with ThreadPoolExecutor(max_workers=concurrency) as executor:
results = executor.map(self.query_vulnerabilities,
self.applications())
for report in results:
#TODO: implement output design here for each application in correct json format
application, vulnerabilities = report
#vulnerabilities = list(lookup_vulnerabilities(application.version,
# vulnerabilities))
if vulnerabilities:
# full_vulnerabilities = [FullVuln(v.id, v.summary, self.query_cve_info(v.id)) for v in vulnerabilities]
yield VulnApp(application.name, application.version,
vulnerabilities)
def query_vulnerabilities(self, application):
self.logger.debug("Quering %s vulnerabilities.", application.name)
name = application.name.lower()
version = application.version
results = []
for cve in self._cvefeed:
vendor_list = cve['cve']['affects']['vendor']['vendor_data']
for vendor in vendor_list:
for product in vendor['product']['product_data']:
if product['product_name'].lower() == name:
product_versions_list = product['version'][
'version_data']
if {'version_value': version} in product_versions_list:
#print(name + ":" + cve['cve']['CVE_data_meta']['ID'] + ":" + version)
results.append(cve)
return application, results
def query_cve_info(self, cve_id):
# query local cve database
result = [
item['cve'] for item in self._cvefeed
if item['cve']['CVE_data_meta']['ID'] == cve_id
]
return result
def applications(self):
return (Application(a['app2_name'], a['app2_version'],
a['app2_publisher'])
for a in self._filesystem.inspect_list_applications2(
self._filesystem._root))
class FSTimeline:
def __init__(self, disk):
self._disk = disk
self._filesystem = None
self._filetype_cache = {}
self._checksum_cache = {}
self.logger = logging.getLogger(
"%s.%s" % (self.__module__, self.__class__.__name__))
def __enter__(self):
self._filesystem = FileSystem(self._disk)
self._filesystem.mount()
return self
def __exit__(self, *_):
self._filesystem.umount()
def __getattr__(self, attr):
return getattr(self._filesystem, attr)
def timeline(self):
self.logger.debug("Extracting File System timeline events.")
events = tuple(
Event(d.inode, d.path, d.size, d.allocated, t, r)
for d in self._visit_filesystem()
for t, r in ((d.atime, 'access'), (d.mtime, 'change'),
(d.ctime, 'attribute_change'), (d.crtime, 'creation'))
if t > 0)
self.logger.debug("Sorting File System timeline events.")
return sorted(events, key=lambda e: e.timestamp)
@lru_cache(maxsize=None)
def file(self, path):
"""Identifies the file type.
Caches the result to reduce overhead on duplicated events.
"""
return self._filesystem.file(path)
@lru_cache(maxsize=None)
def checksum(self, path):
"""Identifies the file type.
Caches the result to reduce overhead on duplicated events.
"""
return self._filesystem.checksum(path)
def _visit_filesystem(self):
"""Walks through the filesystem content."""
self.logger.debug("Parsing File System content.")
root_partition = self._filesystem.inspect_get_roots()[0]
yield from self._root_dirent()
for entry in self._filesystem.filesystem_walk(root_partition):
yield Dirent(
entry['tsk_inode'],
self._filesystem.path('/' + entry['tsk_name']),
entry['tsk_size'], entry['tsk_type'],
True if entry['tsk_flags'] & TSK_ALLOC else False,
timestamp(entry['tsk_atime_sec'], entry['tsk_atime_nsec']),
timestamp(entry['tsk_mtime_sec'], entry['tsk_mtime_nsec']),
timestamp(entry['tsk_ctime_sec'], entry['tsk_ctime_nsec']),
timestamp(entry['tsk_crtime_sec'], entry['tsk_crtime_nsec']))
def _root_dirent(self):
"""Returns the root folder dirent as filesystem_walk API doesn't."""
fstat = self._filesystem.stat('/')
yield Dirent(fstat['ino'], self._filesystem.path('/'), fstat['size'],
'd', True, timestamp(fstat['atime'], 0),
timestamp(fstat['mtime'],
0), timestamp(fstat['ctime'], 0), 0)
class FSTimeline:
def __init__(self, disk):
self._disk = disk
self._filesystem = None
self._filetype_cache = {}
self._checksum_cache = {}
self.logger = logging.getLogger(
"%s.%s" % (self.__module__, self.__class__.__name__))
def __enter__(self):
self._filesystem = FileSystem(self._disk)
self._filesystem.mount()
return self
def __exit__(self, *_):
self._filesystem.umount()
def __getattr__(self, attr):
return getattr(self._filesystem, attr)
def timeline(self):
self.logger.debug("Extracting File System timeline events.")
events = tuple(Event(d.inode, d.path, d.size, d.allocated, t, r)
for d in self._visit_filesystem()
for t, r in ((d.atime, 'access'),
(d.mtime, 'change'),
(d.ctime, 'attribute_change'),
(d.crtime, 'creation'))
if t > 0)
self.logger.debug("Sorting File System timeline events.")
return sorted(events, key=lambda e: e.timestamp)
@lru_cache(maxsize=None)
def file(self, path):
"""Identifies the file type.
Caches the result to reduce overhead on duplicated events.
"""
return self._filesystem.file(path)
@lru_cache(maxsize=None)
def checksum(self, path):
"""Identifies the file type.
Caches the result to reduce overhead on duplicated events.
"""
return self._filesystem.checksum(path)
def _visit_filesystem(self):
"""Walks through the filesystem content."""
self.logger.debug("Parsing File System content.")
root_partition = self._filesystem.inspect_get_roots()[0]
yield from self._root_dirent()
for entry in self._filesystem.filesystem_walk(root_partition):
yield Dirent(
entry['tsk_inode'],
self._filesystem.path('/' + entry['tsk_name']),
entry['tsk_size'], entry['tsk_type'],
True if entry['tsk_flags'] & TSK_ALLOC else False,
timestamp(entry['tsk_atime_sec'], entry['tsk_atime_nsec']),
timestamp(entry['tsk_mtime_sec'], entry['tsk_mtime_nsec']),
timestamp(entry['tsk_ctime_sec'], entry['tsk_ctime_nsec']),
timestamp(entry['tsk_crtime_sec'], entry['tsk_crtime_nsec']))
def _root_dirent(self):
"""Returns the root folder dirent as filesystem_walk API doesn't."""
fstat = self._filesystem.stat('/')
yield Dirent(fstat['ino'], self._filesystem.path('/'),
fstat['size'], 'd', True,
timestamp(fstat['atime'], 0),
timestamp(fstat['mtime'], 0),
timestamp(fstat['ctime'], 0),
0)
class VulnScanner:
"""Vulnerability scanner.
Allows to scan the given disk content and query
a CVE DB for vulnerabilities.
disk must contain the path of a valid disk image.
url must be a valid URL to a REST vulnerability service.
"""
def __init__(self, disk, url):
self._disk = disk
self._filesystem = None
self._url = url.rstrip('/')
self.logger = logging.getLogger(
"%s.%s" % (self.__module__, self.__class__.__name__))
def __enter__(self):
self._filesystem = FileSystem(self._disk)
self._filesystem.mount()
return self
def __exit__(self, *_):
self._filesystem.umount()
def __getattr__(self, attr):
return getattr(self._filesystem, attr)
def scan(self, concurrency=1):
"""Iterates over the applications installed within the disk
and queries the CVE DB to determine whether they are vulnerable.
Concurrency controls the amount of concurrent queries
against the CVE DB.
For each vulnerable application the method yields a namedtuple:
VulnApp(name -> application name
version -> application version
vulnerabilities) -> list of Vulnerabilities
Vulnerability(id -> CVE Id
summary) -> brief description of the vulnerability
"""
self.logger.debug("Scanning FS content.")
with ThreadPoolExecutor(max_workers=concurrency) as executor:
results = executor.map(self.query_vulnerabilities,
self.applications())
for report in results:
application, vulnerabilities = report
vulnerabilities = list(
lookup_vulnerabilities(application.version, vulnerabilities))
if vulnerabilities:
yield VulnApp(application.name, application.version,
vulnerabilities)
def query_vulnerabilities(self, application):
self.logger.debug("Quering %s vulnerabilities.", application.name)
name = application.name.lower()
url = '/'.join((self._url, name, name))
response = requests.get(url)
response.raise_for_status()
return application, response.json()
def applications(self):
return (Application(a['app2_name'], a['app2_version'])
for a in self._filesystem.inspect_list_applications2(
self._filesystem._root))
class VTScanner:
"""VirusTotal scanner.
Allows to scan the given disk content and query VirusTotal.
disk must contain the path of a valid disk image.
apikey must be a valid VT API key.
The attribute batchsize controls the amount of object per VT query.
"""
def __init__(self, disk, apikey):
self._disk = disk
self._apikey = apikey
self._filesystem = None
self.batchsize = 1
self.logger = logging.getLogger(
"%s.%s" % (self.__module__, self.__class__.__name__))
def __enter__(self):
self._filesystem = FileSystem(self._disk)
self._filesystem.mount()
return self
def __exit__(self, *_):
self._filesystem.umount()
def __getattr__(self, attr):
return getattr(self._filesystem, attr)
@property
def apikey(self):
return self._apikey
def scan(self, filetypes=None):
"""Iterates over the content of the disk and queries VirusTotal
to determine whether it's malicious or not.
filetypes is a list containing regular expression patterns.
If given, only the files which type will match with one or more of
the given patterns will be queried against VirusTotal.
For each file which is unknown by VT or positive to any of its engines,
the method yields a namedtuple:
VTReport(path -> C:\\Windows\\System32\\infected.dll
hash -> ab231...
detections) -> dictionary engine -> detection
Files unknown by VirusTotal will contain the string 'unknown'
in the detection field.
"""
self.logger.debug("Scanning FS content.")
checksums = self.filetype_filter(self._filesystem.checksums('/'),
filetypes=filetypes)
self.logger.debug("Querying %d objects to VTotal.", len(checksums))
for files in chunks(checksums, size=self.batchsize):
files = dict((reversed(e) for e in files))
response = vtquery(self._apikey, files.keys())
yield from self.parse_response(files, response)
def filetype_filter(self, files, filetypes=None):
if filetypes is not None:
return [f for f in files
if any((re.match(t, self._filesystem.file(f[0]))
for t in filetypes))]
else:
return files
def parse_response(self, files, response):
response = isinstance(response, list) and response or [response]
for result in response:
yield from self.parse_result(result, files)
def parse_result(self, result, files):
sha1 = result['resource']
path = files[sha1]
if result['response_code'] > 0:
positives = result['positives']
self.logger.debug("%s - %d positives.", path, positives)
if positives > 0:
detections = {engine: detection for engine, detection
in result['scans'].items()
if detection['detected']}
yield VTReport(path, sha1, detections)
else:
self.logger.debug("%s - Unknown file.", path)
yield VTReport(path, sha1, 'UNKNOWN')
