class WinEventLog: """WinEventLog class. Allows to retrieve the Events contained within Windows Event Log files. """ def __init__(self, disk): self._disk = disk self._filesystem = None self.logger = logging.getLogger( "%s.%s" % (self.__module__, self.__class__.__name__)) def __enter__(self): self._filesystem = FileSystem(self._disk) self._filesystem.mount() return self def __exit__(self, *_): self._filesystem.umount() def __getattr__(self, attr): return getattr(self._filesystem, attr) def eventlog(self, path): """Iterates over the Events contained within the log at the given path. For each Event, yields a XML string. """ self.logger.debug("Parsing Event log file %s.", path) with NamedTemporaryFile(buffering=0) as tempfile: self._filesystem.download(path, tempfile.name) file_header = FileHeader(tempfile.read(), 0) for xml_string, _ in evtx_file_xml_view(file_header): yield xml_string
class VTScanner: """VirusTotal scanner. Allows to scan the given disk content and query VirusTotal. disk must contain the path of a valid disk image. apikey must be a valid VT API key. The attribute batchsize controls the amount of object per VT query. """ def __init__(self, disk, apikey): self._disk = disk self._apikey = apikey self._filesystem = None self.batchsize = 1 self.logger = logging.getLogger( "%s.%s" % (self.__module__, self.__class__.__name__)) def __enter__(self): self._filesystem = FileSystem(self._disk) self._filesystem.mount() return self def __exit__(self, *_): self._filesystem.umount() def __getattr__(self, attr): return getattr(self._filesystem, attr) @property def apikey(self): return self._apikey def scan(self, filetypes=None): """Iterates over the content of the disk and queries VirusTotal to determine whether it's malicious or not. filetypes is a list containing regular expression patterns. If given, only the files which type will match with one or more of the given patterns will be queried against VirusTotal. For each file which is unknown by VT or positive to any of its engines, the method yields a namedtuple: VTReport(path -> C:\\Windows\\System32\\infected.dll hash -> ab231... detections) -> dictionary engine -> detection Files unknown by VirusTotal will contain the string 'unknown' in the detection field. """ print("I'm here!1") self.logger.debug("Scanning FS content.") all_checksums = self._filesystem.checksums('/') print("I'm here!2") checksums = self.filetype_filter(all_checksums, filetypes=filetypes) print(len(list(checksums))) print("I'm here!3") # self.logger.debug("Querying %d objects to VTotal.", checksums) # print("checksum: " + checksums) aggregated_files = {} for files in chunks(checksums, size=self.batchsize): files = dict((reversed(e) for e in files)) if len(aggregated_files) < 100: aggregated_files.update(files) else: # print("@@@@@@@@@@@@@@ files @@@@@@@@@@@@@") # for key in files.keys(): # print(key + ", " + files[key]) response = vtquery(self._apikey, aggregated_files.keys()) yield from self.parse_response(aggregated_files, response) aggregated_files = {} def filetype_filter(self, files, filetypes=None): # print(len(list(files))) #for f in files: # print(f) # print(type(filetypes)) # print(filetypes) if filetypes is not None: #return [f for f in files # if any((re.match(t, self._filesystem.file(f[0])) # for t in filetypes))] matched_file_list = [] for f in files: for t in filetypes: if re.match(t, f[0]): # print(self._filesystem.file(f[0])) matched_file_list.append(f) break return matched_file_list else: return files def parse_response(self, files, response): response = isinstance(response, list) and response or [response] for result in response: yield from self.parse_result(result, files) def parse_result(self, result, files): sha1 = result['resource'] path = files[sha1] if result['response_code'] > 0: positives = result['positives'] self.logger.debug("%s - %d positives.", path, positives) if positives > 0: detections = { engine: detection for engine, detection in result['scans'].items() if detection['detected'] } yield VTReport(path, sha1, detections) else: self.logger.debug("%s - Unknown file.", path) yield VTReport(path, sha1, 'UNKNOWN')
class VulnScanner: """Vulnerability scanner. Allows to scan the given disk content and query a CVE DB for vulnerabilities. disk must contain the path of a valid disk image. url must be a valid URL to a REST vulnerability service. """ def __init__(self, disk, url): self._disk = disk self._filesystem = None self._url = url.rstrip('/') self.logger = logging.getLogger( "%s.%s" % (self.__module__, self.__class__.__name__)) def __enter__(self): self._filesystem = FileSystem(self._disk) self._filesystem.mount() return self def __exit__(self, *_): self._filesystem.umount() def __getattr__(self, attr): return getattr(self._filesystem, attr) def scan(self, concurrency=1): """Iterates over the applications installed within the disk and queries the CVE DB to determine whether they are vulnerable. Concurrency controls the amount of concurrent queries against the CVE DB. For each vulnerable application the method yields a namedtuple: VulnApp(name -> application name version -> application version vulnerabilities) -> list of Vulnerabilities Vulnerability(id -> CVE Id summary) -> brief description of the vulnerability """ self.logger.debug("Scanning FS content.") with ThreadPoolExecutor(max_workers=concurrency) as executor: results = executor.map(self.query_vulnerabilities, self.applications()) for report in results: application, vulnerabilities = report vulnerabilities = list(lookup_vulnerabilities(application.version, vulnerabilities)) if vulnerabilities: yield VulnApp(application.name, application.version, vulnerabilities) def query_vulnerabilities(self, application): self.logger.debug("Quering %s vulnerabilities.", application.name) name = application.name.lower() url = '/'.join((self._url, name, name)) response = requests.get(url) response.raise_for_status() return application, response.json() def applications(self): return (Application(a['app2_name'], a['app2_version']) for a in self._filesystem.inspect_list_applications2( self._filesystem._root))
class VulnScanner: """Vulnerability scanner. Allows to scan the given disk content and query a CVE DB for vulnerabilities. disk must contain the path of a valid disk image. """ def __init__(self, disk, cvefeed): self._disk = disk self._filesystem = None self._cvefeed = load_local(cvefeed)['CVE_Items'] self.logger = logging.getLogger( "%s.%s" % (self.__module__, self.__class__.__name__)) self.logger.setLevel(50) def __enter__(self): self._filesystem = FileSystem(self._disk) self._filesystem.mount() return self def __exit__(self, *_): self._filesystem.umount() def __getattr__(self, attr): return getattr(self._filesystem, attr) def scan(self, concurrency=1): """Iterates over the applications installed within the disk and queries the CVE DB to determine whether they are vulnerable. Concurrency controls the amount of concurrent queries against the CVE DB. For each vulnerable application the method yields a namedtuple: VulnApp(name -> application name version -> application version vulnerabilities) -> list of Vulnerabilities Vulnerability(id -> CVE Id summary) -> brief description of the vulnerability """ self.logger.debug("Scanning FS content.") applications = self.applications() #print("#####application versions: ######") #for application in applications: #print(application.name + " : " + application.version + " : " + application.publisher) with ThreadPoolExecutor(max_workers=concurrency) as executor: results = executor.map(self.query_vulnerabilities, self.applications()) for report in results: #TODO: implement output design here for each application in correct json format application, vulnerabilities = report #vulnerabilities = list(lookup_vulnerabilities(application.version, # vulnerabilities)) if vulnerabilities: # full_vulnerabilities = [FullVuln(v.id, v.summary, self.query_cve_info(v.id)) for v in vulnerabilities] yield VulnApp(application.name, application.version, vulnerabilities) def query_vulnerabilities(self, application): self.logger.debug("Quering %s vulnerabilities.", application.name) name = application.name.lower() version = application.version results = [] for cve in self._cvefeed: vendor_list = cve['cve']['affects']['vendor']['vendor_data'] for vendor in vendor_list: for product in vendor['product']['product_data']: if product['product_name'].lower() == name: product_versions_list = product['version'][ 'version_data'] if {'version_value': version} in product_versions_list: #print(name + ":" + cve['cve']['CVE_data_meta']['ID'] + ":" + version) results.append(cve) return application, results def query_cve_info(self, cve_id): # query local cve database result = [ item['cve'] for item in self._cvefeed if item['cve']['CVE_data_meta']['ID'] == cve_id ] return result def applications(self): return (Application(a['app2_name'], a['app2_version'], a['app2_publisher']) for a in self._filesystem.inspect_list_applications2( self._filesystem._root))
class FSTimeline: def __init__(self, disk): self._disk = disk self._filesystem = None self._filetype_cache = {} self._checksum_cache = {} self.logger = logging.getLogger( "%s.%s" % (self.__module__, self.__class__.__name__)) def __enter__(self): self._filesystem = FileSystem(self._disk) self._filesystem.mount() return self def __exit__(self, *_): self._filesystem.umount() def __getattr__(self, attr): return getattr(self._filesystem, attr) def timeline(self): self.logger.debug("Extracting File System timeline events.") events = tuple( Event(d.inode, d.path, d.size, d.allocated, t, r) for d in self._visit_filesystem() for t, r in ((d.atime, 'access'), (d.mtime, 'change'), (d.ctime, 'attribute_change'), (d.crtime, 'creation')) if t > 0) self.logger.debug("Sorting File System timeline events.") return sorted(events, key=lambda e: e.timestamp) @lru_cache(maxsize=None) def file(self, path): """Identifies the file type. Caches the result to reduce overhead on duplicated events. """ return self._filesystem.file(path) @lru_cache(maxsize=None) def checksum(self, path): """Identifies the file type. Caches the result to reduce overhead on duplicated events. """ return self._filesystem.checksum(path) def _visit_filesystem(self): """Walks through the filesystem content.""" self.logger.debug("Parsing File System content.") root_partition = self._filesystem.inspect_get_roots()[0] yield from self._root_dirent() for entry in self._filesystem.filesystem_walk(root_partition): yield Dirent( entry['tsk_inode'], self._filesystem.path('/' + entry['tsk_name']), entry['tsk_size'], entry['tsk_type'], True if entry['tsk_flags'] & TSK_ALLOC else False, timestamp(entry['tsk_atime_sec'], entry['tsk_atime_nsec']), timestamp(entry['tsk_mtime_sec'], entry['tsk_mtime_nsec']), timestamp(entry['tsk_ctime_sec'], entry['tsk_ctime_nsec']), timestamp(entry['tsk_crtime_sec'], entry['tsk_crtime_nsec'])) def _root_dirent(self): """Returns the root folder dirent as filesystem_walk API doesn't.""" fstat = self._filesystem.stat('/') yield Dirent(fstat['ino'], self._filesystem.path('/'), fstat['size'], 'd', True, timestamp(fstat['atime'], 0), timestamp(fstat['mtime'], 0), timestamp(fstat['ctime'], 0), 0)
class FSTimeline: def __init__(self, disk): self._disk = disk self._filesystem = None self._filetype_cache = {} self._checksum_cache = {} self.logger = logging.getLogger( "%s.%s" % (self.__module__, self.__class__.__name__)) def __enter__(self): self._filesystem = FileSystem(self._disk) self._filesystem.mount() return self def __exit__(self, *_): self._filesystem.umount() def __getattr__(self, attr): return getattr(self._filesystem, attr) def timeline(self): self.logger.debug("Extracting File System timeline events.") events = tuple(Event(d.inode, d.path, d.size, d.allocated, t, r) for d in self._visit_filesystem() for t, r in ((d.atime, 'access'), (d.mtime, 'change'), (d.ctime, 'attribute_change'), (d.crtime, 'creation')) if t > 0) self.logger.debug("Sorting File System timeline events.") return sorted(events, key=lambda e: e.timestamp) @lru_cache(maxsize=None) def file(self, path): """Identifies the file type. Caches the result to reduce overhead on duplicated events. """ return self._filesystem.file(path) @lru_cache(maxsize=None) def checksum(self, path): """Identifies the file type. Caches the result to reduce overhead on duplicated events. """ return self._filesystem.checksum(path) def _visit_filesystem(self): """Walks through the filesystem content.""" self.logger.debug("Parsing File System content.") root_partition = self._filesystem.inspect_get_roots()[0] yield from self._root_dirent() for entry in self._filesystem.filesystem_walk(root_partition): yield Dirent( entry['tsk_inode'], self._filesystem.path('/' + entry['tsk_name']), entry['tsk_size'], entry['tsk_type'], True if entry['tsk_flags'] & TSK_ALLOC else False, timestamp(entry['tsk_atime_sec'], entry['tsk_atime_nsec']), timestamp(entry['tsk_mtime_sec'], entry['tsk_mtime_nsec']), timestamp(entry['tsk_ctime_sec'], entry['tsk_ctime_nsec']), timestamp(entry['tsk_crtime_sec'], entry['tsk_crtime_nsec'])) def _root_dirent(self): """Returns the root folder dirent as filesystem_walk API doesn't.""" fstat = self._filesystem.stat('/') yield Dirent(fstat['ino'], self._filesystem.path('/'), fstat['size'], 'd', True, timestamp(fstat['atime'], 0), timestamp(fstat['mtime'], 0), timestamp(fstat['ctime'], 0), 0)
class VulnScanner: """Vulnerability scanner. Allows to scan the given disk content and query a CVE DB for vulnerabilities. disk must contain the path of a valid disk image. url must be a valid URL to a REST vulnerability service. """ def __init__(self, disk, url): self._disk = disk self._filesystem = None self._url = url.rstrip('/') self.logger = logging.getLogger( "%s.%s" % (self.__module__, self.__class__.__name__)) def __enter__(self): self._filesystem = FileSystem(self._disk) self._filesystem.mount() return self def __exit__(self, *_): self._filesystem.umount() def __getattr__(self, attr): return getattr(self._filesystem, attr) def scan(self, concurrency=1): """Iterates over the applications installed within the disk and queries the CVE DB to determine whether they are vulnerable. Concurrency controls the amount of concurrent queries against the CVE DB. For each vulnerable application the method yields a namedtuple: VulnApp(name -> application name version -> application version vulnerabilities) -> list of Vulnerabilities Vulnerability(id -> CVE Id summary) -> brief description of the vulnerability """ self.logger.debug("Scanning FS content.") with ThreadPoolExecutor(max_workers=concurrency) as executor: results = executor.map(self.query_vulnerabilities, self.applications()) for report in results: application, vulnerabilities = report vulnerabilities = list( lookup_vulnerabilities(application.version, vulnerabilities)) if vulnerabilities: yield VulnApp(application.name, application.version, vulnerabilities) def query_vulnerabilities(self, application): self.logger.debug("Quering %s vulnerabilities.", application.name) name = application.name.lower() url = '/'.join((self._url, name, name)) response = requests.get(url) response.raise_for_status() return application, response.json() def applications(self): return (Application(a['app2_name'], a['app2_version']) for a in self._filesystem.inspect_list_applications2( self._filesystem._root))
class VTScanner: """VirusTotal scanner. Allows to scan the given disk content and query VirusTotal. disk must contain the path of a valid disk image. apikey must be a valid VT API key. The attribute batchsize controls the amount of object per VT query. """ def __init__(self, disk, apikey): self._disk = disk self._apikey = apikey self._filesystem = None self.batchsize = 1 self.logger = logging.getLogger( "%s.%s" % (self.__module__, self.__class__.__name__)) def __enter__(self): self._filesystem = FileSystem(self._disk) self._filesystem.mount() return self def __exit__(self, *_): self._filesystem.umount() def __getattr__(self, attr): return getattr(self._filesystem, attr) @property def apikey(self): return self._apikey def scan(self, filetypes=None): """Iterates over the content of the disk and queries VirusTotal to determine whether it's malicious or not. filetypes is a list containing regular expression patterns. If given, only the files which type will match with one or more of the given patterns will be queried against VirusTotal. For each file which is unknown by VT or positive to any of its engines, the method yields a namedtuple: VTReport(path -> C:\\Windows\\System32\\infected.dll hash -> ab231... detections) -> dictionary engine -> detection Files unknown by VirusTotal will contain the string 'unknown' in the detection field. """ self.logger.debug("Scanning FS content.") checksums = self.filetype_filter(self._filesystem.checksums('/'), filetypes=filetypes) self.logger.debug("Querying %d objects to VTotal.", len(checksums)) for files in chunks(checksums, size=self.batchsize): files = dict((reversed(e) for e in files)) response = vtquery(self._apikey, files.keys()) yield from self.parse_response(files, response) def filetype_filter(self, files, filetypes=None): if filetypes is not None: return [f for f in files if any((re.match(t, self._filesystem.file(f[0])) for t in filetypes))] else: return files def parse_response(self, files, response): response = isinstance(response, list) and response or [response] for result in response: yield from self.parse_result(result, files) def parse_result(self, result, files): sha1 = result['resource'] path = files[sha1] if result['response_code'] > 0: positives = result['positives'] self.logger.debug("%s - %d positives.", path, positives) if positives > 0: detections = {engine: detection for engine, detection in result['scans'].items() if detection['detected']} yield VTReport(path, sha1, detections) else: self.logger.debug("%s - Unknown file.", path) yield VTReport(path, sha1, 'UNKNOWN')