def _threaded_malware(dockerDriver, temp_dir, malware_binaries): # Get malware binaries if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('Retrieving malware files from the docker image ...') malware_binaries.extend(malware_extractor.get_malware_included_in_docker_image(docker_driver=dockerDriver, temp_dir=temp_dir)) if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('Malware files from the docker image retrieved')
def _threaded_malware(dockerDriver, temp_dir, malware_binaries): # Get malware binaries if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('Retrieving malware files from the docker image ...') malware_binaries.extend(malware_extractor.get_malware_included_in_docker_image(docker_driver=dockerDriver, temp_dir=temp_dir)) if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('Malware files from the docker image retrieved')
def _threaded_dependencies(dockerDriver, image_name, temp_dir, dependencies): # Get programming language dependencies if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('Retrieving dependencies from the docker image ...') dependencies.extend(dep_info_extractor.get_dependencies_from_docker_image(docker_driver=dockerDriver, image_name=image_name, temp_dir=temp_dir)) if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('Dependencies from the docker image retrieved')
def _threaded_dependencies(dockerDriver, image_name, temp_dir, dependencies): # Get programming language dependencies if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('Retrieving dependencies from the docker image ...') dependencies.extend(dep_info_extractor.get_dependencies_from_docker_image(docker_driver=dockerDriver, image_name=image_name, temp_dir=temp_dir)) if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('Dependencies from the docker image retrieved')
def _init_or_update_db(): try: InternalServer.get_mongodb_driver().insert_init_db_process_status({ 'status': 'Initializing', 'timestamp': datetime.datetime.now().timestamp() }) # Init db db_composer = DBComposer() db_composer.compose_vuln_db() InternalServer.get_mongodb_driver().insert_init_db_process_status({ 'status': 'Updated', 'timestamp': datetime.datetime.now().timestamp() }) except Exception as ex: message = "Unexpected exception of type {0} occurred: {1!r}".format( type(ex).__name__, ex.args) DagdaLogger.get_logger().error(message) if InternalServer.is_debug_logging_enabled(): traceback.print_exc() InternalServer.get_mongodb_driver().insert_init_db_process_status({ 'status': message, 'timestamp': datetime.datetime.now().timestamp() })
def get_rhsa_and_rhba_bz2_archive_files(): if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug( "Collecting RHSA and RHBA .bz2 archive files...") feed_json_file = get_http_resource_content( "https://www.redhat.com/security/data/oval/v2/feed.json") bz2_archive_files = [] for entry in json.loads(feed_json_file)["feed"]["entry"]: if entry["link"][0]["length"] > 850: if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug("Collected " + entry["content"]["src"]) bz2_archive_files.append( get_http_resource_content(entry["content"]["src"])) return bz2_archive_files
def _threaded_cve_gathering(mongoDbDriver, i): if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('... Including CVEs - ' + str(i)) compressed_content = get_http_resource_content( "https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-" + str(i) + ".json.gz") cve_list, cve_ext_info_list = get_cve_list_from_file(compressed_content, i) if len(cve_list) > 0: mongoDbDriver.bulk_insert_cves(cve_list) if len(cve_ext_info_list) > 0: mongoDbDriver.bulk_insert_cves_info(cve_ext_info_list)
def get_rhsa_and_rhba_lists_from_files(bz2_files): if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug("Processing vulnerabilities...") rhsa_list = [] rhba_list = [] rhsa_info_list = [] rhba_info_list = [] count = 0 for bz2_file in bz2_files: _rhsa_list, _rhba_list, _rhsa_info_list, _rhba_info_list = get_rhsa_and_rhba_lists_from_file( bz2_file) rhsa_list.extend(_rhsa_list) rhba_list.extend(_rhba_list) rhsa_info_list.extend(_rhsa_info_list) rhba_info_list.extend(_rhba_info_list) count = count + 1 if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug("Processed: " + str(count) + "/" + str(len(bz2_files))) return rhsa_list, rhba_list, rhsa_info_list, rhba_info_list
def _init_or_update_db(): try: InternalServer.get_mongodb_driver().insert_init_db_process_status( {'status': 'Initializing', 'timestamp': datetime.datetime.now().timestamp()}) # Init db db_composer = DBComposer() db_composer.compose_vuln_db() InternalServer.get_mongodb_driver().insert_init_db_process_status( {'status': 'Updated', 'timestamp': datetime.datetime.now().timestamp()}) except Exception as ex: message = "Unexpected exception of type {0} occurred: {1!r}".format(type(ex).__name__, ex.args) DagdaLogger.get_logger().error(message) if InternalServer.is_debug_logging_enabled(): traceback.print_exc() InternalServer.get_mongodb_driver().insert_init_db_process_status( {'status': message, 'timestamp': datetime.datetime.now().timestamp()})
def _untar_layers(dir, layers): output = {} # Untar layer filesystem bundle for layer in layers: tarfile = TarFile(dir + "/" + layer) for member in tarfile.getmembers(): try: tarfile.extract(member, path=dir, set_attrs=False) except (ValueError, ReadError) as ex: if InternalServer.is_debug_logging_enabled(): message = "Unexpected exception of type {0} occurred while untaring the docker image: {1!r}" \ .format(type(ex).__name__, ex.get_message() if type(ex).__name__ == 'DagdaError' else ex.args) DagdaLogger.get_logger().debug(message) except PermissionError as ex: message = "Unexpected error occurred while untaring the docker image: " + \ "Operation not permitted on {0!r}".format(member.name) DagdaLogger.get_logger().warn(message) # Clean up for layer in layers: clean_up(dir + "/" + layer[:-10])
def _threaded_cve_gathering(mongoDbDriver, i): if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('... Including CVEs - ' + str(i)) compressed_content = get_http_resource_content( "https://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-" + str(i) + ".xml.gz") cve_list = get_cve_list_from_file(compressed_content, i) if len(cve_list) > 0: mongoDbDriver.bulk_insert_cves(cve_list) # Add CVE info collection with additional info like score compressed_content_info = get_http_resource_content( "https://nvd.nist.gov/download/nvdcve-" + str(i) + ".xml.zip") cve_info_list = get_cve_description_from_file(compressed_content_info) compressed_ext_content_info = \ get_http_resource_content("https://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-" + str(i) + ".xml.zip") cve_ext_info_list = get_cve_cweid_from_file( compressed_ext_content_info, cve_info_list) if len(cve_ext_info_list) > 0: mongoDbDriver.bulk_insert_cves_info(cve_ext_info_list)
def run(self): # Read docker daemon events while True: try: for event in self.docker_driver.docker_events(): e = json.loads(event.decode('UTF-8').replace("\n", "")) if 'Actor' in e and 'Attributes' in e['Actor']: iter = list(e['Actor']['Attributes']) for key in iter: if '.' in key: del e['Actor']['Attributes'][key] # Bulk insert self.mongodb_driver.bulk_insert_docker_daemon_events([e]) except requests.packages.urllib3.exceptions.ReadTimeoutError: # Nothing to do pass except bson.errors.InvalidDocument as ex: message = "Unexpected exception of type {0} occurred: {1!r}" \ .format(type(ex).__name__, ex.get_message() if type(ex).__name__ == 'DagdaError' else ex.args) DagdaLogger.get_logger().error(message) if InternalServer.is_debug_logging_enabled(): traceback.print_exc()
def run(self): # Read docker daemon events while True: try: for event in self.docker_driver.docker_events(): e = json.loads(event.decode('UTF-8').replace("\n", "")) if 'Actor' in e and 'Attributes' in e['Actor']: iter = list(e['Actor']['Attributes']) for key in iter: if '.' in key: del e['Actor']['Attributes'][key] # Bulk insert self.mongodb_driver.bulk_insert_docker_daemon_events([e]) except requests.packages.urllib3.exceptions.ReadTimeoutError: # Nothing to do pass except bson.errors.InvalidDocument as ex: message = "Unexpected exception of type {0} occurred: {1!r}" \ .format(type(ex).__name__, ex.get_message() if type(ex).__name__ == 'DagdaError' else ex.args) DagdaLogger.get_logger().error(message) if InternalServer.is_debug_logging_enabled(): traceback.print_exc()
def evaluate_image(self, image_name, container_id): if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug( 'ENTRY to the method for analyzing a docker image') # Init data = {} # -- Static analysis image_name = self.dockerDriver.get_docker_image_name_by_container_id(container_id) if container_id \ else image_name os_packages = [] malware_binaries = [] dependencies = None temp_dir = None try: # Get OS packages if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug( 'Retrieving OS packages from the docker image ...') if container_id is None: # Scans the docker image os_packages = os_info_extractor.get_soft_from_docker_image( docker_driver=self.dockerDriver, image_name=image_name) temp_dir = extract_filesystem_bundle( docker_driver=self.dockerDriver, image_name=image_name) else: # Scans the docker container os_packages = os_info_extractor.get_soft_from_docker_container_id( docker_driver=self.dockerDriver, container_id=container_id) temp_dir = extract_filesystem_bundle( docker_driver=self.dockerDriver, container_id=container_id) if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug( 'OS packages from the docker image retrieved') # Get malware binaries if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug( 'Retrieving malware files from the docker image ...') malware_binaries = malware_extractor.get_malware_included_in_docker_image( docker_driver=self.dockerDriver, temp_dir=temp_dir) if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug( 'Malware files from the docker image retrieved') # Get programming language dependencies if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug( 'Retrieving dependencies from the docker image ...') dependencies = dep_info_extractor.get_dependencies_from_docker_image( docker_driver=self.dockerDriver, image_name=image_name, temp_dir=temp_dir) if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug( 'Dependencies from the docker image retrieved') except Exception as ex: message = "Unexpected exception of type {0} occured: {1!r}"\ .format(type(ex).__name__, ex.get_message() if type(ex).__name__ == 'DagdaError' else ex.args) DagdaLogger.get_logger().error(message) if InternalServer.is_debug_logging_enabled(): traceback.print_exc() data['status'] = message # -- Cleanup if temp_dir is not None: clean_up(temporary_dir=temp_dir) # -- Prepare output if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('Preparing analysis output ...') if dependencies is not None: data['status'] = 'Completed' else: dependencies = [] data['image_name'] = image_name data['timestamp'] = datetime.datetime.now().timestamp() data['static_analysis'] = self.generate_static_analysis( image_name, os_packages, dependencies, malware_binaries) if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('Analysis output completed') # -- Return if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug( 'EXIT from the method for analyzing a docker image') return data
def compose_vuln_db(self): if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('ENTRY to the method for composing VulnDB') # -- CVE # Adding or updating CVEs if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('Updating CVE collection ...') first_year = self.mongoDbDriver.remove_only_cve_for_update() for i in range(first_year, next_year): if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('... Including CVEs - ' + str(i)) compressed_content = get_http_resource_content( "https://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-" + str(i) + ".xml.gz") cve_list = get_cve_list_from_file(compressed_content, i) if len(cve_list) > 0: self.mongoDbDriver.bulk_insert_cves(cve_list) # Add CVE info collection with additional info like score compressed_content_info = get_http_resource_content("https://nvd.nist.gov/download/nvdcve-" + str(i) + ".xml.zip") cve_info_list = get_cve_description_from_file(compressed_content_info) compressed_ext_content_info = \ get_http_resource_content("https://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-" + str(i) + ".xml.zip") cve_ext_info_list = get_cve_cweid_from_file(compressed_ext_content_info, cve_info_list) if len(cve_ext_info_list) > 0: self.mongoDbDriver.bulk_insert_cves_info(cve_ext_info_list) if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('CVE collection updated') # -- Exploit DB # Adding or updating Exploit_db and Exploit_db info if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('Updating Exploit DB collection ...') self.mongoDbDriver.delete_exploit_db_collection() self.mongoDbDriver.delete_exploit_db_info_collection() csv_content = get_http_resource_content( 'https://raw.githubusercontent.com/offensive-security/exploit-database/master/files_exploits.csv') exploit_db_list, exploit_db_info_list = get_exploit_db_list_from_csv(csv_content.decode("utf-8")) self.mongoDbDriver.bulk_insert_exploit_db_ids(exploit_db_list) self.mongoDbDriver.bulk_insert_exploit_db_info(exploit_db_info_list) if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('Exploit DB collection updated') # -- BID # Adding BugTraqs from 20180328_sf_db.json.gz, where 103525 is the max bid in the gz file if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('Updating BugTraqs Id collection ...') max_bid = self.mongoDbDriver.get_max_bid_inserted() if max_bid < 103525: # Clean if max_bid != 0: self.mongoDbDriver.delete_bid_collection() self.mongoDbDriver.delete_bid_info_collection() # Adding BIDs compressed_file = io.BytesIO(get_http_resource_content( "https://github.com/eliasgranderubio/bidDB_downloader/raw/master/bonus_track/20180328_sf_db.json.gz")) bid_items_array, bid_detail_array = get_bug_traqs_lists_from_file(compressed_file) # Insert BIDs for bid_items_list in bid_items_array: self.mongoDbDriver.bulk_insert_bids(bid_items_list) bid_items_list.clear() # Insert BID details self.mongoDbDriver.bulk_insert_bid_info(bid_detail_array) bid_detail_array.clear() # Set the new max bid max_bid = 103525 # Updating BugTraqs from http://www.securityfocus.com/ bid_items_array, bid_detail_array = get_bug_traqs_lists_from_online_mode(bid_downloader(first_bid=max_bid+1, last_bid=104000)) # Insert BIDs if len(bid_items_array) > 0: for bid_items_list in bid_items_array: self.mongoDbDriver.bulk_insert_bids(bid_items_list) bid_items_list.clear() # Insert BID details if len(bid_detail_array) > 0: self.mongoDbDriver.bulk_insert_bid_info(bid_detail_array) bid_detail_array.clear() if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('BugTraqs Id collection updated') # -- RHSA (Red Hat Security Advisory) and RHBA (Red Hat Bug Advisory) # Adding or updating rhsa and rhba collections if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('Updating RHSA & RHBA collections ...') self.mongoDbDriver.delete_rhba_collection() self.mongoDbDriver.delete_rhba_info_collection() self.mongoDbDriver.delete_rhsa_collection() self.mongoDbDriver.delete_rhsa_info_collection() bz2_file = get_http_resource_content('https://www.redhat.com/security/data/oval/rhsa.tar.bz2') rhsa_list, rhba_list, rhsa_info_list, rhba_info_list = get_rhsa_and_rhba_lists_from_file(bz2_file) self.mongoDbDriver.bulk_insert_rhsa(rhsa_list) self.mongoDbDriver.bulk_insert_rhba(rhba_list) self.mongoDbDriver.bulk_insert_rhsa_info(rhsa_info_list) self.mongoDbDriver.bulk_insert_rhba_info(rhba_info_list) if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('RHSA & RHBA collections updated') if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('EXIT from the method for composing VulnDB')
def evaluate_image(self, image_name, container_id): if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('ENTRY to the method for analyzing a docker image') # Init data = {} # -- Static analysis image_name = self.dockerDriver.get_docker_image_name_by_container_id(container_id) if container_id \ else image_name os_packages = [] malware_binaries = [] dependencies = [] temp_dir = None try: # Get OS packages if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('Retrieving OS packages from the docker image ...') if container_id is None: # Scans the docker image os_packages = os_info_extractor.get_soft_from_docker_image(docker_driver=self.dockerDriver, image_name=image_name) temp_dir = extract_filesystem_bundle(docker_driver=self.dockerDriver, image_name=image_name) else: # Scans the docker container os_packages = os_info_extractor.get_soft_from_docker_container_id(docker_driver=self.dockerDriver, container_id=container_id) temp_dir = extract_filesystem_bundle(docker_driver=self.dockerDriver, container_id=container_id) if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('OS packages from the docker image retrieved') # Get malware binaries in a parallel way malware_thread = Thread(target=Analyzer._threaded_malware, args=(self.dockerDriver, temp_dir, malware_binaries)) malware_thread.start() # Get programming language dependencies in a parallel way dependencies_thread = Thread(target=Analyzer._threaded_dependencies, args=(self.dockerDriver, image_name, temp_dir, dependencies)) dependencies_thread.start() # Waiting for the threads malware_thread.join() dependencies_thread.join() except Exception as ex: message = "Unexpected exception of type {0} occurred: {1!r}"\ .format(type(ex).__name__, ex.get_message() if type(ex).__name__ == 'DagdaError' else ex.args) DagdaLogger.get_logger().error(message) if InternalServer.is_debug_logging_enabled(): traceback.print_exc() data['status'] = message # -- Cleanup if temp_dir is not None: clean_up(temporary_dir=temp_dir) # -- Prepare output if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('Preparing analysis output ...') if 'status' not in data or data['status'] is None: data['status'] = 'Completed' data['image_name'] = image_name data['timestamp'] = datetime.datetime.now().timestamp() data['static_analysis'] = self.generate_static_analysis(image_name, os_packages, dependencies, malware_binaries) if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('Analysis output completed') # -- Return if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('EXIT from the method for analyzing a docker image') return data
def compose_vuln_db(self): if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('ENTRY to the method for composing VulnDB') # -- CVE # Adding or updating CVEs if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('Updating CVE collection ...') first_year = self.mongoDbDriver.remove_only_cve_for_update() threads = list() for i in range(first_year, next_year): tmp_thread = Thread(target=DBComposer._threaded_cve_gathering, args=(self.mongoDbDriver, i)) threads.append(tmp_thread) tmp_thread.start() # Waiting for the threads for i in threads: i.join() if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('CVE collection updated') # -- Exploit DB # Adding or updating Exploit_db and Exploit_db info if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('Updating Exploit DB collection ...') self.mongoDbDriver.delete_exploit_db_collection() self.mongoDbDriver.delete_exploit_db_info_collection() csv_content = get_http_resource_content( 'https://raw.githubusercontent.com/offensive-security/exploit-database/master/files_exploits.csv') exploit_db_list, exploit_db_info_list = get_exploit_db_list_from_csv(csv_content.decode("utf-8")) self.mongoDbDriver.bulk_insert_exploit_db_ids(exploit_db_list) self.mongoDbDriver.bulk_insert_exploit_db_info(exploit_db_info_list) if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('Exploit DB collection updated') # -- RHSA (Red Hat Security Advisory) and RHBA (Red Hat Bug Advisory) # Adding or updating rhsa and rhba collections if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('Updating RHSA & RHBA collections ...') self.mongoDbDriver.delete_rhba_collection() self.mongoDbDriver.delete_rhba_info_collection() self.mongoDbDriver.delete_rhsa_collection() self.mongoDbDriver.delete_rhsa_info_collection() bz2_file = get_http_resource_content('https://www.redhat.com/security/data/oval/rhsa.tar.bz2') rhsa_list, rhba_list, rhsa_info_list, rhba_info_list = get_rhsa_and_rhba_lists_from_file(bz2_file) self.mongoDbDriver.bulk_insert_rhsa(rhsa_list) self.mongoDbDriver.bulk_insert_rhba(rhba_list) self.mongoDbDriver.bulk_insert_rhsa_info(rhsa_info_list) self.mongoDbDriver.bulk_insert_rhba_info(rhba_info_list) if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('RHSA & RHBA collections updated') # -- BID # Adding BugTraqs from 20180328_sf_db.json.gz, where 103525 is the max bid in the gz file if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('Updating BugTraqs Id collection ...') max_bid = self.mongoDbDriver.get_max_bid_inserted() bid_thread = Thread(target=DBComposer._threaded_preprocessed_bid_gathering, args=(self.mongoDbDriver, max_bid)) if max_bid < 103525: bid_thread.start() # Set the new max bid max_bid = 103525 # Updating BugTraqs from http://www.securityfocus.com/ bid_items_array, bid_detail_array = get_bug_traqs_lists_from_online_mode(bid_downloader(first_bid=max_bid+1, last_bid=104000)) # Insert BIDs if len(bid_items_array) > 0: for bid_items_list in bid_items_array: self.mongoDbDriver.bulk_insert_bids(bid_items_list) bid_items_list.clear() # Insert BID details if len(bid_detail_array) > 0: self.mongoDbDriver.bulk_insert_bid_info(bid_detail_array) bid_detail_array.clear() # Wait for bid_thread if bid_thread.is_alive(): bid_thread.join() if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('BugTraqs Id collection updated') if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('EXIT from the method for composing VulnDB')