class Mitre: """Mitre connector.""" def __init__(self): # Instantiate the connector helper from config config_file_path = os.path.dirname( os.path.abspath(__file__)) + "/config.yml" config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader) if os.path.isfile(config_file_path) else {}) self.helper = OpenCTIConnectorHelper(config) # Extra config self.mitre_enterprise_file_url = get_config_variable( "MITRE_ENTERPRISE_FILE_URL", ["mitre", "enterprise_file_url"], config) self.mitre_pre_attack_file_url = get_config_variable( "MITRE_PRE_ATTACK_FILE_URL", ["mitre", "pre_attack_file_url"], config) self.mitre_mobile_attack_file_url = get_config_variable( "MITRE_MOBILE_ATTACK_FILE_URL", ["mitre", "mobile_attack_file_url"], config) self.mitre_ics_attack_file_url = get_config_variable( "MITRE_ICS_ATTACK_FILE_URL", ["mitre", "ics_attack_file_url"], config) self.mitre_interval = get_config_variable("MITRE_INTERVAL", ["mitre", "interval"], config, True) self.update_existing_data = get_config_variable( "CONNECTOR_UPDATE_EXISTING_DATA", ["connector", "update_existing_data"], config, ) self.confidence_level = get_config_variable( "CONNECTOR_CONFIDENCE_LEVEL", ["connector", "confidence_level"], config, ) def get_interval(self): return int(self.mitre_interval) * 60 * 60 * 24 def retrieve_data(self, url: str) -> Optional[str]: """ Retrieve data from the given url. Parameters ---------- url : str Url to retrieve. Returns ------- str A string with the content or None in case of failure. """ try: return (urllib.request.urlopen( url, context=ssl.create_default_context(cafile=certifi.where()), ).read().decode("utf-8")) except ( urllib.error.URLError, urllib.error.HTTPError, urllib.error.ContentTooShortError, ) as urllib_error: self.helper.log_error( f"Error retrieving url {url}: {urllib_error}") return None # Add confidence to every object in a bundle def add_confidence_to_bundle_objects(self, serialized_bundle: str) -> str: # the list of object types for which the confidence has to be added # (skip marking-definition, identity, external-reference-as-report) object_types_with_confidence = [ "attack-pattern", "course-of-action", "intrusion-set", "campaign", "malware", "tool", "report", "relationship", ] stix_bundle = json.loads(serialized_bundle) for obj in stix_bundle["objects"]: object_type = obj["type"] if object_type in object_types_with_confidence: # self.helper.log_info(f"Adding confidence to {object_type} object") obj["confidence"] = int(self.confidence_level) return json.dumps(stix_bundle) def process_data(self): try: # Get the current timestamp and check timestamp = int(time.time()) current_state = self.helper.get_state() if current_state is not None and "last_run" in current_state: last_run = current_state["last_run"] self.helper.log_info("Connector last run: " + datetime.utcfromtimestamp(last_run). strftime("%Y-%m-%d %H:%M:%S")) else: last_run = None self.helper.log_info("Connector has never run") # If the last_run is more than interval-1 day if last_run is None or ((timestamp - last_run) > ( (int(self.mitre_interval) - 1) * 60 * 60 * 24)): self.helper.log_info("Connector will run!") now = datetime.utcfromtimestamp(timestamp) friendly_name = "MITRE run @ " + now.strftime( "%Y-%m-%d %H:%M:%S") work_id = self.helper.api.work.initiate_work( self.helper.connect_id, friendly_name) # Mitre enterprise file url if (self.mitre_enterprise_file_url is not None and len(self.mitre_enterprise_file_url) > 0): enterprise_data = self.retrieve_data( self.mitre_enterprise_file_url) enterprise_data_with_confidence = ( self.add_confidence_to_bundle_objects(enterprise_data)) self.send_bundle(work_id, enterprise_data_with_confidence) # Mitre pre attack file url if (self.mitre_pre_attack_file_url is not None and len(self.mitre_pre_attack_file_url) > 0): pre_attack_data = self.retrieve_data( self.mitre_pre_attack_file_url) pre_attack_data_with_confidence = ( self.add_confidence_to_bundle_objects(pre_attack_data)) self.send_bundle(work_id, pre_attack_data_with_confidence) # Mitre mobile attack file url if (self.mitre_mobile_attack_file_url is not None and len(self.mitre_mobile_attack_file_url) > 0): mobile_attack_data = self.retrieve_data( self.mitre_mobile_attack_file_url) mobile_attack_data_with_confidence = ( self.add_confidence_to_bundle_objects( mobile_attack_data)) self.send_bundle(work_id, mobile_attack_data_with_confidence) # Mitre ics attack file url if (self.mitre_ics_attack_file_url is not None and len(self.mitre_ics_attack_file_url) > 0): ics_attack_data = self.retrieve_data( self.mitre_ics_attack_file_url) ics_attack_data_with_confidence = ( self.add_confidence_to_bundle_objects(ics_attack_data)) self.send_bundle(work_id, ics_attack_data_with_confidence) # Store the current timestamp as a last run message = "Connector successfully run, storing last_run as " + str( timestamp) self.helper.log_info(message) self.helper.set_state({"last_run": timestamp}) self.helper.api.work.to_processed(work_id, message) self.helper.log_info( "Last_run stored, next run in: " + str(round(self.get_interval() / 60 / 60 / 24, 2)) + " days") else: new_interval = self.get_interval() - (timestamp - last_run) self.helper.log_info("Connector will not run, next run in: " + str(round(new_interval / 60 / 60 / 24, 2)) + " days") except (KeyboardInterrupt, SystemExit): self.helper.log_info("Connector stop") sys.exit(0) except Exception as e: self.helper.log_error(str(e)) def run(self): self.helper.log_info("Fetching MITRE datasets...") get_run_and_terminate = getattr(self.helper, "get_run_and_terminate", None) if callable( get_run_and_terminate) and self.helper.get_run_and_terminate(): self.process_data() self.helper.force_ping() else: while True: self.process_data() time.sleep(60) def send_bundle(self, work_id: str, serialized_bundle: str) -> None: try: self.helper.send_stix2_bundle( serialized_bundle, entities_types=self.helper.connect_scope, update=self.update_existing_data, work_id=work_id, ) except Exception as e: self.helper.log_error(f"Error while sending bundle: {e}")
class Cve: def __init__(self): # Instantiate the connector helper from config config_file_path = os.path.dirname( os.path.abspath(__file__)) + "/config.yml" config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader) if os.path.isfile(config_file_path) else {}) self.helper = OpenCTIConnectorHelper(config) # Extra config self.cve_import_history = get_config_variable( "CVE_IMPORT_HISTORY", ["cve", "import_history"], config, False) self.cve_nvd_data_feed = get_config_variable("CVE_NVD_DATA_FEED", ["cve", "nvd_data_feed"], config) self.cve_history_data_feed = get_config_variable( "CVE_HISTORY_DATA_FEED", ["cve", "history_data_feed"], config) self.cve_interval = get_config_variable("CVE_INTERVAL", ["cve", "interval"], config, True) self.update_existing_data = get_config_variable( "CONNECTOR_UPDATE_EXISTING_DATA", ["connector", "update_existing_data"], config, ) def get_interval(self): return int(self.cve_interval) * 60 * 60 * 24 def delete_files(self): if os.path.exists("data.json"): os.remove("data.json") if os.path.exists("data.json.gz"): os.remove("data.json.gz") if os.path.exists("data-stix2.json"): os.remove("data-stix2.json") def convert_and_send(self, url, work_id): try: # Downloading json.gz file self.helper.log_info("Requesting the file " + url) response = urllib.request.urlopen( url, context=ssl.create_default_context(cafile=certifi.where())) image = response.read() with open( os.path.dirname(os.path.abspath(__file__)) + "/data.json.gz", "wb") as file: file.write(image) # Unzipping the file self.helper.log_info("Unzipping the file") with gzip.open( os.path.dirname(os.path.abspath(__file__)) + "/data.json.gz", "rb") as f_in: with open("data.json", "wb") as f_out: shutil.copyfileobj(f_in, f_out) # Converting the file to stix2 self.helper.log_info("Converting the file") convert("data.json", "data-stix2.json") with open("data-stix2.json") as stix_json: contents = stix_json.read() self.helper.send_stix2_bundle( contents, entities_types=self.helper.connect_scope, update=self.update_existing_data, work_id=work_id, ) # Remove files self.delete_files() except Exception as e: self.delete_files() self.helper.log_error(str(e)) time.sleep(60) def process_data(self): try: # Get the current timestamp and check timestamp = int(time.time()) current_state = self.helper.get_state() if current_state is not None and "last_run" in current_state: last_run = current_state["last_run"] self.helper.log_info("Connector last run: " + datetime.utcfromtimestamp(last_run). strftime("%Y-%m-%d %H:%M:%S")) else: last_run = None self.helper.log_info("Connector has never run") # If the last_run is more than interval-1 day if last_run is None or ((timestamp - last_run) > ( (int(self.cve_interval) - 1) * 60 * 60 * 24)): timestamp = int(time.time()) now = datetime.utcfromtimestamp(timestamp) friendly_name = "CVE run @ " + now.strftime( "%Y-%m-%d %H:%M:%S") work_id = self.helper.api.work.initiate_work( self.helper.connect_id, friendly_name) self.convert_and_send(self.cve_nvd_data_feed, work_id) # If import history and never run if last_run is None and self.cve_import_history: now = datetime.now() years = list(range(2002, now.year + 1)) for year in years: self.convert_and_send( f"{self.cve_history_data_feed}nvdcve-1.1-{year}.json.gz", work_id, ) # Store the current timestamp as a last run self.helper.log_info( "Connector successfully run, storing last_run as " + str(timestamp)) self.helper.set_state({"last_run": timestamp}) message = ("Last_run stored, next run in: " + str(round(self.get_interval() / 60 / 60 / 24, 2)) + " days") self.helper.api.work.to_processed(work_id, message) self.helper.log_info(message) else: new_interval = self.get_interval() - (timestamp - last_run) self.helper.log_info("Connector will not run, next run in: " + str(round(new_interval / 60 / 60 / 24, 2)) + " days") except (KeyboardInterrupt, SystemExit): self.helper.log_info("Connector stop") exit(0) except Exception as e: self.helper.log_error(str(e)) def run(self): self.helper.log_info("Fetching CVE knowledge...") if self.helper.get_run_and_terminate(): self.process_data() self.helper.force_ping() else: while True: self.process_data() time.sleep(60)
class OpenCTI: def __init__(self): # Instantiate the connector helper from config config_file_path = os.path.dirname( os.path.abspath(__file__)) + "/config.yml" config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader) if os.path.isfile(config_file_path) else {}) self.helper = OpenCTIConnectorHelper(config) # Extra config self.opencti_sectors_file_url = get_config_variable( "CONFIG_SECTORS_FILE_URL", ["config", "sectors_file_url"], config) self.opencti_geography_file_url = get_config_variable( "CONFIG_GEOGRAPHY_FILE_URL", ["config", "geography_file_url"], config) self.opencti_interval = get_config_variable("CONFIG_INTERVAL", ["config", "interval"], config, True) self.update_existing_data = get_config_variable( "CONNECTOR_UPDATE_EXISTING_DATA", ["connector", "update_existing_data"], config, ) def get_interval(self): return int(self.opencti_interval) * 60 * 60 * 24 def process_data(self): try: # Get the current timestamp and check timestamp = int(time.time()) current_state = self.helper.get_state() if current_state is not None and "last_run" in current_state: last_run = current_state["last_run"] self.helper.log_info("Connector last run: " + datetime.utcfromtimestamp(last_run). strftime("%Y-%m-%d %H:%M:%S")) else: last_run = None self.helper.log_info("Connector has never run") # If the last_run is more than interval-1 day if last_run is None or ((timestamp - last_run) > ( (int(self.opencti_interval) - 1) * 60 * 60 * 24)): now = datetime.utcfromtimestamp(timestamp) friendly_name = "OpenCTI datasets run @ " + now.strftime( "%Y-%m-%d %H:%M:%S") work_id = self.helper.api.work.initiate_work( self.helper.connect_id, friendly_name) try: sectors_data = urllib.request.urlopen( self.opencti_sectors_file_url).read() self.helper.send_stix2_bundle( sectors_data.decode("utf-8"), entities_types=self.helper.connect_scope, update=self.update_existing_data, work_id=work_id, ) except Exception as e: self.helper.log_error(str(e)) try: geography_data = urllib.request.urlopen( self.opencti_geography_file_url).read() self.helper.send_stix2_bundle( geography_data.decode("utf-8"), entities_types=self.helper.connect_scope, update=self.update_existing_data, work_id=work_id, ) except Exception as e: self.helper.log_error(str(e)) # Store the current timestamp as a last run message = "Connector successfully run, storing last_run as " + str( timestamp) self.helper.log_info(message) self.helper.set_state({"last_run": timestamp}) self.helper.api.work.to_processed(work_id, message) self.helper.log_info( "Last_run stored, next run in: " + str(round(self.get_interval() / 60 / 60 / 24, 2)) + " days") else: new_interval = self.get_interval() - (timestamp - last_run) self.helper.log_info("Connector will not run, next run in: " + str(round(new_interval / 60 / 60 / 24, 2)) + " days") except (KeyboardInterrupt, SystemExit): self.helper.log_info("Connector stop") exit(0) except Exception as e: self.helper.log_error(str(e)) def run(self): self.helper.log_info("Fetching OpenCTI datasets...") get_run_and_terminate = getattr(self.helper, "get_run_and_terminate", None) if callable( get_run_and_terminate) and self.helper.get_run_and_terminate(): self.process_data() self.helper.force_ping() else: while True: self.process_data() time.sleep(60)