class Cve: def __init__(self): # Instantiate the connector helper from config config_file_path = os.path.dirname( os.path.abspath(__file__)) + "/config.yml" config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader) if os.path.isfile(config_file_path) else {}) self.helper = OpenCTIConnectorHelper(config) # Extra config self.cve_import_history = get_config_variable( "CVE_IMPORT_HISTORY", ["cve", "import_history"], config, False) self.cve_nvd_data_feed = get_config_variable("CVE_NVD_DATA_FEED", ["cve", "nvd_data_feed"], config) self.cve_history_data_feed = get_config_variable( "CVE_HISTORY_DATA_FEED", ["cve", "history_data_feed"], config) self.cve_interval = get_config_variable("CVE_INTERVAL", ["cve", "interval"], config, True) self.update_existing_data = get_config_variable( "CONNECTOR_UPDATE_EXISTING_DATA", ["connector", "update_existing_data"], config, ) def get_interval(self): return int(self.cve_interval) * 60 * 60 * 24 def delete_files(self): if os.path.exists("data.json"): os.remove("data.json") if os.path.exists("data.json.gz"): os.remove("data.json.gz") if os.path.exists("data-stix2.json"): os.remove("data-stix2.json") def convert_and_send(self, url, work_id): try: # Downloading json.gz file self.helper.log_info("Requesting the file " + url) response = urllib.request.urlopen( url, context=ssl.create_default_context(cafile=certifi.where())) image = response.read() with open( os.path.dirname(os.path.abspath(__file__)) + "/data.json.gz", "wb") as file: file.write(image) # Unzipping the file self.helper.log_info("Unzipping the file") with gzip.open( os.path.dirname(os.path.abspath(__file__)) + "/data.json.gz", "rb") as f_in: with open("data.json", "wb") as f_out: shutil.copyfileobj(f_in, f_out) # Converting the file to stix2 self.helper.log_info("Converting the file") convert("data.json", "data-stix2.json") with open("data-stix2.json") as stix_json: contents = stix_json.read() self.helper.send_stix2_bundle( contents, entities_types=self.helper.connect_scope, update=self.update_existing_data, work_id=work_id, ) # Remove files self.delete_files() except Exception as e: self.delete_files() self.helper.log_error(str(e)) time.sleep(60) def process_data(self): try: # Get the current timestamp and check timestamp = int(time.time()) current_state = self.helper.get_state() if current_state is not None and "last_run" in current_state: last_run = current_state["last_run"] self.helper.log_info("Connector last run: " + datetime.utcfromtimestamp(last_run). strftime("%Y-%m-%d %H:%M:%S")) else: last_run = None self.helper.log_info("Connector has never run") # If the last_run is more than interval-1 day if last_run is None or ((timestamp - last_run) > ( (int(self.cve_interval) - 1) * 60 * 60 * 24)): timestamp = int(time.time()) now = datetime.utcfromtimestamp(timestamp) friendly_name = "CVE run @ " + now.strftime( "%Y-%m-%d %H:%M:%S") work_id = self.helper.api.work.initiate_work( self.helper.connect_id, friendly_name) self.convert_and_send(self.cve_nvd_data_feed, work_id) # If import history and never run if last_run is None and self.cve_import_history: now = datetime.now() years = list(range(2002, now.year + 1)) for year in years: self.convert_and_send( f"{self.cve_history_data_feed}nvdcve-1.1-{year}.json.gz", work_id, ) # Store the current timestamp as a last run self.helper.log_info( "Connector successfully run, storing last_run as " + str(timestamp)) self.helper.set_state({"last_run": timestamp}) message = ("Last_run stored, next run in: " + str(round(self.get_interval() / 60 / 60 / 24, 2)) + " days") self.helper.api.work.to_processed(work_id, message) self.helper.log_info(message) else: new_interval = self.get_interval() - (timestamp - last_run) self.helper.log_info("Connector will not run, next run in: " + str(round(new_interval / 60 / 60 / 24, 2)) + " days") except (KeyboardInterrupt, SystemExit): self.helper.log_info("Connector stop") exit(0) except Exception as e: self.helper.log_error(str(e)) def run(self): self.helper.log_info("Fetching CVE knowledge...") if self.helper.get_run_and_terminate(): self.process_data() self.helper.force_ping() else: while True: self.process_data() time.sleep(60)
class OpenCTI: def __init__(self): # Instantiate the connector helper from config config_file_path = os.path.dirname( os.path.abspath(__file__)) + "/config.yml" config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader) if os.path.isfile(config_file_path) else {}) self.helper = OpenCTIConnectorHelper(config) # Extra config self.opencti_sectors_file_url = get_config_variable( "CONFIG_SECTORS_FILE_URL", ["config", "sectors_file_url"], config) self.opencti_geography_file_url = get_config_variable( "CONFIG_GEOGRAPHY_FILE_URL", ["config", "geography_file_url"], config) self.opencti_interval = get_config_variable("CONFIG_INTERVAL", ["config", "interval"], config, True) self.update_existing_data = get_config_variable( "CONNECTOR_UPDATE_EXISTING_DATA", ["connector", "update_existing_data"], config, ) def get_interval(self): return int(self.opencti_interval) * 60 * 60 * 24 def process_data(self): try: # Get the current timestamp and check timestamp = int(time.time()) current_state = self.helper.get_state() if current_state is not None and "last_run" in current_state: last_run = current_state["last_run"] self.helper.log_info("Connector last run: " + datetime.utcfromtimestamp(last_run). strftime("%Y-%m-%d %H:%M:%S")) else: last_run = None self.helper.log_info("Connector has never run") # If the last_run is more than interval-1 day if last_run is None or ((timestamp - last_run) > ( (int(self.opencti_interval) - 1) * 60 * 60 * 24)): now = datetime.utcfromtimestamp(timestamp) friendly_name = "OpenCTI datasets run @ " + now.strftime( "%Y-%m-%d %H:%M:%S") work_id = self.helper.api.work.initiate_work( self.helper.connect_id, friendly_name) try: sectors_data = urllib.request.urlopen( self.opencti_sectors_file_url).read() self.helper.send_stix2_bundle( sectors_data.decode("utf-8"), entities_types=self.helper.connect_scope, update=self.update_existing_data, work_id=work_id, ) except Exception as e: self.helper.log_error(str(e)) try: geography_data = urllib.request.urlopen( self.opencti_geography_file_url).read() self.helper.send_stix2_bundle( geography_data.decode("utf-8"), entities_types=self.helper.connect_scope, update=self.update_existing_data, work_id=work_id, ) except Exception as e: self.helper.log_error(str(e)) # Store the current timestamp as a last run message = "Connector successfully run, storing last_run as " + str( timestamp) self.helper.log_info(message) self.helper.set_state({"last_run": timestamp}) self.helper.api.work.to_processed(work_id, message) self.helper.log_info( "Last_run stored, next run in: " + str(round(self.get_interval() / 60 / 60 / 24, 2)) + " days") else: new_interval = self.get_interval() - (timestamp - last_run) self.helper.log_info("Connector will not run, next run in: " + str(round(new_interval / 60 / 60 / 24, 2)) + " days") except (KeyboardInterrupt, SystemExit): self.helper.log_info("Connector stop") exit(0) except Exception as e: self.helper.log_error(str(e)) def run(self): self.helper.log_info("Fetching OpenCTI datasets...") get_run_and_terminate = getattr(self.helper, "get_run_and_terminate", None) if callable( get_run_and_terminate) and self.helper.get_run_and_terminate(): self.process_data() self.helper.force_ping() else: while True: self.process_data() time.sleep(60)
class Mitre: """Mitre connector.""" def __init__(self): # Instantiate the connector helper from config config_file_path = os.path.dirname( os.path.abspath(__file__)) + "/config.yml" config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader) if os.path.isfile(config_file_path) else {}) self.helper = OpenCTIConnectorHelper(config) # Extra config self.mitre_enterprise_file_url = get_config_variable( "MITRE_ENTERPRISE_FILE_URL", ["mitre", "enterprise_file_url"], config) self.mitre_pre_attack_file_url = get_config_variable( "MITRE_PRE_ATTACK_FILE_URL", ["mitre", "pre_attack_file_url"], config) self.mitre_mobile_attack_file_url = get_config_variable( "MITRE_MOBILE_ATTACK_FILE_URL", ["mitre", "mobile_attack_file_url"], config) self.mitre_ics_attack_file_url = get_config_variable( "MITRE_ICS_ATTACK_FILE_URL", ["mitre", "ics_attack_file_url"], config) self.mitre_interval = get_config_variable("MITRE_INTERVAL", ["mitre", "interval"], config, True) self.update_existing_data = get_config_variable( "CONNECTOR_UPDATE_EXISTING_DATA", ["connector", "update_existing_data"], config, ) self.confidence_level = get_config_variable( "CONNECTOR_CONFIDENCE_LEVEL", ["connector", "confidence_level"], config, ) def get_interval(self): return int(self.mitre_interval) * 60 * 60 * 24 def retrieve_data(self, url: str) -> Optional[str]: """ Retrieve data from the given url. Parameters ---------- url : str Url to retrieve. Returns ------- str A string with the content or None in case of failure. """ try: return (urllib.request.urlopen( url, context=ssl.create_default_context(cafile=certifi.where()), ).read().decode("utf-8")) except ( urllib.error.URLError, urllib.error.HTTPError, urllib.error.ContentTooShortError, ) as urllib_error: self.helper.log_error( f"Error retrieving url {url}: {urllib_error}") return None # Add confidence to every object in a bundle def add_confidence_to_bundle_objects(self, serialized_bundle: str) -> str: # the list of object types for which the confidence has to be added # (skip marking-definition, identity, external-reference-as-report) object_types_with_confidence = [ "attack-pattern", "course-of-action", "intrusion-set", "campaign", "malware", "tool", "report", "relationship", ] stix_bundle = json.loads(serialized_bundle) for obj in stix_bundle["objects"]: object_type = obj["type"] if object_type in object_types_with_confidence: # self.helper.log_info(f"Adding confidence to {object_type} object") obj["confidence"] = int(self.confidence_level) return json.dumps(stix_bundle) def process_data(self): try: # Get the current timestamp and check timestamp = int(time.time()) current_state = self.helper.get_state() if current_state is not None and "last_run" in current_state: last_run = current_state["last_run"] self.helper.log_info("Connector last run: " + datetime.utcfromtimestamp(last_run). strftime("%Y-%m-%d %H:%M:%S")) else: last_run = None self.helper.log_info("Connector has never run") # If the last_run is more than interval-1 day if last_run is None or ((timestamp - last_run) > ( (int(self.mitre_interval) - 1) * 60 * 60 * 24)): self.helper.log_info("Connector will run!") now = datetime.utcfromtimestamp(timestamp) friendly_name = "MITRE run @ " + now.strftime( "%Y-%m-%d %H:%M:%S") work_id = self.helper.api.work.initiate_work( self.helper.connect_id, friendly_name) # Mitre enterprise file url if (self.mitre_enterprise_file_url is not None and len(self.mitre_enterprise_file_url) > 0): enterprise_data = self.retrieve_data( self.mitre_enterprise_file_url) enterprise_data_with_confidence = ( self.add_confidence_to_bundle_objects(enterprise_data)) self.send_bundle(work_id, enterprise_data_with_confidence) # Mitre pre attack file url if (self.mitre_pre_attack_file_url is not None and len(self.mitre_pre_attack_file_url) > 0): pre_attack_data = self.retrieve_data( self.mitre_pre_attack_file_url) pre_attack_data_with_confidence = ( self.add_confidence_to_bundle_objects(pre_attack_data)) self.send_bundle(work_id, pre_attack_data_with_confidence) # Mitre mobile attack file url if (self.mitre_mobile_attack_file_url is not None and len(self.mitre_mobile_attack_file_url) > 0): mobile_attack_data = self.retrieve_data( self.mitre_mobile_attack_file_url) mobile_attack_data_with_confidence = ( self.add_confidence_to_bundle_objects( mobile_attack_data)) self.send_bundle(work_id, mobile_attack_data_with_confidence) # Mitre ics attack file url if (self.mitre_ics_attack_file_url is not None and len(self.mitre_ics_attack_file_url) > 0): ics_attack_data = self.retrieve_data( self.mitre_ics_attack_file_url) ics_attack_data_with_confidence = ( self.add_confidence_to_bundle_objects(ics_attack_data)) self.send_bundle(work_id, ics_attack_data_with_confidence) # Store the current timestamp as a last run message = "Connector successfully run, storing last_run as " + str( timestamp) self.helper.log_info(message) self.helper.set_state({"last_run": timestamp}) self.helper.api.work.to_processed(work_id, message) self.helper.log_info( "Last_run stored, next run in: " + str(round(self.get_interval() / 60 / 60 / 24, 2)) + " days") else: new_interval = self.get_interval() - (timestamp - last_run) self.helper.log_info("Connector will not run, next run in: " + str(round(new_interval / 60 / 60 / 24, 2)) + " days") except (KeyboardInterrupt, SystemExit): self.helper.log_info("Connector stop") sys.exit(0) except Exception as e: self.helper.log_error(str(e)) def run(self): self.helper.log_info("Fetching MITRE datasets...") get_run_and_terminate = getattr(self.helper, "get_run_and_terminate", None) if callable( get_run_and_terminate) and self.helper.get_run_and_terminate(): self.process_data() self.helper.force_ping() else: while True: self.process_data() time.sleep(60) def send_bundle(self, work_id: str, serialized_bundle: str) -> None: try: self.helper.send_stix2_bundle( serialized_bundle, entities_types=self.helper.connect_scope, update=self.update_existing_data, work_id=work_id, ) except Exception as e: self.helper.log_error(f"Error while sending bundle: {e}")
class LastInfoSec: def __init__(self): config_file_path = os.path.dirname( os.path.abspath(__file__)) + "/config.yml" config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader) if os.path.isfile(config_file_path) else {}) self.helper = OpenCTIConnectorHelper(config) self.lastinfosec_cti_url = "https://api.client.lastinfosec.com/v2/stix21/getbyminutes/{}?api_key={}&headers=false&platform=opencti" self.lastinfosec_cve_url = "https://api.client.lastinfosec.com/v2/stix21/vulnerabilities/getlasthour?api_key={}&headers=false&platform=opencti" self.lastinfosec_tactic_url = "https://api.client.lastinfosec.com/v2/stix21/tactic/getlast24hour?api_key={}&headers=false&platform=opencti" self.lastinfosec_apikey = get_config_variable( "CONFIG_LIS_APIKEY", ["lastinfosec", "api_key"], config) self.lastinfosec_cti_enabled = get_config_variable( "CONFIG_LIS_CTI_ENABLED", ["lastinfosec", "cti", "is_enabled"], config) self.lastinfosec_cti_interval = get_config_variable( "CONFIG_LIS_CTI_INTERVAL", ["lastinfosec", "cti", "interval"], config) self.lastinfosec_cve_enabled = get_config_variable( "CONFIG_LIS_CVE_ENABLED", ["lastinfosec", "cve", "is_enabled"], config) self.lastinfosec_tactic_enabled = get_config_variable( "CONFIG_LIS_TACTIC_ENABLED", ["lastinfosec", "tactic", "is_enabled"], config) self.opencti_url = get_config_variable("OPENCTI_URL", ["opencti", "url"], config) self.opencti_id = get_config_variable("OPENCTI_TOKEN", ["opencti", "token"], config) self.update_existing_data = get_config_variable( "OPENCTI_UPDATE_EXISTING_DATA", ["connector", "update_existing_data"], config) self.proxy_http = get_config_variable("PROXY_HTTP", ["opencti", "proxy_http"], config) self.proxy_https = get_config_variable("PROXY_HTTPS", ["opencti", "proxy_https"], config) total_enabled = 0 if self.lastinfosec_cti_enabled: total_enabled += 1 if self.lastinfosec_cve_enabled: total_enabled += 1 if self.lastinfosec_tactic_enabled: total_enabled += 1 if total_enabled == 0: raise Exception("You must enable one feed") elif total_enabled > 1: raise Exception("You can enable only one feed per connector") self.api = OpenCTIApiClient(self.opencti_url, self.opencti_id) def run(self): self.helper.log_info("Fetching lastinfosec datasets...") if not self.helper.get_run_and_terminate(): while True: time_to_sleep = self.process_data() time.sleep(time_to_sleep) else: self.process_data() def process_data(self): time_to_sleep = 0 try: if (self.lastinfosec_cti_enabled and self.lastinfosec_cti_url is not None and self.lastinfosec_apikey is not None): url = self.lastinfosec_cti_url.format( self.lastinfosec_cti_interval, self.lastinfosec_apikey) run_interval = self.lastinfosec_cti_interval * 60 time_to_sleep = self.fetch_data(url, run_interval) elif (self.lastinfosec_cve_enabled and self.lastinfosec_cve_url is not None and self.lastinfosec_apikey is not None): url = self.lastinfosec_cve_url.format(self.lastinfosec_apikey) run_interval = 3600 # 1h in second time_to_sleep = self.fetch_data(url, run_interval) elif (self.lastinfosec_tactic_enabled and self.lastinfosec_tactic_url is not None and self.lastinfosec_apikey is not None): url = self.lastinfosec_tactic_url.format( self.lastinfosec_apikey) run_interval = 86400 # 24h in second time_to_sleep = self.fetch_data(url, run_interval) else: self.helper.log_info("CTI Feed not configured") time.sleep(60) exit(0) except (KeyboardInterrupt, SystemExit): self.helper.log_info("Connector stop") exit(0) except Exception as e: self.helper.log_error("run:" + str(e)) time.sleep(60) return time_to_sleep def fetch_data(self, url: str, run_interval: int): # Get the current timestamp and check start = time.perf_counter() time_to_sleep = 0 timestamp = int(time.time()) now = datetime.datetime.utcfromtimestamp(timestamp) proxy_dic = {} if self.proxy_http is not None: proxy_dic["http"] = self.proxy_http if self.proxy_https is not None: proxy_dic["https"] = self.proxy_https req = requests.get(url, proxies=proxy_dic) if req.status_code == 200: lastinfosec_data = req.json() if isinstance(lastinfosec_data, list) and len(lastinfosec_data) > 0: friendly_name = "LastInfoSec CTI run @ " + now.strftime( "%Y-%m-%d %H:%M:%S") work_id = self.helper.api.work.initiate_work( self.helper.connect_id, friendly_name) self.push_data(lastinfosec_data, timestamp, work_id) stop = time.perf_counter() process_time_seconds = stop - start time_to_sleep = run_interval - process_time_seconds else: message = "Connector error run, storing last_run as {0}".format( timestamp) self.helper.set_state({"last_run": timestamp}) self.helper.log_info(message) time.sleep(150) return time_to_sleep def push_data(self, bundles, timestamp, work_id): for bundle in bundles: sdata = json.dumps(bundle) self.helper.send_stix2_bundle(sdata, work_id=work_id) # Store the current timestamp as a last run message = "Connector successfully run, storing last_run as {0}".format( timestamp) self.helper.set_state({"last_run": timestamp}) self.helper.api.work.to_processed(work_id, message) self.helper.log_info(message)