class Mitre:
"""Mitre connector."""
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
# Extra config
self.mitre_enterprise_file_url = get_config_variable(
"MITRE_ENTERPRISE_FILE_URL", ["mitre", "enterprise_file_url"],
config)
self.mitre_pre_attack_file_url = get_config_variable(
"MITRE_PRE_ATTACK_FILE_URL", ["mitre", "pre_attack_file_url"],
config)
self.mitre_mobile_attack_file_url = get_config_variable(
"MITRE_MOBILE_ATTACK_FILE_URL",
["mitre", "mobile_attack_file_url"], config)
self.mitre_ics_attack_file_url = get_config_variable(
"MITRE_ICS_ATTACK_FILE_URL", ["mitre", "ics_attack_file_url"],
config)
self.mitre_interval = get_config_variable("MITRE_INTERVAL",
["mitre", "interval"],
config, True)
self.update_existing_data = get_config_variable(
"CONNECTOR_UPDATE_EXISTING_DATA",
["connector", "update_existing_data"],
config,
)
self.confidence_level = get_config_variable(
"CONNECTOR_CONFIDENCE_LEVEL",
["connector", "confidence_level"],
config,
)
def get_interval(self):
return int(self.mitre_interval) * 60 * 60 * 24
def retrieve_data(self, url: str) -> Optional[str]:
"""
Retrieve data from the given url.
Parameters
----------
url : str
Url to retrieve.
Returns
-------
str
A string with the content or None in case of failure.
"""
try:
return (urllib.request.urlopen(
url,
context=ssl.create_default_context(cafile=certifi.where()),
).read().decode("utf-8"))
except (
urllib.error.URLError,
urllib.error.HTTPError,
urllib.error.ContentTooShortError,
) as urllib_error:
self.helper.log_error(
f"Error retrieving url {url}: {urllib_error}")
return None
# Add confidence to every object in a bundle
def add_confidence_to_bundle_objects(self, serialized_bundle: str) -> str:
# the list of object types for which the confidence has to be added
# (skip marking-definition, identity, external-reference-as-report)
object_types_with_confidence = [
"attack-pattern",
"course-of-action",
"intrusion-set",
"campaign",
"malware",
"tool",
"report",
"relationship",
]
stix_bundle = json.loads(serialized_bundle)
for obj in stix_bundle["objects"]:
object_type = obj["type"]
if object_type in object_types_with_confidence:
# self.helper.log_info(f"Adding confidence to {object_type} object")
obj["confidence"] = int(self.confidence_level)
return json.dumps(stix_bundle)
def process_data(self):
try:
# Get the current timestamp and check
timestamp = int(time.time())
current_state = self.helper.get_state()
if current_state is not None and "last_run" in current_state:
last_run = current_state["last_run"]
self.helper.log_info("Connector last run: " +
datetime.utcfromtimestamp(last_run).
strftime("%Y-%m-%d %H:%M:%S"))
else:
last_run = None
self.helper.log_info("Connector has never run")
# If the last_run is more than interval-1 day
if last_run is None or ((timestamp - last_run) > (
(int(self.mitre_interval) - 1) * 60 * 60 * 24)):
self.helper.log_info("Connector will run!")
now = datetime.utcfromtimestamp(timestamp)
friendly_name = "MITRE run @ " + now.strftime(
"%Y-%m-%d %H:%M:%S")
work_id = self.helper.api.work.initiate_work(
self.helper.connect_id, friendly_name)
# Mitre enterprise file url
if (self.mitre_enterprise_file_url is not None
and len(self.mitre_enterprise_file_url) > 0):
enterprise_data = self.retrieve_data(
self.mitre_enterprise_file_url)
enterprise_data_with_confidence = (
self.add_confidence_to_bundle_objects(enterprise_data))
self.send_bundle(work_id, enterprise_data_with_confidence)
# Mitre pre attack file url
if (self.mitre_pre_attack_file_url is not None
and len(self.mitre_pre_attack_file_url) > 0):
pre_attack_data = self.retrieve_data(
self.mitre_pre_attack_file_url)
pre_attack_data_with_confidence = (
self.add_confidence_to_bundle_objects(pre_attack_data))
self.send_bundle(work_id, pre_attack_data_with_confidence)
# Mitre mobile attack file url
if (self.mitre_mobile_attack_file_url is not None
and len(self.mitre_mobile_attack_file_url) > 0):
mobile_attack_data = self.retrieve_data(
self.mitre_mobile_attack_file_url)
mobile_attack_data_with_confidence = (
self.add_confidence_to_bundle_objects(
mobile_attack_data))
self.send_bundle(work_id,
mobile_attack_data_with_confidence)
# Mitre ics attack file url
if (self.mitre_ics_attack_file_url is not None
and len(self.mitre_ics_attack_file_url) > 0):
ics_attack_data = self.retrieve_data(
self.mitre_ics_attack_file_url)
ics_attack_data_with_confidence = (
self.add_confidence_to_bundle_objects(ics_attack_data))
self.send_bundle(work_id, ics_attack_data_with_confidence)
# Store the current timestamp as a last run
message = "Connector successfully run, storing last_run as " + str(
timestamp)
self.helper.log_info(message)
self.helper.set_state({"last_run": timestamp})
self.helper.api.work.to_processed(work_id, message)
self.helper.log_info(
"Last_run stored, next run in: " +
str(round(self.get_interval() / 60 / 60 / 24, 2)) +
" days")
else:
new_interval = self.get_interval() - (timestamp - last_run)
self.helper.log_info("Connector will not run, next run in: " +
str(round(new_interval / 60 / 60 /
24, 2)) + " days")
except (KeyboardInterrupt, SystemExit):
self.helper.log_info("Connector stop")
sys.exit(0)
except Exception as e:
self.helper.log_error(str(e))
def run(self):
self.helper.log_info("Fetching MITRE datasets...")
get_run_and_terminate = getattr(self.helper, "get_run_and_terminate",
None)
if callable(
get_run_and_terminate) and self.helper.get_run_and_terminate():
self.process_data()
self.helper.force_ping()
else:
while True:
self.process_data()
time.sleep(60)
def send_bundle(self, work_id: str, serialized_bundle: str) -> None:
try:
self.helper.send_stix2_bundle(
serialized_bundle,
entities_types=self.helper.connect_scope,
update=self.update_existing_data,
work_id=work_id,
)
except Exception as e:
self.helper.log_error(f"Error while sending bundle: {e}")
class Cve:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
# Extra config
self.cve_import_history = get_config_variable(
"CVE_IMPORT_HISTORY", ["cve", "import_history"], config, False)
self.cve_nvd_data_feed = get_config_variable("CVE_NVD_DATA_FEED",
["cve", "nvd_data_feed"],
config)
self.cve_history_data_feed = get_config_variable(
"CVE_HISTORY_DATA_FEED", ["cve", "history_data_feed"], config)
self.cve_interval = get_config_variable("CVE_INTERVAL",
["cve", "interval"], config,
True)
self.update_existing_data = get_config_variable(
"CONNECTOR_UPDATE_EXISTING_DATA",
["connector", "update_existing_data"],
config,
)
def get_interval(self):
return int(self.cve_interval) * 60 * 60 * 24
def delete_files(self):
if os.path.exists("data.json"):
os.remove("data.json")
if os.path.exists("data.json.gz"):
os.remove("data.json.gz")
if os.path.exists("data-stix2.json"):
os.remove("data-stix2.json")
def convert_and_send(self, url, work_id):
try:
# Downloading json.gz file
self.helper.log_info("Requesting the file " + url)
response = urllib.request.urlopen(
url,
context=ssl.create_default_context(cafile=certifi.where()))
image = response.read()
with open(
os.path.dirname(os.path.abspath(__file__)) +
"/data.json.gz", "wb") as file:
file.write(image)
# Unzipping the file
self.helper.log_info("Unzipping the file")
with gzip.open(
os.path.dirname(os.path.abspath(__file__)) +
"/data.json.gz", "rb") as f_in:
with open("data.json", "wb") as f_out:
shutil.copyfileobj(f_in, f_out)
# Converting the file to stix2
self.helper.log_info("Converting the file")
convert("data.json", "data-stix2.json")
with open("data-stix2.json") as stix_json:
contents = stix_json.read()
self.helper.send_stix2_bundle(
contents,
entities_types=self.helper.connect_scope,
update=self.update_existing_data,
work_id=work_id,
)
# Remove files
self.delete_files()
except Exception as e:
self.delete_files()
self.helper.log_error(str(e))
time.sleep(60)
def process_data(self):
try:
# Get the current timestamp and check
timestamp = int(time.time())
current_state = self.helper.get_state()
if current_state is not None and "last_run" in current_state:
last_run = current_state["last_run"]
self.helper.log_info("Connector last run: " +
datetime.utcfromtimestamp(last_run).
strftime("%Y-%m-%d %H:%M:%S"))
else:
last_run = None
self.helper.log_info("Connector has never run")
# If the last_run is more than interval-1 day
if last_run is None or ((timestamp - last_run) > (
(int(self.cve_interval) - 1) * 60 * 60 * 24)):
timestamp = int(time.time())
now = datetime.utcfromtimestamp(timestamp)
friendly_name = "CVE run @ " + now.strftime(
"%Y-%m-%d %H:%M:%S")
work_id = self.helper.api.work.initiate_work(
self.helper.connect_id, friendly_name)
self.convert_and_send(self.cve_nvd_data_feed, work_id)
# If import history and never run
if last_run is None and self.cve_import_history:
now = datetime.now()
years = list(range(2002, now.year + 1))
for year in years:
self.convert_and_send(
f"{self.cve_history_data_feed}nvdcve-1.1-{year}.json.gz",
work_id,
)
# Store the current timestamp as a last run
self.helper.log_info(
"Connector successfully run, storing last_run as " +
str(timestamp))
self.helper.set_state({"last_run": timestamp})
message = ("Last_run stored, next run in: " +
str(round(self.get_interval() / 60 / 60 / 24, 2)) +
" days")
self.helper.api.work.to_processed(work_id, message)
self.helper.log_info(message)
else:
new_interval = self.get_interval() - (timestamp - last_run)
self.helper.log_info("Connector will not run, next run in: " +
str(round(new_interval / 60 / 60 /
24, 2)) + " days")
except (KeyboardInterrupt, SystemExit):
self.helper.log_info("Connector stop")
exit(0)
except Exception as e:
self.helper.log_error(str(e))
def run(self):
self.helper.log_info("Fetching CVE knowledge...")
if self.helper.get_run_and_terminate():
self.process_data()
self.helper.force_ping()
else:
while True:
self.process_data()
time.sleep(60)
class OpenCTI:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
# Extra config
self.opencti_sectors_file_url = get_config_variable(
"CONFIG_SECTORS_FILE_URL", ["config", "sectors_file_url"], config)
self.opencti_geography_file_url = get_config_variable(
"CONFIG_GEOGRAPHY_FILE_URL", ["config", "geography_file_url"],
config)
self.opencti_interval = get_config_variable("CONFIG_INTERVAL",
["config", "interval"],
config, True)
self.update_existing_data = get_config_variable(
"CONNECTOR_UPDATE_EXISTING_DATA",
["connector", "update_existing_data"],
config,
)
def get_interval(self):
return int(self.opencti_interval) * 60 * 60 * 24
def process_data(self):
try:
# Get the current timestamp and check
timestamp = int(time.time())
current_state = self.helper.get_state()
if current_state is not None and "last_run" in current_state:
last_run = current_state["last_run"]
self.helper.log_info("Connector last run: " +
datetime.utcfromtimestamp(last_run).
strftime("%Y-%m-%d %H:%M:%S"))
else:
last_run = None
self.helper.log_info("Connector has never run")
# If the last_run is more than interval-1 day
if last_run is None or ((timestamp - last_run) > (
(int(self.opencti_interval) - 1) * 60 * 60 * 24)):
now = datetime.utcfromtimestamp(timestamp)
friendly_name = "OpenCTI datasets run @ " + now.strftime(
"%Y-%m-%d %H:%M:%S")
work_id = self.helper.api.work.initiate_work(
self.helper.connect_id, friendly_name)
try:
sectors_data = urllib.request.urlopen(
self.opencti_sectors_file_url).read()
self.helper.send_stix2_bundle(
sectors_data.decode("utf-8"),
entities_types=self.helper.connect_scope,
update=self.update_existing_data,
work_id=work_id,
)
except Exception as e:
self.helper.log_error(str(e))
try:
geography_data = urllib.request.urlopen(
self.opencti_geography_file_url).read()
self.helper.send_stix2_bundle(
geography_data.decode("utf-8"),
entities_types=self.helper.connect_scope,
update=self.update_existing_data,
work_id=work_id,
)
except Exception as e:
self.helper.log_error(str(e))
# Store the current timestamp as a last run
message = "Connector successfully run, storing last_run as " + str(
timestamp)
self.helper.log_info(message)
self.helper.set_state({"last_run": timestamp})
self.helper.api.work.to_processed(work_id, message)
self.helper.log_info(
"Last_run stored, next run in: " +
str(round(self.get_interval() / 60 / 60 / 24, 2)) +
" days")
else:
new_interval = self.get_interval() - (timestamp - last_run)
self.helper.log_info("Connector will not run, next run in: " +
str(round(new_interval / 60 / 60 /
24, 2)) + " days")
except (KeyboardInterrupt, SystemExit):
self.helper.log_info("Connector stop")
exit(0)
except Exception as e:
self.helper.log_error(str(e))
def run(self):
self.helper.log_info("Fetching OpenCTI datasets...")
get_run_and_terminate = getattr(self.helper, "get_run_and_terminate",
None)
if callable(
get_run_and_terminate) and self.helper.get_run_and_terminate():
self.process_data()
self.helper.force_ping()
else:
while True:
self.process_data()
time.sleep(60)
