class HistoryConnector:
def __init__(self):
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.SafeLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
self.logger_config = self.helper.api.get_logs_worker_config()
self.elasticsearch = Elasticsearch(
[self.logger_config["elasticsearch_url"]])
self.elasticsearch_index = self.logger_config["elasticsearch_index"]
def _process_message(self, msg):
try:
event_json = json.loads(msg.data)
unix_time = round(int(msg.id.split("-")[0]) / 1000)
event_date = datetime.datetime.fromtimestamp(
unix_time, datetime.timezone.utc)
timestamp = event_date.isoformat().replace("+00:00", "Z")
origin = event_json["origin"]
history_data = {
"internal_id":
msg.id,
"event_type":
msg.event,
"timestamp":
timestamp,
"entity_type":
"history",
"user_id":
origin["user_id"],
"applicant_id":
origin["applicant_id"] if "applicant_id" in origin else None,
"context_data": {
"id":
event_json["data"]["x_opencti_internal_id"]
if "x_opencti_internal_id" in event_json["data"] else
event_json["data"]["x_opencti_id"],
"entity_type":
event_json["data"]["type"],
"from_id":
event_json["data"]["x_opencti_source_ref"]
if "x_opencti_source_ref" in event_json["data"] else None,
"to_id":
event_json["data"]["x_opencti_target_ref"]
if "x_opencti_target_ref" in event_json["data"] else None,
"message":
event_json["message"],
},
}
self.elasticsearch.index(index=self.elasticsearch_index,
id=msg.id,
body=history_data)
except elasticsearch.RequestError as err:
print("Unexpected error:", err, msg)
pass
def start(self):
self.helper.listen_stream(self._process_message)
class SynchronizerConnector:
def __init__(self):
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
# Extra config
self.remote_opencti_url = get_config_variable(
"REMOTE_OPENCTI_URL", ["remote_opencti", "url"], config)
self.remote_opencti_ssl_verify = get_config_variable(
"REMOTE_OPENCTI_SSL_VERIFY", ["remote_opencti", "ssl_verify"],
config)
self.remote_opencti_token = get_config_variable(
"REMOTE_OPENCTI_TOKEN", ["remote_opencti", "token"], config)
self.remote_opencti_events = get_config_variable(
"REMOTE_OPENCTI_EVENTS", ["remote_opencti", "events"],
config).split(",")
self.remote_opencti_start_timestamp = get_config_variable(
"REMOTE_OPENCTI_START_TIMESTAMP",
["remote_opencti", "start_timestamp"],
config,
)
def _process_message(self, msg):
data = json.loads(msg.data)
try:
if "create" in self.remote_opencti_events and msg.event == "create":
bundle = json.dumps({"objects": [data["data"]]})
self.helper.send_stix2_bundle(bundle,
event_version=data["version"])
elif "update" in self.remote_opencti_events and msg.event == "update":
bundle = json.dumps({"objects": [data["data"]]})
self.helper.send_stix2_bundle(bundle,
event_version=data["version"])
elif "delete" in self.remote_opencti_events and msg.event == "delete":
if data["data"]["type"] == "relationship":
self.helper.api.stix_core_relationship.delete(
id=data["data"]["id"])
elif StixCyberObservableTypes.has_value(data["data"]["type"]):
self.helper.api.stix_cyber_observable.delete(
id=data["data"]["id"])
else:
self.helper.api.stix_domain_object.delete(
id=data["data"]["id"])
except:
pass
def start(self):
self.helper.listen_stream(
self._process_message,
self.remote_opencti_url,
self.remote_opencti_token,
self.remote_opencti_ssl_verify,
self.remote_opencti_start_timestamp,
)
class ThreatBusConnector(object):
def __init__(self):
config_file_path = os.path.dirname(os.path.abspath(__file__)) + "/config.yml"
config = (
yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path)
else {}
)
# Connector configuration
self.entity_name = get_config_variable(
"CONNECTOR_ENTITY_NAME", ["connector", "entity_name"], config
)
self.entity_desc = get_config_variable(
"CONNECTOR_ENTITY_DESCRIPTION", ["connector", "entity_description"], config
)
self.forward_all_iocs = get_config_variable(
"CONNECTOR_FORWARD_ALL_IOCS", ["connector", "forward_all_iocs"], config
)
self.threatbus_entity = None
# Custom configuration for Threat Bus ZeroMQ-App plugin endpoint
self.threatbus_zmq_host = get_config_variable(
"THREATBUS_ZMQ_HOST", ["threatbus", "zmq_host"], config
)
self.threatbus_zmq_port = get_config_variable(
"THREATBUS_ZMQ_PORT", ["threatbus", "zmq_port"], config
)
# Helper initialization
self.opencti_helper = OpenCTIConnectorHelper(config)
zmq_endpoint = f"{self.threatbus_zmq_host}:{self.threatbus_zmq_port}"
self.threatbus_helper = ThreatBusConnectorHelper(
zmq_endpoint,
self._report_sighting,
self.opencti_helper.log_info,
self.opencti_helper.log_error,
subscribe_topic="stix2/sighting",
publish_topic="stix2/indicator",
)
def _get_threatbus_entity(self) -> int:
"""
Get the Threat Bus OpenCTI entity. Creates a new entity if it does not
exist yet.
"""
# Use cached:
if self.threatbus_entity is not None:
return self.threatbus_entity
# Try and fetch existing:
threatbus_entity = (
self.opencti_helper.api.stix_domain_object.get_by_stix_id_or_name(
name=self.entity_name
)
)
if threatbus_entity is not None and threatbus_entity.get("id", None):
self.threatbus_entity = threatbus_entity
return self.threatbus_entity
# Create a new one:
self.opencti_helper.log_info(
f"Creating new OpenCTI Threat Bus entity '{self.entity_name}'"
)
self.threatbus_entity = self.opencti_helper.api.identity.create(
type="Organization",
name=self.entity_name,
description=self.entity_desc,
)
return self.threatbus_entity
def _report_sighting(self, msg: str):
"""
Converts a JSON string to a STIX-2 Sighting and reports it to OpenCTI.
@param msg The JSON string
"""
try:
sighting: Sighting = parse(msg, allow_custom=True)
except Exception as e:
self.opencti_helper.log_error(
f"Error parsing message from Threat Bus. Expected a STIX-2 Sighting: {e}"
)
return
if type(sighting) is not Sighting:
self.opencti_helper.log_error(
f"Error parsing message from Threat Bus. Expected a STIX-2 Sighting: {sighting}"
)
return
entity_id = self._get_threatbus_entity().get("id", None)
resp = self.opencti_helper.api.stix_sighting_relationship.create(
fromId=sighting.sighting_of_ref,
toId=entity_id,
createdBy=entity_id,
first_seen=sighting.first_seen.astimezone().strftime("%Y-%m-%dT%H:%M:%SZ")
if sighting.get("first_seen")
else None,
last_seen=sighting.last_seen.astimezone().strftime("%Y-%m-%dT%H:%M:%SZ")
if sighting.get("last_seen")
else None,
confidence=50,
externalReferences=[sighting.sighting_of_ref],
count=1,
)
self.opencti_helper.log_info(f"Created sighting {resp}")
def _map_to_threatbus(
self, data: dict, opencti_action: str
) -> Union[Indicator, None]:
"""
Inspects the given OpenCTI data point and either returns a valid STIX-2
Indicator or None.
@param data A dict object with OpenCTI SSE data
@param opencti_action A string indicating what happened to this item
(either `create`, `update` or `delete`)
@return a STIX-2 Indicator or None
"""
opencti_id: str = data.get("x_opencti_id", None)
if not opencti_id:
self.opencti_helper.log_error(
"Cannot process data without 'x_opencti_id' field"
)
return
indicator: dict = self.opencti_helper.api.indicator.read(id=opencti_id)
if not indicator:
# we are only interested in indicators at this time
return
detection_enabled: bool = indicator.get("x_opencti_detection", False)
if not detection_enabled and self.forward_all_iocs is not True:
# only propagate indicators that are toggled for detection or the
# user enabled forwarding of all indicators regardless of the toggle
return
# overwrite custom OpenCTI ID
indicator["id"] = indicator.get("standard_id")
if opencti_action == "update":
indicator[
ThreatBusSTIX2Constants.X_THREATBUS_UPDATE.value
] = Operation.EDIT.value
if opencti_action == "delete":
indicator[
ThreatBusSTIX2Constants.X_THREATBUS_UPDATE.value
] = Operation.REMOVE.value
return Indicator(**indicator, allow_custom=True)
def _process_message(self, sse_msg: Event):
"""
Invoked for every incoming SSE message from the OpenCTI endpoint
@param sse_msg: the received SSE Event
"""
try:
data: dict = json.loads(sse_msg.data).get("data", None)
if not data:
return
indicator = self._map_to_threatbus(data, sse_msg.event)
if not indicator:
return
self.threatbus_helper.send(indicator.serialize())
except Exception as e:
self.opencti_helper.log_error(
f"Error forwarding indicator to Threat Bus: {e}"
)
def start(self):
self.opencti_helper.log_info("Starting Threat Bus connector")
# Fork a new Thread to communicate with Threat Bus
self.threatbus_helper.start()
atexit.register(self.threatbus_helper.stop)
# Send the main loop into a busy loop for processing OpenCTI events
self.opencti_helper.listen_stream(self._process_message)
class ElasticConnector:
def __init__(self, config: dict = {}, datadir: str = None):
self.shutdown_event: threading.Event = threading.Event()
self.helper = OpenCTIConnectorHelper(config)
logger.info("Connected to OpenCTI")
if (self.helper.connect_live_stream_id is None
or self.helper.connect_live_stream_id == "ChangeMe"):
raise ValueError("Missing Live Stream ID")
self.config = Cut(config)
# Start streaming from 1 second ago
self.helper.set_state({
"connectorLastEventId":
str(int(round(time.time() * 1000)) - 1000)
})
# Get the external URL as configured in OpenCTI Settings page
query = """
query SettingsQuery {
settings {
id
platform_url
}
}
"""
_settings = self.helper.api.query(query)["data"]["settings"]
self.config["opencti.platform_url"] = _settings.get(
"platform_url", None)
self._connect_elasticsearch()
if self.config["connector.mode"] == "ecs":
self.import_manager = IntelManager(self.helper, self.elasticsearch,
self.config, datadir)
self.sightings_manager = SignalsManager(
config=self.config,
shutdown_event=self.shutdown_event,
opencti_client=self.helper,
elasticsearch_client=self.elasticsearch,
)
elif self.config["connector.mode"] == "stix":
self.import_manager = StixManager(self.helper, self.elasticsearch,
self.config, datadir)
self.sightings_manager = None
else:
logger.error(
f"connector.mode: {self.config['connector.mode']} is unsupported. Should be 'ecs' or 'stix'"
)
def _connect_elasticsearch(self) -> None:
_apikey: tuple(str) = None
_httpauth: tuple(str) = None
if self.config.get("cloud.auth", None):
_httpauth = tuple(self.config.get("cloud.auth").split(":"))
elif self.config.get("output.elasticsearch.username",
None) and self.config.get(
"output.elasticsearch.password", None):
_httpauth = (
self.config.get("output.elasticsearch.username"),
self.config.get("output.elasticsearch.password"),
)
if self.config.get("output.elasticsearch.api_key", None):
_apikey = tuple(
self.config.get("output.elasticsearch.api_key").split(":"))
if _httpauth is not None and _apikey is not None:
logger.critical(
"Either username/password auth or api_key auth should be used for Elasticsearch, not both."
)
sys.exit(1)
if self.config.get("cloud.id", None):
logger.debug(
f"Connecting to Elasticsearch using cloud.id {self.config.get('cloud.id')}"
)
self.elasticsearch = Elasticsearch(
cloud_id=self.config.get("cloud.id"),
verify_certs=self.config.get("output.elasticsearch.ssl_verify",
True),
http_auth=_httpauth,
api_key=_apikey,
)
else:
logger.debug(
f"Connecting to Elasticsearch using hosts: {self.config.get('output.elasticsearch.hosts', ['localhost:9200'])}"
)
self.elasticsearch = Elasticsearch(
hosts=self.config.get("output.elasticsearch.hosts",
["localhost:9200"]),
verify_certs=self.config.get("output.elasticsearch.ssl_verify",
True),
http_auth=_httpauth,
api_key=_apikey,
)
logger.info("Connected to Elasticsearch")
return
def handle_create(self, timestamp: datetime, data: dict) -> None:
logger.debug("[CREATE] Processing indicator {" + data["id"] + "}")
self.import_manager.import_cti_event(timestamp, data)
return
def handle_update(self, timestamp, data):
logger.debug("[UPDATE] Processing indicator {" + data["id"] + "}")
self.import_manager.import_cti_event(timestamp, data, is_update=True)
return
def handle_delete(self, timestamp, data):
logger.debug("[DELETE] Processing indicator {" + data["id"] + "}")
self.import_manager.delete_cti_event(data)
return
def _process_message(self, msg) -> None:
logger.debug("_process_message")
try:
event_id = msg.id
timestamp = datetime.fromtimestamp(round(
int(event_id.split("-")[0]) / 1000),
tz=timezone.utc)
data = json.loads(msg.data)["data"]
except ValueError:
logger.error(f"Unable to process the message: {msg}")
raise ValueError("Cannot process the message: " + msg)
logger.debug(f"[PROCESS] Message (id: {event_id}, date: {timestamp})")
if msg.event == "create":
return self.handle_create(timestamp, data)
if msg.event == "update":
return self.handle_update(timestamp, data)
if msg.event == "delete":
return self.handle_delete(timestamp, data)
def start(self) -> None:
self.shutdown_event.clear()
if self.config["connector.mode"] == "ecs":
self.sightings_manager.start()
# Look out, this doesn't block
self.helper.listen_stream(self._process_message)
try:
# Just wait here until someone presses ctrl+c
self.shutdown_event.wait()
except KeyboardInterrupt:
self.shutdown_event.set()
logger.info("Shutting down")
if self.config["connector.mode"] == "ecs":
self.sightings_manager.join(timeout=3)
if self.sightings_manager.is_alive():
logger.warn("Sightings manager didn't shutdown by request")
self.elasticsearch.close()
logger.info(
"Main thread complete. Waiting on background threads to complete. Press CTRL+C to quit."
)
class TaniumConnector:
def __init__(self):
config_file_path = os.path.dirname(os.path.abspath(__file__)) + "/config.yml"
config = (
yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path)
else {}
)
self.helper = OpenCTIConnectorHelper(config)
# Extra config
self.tanium_url = get_config_variable("TANIUM_URL", ["tanium", "url"], config)
self.tanium_ssl_verify = get_config_variable(
"TANIUM_SSL_VERIFY", ["tanium", "ssl_verify"], config, False, True
)
self.tanium_login = get_config_variable(
"TANIUM_LOGIN", ["tanium", "login"], config
)
self.tanium_password = get_config_variable(
"TANIUM_PASSWORD", ["tanium", "password"], config
)
self.tanium_indicator_types = get_config_variable(
"TANIUM_INDICATOR_TYPES", ["tanium", "indicator_types"], config
).split(",")
self.tanium_observable_types = get_config_variable(
"TANIUM_OBSERVABLE_TYPES", ["tanium", "observable_types"], config
).split(",")
self.tanium_import_label = get_config_variable(
"TANIUM_IMPORT_LABEL", ["tanium", "import_label"], config, False, ""
)
self.tanium_import_from_date = get_config_variable(
"TANIUM_IMPORT_FROM_DATE", ["tanium", "import_from_date"], config
)
self.tanium_reputation_blacklist_label = get_config_variable(
"TANIUM_REPUTATION_BLACKLIST_LABEL",
["tanium", "reputation_blacklist_label"],
config,
False,
"",
)
self.tanium_auto_quickscan = get_config_variable(
"TANIUM_AUTO_QUICKSCAN", ["tanium", "auto_quickscan"], config, False, False
)
self.tanium_computer_groups = get_config_variable(
"TANIUM_COMPUTER_GROUPS", ["tanium", "computer_groups"], config, False, ""
).split(",")
# Variables
self.session = None
# Open a session
self._get_session()
# Create the state
if self.tanium_import_from_date:
timestamp = (
parse(self.tanium_import_from_date).timestamp() * 1000
if self.tanium_import_from_date != "now"
else int(round(time.time() * 1000)) - 1000
)
current_state = self.helper.get_state()
if current_state is None:
self.helper.set_state({"connectorLastEventId": timestamp})
# Create the source if not exist
self.source_id = None
sources = self._query("get", "/plugin/products/detect3/api/v1/sources")
for source in sources:
if source["name"] == "OpenCTI":
self.source_id = source["id"]
if self.source_id is None:
source = self._query(
"post",
"/plugin/products/detect3/api/v1/sources",
{
"type": "api-client",
"name": "OpenCTI",
"description": "Cyber Threat Intelligence knowledge imported from OpenCTI.",
"canAutoQuickScan": True,
},
)
self.source_id = source["id"]
def _get_session(self):
payload = {
"username": self.tanium_login,
"password": self.tanium_password,
}
r = requests.post(
self.tanium_url + "/api/v2/session/login",
json=payload,
verify=self.tanium_ssl_verify,
)
if r.status_code == 200:
result = r.json()
self.session = result["data"]["session"]
else:
raise ValueError("Cannot login to the Tanium API")
def _query(
self,
method,
uri,
payload=None,
content_type="application/json",
type=None,
retry=False,
):
self.helper.log_info("Query " + method + " on " + uri)
headers = {"session": self.session}
if method != "upload":
headers["content-type"] = content_type
if type is not None:
headers["type"] = type
if content_type == "application/octet-stream":
headers["content-disposition"] = (
"attachment; filename=" + payload["filename"]
)
if "name" in payload:
headers["name"] = payload["name"]
if "description" in payload:
headers["description"] = payload["description"]
if method == "get":
r = requests.get(
self.tanium_url + uri,
headers=headers,
params=payload,
verify=self.tanium_ssl_verify,
)
elif method == "post":
if content_type == "application/octet-stream":
r = requests.post(
self.tanium_url + uri,
headers=headers,
data=payload["document"],
verify=self.tanium_ssl_verify,
)
elif type is not None:
r = requests.post(
self.tanium_url + uri,
headers=headers,
data=payload["intelDoc"],
verify=self.tanium_ssl_verify,
)
else:
r = requests.post(
self.tanium_url + uri,
headers=headers,
json=payload,
verify=self.tanium_ssl_verify,
)
elif method == "upload":
f = open(payload["filename"], "w")
f.write(payload["content"])
f.close()
files = {"hash": open(payload["filename"], "rb")}
r = requests.post(
self.tanium_url + uri,
headers=headers,
files=files,
verify=self.tanium_ssl_verify,
)
elif method == "put":
if content_type == "application/xml":
r = requests.put(
self.tanium_url + uri,
headers=headers,
data=payload,
verify=self.tanium_ssl_verify,
)
else:
r = requests.put(
self.tanium_url + uri,
headers=headers,
json=payload,
verify=self.tanium_ssl_verify,
)
elif method == "patch":
r = requests.patch(
self.tanium_url + uri,
headers=headers,
json=payload,
verify=self.tanium_ssl_verify,
)
elif method == "delete":
r = requests.delete(
self.tanium_url + uri, headers=headers, verify=self.tanium_ssl_verify
)
else:
raise ValueError("Unspported method")
if r.status_code == 200:
try:
return r.json()
except:
return r.text
elif r.status_code == 401 and not retry:
self._get_session()
return self._query(method, uri, payload, content_type, type, True)
elif r.status_code == 401:
raise ValueError("Query failed, permission denied")
else:
self.helper.log_info(r.text)
def _get_labels(self, labels):
# List labels
tanium_labels = self._query(
"get", "/plugin/products/detect3/api/v1/labels", {"limit": 500}
)
tanium_labels_dict = {}
for tanium_label in tanium_labels:
tanium_labels_dict[tanium_label["name"].lower()] = tanium_label
final_labels = []
for label in labels:
# Label already exists
if label["value"] in tanium_labels_dict:
final_labels.append(tanium_labels_dict[label["value"]])
# Create the label
else:
created_label = self._query(
"post",
"/plugin/products/detect3/api/v1/labels",
{
"name": label["value"],
"description": "Label imported from OpenCTI",
},
)
final_labels.append(created_label)
return final_labels
def _get_by_id(self, internal_id, yara=False):
if yara:
response = self._query(
"get",
"/plugin/products/detect3/api/v1/intels",
{"name": internal_id + ".yara"},
)
else:
response = self._query(
"get",
"/plugin/products/detect3/api/v1/intels",
{"description": internal_id},
)
if response and len(response) > 0:
return response[0]
else:
return None
def _get_reputation_by_hash(self, hash):
response = self._query(
"get",
"/plugin/products/reputation/v3/reputations/custom",
{"search": hash},
)
if response["data"] and len(response["data"]) > 0:
return response["data"][0]
else:
return None
def _create_indicator_stix(self, entity, original_intel_document=None):
if original_intel_document is None:
intel_document = self._get_by_id(entity["id"])
if intel_document is not None:
return intel_document
stix2_bundle = self.helper.api.stix2.export_entity(
entity["entity_type"],
entity["id"],
"simple",
None,
True,
True,
)
initialize_options()
stix_indicator = slide_string(stix2_bundle)
stix_indicator = re.sub(
r"<indicator:Description>(.*?)<\/indicator:Description>",
r"<indicator:Description>" + entity["id"] + "</indicator:Description>",
stix_indicator,
)
stix_indicator = re.sub(
r"<indicator:Description ordinality=\"1\">(.*?)<\/indicator:Description>",
r'<indicator:Description ordinality="1">'
+ entity["id"]
+ "</indicator:Description>",
stix_indicator,
)
payload = {"intelDoc": stix_indicator}
if original_intel_document is not None:
intel_document = self._query(
"put",
"/plugin/products/detect3/api/v1/intels/"
+ str(original_intel_document["id"]),
stix_indicator,
"application/xml",
"stix",
)
else:
intel_document = self._query(
"post",
"/plugin/products/detect3/api/v1/sources/"
+ str(self.source_id)
+ "/intels",
payload,
"application/xml",
"stix",
)
return intel_document
def _create_indicator_yara(self, entity, original_intel_document=None):
if original_intel_document is None:
intel_document = self._get_by_id(entity["id"], True)
if intel_document is not None:
return intel_document
filename = entity["id"] + ".yara"
if original_intel_document is not None:
intel_document = self._query(
"put",
"/plugin/products/detect3/api/v1/intels/"
+ str(original_intel_document["id"]),
{
"filename": filename,
"document": entity["pattern"],
"name": entity["name"],
"description": entity["id"],
},
"application/octet-stream",
"yara",
)
else:
intel_document = self._query(
"post",
"/plugin/products/detect3/api/v1/sources/"
+ str(self.source_id)
+ "/intels",
{
"filename": filename,
"document": entity["pattern"],
"name": entity["name"],
"description": entity["id"],
},
"application/octet-stream",
"yara",
)
return intel_document
def _create_tanium_signal(self, entity, original_intel_document=None):
if original_intel_document is None:
intel_document = self._get_by_id(entity["id"])
if intel_document is not None:
return intel_document
platforms = []
if "x_mitre_platforms" in entity and len(entity["x_mitre_platforms"]) > 0:
for x_mitre_platform in entity["x_mitre_platforms"]:
if x_mitre_platform in ["Linux", "Windows", "macOS"]:
platforms.append(
x_mitre_platform.lower()
if x_mitre_platform != "macOS"
else "mac"
)
if original_intel_document is not None:
intel_document = self._query(
"put",
"/plugin/products/detect3/api/v1/intels/"
+ str(original_intel_document["id"]),
{
"name": entity["name"],
"description": entity["id"],
"platforms": platforms,
"contents": entity["pattern"],
},
)
else:
intel_document = self._query(
"post",
"/plugin/products/detect3/api/v1/sources/"
+ str(self.source_id)
+ "/intels",
{
"name": entity["name"],
"description": entity["id"],
"platforms": platforms,
"contents": entity["pattern"],
},
)
return intel_document
def _create_observable(self, entity, original_intel_document=None):
if original_intel_document is None:
intel_document = self._get_by_id(entity["id"])
if intel_document is not None:
return intel_document
intel_type = None
value = None
name = None
if entity["entity_type"] == "StixFile":
intel_type = "file_hash"
if "hashes" in entity:
for hash in entity["hashes"]:
value = (
value + hash["hash"] + "\n"
if value is not None
else hash["hash"] + "\n"
)
name = hash["hash"]
elif entity["entity_type"] in [
"IPv4-Addr",
"IPv6-Addr",
"Domain-Name",
"X-OpenCTI-Hostname",
]:
intel_type = "ip_or_host"
value = entity["value"]
name = entity["value"]
if intel_type is None or value is None:
return None
openioc = self._query(
"post",
"/plugin/products/detect3/api/v1/intels/quick-add",
{
"exact": True,
"name": name,
"description": entity["id"],
"type": intel_type,
"text": value,
},
)
openioc = re.sub(
r"<description>(.*?)<\/description>",
r"<description>" + entity["id"] + "</description>",
openioc,
)
payload = {"intelDoc": openioc}
if original_intel_document is not None:
intel_document = self._query(
"put",
"/plugin/products/detect3/api/v1/intels/"
+ str(original_intel_document["id"]),
payload,
"application/xml",
"openioc",
)
else:
intel_document = self._query(
"post",
"/plugin/products/detect3/api/v1/sources/"
+ str(self.source_id)
+ "/intels",
payload,
"application/xml",
"openioc",
)
return intel_document
def _post_operations(self, entity, intel_document):
if intel_document is not None and entity is not None:
if self.tanium_auto_quickscan:
for computer_group in self.tanium_computer_groups:
self._query(
"post",
"/plugin/products/detect3/api/v1/quick-scans",
{
"computerGroupId": int(computer_group),
"intelDocId": intel_document["id"],
},
)
external_reference = self.helper.api.external_reference.create(
source_name="Tanium",
url=self.tanium_url
+ "/#/thr_workbench/intel/"
+ str(intel_document["id"]),
external_id=str(intel_document["id"]),
description="Intel document within the Tanium platform.",
)
if entity["entity_type"] == "Indicator":
self.helper.api.stix_domain_object.add_external_reference(
id=entity["id"], external_reference_id=external_reference["id"]
)
else:
self.helper.api.stix_cyber_observable.add_external_reference(
id=entity["id"], external_reference_id=external_reference["id"]
)
if len(entity["objectLabel"]) > 0:
labels = self._get_labels(entity["objectLabel"])
for label in labels:
if label is not None:
self._query(
"put",
"/plugin/products/detect3/api/v1/intels/"
+ str(intel_document["id"])
+ "/labels",
{"id": label["id"]},
)
def _process_intel(self, entity_type, data, original_intel_document=None):
entity = None
intel_document = None
if entity_type == "indicator":
entity = self.helper.api.indicator.read(id=data["data"]["x_opencti_id"])
if (
entity is None
or entity["revoked"]
or entity["pattern_type"] not in self.tanium_indicator_types
):
return {"entity": entity, "intel_document": intel_document}
if entity["pattern_type"] == "stix":
intel_document = self._create_indicator_stix(
entity, original_intel_document
)
elif entity["pattern_type"] == "yara":
intel_document = self._create_indicator_yara(
entity, original_intel_document
)
elif entity["pattern_type"] == "tanium-signal":
intel_document = self._create_tanium_signal(
entity, original_intel_document
)
elif (
StixCyberObservableTypes.has_value(entity_type)
and entity_type.lower() in self.tanium_observable_types
):
entity = self.helper.api.stix_cyber_observable.read(
id=data["data"]["x_opencti_id"]
)
if entity is None:
return {"entity": entity, "intel_document": intel_document}
intel_document = self._create_observable(entity, original_intel_document)
return {"entity": entity, "intel_document": intel_document}
def _process_message(self, msg):
data = json.loads(msg.data)
entity_type = data["data"]["type"]
# If not an indicator, not an observable to import and
if (
entity_type != "indicator"
and entity_type not in self.tanium_observable_types
and (
"labels" in data["data"]
and self.tanium_reputation_blacklist_label not in data["data"]["labels"]
)
and self.tanium_reputation_blacklist_label != "*"
):
self.helper.log_info(
"Not an indicator and not an observable to import, doing nothing"
)
return
# Handle creation
if msg.event == "create":
# No label
if (
"labels" not in data["data"]
and self.tanium_import_label != "*"
and self.tanium_reputation_blacklist_label != "*"
):
self.helper.log_info("No label marked as import, doing nothing")
return
# Import or blacklist labels are not in the given labels
elif (
(
"labels" in data["data"]
and self.tanium_import_label not in data["data"]["labels"]
)
and self.tanium_import_label != "*"
and self.tanium_reputation_blacklist_label not in data["data"]["labels"]
and self.tanium_reputation_blacklist_label != "*"
):
self.helper.log_info(
"No label marked as import or no global label, doing nothing"
)
return
# Revoked is true
elif "revoked" in data["data"] and data["data"]["revoked"]:
return
if (
"labels" in data["data"]
and self.tanium_import_label in data["data"]["labels"]
) or self.tanium_import_label == "*":
# Process intel
processed_intel = self._process_intel(entity_type, data)
intel_document = processed_intel["intel_document"]
entity = processed_intel["entity"]
# Create external reference and add object labels
self._post_operations(entity, intel_document)
if (
"labels" in data["data"]
and self.tanium_reputation_blacklist_label in data["data"]["labels"]
) or self.tanium_reputation_blacklist_label == "*":
if "hashes" in data["data"]:
entry = {"list": "blacklist"}
if "MD5" in data["data"]["hashes"]:
entry["md5"] = data["data"]["hashes"]["MD5"]
entry["uploadedHash"] = data["data"]["hashes"]["MD5"]
else:
entry["md5"] = ""
if "SHA-1" in data["data"]["hashes"]:
entry["sha1"] = data["data"]["hashes"]["SHA-1"]
entry["uploadedHash"] = data["data"]["hashes"]["SHA-1"]
else:
entry["sha1"] = ""
if "SHA-256" in data["data"]["hashes"]:
entry["sha256"] = data["data"]["hashes"]["SHA-256"]
entry["uploadedHash"] = data["data"]["hashes"]["SHA-256"]
else:
entry["sha256"] = ""
entry["notes"] = ",".join(data["data"]["labels"])
self._query(
"post",
"/plugin/products/reputation/v3/reputations/custom/upload?append=true",
[entry],
)
elif msg.event == "update":
if (
"x_data_update" in data["data"]
and "add" in data["data"]["x_data_update"]
and "labels" in data["data"]["x_data_update"]["add"]
):
if self.tanium_reputation_blacklist_label in data["data"][
"x_data_update"
]["add"]["labels"] and StixCyberObservableTypes.has_value(
data["data"]["type"]
):
observable = self.helper.api.stix_cyber_observable.read(
id=data["data"]["id"]
)
observable = self.helper.api.stix2.generate_export(observable)
if "hashes" in observable:
entry = {"list": "blacklist"}
if "MD5" in observable["hashes"]:
entry["md5"] = observable["hashes"]["MD5"]
entry["uploadedHash"] = observable["hashes"]["MD5"]
else:
entry["md5"] = ""
if "SHA-1" in observable["hashes"]:
entry["sha1"] = observable["hashes"]["SHA-1"]
entry["uploadedHash"] = observable["hashes"]["SHA-1"]
else:
entry["sha1"] = ""
if "SHA-256" in observable["hashes"]:
entry["sha256"] = observable["hashes"]["SHA-256"]
entry["uploadedHash"] = observable["hashes"]["SHA-256"]
else:
entry["sha256"] = ""
entry["notes"] = ",".join(observable["labels"])
self._query(
"post",
"/plugin/products/reputation/v3/reputations/custom/upload?append=true",
[entry],
)
if (
self.tanium_import_label
in data["data"]["x_data_update"]["add"]["labels"]
):
# Process intel
processed_intel = self._process_intel(entity_type, data)
intel_document = processed_intel["intel_document"]
entity = processed_intel["entity"]
# Create external reference and add object labels
self._post_operations(entity, intel_document)
else:
entity = self.helper.api.indicator.read(
id=data["data"]["x_opencti_id"],
customAttributes="""
pattern_type
""",
)
intel_document = self._get_by_id(
data["data"]["x_opencti_id"],
yara=True
if entity is not None and entity["pattern_type"] == "yara"
else False,
)
if intel_document:
new_labels = []
for label in data["data"]["x_data_update"]["add"]["labels"]:
new_labels.append({"value": label})
labels = self._get_labels(new_labels)
for label in labels:
self._query(
"put",
"/plugin/products/detect3/api/v1/intels/"
+ str(intel_document["id"])
+ "/labels",
{"id": label["id"]},
)
elif (
"x_data_update" in data["data"]
and "remove" in data["data"]["x_data_update"]
and "labels" in data["data"]["x_data_update"]["remove"]
):
if (
self.tanium_reputation_blacklist_label
in data["data"]["x_data_update"]["remove"]["labels"]
):
if "hashes" in data["data"]:
if "SHA-256" in data["data"]["hashes"]:
self._query(
"post",
"/plugin/products/reputation/v3/reputations/custom/delete",
[data["data"]["hashes"]["SHA-256"]],
)
if "SHA-1" in data["data"]["hashes"]:
self._query(
"post",
"/plugin/products/reputation/v3/reputations/custom/delete",
[data["data"]["hashes"]["SHA-1"]],
)
if "MD5" in data["data"]["hashes"]:
self._query(
"post",
"/plugin/products/reputation/v3/reputations/custom/delete",
[data["data"]["hashes"]["MD5"]],
)
if (
self.tanium_import_label
in data["data"]["x_data_update"]["remove"]["labels"]
):
# Import label has been removed
intel_document = self._get_by_id(data["data"]["x_opencti_id"])
if intel_document is not None:
self._query(
"delete",
"/plugin/products/detect3/api/v1/intels/"
+ str(intel_document["id"]),
)
# Remove external references
if entity_type == "indicator":
entity = self.helper.api.indicator.read(
id=data["data"]["x_opencti_id"]
)
else:
entity = self.helper.api.stix_cyber_observable.read(
id=data["data"]["x_opencti_id"]
)
if (
entity
and "externalReferences" in entity
and len(entity["externalReferences"]) > 0
):
for external_reference in entity["externalReferences"]:
if external_reference["source_name"] == "Tanium":
self.helper.api.external_reference.delete(
external_reference["id"]
)
else:
intel_document = self._get_by_id(data["data"]["x_opencti_id"])
if intel_document:
new_labels = []
for label in data["data"]["x_data_update"]["remove"]["labels"]:
new_labels.append({"value": label})
labels = self._get_labels(new_labels)
for label in labels:
self._query(
"delete",
"/plugin/products/detect3/api/v1/intels/"
+ str(intel_document["id"])
+ "/labels/"
+ str(label["id"]),
)
elif (
"x_data_update" in data["data"]
and "replace" in data["data"]["x_data_update"]
):
if entity_type == "indicator":
if "pattern" in data["data"]["x_data_update"]["replace"]:
intel_document = self._get_by_id(data["data"]["x_opencti_id"])
if intel_document is not None:
self._process_intel(entity_type, data, intel_document)
elif (
"value" in data["data"]["x_data_update"]["replace"]
or "hashes" in data["data"]["x_data_update"]["replace"]
):
intel_document = self._get_by_id(data["data"]["x_opencti_id"])
if intel_document is not None:
self._process_intel(entity_type, data, intel_document)
elif (
"revoked" in data["data"]["x_data_update"]["replace"]
and data["data"]["x_data_update"]["replace"]["revoked"] == True
):
intel_document = self._get_by_id(data["data"]["x_opencti_id"])
if intel_document is not None:
self._query(
"delete",
"/plugin/products/detect3/api/v1/intels/"
+ str(intel_document["id"]),
)
# Remove external references
if entity_type == "indicator":
entity = self.helper.api.indicator.read(
id=data["data"]["x_opencti_id"]
)
else:
entity = self.helper.api.stix_cyber_observable.read(
id=data["data"]["x_opencti_id"]
)
if (
entity
and "externalReferences" in entity
and len(entity["externalReferences"]) > 0
):
for external_reference in entity["externalReferences"]:
if external_reference["source_name"] == "Tanium":
self.helper.api.external_reference.delete(
external_reference["id"]
)
elif msg.event == "delete":
intel_document = self._get_by_id(data["data"]["x_opencti_id"])
if intel_document is not None:
self._query(
"delete",
"/plugin/products/detect3/api/v1/intels/"
+ str(intel_document["id"]),
)
if data["data"]["type"] == "file":
if "hashes" in data["data"]:
if "SHA-256" in data["data"]["hashes"]:
self._query(
"post",
"/plugin/products/reputation/v3/reputations/custom/delete",
[data["data"]["hashes"]["SHA-256"]],
)
if "SHA-1" in data["data"]["hashes"]:
self._query(
"post",
"/plugin/products/reputation/v3/reputations/custom/delete",
[data["data"]["hashes"]["SHA-1"]],
)
if "MD5" in data["data"]["hashes"]:
self._query(
"post",
"/plugin/products/reputation/v3/reputations/custom/delete",
[data["data"]["hashes"]["MD5"]],
)
def start(self):
self.alerts_gatherer = TaniumConnectorAlertsGatherer(
self.helper,
self.tanium_url,
self.tanium_login,
self.tanium_password,
self.tanium_ssl_verify,
)
self.alerts_gatherer.start()
self.helper.listen_stream(self._process_message)
class HistoryConnector:
def __init__(self):
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
self.logger_config = self.helper.api.get_logs_worker_config()
if (self.logger_config["elasticsearch_username"] is not None
and self.logger_config["elasticsearch_password"] is not None):
self.elasticsearch = Elasticsearch(
self.logger_config["elasticsearch_url"],
verify_certs=self.
logger_config["elasticsearch_ssl_reject_unauthorized"],
http_auth=(
self.logger_config["elasticsearch_username"],
self.logger_config["elasticsearch_password"],
),
)
elif self.logger_config["elasticsearch_api_key"] is not None:
self.elasticsearch = Elasticsearch(
self.logger_config["elasticsearch_url"],
verify_certs=self.
logger_config["elasticsearch_ssl_reject_unauthorized"],
api_key=self.logger_config["elasticsearch_api_key"],
)
else:
self.elasticsearch = Elasticsearch(
self.logger_config["elasticsearch_url"],
verify_certs=self.
logger_config["elasticsearch_ssl_reject_unauthorized"],
)
self.elasticsearch_index = self.logger_config["elasticsearch_index"]
def _process_message(self, msg):
try:
event_json = json.loads(msg.data)
unix_time = round(int(msg.id.split("-")[0]) / 1000)
event_date = datetime.datetime.fromtimestamp(
unix_time, datetime.timezone.utc)
timestamp = event_date.isoformat().replace("+00:00", "Z")
origin = event_json["origin"] if "origin" in event_json else {}
history_data = {
"internal_id":
msg.id,
"event_type":
msg.event,
"timestamp":
timestamp,
"entity_type":
"history",
"user_id":
origin["user_id"] if "user_id" in origin else None,
"applicant_id":
origin["applicant_id"] if "applicant_id" in origin else None,
"context_data": {
"id":
event_json["data"]["x_opencti_internal_id"]
if "x_opencti_internal_id" in event_json["data"] else
event_json["data"]["x_opencti_id"],
"entity_type":
event_json["data"]["type"],
"from_id":
event_json["data"]["x_opencti_source_ref"]
if "x_opencti_source_ref" in event_json["data"] else None,
"to_id":
event_json["data"]["x_opencti_target_ref"]
if "x_opencti_target_ref" in event_json["data"] else None,
"message":
event_json["message"],
"commit":
event_json["commit"]["message"] if "commit" in event_json
and "message" in event_json["commit"] else None,
"references":
event_json["commit"]["references"]
if "commit" in event_json
and "references" in event_json["commit"] else None,
},
}
self.elasticsearch.index(index=self.elasticsearch_index,
id=msg.id,
body=history_data)
except Exception as err:
print("Unexpected error:", err, msg)
pass
def start(self):
self.helper.listen_stream(self._process_message)
class ThreatBusConnector(object):
def __init__(self):
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
# Connector configuration
self.entity_name = get_config_variable("CONNECTOR_ENTITY_NAME",
["connector", "entity_name"],
config)
self.entity_desc = get_config_variable(
"CONNECTOR_ENTITY_DESCRIPTION",
["connector", "entity_description"], config)
self.forward_all_iocs = get_config_variable(
"CONNECTOR_FORWARD_ALL_IOCS", ["connector", "forward_all_iocs"],
config)
self.threatbus_entity = None
# Custom configuration for Threat Bus & ZeroMQ plugin endpoint
self.threatbus_zmq_host = get_config_variable(
"THREATBUS_ZMQ_HOST", ["threatbus", "zmq_host"], config)
self.threatbus_zmq_port = get_config_variable(
"THREATBUS_ZMQ_PORT", ["threatbus", "zmq_port"], config)
threatbus_snapshot = get_config_variable(
"THREATBUS_SNAPSHOT",
["threatbus", "snapshot"],
config,
isNumber=True,
default=0,
)
# Helper initialization
self.opencti_helper = OpenCTIConnectorHelper(config)
zmq_endpoint = f"{self.threatbus_zmq_host}:{self.threatbus_zmq_port}"
self.threatbus_helper = ThreatBusConnectorHelper(
zmq_endpoint,
self._handle_threatbus_message,
self.opencti_helper.log_info,
self.opencti_helper.log_error,
subscribe_topics=["stix2/sighting", "stix2/indicator"],
publish_topic="stix2/indicator",
snapshot=threatbus_snapshot,
)
def _get_threatbus_entity(self) -> int:
"""
Get the Threat Bus OpenCTI entity. Creates a new entity if it does not
exist yet.
"""
# Use cached:
if self.threatbus_entity is not None:
return self.threatbus_entity
# Try and fetch existing:
threatbus_entity = (
self.opencti_helper.api.stix_domain_object.get_by_stix_id_or_name(
name=self.entity_name))
if threatbus_entity is not None and threatbus_entity.get("id", None):
self.threatbus_entity = threatbus_entity
return self.threatbus_entity
# Create a new one:
self.opencti_helper.log_info(
f"Creating new OpenCTI Threat Bus entity '{self.entity_name}'")
self.threatbus_entity = self.opencti_helper.api.identity.create(
type="Organization",
name=self.entity_name,
description=self.entity_desc,
)
return self.threatbus_entity
def _handle_threatbus_message(self, msg: str):
"""
Processes a JSON message from Threat Bus (either a serialized STIX-2
Sighting or STIX-2 Indicator) and forwards it to OpenCTI.
"""
try:
stix_msg = parse(msg, allow_custom=True)
except Exception as e:
self.opencti_helper.log_error(
f"Error parsing message from Threat Bus. Expected a STIX-2 Sighting or Indicator: {e}"
)
return
if type(stix_msg) is Sighting:
self._report_sighting(stix_msg)
elif type(stix_msg) is Indicator:
self._ingest_indicator(stix_msg)
else:
self.opencti_helper.log_error(
f"Received STIX object with unexpected type from Threat Bus: {type(stix_msg)}"
)
def _ingest_indicator(self, indicator: Indicator):
"""
Ingests a STIX-2 Indicator into OpenCTI. Does nothing in case the
indicator already exists.
@param indicator The STIX-2 Indicator object to ingest
"""
if type(indicator) is not Indicator:
self.opencti_helper.log_error(
f"Error ingesting indicator from Threat Bus. Expected a STIX-2 Indicator: {indicator}"
)
return
ioc_dct = json.loads(indicator.serialize())
ioc_dct["name"] = ioc_dct.get("name", indicator.id) # default to UUID
ioc_dct["stix_id"] = indicator.id
del ioc_dct["id"]
obs_type = ioc_dct.get("x_opencti_main_observable_type", "Unknown")
ioc_dct["x_opencti_main_observable_type"] = obs_type
resp = self.opencti_helper.api.indicator.create(**ioc_dct)
self.opencti_helper.log_info(f"Created or added to indicator: {resp}")
def _report_sighting(self, sighting: Sighting):
"""
Reports a STIX-2 Sighting to OpenCTI, modeled as a
`OpenCTI.stix_sighting_relation`.
@param sighting The STIX-2 Sighting object to report
"""
if type(sighting) is not Sighting:
self.opencti_helper.log_error(
f"Error reporting sighting from Threat Bus. Expected a STIX-2 Sighting: {sighting}"
)
return
entity_id = self._get_threatbus_entity().get("id", None)
resp = self.opencti_helper.api.stix_sighting_relationship.create(
fromId=sighting.sighting_of_ref,
toId=entity_id,
createdBy=entity_id,
first_seen=sighting.first_seen.astimezone().strftime(
"%Y-%m-%dT%H:%M:%SZ") if sighting.get("first_seen") else None,
last_seen=sighting.last_seen.astimezone().strftime(
"%Y-%m-%dT%H:%M:%SZ") if sighting.get("last_seen") else None,
confidence=50,
externalReferences=[sighting.sighting_of_ref],
count=1,
)
self.opencti_helper.log_info(f"Created sighting {resp}")
def _map_to_threatbus(self, data: dict,
opencti_action: str) -> Union[Indicator, None]:
"""
Inspects the given OpenCTI data point and either returns a valid STIX-2
Indicator or None.
@param data A dict object with OpenCTI SSE data
@param opencti_action A string indicating what happened to this item
(either `create`, `update` or `delete`)
@return a STIX-2 Indicator or None
"""
opencti_id: str = data.get("x_opencti_id", None)
if not opencti_id:
self.opencti_helper.log_error(
"Cannot process data without 'x_opencti_id' field")
return
event_id = data.get("id", None)
update = data.get("x_data_update", {})
added = update.get("add", {})
added_ids = added.get("x_opencti_stix_ids", [])
type_ = data.get("type", None)
if type_ == "indicator" and len(
added_ids) == 1 and added_ids[0] == event_id:
# Discard the update if it was empty. An update is empty when the
# only "changed" attribute is the stix_id and it changed to its own
# already existing value. Example:
# data ~ {'id': 'XXX', 'x_data_update': {'add': {'x_opencti_stix_ids': ['XXX']}}}
return
indicator: dict = self.opencti_helper.api.indicator.read(id=opencti_id)
if not indicator:
# we are only interested in indicators at this time
return
detection_enabled: bool = indicator.get("x_opencti_detection", False)
if not detection_enabled and self.forward_all_iocs is not True:
# only propagate indicators that are toggled for detection or the
# user enabled forwarding of all indicators regardless of the toggle
return
# overwrite custom OpenCTI ID
indicator["id"] = indicator.get("standard_id")
if opencti_action == "update":
indicator[ThreatBusSTIX2Constants.X_THREATBUS_UPDATE.
value] = Operation.EDIT.value
if opencti_action == "delete":
indicator[ThreatBusSTIX2Constants.X_THREATBUS_UPDATE.
value] = Operation.REMOVE.value
return Indicator(**indicator, allow_custom=True)
def _process_message(self, sse_msg: Event):
"""
Invoked for every incoming SSE message from the OpenCTI endpoint
@param sse_msg: the received SSE Event
"""
try:
data: dict = json.loads(sse_msg.data).get("data", None)
if not data:
return
indicator = self._map_to_threatbus(data, sse_msg.event)
if not indicator:
return
self.threatbus_helper.send(indicator.serialize())
except Exception as e:
self.opencti_helper.log_error(
f"Error forwarding indicator to Threat Bus: {e}")
def start(self):
self.opencti_helper.log_info("Starting Threat Bus connector")
# Fork a new Thread to communicate with Threat Bus
self.threatbus_helper.start()
atexit.register(self.threatbus_helper.stop)
# Send the main loop into a busy loop for processing OpenCTI events
self.opencti_helper.listen_stream(self._process_message)
class TestLocalSynchronizer:
def __init__( # pylint: disable=too-many-arguments
self,
source_url,
source_token,
target_url,
target_token,
consuming_count,
start_timestamp,
live_stream_id=None,
):
self.source_url = source_url
self.source_token = source_token
self.target_url = target_url
self.target_token = target_token
self.live_stream_id = live_stream_id
self.count_number = 0
self.consuming_count = consuming_count
self.start_timestamp = start_timestamp
self.stream = None
# Source
config = {
"id": "673ba380-d229-4160-9213-ac5afdaabf96",
"type": "STREAM",
"name": "Synchronizer",
"scope": "synchronizer",
"confidence_level": 15,
"live_stream_id": self.live_stream_id,
"log_level": "info",
}
self.opencti_source_client = OpenCTIApiClient(source_url, source_token)
self.opencti_source_helper = OpenCTIConnectorHelper(
{
"opencti": {"url": self.source_url, "token": self.source_token},
"connector": config,
}
)
# Target
self.opencti_target_client = OpenCTIApiClient(target_url, target_token)
self.opencti_target_helper = OpenCTIConnectorHelper(
{
"opencti": {"url": self.target_url, "token": self.target_token},
"connector": config,
}
)
def _process_message(self, msg):
if msg.event in ("create", "update", "merge", "delete"):
logging.info("%s", f"Processing event {msg.id}")
self.count_number += 1
data = json.loads(msg.data)
if msg.event == "create":
bundle = {
"type": "bundle",
"x_opencti_event_version": data["version"],
"objects": [data["data"]],
}
self.opencti_target_client.stix2.import_bundle(bundle)
elif msg.event == "update":
bundle = {
"type": "bundle",
"x_opencti_event_version": data["version"],
"objects": [data["data"]],
}
self.opencti_target_client.stix2.import_bundle(bundle, True)
elif msg.event == "merge":
sources = data["data"]["x_opencti_context"]["sources"]
object_ids = list(map(lambda element: element["id"], sources))
self.opencti_target_helper.api.stix_core_object.merge(
id=data["data"]["id"], object_ids=object_ids
)
elif msg.event == "delete":
self.opencti_target_helper.api.stix.delete(id=data["data"]["id"])
if self.count_number >= self.consuming_count:
self.stream.stop()
def sync(self):
# Reset the connector state if exists
self.opencti_source_helper.set_state(
{"connectorLastEventId": self.start_timestamp}
)
# Start to listen the stream from start specified parameter
self.stream = self.opencti_source_helper.listen_stream(
self._process_message,
self.source_url,
self.source_token,
False,
self.start_timestamp,
)
self.stream.join()
class BackupFilesConnector:
def __init__(self, conf_data):
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else conf_data)
self.helper = OpenCTIConnectorHelper(config)
# Extra config
self.backup_protocol = get_config_variable("BACKUP_PROTOCOL",
["backup", "protocol"],
config)
self.backup_path = get_config_variable("BACKUP_PATH",
["backup", "path"], config)
def _enrich_with_files(self, current):
entity = current
files = []
if (entity["type"] != "relationship" and entity["type"] != "sighting"
and not StixMetaTypes.has_value(entity["type"])):
files = self.helper.api.stix_core_object.list_files(
id=entity["id"])
elif entity["type"] == "external-reference":
files = self.helper.api.external_reference.list_files(
id=entity["id"])
if len(files) > 0:
entity["x_opencti_files"] = []
for file in files:
url = (self.helper.api.api_url.replace(
"graphql", "storage/get/") + file["id"])
data = self.helper.api.fetch_opencti_file(url,
binary=True,
serialize=True)
entity["x_opencti_files"].append({
"name":
file["name"],
"data":
data,
"mime_type":
file["metaData"]["mimetype"],
"version":
file["metaData"]["version"],
})
return entity
def write_files(self, date_range, entity_id, bundle):
path = self.backup_path + "/opencti_data"
if not os.path.exists(path + "/" + date_range):
os.mkdir(path + "/" + date_range)
path = path + "/" + date_range
with open(path + "/" + entity_id + ".json", "w") as file:
json.dump(bundle, file, indent=4)
def delete_file(self, date_range, entity_id):
path = self.backup_path + "/opencti_data/" + date_range
if not os.path.exists(path):
return
if os.path.isfile(path + "/" + entity_id + ".json"):
os.unlink(path + "/" + entity_id + ".json")
def _process_message(self, msg):
if msg.event == "create" or msg.event == "update" or msg.event == "delete":
data = json.loads(msg.data)
created_at = parser.parse(data["data"]["created_at"])
date_range = round_time(created_at).strftime("%Y%m%dT%H%M%SZ")
if msg.event == "create":
bundle = {
"type": "bundle",
"x_opencti_event_version": data["version"],
"objects": [data["data"]],
}
data["data"] = self._enrich_with_files(data["data"])
self.write_files(date_range, data["data"]["id"], bundle)
elif msg.event == "update":
bundle = {
"type": "bundle",
"x_opencti_event_version": data["version"],
"objects": [data["data"]],
}
data["data"] = self._enrich_with_files(data["data"])
self.write_files(date_range, data["data"]["id"], bundle)
elif msg.event == "delete":
self.delete_file(date_range, data["data"]["id"])
self.helper.log_info("Backup processed event " + msg.id + " in " +
date_range + " / " + data["data"]["id"])
def start(self):
# Check if the directory exists
if not os.path.exists(self.backup_path):
raise ValueError("Backup path does not exist")
if not os.path.exists(self.backup_path + "/opencti_data"):
os.mkdir(self.backup_path + "/opencti_data")
self.helper.listen_stream(self._process_message)
class TaniumConnector:
def __init__(self):
# Initialize parameters and OpenCTI helper
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
# Initialize the Tanium API Handler
self.tanium_url = get_config_variable("TANIUM_URL", ["tanium", "url"],
config)
self.tanium_ssl_verify = get_config_variable("TANIUM_SSL_VERIFY",
["tanium", "ssl_verify"],
config, False, True)
self.tanium_login = get_config_variable("TANIUM_LOGIN",
["tanium", "login"], config)
self.tanium_password = get_config_variable("TANIUM_PASSWORD",
["tanium", "password"],
config)
self.tanium_hashes_in_reputation = get_config_variable(
"TANIUM_HASHES_IN_REPUTATION",
["tanium", "hashes_in_reputation"],
config,
False,
True,
)
self.tanium_no_hashes_in_intels = get_config_variable(
"TANIUM_NO_HASHES_IN_INTELS",
["tanium", "no_hashes_in_intels"],
config,
False,
True,
)
# Launch quickscan automatically (true/false)
self.tanium_auto_quickscan = get_config_variable(
"TANIUM_AUTO_QUICKSCAN", ["tanium", "auto_quickscan"], config,
False, False)
# Target computer groups of the automatic quickscan (if enable)
self.tanium_computer_groups = get_config_variable(
"TANIUM_COMPUTER_GROUPS", ["tanium", "computer_groups"], config,
False, "").split(",")
# Check Live Stream ID
if (self.helper.connect_live_stream_id is None
or self.helper.connect_live_stream_id == "ChangeMe"):
raise ValueError("Missing Live Stream ID")
# Initialize Tanium API
self.tanium_api_handler = TaniumApiHandler(
self.helper,
self.tanium_url,
self.tanium_login,
self.tanium_password,
self.tanium_ssl_verify,
self.tanium_auto_quickscan,
self.tanium_computer_groups,
)
# Initialize managers
self.intel_cache = IntelCache(self.helper)
self.import_manager = IntelManager(self.helper,
self.tanium_api_handler,
self.intel_cache)
def _process_message(self, msg):
try:
data = json.loads(msg.data)["data"]
except:
raise ValueError("Cannot process the message: " + msg)
# Handle creation
if msg.event == "create":
if data["type"] == "indicator":
self.helper.log_info("[CREATE] Processing indicator {" +
data["x_opencti_id"] + "}")
self.import_manager.import_intel_from_indicator(data)
elif data["type"] in [
"ipv4-addr",
"ipv6-addr",
"domain-name",
"x-opencti-hostname",
"process",
]:
self.helper.log_info("[CREATE] Processing observable {" +
data["x_opencti_id"] + "}")
self.import_manager.import_intel_from_observable(data)
elif data["type"] in ["file", "artifact"]:
if self.tanium_hashes_in_reputation:
self.import_manager.import_reputation(data)
if not self.tanium_no_hashes_in_intels:
self.import_manager.import_intel_from_observable(data)
return
# Handle update
if msg.event == "update":
if data["type"] == "indicator":
self.helper.log_info("[UPDATE] Processing indicator {" +
data["x_opencti_id"] + "}")
self.import_manager.import_intel_from_indicator(data, True)
elif data["type"] in [
"ipv4-addr",
"ipv6-addr",
"domain-name",
"x-opencti-hostname",
"file",
"artifact",
"process",
]:
self.helper.log_info("[UPDATE] Processing observable {" +
data["x_opencti_id"] + "}")
self.import_manager.import_intel_from_observable(data, True)
return
# Handle delete
elif msg.event == "delete":
if data["type"] == "indicator":
self.import_manager.delete_intel(data)
elif data["type"] in [
"ipv4-addr",
"ipv6-addr",
"domain-name",
"x-opencti-hostname",
]:
self.import_manager.delete_intel(data)
elif data["type"] in ["file", "artifact"]:
self.import_manager.delete_intel(data)
self.import_manager.delete_reputation(data)
return
return
def start(self):
self.helper.listen_stream(self._process_message)
class TaniumConnector:
def __init__(self):
# Initialize parameters and OpenCTI helper
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
self.helper.set_state(
{"connectorLastEventId": int(round(time.time() * 1000)) - 1000})
# Initialize the Tanium API Handler
tanium_url = get_config_variable("TANIUM_URL", ["tanium", "url"],
config)
tanium_ssl_verify = get_config_variable("TANIUM_SSL_VERIFY",
["tanium", "ssl_verify"],
config, False, True)
tanium_login = get_config_variable("TANIUM_LOGIN", ["tanium", "login"],
config)
tanium_password = get_config_variable("TANIUM_PASSWORD",
["tanium", "password"], config)
# Launch quickscan automatically (true/false)
self.tanium_auto_quickscan = get_config_variable(
"TANIUM_AUTO_QUICKSCAN", ["tanium", "auto_quickscan"], config,
False, False)
# Target computer groups of the automatic quickscan (if enable)
self.tanium_computer_groups = get_config_variable(
"TANIUM_COMPUTER_GROUPS", ["tanium", "computer_groups"], config,
False, "").split(",")
self.tanium_api_handler = TaniumApiHandler(
self.helper,
tanium_url,
tanium_login,
tanium_password,
tanium_ssl_verify,
self.tanium_auto_quickscan,
self.tanium_computer_groups,
)
# Initialize managers
self.intel_cache = IntelCache(self.helper)
self.import_manager = IntelManager(self.helper,
self.tanium_api_handler,
self.intel_cache)
# Filters & conditions
# Types of indicators to synchronize
self.tanium_indicator_types = get_config_variable(
"TANIUM_INDICATOR_TYPES", ["tanium", "indicator_types"],
config).split(",")
# Types of observables to synchronize
self.tanium_observable_types = get_config_variable(
"TANIUM_OBSERVABLE_TYPES", ["tanium", "observable_types"],
config).split(",")
# Synchronize only the given labels (or everything using "*")
self.tanium_import_label = get_config_variable(
"TANIUM_IMPORT_LABEL", ["tanium", "import_label"], config, False,
"")
# Synchronize hashes to Reputation using the given label (or everything using "*")
self.tanium_reputation_blacklist_label = get_config_variable(
"TANIUM_REPUTATION_BLACKLIST_LABEL",
["tanium", "reputation_blacklist_label"],
config,
False,
"",
)
def handle_create_indicator(self, data):
self.helper.log_info("[CREATE] Processing indicator {" + data["id"] +
"}")
# If import everything as intel
if self.tanium_import_label == "*":
self.import_manager.import_intel_from_indicator(data)
return
# If no label in this creation and import if filtered
if "labels" not in data:
self.helper.log_info(
"[CREATE] No label corresponding to import filter, doing nothing"
)
return
# If label corresponds
if self.tanium_import_label in data["labels"]:
self.import_manager.import_intel_from_indicator(data)
return
else:
self.helper.log_info(
"[CREATE] No label corresponding to import filter, doing nothing"
)
return
def handle_create_observable(self, data):
self.helper.log_info("[CREATE] Processing observable {" + data["id"] +
"}")
# if import everything as intel
if self.tanium_import_label == "*":
self.import_manager.import_intel_from_observable(data)
# If file and import everything
if self.tanium_reputation_blacklist_label == "*":
self.import_manager.import_reputation(data)
return
# If no label in this creation and import if filtered
if "labels" not in data:
self.helper.log_info(
"[CREATE] No label corresponding to import filter, doing nothing"
)
return
# If label corresponds
if self.tanium_import_label in data["labels"]:
self.import_manager.import_intel_from_observable(data)
if self.tanium_reputation_blacklist_label in data["labels"]:
self.import_manager.import_reputation(data)
return
def handle_update_indicator(self, data):
self.helper.log_info("[UPDATE] Processing indicator {" + data["id"] +
"}")
# New labels have been added and correspond to filter
if ("x_data_update" in data and "add" in data["x_data_update"]
and "labels" in data["x_data_update"]["add"]
and self.tanium_import_label
in data["x_data_update"]["add"]["labels"]):
# Get the indicator to have the pattern_type
entity = self.helper.api.indicator.read(id=data["x_opencti_id"])
data["name"] = entity["name"]
data["pattern"] = entity["pattern"]
data["pattern_type"] = entity["pattern_type"]
if "x_mitre_platforms" in entity:
data["x_mitre_platforms"] = entity["x_mitre_platforms"]
self.import_manager.import_intel_from_indicator(data)
return
# Labels have been removed and correspond to filter
if ("x_data_update" in data and "remove" in data["x_data_update"]
and "labels" in data["x_data_update"]["remove"]
and self.tanium_import_label
in data["x_data_update"]["remove"]["labels"]):
self.import_manager.delete_intel(data)
return
if ("x_data_update" in data and "replace" in data["x_data_update"]
and "pattern" in data["x_data_update"]["replace"]):
self.import_manager.import_intel_from_indicator(data, True)
return
def handle_update_observable(self, data):
self.helper.log_info("[UPDATE] Processing observable {" + data["id"] +
"}")
# Label has been added and corresponds to filter
if ("x_data_update" in data and "add" in data["x_data_update"]
and "labels" in data["x_data_update"]["add"]):
# For intels
if self.tanium_import_label in data["x_data_update"]["add"][
"labels"]:
# Get the indicator to have the pattern_type
entity = self.helper.api.stix_cyber_observable.read(
id=data["x_opencti_id"])
if "value" in entity:
data["value"] = entity["value"]
if "hashes" in entity:
data["hashes"] = entity["hashes"]
self.import_manager.import_intel_from_observable(data)
# For reputation
if (self.tanium_reputation_blacklist_label
in data["x_data_update"]["add"]["labels"]):
entity = self.helper.api.stix_cyber_observable.read(
id=data["x_opencti_id"])
if "value" in entity:
data["value"] = entity["value"]
if "hashes" in entity:
data["hashes"] = entity["hashes"]
self.import_manager.import_reputation(data)
return
if ("x_data_update" in data and "remove" in data["x_data_update"]
and "labels" in data["x_data_update"]["remove"]
and self.tanium_import_label
in data["x_data_update"]["remove"]["labels"]):
self.import_manager.delete_intel(data)
return
def handle_delete_indicator(self, data):
self.import_manager.delete_intel(data)
return
def handle_delete_observable(self, data):
self.import_manager.delete_intel(data)
self.import_manager.delete_reputation(data)
return
def _process_message(self, msg):
try:
event_id = msg.id
date = datetime.fromtimestamp(
round(int(event_id.split("-")[0]) / 1000))
data = json.loads(msg.data)["data"]
except:
raise ValueError("Cannot process the message: " + msg)
# Ignore types which will not be processed
self.helper.log_info("[PROCESS] Message (id: " + event_id +
", date: " + str(date) + ")")
if ("revoked" in data and data["revoked"] is True) or (
data["type"] != "indicator"
and data["type"] not in self.tanium_observable_types):
self.helper.log_info(
"[PROCESS] Doing nothing, entity type not in import filter or entity revoked"
)
return
# Handle creation
if msg.event == "create":
if (data["type"] == "indicator"
and data["pattern_type"] in self.tanium_indicator_types):
return self.handle_create_indicator(data)
if data["type"] in self.tanium_observable_types:
return self.handle_create_observable(data)
return None
# Handle update
if msg.event == "update":
if data["type"] == "indicator":
return self.handle_update_indicator(data)
if data["type"] in self.tanium_observable_types:
return self.handle_update_observable(data)
return None
# Handle delete
elif msg.event == "delete":
if data["type"] == "indicator":
return self.handle_delete_indicator(data)
if data["type"] in self.tanium_observable_types:
return self.handle_delete_observable(data)
return None
return None
def start(self):
self.helper.listen_stream(self._process_message)
class SplunkConnector:
def __init__(self):
# Initialize parameters and OpenCTI helper
config_file_path = os.path.dirname(os.path.abspath(__file__)) + "/config.yml"
config = (
yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path)
else {}
)
self.helper = OpenCTIConnectorHelper(config)
self.splunk_url = get_config_variable("SPLUNK_URL", ["splunk", "url"], config)
self.splunk_ssl_verify = get_config_variable(
"SPLUNK_SSL_VERIFY", ["splunk", "ssl_verify"], config, False, True
)
self.splunk_login = get_config_variable(
"SPLUNK_LOGIN", ["splunk", "login"], config
)
self.splunk_password = get_config_variable(
"SPLUNK_PASSWORD", ["splunk", "password"], config
)
self.splunk_owner = get_config_variable(
"SPLUNK_OWNER", ["splunk", "owner"], config
)
self.splunk_app = get_config_variable("SPLUNK_APP", ["splunk", "app"], config)
self.splunk_kv_store_name = get_config_variable(
"SPLUNK_KV_STORE_NAME", ["splunk", "kv_store_name"], config
)
if (
self.helper.connect_live_stream_id is None
or self.helper.connect_live_stream_id == "ChangeMe"
):
raise ValueError("Missing Live Stream ID")
# Initialize the KV Store
self._query("post", "/config", {"name": self.splunk_kv_store_name})
def _query(self, method, uri, payload=None, is_json=False):
self.helper.log_info("Query " + method + " on " + uri)
url = (
self.splunk_url
+ "/servicesNS/"
+ self.splunk_owner
+ "/"
+ self.splunk_app
+ "/storage/collections"
+ uri
)
if method == "get":
r = requests.get(
url,
auth=(self.splunk_login, self.splunk_password),
params=payload,
verify=self.splunk_ssl_verify,
)
elif method == "post":
if is_json:
headers = {"content-type": "application/json"}
r = requests.post(
url,
auth=(self.splunk_login, self.splunk_password),
headers=headers,
json=payload,
verify=self.splunk_ssl_verify,
)
else:
r = requests.post(
url,
auth=(self.splunk_login, self.splunk_password),
data=payload,
verify=self.splunk_ssl_verify,
)
elif method == "delete":
r = requests.delete(
url,
auth=(self.splunk_login, self.splunk_password),
verify=self.splunk_ssl_verify,
)
else:
raise ValueError("Unsupported method")
if r.status_code < 500:
print(r.text)
try:
return r.json()
except:
return r.text
else:
self.helper.log_info(r.text)
def _process_message(self, msg):
try:
data = json.loads(msg.data)["data"]
except:
raise ValueError("Cannot process the message: " + msg)
# Handle creation
if msg.event == "create":
self.helper.log_info(
"[CREATE] Processing data {" + data["x_opencti_id"] + "}"
)
data["_key"] = data["x_opencti_id"]
return self._query("post", "/data/" + self.splunk_kv_store_name, data, True)
# Handle update
if msg.event == "update":
self.helper.log_info(
"[UPDATE] Processing data {" + data["x_opencti_id"] + "}"
)
data["_key"] = data["x_opencti_id"]
return self._query(
"post",
"/data/" + self.splunk_kv_store_name + "/" + data["x_opencti_id"],
data,
True,
)
# Handle delete
elif msg.event == "delete":
self.helper.log_info(
"[DELETE] Processing data {" + data["x_opencti_id"] + "}"
)
return self._query(
"delete",
"/data/" + self.splunk_kv_store_name + "/" + data["x_opencti_id"],
data,
)
return None
def start(self):
self.helper.listen_stream(self._process_message)
class ElasticsearchConnector:
def __init__(self):
# Initialize parameters and OpenCTI helper
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
self.elasticsearch_url = get_config_variable("ELASTICSEARCH_URL",
["elasticsearch", "url"],
config)
self.elasticsearch_ssl_verify = get_config_variable(
"ELASTICSEARCH_SSL_VERIFY",
["elasticsearch", "ssl_verify"],
config,
False,
True,
)
self.elasticsearch_login = get_config_variable(
"ELASTICSEARCH_LOGIN", ["elasticsearch", "login"], config)
self.elasticsearch_password = get_config_variable(
"ELASTICSEARCH_PASSWORD", ["elasticsearch", "password"], config)
self.elasticsearch_index = get_config_variable(
"ELASTICSEARCH_INDEX", ["elasticsearch", "index"], config)
if (self.helper.connect_live_stream_id is None
or self.helper.connect_live_stream_id == "ChangeMe"):
raise ValueError("Missing Live Stream ID")
# Initilize connection to Elastic
if (self.elasticsearch_login is not None
and len(self.elasticsearch_login) > 0
and self.elasticsearch_password is not None
and len(self.elasticsearch_password) > 0):
self.elasticsearch = Elasticsearch(
[self.elasticsearch_url],
verify_certs=self.elasticsearch_ssl_verify,
http_auth=(
self.elasticsearch_login,
self.elasticsearch_password,
),
)
else:
self.elasticsearch = Elasticsearch(
[self.elasticsearch_url],
verify_certs=self.elasticsearch_ssl_verify,
)
def _index(self, payload):
self.elasticsearch.index(index=self.elasticsearch_index,
id=payload["x_opencti_id"],
body=payload)
def _delete(self, id):
self.elasticsearch.delete(index=self.elasticsearch_index, id=id)
def _process_message(self, msg):
try:
data = json.loads(msg.data)["data"]
except:
raise ValueError("Cannot process the message: " + msg)
# Handle creation
if msg.event == "create":
self.helper.log_info("[CREATE] Processing data {" +
data["x_opencti_id"] + "}")
return self._index(data)
# Handle update
if msg.event == "update":
self.helper.log_info("[UPDATE] Processing data {" +
data["x_opencti_id"] + "}")
return self._index(data)
# Handle delete
elif msg.event == "delete":
self.helper.log_info("[DELETE] Processing data {" +
data["x_opencti_id"] + "}")
return self._delete(data["x_opencti_id"])
return None
def start(self):
self.helper.listen_stream(self._process_message)
