class Malpedia:
"""OpenCTI Malpedia main class"""
_STATE_LAST_RUN = "state_last_run"
_MALPEDIA_LAST_VERSION = "malpedia_last_version"
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/../config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
# Extra config
self.confidence_level = get_config_variable(
"CONNECTOR_CONFIDENCE_LEVEL",
["connector", "confidence_level"],
config,
isNumber=True,
)
self.update_existing_data = get_config_variable(
"CONNECTOR_UPDATE_EXISTING_DATA",
["connector", "update_existing_data"],
config,
)
self.AUTH_KEY = get_config_variable("MALPEDIA_AUTH_KEY",
["malpedia", "auth_key"], config)
self.INTERVAL_SEC = get_config_variable("MALPEDIA_INTERVAL_SEC",
["malpedia", "interval_sec"],
config)
self.import_intrusion_sets = get_config_variable(
"MALPEDIA_IMPORT_INTRUSION_SETS",
["malpedia", "import_intrusion_sets"],
config,
)
self.import_yara = get_config_variable("MALPEDIA_IMPORT_YARA",
["malpedia", "import_yara"],
config)
self.create_indicators = get_config_variable(
"MALPEDIA_CREATE_INDICATORS", ["malpedia", "create_indicators"],
config)
self.create_observables = get_config_variable(
"MALPEDIA_CREATE_OBSERVABLES", ["malpedia", "create_observables"],
config)
self.helper = OpenCTIConnectorHelper(config)
self.helper.log_info(f"loaded malpedia config: {config}")
# Create Malpedia client and importers
self.client = MalpediaClient(self.AUTH_KEY)
# If we run without API key we can assume all data is TLP:WHITE else we
# default to TLP:AMBER to be safe.
if self.client.unauthenticated:
self.default_marking = self.helper.api.marking_definition.read(
id=TLP_WHITE["id"])
else:
self.default_marking = self.helper.api.marking_definition.read(
id=TLP_AMBER["id"])
self.knowledge_importer = KnowledgeImporter(
self.helper,
self.client,
self.confidence_level,
self.update_existing_data,
self.import_intrusion_sets,
self.import_yara,
self.create_indicators,
self.create_observables,
self.default_marking,
)
def _load_state(self) -> Dict[str, Any]:
current_state = self.helper.get_state()
if not current_state:
return {}
return current_state
@staticmethod
def _get_state_value(state: Optional[Mapping[str, Any]],
key: str,
default: Optional[Any] = None) -> Any:
if state is not None:
return state.get(key, default)
return default
def _is_scheduled(self, last_run: Optional[int],
current_time: int) -> bool:
if last_run is None:
return True
time_diff = current_time - last_run
return time_diff >= int(self.INTERVAL_SEC)
def _check_version(self, last_version: Optional[int],
current_version: int) -> bool:
if last_version is None:
return True
return current_version > last_version
@staticmethod
def _current_unix_timestamp() -> int:
return int(datetime.utcnow().timestamp())
def _get_interval(self):
return int(self.INTERVAL_SEC)
def run(self):
self.helper.log_info("starting Malpedia connector...")
while True:
try:
current_malpedia_version = self.client.current_version()
self.helper.log_info(
f"current Malpedia version: {current_malpedia_version}")
timestamp = self._current_unix_timestamp()
current_state = self._load_state()
self.helper.log_info(f"loaded state: {current_state}")
last_run = self._get_state_value(current_state,
self._STATE_LAST_RUN)
last_malpedia_version = self._get_state_value(
current_state, self._MALPEDIA_LAST_VERSION)
# Only run the connector if:
# 1. It is scheduled to run per interval
# 2. The global Malpedia version from the API is newer than our
# last stored version.
if self._is_scheduled(
last_run, timestamp) and self._check_version(
last_malpedia_version, current_malpedia_version):
self.helper.log_info("running importers")
knowledge_importer_state = self._run_knowledge_importer(
current_state)
self.helper.log_info("done with running importers")
new_state = current_state.copy()
new_state.update(knowledge_importer_state)
new_state[
self._STATE_LAST_RUN] = self._current_unix_timestamp()
new_state[
self._MALPEDIA_LAST_VERSION] = current_malpedia_version
self.helper.log_info(f"storing new state: {new_state}")
self.helper.set_state(new_state)
self.helper.log_info(
f"state stored, next run in: {self._get_interval()} seconds"
)
else:
new_interval = self._get_interval() - (timestamp -
last_run)
self.helper.log_info(
f"connector will not run, next run in: {new_interval} seconds"
)
time.sleep(60)
except (KeyboardInterrupt, SystemExit):
self.helper.log_info("connector stop")
exit(0)
except Exception as e:
self.helper.log_error(str(e))
exit(0)
def _run_knowledge_importer(
self, current_state: Mapping[str, Any]) -> Mapping[str, Any]:
return self.knowledge_importer.run(current_state)
class Cve:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
# Extra config
self.cve_import_history = get_config_variable(
"CVE_IMPORT_HISTORY", ["cve", "import_history"], config, False)
self.cve_nvd_data_feed = get_config_variable("CVE_NVD_DATA_FEED",
["cve", "nvd_data_feed"],
config)
self.cve_history_data_feed = get_config_variable(
"CVE_HISTORY_DATA_FEED", ["cve", "history_data_feed"], config)
self.cve_interval = get_config_variable("CVE_INTERVAL",
["cve", "interval"], config,
True)
self.update_existing_data = get_config_variable(
"CONNECTOR_UPDATE_EXISTING_DATA",
["connector", "update_existing_data"],
config,
)
def get_interval(self):
return int(self.cve_interval) * 60 * 60 * 24
def convert_and_send(self, url):
try:
# Downloading json.gz file
self.helper.log_info("Requesting the file " + url)
urllib.request.urlretrieve(
self.cve_nvd_data_feed,
os.path.dirname(os.path.abspath(__file__)) + "/data.json.gz",
)
# Unzipping the file
self.helper.log_info("Unzipping the file")
with gzip.open("data.json.gz", "rb") as f_in:
with open("data.json", "wb") as f_out:
shutil.copyfileobj(f_in, f_out)
# Converting the file to stix2
self.helper.log_info("Converting the file")
convert("data.json", "data-stix2.json")
with open("data-stix2.json") as stix_json:
contents = stix_json.read()
self.helper.send_stix2_bundle(contents,
self.helper.connect_scope,
self.update_existing_data)
# Remove files
os.remove("data.json")
os.remove("data.json.gz")
os.remove("data-stix2.json")
except Exception as e:
self.helper.log_error(str(e))
time.sleep(60)
def run(self):
self.helper.log_info("Fetching CVE knowledge...")
while True:
try:
# Get the current timestamp and check
timestamp = int(time.time())
current_state = self.helper.get_state()
if current_state is not None and "last_run" in current_state:
last_run = current_state["last_run"]
self.helper.log_info("Connector last run: " +
datetime.utcfromtimestamp(last_run).
strftime("%Y-%m-%d %H:%M:%S"))
else:
last_run = None
self.helper.log_info("Connector has never run")
# If the last_run is more than interval-1 day
if last_run is None or ((timestamp - last_run) > (
(int(self.cve_interval) - 1) * 60 * 60 * 24)):
self.convert_and_send(self.cve_nvd_data_feed)
# If import history and never run
if last_run is None or self.cve_import_history:
now = datetime.now()
years = list(range(2002, now.year + 1))
for year in years:
self.convert_and_send(
f"{self.cve_history_data_feed}nvdcve-1.1-{year}.json.gz"
)
# Store the current timestamp as a last run
self.helper.log_info(
"Connector successfully run, storing last_run as " +
str(timestamp))
self.helper.set_state({"last_run": timestamp})
self.helper.log_info(
"Last_run stored, next run in: " +
str(round(self.get_interval() / 60 / 60 / 24, 2)) +
" days")
time.sleep(60)
else:
new_interval = self.get_interval() - (timestamp - last_run)
self.helper.log_info(
"Connector will not run, next run in: " +
str(round(new_interval / 60 / 60 / 24, 2)) + " days")
time.sleep(60)
except (KeyboardInterrupt, SystemExit):
self.helper.log_info("Connector stop")
exit(0)
except Exception as e:
self.helper.log_error(str(e))
time.sleep(60)
class Cybercrimetracker:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = "{}/config.yml".format(
os.path.dirname(os.path.abspath(__file__)))
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
# Connector Config
self.confidence_level = get_config_variable(
"CONNECTOR_CONFIDENCE_LEVEL",
["connector", "confidence_level"],
config,
isNumber=True,
)
self.update_existing_data = get_config_variable(
"CONNECTOR_UPDATE_EXISTING_DATA",
["connector", "update_existing_data"],
config,
)
# CYBERCRIME-TRACKER.NET Config
self.feed_url = get_config_variable("CYBERCRIMET_RACKER_FEED_URL",
["cybercrime-tracker", "feed_url"],
config)
self.connector_tlp = get_config_variable("CYBERCRIME_TRACKER_TLP",
["cybercrime-tracker", "tlp"],
config)
self.create_indicators = get_config_variable(
"CYBERCRIME_TRACKER_CREATE_INDICATORS",
["cybercrime-tracker", "create_indicators"],
config,
)
self.create_observables = get_config_variable(
"CYBERCRIME_TRACKER_CREATE_OBSERVABLES",
["cybercrime-tracker", "create_observables"],
config,
)
self.interval = get_config_variable(
"CYBERCRIMETRACKER_INTERVAL",
["cybercrime-tracker", "interval"],
config,
isNumber=True,
)
@staticmethod
def _time_to_datetime(input_date: time) -> datetime.datetime:
return datetime.datetime(
input_date.tm_year,
input_date.tm_mon,
input_date.tm_mday,
input_date.tm_hour,
input_date.tm_min,
input_date.tm_sec,
tzinfo=datetime.timezone.utc,
)
def parse_feed_entry(self, entry):
"""
Parses an entry from the feed and returns a dict with:
date: date in iso format
type: name of the malware associated with the C2 server
url: the url of the C2
ip: the IP address of the C2
ext_link: An external link to CYBERCRIME-TRACKER.NET with details
Note: CYBERCRIME-TRACKER.NET does not provide the protocol in the url
as such we always assume 'http'.
"""
parsed_entry = {}
pattern = (
r"(?:\[%{GREEDYDATA:cwhqid}\]\s+Type:\s+%{GREEDYDATA:type}" +
r"\s+-%{GREEDYDATA}:\s+%{IP:ip}|" +
r"\[%{GREEDYDATA:cwhqid}\]\s+Type:\s+%{GREEDYDATA:type})")
entry_summary = Grok(pattern).match(entry["summary"])
if entry_summary:
parsed_entry["date"] = self._time_to_datetime(
entry["published_parsed"])
parsed_entry["type"] = entry_summary["type"]
parsed_entry["ext_link"] = entry["link"]
parsed_entry["url"] = "http://{}".format(quote(entry["title"]))
hostname = urlparse(parsed_entry["url"]).hostname
if entry_summary["ip"] is None:
parsed_entry["ip"] = hostname
else:
parsed_entry["ip"] = entry_summary["ip"]
parsed_entry["domain"] = hostname
self.helper.log_info("Parsed entry: {}".format(entry["title"]))
return parsed_entry
else:
self.helper.log_error("Could not parse: {}".format(entry["title"]))
return False
def gen_indicator_pattern(self, parsed_entry):
if "domain" in parsed_entry.keys():
indicator_pattern = (
"[ipv4-addr:value='{}'] ".format(parsed_entry["ip"]) +
"AND [url:value='{}'] ".format(parsed_entry["url"]) +
"AND [domain-name:value='{}']".format(parsed_entry["domain"]))
else:
indicator_pattern = "[ipv4-addr:value='{}'] ".format(
parsed_entry["ip"]) + "AND [url:value='{}']".format(
parsed_entry["url"])
return indicator_pattern
def run(self):
self.helper.log_info("Fetching data CYBERCRIME-TRACKER.NET...")
tlp = self.helper.api.marking_definition.read(
filters=[{
"key": "definition",
"values": "TLP:{}".format(self.connector_tlp)
}])
while True:
try:
# Get the current timestamp and check
timestamp = int(time.time())
current_state = self.helper.get_state()
if current_state is not None and "last_run" in current_state:
last_run = current_state["last_run"]
self.helper.log_info("Connector last run: {}".format(
datetime.datetime.utcfromtimestamp(last_run).strftime(
"%Y-%m-%d %H:%M:%S")))
else:
last_run = None
self.helper.log_info("Connector has never run")
# Run if it is the first time or we are past the interval
if last_run is None or (
(timestamp - last_run) > self.interval):
self.helper.log_info("Connector will run!")
now = datetime.datetime.utcfromtimestamp(timestamp)
friendly_name = "Cybercrime-Tracker run @ " + now.strftime(
"%Y-%m-%d %H:%M:%S")
work_id = self.helper.api.work.initiate_work(
self.helper.connect_id, friendly_name)
# Get Feed Content
feed = feedparser.parse(self.feed_url)
self.helper.log_info("Found: {} entries.".format(
len(feed["entries"])))
self.feed_summary = {
"Source":
feed["feed"]["title"],
"Date":
self._time_to_datetime(
feed["feed"]["published_parsed"]),
"Details":
feed["feed"]["subtitle"],
"Link":
feed["feed"]["link"],
}
# Create the bundle
bundle_objects = list()
organization = stix2.Identity(
id=OpenCTIStix2Utils.generate_random_stix_id(
"identity"),
name="CYBERCRIME-TRACKER.NET",
identity_class="organization",
description=
"Tracker collecting and sharing daily updates of C2 IPs/Urls. http://cybercrime-tracker.net",
)
bundle_objects.append(organization)
for entry in feed["entries"]:
parsed_entry = self.parse_feed_entry(entry)
external_reference = stix2.ExternalReference(
source_name="{}".format(
self.feed_summary["Source"]),
url=parsed_entry["ext_link"],
)
indicator_pattern = self.gen_indicator_pattern(
parsed_entry)
malware = stix2.Malware(
id=OpenCTIStix2Utils.generate_random_stix_id(
"malware"),
is_family=True,
name=parsed_entry["type"],
description="{} malware.".format(
parsed_entry["type"]),
)
bundle_objects.append(malware)
indicator = None
if self.create_indicators:
indicator = stix2.Indicator(
id=OpenCTIStix2Utils.generate_random_stix_id(
"indicator"),
name=parsed_entry["url"],
description="C2 URL for: {}".format(
parsed_entry["type"]),
labels=["C2 Server"],
pattern_type="stix",
pattern=indicator_pattern,
valid_from=parsed_entry["date"],
created=parsed_entry["date"],
modified=parsed_entry["date"],
created_by_ref=organization.id,
object_marking_refs=[tlp["standard_id"]],
external_references=[external_reference],
custom_properties={
"x_opencti_main_observable_type": "Url"
},
)
bundle_objects.append(indicator)
relation = stix2.Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id(
"relationship"),
source_ref=indicator.id,
target_ref=malware.id,
relationship_type="indicates",
start_time=self._time_to_datetime(
entry["published_parsed"]),
stop_time=self._time_to_datetime(
entry["published_parsed"]) +
datetime.timedelta(0, 3),
description="URLs associated to: " +
parsed_entry["type"],
confidence=self.confidence_level,
created_by_ref=organization.id,
object_marking_refs=[tlp["standard_id"]],
created=parsed_entry["date"],
modified=parsed_entry["date"],
external_references=[external_reference],
)
bundle_objects.append(relation)
if self.create_observables:
observable_url = SimpleObservable(
id=OpenCTIStix2Utils.generate_random_stix_id(
"x-opencti-simple-observable"),
key="Url.value",
labels=["C2 Server"],
value=parsed_entry["url"],
created_by_ref=organization.id,
object_marking_refs=[tlp["standard_id"]],
external_references=[external_reference],
)
bundle_objects.append(observable_url)
observable_ip = SimpleObservable(
id=OpenCTIStix2Utils.generate_random_stix_id(
"x-opencti-simple-observable"),
key="IPv4-Addr.value",
labels=["C2 Server"],
value=parsed_entry["ip"],
created_by_ref=organization.id,
object_marking_refs=[tlp["standard_id"]],
external_references=[external_reference],
)
bundle_objects.append(observable_ip)
observable_domain = None
if "domain" in parsed_entry.keys():
observable_domain = SimpleObservable(
id=OpenCTIStix2Utils.
generate_random_stix_id(
"x-opencti-simple-observable"),
key="Domain-Name.value",
labels=["C2 Server"],
value=parsed_entry["domain"],
created_by_ref=organization.id,
object_marking_refs=[tlp["standard_id"]],
external_references=[external_reference],
)
bundle_objects.append(observable_domain)
if indicator is not None:
relationship_1 = stix2.Relationship(
id=OpenCTIStix2Utils.
generate_random_stix_id("relationship"),
relationship_type="based-on",
created_by_ref=organization.id,
source_ref=indicator.id,
target_ref=observable_url.id,
)
bundle_objects.append(relationship_1)
relationship_2 = stix2.Relationship(
id=OpenCTIStix2Utils.
generate_random_stix_id("relationship"),
relationship_type="based-on",
created_by_ref=organization.id,
source_ref=indicator.id,
target_ref=observable_ip.id,
)
bundle_objects.append(relationship_2)
if observable_domain is not None:
relationship_3 = stix2.Relationship(
id=OpenCTIStix2Utils.
generate_random_stix_id(
"relationship"),
relationship_type="based-on",
created_by_ref=organization.id,
source_ref=indicator.id,
target_ref=observable_domain.id,
)
bundle_objects.append(relationship_3)
# create stix bundle
bundle = stix2.Bundle(objects=bundle_objects)
# send data
self.helper.send_stix2_bundle(
bundle=bundle.serialize(),
update=self.update_existing_data,
work_id=work_id,
)
# Store the current timestamp as a last run
message = (
"Connector successfully run, storing last_run as: {}".
format(str(timestamp)))
self.helper.log_info(message)
self.helper.set_state({"last_run": timestamp})
self.helper.api.work.to_processed(work_id, message)
self.helper.log_info(
"Last_run stored, next run in: {} seconds.".format(
str(round(self.interval, 2))))
time.sleep(60)
else:
new_interval = self.interval - (timestamp - last_run)
self.helper.log_info("Connector will not run. \
Next run in: {} seconds.".format(
str(round(new_interval, 2))))
time.sleep(60)
except (KeyboardInterrupt, SystemExit):
self.helper.log_info("Connector stop")
exit(0)
except Exception as e:
self.helper.log_error(str(e))
time.sleep(60)
class TheHive:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(os.path.abspath(__file__)) + "/config.yml"
config = (
yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path)
else {}
)
self.helper = OpenCTIConnectorHelper(config)
# Extra config
self.thehive_url = get_config_variable(
"THEHIVE_URL", ["thehive", "url"], config
)
self.thehive_api_key = get_config_variable(
"THEHIVE_API_KEY", ["thehive", "api_key"], config
)
self.thehive_check_ssl = get_config_variable(
"THEHIVE_CHECK_SSL", ["thehive", "check_ssl"], config, False, True
)
self.thehive_organization_name = get_config_variable(
"THEHIVE_ORGANIZATION_NAME", ["thehive", "organization_name"], config
)
self.thehive_import_from_date = get_config_variable(
"THEHIVE_IMPORT_FROM_DATE",
["thehive", "import_from_date"],
config,
False,
datetime.utcfromtimestamp(int(time.time())).strftime("%Y-%m-%d %H:%M:%S"),
)
self.update_existing_data = get_config_variable(
"CONNECTOR_UPDATE_EXISTING_DATA",
["connector", "update_existing_data"],
config,
)
self.identity = self.helper.api.identity.create(
type="Organization",
name=self.thehive_organization_name,
description=self.thehive_organization_name,
)
self.thehive_api = TheHiveApi(
self.thehive_url, self.thehive_api_key, cert=self.thehive_check_ssl
)
def generate_case_bundle(self, case):
markings = []
if case["tlp"] == 0:
markings.append(TLP_WHITE)
if case["tlp"] == 1:
markings.append(TLP_GREEN)
if case["tlp"] == 2:
markings.append(TLP_AMBER)
if case["tlp"] == 3:
markings.append(TLP_RED)
if len(markings) == 0:
markings.append(TLP_WHITE)
bundle_objects = []
incident = StixXOpenCTIIncident(
id=OpenCTIStix2Utils.generate_random_stix_id("x-opencti-incident"),
name=case["title"],
description=case["description"],
first_seen=datetime.utcfromtimestamp(
int(case["createdAt"]) / 1000
).strftime("%Y-%m-%dT%H:%M:%SZ"),
last_seen=datetime.utcfromtimestamp(int(case["updatedAt"]) / 1000).strftime(
"%Y-%m-%dT%H:%M:%SZ"
),
object_marking_refs=markings,
labels=case["tags"] if "tags" in case else [],
created_by_ref=self.identity["standard_id"],
)
bundle_objects.append(incident)
# Get observables
observables = self.thehive_api.get_case_observables(case_id=case["id"]).json()
for observable in observables:
if observable["dataType"] == "hash":
if len(observable["data"]) == 32:
data_type = "file_md5"
elif len(observable["data"]) == 40:
data_type = "file_sha1"
elif len(observable["data"]) == 64:
data_type = "file_sha256"
else:
data_type = "unknown"
else:
data_type = observable["dataType"]
observable_key = OBSERVABLES_MAPPING[data_type]
if observable_key is not None:
stix_observable = SimpleObservable(
id=OpenCTIStix2Utils.generate_random_stix_id(
"x-opencti-simple-observable"
),
key=observable_key,
value=observable["data"],
description=observable["message"],
x_opencti_score=80 if observable["ioc"] else 50,
object_marking_refs=markings,
labels=observable["tags"] if "tags" in observable else [],
created_by_ref=self.identity["standard_id"],
x_opencti_create_indicator=observable["ioc"],
)
stix_observable_relation = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id("relationship"),
relationship_type="related-to",
created_by_ref=self.identity["standard_id"],
source_ref=stix_observable.id,
target_ref=incident.id,
object_marking_refs=markings,
)
bundle_objects.append(stix_observable)
bundle_objects.append(stix_observable_relation)
if observable["sighted"]:
fake_indicator_id = (
"indicator--c1034564-a9fb-429b-a1c1-c80116cc8e1e"
)
stix_sighting = Sighting(
id=OpenCTIStix2Utils.generate_random_stix_id("sighting"),
first_seen=datetime.utcfromtimestamp(
int(observable["startDate"] / 1000)
).strftime("%Y-%m-%dT%H:%M:%SZ"),
last_seen=datetime.utcfromtimestamp(
int(observable["startDate"] / 1000 + 3600)
).strftime("%Y-%m-%dT%H:%M:%SZ"),
where_sighted_refs=[self.identity["standard_id"]],
sighting_of_ref=fake_indicator_id,
custom_properties={
"x_opencti_sighting_of_ref": stix_observable.id
},
)
bundle_objects.append(stix_sighting)
bundle = Bundle(objects=bundle_objects).serialize()
return bundle
def run(self):
self.helper.log_info("Starting TheHive Connector...")
while True:
try:
# Get the current timestamp and check
timestamp = int(time.time())
current_state = self.helper.get_state()
if current_state is not None and "last_case_date" in current_state:
last_case_date = current_state["last_case_date"]
self.helper.log_info(
"Connector last_case_date: "
+ datetime.utcfromtimestamp(last_case_date).strftime(
"%Y-%m-%d %H:%M:%S"
)
)
else:
last_case_date = parse(self.thehive_import_from_date).timestamp()
self.helper.log_info("Connector has no last_case_date")
self.helper.log_info(
"Get cases since last run ("
+ datetime.utcfromtimestamp(last_case_date).strftime(
"%Y-%m-%d %H:%M:%S"
)
+ ")"
)
query = Or(
Gt("updatedAt", int(last_case_date * 1000)),
Child("case_task", Gt("createdAt", int(last_case_date * 1000))),
Child("case_artifact", Gt("createdAt", int(last_case_date * 1000))),
)
cases = self.thehive_api.find_cases(
query=query, sort="updatedAt", range="0-100"
).json()
now = datetime.utcfromtimestamp(timestamp)
friendly_name = "TheHive run @ " + now.strftime("%Y-%m-%d %H:%M:%S")
work_id = self.helper.api.work.initiate_work(
self.helper.connect_id, friendly_name
)
try:
for case in cases:
stix_bundle = self.generate_case_bundle(case)
self.helper.send_stix2_bundle(
stix_bundle,
update=self.update_existing_data,
work_id=work_id,
)
except Exception as e:
self.helper.log_error(str(e))
# Store the current timestamp as a last run
message = "Connector successfully run, storing last_run as " + str(
timestamp
)
self.helper.log_info(message)
self.helper.api.work.to_processed(work_id, message)
current_state = self.helper.get_state()
if current_state is None:
current_state = {"last_case_date": timestamp}
else:
current_state["last_case_date"] = timestamp
self.helper.set_state(current_state)
time.sleep(60)
except (KeyboardInterrupt, SystemExit):
self.helper.log_info("Connector stop")
exit(0)
except Exception as e:
self.helper.log_error(str(e))
time.sleep(60)
class CrowdStrike:
"""CrowdStrike connector."""
_CONFIG_NAMESPACE = "crowdstrike"
_CONFIG_BASE_URL = f"{_CONFIG_NAMESPACE}.base_url"
_CONFIG_CLIENT_ID = f"{_CONFIG_NAMESPACE}.client_id"
_CONFIG_CLIENT_SECRET = f"{_CONFIG_NAMESPACE}.client_secret"
_CONFIG_INTERVAL_SEC = f"{_CONFIG_NAMESPACE}.interval_sec"
_CONFIG_SCOPES = f"{_CONFIG_NAMESPACE}.scopes"
_CONFIG_TLP = f"{_CONFIG_NAMESPACE}.tlp"
_CONFIG_CREATE_OBSERVABLES = f"{_CONFIG_NAMESPACE}.create_observables"
_CONFIG_CREATE_INDICATORS = f"{_CONFIG_NAMESPACE}.create_indicators"
_CONFIG_ACTOR_START_TIMESTAMP = f"{_CONFIG_NAMESPACE}.actor_start_timestamp"
_CONFIG_REPORT_START_TIMESTAMP = f"{_CONFIG_NAMESPACE}.report_start_timestamp"
_CONFIG_REPORT_INCLUDE_TYPES = f"{_CONFIG_NAMESPACE}.report_include_types"
_CONFIG_REPORT_STATUS = f"{_CONFIG_NAMESPACE}.report_status"
_CONFIG_REPORT_TYPE = f"{_CONFIG_NAMESPACE}.report_type"
_CONFIG_REPORT_GUESS_MALWARE = f"{_CONFIG_NAMESPACE}.report_guess_malware"
_CONFIG_INDICATOR_START_TIMESTAMP = f"{_CONFIG_NAMESPACE}.indicator_start_timestamp"
_CONFIG_INDICATOR_EXCLUDE_TYPES = f"{_CONFIG_NAMESPACE}.indicator_exclude_types"
_CONFIG_UPDATE_EXISTING_DATA = "connector.update_existing_data"
_CONFIG_SCOPE_ACTOR = "actor"
_CONFIG_SCOPE_REPORT = "report"
_CONFIG_SCOPE_INDICATOR = "indicator"
_CONFIG_SCOPE_YARA_MASTER = "yara_master"
_CONFIG_REPORT_STATUS_MAPPING = {
"new": 0,
"in progress": 1,
"analyzed": 2,
"closed": 3,
}
_DEFAULT_CREATE_OBSERVABLES = True
_DEFAULT_CREATE_INDICATORS = True
_DEFAULT_REPORT_TYPE = "threat-report"
_CONNECTOR_RUN_INTERVAL_SEC = 60
_STATE_LAST_RUN = "last_run"
def __init__(self) -> None:
"""Initialize CrowdStrike connector."""
config = self._read_configuration()
# CrowdStrike connector configuration
base_url = self._get_configuration(config, self._CONFIG_BASE_URL)
client_id = self._get_configuration(config, self._CONFIG_CLIENT_ID)
client_secret = self._get_configuration(config,
self._CONFIG_CLIENT_SECRET)
self.interval_sec = self._get_configuration(config,
self._CONFIG_INTERVAL_SEC,
is_number=True)
scopes_str = self._get_configuration(config, self._CONFIG_SCOPES)
scopes = set()
if scopes_str is not None:
scopes = set(convert_comma_separated_str_to_list(scopes_str))
tlp = self._get_configuration(config, self._CONFIG_TLP)
tlp_marking = self._convert_tlp_to_marking_definition(tlp)
create_observables = self._get_configuration(
config, self._CONFIG_CREATE_OBSERVABLES)
if create_observables is None:
create_observables = self._DEFAULT_CREATE_OBSERVABLES
else:
create_observables = bool(create_observables)
create_indicators = self._get_configuration(
config, self._CONFIG_CREATE_INDICATORS)
if create_indicators is None:
create_indicators = self._DEFAULT_CREATE_INDICATORS
else:
create_indicators = bool(create_indicators)
actor_start_timestamp = self._get_configuration(
config, self._CONFIG_ACTOR_START_TIMESTAMP, is_number=True)
if is_timestamp_in_future(actor_start_timestamp):
raise ValueError("Actor start timestamp is in the future")
report_start_timestamp = self._get_configuration(
config, self._CONFIG_REPORT_START_TIMESTAMP, is_number=True)
if is_timestamp_in_future(report_start_timestamp):
raise ValueError("Report start timestamp is in the future")
report_status_str = self._get_configuration(config,
self._CONFIG_REPORT_STATUS)
report_status = self._convert_report_status_str_to_report_status_int(
report_status_str)
report_type = self._get_configuration(config, self._CONFIG_REPORT_TYPE)
if not report_type:
report_type = self._DEFAULT_REPORT_TYPE
report_include_types_str = self._get_configuration(
config, self._CONFIG_REPORT_INCLUDE_TYPES)
report_include_types = []
if report_include_types_str is not None:
report_include_types = convert_comma_separated_str_to_list(
report_include_types_str)
report_guess_malware = bool(
self._get_configuration(config, self._CONFIG_REPORT_GUESS_MALWARE))
indicator_start_timestamp = self._get_configuration(
config, self._CONFIG_INDICATOR_START_TIMESTAMP, is_number=True)
if is_timestamp_in_future(indicator_start_timestamp):
raise ValueError("Indicator start timestamp is in the future")
indicator_exclude_types_str = self._get_configuration(
config, self._CONFIG_INDICATOR_EXCLUDE_TYPES)
indicator_exclude_types = []
if indicator_exclude_types_str is not None:
indicator_exclude_types = convert_comma_separated_str_to_list(
indicator_exclude_types_str)
update_existing_data = bool(
self._get_configuration(config, self._CONFIG_UPDATE_EXISTING_DATA))
author = self._create_author()
# Create OpenCTI connector helper.
self.helper = OpenCTIConnectorHelper(config)
# Create CrowdStrike client and importers.
client = CrowdStrikeClient(base_url, client_id, client_secret)
# Create importers.
importers: List[BaseImporter] = []
if self._CONFIG_SCOPE_ACTOR in scopes:
actor_importer = ActorImporter(
self.helper,
client.intel_api.actors,
update_existing_data,
author,
actor_start_timestamp,
tlp_marking,
)
importers.append(actor_importer)
if self._CONFIG_SCOPE_REPORT in scopes:
report_importer = ReportImporter(
self.helper,
client.intel_api.reports,
update_existing_data,
author,
report_start_timestamp,
tlp_marking,
report_include_types,
report_status,
report_type,
report_guess_malware,
)
importers.append(report_importer)
if self._CONFIG_SCOPE_INDICATOR in scopes:
indicator_importer = IndicatorImporter(
self.helper,
client.intel_api.indicators,
client.intel_api.reports,
update_existing_data,
author,
indicator_start_timestamp,
tlp_marking,
create_observables,
create_indicators,
indicator_exclude_types,
report_status,
report_type,
)
importers.append(indicator_importer)
if self._CONFIG_SCOPE_YARA_MASTER in scopes:
yara_master_importer = YaraMasterImporter(
self.helper,
client.intel_api.rules,
client.intel_api.reports,
author,
tlp_marking,
update_existing_data,
report_status,
report_type,
)
importers.append(yara_master_importer)
self.importers = importers
@staticmethod
def _read_configuration() -> Dict[str, str]:
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/../config.yml"
if not os.path.isfile(config_file_path):
return {}
return yaml.load(open(config_file_path), Loader=yaml.FullLoader)
@staticmethod
def _create_author() -> Identity:
return create_organization("CrowdStrike")
@staticmethod
def _get_yaml_path(config_name: str) -> List[str]:
return config_name.split(".")
@staticmethod
def _get_environment_variable_name(yaml_path: List[str]) -> str:
return "_".join(yaml_path).upper()
@classmethod
def _get_configuration(cls,
config: Dict[str, Any],
config_name: str,
is_number: bool = False) -> Any:
yaml_path = cls._get_yaml_path(config_name)
env_var_name = cls._get_environment_variable_name(yaml_path)
config_value = get_config_variable(env_var_name,
yaml_path,
config,
isNumber=is_number)
return config_value
@classmethod
def _convert_tlp_to_marking_definition(
cls, tlp_value: Optional[str]) -> MarkingDefinition:
if tlp_value is None:
return DEFAULT_TLP_MARKING_DEFINITION
return get_tlp_string_marking_definition(tlp_value)
@classmethod
def _convert_report_status_str_to_report_status_int(
cls, report_status: str) -> int:
return cls._CONFIG_REPORT_STATUS_MAPPING[report_status.lower()]
def _load_state(self) -> Dict[str, Any]:
current_state = self.helper.get_state()
if not current_state:
return {}
return current_state
@staticmethod
def _get_state_value(state: Optional[Mapping[str, Any]],
key: str,
default: Optional[Any] = None) -> Any:
if state is not None:
return state.get(key, default)
return default
@classmethod
def _sleep(cls, delay_sec: Optional[int] = None) -> None:
sleep_delay = (delay_sec if delay_sec is not None else
cls._CONNECTOR_RUN_INTERVAL_SEC)
time.sleep(sleep_delay)
def _is_scheduled(self, last_run: Optional[int],
current_time: int) -> bool:
if last_run is None:
self._info("CrowdStrike connector clean run")
return True
time_diff = current_time - last_run
return time_diff >= self._get_interval()
@staticmethod
def _current_unix_timestamp() -> int:
return int(time.time())
def run(self):
"""Run CrowdStrike connector."""
self._info("Starting CrowdStrike connector...")
if not self.importers:
self._error("Scope(s) not configured.")
return
while True:
self._info("Running CrowdStrike connector...")
run_interval = self._CONNECTOR_RUN_INTERVAL_SEC
try:
timestamp = self._current_unix_timestamp()
current_state = self._load_state()
self.helper.log_info(f"Loaded state: {current_state}")
last_run = self._get_state_value(current_state,
self._STATE_LAST_RUN)
if self._is_scheduled(last_run, timestamp):
work_id = self._initiate_work(timestamp)
new_state = current_state.copy()
for importer in self.importers:
importer_state = importer.start(work_id, current_state)
new_state.update(importer_state)
new_state[
self._STATE_LAST_RUN] = self._current_unix_timestamp()
self._info("Storing new state: {0}", new_state)
self.helper.set_state(new_state)
message = (
f"State stored, next run in: {self._get_interval()} seconds"
)
self._info(message)
self._complete_work(work_id, message)
else:
next_run = self._get_interval() - (timestamp - last_run)
run_interval = min(run_interval, next_run)
self._info(
"Connector will not run, next run in: {0} seconds",
next_run)
self._sleep(delay_sec=run_interval)
except (KeyboardInterrupt, SystemExit):
self._info("CrowdStrike connector stopping...")
exit(0)
except Exception as e: # noqa: B902
self._error("CrowdStrike connector internal error: {0}",
str(e))
self._sleep()
def _initiate_work(self, timestamp: int) -> str:
datetime_str = timestamp_to_datetime(timestamp)
friendly_name = f"{self.helper.connect_name} @ {datetime_str}"
work_id = self.helper.api.work.initiate_work(self.helper.connect_id,
friendly_name)
self._info("New work '{0}' initiated", work_id)
return work_id
def _complete_work(self, work_id: str, message: str) -> None:
self.helper.api.work.to_processed(work_id, message)
def _get_interval(self) -> int:
return int(self.interval_sec)
def _info(self, msg: str, *args: Any) -> None:
fmt_msg = msg.format(*args)
self.helper.log_info(fmt_msg)
def _error(self, msg: str, *args: Any) -> None:
fmt_msg = msg.format(*args)
self.helper.log_error(fmt_msg)
class Cybercrimetracker:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = "{}/config.yml".format(
os.path.dirname(os.path.abspath(__file__))
)
config = (
yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path)
else {}
)
self.helper = OpenCTIConnectorHelper(config)
# Connector Config
self.confidence_level = get_config_variable(
"CONNECTOR_CONFIDENCE_LEVEL",
["connector", "confidence_level"],
config,
isNumber=True,
)
self.update_data = get_config_variable(
"CONNECTOR_UPDATE_EXISTING_DATA",
["connector", "update_existing_data"],
config,
)
# CYBERCRIME-TRACKER.NET Config
self.feed_url = get_config_variable(
"CYBERCRIMET_RACKER_FEED_URL", ["cybercrime-tracker", "feed_url"], config,
)
self.connector_tlp = get_config_variable(
"CYBERCRIME_TRACKER_TLP", ["cybercrime-tracker", "tlp"], config,
)
self.interval = get_config_variable(
"CYBERCRIMETRACKER_INTERVAL",
["cybercrime-tracker", "interval"],
config,
isNumber=True,
)
@staticmethod
def _time_to_datetime(input_date: time) -> datetime:
return datetime(
input_date.tm_year,
input_date.tm_mon,
input_date.tm_mday,
input_date.tm_hour,
input_date.tm_min,
input_date.tm_sec,
tzinfo=timezone.utc,
).isoformat()
def parse_feed_entry(self, entry):
"""
Parses an entry from the feed and returns a dict with:
date: date in iso format
type: name of the malware associated with the C2 server
url: the url of the C2
ip: the IP address of the C2
ext_link: An external link to CYBERCRIME-TRACKER.NET with details
Note: CYBERCRIME-TRACKER.NET does not provide the protocol in the url
as such we always assume 'http'.
"""
parsed_entry = {}
pattern = (
r"(?:\[%{GREEDYDATA:cwhqid}\]\s+Type:\s+%{GREEDYDATA:type}"
+ r"\s+-%{GREEDYDATA}:\s+%{IP:ip}|"
+ r"\[%{GREEDYDATA:cwhqid}\]\s+Type:\s+%{GREEDYDATA:type})"
)
entry_summary = Grok(pattern).match(entry["summary"])
if entry_summary:
parsed_entry["date"] = self._time_to_datetime(entry["published_parsed"])
parsed_entry["type"] = entry_summary["type"]
parsed_entry["ext_link"] = entry["link"]
parsed_entry["url"] = "http://{}".format(quote(entry["title"]))
hostname = urlparse(parsed_entry["url"]).hostname
if entry_summary["ip"] is None:
parsed_entry["ip"] = hostname
else:
parsed_entry["ip"] = entry_summary["ip"]
parsed_entry["domain"] = hostname
self.helper.log_info("Parsed entry: {}".format(entry["title"]))
return parsed_entry
else:
self.helper.log_error("Could not parse: {}".format(entry["title"]))
return False
def gen_indicator_pattern(self, parsed_entry):
if "domain" in parsed_entry.keys():
indicator_pattern = (
"[ipv4-addr:value='{}'] ".format(parsed_entry["ip"])
+ "AND [url:value='{}'] ".format(parsed_entry["url"])
+ "AND [domain:value='{}']".format(parsed_entry["domain"])
)
else:
indicator_pattern = "[ipv4-addr:value='{}'] ".format(
parsed_entry["ip"]
) + "AND [url:value='{}']".format(parsed_entry["url"])
return indicator_pattern
def run(self):
self.helper.log_info("Fetching data CYBERCRIME-TRACKER.NET...")
tag = self.helper.api.tag.create(
tag_type="C2-Type", value="C2 Server", color="#fc236b",
)
tlp = self.helper.api.marking_definition.read(
filters=[
{"key": "definition", "values": "TLP:{}".format(self.connector_tlp)}
]
)
while True:
try:
# Get the current timestamp and check
timestamp = int(time.time())
current_state = self.helper.get_state()
if current_state is not None and "last_run" in current_state:
last_run = current_state["last_run"]
self.helper.log_info(
"Connector last run: {}".format(
datetime.utcfromtimestamp(last_run).strftime(
"%Y-%m-%d %H:%M:%S"
)
)
)
else:
last_run = None
self.helper.log_info("Connector has never run")
# Run if it is the first time or we are past the interval
if last_run is None or ((timestamp - last_run) > self.interval):
self.helper.log_info("Connector will run!")
# Get Feed Content
feed = feedparser.parse(self.feed_url)
self.helper.log_info(
"Found: {} entries.".format(len(feed["entries"]))
)
self.feed_summary = {
"Source": feed["feed"]["title"],
"Date": self._time_to_datetime(
feed["feed"]["published_parsed"]
),
"Details": feed["feed"]["subtitle"],
"Link": feed["feed"]["link"],
}
# Create entity for the feed.
organization = self.helper.api.identity.create(
type="Organization",
name="CYBERCRIME-TRACKER.NET",
description="Tracker collecting and sharing \
daily updates of C2 IPs/Urls. \
http://cybercrime-tracker.net",
)
for entry in feed["entries"]:
parsed_entry = self.parse_feed_entry(entry)
ext_reference = self.helper.api.external_reference.create(
source_name="{}".format(self.feed_summary["Source"],),
url=parsed_entry["ext_link"],
)
indicator_pattern = self.gen_indicator_pattern(parsed_entry)
# Add malware related to indicator
malware = self.helper.api.malware.create(
name=parsed_entry["type"],
description="{} malware.".format(parsed_entry["type"]),
)
# Add indicator
indicator = self.helper.api.indicator.create(
name=parsed_entry["url"],
description="C2 URL for: {}".format(parsed_entry["type"]),
pattern_type="stix",
indicator_pattern=indicator_pattern,
main_observable_type="URL",
valid_from=parsed_entry["date"],
created=parsed_entry["date"],
modified=parsed_entry["date"],
createdByRef=organization["id"],
markingDefinitions=[tlp["id"]],
update=self.update_data,
)
# Add tag
self.helper.api.stix_entity.add_tag(
id=indicator["id"], tag_id=tag["id"],
)
self.helper.api.stix_entity.add_external_reference(
id=indicator["id"],
external_reference_id=ext_reference["id"],
)
# Add relationship with malware
relation = self.helper.api.stix_relation.create(
fromType="Indicator",
fromId=indicator["id"],
toType="Malware",
toId=malware["id"],
relationship_type="indicates",
first_seen=self._time_to_datetime(
entry["published_parsed"]
),
last_seen=self._time_to_datetime(entry["published_parsed"]),
description="URLs associated to: " + parsed_entry["type"],
weight=self.confidence_level,
role_played="C2 Server",
createdByRef=organization["id"],
created=parsed_entry["date"],
modified=parsed_entry["date"],
update=self.update_data,
)
self.helper.api.stix_entity.add_external_reference(
id=relation["id"],
external_reference_id=ext_reference["id"],
)
# Create Observables and link them to Indicator
observable_url = self.helper.api.stix_observable.create(
type="URL",
observable_value=parsed_entry["url"],
createdByRef=organization["id"],
markingDefinitions=[tlp["id"]],
update=self.update_data,
)
self.helper.api.stix_entity.add_external_reference(
id=observable_url["id"],
external_reference_id=ext_reference["id"],
)
self.helper.api.indicator.add_stix_observable(
id=indicator["id"], stix_observable_id=observable_url["id"],
)
observable_ip = self.helper.api.stix_observable.create(
type="IPv4-Addr",
observable_value=parsed_entry["ip"],
createdByRef=organization["id"],
markingDefinitions=[tlp["id"]],
update=self.update_data,
)
self.helper.api.stix_entity.add_external_reference(
id=observable_ip["id"],
external_reference_id=ext_reference["id"],
)
self.helper.api.indicator.add_stix_observable(
id=indicator["id"], stix_observable_id=observable_ip["id"],
)
if "domain" in parsed_entry.keys():
observable_domain = self.helper.api.stix_observable.create(
type="Domain",
observable_value=parsed_entry["domain"],
createdByRef=organization["id"],
markingDefinitions=[tlp["id"]],
update=self.update_data,
)
self.helper.api.stix_entity.add_external_reference(
id=observable_domain["id"],
external_reference_id=ext_reference["id"],
)
self.helper.api.indicator.add_stix_observable(
id=indicator["id"],
stix_observable_id=observable_domain["id"],
)
self.helper.api.stix_relation.create(
fromType="Domain",
fromId=observable_domain["id"],
toType="IPv4-Addr",
toId=observable_ip["id"],
relationship_type="resolves",
last_seen=self._time_to_datetime(
entry["published_parsed"]
),
weight=self.confidence_level,
createdByRef=organization["id"],
created=parsed_entry["date"],
modified=parsed_entry["date"],
update=self.update_data,
)
# Store the current timestamp as a last run
self.helper.log_info(
"Connector successfully run, \
storing last_run as: {}".format(
str(timestamp)
)
)
self.helper.set_state({"last_run": timestamp})
self.helper.log_info(
"Last_run stored, next run in: {} seconds.".format(
str(round(self.interval, 2))
)
)
new_state = {"last_run": int(time.time())}
self.helper.set_state(new_state)
time.sleep(60)
else:
new_interval = self.interval - (timestamp - last_run)
self.helper.log_info(
"Connector will not run. \
Next run in: {} seconds.".format(
str(round(new_interval, 2))
)
)
time.sleep(60)
except (KeyboardInterrupt, SystemExit):
self.helper.log_info("Connector stop")
exit(0)
except Exception as e:
self.helper.log_error(str(e))
time.sleep(60)
class FireEye:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.SafeLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
# Extra config
self.fireeye_api_url = get_config_variable("FIREEYE_API_URL",
["fireeye", "api_url"],
config)
self.fireeye_api_v3_public = get_config_variable(
"FIREEYE_API_V3_PUBLIC", ["fireeye", "api_v3_public"], config)
self.fireeye_api_v3_secret = get_config_variable(
"FIREEYE_API_V3_SECRET", ["fireeye", "api_v3_secret"], config)
self.fireeye_collections = get_config_variable(
"FIREEYE_COLLECTIONS", ["fireeye", "collections"],
config).split(",")
self.fireeye_import_start_date = get_config_variable(
"FIREEYE_IMPORT_START_DATE",
["fireeye", "import_start_date"],
config,
)
self.fireeye_interval = get_config_variable("FIREEYE_INTERVAL",
["fireeye", "interval"],
config, True)
self.update_existing_data = get_config_variable(
"CONNECTOR_UPDATE_EXISTING_DATA",
["connector", "update_existing_data"],
config,
)
self.added_after = parse(self.fireeye_import_start_date).timestamp()
self.identity = self.helper.api.identity.create(
type="Organization",
name="FireEye, Inc.",
description=
"FireEye is a publicly traded cybersecurity company headquartered in Milpitas, California. It has been involved in the detection and prevention of major cyber attacks. It provides hardware, software, and services to investigate cybersecurity attacks, protect against malicious software, and analyze IT security risks. FireEye was founded in 2004.",
)
self.marking = self.helper.api.marking_definition.create(
definition_type="COMMERCIAL",
definition="FIREEYE",
x_opencti_order=99,
x_opencti_color="#a01526",
)
# Init variables
self.auth_token = None
self._get_token()
def get_interval(self):
return int(self.fireeye_interval) * 60
def _get_token(self):
r = requests.post(
self.fireeye_api_url + "/token",
auth=HTTPBasicAuth(self.fireeye_api_v3_public,
self.fireeye_api_v3_secret),
data={"grant_type": "client_credentials"},
)
if r.status_code != 200:
raise ValueError("FireEye Authentication failed")
data = r.json()
self.auth_token = data.get("access_token")
def _search(self, stix_id, retry=False):
self.helper.log_info("Searching for " + stix_id)
headers = {
"authorization": "Bearer " + self.auth_token,
"accept": "application/vnd.oasis.stix+json; version=2.1",
"x-app-name": "opencti-connector-4.2.2",
}
body = """
{
"queries": [
{
"type": "ENTITY_TYPE",
"query": "id = 'ENTITY_ID'"
}
],
"include_connected_objects": false
}
"""
entity_type = stix_id.split("--")[0]
if entity_type not in searchable_types:
return None
body = body.replace("ENTITY_TYPE",
entity_type).replace("ENTITY_ID", stix_id)
r = requests.post(self.fireeye_api_url + "/collections/search",
data=body,
headers=headers)
if r.status_code == 200:
return r
elif (r.status_code == 401 or r.status_code == 403) and not retry:
self._get_token()
return self._search(stix_id, True)
elif r.status_code == 204 or r.status_code == 205:
return None
elif r.status_code == 401 or r.status_code == 403:
raise ValueError("Query failed, permission denied")
else:
print(r)
raise ValueError("An unknown error occurred")
def _query(self, url, retry=False):
headers = {
"authorization": "Bearer " + self.auth_token,
"accept": "application/vnd.oasis.stix+json; version=2.1",
"x-app-name": "opencti-connector-4.2.2",
}
r = requests.get(url, headers=headers)
if r.status_code == 200:
return r
elif (r.status_code == 401 or r.status_code == 403) and not retry:
self._get_token()
return self._query(url, True)
elif r.status_code == 401 or r.status_code == 403:
raise ValueError("Query failed, permission denied")
else:
raise ValueError("An unknown error occurred")
def _send_entity(self, bundle, work_id):
if "objects" in bundle and len(bundle) > 0:
final_objects = []
for stix_object in bundle["objects"]:
if stix_object["type"] == "threat-actor":
stix_object["type"] = "intrusion-set"
stix_object["id"] = stix_object["id"].replace(
"threat-actor", "intrusion-set")
if "created_by_ref" not in stix_object:
stix_object["created_by_ref"] = self.identity[
"standard_id"]
if stix_object["type"] != "marking-definition":
stix_object["object_marking_refs"] = [
"marking-definition--f88d31f6-486f-44da-b317-01333bde0b82"
]
stix_object["object_marking_refs"].append(
self.marking["standard_id"])
final_objects.append(stix_object)
final_bundle = {"type": "bundle", "objects": final_objects}
self.helper.send_stix2_bundle(
json.dumps(final_bundle),
update=self.update_existing_data,
work_id=work_id,
)
def _import_collection(self,
collection,
last_id_modified_timestamp=None,
last_id=None,
work_id=None):
have_next_page = True
url = None
last_object = None
while have_next_page:
if url is None:
if last_id_modified_timestamp is not None:
url = (self.fireeye_api_url + "/collections/" +
collection + "/objects" + "?added_after=" +
str(self.added_after) + "&length=100" +
"&last_id_modified_timestamp=" +
str(last_id_modified_timestamp))
else:
url = (self.fireeye_api_url + "/collections/" +
collection + "/objects" + "?added_after=" +
str(self.added_after) + "&length=100")
result = self._query(url)
parsed_result = json.loads(result.text)
if "objects" in parsed_result and len(parsed_result) > 0:
last_object = parsed_result["objects"][-1]
object_ids = [
stix_object["id"]
for stix_object in parsed_result["objects"]
]
if last_object["id"] != last_id:
final_objects = []
for stix_object in parsed_result["objects"]:
if stix_object["type"] == "threat-actor":
stix_object["type"] = "intrusion-set"
stix_object["id"] = stix_object["id"].replace(
"threat-actor", "intrusion-set")
if stix_object["type"] == "relationship":
# If the source_ref is not in the current bundle
if stix_object["source_ref"] not in object_ids:
# Search entity in OpenCTI
opencti_entity = (
self.helper.api.stix_domain_object.read(
id=stix_object["source_ref"]))
# If the entity is not found
if opencti_entity is None:
# Search the entity in FireEye
fireeye_entity = self._search(
stix_object["source_ref"])
# If the entity is found
if fireeye_entity is not None:
fireeye_entity_decoded = json.loads(
fireeye_entity.text)
# Send the entity before this bundle
self._send_entity(
fireeye_entity_decoded, work_id)
stix_object["source_ref"] = stix_object[
"source_ref"].replace("threat-actor",
"intrusion-set")
# Search if the entity is not in bundle
if stix_object["target_ref"] not in object_ids:
opencti_entity = (
self.helper.api.stix_domain_object.read(
id=stix_object["target_ref"]))
if opencti_entity is None:
fireeye_entity = self._search(
stix_object["target_ref"])
if fireeye_entity is not None:
fireeye_entity_decoded = json.loads(
fireeye_entity.text)
self._send_entity(
fireeye_entity_decoded, work_id)
stix_object["target_ref"] = stix_object[
"target_ref"].replace("threat-actor",
"intrusion-set")
if ("object_refs" in stix_object
and len(stix_object["object_refs"]) > 0):
for object_ref in stix_object["object_refs"]:
if object_ref not in object_ids:
opencti_entity = (self.helper.api.
stix_domain_object.read(
id=object_ref))
if opencti_entity is None:
fireeye_entity = self._search(
object_ref)
if fireeye_entity is not None:
fireeye_entity_decoded = json.loads(
fireeye_entity.text)
self._send_entity(
fireeye_entity_decoded,
work_id)
if "created_by_ref" not in stix_object:
stix_object["created_by_ref"] = self.identity[
"standard_id"]
if stix_object["type"] != "marking-definition":
stix_object["object_marking_refs"] = [
"marking-definition--f88d31f6-486f-44da-b317-01333bde0b82"
]
stix_object["object_marking_refs"].append(
self.marking["standard_id"])
final_objects.append(stix_object)
final_bundle = {"type": "bundle", "objects": final_objects}
self.helper.send_stix2_bundle(
json.dumps(final_bundle),
update=self.update_existing_data,
work_id=work_id,
)
headers = result.headers
if "Link" in headers:
have_next_page = True
link = headers["Link"].split(";")
url = link[0][1:-1]
last_id_modified_timestamp = parse_qs(
urlparse(
url).query)["last_id_modified_timestamp"][0]
else:
have_next_page = False
else:
have_next_page = False
return {
"last_id_modified_timestamp": last_id_modified_timestamp,
"last_id": last_object["id"] if "id" in last_object else None,
}
def run(self):
while True:
try:
self.helper.log_info("Synchronizing with FireEye API...")
timestamp = int(time.time())
now = datetime.datetime.utcfromtimestamp(timestamp)
friendly_name = "FireEye run @ " + now.strftime(
"%Y-%m-%d %H:%M:%S")
work_id = self.helper.api.work.initiate_work(
self.helper.connect_id, friendly_name)
current_state = self.helper.get_state()
if (current_state is None
or "last_id_modified_timestamp" not in current_state):
self.helper.set_state({
"last_id_modified_timestamp": {
"indicators": None,
"reports": None,
},
"last_id": {
"indicators": None,
"reports": None,
},
})
current_state = self.helper.get_state()
last_id_modified_timestamp = current_state[
"last_id_modified_timestamp"]
last_id = current_state["last_id"]
if "indicators" in self.fireeye_collections:
self.helper.log_info(
"Get indicators created after " +
str(last_id_modified_timestamp["indicators"]))
indicators_last = self._import_collection(
"indicators",
last_id_modified_timestamp["indicators"],
last_id["indicators"],
work_id,
)
current_state = self.helper.get_state()
self.helper.set_state({
"last_id_modified_timestamp": {
"indicators":
indicators_last["last_id_modified_timestamp"],
"reports":
current_state["last_id_modified_timestamp"]
["reports"],
},
"last_id": {
"indicators": indicators_last["last_id"],
"reports": current_state["last_id"]["reports"],
},
})
if "reports" in self.fireeye_collections:
self.helper.log_info(
"Get reports created after " +
str(last_id_modified_timestamp["reports"]))
reports_last = self._import_collection(
"reports",
last_id_modified_timestamp["reports"],
last_id["reports"],
work_id,
)
current_state = self.helper.get_state()
self.helper.set_state({
"last_id_modified_timestamp": {
"indicators":
current_state["last_id_modified_timestamp"]
["indicators"],
"reports":
reports_last["last_id_modified_timestamp"],
},
"last_id": {
"indicators":
current_state["last_id"]["indicators"],
"reports": reports_last["last_id"],
},
})
message = "End of synchronization"
self.helper.api.work.to_processed(work_id, message)
self.helper.log_info(message)
time.sleep(self.get_interval())
except (KeyboardInterrupt, SystemExit):
self.helper.log_info("Connector stop")
exit(0)
except Exception as e:
self.helper.log_error(str(e))
time.sleep(60)
class VXVault:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
# Extra config
self.vxvault_url = get_config_variable("VXVAULT_URL",
["vxvault", "url"], config)
self.vxvault_interval = get_config_variable("VXVAULT_INTERVAL",
["vxvault", "interval"],
config, True)
self.create_indicators = get_config_variable(
"VXVAULT_CREATE_INDICATORS", ["vxvault", "create_indicators"],
config, True)
self.update_existing_data = get_config_variable(
"CONNECTOR_UPDATE_EXISTING_DATA",
["connector", "update_existing_data"],
config,
)
self.identity = self.helper.api.identity.create(
type="Organization",
name="VX Vault",
description=
"VX Vault is providing URLs of potential malicious payload.",
)
def get_interval(self):
return int(self.vxvault_interval) * 60 * 60 * 24
def next_run(self, seconds):
return
def run(self):
self.helper.log_info("Fetching VXVault dataset...")
while True:
try:
# Get the current timestamp and check
timestamp = int(time.time())
current_state = self.helper.get_state()
if current_state is not None and "last_run" in current_state:
last_run = current_state["last_run"]
self.helper.log_info("Connector last run: " +
datetime.utcfromtimestamp(last_run).
strftime("%Y-%m-%d %H:%M:%S"))
else:
last_run = None
self.helper.log_info("Connector has never run")
# If the last_run is more than interval-1 day
if last_run is None or ((timestamp - last_run) > (
(int(self.vxvault_interval) - 1) * 60 * 60 * 24)):
self.helper.log_info("Connector will run!")
now = datetime.utcfromtimestamp(timestamp)
friendly_name = "VXVault run @ " + now.strftime(
"%Y-%m-%d %H:%M:%S")
work_id = self.helper.api.work.initiate_work(
self.helper.connect_id, friendly_name)
try:
response = urllib.request.urlopen(
self.vxvault_url,
context=ssl.create_default_context(
cafile=certifi.where()),
)
image = response.read()
with open(
os.path.dirname(os.path.abspath(__file__)) +
"/data.txt",
"wb",
) as file:
file.write(image)
count = 0
bundle_objects = []
with open(
os.path.dirname(os.path.abspath(__file__)) +
"/data.txt") as fp:
for line in fp:
count += 1
if count <= 3:
continue
external_reference = ExternalReference(
source_name="VX Vault",
url="http://vxvault.net",
description="VX Vault repository URL",
)
stix_observable = SimpleObservable(
id=OpenCTIStix2Utils.
generate_random_stix_id(
"x-opencti-simple-observable"),
key="Url.value",
value=line,
description="VX Vault URL",
x_opencti_score=80,
object_marking_refs=[TLP_WHITE],
created_by_ref=self.
identity["standard_id"],
x_opencti_create_indicator=self.
create_indicators,
external_references=[external_reference],
)
bundle_objects.append(stix_observable)
bundle = Bundle(objects=bundle_objects).serialize()
self.helper.send_stix2_bundle(
bundle,
update=self.update_existing_data,
work_id=work_id,
)
if os.path.exists(
os.path.dirname(os.path.abspath(__file__)) +
"/data.txt"):
os.remove(
os.path.dirname(os.path.abspath(__file__)) +
"/data.txt")
except Exception as e:
self.helper.log_error(str(e))
# Store the current timestamp as a last run
message = "Connector successfully run, storing last_run as " + str(
timestamp)
self.helper.log_info(message)
self.helper.set_state({"last_run": timestamp})
self.helper.api.work.to_processed(work_id, message)
self.helper.log_info(
"Last_run stored, next run in: " +
str(round(self.get_interval() / 60 / 60 / 24, 2)) +
" days")
time.sleep(60)
else:
new_interval = self.get_interval() - (timestamp - last_run)
self.helper.log_info(
"Connector will not run, next run in: " +
str(round(new_interval / 60 / 60 / 24, 2)) + " days")
time.sleep(60)
except (KeyboardInterrupt, SystemExit):
self.helper.log_info("Connector stop")
exit(0)
except Exception as e:
self.helper.log_error(str(e))
time.sleep(60)
class Amitt:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.SafeLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
# Extra config
self.amitt_file_url = get_config_variable("AMITT_FILE_URL",
["amitt", "amitt_file_url"],
config)
self.pre_amitt_file_url = get_config_variable(
"PRE_AMITT_FILE_URL", ["amitt", "pre_amitt_file_url"], config)
self.amitt_interval = get_config_variable("AMITT_INTERVAL",
["amitt", "interval"],
config, True)
self.update_existing_data = get_config_variable(
"CONNECTOR_UPDATE_EXISTING_DATA",
["connector", "update_existing_data"],
config,
)
def get_interval(self):
return int(self.amitt_interval) * 60 * 60 * 24
def run(self):
self.helper.log_info("Fetching AMITT datasets...")
while True:
try:
# Get the current timestamp and check
timestamp = int(time.time())
current_state = self.helper.get_state()
if current_state is not None and "last_run" in current_state:
last_run = current_state["last_run"]
self.helper.log_info("Connector last run: " +
datetime.utcfromtimestamp(last_run).
strftime("%Y-%m-%d %H:%M:%S"))
else:
last_run = None
self.helper.log_info("Connector has never run")
# If the last_run is more than interval-1 day
if last_run is None or ((timestamp - last_run) > (
(int(self.amitt_interval) - 1) * 60 * 60 * 24)):
self.helper.log_info("Connector will run!")
amitt_data = (urllib.request.urlopen(
self.amitt_file_url).read().decode("utf-8"))
self.helper.send_stix2_bundle(
amitt_data,
entities_types=self.helper.connect_scope,
update=self.update_existing_data,
)
pre_amitt_data = urllib.request.urlopen(
self.pre_amitt_file_url).read()
self.helper.send_stix2_bundle(
pre_amitt_data.decode("utf-8"),
entities_types=self.helper.connect_scope,
update=self.update_existing_data,
)
# Store the current timestamp as a last run
self.helper.log_info(
"Connector successfully run, storing last_run as " +
str(timestamp))
self.helper.set_state({"last_run": timestamp})
self.helper.log_info(
"Last_run stored, next run in: " +
str(round(self.get_interval() / 60 / 60 / 24, 2)) +
" days")
time.sleep(60)
else:
new_interval = self.get_interval() - (timestamp - last_run)
self.helper.log_info(
"Connector will not run, next run in: " +
str(round(new_interval / 60 / 60 / 24, 2)) + " days")
time.sleep(60)
except (KeyboardInterrupt, SystemExit):
self.helper.log_info("Connector stop")
exit(0)
except Exception as e:
self.helper.log_error(str(e))
time.sleep(60)
class Amitt:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + '/config.yml'
config = yaml.load(open(config_file_path), Loader=yaml.FullLoader
) if os.path.isfile(config_file_path) else {}
self.helper = OpenCTIConnectorHelper(config)
# Extra config
self.amitt_file_url = get_config_variable('AMITT_FILE_URL',
['amitt', 'amitt_file_url'],
config)
self.pre_amitt_file_url = get_config_variable(
'PRE_AMITT_FILE_URL', ['amitt', 'pre_amitt_file_url'], config)
self.amitt_interval = get_config_variable('AMITT_INTERVAL',
['amitt', 'interval'],
config, True)
self.update_existing_data = get_config_variable(
'CONNECTOR_UPDATE_EXISTING_DATA',
['connector', 'update_existing_data'], config)
def get_interval(self):
return int(self.amitt_interval) * 60 * 60 * 24
def run(self):
self.helper.log_info('Fetching AMITT datasets...')
while True:
try:
# Get the current timestamp and check
timestamp = int(time.time())
current_state = self.helper.get_state()
if current_state is not None and 'last_run' in current_state:
last_run = current_state['last_run']
self.helper.log_info('Connector last run: ' +
datetime.utcfromtimestamp(last_run).
strftime('%Y-%m-%d %H:%M:%S'))
else:
last_run = None
self.helper.log_info('Connector has never run')
# If the last_run is more than interval-1 day
if last_run is None or ((timestamp - last_run) > (
(int(self.amitt_interval) - 1) * 60 * 60 * 24)):
self.helper.log_info('Connector will run!')
amitt_data = urllib.request.urlopen(
self.amitt_file_url).read().decode('utf-8')
self.helper.send_stix2_bundle(amitt_data,
self.helper.connect_scope,
self.update_existing_data)
pre_amitt_data = urllib.request.urlopen(
self.pre_amitt_file_url).read()
self.helper.send_stix2_bundle(
pre_amitt_data.decode('utf-8'),
self.helper.connect_scope, self.update_existing_data)
# Store the current timestamp as a last run
self.helper.log_info(
'Connector successfully run, storing last_run as ' +
str(timestamp))
self.helper.set_state({'last_run': timestamp})
self.helper.log_info(
'Last_run stored, next run in: ' +
str(round(self.get_interval() / 60 / 60 / 24, 2)) +
' days')
time.sleep(60)
else:
new_interval = self.get_interval() - (timestamp - last_run)
self.helper.log_info(
'Connector will not run, next run in: ' +
str(round(new_interval / 60 / 60 / 24, 2)) + ' days')
time.sleep(60)
except (KeyboardInterrupt, SystemExit):
self.helper.log_info('Connector stop')
exit(0)
except Exception as e:
self.helper.log_error(str(e))
time.sleep(60)
class TaniumConnector:
def __init__(self):
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
# Extra config
self.tanium_url = get_config_variable("TANIUM_URL", ["tanium", "url"],
config)
self.tanium_ssl_verify = get_config_variable("TANIUM_SSL_VERIFY",
["tanium", "ssl_verify"],
config, False, True)
self.tanium_login = get_config_variable("TANIUM_LOGIN",
["tanium", "login"], config)
self.tanium_password = get_config_variable("TANIUM_PASSWORD",
["tanium", "password"],
config)
self.tanium_indicator_types = get_config_variable(
"TANIUM_INDICATOR_TYPES", ["tanium", "indicator_types"],
config).split(",")
self.tanium_observable_types = get_config_variable(
"TANIUM_OBSERVABLE_TYPES", ["tanium", "observable_types"],
config).split(",")
self.tanium_import_label = get_config_variable(
"TANIUM_IMPORT_LABEL", ["tanium", "import_label"], config, False,
"")
self.tanium_import_from_date = get_config_variable(
"TANIUM_IMPORT_FROM_DATE", ["tanium", "import_from_date"], config)
self.tanium_reputation_blacklist_label = get_config_variable(
"TANIUM_REPUTATION_BLACKLIST_LABEL",
["tanium", "reputation_blacklist_label"],
config,
False,
"",
)
self.tanium_auto_quickscan = get_config_variable(
"TANIUM_AUTO_QUICKSCAN", ["tanium", "auto_quickscan"], config,
False, False)
self.tanium_computer_groups = get_config_variable(
"TANIUM_COMPUTER_GROUPS", ["tanium", "computer_groups"], config,
False, "").split(",")
# Variables
self.session = None
# Open a session
self._get_session()
# Create the state
if self.tanium_import_from_date:
timestamp = (parse(self.tanium_import_from_date).timestamp() *
1000 if self.tanium_import_from_date != "now" else
int(round(time.time() * 1000)) - 1000)
current_state = self.helper.get_state()
if current_state is None:
self.helper.set_state({"connectorLastEventId": timestamp})
# Create the source if not exist
self.source_id = None
sources = self._query("get", "/plugin/products/detect3/api/v1/sources")
for source in sources:
if source["name"] == "OpenCTI":
self.source_id = source["id"]
if self.source_id is None:
source = self._query(
"post",
"/plugin/products/detect3/api/v1/sources",
{
"type": "api-client",
"name": "OpenCTI",
"description":
"Cyber Threat Intelligence knowledge imported from OpenCTI.",
"canAutoQuickScan": True,
},
)
self.source_id = source["id"]
def _get_session(self):
payload = {
"username": self.tanium_login,
"password": self.tanium_password,
}
r = requests.post(
self.tanium_url + "/api/v2/session/login",
json=payload,
verify=self.tanium_ssl_verify,
)
if r.status_code == 200:
result = r.json()
self.session = result["data"]["session"]
else:
raise ValueError("Cannot login to the Tanium API")
def _query(
self,
method,
uri,
payload=None,
content_type="application/json",
type=None,
retry=False,
):
self.helper.log_info("Query " + method + " on " + uri)
headers = {"session": self.session}
if method != "upload":
headers["content-type"] = content_type
if type is not None:
headers["type"] = type
if content_type == "application/octet-stream":
headers["content-disposition"] = ("attachment; filename=" +
payload["filename"])
if "name" in payload:
headers["name"] = payload["name"]
if "description" in payload:
headers["description"] = payload["description"]
if method == "get":
r = requests.get(
self.tanium_url + uri,
headers=headers,
params=payload,
verify=self.tanium_ssl_verify,
)
elif method == "post":
if content_type == "application/octet-stream":
r = requests.post(
self.tanium_url + uri,
headers=headers,
data=payload["document"],
verify=self.tanium_ssl_verify,
)
elif type is not None:
r = requests.post(
self.tanium_url + uri,
headers=headers,
data=payload["intelDoc"],
verify=self.tanium_ssl_verify,
)
else:
r = requests.post(
self.tanium_url + uri,
headers=headers,
json=payload,
verify=self.tanium_ssl_verify,
)
elif method == "upload":
f = open(payload["filename"], "w")
f.write(payload["content"])
f.close()
files = {"hash": open(payload["filename"], "rb")}
r = requests.post(
self.tanium_url + uri,
headers=headers,
files=files,
verify=self.tanium_ssl_verify,
)
elif method == "put":
if content_type == "application/xml":
r = requests.put(
self.tanium_url + uri,
headers=headers,
data=payload,
verify=self.tanium_ssl_verify,
)
else:
r = requests.put(
self.tanium_url + uri,
headers=headers,
json=payload,
verify=self.tanium_ssl_verify,
)
elif method == "patch":
r = requests.patch(
self.tanium_url + uri,
headers=headers,
json=payload,
verify=self.tanium_ssl_verify,
)
elif method == "delete":
r = requests.delete(self.tanium_url + uri,
headers=headers,
verify=self.tanium_ssl_verify)
else:
raise ValueError("Unspported method")
if r.status_code == 200:
try:
return r.json()
except:
return r.text
elif r.status_code == 401 and not retry:
self._get_session()
return self._query(method, uri, payload, content_type, type, True)
elif r.status_code == 401:
raise ValueError("Query failed, permission denied")
else:
self.helper.log_info(r.text)
def _get_labels(self, labels):
# List labels
tanium_labels = self._query("get",
"/plugin/products/detect3/api/v1/labels",
{"limit": 500})
tanium_labels_dict = {}
for tanium_label in tanium_labels:
tanium_labels_dict[tanium_label["name"].lower()] = tanium_label
final_labels = []
for label in labels:
# Label already exists
if label["value"] in tanium_labels_dict:
final_labels.append(tanium_labels_dict[label["value"]])
# Create the label
else:
created_label = self._query(
"post",
"/plugin/products/detect3/api/v1/labels",
{
"name": label["value"],
"description": "Label imported from OpenCTI",
},
)
final_labels.append(created_label)
return final_labels
def _get_by_id(self, internal_id, yara=False):
if yara:
response = self._query(
"get",
"/plugin/products/detect3/api/v1/intels",
{"name": internal_id + ".yara"},
)
else:
response = self._query(
"get",
"/plugin/products/detect3/api/v1/intels",
{"description": internal_id},
)
if response and len(response) > 0:
return response[0]
else:
return None
def _get_reputation_by_hash(self, hash):
response = self._query(
"get",
"/plugin/products/reputation/v3/reputations/custom",
{"search": hash},
)
if response["data"] and len(response["data"]) > 0:
return response["data"][0]
else:
return None
def _create_indicator_stix(self, entity, original_intel_document=None):
if original_intel_document is None:
intel_document = self._get_by_id(entity["id"])
if intel_document is not None:
return intel_document
stix2_bundle = self.helper.api.stix2.export_entity(
entity["entity_type"],
entity["id"],
"simple",
None,
True,
True,
)
initialize_options()
stix_indicator = slide_string(stix2_bundle)
stix_indicator = re.sub(
r"<indicator:Description>(.*?)<\/indicator:Description>",
r"<indicator:Description>" + entity["id"] +
"</indicator:Description>",
stix_indicator,
)
stix_indicator = re.sub(
r"<indicator:Description ordinality=\"1\">(.*?)<\/indicator:Description>",
r'<indicator:Description ordinality="1">' + entity["id"] +
"</indicator:Description>",
stix_indicator,
)
payload = {"intelDoc": stix_indicator}
if original_intel_document is not None:
intel_document = self._query(
"put",
"/plugin/products/detect3/api/v1/intels/" +
str(original_intel_document["id"]),
stix_indicator,
"application/xml",
"stix",
)
else:
intel_document = self._query(
"post",
"/plugin/products/detect3/api/v1/sources/" +
str(self.source_id) + "/intels",
payload,
"application/xml",
"stix",
)
return intel_document
def _create_indicator_yara(self, entity, original_intel_document=None):
if original_intel_document is None:
intel_document = self._get_by_id(entity["id"], True)
if intel_document is not None:
return intel_document
filename = entity["id"] + ".yara"
if original_intel_document is not None:
intel_document = self._query(
"put",
"/plugin/products/detect3/api/v1/intels/" +
str(original_intel_document["id"]),
{
"filename": filename,
"document": entity["pattern"],
"name": entity["name"],
"description": entity["id"],
},
"application/octet-stream",
"yara",
)
else:
intel_document = self._query(
"post",
"/plugin/products/detect3/api/v1/sources/" +
str(self.source_id) + "/intels",
{
"filename": filename,
"document": entity["pattern"],
"name": entity["name"],
"description": entity["id"],
},
"application/octet-stream",
"yara",
)
return intel_document
def _create_tanium_signal(self, entity, original_intel_document=None):
if original_intel_document is None:
intel_document = self._get_by_id(entity["id"])
if intel_document is not None:
return intel_document
platforms = []
if "x_mitre_platforms" in entity and len(
entity["x_mitre_platforms"]) > 0:
for x_mitre_platform in entity["x_mitre_platforms"]:
if x_mitre_platform in ["Linux", "Windows", "macOS"]:
platforms.append(x_mitre_platform.lower(
) if x_mitre_platform != "macOS" else "mac")
if original_intel_document is not None:
intel_document = self._query(
"put",
"/plugin/products/detect3/api/v1/intels/" +
str(original_intel_document["id"]),
{
"name": entity["name"],
"description": entity["id"],
"platforms": platforms,
"contents": entity["pattern"],
},
)
else:
intel_document = self._query(
"post",
"/plugin/products/detect3/api/v1/sources/" +
str(self.source_id) + "/intels",
{
"name": entity["name"],
"description": entity["id"],
"platforms": platforms,
"contents": entity["pattern"],
},
)
return intel_document
def _create_observable(self, entity, original_intel_document=None):
if original_intel_document is None:
intel_document = self._get_by_id(entity["id"])
if intel_document is not None:
return intel_document
intel_type = None
value = None
name = None
if entity["entity_type"] == "StixFile":
intel_type = "file_hash"
if "hashes" in entity:
for hash in entity["hashes"]:
value = (value + hash["hash"] +
"\n" if value is not None else hash["hash"] +
"\n")
name = hash["hash"]
elif entity["entity_type"] in [
"IPv4-Addr",
"IPv6-Addr",
"Domain-Name",
"X-OpenCTI-Hostname",
]:
intel_type = "ip_or_host"
value = entity["value"]
name = entity["value"]
if intel_type is None or value is None:
return None
openioc = self._query(
"post",
"/plugin/products/detect3/api/v1/intels/quick-add",
{
"exact": True,
"name": name,
"description": entity["id"],
"type": intel_type,
"text": value,
},
)
openioc = re.sub(
r"<description>(.*?)<\/description>",
r"<description>" + entity["id"] + "</description>",
openioc,
)
payload = {"intelDoc": openioc}
if original_intel_document is not None:
intel_document = self._query(
"put",
"/plugin/products/detect3/api/v1/intels/" +
str(original_intel_document["id"]),
payload,
"application/xml",
"openioc",
)
else:
intel_document = self._query(
"post",
"/plugin/products/detect3/api/v1/sources/" +
str(self.source_id) + "/intels",
payload,
"application/xml",
"openioc",
)
return intel_document
def _post_operations(self, entity, intel_document):
if intel_document is not None and entity is not None:
if self.tanium_auto_quickscan:
for computer_group in self.tanium_computer_groups:
self._query(
"post",
"/plugin/products/detect3/api/v1/quick-scans",
{
"computerGroupId": int(computer_group),
"intelDocId": intel_document["id"],
},
)
external_reference = self.helper.api.external_reference.create(
source_name="Tanium",
url=self.tanium_url + "/#/thr_workbench/intel/" +
str(intel_document["id"]),
external_id=str(intel_document["id"]),
description="Intel document within the Tanium platform.",
)
if entity["entity_type"] == "Indicator":
self.helper.api.stix_domain_object.add_external_reference(
id=entity["id"],
external_reference_id=external_reference["id"])
else:
self.helper.api.stix_cyber_observable.add_external_reference(
id=entity["id"],
external_reference_id=external_reference["id"])
if len(entity["objectLabel"]) > 0:
labels = self._get_labels(entity["objectLabel"])
for label in labels:
if label is not None:
self._query(
"put",
"/plugin/products/detect3/api/v1/intels/" +
str(intel_document["id"]) + "/labels",
{"id": label["id"]},
)
def _process_intel(self, entity_type, data, original_intel_document=None):
entity = None
intel_document = None
if entity_type == "indicator":
entity = self.helper.api.indicator.read(
id=data["data"]["x_opencti_id"])
if (entity is None or entity["revoked"] or entity["pattern_type"]
not in self.tanium_indicator_types):
return {"entity": entity, "intel_document": intel_document}
if entity["pattern_type"] == "stix":
intel_document = self._create_indicator_stix(
entity, original_intel_document)
elif entity["pattern_type"] == "yara":
intel_document = self._create_indicator_yara(
entity, original_intel_document)
elif entity["pattern_type"] == "tanium-signal":
intel_document = self._create_tanium_signal(
entity, original_intel_document)
elif (StixCyberObservableTypes.has_value(entity_type)
and entity_type.lower() in self.tanium_observable_types):
entity = self.helper.api.stix_cyber_observable.read(
id=data["data"]["x_opencti_id"])
if entity is None:
return {"entity": entity, "intel_document": intel_document}
intel_document = self._create_observable(entity,
original_intel_document)
return {"entity": entity, "intel_document": intel_document}
def _process_message(self, msg):
data = json.loads(msg.data)
entity_type = data["data"]["type"]
# If not an indicator, not an observable to import and
if (entity_type != "indicator"
and entity_type not in self.tanium_observable_types
and ("labels" in data["data"]
and self.tanium_reputation_blacklist_label
not in data["data"]["labels"])
and self.tanium_reputation_blacklist_label != "*"):
self.helper.log_info(
"Not an indicator and not an observable to import, doing nothing"
)
return
# Handle creation
if msg.event == "create":
# No label
if ("labels" not in data["data"]
and self.tanium_import_label != "*"
and self.tanium_reputation_blacklist_label != "*"):
self.helper.log_info(
"No label marked as import, doing nothing")
return
# Import or blacklist labels are not in the given labels
elif (("labels" in data["data"]
and self.tanium_import_label not in data["data"]["labels"])
and self.tanium_import_label != "*"
and self.tanium_reputation_blacklist_label
not in data["data"]["labels"]
and self.tanium_reputation_blacklist_label != "*"):
self.helper.log_info(
"No label marked as import or no global label, doing nothing"
)
return
# Revoked is true
elif "revoked" in data["data"] and data["data"]["revoked"]:
return
if ("labels" in data["data"]
and self.tanium_import_label in data["data"]["labels"]
) or self.tanium_import_label == "*":
# Process intel
processed_intel = self._process_intel(entity_type, data)
intel_document = processed_intel["intel_document"]
entity = processed_intel["entity"]
# Create external reference and add object labels
self._post_operations(entity, intel_document)
if ("labels" in data["data"]
and self.tanium_reputation_blacklist_label in data["data"]
["labels"]) or self.tanium_reputation_blacklist_label == "*":
if "hashes" in data["data"]:
entry = {"list": "blacklist"}
if "MD5" in data["data"]["hashes"]:
entry["md5"] = data["data"]["hashes"]["MD5"]
entry["uploadedHash"] = data["data"]["hashes"]["MD5"]
else:
entry["md5"] = ""
if "SHA-1" in data["data"]["hashes"]:
entry["sha1"] = data["data"]["hashes"]["SHA-1"]
entry["uploadedHash"] = data["data"]["hashes"]["SHA-1"]
else:
entry["sha1"] = ""
if "SHA-256" in data["data"]["hashes"]:
entry["sha256"] = data["data"]["hashes"]["SHA-256"]
entry["uploadedHash"] = data["data"]["hashes"][
"SHA-256"]
else:
entry["sha256"] = ""
entry["notes"] = ",".join(data["data"]["labels"])
self._query(
"post",
"/plugin/products/reputation/v3/reputations/custom/upload?append=true",
[entry],
)
elif msg.event == "update":
if ("x_data_update" in data["data"]
and "add" in data["data"]["x_data_update"]
and "labels" in data["data"]["x_data_update"]["add"]):
if self.tanium_reputation_blacklist_label in data["data"][
"x_data_update"]["add"][
"labels"] and StixCyberObservableTypes.has_value(
data["data"]["type"]):
observable = self.helper.api.stix_cyber_observable.read(
id=data["data"]["id"])
observable = self.helper.api.stix2.generate_export(
observable)
if "hashes" in observable:
entry = {"list": "blacklist"}
if "MD5" in observable["hashes"]:
entry["md5"] = observable["hashes"]["MD5"]
entry["uploadedHash"] = observable["hashes"]["MD5"]
else:
entry["md5"] = ""
if "SHA-1" in observable["hashes"]:
entry["sha1"] = observable["hashes"]["SHA-1"]
entry["uploadedHash"] = observable["hashes"][
"SHA-1"]
else:
entry["sha1"] = ""
if "SHA-256" in observable["hashes"]:
entry["sha256"] = observable["hashes"]["SHA-256"]
entry["uploadedHash"] = observable["hashes"][
"SHA-256"]
else:
entry["sha256"] = ""
entry["notes"] = ",".join(observable["labels"])
self._query(
"post",
"/plugin/products/reputation/v3/reputations/custom/upload?append=true",
[entry],
)
if (self.tanium_import_label
in data["data"]["x_data_update"]["add"]["labels"]):
# Process intel
processed_intel = self._process_intel(entity_type, data)
intel_document = processed_intel["intel_document"]
entity = processed_intel["entity"]
# Create external reference and add object labels
self._post_operations(entity, intel_document)
else:
entity = self.helper.api.indicator.read(
id=data["data"]["x_opencti_id"],
customAttributes="""
pattern_type
""",
)
intel_document = self._get_by_id(
data["data"]["x_opencti_id"],
yara=True if entity is not None
and entity["pattern_type"] == "yara" else False,
)
if intel_document:
new_labels = []
for label in data["data"]["x_data_update"]["add"][
"labels"]:
new_labels.append({"value": label})
labels = self._get_labels(new_labels)
for label in labels:
self._query(
"put",
"/plugin/products/detect3/api/v1/intels/" +
str(intel_document["id"]) + "/labels",
{"id": label["id"]},
)
elif ("x_data_update" in data["data"]
and "remove" in data["data"]["x_data_update"]
and "labels" in data["data"]["x_data_update"]["remove"]):
if (self.tanium_reputation_blacklist_label
in data["data"]["x_data_update"]["remove"]["labels"]):
if "hashes" in data["data"]:
if "SHA-256" in data["data"]["hashes"]:
self._query(
"post",
"/plugin/products/reputation/v3/reputations/custom/delete",
[data["data"]["hashes"]["SHA-256"]],
)
if "SHA-1" in data["data"]["hashes"]:
self._query(
"post",
"/plugin/products/reputation/v3/reputations/custom/delete",
[data["data"]["hashes"]["SHA-1"]],
)
if "MD5" in data["data"]["hashes"]:
self._query(
"post",
"/plugin/products/reputation/v3/reputations/custom/delete",
[data["data"]["hashes"]["MD5"]],
)
if (self.tanium_import_label
in data["data"]["x_data_update"]["remove"]["labels"]):
# Import label has been removed
intel_document = self._get_by_id(
data["data"]["x_opencti_id"])
if intel_document is not None:
self._query(
"delete",
"/plugin/products/detect3/api/v1/intels/" +
str(intel_document["id"]),
)
# Remove external references
if entity_type == "indicator":
entity = self.helper.api.indicator.read(
id=data["data"]["x_opencti_id"])
else:
entity = self.helper.api.stix_cyber_observable.read(
id=data["data"]["x_opencti_id"])
if (entity and "externalReferences" in entity
and len(entity["externalReferences"]) > 0):
for external_reference in entity["externalReferences"]:
if external_reference["source_name"] == "Tanium":
self.helper.api.external_reference.delete(
external_reference["id"])
else:
intel_document = self._get_by_id(
data["data"]["x_opencti_id"])
if intel_document:
new_labels = []
for label in data["data"]["x_data_update"]["remove"][
"labels"]:
new_labels.append({"value": label})
labels = self._get_labels(new_labels)
for label in labels:
self._query(
"delete",
"/plugin/products/detect3/api/v1/intels/" +
str(intel_document["id"]) + "/labels/" +
str(label["id"]),
)
elif ("x_data_update" in data["data"]
and "replace" in data["data"]["x_data_update"]):
if entity_type == "indicator":
if "pattern" in data["data"]["x_data_update"]["replace"]:
intel_document = self._get_by_id(
data["data"]["x_opencti_id"])
if intel_document is not None:
self._process_intel(entity_type, data,
intel_document)
elif ("value" in data["data"]["x_data_update"]["replace"]
or "hashes"
in data["data"]["x_data_update"]["replace"]):
intel_document = self._get_by_id(
data["data"]["x_opencti_id"])
if intel_document is not None:
self._process_intel(entity_type, data,
intel_document)
elif ("revoked" in data["data"]["x_data_update"]["replace"]
and
data["data"]["x_data_update"]["replace"]["revoked"]
is True):
intel_document = self._get_by_id(
data["data"]["x_opencti_id"])
if intel_document is not None:
self._query(
"delete",
"/plugin/products/detect3/api/v1/intels/" +
str(intel_document["id"]),
)
# Remove external references
if entity_type == "indicator":
entity = self.helper.api.indicator.read(
id=data["data"]["x_opencti_id"])
else:
entity = self.helper.api.stix_cyber_observable.read(
id=data["data"]["x_opencti_id"])
if (entity and "externalReferences" in entity
and len(entity["externalReferences"]) > 0):
for external_reference in entity[
"externalReferences"]:
if external_reference[
"source_name"] == "Tanium":
self.helper.api.external_reference.delete(
external_reference["id"])
elif msg.event == "delete":
intel_document = self._get_by_id(data["data"]["x_opencti_id"])
if intel_document is not None:
self._query(
"delete",
"/plugin/products/detect3/api/v1/intels/" +
str(intel_document["id"]),
)
if data["data"]["type"] == "file":
if "hashes" in data["data"]:
if "SHA-256" in data["data"]["hashes"]:
self._query(
"post",
"/plugin/products/reputation/v3/reputations/custom/delete",
[data["data"]["hashes"]["SHA-256"]],
)
if "SHA-1" in data["data"]["hashes"]:
self._query(
"post",
"/plugin/products/reputation/v3/reputations/custom/delete",
[data["data"]["hashes"]["SHA-1"]],
)
if "MD5" in data["data"]["hashes"]:
self._query(
"post",
"/plugin/products/reputation/v3/reputations/custom/delete",
[data["data"]["hashes"]["MD5"]],
)
def start(self):
self.alerts_gatherer = TaniumConnectorAlertsGatherer(
self.helper,
self.tanium_url,
self.tanium_login,
self.tanium_password,
self.tanium_ssl_verify,
)
self.alerts_gatherer.start()
self.helper.listen_stream(self._process_message)
class LastInfoSec:
def __init__(self):
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
self.lastinfosec_cti_url = "https://api.client.lastinfosec.com/v2/stix21/getbyminutes/{}?api_key={}&headers=false&platform=opencti"
self.lastinfosec_cve_url = "https://api.client.lastinfosec.com/v2/stix21/vulnerabilities/getlasthour?api_key={}&headers=false&platform=opencti"
self.lastinfosec_tactic_url = "https://api.client.lastinfosec.com/v2/stix21/tactic/getlast24hour?api_key={}&headers=false&platform=opencti"
self.lastinfosec_apikey = get_config_variable(
"CONFIG_LIS_APIKEY", ["lastinfosec", "api_key"], config)
self.lastinfosec_cti_enabled = get_config_variable(
"CONFIG_LIS_CTI_ENABLED", ["lastinfosec", "cti", "is_enabled"],
config)
self.lastinfosec_cti_interval = get_config_variable(
"CONFIG_LIS_CTI_INTERVAL", ["lastinfosec", "cti", "interval"],
config)
self.lastinfosec_cve_enabled = get_config_variable(
"CONFIG_LIS_CVE_ENABLED", ["lastinfosec", "cve", "is_enabled"],
config)
self.lastinfosec_tactic_enabled = get_config_variable(
"CONFIG_LIS_TACTIC_ENABLED",
["lastinfosec", "tactic", "is_enabled"], config)
self.opencti_url = get_config_variable("OPENCTI_URL",
["opencti", "url"], config)
self.opencti_id = get_config_variable("OPENCTI_TOKEN",
["opencti", "token"], config)
self.update_existing_data = get_config_variable(
"OPENCTI_UPDATE_EXISTING_DATA",
["connector", "update_existing_data"], config)
self.proxy_http = get_config_variable("PROXY_HTTP",
["opencti", "proxy_http"],
config)
self.proxy_https = get_config_variable("PROXY_HTTPS",
["opencti", "proxy_https"],
config)
total_enabled = 0
if self.lastinfosec_cti_enabled:
total_enabled += 1
if self.lastinfosec_cve_enabled:
total_enabled += 1
if self.lastinfosec_tactic_enabled:
total_enabled += 1
if total_enabled == 0:
raise Exception("You must enable one feed")
elif total_enabled > 1:
raise Exception("You can enable only one feed per connector")
self.api = OpenCTIApiClient(self.opencti_url, self.opencti_id)
def run(self):
self.helper.log_info("Fetching lastinfosec datasets...")
if not self.helper.get_run_and_terminate():
while True:
time_to_sleep = self.process_data()
time.sleep(time_to_sleep)
else:
self.process_data()
def process_data(self):
time_to_sleep = 0
try:
if (self.lastinfosec_cti_enabled
and self.lastinfosec_cti_url is not None
and self.lastinfosec_apikey is not None):
url = self.lastinfosec_cti_url.format(
self.lastinfosec_cti_interval, self.lastinfosec_apikey)
run_interval = self.lastinfosec_cti_interval * 60
time_to_sleep = self.fetch_data(url, run_interval)
elif (self.lastinfosec_cve_enabled
and self.lastinfosec_cve_url is not None
and self.lastinfosec_apikey is not None):
url = self.lastinfosec_cve_url.format(self.lastinfosec_apikey)
run_interval = 3600 # 1h in second
time_to_sleep = self.fetch_data(url, run_interval)
elif (self.lastinfosec_tactic_enabled
and self.lastinfosec_tactic_url is not None
and self.lastinfosec_apikey is not None):
url = self.lastinfosec_tactic_url.format(
self.lastinfosec_apikey)
run_interval = 86400 # 24h in second
time_to_sleep = self.fetch_data(url, run_interval)
else:
self.helper.log_info("CTI Feed not configured")
time.sleep(60)
exit(0)
except (KeyboardInterrupt, SystemExit):
self.helper.log_info("Connector stop")
exit(0)
except Exception as e:
self.helper.log_error("run:" + str(e))
time.sleep(60)
return time_to_sleep
def fetch_data(self, url: str, run_interval: int):
# Get the current timestamp and check
start = time.perf_counter()
time_to_sleep = 0
timestamp = int(time.time())
now = datetime.datetime.utcfromtimestamp(timestamp)
proxy_dic = {}
if self.proxy_http is not None:
proxy_dic["http"] = self.proxy_http
if self.proxy_https is not None:
proxy_dic["https"] = self.proxy_https
req = requests.get(url, proxies=proxy_dic)
if req.status_code == 200:
lastinfosec_data = req.json()
if isinstance(lastinfosec_data,
list) and len(lastinfosec_data) > 0:
friendly_name = "LastInfoSec CTI run @ " + now.strftime(
"%Y-%m-%d %H:%M:%S")
work_id = self.helper.api.work.initiate_work(
self.helper.connect_id, friendly_name)
self.push_data(lastinfosec_data, timestamp, work_id)
stop = time.perf_counter()
process_time_seconds = stop - start
time_to_sleep = run_interval - process_time_seconds
else:
message = "Connector error run, storing last_run as {0}".format(
timestamp)
self.helper.set_state({"last_run": timestamp})
self.helper.log_info(message)
time.sleep(150)
return time_to_sleep
def push_data(self, bundles, timestamp, work_id):
for bundle in bundles:
sdata = json.dumps(bundle)
self.helper.send_stix2_bundle(sdata, work_id=work_id)
# Store the current timestamp as a last run
message = "Connector successfully run, storing last_run as {0}".format(
timestamp)
self.helper.set_state({"last_run": timestamp})
self.helper.api.work.to_processed(work_id, message)
self.helper.log_info(message)
