def create_indicator(ioc_type, ioc_value, scan_results):
if (ioc_type == "url") or (ioc_type == "ip-src") or (ioc_type == "ip_dst") or (ioc_type == "domain") or (ioc_type == "hostname"):
ioc_pattern = "[url:value = '" + ioc_value + "']"
indicator = Indicator(labels=["malicious-activity"], pattern=ioc_pattern, description=scan_results)
if (ioc_type == "md5") or (ioc_type == "sha256"):
ioc_pattern = "[file:hashes." + ioc_type + " = '" + ioc_value + "']"
indicator = Indicator(labels=["malicious-activity"], pattern=ioc_pattern, description=scan_results)
return indicator
def create_indicator(content):
name = "SHA256 for malware object"
description = "This hash indicates the present of Malware."
pattern = ''
# temp=re.findall('"sha256":"(.*?)"',content)
# if len(temp)!=0:
value = get(content, 'sha256')
if (value != -1):
pattern = "[file:hashes.'SHA-256'='" + value + "']"
else:
return None
indicator_types = ["malicious-activity"]
pattern_type = 'stix'
indicator_list = []
indicator = Indicator(name=name,
description=description,
pattern_type=pattern_type,
pattern=pattern,
indicator_types=indicator_types)
indicator_list.append(indicator)
name = "MD5 for malware object"
description = "This hash indicates the present of Malware."
md5 = content['md5']
md5_pattern = "[file:hashes.'MD5'='" + md5 + "']"
indicator1 = Indicator(name=name,
description=description,
pattern_type=pattern_type,
pattern=md5_pattern,
indicator_types=indicator_types)
indicator_list.append(indicator1)
name = "SHA1 for malware object"
description = "This hash indicates the present of Malware."
sha1 = content['sha1']
sha1_pattern = "[file:hashes.'SHA-1'='" + sha1 + "']"
indicator2 = Indicator(name=name,
description=description,
pattern_type=pattern_type,
pattern=sha1_pattern,
indicator_types=indicator_types)
indicator_list.append(indicator2)
name = "Size of malware object"
description = "Malware size."
size = content['static_analysis']['binary_info']['size']
size_pattern = "[file:size=" + str(size) + "]"
indicator3 = Indicator(name=name,
description=description,
pattern_type=pattern_type,
pattern=size_pattern,
indicator_types=indicator_types)
indicator_list.append(indicator3)
return indicator_list
def setUp(self):
self.ts = datetime.now(timezone.utc).astimezone()
self.indicator_id = "indicator--de0c3d3f-02ee-4086-88f1-51200ac831f7"
self.point_ioc = "evil.com"
self.pattern = f"[domain-name:value = '{self.point_ioc}']"
self.indicator = Indicator(
id=self.indicator_id,
created=self.ts,
modified=self.ts,
pattern_type="stix",
pattern=self.pattern,
)
self.module_namespace = "TestNamespace"
self.logger = getLogger("test")
def rel_fs_store():
cam = Campaign(id=CAMPAIGN_ID, **CAMPAIGN_KWARGS)
idy = Identity(id=IDENTITY_ID, **IDENTITY_KWARGS)
ind = Indicator(id=INDICATOR_ID, **INDICATOR_KWARGS)
mal = Malware(id=MALWARE_ID, **MALWARE_KWARGS)
rel1 = Relationship(ind, 'indicates', mal, id=RELATIONSHIP_IDS[0])
rel2 = Relationship(mal, 'targets', idy, id=RELATIONSHIP_IDS[1])
rel3 = Relationship(cam, 'uses', mal, id=RELATIONSHIP_IDS[2])
stix_objs = [cam, idy, ind, mal, rel1, rel2, rel3]
fs = FileSystemStore(FS_PATH)
for o in stix_objs:
fs.add(o)
yield fs
for o in stix_objs:
filepath = os.path.join(FS_PATH, o.type, o.id,
_timestamp2filename(o.modified) + '.json')
# Some test-scoped fixtures (e.g. fs_store) delete all campaigns, so by
# the time this module-scoped fixture tears itself down, it may find
# its campaigns already gone, which causes not-found errors.
try:
os.remove(filepath)
except OSError as e:
# 3 is the ERROR_PATH_NOT_FOUND windows error code. Which has an
# errno symbolic value, but not the windows meaning...
if e.errno in (errno.ENOENT, 3):
continue
raise
def create_indicator(
pattern: str,
pattern_type: str,
created_by: Optional[Identity] = None,
name: Optional[str] = None,
description: Optional[str] = None,
valid_from: Optional[datetime] = None,
labels: Optional[List[str]] = None,
confidence: Optional[int] = None,
object_markings: Optional[List[MarkingDefinition]] = None,
x_opencti_main_observable_type: Optional[str] = None,
) -> Indicator:
"""Create an indicator."""
custom_properties: Dict[str, Any] = {
X_OPENCTI_SCORE: DEFAULT_X_OPENCTI_SCORE
}
if x_opencti_main_observable_type is not None:
custom_properties[
X_OPENCTI_MAIN_OBSERVABLE_TYPE] = x_opencti_main_observable_type
return Indicator(
id=_create_random_identifier("indicator"),
created_by_ref=created_by,
name=name,
description=description,
pattern=pattern,
pattern_type=pattern_type,
valid_from=valid_from,
labels=labels,
confidence=confidence,
object_marking_refs=object_markings,
custom_properties=custom_properties,
)
def create_indicator(
name: str,
author: Identity,
description: str,
valid_from: datetime,
observable_type: str,
observable_value: str,
pattern_type: str,
pattern_value: str,
indicator_pattern: str,
object_marking_refs: List[MarkingDefinition],
) -> Indicator:
"""Create an indicator."""
return Indicator(
created_by_ref=author,
name=name,
description=description,
pattern=str(pattern_value),
valid_from=valid_from,
labels=["malicious-activity"],
object_marking_refs=object_marking_refs,
custom_properties={
CustomProperties.OBSERVABLE_TYPE: observable_type,
CustomProperties.OBSERVABLE_VALUE: observable_value,
CustomProperties.PATTERN_TYPE: pattern_type,
CustomProperties.INDICATOR_PATTERN: indicator_pattern,
},
)
def create_indicator(
pattern: str,
pattern_type: str,
created_by: Optional[Identity] = None,
created: Optional[datetime] = None,
modified: Optional[datetime] = None,
name: Optional[str] = None,
description: Optional[str] = None,
valid_from: Optional[datetime] = None,
labels: Optional[List[str]] = None,
confidence: Optional[int] = None,
object_markings: Optional[List[MarkingDefinition]] = None,
) -> Indicator:
"""Create an indicator."""
return Indicator(
id=_create_random_identifier("indicator"),
created_by_ref=created_by,
created=created,
modified=modified,
name=name,
description=description,
pattern=pattern,
pattern_type=pattern_type,
valid_from=valid_from,
labels=labels,
confidence=confidence,
object_marking_refs=object_markings,
custom_properties={X_OPENCTI_SCORE: DEFAULT_X_OPENCTI_SCORE},
)
def test_zmq_app_plugin_message_roundtrip(self):
"""
Backend-agnostic message passing scenario. Sends a fixed amount of
messages via the threatbus ZeroMQ app plugin, subscribes to Threat Bus,
and checks if the initially sent messages can be retrieved back.
"""
result_q = queue.Queue()
items = 2
topics = ["stix2/indicator", "stix2/sighting"]
rec = threading.Thread(target=zmq_receiver.forward,
args=(items, topics, result_q),
daemon=False)
rec.start()
ioc = Indicator(pattern_type="stix",
pattern="[ipv4-addr:value = '6.6.6.6']")
zmq_sender.send(
"stix2/indicator",
ioc.serialize(),
port=13372,
bind=False,
)
sighting = Sighting(sighting_of_ref=ioc.id)
zmq_sender.send(
"stix2/sighting",
sighting.serialize(),
port=13372,
bind=False,
)
time.sleep(1)
self.assertEqual(result_q.qsize(), items)
event = result_q.get(timeout=1)
self.assertIsNotNone(event)
self.assertEqual(parse(event), ioc)
result_q.task_done()
event = result_q.get(timeout=1)
self.assertIsNotNone(event)
self.assertEqual(parse(event), sighting)
result_q.task_done()
self.assertEqual(0, result_q.qsize())
result_q.join()
rec.join(timeout=1)
def rel_mem_store():
cam = Campaign(id=CAMPAIGN_ID, **CAMPAIGN_KWARGS)
idy = Identity(id=IDENTITY_ID, **IDENTITY_KWARGS)
ind = Indicator(id=INDICATOR_ID, **INDICATOR_KWARGS)
mal = Malware(id=MALWARE_ID, **MALWARE_KWARGS)
rel1 = Relationship(ind, 'indicates', mal, id=RELATIONSHIP_IDS[0])
rel2 = Relationship(mal, 'targets', idy, id=RELATIONSHIP_IDS[1])
rel3 = Relationship(cam, 'uses', mal, id=RELATIONSHIP_IDS[2])
stix_objs = [cam, idy, ind, mal, rel1, rel2, rel3]
yield MemoryStore(stix_objs)
def setUp(self):
self.ts = datetime.now().astimezone()
self.ioc_id = "indicator--42d31a5b-2da0-4bdd-9823-1723a98fc2fb"
self.ioc_value = "example.com"
self.ioc = Indicator(
id=self.ioc_id,
pattern_type="stix",
pattern=f"[domain-name:value = '{self.ioc_value}']",
)
self.sighting_context = {
"ts": "2017-03-03T23:56:09.652643840",
"uid": "CMeLkt11aTqwgN4FI9",
"id.orig_h": "172.31.130.19",
"id.orig_p": 43872,
"id.resp_h": "172.31.129.17",
"id.resp_p": 20004,
"proto": "tcp",
"service": None,
"duration": 0.025249,
"orig_bytes": 311,
"resp_bytes": 999,
"conn_state": "SF",
"local_orig": None,
"local_resp": None,
"missed_bytes": 0,
"history": "ShADadFf",
"orig_pkts": 9,
"orig_ip_bytes": 787,
"resp_pkts": 7,
"resp_ip_bytes": 1371,
"tunnel_parents": [],
"alert": {
"signature": "VAST-RETRO Generic IoC match for: 172.31.129.17",
"category": "Potentially Bad Traffic",
"action": "allowed",
},
"event_type": "alert",
"_extra": {
"vast-ioc": "172.31.129.17"
},
"source": "VAST",
}
self.sighting = Sighting(sighting_of_ref=self.ioc_id)
self.snapshot_id = "SNAPSHOT_UUID"
self.snapshot = timedelta(days=42, hours=23, minutes=13, seconds=37)
self.snapshot_request = SnapshotRequest(MessageType.SIGHTING,
self.snapshot_id,
self.snapshot)
self.snapshot_envelope_indicator = SnapshotEnvelope(
MessageType.INDICATOR, self.snapshot_id, self.ioc)
self.snapshot_envelope_sighting = SnapshotEnvelope(
MessageType.SIGHTING, self.snapshot_id, self.sighting)
def _map_to_threatbus(
self, data: dict, opencti_action: str
) -> Union[Indicator, None]:
"""
Inspects the given OpenCTI data point and either returns a valid STIX-2
Indicator or None.
@param data A dict object with OpenCTI SSE data
@param opencti_action A string indicating what happened to this item
(either `create`, `update` or `delete`)
@return a STIX-2 Indicator or None
"""
opencti_id: str = data.get("x_opencti_id", None)
if not opencti_id:
self.opencti_helper.log_error(
"Cannot process data without 'x_opencti_id' field"
)
return
event_id = data.get("id", None)
update = data.get("x_data_update", {})
added = update.get("add", {})
added_ids = added.get("x_opencti_stix_ids", [])
type_ = data.get("type", None)
if type_ == "indicator" and len(added_ids) == 1 and added_ids[0] == event_id:
# Discard the update if it was empty. An update is empty when the
# only "changed" attribute is the stix_id and it changed to its own
# already existing value. Example:
# data ~ {'id': 'XXX', 'x_data_update': {'add': {'x_opencti_stix_ids': ['XXX']}}}
return
if opencti_action == "delete":
indicator: dict = data
indicator[
ThreatBusSTIX2Constants.X_THREATBUS_UPDATE.value
] = Operation.REMOVE.value
else:
indicator: dict = self.opencti_helper.api.indicator.read(id=opencti_id)
if not indicator:
# we are only interested in indicators at this time
return
# overwrite custom OpenCTI ID
indicator["id"] = indicator.get("standard_id")
if opencti_action == "update":
indicator[
ThreatBusSTIX2Constants.X_THREATBUS_UPDATE.value
] = Operation.EDIT.value
# only propagate indicators that are toggled for detection or the user
# enabled forwarding of all indicators regardless of the toggle
detection_enabled: bool = indicator.get("x_opencti_detection", False)
if not detection_enabled and self.forward_all_iocs is not True:
return
return Indicator(**indicator, allow_custom=True)
def forge_indicator(url, score):
indicator = Indicator(
name="Suspicious URL",
labels=["malicious-activity"],
created=get_time(),
description="This URL has been detected as malicious with a score of "
+ str(score) + ".",
pattern="[url:value = '" + url + "']")
return indicator
def test_indicator_to_vast_query(self):
## test IP
ip = "6.6.6.6"
expected_vast_query = f"{ip}"
ioc = Indicator(pattern_type="stix",
pattern=f"[ipv4-addr:value = '{ip}']")
self.assertEqual(expected_vast_query, indicator_to_vast_query(ioc))
## test URL
url = "example.com/foo/bar?query=123"
expected_vast_query = f'"{url}" == net.uri'
ioc = Indicator(pattern_type="stix", pattern=f"[url:value = '{url}']")
self.assertEqual(expected_vast_query, indicator_to_vast_query(ioc))
## test domain
domain = "example.com"
expected_vast_query = f'"{domain}" == net.domain || "{domain}" == net.hostname'
ioc = Indicator(pattern_type="stix",
pattern=f"[domain-name:value = '{domain}']")
self.assertEqual(expected_vast_query, indicator_to_vast_query(ioc))
def test_indicator_to_vast_query(self):
## test IP
expected_vast_query = f"{self.ioc_value}"
self.assertEqual(expected_vast_query,
indicator_to_vast_query(self.indicator))
## test URL
url = "example.com/foo/bar?query=123"
expected_vast_query = f'"{url}" in net.uri'
other_ioc = Indicator(pattern_type="stix",
pattern=f"[url:value = '{url}']")
self.assertEqual(expected_vast_query,
indicator_to_vast_query(other_ioc))
## test domain
domain = "example.com"
expected_vast_query = f'"{domain}" == net.domain || "{domain}" == net.hostname'
other_ioc = Indicator(pattern_type="stix",
pattern=f"[domain-name:value = '{domain}']")
self.assertEqual(expected_vast_query,
indicator_to_vast_query(other_ioc))
def _handle_indicator(self, indicator: Indicator):
"""
Handles a STIX-2 Indicator update received via Threat Bus. Does nothing
in case the indicator already exists and the new indicator does not add
any new fields/values to the existing indicator. By doing so, this
function effectively avoids double updates that otherwise would result
in SSE events without a real change.
@param indicator The STIX-2 Indicator received from Threat Bus
"""
if type(indicator) is not Indicator:
self.opencti_helper.log_error(
f"Error ingesting indicator from Threat Bus. Expected a STIX-2 Indicator: {indicator}"
)
return
if (
ThreatBusSTIX2Constants.X_THREATBUS_UPDATE.value
in indicator.object_properties()
and indicator.x_threatbus_update == Operation.REMOVE.value
):
# OpenCTI does not support indicator removal via API calls (yet)
return
lookup_resp = self.opencti_helper.api.indicator.read(id=indicator.id)
if not lookup_resp:
# No indicator with that ID exists already.
self._create_or_update_indicator(indicator)
return
lookup_resp["id"] = lookup_resp["standard_id"]
lookup_indicator = Indicator(**lookup_resp, allow_custom=True)
# We found an existing indicator. To avoid double updates in the SSE
# stream we check if the indicator from Threat Bus adds anything new.
for prop, new_value in indicator.items():
if prop == "id" or prop.startswith("x_"):
continue
existing_value = lookup_indicator.get(prop, None)
if existing_value is None or new_value != existing_value:
self._create_or_update_indicator(indicator)
return
def _create_or_update_indicator(self, indicator: Indicator):
"""
Creates or updates a STIX-2 Indicator in OpenCTI
@param indicator The STIX-2 Indicator
"""
ioc_dct = json.loads(indicator.serialize())
ioc_dct["name"] = ioc_dct.get("name", indicator.id) # default to UUID
ioc_dct["stix_id"] = indicator.id
del ioc_dct["id"]
obs_type = ioc_dct.get("x_opencti_main_observable_type", "Unknown")
ioc_dct["x_opencti_main_observable_type"] = obs_type
resp = self.opencti_helper.api.indicator.create(**ioc_dct)
self.opencti_helper.log_info(f"Created or added to indicator: {resp}")
def setUp(self):
self.ts = datetime.now(timezone.utc).astimezone()
self.indicator_id = "indicator--46b3f973-5c03-41fc-9efe-49598a267a35"
self.operation = Operation.REMOVE
self.ioc_value = "6.6.6.6"
self.pattern = f"[ipv4-addr:value = '{self.ioc_value}']"
self.indicator = Indicator(
pattern_type="stix",
pattern=self.pattern,
created=self.ts,
id=self.indicator_id,
)
self.valid_query_result = f'{{"timestamp": "{self.ts}", "flow_id": 1840147514011873, "pcap_cnt": 626, "src_ip": "{self.ioc_value}", "src_port": 1193, "dest_ip": "65.54.95.64", "dest_port": 80, "proto": "TCP", "event_type": "http", "community_id": "1:AzSEWwmsqEKUX5qrReAHI3Rpizg=", "http": {{"hostname": "download.windowsupdate.com", "url": "/v9/windowsupdate/a/selfupdate/WSUS3/x86/Other/wsus3setup.cab?0911180916", "http_port": null, "http_user_agent": "Windows-Update-Agent", "http_content_type": "application/octet-stream", "http_method": "HEAD", "http_refer": null, "protocol": "HTTP/1.1", "status": 200, "redirect": null, "length": 0}}, "tx_id": 0}}'
self.valid_matcher_result = f'{{"ts": "{self.ts}", "data_id": 8, "indicator_id": 5, "matcher": "threatbus-syeocdkfcy", "value": "{self.ioc_value}", "reference": "threatbus__{self.indicator_id}"}}'
def setUp(self):
self.ts = datetime.now(timezone.utc).astimezone()
self.indicator_id = "indicator--46b3f973-5c03-41fc-9efe-49598a267a35"
self.operation = Operation.REMOVE
self.ioc_value = "evil.com"
self.pattern = f"[domain-name:value = '{self.ioc_value}']"
self.indicator = Indicator(
pattern_type="stix",
pattern=self.pattern,
created=self.ts,
id=self.indicator_id,
)
self.valid_query_result = f'{{"timestamp": "{self.ts}", "flow_id": 1840147514011873, "pcap_cnt": 626, "src_ip": "{self.ioc_value}", "src_port": 1193, "dest_ip": "65.54.95.64", "dest_port": 80, "proto": "TCP", "event_type": "http", "community_id": "1:AzSEWwmsqEKUX5qrReAHI3Rpizg=", "http": {{"hostname": "download.windowsupdate.com", "url": "/v9/windowsupdate/a/selfupdate/WSUS3/x86/Other/wsus3setup.cab?0911180916", "http_port": null, "http_user_agent": "Windows-Update-Agent", "http_content_type": "application/octet-stream", "http_method": "HEAD", "http_refer": null, "protocol": "HTTP/1.1", "status": 200, "redirect": null, "length": 0}}, "tx_id": 0}}'
self.valid_matcher_result = f'{{"indicator": {{"value": "{self.ioc_value}", "context": "\\"threatbus__{self.indicator_id}\\""}}, "type": "zeek.dns", "event": {{"ts": "{self.ts}", "uid": "CGG3VBvnp9JKfVP56", "id.orig_h": "172.19.0.2", "id.orig_p": 37241, "id.resp_h": "1.1.1.1", "id.resp_p": 53, "proto": "udp", "trans_id": 53396, "rtt": "18.42ms", "query": "reddit.com", "qclass": 1, "qclass_name": "C_INTERNET", "qtype": 1, "qtype_name": "A", "rcode": 0, "rcode_name": "NOERROR", "AA": false, "TC": false, "RD": true, "RA": true, "Z": 2, "answers": ["151.101.193.140", "151.101.65.140", "151.101.1.140", "151.101.129.140"], "TTLs": ["2.47m", "2.47m", "2.47m", "2.47m"], "rejected": false}} }}'
def outputstix2(results, config):
indicators = []
if results is None:
return
if config.debug:
print("Creating STIX2 indicators", file=sys.stderr)
for result in results:
# Indicator setup based on https://oasis-open.github.io/cti-documentation/examples/indicator-for-malicious-url
description = '|'.join(
result.get('threat_types', []) + result.get('last_seen_as', []) +
result.get('malware_family', []))
if result['type'] == 'url':
indicator = Indicator(valid_from=result.get(
'last_seen', result['last_published']),
labels="malicious-activity",
description=description,
pattern="[url:value = '%s']" % result["key"],
pattern_type="stix")
elif result['type'] == 'domain':
indicator = Indicator(
valid_from=result.get('last_seen', result['last_published']),
labels="malicious-activity",
description=description,
pattern="[domain-name:value = '%s']" % result["key"],
pattern_type="stix")
elif result['type'] == 'ip':
indicator = Indicator(
valid_from=result.get('last_seen', result['last_published']),
labels="malicious-activity",
description=description,
pattern="[ipv4-addr:value = '%s']" % result["key"],
pattern_type="stix")
indicators.append(indicator)
if config.debug:
print("Creating STIX2 bundle", file=sys.stderr)
bundle = Bundle(indicators) # This takes WAY too long, needs measurement
return bundle
def map_to_cif(indicator: Indicator, confidence: int, tags: List[str],
tlp: str, group: str, logger) -> Union[CIFIndicator, None]:
"""
Maps a STIX-2 Indicator to a CIFv3 compatible indicator format.
@param indicator The STIX-2 Indicator to map
@param confidence The confidence to use when building the CIF indicator
@param tags The tags to use when building the CIF indicator
@param tlp The tlp to use when building the CIF indicator
@param group The group to use when building the CIF indicator
@return the mapped intel item or None
"""
if not indicator or type(indicator) is not Indicator:
logger.debug(f"Expected STIX-2 indicator, discarding {indicator}")
return None
if (ThreatBusSTIX2Constants.X_THREATBUS_UPDATE.value
in indicator.object_properties()):
logger.debug(
f"CIFv3 only supports adding indicators, not deleting / editing. Discardig {indicator}"
)
return None
if not is_point_equality_ioc(indicator.pattern):
logger.debug(
f"CIFv3 only supports point indicators, discardig {indicator}")
return None
object_path, ioc_value = split_object_path_and_value(indicator.pattern)
if object_path not in cif_supported_types:
logger.debug(
f"Discardig indicator with unsupported object-path {indicator}")
return None
# convert lasttime
lasttime = indicator.created.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
ioc_dict = {
"indicator": ioc_value,
"confidence": confidence,
"tags": tags,
"tlp": tlp,
"group": group,
"lasttime": lasttime,
}
try:
return CIFIndicator(**ioc_dict)
except InvalidIndicator as e:
logger.error(f"Invalid CIF indicator {e}")
except Exception as e:
logger.error(f"CIF indicator error: {e}")
def setUp(self):
self.observations = [
{
"type": "identity",
},
{"type": "observed-data", "some-prop": "value"},
{
"type": "observed-data",
"some-prop": "value",
"last_observed": "2021-05-04T15:15:58.919Z",
},
]
self.indicator = Indicator(
pattern="[ipv4-addr:value = '6.6.6.6']", pattern_type="stix"
)
def map_indicator_to_broker_event(indicator: Indicator, module_namespace: str,
logger) -> Union[broker.zeek.Event, None]:
"""
Maps STIX-2 Indicators to Broker events using the Zeek Intel format
@see https://docs.zeek.org/en/current/scripts/base/frameworks/intel/main.zeek.html#type-Intel::Type
@param indicator The STIX-2 Indicator to convert
@param module_namespace A Zeek namespace to use for sending the event
@return The mapped broker event or None
"""
if type(indicator) is not Indicator:
logger.debug(
f"Discarding message, expected STIX-2 Indicator: {indicator}")
return None
if not is_point_equality_ioc(indicator.pattern):
logger.debug(
f"Zeek only supports point-IoCs. Cannot map compound pattern to a Zeek Intel item: {indicator.pattern}"
)
return None
object_path, ioc_value = split_object_path_and_value(indicator.pattern)
# get matching Zeek intel type
zeek_type = zeek_intel_type_map.get(object_path, None)
if not zeek_type:
logger.debug(
f"No matching Zeek type found for STIX-2 indicator type '{object_path}'"
)
return None
if zeek_type == "URL":
# remove leading protocol, if any
parsed = urlparse(ioc_value)
scheme = f"{parsed.scheme}://"
ioc_value = parsed.geturl().replace(scheme, "", 1)
elif zeek_type == "ADDR" and re.match(".+/.+", ioc_value):
# elevate to subnet if possible
zeek_type = "SUBNET"
operation = "ADD" ## Zeek operation to add a new Intel item
if (ThreatBusSTIX2Constants.X_THREATBUS_UPDATE.value
in indicator.object_properties()
and indicator.x_threatbus_update == Operation.REMOVE.value):
operation = "REMOVE"
return broker.zeek.Event(
f"{module_namespace}::intel",
(indicator.created, str(
indicator.id), zeek_type, ioc_value, operation),
)
def writeStix(shadyUrl, score):
# a mettre dans ta callback
indicator = Indicator(
name="Malicious site (phishing)",
labels=["malicious-activity"],
pattern="[url:value = \'" + shadyUrl + "\']",
created="{}".format(datetime.datetime.now().date()),
valid_from="{}".format(datetime.datetime.now().date()),
description=
"Possible malicious site detected by LID-TP1 with a score of " +
str(score) + ", shady url : " + shadyUrl)
mem.add(indicator)
mem.save_to_file(
"./database/{}.json".format(datetime.datetime.now().date())
) # end of the function, if the site is a phishing site (score > 0) -> write in STIXv2.0 file
def rel_fs_store():
cam = Campaign(id=CAMPAIGN_ID, **CAMPAIGN_KWARGS)
idy = Identity(id=IDENTITY_ID, **IDENTITY_KWARGS)
ind = Indicator(id=INDICATOR_ID, **INDICATOR_KWARGS)
mal = Malware(id=MALWARE_ID, **MALWARE_KWARGS)
rel1 = Relationship(ind, 'indicates', mal, id=RELATIONSHIP_IDS[0])
rel2 = Relationship(mal, 'targets', idy, id=RELATIONSHIP_IDS[1])
rel3 = Relationship(cam, 'uses', mal, id=RELATIONSHIP_IDS[2])
stix_objs = [cam, idy, ind, mal, rel1, rel2, rel3]
fs = FileSystemStore(FS_PATH)
for o in stix_objs:
fs.add(o)
yield fs
for o in stix_objs:
os.remove(os.path.join(FS_PATH, o.type, o.id + '.json'))
def setUp(self):
# self.timestamp = datetime.now(timezone.utc).astimezone()
self.raw_ts = 1579104545
self.timestamp = datetime.fromtimestamp(self.raw_ts).astimezone(
timezone.utc)
self.ioc_id = "indicator--42d31a5b-2da0-4bdd-9823-1723a98fc2fb"
self.ioc_value = "example.com"
self.ioc = Indicator(
id=self.ioc_id,
created=self.timestamp,
pattern_type="stix",
pattern=f"[domain-name:value = '{self.ioc_value}']",
)
self.ep = broker.Endpoint()
self.ep.peer(self.host, self.port)
self.subscribe_callback.reset_mock()
self.unsubscribe_callback.reset_mock()
def newioc_view(request):
#if request.method == 'POST':
form = IndicatorForm(request.POST)
if form.is_valid():
name = form.cleaned_data['name']
labels = form.cleaned_data['labels']
pattern = form.cleaned_data['pattern']
indicator = Indicator(name=name, labels=labels, pattern=pattern)
print(indicator)
#saving the indicator to database
return redirect('/dashboard')
context = {'form': form}
return render(request, 'newioc.html', context)
def _ingest_indicator(self, indicator: Indicator):
"""
Ingests a STIX-2 Indicator into OpenCTI. Does nothing in case the
indicator already exists.
@param indicator The STIX-2 Indicator object to ingest
"""
if type(indicator) is not Indicator:
self.opencti_helper.log_error(
f"Error ingesting indicator from Threat Bus. Expected a STIX-2 Indicator: {indicator}"
)
return
ioc_dct = json.loads(indicator.serialize())
ioc_dct["name"] = ioc_dct.get("name", indicator.id) # default to UUID
ioc_dct["stix_id"] = indicator.id
del ioc_dct["id"]
obs_type = ioc_dct.get("x_opencti_main_observable_type", "Unknown")
ioc_dct["x_opencti_main_observable_type"] = obs_type
resp = self.opencti_helper.api.indicator.create(**ioc_dct)
self.opencti_helper.log_info(f"Created or added to indicator: {resp}")
def _map_to_threatbus(
self, data: dict, opencti_action: str
) -> Union[Indicator, None]:
"""
Inspects the given OpenCTI data point and either returns a valid STIX-2
Indicator or None.
@param data A dict object with OpenCTI SSE data
@param opencti_action A string indicating what happened to this item
(either `create`, `update` or `delete`)
@return a STIX-2 Indicator or None
"""
opencti_id: str = data.get("x_opencti_id", None)
if not opencti_id:
self.opencti_helper.log_error(
"Cannot process data without 'x_opencti_id' field"
)
return
indicator: dict = self.opencti_helper.api.indicator.read(id=opencti_id)
if not indicator:
# we are only interested in indicators at this time
return
detection_enabled: bool = indicator.get("x_opencti_detection", False)
if not detection_enabled and self.forward_all_iocs is not True:
# only propagate indicators that are toggled for detection or the
# user enabled forwarding of all indicators regardless of the toggle
return
# overwrite custom OpenCTI ID
indicator["id"] = indicator.get("standard_id")
if opencti_action == "update":
indicator[
ThreatBusSTIX2Constants.X_THREATBUS_UPDATE.value
] = Operation.EDIT.value
if opencti_action == "delete":
indicator[
ThreatBusSTIX2Constants.X_THREATBUS_UPDATE.value
] = Operation.REMOVE.value
return Indicator(**indicator, allow_custom=True)
def setUp(self):
self.created = datetime.now().astimezone()
self.indicator_id = "indicator--df0f9a0e-c3b6-4f53-b0cf-5e9c454ee0cc"
self.pattern = "[ipv4-addr:value = '6.6.6.6']"
self.pattern_type = "stix2"
self.operation = Operation.REMOVE
self.indicator = Indicator(
id=self.indicator_id,
pattern=self.pattern,
pattern_type=self.pattern_type,
created=self.created,
valid_from=self.created,
modified=self.created,
)
self.sighting_source = "VAST"
self.sighting_id = "sighting--df0f9a0e-c3b6-4f53-b0cf-5e9c454ee0cc"
self.sighting_context = {
"ts": "2017-03-03T23:56:09.652643840",
"uid": "CMeLkt11aTqwgN4FI9",
"id.orig_h": "172.31.130.19",
"id.orig_p": 43872,
"id.resp_h": "172.31.129.17",
"id.resp_p": 20004,
"proto": "tcp",
"service": None,
"duration": 0.025249,
"orig_bytes": 311,
"resp_bytes": 999,
"conn_state": "SF",
"local_orig": None,
"local_resp": None,
"missed_bytes": 0,
"history": "ShADadFf",
"orig_pkts": 9,
"orig_ip_bytes": 787,
"resp_pkts": 7,
"resp_ip_bytes": 1371,
"tunnel_parents": [],
"alert": {
"signature": "VAST-RETRO Generic IoC match for: 172.31.129.17",
"category": "Potentially Bad Traffic",
"action": "allowed",
},
"event_type": "alert",
"_extra": {
"vast-ioc": "172.31.129.17"
},
}
self.sighting = Sighting(
id=self.sighting_id,
created=self.created,
modified=self.created,
sighting_of_ref=self.indicator_id,
custom_properties={
ThreatBusSTIX2Constants.X_THREATBUS_SIGHTING_CONTEXT.value:
self.sighting_context,
ThreatBusSTIX2Constants.X_THREATBUS_INDICATOR.value:
self.indicator,
ThreatBusSTIX2Constants.X_THREATBUS_SOURCE.value:
self.sighting_source,
},
)
self.snapshot_id = "SNAPSHOT_UUID"
self.snapshot = timedelta(days=42, hours=23, minutes=13, seconds=37)
self.snapshot_request = SnapshotRequest(MessageType.SIGHTING,
self.snapshot_id,
self.snapshot)
self.snapshot_envelope_indicator = SnapshotEnvelope(
MessageType.INDICATOR, self.snapshot_id, self.indicator)
self.snapshot_envelope_sighting = SnapshotEnvelope(
MessageType.SIGHTING, self.snapshot_id, self.sighting)
def process_attribute(
self,
author,
event_elements,
event_markings,
attribute_external_references,
attribute,
):
try:
resolved_attributes = self.resolve_type(
attribute["type"], attribute["value"]
)
if resolved_attributes is None:
return None
for resolved_attribute in resolved_attributes:
### Pre-process
# Elements
attribute_elements = self.prepare_elements(attribute["Galaxy"], author)
# Markings & Tags
attribute_tags = []
if "Tag" in attribute:
attribute_markings = self.resolve_markings(
attribute["Tag"], with_default=False
)
attribute_tags = self.resolve_tags(attribute["Tag"])
if len(attribute_markings) == 0:
attribute_markings = event_markings
else:
attribute_markings = event_markings
### Create the indicator
observable_type = resolved_attribute["type"]
observable_value = resolved_attribute["value"]
name = resolved_attribute["value"]
pattern_type = "stix"
# observable type is yara for instance
if observable_type in PATTERNTYPES:
pattern_type = observable_type
observable_type = "Unknown"
genuine_pattern = (
"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']"
)
pattern = observable_value
name = (
attribute["comment"]
if len(attribute["comment"]) > 0
else observable_type
)
# observable type is not in stix 2
elif observable_type not in OPENCTISTIX2:
return None
# observable type is in stix
else:
if "transform" in OPENCTISTIX2[observable_type]:
if (
OPENCTISTIX2[observable_type]["transform"]["operation"]
== "remove_string"
):
observable_value = observable_value.replace(
OPENCTISTIX2[observable_type]["transform"]["value"], ""
)
lhs = ObjectPath(
OPENCTISTIX2[observable_type]["type"],
OPENCTISTIX2[observable_type]["path"],
)
genuine_pattern = str(
ObservationExpression(
EqualityComparisonExpression(lhs, observable_value)
)
)
pattern = genuine_pattern
indicator = Indicator(
name=name,
description=attribute["comment"],
pattern=genuine_pattern,
valid_from=datetime.utcfromtimestamp(
int(attribute["timestamp"])
).strftime("%Y-%m-%dT%H:%M:%SZ"),
labels=["malicious-activity"],
created_by_ref=author,
object_marking_refs=attribute_markings,
external_references=attribute_external_references,
custom_properties={
"x_opencti_indicator_pattern": pattern,
"x_opencti_observable_type": observable_type,
"x_opencti_observable_value": observable_value,
"x_opencti_pattern_type": pattern_type,
"x_opencti_tags": attribute_tags,
},
)
### Create the relationships
relationships = []
# Event threats
for threat in (
event_elements["intrusion_sets"]
+ event_elements["malwares"]
+ event_elements["tools"]
):
relationships.append(
Relationship(
relationship_type="indicates",
created_by_ref=author,
source_ref=indicator.id,
target_ref=threat.id,
description=attribute["comment"],
object_marking_refs=attribute_markings,
custom_properties={
"x_opencti_first_seen": datetime.utcfromtimestamp(
int(attribute["timestamp"])
).strftime("%Y-%m-%dT%H:%M:%SZ"),
"x_opencti_last_seen": datetime.utcfromtimestamp(
int(attribute["timestamp"])
).strftime("%Y-%m-%dT%H:%M:%SZ"),
"x_opencti_weight": self.helper.connect_confidence_level,
},
)
)
# Attribute threats
for threat in (
attribute_elements["intrusion_sets"]
+ attribute_elements["malwares"]
+ attribute_elements["tools"]
):
relationships.append(
Relationship(
relationship_type="indicates",
created_by_ref=author,
source_ref=indicator.id,
target_ref=threat.id,
description=attribute["comment"],
object_marking_refs=attribute_markings,
custom_properties={
"x_opencti_first_seen": datetime.utcfromtimestamp(
int(attribute["timestamp"])
).strftime("%Y-%m-%dT%H:%M:%SZ"),
"x_opencti_last_seen": datetime.utcfromtimestamp(
int(attribute["timestamp"])
).strftime("%Y-%m-%dT%H:%M:%SZ"),
"x_opencti_weight": self.helper.connect_confidence_level,
},
)
)
# Event Attack Patterns
for attack_pattern in event_elements["attack_patterns"]:
if len(event_elements["malwares"]) > 0:
threats = event_elements["malwares"]
elif len(event_elements["intrusion_sets"]) > 0:
threats = event_elements["intrusion_sets"]
else:
threats = []
for threat in threats:
relationship_uses = Relationship(
relationship_type="uses",
created_by_ref=author,
source_ref=threat.id,
target_ref=attack_pattern.id,
description=attribute["comment"],
object_marking_refs=attribute_markings,
custom_properties={
"x_opencti_first_seen": datetime.utcfromtimestamp(
int(attribute["timestamp"])
).strftime("%Y-%m-%dT%H:%M:%SZ"),
"x_opencti_last_seen": datetime.utcfromtimestamp(
int(attribute["timestamp"])
).strftime("%Y-%m-%dT%H:%M:%SZ"),
"x_opencti_weight": self.helper.connect_confidence_level,
"x_opencti_ignore_dates": True,
},
)
relationships.append(relationship_uses)
relationship_indicates = Relationship(
relationship_type="indicates",
created_by_ref=author,
source_ref=indicator.id,
target_ref="malware--fa42a846-8d90-4e51-bc29-71d5b4802168", # Fake
description=attribute["comment"],
object_marking_refs=attribute_markings,
custom_properties={
"x_opencti_first_seen": datetime.utcfromtimestamp(
int(attribute["timestamp"])
).strftime("%Y-%m-%dT%H:%M:%SZ"),
"x_opencti_last_seen": datetime.utcfromtimestamp(
int(attribute["timestamp"])
).strftime("%Y-%m-%dT%H:%M:%SZ"),
"x_opencti_weight": self.helper.connect_confidence_level,
"x_opencti_source_ref": indicator.id,
"x_opencti_target_ref": relationship_uses.id,
},
)
relationships.append(relationship_indicates)
# Attribute Attack Patterns
for attack_pattern in attribute_elements["attack_patterns"]:
if len(attribute_elements["malwares"]) > 0:
threats = attribute_elements["malwares"]
elif len(attribute_elements["intrusion_sets"]) > 0:
threats = attribute_elements["intrusion_sets"]
else:
threats = []
for threat in threats:
relationship_uses = Relationship(
relationship_type="uses",
created_by_ref=author,
source_ref=threat.id,
target_ref=attack_pattern.id,
description=attribute["comment"],
object_marking_refs=attribute_markings,
custom_properties={
"x_opencti_first_seen": datetime.utcfromtimestamp(
int(attribute["timestamp"])
).strftime("%Y-%m-%dT%H:%M:%SZ"),
"x_opencti_last_seen": datetime.utcfromtimestamp(
int(attribute["timestamp"])
).strftime("%Y-%m-%dT%H:%M:%SZ"),
"x_opencti_weight": self.helper.connect_confidence_level,
"x_opencti_ignore_dates": True,
},
)
relationships.append(relationship_uses)
relationship_indicates = Relationship(
relationship_type="indicates",
created_by_ref=author,
source_ref=indicator.id,
target_ref="malware--fa42a846-8d90-4e51-bc29-71d5b4802168", # Fake
description=attribute["comment"],
object_marking_refs=attribute_markings,
custom_properties={
"x_opencti_first_seen": datetime.utcfromtimestamp(
int(attribute["timestamp"])
).strftime("%Y-%m-%dT%H:%M:%SZ"),
"x_opencti_last_seen": datetime.utcfromtimestamp(
int(attribute["timestamp"])
).strftime("%Y-%m-%dT%H:%M:%SZ"),
"x_opencti_weight": self.helper.connect_confidence_level,
"x_opencti_source_ref": indicator.id,
"x_opencti_target_ref": relationship_uses.id,
"x_opencti_ignore_dates": True,
},
)
relationships.append(relationship_indicates)
return {
"indicator": indicator,
"relationships": relationships,
"attribute_elements": attribute_elements,
"markings": attribute_markings,
}
except:
return None
