def create_indicator(ioc_type, ioc_value, scan_results):
if (ioc_type == "url") or (ioc_type == "ip-src") or (ioc_type == "ip_dst") or (ioc_type == "domain") or (ioc_type == "hostname"):
ioc_pattern = "[url:value = '" + ioc_value + "']"
indicator = Indicator(labels=["malicious-activity"], pattern=ioc_pattern, description=scan_results)
if (ioc_type == "md5") or (ioc_type == "sha256"):
ioc_pattern = "[file:hashes." + ioc_type + " = '" + ioc_value + "']"
indicator = Indicator(labels=["malicious-activity"], pattern=ioc_pattern, description=scan_results)
return indicator
def create_indicator(content):
name = "SHA256 for malware object"
description = "This hash indicates the present of Malware."
pattern = ''
# temp=re.findall('"sha256":"(.*?)"',content)
# if len(temp)!=0:
value = get(content, 'sha256')
if (value != -1):
pattern = "[file:hashes.'SHA-256'='" + value + "']"
else:
return None
indicator_types = ["malicious-activity"]
pattern_type = 'stix'
indicator_list = []
indicator = Indicator(name=name,
description=description,
pattern_type=pattern_type,
pattern=pattern,
indicator_types=indicator_types)
indicator_list.append(indicator)
name = "MD5 for malware object"
description = "This hash indicates the present of Malware."
md5 = content['md5']
md5_pattern = "[file:hashes.'MD5'='" + md5 + "']"
indicator1 = Indicator(name=name,
description=description,
pattern_type=pattern_type,
pattern=md5_pattern,
indicator_types=indicator_types)
indicator_list.append(indicator1)
name = "SHA1 for malware object"
description = "This hash indicates the present of Malware."
sha1 = content['sha1']
sha1_pattern = "[file:hashes.'SHA-1'='" + sha1 + "']"
indicator2 = Indicator(name=name,
description=description,
pattern_type=pattern_type,
pattern=sha1_pattern,
indicator_types=indicator_types)
indicator_list.append(indicator2)
name = "Size of malware object"
description = "Malware size."
size = content['static_analysis']['binary_info']['size']
size_pattern = "[file:size=" + str(size) + "]"
indicator3 = Indicator(name=name,
description=description,
pattern_type=pattern_type,
pattern=size_pattern,
indicator_types=indicator_types)
indicator_list.append(indicator3)
return indicator_list
def create_indicator(
pattern: str,
pattern_type: str,
created_by: Optional[Identity] = None,
name: Optional[str] = None,
description: Optional[str] = None,
valid_from: Optional[datetime] = None,
labels: Optional[List[str]] = None,
confidence: Optional[int] = None,
object_markings: Optional[List[MarkingDefinition]] = None,
x_opencti_main_observable_type: Optional[str] = None,
) -> Indicator:
"""Create an indicator."""
custom_properties: Dict[str, Any] = {
X_OPENCTI_SCORE: DEFAULT_X_OPENCTI_SCORE
}
if x_opencti_main_observable_type is not None:
custom_properties[
X_OPENCTI_MAIN_OBSERVABLE_TYPE] = x_opencti_main_observable_type
return Indicator(
id=_create_random_identifier("indicator"),
created_by_ref=created_by,
name=name,
description=description,
pattern=pattern,
pattern_type=pattern_type,
valid_from=valid_from,
labels=labels,
confidence=confidence,
object_marking_refs=object_markings,
custom_properties=custom_properties,
)
def create_indicator(
name: str,
author: Identity,
description: str,
valid_from: datetime,
observable_type: str,
observable_value: str,
pattern_type: str,
pattern_value: str,
indicator_pattern: str,
object_marking_refs: List[MarkingDefinition],
) -> Indicator:
"""Create an indicator."""
return Indicator(
created_by_ref=author,
name=name,
description=description,
pattern=str(pattern_value),
valid_from=valid_from,
labels=["malicious-activity"],
object_marking_refs=object_marking_refs,
custom_properties={
CustomProperties.OBSERVABLE_TYPE: observable_type,
CustomProperties.OBSERVABLE_VALUE: observable_value,
CustomProperties.PATTERN_TYPE: pattern_type,
CustomProperties.INDICATOR_PATTERN: indicator_pattern,
},
)
def create_indicator(
pattern: str,
pattern_type: str,
created_by: Optional[Identity] = None,
created: Optional[datetime] = None,
modified: Optional[datetime] = None,
name: Optional[str] = None,
description: Optional[str] = None,
valid_from: Optional[datetime] = None,
labels: Optional[List[str]] = None,
confidence: Optional[int] = None,
object_markings: Optional[List[MarkingDefinition]] = None,
) -> Indicator:
"""Create an indicator."""
return Indicator(
id=_create_random_identifier("indicator"),
created_by_ref=created_by,
created=created,
modified=modified,
name=name,
description=description,
pattern=pattern,
pattern_type=pattern_type,
valid_from=valid_from,
labels=labels,
confidence=confidence,
object_marking_refs=object_markings,
custom_properties={X_OPENCTI_SCORE: DEFAULT_X_OPENCTI_SCORE},
)
def rel_fs_store():
cam = Campaign(id=CAMPAIGN_ID, **CAMPAIGN_KWARGS)
idy = Identity(id=IDENTITY_ID, **IDENTITY_KWARGS)
ind = Indicator(id=INDICATOR_ID, **INDICATOR_KWARGS)
mal = Malware(id=MALWARE_ID, **MALWARE_KWARGS)
rel1 = Relationship(ind, 'indicates', mal, id=RELATIONSHIP_IDS[0])
rel2 = Relationship(mal, 'targets', idy, id=RELATIONSHIP_IDS[1])
rel3 = Relationship(cam, 'uses', mal, id=RELATIONSHIP_IDS[2])
stix_objs = [cam, idy, ind, mal, rel1, rel2, rel3]
fs = FileSystemStore(FS_PATH)
for o in stix_objs:
fs.add(o)
yield fs
for o in stix_objs:
filepath = os.path.join(FS_PATH, o.type, o.id,
_timestamp2filename(o.modified) + '.json')
# Some test-scoped fixtures (e.g. fs_store) delete all campaigns, so by
# the time this module-scoped fixture tears itself down, it may find
# its campaigns already gone, which causes not-found errors.
try:
os.remove(filepath)
except OSError as e:
# 3 is the ERROR_PATH_NOT_FOUND windows error code. Which has an
# errno symbolic value, but not the windows meaning...
if e.errno in (errno.ENOENT, 3):
continue
raise
def rel_mem_store():
cam = Campaign(id=CAMPAIGN_ID, **CAMPAIGN_KWARGS)
idy = Identity(id=IDENTITY_ID, **IDENTITY_KWARGS)
ind = Indicator(id=INDICATOR_ID, **INDICATOR_KWARGS)
mal = Malware(id=MALWARE_ID, **MALWARE_KWARGS)
rel1 = Relationship(ind, 'indicates', mal, id=RELATIONSHIP_IDS[0])
rel2 = Relationship(mal, 'targets', idy, id=RELATIONSHIP_IDS[1])
rel3 = Relationship(cam, 'uses', mal, id=RELATIONSHIP_IDS[2])
stix_objs = [cam, idy, ind, mal, rel1, rel2, rel3]
yield MemoryStore(stix_objs)
def setUp(self):
self.ts = datetime.now().astimezone()
self.ioc_id = "indicator--42d31a5b-2da0-4bdd-9823-1723a98fc2fb"
self.ioc_value = "example.com"
self.ioc = Indicator(
id=self.ioc_id,
pattern_type="stix",
pattern=f"[domain-name:value = '{self.ioc_value}']",
)
self.sighting_context = {
"ts": "2017-03-03T23:56:09.652643840",
"uid": "CMeLkt11aTqwgN4FI9",
"id.orig_h": "172.31.130.19",
"id.orig_p": 43872,
"id.resp_h": "172.31.129.17",
"id.resp_p": 20004,
"proto": "tcp",
"service": None,
"duration": 0.025249,
"orig_bytes": 311,
"resp_bytes": 999,
"conn_state": "SF",
"local_orig": None,
"local_resp": None,
"missed_bytes": 0,
"history": "ShADadFf",
"orig_pkts": 9,
"orig_ip_bytes": 787,
"resp_pkts": 7,
"resp_ip_bytes": 1371,
"tunnel_parents": [],
"alert": {
"signature": "VAST-RETRO Generic IoC match for: 172.31.129.17",
"category": "Potentially Bad Traffic",
"action": "allowed",
},
"event_type": "alert",
"_extra": {
"vast-ioc": "172.31.129.17"
},
"source": "VAST",
}
self.sighting = Sighting(sighting_of_ref=self.ioc_id)
self.snapshot_id = "SNAPSHOT_UUID"
self.snapshot = timedelta(days=42, hours=23, minutes=13, seconds=37)
self.snapshot_request = SnapshotRequest(MessageType.SIGHTING,
self.snapshot_id,
self.snapshot)
self.snapshot_envelope_indicator = SnapshotEnvelope(
MessageType.INDICATOR, self.snapshot_id, self.ioc)
self.snapshot_envelope_sighting = SnapshotEnvelope(
MessageType.SIGHTING, self.snapshot_id, self.sighting)
def _map_to_threatbus(
self, data: dict, opencti_action: str
) -> Union[Indicator, None]:
"""
Inspects the given OpenCTI data point and either returns a valid STIX-2
Indicator or None.
@param data A dict object with OpenCTI SSE data
@param opencti_action A string indicating what happened to this item
(either `create`, `update` or `delete`)
@return a STIX-2 Indicator or None
"""
opencti_id: str = data.get("x_opencti_id", None)
if not opencti_id:
self.opencti_helper.log_error(
"Cannot process data without 'x_opencti_id' field"
)
return
event_id = data.get("id", None)
update = data.get("x_data_update", {})
added = update.get("add", {})
added_ids = added.get("x_opencti_stix_ids", [])
type_ = data.get("type", None)
if type_ == "indicator" and len(added_ids) == 1 and added_ids[0] == event_id:
# Discard the update if it was empty. An update is empty when the
# only "changed" attribute is the stix_id and it changed to its own
# already existing value. Example:
# data ~ {'id': 'XXX', 'x_data_update': {'add': {'x_opencti_stix_ids': ['XXX']}}}
return
if opencti_action == "delete":
indicator: dict = data
indicator[
ThreatBusSTIX2Constants.X_THREATBUS_UPDATE.value
] = Operation.REMOVE.value
else:
indicator: dict = self.opencti_helper.api.indicator.read(id=opencti_id)
if not indicator:
# we are only interested in indicators at this time
return
# overwrite custom OpenCTI ID
indicator["id"] = indicator.get("standard_id")
if opencti_action == "update":
indicator[
ThreatBusSTIX2Constants.X_THREATBUS_UPDATE.value
] = Operation.EDIT.value
# only propagate indicators that are toggled for detection or the user
# enabled forwarding of all indicators regardless of the toggle
detection_enabled: bool = indicator.get("x_opencti_detection", False)
if not detection_enabled and self.forward_all_iocs is not True:
return
return Indicator(**indicator, allow_custom=True)
def forge_indicator(url, score):
indicator = Indicator(
name="Suspicious URL",
labels=["malicious-activity"],
created=get_time(),
description="This URL has been detected as malicious with a score of "
+ str(score) + ".",
pattern="[url:value = '" + url + "']")
return indicator
def test_indicator_to_vast_query(self):
## test IP
ip = "6.6.6.6"
expected_vast_query = f"{ip}"
ioc = Indicator(pattern_type="stix",
pattern=f"[ipv4-addr:value = '{ip}']")
self.assertEqual(expected_vast_query, indicator_to_vast_query(ioc))
## test URL
url = "example.com/foo/bar?query=123"
expected_vast_query = f'"{url}" == net.uri'
ioc = Indicator(pattern_type="stix", pattern=f"[url:value = '{url}']")
self.assertEqual(expected_vast_query, indicator_to_vast_query(ioc))
## test domain
domain = "example.com"
expected_vast_query = f'"{domain}" == net.domain || "{domain}" == net.hostname'
ioc = Indicator(pattern_type="stix",
pattern=f"[domain-name:value = '{domain}']")
self.assertEqual(expected_vast_query, indicator_to_vast_query(ioc))
def test_indicator_to_vast_query(self):
## test IP
expected_vast_query = f"{self.ioc_value}"
self.assertEqual(expected_vast_query,
indicator_to_vast_query(self.indicator))
## test URL
url = "example.com/foo/bar?query=123"
expected_vast_query = f'"{url}" in net.uri'
other_ioc = Indicator(pattern_type="stix",
pattern=f"[url:value = '{url}']")
self.assertEqual(expected_vast_query,
indicator_to_vast_query(other_ioc))
## test domain
domain = "example.com"
expected_vast_query = f'"{domain}" == net.domain || "{domain}" == net.hostname'
other_ioc = Indicator(pattern_type="stix",
pattern=f"[domain-name:value = '{domain}']")
self.assertEqual(expected_vast_query,
indicator_to_vast_query(other_ioc))
def setUp(self):
self.ts = datetime.now(timezone.utc).astimezone()
self.indicator_id = "indicator--46b3f973-5c03-41fc-9efe-49598a267a35"
self.operation = Operation.REMOVE
self.ioc_value = "6.6.6.6"
self.pattern = f"[ipv4-addr:value = '{self.ioc_value}']"
self.indicator = Indicator(
pattern_type="stix",
pattern=self.pattern,
created=self.ts,
id=self.indicator_id,
)
self.valid_query_result = f'{{"timestamp": "{self.ts}", "flow_id": 1840147514011873, "pcap_cnt": 626, "src_ip": "{self.ioc_value}", "src_port": 1193, "dest_ip": "65.54.95.64", "dest_port": 80, "proto": "TCP", "event_type": "http", "community_id": "1:AzSEWwmsqEKUX5qrReAHI3Rpizg=", "http": {{"hostname": "download.windowsupdate.com", "url": "/v9/windowsupdate/a/selfupdate/WSUS3/x86/Other/wsus3setup.cab?0911180916", "http_port": null, "http_user_agent": "Windows-Update-Agent", "http_content_type": "application/octet-stream", "http_method": "HEAD", "http_refer": null, "protocol": "HTTP/1.1", "status": 200, "redirect": null, "length": 0}}, "tx_id": 0}}'
self.valid_matcher_result = f'{{"ts": "{self.ts}", "data_id": 8, "indicator_id": 5, "matcher": "threatbus-syeocdkfcy", "value": "{self.ioc_value}", "reference": "threatbus__{self.indicator_id}"}}'
def setUp(self):
self.ts = datetime.now(timezone.utc).astimezone()
self.indicator_id = "indicator--de0c3d3f-02ee-4086-88f1-51200ac831f7"
self.point_ioc = "evil.com"
self.pattern = f"[domain-name:value = '{self.point_ioc}']"
self.indicator = Indicator(
id=self.indicator_id,
created=self.ts,
modified=self.ts,
pattern_type="stix",
pattern=self.pattern,
)
self.module_namespace = "TestNamespace"
self.logger = getLogger("test")
def setUp(self):
self.ts = datetime.now(timezone.utc).astimezone()
self.indicator_id = "indicator--46b3f973-5c03-41fc-9efe-49598a267a35"
self.operation = Operation.REMOVE
self.ioc_value = "evil.com"
self.pattern = f"[domain-name:value = '{self.ioc_value}']"
self.indicator = Indicator(
pattern_type="stix",
pattern=self.pattern,
created=self.ts,
id=self.indicator_id,
)
self.valid_query_result = f'{{"timestamp": "{self.ts}", "flow_id": 1840147514011873, "pcap_cnt": 626, "src_ip": "{self.ioc_value}", "src_port": 1193, "dest_ip": "65.54.95.64", "dest_port": 80, "proto": "TCP", "event_type": "http", "community_id": "1:AzSEWwmsqEKUX5qrReAHI3Rpizg=", "http": {{"hostname": "download.windowsupdate.com", "url": "/v9/windowsupdate/a/selfupdate/WSUS3/x86/Other/wsus3setup.cab?0911180916", "http_port": null, "http_user_agent": "Windows-Update-Agent", "http_content_type": "application/octet-stream", "http_method": "HEAD", "http_refer": null, "protocol": "HTTP/1.1", "status": 200, "redirect": null, "length": 0}}, "tx_id": 0}}'
self.valid_matcher_result = f'{{"indicator": {{"value": "{self.ioc_value}", "context": "\\"threatbus__{self.indicator_id}\\""}}, "type": "zeek.dns", "event": {{"ts": "{self.ts}", "uid": "CGG3VBvnp9JKfVP56", "id.orig_h": "172.19.0.2", "id.orig_p": 37241, "id.resp_h": "1.1.1.1", "id.resp_p": 53, "proto": "udp", "trans_id": 53396, "rtt": "18.42ms", "query": "reddit.com", "qclass": 1, "qclass_name": "C_INTERNET", "qtype": 1, "qtype_name": "A", "rcode": 0, "rcode_name": "NOERROR", "AA": false, "TC": false, "RD": true, "RA": true, "Z": 2, "answers": ["151.101.193.140", "151.101.65.140", "151.101.1.140", "151.101.129.140"], "TTLs": ["2.47m", "2.47m", "2.47m", "2.47m"], "rejected": false}} }}'
def outputstix2(results, config):
indicators = []
if results is None:
return
if config.debug:
print("Creating STIX2 indicators", file=sys.stderr)
for result in results:
# Indicator setup based on https://oasis-open.github.io/cti-documentation/examples/indicator-for-malicious-url
description = '|'.join(
result.get('threat_types', []) + result.get('last_seen_as', []) +
result.get('malware_family', []))
if result['type'] == 'url':
indicator = Indicator(valid_from=result.get(
'last_seen', result['last_published']),
labels="malicious-activity",
description=description,
pattern="[url:value = '%s']" % result["key"],
pattern_type="stix")
elif result['type'] == 'domain':
indicator = Indicator(
valid_from=result.get('last_seen', result['last_published']),
labels="malicious-activity",
description=description,
pattern="[domain-name:value = '%s']" % result["key"],
pattern_type="stix")
elif result['type'] == 'ip':
indicator = Indicator(
valid_from=result.get('last_seen', result['last_published']),
labels="malicious-activity",
description=description,
pattern="[ipv4-addr:value = '%s']" % result["key"],
pattern_type="stix")
indicators.append(indicator)
if config.debug:
print("Creating STIX2 bundle", file=sys.stderr)
bundle = Bundle(indicators) # This takes WAY too long, needs measurement
return bundle
def setUp(self):
self.observations = [
{
"type": "identity",
},
{"type": "observed-data", "some-prop": "value"},
{
"type": "observed-data",
"some-prop": "value",
"last_observed": "2021-05-04T15:15:58.919Z",
},
]
self.indicator = Indicator(
pattern="[ipv4-addr:value = '6.6.6.6']", pattern_type="stix"
)
def writeStix(shadyUrl, score):
# a mettre dans ta callback
indicator = Indicator(
name="Malicious site (phishing)",
labels=["malicious-activity"],
pattern="[url:value = \'" + shadyUrl + "\']",
created="{}".format(datetime.datetime.now().date()),
valid_from="{}".format(datetime.datetime.now().date()),
description=
"Possible malicious site detected by LID-TP1 with a score of " +
str(score) + ", shady url : " + shadyUrl)
mem.add(indicator)
mem.save_to_file(
"./database/{}.json".format(datetime.datetime.now().date())
) # end of the function, if the site is a phishing site (score > 0) -> write in STIXv2.0 file
def rel_fs_store():
cam = Campaign(id=CAMPAIGN_ID, **CAMPAIGN_KWARGS)
idy = Identity(id=IDENTITY_ID, **IDENTITY_KWARGS)
ind = Indicator(id=INDICATOR_ID, **INDICATOR_KWARGS)
mal = Malware(id=MALWARE_ID, **MALWARE_KWARGS)
rel1 = Relationship(ind, 'indicates', mal, id=RELATIONSHIP_IDS[0])
rel2 = Relationship(mal, 'targets', idy, id=RELATIONSHIP_IDS[1])
rel3 = Relationship(cam, 'uses', mal, id=RELATIONSHIP_IDS[2])
stix_objs = [cam, idy, ind, mal, rel1, rel2, rel3]
fs = FileSystemStore(FS_PATH)
for o in stix_objs:
fs.add(o)
yield fs
for o in stix_objs:
os.remove(os.path.join(FS_PATH, o.type, o.id + '.json'))
def setUp(self):
# self.timestamp = datetime.now(timezone.utc).astimezone()
self.raw_ts = 1579104545
self.timestamp = datetime.fromtimestamp(self.raw_ts).astimezone(
timezone.utc)
self.ioc_id = "indicator--42d31a5b-2da0-4bdd-9823-1723a98fc2fb"
self.ioc_value = "example.com"
self.ioc = Indicator(
id=self.ioc_id,
created=self.timestamp,
pattern_type="stix",
pattern=f"[domain-name:value = '{self.ioc_value}']",
)
self.ep = broker.Endpoint()
self.ep.peer(self.host, self.port)
self.subscribe_callback.reset_mock()
self.unsubscribe_callback.reset_mock()
def newioc_view(request):
#if request.method == 'POST':
form = IndicatorForm(request.POST)
if form.is_valid():
name = form.cleaned_data['name']
labels = form.cleaned_data['labels']
pattern = form.cleaned_data['pattern']
indicator = Indicator(name=name, labels=labels, pattern=pattern)
print(indicator)
#saving the indicator to database
return redirect('/dashboard')
context = {'form': form}
return render(request, 'newioc.html', context)
def test_zmq_app_plugin_message_roundtrip(self):
"""
Backend-agnostic message passing scenario. Sends a fixed amount of
messages via the threatbus ZeroMQ app plugin, subscribes to Threat Bus,
and checks if the initially sent messages can be retrieved back.
"""
result_q = queue.Queue()
items = 2
topics = ["stix2/indicator", "stix2/sighting"]
rec = threading.Thread(target=zmq_receiver.forward,
args=(items, topics, result_q),
daemon=False)
rec.start()
ioc = Indicator(pattern_type="stix",
pattern="[ipv4-addr:value = '6.6.6.6']")
zmq_sender.send(
"stix2/indicator",
ioc.serialize(),
port=13372,
bind=False,
)
sighting = Sighting(sighting_of_ref=ioc.id)
zmq_sender.send(
"stix2/sighting",
sighting.serialize(),
port=13372,
bind=False,
)
time.sleep(1)
self.assertEqual(result_q.qsize(), items)
event = result_q.get(timeout=1)
self.assertIsNotNone(event)
self.assertEqual(parse(event), ioc)
result_q.task_done()
event = result_q.get(timeout=1)
self.assertIsNotNone(event)
self.assertEqual(parse(event), sighting)
result_q.task_done()
self.assertEqual(0, result_q.qsize())
result_q.join()
rec.join(timeout=1)
def _handle_indicator(self, indicator: Indicator):
"""
Handles a STIX-2 Indicator update received via Threat Bus. Does nothing
in case the indicator already exists and the new indicator does not add
any new fields/values to the existing indicator. By doing so, this
function effectively avoids double updates that otherwise would result
in SSE events without a real change.
@param indicator The STIX-2 Indicator received from Threat Bus
"""
if type(indicator) is not Indicator:
self.opencti_helper.log_error(
f"Error ingesting indicator from Threat Bus. Expected a STIX-2 Indicator: {indicator}"
)
return
if (
ThreatBusSTIX2Constants.X_THREATBUS_UPDATE.value
in indicator.object_properties()
and indicator.x_threatbus_update == Operation.REMOVE.value
):
# OpenCTI does not support indicator removal via API calls (yet)
return
lookup_resp = self.opencti_helper.api.indicator.read(id=indicator.id)
if not lookup_resp:
# No indicator with that ID exists already.
self._create_or_update_indicator(indicator)
return
lookup_resp["id"] = lookup_resp["standard_id"]
lookup_indicator = Indicator(**lookup_resp, allow_custom=True)
# We found an existing indicator. To avoid double updates in the SSE
# stream we check if the indicator from Threat Bus adds anything new.
for prop, new_value in indicator.items():
if prop == "id" or prop.startswith("x_"):
continue
existing_value = lookup_indicator.get(prop, None)
if existing_value is None or new_value != existing_value:
self._create_or_update_indicator(indicator)
return
def _map_to_threatbus(
self, data: dict, opencti_action: str
) -> Union[Indicator, None]:
"""
Inspects the given OpenCTI data point and either returns a valid STIX-2
Indicator or None.
@param data A dict object with OpenCTI SSE data
@param opencti_action A string indicating what happened to this item
(either `create`, `update` or `delete`)
@return a STIX-2 Indicator or None
"""
opencti_id: str = data.get("x_opencti_id", None)
if not opencti_id:
self.opencti_helper.log_error(
"Cannot process data without 'x_opencti_id' field"
)
return
indicator: dict = self.opencti_helper.api.indicator.read(id=opencti_id)
if not indicator:
# we are only interested in indicators at this time
return
detection_enabled: bool = indicator.get("x_opencti_detection", False)
if not detection_enabled and self.forward_all_iocs is not True:
# only propagate indicators that are toggled for detection or the
# user enabled forwarding of all indicators regardless of the toggle
return
# overwrite custom OpenCTI ID
indicator["id"] = indicator.get("standard_id")
if opencti_action == "update":
indicator[
ThreatBusSTIX2Constants.X_THREATBUS_UPDATE.value
] = Operation.EDIT.value
if opencti_action == "delete":
indicator[
ThreatBusSTIX2Constants.X_THREATBUS_UPDATE.value
] = Operation.REMOVE.value
return Indicator(**indicator, allow_custom=True)
def setUp(self):
self.created = datetime.now().astimezone()
self.indicator_id = "indicator--df0f9a0e-c3b6-4f53-b0cf-5e9c454ee0cc"
self.pattern = "[ipv4-addr:value = '6.6.6.6']"
self.pattern_type = "stix2"
self.operation = Operation.REMOVE
self.indicator = Indicator(
id=self.indicator_id,
pattern=self.pattern,
pattern_type=self.pattern_type,
created=self.created,
valid_from=self.created,
modified=self.created,
)
self.sighting_source = "VAST"
self.sighting_id = "sighting--df0f9a0e-c3b6-4f53-b0cf-5e9c454ee0cc"
self.sighting_context = {
"ts": "2017-03-03T23:56:09.652643840",
"uid": "CMeLkt11aTqwgN4FI9",
"id.orig_h": "172.31.130.19",
"id.orig_p": 43872,
"id.resp_h": "172.31.129.17",
"id.resp_p": 20004,
"proto": "tcp",
"service": None,
"duration": 0.025249,
"orig_bytes": 311,
"resp_bytes": 999,
"conn_state": "SF",
"local_orig": None,
"local_resp": None,
"missed_bytes": 0,
"history": "ShADadFf",
"orig_pkts": 9,
"orig_ip_bytes": 787,
"resp_pkts": 7,
"resp_ip_bytes": 1371,
"tunnel_parents": [],
"alert": {
"signature": "VAST-RETRO Generic IoC match for: 172.31.129.17",
"category": "Potentially Bad Traffic",
"action": "allowed",
},
"event_type": "alert",
"_extra": {
"vast-ioc": "172.31.129.17"
},
}
self.sighting = Sighting(
id=self.sighting_id,
created=self.created,
modified=self.created,
sighting_of_ref=self.indicator_id,
custom_properties={
ThreatBusSTIX2Constants.X_THREATBUS_SIGHTING_CONTEXT.value:
self.sighting_context,
ThreatBusSTIX2Constants.X_THREATBUS_INDICATOR.value:
self.indicator,
ThreatBusSTIX2Constants.X_THREATBUS_SOURCE.value:
self.sighting_source,
},
)
self.snapshot_id = "SNAPSHOT_UUID"
self.snapshot = timedelta(days=42, hours=23, minutes=13, seconds=37)
self.snapshot_request = SnapshotRequest(MessageType.SIGHTING,
self.snapshot_id,
self.snapshot)
self.snapshot_envelope_indicator = SnapshotEnvelope(
MessageType.INDICATOR, self.snapshot_id, self.indicator)
self.snapshot_envelope_sighting = SnapshotEnvelope(
MessageType.SIGHTING, self.snapshot_id, self.sighting)
def main(message, context):
if message['message_type'] == "heartbeat":
return
if message['message_type'] == "certificate_update":
all_domains = message['data']['leaf_cert']['all_domains']
if len(all_domains) == 0:
return
else:
domain = all_domains[0]
rows, columns = os.popen('stty size', 'r').read().split()
print(" " * int(columns), end='\r')
print(WHITE + domain, end='\r')
url = ""
sp = domain.split('.')
if "fr" not in sp:
return
else:
url = all_domains[0]
if url.startswith("*."):
url = url[2:]
print(" " * int(columns), end='\r')
print(GRAY + url + WHITE, end="\r") # URL acceptee et en attente
# start of url check
score = 0
# Detect phishing
# etape 1 : Verification de l'url
sc, site = verif_url_alter(url)
score += (sc * 1) #max:100
assert site != "" and site != None, "Aucun site correspondant"
# etape 2 : Verification de la distance syntaxique de l'auteur
score_whois, date_off, date_fake = check_whois(url, site)
score_whois = score_whois * 0.3 # max: 21
score += score_whois
# etape 3 : Verification par virustotal
score_antivirus = antivirus(url)
score_antivirus = (score_antivirus * 1)
score += score_antivirus # max: 100
# etape 4 - 5 : Verification de la presence d'un formulaire "login/password" et d'import javascript externe
score_ident, score_js = require_ident(url)
score_ident = score_ident * 0.5
score_js = score_js * 0.5
score += score_ident # max: 50
score += score_js # max: 50
# etape 6 : Verification date de creation des domaines officiels et scanne
score_domain_date = check_domain_date(date_off, date_fake)
score_domain_date = score_domain_date * 0.2
score += score_domain_date # max: 20
# etape 7: Verification de mots suspects dans l'url
score_suspects = mots_suspects(url)
score += (score_suspects * 1) # max: 1925 # 3 mots en moyenne ?
#score max : 2266
b, s = isPhishing(score)
if b:
print(
RED + url + WHITE + " : " + s +
" (score=%.2f -- mot-suspects:%.2f url:%.2f whois:%.2f antivirus:%.2f identifiants:%.2f javascript:%.2f date_domaines:%.2f)"
% (score, score_suspects, sc, score_whois, score_antivirus,
score_ident, score_js, score_domain_date))
date = datetime.datetime.now().strftime('%m/%d/%y %H:%M:%S')
date_until_valid = datetime.datetime.now() + datetime.timedelta(
days=30)
# add in STIXv2.0 file
indicator = Indicator(
name="Malicious site (phishing)",
labels=["malicious-activity"],
pattern="[url:value = '{}']".format(url),
created="{}".format(date),
valid_from="{}".format(date),
valid_until="{}".format(date_until_valid),
description=
"score=%.2f -- mot-suspects:%.2f url:%.2f whois:%.2f antivirus:%.2f identifiants:%.2f javascript:%.2f date_domaines:%.2f)"
% (score, score_suspects, sc, score_whois, score_antivirus,
score_ident, score_js, score_domain_date))
mem.add(indicator)
mem.save_to_file(
"../database/site_database{}.json".format(
datetime.datetime.now().date())
) #end of the function, if the site is a phishing site (score > 0) -> write in STIXv2.0 file
else:
print(GREEN + url + WHITE + " : " + s + " (score=%.2f)" % score)
print(WHITE)
def process_attribute(
self,
author,
event_elements,
event_markings,
attribute_external_references,
attribute,
):
try:
resolved_attributes = self.resolve_type(
attribute["type"], attribute["value"]
)
if resolved_attributes is None:
return None
for resolved_attribute in resolved_attributes:
### Pre-process
# Elements
attribute_elements = self.prepare_elements(attribute["Galaxy"], author)
# Markings & Tags
attribute_tags = []
if "Tag" in attribute:
attribute_markings = self.resolve_markings(
attribute["Tag"], with_default=False
)
attribute_tags = self.resolve_tags(attribute["Tag"])
if len(attribute_markings) == 0:
attribute_markings = event_markings
else:
attribute_markings = event_markings
### Create the indicator
observable_type = resolved_attribute["type"]
observable_value = resolved_attribute["value"]
name = resolved_attribute["value"]
pattern_type = "stix"
# observable type is yara for instance
if observable_type in PATTERNTYPES:
pattern_type = observable_type
observable_type = "Unknown"
genuine_pattern = (
"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']"
)
pattern = observable_value
name = (
attribute["comment"]
if len(attribute["comment"]) > 0
else observable_type
)
# observable type is not in stix 2
elif observable_type not in OPENCTISTIX2:
return None
# observable type is in stix
else:
if "transform" in OPENCTISTIX2[observable_type]:
if (
OPENCTISTIX2[observable_type]["transform"]["operation"]
== "remove_string"
):
observable_value = observable_value.replace(
OPENCTISTIX2[observable_type]["transform"]["value"], ""
)
lhs = ObjectPath(
OPENCTISTIX2[observable_type]["type"],
OPENCTISTIX2[observable_type]["path"],
)
genuine_pattern = str(
ObservationExpression(
EqualityComparisonExpression(lhs, observable_value)
)
)
pattern = genuine_pattern
indicator = Indicator(
name=name,
description=attribute["comment"],
pattern=genuine_pattern,
valid_from=datetime.utcfromtimestamp(
int(attribute["timestamp"])
).strftime("%Y-%m-%dT%H:%M:%SZ"),
labels=["malicious-activity"],
created_by_ref=author,
object_marking_refs=attribute_markings,
external_references=attribute_external_references,
custom_properties={
"x_opencti_indicator_pattern": pattern,
"x_opencti_observable_type": observable_type,
"x_opencti_observable_value": observable_value,
"x_opencti_pattern_type": pattern_type,
"x_opencti_tags": attribute_tags,
},
)
### Create the relationships
relationships = []
# Event threats
for threat in (
event_elements["intrusion_sets"]
+ event_elements["malwares"]
+ event_elements["tools"]
):
relationships.append(
Relationship(
relationship_type="indicates",
created_by_ref=author,
source_ref=indicator.id,
target_ref=threat.id,
description=attribute["comment"],
object_marking_refs=attribute_markings,
custom_properties={
"x_opencti_first_seen": datetime.utcfromtimestamp(
int(attribute["timestamp"])
).strftime("%Y-%m-%dT%H:%M:%SZ"),
"x_opencti_last_seen": datetime.utcfromtimestamp(
int(attribute["timestamp"])
).strftime("%Y-%m-%dT%H:%M:%SZ"),
"x_opencti_weight": self.helper.connect_confidence_level,
},
)
)
# Attribute threats
for threat in (
attribute_elements["intrusion_sets"]
+ attribute_elements["malwares"]
+ attribute_elements["tools"]
):
relationships.append(
Relationship(
relationship_type="indicates",
created_by_ref=author,
source_ref=indicator.id,
target_ref=threat.id,
description=attribute["comment"],
object_marking_refs=attribute_markings,
custom_properties={
"x_opencti_first_seen": datetime.utcfromtimestamp(
int(attribute["timestamp"])
).strftime("%Y-%m-%dT%H:%M:%SZ"),
"x_opencti_last_seen": datetime.utcfromtimestamp(
int(attribute["timestamp"])
).strftime("%Y-%m-%dT%H:%M:%SZ"),
"x_opencti_weight": self.helper.connect_confidence_level,
},
)
)
# Event Attack Patterns
for attack_pattern in event_elements["attack_patterns"]:
if len(event_elements["malwares"]) > 0:
threats = event_elements["malwares"]
elif len(event_elements["intrusion_sets"]) > 0:
threats = event_elements["intrusion_sets"]
else:
threats = []
for threat in threats:
relationship_uses = Relationship(
relationship_type="uses",
created_by_ref=author,
source_ref=threat.id,
target_ref=attack_pattern.id,
description=attribute["comment"],
object_marking_refs=attribute_markings,
custom_properties={
"x_opencti_first_seen": datetime.utcfromtimestamp(
int(attribute["timestamp"])
).strftime("%Y-%m-%dT%H:%M:%SZ"),
"x_opencti_last_seen": datetime.utcfromtimestamp(
int(attribute["timestamp"])
).strftime("%Y-%m-%dT%H:%M:%SZ"),
"x_opencti_weight": self.helper.connect_confidence_level,
"x_opencti_ignore_dates": True,
},
)
relationships.append(relationship_uses)
relationship_indicates = Relationship(
relationship_type="indicates",
created_by_ref=author,
source_ref=indicator.id,
target_ref="malware--fa42a846-8d90-4e51-bc29-71d5b4802168", # Fake
description=attribute["comment"],
object_marking_refs=attribute_markings,
custom_properties={
"x_opencti_first_seen": datetime.utcfromtimestamp(
int(attribute["timestamp"])
).strftime("%Y-%m-%dT%H:%M:%SZ"),
"x_opencti_last_seen": datetime.utcfromtimestamp(
int(attribute["timestamp"])
).strftime("%Y-%m-%dT%H:%M:%SZ"),
"x_opencti_weight": self.helper.connect_confidence_level,
"x_opencti_source_ref": indicator.id,
"x_opencti_target_ref": relationship_uses.id,
},
)
relationships.append(relationship_indicates)
# Attribute Attack Patterns
for attack_pattern in attribute_elements["attack_patterns"]:
if len(attribute_elements["malwares"]) > 0:
threats = attribute_elements["malwares"]
elif len(attribute_elements["intrusion_sets"]) > 0:
threats = attribute_elements["intrusion_sets"]
else:
threats = []
for threat in threats:
relationship_uses = Relationship(
relationship_type="uses",
created_by_ref=author,
source_ref=threat.id,
target_ref=attack_pattern.id,
description=attribute["comment"],
object_marking_refs=attribute_markings,
custom_properties={
"x_opencti_first_seen": datetime.utcfromtimestamp(
int(attribute["timestamp"])
).strftime("%Y-%m-%dT%H:%M:%SZ"),
"x_opencti_last_seen": datetime.utcfromtimestamp(
int(attribute["timestamp"])
).strftime("%Y-%m-%dT%H:%M:%SZ"),
"x_opencti_weight": self.helper.connect_confidence_level,
"x_opencti_ignore_dates": True,
},
)
relationships.append(relationship_uses)
relationship_indicates = Relationship(
relationship_type="indicates",
created_by_ref=author,
source_ref=indicator.id,
target_ref="malware--fa42a846-8d90-4e51-bc29-71d5b4802168", # Fake
description=attribute["comment"],
object_marking_refs=attribute_markings,
custom_properties={
"x_opencti_first_seen": datetime.utcfromtimestamp(
int(attribute["timestamp"])
).strftime("%Y-%m-%dT%H:%M:%SZ"),
"x_opencti_last_seen": datetime.utcfromtimestamp(
int(attribute["timestamp"])
).strftime("%Y-%m-%dT%H:%M:%SZ"),
"x_opencti_weight": self.helper.connect_confidence_level,
"x_opencti_source_ref": indicator.id,
"x_opencti_target_ref": relationship_uses.id,
"x_opencti_ignore_dates": True,
},
)
relationships.append(relationship_indicates)
return {
"indicator": indicator,
"relationships": relationships,
"attribute_elements": attribute_elements,
"markings": attribute_markings,
}
except:
return None
def process_attribute(self, author, event_elements, event_markings, attribute):
resolved_attributes = self.resolve_type(attribute['type'], attribute['value'])
if resolved_attributes is None:
return None
for resolved_attribute in resolved_attributes:
### Pre-process
# Elements
attribute_elements = self.prepare_elements(attribute['Galaxy'])
# Markings
if 'Tag' in attribute:
attribute_markings = self.resolve_markings(attribute['Tag'], with_default=False)
if len(attribute_markings) == 0:
attribute_markings = event_markings
else:
attribute_markings = event_markings
### Create the indicator
observable_type = resolved_attribute['type']
observable_value = resolved_attribute['value']
pattern_type = 'stix'
if observable_type in PATTERNTYPES:
pattern_type = observable_type
elif observable_type not in OPENCTISTIX2:
return None
else:
if 'transform' in OPENCTISTIX2[observable_type]:
if OPENCTISTIX2[observable_type]['transform']['operation'] == 'remove_string':
observable_value = observable_value.replace(OPENCTISTIX2[observable_type]['transform']['value'], '')
lhs = ObjectPath(OPENCTISTIX2[observable_type]['type'], OPENCTISTIX2[observable_type]['path'])
observable_value = ObservationExpression(EqualityComparisonExpression(lhs, observable_value))
try:
indicator = Indicator(
name=resolved_attribute['value'],
description=attribute['comment'],
pattern=str(observable_value),
valid_from=datetime.utcfromtimestamp(int(attribute['timestamp'])).strftime('%Y-%m-%dT%H:%M:%SZ'),
labels=['malicious-activity'],
created_by_ref=author,
object_marking_refs=attribute_markings,
custom_properties={
'x_opencti_observable_type': resolved_attribute['type'],
'x_opencti_observable_value': resolved_attribute['value'],
'x_opencti_pattern_type': pattern_type
}
)
except:
return None
### Create the relationships
relationships = []
# Event threats
for threat in (event_elements['intrusion_sets'] + event_elements['malwares'] + event_elements['tools']):
relationships.append(
Relationship(
relationship_type='indicates',
created_by_ref=author,
source_ref=indicator.id,
target_ref=threat.id,
description=attribute['comment'],
object_marking_refs=attribute_markings,
custom_properties={
'x_opencti_first_seen': datetime.utcfromtimestamp(int(attribute['timestamp'])).strftime(
'%Y-%m-%dT%H:%M:%SZ'),
'x_opencti_last_seen': datetime.utcfromtimestamp(int(attribute['timestamp'])).strftime(
'%Y-%m-%dT%H:%M:%SZ'),
'x_opencti_weight': self.helper.connect_confidence_level
}
)
)
# Attribute threats
for threat in (attribute_elements['intrusion_sets'] + attribute_elements['malwares'] + attribute_elements[
'tools']):
relationships.append(
Relationship(
relationship_type='indicates',
created_by_ref=author,
source_ref=indicator.id,
target_ref=threat.id,
description=attribute['comment'],
object_marking_refs=attribute_markings,
custom_properties={
'x_opencti_first_seen': datetime.utcfromtimestamp(int(attribute['timestamp'])).strftime(
'%Y-%m-%dT%H:%M:%SZ'),
'x_opencti_last_seen': datetime.utcfromtimestamp(int(attribute['timestamp'])).strftime(
'%Y-%m-%dT%H:%M:%SZ'),
'x_opencti_weight': self.helper.connect_confidence_level
}
)
)
# Event Attack Patterns
for attack_pattern in event_elements['attack_patterns']:
if len(event_elements['malwares']) > 0:
threats = event_elements['malwares']
elif len(event_elements['intrusion_sets']) > 0:
threats = event_elements['intrusion_sets']
else:
threats = []
for threat in threats:
relationship_uses = Relationship(
relationship_type='uses',
created_by_ref=author,
source_ref=threat.id,
target_ref=attack_pattern.id,
description=attribute['comment'],
object_marking_refs=attribute_markings,
custom_properties={
'x_opencti_first_seen': datetime.utcfromtimestamp(int(attribute['timestamp'])).strftime(
'%Y-%m-%dT%H:%M:%SZ'),
'x_opencti_last_seen': datetime.utcfromtimestamp(int(attribute['timestamp'])).strftime(
'%Y-%m-%dT%H:%M:%SZ'),
'x_opencti_weight': self.helper.connect_confidence_level,
'x_opencti_ignore_dates': True
}
)
relationships.append(relationship_uses)
relationship_indicates = Relationship(
relationship_type='indicates',
created_by_ref=author,
source_ref=indicator.id,
target_ref='malware--fa42a846-8d90-4e51-bc29-71d5b4802168', # Fake
description=attribute['comment'],
object_marking_refs=attribute_markings,
custom_properties={
'x_opencti_first_seen': datetime.utcfromtimestamp(int(attribute['timestamp'])).strftime(
'%Y-%m-%dT%H:%M:%SZ'),
'x_opencti_last_seen': datetime.utcfromtimestamp(int(attribute['timestamp'])).strftime(
'%Y-%m-%dT%H:%M:%SZ'),
'x_opencti_weight': self.helper.connect_confidence_level,
'x_opencti_source_ref': indicator.id,
'x_opencti_target_ref': relationship_uses.id
}
)
relationships.append(relationship_indicates)
# Attribute Attack Patterns
for attack_pattern in attribute_elements['attack_patterns']:
if len(attribute_elements['malwares']) > 0:
threats = attribute_elements['malwares']
elif len(attribute_elements['intrusion_sets']) > 0:
threats = attribute_elements['intrusion_sets']
else:
threats = []
for threat in threats:
relationship_uses = Relationship(
relationship_type='uses',
created_by_ref=author,
source_ref=threat.id,
target_ref=attack_pattern.id,
description=attribute['comment'],
object_marking_refs=attribute_markings,
custom_properties={
'x_opencti_first_seen': datetime.utcfromtimestamp(int(attribute['timestamp'])).strftime(
'%Y-%m-%dT%H:%M:%SZ'),
'x_opencti_last_seen': datetime.utcfromtimestamp(int(attribute['timestamp'])).strftime(
'%Y-%m-%dT%H:%M:%SZ'),
'x_opencti_weight': self.helper.connect_confidence_level,
'x_opencti_ignore_dates': True
}
)
relationships.append(relationship_uses)
relationship_indicates = Relationship(
relationship_type='indicates',
created_by_ref=author,
source_ref=indicator.id,
target_ref='malware--fa42a846-8d90-4e51-bc29-71d5b4802168', # Fake
description=attribute['comment'],
object_marking_refs=attribute_markings,
custom_properties={
'x_opencti_first_seen': datetime.utcfromtimestamp(int(attribute['timestamp'])).strftime(
'%Y-%m-%dT%H:%M:%SZ'),
'x_opencti_last_seen': datetime.utcfromtimestamp(int(attribute['timestamp'])).strftime(
'%Y-%m-%dT%H:%M:%SZ'),
'x_opencti_weight': self.helper.connect_confidence_level,
'x_opencti_source_ref': indicator.id,
'x_opencti_target_ref': relationship_uses.id,
'x_opencti_ignore_dates': True
}
)
relationships.append(relationship_indicates)
return {
'indicator': indicator,
'relationships': relationships,
'attribute_elements': attribute_elements,
'markings': attribute_markings
}
aps = []
if args.mitre:
aps = ids_to_mitre_attack_patterns(args.mitre)
objects.extend(aps)
for ioc in all_ioc:
try:
title, pattern = ioc_to_title_and_pattern(ioc)
except Exception as e:
logging.error(f"Skipping indicator: {e}")
continue
description = " ".join(
title.split()[:2]) + f" involved with {args.threat_name}"
indicator = Indicator(labels="malicious-activity",
pattern_type='stix',
pattern=pattern,
valid_from=datetime.now(),
description=description,
name=title,
created_by_ref=identity)
relationship = Relationship(relationship_type='indicates',
source_ref=indicator.id,
target_ref=malware.id)
objects.append(indicator)
objects.append(relationship)
for ap in aps:
relationship = Relationship(relationship_type='indicates',
source_ref=indicator.id,
target_ref=ap.id)
objects.append(relationship)
bundle = Bundle(objects=objects)
def STIX2(url, score, description):
indik = Indicator(name="URL Suspecte", labels=["malicious-activity"], created=datetime.datetime.now(), description="Score = " + str(score) + " | " + description, pattern="[url:value = '" + url + "']")
return indik
