def _process_message(self, data: Dict) -> str:
file_fetch = data["file_fetch"]
file_uri = self.helper.opencti_url + file_fetch
# Downloading and saving file to connector
self.helper.log_info("Importing the file " + file_uri)
observable = SimpleObservable(
id=OpenCTIStix2Utils.generate_random_stix_id(
"x-opencti-simple-observable"),
key=self.data["simple_observable_key"],
value=self.data["simple_observable_value"],
)
bundle_objects = [observable]
entity_id = data.get("entity_id", None)
report = self.helper.api.report.read(id=entity_id)
report = Report(
id=report["standard_id"],
name=report["name"],
description=report["description"],
published=self.helper.api.stix2.format_date(report["published"]),
report_types=report["report_types"],
object_refs=bundle_objects,
)
bundle_objects.append(report)
# create stix bundle
bundle = Bundle(objects=bundle_objects).serialize()
# send data
self.helper.send_stix2_bundle(bundle=bundle)
return "foo"
def _process_message(self, data):
file_fetch = data["file_fetch"]
file_uri = self.helper.opencti_url + file_fetch
file_name = os.path.basename(file_fetch)
entity_id = data["entity_id"]
# Get context
is_context = entity_id is not None and len(entity_id) > 0
if self.helper.get_only_contextual() and not is_context:
raise ValueError(
"No context defined, connector is get_only_contextual true"
)
self.helper.log_info("Importing the file " + file_uri)
# Get the file
file_content = self.helper.api.fetch_opencti_file(file_uri, True)
# Write the file
f = open(file_name, "wb")
f.write(file_content)
f.close()
# Parse
bundle_objects = []
i = 0
parser = iocp.IOC_Parser(None, "pdf", True, "pdfminer", "json")
parsed = parser.parse(file_name)
os.remove(file_name)
if parsed != []:
for file in parsed:
if file != None:
for page in file:
if page != []:
for match in page:
resolved_match = self.resolve_match(match)
if resolved_match:
observable = SimpleObservable(
id=OpenCTIStix2Utils.generate_random_stix_id(
"x-opencti-simple-observable"
),
key=resolved_match["type"],
value=resolved_match["value"],
x_opencti_create_indicator=self.create_indicator,
)
bundle_objects.append(observable)
i += 1
else:
self.helper.log_error("Could not parse the report!")
if is_context:
entity = self.helper.api.stix_domain_object.read(id=entity_id)
if entity is not None:
if entity["entity_type"] == "Report" and len(bundle_objects) > 0:
report = Report(
id=entity["standard_id"],
name=entity["name"],
description=entity["description"],
published=self.helper.api.stix2.format_date(entity["created"]),
report_types=entity["report_types"],
object_refs=bundle_objects,
)
bundle_objects.append(report)
bundle = Bundle(objects=bundle_objects).serialize()
bundles_sent = self.helper.send_stix2_bundle(bundle)
return "Sent " + str(len(bundles_sent)) + " stix bundle(s) for worker import"
def generate_case_bundle(self, case):
markings = []
if case["tlp"] == 0:
markings.append(TLP_WHITE)
if case["tlp"] == 1:
markings.append(TLP_GREEN)
if case["tlp"] == 2:
markings.append(TLP_AMBER)
if case["tlp"] == 3:
markings.append(TLP_RED)
if len(markings) == 0:
markings.append(TLP_WHITE)
bundle_objects = []
incident = StixXOpenCTIIncident(
id=OpenCTIStix2Utils.generate_random_stix_id("x-opencti-incident"),
name=case["title"],
description=case["description"],
first_seen=datetime.utcfromtimestamp(
int(case["createdAt"]) / 1000).strftime("%Y-%m-%dT%H:%M:%SZ"),
last_seen=datetime.utcfromtimestamp(
int(case["updatedAt"]) / 1000).strftime("%Y-%m-%dT%H:%M:%SZ"),
object_marking_refs=markings,
labels=case["tags"] if "tags" in case else [],
created_by_ref=self.identity["standard_id"],
)
bundle_objects.append(incident)
# Get observables
observables = self.thehive_api.get_case_observables(
case_id=case["id"]).json()
for observable in observables:
if observable["dataType"] == "hash":
if len(observable["data"]) == 32:
data_type = "file_md5"
elif len(observable["data"]) == 40:
data_type = "file_sha1"
elif len(observable["data"]) == 64:
data_type = "file_sha256"
else:
data_type = "unknown"
else:
data_type = observable["dataType"]
observable_key = OBSERVABLES_MAPPING[data_type]
if observable_key is not None:
stix_observable = SimpleObservable(
id=OpenCTIStix2Utils.generate_random_stix_id(
"x-opencti-simple-observable"),
key=observable_key,
value=observable["data"],
description=observable["message"],
x_opencti_score=80 if observable["ioc"] else 50,
object_marking_refs=markings,
labels=observable["tags"] if "tags" in observable else [],
created_by_ref=self.identity["standard_id"],
x_opencti_create_indicator=observable["ioc"],
)
stix_observable_relation = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id(
"relationship"),
relationship_type="related-to",
created_by_ref=self.identity["standard_id"],
source_ref=stix_observable.id,
target_ref=incident.id,
object_marking_refs=markings,
)
bundle_objects.append(stix_observable)
bundle_objects.append(stix_observable_relation)
if observable["sighted"]:
fake_indicator_id = (
"indicator--c1034564-a9fb-429b-a1c1-c80116cc8e1e")
stix_sighting = Sighting(
id=OpenCTIStix2Utils.generate_random_stix_id(
"sighting"),
first_seen=datetime.utcfromtimestamp(
int(observable["startDate"] /
1000)).strftime("%Y-%m-%dT%H:%M:%SZ"),
last_seen=datetime.utcfromtimestamp(
int(observable["startDate"] / 1000 +
3600)).strftime("%Y-%m-%dT%H:%M:%SZ"),
where_sighted_refs=[self.identity["standard_id"]],
sighting_of_ref=fake_indicator_id,
custom_properties={
"x_opencti_sighting_of_ref": stix_observable.id
},
)
bundle_objects.append(stix_sighting)
bundle = Bundle(objects=bundle_objects).serialize()
return bundle
def _send_knowledge(self, observable, report):
bundle_objects = []
final_observable = observable
if observable["entity_type"] in ["StixFile", "Artifact"]:
final_observable = self.helper.api.stix_cyber_observable.update_field(
id=final_observable["id"],
key="hashes.MD5",
value=report["md5"])
final_observable = self.helper.api.stix_cyber_observable.update_field(
id=final_observable["id"],
key="hashes.SHA-1",
value=report["sha1"])
final_observable = self.helper.api.stix_cyber_observable.update_field(
id=final_observable["id"],
key="hashes.SHA-256",
value=report["sha256"],
)
if "name" not in final_observable or final_observable[
"name"] is None:
self.helper.api.stix_cyber_observable.update_field(
id=final_observable["id"],
key="x_opencti_additional_names",
value=report["submit_name"],
operation="add",
)
if final_observable["entity_type"] == "StixFile":
self.helper.api.stix_cyber_observable.update_field(
id=final_observable["id"],
key="size",
value=str(report["size"]),
)
self.helper.api.stix_cyber_observable.update_field(
id=final_observable["id"],
key="x_opencti_score",
value=str(report["threat_score"]),
)
# Create external reference
external_reference = self.helper.api.external_reference.create(
source_name="Hybrid Analysis",
url="https://www.hybrid-analysis.com/sample/" + report["sha256"],
description="Hybrid Analysis Report",
)
self.helper.api.stix_cyber_observable.add_external_reference(
id=final_observable["id"],
external_reference_id=external_reference["id"],
)
# Create tags
for tag in report["type_short"]:
tag_ha = self.helper.api.label.create(value=tag, color="#0059f7")
self.helper.api.stix_cyber_observable.add_label(
id=final_observable["id"], label_id=tag_ha["id"])
# Attach the TTPs
for tactic in report["mitre_attcks"]:
if (tactic["malicious_identifiers_count"] > 0
or tactic["suspicious_identifiers_count"] > 0):
attack_pattern = AttackPattern(
id=OpenCTIStix2Utils.generate_random_stix_id(
"attack-pattern"),
created_by_ref=self.identity,
name=tactic["technique"],
custom_properties={
"x_mitre_id": tactic["attck_id"],
},
object_marking_refs=[TLP_WHITE],
)
relationship = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id(
"relationship"),
relationship_type="uses",
created_by_ref=self.identity,
source_ref=final_observable["standard_id"],
target_ref=attack_pattern.id,
object_marking_refs=[TLP_WHITE],
)
bundle_objects.append(attack_pattern)
bundle_objects.append(relationship)
# Attach the domains
for domain in report["domains"]:
domain_stix = SimpleObservable(
id=OpenCTIStix2Utils.generate_random_stix_id(
"x-opencti-simple-observable"),
key="Domain-Name.value",
value=domain,
created_by_ref=self.identity,
object_marking_refs=[TLP_WHITE],
)
relationship = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id("relationship"),
relationship_type="communicates-with",
created_by_ref=self.identity,
source_ref=final_observable["standard_id"],
target_ref=domain_stix.id,
object_marking_refs=[TLP_WHITE],
)
bundle_objects.append(domain_stix)
bundle_objects.append(relationship)
# Attach the IP addresses
for host in report["hosts"]:
host_stix = SimpleObservable(
id=OpenCTIStix2Utils.generate_random_stix_id(
"x-opencti-simple-observable"),
key=self.detect_ip_version(host) + ".value",
value=host,
created_by_ref=self.identity,
object_marking_refs=[TLP_WHITE],
)
relationship = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id("relationship"),
relationship_type="communicates-with",
created_by_ref=self.identity,
source_ref=final_observable["standard_id"],
target_ref=host_stix.id,
object_marking_refs=[TLP_WHITE],
)
bundle_objects.append(host_stix)
bundle_objects.append(relationship)
# Attach other files
for file in report["extracted_files"]:
if file["threat_level"] > 0:
file_stix = File(
id=OpenCTIStix2Utils.generate_random_stix_id("file"),
hashes={
"MD5": file["md5"],
"SHA-1": file["sha1"],
"SHA-256": file["sha256"],
},
size=file["size"],
name=file["name"],
custom_properties={"x_opencti_labels": file["type_tags"]},
created_by_ref=self.identity,
object_marking_refs=[TLP_WHITE],
)
relationship = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id(
"relationship"),
relationship_type="drops",
created_by_ref=self.identity,
source_ref=final_observable["standard_id"],
target_ref=file_stix.id,
)
bundle_objects.append(file_stix)
bundle_objects.append(relationship)
for tactic in report["mitre_attcks"]:
if (tactic["malicious_identifiers_count"] > 0
or tactic["suspicious_identifiers_count"] > 0):
attack_pattern = AttackPattern(
id=OpenCTIStix2Utils.generate_random_stix_id(
"attack-pattern"),
created_by_ref=self.identity,
name=tactic["technique"],
custom_properties={
"x_mitre_id": tactic["attck_id"],
},
)
relationship = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id(
"relationship"),
relationship_type="uses",
created_by_ref=self.identity,
source_ref=final_observable["standard_id"],
target_ref=attack_pattern.id,
)
bundle_objects.append(attack_pattern)
bundle_objects.append(relationship)
if len(bundle_objects) > 0:
bundle = Bundle(objects=bundle_objects).serialize()
bundles_sent = self.helper.send_stix2_bundle(bundle)
return ("Sent " + str(len(bundles_sent)) +
" stix bundle(s) for worker import")
else:
return "Nothing to attach"
def _process_parsing_results(
self, parsed: List[Dict],
context_entity: Dict) -> (List[SimpleObservable], List[str]):
observables = []
entities = []
if context_entity is not None:
object_markings = [
x["standard_id"]
for x in context_entity.get("objectMarking", [])
]
# external_references = [x['standard_id'] for x in report.get('externalReferences', [])]
# labels = [x['standard_id'] for x in report.get('objectLabel', [])]
author = context_entity.get("createdBy")
else:
object_markings = []
author = None
if author is not None:
author = author.get("standard_id", None)
for match in parsed:
if match[RESULT_FORMAT_TYPE] == OBSERVABLE_CLASS:
if match[RESULT_FORMAT_CATEGORY] == "Vulnerability.name":
entity = self.helper.api.vulnerability.read(
filters={
"key": "name",
"values": [match[RESULT_FORMAT_MATCH]]
})
if entity is None:
self.helper.log_info(
f"Vulnerability with name '{match[RESULT_FORMAT_MATCH]}' could not be "
f"found. Is the CVE Connector activated?")
continue
entities.append(entity["standard_id"])
elif match[
RESULT_FORMAT_CATEGORY] == "Attack-Pattern.x_mitre_id":
entity = self.helper.api.attack_pattern.read(
filters={
"key": "x_mitre_id",
"values": [match[RESULT_FORMAT_MATCH]],
})
if entity is None:
self.helper.log_info(
f"AttackPattern with MITRE ID '{match[RESULT_FORMAT_MATCH]}' could not be "
f"found. Is the MITRE Connector activated?")
continue
entities.append(entity["standard_id"])
else:
observable = SimpleObservable(
id=OpenCTIStix2Utils.generate_random_stix_id(
"x-opencti-simple-observable"),
key=match[RESULT_FORMAT_CATEGORY],
value=match[RESULT_FORMAT_MATCH],
x_opencti_create_indicator=self.create_indicator,
object_marking_refs=object_markings,
created_by_ref=author,
# labels=labels,
# external_references=external_references
)
observables.append(observable)
elif match[RESULT_FORMAT_TYPE] == ENTITY_CLASS:
entities.append(match[RESULT_FORMAT_MATCH])
else:
self.helper.log_info("Odd data received: {}".format(match))
return observables, entities
def run(self):
self.helper.log_info("Fetching URLhaus dataset...")
while True:
try:
# Get the current timestamp and check
timestamp = int(time.time())
current_state = self.helper.get_state()
if current_state is not None and "last_run" in current_state:
last_run = current_state["last_run"]
self.helper.log_info("Connector last run: " +
datetime.utcfromtimestamp(last_run).
strftime("%Y-%m-%d %H:%M:%S"))
else:
last_run = None
self.helper.log_info("Connector has never run")
# If the last_run is more than interval-1 day
if last_run is None or ((timestamp - last_run) > (
(int(self.urlhaus_interval) - 1) * 60 * 60 * 24)):
self.helper.log_info("Connector will run!")
now = datetime.utcfromtimestamp(timestamp)
friendly_name = "URLhaus run @ " + now.strftime(
"%Y-%m-%d %H:%M:%S")
work_id = self.helper.api.work.initiate_work(
self.helper.connect_id, friendly_name)
try:
response = urllib.request.urlopen(
self.urlhaus_csv_url,
context=ssl.create_default_context(
cafile=certifi.where()),
)
image = response.read()
with open(
os.path.dirname(os.path.abspath(__file__)) +
"/data.csv",
"wb",
) as file:
file.write(image)
fp = open(
os.path.dirname(os.path.abspath(__file__)) +
"/data.csv",
"r",
)
rdr = csv.reader(filter(lambda row: row[0] != "#", fp))
bundle_objects = []
for row in rdr:
if row[3] == "online" or self.urlhaus_import_offline:
external_reference = ExternalReference(
source_name="Abuse.ch URLhaus",
url=row[6],
description="URLhaus repository URL",
)
stix_observable = SimpleObservable(
id=OpenCTIStix2Utils.
generate_random_stix_id(
"x-opencti-simple-observable"),
key="Url.value",
value=row[2],
description=row[4],
x_opencti_score=80,
object_marking_refs=[TLP_WHITE],
labels=row[5].split(","),
created_by_ref=self.
identity["standard_id"],
x_opencti_create_indicator=True,
external_references=[external_reference],
)
bundle_objects.append(stix_observable)
fp.close()
bundle = Bundle(objects=bundle_objects).serialize()
self.helper.send_stix2_bundle(
bundle,
entities_types=self.helper.connect_scope,
update=self.update_existing_data,
work_id=work_id,
)
if os.path.exists(
os.path.dirname(os.path.abspath(__file__)) +
"/data.csv"):
os.remove(
os.path.dirname(os.path.abspath(__file__)) +
"/data.csv")
except Exception as e:
self.helper.log_error(str(e))
# Store the current timestamp as a last run
message = "Connector successfully run, storing last_run as " + str(
timestamp)
self.helper.log_info(message)
self.helper.set_state({"last_run": timestamp})
self.helper.api.work.to_processed(work_id, message)
self.helper.log_info(
"Last_run stored, next run in: " +
str(round(self.get_interval() / 60 / 60 / 24, 2)) +
" days")
time.sleep(60)
else:
new_interval = self.get_interval() - (timestamp - last_run)
self.helper.log_info(
"Connector will not run, next run in: " +
str(round(new_interval / 60 / 60 / 24, 2)) + " days")
time.sleep(60)
except (KeyboardInterrupt, SystemExit):
self.helper.log_info("Connector stop")
exit(0)
except Exception as e:
self.helper.log_error(str(e))
time.sleep(60)
def run(self):
self.helper.log_info("Fetching VXVault dataset...")
while True:
try:
# Get the current timestamp and check
timestamp = int(time.time())
current_state = self.helper.get_state()
if current_state is not None and "last_run" in current_state:
last_run = current_state["last_run"]
self.helper.log_info("Connector last run: " +
datetime.utcfromtimestamp(last_run).
strftime("%Y-%m-%d %H:%M:%S"))
else:
last_run = None
self.helper.log_info("Connector has never run")
# If the last_run is more than interval-1 day
if last_run is None or ((timestamp - last_run) > (
(int(self.vxvault_interval) - 1) * 60 * 60 * 24)):
self.helper.log_info("Connector will run!")
now = datetime.utcfromtimestamp(timestamp)
friendly_name = "VXVault run @ " + now.strftime(
"%Y-%m-%d %H:%M:%S")
work_id = self.helper.api.work.initiate_work(
self.helper.connect_id, friendly_name)
try:
response = urllib.request.urlopen(
self.vxvault_url,
context=ssl.create_default_context(
cafile=certifi.where()),
)
image = response.read()
with open(
os.path.dirname(os.path.abspath(__file__)) +
"/data.txt",
"wb",
) as file:
file.write(image)
count = 0
bundle_objects = []
with open(
os.path.dirname(os.path.abspath(__file__)) +
"/data.txt") as fp:
for line in fp:
count += 1
if count <= 3:
continue
external_reference = ExternalReference(
source_name="VX Vault",
url="http://vxvault.net",
description="VX Vault repository URL",
)
stix_observable = SimpleObservable(
id=OpenCTIStix2Utils.
generate_random_stix_id(
"x-opencti-simple-observable"),
key="Url.value",
value=line,
description="VX Vault URL",
x_opencti_score=80,
object_marking_refs=[TLP_WHITE],
created_by_ref=self.
identity["standard_id"],
x_opencti_create_indicator=self.
create_indicators,
external_references=[external_reference],
)
bundle_objects.append(stix_observable)
bundle = Bundle(objects=bundle_objects,
allow_custom=True).serialize()
self.helper.send_stix2_bundle(
bundle,
update=self.update_existing_data,
work_id=work_id,
)
if os.path.exists(
os.path.dirname(os.path.abspath(__file__)) +
"/data.txt"):
os.remove(
os.path.dirname(os.path.abspath(__file__)) +
"/data.txt")
except Exception as e:
self.helper.log_error(str(e))
# Store the current timestamp as a last run
message = "Connector successfully run, storing last_run as " + str(
timestamp)
self.helper.log_info(message)
self.helper.set_state({"last_run": timestamp})
self.helper.api.work.to_processed(work_id, message)
self.helper.log_info(
"Last_run stored, next run in: " +
str(round(self.get_interval() / 60 / 60 / 24, 2)) +
" days")
time.sleep(60)
else:
new_interval = self.get_interval() - (timestamp - last_run)
self.helper.log_info(
"Connector will not run, next run in: " +
str(round(new_interval / 60 / 60 / 24, 2)) + " days")
time.sleep(60)
except (KeyboardInterrupt, SystemExit):
self.helper.log_info("Connector stop")
exit(0)
except Exception as e:
self.helper.log_error(str(e))
time.sleep(60)
def _send_knowledge(self, observable, report):
bundle_objects = []
final_observable = observable
target = report["target"]["file"]
task_id = report["info"]["id"]
final_observable = self.helper.api.stix_cyber_observable.update_field(
id=final_observable["id"],
input={
"key": "hashes.MD5",
"value": target["md5"]
},
)
final_observable = self.helper.api.stix_cyber_observable.update_field(
id=final_observable["id"],
input={
"key": "hashes.SHA-1",
"value": target["sha1"]
},
)
final_observable = self.helper.api.stix_cyber_observable.update_field(
id=final_observable["id"],
input={
"key": "hashes.SHA-256",
"value": target["sha256"],
},
)
self.helper.api.stix_cyber_observable.update_field(
id=final_observable["id"],
input={
"key": "x_opencti_score",
"value": str(int(report["malscore"] * 10)),
},
)
# Create external references
# Analysis URL
external_reference = self.helper.api.external_reference.create(
source_name="CAPEv2 Sandbox Analysis",
url=f"{self.cape_url}/analysis/{task_id}/",
description="CAPEv2 Sandbox Analysis",
)
self.helper.api.stix_cyber_observable.add_external_reference(
id=final_observable["id"],
external_reference_id=external_reference["id"],
)
# JSON report
external_reference = self.helper.api.external_reference.create(
source_name="CAPEv2 Sandbox JSON Report",
url=f"{self.cape_url}/filereport/{task_id}/json/",
description="CAPEv2 Sandbox JSON Report",
)
self.helper.api.stix_cyber_observable.add_external_reference(
id=final_observable["id"],
external_reference_id=external_reference["id"],
)
# HTML Report
external_reference = self.helper.api.external_reference.create(
source_name="CAPEv2 Sandbox HTML Report",
url=f"{self.cape_url}/filereport/{task_id}/html/",
description="CAPEv2 Sandbox HTML Report",
)
self.helper.api.stix_cyber_observable.add_external_reference(
id=final_observable["id"],
external_reference_id=external_reference["id"],
)
# Create label if family was detected
if "detections" in report and report["detections"]:
label = self.helper.api.label.create(value=report["detections"],
color="#0059f7")
self.helper.api.stix_cyber_observable.add_label(
id=final_observable["id"], label_id=label["id"])
# Create a Note containing the TrID results
trid_json = json.dumps(report["trid"], indent=2)
note = Note(
abstract="TrID Analysis",
content=f"```\n{trid_json}\n```",
created_by_ref=self.identity,
object_refs=[final_observable["standard_id"]],
)
bundle_objects.append(note)
# Attach the TTPs
for tactic_dict in report["ttps"]:
attack_id = tactic_dict["ttp"]
signature = tactic_dict["signature"]
attack_pattern = AttackPattern(
id=OpenCTIStix2Utils.generate_random_stix_id("attack-pattern"),
created_by_ref=self.identity,
name=signature,
custom_properties={
"x_mitre_id": attack_id,
},
object_marking_refs=[TLP_WHITE],
)
relationship = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id("relationship"),
relationship_type="uses",
created_by_ref=self.identity,
source_ref=final_observable["standard_id"],
target_ref=attack_pattern.id,
object_marking_refs=[TLP_WHITE],
)
bundle_objects.append(attack_pattern)
bundle_objects.append(relationship)
# Handle procdumps and attach any Flare CAPA TTPs
if "procdump" in report and report["procdump"]:
# Download the zip archive of procdump files
zip_contents = self._get_procdump_zip(task_id)
zip_obj = io.BytesIO(zip_contents)
# Extract with "infected" password
zip_file = pyzipper.AESZipFile(zip_obj)
zip_file.setpassword(b"infected")
# Process each entry in the procdump key
for procdump_dict in report["procdump"]:
# If less noise was specified
if self.less_noise:
# and no Yara matches, skip this procdump
if not procdump_dict["cape_yara"] or not procdump_dict[
"yara"]:
continue
sha256 = procdump_dict["sha256"]
cape_type = procdump_dict["cape_type"]
module_path = procdump_dict["module_path"]
procdump_contents = zip_file.read(sha256)
mime_type = magic.from_buffer(procdump_contents, mime=True)
kwargs = {
"file_name": module_path,
"data": procdump_contents,
"mime_type": mime_type,
"x_opencti_description": cape_type,
}
response = self.helper.api.stix_cyber_observable.upload_artifact(
**kwargs)
self.helper.log_info(
f'Uploaded procdump with sha256 "{sha256}" and type "{cape_type}".'
)
# Build labels
yara_rules = []
cape_yara_rules = []
yara_rules.extend(
[yara_dict["name"] for yara_dict in procdump_dict["yara"]])
cape_yara_rules.extend([
yara_dict["name"]
for yara_dict in procdump_dict["cape_yara"]
])
cape_yara_rules.extend([
yara_dict["meta"]["cape_type"]
for yara_dict in procdump_dict["cape_yara"]
])
# Create and apply yara rule based labels
for yara_rule in yara_rules:
label = self.helper.api.label.create(value=yara_rule,
color="#0059f7")
self.helper.api.stix_cyber_observable.add_label(
id=response["id"], label_id=label["id"])
# Create and apply cape yara rule based labels
for cape_yara_rule in cape_yara_rules:
label = self.helper.api.label.create(value=cape_yara_rule,
color="#ff8178")
self.helper.api.stix_cyber_observable.add_label(
id=response["id"], label_id=label["id"])
# Create label for cape_type
label = self.helper.api.label.create(value=cape_type,
color="#0059f7")
self.helper.api.stix_cyber_observable.add_label(
id=response["id"], label_id=label["id"])
# Create relationship between uploaded procdump Artifact and original
relationship = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id(
"relationship"),
relationship_type="related-to",
created_by_ref=self.identity,
source_ref=response["standard_id"],
target_ref=final_observable["standard_id"],
)
bundle_objects.append(relationship)
# Handle Flare CAPA TTPs
if "flare_capa" in procdump_dict and procdump_dict[
"flare_capa"]:
attck_dict = procdump_dict["flare_capa"]["ATTCK"]
for tactic in attck_dict:
tp_list = attck_dict[tactic]
for tp in tp_list:
attack_pattern = AttackPattern(
id=OpenCTIStix2Utils.generate_random_stix_id(
"attack-pattern"),
created_by_ref=self.identity,
name=tactic,
custom_properties={
"x_mitre_id": tp,
},
object_marking_refs=[TLP_WHITE],
)
relationship = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id(
"relationship"),
relationship_type="uses",
created_by_ref=self.identity,
source_ref=response["standard_id"],
target_ref=attack_pattern.id,
object_marking_refs=[TLP_WHITE],
)
bundle_objects.append(attack_pattern)
bundle_objects.append(relationship)
else:
self.helper.log_info(
f"Could not find flare_capa key or was empty in procdump {sha256}."
)
# Close the zip file
zip_file.close()
else:
self.helper.log_info(
"Skipping processing process dumps, procdump key is empty or not exists."
)
# Attach CnC addresses if any configs were found
if "CAPE" in report and report["CAPE"]["configs"]:
configs_list = report["CAPE"]["configs"]
for config_dict in configs_list:
for detection_name in config_dict:
# Create a Note containing the config
note = Note(
abstract=f"{detection_name} Config",
content=
f"```\n{json.dumps(config_dict, indent=2)}\n```",
created_by_ref=self.identity,
object_refs=[final_observable["standard_id"]],
)
bundle_objects.append(note)
if not "address" in config_dict[detection_name]:
self.helper.log_info(
f'Could not find an "address" key in {detection_name} config.'
)
continue
address_list = config_dict[detection_name]["address"]
for address in address_list:
parsed = address.rsplit(":", 1)[0]
if self._is_ipv4_address(parsed):
host_stix = SimpleObservable(
id=OpenCTIStix2Utils.generate_random_stix_id(
"x-opencti-simple-observable"),
labels=[detection_name, "c2 server"],
key="IPv4-Addr.value",
value=parsed,
created_by_ref=self.identity,
)
relationship = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id(
"relationship"),
relationship_type="communicates-with",
created_by_ref=self.identity,
source_ref=final_observable["standard_id"],
target_ref=host_stix.id,
)
bundle_objects.append(host_stix)
bundle_objects.append(relationship)
else:
domain = urlparse(address).hostname
domain_stix = SimpleObservable(
id=OpenCTIStix2Utils.generate_random_stix_id(
"x-opencti-simple-observable"),
labels=[detection_name, "c2 server"],
key="Domain-Name.value",
value=domain,
created_by_ref=self.identity,
object_marking_refs=[TLP_WHITE],
)
relationship = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id(
"relationship"),
relationship_type="communicates-with",
created_by_ref=self.identity,
source_ref=final_observable["standard_id"],
target_ref=domain_stix.id,
)
bundle_objects.append(domain_stix)
bundle_objects.append(relationship)
# Attach the domains
for domain_dict in report.get("network", {}).get("domains", []):
domain_stix = SimpleObservable(
id=OpenCTIStix2Utils.generate_random_stix_id(
"x-opencti-simple-observable"),
key="Domain-Name.value",
value=domain_dict["domain"],
created_by_ref=self.identity,
object_marking_refs=[TLP_WHITE],
)
relationship = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id("relationship"),
relationship_type="communicates-with",
created_by_ref=self.identity,
source_ref=final_observable["standard_id"],
target_ref=domain_stix.id,
)
bundle_objects.append(domain_stix)
bundle_objects.append(relationship)
# Attach the IP addresses
for host_dict in report.get("network", {}).get("hosts", []):
host = host_dict["ip"]
host_stix = SimpleObservable(
id=OpenCTIStix2Utils.generate_random_stix_id(
"x-opencti-simple-observable"),
key="IPv4-Addr.value",
value=host,
created_by_ref=self.identity,
)
relationship = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id("relationship"),
relationship_type="communicates-with",
created_by_ref=self.identity,
source_ref=final_observable["standard_id"],
target_ref=host_stix.id,
)
bundle_objects.append(host_stix)
bundle_objects.append(relationship)
# Handle CAPE payloads and attach Flare CAPA TTPs if any
if ("CAPE" in report and "payloads" in report["CAPE"]
and report["CAPE"]["payloads"]):
# Download the zip archive of payloads
zip_contents = self._get_payloads_zip(task_id)
zip_obj = io.BytesIO(zip_contents)
# Extract with "infected" password
zip_file = pyzipper.AESZipFile(zip_obj)
zip_file.setpassword(b"infected")
# Process each payload
for payload_dict in report["CAPE"]["payloads"]:
module_path = payload_dict["module_path"]
sha256 = payload_dict["sha256"]
cape_type = payload_dict["cape_type"]
payload_contents = zip_file.read(sha256)
mime_type = magic.from_buffer(payload_contents, mime=True)
kwargs = {
"file_name": module_path,
"data": payload_contents,
"mime_type": mime_type,
"x_opencti_description": cape_type,
}
response = self.helper.api.stix_cyber_observable.upload_artifact(
**kwargs)
self.helper.log_info(
f'Uploaded CAPE payload with sha256 "{sha256}" and type "{cape_type}".'
)
# Create and apply label
label = self.helper.api.label.create(value=cape_type,
color="#ff8178")
self.helper.api.stix_cyber_observable.add_label(
id=response["id"], label_id=label["id"])
# Create relationship between uploaded payload Artifact and original
relationship = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id(
"relationship"),
relationship_type="related-to",
created_by_ref=self.identity,
source_ref=response["standard_id"],
target_ref=final_observable["standard_id"],
)
bundle_objects.append(relationship)
# Handle Flare CAPA TTPs if any
if "flare_capa" in payload_dict and payload_dict["flare_capa"]:
attck_dict = payload_dict["flare_capa"]["ATTCK"]
for tactic in attck_dict:
for tp in attck_dict[tactic]:
attack_pattern = AttackPattern(
id=OpenCTIStix2Utils.generate_random_stix_id(
"attack-pattern"),
created_by_ref=self.identity,
name=tactic,
custom_properties={
"x_mitre_id": tp,
},
object_marking_refs=[TLP_WHITE],
)
relationship = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id(
"relationship"),
relationship_type="uses",
created_by_ref=self.identity,
source_ref=response["standard_id"],
target_ref=attack_pattern.id,
object_marking_refs=[TLP_WHITE],
)
bundle_objects.append(attack_pattern)
bundle_objects.append(relationship)
else:
self.helper.log_info(
f"Could not find flare_capa key or was empty in CAPE payload {sha256}."
)
# Close the zip file
zip_file.close()
else:
self.helper.log_info(
"Skipping processing CAPE payloads, payloads key is empty or not exists."
)
# Serialize and send all bundles
if bundle_objects:
bundle = Bundle(objects=bundle_objects,
allow_custom=True).serialize()
bundles_sent = self.helper.send_stix2_bundle(bundle)
return f"Sent {len(bundles_sent)} stix bundle(s) for worker import"
else:
return "Nothing to attach"
def _process_overview_report(self, observable, overview_dict, sample_id):
bundle_objects = []
# Create external reference
# Analysis URL
external_reference = self.helper.api.external_reference.create(
source_name="Hatching Triage Sandbox Analysis",
url=f"https://tria.ge/{sample_id}/",
description="Hatching Triage Sandbox Analysis",
)
self.helper.api.stix_cyber_observable.add_external_reference(
id=observable["id"],
external_reference_id=external_reference["id"],
)
# Create labels from the tags
if "tags" in overview_dict["analysis"]:
for tag in overview_dict["analysis"]["tags"]:
# Set the label color depending on tag type
# Note: Only certain tags are separated by a colon
# Those are the tags we are colorizing
tag_split = tag.split(":")
tag_value = tag
label_color = self.default_tag_color
if len(tag_split) == 2:
if "family" == tag_split[0]:
label_color = self.family_color
elif "botnet" == tag_split[0]:
label_color = self.botnet_color
elif "campaign" == tag_split[0]:
label_color = self.campaign_color
tag_value = tag_split[1]
# Create and add the label
label = self.helper.api.label.create(value=tag_value, color=label_color)
self.helper.api.stix_cyber_observable.add_label(
id=observable["id"], label_id=label["id"]
)
# Create default labels
self.helper.api.label.create(value="c2 server", color=self.default_tag_color)
self.helper.api.label.create(value="credentials", color=self.default_tag_color)
self.helper.api.label.create(value="dynamic", color=self.default_tag_color)
extracted_label = self.helper.api.label.create(
value="extracted", color=self.default_tag_color
)
self.helper.api.label.create(value="dropper", color=self.default_tag_color)
# Process extracted key
extracted_list = overview_dict.get("extracted", [])
for extracted_dict in extracted_list:
# Handle config
if "config" in extracted_dict:
if "rule" not in extracted_dict["config"]:
self.helper.api.log_info("rule key not found, skipping...")
continue
# Create a Note
config_json = json.dumps(extracted_dict, indent=2)
config_rule = extracted_dict["config"]["rule"]
note = Note(
abstract=f"{config_rule} Config",
content=f"```\n{config_json}\n```",
created_by_ref=self.identity,
object_refs=[observable["standard_id"]],
)
bundle_objects.append(note)
# Create Observables and Relationships for C2s
c2_list = extracted_dict.get("config").get("c2", [])
for c2 in c2_list:
# Differentiate between C2 IP and URL
parsed = c2.split(":")[0]
key = "Url"
relationship_type = "communicates-with"
if self._is_ipv4_address(parsed):
key = "IPv4-Addr"
else:
parsed = c2
relationship_type = "related-to"
host_stix = SimpleObservable(
id=OpenCTIStix2Utils.generate_random_stix_id(
"x-opencti-simple-observable"
),
labels=[config_rule, "c2 server"],
key=f"{key}.value",
value=parsed,
created_by_ref=self.identity,
)
relationship = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id("relationship"),
relationship_type=relationship_type,
created_by_ref=self.identity,
source_ref=observable["standard_id"],
target_ref=host_stix.id,
allow_custom=True,
)
bundle_objects.append(host_stix)
bundle_objects.append(relationship)
# Create Observables and Relationships for credentials
creds_list = extracted_dict.get("config").get("credentials", [])
for cred_dict in creds_list:
host = cred_dict.get("host", None)
username = cred_dict.get("username")
protocol = cred_dict.get("protocol")
if host:
# Add Host Observable
host_stix = SimpleObservable(
id=OpenCTIStix2Utils.generate_random_stix_id(
"x-opencti-simple-observable"
),
labels=[config_rule, "credentials"],
key="X-OpenCTI-Hostname.value",
value=host,
created_by_ref=self.identity,
)
relationship = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id(
"relationship"
),
relationship_type="related-to",
created_by_ref=self.identity,
source_ref=observable["standard_id"],
target_ref=host_stix.id,
allow_custom=True,
)
bundle_objects.append(host_stix)
bundle_objects.append(relationship)
if protocol == "smtp" and username:
# Add Email Address Observable
email_stix = SimpleObservable(
id=OpenCTIStix2Utils.generate_random_stix_id(
"x-opencti-simple-observable"
),
labels=[config_rule, "credentials"],
key="Email-Addr.value",
value=username,
created_by_ref=self.identity,
)
relationship = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id(
"relationship"
),
relationship_type="related-to",
created_by_ref=self.identity,
source_ref=observable["standard_id"],
target_ref=email_stix.id,
allow_custom=True,
)
bundle_objects.append(email_stix)
bundle_objects.append(relationship)
# Download task file, wait for it to become available
# Give up after 5 tries
task_id = extracted_dict["tasks"][0]
filename = extracted_dict["dumped_file"]
file_contents = self.triage_client.sample_task_file(
sample_id, task_id, filename
)
for x in range(5):
# Sample not yet available, sleep
if b"NOT_AVAILABLE" in file_contents[:30]:
time.sleep(10)
continue
file_contents = self.triage_client.sample_task_file(
sample_id, task_id, filename
)
if b"NOT_AVAILABLE" in file_contents[:30]:
self.helper.api.log_info(
"Maximum attempts tried to obtain extracted file, skipping..."
)
continue
# Upload task file to OpenCTI
mime_type = magic.from_buffer(file_contents, mime=True)
kwargs = {
"file_name": f"{sample_id}_{filename}",
"data": file_contents,
"mime_type": mime_type,
"x_opencti_description": f"Extracted file from sample ID {sample_id} and task ID {task_id}.",
}
response = self.helper.api.stix_cyber_observable.upload_artifact(
**kwargs
)
# Create Relationship between original Observable and the extracted
relationship = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id("relationship"),
relationship_type="related-to",
created_by_ref=self.identity,
source_ref=response["standard_id"],
target_ref=observable["standard_id"],
allow_custom=True,
)
bundle_objects.append(relationship)
# Create and apply labels to extracted file
label = self.helper.api.label.create(
value=config_rule, color=self.family_color
)
self.helper.api.stix_cyber_observable.add_label(
id=response["id"], label_id=label["id"]
)
self.helper.api.stix_cyber_observable.add_label(
id=response["id"], label_id=extracted_label["id"]
)
# Handle dropper
if "dropper" in extracted_dict:
dropper_dict = extracted_dict["dropper"]
# Create Url Observables and Relationships
dropper_urls = dropper_dict["urls"]
for url_dict in dropper_urls:
dropper_type = url_dict["type"]
url = url_dict["url"]
url_stix = SimpleObservable(
id=OpenCTIStix2Utils.generate_random_stix_id(
"x-opencti-simple-observable"
),
labels=[dropper_type],
key="Url.value",
value=url.rstrip(),
created_by_ref=self.identity,
object_marking_refs=[TLP_WHITE],
)
relationship = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id("relationship"),
relationship_type="related-to",
created_by_ref=self.identity,
source_ref=observable["standard_id"],
target_ref=url_stix.id,
allow_custom=True,
)
bundle_objects.append(url_stix)
bundle_objects.append(relationship)
# Create dropper type label
self.helper.api.label.create(
value=dropper_type, color=self.default_tag_color
)
# Attach domains
domains = [
task_dict.get("iocs", {}).get("domains", [])
for task_dict in overview_dict["targets"]
]
for domain in domains[0]:
domain_stix = SimpleObservable(
id=OpenCTIStix2Utils.generate_random_stix_id(
"x-opencti-simple-observable"
),
labels=["dynamic"],
key="Domain-Name.value",
value=domain,
created_by_ref=self.identity,
object_marking_refs=[TLP_WHITE],
)
relationship = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id("relationship"),
relationship_type="communicates-with",
created_by_ref=self.identity,
source_ref=observable["standard_id"],
target_ref=domain_stix.id,
allow_custom=True,
)
bundle_objects.append(domain_stix)
bundle_objects.append(relationship)
# Attach IP addresses
ips = [
task_dict.get("iocs", {}).get("ips", [])
for task_dict in overview_dict["targets"]
]
for ip in ips[0]:
# Filter out non-global and known DNS IPs
if not ipaddress.ip_address(ip).is_global:
continue
if ip in ["8.8.8.8", "8.8.4.4"]:
continue
host_stix = SimpleObservable(
id=OpenCTIStix2Utils.generate_random_stix_id(
"x-opencti-simple-observable"
),
labels=["dynamic"],
key="IPv4-Addr.value",
value=ip,
created_by_ref=self.identity,
)
relationship = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id("relationship"),
relationship_type="communicates-with",
created_by_ref=self.identity,
source_ref=observable["standard_id"],
target_ref=host_stix.id,
allow_custom=True,
)
bundle_objects.append(host_stix)
bundle_objects.append(relationship)
# Attach the TTPs
if "signatures" in overview_dict:
for signature_dict in overview_dict["signatures"]:
# Skip any dicts without a ttps key
if "ttp" not in signature_dict:
continue
name = signature_dict["name"]
ttps = signature_dict["ttp"]
for ttp in ttps:
attack_pattern = AttackPattern(
id=OpenCTIStix2Utils.generate_random_stix_id("attack-pattern"),
created_by_ref=self.identity,
name=name,
custom_properties={
"x_mitre_id": ttp,
},
object_marking_refs=[TLP_WHITE],
)
relationship = Relationship(
id=OpenCTIStix2Utils.generate_random_stix_id("relationship"),
relationship_type="uses",
created_by_ref=self.identity,
source_ref=observable["standard_id"],
target_ref=attack_pattern.id,
object_marking_refs=[TLP_WHITE],
allow_custom=True,
)
bundle_objects.append(attack_pattern)
bundle_objects.append(relationship)
# Serialize and send all bundles
if bundle_objects:
bundle = Bundle(objects=bundle_objects, allow_custom=True).serialize()
bundles_sent = self.helper.send_stix2_bundle(bundle)
return f"Sent {len(bundles_sent)} stix bundle(s) for worker import"
else:
return "Nothing to attach"
def _process_message(self, data):
file_fetch = data["file_fetch"]
file_uri = self.helper.opencti_url + file_fetch
file_name = os.path.basename(file_fetch)
entity_id = data.get("entity_id", None)
self.helper.log_info(entity_id)
# Get context
is_context = entity_id is not None and len(entity_id) > 0
self.helper.log_info("Context: {}".format(is_context))
self.helper.log_info(
"get_only_contextual: {}".format(self.helper.get_only_contextual())
)
if self.helper.get_only_contextual() and not is_context:
raise ValueError(
"No context defined, connector is get_only_contextual true"
)
self.helper.log_info("Importing the file " + file_uri)
# Get the file
file_content = self.helper.api.fetch_opencti_file(file_uri, True)
# Write the file
f = open(file_name, "wb")
f.write(file_content)
f.close()
# Parse
bundle_objects = []
stix_objects = set()
i = 0
custom_indicators = self._get_entities()
mime_type = self._get_mime_type(file_name)
print(mime_type)
if mime_type is None:
raise ValueError("Invalid file format of {}".format(file_name))
parser = iocp.IOC_Parser(
None,
mime_type,
True,
"pdfminer",
"json",
custom_indicators=custom_indicators,
)
parsed = parser.parse(file_name)
os.remove(file_name)
if parsed != []:
for file in parsed:
if file != None:
for page in file:
if page != []:
for match in page:
resolved_match = self._resolve_match(match)
if resolved_match:
# For the creation of relationships
if resolved_match[
"type"
] not in self.types.values() and self._is_uuid(
resolved_match["value"]
):
stix_objects.add(resolved_match["value"])
# For CVEs since SimpleObservable doesn't support Vulnerabilities yet
elif resolved_match["type"] == "Vulnerability.name":
vulnerability = Vulnerability(
name=resolved_match["value"]
)
bundle_objects.append(vulnerability)
# Other observables
elif resolved_match["type"] in self.types.values():
observable = SimpleObservable(
id=OpenCTIStix2Utils.generate_random_stix_id(
"x-opencti-simple-observable"
),
key=resolved_match["type"],
value=resolved_match["value"],
x_opencti_create_indicator=self.create_indicator,
)
bundle_objects.append(observable)
else:
self.helper.log_info(
"Odd data received: {}".format(
resolved_match
)
)
i += 1
else:
self.helper.log_error("Could not parse the report!")
if is_context:
entity = self.helper.api.stix_domain_object.read(id=entity_id)
if entity is not None:
if entity["entity_type"] == "Report" and len(bundle_objects) > 0:
report = Report(
id=entity["standard_id"],
name=entity["name"],
description=entity["description"],
published=self.helper.api.stix2.format_date(entity["created"]),
report_types=entity["report_types"],
object_refs=bundle_objects,
)
bundle_objects.append(report)
bundles_sent = []
if len(bundle_objects) > 0:
bundle = Bundle(objects=bundle_objects).serialize()
bundles_sent = self.helper.send_stix2_bundle(bundle)
if len(stix_objects) > 0 and entity_id is not None:
report = self.helper.api.report.read(id=entity_id)
if report:
for stix_object in stix_objects:
self.helper.log_info(stix_object)
self.helper.api.report.add_stix_object_or_stix_relationship(
id=report["id"], stixObjectOrStixRelationshipId=stix_object
)
return "Sent " + str(len(bundles_sent)) + " stix bundle(s) for worker import"