class ReportImporter:
def __init__(self) -> None:
# Instantiate the connector helper from config
base_path = os.path.dirname(os.path.abspath(__file__))
config_file_path = base_path + "/../config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
self.helper = OpenCTIConnectorHelper(config)
self.create_indicator = get_config_variable(
"IMPORT_DOCUMENT_CREATE_INDICATOR",
["import_document", "create_indicator"],
config,
)
# Load Entity and Observable configs
observable_config_file = base_path + "/config/observable_config.ini"
entity_config_file = base_path + "/config/entity_config.ini"
if os.path.isfile(observable_config_file) and os.path.isfile(
entity_config_file):
self.observable_config = self._parse_config(
observable_config_file, Observable)
else:
raise FileNotFoundError(f"{observable_config_file} was not found")
if os.path.isfile(entity_config_file):
self.entity_config = self._parse_config(entity_config_file,
EntityConfig)
else:
raise FileNotFoundError(f"{entity_config_file} was not found")
def _process_message(self, data: Dict) -> str:
self.helper.log_info("Processing new message")
file_name = self._download_import_file(data)
entity_id = data.get("entity_id", None)
bypass_validation = data.get("bypass_validation", False)
entity = (self.helper.api.stix_domain_object.read(
id=entity_id) if entity_id is not None else None)
if self.helper.get_only_contextual() and entity is None:
return "Connector is only contextual and entity is not defined. Nothing was imported"
# Retrieve entity set from OpenCTI
entity_indicators = self._collect_stix_objects(self.entity_config)
# Parse report
parser = ReportParser(self.helper, entity_indicators,
self.observable_config)
parsed = parser.run_parser(file_name, data["file_mime"])
os.remove(file_name)
if not parsed:
return "No information extracted from report"
# Process parsing results
self.helper.log_debug("Results: {}".format(parsed))
observables, entities = self._process_parsing_results(parsed, entity)
# Send results to OpenCTI
observable_cnt = self._process_parsed_objects(entity, observables,
entities,
bypass_validation,
file_name)
entity_cnt = len(entities)
if self.helper.get_validate_before_import() and not bypass_validation:
return "Generated bundle sent for validation"
else:
return (
f"Sent {observable_cnt} observables, 1 report update and {entity_cnt} entity connections as stix "
f"bundle for worker import ")
def start(self) -> None:
self.helper.listen(self._process_message)
def _download_import_file(self, data: Dict) -> str:
file_fetch = data["file_fetch"]
file_uri = self.helper.opencti_url + file_fetch
# Downloading and saving file to connector
self.helper.log_info("Importing the file " + file_uri)
file_name = os.path.basename(file_fetch)
file_content = self.helper.api.fetch_opencti_file(file_uri, True)
with open(file_name, "wb") as f:
f.write(file_content)
return file_name
def _collect_stix_objects(
self, entity_config_list: List[EntityConfig]) -> List[Entity]:
base_func = self.helper.api
entity_list = []
for entity_config in entity_config_list:
func_format = entity_config.stix_class
try:
custom_function = getattr(base_func, func_format)
entries = custom_function.list(
getAll=True,
filters=entity_config.filter,
customAttributes=entity_config.custom_attributes,
)
entity_list += entity_config.convert_to_entity(
entries, self.helper)
except AttributeError:
e = "Selected parser format is not supported: {}".format(
func_format)
raise NotImplementedError(e)
return entity_list
@staticmethod
def _parse_config(config_file: str,
file_class: Callable) -> List[BaseModel]:
config = MyConfigParser()
config.read(config_file)
config_list = []
for section, content in config.as_dict().items():
content["name"] = section
config_object = file_class(**content)
config_list.append(config_object)
return config_list
def _process_parsing_results(
self, parsed: List[Dict],
context_entity: Dict) -> (List[SimpleObservable], List[str]):
observables = []
entities = []
if context_entity is not None:
object_markings = [
x["standard_id"]
for x in context_entity.get("objectMarking", [])
]
# external_references = [x['standard_id'] for x in report.get('externalReferences', [])]
# labels = [x['standard_id'] for x in report.get('objectLabel', [])]
author = context_entity.get("createdBy")
else:
object_markings = []
author = None
if author is not None:
author = author.get("standard_id", None)
for match in parsed:
if match[RESULT_FORMAT_TYPE] == OBSERVABLE_CLASS:
if match[RESULT_FORMAT_CATEGORY] == "Vulnerability.name":
entity = self.helper.api.vulnerability.read(
filters={
"key": "name",
"values": [match[RESULT_FORMAT_MATCH]]
})
if entity is None:
self.helper.log_info(
f"Vulnerability with name '{match[RESULT_FORMAT_MATCH]}' could not be "
f"found. Is the CVE Connector activated?")
continue
entities.append(entity["standard_id"])
elif match[
RESULT_FORMAT_CATEGORY] == "Attack-Pattern.x_mitre_id":
entity = self.helper.api.attack_pattern.read(
filters={
"key": "x_mitre_id",
"values": [match[RESULT_FORMAT_MATCH]],
})
if entity is None:
self.helper.log_info(
f"AttackPattern with MITRE ID '{match[RESULT_FORMAT_MATCH]}' could not be "
f"found. Is the MITRE Connector activated?")
continue
entities.append(entity["standard_id"])
else:
observable = SimpleObservable(
id=OpenCTIStix2Utils.generate_random_stix_id(
"x-opencti-simple-observable"),
key=match[RESULT_FORMAT_CATEGORY],
value=match[RESULT_FORMAT_MATCH],
x_opencti_create_indicator=self.create_indicator,
object_marking_refs=object_markings,
created_by_ref=author,
# labels=labels,
# external_references=external_references
)
observables.append(observable)
elif match[RESULT_FORMAT_TYPE] == ENTITY_CLASS:
entities.append(match[RESULT_FORMAT_MATCH])
else:
self.helper.log_info("Odd data received: {}".format(match))
return observables, entities
def _process_parsed_objects(
self,
entity: Dict,
observables: List,
entities: List,
bypass_validation: bool,
file_name: str,
) -> int:
if len(observables) == 0 and len(entities) == 0:
return 0
if entity is not None and entity["entity_type"] == "Report":
report = Report(
id=entity["standard_id"],
name=entity["name"],
description=entity["description"],
published=self.helper.api.stix2.format_date(entity["created"]),
report_types=entity["report_types"],
object_refs=observables + entities,
allow_custom=True,
)
observables.append(report)
elif entity is not None:
# TODO, relate all object to the entity
entity_stix_bundle = self.helper.api.stix2.export_entity(
entity["entity_type"], entity["id"])
observables = observables + entity_stix_bundle["objects"]
else:
timestamp = int(time.time())
now = datetime.utcfromtimestamp(timestamp)
report = Report(
name=file_name,
description="Automatic import",
published=now,
report_types=["threat-report"],
object_refs=observables + entities,
allow_custom=True,
)
observables.append(report)
bundles_sent = []
if len(observables) > 0:
bundle = Bundle(objects=observables, allow_custom=True).serialize()
bundles_sent = self.helper.send_stix2_bundle(
bundle=bundle,
update=True,
bypass_validation=bypass_validation,
file_name=file_name + ".json",
entity_id=entity["id"] if entity is not None else None,
)
# len() - 1 because the report update increases the count by one
return len(bundles_sent) - 1
class ImportFilePdfObservables:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(os.path.abspath(__file__)) + "/config.yml"
config = (
yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path)
else {}
)
self.helper = OpenCTIConnectorHelper(config)
self.create_indicator = get_config_variable(
"PDF_OBSERVABLES_CREATE_INDICATOR",
["pdf_observables", "create_indicator"],
config,
)
def _process_message(self, data):
file_fetch = data["file_fetch"]
file_uri = self.helper.opencti_url + file_fetch
file_name = os.path.basename(file_fetch)
entity_id = data["entity_id"]
# Get context
is_context = entity_id is not None and len(entity_id) > 0
if self.helper.get_only_contextual() and not is_context:
raise ValueError(
"No context defined, connector is get_only_contextual true"
)
self.helper.log_info("Importing the file " + file_uri)
# Get the file
file_content = self.helper.api.fetch_opencti_file(file_uri, True)
# Write the file
f = open(file_name, "wb")
f.write(file_content)
f.close()
# Parse
bundle_objects = []
i = 0
parser = iocp.IOC_Parser(None, "pdf", True, "pdfminer", "json")
parsed = parser.parse(file_name)
os.remove(file_name)
if parsed != []:
for file in parsed:
if file != None:
for page in file:
if page != []:
for match in page:
resolved_match = self.resolve_match(match)
if resolved_match:
observable = SimpleObservable(
id=OpenCTIStix2Utils.generate_random_stix_id(
"x-opencti-simple-observable"
),
key=resolved_match["type"],
value=resolved_match["value"],
x_opencti_create_indicator=self.create_indicator,
)
bundle_objects.append(observable)
i += 1
else:
self.helper.log_error("Could not parse the report!")
if is_context:
entity = self.helper.api.stix_domain_object.read(id=entity_id)
if entity is not None:
if entity["entity_type"] == "Report" and len(bundle_objects) > 0:
report = Report(
id=entity["standard_id"],
name=entity["name"],
description=entity["description"],
published=self.helper.api.stix2.format_date(entity["created"]),
report_types=entity["report_types"],
object_refs=bundle_objects,
)
bundle_objects.append(report)
bundle = Bundle(objects=bundle_objects).serialize()
bundles_sent = self.helper.send_stix2_bundle(bundle)
return "Sent " + str(len(bundles_sent)) + " stix bundle(s) for worker import"
# Start the main loop
def start(self):
self.helper.listen(self._process_message)
def resolve_match(self, match):
types = {
"MD5": "File.hashes.MD5",
"SHA1": "File.hashes.SHA-1",
"SHA256": "File.hashes.SHA-256",
"Filename": "File.name",
"IP": "IPv4-Addr.value",
"Host": "X-OpenCTI-Hostname.value",
"Filepath": "File.path",
"URL": "Url.value",
"Email": "Email-Addr.value",
}
type = match["type"]
value = match["match"]
if type in types:
resolved_type = types[type]
if resolved_type == "IPv4-Addr.value":
# Demilitarized IP
if "[.]" in value:
value = value.replace("[.]", ".")
type_0 = self.detect_ip_version(value)
elif resolved_type == "Url.value":
# Demilitarized URL
if "hxxp://" in value:
value = value.replace("hxxp://", "http://")
if "hxxps://" in value:
value = value.replace("hxxps://", "https://")
if "hxxxs://" in value:
value = value.replace("hxxxs://", "https://")
if "[.]" in value:
value = value.replace("[.]", ".")
type_0 = resolved_type
elif resolved_type == "X-OpenCTI-Hostname.value":
# Demilitarized Host
if "[.]" in value:
value = value.replace("[.]", ".")
type_0 = resolved_type
else:
type_0 = resolved_type
return {"type": type_0, "value": value}
else:
return False
def detect_ip_version(self, value):
if len(value) > 16:
return "IPv6-Addr.value"
else:
return "IPv4-Addr.value"
class ReportImporter:
def __init__(self) -> None:
# Instantiate the connector helper from config
base_path = os.path.dirname(os.path.abspath(__file__))
config_file_path = base_path + "/../config.yml"
config = (
yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path)
else {}
)
self.helper = OpenCTIConnectorHelper(config)
self.create_indicator = get_config_variable(
"IMPORT_REPORT_CREATE_INDICATOR",
["import_report", "create_indicator"],
config,
)
# Load Entity and Observable configs
observable_config_file = base_path + "/config/observable_config.ini"
entity_config_file = base_path + "/config/entity_config.ini"
if os.path.isfile(observable_config_file) and os.path.isfile(
entity_config_file
):
self.observable_config = self._parse_config(
observable_config_file, Observable
)
else:
raise FileNotFoundError(f"{observable_config_file} was not found")
if os.path.isfile(entity_config_file):
self.entity_config = self._parse_config(entity_config_file, EntityConfig)
else:
raise FileNotFoundError(f"{entity_config_file} was not found")
def _process_message(self, data: Dict) -> str:
file_name = self._download_import_file(data)
entity_id = data.get("entity_id", None)
if self._check_context(entity_id):
raise ValueError(
"No context defined, connector is get_only_contextual true"
)
# Retrieve entity set from OpenCTI
entity_indicators = self._collect_stix_objects(self.entity_config)
# Parse peport
parser = ReportParser(self.helper, entity_indicators, self.observable_config)
parsed = parser.run_parser(file_name, data["file_mime"])
os.remove(file_name)
if not parsed:
return "No information extracted from report"
# Process parsing results
self.helper.log_debug("Results: {}".format(parsed))
observables, entities = self._process_parsing_results(parsed)
report = self.helper.api.report.read(id=entity_id)
# Send results to OpenCTI
observable_cnt = self._process_observables(report, observables)
entity_cnt = self._process_entities(report, entities)
return f"Sent {observable_cnt} stix bundle(s) and {entity_cnt} entity connections for worker import"
def start(self) -> None:
self.helper.listen(self._process_message)
def _download_import_file(self, data: Dict) -> str:
file_fetch = data["file_fetch"]
file_uri = self.helper.opencti_url + file_fetch
# Downloading and saving file to connector
self.helper.log_info("Importing the file " + file_uri)
file_name = os.path.basename(file_fetch)
file_content = self.helper.api.fetch_opencti_file(file_uri, True)
with open(file_name, "wb") as f:
f.write(file_content)
return file_name
def _check_context(self, entity_id: str) -> bool:
is_context = entity_id and len(entity_id) > 0
return self.helper.get_only_contextual() and not is_context
def _collect_stix_objects(
self, entity_config_list: List[EntityConfig]
) -> List[Entity]:
base_func = self.helper.api
entity_list = []
for entity_config in entity_config_list:
func_format = entity_config.stix_class
try:
custom_function = getattr(base_func, func_format)
entries = custom_function.list(
getAll=True, filters=entity_config.filter
)
entity_list += entity_config.convert_to_entity(entries)
except AttributeError:
e = "Selected parser format is not supported: {}".format(func_format)
raise NotImplementedError(e)
return entity_list
@staticmethod
def _parse_config(config_file: str, file_class: Callable) -> List[BaseModel]:
config = MyConfigParser()
config.read(config_file)
config_list = []
for section, content in config.as_dict().items():
content["name"] = section
config_object = file_class(**content)
config_list.append(config_object)
return config_list
def _process_parsing_results(
self, parsed: List[Dict]
) -> (List[SimpleObservable], List[str]):
observables = []
entities = []
for match in parsed:
if match[RESULT_FORMAT_TYPE] == OBSERVABLE_CLASS:
# Hardcoded exceptions since SimpleObservable doesn't support those types yet
if match[RESULT_FORMAT_CATEGORY] == "Vulnerability.name":
observable = self.helper.api.vulnerability.read(
filters={"key": "name", "values": [match[RESULT_FORMAT_MATCH]]}
)
if observable is None:
self.helper.log_info(
f"Vulnerability with name '{match[RESULT_FORMAT_MATCH]}' could not be "
f"found. Is the CVE Connector activated?"
)
continue
elif match[RESULT_FORMAT_CATEGORY] == "Attack-Pattern.x_mitre_id":
observable = self.helper.api.attack_pattern.read(
filters={
"key": "x_mitre_id",
"values": [match[RESULT_FORMAT_MATCH]],
}
)
if observable is None:
self.helper.log_info(
f"AttackPattern with MITRE ID '{match[RESULT_FORMAT_MATCH]}' could not be "
f"found. Is the MITRE Connector activated?"
)
continue
else:
observable = self.helper.api.stix_cyber_observable.create(
simple_observable_id=OpenCTIStix2Utils.generate_random_stix_id(
"x-opencti-simple-observable"
),
simple_observable_key=match[RESULT_FORMAT_CATEGORY],
simple_observable_value=match[RESULT_FORMAT_MATCH],
createIndicator=self.create_indicator,
)
observables.append(observable["id"])
elif match[RESULT_FORMAT_TYPE] == ENTITY_CLASS:
entities.append(match[RESULT_FORMAT_MATCH])
else:
self.helper.log_info("Odd data received: {}".format(match))
return observables, entities
def _process_observables(self, report: Dict, observables: List) -> int:
if report is None:
self.helper.log_error(
"No report found! This is a purely contextual connector and this should not happen"
)
if len(observables) == 0:
return 0
report = self.helper.api.report.create(
id=report["standard_id"],
name=report["name"],
description=report["description"],
published=self.helper.api.stix2.format_date(report["created"]),
report_types=report["report_types"],
update=True,
)
for observable in observables:
self.helper.api.report.add_stix_object_or_stix_relationship(
id=report["id"], stixObjectOrStixRelationshipId=observable
)
return len(observables)
def _process_entities(self, report: Dict, entities: List) -> int:
if report:
for stix_object in entities:
self.helper.api.report.add_stix_object_or_stix_relationship(
id=report["id"], stixObjectOrStixRelationshipId=stix_object
)
return len(entities)
class ImportFilePdfObservables:
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(os.path.abspath(__file__)) + "/config.yml"
config = (
yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path)
else {}
)
self.helper = OpenCTIConnectorHelper(config)
self.create_indicator = get_config_variable(
"PDF_OBSERVABLES_CREATE_INDICATOR",
["pdf_observables", "create_indicator"],
config,
)
self.types = {
"MD5": "File.hashes.MD5",
"SHA1": "File.hashes.SHA-1",
"SHA256": "File.hashes.SHA-256",
"Filename": "File.name",
"IP": "IPv4-Addr.value",
"DomainName": "Domain-Name.value",
# Directory is not yet fully functional
# "Directory": "Directory.path",
"URL": "Url.value",
"Email": "Email-Addr.value",
"CVE": "Vulnerability.name",
"Registry": "Windows-Registry-Key.key",
}
def _process_message(self, data):
file_fetch = data["file_fetch"]
file_uri = self.helper.opencti_url + file_fetch
file_name = os.path.basename(file_fetch)
entity_id = data.get("entity_id", None)
self.helper.log_info(entity_id)
# Get context
is_context = entity_id is not None and len(entity_id) > 0
self.helper.log_info("Context: {}".format(is_context))
self.helper.log_info(
"get_only_contextual: {}".format(self.helper.get_only_contextual())
)
if self.helper.get_only_contextual() and not is_context:
raise ValueError(
"No context defined, connector is get_only_contextual true"
)
self.helper.log_info("Importing the file " + file_uri)
# Get the file
file_content = self.helper.api.fetch_opencti_file(file_uri, True)
# Write the file
f = open(file_name, "wb")
f.write(file_content)
f.close()
# Parse
bundle_objects = []
stix_objects = set()
i = 0
custom_indicators = self._get_entities()
mime_type = self._get_mime_type(file_name)
print(mime_type)
if mime_type is None:
raise ValueError("Invalid file format of {}".format(file_name))
parser = iocp.IOC_Parser(
None,
mime_type,
True,
"pdfminer",
"json",
custom_indicators=custom_indicators,
)
parsed = parser.parse(file_name)
os.remove(file_name)
if parsed != []:
for file in parsed:
if file != None:
for page in file:
if page != []:
for match in page:
resolved_match = self._resolve_match(match)
if resolved_match:
# For the creation of relationships
if resolved_match[
"type"
] not in self.types.values() and self._is_uuid(
resolved_match["value"]
):
stix_objects.add(resolved_match["value"])
# For CVEs since SimpleObservable doesn't support Vulnerabilities yet
elif resolved_match["type"] == "Vulnerability.name":
vulnerability = Vulnerability(
name=resolved_match["value"]
)
bundle_objects.append(vulnerability)
# Other observables
elif resolved_match["type"] in self.types.values():
observable = SimpleObservable(
id=OpenCTIStix2Utils.generate_random_stix_id(
"x-opencti-simple-observable"
),
key=resolved_match["type"],
value=resolved_match["value"],
x_opencti_create_indicator=self.create_indicator,
)
bundle_objects.append(observable)
else:
self.helper.log_info(
"Odd data received: {}".format(
resolved_match
)
)
i += 1
else:
self.helper.log_error("Could not parse the report!")
if is_context:
entity = self.helper.api.stix_domain_object.read(id=entity_id)
if entity is not None:
if entity["entity_type"] == "Report" and len(bundle_objects) > 0:
report = Report(
id=entity["standard_id"],
name=entity["name"],
description=entity["description"],
published=self.helper.api.stix2.format_date(entity["created"]),
report_types=entity["report_types"],
object_refs=bundle_objects,
)
bundle_objects.append(report)
bundles_sent = []
if len(bundle_objects) > 0:
bundle = Bundle(objects=bundle_objects).serialize()
bundles_sent = self.helper.send_stix2_bundle(bundle)
if len(stix_objects) > 0 and entity_id is not None:
report = self.helper.api.report.read(id=entity_id)
if report:
for stix_object in stix_objects:
self.helper.log_info(stix_object)
self.helper.api.report.add_stix_object_or_stix_relationship(
id=report["id"], stixObjectOrStixRelationshipId=stix_object
)
return "Sent " + str(len(bundles_sent)) + " stix bundle(s) for worker import"
def start(self):
self.helper.listen(self._process_message)
def _resolve_match(self, match):
type = match["type"]
value = match["match"]
if type in self.types:
resolved_type = self.types[type]
if resolved_type == "IPv4-Addr.value":
# Demilitarized IP
if "[.]" in value:
value = value.replace("[.]", ".")
type_0 = self._detect_ip_version(value)
elif resolved_type == "Url.value":
# Demilitarized URL
if "hxxp://" in value:
value = value.replace("hxxp://", "http://")
if "hxxps://" in value:
value = value.replace("hxxps://", "https://")
if "hxxxs://" in value:
value = value.replace("hxxxs://", "https://")
if "[.]" in value:
value = value.replace("[.]", ".")
type_0 = resolved_type
elif resolved_type == "DomainName.value":
# Demilitarized Host
if "[.]" in value:
value = value.replace("[.]", ".")
type_0 = resolved_type
else:
type_0 = resolved_type
return {"type": type_0, "value": value}
elif self._is_uuid(value):
return {"type": type, "value": value}
else:
self.helper.log_info("Some odd info received: {}".format(match))
return False
def _detect_ip_version(self, value):
if len(value) > 16:
return "IPv6-Addr.value"
else:
return "IPv4-Addr.value"
def _is_uuid(self, value):
try:
uuid.UUID(value)
except ValueError:
return False
return True
def _get_entities(self):
setup = {
"attack_pattern": {
"entity_filter": None,
"entity_fields": ["x_mitre_id"],
"type": "entity",
},
"identity": {
"entity_filter": None,
"entity_fields": ["aliases", "name"],
"type": "entity",
},
"location": {
"entity_filter": [{"key": "entity_type", "values": ["Country"]}],
"entity_fields": ["aliases", "name"],
"type": "entity",
},
"intrusion_set": {
"entity_filter": None,
"entity_fields": ["aliases", "name"],
"type": "entity",
},
"malware": {
"entity_filter": None,
"entity_fields": ["aliases", "name"],
"type": "entity",
},
"tool": {
"entity_filter": None,
"entity_fields": ["aliases", "name"],
"type": "entity",
},
}
return self._collect_stix_objects(setup)
def _collect_stix_objects(self, setup_dict: Dict):
base_func = self.helper.api
information_list = {}
for entity, args in setup_dict.items():
func_format = entity
try:
custom_function = getattr(base_func, func_format)
entries = custom_function.list(
getAll=True, filters=args["entity_filter"]
)
information_list[entity] = self._make_1d_list(
entries, args["entity_fields"]
)
except AttributeError:
e = "Selected parser format is not supported: %s" % func_format
raise NotImplementedError(e)
return information_list
def _make_1d_list(self, values, keys):
items = {}
for item in values:
_id = item.get("id")
sub_items = set()
if (
item.get("externalReferences", None) is None
or len(item["externalReferences"]) == 0
):
continue
for key in keys:
elem = item.get(key, [])
if elem:
if type(elem) == list:
sub_items.update(elem)
elif type(elem) == str:
sub_items.add(elem)
items[_id] = sub_items
return items
def _get_mime_type(self, file_name):
mime = magic.Magic(mime=True)
translation = {
"application/pdf": "pdf",
# 'text/html': 'html',
# 'text/plain': 'txt'
}
mimetype = mime.from_file(file_name)
return translation.get(mimetype, None)
