def create():
# Start measuring execution time for evaluation purposes
start_time = time.time()
# Get STIX 2.1 bundle in json format
data = request.get_json() or {}
# Validate input
validation = results = validate_instance(data)
# Return errors if not valid STIX 2.1 format
if not validation.is_valid:
return bad_request(validation.errors[0].message)
# Initialize kafka broker
producer = KafkaProducer(
bootstrap_servers=[kafka_broker],
value_serializer=lambda x: dumps(x).encode('utf-8'))
# Send content to broker
producer.send(kafka_topic, value=data)
# Finish measuring execution time for evaluation purposes
end_time = time.time()
ellapsed_time = end_time - start_time
# Return status code
print("--- %s seconds ---" % (ellapsed_time))
return {'message': 'Success', 'time': ellapsed_time}
def create_results_connection(self, search_id, offset, length):
#search_id is the pattern
observations = []
if "http_user" in self.configuration:
response = requests.get(self.configuration["bundle_url"],
auth=(self.configuration["http_user"],
self.configuration["http_password"]))
else:
response = requests.get(self.configuration["bundle_url"])
if response.status_code != 200:
response.raise_for_status()
bundle = response.json()
if "validate" in self.configuration and self.configuration[
"validate"] is True:
results = validate_instance(bundle)
if results.is_valid is not True:
return {
"success": False,
"message": "Invalid STIX recieved: " + json.dumps(results)
}
for obj in bundle["objects"]:
if obj["type"] == "observed-data":
observations.append(obj)
#Pattern match
results = self.match(search_id, observations, False)
return results[int(offset):int(offset + length)]
def transform(self, obj):
"""
Transforms the given object in to a STIX observation based on the mapping file and transform functions
:param obj: the datasource object that is being converted to stix
:return: the input object converted to stix valid json
"""
object_map = {}
stix_type = 'observed-data'
ds_map = self.ds_to_stix_map
observation = {
'id': stix_type + '--' + str(uuid.uuid4()),
'type': stix_type,
'created_by_ref': self.identity_id,
'objects': {}
}
# create normal type objects
if isinstance(obj, dict):
for ds_key in obj.keys():
self._transform(object_map, observation, ds_map, ds_key, obj)
else:
print("Not a dict: {}".format(obj))
# Validate each STIX object
if self.stix_validator:
validated_result = validate_instance(observation)
print_results(validated_result)
return observation
def create_infrastructure_object_sdo(self, infrastructure_object, enriched_ioc, indicator_id):
try:
stix_type = 'infrastructure'
DEFAULT_SPEC_VERSION = "2.1"
now = "{}Z".format(datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3])
infrastructure = {
'type': stix_type,
'spec_version' : DEFAULT_SPEC_VERSION,
'id': stix_type + '--' + str(uuid.uuid4()),
'created': now,
'modified': now,
'name': 'Infrastructure related to ' + enriched_ioc,
'infrastructure_types': infrastructure_object['infrastructure_types'],
'description' : infrastructure_object['description'] if infrastructure_object.get('description') is not None else ','.join(infrastructure_object.get('infrastructure_types'))
}
infrastructure_types = self.normalized_infra_type(infrastructure['infrastructure_types'])
infrastructure['infrastructure_types'] = infrastructure_types
if self.stix_validator:
options = ValidationOptions(version="2.1")
results = validate_instance(infrastructure, options)
if results.is_valid is False:
print_results(results)
raise Exception(f'Invalid parameter set in infrastructure SDO. Please follow STIX 2.1 spec for properties')
infrastructure_array = [infrastructure]
relationship = self.createRelationship(infrastructure_array, indicator_id)
infrastructure_array += relationship
return infrastructure_array
except Exception as err:
raise Exception(f'Exception occurred in create_infrastructure_object_sdo : {err}')
def create_identity_sdo(self, data_source, namespace):
try:
DETERMINISTIC_IDENTITY_ID = uuid.uuid5(uuid.UUID(namespace), data_source['name'])
DEFAULT_SPEC_VERSION = '2.1'
stix_type = 'identity'
now = "{}Z".format(datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3])
stix_identity_sdo = {
'type': stix_type,
'name': data_source['name'],
'spec_version': DEFAULT_SPEC_VERSION,
'id': stix_type + '--' + str(DETERMINISTIC_IDENTITY_ID),
'created': now,
'modified': now,
}
if data_source.get('description'): stix_identity_sdo['description'] = data_source['description']
if data_source.get('roles'): stix_identity_sdo['roles'] = data_source['roles']
if data_source.get('identity_class'): stix_identity_sdo['identity_class'] = data_source['identity_class']
if data_source.get('sectors'): stix_identity_sdo['sectors'] = data_source['sectors']
if data_source.get('sectors'): stix_identity_sdo['sectors'] = data_source['sectors']
if data_source.get('contact_information'): stix_identity_sdo['contact_information'] = data_source['contact_information']
if self.stix_validator:
options = ValidationOptions(version="2.1")
results = validate_instance(stix_identity_sdo, options)
if results.is_valid is False:
print_results(results)
raise Exception(f'Invalid parameter set in identity SDO. Please follow STIX 2.1 spec for properties')
return [stix_identity_sdo]
except Exception as err:
raise Exception(f'Exception occurred in create_identity_sdo in BaseNormalization : {err}')
def test_certificate_cim_to_stix(self):
count = 1
time = "2018-08-21T15:11:55.000+00:00"
serial = "1234"
version = "1"
sig_algorithm = "md5WithRSAEncryption"
key_algorithm = "rsaEncryption"
issuer = "C=US, ST=California, O=www.example.com, OU=new, CN=new"
subject = "C=US, ST=Maryland, L=Baltimore, O=John Doe, OU=ExampleCorp, CN=www.example.com/[email protected]"
ssl_hash = "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f"
data = {
"event_count": count,
"_time": time,
"ssl_serial": serial,
"ssl_version": version,
"ssl_signature_algorithm": sig_algorithm,
"ssl_issuer": issuer,
"ssl_subject": subject,
"ssl_hash": ssl_hash,
"ssl_publickey_algorithm": key_algorithm
}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], get_module_transformers(MODULE),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
validated_result = validate_instance(observed_data)
assert (validated_result.is_valid == True)
assert ('objects' in observed_data)
objects = observed_data['objects']
# Test objects in Stix observable data model after transform
cert_obj = TestTransform.get_first_of_type(objects.values(),
'x509-certificate')
assert (cert_obj is not None), 'x509-certificate object type not found'
assert (cert_obj.keys() == {
'type', 'serial_number', 'version', "signature_algorithm",
"subject_public_key_algorithm", "issuer", "subject", "hashes"
})
assert (cert_obj['serial_number'] == "1234")
assert (cert_obj['version'] == "1")
assert (cert_obj['signature_algorithm'] == "md5WithRSAEncryption")
assert (cert_obj['issuer'] ==
"C=US, ST=California, O=www.example.com, OU=new, CN=new")
assert (
cert_obj['subject'] ==
"C=US, ST=Maryland, L=Baltimore, O=John Doe, OU=ExampleCorp, CN=www.example.com/[email protected]"
)
assert (cert_obj['subject_public_key_algorithm'] == "rsaEncryption")
assert (
cert_obj['hashes']['SHA-256'] ==
"aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f")
assert (objects.keys() == set(map(str, range(0, 1))))
def create_sighting_sdo(self, sighting_object, indicator_id):
try:
stix_type = 'sighting'
DEFAULT_SPEC_VERSION = "2.1"
now = "{}Z".format(datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3])
sighting = {
'type': stix_type,
'spec_version' : DEFAULT_SPEC_VERSION,
'id': stix_type + '--' + str(uuid.uuid4()),
'sighting_of_ref': indicator_id,
'count': sighting_object['count'],
'created': now,
'modified': now
}
if self.stix_validator:
options = ValidationOptions(version="2.1")
results = validate_instance(sighting, options)
if results.is_valid is False:
print_results(results)
raise Exception(f'Invalid parameter set in sighting SDO. Please follow STIX 2.1 spec for properties')
return [sighting]
except Exception as err:
raise Exception(f'Exception occurred in create_sighting_sdo in BaseNormalization : {err}')
def test_network_cim_to_stix(self):
count = 2
time = "2018-08-21T15:11:55.000+00:00"
user = "******"
dest_ip = "127.0.0.1"
dest_port = "8090"
src_ip = "2001:0db8:85a3:0000:0000:8a2e:0370:7334"
src_port = "8080"
transport = "http"
data = {
"event_count": count,
"_time": time,
"user": user,
"dest_ip": dest_ip,
"dest_port": dest_port,
"src_ip": src_ip,
"src_port": src_port,
"protocol": transport
}
print(data)
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], get_module_transformers(MODULE),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
validated_result = validate_instance(observed_data)
assert (validated_result.is_valid == True)
assert ('objects' in observed_data)
objects = observed_data['objects']
nt_obj = TestTransform.get_first_of_type(objects.values(),
'network-traffic')
assert (nt_obj is not None), 'network-traffic object type not found'
assert (nt_obj.keys() == {
'type', 'src_port', 'dst_port', 'src_ref', 'dst_ref', 'protocols'
})
assert (nt_obj['src_port'] == 8080)
assert (nt_obj['dst_port'] == 8090)
assert (nt_obj['protocols'] == ['http'])
ip_ref = nt_obj['dst_ref']
assert (ip_ref
in objects), f"dst_ref with key {nt_obj['dst_ref']} not found"
ip_obj = objects[ip_ref]
assert (ip_obj.keys() == {'type', 'value'})
assert (ip_obj['type'] == 'ipv4-addr')
assert (ip_obj['value'] == dest_ip)
ip_ref = nt_obj['src_ref']
assert (ip_ref
in objects), f"src_ref with key {nt_obj['src_ref']} not found"
ip_obj = objects[ip_ref]
assert (ip_obj.keys() == {'type', 'value'})
assert (ip_obj['type'] == 'ipv6-addr')
assert (ip_obj['value'] == src_ip)
def test_email_cim_to_stix(self):
tag = "email"
count = 3
time = "2018-08-21T15:11:55.000+00:00"
src_user = "******"
subject = "Test Subject"
multi = "False"
data = {
"tag": tag,
"event_count": count,
"_time": time,
"src_user": src_user,
"subject": subject,
"is_multipart": multi
}
result_bundle = cim_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
validated_result = validate_instance(observed_data)
assert (validated_result.is_valid == True)
assert ('objects' in observed_data)
objects = observed_data['objects']
msg_obj = TestTransform.get_first_of_type(objects.values(),
'email-message')
assert (msg_obj is not None), 'email-message object type not found'
assert (msg_obj.keys() == {
'type', 'subject', 'sender_ref', 'from_ref', 'is_multipart'
})
assert (msg_obj['subject'] == "Test Subject")
assert (msg_obj['is_multipart'] == False)
sender_ref = msg_obj['sender_ref']
assert (sender_ref in objects
), f"sender_ref with key {msg_obj['sender_ref']} not found"
addr_obj = objects[sender_ref]
assert (addr_obj.keys() == {'type', 'value'})
assert (addr_obj['type'] == 'email-addr')
assert (addr_obj['value'] == src_user)
from_ref = msg_obj['from_ref']
assert (
sender_ref
in objects), f"from_ref with key {msg_obj['from_ref']} not found"
addr_obj = objects[from_ref]
assert (addr_obj.keys() == {'type', 'value'})
assert (addr_obj['type'] == 'email-addr')
assert (addr_obj['value'] == src_user)
def create_results_connection(self, search_id, offset, length):
# search_id is the pattern
observations = []
return_obj = dict()
response = self.client.call_api(self.bundle_url,
'get',
timeout=self.timeout)
if response.code != 200:
response_txt = response.raise_for_status()
if ErrorResponder.is_plain_string(response_txt):
ErrorResponder.fill_error(return_obj, message=response_txt)
elif ErrorResponder.is_json_string(response_txt):
response_json = json.loads(response_txt)
ErrorResponder.fill_error(return_obj, response_json,
['reason'])
else:
raise UnexpectedResponseException
else:
try:
response_txt = response.read().decode('utf-8')
bundle = json.loads(response_txt)
if "stix_validator" in self.connection[
'options'] and self.connection['options'].get(
"stix_validator") is True:
results = validate_instance(bundle)
if results.is_valid is not True:
ErrorResponder.fill_error(
return_obj,
message='Invalid Objects in STIX Bundle.')
return return_obj
for obj in bundle["objects"]:
if obj["type"] == "observed-data":
observations.append(obj)
# Pattern match
try:
results = self.match(search_id, observations, False)
if len(results) != 0:
return_obj['success'] = True
return_obj['data'] = results[int(offset):int(offset +
length)]
else:
return_obj['success'] = True
return_obj['data'] = []
except Exception as ex:
ErrorResponder.fill_error(
return_obj,
message='Object matching error: ' + str(ex))
except Exception as ex:
ErrorResponder.fill_error(
return_obj,
message='Invalid STIX bundle. Malformed JSON: ' + str(ex))
return return_obj
def create_results_connection(self, search_id, offset, length):
# search_id is the pattern
observations = []
return_obj = dict()
bundle_url = self.connection.get('host')
auth = self.configuration.get('auth')
if auth is not None:
response = requests.get(bundle_url,
auth=(auth.get('username'),
auth.get('password')))
else:
response = requests.get(bundle_url)
response_code = response.status_code
if response_code != 200:
response_txt = response.raise_for_status()
if ErrorResponder.is_plain_string(response_txt):
ErrorResponder.fill_error(return_obj, message=response_txt)
elif ErrorResponder.is_json_string(response_txt):
response_json = json.loads(response_txt)
ErrorResponder.fill_error(return_obj, response_json,
['reason'])
else:
raise UnexpectedResponseException
else:
bundle = response.json()
if "validate" in self.configuration and self.configuration[
"validate"] is True:
results = validate_instance(bundle)
if results.is_valid is not True:
return {
"success": False,
"message":
"Invalid STIX received: " + json.dumps(results)
}
for obj in bundle["objects"]:
if obj["type"] == "observed-data":
observations.append(obj)
# Pattern match
results = self.match(search_id, observations, False)
if len(results) != 0:
return_obj['success'] = True
return_obj['data'] = results[int(offset):int(offset + length)]
else:
return_obj['success'] = True
return_obj['data'] = []
return return_obj
def bundle_analysis(db, bundle, collection_id):
if bundle['type'] == 'bundle':
status = {
"id": str(uuid.uuid4()),
"status": None,
"request_timestamp": datetime.utcnow(),
"total_count": len(bundle['objects']),
"success_count": 0,
"successes": [],
"failure_count": 0,
"failures": [],
"pending_count": 0,
"pendings": []
}
for object in bundle['objects']:
results = validate_instance(object)
if results.is_valid:
date_added = datetime.utcnow()
print(object["id"])
manifest = {
"id": object["id"],
"date_added": date_added,
"versions": [],
"media_types":
["application/vnd.oasis.stix+json; version=2.0"],
"_collection_id": collection_id
}
manifest['versions'].append(date_added)
# TODO check if object exists and update manifest instead of adding new
db.manifest.insert_one(manifest)
status["success_count"] += 1
status["successes"].append(object["id"])
# insert collection_id field and insert object in database
object['_collection_id'] = collection_id
db.objects.insert_one(object)
else:
status["failure_count"] += 1
status["failures"].append(object["id"])
status["status"] = 'complete'
db.status.insert_one(status)
status.pop('_id')
resp = jsonify(status)
resp.status_code = 200
return resp
else:
# Respond that submitted input is not a STIX2 bundle
debugstr = "Submitted Input is not a valid STIX bundle"
message = {
'status': 422,
'message': debugstr,
}
resp = jsonify(message)
resp.status_code = 404
return resp
def create_results_connection(self, search_id, offset, length):
observations = []
return_obj = dict()
response = None
if self.connection['options'].get('error_type') == ERROR_TYPE_TIMEOUT:
# httpstat.us/200?sleep=60000 for slow connection that is valid
self.client.call_api('https://httpstat.us/200?sleep=60000', 'get', timeout=self.timeout)
elif self.connection['options'].get('error_type') == ERROR_TYPE_BAD_CONNECTION:
# www.google.com:81 for a bad connection that will timeout
response = self.client.call_api('https://www.google.com:81', 'get', timeout=self.timeout)
if not response:
response = self.client.call_api(self.bundle_url, 'get', timeout=self.timeout)
if response.code != 200:
response_txt = response.raise_for_status()
if ErrorResponder.is_plain_string(response_txt):
ErrorResponder.fill_error(return_obj, message=response_txt, connector=self.connector)
elif ErrorResponder.is_json_string(response_txt):
response_json = json.loads(response_txt)
ErrorResponder.fill_error(return_obj, response_json, ['reason'], connector=self.connector)
else:
raise UnexpectedResponseException
else:
try:
response_txt = response.read().decode('utf-8')
bundle = json.loads(response_txt)
if "stix_validator" in self.connection['options'] and self.connection['options'].get("stix_validator") is True:
results = validate_instance(bundle)
if results.is_valid is not True:
ErrorResponder.fill_error(return_obj, message='Invalid Objects in STIX Bundle.', connector=self.connector)
return return_obj
for obj in bundle["objects"]:
if obj["type"] == "observed-data":
observations.append(obj)
# Pattern match
try:
results = self.match(search_id, observations, False)
if len(results) != 0:
return_obj['success'] = True
return_obj['data'] = results[int(offset):int(offset + length)]
else:
return_obj['success'] = True
return_obj['data'] = []
except Exception as ex:
ErrorResponder.fill_error(return_obj, message='Object matching error: ' + str(ex), connector=self.connector)
except Exception as ex:
ErrorResponder.fill_error(return_obj, message='Invalid STIX bundle. Malformed JSON: ' + str(ex), connector=self.connector)
return return_obj
def transform(self, obj):
"""
Transforms the given object in to a STIX observation based on the mapping file and transform functions
:param obj: the datasource object that is being converted to stix
:return: the input object converted to stix valid json
"""
NUMBER_OBSERVED_KEY = 'number_observed'
object_map = {}
stix_type = 'observed-data'
ds_map = self.ds_to_stix_map
observation = {
'id':
stix_type + '--' + str(uuid.uuid4()),
'type':
stix_type,
'created_by_ref':
self.identity_id,
'created':
"{}Z".format(
datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3]),
'modified':
"{}Z".format(
datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3]),
'objects': {}
}
# create normal type objects
if isinstance(obj, dict):
for ds_key in obj.keys():
self._transform(object_map, observation, ds_map, ds_key, obj)
else:
self.logger.debug("Not a dict: {}".format(obj))
# Add required property to the observation if it wasn't added via the mapping
if self.options.get('unmapped_fallback'):
if "first_observed" not in observation and "last_observed" not in observation:
observation['first_observed'] = "{}Z".format(
datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3])
observation['last_observed'] = "{}Z".format(
datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3])
# Add required property to the observation if it wasn't added via the mapping
if NUMBER_OBSERVED_KEY not in observation:
observation[NUMBER_OBSERVED_KEY] = 1
# Validate each STIX object
if self.stix_validator:
validated_result = validate_instance(observation)
print_results(validated_result)
return observation
def create_malware_sdo(self,malware_object, indicator_id, enriched_ioc):
try:
malware_array=[]
if isinstance(malware_object, list):
for data in malware_object:
#print(data)
stix_type = 'malware'
DEFAULT_SPEC_VERSION = "2.1"
now = "{}Z".format(datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3])
malware = {
'type': stix_type,
'name': data.get('name') if data.get('name') is not None else 'Malware related to ' + enriched_ioc,
'spec_version': DEFAULT_SPEC_VERSION,
'id': stix_type + '--' + str(uuid.uuid4()),
'created': now,
'modified': now,
'malware_types': data.get('malware_types') if data.get('malware_types') is not None else ['unknown'],
'is_family' : data.get('is_family') if data.get('is_family') is not None else False
}
# right now its iterates additional attributes of malware SDO and no null, empty list is not checked. Developer has to ensure not to send such data
for key,value in data.items():
if key is not malware:
malware[key] = value
# set the description same as malware type returns from threat feed if description property is not provided.
if data.get('description'):
malware['description'] = data.get('description')
elif data.get('malware_types') and 'unknown' not in data.get('malware_types'):
malware['description'] = ','.join(data.get('malware_types')) if isinstance(data.get('malware_types'),list) else data.get('malware_types')
malware_types = self.normalized_malware_type(malware['malware_types'])
malware['malware_types'] = malware_types
# malware SDO properties validation
if self.stix_validator:
options = ValidationOptions(version="2.1")
results = validate_instance(malware, options)
if results.is_valid is False:
print_results(results)
raise Exception(f'Invalid parameter set in malware SDO. Please follow STIX 2.1 spec for properties')
# if name is not present then compare only malware_types to remove duplicate else check malware types and name.
if (len([i for i in malware_array if (i['malware_types'] == malware ['malware_types'] and i['name'] == malware ['name'])]) == 0):
malware_array.append(malware)
relationship = self.createRelationship(malware_array, indicator_id)
malware_array += relationship
return malware_array
except Exception as err:
raise Exception(f'Exception occurred in create_malware_sdo in BaseNormalization : {err}')
def create_indicator_sdo(self, indicator_object: dict, identity_id: str, extension_id:str=None, nested_properties:list=None, top_properties:list=None):
try:
# Param: Dictionary
stix_type = 'indicator'
pattern_type = 'stix'
DEFAULT_SPEC_VERSION = "2.1"
now = "{}Z".format(datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3])
# Exception handle required property
if 'pattern' not in indicator_object:
raise ValueError(f'Missing required indicator property: pattern')
indicator = {
'type': stix_type,
'spec_version': DEFAULT_SPEC_VERSION,
'id': stix_type + '--' + str(uuid.uuid4()),
'pattern': indicator_object['pattern'],
'pattern_type': pattern_type,
'created_by_ref': identity_id,
'created': now,
'modified': now,
'valid_from': now,
}
if indicator_object.get('name'): indicator['name'] = indicator_object['name']
if indicator_object.get('description'): indicator['description'] = indicator_object['description']
if indicator_object.get('pattern_version'): indicator['pattern_version'] = indicator_object['pattern_version']
if indicator_object.get('valid_until'): indicator['valid_until'] = indicator_object['valid_until']
if indicator_object.get('kill_chain_phases'): indicator['kill_chain_phases'] = indicator_object['kill_chain_phases']
if indicator_object.get('indicator_types'): indicator['indicator_types'] = indicator_object['indicator_types']
if indicator_object.get('external_references'): indicator['external_references'] = indicator_object['external_references']
if (extension_id):
indicator = self.add_extension(indicator, extension_id, nested_properties, top_properties)
# indicator SDO properties validation
if self.stix_validator:
options = ValidationOptions(version="2.1")
results = validate_instance(indicator, options)
if results.is_valid is False:
print_results(results)
raise Exception(f'Invalid parameter set in indicator SDO. Please follow STIX 2.1 spec for properties')
return [indicator]
except ValueError as err:
raise ValueError(err)
def __main__():
bundle_file = sys.argv[1]
try:
with open(bundle_file) as f:
bundle = json.load(f)
results = validate_instance(bundle)
if results.is_valid is not True:
print_results(results)
raise Exception()
print('*** STIX Bundle validated!!\n')
except ValueError as ex:
print("*** Malformed JSON in the STIX Bundle: " + str(ex))
except Exception as ex:
print(
"\n *** Invalid STIX Objects found in the bundle. Please fix the error marked as Red[X]. Warnings marked as yellow [!] can be ingnored but recommended to fix ***\n"
)
def create_extension_sdo(self, identity_object, namespace, nested_properties=[], toplevel_properties=[], schema='https://www.ibm.com/cp4s'):
try:
# Create an extension-definition object to be used in conjunction with STIX Indicator object
stix_type = 'extension-definition'
DEFAULT_SPEC_VERSION = "2.1"
EXTENSION_VERSION = '1.2.1'
extension_object = {
'id': stix_type + '--' + str(uuid.uuid5(uuid.UUID(namespace), 'extension-definition')),
'type': stix_type,
'spec_version': DEFAULT_SPEC_VERSION,
'name': (identity_object.get('name') + ' extension') if identity_object.get('name') is not None else "extension definition object",
'created': "{}Z".format(datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3]),
'modified': "{}Z".format(datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3]),
'created_by_ref': identity_object['id'],
'schema': schema,
'version': EXTENSION_VERSION,
}
if identity_object.get('description'): extension_object['description'] = 'Extension object for ' + identity_object.get('description')
if (len(nested_properties) > 0 or len(toplevel_properties) > 0):
extension_object['extension_types'] = []
extension_object['extension_properties'] = []
if (len(toplevel_properties) > 0):
extension_object['extension_types'].append('toplevel-property-extension')
for prop in toplevel_properties:
extension_object['extension_properties'].append(prop)
if (len(nested_properties) > 0):
extension_object['extension_types'].append('property-extension')
if (not len(extension_object['extension_properties']) > 0):
del extension_object['extension_properties']
if self.stix_validator:
options = ValidationOptions(version="2.1")
results = validate_instance(extension_object, options)
if results.is_valid is False:
print_results(results)
raise Exception(f'Invalid parameter set in extension_object SDO. Please follow STIX 2.1 spec for properties')
stix_extension_sdo = [extension_object]
return stix_extension_sdo
except Exception as err:
raise Exception(f'Exception occurred in create_extension_sdo in BaseNormalization : {err}')
def __main__():
bundle_file = sys.argv[1]
try:
with open(bundle_file) as f:
bundle = json.load(f)
results = validate_instance(bundle)
if results.is_valid:
print_results(results)
print(
"\n *** STIX bundle is valid but may contain warnings. Warnings marked as yellow [!] can be ignored but recommended to fix ***\n"
)
else:
print_results(results)
print(
"\n *** Invalid STIX Objects found in the bundle. Please fix the error marked as Red[X]. Warnings marked as yellow [!] can be ignored but recommended to fix ***\n"
)
except ValueError as ex:
print("*** Malformed JSON in the STIX Bundle: " + str(ex))
def transform(self, obj):
"""
Transforms the given object in to a STIX observation based on the mapping file and transform functions
:param obj: the datasource object that is being converted to stix
:return: the input object converted to stix valid json
"""
object_map = {}
stix_type = 'observed-data'
ds_map = self.ds_to_stix_map
transformers = self.transformers
observation = {
'id': stix_type + '--' + str(uuid.uuid4()),
'type': stix_type,
'created_by_ref': self.identity_id,
'objects': {}
}
# create normal type objects
for ds_key in obj:
if ds_key not in ds_map:
logging.debug(
'{} is not found in map, skipping'.format(ds_key))
continue
# get the stix keys that are mapped
ds_key_def_obj = self.ds_to_stix_map[ds_key]
ds_key_def_list = ds_key_def_obj if isinstance(
ds_key_def_obj, list) else [ds_key_def_obj]
for ds_key_def in ds_key_def_list:
if ds_key_def is None or 'key' not in ds_key_def:
logging.debug(
'{} is not valid (None, or missing key)'.format(
ds_key_def))
continue
key_to_add = ds_key_def['key']
transformer = transformers[ds_key_def[
'transformer']] if 'transformer' in ds_key_def else None
if ds_key_def.get('cybox', self.cybox_default):
object_name = ds_key_def.get('object')
if 'references' in ds_key_def:
stix_value = object_map[ds_key_def['references']]
else:
stix_value = DataSourceObjToStixObj._get_value(
obj, ds_key, transformer)
if not DataSourceObjToStixObj._valid_stix_value(
self.properties, key_to_add, stix_value):
continue
DataSourceObjToStixObj._handle_cybox_key_def(
key_to_add, observation, stix_value, object_map,
object_name)
else:
stix_value = DataSourceObjToStixObj._get_value(
obj, ds_key, transformer)
if not DataSourceObjToStixObj._valid_stix_value(
self.properties, key_to_add, stix_value):
continue
DataSourceObjToStixObj._add_property(
observation, key_to_add, stix_value)
# Validate each STIX object
if self.stix_validator:
validated_result = validate_instance(observation)
print_results(validated_result)
return observation
def test_change_cim_to_stix(self):
count = 1
time = "2018-08-21T15:11:55.000+00:00"
file_bytes = "300"
user = "******"
objPath = "hkey_local_machine\\system\\bar\\foo"
filePath = "C:\\Users\\someuser\\sample.dll"
create_time = "2018-08-15T15:11:55.676+00:00"
modify_time = "2018-08-15T18:10:30.456+00:00"
file_hash = "41a26255d16d121dc525a6445144b895"
file_name = "sample.dll"
file_size = 25536
data = {
"event_count": count,
"_time": time,
"user": user,
"bytes": file_bytes,
"object_path": objPath,
"file_path": filePath,
"file_create_time": create_time,
"file_modify_time": modify_time,
"file_hash": file_hash,
"file_size": file_size,
"file_name": file_name
}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source,
map_data, [data],
get_module_transformers(MODULE),
options,
callback=hash_type_lookup)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
validated_result = validate_instance(observed_data)
assert (validated_result.is_valid == True)
assert ('objects' in observed_data)
objects = observed_data['objects']
# Test objects in Stix observable data model after transform
wrk_obj = TestTransform.get_first_of_type(objects.values(),
'windows-registry-key')
assert (wrk_obj is not None)
assert (wrk_obj.keys() == {'type', 'key'})
assert (wrk_obj['key'] == "hkey_local_machine\\system\\bar\\foo")
user_obj = TestTransform.get_first_of_type(objects.values(),
'user-account')
assert (user_obj is not None), 'user-account object type not found'
assert (user_obj.keys() == {'type', 'account_login', 'user_id'})
assert (user_obj['account_login'] == "ibm_user")
assert (user_obj['user_id'] == "ibm_user")
file_obj = TestTransform.get_first_of_type(objects.values(), 'file')
assert (file_obj is not None), 'file object type not found'
assert (file_obj.keys() == {
'type', 'parent_directory_ref', 'created', 'modified', 'hashes',
'name', 'size'
})
assert (file_obj['created'] == "2018-08-15T15:11:55.676Z")
assert (file_obj['modified'] == "2018-08-15T18:10:30.456Z")
assert (file_obj['name'] == "sample.dll")
assert (file_obj['size'] == 25536)
assert (
file_obj['hashes']['MD5'] == "41a26255d16d121dc525a6445144b895")
dir_ref = file_obj['parent_directory_ref']
assert (
dir_ref in objects
), f"parent_directory_ref with key {file_obj['parent_directory_ref']} not found"
dir_obj = objects[dir_ref]
assert (dir_obj is not None), 'directory object type not found'
assert (dir_obj.keys() == {'type', 'path', 'created', 'modified'})
assert (dir_obj['path'] == "C:\\Users\\someuser\\sample.dll")
assert (dir_obj['created'] == "2018-08-15T15:11:55.676Z")
assert (dir_obj['modified'] == "2018-08-15T18:10:30.456Z")
print(objects.keys())
print(result_bundle_objects)
assert (objects.keys() == set(map(str, range(0, 5))))
def transform(self, obj):
"""
Transforms the given object in to a STIX observation based on the mapping file and transform functions
:param obj: the datasource object that is being converted to stix
:return: the input object converted to stix valid json
"""
index = 0
ref_objs = {}
linked_objs = {}
stix_type = 'observed-data'
uniq_id = str(uuid.uuid4())
ds_map = self.dsToStixMap
xformers = self.transformers
observation = {
'x_com_ibm_uds_datasource': {
'id': self.datasource['id'],
'name': self.datasource['name']
},
'id': stix_type + '--' + uniq_id,
'type': stix_type,
'objects': {},
}
# create normal type objects
for ds_key in ds_map:
# get the stix keys that are mapped
ds_key_def_obj = self.dsToStixMap[ds_key]
ds_key_def_list = ds_key_def_obj if isinstance(
ds_key_def_obj, list) else [ds_key_def_obj]
for ds_key_def in ds_key_def_list:
if ds_key_def is None or 'key' not in ds_key_def or 'type' not in ds_key_def:
logging.debug(
'{} is not valid (None, or missing key and type)'.
format(ds_key_def))
continue
if ds_key_def['type'] != 'value' or 'cybox' in ds_key_def:
continue
key_to_add = ds_key_def['key']
transformer = xformers[ds_key_def[
'transformer']] if 'transformer' in ds_key_def else None
linked = ds_key_def[
'linked'] if 'linked' in ds_key_def else None
stix_value = DataSourceObjToStixObj._get_value(
obj, ds_key, transformer)
if stix_value is None:
continue
prop_obj = DataSourceObjToStixObj._determine_prop_attr(
key_to_add, self.outer_props, self.simple_props)
if prop_obj[0] is not None and 'valid_regex' in prop_obj[0]:
pattern = re.compile(prop_obj[0]['valid_regex'])
if not pattern.match(str(stix_value)):
continue
# handle when object is linked
if linked is not None:
observation = DataSourceObjToStixObj._handle_linked(
key_to_add, observation, stix_value)
elif prop_obj[1] == 'OUTER':
observation.update({key_to_add: stix_value})
else:
index = (self._add_to_objects(key_to_add, stix_value,
observation, index, ds_key,
ref_objs, linked,
linked_objs, True, None))
# create complex type objects
DataSourceObjToStixObj._create_complex_objects(ds_map, xformers, index,
observation, ref_objs,
linked_objs, obj)
# Validate each STIX object
if self.stix_validator:
validated_result = validate_instance(observation)
print_results(validated_result)
return observation
def test_process_cim_to_stix(self):
count = 1
time = "2018-08-21T15:11:55.000+00:00"
user = "******"
pid = 0
name = "test_process"
filePath = "C:\\Users\\someuser\\sample.dll"
create_time = "2018-08-15T15:11:55.676+00:00"
modify_time = "2018-08-15T18:10:30.456+00:00"
file_hash = "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f"
file_name = "sample.dll"
file_size = 25536
data = {
"event_count": count,
"_time": time,
"user": user,
"process_name": name,
"process_id": pid,
"file_path": filePath,
"file_create_time": create_time,
"file_modify_time": modify_time,
"file_hash": file_hash,
"file_size": file_size,
"file_name": file_name
}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source,
map_data, [data],
get_module_transformers(MODULE),
options,
callback=hash_type_lookup)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
validated_result = validate_instance(observed_data)
assert (validated_result.is_valid == True)
assert ('objects' in observed_data)
objects = observed_data['objects']
# Test objects in Stix observable data model after transform
proc_obj = TestTransform.get_first_of_type(objects.values(), 'process')
assert (proc_obj is not None), 'process object type not found'
assert (proc_obj.keys() == {'type', 'name', 'pid', 'binary_ref'})
assert (proc_obj['name'] == "test_process")
assert (proc_obj['pid'] == 0)
user_obj = TestTransform.get_first_of_type(objects.values(),
'user-account')
assert (user_obj is not None), 'user-account object type not found'
assert (user_obj.keys() == {'type', 'account_login', 'user_id'})
assert (user_obj['account_login'] == "test_user")
assert (user_obj['user_id'] == "test_user")
bin_ref = proc_obj['binary_ref']
assert (bin_ref in objects
), f"binary_ref with key {proc_obj['binary_ref']} not found"
file_obj = objects[bin_ref]
assert (file_obj is not None), 'file object type not found'
assert (file_obj.keys() == {
'type', 'parent_directory_ref', 'created', 'modified', 'size',
'name', 'hashes'
})
assert (file_obj['created'] == "2018-08-15T15:11:55.676Z")
assert (file_obj['modified'] == "2018-08-15T18:10:30.456Z")
assert (file_obj['name'] == "sample.dll")
assert (file_obj['size'] == 25536)
assert (
file_obj['hashes']['SHA-256'] ==
"aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f")
dir_ref = file_obj['parent_directory_ref']
assert (
dir_ref in objects
), f"parent_directory_ref with key {file_obj['parent_directory_ref']} not found"
dir_obj = objects[dir_ref]
assert (dir_obj is not None), 'directory object type not found'
assert (dir_obj.keys() == {'type', 'path', 'created', 'modified'})
assert (dir_obj['path'] == "C:\\Users\\someuser\\sample.dll")
assert (dir_obj['created'] == "2018-08-15T15:11:55.676Z")
assert (dir_obj['modified'] == "2018-08-15T18:10:30.456Z")
assert (objects.keys() == set(map(str, range(0, 4))))
def test_cim_to_stix_no_tags(self):
data = {"src_ip": "169.250.0.1", "src_port": "1220", "src_mac": "aa:bb:cc:dd:11:22",
"dest_ip": "127.0.0.1", "dest_port": "1120", "dest_mac": "ee:dd:bb:aa:cc:11",
"file_hash": "cf23df2207d99a74fbe169e3eba035e633b65d94",
"user": "******", "url": "https://wally.fireeye.com/malware_analysis/analyses?maid=1",
"protocol": "tcp", "_bkt": "main~44~6D3E49A0-31FE-44C3-8373-C3AC6B1ABF06", "_cd": "44:12606114",
"_indextime": "1546960685",
"_raw": "Jan 08 2019 15:18:04 192.168.33.131 fenotify-2.alert: CEF:0|FireEye|MAS|6.2.0.74298|MO|"
"malware-object|4|rt=Jan 08 2019 15:18:04 Z src=169.250.0.1 dpt=1120 dst=127.0.0.1"
" spt=1220 smac=AA:BB:CC:DD:11:22 dmac=EE:DD:BB:AA:CC:11 cn2Label=sid cn2=111"
" fileHash=41a26255d16d121dc525a6445144b895 proto=tcp "
"request=http://qa-server.eng.fireeye.com/QE/NotificationPcaps/"
"58.253.68.29_80-192.168.85.128_1165-2119283109_T.exe cs3Label=osinfo"
" cs3=Microsoft Windows7 Professional 6.1 sp1 dvchost=wally dvc=10.2.101.101 cn1Label=vlan"
" cn1=0 externalId=1 cs4Label=link "
"cs4=https://wally.fireeye.com/malware_analysis/analyses?maid=1 cs2Label=anomaly"
" cs2=misc-anomaly cs1Label=sname cs1=FE_UPX;Trojan.PWS.OnlineGames",
"_serial": "0", "_si": ["splunk3-01.internal.resilientsystems.com", "main"],
"_sourcetype": "fe_cef_syslog", "_time": "2019-01-08T15:18:04.000+00:00", "event_count": 1
}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(), options, callback=hash_type_lookup)
assert(result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
validated_result = validate_instance(observed_data)
assert(validated_result.is_valid == True)
assert('objects' in observed_data)
objects = observed_data['objects']
nt_obj = TestTransform.get_first_of_type(objects.values(), 'network-traffic')
assert(nt_obj is not None), 'network-traffic object type not found'
assert(nt_obj.keys() == {'type', 'src_ref', 'src_port', 'dst_ref', 'dst_port', 'protocols'})
assert(nt_obj['src_port'] == 1220)
assert(nt_obj['dst_port'] == 1120)
assert(nt_obj['protocols'] == ['tcp'])
nt_obj_2 = objects['2']
assert (nt_obj_2 is not None), 'network-traffic object type not found'
assert (nt_obj_2.keys() == {'type', 'src_ref', 'src_port', 'dst_ref', 'dst_port', 'protocols'})
assert (nt_obj_2['src_port'] == 1220)
assert (nt_obj_2['dst_port'] == 1120)
assert (nt_obj_2['protocols'] == ['tcp'])
mac_ref = nt_obj_2['dst_ref']
assert(mac_ref in objects), "dst_ref with key {nt_obj['dst_ref']} not found"
mac_obj = objects[mac_ref]
assert(mac_obj.keys() == {'type', 'value'})
assert(mac_obj['type'] == 'mac-addr')
assert(mac_obj['value'] == 'ee:dd:bb:aa:cc:11')
mac_ref = nt_obj_2['src_ref']
assert(mac_ref in objects), "src_ref with key {nt_obj['dst_ref']} not found"
mac_obj = objects[mac_ref]
assert(mac_obj.keys() == {'type', 'value'})
assert(mac_obj['type'] == 'mac-addr')
assert(mac_obj['value'] == 'aa:bb:cc:dd:11:22')
ip_ref = nt_obj['dst_ref']
assert(ip_ref in objects), "dst_ref with key {nt_obj['dst_ref']} not found"
ip_obj = objects[ip_ref]
assert(ip_obj.keys() == {'type', 'value'})
assert(ip_obj['type'] == 'ipv4-addr')
assert(ip_obj['value'] == '127.0.0.1')
ip_ref = nt_obj['src_ref']
assert(ip_ref in objects), "src_ref with key {nt_obj['src_ref']} not found"
ip_obj = objects[ip_ref]
assert(ip_obj.keys() == {'type', 'value'})
assert(ip_obj['type'] == 'ipv4-addr')
assert(ip_obj['value'] == '169.250.0.1')
file_obj = TestTransform.get_first_of_type(objects.values(), 'file')
assert (file_obj is not None), 'file object type not found'
assert (file_obj.keys() == {'type', 'hashes'})
assert (file_obj['hashes']['SHA-1'] == "cf23df2207d99a74fbe169e3eba035e633b65d94")
user_obj = TestTransform.get_first_of_type(objects.values(), 'user-account')
assert (user_obj is not None), 'user object type not found'
assert (user_obj.keys() == {'type', 'account_login', 'user_id'})
assert (user_obj['account_login'] == "sname")
assert (user_obj['user_id'] == "sname")
url_obj = TestTransform.get_first_of_type(objects.values(), 'url')
assert (url_obj is not None), 'url object type not found'
assert (url_obj.keys() == {'type', 'value'})
assert (url_obj['value'] == "https://wally.fireeye.com/malware_analysis/analyses?maid=1")
domain_obj = TestTransform.get_first_of_type(objects.values(), 'domain-name')
assert (domain_obj is not None), 'domain object type not found'
assert (domain_obj.keys() == {'type', 'value'})
assert (domain_obj['value'] == "wally.fireeye.com")
payload_obj = TestTransform.get_first_of_type(objects.values(), 'artifact')
assert (payload_obj is not None), 'payload object type not found'
assert (payload_obj.keys() == {'type', 'payload_bin'})
payload = 'SmFuIDA4IDIwMTkgMTU6MTg6MDQgMTkyLjE2OC4zMy4xMzEgZmVub3RpZnktMi5hbGVydDogQ0VGOjB8RmlyZUV5ZXxNQV' \
'N8Ni4yLjAuNzQyOTh8TU98bWFsd2FyZS1vYmplY3R8NHxydD1KYW4gMDggMjAxOSAxNToxODowNCBaIHNyYz0xNjkuMjUw' \
'LjAuMSBkcHQ9MTEyMCBkc3Q9MTI3LjAuMC4xIHNwdD0xMjIwIHNtYWM9QUE6QkI6Q0M6REQ6MTE6MjIgZG1hYz1FRTpERD' \
'pCQjpBQTpDQzoxMSBjbjJMYWJlbD1zaWQgY24yPTExMSBmaWxlSGFzaD00MWEyNjI1NWQxNmQxMjFkYzUyNWE2NDQ1MTQ0' \
'Yjg5NSBwcm90bz10Y3AgcmVxdWVzdD1odHRwOi8vcWEtc2VydmVyLmVuZy5maXJlZXllLmNvbS9RRS9Ob3RpZmljYXRpb2' \
'5QY2Fwcy81OC4yNTMuNjguMjlfODAtMTkyLjE2OC44NS4xMjhfMTE2NS0yMTE5MjgzMTA5X1QuZXhlIGNzM0xhYmVsPW9z' \
'aW5mbyBjczM9TWljcm9zb2Z0IFdpbmRvd3M3IFByb2Zlc3Npb25hbCA2LjEgc3AxIGR2Y2hvc3Q9d2FsbHkgZHZjPTEwLj' \
'IuMTAxLjEwMSBjbjFMYWJlbD12bGFuIGNuMT0wIGV4dGVybmFsSWQ9MSBjczRMYWJlbD1saW5rIGNzND1odHRwczovL3dh' \
'bGx5LmZpcmVleWUuY29tL21hbHdhcmVfYW5hbHlzaXMvYW5hbHlzZXM/bWFpZD0xIGNzMkxhYmVsPWFub21hbHkgY3MyPW' \
'1pc2MtYW5vbWFseSBjczFMYWJlbD1zbmFtZSBjczE9RkVfVVBYO1Ryb2phbi5QV1MuT25saW5lR2FtZXM='
assert (payload_obj['payload_bin'] == payload)
def test_change_cim_to_stix(self):
tag = "change"
count = 1
time = "2018-08-21T15:11:55.000+00:00"
file_bytes = "300"
user = "******"
objPath = "hkey_local_machine\\system\\bar\\foo"
filePath = "C:\\Users\\someuser\\sample.dll"
create_time = "2018-08-15T15:11:55.676+00:00"
modify_time = "2018-08-15T18:10:30.456+00:00"
file_hash = "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f"
file_name = "sample.dll"
file_size = 25536
data = {
"tag": tag,
"event_count": count,
"_time": time,
"user": user,
"bytes": file_bytes,
"object_path": objPath,
"file_path": filePath,
"file_create_time": create_time,
"file_modify_time": modify_time,
"file_hash": file_hash,
"file_size": file_size,
"file_name": file_name
}
result_bundle = cim_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
validated_result = validate_instance(observed_data)
assert (validated_result.is_valid == True)
assert ('objects' in observed_data)
objects = observed_data['objects']
# Test objects in Stix observable data model after transform
wrk_obj = TestTransform.get_first_of_type(objects.values(),
'windows-registry-key')
assert (wrk_obj
is not None), 'windows-registry-key object type not found'
assert (wrk_obj.keys() == {'type', 'creator_user_ref', 'key'})
assert (wrk_obj['key'] == "hkey_local_machine\\system\\bar\\foo")
user_ref = wrk_obj['creator_user_ref']
assert (
user_ref in objects
), f"creator_user_ref with key {wrk_obj['creator_user_ref']} not found"
user_obj = objects[user_ref]
assert (user_obj is not None), 'user-account object type not found'
assert (user_obj.keys() == {'type', 'account_login', 'user_id'})
assert (user_obj['account_login'] == "ibm_user")
assert (user_obj['user_id'] == "ibm_user")
file_obj = TestTransform.get_first_of_type(objects.values(), 'file')
assert (file_obj is not None), 'file object type not found'
assert (file_obj.keys() == {
'type', 'parent_directory_ref', 'created', 'modified', 'hashes',
'name', 'size'
})
assert (file_obj['created'] == "2018-08-15T15:11:55.676Z")
assert (file_obj['modified'] == "2018-08-15T18:10:30.456Z")
assert (file_obj['name'] == "sample.dll")
assert (file_obj['size'] == 25536)
assert (
file_obj['hashes']['SHA-256'] ==
"aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f")
dir_ref = file_obj['parent_directory_ref']
assert (
dir_ref in objects
), f"parent_directory_ref with key {file_obj['parent_directory_ref']} not found"
dir_obj = objects[dir_ref]
assert (dir_obj is not None), 'directory object type not found'
assert (dir_obj.keys() == {'type', 'path', 'created', 'modified'})
assert (dir_obj['path'] == "C:\\Users\\someuser\\sample.dll")
assert (dir_obj['created'] == "2018-08-15T15:11:55.676Z")
assert (dir_obj['modified'] == "2018-08-15T18:10:30.456Z")
assert (objects.keys() == set(map(str, range(0, 4))))
def transform(self, obj):
"""
Transforms the given object in to a STIX observation based on the mapping file and transform functions
:param obj: the datasource object that is being converted to stix
:return: the input object converted to stix valid json
"""
object_map = {}
stix_type = 'observed-data'
ds_map = self.ds_to_stix_map
now = "{}Z".format(
datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3])
object_id_map = {}
observation = {
'id': stix_type + '--' + str(uuid.uuid4()),
'type': stix_type,
'created_by_ref': self.identity_id,
'created': now,
'modified': now,
'objects': {}
}
# create normal type objects
if isinstance(obj, dict):
for ds_key in obj.keys():
self._transform(object_map, observation, ds_map, ds_key, obj)
else:
self.logger.debug("Not a dict: {}".format(obj))
# special case:
# remove object if:
# a reference attribute object does not contain at least one property other than 'type'
self._cleanup_references(object_map, observation, ds_map)
# Add required properties to the observation if it wasn't added from the mapping
if FIRST_OBSERVED_KEY not in observation:
observation[FIRST_OBSERVED_KEY] = now
if LAST_OBSERVED_KEY not in observation:
observation[LAST_OBSERVED_KEY] = now
if NUMBER_OBSERVED_KEY not in observation:
observation[NUMBER_OBSERVED_KEY] = 1
if self.spec_version == "2.1":
cybox_objects = observation["objects"]
self._generate_and_apply_deterministic_id(object_id_map,
cybox_objects)
self._replace_references(object_id_map, cybox_objects)
object_refs = []
# add cybox references to observed-data object
for key, value in object_id_map.items():
object_refs.append(value)
observation["object_refs"] = object_refs
observation["spec_version"] = "2.1"
self._collect_unique_cybox_objects(cybox_objects)
# Validate each STIX object
if self.stix_validator:
validated_result = validate_instance(observation)
print_results(validated_result)
return observation