def dupehash(cls, apiobj): properties = rgetattr(apiobj, ['object_', 'properties'], None) obs_type = rgetattr(properties, ['_XSI_TYPE'], None) if obs_type is not None: if obs_type in OBS_HASH_PATHS: # we can produce a meaningful hash for selected properties to_hash = "%s|%s|%s" % (cls.SHORT_NAME, obs_type, '|'.join( [ str(rgetattr(properties, str(path).split('.'), '')) for path in OBS_HASH_PATHS[obs_type] ])) hash_ = "certuk:%s" % hashlib.sha1(to_hash).hexdigest() else: # we can't produce a meaningful hash on selected properties, so hash them all to_hash = scrubcopy( nested_get(apiobj.to_dict(), ['object', 'properties'], {}), ['xsi:type']) hash_ = "certuk:%s" % dicthash_sha1(to_hash, salt=cls.SHORT_NAME) else: # we don't know what this is, so let the existing code deal with it hash_ = super(DBObservablePatch, cls).dupehash(apiobj) return hash_
def summary_value_generator(self, obj): http_request_response_list = rgetattr( obj, ['_object', 'properties', 'http_request_response']) value = rgetattr(http_request_response_list[0], [ 'http_client_request', 'http_request_header', 'parsed_header', 'user_agent' ], '(undefined)') return str(value)
def summary_value_generator(self, obj): network_connection_object = "Source Socket Address: " network_connection_object += self.get_socket_summary( rgetattr(obj, ['_object', 'properties', 'source_socket_address'])) network_connection_object += ": Destination Socket Address: " network_connection_object += self.get_socket_summary( rgetattr(obj, ['_object', 'properties', 'destination_socket_address'])) return network_connection_object
def to_draft_handler(self, observable, tg, load_by_id, id_ns=''): return { 'objectType': 'HTTP Session', 'id': rgetattr(observable, ['id_'], ''), 'id_ns': id_ns, 'title': rgetattr(observable, ['title'], ''), 'description': str(rgetattr(observable, ['description'], '')), 'user_agent': self.summary_value_generator(observable) }
def _package_objects_to_consider(contents, local, type_, property_): ids_to_objects_to_consider = {} for id_, io in contents.iteritems(): is_local = rgetattr(contents.get(id_, None), ['api_object', 'obj', 'id_ns'], '') == LOCAL_NAMESPACE correct_ns = is_local if local else (not is_local) if not correct_ns: continue if rgetattr(contents.get(id_, None), ['api_object', 'ty'], '') == type_ and len( rgetattr(contents.get(id_, None), property_, '')) > 0: ids_to_objects_to_consider.setdefault(id_, []).append(io) return ids_to_objects_to_consider
def get_socket(self, socket_object): address = rgetattr(socket_object, ['ip_address', 'address_value'], '') hostname = rgetattr(socket_object, ['hostname', 'hostname_value'], '') port = rgetattr(socket_object, ['port', 'port_value'], '') protocol = rgetattr(socket_object, ['port', 'layer4_protocol'], '') socket = { "ip_address": str(address), "hostname": str(hostname), "port": str(port), "protocol": str(protocol) } return socket
def _merge_properties(api_object, id_, count, additional_file_hashes): api_object.obj.sighting_count = _get_sighting_count(api_object.obj) + count if id_ in additional_file_hashes: file_properties = rgetattr(api_object, ['obj', 'object_', 'properties'], None) for hash_type, hash_value in additional_file_hashes[id_].iteritems(): if getattr(file_properties, hash_type, None) is None: setattr(file_properties, hash_type, hash_value)
def to_draft(cls, inc, tg, load_by_id, id_ns=''): draft = super(DBIncidentPatch, cls).to_draft(inc, tg, load_by_id, id_ns) if 'responder' in draft: # fix unbalanced save / load keys in incident.py del draft['responder'] draft['responders'] = [EdgeInformationSource.clone(responder).to_draft() for responder in inc.responders] draft['coordinators'] = [EdgeInformationSource.clone(coordinator).to_draft() for coordinator in inc.coordinators] draft['categories'] = [c.value for c in rgetattr(inc, ['categories'], [])] if inc.time: draft['time'] = inc.time.to_dict() for key, value in draft.get('time').iteritems(): if isinstance(value, basestring): new_value = DBIncidentPatch.convert_to_and_strip_config_timezone(value) draft.get('time')[key] = new_value else: value['value'] = DBIncidentPatch.convert_to_and_strip_config_timezone(value['value']) if inc.external_ids: draft['external_ids'] = [] for ex_id in inc.external_ids: draft['external_ids'].append({'source': ex_id.source, 'id': ex_id.value}) # Redoing to use correct patched function, can't guarantee correct function if monkey patched draft["markings"] = DBIncidentPatch.handling_to_draft(inc, "statement") draft["tlp"] = DBIncidentPatch.handling_to_draft(inc, "color") return draft
class DBObservablePatch(DBObservable): def __init__(self, obj=None, item=None, id_=None): super(DBObservablePatch, self).__init__(obj, item, id_) CUSTOM_DRAFT_HANDLERS = custom_draft_handler_map @staticmethod def _get_custom_to_draft_handler(object_type): try: return DBObservablePatch.CUSTOM_DRAFT_HANDLERS[object_type] except KeyError: raise ValueError("Unexpected Object Type %s" % object_type) @classmethod def to_draft(cls, observable, tg, load_by_id, id_ns=''): try: draft = super(DBObservablePatch, cls).to_draft(observable, tg, load_by_id, id_ns=id_ns) # When creating a draft from a fully formed object with None for some fields, the draft contains # fields with string 'None' which can cause parsing issues e.g. vlan_num for attr, value in draft.iteritems(): if value == 'None': draft[attr] = '' # to_draft sets the hash_type to lower case. This is not what is expected during Inboxing or Validation! hash_type_map = { 'md5': 'MD5', 'md6': 'MD6', 'sha1': 'SHA1', 'sha224': 'SHA224', 'sha256': 'SHA256', 'sha384': 'SHA384', 'sha512': 'SHA512', 'ssdeep': 'SSDEEP', 'other': 'Other' } if 'hashes' in draft: for hash_ in draft['hashes']: hash_['hash_type'] = hash_type_map.get( hash_['hash_type'].lower(), 'Other') return draft except ValueError, v: if v.__class__ != ValueError: raise object_type = rgetattr(observable, ['_object', '_properties', '_XSI_TYPE'], 'None') draft_handler = DBObservablePatch._get_custom_to_draft_handler( object_type) return draft_handler(observable, tg, load_by_id, id_ns)
def to_draft(cls, indicator, tg, load_by_id, id_ns=''): draft = super(DBIndicatorPatch, cls).to_draft(indicator, tg, load_by_id, id_ns) kill_chain_phases = rgetattr(indicator, ['kill_chain_phases']) if kill_chain_phases: draft['kill_chain_phase'] = rgetattr(kill_chain_phases[0], ['phase_id'], '') test_mechanisms = rgetattr(indicator, ['test_mechanisms']) if test_mechanisms: for test_mechanism in test_mechanisms: if isinstance(test_mechanism, SnortTestMechanism): snort_dict = SnortTestMechanism.to_dict(test_mechanism) snort_dict['type'] = 'Snort' draft.setdefault('test_mechanisms', []).append(snort_dict) if isinstance(test_mechanism, YaraTestMechanism): yara_dict = YaraTestMechanism.to_dict(test_mechanism) yara_dict['type'] = 'Yara' draft.setdefault('test_mechanisms', []).append(yara_dict) return draft
def dedup(collection, idref_path='item.idref'): if not collection or len(collection) == 0: return seen = set() idx = 0 for edge in collection: idref = rgetattr(edge, idref_path.split('.')) if idref not in seen: seen.add(idref) collection[idx] = edge idx += 1 del collection[idx:]
def to_draft_handler(self, observable, tg, load_by_id, id_ns=''): return { 'objectType': 'Network Connection', 'id': rgetattr(observable, ['id_'], ''), 'id_ns': id_ns, 'title': rgetattr(observable, ['title'], ''), 'description': str(rgetattr(observable, ['description'], '')), 'source_socket_address': self.get_socket( rgetattr(observable, ['_object', 'properties', 'source_socket_address'])), 'destination_socket_address': self.get_socket( rgetattr( observable, ['_object', 'properties', 'destination_socket_address'])) }
def _generate_map_table_on_hash(contents, hashes, type_): hash_to_ids = {} for id_, hash_ in sorted(hashes.iteritems()): if rgetattr(contents.get(id_, None), ['api_object', 'ty'], '') == type_: hash_to_ids.setdefault(hash_, []).append(id_) map_table = {} for hash_, ids in hash_to_ids.iteritems(): if len(ids) > 1: master = _get_id_with_lowest_tlp(contents, ids) ids.remove(master) for dup in ids: map_table[dup] = master return map_table
def handling_to_draft(construct, structure): draft = {} draft[structure] = [] handling = rgetattr(construct, ['handling'], '') if handling: for marking in handling.markings: for marking_structure in marking.marking_structures: # stops it adding handling_caveats to marking even though both are in simple marking structures if getattr(marking_structure, 'marking_model_name', None): # retrieve all simple marking statements for handling_caveats if structure == "handling_caveat": draft[structure].append(getattr(marking_structure, 'statement', None)) else: if getattr(marking_structure, structure, None): draft[structure].append(getattr(marking_structure, structure, None)) return draft[structure]
def get_additional_file_hashes(original, duplicates): additional_file_hashes = {} api_object = EdgeObject.load(original).to_ApiObject() if rgetattr(api_object, STIXDedup.PROPERTY_TYPE, None) == 'FileObjectType': for dup in duplicates: try: api_obj, tlp, esms, etou = STIXDedup.load_eo(dup) except EdgeError as e: continue io = InboxItem(api_object=api_obj, etlp=tlp, esms=esms, etou=etou) add_additional_file_hashes(io, additional_file_hashes, original) return additional_file_hashes
def add_additional_file_hashes(io, additional_file_hashes, existing_id): def add_missing_file_hash(inbox_object, file_hashes, property_name): hash_type = property_name[-1] if hash_type not in file_hashes: hash_value = rgetattr(inbox_object, property_name, None) if hash_value is not None: file_hashes[hash_type] = hash_value if rgetattr(io, PROPERTY_TYPE, None) == 'FileObjectType': if existing_id not in additional_file_hashes: additional_file_hashes[existing_id] = {} add_missing_file_hash(io, additional_file_hashes[existing_id], PROPERTY_MD5) add_missing_file_hash(io, additional_file_hashes[existing_id], PROPERTY_SHA1) add_missing_file_hash(io, additional_file_hashes[existing_id], PROPERTY_SHA224) add_missing_file_hash(io, additional_file_hashes[existing_id], PROPERTY_SHA256) add_missing_file_hash(io, additional_file_hashes[existing_id], PROPERTY_SHA384) add_missing_file_hash(io, additional_file_hashes[existing_id], PROPERTY_SHA512)
def _add_matching_file_observables(db, map_table, contents): # identify file observables in contents excluding any which are already in map_table new_file_obs = {id_: inbox_item for (id_, inbox_item) in contents.iteritems() if inbox_item.api_object.ty == 'obs' and id_ not in map_table and # exclude perfect matches which have already been discovered via data hash rgetattr(inbox_item, PROPERTY_TYPE, 'Unknown') == 'FileObjectType'} if not new_file_obs: # if we have no new file observables, we can bail out return existing_file_obs = _find_matching_db_file_obs(db, new_file_obs) for existing_file in existing_file_obs: existing_file_obj = EdgeObject(existing_file).to_ApiObject() for (new_id, new_file) in new_file_obs.iteritems(): if new_id in map_table: break if _is_matching_file(existing_file_obj, new_file.api_object): map_table[new_id] = existing_file['_id']
def matches(existing, new, property_path): # NOTE: need to ignore the `api_object` part of the property path here - hence `[1:]` existing_value = rgetattr(existing, property_path[1:], None) new_value = rgetattr(new, property_path[1:], None) return existing_value is not None and new_value is not None and existing_value == new_value
def extract_properties(inbox_items, property_path): return list({ str(rgetattr(inbox_item, property_path, None)) for inbox_item in inbox_items.itervalues() if rgetattr(inbox_item, property_path, None) is not None })
def custom_get_obs_value(obj): type_ = rgetattr(obj, ['object_', 'properties', '_XSI_TYPE'], None) custom_type_handler = custom_object_value_map.get(type_) if custom_type_handler is None: return original_get_obs_value(obj) return custom_type_handler(obj)
def add_missing_file_hash(inbox_object, file_hashes, property_name): hash_type = property_name[-1] if hash_type not in file_hashes: hash_value = rgetattr(inbox_object, property_name, None) if hash_value is not None: file_hashes[hash_type] = hash_value
def collapse_nested_values(value): return rgetattr(value, ['value'], value)