def to_cybox_observable(self): """ Convert a Domain to a CybOX Observables. Returns a tuple of (CybOX object, releasability list). To get the cybox object as xml or json, call to_xml() or to_json(), respectively, on the resulting CybOX object. """ obj = DomainName() obj.value = self.domain obj.type_ = self.record_type return ([Observable(obj)], self.releasability)
def to_cybox_observable(self): """ Convert a Domain to a CybOX Observables. Returns a tuple of (CybOX object, releasability list). To get the cybox object as xml or json, call to_xml() or to_json(), respectively, on the resulting CybOX object. """ obj = DomainName() obj.value = self.domain obj.type_ = self.record_type return ([Observable(obj)], self.releasability)
def json2observable(config, src, dest, endpoint, json_, crits_id): # TODO split into smaller functions '''transform crits observables into cybox''' try: set_id_method(IDGenerator.METHOD_UUID) xmlns_url = config['edge']['sites'][dest]['stix']['xmlns_url'] xmlns_name = config['edge']['sites'][dest]['stix']['xmlns_name'] set_cybox_id_namespace(Namespace(xmlns_url, xmlns_name)) if endpoint == 'ips': crits_types = {'Address - cidr': 'cidr', 'Address - ipv4-addr': 'ipv4-addr', 'Address - ipv4-net': 'ipv4-net', 'Address - ipv4-net-mask': 'ipv4-netmask', 'Address - ipv6-addr': 'ipv6-addr', 'Address - ipv6-net': 'ipv6-net', 'Address - ipv6-net-mask': 'ipv6-netmask'} addr = Address(address_value=json_['ip'], category=crits_types[json_['type']]) addr.condition = 'Equals' observable_ = Observable(addr) elif endpoint == 'domains': domain = DomainName() domain.type_ = 'FQDN' domain.value = json_['domain'] domain.condition = 'Equals' observable_ = Observable(domain) elif endpoint == 'samples': crits_types = {'md5': 'MD5', 'sha1': 'SHA1', 'sha224': 'SHA224', 'sha256': 'SHA256', 'sha384': 'SHA384', 'sha512': 'SHA512', 'ssdeep': 'SSDEEP'} file_object = File() file_object.file_name = json_['filename'] for hash in crits_types.keys(): if hash in json_: file_object.add_hash(Hash(json_[hash], type_=crits_types[hash])) for i in file_object.hashes: i.simple_hash_value.condition = "Equals" observable_ = Observable(file_object) elif endpoint == 'emails': crits_types = {'subject': 'subject', 'to': 'to', 'cc': 'cc', 'from_address': 'from_', 'sender': 'sender', 'date': 'date', 'message_id': 'message_id', 'reply_to': 'reply_to', 'boundary': 'boundary', 'x_mailer': 'x_mailer', 'x_originating_ip': 'x_originating_ip'} email = EmailMessage() email.header = EmailHeader() for k in crits_types.keys(): val = json_.get(k, None) if val: email.header.__setattr__(crits_types[k], val) email.header.__getattribute__(crits_types[k]).condition = \ 'Equals' observable_ = Observable(email) else: config['logger'].error( log.log_messages['unsupported_object_error'].format( type_='crits', obj_type=endpoint, id_=crits_id)) return(None) observable_.id = xmlns_name + ':observable-' + crits_id observable_.id_ = observable_.id return(observable_) except: e = sys.exc_info()[0] config['logger'].error( log.log_messages['obj_convert_error'].format( src_type='crits', src_obj='observable', id_=crits_id, dest_type='cybox', dest_obj='observable')) config['logger'].exception(e) return(None)
def json2observable(config, src, dest, endpoint, json_, crits_id): # TODO split into smaller functions '''transform crits observables into cybox''' try: set_id_method(IDGenerator.METHOD_UUID) xmlns_url = config['edge']['sites'][dest]['stix']['xmlns_url'] xmlns_name = config['edge']['sites'][dest]['stix']['xmlns_name'] set_cybox_id_namespace(Namespace(xmlns_url, xmlns_name)) if endpoint == 'ips': crits_types = { 'Address - cidr': 'cidr', 'Address - ipv4-addr': 'ipv4-addr', 'Address - ipv4-net': 'ipv4-net', 'Address - ipv4-net-mask': 'ipv4-netmask', 'Address - ipv6-addr': 'ipv6-addr', 'Address - ipv6-net': 'ipv6-net', 'Address - ipv6-net-mask': 'ipv6-netmask' } addr = Address(address_value=json_['ip'], category=crits_types[json_['type']]) addr.condition = 'Equals' observable_ = Observable(addr) elif endpoint == 'domains': domain = DomainName() domain.type_ = 'FQDN' domain.value = json_['domain'] domain.condition = 'Equals' observable_ = Observable(domain) elif endpoint == 'samples': crits_types = { 'md5': 'MD5', 'sha1': 'SHA1', 'sha224': 'SHA224', 'sha256': 'SHA256', 'sha384': 'SHA384', 'sha512': 'SHA512', 'ssdeep': 'SSDEEP' } file_object = File() file_object.file_name = json_['filename'] for hash in crits_types.keys(): if hash in json_: file_object.add_hash( Hash(json_[hash], type_=crits_types[hash])) for i in file_object.hashes: i.simple_hash_value.condition = "Equals" observable_ = Observable(file_object) elif endpoint == 'emails': crits_types = { 'subject': 'subject', 'to': 'to', 'cc': 'cc', 'from_address': 'from_', 'sender': 'sender', 'date': 'date', 'message_id': 'message_id', 'reply_to': 'reply_to', 'boundary': 'boundary', 'x_mailer': 'x_mailer', 'x_originating_ip': 'x_originating_ip' } email = EmailMessage() email.header = EmailHeader() for k in crits_types.keys(): val = json_.get(k, None) if val: email.header.__setattr__(crits_types[k], val) email.header.__getattribute__(crits_types[k]).condition = \ 'Equals' observable_ = Observable(email) else: config['logger'].error( log.log_messages['unsupported_object_error'].format( type_='crits', obj_type=endpoint, id_=crits_id)) return (None) observable_.id = xmlns_name + ':observable-' + crits_id observable_.id_ = observable_.id return (observable_) except: e = sys.exc_info()[0] config['logger'].error(log.log_messages['obj_convert_error'].format( src_type='crits', src_obj='observable', id_=crits_id, dest_type='cybox', dest_obj='observable')) config['logger'].exception(e) return (None)
def gen_stix_observable_sample(config, target=None, datatype=None, title='random test data', description='random test data', package_intents='Indicators - Watchlist', tlp_color='WHITE'): '''generate sample stix data comprised of indicator_count indicators of type datatype''' # setup the xmlns... xmlns_url = config['edge']['sites'][target]['stix']['xmlns_url'] xmlns_name = config['edge']['sites'][target]['stix']['xmlns_name'] set_stix_id_namespace({xmlns_url: xmlns_name}) set_cybox_id_namespace(Namespace(xmlns_url, xmlns_name)) # construct a stix package... stix_package = STIXPackage() stix_header = STIXHeader() stix_header.title = title stix_header.description = description stix_header.package_intents = package_intents marking = MarkingSpecification() marking.controlled_structure = '../../../../descendant-or-self::node()' tlp_marking = TLPMarkingStructure() tlp_marking.color = tlp_color marking.marking_structures.append(tlp_marking) stix_package.stix_header = stix_header stix_package.stix_header.handling = Marking() stix_package.stix_header.handling.add_marking(marking) # ...and stuff it full of random sample data :-) if datatype == 'ip': addr = Address(address_value=datagen.generate_random_ip_address(), category='ipv4-addr') addr.condition = 'Equals' stix_package.add_observable(Observable(addr)) elif datatype == 'domain': domain = DomainName() domain.type_ = 'FQDN' domain.value = datagen.generate_random_domain(config) domain.condition = 'Equals' stix_package.add_observable(Observable(domain)) elif datatype == 'filehash': file_object = File() file_object.file_name = str(uuid.uuid4()) + '.exe' hashes = datagen.generate_random_hashes() for hash in hashes.keys(): file_object.add_hash(Hash(hashes[hash], type_=hash.upper())) for i in file_object.hashes: i.simple_hash_value.condition = "Equals" stix_package.add_observable(Observable(file_object)) elif datatype == 'email': try: msg = datagen.get_random_spam_msg(config) email = EmailMessage() email.header = EmailHeader() header_map = {'Subject': 'subject', 'To': 'to', 'Cc': 'cc', 'Bcc': 'bcc', 'From': 'from_', 'Sender': 'sender', 'Date': 'date', 'Message-ID': 'message_id', 'Reply-To': 'reply_to', 'In-Reply-To': 'in_reply_to', 'Content-Type': 'content_type', 'Errors-To': 'errors_to', 'Precedence': 'precedence', 'Boundary': 'boundary', 'MIME-Version': 'mime_version', 'X-Mailer': 'x_mailer', 'User-Agent': 'user_agent', 'X-Originating-IP': 'x_originating_ip', 'X-Priority': 'x_priority'} # TODO handle received_lines for key in header_map.keys(): val = msg.get(key, None) if val: email.header.__setattr__(header_map[key], val) email.header.__getattribute__(header_map[key]).condition = \ 'Equals' # TODO handle email bodies (it's mostly all there except for # handling weird text encoding problems that were making # libcybox stacktrace) # body = get_email_payload(random_spam_msg) # if body: # email.raw_body = body stix_package.add_observable(Observable(email)) except: return(None) observable_id = stix_package.observables.observables[0].id_ return(observable_id, stix_package)
def build(self): """Define the STIX report.""" self.stix_header.title = self.pulse["name"] self.stix_header.description = self.pulse["description"] self.stix_header.package_intents = "Indicators" self.stix_header.information_source = InformationSource() self.stix_header.information_source.time = Time() self.stix_header.information_source.time.received_time = self.pulse[ "modified"] self.stix_header.information_source.time.produced_time = self.stix_package.timestamp self.stix_header.information_source.identity = Identity() self.stix_header.information_source.identity.name = IDENTITY_NAME # self.stix_package.stix_header = self.stix_header # self.stix_package.stix_header.handling = self._marking() # self.report = Report() # self.report.header = Header() # self.report.header.title = self.pulse["name"] # self.report.header.descriptions = self.pulse["description"] # self.report.header.intents = "Indicators" # self.report.header.short_description = "%spulse/%s" % ( # PULSE_SERVER_BASE, str(self.pulse["id"])) # self.report.header.information_source = InformationSource() # self.report.header.information_source.time = Time() # self.report.header.information_source.time.received_time = self.pulse[ # "modified"] # self.report.header.information_source.time.produced_time = self.report.timestamp # self.report.header.information_source.identity = Identity() # self.report.header.information_source.identity.name = IDENTITY_NAME hashes = False addresses = False emails = False domains = False urls = False mutex = False hash_indicator = Indicator() hash_indicator.set_producer_identity(IDENTITY_NAME) hash_indicator.set_produced_time(hash_indicator.timestamp) hash_indicator.set_received_time(self.pulse["modified"]) hash_indicator.title = "[OTX] [Files] " + self.pulse["name"] hash_indicator.add_indicator_type("File Hash Watchlist") hash_indicator.confidence = "Low" address_indicator = Indicator() address_indicator.set_producer_identity(IDENTITY_NAME) address_indicator.set_produced_time(address_indicator.timestamp) address_indicator.set_received_time(self.pulse["modified"]) address_indicator.title = "[OTX] [IP] " + self.pulse["name"] address_indicator.add_indicator_type("IP Watchlist") address_indicator.confidence = "Low" domain_indicator = Indicator() domain_indicator.set_producer_identity(IDENTITY_NAME) domain_indicator.set_produced_time(domain_indicator.timestamp) domain_indicator.set_received_time(self.pulse["modified"]) domain_indicator.title = "[OTX] [Domain] " + self.pulse["name"] domain_indicator.add_indicator_type("Domain Watchlist") domain_indicator.confidence = "Low" url_indicator = Indicator() url_indicator.set_producer_identity(IDENTITY_NAME) url_indicator.set_produced_time(url_indicator.timestamp) url_indicator.set_received_time(self.pulse["modified"]) url_indicator.title = "[OTX] [URL] " + self.pulse["name"] url_indicator.add_indicator_type("URL Watchlist") url_indicator.confidence = "Low" email_indicator = Indicator() email_indicator.set_producer_identity(IDENTITY_NAME) email_indicator.set_produced_time(email_indicator.timestamp) email_indicator.set_received_time(self.pulse["modified"]) email_indicator.title = "[OTX] [Email] " + self.pulse["name"] email_indicator.add_indicator_type("Malicious E-mail") email_indicator.confidence = "Low" mutex_indicator = Indicator() mutex_indicator.set_producer_identity(IDENTITY_NAME) mutex_indicator.set_produced_time(mutex_indicator.timestamp) mutex_indicator.set_received_time(self.pulse["modified"]) mutex_indicator.title = "[OTX] [Mutex] " + self.pulse["name"] mutex_indicator.add_indicator_type("Malware Artifacts") mutex_indicator.confidence = "Low" for p_indicator in self.pulse["indicators"]: if p_indicator["type"] in self.hash_translation: file_object = File() file_object.add_hash(Hash(p_indicator["indicator"])) file_object.hashes[0].simple_hash_value.condition = "Equals" file_object.hashes[0].type_.condition = "Equals" file_obs = Observable(file_object) file_obs.title = "File: " + \ str(file_object.hashes[0].type_) + \ " - " + p_indicator["indicator"] hash_indicator.add_observable(file_obs) try: hash_indicator.description = p_indicator["description"] except KeyError: hash_indicator.description = "" hashes = True elif p_indicator["type"] in self.address_translation: ip = Address() ip.address_value = p_indicator["indicator"] ip.category = self.address_translation[p_indicator["type"]] ip.address_value.condition = "Equals" ip_obs = Observable(ip) ip_obs.title = "Address: " + str(ip.address_value) address_indicator.add_observable(ip_obs) try: address_indicator.description = p_indicator["description"] except KeyError: address_indicator.description = "" addresses = True elif p_indicator["type"] in self.name_translation: domain = DomainName() domain.value = p_indicator["indicator"] domain.type_ = "FQDN" domain.value.condition = "Equals" domain_obs = Observable(domain) domain_obs.title = "Domain: " + str(domain.value) domain_indicator.add_observable(domain_obs) try: domain_indicator.description = p_indicator["description"] except KeyError: domain_indicator.description = "" domains = True elif p_indicator["type"] == "URL": url = URI() url.value = p_indicator["indicator"] url.type_ = URI.TYPE_URL url.value.condition = "Equals" url_obs = Observable(url) url_obs.title = "URI: " + str(url.value) url_indicator.add_observable(url_obs) try: url_indicator.description = p_indicator["description"] except KeyError: url_indicator.description = "" urls = True elif p_indicator["type"] == "email": email = Address() email.address_value = p_indicator["indicator"] email.category = "e-mail" email.address_value.condition = "Equals" email_obs = Observable(email) email_obs.title = "Address: " + str(email.address_value) email_indicator.add_observable(email_obs) try: email_indicator.description = p_indicator["indicator"] except KeyError: email_indicator.description = "" emails = True elif p_indicator["type"] == "Mutex": mutex = Mutex() mutex.name = p_indicator["indicator"] mutex.named = True mutex_obs = Observable(mutex) mutex_obs.title = "Mutex: " + str(mutex.name) mutex_indicator.add_observable(mutex_obs) try: mutex_indicator.description = p_indicator["indicator"] except KeyError: mutex_indicator.description = "" mutex = True else: continue if hashes: self.stix_package.add_indicator(hash_indicator) if addresses: self.stix_package.add_indicator(address_indicator) if domains: self.stix_package.add_indicator(domain_indicator) if urls: self.stix_package.add_indicator(url_indicator) if emails: self.stix_package.add_indicator(email_indicator) if mutex: self.stix_package.add_indicator(mutex_indicator)
def gen_stix_observable_sample(config, target=None, datatype=None, title='random test data', description='random test data', package_intents='Indicators - Watchlist', tlp_color='WHITE'): '''generate sample stix data comprised of indicator_count indicators of type datatype''' # setup the xmlns... xmlns_url = config['edge']['sites'][target]['stix']['xmlns_url'] xmlns_name = config['edge']['sites'][target]['stix']['xmlns_name'] set_stix_id_namespace({xmlns_url: xmlns_name}) set_cybox_id_namespace(Namespace(xmlns_url, xmlns_name)) # construct a stix package... stix_package = STIXPackage() stix_header = STIXHeader() stix_header.title = title stix_header.description = description stix_header.package_intents = package_intents marking = MarkingSpecification() marking.controlled_structure = '../../../../descendant-or-self::node()' tlp_marking = TLPMarkingStructure() tlp_marking.color = tlp_color marking.marking_structures.append(tlp_marking) stix_package.stix_header = stix_header stix_package.stix_header.handling = Marking() stix_package.stix_header.handling.add_marking(marking) # ...and stuff it full of random sample data :-) if datatype == 'ip': addr = Address(address_value=datagen.generate_random_ip_address(), category='ipv4-addr') addr.condition = 'Equals' stix_package.add_observable(Observable(addr)) elif datatype == 'domain': domain = DomainName() domain.type_ = 'FQDN' domain.value = datagen.generate_random_domain(config) domain.condition = 'Equals' stix_package.add_observable(Observable(domain)) elif datatype == 'filehash': file_object = File() file_object.file_name = str(uuid.uuid4()) + '.exe' hashes = datagen.generate_random_hashes() for hash in hashes.keys(): file_object.add_hash(Hash(hashes[hash], type_=hash.upper())) for i in file_object.hashes: i.simple_hash_value.condition = "Equals" stix_package.add_observable(Observable(file_object)) elif datatype == 'email': try: msg = datagen.get_random_spam_msg(config) email = EmailMessage() email.header = EmailHeader() header_map = { 'Subject': 'subject', 'To': 'to', 'Cc': 'cc', 'Bcc': 'bcc', 'From': 'from_', 'Sender': 'sender', 'Date': 'date', 'Message-ID': 'message_id', 'Reply-To': 'reply_to', 'In-Reply-To': 'in_reply_to', 'Content-Type': 'content_type', 'Errors-To': 'errors_to', 'Precedence': 'precedence', 'Boundary': 'boundary', 'MIME-Version': 'mime_version', 'X-Mailer': 'x_mailer', 'User-Agent': 'user_agent', 'X-Originating-IP': 'x_originating_ip', 'X-Priority': 'x_priority' } # TODO handle received_lines for key in header_map.keys(): val = msg.get(key, None) if val: email.header.__setattr__(header_map[key], val) email.header.__getattribute__(header_map[key]).condition = \ 'Equals' # TODO handle email bodies (it's mostly all there except for # handling weird text encoding problems that were making # libcybox stacktrace) # body = get_email_payload(random_spam_msg) # if body: # email.raw_body = body stix_package.add_observable(Observable(email)) except: return (None) observable_id = stix_package.observables.observables[0].id_ return (observable_id, stix_package)
def gen_stix_observable_sample( config, target=None, datatype=None, title="random test data", description="random test data", package_intents="Indicators - Watchlist", tlp_color="WHITE", ): """generate sample stix data comprised of indicator_count indicators of type datatype""" # setup the xmlns... xmlns_url = config["edge"]["sites"][target]["stix"]["xmlns_url"] xmlns_name = config["edge"]["sites"][target]["stix"]["xmlns_name"] set_stix_id_namespace({xmlns_url: xmlns_name}) set_cybox_id_namespace(Namespace(xmlns_url, xmlns_name)) # construct a stix package... stix_package = STIXPackage() stix_header = STIXHeader() stix_header.title = title stix_header.description = description stix_header.package_intents = package_intents marking = MarkingSpecification() marking.controlled_structure = "../../../../descendant-or-self::node()" tlp_marking = TLPMarkingStructure() tlp_marking.color = tlp_color marking.marking_structures.append(tlp_marking) stix_package.stix_header = stix_header stix_package.stix_header.handling = Marking() stix_package.stix_header.handling.add_marking(marking) # ...and stuff it full of random sample data :-) if datatype == "ip": addr = Address(address_value=datagen_.generate_random_ip_address(), category="ipv4-addr") addr.condition = "Equals" stix_package.add_observable(Observable(addr)) elif datatype == "domain": domain = DomainName() domain.type_ = "FQDN" domain.value = datagen_.generate_random_domain(config) domain.condition = "Equals" stix_package.add_observable(Observable(domain)) elif datatype == "filehash": file_object = File() file_object.file_name = str(uuid.uuid4()) + ".exe" hashes = datagen_.generate_random_hashes() for hash in hashes.keys(): file_object.add_hash(Hash(hashes[hash], type_=hash.upper())) for i in file_object.hashes: i.simple_hash_value.condition = "Equals" stix_package.add_observable(Observable(file_object)) elif datatype == "email": try: msg = datagen_.get_random_spam_msg(config) email = EmailMessage() email.header = EmailHeader() header_map = { "Subject": "subject", "To": "to", "Cc": "cc", "Bcc": "bcc", "From": "from_", "Sender": "sender", "Date": "date", "Message-ID": "message_id", "Reply-To": "reply_to", "In-Reply-To": "in_reply_to", "Content-Type": "content_type", "Errors-To": "errors_to", "Precedence": "precedence", "Boundary": "boundary", "MIME-Version": "mime_version", "X-Mailer": "x_mailer", "User-Agent": "user_agent", "X-Originating-IP": "x_originating_ip", "X-Priority": "x_priority", } # TODO handle received_lines for key in header_map.keys(): val = msg.get(key, None) if val: email.header.__setattr__(header_map[key], val) email.header.__getattribute__(header_map[key]).condition = "Equals" # TODO handle email bodies (it's mostly all there except for # handling weird text encoding problems that were making # libcybox stacktrace) # body = get_email_payload(random_spam_msg) # if body: # email.raw_body = body stix_package.add_observable(Observable(email)) except: return None observable_id = stix_package.observables.observables[0].id_ return (observable_id, stix_package)