def process(self, task: Task) -> None: # type: ignore config = task.get_payload("config") family = task.headers["family"] dhash = config_dhash(config) # Parse the config using iocextract library iocs = parse(family, config) if not iocs: # Nothing actionable found - skip the config return # Upload structured data to MISP event = MISPEvent() event.uuid = str(uuid5(self.CONFIG_NAMESPACE, dhash)) event.add_tag(f"mwdb:family:{family}") event.info = f"Malware configuration ({family})" if self.mwdb_url is not None: event.add_attribute("link", f"{self.mwdb_url}/config/{dhash}") for o in iocs.to_misp(): event.add_object(o) misp = ExpandedPyMISP(self.misp_url, self.misp_key, self.misp_verifycert) misp.add_event(event)
def export_to_misp(user_details, report,indicators): misp_url = user_details.get('misp_url') misp_key = user_details.get('misp_api_key') misp_verifycert = True misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert) event = MISPEvent() event.info = report.title event = misp.add_event(event, pythonify=True) print(event) created = json.loads(event.to_json()) event_id = created.get('id') report_id = report.id for indicator in indicators: indicator_value = indicator[1] if indicator[2] == 'IP': indicator_type = "ip-dst" elif indicator[2] == 'Domain': indicator_type = 'domain' elif indicator[2] == 'Email': indicator_type = 'email-src' elif indicator[2] == 'CVE': indicator_value = indicator[1].replace('_', '-') indicator_type = 'vulnerability' elif indicator[2] == 'MD5 Hash': indicator_type = 'md5' elif indicator[2] == 'SHA256 Hash': indicator_type = 'sha256' elif indicator[2] == 'URL': indicator_type = 'url' try: misp.add_attribute(event_id,{'type':indicator_type,'value':indicator_value},pythonify=True) except: pass
def submit_tf_update(misp: ExpandedPyMISP, attributes: list) -> MISPEvent: """ create/update abuse.ch MISP-Event and append the new attributes """ eventinfo = event_info_template.format( datetime.now().strftime(info_dateformat)) # logging.debug(eventinfo) events = misp.search(controller='events', eventinfo=eventinfo, org=1, pythonify=True) if events: # current event exists already event = events[0] else: # create a new event event = MISPEvent() event.distribution = event_distribution event.threat_level_id = event_threat_level event.analysis = 2 event.info = eventinfo for tag in tagging: event.add_tag(tag) event = misp.add_event(event, pythonify=True) for att in attributes: event.add_attribute(**att) event.published = autopublish return misp.update_event(event)
def save(self): site_id = self.validated_data['id'] site = Site.objects.get(pk=site_id) # Check if there is already an Event if DnsTwisted.objects.filter(domain_name=site.domain_name): dns_twisted = DnsTwisted.objects.get(domain_name=site.domain_name) if site.misp_event_id is None: site.misp_event_id = dns_twisted.misp_event_id # Save the case id in database Site.objects.filter(pk=site.pk).update( misp_event_id=dns_twisted.misp_event_id) # Test MISP instance connection try: requests.get(settings.MISP_URL, verify=settings.MISP_VERIFY_SSL) except requests.exceptions.SSLError as e: print(str(timezone.now()) + " - ", e) raise AuthenticationFailed("SSL Error: " + settings.MISP_URL) except requests.exceptions.RequestException as e: print(str(timezone.now()) + " - ", e) raise NotFound("Not Found: " + settings.MISP_URL) misp_api = ExpandedPyMISP(settings.MISP_URL, settings.MISP_KEY, settings.MISP_VERIFY_SSL) if site.misp_event_id is not None: # If the event already exist, then we update IOCs update_attributes(misp_api, site) else: # If the event does not exist, then we create it # Prepare MISP Event event = MISPEvent() event.distribution = 0 event.threat_level_id = 2 event.analysis = 0 event.info = "Suspicious domain name " + site.domain_name event.tags = create_misp_tags(misp_api) # Create MISP Event print(str(timezone.now()) + " - " + 'Create MISP Event') print('-----------------------------') event = misp_api.add_event(event, pythonify=True) # Store Event Id in database Site.objects.filter(pk=site.pk).update(misp_event_id=event.id) if DnsTwisted.objects.filter(domain_name=site.domain_name): DnsTwisted.objects.filter(domain_name=site.domain_name).update( misp_event_id=event.id) # Create MISP Attributes create_attributes(misp_api, event.id, site)
def register_misp(misp: ExpandedPyMISP, misp_event_dict: dict) -> None: """ MISPイベントデータを受け取り、MISPに登録する """ misp_event = MISPEvent() misp_event.from_dict(**misp_event_dict) threat_level_id = misp_event.get('threat_level_id') threat_level = f'anyrun:threat_level:{threat_level_id}' misp_event.add_tag('anyrun') misp_event.add_tag(threat_level) retry_count = 0 while True: try: # pymispをインスタンス化してイベント登録 event_data = misp.add_event(misp_event) if event_data.get('errors'): raise Exception(event_data['errors']) # MISPに登録されるイベントIDを取得し、出力 event_id = event_data['Event']['id'] print(f'新規に登録されたEvent_ID: {event_id}') return except: except_return = traceback.format_exc() # インポート済みの場合 if const.DUPLICATE_EVENT_CONFIRM_WORD in except_return: print('Importを行おうとしたイベントは既にMISPに登録されています') return # リトライ回数チェック retry_count += 1 if retry_count >= const.RETRY_MAXIMUM_LIMIT: raise # インターバル処理 print('MISPへのイベントインポートをリトライします') time.sleep(const.COMMAND_INTERVAL_TIME)
class Output(cowrie.core.output.Output): """ MISP Upload Plugin for Cowrie. This Plugin creates a new event for unseen file uploads or adds sightings for previously seen files. The decision is done by searching for the SHA 256 sum in all matching attributes. """ @ignore_warnings def start(self): """ Start output plugin """ misp_url = CowrieConfig().get('output_misp', 'base_url') misp_key = CowrieConfig().get('output_misp', 'api_key') misp_verifycert = ("true" == CowrieConfig().get('output_misp', 'verify_cert').lower()) self.misp_api = PyMISP(url=misp_url, key=misp_key, ssl=misp_verifycert, debug=False) self.is_python2 = sys.version_info[0] < 3 self.debug = CowrieConfig().getboolean('output_misp', 'debug', fallback=False) self.publish = CowrieConfig().getboolean('output_misp', 'publish_event', fallback=False) def stop(self): """ Stop output plugin """ pass def write(self, entry): """ Push file download to MISP """ if entry['eventid'] == 'cowrie.session.file_download': file_sha_attrib = self.find_attribute("sha256", entry["shasum"]) if file_sha_attrib: # file is known, add sighting! if self.debug: log.msg("File known, add sighting") self.add_sighting(entry, file_sha_attrib) else: # file is unknown, new event with upload if self.debug: log.msg("File unknwon, add new event") self.create_new_event(entry) @ignore_warnings def find_attribute(self, attribute_type, searchterm): """ Returns a matching attribute or None if nothing was found. """ result = self.misp_api.search( controller="attributes", type_attribute=attribute_type, value=searchterm ) # legacy PyMISP returns the Attribute wrapped in a response if self.is_python2: result = result["response"] if result["Attribute"]: return result["Attribute"][0] else: return None @ignore_warnings def create_new_event(self, entry): if self.is_python2: self.misp_api.upload_sample( entry["shasum"], entry["outfile"], None, distribution=1, info="File uploaded to Cowrie ({})".format(entry["sensor"]), analysis=0, threat_level_id=2 ) else: attribute = MISPAttribute() attribute.type = "malware-sample" attribute.value = entry["shasum"] attribute.data = Path(entry["outfile"]) attribute.comment = "File uploaded to Cowrie ({})".format(entry["sensor"]) attribute.expand = "binary" event = MISPEvent() event.info = "File uploaded to Cowrie ({})".format(entry["sensor"]) event.attributes = [attribute] event.run_expansions() if self.publish: event.publish() result = self.misp_api.add_event(event) if self.debug: log.msg("Event creation result: \n%s" % result) @ignore_warnings def add_sighting(self, entry, attribute): if self.is_python2: self.misp_api.sighting( uuid=attribute["uuid"], source="{} (Cowrie)".format(entry["sensor"]) ) else: sighting = MISPSighting() sighting.source = "{} (Cowrie)".format(entry["sensor"]) self.misp_api.add_sighting(sighting, attribute)
class Mail2MISP(): def __init__(self, misp_url, misp_key, verifycert, config, offline=False, urlsonly=False): self.offline = offline if not self.offline: self.misp = ExpandedPyMISP(misp_url, misp_key, verifycert, debug=config.debug) self.config = config self.urlsonly = urlsonly if not hasattr(self.config, 'enable_dns'): setattr(self.config, 'enable_dns', True) if self.urlsonly is False: setattr(self.config, 'enable_dns', False) self.debug = self.config.debug self.config_from_email_body = {} # Init Faup self.f = Faup() self.sightings_to_add = [] def load_email(self, pseudofile): self.pseudofile = pseudofile self.original_mail = message_from_bytes(self.pseudofile.getvalue(), policy=policy.default) self.subject = self.original_mail.get('Subject') try: self.sender = self.original_mail.get('From') except: self.sender = "<unknown sender>" # Remove words from subject for removeword in self.config.removelist: self.subject = re.sub(removeword, "", self.subject).strip() # Initialize the MISP event self.misp_event = MISPEvent() self.misp_event.info = f'{self.config.email_subject_prefix} - {self.subject}' self.misp_event.distribution = self.config.default_distribution self.misp_event.threat_level_id = self.config.default_threat_level self.misp_event.analysis = self.config.default_analysis def sighting(self, value, source): if self.offline: raise Exception('The script is running in offline mode, ') '''Add a sighting''' s = MISPSighting() s.from_dict(value=value, source=source) self.misp.add_sighting(s) def _find_inline_forward(self): '''Does the body contains a forwarded email?''' for identifier in self.config.forward_identifiers: if identifier in self.clean_email_body: self.clean_email_body, fw_email = self.clean_email_body.split( identifier) return self.forwarded_email( pseudofile=BytesIO(fw_email.encode())) def _find_attached_forward(self): forwarded_emails = [] for attachment in self.original_mail.iter_attachments(): attachment_content = attachment.get_content() # Search for email forwarded as attachment # I could have more than one, attaching everything. if isinstance(attachment_content, message.EmailMessage): forwarded_emails.append( self.forwarded_email( pseudofile=BytesIO(attachment_content.as_bytes()))) else: if isinstance(attachment_content, str): attachment_content = attachment_content.encode() filename = attachment.get_filename() if not filename: filename = 'missing_filename' if self.config_from_email_body.get( 'attachment' ) == self.config.m2m_benign_attachment_keyword: # Attach sane file self.misp_event.add_attribute( 'attachment', value=filename, data=BytesIO(attachment_content)) else: f_object, main_object, sections = make_binary_objects( pseudofile=BytesIO(attachment_content), filename=filename, standalone=False) self.misp_event.add_object(f_object) if main_object: self.misp_event.add_object(main_object) [ self.misp_event.add_object(section) for section in sections ] return forwarded_emails def email_from_spamtrap(self): '''The email comes from a spamtrap and should be attached as-is.''' raw_body = self.original_mail.get_body(preferencelist=('html', 'plain')) if raw_body: self.clean_email_body = html.unescape( raw_body.get_payload(decode=True).decode( 'utf8', 'surrogateescape')) else: self.clean_email_body = '' return self.forwarded_email(self.pseudofile) def forwarded_email(self, pseudofile: BytesIO): '''Extracts all possible indicators out of an email and create a MISP event out of it. * Gets all relevant Headers * Attach the body * Create MISP file objects (uses lief if possible) * Set all references ''' email_object = EMailObject(pseudofile=pseudofile, attach_original_mail=True, standalone=False) if email_object.attachments: # Create file objects for the attachments for attachment_name, attachment in email_object.attachments: if not attachment_name: attachment_name = 'NameMissing.txt' if self.config_from_email_body.get( 'attachment' ) == self.config.m2m_benign_attachment_keyword: a = self.misp_event.add_attribute('attachment', value=attachment_name, data=attachment) email_object.add_reference(a.uuid, 'related-to', 'Email attachment') else: f_object, main_object, sections = make_binary_objects( pseudofile=attachment, filename=attachment_name, standalone=False) if self.config.vt_key: try: vt_object = VTReportObject( self.config.vt_key, f_object.get_attributes_by_relation( 'sha256')[0].value, standalone=False) self.misp_event.add_object(vt_object) f_object.add_reference(vt_object.uuid, 'analysed-with') except InvalidMISPObject as e: print(e) pass self.misp_event.add_object(f_object) if main_object: self.misp_event.add_object(main_object) for section in sections: self.misp_event.add_object(section) email_object.add_reference(f_object.uuid, 'related-to', 'Email attachment') self.process_body_iocs(email_object) if self.config.spamtrap or self.config.attach_original_mail or self.config_from_email_body.get( 'attach_original_mail'): self.misp_event.add_object(email_object) return email_object def process_email_body(self): mail_as_bytes = self.original_mail.get_body( preferencelist=('html', 'plain')).get_payload(decode=True) if mail_as_bytes: self.clean_email_body = html.unescape( mail_as_bytes.decode('utf8', 'surrogateescape')) # Check if there are config lines in the body & convert them to a python dictionary: # <config.body_config_prefix>:<key>:<value> => {<key>: <value>} self.config_from_email_body = { k.strip(): v.strip() for k, v in re.findall( f'{self.config.body_config_prefix}:(.*):(.*)', self.clean_email_body) } if self.config_from_email_body: # ... remove the config lines from the body self.clean_email_body = re.sub( rf'^{self.config.body_config_prefix}.*\n?', '', html.unescape( self.original_mail.get_body( preferencelist=('html', 'plain')).get_payload( decode=True).decode('utf8', 'surrogateescape')), flags=re.MULTILINE) # Check if autopublish key is present and valid if self.config_from_email_body.get( 'm2mkey') == self.config.m2m_key: if self.config_from_email_body.get('distribution') is not None: self.misp_event.distribution = self.config_from_email_body.get( 'distribution') if self.config_from_email_body.get('threat_level') is not None: self.misp_event.threat_level_id = self.config_from_email_body.get( 'threat_level') if self.config_from_email_body.get('analysis') is not None: self.misp_event.analysis = self.config_from_email_body.get( 'analysis') if self.config_from_email_body.get('publish'): self.misp_event.publish() self._find_inline_forward() else: self.clean_email_body = '' self._find_attached_forward() def process_body_iocs(self, email_object=None): if email_object: body = html.unescape( email_object.email.get_body( preferencelist=('html', 'plain')).get_payload(decode=True).decode( 'utf8', 'surrogateescape')) else: body = self.clean_email_body # Cleanup body content # Depending on the source of the mail, there is some cleanup to do. Ignore lines in body of message for ignoreline in self.config.ignorelist: body = re.sub(rf'^{ignoreline}.*\n?', '', body, flags=re.MULTILINE) # Remove everything after the stopword from the body body = body.split(self.config.stopword, 1)[0] # Add tags to the event if keywords are found in the mail for tag in self.config.tlptags: for alternativetag in self.config.tlptags[tag]: if alternativetag in body.lower(): self.misp_event.add_tag(tag) # Prepare extraction of IOCs # Refang email data body = refang(body) # Extract and add hashes contains_hash = False for h in set(re.findall(hashmarker.MD5_REGEX, body)): contains_hash = True attribute = self.misp_event.add_attribute( 'md5', h, enforceWarninglist=self.config.enforcewarninglist) if email_object: email_object.add_reference(attribute.uuid, 'contains') if self.config.sighting: self.sightings_to_add.append((h, self.config.sighting_source)) for h in set(re.findall(hashmarker.SHA1_REGEX, body)): contains_hash = True attribute = self.misp_event.add_attribute( 'sha1', h, enforceWarninglist=self.config.enforcewarninglist) if email_object: email_object.add_reference(attribute.uuid, 'contains') if self.config.sighting: self.sightings_to_add.append((h, self.config.sighting_source)) for h in set(re.findall(hashmarker.SHA256_REGEX, body)): contains_hash = True attribute = self.misp_event.add_attribute( 'sha256', h, enforceWarninglist=self.config.enforcewarninglist) if email_object: email_object.add_reference(attribute.uuid, 'contains') if self.config.sighting: self.sightings_to_add.append((h, self.config.sighting_source)) if contains_hash: [ self.misp_event.add_tag(tag) for tag in self.config.hash_only_tags ] # # Extract network IOCs urllist = [] urllist += re.findall(urlmarker.WEB_URL_REGEX, body) urllist += re.findall(urlmarker.IP_REGEX, body) if self.debug: syslog.syslog(str(urllist)) hostname_processed = [] # Add IOCs and expanded information to MISP for entry in set(urllist): ids_flag = True self.f.decode(entry) domainname = self.f.get_domain() if domainname in self.config.excludelist: # Ignore the entry continue hostname = self.f.get_host() scheme = self.f.get_scheme() if scheme: scheme = scheme resource_path = self.f.get_resource_path() if resource_path: resource_path = resource_path if self.debug: syslog.syslog(domainname) if domainname in self.config.internallist and self.urlsonly is False: # Add link to internal reference unless in urlsonly mode attribute = self.misp_event.add_attribute( 'link', entry, category='Internal reference', to_ids=False, enforceWarninglist=False) if email_object: email_object.add_reference(attribute.uuid, 'contains') elif domainname in self.config.externallist or self.urlsonly is False: # External analysis attribute = self.misp_event.add_attribute( 'link', entry, category='External analysis', to_ids=False, enforceWarninglist=False) if email_object: email_object.add_reference(attribute.uuid, 'contains') elif domainname in self.config.externallist or self.urlsonly: # External analysis if self.urlsonly: comment = self.subject + " (from: " + self.sender + ")" else: comment = "" attribute = self.misp.add_attribute( self.urlsonly, { "type": 'link', "value": entry, "category": 'External analysis', "to_ids": False, "comment": comment }) for tag in self.config.tlptags: for alternativetag in self.config.tlptags[tag]: if alternativetag in self.subject.lower(): self.misp.tag(attribute["uuid"], tag) new_subject = comment.replace(alternativetag, '') self.misp.change_comment(attribute["uuid"], new_subject) else: # The URL is probably an indicator. comment = "" if (domainname in self.config.noidsflaglist) or ( hostname in self.config.noidsflaglist): ids_flag = False comment = "Known host (mostly for connectivity test or IP lookup)" if self.debug: syslog.syslog(str(entry)) if scheme: if is_ip(hostname): attribute = self.misp_event.add_attribute( 'url', entry, to_ids=False, enforceWarninglist=self.config.enforcewarninglist) if email_object: email_object.add_reference(attribute.uuid, 'contains') else: if resource_path: # URL has path, ignore warning list attribute = self.misp_event.add_attribute( 'url', entry, to_ids=ids_flag, enforceWarninglist=False, comment=comment) if email_object: email_object.add_reference( attribute.uuid, 'contains') else: # URL has no path attribute = self.misp_event.add_attribute( 'url', entry, to_ids=ids_flag, enforceWarninglist=self.config. enforcewarninglist, comment=comment) if email_object: email_object.add_reference( attribute.uuid, 'contains') if self.config.sighting: self.sightings_to_add.append( (entry, self.config.sighting_source)) if hostname in hostname_processed: # Hostname already processed. continue hostname_processed.append(hostname) if self.config.sighting: self.sightings_to_add.append( (hostname, self.config.sighting_source)) if self.debug: syslog.syslog(hostname) comment = '' port = self.f.get_port() if port: port = port comment = f'on port: {port}' if is_ip(hostname): attribute = self.misp_event.add_attribute( 'ip-dst', hostname, to_ids=ids_flag, enforceWarninglist=self.config.enforcewarninglist, comment=comment) if email_object: email_object.add_reference(attribute.uuid, 'contains') else: related_ips = [] if HAS_DNS and self.config.enable_dns: try: syslog.syslog(hostname) for rdata in dns.resolver.query(hostname, 'A'): if self.debug: syslog.syslog(str(rdata)) related_ips.append(rdata.to_text()) except Exception as e: if self.debug: syslog.syslog(str(e)) if related_ips: hip = MISPObject(name='ip-port') hip.add_attribute( 'hostname', value=hostname, to_ids=ids_flag, enforceWarninglist=self.config.enforcewarninglist, comment=comment) for ip in set(related_ips): hip.add_attribute('ip', type='ip-dst', value=ip, to_ids=False, enforceWarninglist=self.config. enforcewarninglist) self.misp_event.add_object(hip) if email_object: email_object.add_reference(hip.uuid, 'contains') else: if self.urlsonly is False: attribute = self.misp_event.add_attribute( 'hostname', value=hostname, to_ids=ids_flag, enforceWarninglist=self.config. enforcewarninglist, comment=comment) if email_object: email_object.add_reference(attribute.uuid, 'contains') def add_event(self): '''Add event on the remote MISP instance.''' # Add additional tags depending on others tags = [] for tag in [t.name for t in self.misp_event.tags]: if self.config.dependingtags.get(tag): tags += self.config.dependingtags.get(tag) # Add additional tags according to configuration for malware in self.config.malwaretags: if malware.lower() in self.subject.lower(): tags += self.config.malwaretags.get(malware) if tags: [self.misp_event.add_tag(tag) for tag in tags] has_tlp_tag = False for tag in [t.name for t in self.misp_event.tags]: if tag.lower().startswith('tlp'): has_tlp_tag = True if not has_tlp_tag: self.misp_event.add_tag(self.config.tlptag_default) if self.offline: return self.misp_event.to_json() event = self.misp.add_event(self.misp_event, pythonify=True) if self.config.sighting: for value, source in self.sightings_to_add: self.sighting(value, source) return event
class MispInstance: ddosch_tag_name = 'DDoSCH' ddosch_tag_colour = '#ff7dfd' def __init__(self, host: str, token: str, protocol: str, verify_tls: bool, sharing_group: str, publish: bool = False): self.host = host self.token = token self.protocol = protocol self.verify_tls = verify_tls self.misp: ExpandedPyMISP self.sharing_group = sharing_group self.publish = publish try: self.misp = ExpandedPyMISP(f'{self.protocol}://{self.host}', self.token, ssl=self.verify_tls, tool='dissector') except PyMISPError: LOGGER.critical( f'Could not connect to MISP instance at "{self.protocol}://{self.host}".' ) self.misp = None def search_misp_events(self, misp_filter: dict = None) -> Optional[dict]: """ Search for MISP events :param misp_filter: fields by which to filter retrieved MISP events :return: MISP events if found, else None """ LOGGER.debug(f'Searching MISP events with filter: {misp_filter}') if not self.verify_tls: urllib3.disable_warnings() response = requests.post(f'{self.protocol}://{self.host}/events/index', json=misp_filter or dict(), headers={ 'Authorization': self.token, 'Accept': 'application/json' }, timeout=10, verify=self.verify_tls) try: response.raise_for_status() return response.json() except requests.HTTPError: LOGGER.critical( f'Retrieving MISP events responded with status code:{response.status_code}' ) return None def get_misp_tag(self, tag_name) -> Optional[dict]: """ Search in MISP for a tag with tag_name, return the tag dict if it exists. :param tag_name: Name of the tag to retrieve :return: Tag or None """ LOGGER.info(f'Searching for {tag_name} tag in MISP') if not self.verify_tls: urllib3.disable_warnings() response = requests.get( f'{self.protocol}://{self.host}/tags/search/{self.ddosch_tag_name}', headers={ 'Authorization': self.token, 'Accept': 'application/json' }, timeout=10, verify=self.verify_tls) try: response.raise_for_status() for item in response.json(): if item['Tag']['name'] == self.ddosch_tag_name: return item else: LOGGER.info( f'No tags found with name "{self.ddosch_tag_name}"') return None except requests.HTTPError: LOGGER.critical( f'Searching for MISP Tag responded with status code:{response.status_code}' ) return None def add_misp_tag(self, tag_name, tag_color) -> Optional[dict]: """ Create a new tag in MISP :param tag_name: Name of the new tag :param tag_color: Color of the new tag :return: Server response if succesful, else None """ LOGGER.debug(f'Creating a {tag_name} tag in MISP') if not self.verify_tls: urllib3.disable_warnings() response = requests.post(f'{self.protocol}://{self.host}/tags/add', json={ 'name': tag_name, 'colour': tag_color }, headers={ 'Authorization': self.token, 'Accept': 'application/json' }, timeout=10, verify=self.verify_tls) try: response.raise_for_status() return response.json() except requests.HTTPError: LOGGER.critical( f'Creating MISP Tag responded with status code:{response.status_code}' ) return None def add_misp_fingerprint(self, fingerprint_json: dict): """ Upload fingerprint to MISP :param fingerprint_json: fingerprint to upload :return: """ LOGGER.info('Uploading the fingerprint to MISP') start = time.time() # Maximum number of source IPs to include # 0 means all (no limit) source_ips_limit = 10 # Possible dicts in each attack_vector of the fingerprint # that will be added as comments (with the dict as value) to the event (not the ddos objects) attack_vector_dicts = [ 'ttl', 'tcp_flags', 'fragmentation_offset', 'ethernet_type', 'frame_len', 'dns_query_name', 'dns_query_type', 'ICMP type', 'ntp_requestcode', 'http_uri', 'http_method', 'http_user_agent', ] # Possible fields in each attack_vector of the fingerprint # that will be added as comments to the event (not the ddos objects) attack_vector_fields = [ 'service', 'fraction_of_attack', 'nr_flows', 'nr_packets', 'nr_megabytes', 'time_start', 'duration_seconds', ] # Possible fields in the fingerprint # that will be added as comments to the event fingerprint_fields = [ 'time_start', 'time_end', 'duration_seconds', 'total_flows', 'total_megabytes', 'total_packets', 'total_ips', 'avg_bps', 'avg_pps', 'avg_Bpp', ] # Create the DDoSCH tag (returns existing one if already present) ddosch_tag = self.get_misp_tag(self.ddosch_tag_name) if ddosch_tag is None: ddosch_tag = self.add_misp_tag(self.ddosch_tag_name, self.ddosch_tag_colour) LOGGER.debug(ddosch_tag) # Retrieve (or create) the sharing group if specified misp_sharing_group = None if self.sharing_group: misp_sharing_group = [ sh_grp for sh_grp in self.misp.sharing_groups(pythonify=True) if sh_grp.name == self.sharing_group ] if len(misp_sharing_group) == 0: misp_sharing_group = self.misp.add_sharing_group( {'name': self.sharing_group}, pythonify=True) else: misp_sharing_group = misp_sharing_group[0] # Create an event to link everything to LOGGER.debug('Creating a new event for the fingerprint') event = MISPEvent() event.info = fingerprint_json['key'] # TARGET event.add_attribute(category='Network activity', type='ip-dst', value=fingerprint_json['target'], comment='target') # KEY event.add_attribute(category='Network activity', type='md5', value=fingerprint_json['key'], comment='attack key') LOGGER.debug('Adding fingerprint fields') for fp_field in fingerprint_fields: if fp_field in fingerprint_json: event.add_attribute(category='Network activity', type='comment', value=fingerprint_json[fp_field], comment=fp_field) # TAGS if 'tags' in fingerprint_json: LOGGER.debug('Adding fingerprint tags') for tag in fingerprint_json['tags']: event.add_tag(tag=tag) event.add_tag(tag='validated') if ddosch_tag is not None: event.add_tag(tag=self.ddosch_tag_name) # Add each attack vector as a MISP object to the MISP event for v_i, attack_vector in enumerate( fingerprint_json['attack_vectors']): LOGGER.debug(f'Processing Attack Vector #{v_i}') ddos_object = MISPObject(name='ddos') # ATTACK VECTOR PROTOCOL ddos_object.add_attribute('protocol', attack_vector['protocol'], comment=f'vector {v_i}') for av_dict in attack_vector_dicts: if av_dict in attack_vector and type( attack_vector[av_dict]) == dict: LOGGER.debug(f'Adding dict {av_dict}') event.add_attribute( category='Network activity', type='comment', value=json.dumps(attack_vector[av_dict]), comment=f'vector {v_i} {av_dict} ({av_dict}:fraction)') for av_field in attack_vector_fields: if av_field in attack_vector and attack_vector[ av_field] is not None: LOGGER.debug(f'Adding field {av_field}') event.add_attribute(category='Network activity', type='comment', value=attack_vector[av_field], comment=f'vector {v_i} {av_field}') # ATTACK VECTOR SOURCE_PORT if type(attack_vector['source_port']) == int: LOGGER.debug('Adding source ports') ddos_object.add_attribute('src-port', attack_vector['source_port'], comment=f'vector {v_i} src-port') # ATTACK VECTOR DESTINATION PORTS if type(attack_vector['destination_ports']) == dict: LOGGER.debug('Adding destination ports') for port in attack_vector['destination_ports'].keys(): ddos_object.add_attribute( 'dst-port', int(port), comment=f'vector {v_i} destination port ' f'(fraction:{attack_vector["destination_ports"][port]}' ) # ATTACK VECTOR DNS if 'dns_query_name' in attack_vector or 'dns_query_type' in attack_vector: ddos_object.add_attribute( 'type', 'dns', comment=f'vector {v_i} type of attack vector') ddos_object.add_attribute( 'type', 'dns-amplification', comment=f'vector {v_i} type of attack vector') # ATTACK VECTOR ICMP if 'ICMP type' in attack_vector: ddos_object.add_attribute( 'type', 'icmp', comment=f'vector {v_i} type of attack vector') # ATTACK VECTOR NTP if 'ntp_requestcode' in attack_vector: ddos_object.add_attribute( 'type', 'ntp-amplification', comment=f'vector {v_i} type of attack vector') # ATTACK VECTOR SOURCE IPS if 'source_ips' in attack_vector: for i, src_ip in enumerate(attack_vector['source_ips'], start=1): ddos_object.add_attribute( 'ip-src', src_ip, comment=f'vector {v_i} source IP') if i >= source_ips_limit > 0: break event.add_object(ddos_object, pythonify=True) if misp_sharing_group: self.misp.change_sharing_group_on_entity(event, misp_sharing_group.id, pythonify=True) if self.publish: event.publish() event = self.misp.add_event(event, pythonify=True) LOGGER.info(f'event: {event}') LOGGER.debug('That took {} seconds'.format(time.time() - start))
def save(self): alert_id = self.validated_data['id'] alert = Alert.objects.get(pk=alert_id) dns_twisted = DnsTwisted.objects.get(pk=alert.dns_twisted.pk) # Getting IOCs related to the new twisted domain if Site.objects.filter(domain_name=dns_twisted.domain_name): already_in_monitoring = True site = Site.objects.get(domain_name=dns_twisted.domain_name) # Store Event Id in database DnsTwisted.objects.filter(pk=dns_twisted.pk).update(misp_event_id=site.misp_event_id) else: already_in_monitoring = False site = Site.objects.create(domain_name=dns_twisted.domain_name, rtir=-999999999) monitoring_init(site) site = Site.objects.get(pk=site.pk) # We now hav the IOCs related to the domain, we can remove it from monitoring if not already_in_monitoring: Site.objects.filter(pk=site.pk).delete() if site.misp_event_id is None: site.misp_event_id = dns_twisted.misp_event_id # Test MISP instance connection try: requests.get(settings.MISP_URL, verify=settings.MISP_VERIFY_SSL) except requests.exceptions.SSLError as e: print(str(timezone.now()) + " - ", e) raise AuthenticationFailed("SSL Error: " + settings.MISP_URL) except requests.exceptions.RequestException as e: print(str(timezone.now()) + " - ", e) raise NotFound("Not Found: " + settings.MISP_URL) misp_api = ExpandedPyMISP(settings.MISP_URL, settings.MISP_KEY, settings.MISP_VERIFY_SSL) if site.misp_event_id is not None: # If the event already exist, then we update IOCs update_attributes(misp_api, site) else: # If the event does not exist, then we create it # Prepare MISP Event event = MISPEvent() event.distribution = 0 event.threat_level_id = 2 event.analysis = 0 event.info = "Suspicious domain name " + site.domain_name event.tags = create_misp_tags(misp_api) # Create MISP Event print(str(timezone.now()) + " - " + 'Create MISP Event') print('-----------------------------') event = misp_api.add_event(event, pythonify=True) # Store Event Id in database DnsTwisted.objects.filter(pk=dns_twisted.pk).update(misp_event_id=event.id) if Site.objects.filter(domain_name=dns_twisted.domain_name): Site.objects.filter(pk=site.pk).update(misp_event_id=event.id) # Create MISP Attributes create_attributes(misp_api, event.id, site)
"to_ids": False, "value": feed, "disable_correlation": False, "object_relation": "", "type": "ip-src" } attr.append(attribute) EventDict = { "Event": { "info": "Cyber Cure Known Bad", "publish timestamp": ts, "analysis": 2, "Tag": { "colour": "ff0000", "exportable": True, "name": "Malicious IPs" }, "Attribute": attr, "threat_level_id": "1", "extends_uuid": "", "published": False, "date": now.strftime("%Y-%m-%d"), "Orgc": { "uuid": "Your UUID", "name": "Cyber Cure" }, } } misp.add_event(EventDict)
class MISP(): def __init__(self, query): self.misp = ExpandedPyMISP(misp_url, misp_key, ssl=misp_verify, debug=False) self.misp_tag = misp_tag self.tags = self.misp.tags() self.query = query self.mainfile = query['Summary']['Subject']['Name'] self.attributes = [] def form_attr_obj(self, type, value, file=None): try: attr = MISPAttribute() attr.type = type attr.value = value if file is not None: path = Path(file) attr.data = path self.attributes.append(attr) except Exception as e: exc_type, exc_obj, exc_tb = sys.exc_info() print("ERROR: Error in {location}.{funct_name}() - line {line_no} : {error}" .format(location=__name__, funct_name=sys._getframe().f_code.co_name, line_no=exc_tb.tb_lineno, error=str(e))) def create_attr_obj(self): try: self.form_attr_obj('filename', self.mainfile) atdip = self.query['Summary']['ATD IP'] if not atdip: pass else: self.form_attr_obj('comment', 'ATD IP {0}'.format(atdip)) dstip = self.query['Summary']['Dst IP'] if not dstip: pass else: self.form_attr_obj('ip-dst', dstip) taskid = self.query['Summary']['TaskId'] if not taskid: pass else: self.form_attr_obj('comment', 'ATD TaskID: {0}'.format(taskid)) md5 = self.query['Summary']['Subject']['md5'] if not md5: pass else: self.form_attr_obj('md5', md5) sha1 = self.query['Summary']['Subject']['sha-1'] if not sha1: pass else: self.form_attr_obj('sha1', sha1) sha256 = self.query['Summary']['Subject']['sha-256'] if not sha256: pass else: self.form_attr_obj('sha256', sha256) size = self.query['Summary']['Subject']['size'] if not size: pass else: self.form_attr_obj('comment', 'File size is: {0}'.format(size)) verdict = self.query['Summary']['Verdict']['Description'] if not verdict: pass else: self.form_attr_obj('comment', verdict) # Add process information to MISP try: for processes in self.query['Summary']['Processes']: name = processes['Name'] md5 = processes['Md5'] sha1 = processes['Sha1'] sha256 = processes['Sha256'] if not name: pass else: self.form_attr_obj('filename', name) if not md5: pass else: self.form_attr_obj('md5', md5) if not sha1: pass else: self.form_attr_obj('sha1', sha1) if not sha256: pass else: self.form_attr_obj('sha256', sha256) except: pass # Add files information to MISP try: for files in self.query['Summary']['Files']: name = files['Name'] md5 = files['Md5'] sha1 = files['Sha1'] sha256 = files['Sha256'] if not name: pass else: self.form_attr_obj('filename', name) if not md5: pass else: self.form_attr_obj('md5', md5) if not sha1: pass else: self.form_attr_obj('sha1', sha1) if not sha256: pass else: self.form_attr_obj('sha256', sha256) except: pass # Add URL information to MISP try: for url in self.query['Summary']['Urls']: url = url['Url'] if not url: pass else: self.form_attr_obj('url', url) except: pass # Add ips information to MISP try: for ips in self.query['Summary']['Ips']: ipv4 = ips['Ipv4'] port = ips['Port'] if not ipv4: pass else: self.form_attr_obj('ip_dst', ipv4) if not port: pass else: self.form_attr_obj('port', port) self.form_attr_obj('url','{0}:{1}'.format(ipv4, port)) except: pass # Add stats Information to MISP try: for stats in self.query['Summary']['Stats']: category = stats['Category'] if not category: pass else: self.form_attr_obj('comment', category) except: pass # Add behaviour information to MISP try: for behave in self.query['Summary']['Behavior']: behave = behave['Analysis'] if not behave: pass else: self.form_attr_obj('comment', behave) except: pass # Download original sample from ATD analysis time.sleep(10) atd = ATD() atd.get_report(taskid, 'pdf') atd.get_report(taskid, 'sample') atd.logout() pdf = '{0}.pdf'.format(taskid) samplefile = '{0}.zip'.format(taskid) # add attributes for analysis report self.form_attr_obj('attachment', '{0}.pdf'.format(str(self.mainfile)), file=pdf) # add attributes for malware sample self.form_attr_obj('malware-sample', '{0}.zip'.format(str(self.mainfile)), file=samplefile) # Delete original sample local - potentially nice to have locally as well os.remove(pdf) os.remove(samplefile) except Exception as e: exc_type, exc_obj, exc_tb = sys.exc_info() print("ERROR: Error in {location}.{funct_name}() - line {line_no} : {error}" .format(location=__name__, funct_name=sys._getframe().f_code.co_name, line_no=exc_tb.tb_lineno, error=str(e))) def add_event(self): try: event = MISPEvent() event.distribution = 0 # ATD Threat mapping to MISP Threat Level atd_threat_level = self.query['Summary']['Verdict']['Severity'] if not atd_threat_level: pass else: if atd_threat_level == '3': event.threat_level_id = 1 elif atd_threat_level == '4': event.threat_level_id = 2 elif atd_threat_level == '5': event.threat_level_id = 3 else: event.threat_level_id = 0 event.analysis = 0 # initial event.info = "ATD Analysis Report - {0}".format(self.mainfile) event.attributes = self.attributes event.Tag = 'ATD:Report' event = self.misp.add_event(event, pythonify=True) self.evenid = event.id print('SUCCESS: New MISP Event got created with ID: {}'.format(str(event.id))) except Exception as e: exc_type, exc_obj, exc_tb = sys.exc_info() print("ERROR: Error in {location}.{funct_name}() - line {line_no} : {error}" .format(location=__name__, funct_name=sys._getframe().f_code.co_name, line_no=exc_tb.tb_lineno, error=str(e))) def assign_tag(self): try: uuid = None results = self.misp.search(eventid=self.evenid) for event in results: uuid = event['Event']['uuid'] self.misp.tag(uuid, self.misp_tag) break if uuid is None: print('STATUS: Could not tag the MISP Event. Continuing...') #MITRE tags assign to McAfee ATD findings version = self.query['Summary']['SUMversion'] version = str(version).replace('.', '') if int(version) >= 46021: mitre_list = self.query['Summary']['Mitre'] for val in mitre_list: mitre_tech = val['Techniques'] for tag in self.tags: if 'mitre-attack-pattern="{0}'.format(mitre_tech) in tag['name']: self.misp.tag(uuid, tag['id']) except Exception as e: exc_type, exc_obj, exc_tb = sys.exc_info() print("ERROR: Error in {location}.{funct_name}() - line {line_no} : {error}" .format(location=__name__, funct_name=sys._getframe().f_code.co_name, line_no=exc_tb.tb_lineno, error=str(e))) def main(self): self.create_attr_obj() self.add_event() self.assign_tag()
objects = csv_loader.load() if args.dump: for o in objects: print(o.to_json()) else: if offline: print('You are in offline mode, quitting.') else: misp = ExpandedPyMISP(url=misp_url, key=misp_key, ssl=misp_verifycert) if args.new_event: event = MISPEvent() event.info = args.new_event for o in objects: event.add_object(**o) new_event = misp.add_event(event) if isinstance(new_event, str): print(new_event) elif 'id' in new_event: print(f'Created new event {new_event.id}') else: print('Something went wrong:') print(new_event) else: for o in objects: new_object = misp.add_object(args.update_event, o) if isinstance(new_object, str): print(new_object) elif new_object.attributes: print(f'New {new_object.name} object added to {args.update_event}') else:
class MISPController(object): ''' MISP Controller ''' def __init__(self, misp_param, debug=False): self.misp_param = misp_param self.debug = debug if misp_param.get('connect_immediately', False): self._connect() else: self.misp = None def import_event(self, event_data): ''' Import event ''' # Check registered same event info print('importing: {}'.format(event_data['title'])) events = self._search_event(eventinfo=event_data['title']) if events != None: for event in events: if event_data['title'] == event['Event']['info']: self._remove_event(event['Event']['id']) event = self._add_event(event_data) if event: print('created event: {}'.format(event.id)) else: print("Import failed.Please retry: {}".format(event_data['title'])) def _connect(self): self.debug_print('URL: {}'.format(self.misp_param['url'])) self.debug_print('authkey: {}'.format(self.misp_param['authkey'])) self.misp = ExpandedPyMISP(self.misp_param['url'], self.misp_param['authkey'], ssl=False, debug=False) self._registered_tags = self.misp.tags() def _check_tag(self, target_tag): if self.misp == None: self._connect() if self._registered_tags == None: self._get_tags() for tag_info in self._registered_tags: if tag_info.get('name', '') == target_tag: return True self.debug_print('new tag: {}'.format(target_tag)) cnt = 0 while True: try: if self.misp == None: self._connect() tmp = MISPTag() tmp.from_dict(name=target_tag) self.misp.add_tag(tmp) self._get_tags() return True except: print(traceback.format_exc()) if cnt < int(self.misp_param.get('max_retry_count', '0')): print('add new tag retry: {}'.format(cnt)) cnt = cnt + 1 time.sleep(10) else: return False def _add_event(self, value): for tag in value['event_tags']: self._check_tag(tag) for attribute in value['attributes']: for tag in attribute['tags']: self._check_tag(tag) cnt = 0 while True: try: if self.misp == None: self._connect() tmp = MISPEvent() tmp.from_dict( distribution=self.misp_param['distribution'], threat_level_id=self.misp_param['threat_level_id'], analysis=self.misp_param['analysis'], info=value['title'], date=value['date'], published=False) response = self.misp.add_event(tmp) if response.get('errors'): raise Exception(str(response['errors'])) event = MISPEvent() event.load(response) break except: print(traceback.format_exc()) if cnt < int(self.misp_param.get('max_retry_count', '0')): print('add new event retry: {}'.format(cnt)) cnt = cnt + 1 time.sleep(10) else: return None self.debug_print(event.id) for tag in value['event_tags']: event.add_tag(tag) for attribute in value['attributes']: attribute_tags = [] event.add_attribute(type=attribute['type'], value=attribute['value'], category=attribute['category'], comment=attribute.get('comment', ''), distribution=self.misp_param['distribution'], Tag=self._create_tags(attribute['tags'])) event.published = True if self._update_event(event): self.debug_print('completed') return event else: self.debug_print('add failed') return None def _get_event(self, id): cnt = 0 while True: try: if self.misp == None: self._connect() self.debug_print('get event start: {}'.format(id)) event = self.misp.get_event(id) if event.get('errors'): raise Exception(str(event['errors'])) self.debug_print('get event end: {}'.format(id)) return event except: print(traceback.format_exc()) if cnt < int(self.misp_param.get('max_retry_count', '0')): print('get event retry: {}'.format(cnt)) cnt = cnt + 1 time.sleep(10) else: return None def _remove_event(self, id): if id: print('delete event: {}'.format(id)) cnt = 0 while True: try: if self.misp == None: self._connect() response = self.misp.delete_event(id) if response.get('errors'): raise Exception(str(response['errors'])) return True except: print(traceback.format_exc()) if cnt < int(self.misp_param.get('max_retry_count', '0')): print('remove event retry: {}'.format(cnt)) cnt = cnt + 1 time.sleep(10) else: return False def _search_event(self, **cons): cnt = 0 while True: try: if self.misp == None: self._connect() self.debug_print('search event start') response = self.misp.search_index(**cons) self.debug_print('search event end') results = [] for json in response: if json.get('id', ''): results.append(self._get_event(json['id'])) else: print('no event ID') print(json) return results except: print(traceback.format_exc()) if cnt < int(self.misp_param.get('max_retry_count', '0')): print('search event retry: {}'.format(cnt)) cnt = cnt + 1 time.sleep(10) else: return None def _update_event(self, event): cnt = 0 while True: try: if self.misp == None: self._connect() self.debug_print('event update start: {}'.format(event.id)) response = self.misp.update_event(event) if response.get('errors'): raise Exception(str(response['errors'])) self.debug_print('{} updated'.format(event.id)) return True except: print(traceback.format_exc()) if cnt < int(self.misp_param.get('max_retry_count', '0')): print('retry: {}'.format(cnt)) cnt = cnt + 1 time.sleep(10) else: print('event update failed: {}'.format(event.info)) try: self._remove_event(event.id) except: pass return False def _create_tags(self, values): tags = [] for value in values: if value: tags.append({'name': value}) return tags def debug_print(self, message): if self.debug == False: return # nowstr = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') nowstr = datetime.datetime.now().strftime('%H:%M:%S') print('{}\t{}'.format(nowstr, message))
class Scrippts: def __init__(self): self.misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert) def geolocation_alt(self) -> MISPObject: # Alert, NWT, Canada location = MISPObject('geolocation', standalone=False) location.add_attribute('latitude', 82.3) location.add_attribute('longitude', 62.3) location.add_attribute('altitude', 210) location.add_attribute('text', 'Alert, NWT, Canada') return location def tag_alt(self) -> MISPTag: tag = MISPTag() tag.name = 'scrippsco2-sampling-stations:ALT' return tag def geolocation_ptb(self): # Point Barrow, Alaska location = MISPObject('geolocation') location.add_attribute('latitude', 71.3) location.add_attribute('longitude', 156.6) location.add_attribute('altitude', 11) location.add_attribute('text', 'Point Barrow, Alaska') return location def tag_ptb(self): tag = MISPTag() tag.name = 'scrippsco2-sampling-stations:PTB' return tag def geolocation_stp(self) -> MISPObject: # Station P location = MISPObject('geolocation') location.add_attribute('latitude', 50) location.add_attribute('longitude', 145) location.add_attribute('altitude', 0) location.add_attribute('text', 'Station P') return location def tag_stp(self): tag = MISPTag() tag.name = 'scrippsco2-sampling-stations:STP' return tag def geolocation_ljo(self) -> MISPObject: # La Jolla Pier, California location = MISPObject('geolocation') location.add_attribute('latitude', 32.9) location.add_attribute('longitude', 117.3) location.add_attribute('altitude', 10) location.add_attribute('text', 'La Jolla Pier, California') return location def tag_ljo(self): tag = MISPTag() tag.name = 'scrippsco2-sampling-stations:LJO' return tag def geolocation_bcs(self) -> MISPObject: # Baja California Sur, Mexico location = MISPObject('geolocation') location.add_attribute('latitude', 23.3) location.add_attribute('longitude', 110.2) location.add_attribute('altitude', 4) location.add_attribute('text', 'Baja California Sur, Mexico') return location def tag_bcs(self): tag = MISPTag() tag.name = 'scrippsco2-sampling-stations:BCS' return tag def geolocation_mlo(self) -> MISPObject: # Mauna Loa Observatory, Hawaii location = MISPObject('geolocation') location.add_attribute('latitude', 19.5) location.add_attribute('longitude', 155.6) location.add_attribute('altitude', 3397) location.add_attribute('text', 'Mauna Loa Observatory, Hawaii') return location def tag_mlo(self): tag = MISPTag() tag.name = 'scrippsco2-sampling-stations:MLO' return tag def geolocation_kum(self) -> MISPObject: # Cape Kumukahi, Hawaii location = MISPObject('geolocation') location.add_attribute('latitude', 19.5) location.add_attribute('longitude', 154.8) location.add_attribute('altitude', 3) location.add_attribute('text', 'Cape Kumukahi, Hawaii') return location def tag_kum(self): tag = MISPTag() tag.name = 'scrippsco2-sampling-stations:KUM' return tag def geolocation_chr(self): # Christmas Island, Fanning Island location = MISPObject('geolocation') location.add_attribute('latitude', 2) location.add_attribute('longitude', 157.3) location.add_attribute('altitude', 2) location.add_attribute('text', 'Christmas Island, Fanning Island') return location def tag_chr(self): tag = MISPTag() tag.name = 'scrippsco2-sampling-stations:CHR' return tag def geolocation_sam(self): # American Samoa location = MISPObject('geolocation') location.add_attribute('latitude', 14.2) location.add_attribute('longitude', 170.6) location.add_attribute('altitude', 30) location.add_attribute('text', 'American Samoa') return location def tag_sam(self): tag = MISPTag() tag.name = 'scrippsco2-sampling-stations:SAM' return tag def geolocation_ker(self): # Kermadec Islands, Raoul Island location = MISPObject('geolocation') location.add_attribute('latitude', 29.2) location.add_attribute('longitude', 177.9) location.add_attribute('altitude', 2) location.add_attribute('text', 'Kermadec Islands, Raoul Island') return location def tag_ker(self): tag = MISPTag() tag.name = 'scrippsco2-sampling-stations:KER' return tag def geolocation_nzd(self): # Baring Head, New Zealand location = MISPObject('geolocation') location.add_attribute('latitude', 41.4) location.add_attribute('longitude', 174.9) location.add_attribute('altitude', 85) location.add_attribute('text', 'Baring Head, New Zealand') return location def tag_nzd(self): tag = MISPTag() tag.name = 'scrippsco2-sampling-stations:NZD' return tag def geolocation_psa(self): # Palmer Station, Antarctica location = MISPObject('geolocation') location.add_attribute('latitude', 64.9) location.add_attribute('longitude', 64) location.add_attribute('altitude', 10) location.add_attribute('text', 'Palmer Station, Antarctica') return location def tag_psa(self): tag = MISPTag() tag.name = 'scrippsco2-sampling-stations:PSA' return tag def geolocation_spo(self): # South Pole location = MISPObject('geolocation') location.add_attribute('latitude', 90) location.add_attribute('longitude', 0) location.add_attribute('altitude', 2810) location.add_attribute('text', 'South Pole') return location def tag_spo(self): tag = MISPTag() tag.name = 'scrippsco2-sampling-stations:SPO' return tag def fetch(self, url): filepath = Path('scrippts') / Path(url).name if filepath.exists(): return filepath r = requests.get(url) if r.status_code != 200 or r.text[0] != '"': print(url) return False with filepath.open('w') as f: f.write(r.text) return filepath def get_existing_event_to_update(self, infofield): found = self.misp.search(eventinfo=infofield, pythonify=True) if found: event = found[0] return event return False def import_all(self, stations_short_names, interval, data_type): object_creator = getattr(self, f'{interval}_flask_{data_type}') if data_type == 'co2': base_url = 'http://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_co2/' elif data_type in ['c13', 'o18']: base_url = 'http://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_isotopic/' for station in stations_short_names: url = f'{base_url}/{interval}/{interval}_flask_{data_type}_{station}.csv' infofield = f'[{station.upper()}] {interval} average atmospheric {data_type} concentrations' filepath = self.fetch(url) if not filepath: continue update = True event = self.get_existing_event_to_update(infofield) if event: location = event.get_objects_by_name('geolocation')[0] if not event: event = MISPEvent() event.info = infofield event.add_tag(getattr(self, f'tag_{station}')()) location = getattr(self, f'geolocation_{station}')() event.add_object(location) event.add_attribute('link', f'http://scrippsco2.ucsd.edu/data/atmospheric_co2/{station}') update = False object_creator(event, location, filepath, update) if update: self.misp.update_event(event) else: self.misp.add_event(event) def import_monthly_co2_all(self): to_import = ['alt', 'ptb', 'stp', 'ljo', 'bcs', 'mlo', 'kum', 'chr', 'sam', 'ker', 'nzd'] self.import_all(to_import, 'monthly', 'co2') def import_monthly_c13_all(self): to_import = ['alt', 'ptb', 'stp', 'ljo', 'bcs', 'mlo', 'kum', 'chr', 'sam', 'ker', 'nzd', 'psa', 'spo'] self.import_all(to_import, 'monthly', 'c13') def import_monthly_o18_all(self): to_import = ['alt', 'ptb', 'stp', 'ljo', 'bcs', 'mlo', 'kum', 'chr', 'sam', 'ker', 'nzd', 'spo'] self.import_all(to_import, 'monthly', 'o18') def import_daily_co2_all(self): to_import = ['alt', 'ptb', 'stp', 'ljo', 'bcs', 'mlo', 'kum', 'chr', 'sam', 'ker', 'nzd'] self.import_all(to_import, 'daily', 'co2') def import_daily_c13_all(self): to_import = ['alt', 'ptb', 'ljo', 'bcs', 'mlo', 'kum', 'chr', 'sam', 'ker', 'nzd', 'spo'] self.import_all(to_import, 'daily', 'c13') def import_daily_o18_all(self): to_import = ['alt', 'ptb', 'ljo', 'bcs', 'mlo', 'kum', 'chr', 'sam', 'ker', 'nzd', 'spo'] self.import_all(to_import, 'daily', 'o18') def split_data_comment(self, csv_file, update, event): comment = '' data = [] with csv_file.open() as f: for line in f: if line[0] == '"': if update: continue if '----------' in line: event.add_attribute('comment', comment, disable_correlation=True) comment = '' continue comment += line[1:-1].strip() else: data.append(line) if not update: event.add_attribute('comment', comment, disable_correlation=True) return data def monthly_flask_co2(self, event, location, csv_file, update): data = self.split_data_comment(csv_file, update, event) dates_already_imported = [] if update: # get all datetime from existing event for obj in event.get_objects_by_name('scrippsco2-co2-monthly'): date_attribute = obj.get_attributes_by_relation('sample-datetime')[0] dates_already_imported.append(date_attribute.value) reader = csv.reader(data) for row in reader: if not row[0].isdigit(): # This file has f****d up headers continue sample_date = parse(f'{row[0]}-{row[1]}-16T00:00:00') if sample_date in dates_already_imported: continue obj = MISPObject('scrippsco2-co2-monthly', standalone=False) obj.add_attribute('sample-datetime', sample_date) obj.add_attribute('sample-date-excel', float(row[2])) obj.add_attribute('sample-date-fractional', float(row[3])) obj.add_attribute('monthly-co2', float(row[4])) obj.add_attribute('monthly-co2-seasonal-adjustment', float(row[5])) obj.add_attribute('monthly-co2-smoothed', float(row[6])) obj.add_attribute('monthly-co2-smoothed-seasonal-adjustment', float(row[7])) obj.add_reference(location, 'sampling-location') event.add_object(obj) def monthly_flask_c13(self, event, location, csv_file, update): data = self.split_data_comment(csv_file, update, event) dates_already_imported = [] if update: # get all datetime from existing event for obj in event.get_objects_by_name('scrippsco2-c13-monthly'): date_attribute = obj.get_attributes_by_relation('sample-datetime')[0] dates_already_imported.append(date_attribute.value) reader = csv.reader(data) for row in reader: if not row[0].isdigit(): # This file has f****d up headers continue sample_date = parse(f'{row[0]}-{row[1]}-16T00:00:00') if sample_date in dates_already_imported: continue obj = MISPObject('scrippsco2-c13-monthly', standalone=False) obj.add_attribute('sample-datetime', sample_date) obj.add_attribute('sample-date-excel', float(row[2])) obj.add_attribute('sample-date-fractional', float(row[3])) obj.add_attribute('monthly-c13', float(row[4])) obj.add_attribute('monthly-c13-seasonal-adjustment', float(row[5])) obj.add_attribute('monthly-c13-smoothed', float(row[6])) obj.add_attribute('monthly-c13-smoothed-seasonal-adjustment', float(row[7])) obj.add_reference(location, 'sampling-location') event.add_object(obj) def monthly_flask_o18(self, event, location, csv_file, update): data = self.split_data_comment(csv_file, update, event) dates_already_imported = [] if update: # get all datetime from existing event for obj in event.get_objects_by_name('scrippsco2-o18-monthly'): date_attribute = obj.get_attributes_by_relation('sample-datetime')[0] dates_already_imported.append(date_attribute.value) reader = csv.reader(data) for row in reader: if not row[0].isdigit(): # This file has f****d up headers continue sample_date = parse(f'{row[0]}-{row[1]}-16T00:00:00') if sample_date in dates_already_imported: continue obj = MISPObject('scrippsco2-o18-monthly', standalone=False) obj.add_attribute('sample-datetime', sample_date) obj.add_attribute('sample-date-excel', float(row[2])) obj.add_attribute('sample-date-fractional', float(row[3])) obj.add_attribute('monthly-o18', float(row[4])) obj.add_attribute('monthly-o18-seasonal-adjustment', float(row[5])) obj.add_attribute('monthly-o18-smoothed', float(row[6])) obj.add_attribute('monthly-o18-smoothed-seasonal-adjustment', float(row[7])) obj.add_reference(location, 'sampling-location') event.add_object(obj) def daily_flask_co2(self, event, location, csv_file, update): data = self.split_data_comment(csv_file, update, event) dates_already_imported = [] if update: # get all datetime from existing event for obj in event.get_objects_by_name('scrippsco2-co2-daily'): date_attribute = obj.get_attributes_by_relation('sample-datetime')[0] dates_already_imported.append(date_attribute.value) reader = csv.reader(data) for row in reader: sample_date = parse(f'{row[0]}-{row[1]}') if sample_date in dates_already_imported: continue obj = MISPObject('scrippsco2-co2-daily', standalone=False) obj.add_attribute('sample-datetime', sample_date) obj.add_attribute('sample-date-excel', float(row[2])) obj.add_attribute('sample-date-fractional', float(row[3])) obj.add_attribute('number-flask', int(row[4])) obj.add_attribute('flag', int(row[5])) attr = obj.add_attribute('co2-value', float(row[6])) attr.add_tag(f'scrippsco2-fgc:{int(row[5])}') obj.add_reference(location, 'sampling-location') event.add_object(obj) def daily_flask_c13(self, event, location, csv_file, update): data = self.split_data_comment(csv_file, update, event) dates_already_imported = [] if update: # get all datetime from existing event for obj in event.get_objects_by_name('scrippsco2-c13-daily'): date_attribute = obj.get_attributes_by_relation('sample-datetime')[0] dates_already_imported.append(date_attribute.value) reader = csv.reader(data) for row in reader: sample_date = parse(f'{row[0]}-{row[1]}') if sample_date in dates_already_imported: continue obj = MISPObject('scrippsco2-c13-daily', standalone=False) obj.add_attribute('sample-datetime', sample_date) obj.add_attribute('sample-date-excel', float(row[2])) obj.add_attribute('sample-date-fractional', float(row[3])) obj.add_attribute('number-flask', int(row[4])) obj.add_attribute('flag', int(row[5])) attr = obj.add_attribute('c13-value', float(row[6])) attr.add_tag(f'scrippsco2-fgi:{int(row[5])}') obj.add_reference(location, 'sampling-location') event.add_object(obj) def daily_flask_o18(self, event, location, csv_file, update): data = self.split_data_comment(csv_file, update, event) dates_already_imported = [] if update: # get all datetime from existing event for obj in event.get_objects_by_name('scrippsco2-o18-daily'): date_attribute = obj.get_attributes_by_relation('sample-datetime')[0] dates_already_imported.append(date_attribute.value) reader = csv.reader(data) for row in reader: sample_date = parse(f'{row[0]}-{row[1]}') if sample_date in dates_already_imported: continue obj = MISPObject('scrippsco2-o18-daily', standalone=False) obj.add_attribute('sample-datetime', sample_date) obj.add_attribute('sample-date-excel', float(row[2])) obj.add_attribute('sample-date-fractional', float(row[3])) obj.add_attribute('number-flask', int(row[4])) obj.add_attribute('flag', int(row[5])) attr = obj.add_attribute('o18-value', float(row[6])) attr.add_tag(f'scrippsco2-fgi:{int(row[5])}') obj.add_reference(location, 'sampling-location') event.add_object(obj)
class MispHandler: def __init__(self, config, logger): self.logger = logger MISP_KEY = config['MISP_KEY'] MISP_URL = config['MISP_URL'] MISP_VERIFYCERT = config['MISP_VERIFYCERT'] self.config = config self.misp = ExpandedPyMISP(MISP_URL, MISP_KEY, MISP_VERIFYCERT) self.orgc = MISPOrganisation() self.orgc.name = config['MISP_ORG_NAME'] self.orgc.id = config['MISP_ORG_ID'] self.orgc.uuid = config['MISP_ORG_UUID'] self.tags = config['tags'] self.galaxy_synonyms = {} self.enabled_clusters = self.config['galaxies'] self.galaxy_tags = {} self._init_galaxies() def get_event_id(self, event): try: return event['Event']['id'] except KeyError: return event.id except TypeError: return event.id def get_attributes(self, event): try: attributes = event['Event']['Attribute'] except KeyError: attributes = event.attributes return attributes def add_sigthing(self, id): sighting = MISPSighting() self.misp.add_sighting(sighting, id) def get_event(self, malware_type, feed_tag): malware_tag = "malware:" + malware_type.lower() res = self.misp.search(tags=[feed_tag], controller='events', pythonify=True) for event in res: for tag in event.tags: if tag['name'].lower() == malware_tag.lower(): return event return None def _init_galaxies(self): i = 1 cont = True for cluster in self.enabled_clusters: self.galaxy_tags[cluster] = [] while cont: g = self.misp.get_galaxy(i) try: galaxy_cluster = g['Galaxy']['name'] except KeyError: cont = False continue if galaxy_cluster.lower() in self.enabled_clusters: elements = g['GalaxyCluster'] for element in elements: self.galaxy_tags[galaxy_cluster.lower()].append(element['tag_name']) for inner_element in element['GalaxyElement']: if inner_element['key'] == 'synonyms': if not element['tag_name'] in self.galaxy_synonyms: self.galaxy_synonyms[element['tag_name']] = [] self.galaxy_synonyms[element['tag_name']].append(inner_element['value']) i = i + 1 def get_galaxies(self, malware_tag): res = [] for cluster in self.enabled_clusters: for galaxy_tag in self.galaxy_tags[cluster]: malware = malware_tag.split(':')[1].lower().replace(" ", "") galaxy_value = galaxy_tag.split('"')[1].lower().replace(" ", "") if malware == galaxy_value: res.append(galaxy_tag) break else: if galaxy_tag in self.galaxy_synonyms: for synonym in self.galaxy_synonyms[galaxy_tag]: galaxy_value = synonym.lower().replace(" ", "") if malware == galaxy_value: res.append(galaxy_tag) break return res def get_file_taxonomy(self, ft): if ft == 'exe': return 'file-type:type="peexe"' elif ft == 'dll': return 'file-type:type="pedll"' elif ft == 'zip': return 'file-type:type="zip"' elif ft == 'apk': return 'file-type:type="android"' elif ft == 'rar': return 'file-type:type="rar"' elif ft == 'xls': return 'file-type:type="xls"' elif ft == 'xlsx': return 'file-type:type="xlsx"' elif ft == 'doc': return 'file-type:type="doc"' elif ft == 'docx': return 'file-type:type="docx"' elif ft == '7z' or ft == '7zip': return 'file-type:type="7zip"' elif ft == 'gz' or ft == 'gzip': return 'file-type:type="gzip"' file_types = self.misp.get_taxonomy(52)['entries'] for file_type in file_types: if ft == file_type['tag'].split('"')[1].strip(): return file_type['tag'] print("Unknown Filetype: " + ft) return '' def new_misp_event(self, malware_type, feed_tag, event_info, additional_tags=[], info_cred='admirality-scale:information-credibility="2"'): malware_tag = "malware:" + malware_type.lower() misp_event_obj = MISPEvent() misp_event_obj.info = event_info misp_event_obj.add_tag(feed_tag) misp_event_obj.add_tag(info_cred) if len(malware_tag) > 0: misp_event_obj.add_tag(malware_tag.lower()) for tag in self.config['tags']: misp_event_obj.add_tag(tag) for tag in additional_tags: misp_event_obj.add_tag(tag) misp_event_obj.orgc = self.orgc galaxies = self.get_galaxies(malware_tag) for galaxy in galaxies: misp_event_obj.add_tag(galaxy) misp_event = self.misp.add_event(misp_event_obj) return misp_event
if last_event_date < date.today() or int(nb_attr) > 1000: me = create_new_event() else: event_id = response[0].id else: me = create_new_event() parameters = {'banned-ip': args.banned_ip, 'attack-type': args.attack_type} if args.processing_timestamp: parameters['processing-timestamp'] = args.processing_timestamp if args.failures: parameters['failures'] = args.failures if args.sensor: parameters['sensor'] = args.sensor if args.victim: parameters['victim'] = args.victim if args.logline: parameters['logline'] = b64decode(args.logline).decode() if args.logfile: with open(args.logfile, 'rb') as f: parameters['logfile'] = { 'value': os.path.basename(args.logfile), 'data': BytesIO(f.read()) } f2b = Fail2BanObject(parameters=parameters, standalone=False) if me: me.add_object(f2b) pymisp.add_event(me) elif event_id: a = pymisp.add_object(event_id, f2b)
if args.is_malware: arg_type = 'malware-sample' else: arg_type = 'attachment' # Create attributes attributes = [] for f in files: a = MISPAttribute() a.type = arg_type a.value = f.name a.data = f a.comment = args.comment a.distribution = args.distrib if args.expand and arg_type == 'malware-sample': a.expand = 'binary' attributes.append(a) if args.event: for a in attributes: misp.add_attribute(args.event, a) else: m = MISPEvent() m.info = args.info m.distribution = args.distrib m.attributes = attributes if args.expand and arg_type == 'malware-sample': m.run_expansions() misp.add_event(m)
class OutputMISPBot(Bot): def init(self): if ExpandedPyMISP is None: raise ValueError('Could not import pymisp. Please install it.') # Initialize MISP connection self.misp = ExpandedPyMISP(self.parameters.misp_url, self.parameters.misp_key, self.parameters.misp_verify) # Handle the InsecureRequestWarning if not self.parameters.misp_verify: urllib3.disable_warnings() self.logger.info ("Connected to MISP") def process(self): # Grab raw event report = self.receive_message() raw_report = utils.base64_decode(report.get('raw')) atd_event = json.loads(raw_report) if (atd_event['Summary']['Event_Type'] == 'ATD File Report'): if (int(atd_event['Summary']['Verdict']['Severity']) >= self.parameters.atd_verdict_severity): # Generate MISP Event event = self.misp_add_event(atd_event) # Add event to MISP instance event = self.misp.add_event(event) if self.parameters.atd_event_publish: # Publish new MISP event self.misp.publish(event) self.acknowledge_message() def misp_add_attribute(self, event, a_type, a_value): # Create Attribute object in MISP misp_attribute = MISPAttribute() if a_type: misp_attribute.type = a_type if a_value: misp_attribute.value = a_value event.add_attribute(misp_attribute.type, misp_attribute.value) def misp_add_tag(self, event, a_value): # Create Tag object in MISP misp_tag = MISPTag() if a_value: misp_tag.name = a_value event.add_tag(misp_tag) def misp_add_fileObject (self, event, filename, md5, sha1, sha256): # Create new MISPFileObject misp_object = MISPObject (name = 'file') # Add attributes misp_object.add_attribute("filename", value = filename) misp_object.add_attribute("md5", value = md5) misp_object.add_attribute("sha1", value = sha1) misp_object.add_attribute("sha256", value = sha256) # Add MISPFileObject to event event.add_object (misp_object) def misp_add_event(self, query): # Parse out all data from json mainfile = query['Summary']['Subject']['Name'] # Set Distribution = Organization Only distribution=self.parameters.misp_distribution # Set Threat level = getting the threat level from ATD threat_level_id=query['Summary']['Verdict']['Severity'] # Set Analysis status = completed analysis_status=2 # Creat Event object in MISP misp_event = MISPEvent() misp_event.info = "McAfee ATD Sandbox Analysis Report - " + mainfile misp_event.distribution = distribution misp_event.threat_level_id = atd_to_misp_confidence(threat_level_id) misp_event.analysis = analysis_status # Add main Information to MISP atdip = query['Summary']['ATD IP'] if not atdip: pass else: self.misp_add_attribute(misp_event, "comment", "ATD IP " + atdip) dstip = query['Summary']['Dst IP'] if not dstip: pass else: self.misp_add_attribute(misp_event, "ip-dst", dstip) taskid = query['Summary']['TaskId'] if not taskid: pass else: self.misp_add_attribute(misp_event, "comment", "ATD TaskID: " + taskid) size = query['Summary']['Subject']['size'] if not size: pass else: self.misp_add_attribute(misp_event, "comment", "File size is " + size) verdict = query['Summary']['Verdict']['Description'] if not verdict: pass else: self.misp_add_attribute(misp_event, "comment", verdict) # Add file object to MISP Event self.misp_add_fileObject (misp_event, mainfile, query['Summary']['Subject']['md5'], query['Summary']['Subject']['sha-1'], query['Summary']['Subject']['sha-256'] ) # Add process information to MISP try: for processes in query['Summary']['Processes']: name = processes['Name'] md5 = processes['Md5'] sha1 = processes['Sha1'] sha256 = processes['Sha256'] if not name: pass else: self.misp_add_attribute(misp_event, "filename", name) if not md5: pass else: self.misp_add_attribute(misp_event, "md5", md5) if not sha1: pass else: self.misp_add_attribute(misp_event, "sha1", sha1) if not sha256: pass else: self.misp_add_attribute(misp_event, "sha256", sha256) except: pass # Add files information to MISP try: for files in query['Summary']['Files']: # Evaluate attributes name = files['Name'] md5 = files['Md5'] sha1 = files['Sha1'] sha256 = files['Sha256'] # Add attributes as FileObject to event self.misp_add_fileObject (misp_event, name, md5, sha1, sha256) except: pass # Add URL information to MISP try: for url in query['Summary']['Urls']: url = url['Url'] if not url: pass else: self.misp_add_attribute(misp_event, "url", url) except: pass # Add ips information to MISP try: for ips in query['Summary']['Ips']: ipv4 = ips['Ipv4'] port = ips['Port'] if not ipv4: pass else: self.misp_add_attribute(misp_event, "ip-dst", ipv4) if not port: pass else: self.misp_add_attribute(misp_event, "url", ipv4 + ":" + port) except: pass # Add stats Information to MISP try: for stats in query['Summary']['Stats']: category = stats['Category'] if not category: pass else: self.misp_add_attribute(misp_event, "comment", category) except: pass # Add behaviour information to MISP try: for behave in query['Summary']['Behavior']: behave = behave['Analysis'] if not category: pass else: self.misp_add_attribute(misp_event, "comment", behave) except: pass # Add Confidence level from ATD to MISP self.misp_add_tag(misp_event, str(atd_to_veris_confidence(threat_level_id))) # Add TLP info to MISP self.misp_add_tag(misp_event, str("tlp:amber")) self.misp_add_tag(misp_event, str("McAfee ATD Analysis")) # Add tag to event self.misp_add_tag(misp_event, str("cssa:origin=\"sandbox\"")) self.misp_add_tag(misp_event, str("cssa:sharing-class=\"unvetted\"")) # Add actual event to MISP instance # Moved to calling routine # misp_event = self.misp.add_event(misp_event) return misp_event
customerUserId = event.add_attribute( 'other', alert["identity"]["customerUserId"], comment="Customer User Id") if alert["identity"]["department"] is not None: department = event.add_attribute(alert['other', "identity"]["department"], comment="Department") if alert["identity"]["location"] is not None: location = event.add_attribute('other', alert["identity"]["location"], comment="Location") if alert["identity"]["name"] is not None: name = event.add_attribute('target-user', alert["identity"]["name"], comment="Name") if alert["identity"]["title"] is not None: title = event.add_attribute('other', alert["identity"]["title"], comment="Title") event.add_tag("VAP") misp.add_event(event.to_json())
if args.dump: for o in objects: print(o.to_json()) else: if offline: print('You are in offline mode, quitting.') else: misp = ExpandedPyMISP(url=misp_url, key=misp_key, ssl=misp_verifycert) if args.new_event: event = MISPEvent() event.info = args.new_event for o in objects: event.add_object(**o) new_event = misp.add_event(event) if isinstance(new_event, str): print(new_event) elif 'id' in new_event: print(f'Created new event {new_event.id}') else: print('Something went wrong:') print(new_event) else: for o in objects: new_object = misp.add_object(args.update_event, o) if isinstance(new_object, str): print(new_object) elif new_object.attributes: print( f'New {new_object.name} object added to {args.update_event}'
def main(): # Process arguments and initialize TRAP interface try: args, _ = process_arguments() except SystemExit as e: args = None # Error in arguments or help message trap = pytrap.TrapCtx() try: trap.init(sys.argv, 1, 0, module_name="Backscatter classifier common TRAP help") # Allow trap to print it's own help but terminate script anyway due to error in arguments if args is None: sys.exit(PYTHON_ARGS_ERROR_EXIT) trap.setRequiredFmt(0) except Exception as e: # Trap error message print(e) sys.exit(TRAP_ARGS_ERROR_EXIT) # Logging settings logger = logging.getLogger("backscatter_classifier") logging.basicConfig( level=logging.DEBUG, filename=args.logfile, filemode='w', format= "[%(levelname)s], %(asctime)s, %(name)s, %(funcName)s, line %(lineno)d: %(message)s" ) # ASN and city databases try: geoip_db = Geoip2Wrapper(args.agp, args.cgp) except Exception as e: logger.error(e) logger.error("Error while create GeoIP2 wrapper") print(str(e), file=sys.stderr) sys.exit(EXIT_FAILURE) if args.export_to in ("c3isp", "c3isp-misp"): c3isp_upload.read_config(args.c3isp_config) if args.export_to == "misp": # MISP instance try: misp_instance = ExpandedPyMISP(args.url, args.key, args.ssl) except Exception as e: logger.error(e) logger.error("Error while creating MISP instance") print(str(e), file=sys.stderr) sys.exit(EXIT_FAILURE) # DDoS model ddos_model = pickle.load(args.model) ddos_classifier = DDoSClassifier(ddos_model, args.min_threshold) # *** MAIN PROCESSING LOOP *** while True: try: # Receive data try: data = trap.recv() except pytrap.FormatChanged as e: # Automatically detect format fmttype, fmtspec = trap.getDataFmt(0) rec = pytrap.UnirecTemplate(fmtspec) data = e.data if len(data) <= 1: # Terminating message break # Decode received data into python object rec.setData(data) # Only Ipv4 try: victim_ip = IPv4Address(rec.SRC_IP) except Exception as e: logger.info("Received IPv6 address, skipping") continue # Predict class of backscatter like traffic try: duration = DDoSClassifier.DURATION(rec) if duration < args.min_duration: # attack is too short lived to be reported continue if duration > args.max_duration: # attack is too long continue if rec.FLOW_COUNT < args.min_flows: # attack does not have enough flows continue for subnet in CESNET_NET: if victim_ip in subnet: continue ddos = ddos_classifier.predict(rec) except Exception as e: logger.error(e) continue # Report attack using MISP try: if ddos: try: domain = gethostbyaddr(str(victim_ip)) except herror as e: # Do not report for unknown domains continue event = create_ddos_event(rec, geoip_db, victim_ip, domain[0], args.misp_templates_dir) if args.export_to == "misp": try: event_id = misp_instance.add_event( event)['Event']['id'] misp_instance.publish(event_id) except Exception as e: logger.error(e) elif args.export_to in ("c3isp", "c3isp-misp"): logger.debug(f"Uploading event to C3ISP platform.") event_file_path = Path( "misp_event_to_c3isp.json").absolute() with temporary_open(event_file_path, 'w') as event_file: json.dump(event.to_json(), event_file) response = c3isp_upload.upload_to_c3isp( event_file_path) logger.debug(f"Response: {response}") if not response or ('status' in response and response['status'] == 'ERROR'): logger.error("ERROR during upload!") continue if args.export_to == "c3isp-misp": dpo_id = response['content'][ 'additionalProperties']['dposId'] logger.debug(f'Exporting DPO {dpo_id} to MISP.') c3isp_upload.export_misp(dpo_id, logger) except Exception as e: logger.error(str(e)) continue except Exception as e: # Log and re-raise exception logger.error(e) raise e # *** END OF MAIN PROCESSING LOOP *** trap.finalize()
from pymisp import ExpandedPyMISP, MISPEvent from pymisp import MISPObject from keys import misp_url, misp_key, misp_verifycert from datetime import date misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert) event = MISPEvent() event.info = 'IoT malware' # Event Title event.distribution = 1 # 0 = Your Organisation Only, 1 = Community event.threat_level_id = 2 # 1 = High, 2 = Medium, 3 = Low event.analysis = 2 # 0 (initial analysis), 1 (On-Going), 2 (Complete) event.add_tag('malware_classification:malware-category="Botnet"') event.add_tag('tlp:amber') d = date.today() event.set_date(d) attribute_second = event.add_attribute('url', 'http://1.2.3.4/example', disable_correlation=False, comment="Botnet example text", to_ids=False) event = misp.add_event(event, pythonify=True) # Publish event event.publish()
org_new.sector = "Government" org.id = api.add_organisation(org_new, pythonify=True).id # Create the MISP event by loading the JSON file # This will not add the attributes, but does add the event tags and galaxies # We also add a random UUID for uniqueness event = MISPEvent() event.load_file(json_import) event.uuid = event_import_uuid if not event_import_info: event_import_info = event.info event.info = event_import_info event.date = event_import_date event.distribution = event_import_distribution event.orgc = api.get_organisation(org, pythonify=True) event = api.add_event(event, pythonify=True) # Check if the event was created if (int(event.id) > 0): # Yes, read the content and add attributes and objects # Include a sleep so for the scheduler count_attributes = 0 count_objects = 0 with open(json_import) as json_file: data = json.load(json_file) if 'Attribute' in data.get("response")[0].get("Event"): attributes = data.get("response")[0].get("Event").get("Attribute") for attribute in attributes: misp_tag = [] if 'Tag' in attribute:
class MISPCaseManagement: def __init__( self, config_file='C:\\automation-hunting\\misp\\conf\\misp-case-provider.yaml', debug=False): self.misp_url = None self.api_key = None self.verify_cert = False if not self.get_config_data(config_file): raise Exception('Invalid Configuration File') self.misp_api = ExpandedPyMISP(self.misp_url, self.api_key, self.verify_cert, debug=debug) self.add_ioc_functions = [ self.misp_api.add_ipdst, self.misp_api.add_ipsrc, self.misp_api.add_internal_comment, self.misp_api.add_internal_text, self.misp_api.add_email_subject, self.misp_api.add_mutex, self.misp_api.add_filename, self.misp_api.add_hostname, self.misp_api.add_domain, self.misp_api.add_domain_ip, self.misp_api.add_email_src, self.misp_api.add_email_dst, self.misp_api.add_hashes, self.misp_api.add_object, self.misp_api.add_regkey, self.misp_api.add_url, self.misp_api.add_user, self.misp_api.add_useragent, self.misp_api.add_target_user ] def get_config_data(self, yaml_file): with open(yaml_file, 'r') as ymlfile: cfg = yaml.load(ymlfile, Loader=yaml.FullLoader) valid = False if self.validate_cfg_yml(cfg): self.misp_url = cfg['misp']['misp_url'] self.api_key = cfg['misp']['api_key'] self.verify_cert = cfg['misp']['verify_cert'] valid = True return valid @staticmethod def validate_cfg_yml(cfg): if 'misp' not in cfg: print('Not main') return False else: if 'misp_url' not in cfg['misp'] or 'api_key' not in cfg['misp']: return False return True def create_full_event( self, info, distribution: MISPDistribution = MISPDistribution.ORGANIZATION, threat_level: MISPThreatLevel = MISPThreatLevel.MEDIUM, analysis: MISPAnalysis = MISPAnalysis.INITIAL, attributes: list = None, tags: list = None): new_event = MISPEvent() new_event.distribution = distribution.value new_event.threat_level_id = threat_level.value new_event.analysis = analysis.value new_event.info = info if attributes is not None: new_event.Attribute = list() if tags is not None: new_event.Tag = list() event = self.misp_api.add_event(new_event) self.misp_api.get_all_tags() print(event.to_json()) return event def search_tag(self, tag_value, operator: StringOperator = StringOperator.EQ, sensitive=True, **kwargs): tags = self.misp_api.get_tags_list() ret_tags = list() for tag in tags: found = False if operator == StringOperator.EQ: if sensitive: if tag['name'] == tag_value: found = True else: if tag['name'].lower() == tag_value.lower(): found = True elif operator == StringOperator.CONTAINS: if sensitive: if tag_value in tag['name']: found = True else: if tag['name'].lower() == tag_value.lower(): found = True elif operator == StringOperator.REGEX: if sensitive: if re.search(tag_value, tag['name']) is not None: found = True else: if re.search(tag_value.lower(), tag['name'].lower()) is not None: found = True else: print('Unsupported') if found and len(kwargs) > 0: if 'exportable' in kwargs: if tag['exportable'] == kwargs['exportable']: found = True else: found = False if 'org_id' in kwargs: if tag['org_id'] == str(kwargs['org_id']): found = True else: found = False if 'user_id' in kwargs: if tag['user_id'] == str(kwargs['user_id']): found = True else: found = False if 'hide_tag' in kwargs: if tag['hide_tag'] == kwargs['hide_tag']: found = True else: found = False if found: ret_tags.append(tag) return ret_tags def create_tag(self, tag_data): new_tag = self.misp_api.new_tag(name=tag_data['name'], colour=tag_data['colour'], exportable=tag_data['exportable'], hide_tag=tag_data['hide_tag']) return new_tag def search_event(self, **kwargs): result = None if 'event_name' in kwargs: result = self.misp_api.search(eventinfo=kwargs['event_name']) elif 'event_id' in kwargs: result = self.misp_api.search(eventid=kwargs['event_id']) elif 'uuid' in kwargs: result = self.misp_api.search(uuid=kwargs['uuid']) else: print('Unsupported search filter') return result def add_ioc(self, event, data_type: MISPDataType, **kwargs): if data_type.value == 12: if 'domain' in kwargs and 'ip' in kwargs: self.add_ioc_functions[data_type.value](event, kwargs['domain'], kwargs['ip']) else: print('Missing parameters for: ' + data_type.name) elif data_type.value == 9: if 'md5' in kwargs: self.add_ioc_functions[data_type.value](event, md5=kwargs['md5']) if 'sha1' in kwargs: self.add_ioc_functions[data_type.value](event, sha1=kwargs['sha1']) if 'sha256' in kwargs: self.add_ioc_functions[data_type.value]( event, sha256=kwargs['sha256']) if 'ssdeep' in kwargs: self.add_ioc_functions[data_type.value]( event, ssdeep=kwargs['ssdeep']) else: if 'value' in kwargs: print('ADDING ATTRIBUTE') self.add_ioc_functions[data_type.value](event, kwargs['value']) else: print('Missing parameters for: ' + data_type.name)
class Output(cowrie.core.output.Output): """ MISP Upload Plugin for Cowrie. This Plugin creates a new event for unseen file uploads or adds sightings for previously seen files. The decision is done by searching for the SHA 256 sum in all matching attributes. """ @ignore_warnings def start(self): """ Start output plugin """ misp_url = CowrieConfig.get("output_misp", "base_url") misp_key = CowrieConfig.get("output_misp", "api_key") misp_verifycert = ("true" == CowrieConfig.get("output_misp", "verify_cert").lower()) self.misp_api = PyMISP(url=misp_url, key=misp_key, ssl=misp_verifycert, debug=False) self.debug = CowrieConfig.getboolean("output_misp", "debug", fallback=False) self.publish = CowrieConfig.getboolean("output_misp", "publish_event", fallback=False) def stop(self): """ Stop output plugin """ pass def write(self, entry): """ Push file download to MISP """ if entry["eventid"] == "cowrie.session.file_download": file_sha_attrib = self.find_attribute("sha256", entry["shasum"]) if file_sha_attrib: # file is known, add sighting! if self.debug: log.msg("File known, add sighting") self.add_sighting(entry, file_sha_attrib) else: # file is unknown, new event with upload if self.debug: log.msg("File unknwon, add new event") self.create_new_event(entry) @ignore_warnings def find_attribute(self, attribute_type, searchterm): """ Returns a matching attribute or None if nothing was found. """ result = self.misp_api.search(controller="attributes", type_attribute=attribute_type, value=searchterm) if result["Attribute"]: return result["Attribute"][0] else: return None @ignore_warnings def create_new_event(self, entry): attribute = MISPAttribute() attribute.type = "malware-sample" attribute.value = entry["shasum"] attribute.data = Path(entry["outfile"]) attribute.comment = "File uploaded to Cowrie ({})".format( entry["sensor"]) attribute.expand = "binary" event = MISPEvent() event.info = "File uploaded to Cowrie ({})".format(entry["sensor"]) event.attributes = [attribute] event.run_expansions() if self.publish: event.publish() result = self.misp_api.add_event(event) if self.debug: log.msg("Event creation result: \n%s" % result) @ignore_warnings def add_sighting(self, entry, attribute): sighting = MISPSighting() sighting.source = "{} (Cowrie)".format(entry["sensor"]) self.misp_api.add_sighting(sighting, attribute)
class MispConnector(): # TODO: Switch to ExpandedPyMISP misp = None tagsGenerated = False tags = None def __init__(self, url, apiKey): self.misp = ExpandedPyMISP(url, apiKey, MISP_VERIFYCERT, debug=False) self.tags = self.generate_misp_tags() def generate_misp_tags(self) -> List[MISPTag]: tagList = [] tags = [ { 'name': 'AutoGenerated', 'colour': '#00ace6', 'exportable': True }, { 'name': 'HoneytrapEvent', 'colour': '#581845', 'exportable': True }, { 'name': 'ModSecurity', 'colour': '#a04000', 'exportable': True }, ] try: for item in tags: tag = MISPTag() for key in item: tag[key] = item[key] self.misp.add_tag(tag, pythonify=True) tagList.append(tag) log.info("MISP tags generated") return tagList except PyMISPError as e: raise RuntimeError("Failed to initialise MISP tags") def send_misp_event(self, json_log): logEvent = ModsecLog(json_log) event = self.generate_event(logEvent) self.misp.add_event(event) log.debug(event) def generate_event(self, modsecLog: ModsecLog) -> MISPEvent: event = MISPEvent() event.info = modsecLog.generateInfoLine() [http_method, url, http_version] = modsecLog.getRequestLine() event.add_attribute( type='ip-src|port', value=str(modsecLog.log['transaction']['remote_address']) + "|" + str(modsecLog.log['transaction']['remote_port']), comment="Attacker", pythonify=True) event.add_attribute( type='ip-dst|port', value=str(modsecLog.log['transaction']['local_address']) + "|" + str(modsecLog.log['transaction']['local_port']), comment="Server", pythonify=True) event.add_attribute(type='http-method', value=http_method, pythonify=True) event.add_attribute(type='url', value=url, pythonify=True) event.add_attribute(type='datetime', value=modsecLog.log['@timestamp'], pythonify=True) event.add_attribute(type='other', value=modsecLog.log['audit_data']['producer'], comment="Producer", pythonify=True) event.add_attribute(type='text', value=json.dumps(modsecLog.log, indent=2), comment="Json log", pythonify=True) #event.add_attribute(type='vulnerability', value=json_log['transaction']['time'], pythonify=True) for tag in self.tags: event.add_tag(tag) log.debug("elasticsearch data") log.debug(json.dumps(modsecLog.log, indent=2)) return event
class MISPApi: def __init__(self, api_url: str, api_key: str, verify_cert: bool = True): self.pymisp = ExpandedPyMISP(api_url, api_key, verify_cert) def search(self, controller: str = 'attributes', **kwargs): return self.pymisp.search(controller, **kwargs) def searchall(self, value: str, controller: str = 'attributes') -> List[Any]: result = self.pymisp.search(controller, value=value, searchall=True) assert isinstance(result, dict) # Please mypy return result.get('Attribute', []) def search_sightings(self, context_id: str, context: str = 'attribute', source: Optional[str] = None): return self.pymisp.search_sightings(context=context, context_id=context_id, source=source) def org_name_id_mapping(self): pass def attr_search(self, attr: Attr) -> List[Any]: result = [] for typ in attr.search_types: result += self.search(type=typ.value, value=attr.value).get('Attribute', []) return result def domain_name_search( self, domain_name: str, searchall: bool = False, publish_timestamp: Optional[datetime] = None, limit: Optional[int] = None, ) -> List[Any]: result = self.search(type='domain', value=domain_name, searchall=searchall, publish_timestamp=publish_timestamp, limit=limit) return result.get('Attribute', []) def url_search(self, url: str, searchall: bool = False) -> List[Any]: result = self.search(type='url', value=url, searchall=searchall) return result.get('Attribute', []) def sighting_lookup(self, attribute_id: str, source: Optional[str] = None) -> List[Any]: result = self.search_sightings(context_id=attribute_id, source=source) return [item['Sighting'] for item in result] def _get_report_category(self, attr: Attr) -> EventCategory: if any(True for t in attr.report_types if t in [AttrType.DOMAIN, AttrType.URL, AttrType.IP_SRC]): return EventCategory.NETWORK_ACTIVITY elif any(True for t in attr.report_types if t in [AttrType.MD5, AttrType.SHA1]): return EventCategory.PAYLOAD_DELIVERY else: raise NotImplementedError( f'EventCategory for {attr.report_types} not implemented') def add_event( self, attr_items: List[Attr], info: str, tags: List, comment: str, to_ids: bool, reference: Optional[str], ts: Optional[int] = None, published: Optional[bool] = False, ): attrs = [] for item in attr_items: category = self._get_report_category(item) for report_type in item.report_types: report_attr = MISPAttribute() report_attr.from_dict( type=report_type.value, category=category.value, to_ids=to_ids, value=item.value, comment=comment, timestamp=ts, ) attrs.append(report_attr) if reference: reference_attr = MISPAttribute() reference_attr.from_dict( type='text', category=EventCategory.INTERNAL_REFERENCE.value, value=reference, disable_correlation=True, comment=comment, timestamp=ts, ) attrs.append(reference_attr) event = MISPEvent() event.from_dict(info=info, Attribute=attrs, Tag=tags, date=date.today(), published=published, threat_level_id=2) logger.debug(event) return self.pymisp.add_event(event) def add_sighting(self, attr: Attr, sighting_type: str, source: str) -> MISPSighting: sighting = MISPSighting() sighting['value'] = attr.value sighting['type'] = sighting_type sighting['source'] = source res = self.pymisp.add_sighting(sighting, pythonify=True) assert isinstance(res, MISPSighting) # please mypy return res def remove_sighting( self, attr: Attr, sighting_type: str, source: str, date_from: Optional[datetime] = None, date_to: Optional[datetime] = None, ): sightings: List[MISPSighting] = [] for item in self.attr_search(attr): res = self.pymisp.search_sightings( context='attribute', context_id=item['id'], type_sighting=sighting_type, source=source, date_from=date_from, date_to=date_to, pythonify=True, ) # Can't get mypy to understand that sighting contains a MISPSighting sightings.extend([ d['sighting'] for d in res if d.get('sighting') is not None ]) # type: ignore # Please mypy for sighting in sightings: self.pymisp.delete_sighting(sighting)