def dotransform(request, response): emailaddr = [] msgfile = request.value lookFor = ['To', 'From'] tmpfolder = request.fields['sniffMyPackets.outputfld'] with open(msgfile, mode='r') as msgfile: reader = msgfile.read() reader = str(reader) for x in lookFor: if x in reader: for s in re.finditer('RCPT TO: <([\w.-]+@[\w.-]+)>', reader): to_addr = s.group(1), 'mail_to' emailaddr.append(to_addr) for t in re.finditer('MAIL FROM: <([\w.-]+@[\w.-]+)>', reader): from_addr = t.group(1), 'mail_from' emailaddr.append(from_addr) for addr, addrfield in emailaddr: e = EmailAddress(addr) e.linklabel = addrfield e += Field('filelocation', request.value, displayname='File Location', matchingrule='loose') e += Field('emailaddr', addrfield, displayname='Header Info') response += e return response
def do_transform(self, request, response, config): person = request.entity fields = person.fields if fields.get("properties.url"): url = fields.get("properties.url").value else: url = None soup = scrape(url) if soup: email_addresses = soup.find_all(attrs={"class": "__cf_email__"}) for email_address in email_addresses: fp = email_address['data-cfemail'] r = int(fp[:2], 16) email = ''.join([ chr(int(fp[i:i + 2], 16) ^ r) for i in range(2, len(fp), 2) ]) response += EmailAddress(email) phone_numbers = soup.find_all(attrs={"data-link-to-more": "phone"}) for phone_number in phone_numbers: response += PhoneNumber(phone_number.get_text()) return response
def dotransform(request, response, config): error, found = lookup_whois(request.value) if not error and found: if dict == type(found): for result, value in found.iteritems(): if set == type(value): if "whois_domains" == result: for d in value: if d: e = Domain(d) e.fqdn = d response += e if "whois_emails" == result: for em in value: if em: e = EmailAddress(em) response += e if "whois_nameservers" == result: for w in value: if w: e = NSRecord(w) response += e #Display error message in Transform Output response += UIMessage(error) return response
def do_transform(self, request, response, config): person = request.entity fields = person.fields if fields.get("properties.url"): url = fields.get("properties.url").value else: url = None soup = scrape(url) if soup: email_addresses = soup.find_all(attrs={"class": "__cf_email__"}) for email_address in email_addresses: fp = email_address['data-cfemail'] r = int(fp[:2], 16) email = ''.join([ chr(int(fp[i:i + 2], 16) ^ r) for i in range(2, len(fp), 2) ]) response += EmailAddress(email) phone_numbers = soup.find_all(attrs={"data-link-to-more": "phone"}) for phone_number in phone_numbers: response += PhoneNumber(phone_number.get_text()) addresses = soup.find_all(attrs={"data-link-to-more": "address"}) for unformated_address in addresses: address = unformated_address.get_text().split(" ") address[-1] = address[-1].split("-")[0] response += Location(" ".join(address)) relatives = soup.find_all(attrs={"data-link-to-more": "relative"}) for relative in relatives: response += TruePerson( relative.get_text(), properties_url=config['TruePeopleSearch.local.base_url'] + relative['href']) associates = soup.find_all( attrs={"data-link-to-more": "associate"}) for associate in associates: response += TruePerson( associate.get_text(), properties_url=config['TruePeopleSearch.local.base_url'] + associate['href']) return response
def dotransform(request, response): pcap = request.value lookfor = ['MAIL FROM:', 'RCPT TO:'] pkts = rdpcap(pcap) usedb = config['working/usedb'] # Check to see if we are using the database or not if usedb > 0: d = mongo_connect() c = d['CREDS'] # Hash the pcap file try: md5pcap = md5_for_file(pcap) except Exception as e: return response + UIMessage(str(e)) x = find_session(md5pcap) pcap_id = x[0] else: pass addr = [] try: for p in pkts: for m in lookfor: if p.haslayer(TCP) and p.haslayer(Raw): raw = p[Raw].load if m in raw: for s in re.finditer('<([\S.-]+@[\S-]+)>', raw): addr.append(s.group(1)) except Exception as e: return response + UIMessage(str(e)) for x in addr: if usedb > 0: data = {'PCAP ID': pcap_id, 'Type': 'Email Address', 'Record': x} t = d.CREDS.find({'Record': x}).count() if t > 0: pass else: c.insert(data) else: pass e = EmailAddress(x) response += e return response
def detType(in_val): val = str(in_val) #::Email email = re.compile(".*\[@\][a-z0-9\-]{1,}\.[a-z0-9\-]{1,}") #::IP ipv4 = re.compile("^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$") #::CIDR cidr = re.compile("^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/\d{1,2}$") #::Range v4range = re.compile( "^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\-\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$" ) #::Domain dom = re.compile("([a-z0-9\-]{1,}\.?)+\.[a-z0-9\-]{1,}$") if email.match(val): e = EmailAddress(val) return e if ipv4.match(val): e = IPv4Address(val) return e if cidr.match(val): e = CIDR(val) return e if v4range.match(val): e = Range(val) return e if dom.match(val): e = Domain(val) return e if re.match("^([a-z]*)://", val, re.M | re.I): e = URL(val) e.url = val return e
def dotransform(request, response, config): if 'workspace' in request.fields: workspace = request.fields['workspace'] else: workspace = request.value dbcon = db_connect(workspace) contact_list = get_contacts(dbcon) for email in contact_list: if email[0] == request.fields["fname"] and email[1] == request.fields[ "lname"]: e = EmailAddress(email[2]) e += Field("workspace", workspace, displayname='Workspace') e += Field("fullname", email[0] + ' ' + email[1], displayname='Fullname') response += e else: pass return response
def dotransform(request, response, config): tr_details = ['Reference', 'Source', 'KillChain', 'Firstseen', 'Lastseen', 'Attribution', 'ProcessType', 'Rrname', 'Rdata', 'Country', 'Tags', 'Comment', 'RootNode', 'Confidence'] #Disable cache to get actual data from Threat Recon cache, found = search(request.value, cache=False) #Default linkcolor linkcolor = "0x000000" if found: if defaultdict == type(found): for rootnode, value in found.iteritems(): #If the RootNode is empty, display attributes if len(rootnode) == 0: for indicator in value: #debug(indicator) e = '' indtype = indicator['Type'].lower().strip() if "whois email" == indtype: e = EmailAddress(indicator['Indicator']) if "name server" == indtype: e = NSRecord(indicator['Indicator']) if "domain" == indtype: e = Domain(indicator['Indicator']) e.fqdn = indicator['Indicator'] if "ip" == indtype: e = IPv4Address(indicator['Indicator']) if "phone or fax no." == indtype: e = PhoneNumber(indicator['Indicator']) if "whois address component" == indtype: e = Phrase(indicator['Indicator']) if "email" == indtype: e = EmailAddress(indicator['Indicator']) if "netname" == indtype: e = NetNameThreatRecon(indicator['Indicator']) if "cidr" == indtype: e = IPv4Address(indicator['Indicator']) if "netrange" == indtype: e = Netblock(indicator['Indicator']) if e: #Set linkcolor e.linkcolor = linkcolor #Set comments if indicator['Comment']: e.notes = string_filter(indicator['Comment']) #Set Details for detail in tr_details: if detail in indicator: if indicator[detail]: e += Label(name=detail, value=string_filter(indicator[detail])) response += e else: #Display the RootNodes e = ThreatRecon(rootnode) response += e return response
def dotransform(request, response, config): tr_details = [ 'Reference', 'Source', 'KillChain', 'Firstseen', 'Lastseen', 'Attribution', 'ProcessType', 'Rrname', 'Rdata', 'Country', 'Tags', 'Comment', 'RootNode', 'Confidence' ] #Default link color is black linkcolor = "0x000000" cache, found = search(request.value) if found: if list == type(found): for indicator in found: debug(indicator) e = '' indtype = indicator['Type'].lower().strip() if "whois email" == indtype: e = EmailAddress(indicator['Indicator']) #response += e if "name server" == indtype: e = NSRecord(indicator['Indicator']) #response += e if "domain" == indtype: e = Domain(indicator['Indicator']) e.fqdn = indicator['Indicator'] #response += e #IF Type is not domain, check if Rrname is not empty elif indicator['Rrname'] and indicator['Rrname'] != 'NA': d = Domain(indicator['Rrname']) d.fqdn = indicator['Rrname'] response += d if "ip" == indtype: e = IPv4Address(indicator['Indicator']) #response += e #IF Type is not IP, check if Rdata is not empty elif indicator['Rdata']: i = IPv4Address(indicator['Rdata']) response += i if "phone or fax no." == indtype: e = PhoneNumber(indicator['Indicator']) #response += e if "whois address component" == indtype: e = Phrase(indicator['Indicator']) #response += e if "email" == indtype: e = EmailAddress(indicator['Indicator']) #response += e if "netname" == indtype: e = NetNameThreatRecon(indicator['Indicator']) #response += e if "cidr" == indtype: e = IPv4Address(indicator['Indicator']) #response += e if "netrange" == indtype: e = Netblock(indicator['Indicator']) #response += e if indicator['Country']: l = Location(indicator['Country']) response += l #Add Comments and details to own Entity entity = e #request.entity #Set comments if indicator['Comment']: entity.notes = string_filter(indicator['Comment']) #Set Details for detail in tr_details: if detail in indicator: if indicator[detail]: entity += Label(name=detail, value=string_filter( indicator[detail])) #Set link color if "Confidence" in indicator: if indicator['Confidence'] >= 70: linkcolor = "0xff0000" entity.linkcolor = linkcolor response += entity return response
def dotransform(request, response, config): try: actor = get_actor(request.fields['ThreatCentral.resourceId']) except ThreatCentralError as err: response += UIMessage(err.value, type='PartialError') return response except KeyError: try: actors = search_actor(request.value) except ThreatCentralError as err: response += UIMessage(err.value, type='PartialError') return response else: i = 0 for actor in actors: try: rtype = lower(actor.get('type')) actor = actor.get('resource') if actor.get('tcScore'): weight = int(actor.get('tcScore')) else: weight = 1 if len(actor) is not 0: if rtype == 'actor': if actor.get('name'): e = Actor(encode_to_utf8(actor.get('name')), weight=weight) e.name = encode_to_utf8(actor.get('name')) e.actor = encode_to_utf8(actor.get('name')) elif actor.get('title'): e = Actor(encode_to_utf8(actor.get('title'))) e.title = encode_to_utf8(actor.get('title')) e.resourceId = actor.get('resourceId') if actor.get('organization'): e.organization = encode_to_utf8( actor.get('organization')) if actor.get('aliases'): e.aliases = ', '.join([ encode_to_utf8(_) for _ in actor.get('aliases') ]) if actor.get('country'): e.country = encode_to_utf8( actor.get('country', dict()).get('displayName')) if actor.get('score'): e.score = actor.get('score') if actor.get('links'): e += Label( 'Links', '<br/>'.join([ '<a href="{}">{}</a>'.format( _.get('href'), _.get('href')) for _ in actor.get('links') ])) if actor.get('hyperlinks'): e += Label( 'Hyperlinks', '<br/>'.join([ '<a href="{}">{}</a>'.format( _.get('url'), _.get('title')) for _ in actor.get('hyperlinks') ])) if actor.get('title'): e += Label('Title', encode_to_utf8(actor.get('title'))) if actor.get('resourceId'): e += Label('ResourceID', actor.get('resourceId')) if actor.get('aliases'): e += Label( 'Aliases', '<br/>'.join([ encode_to_utf8(_) for _ in actor.get('aliases', '') ])) if actor.get('description'): e += Label( 'Description', '<br/>'.join( encode_to_utf8( actor.get('description', '')).split('\n'))) if actor.get('country'): e += Label( 'Country', encode_to_utf8( actor.get('country', dict()).get('displayName'))) if actor.get('organization'): e += Label( 'Organization', encode_to_utf8(actor.get('organization'))) if actor.get('types'): e += Label( 'Types', '<br/>'.join([ encode_to_utf8(_.get('displayName')) for _ in actor.get('types') ])) if actor.get('motivations'): e += Label( 'Motivations', '<br/>'.join([ encode_to_utf8(_.get('displayName')) for _ in actor.get('motivations') ])) if actor.get('intendedEffects'): e += Label( 'Intended Effects', '<br/>'.join([ encode_to_utf8(_.get('displayName')) for _ in actor.get('intendedEffects') ])) if actor.get('sophistication'): e += Label( 'Sophistication', actor.get('sophistication', dict()).get('displayName')) if actor.get('socialMediaText'): e += Label( 'Social Media', '<br/>'.join( encode_to_utf8( actor.get('socialMediaText', '')).split('\n'))) if actor.get('moreInfo'): e += Label( 'More Info', '<br/>'.join( encode_to_utf8( actor.get('moreInfo', '')).split('\n'))) if actor.get('score'): e += Label('Score', actor.get('score')) if i < 1: i += 1 e.linkcolor = "0xf90000" response += e except AttributeError as err: response += UIMessage(err, type='PartialError') continue except ThreatCentralError as err: response += UIMessage(err.value, type='PartialError') except TypeError: return response else: if actor: try: if actor.get('tcScore'): weight = int(actor.get('tcScore')) else: weight = 1 # Update entity? e = Actor(request.value, weight=weight) if actor.get('name'): e.name = encode_to_utf8(actor.get('name')) e.actor = encode_to_utf8(actor.get('name')) e.title = encode_to_utf8(actor.get('title')) e.resourceId = actor.get('resourceId') if actor.get('organization'): e.organization = encode_to_utf8(actor.get('organization')) if actor.get('aliases'): e.aliases = ', '.join( [encode_to_utf8(_) for _ in actor.get('aliases')]) if actor.get('country'): e.country = encode_to_utf8( actor.get('country', dict()).get('displayName')) # Add Location entitie l = Location( encode_to_utf8( actor.get('country', dict()).get('displayName'))) response += l if actor.get('score'): e.score = actor.get('score') if actor.get('links'): e += Label( 'Links', '<br/>'.join([ '<a href="{}">{}</a>'.format( _.get('href'), _.get('href')) for _ in actor.get('links') ])) if actor.get('hyperlinks'): e += Label( 'Hyperlinks', '<br/>'.join([ '<a href="{}">{}</a>'.format( _.get('url'), _.get('title')) for _ in actor.get('hyperlinks') ])) if actor.get('title'): e += Label('Title', encode_to_utf8(actor.get('title'))) if actor.get('resourceId'): e += Label('ResourceID', actor.get('resourceId')) if actor.get('aliases'): e += Label( 'Aliases', '<br/>'.join([ encode_to_utf8(_) for _ in actor.get('aliases', '') ])) if actor.get('description'): e += Label( 'Description', '<br/>'.join( encode_to_utf8(actor.get('description', '')).split('\n'))) if actor.get('country'): e += Label( 'Country', encode_to_utf8( actor.get('country', dict()).get('displayName'))) if actor.get('organization'): e += Label('Organization', encode_to_utf8(actor.get('organization'))) if actor.get('types'): e += Label( 'Types', '<br/>'.join([ encode_to_utf8(_.get('displayName')) for _ in actor.get('types') ])) if actor.get('motivations'): e += Label( 'Motivations', '<br/>'.join([ encode_to_utf8(_.get('displayName')) for _ in actor.get('motivations') ])) if actor.get('intendedEffects'): e += Label( 'Intended Effects', '<br/>'.join([ encode_to_utf8(_.get('displayName')) for _ in actor.get('intendedEffects') ])) if actor.get('sophistication'): e += Label( 'Sophistication', encode_to_utf8( actor.get('sophistication', dict()).get('displayName'))) if actor.get('socialMediaText'): e += Label( 'Social Media', '<br/>'.join( encode_to_utf8(actor.get('socialMediaText', '')).split('\n'))) if actor.get('moreInfo'): e += Label( 'More Info', '<br/>'.join( encode_to_utf8(actor.get('moreInfo', '')).split('\n'))) if actor.get('score'): e += Label('Score', actor.get('score')) response += e # Extract email addresses usable_info = search_for_usable_info('{} {} {}'.format( encode_to_utf8(actor.get('description')), encode_to_utf8(actor.get('socialMediaText')), encode_to_utf8(actor.get('moreInfo')))) if usable_info: debug(usable_info) try: urls = usable_info.get('url', dict()) for twitter in urls.get('twitter', list()): t = Twitter(twitter.get('name')) t.uid = twitter.get('name') t.set_field('affiliation.profile-url', twitter.get('url')) response += t for facebook in urls.get('facebook', list()): f = Facebook(facebook.get('name')) f.uid = facebook.get('name') f.set_field('affiliation.profile-url', facebook.get('url')) response += f for other in urls.get('other', list()): u = URL(other) u.url = other response += u emailaddr = usable_info.get('email', list()) for email in emailaddr: e = EmailAddress(email) response += e except AttributeError as err: response += UIMessage('Error: {}'.format(err)) except AttributeError as err: response += UIMessage('Error: {}'.format(err), type='PartialError') except ThreatCentralError as err: response += UIMessage(err.value, type='PartialError') except TypeError: return response return response
def dotransform(request, response, config): tr_details = [ 'Reference', 'Source', 'KillChain', 'Firstseen', 'Lastseen', 'Attribution', 'ProcessType', 'Rrname', 'Rdata', 'Country', 'Tags', 'Comment', 'RootNode', 'Confidence' ] #Disable cache to get actual data from Threat Recon cache, found = search(request.value, cache=False) #Default linkcolor linkcolor = "0x000000" if found: if defaultdict == type(found): for rootnode, value in found.iteritems(): #If the RootNode is empty, display attributes if len(rootnode) == 0: for indicator in value: #debug(indicator) e = '' indtype = indicator['Type'].lower().strip() if "whois email" == indtype: e = EmailAddress(indicator['Indicator']) if "name server" == indtype: e = NSRecord(indicator['Indicator']) if "domain" == indtype: e = Domain(indicator['Indicator']) e.fqdn = indicator['Indicator'] if "ip" == indtype: e = IPv4Address(indicator['Indicator']) if "phone or fax no." == indtype: e = PhoneNumber(indicator['Indicator']) if "whois address component" == indtype: e = Phrase(indicator['Indicator']) if "email" == indtype: e = EmailAddress(indicator['Indicator']) if "netname" == indtype: e = NetNameThreatRecon(indicator['Indicator']) if "cidr" == indtype: e = IPv4Address(indicator['Indicator']) if "netrange" == indtype: e = Netblock(indicator['Indicator']) if e: #Set linkcolor e.linkcolor = linkcolor #Set comments if indicator['Comment']: e.notes = string_filter(indicator['Comment']) #Set Details for detail in tr_details: if detail in indicator: if indicator[detail]: e += Label(name=detail, value=string_filter( indicator[detail])) response += e else: #Display the RootNodes e = ThreatRecon(rootnode) response += e return response