def dotransform(request, response):
emailaddr = []
msgfile = request.value
lookFor = ['To', 'From']
tmpfolder = request.fields['sniffMyPackets.outputfld']
with open(msgfile, mode='r') as msgfile:
reader = msgfile.read()
reader = str(reader)
for x in lookFor:
if x in reader:
for s in re.finditer('RCPT TO: <([\w.-]+@[\w.-]+)>', reader):
to_addr = s.group(1), 'mail_to'
emailaddr.append(to_addr)
for t in re.finditer('MAIL FROM: <([\w.-]+@[\w.-]+)>', reader):
from_addr = t.group(1), 'mail_from'
emailaddr.append(from_addr)
for addr, addrfield in emailaddr:
e = EmailAddress(addr)
e.linklabel = addrfield
e += Field('filelocation', request.value, displayname='File Location', matchingrule='loose')
e += Field('emailaddr', addrfield, displayname='Header Info')
response += e
return response
def dotransform(request, response):
emailaddr = []
msgfile = request.value
lookFor = ['To', 'From']
tmpfolder = request.fields['sniffMyPackets.outputfld']
with open(msgfile, mode='r') as msgfile:
reader = msgfile.read()
reader = str(reader)
for x in lookFor:
if x in reader:
for s in re.finditer('RCPT TO: <([\w.-]+@[\w.-]+)>', reader):
to_addr = s.group(1), 'mail_to'
emailaddr.append(to_addr)
for t in re.finditer('MAIL FROM: <([\w.-]+@[\w.-]+)>', reader):
from_addr = t.group(1), 'mail_from'
emailaddr.append(from_addr)
for addr, addrfield in emailaddr:
e = EmailAddress(addr)
e.linklabel = addrfield
e += Field('filelocation',
request.value,
displayname='File Location',
matchingrule='loose')
e += Field('emailaddr', addrfield, displayname='Header Info')
response += e
return response
def do_transform(self, request, response, config):
person = request.entity
fields = person.fields
if fields.get("properties.url"):
url = fields.get("properties.url").value
else:
url = None
soup = scrape(url)
if soup:
email_addresses = soup.find_all(attrs={"class": "__cf_email__"})
for email_address in email_addresses:
fp = email_address['data-cfemail']
r = int(fp[:2], 16)
email = ''.join([
chr(int(fp[i:i + 2], 16) ^ r)
for i in range(2, len(fp), 2)
])
response += EmailAddress(email)
phone_numbers = soup.find_all(attrs={"data-link-to-more": "phone"})
for phone_number in phone_numbers:
response += PhoneNumber(phone_number.get_text())
return response
def dotransform(request, response, config):
error, found = lookup_whois(request.value)
if not error and found:
if dict == type(found):
for result, value in found.iteritems():
if set == type(value):
if "whois_domains" == result:
for d in value:
if d:
e = Domain(d)
e.fqdn = d
response += e
if "whois_emails" == result:
for em in value:
if em:
e = EmailAddress(em)
response += e
if "whois_nameservers" == result:
for w in value:
if w:
e = NSRecord(w)
response += e
#Display error message in Transform Output
response += UIMessage(error)
return response
def do_transform(self, request, response, config):
person = request.entity
fields = person.fields
if fields.get("properties.url"):
url = fields.get("properties.url").value
else:
url = None
soup = scrape(url)
if soup:
email_addresses = soup.find_all(attrs={"class": "__cf_email__"})
for email_address in email_addresses:
fp = email_address['data-cfemail']
r = int(fp[:2], 16)
email = ''.join([
chr(int(fp[i:i + 2], 16) ^ r)
for i in range(2, len(fp), 2)
])
response += EmailAddress(email)
phone_numbers = soup.find_all(attrs={"data-link-to-more": "phone"})
for phone_number in phone_numbers:
response += PhoneNumber(phone_number.get_text())
addresses = soup.find_all(attrs={"data-link-to-more": "address"})
for unformated_address in addresses:
address = unformated_address.get_text().split(" ")
address[-1] = address[-1].split("-")[0]
response += Location(" ".join(address))
relatives = soup.find_all(attrs={"data-link-to-more": "relative"})
for relative in relatives:
response += TruePerson(
relative.get_text(),
properties_url=config['TruePeopleSearch.local.base_url'] +
relative['href'])
associates = soup.find_all(
attrs={"data-link-to-more": "associate"})
for associate in associates:
response += TruePerson(
associate.get_text(),
properties_url=config['TruePeopleSearch.local.base_url'] +
associate['href'])
return response
def dotransform(request, response):
pcap = request.value
lookfor = ['MAIL FROM:', 'RCPT TO:']
pkts = rdpcap(pcap)
usedb = config['working/usedb']
# Check to see if we are using the database or not
if usedb > 0:
d = mongo_connect()
c = d['CREDS']
# Hash the pcap file
try:
md5pcap = md5_for_file(pcap)
except Exception as e:
return response + UIMessage(str(e))
x = find_session(md5pcap)
pcap_id = x[0]
else:
pass
addr = []
try:
for p in pkts:
for m in lookfor:
if p.haslayer(TCP) and p.haslayer(Raw):
raw = p[Raw].load
if m in raw:
for s in re.finditer('<([\S.-]+@[\S-]+)>', raw):
addr.append(s.group(1))
except Exception as e:
return response + UIMessage(str(e))
for x in addr:
if usedb > 0:
data = {'PCAP ID': pcap_id, 'Type': 'Email Address', 'Record': x}
t = d.CREDS.find({'Record': x}).count()
if t > 0:
pass
else:
c.insert(data)
else:
pass
e = EmailAddress(x)
response += e
return response
def detType(in_val):
val = str(in_val)
#::Email
email = re.compile(".*\[@\][a-z0-9\-]{1,}\.[a-z0-9\-]{1,}")
#::IP
ipv4 = re.compile("^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$")
#::CIDR
cidr = re.compile("^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/\d{1,2}$")
#::Range
v4range = re.compile(
"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\-\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$"
)
#::Domain
dom = re.compile("([a-z0-9\-]{1,}\.?)+\.[a-z0-9\-]{1,}$")
if email.match(val):
e = EmailAddress(val)
return e
if ipv4.match(val):
e = IPv4Address(val)
return e
if cidr.match(val):
e = CIDR(val)
return e
if v4range.match(val):
e = Range(val)
return e
if dom.match(val):
e = Domain(val)
return e
if re.match("^([a-z]*)://", val, re.M | re.I):
e = URL(val)
e.url = val
return e
def dotransform(request, response, config):
if 'workspace' in request.fields:
workspace = request.fields['workspace']
else:
workspace = request.value
dbcon = db_connect(workspace)
contact_list = get_contacts(dbcon)
for email in contact_list:
if email[0] == request.fields["fname"] and email[1] == request.fields[
"lname"]:
e = EmailAddress(email[2])
e += Field("workspace", workspace, displayname='Workspace')
e += Field("fullname",
email[0] + ' ' + email[1],
displayname='Fullname')
response += e
else:
pass
return response
def dotransform(request, response, config):
tr_details = ['Reference', 'Source', 'KillChain', 'Firstseen', 'Lastseen', 'Attribution',
'ProcessType', 'Rrname', 'Rdata', 'Country', 'Tags', 'Comment',
'RootNode', 'Confidence']
#Disable cache to get actual data from Threat Recon
cache, found = search(request.value, cache=False)
#Default linkcolor
linkcolor = "0x000000"
if found:
if defaultdict == type(found):
for rootnode, value in found.iteritems():
#If the RootNode is empty, display attributes
if len(rootnode) == 0:
for indicator in value:
#debug(indicator)
e = ''
indtype = indicator['Type'].lower().strip()
if "whois email" == indtype:
e = EmailAddress(indicator['Indicator'])
if "name server" == indtype:
e = NSRecord(indicator['Indicator'])
if "domain" == indtype:
e = Domain(indicator['Indicator'])
e.fqdn = indicator['Indicator']
if "ip" == indtype:
e = IPv4Address(indicator['Indicator'])
if "phone or fax no." == indtype:
e = PhoneNumber(indicator['Indicator'])
if "whois address component" == indtype:
e = Phrase(indicator['Indicator'])
if "email" == indtype:
e = EmailAddress(indicator['Indicator'])
if "netname" == indtype:
e = NetNameThreatRecon(indicator['Indicator'])
if "cidr" == indtype:
e = IPv4Address(indicator['Indicator'])
if "netrange" == indtype:
e = Netblock(indicator['Indicator'])
if e:
#Set linkcolor
e.linkcolor = linkcolor
#Set comments
if indicator['Comment']:
e.notes = string_filter(indicator['Comment'])
#Set Details
for detail in tr_details:
if detail in indicator:
if indicator[detail]:
e += Label(name=detail, value=string_filter(indicator[detail]))
response += e
else:
#Display the RootNodes
e = ThreatRecon(rootnode)
response += e
return response
def dotransform(request, response, config):
tr_details = [
'Reference', 'Source', 'KillChain', 'Firstseen', 'Lastseen',
'Attribution', 'ProcessType', 'Rrname', 'Rdata', 'Country', 'Tags',
'Comment', 'RootNode', 'Confidence'
]
#Default link color is black
linkcolor = "0x000000"
cache, found = search(request.value)
if found:
if list == type(found):
for indicator in found:
debug(indicator)
e = ''
indtype = indicator['Type'].lower().strip()
if "whois email" == indtype:
e = EmailAddress(indicator['Indicator'])
#response += e
if "name server" == indtype:
e = NSRecord(indicator['Indicator'])
#response += e
if "domain" == indtype:
e = Domain(indicator['Indicator'])
e.fqdn = indicator['Indicator']
#response += e
#IF Type is not domain, check if Rrname is not empty
elif indicator['Rrname'] and indicator['Rrname'] != 'NA':
d = Domain(indicator['Rrname'])
d.fqdn = indicator['Rrname']
response += d
if "ip" == indtype:
e = IPv4Address(indicator['Indicator'])
#response += e
#IF Type is not IP, check if Rdata is not empty
elif indicator['Rdata']:
i = IPv4Address(indicator['Rdata'])
response += i
if "phone or fax no." == indtype:
e = PhoneNumber(indicator['Indicator'])
#response += e
if "whois address component" == indtype:
e = Phrase(indicator['Indicator'])
#response += e
if "email" == indtype:
e = EmailAddress(indicator['Indicator'])
#response += e
if "netname" == indtype:
e = NetNameThreatRecon(indicator['Indicator'])
#response += e
if "cidr" == indtype:
e = IPv4Address(indicator['Indicator'])
#response += e
if "netrange" == indtype:
e = Netblock(indicator['Indicator'])
#response += e
if indicator['Country']:
l = Location(indicator['Country'])
response += l
#Add Comments and details to own Entity
entity = e #request.entity
#Set comments
if indicator['Comment']:
entity.notes = string_filter(indicator['Comment'])
#Set Details
for detail in tr_details:
if detail in indicator:
if indicator[detail]:
entity += Label(name=detail,
value=string_filter(
indicator[detail]))
#Set link color
if "Confidence" in indicator:
if indicator['Confidence'] >= 70:
linkcolor = "0xff0000"
entity.linkcolor = linkcolor
response += entity
return response
def dotransform(request, response, config):
try:
actor = get_actor(request.fields['ThreatCentral.resourceId'])
except ThreatCentralError as err:
response += UIMessage(err.value, type='PartialError')
return response
except KeyError:
try:
actors = search_actor(request.value)
except ThreatCentralError as err:
response += UIMessage(err.value, type='PartialError')
return response
else:
i = 0
for actor in actors:
try:
rtype = lower(actor.get('type'))
actor = actor.get('resource')
if actor.get('tcScore'):
weight = int(actor.get('tcScore'))
else:
weight = 1
if len(actor) is not 0:
if rtype == 'actor':
if actor.get('name'):
e = Actor(encode_to_utf8(actor.get('name')),
weight=weight)
e.name = encode_to_utf8(actor.get('name'))
e.actor = encode_to_utf8(actor.get('name'))
elif actor.get('title'):
e = Actor(encode_to_utf8(actor.get('title')))
e.title = encode_to_utf8(actor.get('title'))
e.resourceId = actor.get('resourceId')
if actor.get('organization'):
e.organization = encode_to_utf8(
actor.get('organization'))
if actor.get('aliases'):
e.aliases = ', '.join([
encode_to_utf8(_)
for _ in actor.get('aliases')
])
if actor.get('country'):
e.country = encode_to_utf8(
actor.get('country',
dict()).get('displayName'))
if actor.get('score'):
e.score = actor.get('score')
if actor.get('links'):
e += Label(
'Links', '<br/>'.join([
'<a href="{}">{}</a>'.format(
_.get('href'), _.get('href'))
for _ in actor.get('links')
]))
if actor.get('hyperlinks'):
e += Label(
'Hyperlinks', '<br/>'.join([
'<a href="{}">{}</a>'.format(
_.get('url'), _.get('title'))
for _ in actor.get('hyperlinks')
]))
if actor.get('title'):
e += Label('Title',
encode_to_utf8(actor.get('title')))
if actor.get('resourceId'):
e += Label('ResourceID',
actor.get('resourceId'))
if actor.get('aliases'):
e += Label(
'Aliases', '<br/>'.join([
encode_to_utf8(_)
for _ in actor.get('aliases', '')
]))
if actor.get('description'):
e += Label(
'Description', '<br/>'.join(
encode_to_utf8(
actor.get('description',
'')).split('\n')))
if actor.get('country'):
e += Label(
'Country',
encode_to_utf8(
actor.get('country',
dict()).get('displayName')))
if actor.get('organization'):
e += Label(
'Organization',
encode_to_utf8(actor.get('organization')))
if actor.get('types'):
e += Label(
'Types', '<br/>'.join([
encode_to_utf8(_.get('displayName'))
for _ in actor.get('types')
]))
if actor.get('motivations'):
e += Label(
'Motivations', '<br/>'.join([
encode_to_utf8(_.get('displayName'))
for _ in actor.get('motivations')
]))
if actor.get('intendedEffects'):
e += Label(
'Intended Effects', '<br/>'.join([
encode_to_utf8(_.get('displayName'))
for _ in actor.get('intendedEffects')
]))
if actor.get('sophistication'):
e += Label(
'Sophistication',
actor.get('sophistication',
dict()).get('displayName'))
if actor.get('socialMediaText'):
e += Label(
'Social Media', '<br/>'.join(
encode_to_utf8(
actor.get('socialMediaText',
'')).split('\n')))
if actor.get('moreInfo'):
e += Label(
'More Info', '<br/>'.join(
encode_to_utf8(
actor.get('moreInfo',
'')).split('\n')))
if actor.get('score'):
e += Label('Score', actor.get('score'))
if i < 1:
i += 1
e.linkcolor = "0xf90000"
response += e
except AttributeError as err:
response += UIMessage(err, type='PartialError')
continue
except ThreatCentralError as err:
response += UIMessage(err.value, type='PartialError')
except TypeError:
return response
else:
if actor:
try:
if actor.get('tcScore'):
weight = int(actor.get('tcScore'))
else:
weight = 1
# Update entity?
e = Actor(request.value, weight=weight)
if actor.get('name'):
e.name = encode_to_utf8(actor.get('name'))
e.actor = encode_to_utf8(actor.get('name'))
e.title = encode_to_utf8(actor.get('title'))
e.resourceId = actor.get('resourceId')
if actor.get('organization'):
e.organization = encode_to_utf8(actor.get('organization'))
if actor.get('aliases'):
e.aliases = ', '.join(
[encode_to_utf8(_) for _ in actor.get('aliases')])
if actor.get('country'):
e.country = encode_to_utf8(
actor.get('country', dict()).get('displayName'))
# Add Location entitie
l = Location(
encode_to_utf8(
actor.get('country', dict()).get('displayName')))
response += l
if actor.get('score'):
e.score = actor.get('score')
if actor.get('links'):
e += Label(
'Links', '<br/>'.join([
'<a href="{}">{}</a>'.format(
_.get('href'), _.get('href'))
for _ in actor.get('links')
]))
if actor.get('hyperlinks'):
e += Label(
'Hyperlinks', '<br/>'.join([
'<a href="{}">{}</a>'.format(
_.get('url'), _.get('title'))
for _ in actor.get('hyperlinks')
]))
if actor.get('title'):
e += Label('Title', encode_to_utf8(actor.get('title')))
if actor.get('resourceId'):
e += Label('ResourceID', actor.get('resourceId'))
if actor.get('aliases'):
e += Label(
'Aliases', '<br/>'.join([
encode_to_utf8(_)
for _ in actor.get('aliases', '')
]))
if actor.get('description'):
e += Label(
'Description', '<br/>'.join(
encode_to_utf8(actor.get('description',
'')).split('\n')))
if actor.get('country'):
e += Label(
'Country',
encode_to_utf8(
actor.get('country', dict()).get('displayName')))
if actor.get('organization'):
e += Label('Organization',
encode_to_utf8(actor.get('organization')))
if actor.get('types'):
e += Label(
'Types', '<br/>'.join([
encode_to_utf8(_.get('displayName'))
for _ in actor.get('types')
]))
if actor.get('motivations'):
e += Label(
'Motivations', '<br/>'.join([
encode_to_utf8(_.get('displayName'))
for _ in actor.get('motivations')
]))
if actor.get('intendedEffects'):
e += Label(
'Intended Effects', '<br/>'.join([
encode_to_utf8(_.get('displayName'))
for _ in actor.get('intendedEffects')
]))
if actor.get('sophistication'):
e += Label(
'Sophistication',
encode_to_utf8(
actor.get('sophistication',
dict()).get('displayName')))
if actor.get('socialMediaText'):
e += Label(
'Social Media', '<br/>'.join(
encode_to_utf8(actor.get('socialMediaText',
'')).split('\n')))
if actor.get('moreInfo'):
e += Label(
'More Info', '<br/>'.join(
encode_to_utf8(actor.get('moreInfo',
'')).split('\n')))
if actor.get('score'):
e += Label('Score', actor.get('score'))
response += e
# Extract email addresses
usable_info = search_for_usable_info('{} {} {}'.format(
encode_to_utf8(actor.get('description')),
encode_to_utf8(actor.get('socialMediaText')),
encode_to_utf8(actor.get('moreInfo'))))
if usable_info:
debug(usable_info)
try:
urls = usable_info.get('url', dict())
for twitter in urls.get('twitter', list()):
t = Twitter(twitter.get('name'))
t.uid = twitter.get('name')
t.set_field('affiliation.profile-url',
twitter.get('url'))
response += t
for facebook in urls.get('facebook', list()):
f = Facebook(facebook.get('name'))
f.uid = facebook.get('name')
f.set_field('affiliation.profile-url',
facebook.get('url'))
response += f
for other in urls.get('other', list()):
u = URL(other)
u.url = other
response += u
emailaddr = usable_info.get('email', list())
for email in emailaddr:
e = EmailAddress(email)
response += e
except AttributeError as err:
response += UIMessage('Error: {}'.format(err))
except AttributeError as err:
response += UIMessage('Error: {}'.format(err),
type='PartialError')
except ThreatCentralError as err:
response += UIMessage(err.value, type='PartialError')
except TypeError:
return response
return response
def dotransform(request, response, config):
tr_details = [
'Reference', 'Source', 'KillChain', 'Firstseen', 'Lastseen',
'Attribution', 'ProcessType', 'Rrname', 'Rdata', 'Country', 'Tags',
'Comment', 'RootNode', 'Confidence'
]
#Disable cache to get actual data from Threat Recon
cache, found = search(request.value, cache=False)
#Default linkcolor
linkcolor = "0x000000"
if found:
if defaultdict == type(found):
for rootnode, value in found.iteritems():
#If the RootNode is empty, display attributes
if len(rootnode) == 0:
for indicator in value:
#debug(indicator)
e = ''
indtype = indicator['Type'].lower().strip()
if "whois email" == indtype:
e = EmailAddress(indicator['Indicator'])
if "name server" == indtype:
e = NSRecord(indicator['Indicator'])
if "domain" == indtype:
e = Domain(indicator['Indicator'])
e.fqdn = indicator['Indicator']
if "ip" == indtype:
e = IPv4Address(indicator['Indicator'])
if "phone or fax no." == indtype:
e = PhoneNumber(indicator['Indicator'])
if "whois address component" == indtype:
e = Phrase(indicator['Indicator'])
if "email" == indtype:
e = EmailAddress(indicator['Indicator'])
if "netname" == indtype:
e = NetNameThreatRecon(indicator['Indicator'])
if "cidr" == indtype:
e = IPv4Address(indicator['Indicator'])
if "netrange" == indtype:
e = Netblock(indicator['Indicator'])
if e:
#Set linkcolor
e.linkcolor = linkcolor
#Set comments
if indicator['Comment']:
e.notes = string_filter(indicator['Comment'])
#Set Details
for detail in tr_details:
if detail in indicator:
if indicator[detail]:
e += Label(name=detail,
value=string_filter(
indicator[detail]))
response += e
else:
#Display the RootNodes
e = ThreatRecon(rootnode)
response += e
return response
