def do_transform(self, request, response, config):
person = request.entity
fields = person.fields
if fields.get("properties.url"):
url = fields.get("properties.url").value
else:
url = None
soup = scrape(url)
if soup:
email_addresses = soup.find_all(attrs={"class": "__cf_email__"})
for email_address in email_addresses:
fp = email_address['data-cfemail']
r = int(fp[:2], 16)
email = ''.join([
chr(int(fp[i:i + 2], 16) ^ r)
for i in range(2, len(fp), 2)
])
response += EmailAddress(email)
phone_numbers = soup.find_all(attrs={"data-link-to-more": "phone"})
for phone_number in phone_numbers:
response += PhoneNumber(phone_number.get_text())
return response
def dotransform(request, response):
client = login()
user = client.users(request.fields['affiliation.uid']).get('user', {})
if 'contact' in user and 'phone' in user['contact']:
e = PhoneNumber(user['contact']['phone'])
response += e
return response
def do_transform(self, request, response, config):
person = request.entity
fields = person.fields
if fields.get("properties.url"):
url = fields.get("properties.url").value
else:
url = None
soup = scrape(url)
if soup:
email_addresses = soup.find_all(attrs={"class": "__cf_email__"})
for email_address in email_addresses:
fp = email_address['data-cfemail']
r = int(fp[:2], 16)
email = ''.join([
chr(int(fp[i:i + 2], 16) ^ r)
for i in range(2, len(fp), 2)
])
response += EmailAddress(email)
phone_numbers = soup.find_all(attrs={"data-link-to-more": "phone"})
for phone_number in phone_numbers:
response += PhoneNumber(phone_number.get_text())
addresses = soup.find_all(attrs={"data-link-to-more": "address"})
for unformated_address in addresses:
address = unformated_address.get_text().split(" ")
address[-1] = address[-1].split("-")[0]
response += Location(" ".join(address))
relatives = soup.find_all(attrs={"data-link-to-more": "relative"})
for relative in relatives:
response += TruePerson(
relative.get_text(),
properties_url=config['TruePeopleSearch.local.base_url'] +
relative['href'])
associates = soup.find_all(
attrs={"data-link-to-more": "associate"})
for associate in associates:
response += TruePerson(
associate.get_text(),
properties_url=config['TruePeopleSearch.local.base_url'] +
associate['href'])
return response
def do_transform(self, request, response, config):
person = request.entity
fields = person.fields
if fields.get("properties.url"):
url = fields.get("properties.url").value
else:
url = None
soup = scrape(url)
if soup:
phone_numbers = soup.find_all(attrs={"data-link-to-more": "phone"})
for phone_number in phone_numbers:
response += PhoneNumber(phone_number.get_text())
return response
def dotransform(request, response, config):
tr_details = [
'Reference', 'Source', 'KillChain', 'Firstseen', 'Lastseen',
'Attribution', 'ProcessType', 'Rrname', 'Rdata', 'Country', 'Tags',
'Comment', 'RootNode', 'Confidence'
]
#Default link color is black
linkcolor = "0x000000"
cache, found = search(request.value)
if found:
if list == type(found):
for indicator in found:
debug(indicator)
e = ''
indtype = indicator['Type'].lower().strip()
if "whois email" == indtype:
e = EmailAddress(indicator['Indicator'])
#response += e
if "name server" == indtype:
e = NSRecord(indicator['Indicator'])
#response += e
if "domain" == indtype:
e = Domain(indicator['Indicator'])
e.fqdn = indicator['Indicator']
#response += e
#IF Type is not domain, check if Rrname is not empty
elif indicator['Rrname'] and indicator['Rrname'] != 'NA':
d = Domain(indicator['Rrname'])
d.fqdn = indicator['Rrname']
response += d
if "ip" == indtype:
e = IPv4Address(indicator['Indicator'])
#response += e
#IF Type is not IP, check if Rdata is not empty
elif indicator['Rdata']:
i = IPv4Address(indicator['Rdata'])
response += i
if "phone or fax no." == indtype:
e = PhoneNumber(indicator['Indicator'])
#response += e
if "whois address component" == indtype:
e = Phrase(indicator['Indicator'])
#response += e
if "email" == indtype:
e = EmailAddress(indicator['Indicator'])
#response += e
if "netname" == indtype:
e = NetNameThreatRecon(indicator['Indicator'])
#response += e
if "cidr" == indtype:
e = IPv4Address(indicator['Indicator'])
#response += e
if "netrange" == indtype:
e = Netblock(indicator['Indicator'])
#response += e
if indicator['Country']:
l = Location(indicator['Country'])
response += l
#Add Comments and details to own Entity
entity = e #request.entity
#Set comments
if indicator['Comment']:
entity.notes = string_filter(indicator['Comment'])
#Set Details
for detail in tr_details:
if detail in indicator:
if indicator[detail]:
entity += Label(name=detail,
value=string_filter(
indicator[detail]))
#Set link color
if "Confidence" in indicator:
if indicator['Confidence'] >= 70:
linkcolor = "0xff0000"
entity.linkcolor = linkcolor
response += entity
return response
def dotransform(request, response, config):
tr_details = [
'Reference', 'Source', 'KillChain', 'Firstseen', 'Lastseen',
'Attribution', 'ProcessType', 'Rrname', 'Rdata', 'Country', 'Tags',
'Comment', 'RootNode', 'Confidence'
]
#Disable cache to get actual data from Threat Recon
cache, found = search(request.value, cache=False)
#Default linkcolor
linkcolor = "0x000000"
if found:
if defaultdict == type(found):
for rootnode, value in found.iteritems():
#If the RootNode is empty, display attributes
if len(rootnode) == 0:
for indicator in value:
#debug(indicator)
e = ''
indtype = indicator['Type'].lower().strip()
if "whois email" == indtype:
e = EmailAddress(indicator['Indicator'])
if "name server" == indtype:
e = NSRecord(indicator['Indicator'])
if "domain" == indtype:
e = Domain(indicator['Indicator'])
e.fqdn = indicator['Indicator']
if "ip" == indtype:
e = IPv4Address(indicator['Indicator'])
if "phone or fax no." == indtype:
e = PhoneNumber(indicator['Indicator'])
if "whois address component" == indtype:
e = Phrase(indicator['Indicator'])
if "email" == indtype:
e = EmailAddress(indicator['Indicator'])
if "netname" == indtype:
e = NetNameThreatRecon(indicator['Indicator'])
if "cidr" == indtype:
e = IPv4Address(indicator['Indicator'])
if "netrange" == indtype:
e = Netblock(indicator['Indicator'])
if e:
#Set linkcolor
e.linkcolor = linkcolor
#Set comments
if indicator['Comment']:
e.notes = string_filter(indicator['Comment'])
#Set Details
for detail in tr_details:
if detail in indicator:
if indicator[detail]:
e += Label(name=detail,
value=string_filter(
indicator[detail]))
response += e
else:
#Display the RootNodes
e = ThreatRecon(rootnode)
response += e
return response