def dotransform(request, response):
r = geoip(request.value)
if r is not None:
if 'error' in r:
response += UIMessage(r['error'])
return response
locname = ''
cityf = None
countryf = None
if 'city' in r:
locname += r['city']
cityf = r['city']
if 'countryName' in r:
locname += ', %s' % r['countryName']
countryf = r['countryName']
e = Location(locname)
if 'longitude' in r and 'latitude' in r:
e.longitude = r['longitude']
e.latitude = r['latitude']
link = maplink(r)
e += Label('Map It', A(link, link), type='text/html')
if 'region' in r:
e.area = r['region']
if cityf is not None:
e.city = cityf
if countryf is not None:
e.country = countryf
e.iconurl = flag(countryf)
if 'countryCode' in r:
e.countrycode = r['countryCode']
if e.iconurl is None:
e.iconurl = flag(r['countryCode'])
response += e
return response
def dotransform(request, response):
# Download GeoIP Database from MaxMinds
if not os.path.exists('/opt/geoipdb/geoipdb.dat'):
return response + UIMessage('Need local install of MaxMinds Geo IP database, use the download script in resource/external/geoipdownload.sh')
gi = pygeoip.GeoIP('/opt/geoipdb/geoipdb.dat')
pcap = request.value
pkts = rdpcap(pcap)
ip_raw = []
ip_geo = []
ip_exclusions = ['192.168.', '172.16.', '10.']
for x in pkts:
if x.haslayer(IP):
src = x.getlayer(IP).src
if src != '0.0.0.0':
if src not in ip_raw:
ip_raw.append(src)
for s in ip_raw:
if ip_exclusions[0] in s or ip_exclusions[1] in s or ip_exclusions[2] in s:
pass
else:
rec = gi.record_by_addr(s)
city = rec['city']
postcode = rec['postal_code']
country = rec['country_name']
lng = rec['longitude']
lat = rec['latitude']
ccode = rec['country_code']
google_map_url = 'https://maps.google.co.uk/maps?z=20&q=%s,%s' %(lat, lng)
geo_ip = s,city, postcode, country, ccode, str(lng), str(lat), google_map_url
if geo_ip not in ip_geo:
ip_geo.append(geo_ip)
for ip, city, postcode, country, ccode, lng, lat, gmap in ip_geo:
e = Location(country)
e.country = country
e.city = city
e.linkcolor = 0x2314CA
e.linklabel = ip
e.areacode = postcode
e.longitude = float(lng)
e.latitude = float(lat)
e.countrycode = ccode
e += Field('ipaddress', ip, displayname='IP Address')
e += Field('geomapurl', gmap, displayname='Google Map URL')
e += Field('pcapsrc', pcap, displayname='Original pcap File')
response += e
return response
def getlocbymac(mac):
ll = geomac(mac)
gcr = reversegeo(ll['latitude'], ll['longitude'])[0]
l = Location('-, -')
l.city = '-'
l.country = '-'
for i in gcr['address_components']:
if 'locality' in i['types']:
l.city = i['long_name']
if 'administrative_area_level_1' in i['types']:
l.area = i['long_name']
if 'country' in i['types']:
l.country = i['long_name']
l.latitude = gcr['geometry']['location']['lat']
l.longitude = gcr['geometry']['location']['lng']
l.value = '%s, %s' % (l.city, l.country)
return l
def do_transform(self, request, response, config):
tweet = request.entity
_body = {
'query': {
'match': {
'id': tweet.id
}
},
'size': request.limits.hard
}
res = es.search(index="twinttweets", body=_body)
for hit in res['hits']['hits']:
tweet = hit['_source']
r = Location()
if tweet['location']:
r.name = tweet['location']
response += r
return response
def dotransform(request, response):
p = JSONDecoder().decode(
pipljsonsearch(first_name=request.entity.firstnames or '',
last_name=request.entity.lastname or ''))
if 'error' in p:
response += UIMessage(p['error'])
for r in p['results']['records']:
if 'addresses' in r:
for a in r['addresses']:
e = Location(a['display'])
e.countrycode = a['country']
e += Label('Source',
'<a href="%s">%s</a>' %
(r['source']['url'], r['source']['@ds_name']),
type='text/html')
response += e
return response
def dotransform(request, response, config):
if 'workspace' in request.fields:
workspace = request.fields['workspace']
else:
workspace = request.value
e = Location(request.fields["location"].decode('ascii'))
e += Field("workspace", workspace, displayname='Workspace')
response += e
return response
def do_transform(self, request, response, config):
tweet = request.entity
_body = {
'query': {
'match': {
'id': tweet.id
}
},
'size': request.limits.hard
}
res = es.search(index="twinttweets", body=_body)
for hit in res['hits']['hits']:
tweet = hit['_source']
r = Location()
try:
r.longitude = tweet['geo_near']['lon']
r.latitude = tweet['geo_near']['lon']
r.city = tweet['near']
response += r
except KeyError:
pass
return response
def dotransform(request, response):
p = JSONDecoder().decode(
pipljsonsearch(
first_name=request.fields['firstname'],
last_name=request.fields['lastname']
)
)
if 'error' in p:
response += UIMessage(p['error'])
for r in p['results']['records']:
if 'addresses' in r:
for a in r['addresses']:
e = Location(a['display'])
e.countrycode = a['country']
e += Label(
'Source', '<a href="%s">%s</a>' % (r['source']['url'], r['source']['@ds_name']), type='text/html'
)
response += e
return response
def do_transform(self, request, response, config):
person = request.entity
fields = person.fields
if fields.get("properties.url"):
url = fields.get("properties.url").value
else:
url = None
soup = scrape(url)
if soup:
email_addresses = soup.find_all(attrs={"class": "__cf_email__"})
for email_address in email_addresses:
fp = email_address['data-cfemail']
r = int(fp[:2], 16)
email = ''.join([
chr(int(fp[i:i + 2], 16) ^ r)
for i in range(2, len(fp), 2)
])
response += EmailAddress(email)
phone_numbers = soup.find_all(attrs={"data-link-to-more": "phone"})
for phone_number in phone_numbers:
response += PhoneNumber(phone_number.get_text())
addresses = soup.find_all(attrs={"data-link-to-more": "address"})
for unformated_address in addresses:
address = unformated_address.get_text().split(" ")
address[-1] = address[-1].split("-")[0]
response += Location(" ".join(address))
relatives = soup.find_all(attrs={"data-link-to-more": "relative"})
for relative in relatives:
response += TruePerson(
relative.get_text(),
properties_url=config['TruePeopleSearch.local.base_url'] +
relative['href'])
associates = soup.find_all(
attrs={"data-link-to-more": "associate"})
for associate in associates:
response += TruePerson(
associate.get_text(),
properties_url=config['TruePeopleSearch.local.base_url'] +
associate['href'])
return response
def do_transform(self, request, response, config):
person = request.entity
fields = person.fields
if fields.get("properties.url"):
url = fields.get("properties.url").value
else:
url = None
soup = scrape(url)
if soup:
addresses = soup.find_all(attrs={"data-link-to-more": "address"})
for address in addresses:
response += Location(address.get_text())
return response
def do_transform(self, request, response, config):
person = request.entity
fields = person.fields
if fields.get("properties.url"):
url = fields.get("properties.url").value
else:
url = None
soup = scrape(url)
if soup:
addresses = soup.find_all(attrs={"data-link-to-more": "address"})
if addresses:
address = addresses[0].get_text().split(" ")
address[-1] = address[-1].split("-")[0]
response += Location(" ".join(address))
return response
def dotransform(request, response, config):
if 'workspace' in request.fields:
workspace = request.fields['workspace']
else:
workspace = request.value
dbcon = db_connect(workspace)
host_list = get_hosts(dbcon)
for loc in host_list:
if loc[0] == request.value or loc[1] == request.value:
e = Location(str(loc[2]) + ', ' + str(loc[3]),
latitude=float(int(loc[4])),
longitude=float(int(loc[5])))
e += Field("workspace", workspace, displayname='Workspace')
response += e
return response
def do_transform(self, request, response, config):
# don't forget to add `from maltego.message import Bookmark`
ip_address = request.entity.value
url_template = config['hello.local.geo_ip_url']
geoip_str = urlopen(url_template.format(ip=ip_address)).read()
geoip_json = json.loads(geoip_str)
country_code = geoip_json.get('country_code').lower()
response += Location(
country=geoip_json.get('country_name', 'Unknown'),
city=geoip_json.get('city'),
countrycode=geoip_json.get('country_code'),
latitude=geoip_json.get('latitude'),
longitude=geoip_json.get('longitude'),
area=geoip_json.get('region_name'),
link_label='From FreeGeoIP',
bookmark=Bookmark.Orange,
icon_url='http://www.geoips.com/assets/img/flag/256/%s.png' %
country_code)
return response
def getlocs(response, data):
for loc in data:
l = loc['place']['location']
e = Location('%s, %s' % (l.get('city', ''), l.get('country', '')))
if 'country' in l:
e.country = l['country']
if 'city' in l:
e.city = l['city']
if 'state' in l:
e.area = l['state']
if 'longitude' in l and 'latitude' in l:
e.longitude = l['longitude']
e.latitude = l['latitude']
sa = ''
if 'name' in loc['place']:
sa = loc['place']['name']
if 'street' in l:
sa += ', %s' % l['street']
if 'zip' in l:
sa += ', %s' % l['zip']
e += Field('streetaddress', sa)
response += e
def dotransform(request, response, config):
tr_details = [
'Reference', 'Source', 'KillChain', 'Firstseen', 'Lastseen',
'Attribution', 'ProcessType', 'Rrname', 'Rdata', 'Country', 'Tags',
'Comment', 'RootNode', 'Confidence'
]
#Default link color is black
linkcolor = "0x000000"
cache, found = search(request.value)
if found:
if list == type(found):
for indicator in found:
debug(indicator)
e = ''
indtype = indicator['Type'].lower().strip()
if "whois email" == indtype:
e = EmailAddress(indicator['Indicator'])
#response += e
if "name server" == indtype:
e = NSRecord(indicator['Indicator'])
#response += e
if "domain" == indtype:
e = Domain(indicator['Indicator'])
e.fqdn = indicator['Indicator']
#response += e
#IF Type is not domain, check if Rrname is not empty
elif indicator['Rrname'] and indicator['Rrname'] != 'NA':
d = Domain(indicator['Rrname'])
d.fqdn = indicator['Rrname']
response += d
if "ip" == indtype:
e = IPv4Address(indicator['Indicator'])
#response += e
#IF Type is not IP, check if Rdata is not empty
elif indicator['Rdata']:
i = IPv4Address(indicator['Rdata'])
response += i
if "phone or fax no." == indtype:
e = PhoneNumber(indicator['Indicator'])
#response += e
if "whois address component" == indtype:
e = Phrase(indicator['Indicator'])
#response += e
if "email" == indtype:
e = EmailAddress(indicator['Indicator'])
#response += e
if "netname" == indtype:
e = NetNameThreatRecon(indicator['Indicator'])
#response += e
if "cidr" == indtype:
e = IPv4Address(indicator['Indicator'])
#response += e
if "netrange" == indtype:
e = Netblock(indicator['Indicator'])
#response += e
if indicator['Country']:
l = Location(indicator['Country'])
response += l
#Add Comments and details to own Entity
entity = e #request.entity
#Set comments
if indicator['Comment']:
entity.notes = string_filter(indicator['Comment'])
#Set Details
for detail in tr_details:
if detail in indicator:
if indicator[detail]:
entity += Label(name=detail,
value=string_filter(
indicator[detail]))
#Set link color
if "Confidence" in indicator:
if indicator['Confidence'] >= 70:
linkcolor = "0xff0000"
entity.linkcolor = linkcolor
response += entity
return response
def dotransform(request, response, config):
try:
actor = get_actor(request.fields['ThreatCentral.resourceId'])
except ThreatCentralError as err:
response += UIMessage(err.value, type='PartialError')
return response
except KeyError:
try:
actors = search_actor(request.value)
except ThreatCentralError as err:
response += UIMessage(err.value, type='PartialError')
return response
else:
i = 0
for actor in actors:
try:
rtype = lower(actor.get('type'))
actor = actor.get('resource')
if actor.get('tcScore'):
weight = int(actor.get('tcScore'))
else:
weight = 1
if len(actor) is not 0:
if rtype == 'actor':
if actor.get('name'):
e = Actor(encode_to_utf8(actor.get('name')),
weight=weight)
e.name = encode_to_utf8(actor.get('name'))
e.actor = encode_to_utf8(actor.get('name'))
elif actor.get('title'):
e = Actor(encode_to_utf8(actor.get('title')))
e.title = encode_to_utf8(actor.get('title'))
e.resourceId = actor.get('resourceId')
if actor.get('organization'):
e.organization = encode_to_utf8(
actor.get('organization'))
if actor.get('aliases'):
e.aliases = ', '.join([
encode_to_utf8(_)
for _ in actor.get('aliases')
])
if actor.get('country'):
e.country = encode_to_utf8(
actor.get('country',
dict()).get('displayName'))
if actor.get('score'):
e.score = actor.get('score')
if actor.get('links'):
e += Label(
'Links', '<br/>'.join([
'<a href="{}">{}</a>'.format(
_.get('href'), _.get('href'))
for _ in actor.get('links')
]))
if actor.get('hyperlinks'):
e += Label(
'Hyperlinks', '<br/>'.join([
'<a href="{}">{}</a>'.format(
_.get('url'), _.get('title'))
for _ in actor.get('hyperlinks')
]))
if actor.get('title'):
e += Label('Title',
encode_to_utf8(actor.get('title')))
if actor.get('resourceId'):
e += Label('ResourceID',
actor.get('resourceId'))
if actor.get('aliases'):
e += Label(
'Aliases', '<br/>'.join([
encode_to_utf8(_)
for _ in actor.get('aliases', '')
]))
if actor.get('description'):
e += Label(
'Description', '<br/>'.join(
encode_to_utf8(
actor.get('description',
'')).split('\n')))
if actor.get('country'):
e += Label(
'Country',
encode_to_utf8(
actor.get('country',
dict()).get('displayName')))
if actor.get('organization'):
e += Label(
'Organization',
encode_to_utf8(actor.get('organization')))
if actor.get('types'):
e += Label(
'Types', '<br/>'.join([
encode_to_utf8(_.get('displayName'))
for _ in actor.get('types')
]))
if actor.get('motivations'):
e += Label(
'Motivations', '<br/>'.join([
encode_to_utf8(_.get('displayName'))
for _ in actor.get('motivations')
]))
if actor.get('intendedEffects'):
e += Label(
'Intended Effects', '<br/>'.join([
encode_to_utf8(_.get('displayName'))
for _ in actor.get('intendedEffects')
]))
if actor.get('sophistication'):
e += Label(
'Sophistication',
actor.get('sophistication',
dict()).get('displayName'))
if actor.get('socialMediaText'):
e += Label(
'Social Media', '<br/>'.join(
encode_to_utf8(
actor.get('socialMediaText',
'')).split('\n')))
if actor.get('moreInfo'):
e += Label(
'More Info', '<br/>'.join(
encode_to_utf8(
actor.get('moreInfo',
'')).split('\n')))
if actor.get('score'):
e += Label('Score', actor.get('score'))
if i < 1:
i += 1
e.linkcolor = "0xf90000"
response += e
except AttributeError as err:
response += UIMessage(err, type='PartialError')
continue
except ThreatCentralError as err:
response += UIMessage(err.value, type='PartialError')
except TypeError:
return response
else:
if actor:
try:
if actor.get('tcScore'):
weight = int(actor.get('tcScore'))
else:
weight = 1
# Update entity?
e = Actor(request.value, weight=weight)
if actor.get('name'):
e.name = encode_to_utf8(actor.get('name'))
e.actor = encode_to_utf8(actor.get('name'))
e.title = encode_to_utf8(actor.get('title'))
e.resourceId = actor.get('resourceId')
if actor.get('organization'):
e.organization = encode_to_utf8(actor.get('organization'))
if actor.get('aliases'):
e.aliases = ', '.join(
[encode_to_utf8(_) for _ in actor.get('aliases')])
if actor.get('country'):
e.country = encode_to_utf8(
actor.get('country', dict()).get('displayName'))
# Add Location entitie
l = Location(
encode_to_utf8(
actor.get('country', dict()).get('displayName')))
response += l
if actor.get('score'):
e.score = actor.get('score')
if actor.get('links'):
e += Label(
'Links', '<br/>'.join([
'<a href="{}">{}</a>'.format(
_.get('href'), _.get('href'))
for _ in actor.get('links')
]))
if actor.get('hyperlinks'):
e += Label(
'Hyperlinks', '<br/>'.join([
'<a href="{}">{}</a>'.format(
_.get('url'), _.get('title'))
for _ in actor.get('hyperlinks')
]))
if actor.get('title'):
e += Label('Title', encode_to_utf8(actor.get('title')))
if actor.get('resourceId'):
e += Label('ResourceID', actor.get('resourceId'))
if actor.get('aliases'):
e += Label(
'Aliases', '<br/>'.join([
encode_to_utf8(_)
for _ in actor.get('aliases', '')
]))
if actor.get('description'):
e += Label(
'Description', '<br/>'.join(
encode_to_utf8(actor.get('description',
'')).split('\n')))
if actor.get('country'):
e += Label(
'Country',
encode_to_utf8(
actor.get('country', dict()).get('displayName')))
if actor.get('organization'):
e += Label('Organization',
encode_to_utf8(actor.get('organization')))
if actor.get('types'):
e += Label(
'Types', '<br/>'.join([
encode_to_utf8(_.get('displayName'))
for _ in actor.get('types')
]))
if actor.get('motivations'):
e += Label(
'Motivations', '<br/>'.join([
encode_to_utf8(_.get('displayName'))
for _ in actor.get('motivations')
]))
if actor.get('intendedEffects'):
e += Label(
'Intended Effects', '<br/>'.join([
encode_to_utf8(_.get('displayName'))
for _ in actor.get('intendedEffects')
]))
if actor.get('sophistication'):
e += Label(
'Sophistication',
encode_to_utf8(
actor.get('sophistication',
dict()).get('displayName')))
if actor.get('socialMediaText'):
e += Label(
'Social Media', '<br/>'.join(
encode_to_utf8(actor.get('socialMediaText',
'')).split('\n')))
if actor.get('moreInfo'):
e += Label(
'More Info', '<br/>'.join(
encode_to_utf8(actor.get('moreInfo',
'')).split('\n')))
if actor.get('score'):
e += Label('Score', actor.get('score'))
response += e
# Extract email addresses
usable_info = search_for_usable_info('{} {} {}'.format(
encode_to_utf8(actor.get('description')),
encode_to_utf8(actor.get('socialMediaText')),
encode_to_utf8(actor.get('moreInfo'))))
if usable_info:
debug(usable_info)
try:
urls = usable_info.get('url', dict())
for twitter in urls.get('twitter', list()):
t = Twitter(twitter.get('name'))
t.uid = twitter.get('name')
t.set_field('affiliation.profile-url',
twitter.get('url'))
response += t
for facebook in urls.get('facebook', list()):
f = Facebook(facebook.get('name'))
f.uid = facebook.get('name')
f.set_field('affiliation.profile-url',
facebook.get('url'))
response += f
for other in urls.get('other', list()):
u = URL(other)
u.url = other
response += u
emailaddr = usable_info.get('email', list())
for email in emailaddr:
e = EmailAddress(email)
response += e
except AttributeError as err:
response += UIMessage('Error: {}'.format(err))
except AttributeError as err:
response += UIMessage('Error: {}'.format(err),
type='PartialError')
except ThreatCentralError as err:
response += UIMessage(err.value, type='PartialError')
except TypeError:
return response
return response
def dotransform(request, response):
try:
s = SNMPManager(*snmpargs(request))
response += Location(s.location)
except SNMPError, s:
response += UIMessage(str(s))