def dotransform(request, response): r = geoip(request.value) if r is not None: if 'error' in r: response += UIMessage(r['error']) return response locname = '' cityf = None countryf = None if 'city' in r: locname += r['city'] cityf = r['city'] if 'countryName' in r: locname += ', %s' % r['countryName'] countryf = r['countryName'] e = Location(locname) if 'longitude' in r and 'latitude' in r: e.longitude = r['longitude'] e.latitude = r['latitude'] link = maplink(r) e += Label('Map It', A(link, link), type='text/html') if 'region' in r: e.area = r['region'] if cityf is not None: e.city = cityf if countryf is not None: e.country = countryf e.iconurl = flag(countryf) if 'countryCode' in r: e.countrycode = r['countryCode'] if e.iconurl is None: e.iconurl = flag(r['countryCode']) response += e return response
def dotransform(request, response): # Download GeoIP Database from MaxMinds if not os.path.exists('/opt/geoipdb/geoipdb.dat'): return response + UIMessage('Need local install of MaxMinds Geo IP database, use the download script in resource/external/geoipdownload.sh') gi = pygeoip.GeoIP('/opt/geoipdb/geoipdb.dat') pcap = request.value pkts = rdpcap(pcap) ip_raw = [] ip_geo = [] ip_exclusions = ['192.168.', '172.16.', '10.'] for x in pkts: if x.haslayer(IP): src = x.getlayer(IP).src if src != '0.0.0.0': if src not in ip_raw: ip_raw.append(src) for s in ip_raw: if ip_exclusions[0] in s or ip_exclusions[1] in s or ip_exclusions[2] in s: pass else: rec = gi.record_by_addr(s) city = rec['city'] postcode = rec['postal_code'] country = rec['country_name'] lng = rec['longitude'] lat = rec['latitude'] ccode = rec['country_code'] google_map_url = 'https://maps.google.co.uk/maps?z=20&q=%s,%s' %(lat, lng) geo_ip = s,city, postcode, country, ccode, str(lng), str(lat), google_map_url if geo_ip not in ip_geo: ip_geo.append(geo_ip) for ip, city, postcode, country, ccode, lng, lat, gmap in ip_geo: e = Location(country) e.country = country e.city = city e.linkcolor = 0x2314CA e.linklabel = ip e.areacode = postcode e.longitude = float(lng) e.latitude = float(lat) e.countrycode = ccode e += Field('ipaddress', ip, displayname='IP Address') e += Field('geomapurl', gmap, displayname='Google Map URL') e += Field('pcapsrc', pcap, displayname='Original pcap File') response += e return response
def getlocbymac(mac): ll = geomac(mac) gcr = reversegeo(ll['latitude'], ll['longitude'])[0] l = Location('-, -') l.city = '-' l.country = '-' for i in gcr['address_components']: if 'locality' in i['types']: l.city = i['long_name'] if 'administrative_area_level_1' in i['types']: l.area = i['long_name'] if 'country' in i['types']: l.country = i['long_name'] l.latitude = gcr['geometry']['location']['lat'] l.longitude = gcr['geometry']['location']['lng'] l.value = '%s, %s' % (l.city, l.country) return l
def do_transform(self, request, response, config): tweet = request.entity _body = { 'query': { 'match': { 'id': tweet.id } }, 'size': request.limits.hard } res = es.search(index="twinttweets", body=_body) for hit in res['hits']['hits']: tweet = hit['_source'] r = Location() if tweet['location']: r.name = tweet['location'] response += r return response
def dotransform(request, response): p = JSONDecoder().decode( pipljsonsearch(first_name=request.entity.firstnames or '', last_name=request.entity.lastname or '')) if 'error' in p: response += UIMessage(p['error']) for r in p['results']['records']: if 'addresses' in r: for a in r['addresses']: e = Location(a['display']) e.countrycode = a['country'] e += Label('Source', '<a href="%s">%s</a>' % (r['source']['url'], r['source']['@ds_name']), type='text/html') response += e return response
def dotransform(request, response, config): if 'workspace' in request.fields: workspace = request.fields['workspace'] else: workspace = request.value e = Location(request.fields["location"].decode('ascii')) e += Field("workspace", workspace, displayname='Workspace') response += e return response
def do_transform(self, request, response, config): tweet = request.entity _body = { 'query': { 'match': { 'id': tweet.id } }, 'size': request.limits.hard } res = es.search(index="twinttweets", body=_body) for hit in res['hits']['hits']: tweet = hit['_source'] r = Location() try: r.longitude = tweet['geo_near']['lon'] r.latitude = tweet['geo_near']['lon'] r.city = tweet['near'] response += r except KeyError: pass return response
def dotransform(request, response): p = JSONDecoder().decode( pipljsonsearch( first_name=request.fields['firstname'], last_name=request.fields['lastname'] ) ) if 'error' in p: response += UIMessage(p['error']) for r in p['results']['records']: if 'addresses' in r: for a in r['addresses']: e = Location(a['display']) e.countrycode = a['country'] e += Label( 'Source', '<a href="%s">%s</a>' % (r['source']['url'], r['source']['@ds_name']), type='text/html' ) response += e return response
def do_transform(self, request, response, config): person = request.entity fields = person.fields if fields.get("properties.url"): url = fields.get("properties.url").value else: url = None soup = scrape(url) if soup: email_addresses = soup.find_all(attrs={"class": "__cf_email__"}) for email_address in email_addresses: fp = email_address['data-cfemail'] r = int(fp[:2], 16) email = ''.join([ chr(int(fp[i:i + 2], 16) ^ r) for i in range(2, len(fp), 2) ]) response += EmailAddress(email) phone_numbers = soup.find_all(attrs={"data-link-to-more": "phone"}) for phone_number in phone_numbers: response += PhoneNumber(phone_number.get_text()) addresses = soup.find_all(attrs={"data-link-to-more": "address"}) for unformated_address in addresses: address = unformated_address.get_text().split(" ") address[-1] = address[-1].split("-")[0] response += Location(" ".join(address)) relatives = soup.find_all(attrs={"data-link-to-more": "relative"}) for relative in relatives: response += TruePerson( relative.get_text(), properties_url=config['TruePeopleSearch.local.base_url'] + relative['href']) associates = soup.find_all( attrs={"data-link-to-more": "associate"}) for associate in associates: response += TruePerson( associate.get_text(), properties_url=config['TruePeopleSearch.local.base_url'] + associate['href']) return response
def do_transform(self, request, response, config): person = request.entity fields = person.fields if fields.get("properties.url"): url = fields.get("properties.url").value else: url = None soup = scrape(url) if soup: addresses = soup.find_all(attrs={"data-link-to-more": "address"}) for address in addresses: response += Location(address.get_text()) return response
def do_transform(self, request, response, config): person = request.entity fields = person.fields if fields.get("properties.url"): url = fields.get("properties.url").value else: url = None soup = scrape(url) if soup: addresses = soup.find_all(attrs={"data-link-to-more": "address"}) if addresses: address = addresses[0].get_text().split(" ") address[-1] = address[-1].split("-")[0] response += Location(" ".join(address)) return response
def dotransform(request, response, config): if 'workspace' in request.fields: workspace = request.fields['workspace'] else: workspace = request.value dbcon = db_connect(workspace) host_list = get_hosts(dbcon) for loc in host_list: if loc[0] == request.value or loc[1] == request.value: e = Location(str(loc[2]) + ', ' + str(loc[3]), latitude=float(int(loc[4])), longitude=float(int(loc[5]))) e += Field("workspace", workspace, displayname='Workspace') response += e return response
def do_transform(self, request, response, config): # don't forget to add `from maltego.message import Bookmark` ip_address = request.entity.value url_template = config['hello.local.geo_ip_url'] geoip_str = urlopen(url_template.format(ip=ip_address)).read() geoip_json = json.loads(geoip_str) country_code = geoip_json.get('country_code').lower() response += Location( country=geoip_json.get('country_name', 'Unknown'), city=geoip_json.get('city'), countrycode=geoip_json.get('country_code'), latitude=geoip_json.get('latitude'), longitude=geoip_json.get('longitude'), area=geoip_json.get('region_name'), link_label='From FreeGeoIP', bookmark=Bookmark.Orange, icon_url='http://www.geoips.com/assets/img/flag/256/%s.png' % country_code) return response
def getlocs(response, data): for loc in data: l = loc['place']['location'] e = Location('%s, %s' % (l.get('city', ''), l.get('country', ''))) if 'country' in l: e.country = l['country'] if 'city' in l: e.city = l['city'] if 'state' in l: e.area = l['state'] if 'longitude' in l and 'latitude' in l: e.longitude = l['longitude'] e.latitude = l['latitude'] sa = '' if 'name' in loc['place']: sa = loc['place']['name'] if 'street' in l: sa += ', %s' % l['street'] if 'zip' in l: sa += ', %s' % l['zip'] e += Field('streetaddress', sa) response += e
def dotransform(request, response, config): tr_details = [ 'Reference', 'Source', 'KillChain', 'Firstseen', 'Lastseen', 'Attribution', 'ProcessType', 'Rrname', 'Rdata', 'Country', 'Tags', 'Comment', 'RootNode', 'Confidence' ] #Default link color is black linkcolor = "0x000000" cache, found = search(request.value) if found: if list == type(found): for indicator in found: debug(indicator) e = '' indtype = indicator['Type'].lower().strip() if "whois email" == indtype: e = EmailAddress(indicator['Indicator']) #response += e if "name server" == indtype: e = NSRecord(indicator['Indicator']) #response += e if "domain" == indtype: e = Domain(indicator['Indicator']) e.fqdn = indicator['Indicator'] #response += e #IF Type is not domain, check if Rrname is not empty elif indicator['Rrname'] and indicator['Rrname'] != 'NA': d = Domain(indicator['Rrname']) d.fqdn = indicator['Rrname'] response += d if "ip" == indtype: e = IPv4Address(indicator['Indicator']) #response += e #IF Type is not IP, check if Rdata is not empty elif indicator['Rdata']: i = IPv4Address(indicator['Rdata']) response += i if "phone or fax no." == indtype: e = PhoneNumber(indicator['Indicator']) #response += e if "whois address component" == indtype: e = Phrase(indicator['Indicator']) #response += e if "email" == indtype: e = EmailAddress(indicator['Indicator']) #response += e if "netname" == indtype: e = NetNameThreatRecon(indicator['Indicator']) #response += e if "cidr" == indtype: e = IPv4Address(indicator['Indicator']) #response += e if "netrange" == indtype: e = Netblock(indicator['Indicator']) #response += e if indicator['Country']: l = Location(indicator['Country']) response += l #Add Comments and details to own Entity entity = e #request.entity #Set comments if indicator['Comment']: entity.notes = string_filter(indicator['Comment']) #Set Details for detail in tr_details: if detail in indicator: if indicator[detail]: entity += Label(name=detail, value=string_filter( indicator[detail])) #Set link color if "Confidence" in indicator: if indicator['Confidence'] >= 70: linkcolor = "0xff0000" entity.linkcolor = linkcolor response += entity return response
def dotransform(request, response, config): try: actor = get_actor(request.fields['ThreatCentral.resourceId']) except ThreatCentralError as err: response += UIMessage(err.value, type='PartialError') return response except KeyError: try: actors = search_actor(request.value) except ThreatCentralError as err: response += UIMessage(err.value, type='PartialError') return response else: i = 0 for actor in actors: try: rtype = lower(actor.get('type')) actor = actor.get('resource') if actor.get('tcScore'): weight = int(actor.get('tcScore')) else: weight = 1 if len(actor) is not 0: if rtype == 'actor': if actor.get('name'): e = Actor(encode_to_utf8(actor.get('name')), weight=weight) e.name = encode_to_utf8(actor.get('name')) e.actor = encode_to_utf8(actor.get('name')) elif actor.get('title'): e = Actor(encode_to_utf8(actor.get('title'))) e.title = encode_to_utf8(actor.get('title')) e.resourceId = actor.get('resourceId') if actor.get('organization'): e.organization = encode_to_utf8( actor.get('organization')) if actor.get('aliases'): e.aliases = ', '.join([ encode_to_utf8(_) for _ in actor.get('aliases') ]) if actor.get('country'): e.country = encode_to_utf8( actor.get('country', dict()).get('displayName')) if actor.get('score'): e.score = actor.get('score') if actor.get('links'): e += Label( 'Links', '<br/>'.join([ '<a href="{}">{}</a>'.format( _.get('href'), _.get('href')) for _ in actor.get('links') ])) if actor.get('hyperlinks'): e += Label( 'Hyperlinks', '<br/>'.join([ '<a href="{}">{}</a>'.format( _.get('url'), _.get('title')) for _ in actor.get('hyperlinks') ])) if actor.get('title'): e += Label('Title', encode_to_utf8(actor.get('title'))) if actor.get('resourceId'): e += Label('ResourceID', actor.get('resourceId')) if actor.get('aliases'): e += Label( 'Aliases', '<br/>'.join([ encode_to_utf8(_) for _ in actor.get('aliases', '') ])) if actor.get('description'): e += Label( 'Description', '<br/>'.join( encode_to_utf8( actor.get('description', '')).split('\n'))) if actor.get('country'): e += Label( 'Country', encode_to_utf8( actor.get('country', dict()).get('displayName'))) if actor.get('organization'): e += Label( 'Organization', encode_to_utf8(actor.get('organization'))) if actor.get('types'): e += Label( 'Types', '<br/>'.join([ encode_to_utf8(_.get('displayName')) for _ in actor.get('types') ])) if actor.get('motivations'): e += Label( 'Motivations', '<br/>'.join([ encode_to_utf8(_.get('displayName')) for _ in actor.get('motivations') ])) if actor.get('intendedEffects'): e += Label( 'Intended Effects', '<br/>'.join([ encode_to_utf8(_.get('displayName')) for _ in actor.get('intendedEffects') ])) if actor.get('sophistication'): e += Label( 'Sophistication', actor.get('sophistication', dict()).get('displayName')) if actor.get('socialMediaText'): e += Label( 'Social Media', '<br/>'.join( encode_to_utf8( actor.get('socialMediaText', '')).split('\n'))) if actor.get('moreInfo'): e += Label( 'More Info', '<br/>'.join( encode_to_utf8( actor.get('moreInfo', '')).split('\n'))) if actor.get('score'): e += Label('Score', actor.get('score')) if i < 1: i += 1 e.linkcolor = "0xf90000" response += e except AttributeError as err: response += UIMessage(err, type='PartialError') continue except ThreatCentralError as err: response += UIMessage(err.value, type='PartialError') except TypeError: return response else: if actor: try: if actor.get('tcScore'): weight = int(actor.get('tcScore')) else: weight = 1 # Update entity? e = Actor(request.value, weight=weight) if actor.get('name'): e.name = encode_to_utf8(actor.get('name')) e.actor = encode_to_utf8(actor.get('name')) e.title = encode_to_utf8(actor.get('title')) e.resourceId = actor.get('resourceId') if actor.get('organization'): e.organization = encode_to_utf8(actor.get('organization')) if actor.get('aliases'): e.aliases = ', '.join( [encode_to_utf8(_) for _ in actor.get('aliases')]) if actor.get('country'): e.country = encode_to_utf8( actor.get('country', dict()).get('displayName')) # Add Location entitie l = Location( encode_to_utf8( actor.get('country', dict()).get('displayName'))) response += l if actor.get('score'): e.score = actor.get('score') if actor.get('links'): e += Label( 'Links', '<br/>'.join([ '<a href="{}">{}</a>'.format( _.get('href'), _.get('href')) for _ in actor.get('links') ])) if actor.get('hyperlinks'): e += Label( 'Hyperlinks', '<br/>'.join([ '<a href="{}">{}</a>'.format( _.get('url'), _.get('title')) for _ in actor.get('hyperlinks') ])) if actor.get('title'): e += Label('Title', encode_to_utf8(actor.get('title'))) if actor.get('resourceId'): e += Label('ResourceID', actor.get('resourceId')) if actor.get('aliases'): e += Label( 'Aliases', '<br/>'.join([ encode_to_utf8(_) for _ in actor.get('aliases', '') ])) if actor.get('description'): e += Label( 'Description', '<br/>'.join( encode_to_utf8(actor.get('description', '')).split('\n'))) if actor.get('country'): e += Label( 'Country', encode_to_utf8( actor.get('country', dict()).get('displayName'))) if actor.get('organization'): e += Label('Organization', encode_to_utf8(actor.get('organization'))) if actor.get('types'): e += Label( 'Types', '<br/>'.join([ encode_to_utf8(_.get('displayName')) for _ in actor.get('types') ])) if actor.get('motivations'): e += Label( 'Motivations', '<br/>'.join([ encode_to_utf8(_.get('displayName')) for _ in actor.get('motivations') ])) if actor.get('intendedEffects'): e += Label( 'Intended Effects', '<br/>'.join([ encode_to_utf8(_.get('displayName')) for _ in actor.get('intendedEffects') ])) if actor.get('sophistication'): e += Label( 'Sophistication', encode_to_utf8( actor.get('sophistication', dict()).get('displayName'))) if actor.get('socialMediaText'): e += Label( 'Social Media', '<br/>'.join( encode_to_utf8(actor.get('socialMediaText', '')).split('\n'))) if actor.get('moreInfo'): e += Label( 'More Info', '<br/>'.join( encode_to_utf8(actor.get('moreInfo', '')).split('\n'))) if actor.get('score'): e += Label('Score', actor.get('score')) response += e # Extract email addresses usable_info = search_for_usable_info('{} {} {}'.format( encode_to_utf8(actor.get('description')), encode_to_utf8(actor.get('socialMediaText')), encode_to_utf8(actor.get('moreInfo')))) if usable_info: debug(usable_info) try: urls = usable_info.get('url', dict()) for twitter in urls.get('twitter', list()): t = Twitter(twitter.get('name')) t.uid = twitter.get('name') t.set_field('affiliation.profile-url', twitter.get('url')) response += t for facebook in urls.get('facebook', list()): f = Facebook(facebook.get('name')) f.uid = facebook.get('name') f.set_field('affiliation.profile-url', facebook.get('url')) response += f for other in urls.get('other', list()): u = URL(other) u.url = other response += u emailaddr = usable_info.get('email', list()) for email in emailaddr: e = EmailAddress(email) response += e except AttributeError as err: response += UIMessage('Error: {}'.format(err)) except AttributeError as err: response += UIMessage('Error: {}'.format(err), type='PartialError') except ThreatCentralError as err: response += UIMessage(err.value, type='PartialError') except TypeError: return response return response
def dotransform(request, response): try: s = SNMPManager(*snmpargs(request)) response += Location(s.location) except SNMPError, s: response += UIMessage(str(s))