Beispiel #1
0
def dotransform(request, response):
    r = geoip(request.value)
    if r is not None:
        if 'error' in r:
            response += UIMessage(r['error'])
            return response
        locname = ''
        cityf = None
        countryf = None
        if 'city' in r:
            locname += r['city']
            cityf = r['city']
        if 'countryName' in r:
            locname += ', %s' % r['countryName']
            countryf = r['countryName']
        e = Location(locname)
        if 'longitude' in r and 'latitude' in r:
            e.longitude = r['longitude']
            e.latitude = r['latitude']
            link = maplink(r)
            e += Label('Map It', A(link, link), type='text/html')
        if 'region' in r:
            e.area = r['region']
        if cityf is not None:
            e.city = cityf
        if countryf is not None:
            e.country = countryf
            e.iconurl = flag(countryf)
        if 'countryCode' in r:
            e.countrycode =  r['countryCode']
            if e.iconurl is None:
                e.iconurl = flag(r['countryCode'])
        response += e
    return response
Beispiel #2
0
def dotransform(request, response):

    # Download GeoIP Database from MaxMinds
    if not os.path.exists('/opt/geoipdb/geoipdb.dat'): 
        return response + UIMessage('Need local install of MaxMinds Geo IP database, use the download script in resource/external/geoipdownload.sh')

    gi = pygeoip.GeoIP('/opt/geoipdb/geoipdb.dat')

    pcap = request.value
    pkts = rdpcap(pcap)

    ip_raw = []
    ip_geo = []
    ip_exclusions = ['192.168.', '172.16.', '10.']

    for x in pkts:
        if x.haslayer(IP):
            src = x.getlayer(IP).src
            if src != '0.0.0.0':
                if src not in ip_raw:
                    ip_raw.append(src)

    for s in ip_raw:
        if ip_exclusions[0] in s or ip_exclusions[1] in s or ip_exclusions[2] in s:
            pass
        else:
            rec = gi.record_by_addr(s)
            city = rec['city']
            postcode = rec['postal_code']
            country = rec['country_name']
            lng = rec['longitude']
            lat = rec['latitude']
            ccode = rec['country_code']
            google_map_url = 'https://maps.google.co.uk/maps?z=20&q=%s,%s' %(lat, lng)
            geo_ip = s,city, postcode, country, ccode, str(lng), str(lat), google_map_url
            if geo_ip not in ip_geo:
                ip_geo.append(geo_ip)

    for ip, city, postcode, country, ccode, lng, lat, gmap in ip_geo:
        e = Location(country)
        e.country = country
        e.city = city
        e.linkcolor = 0x2314CA
        e.linklabel = ip
        e.areacode = postcode
        e.longitude = float(lng)
        e.latitude = float(lat)
        e.countrycode = ccode
        e += Field('ipaddress', ip, displayname='IP Address')
        e += Field('geomapurl', gmap, displayname='Google Map URL')
        e += Field('pcapsrc', pcap, displayname='Original pcap File')
        response += e
    return response
Beispiel #3
0
def getlocbymac(mac):
    ll = geomac(mac)
    gcr = reversegeo(ll['latitude'], ll['longitude'])[0]
    l = Location('-, -')
    l.city = '-'
    l.country = '-'
    for i in gcr['address_components']:
        if 'locality' in i['types']:
            l.city = i['long_name']
        if 'administrative_area_level_1' in i['types']:
            l.area = i['long_name']
        if 'country' in i['types']:
            l.country = i['long_name']
    l.latitude = gcr['geometry']['location']['lat']
    l.longitude = gcr['geometry']['location']['lng']
    l.value = '%s, %s' % (l.city, l.country)
    return l
 def do_transform(self, request, response, config):
     tweet = request.entity
     _body = {
         'query': {
             'match': {
                 'id': tweet.id
             }
         },
         'size': request.limits.hard
     }
     res = es.search(index="twinttweets", body=_body)
     for hit in res['hits']['hits']:
         tweet = hit['_source']
         r = Location()
         if tweet['location']:
             r.name = tweet['location']
             response += r
     return response
Beispiel #5
0
def dotransform(request, response):
    p = JSONDecoder().decode(
        pipljsonsearch(first_name=request.entity.firstnames or '',
                       last_name=request.entity.lastname or ''))

    if 'error' in p:
        response += UIMessage(p['error'])

    for r in p['results']['records']:
        if 'addresses' in r:
            for a in r['addresses']:
                e = Location(a['display'])
                e.countrycode = a['country']
                e += Label('Source',
                           '<a href="%s">%s</a>' %
                           (r['source']['url'], r['source']['@ds_name']),
                           type='text/html')
                response += e

    return response
Beispiel #6
0
def dotransform(request, response, config):

    if 'workspace' in request.fields:
        workspace = request.fields['workspace']
    else:
        workspace = request.value

    e = Location(request.fields["location"].decode('ascii'))
    e += Field("workspace", workspace, displayname='Workspace')
    response += e

    return response
 def do_transform(self, request, response, config):
     tweet = request.entity
     _body = {
         'query': {
             'match': {
                 'id': tweet.id
             }
         },
         'size': request.limits.hard
     }
     res = es.search(index="twinttweets", body=_body)
     for hit in res['hits']['hits']:
         tweet = hit['_source']
         r = Location()
         try:
             r.longitude = tweet['geo_near']['lon']
             r.latitude = tweet['geo_near']['lon']
             r.city = tweet['near']
             response += r
         except KeyError:
             pass
     return response
Beispiel #8
0
def dotransform(request, response):
    p = JSONDecoder().decode(
        pipljsonsearch(
            first_name=request.fields['firstname'],
            last_name=request.fields['lastname']
        )
    )

    if 'error' in p:
        response += UIMessage(p['error'])

    for r in p['results']['records']:
        if 'addresses' in r:
            for a in r['addresses']:
                e = Location(a['display'])
                e.countrycode = a['country']
                e += Label(
                    'Source', '<a href="%s">%s</a>' % (r['source']['url'], r['source']['@ds_name']), type='text/html'
                )
                response += e

    return response
    def do_transform(self, request, response, config):
        person = request.entity
        fields = person.fields

        if fields.get("properties.url"):
            url = fields.get("properties.url").value
        else:
            url = None

        soup = scrape(url)

        if soup:
            email_addresses = soup.find_all(attrs={"class": "__cf_email__"})
            for email_address in email_addresses:
                fp = email_address['data-cfemail']
                r = int(fp[:2], 16)
                email = ''.join([
                    chr(int(fp[i:i + 2], 16) ^ r)
                    for i in range(2, len(fp), 2)
                ])
                response += EmailAddress(email)

            phone_numbers = soup.find_all(attrs={"data-link-to-more": "phone"})
            for phone_number in phone_numbers:
                response += PhoneNumber(phone_number.get_text())

            addresses = soup.find_all(attrs={"data-link-to-more": "address"})
            for unformated_address in addresses:
                address = unformated_address.get_text().split(" ")
                address[-1] = address[-1].split("-")[0]
                response += Location(" ".join(address))

            relatives = soup.find_all(attrs={"data-link-to-more": "relative"})
            for relative in relatives:
                response += TruePerson(
                    relative.get_text(),
                    properties_url=config['TruePeopleSearch.local.base_url'] +
                    relative['href'])

            associates = soup.find_all(
                attrs={"data-link-to-more": "associate"})
            for associate in associates:
                response += TruePerson(
                    associate.get_text(),
                    properties_url=config['TruePeopleSearch.local.base_url'] +
                    associate['href'])

        return response
    def do_transform(self, request, response, config):
        person = request.entity
        fields = person.fields

        if fields.get("properties.url"):
            url = fields.get("properties.url").value
        else:
            url = None

        soup = scrape(url)

        if soup:
            addresses = soup.find_all(attrs={"data-link-to-more": "address"})
            for address in addresses:
                response += Location(address.get_text())

        return response
Beispiel #11
0
    def do_transform(self, request, response, config):
        person = request.entity
        fields = person.fields

        if fields.get("properties.url"):
            url = fields.get("properties.url").value
        else:
            url = None

        soup = scrape(url)

        if soup:
            addresses = soup.find_all(attrs={"data-link-to-more": "address"})
            if addresses:
                address = addresses[0].get_text().split(" ")
                address[-1] = address[-1].split("-")[0]
                response += Location(" ".join(address))

        return response
def dotransform(request, response, config):

    if 'workspace' in request.fields:
        workspace = request.fields['workspace']
    else:
        workspace = request.value

    dbcon = db_connect(workspace)
    host_list = get_hosts(dbcon)

    for loc in host_list:
        if loc[0] == request.value or loc[1] == request.value:
            e = Location(str(loc[2]) + ', ' + str(loc[3]),
                         latitude=float(int(loc[4])),
                         longitude=float(int(loc[5])))
            e += Field("workspace", workspace, displayname='Workspace')
            response += e

    return response
    def do_transform(self, request, response, config):
        # don't forget to add `from maltego.message import Bookmark`
        ip_address = request.entity.value

        url_template = config['hello.local.geo_ip_url']
        geoip_str = urlopen(url_template.format(ip=ip_address)).read()
        geoip_json = json.loads(geoip_str)
        country_code = geoip_json.get('country_code').lower()

        response += Location(
            country=geoip_json.get('country_name', 'Unknown'),
            city=geoip_json.get('city'),
            countrycode=geoip_json.get('country_code'),
            latitude=geoip_json.get('latitude'),
            longitude=geoip_json.get('longitude'),
            area=geoip_json.get('region_name'),
            link_label='From FreeGeoIP',
            bookmark=Bookmark.Orange,
            icon_url='http://www.geoips.com/assets/img/flag/256/%s.png' %
            country_code)

        return response
def getlocs(response, data):

    for loc in data:
        l = loc['place']['location']
        e = Location('%s, %s' % (l.get('city', ''), l.get('country', '')))
        if  'country' in l:
            e.country = l['country']
        if 'city' in l:
            e.city = l['city']
        if 'state' in l:
            e.area = l['state']
        if 'longitude' in l and 'latitude' in l:
            e.longitude = l['longitude']
            e.latitude = l['latitude']
        sa = ''
        if 'name' in loc['place']:
            sa = loc['place']['name']
        if 'street' in l:
            sa += ', %s' % l['street']
        if 'zip' in l:
            sa += ', %s' % l['zip']
        e += Field('streetaddress', sa)
        response += e
def dotransform(request, response, config):

    tr_details = [
        'Reference', 'Source', 'KillChain', 'Firstseen', 'Lastseen',
        'Attribution', 'ProcessType', 'Rrname', 'Rdata', 'Country', 'Tags',
        'Comment', 'RootNode', 'Confidence'
    ]

    #Default link color is black
    linkcolor = "0x000000"

    cache, found = search(request.value)

    if found:
        if list == type(found):
            for indicator in found:
                debug(indicator)
                e = ''
                indtype = indicator['Type'].lower().strip()

                if "whois email" == indtype:
                    e = EmailAddress(indicator['Indicator'])
                    #response += e

                if "name server" == indtype:
                    e = NSRecord(indicator['Indicator'])
                    #response += e

                if "domain" == indtype:
                    e = Domain(indicator['Indicator'])
                    e.fqdn = indicator['Indicator']
                    #response += e
                #IF Type is not domain, check if Rrname is not empty
                elif indicator['Rrname'] and indicator['Rrname'] != 'NA':
                    d = Domain(indicator['Rrname'])
                    d.fqdn = indicator['Rrname']
                    response += d

                if "ip" == indtype:
                    e = IPv4Address(indicator['Indicator'])
                    #response += e
                #IF Type is not IP, check if Rdata is not empty
                elif indicator['Rdata']:
                    i = IPv4Address(indicator['Rdata'])
                    response += i

                if "phone or fax no." == indtype:
                    e = PhoneNumber(indicator['Indicator'])
                    #response += e

                if "whois address component" == indtype:
                    e = Phrase(indicator['Indicator'])
                    #response += e

                if "email" == indtype:
                    e = EmailAddress(indicator['Indicator'])
                    #response += e

                if "netname" == indtype:
                    e = NetNameThreatRecon(indicator['Indicator'])
                    #response += e

                if "cidr" == indtype:
                    e = IPv4Address(indicator['Indicator'])
                    #response += e

                if "netrange" == indtype:
                    e = Netblock(indicator['Indicator'])
                    #response += e

                if indicator['Country']:
                    l = Location(indicator['Country'])
                    response += l

                #Add Comments and details to own Entity
                entity = e  #request.entity

                #Set comments
                if indicator['Comment']:
                    entity.notes = string_filter(indicator['Comment'])

                    #Set Details
                for detail in tr_details:
                    if detail in indicator:
                        if indicator[detail]:
                            entity += Label(name=detail,
                                            value=string_filter(
                                                indicator[detail]))

                #Set link color
                if "Confidence" in indicator:
                    if indicator['Confidence'] >= 70:
                        linkcolor = "0xff0000"

                entity.linkcolor = linkcolor

                response += entity

    return response
Beispiel #16
0
def dotransform(request, response, config):

    try:
        actor = get_actor(request.fields['ThreatCentral.resourceId'])
    except ThreatCentralError as err:
        response += UIMessage(err.value, type='PartialError')
        return response
    except KeyError:
        try:
            actors = search_actor(request.value)
        except ThreatCentralError as err:
            response += UIMessage(err.value, type='PartialError')
            return response
        else:
            i = 0
            for actor in actors:
                try:
                    rtype = lower(actor.get('type'))
                    actor = actor.get('resource')

                    if actor.get('tcScore'):
                        weight = int(actor.get('tcScore'))
                    else:
                        weight = 1

                    if len(actor) is not 0:

                        if rtype == 'actor':
                            if actor.get('name'):
                                e = Actor(encode_to_utf8(actor.get('name')),
                                          weight=weight)
                                e.name = encode_to_utf8(actor.get('name'))
                                e.actor = encode_to_utf8(actor.get('name'))
                            elif actor.get('title'):
                                e = Actor(encode_to_utf8(actor.get('title')))

                            e.title = encode_to_utf8(actor.get('title'))
                            e.resourceId = actor.get('resourceId')
                            if actor.get('organization'):
                                e.organization = encode_to_utf8(
                                    actor.get('organization'))
                            if actor.get('aliases'):
                                e.aliases = ', '.join([
                                    encode_to_utf8(_)
                                    for _ in actor.get('aliases')
                                ])
                            if actor.get('country'):
                                e.country = encode_to_utf8(
                                    actor.get('country',
                                              dict()).get('displayName'))
                            if actor.get('score'):
                                e.score = actor.get('score')

                            if actor.get('links'):
                                e += Label(
                                    'Links', '<br/>'.join([
                                        '<a href="{}">{}</a>'.format(
                                            _.get('href'), _.get('href'))
                                        for _ in actor.get('links')
                                    ]))
                            if actor.get('hyperlinks'):
                                e += Label(
                                    'Hyperlinks', '<br/>'.join([
                                        '<a href="{}">{}</a>'.format(
                                            _.get('url'), _.get('title'))
                                        for _ in actor.get('hyperlinks')
                                    ]))

                            if actor.get('title'):
                                e += Label('Title',
                                           encode_to_utf8(actor.get('title')))
                            if actor.get('resourceId'):
                                e += Label('ResourceID',
                                           actor.get('resourceId'))

                            if actor.get('aliases'):
                                e += Label(
                                    'Aliases', '<br/>'.join([
                                        encode_to_utf8(_)
                                        for _ in actor.get('aliases', '')
                                    ]))
                            if actor.get('description'):
                                e += Label(
                                    'Description', '<br/>'.join(
                                        encode_to_utf8(
                                            actor.get('description',
                                                      '')).split('\n')))

                            if actor.get('country'):
                                e += Label(
                                    'Country',
                                    encode_to_utf8(
                                        actor.get('country',
                                                  dict()).get('displayName')))
                            if actor.get('organization'):
                                e += Label(
                                    'Organization',
                                    encode_to_utf8(actor.get('organization')))
                            if actor.get('types'):
                                e += Label(
                                    'Types', '<br/>'.join([
                                        encode_to_utf8(_.get('displayName'))
                                        for _ in actor.get('types')
                                    ]))

                            if actor.get('motivations'):
                                e += Label(
                                    'Motivations', '<br/>'.join([
                                        encode_to_utf8(_.get('displayName'))
                                        for _ in actor.get('motivations')
                                    ]))

                            if actor.get('intendedEffects'):
                                e += Label(
                                    'Intended Effects', '<br/>'.join([
                                        encode_to_utf8(_.get('displayName'))
                                        for _ in actor.get('intendedEffects')
                                    ]))

                            if actor.get('sophistication'):
                                e += Label(
                                    'Sophistication',
                                    actor.get('sophistication',
                                              dict()).get('displayName'))

                            if actor.get('socialMediaText'):
                                e += Label(
                                    'Social Media', '<br/>'.join(
                                        encode_to_utf8(
                                            actor.get('socialMediaText',
                                                      '')).split('\n')))

                            if actor.get('moreInfo'):
                                e += Label(
                                    'More Info', '<br/>'.join(
                                        encode_to_utf8(
                                            actor.get('moreInfo',
                                                      '')).split('\n')))

                            if actor.get('score'):
                                e += Label('Score', actor.get('score'))

                            if i < 1:
                                i += 1
                                e.linkcolor = "0xf90000"

                            response += e

                except AttributeError as err:
                    response += UIMessage(err, type='PartialError')
                    continue
                except ThreatCentralError as err:
                    response += UIMessage(err.value, type='PartialError')
                except TypeError:
                    return response
    else:
        if actor:
            try:
                if actor.get('tcScore'):
                    weight = int(actor.get('tcScore'))
                else:
                    weight = 1

                # Update entity?
                e = Actor(request.value, weight=weight)
                if actor.get('name'):
                    e.name = encode_to_utf8(actor.get('name'))
                    e.actor = encode_to_utf8(actor.get('name'))

                e.title = encode_to_utf8(actor.get('title'))
                e.resourceId = actor.get('resourceId')
                if actor.get('organization'):
                    e.organization = encode_to_utf8(actor.get('organization'))
                if actor.get('aliases'):
                    e.aliases = ', '.join(
                        [encode_to_utf8(_) for _ in actor.get('aliases')])
                if actor.get('country'):
                    e.country = encode_to_utf8(
                        actor.get('country', dict()).get('displayName'))
                    # Add Location entitie
                    l = Location(
                        encode_to_utf8(
                            actor.get('country', dict()).get('displayName')))
                    response += l
                if actor.get('score'):
                    e.score = actor.get('score')

                if actor.get('links'):
                    e += Label(
                        'Links', '<br/>'.join([
                            '<a href="{}">{}</a>'.format(
                                _.get('href'), _.get('href'))
                            for _ in actor.get('links')
                        ]))
                if actor.get('hyperlinks'):
                    e += Label(
                        'Hyperlinks', '<br/>'.join([
                            '<a href="{}">{}</a>'.format(
                                _.get('url'), _.get('title'))
                            for _ in actor.get('hyperlinks')
                        ]))

                if actor.get('title'):
                    e += Label('Title', encode_to_utf8(actor.get('title')))
                if actor.get('resourceId'):
                    e += Label('ResourceID', actor.get('resourceId'))
                if actor.get('aliases'):
                    e += Label(
                        'Aliases', '<br/>'.join([
                            encode_to_utf8(_)
                            for _ in actor.get('aliases', '')
                        ]))
                if actor.get('description'):
                    e += Label(
                        'Description', '<br/>'.join(
                            encode_to_utf8(actor.get('description',
                                                     '')).split('\n')))
                if actor.get('country'):
                    e += Label(
                        'Country',
                        encode_to_utf8(
                            actor.get('country', dict()).get('displayName')))
                if actor.get('organization'):
                    e += Label('Organization',
                               encode_to_utf8(actor.get('organization')))
                if actor.get('types'):
                    e += Label(
                        'Types', '<br/>'.join([
                            encode_to_utf8(_.get('displayName'))
                            for _ in actor.get('types')
                        ]))

                if actor.get('motivations'):
                    e += Label(
                        'Motivations', '<br/>'.join([
                            encode_to_utf8(_.get('displayName'))
                            for _ in actor.get('motivations')
                        ]))
                if actor.get('intendedEffects'):
                    e += Label(
                        'Intended Effects', '<br/>'.join([
                            encode_to_utf8(_.get('displayName'))
                            for _ in actor.get('intendedEffects')
                        ]))
                if actor.get('sophistication'):
                    e += Label(
                        'Sophistication',
                        encode_to_utf8(
                            actor.get('sophistication',
                                      dict()).get('displayName')))
                if actor.get('socialMediaText'):
                    e += Label(
                        'Social Media', '<br/>'.join(
                            encode_to_utf8(actor.get('socialMediaText',
                                                     '')).split('\n')))
                if actor.get('moreInfo'):
                    e += Label(
                        'More Info', '<br/>'.join(
                            encode_to_utf8(actor.get('moreInfo',
                                                     '')).split('\n')))

                if actor.get('score'):
                    e += Label('Score', actor.get('score'))

                response += e

                # Extract email addresses
                usable_info = search_for_usable_info('{} {} {}'.format(
                    encode_to_utf8(actor.get('description')),
                    encode_to_utf8(actor.get('socialMediaText')),
                    encode_to_utf8(actor.get('moreInfo'))))
                if usable_info:
                    debug(usable_info)
                    try:
                        urls = usable_info.get('url', dict())
                        for twitter in urls.get('twitter', list()):
                            t = Twitter(twitter.get('name'))
                            t.uid = twitter.get('name')
                            t.set_field('affiliation.profile-url',
                                        twitter.get('url'))
                            response += t

                        for facebook in urls.get('facebook', list()):
                            f = Facebook(facebook.get('name'))
                            f.uid = facebook.get('name')
                            f.set_field('affiliation.profile-url',
                                        facebook.get('url'))
                            response += f

                        for other in urls.get('other', list()):
                            u = URL(other)
                            u.url = other
                            response += u

                        emailaddr = usable_info.get('email', list())
                        for email in emailaddr:
                            e = EmailAddress(email)
                            response += e

                    except AttributeError as err:
                        response += UIMessage('Error: {}'.format(err))

            except AttributeError as err:
                response += UIMessage('Error: {}'.format(err),
                                      type='PartialError')
            except ThreatCentralError as err:
                response += UIMessage(err.value, type='PartialError')
            except TypeError:
                return response

    return response
Beispiel #17
0
def dotransform(request, response):
    try:
        s = SNMPManager(*snmpargs(request))
        response += Location(s.location)
    except SNMPError, s:
        response += UIMessage(str(s))