def dotransform(request, response, config): error, found = lookup_whois(request.value) if not error and found: if dict == type(found): for result, value in found.iteritems(): if set == type(value): if "whois_domains" == result: for d in value: if d: e = Domain(d) e.fqdn = d response += e if "whois_emails" == result: for em in value: if em: e = EmailAddress(em) response += e if "whois_nameservers" == result: for w in value: if w: e = NSRecord(w) response += e #Display error message in Transform Output response += UIMessage(error) return response
def dotransform(request, response): dns = request.value if '.' in dns: response += Domain('.'.join(dns.split('.')[-2:])) else: response += Domain(request.value) return response
def dotransform(request, response): page = build(request.value) try: single = page.find( text='The following Host Name was requested from a host database:' ).findNext() except: single = None pass try: single = page.find( text='The following Host Names were requested from a host database:' ).findNext() except: single = None pass try: single2 = page.find( text='The following Internet Connection was established:' ).findNext() except: single2 = None pass try: multi = page.find( text='The following Internet Connections were established:' ).findNext('table') except: multi = None pass if single is not None: for dom in single.findAll("li"): text = dom.text response += Domain(text) if single2 is not None: dom = single2.findNext('tr').findNext('tr').findNext('td') text = dom.text response += Domain(text) if multi is not None: for entry in multi.findAll('tr')[1::]: dom = entry.findNext('td') text = dom.text response += Domain(text) return response
def dotransform(request, response): # Report transform progress progress(50) ip = request.value total = "" urldom = 'https://www.virustotal.com/en/ip-address/' + ip + '/information/' soup = BeautifulSoup(urllib2.urlopen(urldom).read()) try: links = soup.findAll('div', attrs={'class': 'enum'}) for link in links: total += str(link) total = BeautifulSoup(total) for totals in total.findAll('a', href=True): totals = totals['href'] theIP = totals.replace("/en/domain/", "") e = theIP.replace("/information/", "") e = Domain(e) response += e except IOError: print 'IO Error' # Update progress progress(100) # Return response for visualization return response
def dotransform(request, response): # Report transform progress progress(50) ip = request.value url = 'https://www.virustotal.com/vtapi/v2/ip-address/report' parameters = {'ip': ip, 'apikey': config['virustotal/apikey']} resp = urllib2.urlopen('%s?%s' % (url, urllib.urlencode(parameters))).read() response_dict = json.loads(resp) #Latest detected URLs" try: for i in range(0, len(response_dict['resolutions'])): host = response_dict['resolutions'][i]['hostname'] host = Domain(host) response += host except IOError: response = 'IO Error' except KeyError: response = 'Not Found' # Update progress progress(100) # Return response for visualization return response
def dotransform(request, response): page = build(request.value) doms = [] if page.find("span", {"id": "sharedsub"}): section = page.find("span", {"id": "sharedsub"}).findNext('ul') for entry in section.findAll("li"): response += Domain(entry.text) elif page.find("span", {"id": "sharedsubv"}): section = page.find("span", {"id": "sharedsubv"}).findNext('ul') for entry in section.findAll("li"): response += Domain(entry.text) else: response += UIMessage('No subdomains in robtex') return response
def dotransform(request, response): try: items=ast.literal_eval(request.fields['resolutions']) except: return response for item in items: last=item['last_resolved'] host=item['hostname'] r=Domain(host) r.linklabel=last response+=r return response
def do_transform(self, request, response, config): be = BinaryEdge(config['binaryedge.local.api_key']) domain = request.entity.value try: # Only consider the fist page res = be.domain_dns(domain) except BinaryEdgeException as e: raise MaltegoException('BinaryEdge error: %s' % e.msg) else: already = [domain] for event in res['events']: if 'A' in event: for ip in event['A']: if ip not in already: response += IPv4Address(ip) already.append(ip) if 'domain' in event: if event['domain'] not in already: response += Domain(event['domain']) already.append(event['domain']) if 'MX' in event: for mx in event['MX']: if mx not in already: response += MXRecord(mx) already.append(mx) if 'NS' in event: for ns in event['NS']: if ns not in already: response += NSRecord(ns) already.append(ns) return response return response
def dotransform(request, response): if request.fields['behavioral'] != "": try: behavior = ast.literal_eval(request.fields['behavior_data']) except Exception as e: debug("Entity has no behavioral data") return response if behavior.has_key("network"): if behavior['network'].has_key('dns'): for item in behavior['network']['dns']: host = Domain(item['hostname']) host.linklabel = "vt_behav->hosts" response += host if item.has_key('ip'): ip = IPv4Address(item['ip']) ip.linklabel = "vt_behav->hosts" response += ip if behavior['network'].has_key('tcp'): for item in behavior['network']['tcp']: conn = item.split(":") r = IPv4Address(conn[0]) r.linklabel = "vt_behav->hosts_tcp (%s)" % str(conn[1]) response += r if behavior['network'].has_key('udp'): for item in behavior['network']['udp']: conn = item.split(":") r = IPv4Address(conn[0]) r.linklabel = "vt_behav->hosts_udp (%s)" % str(conn[1]) response += r if behavior['network'].has_key('http'): for item in behavior['network']['http']: r = URL(item['url']) r.url = item['url'] r.linklabel = "vt_behav->hosts_http (%s)" % item['method'] response += r else: debug("ripVT: No behavioral for %s" % request.value) return response
def dotransform(request, response, config): if 'taskid' in request.fields: task = request.fields['taskid'] else: task = request.value netw = network(report(task)) for d in netw['domains']: response += Domain(d['domain'].decode('ascii'), taskid=task) return response
def do_transform(self, request, response, config): be = BinaryEdge(config['binaryedge.local.api_key']) ip = request.entity.value try: res = be.host(ip) except BinaryEdgeException as e: raise MaltegoException('BinaryEdge error: %s' % e.msg) else: already = [] for port in res['events']: response += Port(port['port']) for result in port['results']: if result['origin']['type'] == 'ssl': cert = result['result']['data']['cert_info'][ 'certificate_chain'][0] # How to return a certificate ? if 'commonName' in cert['as_dict']['subject']: d = cert['as_dict']['subject']['commonName'] if d not in already: response += Domain(d) already.append(d) if 'extensions' in cert['as_dict']: if 'X509v3 Subject Alternative Name' in cert[ 'as_dict']['extensions']: for domain in cert['as_dict']['extensions'][ 'X509v3 Subject Alternative Name'][ 'DNS']: if domain not in already: response += Domain(domain) already.append(domain) if result['origin']['type'] in ['http', 'grabber']: if 'server' in result['result']['data']['response'][ 'headers']: banner = result['result']['data']['response'][ 'headers']['server'] if banner not in already: response += Banner(banner) already.append(banner) return response
def dotransform(request, response): ns = request.value results = query('-n', ns, 0, 'n') for result in results: data = json.loads(result) if data.has_key('rrname'): if data.has_key('time_first'): first = data['time_first'] last = data['time_last'] elif data.has_key('zone_time_first'): first = data['zone_time_first'] last = data['zone_time_last'] fnice = datetime.datetime.fromtimestamp(int(first)).strftime('%m-%d-%Y') lnice = datetime.datetime.fromtimestamp(int(last)).strftime('%m-%d-%Y') e = Domain(data['rrname'].rstrip('.')) e.linklabel = fnice + ' - ' + lnice response += e return response
def dotransform(request, response): try: items = ast.literal_eval(request.fields['subdomains']) except: return response for item in items: r = Domain(item) response += r return response
def do_transform(self, request, response, config): be = BinaryEdge(config['binaryedge.local.api_key']) domain = request.entity.value try: # Only consider the fist page res = be.domain_subdomains(domain) except BinaryEdgeException as e: raise MaltegoException('BinaryEdge error: %s' % e.msg) else: for e in res["events"]: if e != domain: response += Domain(e) return response
def dotransform(request, response): data = getbehavior(request.value) try: try: network = data['network'] except: #no network data pass try: for result in network['dns']: dom = result['hostname'] ip = result['ip'] response += Domain(dom) response += IPv4Address['ip'] except: #no dns data pass try: for request in network['http']: uri = URL(request['uri']) uri.url = request['uri'] ua = UserAgent(request['user-agent']) #req = HTTPRequest(request['data']) port = Port(request['port']) response += uri response += ua #response += req response += port except: #no http data pass try: for entry in network['tcp']: e = entry['dst'] if e.startswith('10.'): pass else: conn = IPv4Address(e) response += conn except: #no tcp data pass except: response += UIMessage(data['verbose_msg']) return response
def dotransform(request, response): #Build the request type = 'hash' page = build(request.value, type) try: list = page.find(text='PCAP Raw DNS Queries').previous.previous.parent.findAll('p') except: raise MaltegoException('No DNS Queries') for item in list: if item.text != 'none': response += Domain(item.text) return response
def dotransform(request, response): #Build Request page = build(request.value) #Finds the DNS section and extracts domains try: table = page.find("div", {"id": "network_dns"}).findNext('table') elements = table.findAll("span", {"class": "mono"}) for element in elements: text = element.find(text=True) response += Domain(text) except: return response return response
def dotransform(request, response, config): hosts_gather(request.value) hosts_enum(request.value) hosts_geo(request.value) dbcon = db_connect(request.value) host_list = get_hosts(dbcon) for host in host_list: e = Domain(host[0]) e += Field("workspace", request.value, displayname='Workspace') response += e return response
def dotransform(request, response): ns = request.value results = query('-n', ns, 0, 'n') for result in results: data = json.loads(result) if data.has_key('rrname'): if data.has_key('time_first'): first = data['time_first'] last = data['time_last'] elif data.has_key('zone_time_first'): first = data['zone_time_first'] last = data['zone_time_last'] fnice = datetime.datetime.fromtimestamp( int(first)).strftime('%m-%d-%Y') lnice = datetime.datetime.fromtimestamp( int(last)).strftime('%m-%d-%Y') e = Domain(data['rrname'].rstrip('.')) e.linklabel = fnice + ' - ' + lnice response += e return response
def do_transform(self, request, response, config): ip = request.entity.value be = BinaryEdge(config['binaryedge.local.api_key']) try: res = be.domain_ip(ip) except BinaryEdgeException as e: raise MaltegoException('BinaryEdge error: %s' % e.msg) else: already = [] for e in res['events']: if e['domain'] not in already: response += Domain(e['domain']) already.append(e['domain']) return response
def dotransform(request, response, config): if 'workspace' in request.fields: workspace = request.fields['workspace'] else: workspace = request.value dbcon = db_connect(workspace) host_list = get_hosts(dbcon) for host in host_list: e = Domain(host[0]) e += Field("workspace", workspace, displayname='Workspace') response += e return response
def detType(in_val): val = str(in_val) #::Email email = re.compile(".*\[@\][a-z0-9\-]{1,}\.[a-z0-9\-]{1,}") #::IP ipv4 = re.compile("^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$") #::CIDR cidr = re.compile("^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/\d{1,2}$") #::Range v4range = re.compile( "^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\-\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$" ) #::Domain dom = re.compile("([a-z0-9\-]{1,}\.?)+\.[a-z0-9\-]{1,}$") if email.match(val): e = EmailAddress(val) return e if ipv4.match(val): e = IPv4Address(val) return e if cidr.match(val): e = CIDR(val) return e if v4range.match(val): e = Range(val) return e if dom.match(val): e = Domain(val) return e if re.match("^([a-z]*)://", val, re.M | re.I): e = URL(val) e.url = val return e
def dotransform(request, response): dns_results = [] pcap = request.value pkt = rdpcap(pcap) for pkts in pkt: if pkts.haslayer(DNSQR): drec = pkts.getlayer(DNSQR).qname if drec not in dns_results: dns_results.append(drec) for drec in dns_results: e = Domain(drec.strip('.')) e += Field('pcapsrc', pcap, displayname='Original pcap File') response += e return response
def dotransform(request, response): domain = request.value results = query('-r', domain, 0, 'n') for result in results: data = json.loads(result) if data.has_key('time_first'): first = data['time_first'] last = data['time_last'] elif data.has_key('zone_time_first'): first = data['zone_time_first'] last = data['zone_time_last'] fnice = datetime.datetime.fromtimestamp( int(first)).strftime('%m-%d-%Y') lnice = datetime.datetime.fromtimestamp(int(last)).strftime('%m-%d-%Y') if data['rrtype'] == 'NS': for item in data['rdata']: e = NSRecord(item) e.linklabel = fnice + ' - ' + lnice response += e elif data['rrtype'] == 'MX': for item in data['rdata']: e = MXRecord(item) e.linklabel = fnice + ' - ' + lnice response += e elif data['rrtype'] == 'CNAME': for item in data['rdata']: e = Domain(item.rstrip('.')) e.linklabel = fnice + ' - ' + lnice response += e elif data['rrtype'] == 'A': pass else: type = data['rrtype'] for item in data['rdata']: label = type + ' ' + item e = Phrase(label) e.linklabel = fnice + ' - ' + lnice response += e return response
def dotransform(request, response): # Store the pcap file as a variable pcap = request.value usedb = config['working/usedb'] # Check to see if we are using the database or not if usedb > 0: # Connect to the database so we can insert the record created below x = mongo_connect() c = x['DNS'] # Hash the pcap file try: md5hash = md5_for_file(pcap) except Exception as e: return response + UIMessage(str(e)) # Get the session and/or pcap id d = find_session(md5hash) pcap_id = d[0] session_id = d[1] else: pass try: pkts = rdpcap(pcap) dns_requests = [] for p in pkts: if p.haslayer(DNSQR): timestamp = datetime.datetime.fromtimestamp( p.time).strftime('%Y-%m-%d %H:%M:%S.%f') r = p[DNSQR].qname[:-1] tld = tldextract.extract(r) domain = tld.registered_domain if usedb > 0: dns = OrderedDict({ 'PCAP ID': pcap_id, 'Stream ID': session_id, 'Time Stamp': timestamp, 'Type': 'Request', 'IP': { 'src': p[IP].src, 'dst': p[IP].dst, 'length': p[IP].len }, 'Request Details': { 'Query Type': p[DNSQR].qtype, 'Query Name': r, 'Domain': domain } }) t = x.DNS.find({'Time Stamp': timestamp}).count() if t > 0: pass else: c.insert(dns) else: pass if r not in dns_requests: dns_requests.append(domain) else: pass for d in dns_requests: x = Domain(d) response += x return response except Exception as e: if usedb > 0: error_logging(str(e), 'DNS Requests') else: return response + UIMessage(str(e))
def dotransform(request, response, config): command = "--hash_to_domain " + request.value qradio_output = get_qradio_data(command, 0) for entry in qradio_output: response += Domain(entry) return response
def dotransform(request, response): tmp = ".".join(str(request.value).split(".")[-2:]) response += Domain(tmp) return response
def dotransform(request, response, config): if 'ThreatCentral.resourceId' in request.fields: try: indicator = get_indicator( request.fields['ThreatCentral.resourceId']) except ThreatCentralError as err: response += UIMessage(err.value, type='PartialError') else: try: # Update Indicator entity ? e = Indicator(request.value) e.title = encode_to_utf8(indicator.get('title')) e.resourceId = indicator.get('resourceId') e.severity = indicator.get('severity', dict()).get('displayName') e.confidence = indicator.get('confidence', dict()).get('displayName') e.indicatorType = indicator.get('indicatorType', dict()).get('displayName') e += Label( 'Severity', indicator.get('severity', dict()).get('displayName')) e += Label( 'Confidence', indicator.get('confidence', dict()).get('displayName')) e += Label( 'Indicator Type', indicator.get('indicatorType', dict()).get('displayName')) if indicator.get('description'): e += Label( 'Description', '<br/>'.join( encode_to_utf8( indicator.get('description')).split('\n'))) response += e if len(indicator.get('observables', list())) is not 0: for observable in indicator.get('observables'): if upper(observable.get( 'type', dict()).get('value')) == 'DOMAIN': if observable.get('tcScore'): weight = int(observable.get('tcScore')) else: weight = 1 e = Domain(observable.get('value'), weight=weight) if upper( observable.get('location', dict()).get('city') ) != 'UNDEFINED_GEO_LOCATION_STRING': e += Label( 'Location', '<br/>'.join([ '{}:{}'.format(encode_to_utf8(k), encode_to_utf8(v)) for k, v in observable.get( 'location', dict()).iteritems() ])) response += e except AttributeError as err: response += UIMessage('Error: {}'.format(err), type='PartialError') except ThreatCentralError as err: response += UIMessage(err.value, type='PartialError') except TypeError: return response return response
def dotransform(request, response, config): tr_details = [ 'Reference', 'Source', 'KillChain', 'Firstseen', 'Lastseen', 'Attribution', 'ProcessType', 'Rrname', 'Rdata', 'Country', 'Tags', 'Comment', 'RootNode', 'Confidence' ] #Disable cache to get actual data from Threat Recon cache, found = search(request.value, cache=False) #Default linkcolor linkcolor = "0x000000" if found: if defaultdict == type(found): for rootnode, value in found.iteritems(): #If the RootNode is empty, display attributes if len(rootnode) == 0: for indicator in value: #debug(indicator) e = '' indtype = indicator['Type'].lower().strip() if "whois email" == indtype: e = EmailAddress(indicator['Indicator']) if "name server" == indtype: e = NSRecord(indicator['Indicator']) if "domain" == indtype: e = Domain(indicator['Indicator']) e.fqdn = indicator['Indicator'] if "ip" == indtype: e = IPv4Address(indicator['Indicator']) if "phone or fax no." == indtype: e = PhoneNumber(indicator['Indicator']) if "whois address component" == indtype: e = Phrase(indicator['Indicator']) if "email" == indtype: e = EmailAddress(indicator['Indicator']) if "netname" == indtype: e = NetNameThreatRecon(indicator['Indicator']) if "cidr" == indtype: e = IPv4Address(indicator['Indicator']) if "netrange" == indtype: e = Netblock(indicator['Indicator']) if e: #Set linkcolor e.linkcolor = linkcolor #Set comments if indicator['Comment']: e.notes = string_filter(indicator['Comment']) #Set Details for detail in tr_details: if detail in indicator: if indicator[detail]: e += Label(name=detail, value=string_filter( indicator[detail])) response += e else: #Display the RootNodes e = ThreatRecon(rootnode) response += e return response
def dotransform(request, response, config): tr_details = [ 'Reference', 'Source', 'KillChain', 'Firstseen', 'Lastseen', 'Attribution', 'ProcessType', 'Rrname', 'Rdata', 'Country', 'Tags', 'Comment', 'RootNode', 'Confidence' ] #Default link color is black linkcolor = "0x000000" cache, found = search(request.value) if found: if list == type(found): for indicator in found: debug(indicator) e = '' indtype = indicator['Type'].lower().strip() if "whois email" == indtype: e = EmailAddress(indicator['Indicator']) #response += e if "name server" == indtype: e = NSRecord(indicator['Indicator']) #response += e if "domain" == indtype: e = Domain(indicator['Indicator']) e.fqdn = indicator['Indicator'] #response += e #IF Type is not domain, check if Rrname is not empty elif indicator['Rrname'] and indicator['Rrname'] != 'NA': d = Domain(indicator['Rrname']) d.fqdn = indicator['Rrname'] response += d if "ip" == indtype: e = IPv4Address(indicator['Indicator']) #response += e #IF Type is not IP, check if Rdata is not empty elif indicator['Rdata']: i = IPv4Address(indicator['Rdata']) response += i if "phone or fax no." == indtype: e = PhoneNumber(indicator['Indicator']) #response += e if "whois address component" == indtype: e = Phrase(indicator['Indicator']) #response += e if "email" == indtype: e = EmailAddress(indicator['Indicator']) #response += e if "netname" == indtype: e = NetNameThreatRecon(indicator['Indicator']) #response += e if "cidr" == indtype: e = IPv4Address(indicator['Indicator']) #response += e if "netrange" == indtype: e = Netblock(indicator['Indicator']) #response += e if indicator['Country']: l = Location(indicator['Country']) response += l #Add Comments and details to own Entity entity = e #request.entity #Set comments if indicator['Comment']: entity.notes = string_filter(indicator['Comment']) #Set Details for detail in tr_details: if detail in indicator: if indicator[detail]: entity += Label(name=detail, value=string_filter( indicator[detail])) #Set link color if "Confidence" in indicator: if indicator['Confidence'] >= 70: linkcolor = "0xff0000" entity.linkcolor = linkcolor response += entity return response