def dotransform(request, response, config):
error, found = lookup_whois(request.value)
if not error and found:
if dict == type(found):
for result, value in found.iteritems():
if set == type(value):
if "whois_domains" == result:
for d in value:
if d:
e = Domain(d)
e.fqdn = d
response += e
if "whois_emails" == result:
for em in value:
if em:
e = EmailAddress(em)
response += e
if "whois_nameservers" == result:
for w in value:
if w:
e = NSRecord(w)
response += e
#Display error message in Transform Output
response += UIMessage(error)
return response
def dotransform(request, response):
dns = request.value
if '.' in dns:
response += Domain('.'.join(dns.split('.')[-2:]))
else:
response += Domain(request.value)
return response
def dotransform(request, response, config):
error, found = lookup_whois(request.value)
if not error and found:
if dict == type(found):
for result, value in found.iteritems():
if set == type(value):
if "whois_domains" == result:
for d in value:
if d:
e = Domain(d)
e.fqdn = d
response += e
if "whois_emails" == result:
for em in value:
if em:
e = EmailAddress(em)
response += e
if "whois_nameservers" == result:
for w in value:
if w:
e = NSRecord(w)
response += e
#Display error message in Transform Output
response += UIMessage(error)
return response
def dotransform(request, response):
page = build(request.value)
try:
single = page.find(
text='The following Host Name was requested from a host database:'
).findNext()
except:
single = None
pass
try:
single = page.find(
text='The following Host Names were requested from a host database:'
).findNext()
except:
single = None
pass
try:
single2 = page.find(
text='The following Internet Connection was established:'
).findNext()
except:
single2 = None
pass
try:
multi = page.find(
text='The following Internet Connections were established:'
).findNext('table')
except:
multi = None
pass
if single is not None:
for dom in single.findAll("li"):
text = dom.text
response += Domain(text)
if single2 is not None:
dom = single2.findNext('tr').findNext('tr').findNext('td')
text = dom.text
response += Domain(text)
if multi is not None:
for entry in multi.findAll('tr')[1::]:
dom = entry.findNext('td')
text = dom.text
response += Domain(text)
return response
def dotransform(request, response):
# Report transform progress
progress(50)
ip = request.value
total = ""
urldom = 'https://www.virustotal.com/en/ip-address/' + ip + '/information/'
soup = BeautifulSoup(urllib2.urlopen(urldom).read())
try:
links = soup.findAll('div', attrs={'class': 'enum'})
for link in links:
total += str(link)
total = BeautifulSoup(total)
for totals in total.findAll('a', href=True):
totals = totals['href']
theIP = totals.replace("/en/domain/", "")
e = theIP.replace("/information/", "")
e = Domain(e)
response += e
except IOError:
print 'IO Error'
# Update progress
progress(100)
# Return response for visualization
return response
def dotransform(request, response):
# Report transform progress
progress(50)
ip = request.value
url = 'https://www.virustotal.com/vtapi/v2/ip-address/report'
parameters = {'ip': ip, 'apikey': config['virustotal/apikey']}
resp = urllib2.urlopen('%s?%s' %
(url, urllib.urlencode(parameters))).read()
response_dict = json.loads(resp)
#Latest detected URLs"
try:
for i in range(0, len(response_dict['resolutions'])):
host = response_dict['resolutions'][i]['hostname']
host = Domain(host)
response += host
except IOError:
response = 'IO Error'
except KeyError:
response = 'Not Found'
# Update progress
progress(100)
# Return response for visualization
return response
def dotransform(request, response):
page = build(request.value)
doms = []
if page.find("span", {"id": "sharedsub"}):
section = page.find("span", {"id": "sharedsub"}).findNext('ul')
for entry in section.findAll("li"):
response += Domain(entry.text)
elif page.find("span", {"id": "sharedsubv"}):
section = page.find("span", {"id": "sharedsubv"}).findNext('ul')
for entry in section.findAll("li"):
response += Domain(entry.text)
else:
response += UIMessage('No subdomains in robtex')
return response
def dotransform(request, response):
try:
items=ast.literal_eval(request.fields['resolutions'])
except:
return response
for item in items:
last=item['last_resolved']
host=item['hostname']
r=Domain(host)
r.linklabel=last
response+=r
return response
def do_transform(self, request, response, config):
be = BinaryEdge(config['binaryedge.local.api_key'])
domain = request.entity.value
try:
# Only consider the fist page
res = be.domain_dns(domain)
except BinaryEdgeException as e:
raise MaltegoException('BinaryEdge error: %s' % e.msg)
else:
already = [domain]
for event in res['events']:
if 'A' in event:
for ip in event['A']:
if ip not in already:
response += IPv4Address(ip)
already.append(ip)
if 'domain' in event:
if event['domain'] not in already:
response += Domain(event['domain'])
already.append(event['domain'])
if 'MX' in event:
for mx in event['MX']:
if mx not in already:
response += MXRecord(mx)
already.append(mx)
if 'NS' in event:
for ns in event['NS']:
if ns not in already:
response += NSRecord(ns)
already.append(ns)
return response
return response
def dotransform(request, response):
if request.fields['behavioral'] != "":
try:
behavior = ast.literal_eval(request.fields['behavior_data'])
except Exception as e:
debug("Entity has no behavioral data")
return response
if behavior.has_key("network"):
if behavior['network'].has_key('dns'):
for item in behavior['network']['dns']:
host = Domain(item['hostname'])
host.linklabel = "vt_behav->hosts"
response += host
if item.has_key('ip'):
ip = IPv4Address(item['ip'])
ip.linklabel = "vt_behav->hosts"
response += ip
if behavior['network'].has_key('tcp'):
for item in behavior['network']['tcp']:
conn = item.split(":")
r = IPv4Address(conn[0])
r.linklabel = "vt_behav->hosts_tcp (%s)" % str(conn[1])
response += r
if behavior['network'].has_key('udp'):
for item in behavior['network']['udp']:
conn = item.split(":")
r = IPv4Address(conn[0])
r.linklabel = "vt_behav->hosts_udp (%s)" % str(conn[1])
response += r
if behavior['network'].has_key('http'):
for item in behavior['network']['http']:
r = URL(item['url'])
r.url = item['url']
r.linklabel = "vt_behav->hosts_http (%s)" % item['method']
response += r
else:
debug("ripVT: No behavioral for %s" % request.value)
return response
def dotransform(request, response, config):
if 'taskid' in request.fields:
task = request.fields['taskid']
else:
task = request.value
netw = network(report(task))
for d in netw['domains']:
response += Domain(d['domain'].decode('ascii'), taskid=task)
return response
def do_transform(self, request, response, config):
be = BinaryEdge(config['binaryedge.local.api_key'])
ip = request.entity.value
try:
res = be.host(ip)
except BinaryEdgeException as e:
raise MaltegoException('BinaryEdge error: %s' % e.msg)
else:
already = []
for port in res['events']:
response += Port(port['port'])
for result in port['results']:
if result['origin']['type'] == 'ssl':
cert = result['result']['data']['cert_info'][
'certificate_chain'][0]
# How to return a certificate ?
if 'commonName' in cert['as_dict']['subject']:
d = cert['as_dict']['subject']['commonName']
if d not in already:
response += Domain(d)
already.append(d)
if 'extensions' in cert['as_dict']:
if 'X509v3 Subject Alternative Name' in cert[
'as_dict']['extensions']:
for domain in cert['as_dict']['extensions'][
'X509v3 Subject Alternative Name'][
'DNS']:
if domain not in already:
response += Domain(domain)
already.append(domain)
if result['origin']['type'] in ['http', 'grabber']:
if 'server' in result['result']['data']['response'][
'headers']:
banner = result['result']['data']['response'][
'headers']['server']
if banner not in already:
response += Banner(banner)
already.append(banner)
return response
def dotransform(request, response):
ns = request.value
results = query('-n', ns, 0, 'n')
for result in results:
data = json.loads(result)
if data.has_key('rrname'):
if data.has_key('time_first'):
first = data['time_first']
last = data['time_last']
elif data.has_key('zone_time_first'):
first = data['zone_time_first']
last = data['zone_time_last']
fnice = datetime.datetime.fromtimestamp(int(first)).strftime('%m-%d-%Y')
lnice = datetime.datetime.fromtimestamp(int(last)).strftime('%m-%d-%Y')
e = Domain(data['rrname'].rstrip('.'))
e.linklabel = fnice + ' - ' + lnice
response += e
return response
def dotransform(request, response):
try:
items = ast.literal_eval(request.fields['subdomains'])
except:
return response
for item in items:
r = Domain(item)
response += r
return response
def do_transform(self, request, response, config):
be = BinaryEdge(config['binaryedge.local.api_key'])
domain = request.entity.value
try:
# Only consider the fist page
res = be.domain_subdomains(domain)
except BinaryEdgeException as e:
raise MaltegoException('BinaryEdge error: %s' % e.msg)
else:
for e in res["events"]:
if e != domain:
response += Domain(e)
return response
def dotransform(request, response):
data = getbehavior(request.value)
try:
try:
network = data['network']
except:
#no network data
pass
try:
for result in network['dns']:
dom = result['hostname']
ip = result['ip']
response += Domain(dom)
response += IPv4Address['ip']
except:
#no dns data
pass
try:
for request in network['http']:
uri = URL(request['uri'])
uri.url = request['uri']
ua = UserAgent(request['user-agent'])
#req = HTTPRequest(request['data'])
port = Port(request['port'])
response += uri
response += ua
#response += req
response += port
except:
#no http data
pass
try:
for entry in network['tcp']:
e = entry['dst']
if e.startswith('10.'):
pass
else:
conn = IPv4Address(e)
response += conn
except:
#no tcp data
pass
except:
response += UIMessage(data['verbose_msg'])
return response
def dotransform(request, response):
#Build the request
type = 'hash'
page = build(request.value, type)
try:
list = page.find(text='PCAP Raw DNS Queries').previous.previous.parent.findAll('p')
except:
raise MaltegoException('No DNS Queries')
for item in list:
if item.text != 'none':
response += Domain(item.text)
return response
def dotransform(request, response):
#Build Request
page = build(request.value)
#Finds the DNS section and extracts domains
try:
table = page.find("div", {"id": "network_dns"}).findNext('table')
elements = table.findAll("span", {"class": "mono"})
for element in elements:
text = element.find(text=True)
response += Domain(text)
except:
return response
return response
def dotransform(request, response, config):
hosts_gather(request.value)
hosts_enum(request.value)
hosts_geo(request.value)
dbcon = db_connect(request.value)
host_list = get_hosts(dbcon)
for host in host_list:
e = Domain(host[0])
e += Field("workspace", request.value, displayname='Workspace')
response += e
return response
def dotransform(request, response):
ns = request.value
results = query('-n', ns, 0, 'n')
for result in results:
data = json.loads(result)
if data.has_key('rrname'):
if data.has_key('time_first'):
first = data['time_first']
last = data['time_last']
elif data.has_key('zone_time_first'):
first = data['zone_time_first']
last = data['zone_time_last']
fnice = datetime.datetime.fromtimestamp(
int(first)).strftime('%m-%d-%Y')
lnice = datetime.datetime.fromtimestamp(
int(last)).strftime('%m-%d-%Y')
e = Domain(data['rrname'].rstrip('.'))
e.linklabel = fnice + ' - ' + lnice
response += e
return response
def do_transform(self, request, response, config):
ip = request.entity.value
be = BinaryEdge(config['binaryedge.local.api_key'])
try:
res = be.domain_ip(ip)
except BinaryEdgeException as e:
raise MaltegoException('BinaryEdge error: %s' % e.msg)
else:
already = []
for e in res['events']:
if e['domain'] not in already:
response += Domain(e['domain'])
already.append(e['domain'])
return response
def dotransform(request, response, config):
if 'workspace' in request.fields:
workspace = request.fields['workspace']
else:
workspace = request.value
dbcon = db_connect(workspace)
host_list = get_hosts(dbcon)
for host in host_list:
e = Domain(host[0])
e += Field("workspace", workspace, displayname='Workspace')
response += e
return response
def detType(in_val):
val = str(in_val)
#::Email
email = re.compile(".*\[@\][a-z0-9\-]{1,}\.[a-z0-9\-]{1,}")
#::IP
ipv4 = re.compile("^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$")
#::CIDR
cidr = re.compile("^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/\d{1,2}$")
#::Range
v4range = re.compile(
"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\-\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$"
)
#::Domain
dom = re.compile("([a-z0-9\-]{1,}\.?)+\.[a-z0-9\-]{1,}$")
if email.match(val):
e = EmailAddress(val)
return e
if ipv4.match(val):
e = IPv4Address(val)
return e
if cidr.match(val):
e = CIDR(val)
return e
if v4range.match(val):
e = Range(val)
return e
if dom.match(val):
e = Domain(val)
return e
if re.match("^([a-z]*)://", val, re.M | re.I):
e = URL(val)
e.url = val
return e
def dotransform(request, response):
dns_results = []
pcap = request.value
pkt = rdpcap(pcap)
for pkts in pkt:
if pkts.haslayer(DNSQR):
drec = pkts.getlayer(DNSQR).qname
if drec not in dns_results:
dns_results.append(drec)
for drec in dns_results:
e = Domain(drec.strip('.'))
e += Field('pcapsrc', pcap, displayname='Original pcap File')
response += e
return response
def dotransform(request, response):
domain = request.value
results = query('-r', domain, 0, 'n')
for result in results:
data = json.loads(result)
if data.has_key('time_first'):
first = data['time_first']
last = data['time_last']
elif data.has_key('zone_time_first'):
first = data['zone_time_first']
last = data['zone_time_last']
fnice = datetime.datetime.fromtimestamp(
int(first)).strftime('%m-%d-%Y')
lnice = datetime.datetime.fromtimestamp(int(last)).strftime('%m-%d-%Y')
if data['rrtype'] == 'NS':
for item in data['rdata']:
e = NSRecord(item)
e.linklabel = fnice + ' - ' + lnice
response += e
elif data['rrtype'] == 'MX':
for item in data['rdata']:
e = MXRecord(item)
e.linklabel = fnice + ' - ' + lnice
response += e
elif data['rrtype'] == 'CNAME':
for item in data['rdata']:
e = Domain(item.rstrip('.'))
e.linklabel = fnice + ' - ' + lnice
response += e
elif data['rrtype'] == 'A':
pass
else:
type = data['rrtype']
for item in data['rdata']:
label = type + ' ' + item
e = Phrase(label)
e.linklabel = fnice + ' - ' + lnice
response += e
return response
def dotransform(request, response):
# Store the pcap file as a variable
pcap = request.value
usedb = config['working/usedb']
# Check to see if we are using the database or not
if usedb > 0:
# Connect to the database so we can insert the record created below
x = mongo_connect()
c = x['DNS']
# Hash the pcap file
try:
md5hash = md5_for_file(pcap)
except Exception as e:
return response + UIMessage(str(e))
# Get the session and/or pcap id
d = find_session(md5hash)
pcap_id = d[0]
session_id = d[1]
else:
pass
try:
pkts = rdpcap(pcap)
dns_requests = []
for p in pkts:
if p.haslayer(DNSQR):
timestamp = datetime.datetime.fromtimestamp(
p.time).strftime('%Y-%m-%d %H:%M:%S.%f')
r = p[DNSQR].qname[:-1]
tld = tldextract.extract(r)
domain = tld.registered_domain
if usedb > 0:
dns = OrderedDict({
'PCAP ID': pcap_id,
'Stream ID': session_id,
'Time Stamp': timestamp,
'Type': 'Request',
'IP': {
'src': p[IP].src,
'dst': p[IP].dst,
'length': p[IP].len
},
'Request Details': {
'Query Type': p[DNSQR].qtype,
'Query Name': r,
'Domain': domain
}
})
t = x.DNS.find({'Time Stamp': timestamp}).count()
if t > 0:
pass
else:
c.insert(dns)
else:
pass
if r not in dns_requests:
dns_requests.append(domain)
else:
pass
for d in dns_requests:
x = Domain(d)
response += x
return response
except Exception as e:
if usedb > 0:
error_logging(str(e), 'DNS Requests')
else:
return response + UIMessage(str(e))
def dotransform(request, response, config):
command = "--hash_to_domain " + request.value
qradio_output = get_qradio_data(command, 0)
for entry in qradio_output:
response += Domain(entry)
return response
def dotransform(request, response):
tmp = ".".join(str(request.value).split(".")[-2:])
response += Domain(tmp)
return response
def dotransform(request, response, config):
if 'ThreatCentral.resourceId' in request.fields:
try:
indicator = get_indicator(
request.fields['ThreatCentral.resourceId'])
except ThreatCentralError as err:
response += UIMessage(err.value, type='PartialError')
else:
try:
# Update Indicator entity ?
e = Indicator(request.value)
e.title = encode_to_utf8(indicator.get('title'))
e.resourceId = indicator.get('resourceId')
e.severity = indicator.get('severity',
dict()).get('displayName')
e.confidence = indicator.get('confidence',
dict()).get('displayName')
e.indicatorType = indicator.get('indicatorType',
dict()).get('displayName')
e += Label(
'Severity',
indicator.get('severity', dict()).get('displayName'))
e += Label(
'Confidence',
indicator.get('confidence', dict()).get('displayName'))
e += Label(
'Indicator Type',
indicator.get('indicatorType', dict()).get('displayName'))
if indicator.get('description'):
e += Label(
'Description', '<br/>'.join(
encode_to_utf8(
indicator.get('description')).split('\n')))
response += e
if len(indicator.get('observables', list())) is not 0:
for observable in indicator.get('observables'):
if upper(observable.get(
'type', dict()).get('value')) == 'DOMAIN':
if observable.get('tcScore'):
weight = int(observable.get('tcScore'))
else:
weight = 1
e = Domain(observable.get('value'), weight=weight)
if upper(
observable.get('location',
dict()).get('city')
) != 'UNDEFINED_GEO_LOCATION_STRING':
e += Label(
'Location', '<br/>'.join([
'{}:{}'.format(encode_to_utf8(k),
encode_to_utf8(v))
for k, v in observable.get(
'location', dict()).iteritems()
]))
response += e
except AttributeError as err:
response += UIMessage('Error: {}'.format(err),
type='PartialError')
except ThreatCentralError as err:
response += UIMessage(err.value, type='PartialError')
except TypeError:
return response
return response
def dotransform(request, response, config):
tr_details = [
'Reference', 'Source', 'KillChain', 'Firstseen', 'Lastseen',
'Attribution', 'ProcessType', 'Rrname', 'Rdata', 'Country', 'Tags',
'Comment', 'RootNode', 'Confidence'
]
#Disable cache to get actual data from Threat Recon
cache, found = search(request.value, cache=False)
#Default linkcolor
linkcolor = "0x000000"
if found:
if defaultdict == type(found):
for rootnode, value in found.iteritems():
#If the RootNode is empty, display attributes
if len(rootnode) == 0:
for indicator in value:
#debug(indicator)
e = ''
indtype = indicator['Type'].lower().strip()
if "whois email" == indtype:
e = EmailAddress(indicator['Indicator'])
if "name server" == indtype:
e = NSRecord(indicator['Indicator'])
if "domain" == indtype:
e = Domain(indicator['Indicator'])
e.fqdn = indicator['Indicator']
if "ip" == indtype:
e = IPv4Address(indicator['Indicator'])
if "phone or fax no." == indtype:
e = PhoneNumber(indicator['Indicator'])
if "whois address component" == indtype:
e = Phrase(indicator['Indicator'])
if "email" == indtype:
e = EmailAddress(indicator['Indicator'])
if "netname" == indtype:
e = NetNameThreatRecon(indicator['Indicator'])
if "cidr" == indtype:
e = IPv4Address(indicator['Indicator'])
if "netrange" == indtype:
e = Netblock(indicator['Indicator'])
if e:
#Set linkcolor
e.linkcolor = linkcolor
#Set comments
if indicator['Comment']:
e.notes = string_filter(indicator['Comment'])
#Set Details
for detail in tr_details:
if detail in indicator:
if indicator[detail]:
e += Label(name=detail,
value=string_filter(
indicator[detail]))
response += e
else:
#Display the RootNodes
e = ThreatRecon(rootnode)
response += e
return response
def dotransform(request, response, config):
tr_details = [
'Reference', 'Source', 'KillChain', 'Firstseen', 'Lastseen',
'Attribution', 'ProcessType', 'Rrname', 'Rdata', 'Country', 'Tags',
'Comment', 'RootNode', 'Confidence'
]
#Default link color is black
linkcolor = "0x000000"
cache, found = search(request.value)
if found:
if list == type(found):
for indicator in found:
debug(indicator)
e = ''
indtype = indicator['Type'].lower().strip()
if "whois email" == indtype:
e = EmailAddress(indicator['Indicator'])
#response += e
if "name server" == indtype:
e = NSRecord(indicator['Indicator'])
#response += e
if "domain" == indtype:
e = Domain(indicator['Indicator'])
e.fqdn = indicator['Indicator']
#response += e
#IF Type is not domain, check if Rrname is not empty
elif indicator['Rrname'] and indicator['Rrname'] != 'NA':
d = Domain(indicator['Rrname'])
d.fqdn = indicator['Rrname']
response += d
if "ip" == indtype:
e = IPv4Address(indicator['Indicator'])
#response += e
#IF Type is not IP, check if Rdata is not empty
elif indicator['Rdata']:
i = IPv4Address(indicator['Rdata'])
response += i
if "phone or fax no." == indtype:
e = PhoneNumber(indicator['Indicator'])
#response += e
if "whois address component" == indtype:
e = Phrase(indicator['Indicator'])
#response += e
if "email" == indtype:
e = EmailAddress(indicator['Indicator'])
#response += e
if "netname" == indtype:
e = NetNameThreatRecon(indicator['Indicator'])
#response += e
if "cidr" == indtype:
e = IPv4Address(indicator['Indicator'])
#response += e
if "netrange" == indtype:
e = Netblock(indicator['Indicator'])
#response += e
if indicator['Country']:
l = Location(indicator['Country'])
response += l
#Add Comments and details to own Entity
entity = e #request.entity
#Set comments
if indicator['Comment']:
entity.notes = string_filter(indicator['Comment'])
#Set Details
for detail in tr_details:
if detail in indicator:
if indicator[detail]:
entity += Label(name=detail,
value=string_filter(
indicator[detail]))
#Set link color
if "Confidence" in indicator:
if indicator['Confidence'] >= 70:
linkcolor = "0xff0000"
entity.linkcolor = linkcolor
response += entity
return response
