def do_transform(self, request, response, config):
be = BinaryEdge(config['binaryedge.local.api_key'])
domain = request.entity.value
try:
# Only consider the fist page
res = be.domain_dns(domain)
except BinaryEdgeException as e:
raise MaltegoException('BinaryEdge error: %s' % e.msg)
else:
already = [domain]
for event in res['events']:
if 'A' in event:
for ip in event['A']:
if ip not in already:
response += IPv4Address(ip)
already.append(ip)
if 'domain' in event:
if event['domain'] not in already:
response += Domain(event['domain'])
already.append(event['domain'])
if 'MX' in event:
for mx in event['MX']:
if mx not in already:
response += MXRecord(mx)
already.append(mx)
if 'NS' in event:
for ns in event['NS']:
if ns not in already:
response += NSRecord(ns)
already.append(ns)
return response
return response
def dotransform(request, response, config):
error, found = lookup_whois(request.value)
if not error and found:
if dict == type(found):
for result, value in found.iteritems():
if set == type(value):
if "whois_domains" == result:
for d in value:
if d:
e = Domain(d)
e.fqdn = d
response += e
if "whois_emails" == result:
for em in value:
if em:
e = EmailAddress(em)
response += e
if "whois_nameservers" == result:
for w in value:
if w:
e = NSRecord(w)
response += e
#Display error message in Transform Output
response += UIMessage(error)
return response
def dotransform(request, response):
ans = nslookup(request.value, 'NS')
if ans is not None and DNS in ans:
for i in range(0, ans[DNS].ancount):
if ans[DNS].an[i].type == 2:
response += NSRecord(ans[DNS].an[i].rdata.rstrip('.'))
return response
def addrecord(record, response):
if record.type == 2:
response += NSRecord(record.rdata.rstrip('.'))
elif record.type == 15:
e = MXRecord(record.rdata.rstrip('.'))
e += Field('mxrecord.priority', record.mxpriority)
response += e
elif record.type in [1, 5]:
response += DNSName(record.rrname.rstrip('.'))
elif record.type == 16:
response += Phrase(record.rdata)
return response
def dotransform(request, response):
domain = request.value
results = query('-r', domain, 0, 'n')
for result in results:
data = json.loads(result)
if data.has_key('time_first'):
first = data['time_first']
last = data['time_last']
elif data.has_key('zone_time_first'):
first = data['zone_time_first']
last = data['zone_time_last']
fnice = datetime.datetime.fromtimestamp(
int(first)).strftime('%m-%d-%Y')
lnice = datetime.datetime.fromtimestamp(int(last)).strftime('%m-%d-%Y')
if data['rrtype'] == 'NS':
for item in data['rdata']:
e = NSRecord(item)
e.linklabel = fnice + ' - ' + lnice
response += e
elif data['rrtype'] == 'MX':
for item in data['rdata']:
e = MXRecord(item)
e.linklabel = fnice + ' - ' + lnice
response += e
elif data['rrtype'] == 'CNAME':
for item in data['rdata']:
e = Domain(item.rstrip('.'))
e.linklabel = fnice + ' - ' + lnice
response += e
elif data['rrtype'] == 'A':
pass
else:
type = data['rrtype']
for item in data['rdata']:
label = type + ' ' + item
e = Phrase(label)
e.linklabel = fnice + ' - ' + lnice
response += e
return response
def dotransform(request, response, config):
if 'taskid' in request.fields:
task = request.fields['taskid']
else:
task = request.value
netw = network(report(task))
dns_lst = []
for d in netw['dns']:
if d['request'] not in dns_lst:
response += NSRecord(d['request'].decode('ascii'), taskid=task)
dns_lst.append(d['request'])
return response
def dotransform(request, response):
domain = request.value
results = query("-r", domain, 0, "n")
for result in results:
data = json.loads(result)
if data.has_key("time_first"):
first = data["time_first"]
last = data["time_last"]
elif data.has_key("zone_time_first"):
first = data["zone_time_first"]
last = data["zone_time_last"]
fnice = datetime.datetime.fromtimestamp(int(first)).strftime("%m-%d-%Y")
lnice = datetime.datetime.fromtimestamp(int(last)).strftime("%m-%d-%Y")
if data["rrtype"] == "NS":
for item in data["rdata"]:
e = NSRecord(item)
e.linklabel = fnice + " - " + lnice
response += e
elif data["rrtype"] == "MX":
for item in data["rdata"]:
e = MXRecord(item)
e.linklabel = fnice + " - " + lnice
response += e
elif data["rrtype"] == "CNAME":
for item in data["rdata"]:
e = Domain(item.rstrip("."))
e.linklabel = fnice + " - " + lnice
response += e
elif data["rrtype"] == "A":
pass
else:
type = data["rrtype"]
for item in data["rdata"]:
label = type + " " + item
e = Phrase(label)
e.linklabel = fnice + " - " + lnice
response += e
return response
def dotransform(request, response, config):
tr_details = [
'Reference', 'Source', 'KillChain', 'Firstseen', 'Lastseen',
'Attribution', 'ProcessType', 'Rrname', 'Rdata', 'Country', 'Tags',
'Comment', 'RootNode', 'Confidence'
]
#Default link color is black
linkcolor = "0x000000"
cache, found = search(request.value)
if found:
if list == type(found):
for indicator in found:
debug(indicator)
e = ''
indtype = indicator['Type'].lower().strip()
if "whois email" == indtype:
e = EmailAddress(indicator['Indicator'])
#response += e
if "name server" == indtype:
e = NSRecord(indicator['Indicator'])
#response += e
if "domain" == indtype:
e = Domain(indicator['Indicator'])
e.fqdn = indicator['Indicator']
#response += e
#IF Type is not domain, check if Rrname is not empty
elif indicator['Rrname'] and indicator['Rrname'] != 'NA':
d = Domain(indicator['Rrname'])
d.fqdn = indicator['Rrname']
response += d
if "ip" == indtype:
e = IPv4Address(indicator['Indicator'])
#response += e
#IF Type is not IP, check if Rdata is not empty
elif indicator['Rdata']:
i = IPv4Address(indicator['Rdata'])
response += i
if "phone or fax no." == indtype:
e = PhoneNumber(indicator['Indicator'])
#response += e
if "whois address component" == indtype:
e = Phrase(indicator['Indicator'])
#response += e
if "email" == indtype:
e = EmailAddress(indicator['Indicator'])
#response += e
if "netname" == indtype:
e = NetNameThreatRecon(indicator['Indicator'])
#response += e
if "cidr" == indtype:
e = IPv4Address(indicator['Indicator'])
#response += e
if "netrange" == indtype:
e = Netblock(indicator['Indicator'])
#response += e
if indicator['Country']:
l = Location(indicator['Country'])
response += l
#Add Comments and details to own Entity
entity = e #request.entity
#Set comments
if indicator['Comment']:
entity.notes = string_filter(indicator['Comment'])
#Set Details
for detail in tr_details:
if detail in indicator:
if indicator[detail]:
entity += Label(name=detail,
value=string_filter(
indicator[detail]))
#Set link color
if "Confidence" in indicator:
if indicator['Confidence'] >= 70:
linkcolor = "0xff0000"
entity.linkcolor = linkcolor
response += entity
return response
def dotransform(request, response, config):
tr_details = [
'Reference', 'Source', 'KillChain', 'Firstseen', 'Lastseen',
'Attribution', 'ProcessType', 'Rrname', 'Rdata', 'Country', 'Tags',
'Comment', 'RootNode', 'Confidence'
]
#Disable cache to get actual data from Threat Recon
cache, found = search(request.value, cache=False)
#Default linkcolor
linkcolor = "0x000000"
if found:
if defaultdict == type(found):
for rootnode, value in found.iteritems():
#If the RootNode is empty, display attributes
if len(rootnode) == 0:
for indicator in value:
#debug(indicator)
e = ''
indtype = indicator['Type'].lower().strip()
if "whois email" == indtype:
e = EmailAddress(indicator['Indicator'])
if "name server" == indtype:
e = NSRecord(indicator['Indicator'])
if "domain" == indtype:
e = Domain(indicator['Indicator'])
e.fqdn = indicator['Indicator']
if "ip" == indtype:
e = IPv4Address(indicator['Indicator'])
if "phone or fax no." == indtype:
e = PhoneNumber(indicator['Indicator'])
if "whois address component" == indtype:
e = Phrase(indicator['Indicator'])
if "email" == indtype:
e = EmailAddress(indicator['Indicator'])
if "netname" == indtype:
e = NetNameThreatRecon(indicator['Indicator'])
if "cidr" == indtype:
e = IPv4Address(indicator['Indicator'])
if "netrange" == indtype:
e = Netblock(indicator['Indicator'])
if e:
#Set linkcolor
e.linkcolor = linkcolor
#Set comments
if indicator['Comment']:
e.notes = string_filter(indicator['Comment'])
#Set Details
for detail in tr_details:
if detail in indicator:
if indicator[detail]:
e += Label(name=detail,
value=string_filter(
indicator[detail]))
response += e
else:
#Display the RootNodes
e = ThreatRecon(rootnode)
response += e
return response
def nslookup(name, type_, response, resolvers=None, recursive=True):
name = name.rstrip('.')
if isinstance(type_, basestring):
type_ = dns.rdatatype.from_text(type_)
if type_ == dns.rdatatype.PTR:
name = dns.reversename.from_address(name)
if not resolvers:
resolvers = dns.resolver.get_default_resolver().nameservers
elif isinstance(resolvers, basestring):
resolvers = [resolvers]
if type_ in [dns.rdatatype.AXFR, dns.rdatatype.IXFR]:
try:
discovered_names = []
for ns in dns.resolver.query(name, dns.rdatatype.NS):
xfr(ns.to_text(), name, response, discovered_names=discovered_names)
return True
except dns.resolver.NXDOMAIN:
response += UIMessage("DNS records for %s do not exist." % repr(name))
except dns.resolver.NoNameservers:
response += UIMessage("No nameservers found for %s." % repr(name))
except dns.resolver.Timeout:
response += UIMessage("DNS request for %s timed out." % repr(name))
except dns.resolver.NoAnswer:
response += UIMessage("DNS request for %s resulted in no response." % repr(name))
except socket.error:
response += UIMessage("A socket error has occurred. Make sure you are connected or the traffic is allowed.")
return False
try:
request = dns.message.make_query(name, type_, dns.rdataclass.IN)
if not recursive:
request.flags ^= dns.flags.RD
for resolver in resolvers:
ans = dns.query.udp(request, resolver).answer
if ans:
for rrset in ans:
for rr in rrset:
if rr.rdtype == type_:
if type_ == dns.rdatatype.A:
response += IPv4Address(rr.to_text(True))
elif type_ == dns.rdatatype.NS:
response += UIMessage(repr(rr))
response += NSRecord(rr.to_text()[:-1])
elif type_ == dns.rdatatype.CNAME:
response += DNSName(rr.to_text())
elif type_ == dns.rdatatype.SOA:
e = NSRecord(rr.mname.to_text(True))
e += Field('mailaddr', rr.rname.to_text(True), displayname='Authority')
response += e
elif type_ == dns.rdatatype.PTR:
response += DNSName(rr.to_text()[:-1])
elif type_ == dns.rdatatype.MX:
e = MXRecord(rr.exchange.to_text(True))
e.mxpriority = rr.preference
response += e
elif type_ == dns.rdatatype.TXT:
response += Phrase(rr.to_text(True))
elif type_ == dns.rdatatype.AAAA:
response += IPv6Address(rr.to_text(True))
else:
response += Phrase(rr.to_text(True))
return True
except dns.resolver.NXDOMAIN:
response += UIMessage("DNS records for %s do not exist." % repr(name))
except dns.resolver.Timeout:
response += UIMessage("DNS request for %s timed out." % repr(name))
except dns.resolver.NoNameservers:
response += UIMessage("No name servers found for %s." % repr(name))
except dns.resolver.NoAnswer:
response += UIMessage("The DNS server returned with no response for %s." % repr(name))
except socket.error:
response += UIMessage("A socket error has occurred. Make sure you are connected or the traffic is allowed.")
return False
def nslookup(name, type_, response, resolvers=None, recursive=True):
name = name.rstrip('.')
if isinstance(type_, basestring):
type_ = dns.rdatatype.from_text(type_)
if type_ == dns.rdatatype.PTR:
name = dns.reversename.from_address(name)
if not resolvers:
resolvers = dns.resolver.get_default_resolver().nameservers
elif isinstance(resolvers, basestring):
resolvers = [resolvers]
if type_ in [dns.rdatatype.AXFR, dns.rdatatype.IXFR]:
try:
discovered_names = []
for ns in dns.resolver.query(name, dns.rdatatype.NS):
xfr(ns.to_text(),
name,
response,
discovered_names=discovered_names)
return True
except dns.resolver.NXDOMAIN:
response += UIMessage("DNS records for %s do not exist." %
repr(name))
except dns.resolver.NoNameservers:
response += UIMessage("No nameservers found for %s." % repr(name))
except dns.resolver.Timeout:
response += UIMessage("DNS request for %s timed out." % repr(name))
except dns.resolver.NoAnswer:
response += UIMessage(
"DNS request for %s resulted in no response." % repr(name))
except socket.error:
response += UIMessage(
"A socket error has occurred. Make sure you are connected or the traffic is allowed."
)
return False
try:
request = dns.message.make_query(name, type_, dns.rdataclass.IN)
if not recursive:
request.flags ^= dns.flags.RD
for resolver in resolvers:
ans = dns.query.udp(request, resolver).answer
if ans:
for rrset in ans:
for rr in rrset:
if rr.rdtype == type_:
if type_ == dns.rdatatype.A:
response += IPv4Address(rr.to_text(True))
elif type_ == dns.rdatatype.NS:
response += UIMessage(repr(rr))
response += NSRecord(rr.to_text()[:-1])
elif type_ == dns.rdatatype.CNAME:
response += DNSName(rr.to_text())
elif type_ == dns.rdatatype.SOA:
e = NSRecord(rr.mname.to_text(True))
e += Field('mailaddr',
rr.rname.to_text(True),
displayname='Authority')
response += e
elif type_ == dns.rdatatype.PTR:
response += DNSName(rr.to_text()[:-1])
elif type_ == dns.rdatatype.MX:
e = MXRecord(rr.exchange.to_text(True))
e.mxpriority = rr.preference
response += e
elif type_ == dns.rdatatype.TXT:
response += Phrase(rr.to_text(True))
elif type_ == dns.rdatatype.AAAA:
response += IPv6Address(rr.to_text(True))
else:
response += Phrase(rr.to_text(True))
return True
except dns.resolver.NXDOMAIN:
response += UIMessage("DNS records for %s do not exist." % repr(name))
except dns.resolver.Timeout:
response += UIMessage("DNS request for %s timed out." % repr(name))
except dns.resolver.NoNameservers:
response += UIMessage("No name servers found for %s." % repr(name))
except dns.resolver.NoAnswer:
response += UIMessage(
"The DNS server returned with no response for %s." % repr(name))
except socket.error:
response += UIMessage(
"A socket error has occurred. Make sure you are connected or the traffic is allowed."
)
return False
def dotransform(request, response, config):
"""
The dotransform function is our transform's entry point. The request object has the following properties:
- value: a string containing the value of the input entity.
- fields: a dictionary of entity field names and their respective values of the input entity.
- params: any additional command-line arguments to be passed to the transform.
- entity: the information above is serialized into an Entity object. The entity type is determined
by the inputs field in @configure for local transforms. For remote transforms, the entity
type is determined by the information in the body of the request. Local transforms suffer
from one limitation: if more than one entity type is listed in the inputs field of @configure,
the entity type might not be resolvable. Therefore, this should not be referenced in local
transforms if there is more than one input entity type defined in @configure.
The response object is a container for output entities, UI messages, and exception messages. The config object
contains a key-value store of the configuration file.
TODO: write your data mining logic below.
"""
client = get_client(config)
prog = 10
progress(prog)
debug('Starting RiskIQ passive dns lookup...')
value = request.entities[0].value
if IP_REGEX.match(value):
api_response = client.get_dns_ptr_by_ip(value, rrtype=None)
else:
api_response = client.get_dns_data_by_name(value, rrtype=None)
if not api_response:
progress(100)
return response
dns_data = api_response['records']
a_responses = set()
ns_responses = set()
mx_responses = set()
aaaa_responses = set()
cname_responses = set()
responses = set()
for dns_datum in dns_data:
data = dns_datum['data']
if dns_datum.get('rrtype') == 'A':
a_responses |= set(data)
elif dns_datum.get('rrtype') == 'CNAME':
cname_responses |= set(data)
elif dns_datum.get('rrtype') == 'NS':
ns_responses |= set(data)
elif dns_datum.get('rrtype') == 'MX':
mx_responses |= set(data)
elif dns_datum.get('rrtype') == 'AAAA':
aaaa_responses |= set(data)
elif dns_datum.get('rrtype') == 'TXT':
pass
else:
responses |= set(data)
prog += 40
progress(prog)
for rec in a_responses:
e = IPv4Address(rec)
e.ip = rec
response += e
prog += 10
progress(prog)
"""
for rec in aaaa_responses:
e = IPv6Address(rec)
e.ip = rec
response += e
prog += 10
progress(prog)
"""
for _rec in ns_responses:
rec = fix_dom(_rec)
e = NSRecord(rec)
e.fqdn = rec
response += e
prog += 10
progress(prog)
for _rec in mx_responses:
rec = fix_dom(_rec)
e = MXRecord(rec)
e.fqdn = rec
response += e
prog += 10
progress(prog)
for _rec in cname_responses:
rec = fix_dom(_rec)
e = DNSName(rec)
e.fqdn = rec
response += e
prog += 10
progress(prog)
for _rec in responses:
rec = fix_dom(_rec)
if IP_REGEX.match(rec):
e = IPv4Address(rec)
e.ip = rec
else:
e = DNSName(rec)
e.fqdn = rec
response += e
progress(100)
return response
