def to_clients(response, output):
cat = None
for line in output.split('\n'):
if not line:
continue
elif line.startswith(' '):
e = None
if cat in range(Category.AlternativeTargetInterfaces,
Category.OtherAssociations):
for ip in ip_matcher.findall(line):
e = IPv4Address(ip)
e += Field('category',
Category.name(cat),
displayname='Category')
response += e
elif cat == Category.OtherAssociations:
ip, desc = line.strip().split(' ', 1)
e = IPv4Address(ip)
e += Label('Additional Info', desc)
e += Field('category',
Category.name(cat),
displayname='Category')
response += e
elif line.startswith(' '):
for id in range(Category.AlternativeTargetInterfaces,
Category.OtherAssociations + 1):
if Category.name(id) in line:
cat = id
break
def passivescan(network, response):
nodes = {}
debug('Sniffing network traffic for more hosts.')
ans = sniff(count=config['scapy/sniffcount'],
timeout=config['scapy/snifftimeout'])
debug('Analyzing traffic.')
for i in ans:
src = None
dst = None
if IP in i:
src = i[IP].src
dst = i[IP].dst
elif ARP in i:
src = i[ARP].psrc
dst = i[ARP].pdst
else:
continue
if src in network and src not in nodes:
nodes[src] = True
e = IPv4Address(src, internal=True)
e += Field('ethernet.hwaddr',
i.src,
displayname='Hardware Address')
response += e
if dst in network and dst not in nodes and i.dst != 'ff:ff:ff:ff:ff:ff':
nodes[dst] = True
e = IPv4Address(dst, internal=True)
e += Field('ethernet.hwaddr',
i.dst,
displayname='Hardware Address')
response += e
def dotransform(request, response):
# Report transform progress
progress(50)
total = ""
urldom = 'https://www.virustotal.com/en/domain/' + request.value + '/information/'
soup = BeautifulSoup(urllib2.urlopen(urldom).read())
try:
links = soup.findAll('div', attrs={'class': 'enum'})
for link in links:
total += str(link)
total = BeautifulSoup(total)
for totals in total.findAll('a', href=True):
totals = totals['href']
theIP = totals.replace("/en/ip-address/", "")
e = theIP.replace("/information/", "")
response += IPv4Address(e)
except IOError:
print 'IO Error'
# Update progress
progress(100)
# Return response for visualization
return response
def dotransform(request, response, config):
"""
The dotransform function is our transform's entry point. The request object has the following properties:
- value: a string containing the value of the input entity.
- fields: a dictionary of entity field names and their respective values of the input entity.
- params: any additional command-line arguments to be passed to the transform.
- entity: the information above is serialized into an Entity object. The entity type is determined
by the inputs field in @configure for local transforms. For remote transforms, the entity
type is determined by the information in the body of the request. Local transforms suffer
from one limitation: if more than one entity type is listed in the inputs field of @configure,
the entity type might not be resolvable. Therefore, this should not be referenced in local
transforms if there is more than one input entity type defined in @configure.
The response object is a container for output entities, UI messages, and exception messages. The config object
contains a key-value store of the configuration file.
TODO: write your data mining logic below.
"""
progress(10)
debug('Extracting IP')
val = request.entities[0].fields['malriq.ip'].value
ipe = IPv4Address(val)
ipe.ip = val
response += [ipe]
progress(100)
return response
def findlocalneighbors(network, response):
debug('ARP sweeping %s' % network.netblock)
# e = Netblock(network.netblock)
# e += Label('CIDR Notation', repr(network))
# e += Label('Network Mask', network.netmask)
# e += Label('Number of Hosts', int(~network.netmask) - 1)
# response += e
ans = arping(repr(network),
timeout=config['scapy/sr_timeout'],
verbose=config['scapy/sr_verbose'])[0]
for i in ans:
e = IPv4Address(i[1].psrc)
e.internal = True
e += Field('ethernet.hwaddr',
i[1].hwsrc,
displayname='Hardware Address')
response += e
if len(ans) <= 1:
passivescan(network, response)
return response
def do_transform(self, request, response, config):
be = BinaryEdge(config['binaryedge.local.api_key'])
domain = request.entity.value
try:
# Only consider the fist page
res = be.domain_dns(domain)
except BinaryEdgeException as e:
raise MaltegoException('BinaryEdge error: %s' % e.msg)
else:
already = [domain]
for event in res['events']:
if 'A' in event:
for ip in event['A']:
if ip not in already:
response += IPv4Address(ip)
already.append(ip)
if 'domain' in event:
if event['domain'] not in already:
response += Domain(event['domain'])
already.append(event['domain'])
if 'MX' in event:
for mx in event['MX']:
if mx not in already:
response += MXRecord(mx)
already.append(mx)
if 'NS' in event:
for ns in event['NS']:
if ns not in already:
response += NSRecord(ns)
already.append(ns)
return response
return response
def dotransform(request, response):
interface = request.value
conf.iface = interface
subnet = ''
network = ''
cidr = ''
arpscan = []
for x in conf.route.routes:
if x[3] == interface and x[2] == '0.0.0.0':
subnet = x[1]
network = x[0]
subnet = subnetAddress(subnet)
cidr = cidr2subnet(subnet)
network = networkAddress(network)
ans, uans = arping(str(network) + '/' + str(cidr), verbose=0)
for send, rcv in ans:
e = IPv4Address(rcv.sprintf("%ARP.psrc%"))
e.internal = True
e += Field('ethernet.hwaddr',
rcv.sprintf("%Ether.src%"),
displayname='Hardware Address')
response += e
return response
def dotransform(request, response):
pcap = request.value
pkts = rdpcap(pcap)
tcp_srcip = []
udp_srcip = []
convo = []
for p in pkts:
if p.haslayer(TCP):
tcp_srcip.append(p.getlayer(IP).src)
if p.haslayer(IP) and p.haslayer(UDP):
udp_srcip.append(p.getlayer(IP).src)
for x in tcp_srcip:
talker = x, str(tcp_srcip.count(x)), 'tcp'
if talker not in convo:
convo.append(talker)
for y in udp_srcip:
talker = y, str(udp_srcip.count(y)), 'udp'
if talker not in convo:
convo.append(talker)
for srcip, count, proto in convo:
e = IPv4Address(srcip)
e.linkcolor = 0x2314CA
e.linklabel = proto
e += Field('pcapsrc', pcap, displayname='Original pcap File')
e += Field('proto', proto, displayname='Protocol')
response += e
return response
def dotransform(request, response):
ans = nslookup(request.value)
if ans is not None and DNS in ans:
for i in range(0, ans[DNS].ancount):
if ans[DNS].an[i].type == 1:
response += IPv4Address(ans[DNS].an[i].rdata)
return response
def dotransform(request, response):
filename = request.value
usedb = config['working/usedb']
# Check to see if we are using the database or not
if usedb == 0:
return response + UIMessage(
'No database support configured, check your config file')
else:
pass
# Connect to the database so we can search for IP addresses.
x = mongo_connect()
c = x['STREAMS']
try:
hosts = []
r = x.STREAMS.find({'File Name': {'$regex': filename}})
if r > 0:
for x in r:
hosts.append(x['Packet']['Source IP'])
hosts.append(x['Packet']['Destination IP'])
# streamid = x['Stream ID']
else:
return response + UIMessage(
'No records found, please make sure the pcap stream file is indexed'
)
for h in hosts:
e = IPv4Address(h)
# e += Field('streamid', streamid, displayname='Stream ID', MatchingRule='Loose')
response += e
return response
except Exception as e:
return response + UIMessage(str(e))
def dotransform(request, response):
iface = request.value
conf.iface = iface
subnet = ''
network = ''
for x in conf.route.routes:
if x[3] == iface and x[2] == '0.0.0.0':
subnet = x[1]
network = x[0]
subnet = subnetAddress(subnet)
cidr = cidr2subnet(subnet)
network = networkAddress(network)
ans, uans = arping(network + '/' + str(cidr), verbose=0)
for send, rcv in ans:
e = IPv4Address(rcv[ARP].psrc)
e += Field('ethernet.hwaddr',
rcv[Ether].src,
displayname='Hardware Address')
e.internal = True
response += e
return response
def dotransform(request, response):
pcap = request.value
usedb = config['working/usedb']
# Check to see if we are using the database or not
if usedb == 0:
return response + UIMessage(
'No database support configured, check your config file')
else:
pass
x = mongo_connect()
ipaddr = []
try:
r = x.STREAMS.find({"File Name": pcap}).count()
if r > 0:
p = x.STREAMS.find({"File Name": pcap}, {
"Packet.Source IP": 1,
"Packet.Destination IP": 1,
"_id": 0
})
for i in p:
sip = i['Packet']['Source IP']
dip = i['Packet']['Destination IP']
ipaddr.append(sip)
ipaddr.append(dip)
else:
return response + UIMessage(
'This needs to be run from a TCP/UDP stream')
except Exception as e:
return response + UIMessage(str(e))
for t in ipaddr:
e = IPv4Address(t)
response += e
return response
def dotransform(request, response):
sip = request.value
dump = request.fields['dumpfile']
x = parse_netflow(dump)
for i in x:
srcip = i[4]
srcip = srcip.split(':')[0]
proto = i[3]
if sip in srcip:
dip = i[6]
dip = dip.split(':')[0]
e = IPv4Address(dip)
e += Field('dumpfile',
dump,
displayname='Dump File',
matchingrule='loose')
# e.linklabel = proto
if proto == 'TCP':
e.linkcolor = 0xff0000
if proto == 'UDP':
e.linkcolor = 0x002bff
if proto == 'ICMP':
e.linkcolor = 0x2f9a0d
response += e
else:
pass
return response
def dotransform(request, response):
pcap = request.fields['pcapsrc']
srcip = request.fields['ipaddress']
e = IPv4Address(srcip)
e += Field('pcapsrc', pcap, displayname='Original pcap File')
response += e
return response
def dotransform(request, response):
e = IPv4Address(IP(dst='4.2.2.1').src)
e.internal = True
e += Field("ethernet.hwaddr", (Ether() / IP(dst='4.2.2.1')).src,
displayname="Hardware Address",
matching_rule=MatchingRule.Loose)
response += e
return response
def dotransform(request, response):
host = request.fields['kippodatabase']
x = db_connect(host)
cursor = x.cursor()
query = "select ip from sessions"
cursor.execute(query)
for ip in cursor:
e = IPv4Address('%s' % ip)
e += Field('kippodatabase', host, displayname='Kippo Database')
response += e
return response
def dotransform(request, response, config):
host = request.value
x = db_connect(host)
cursor = x.cursor()
query = ("select ip from sessions")
cursor.execute(query)
for ip in cursor:
e = IPv4Address('%s' % (ip))
e += Field('kippoip', host, displayname='Kippo IP')
response += e
return response
def dotransform(request, response):
if request.fields['behavioral'] != "":
try:
behavior = ast.literal_eval(request.fields['behavior_data'])
except Exception as e:
debug("Entity has no behavioral data")
return response
if behavior.has_key("network"):
if behavior['network'].has_key('dns'):
for item in behavior['network']['dns']:
host = Domain(item['hostname'])
host.linklabel = "vt_behav->hosts"
response += host
if item.has_key('ip'):
ip = IPv4Address(item['ip'])
ip.linklabel = "vt_behav->hosts"
response += ip
if behavior['network'].has_key('tcp'):
for item in behavior['network']['tcp']:
conn = item.split(":")
r = IPv4Address(conn[0])
r.linklabel = "vt_behav->hosts_tcp (%s)" % str(conn[1])
response += r
if behavior['network'].has_key('udp'):
for item in behavior['network']['udp']:
conn = item.split(":")
r = IPv4Address(conn[0])
r.linklabel = "vt_behav->hosts_udp (%s)" % str(conn[1])
response += r
if behavior['network'].has_key('http'):
for item in behavior['network']['http']:
r = URL(item['url'])
r.url = item['url']
r.linklabel = "vt_behav->hosts_http (%s)" % item['method']
response += r
else:
debug("ripVT: No behavioral for %s" % request.value)
return response
def dotransform(request, response):
nexthop = conf.route6.route(
request.value)[2] if ':' in request.value else conf.route.route(
request.value)[2]
e = IPv4Address(nexthop)
e.internal = True
if ':' not in nexthop:
e += Field('ethernet.hwaddr',
getmacbyip(nexthop),
displayname='Hardware Address')
response += e
return response
def dotransform(request, response, config):
if 'taskid' in request.fields:
task = request.fields['taskid']
else:
task = request.value
netw = network(report(task))
for d in netw['domains']:
response += IPv4Address(d['ip'].decode('ascii'), taskid=task)
return response
def dotransform(request, response):
try:
s_ip = request.fields['source_ip']
pcap = request.fields['pcapsrc']
except:
return response + UIMessage('Sorry this isnt a SmP OS Type')
e = IPv4Address(s_ip)
e += Field('pcapsrc', pcap, displayname='Original pcap File')
response += e
return response
def dotransform(request, response, config):
if 'workspace' in request.fields:
workspace = request.fields['workspace']
else:
workspace = request.value
dbcon = db_connect(workspace)
host_list = get_hosts(dbcon)
for ip in host_list:
if ip[0] == request.value:
e = IPv4Address(ip[1])
e += Field("workspace", workspace, displayname='Workspace')
e += Field("domainname", request.value, displayname='Domain Name')
response += e
else:
e = IPv4Address(ip[1])
e += Field("workspace", workspace, displayname='Workspace')
response += e
return response
def dotransform(request, response):
# Build the request
page = build(request.value)
# Search the page to extract all IP addresses present
try:
for element in page.findAll(text=re.compile(
"^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$"
)):
response += IPv4Address(element)
except:
pass
return response
def dotransform(request, response):
data = getbehavior(request.value)
try:
try:
network = data['network']
except:
#no network data
pass
try:
for result in network['dns']:
dom = result['hostname']
ip = result['ip']
response += Domain(dom)
response += IPv4Address['ip']
except:
#no dns data
pass
try:
for request in network['http']:
uri = URL(request['uri'])
uri.url = request['uri']
ua = UserAgent(request['user-agent'])
#req = HTTPRequest(request['data'])
port = Port(request['port'])
response += uri
response += ua
#response += req
response += port
except:
#no http data
pass
try:
for entry in network['tcp']:
e = entry['dst']
if e.startswith('10.'):
pass
else:
conn = IPv4Address(e)
response += conn
except:
#no tcp data
pass
except:
response += UIMessage(data['verbose_msg'])
return response
def dotransform(request, response):
#Build request
page = build(request.value)
#Find the Hosts section and extract IPs
try:
table = page.find("div", {"id": "network_hosts"}).findNext('table')
elements = table.findAll('td', {"class": "row"})
for element in elements:
text = element.find(text=True)
response += IPv4Address(text)
except:
return response
return response
def dotransform(request, response):
pcap = request.fields['pcapsrc']
try:
srcip = request.fields['hostdst']
except:
srcip = request.fields['sniffMyPackets.hostdst']
if srcip is not None:
e = IPv4Address(srcip)
e += Field('pcapsrc', pcap, displayname='Original pcap File')
response += e
return response
else:
return response + UIMessage('Does not contain Source IP field')
def do_transform(self, request, response, config):
base_url = config['OTX_transform.local.otx_url']
api_key = config['OTX_transform.local.api_key']
host = request.entity.value
url = '%s/indicators/hostname/%s/passive_dns' % \
(base_url, host)
r = requests.get(url, headers={'X-OTX-API-KEY': api_key})
if r.status_code == 200:
res = r.json()
for pdns in res['passive_dns']:
ip = IPv4Address(pdns['address'])
ip.link_label = 'last: %s' % pdns['last']
response += ip
return response
def detType(in_val):
val = str(in_val)
#::Email
email = re.compile(".*\[@\][a-z0-9\-]{1,}\.[a-z0-9\-]{1,}")
#::IP
ipv4 = re.compile("^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$")
#::CIDR
cidr = re.compile("^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/\d{1,2}$")
#::Range
v4range = re.compile(
"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\-\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$"
)
#::Domain
dom = re.compile("([a-z0-9\-]{1,}\.?)+\.[a-z0-9\-]{1,}$")
if email.match(val):
e = EmailAddress(val)
return e
if ipv4.match(val):
e = IPv4Address(val)
return e
if cidr.match(val):
e = CIDR(val)
return e
if v4range.match(val):
e = Range(val)
return e
if dom.match(val):
e = Domain(val)
return e
if re.match("^([a-z]*)://", val, re.M | re.I):
e = URL(val)
e.url = val
return e
def dotransform(request, response):
# Check PAN Authentication AND KEY
key = pamod.get_login()
# Create and submit the query to the API and return the jobid
tid = request.fields['tid']
query = '(threatid eq %s)' % (tid)
jobid = pamod.pa_log_query('threat', key, query)
sleep(5)
# Loop function to check if the log query job is done
root = ET.fromstring(pamod.pa_log_get(jobid, key))
for status in root.findall(".//job/status"):
while status.text == 'ACT':
sleep(5)
root = ET.fromstring(pamod.pa_log_get(jobid, key))
for status in root.findall(".//job/status"):
if status.text == 'FIN':
break
# parse the log data and create dictionaries stored in a list for each individual log
log_list = []
for entry in root.findall(".//log/logs/entry"):
entry_dic = {}
for data in entry:
entry_dic[data.tag] = data.text
log_list.append(entry_dic)
# Create the Maltego Entity
ip_list = []
for d in log_list:
if d['src'] not in ip_list:
response += IPv4Address(
d['src'],
tid=d['tid'],
ipsrc=d['dst'],
)
ip_list.append(d['src'])
return response
def dotransform(request, response):
page = build(request.value)
if page.find('span', {'id' : 'error'}):
# No Matches in Malc0de
return response
else:
for hit in page.findAll('tr', {'class' : 'class1'}):
temp = []
for column in hit.findAll('td'):
temp.append(column.text)
e = IPv4Address(temp[2])
e += Field('URL', temp[1], displayname='URL')
e += Field('AS', temp[4], displayname='AS')
e += Field('Date', temp[0], displayname='Date')
response += e
return response
