def to_clients(response, output): cat = None for line in output.split('\n'): if not line: continue elif line.startswith(' '): e = None if cat in range(Category.AlternativeTargetInterfaces, Category.OtherAssociations): for ip in ip_matcher.findall(line): e = IPv4Address(ip) e += Field('category', Category.name(cat), displayname='Category') response += e elif cat == Category.OtherAssociations: ip, desc = line.strip().split(' ', 1) e = IPv4Address(ip) e += Label('Additional Info', desc) e += Field('category', Category.name(cat), displayname='Category') response += e elif line.startswith(' '): for id in range(Category.AlternativeTargetInterfaces, Category.OtherAssociations + 1): if Category.name(id) in line: cat = id break
def passivescan(network, response): nodes = {} debug('Sniffing network traffic for more hosts.') ans = sniff(count=config['scapy/sniffcount'], timeout=config['scapy/snifftimeout']) debug('Analyzing traffic.') for i in ans: src = None dst = None if IP in i: src = i[IP].src dst = i[IP].dst elif ARP in i: src = i[ARP].psrc dst = i[ARP].pdst else: continue if src in network and src not in nodes: nodes[src] = True e = IPv4Address(src, internal=True) e += Field('ethernet.hwaddr', i.src, displayname='Hardware Address') response += e if dst in network and dst not in nodes and i.dst != 'ff:ff:ff:ff:ff:ff': nodes[dst] = True e = IPv4Address(dst, internal=True) e += Field('ethernet.hwaddr', i.dst, displayname='Hardware Address') response += e
def dotransform(request, response): # Report transform progress progress(50) total = "" urldom = 'https://www.virustotal.com/en/domain/' + request.value + '/information/' soup = BeautifulSoup(urllib2.urlopen(urldom).read()) try: links = soup.findAll('div', attrs={'class': 'enum'}) for link in links: total += str(link) total = BeautifulSoup(total) for totals in total.findAll('a', href=True): totals = totals['href'] theIP = totals.replace("/en/ip-address/", "") e = theIP.replace("/information/", "") response += IPv4Address(e) except IOError: print 'IO Error' # Update progress progress(100) # Return response for visualization return response
def dotransform(request, response, config): """ The dotransform function is our transform's entry point. The request object has the following properties: - value: a string containing the value of the input entity. - fields: a dictionary of entity field names and their respective values of the input entity. - params: any additional command-line arguments to be passed to the transform. - entity: the information above is serialized into an Entity object. The entity type is determined by the inputs field in @configure for local transforms. For remote transforms, the entity type is determined by the information in the body of the request. Local transforms suffer from one limitation: if more than one entity type is listed in the inputs field of @configure, the entity type might not be resolvable. Therefore, this should not be referenced in local transforms if there is more than one input entity type defined in @configure. The response object is a container for output entities, UI messages, and exception messages. The config object contains a key-value store of the configuration file. TODO: write your data mining logic below. """ progress(10) debug('Extracting IP') val = request.entities[0].fields['malriq.ip'].value ipe = IPv4Address(val) ipe.ip = val response += [ipe] progress(100) return response
def findlocalneighbors(network, response): debug('ARP sweeping %s' % network.netblock) # e = Netblock(network.netblock) # e += Label('CIDR Notation', repr(network)) # e += Label('Network Mask', network.netmask) # e += Label('Number of Hosts', int(~network.netmask) - 1) # response += e ans = arping(repr(network), timeout=config['scapy/sr_timeout'], verbose=config['scapy/sr_verbose'])[0] for i in ans: e = IPv4Address(i[1].psrc) e.internal = True e += Field('ethernet.hwaddr', i[1].hwsrc, displayname='Hardware Address') response += e if len(ans) <= 1: passivescan(network, response) return response
def do_transform(self, request, response, config): be = BinaryEdge(config['binaryedge.local.api_key']) domain = request.entity.value try: # Only consider the fist page res = be.domain_dns(domain) except BinaryEdgeException as e: raise MaltegoException('BinaryEdge error: %s' % e.msg) else: already = [domain] for event in res['events']: if 'A' in event: for ip in event['A']: if ip not in already: response += IPv4Address(ip) already.append(ip) if 'domain' in event: if event['domain'] not in already: response += Domain(event['domain']) already.append(event['domain']) if 'MX' in event: for mx in event['MX']: if mx not in already: response += MXRecord(mx) already.append(mx) if 'NS' in event: for ns in event['NS']: if ns not in already: response += NSRecord(ns) already.append(ns) return response return response
def dotransform(request, response): interface = request.value conf.iface = interface subnet = '' network = '' cidr = '' arpscan = [] for x in conf.route.routes: if x[3] == interface and x[2] == '0.0.0.0': subnet = x[1] network = x[0] subnet = subnetAddress(subnet) cidr = cidr2subnet(subnet) network = networkAddress(network) ans, uans = arping(str(network) + '/' + str(cidr), verbose=0) for send, rcv in ans: e = IPv4Address(rcv.sprintf("%ARP.psrc%")) e.internal = True e += Field('ethernet.hwaddr', rcv.sprintf("%Ether.src%"), displayname='Hardware Address') response += e return response
def dotransform(request, response): pcap = request.value pkts = rdpcap(pcap) tcp_srcip = [] udp_srcip = [] convo = [] for p in pkts: if p.haslayer(TCP): tcp_srcip.append(p.getlayer(IP).src) if p.haslayer(IP) and p.haslayer(UDP): udp_srcip.append(p.getlayer(IP).src) for x in tcp_srcip: talker = x, str(tcp_srcip.count(x)), 'tcp' if talker not in convo: convo.append(talker) for y in udp_srcip: talker = y, str(udp_srcip.count(y)), 'udp' if talker not in convo: convo.append(talker) for srcip, count, proto in convo: e = IPv4Address(srcip) e.linkcolor = 0x2314CA e.linklabel = proto e += Field('pcapsrc', pcap, displayname='Original pcap File') e += Field('proto', proto, displayname='Protocol') response += e return response
def dotransform(request, response): ans = nslookup(request.value) if ans is not None and DNS in ans: for i in range(0, ans[DNS].ancount): if ans[DNS].an[i].type == 1: response += IPv4Address(ans[DNS].an[i].rdata) return response
def dotransform(request, response): filename = request.value usedb = config['working/usedb'] # Check to see if we are using the database or not if usedb == 0: return response + UIMessage( 'No database support configured, check your config file') else: pass # Connect to the database so we can search for IP addresses. x = mongo_connect() c = x['STREAMS'] try: hosts = [] r = x.STREAMS.find({'File Name': {'$regex': filename}}) if r > 0: for x in r: hosts.append(x['Packet']['Source IP']) hosts.append(x['Packet']['Destination IP']) # streamid = x['Stream ID'] else: return response + UIMessage( 'No records found, please make sure the pcap stream file is indexed' ) for h in hosts: e = IPv4Address(h) # e += Field('streamid', streamid, displayname='Stream ID', MatchingRule='Loose') response += e return response except Exception as e: return response + UIMessage(str(e))
def dotransform(request, response): iface = request.value conf.iface = iface subnet = '' network = '' for x in conf.route.routes: if x[3] == iface and x[2] == '0.0.0.0': subnet = x[1] network = x[0] subnet = subnetAddress(subnet) cidr = cidr2subnet(subnet) network = networkAddress(network) ans, uans = arping(network + '/' + str(cidr), verbose=0) for send, rcv in ans: e = IPv4Address(rcv[ARP].psrc) e += Field('ethernet.hwaddr', rcv[Ether].src, displayname='Hardware Address') e.internal = True response += e return response
def dotransform(request, response): pcap = request.value usedb = config['working/usedb'] # Check to see if we are using the database or not if usedb == 0: return response + UIMessage( 'No database support configured, check your config file') else: pass x = mongo_connect() ipaddr = [] try: r = x.STREAMS.find({"File Name": pcap}).count() if r > 0: p = x.STREAMS.find({"File Name": pcap}, { "Packet.Source IP": 1, "Packet.Destination IP": 1, "_id": 0 }) for i in p: sip = i['Packet']['Source IP'] dip = i['Packet']['Destination IP'] ipaddr.append(sip) ipaddr.append(dip) else: return response + UIMessage( 'This needs to be run from a TCP/UDP stream') except Exception as e: return response + UIMessage(str(e)) for t in ipaddr: e = IPv4Address(t) response += e return response
def dotransform(request, response): sip = request.value dump = request.fields['dumpfile'] x = parse_netflow(dump) for i in x: srcip = i[4] srcip = srcip.split(':')[0] proto = i[3] if sip in srcip: dip = i[6] dip = dip.split(':')[0] e = IPv4Address(dip) e += Field('dumpfile', dump, displayname='Dump File', matchingrule='loose') # e.linklabel = proto if proto == 'TCP': e.linkcolor = 0xff0000 if proto == 'UDP': e.linkcolor = 0x002bff if proto == 'ICMP': e.linkcolor = 0x2f9a0d response += e else: pass return response
def dotransform(request, response): pcap = request.fields['pcapsrc'] srcip = request.fields['ipaddress'] e = IPv4Address(srcip) e += Field('pcapsrc', pcap, displayname='Original pcap File') response += e return response
def dotransform(request, response): e = IPv4Address(IP(dst='4.2.2.1').src) e.internal = True e += Field("ethernet.hwaddr", (Ether() / IP(dst='4.2.2.1')).src, displayname="Hardware Address", matching_rule=MatchingRule.Loose) response += e return response
def dotransform(request, response): host = request.fields['kippodatabase'] x = db_connect(host) cursor = x.cursor() query = "select ip from sessions" cursor.execute(query) for ip in cursor: e = IPv4Address('%s' % ip) e += Field('kippodatabase', host, displayname='Kippo Database') response += e return response
def dotransform(request, response, config): host = request.value x = db_connect(host) cursor = x.cursor() query = ("select ip from sessions") cursor.execute(query) for ip in cursor: e = IPv4Address('%s' % (ip)) e += Field('kippoip', host, displayname='Kippo IP') response += e return response
def dotransform(request, response): if request.fields['behavioral'] != "": try: behavior = ast.literal_eval(request.fields['behavior_data']) except Exception as e: debug("Entity has no behavioral data") return response if behavior.has_key("network"): if behavior['network'].has_key('dns'): for item in behavior['network']['dns']: host = Domain(item['hostname']) host.linklabel = "vt_behav->hosts" response += host if item.has_key('ip'): ip = IPv4Address(item['ip']) ip.linklabel = "vt_behav->hosts" response += ip if behavior['network'].has_key('tcp'): for item in behavior['network']['tcp']: conn = item.split(":") r = IPv4Address(conn[0]) r.linklabel = "vt_behav->hosts_tcp (%s)" % str(conn[1]) response += r if behavior['network'].has_key('udp'): for item in behavior['network']['udp']: conn = item.split(":") r = IPv4Address(conn[0]) r.linklabel = "vt_behav->hosts_udp (%s)" % str(conn[1]) response += r if behavior['network'].has_key('http'): for item in behavior['network']['http']: r = URL(item['url']) r.url = item['url'] r.linklabel = "vt_behav->hosts_http (%s)" % item['method'] response += r else: debug("ripVT: No behavioral for %s" % request.value) return response
def dotransform(request, response): nexthop = conf.route6.route( request.value)[2] if ':' in request.value else conf.route.route( request.value)[2] e = IPv4Address(nexthop) e.internal = True if ':' not in nexthop: e += Field('ethernet.hwaddr', getmacbyip(nexthop), displayname='Hardware Address') response += e return response
def dotransform(request, response, config): if 'taskid' in request.fields: task = request.fields['taskid'] else: task = request.value netw = network(report(task)) for d in netw['domains']: response += IPv4Address(d['ip'].decode('ascii'), taskid=task) return response
def dotransform(request, response): try: s_ip = request.fields['source_ip'] pcap = request.fields['pcapsrc'] except: return response + UIMessage('Sorry this isnt a SmP OS Type') e = IPv4Address(s_ip) e += Field('pcapsrc', pcap, displayname='Original pcap File') response += e return response
def dotransform(request, response, config): if 'workspace' in request.fields: workspace = request.fields['workspace'] else: workspace = request.value dbcon = db_connect(workspace) host_list = get_hosts(dbcon) for ip in host_list: if ip[0] == request.value: e = IPv4Address(ip[1]) e += Field("workspace", workspace, displayname='Workspace') e += Field("domainname", request.value, displayname='Domain Name') response += e else: e = IPv4Address(ip[1]) e += Field("workspace", workspace, displayname='Workspace') response += e return response
def dotransform(request, response): # Build the request page = build(request.value) # Search the page to extract all IP addresses present try: for element in page.findAll(text=re.compile( "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$" )): response += IPv4Address(element) except: pass return response
def dotransform(request, response): data = getbehavior(request.value) try: try: network = data['network'] except: #no network data pass try: for result in network['dns']: dom = result['hostname'] ip = result['ip'] response += Domain(dom) response += IPv4Address['ip'] except: #no dns data pass try: for request in network['http']: uri = URL(request['uri']) uri.url = request['uri'] ua = UserAgent(request['user-agent']) #req = HTTPRequest(request['data']) port = Port(request['port']) response += uri response += ua #response += req response += port except: #no http data pass try: for entry in network['tcp']: e = entry['dst'] if e.startswith('10.'): pass else: conn = IPv4Address(e) response += conn except: #no tcp data pass except: response += UIMessage(data['verbose_msg']) return response
def dotransform(request, response): #Build request page = build(request.value) #Find the Hosts section and extract IPs try: table = page.find("div", {"id": "network_hosts"}).findNext('table') elements = table.findAll('td', {"class": "row"}) for element in elements: text = element.find(text=True) response += IPv4Address(text) except: return response return response
def dotransform(request, response): pcap = request.fields['pcapsrc'] try: srcip = request.fields['hostdst'] except: srcip = request.fields['sniffMyPackets.hostdst'] if srcip is not None: e = IPv4Address(srcip) e += Field('pcapsrc', pcap, displayname='Original pcap File') response += e return response else: return response + UIMessage('Does not contain Source IP field')
def do_transform(self, request, response, config): base_url = config['OTX_transform.local.otx_url'] api_key = config['OTX_transform.local.api_key'] host = request.entity.value url = '%s/indicators/hostname/%s/passive_dns' % \ (base_url, host) r = requests.get(url, headers={'X-OTX-API-KEY': api_key}) if r.status_code == 200: res = r.json() for pdns in res['passive_dns']: ip = IPv4Address(pdns['address']) ip.link_label = 'last: %s' % pdns['last'] response += ip return response
def detType(in_val): val = str(in_val) #::Email email = re.compile(".*\[@\][a-z0-9\-]{1,}\.[a-z0-9\-]{1,}") #::IP ipv4 = re.compile("^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$") #::CIDR cidr = re.compile("^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/\d{1,2}$") #::Range v4range = re.compile( "^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\-\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$" ) #::Domain dom = re.compile("([a-z0-9\-]{1,}\.?)+\.[a-z0-9\-]{1,}$") if email.match(val): e = EmailAddress(val) return e if ipv4.match(val): e = IPv4Address(val) return e if cidr.match(val): e = CIDR(val) return e if v4range.match(val): e = Range(val) return e if dom.match(val): e = Domain(val) return e if re.match("^([a-z]*)://", val, re.M | re.I): e = URL(val) e.url = val return e
def dotransform(request, response): # Check PAN Authentication AND KEY key = pamod.get_login() # Create and submit the query to the API and return the jobid tid = request.fields['tid'] query = '(threatid eq %s)' % (tid) jobid = pamod.pa_log_query('threat', key, query) sleep(5) # Loop function to check if the log query job is done root = ET.fromstring(pamod.pa_log_get(jobid, key)) for status in root.findall(".//job/status"): while status.text == 'ACT': sleep(5) root = ET.fromstring(pamod.pa_log_get(jobid, key)) for status in root.findall(".//job/status"): if status.text == 'FIN': break # parse the log data and create dictionaries stored in a list for each individual log log_list = [] for entry in root.findall(".//log/logs/entry"): entry_dic = {} for data in entry: entry_dic[data.tag] = data.text log_list.append(entry_dic) # Create the Maltego Entity ip_list = [] for d in log_list: if d['src'] not in ip_list: response += IPv4Address( d['src'], tid=d['tid'], ipsrc=d['dst'], ) ip_list.append(d['src']) return response
def dotransform(request, response): page = build(request.value) if page.find('span', {'id' : 'error'}): # No Matches in Malc0de return response else: for hit in page.findAll('tr', {'class' : 'class1'}): temp = [] for column in hit.findAll('td'): temp.append(column.text) e = IPv4Address(temp[2]) e += Field('URL', temp[1], displayname='URL') e += Field('AS', temp[4], displayname='AS') e += Field('Date', temp[0], displayname='Date') response += e return response