def do_transform(self, request, response, config): be = BinaryEdge(config['binaryedge.local.api_key']) domain = request.entity.value try: # Only consider the fist page res = be.domain_dns(domain) except BinaryEdgeException as e: raise MaltegoException('BinaryEdge error: %s' % e.msg) else: already = [domain] for event in res['events']: if 'A' in event: for ip in event['A']: if ip not in already: response += IPv4Address(ip) already.append(ip) if 'domain' in event: if event['domain'] not in already: response += Domain(event['domain']) already.append(event['domain']) if 'MX' in event: for mx in event['MX']: if mx not in already: response += MXRecord(mx) already.append(mx) if 'NS' in event: for ns in event['NS']: if ns not in already: response += NSRecord(ns) already.append(ns) return response return response
def dotransform(request, response, config): error, found = lookup_whois(request.value) if not error and found: if dict == type(found): for result, value in found.iteritems(): if set == type(value): if "whois_domains" == result: for d in value: if d: e = Domain(d) e.fqdn = d response += e if "whois_emails" == result: for em in value: if em: e = EmailAddress(em) response += e if "whois_nameservers" == result: for w in value: if w: e = NSRecord(w) response += e #Display error message in Transform Output response += UIMessage(error) return response
def dotransform(request, response): ans = nslookup(request.value, 'NS') if ans is not None and DNS in ans: for i in range(0, ans[DNS].ancount): if ans[DNS].an[i].type == 2: response += NSRecord(ans[DNS].an[i].rdata.rstrip('.')) return response
def addrecord(record, response): if record.type == 2: response += NSRecord(record.rdata.rstrip('.')) elif record.type == 15: e = MXRecord(record.rdata.rstrip('.')) e += Field('mxrecord.priority', record.mxpriority) response += e elif record.type in [1, 5]: response += DNSName(record.rrname.rstrip('.')) elif record.type == 16: response += Phrase(record.rdata) return response
def dotransform(request, response): domain = request.value results = query('-r', domain, 0, 'n') for result in results: data = json.loads(result) if data.has_key('time_first'): first = data['time_first'] last = data['time_last'] elif data.has_key('zone_time_first'): first = data['zone_time_first'] last = data['zone_time_last'] fnice = datetime.datetime.fromtimestamp( int(first)).strftime('%m-%d-%Y') lnice = datetime.datetime.fromtimestamp(int(last)).strftime('%m-%d-%Y') if data['rrtype'] == 'NS': for item in data['rdata']: e = NSRecord(item) e.linklabel = fnice + ' - ' + lnice response += e elif data['rrtype'] == 'MX': for item in data['rdata']: e = MXRecord(item) e.linklabel = fnice + ' - ' + lnice response += e elif data['rrtype'] == 'CNAME': for item in data['rdata']: e = Domain(item.rstrip('.')) e.linklabel = fnice + ' - ' + lnice response += e elif data['rrtype'] == 'A': pass else: type = data['rrtype'] for item in data['rdata']: label = type + ' ' + item e = Phrase(label) e.linklabel = fnice + ' - ' + lnice response += e return response
def dotransform(request, response, config): if 'taskid' in request.fields: task = request.fields['taskid'] else: task = request.value netw = network(report(task)) dns_lst = [] for d in netw['dns']: if d['request'] not in dns_lst: response += NSRecord(d['request'].decode('ascii'), taskid=task) dns_lst.append(d['request']) return response
def dotransform(request, response): domain = request.value results = query("-r", domain, 0, "n") for result in results: data = json.loads(result) if data.has_key("time_first"): first = data["time_first"] last = data["time_last"] elif data.has_key("zone_time_first"): first = data["zone_time_first"] last = data["zone_time_last"] fnice = datetime.datetime.fromtimestamp(int(first)).strftime("%m-%d-%Y") lnice = datetime.datetime.fromtimestamp(int(last)).strftime("%m-%d-%Y") if data["rrtype"] == "NS": for item in data["rdata"]: e = NSRecord(item) e.linklabel = fnice + " - " + lnice response += e elif data["rrtype"] == "MX": for item in data["rdata"]: e = MXRecord(item) e.linklabel = fnice + " - " + lnice response += e elif data["rrtype"] == "CNAME": for item in data["rdata"]: e = Domain(item.rstrip(".")) e.linklabel = fnice + " - " + lnice response += e elif data["rrtype"] == "A": pass else: type = data["rrtype"] for item in data["rdata"]: label = type + " " + item e = Phrase(label) e.linklabel = fnice + " - " + lnice response += e return response
def dotransform(request, response, config): tr_details = [ 'Reference', 'Source', 'KillChain', 'Firstseen', 'Lastseen', 'Attribution', 'ProcessType', 'Rrname', 'Rdata', 'Country', 'Tags', 'Comment', 'RootNode', 'Confidence' ] #Default link color is black linkcolor = "0x000000" cache, found = search(request.value) if found: if list == type(found): for indicator in found: debug(indicator) e = '' indtype = indicator['Type'].lower().strip() if "whois email" == indtype: e = EmailAddress(indicator['Indicator']) #response += e if "name server" == indtype: e = NSRecord(indicator['Indicator']) #response += e if "domain" == indtype: e = Domain(indicator['Indicator']) e.fqdn = indicator['Indicator'] #response += e #IF Type is not domain, check if Rrname is not empty elif indicator['Rrname'] and indicator['Rrname'] != 'NA': d = Domain(indicator['Rrname']) d.fqdn = indicator['Rrname'] response += d if "ip" == indtype: e = IPv4Address(indicator['Indicator']) #response += e #IF Type is not IP, check if Rdata is not empty elif indicator['Rdata']: i = IPv4Address(indicator['Rdata']) response += i if "phone or fax no." == indtype: e = PhoneNumber(indicator['Indicator']) #response += e if "whois address component" == indtype: e = Phrase(indicator['Indicator']) #response += e if "email" == indtype: e = EmailAddress(indicator['Indicator']) #response += e if "netname" == indtype: e = NetNameThreatRecon(indicator['Indicator']) #response += e if "cidr" == indtype: e = IPv4Address(indicator['Indicator']) #response += e if "netrange" == indtype: e = Netblock(indicator['Indicator']) #response += e if indicator['Country']: l = Location(indicator['Country']) response += l #Add Comments and details to own Entity entity = e #request.entity #Set comments if indicator['Comment']: entity.notes = string_filter(indicator['Comment']) #Set Details for detail in tr_details: if detail in indicator: if indicator[detail]: entity += Label(name=detail, value=string_filter( indicator[detail])) #Set link color if "Confidence" in indicator: if indicator['Confidence'] >= 70: linkcolor = "0xff0000" entity.linkcolor = linkcolor response += entity return response
def dotransform(request, response, config): tr_details = [ 'Reference', 'Source', 'KillChain', 'Firstseen', 'Lastseen', 'Attribution', 'ProcessType', 'Rrname', 'Rdata', 'Country', 'Tags', 'Comment', 'RootNode', 'Confidence' ] #Disable cache to get actual data from Threat Recon cache, found = search(request.value, cache=False) #Default linkcolor linkcolor = "0x000000" if found: if defaultdict == type(found): for rootnode, value in found.iteritems(): #If the RootNode is empty, display attributes if len(rootnode) == 0: for indicator in value: #debug(indicator) e = '' indtype = indicator['Type'].lower().strip() if "whois email" == indtype: e = EmailAddress(indicator['Indicator']) if "name server" == indtype: e = NSRecord(indicator['Indicator']) if "domain" == indtype: e = Domain(indicator['Indicator']) e.fqdn = indicator['Indicator'] if "ip" == indtype: e = IPv4Address(indicator['Indicator']) if "phone or fax no." == indtype: e = PhoneNumber(indicator['Indicator']) if "whois address component" == indtype: e = Phrase(indicator['Indicator']) if "email" == indtype: e = EmailAddress(indicator['Indicator']) if "netname" == indtype: e = NetNameThreatRecon(indicator['Indicator']) if "cidr" == indtype: e = IPv4Address(indicator['Indicator']) if "netrange" == indtype: e = Netblock(indicator['Indicator']) if e: #Set linkcolor e.linkcolor = linkcolor #Set comments if indicator['Comment']: e.notes = string_filter(indicator['Comment']) #Set Details for detail in tr_details: if detail in indicator: if indicator[detail]: e += Label(name=detail, value=string_filter( indicator[detail])) response += e else: #Display the RootNodes e = ThreatRecon(rootnode) response += e return response
def nslookup(name, type_, response, resolvers=None, recursive=True): name = name.rstrip('.') if isinstance(type_, basestring): type_ = dns.rdatatype.from_text(type_) if type_ == dns.rdatatype.PTR: name = dns.reversename.from_address(name) if not resolvers: resolvers = dns.resolver.get_default_resolver().nameservers elif isinstance(resolvers, basestring): resolvers = [resolvers] if type_ in [dns.rdatatype.AXFR, dns.rdatatype.IXFR]: try: discovered_names = [] for ns in dns.resolver.query(name, dns.rdatatype.NS): xfr(ns.to_text(), name, response, discovered_names=discovered_names) return True except dns.resolver.NXDOMAIN: response += UIMessage("DNS records for %s do not exist." % repr(name)) except dns.resolver.NoNameservers: response += UIMessage("No nameservers found for %s." % repr(name)) except dns.resolver.Timeout: response += UIMessage("DNS request for %s timed out." % repr(name)) except dns.resolver.NoAnswer: response += UIMessage("DNS request for %s resulted in no response." % repr(name)) except socket.error: response += UIMessage("A socket error has occurred. Make sure you are connected or the traffic is allowed.") return False try: request = dns.message.make_query(name, type_, dns.rdataclass.IN) if not recursive: request.flags ^= dns.flags.RD for resolver in resolvers: ans = dns.query.udp(request, resolver).answer if ans: for rrset in ans: for rr in rrset: if rr.rdtype == type_: if type_ == dns.rdatatype.A: response += IPv4Address(rr.to_text(True)) elif type_ == dns.rdatatype.NS: response += UIMessage(repr(rr)) response += NSRecord(rr.to_text()[:-1]) elif type_ == dns.rdatatype.CNAME: response += DNSName(rr.to_text()) elif type_ == dns.rdatatype.SOA: e = NSRecord(rr.mname.to_text(True)) e += Field('mailaddr', rr.rname.to_text(True), displayname='Authority') response += e elif type_ == dns.rdatatype.PTR: response += DNSName(rr.to_text()[:-1]) elif type_ == dns.rdatatype.MX: e = MXRecord(rr.exchange.to_text(True)) e.mxpriority = rr.preference response += e elif type_ == dns.rdatatype.TXT: response += Phrase(rr.to_text(True)) elif type_ == dns.rdatatype.AAAA: response += IPv6Address(rr.to_text(True)) else: response += Phrase(rr.to_text(True)) return True except dns.resolver.NXDOMAIN: response += UIMessage("DNS records for %s do not exist." % repr(name)) except dns.resolver.Timeout: response += UIMessage("DNS request for %s timed out." % repr(name)) except dns.resolver.NoNameservers: response += UIMessage("No name servers found for %s." % repr(name)) except dns.resolver.NoAnswer: response += UIMessage("The DNS server returned with no response for %s." % repr(name)) except socket.error: response += UIMessage("A socket error has occurred. Make sure you are connected or the traffic is allowed.") return False
def nslookup(name, type_, response, resolvers=None, recursive=True): name = name.rstrip('.') if isinstance(type_, basestring): type_ = dns.rdatatype.from_text(type_) if type_ == dns.rdatatype.PTR: name = dns.reversename.from_address(name) if not resolvers: resolvers = dns.resolver.get_default_resolver().nameservers elif isinstance(resolvers, basestring): resolvers = [resolvers] if type_ in [dns.rdatatype.AXFR, dns.rdatatype.IXFR]: try: discovered_names = [] for ns in dns.resolver.query(name, dns.rdatatype.NS): xfr(ns.to_text(), name, response, discovered_names=discovered_names) return True except dns.resolver.NXDOMAIN: response += UIMessage("DNS records for %s do not exist." % repr(name)) except dns.resolver.NoNameservers: response += UIMessage("No nameservers found for %s." % repr(name)) except dns.resolver.Timeout: response += UIMessage("DNS request for %s timed out." % repr(name)) except dns.resolver.NoAnswer: response += UIMessage( "DNS request for %s resulted in no response." % repr(name)) except socket.error: response += UIMessage( "A socket error has occurred. Make sure you are connected or the traffic is allowed." ) return False try: request = dns.message.make_query(name, type_, dns.rdataclass.IN) if not recursive: request.flags ^= dns.flags.RD for resolver in resolvers: ans = dns.query.udp(request, resolver).answer if ans: for rrset in ans: for rr in rrset: if rr.rdtype == type_: if type_ == dns.rdatatype.A: response += IPv4Address(rr.to_text(True)) elif type_ == dns.rdatatype.NS: response += UIMessage(repr(rr)) response += NSRecord(rr.to_text()[:-1]) elif type_ == dns.rdatatype.CNAME: response += DNSName(rr.to_text()) elif type_ == dns.rdatatype.SOA: e = NSRecord(rr.mname.to_text(True)) e += Field('mailaddr', rr.rname.to_text(True), displayname='Authority') response += e elif type_ == dns.rdatatype.PTR: response += DNSName(rr.to_text()[:-1]) elif type_ == dns.rdatatype.MX: e = MXRecord(rr.exchange.to_text(True)) e.mxpriority = rr.preference response += e elif type_ == dns.rdatatype.TXT: response += Phrase(rr.to_text(True)) elif type_ == dns.rdatatype.AAAA: response += IPv6Address(rr.to_text(True)) else: response += Phrase(rr.to_text(True)) return True except dns.resolver.NXDOMAIN: response += UIMessage("DNS records for %s do not exist." % repr(name)) except dns.resolver.Timeout: response += UIMessage("DNS request for %s timed out." % repr(name)) except dns.resolver.NoNameservers: response += UIMessage("No name servers found for %s." % repr(name)) except dns.resolver.NoAnswer: response += UIMessage( "The DNS server returned with no response for %s." % repr(name)) except socket.error: response += UIMessage( "A socket error has occurred. Make sure you are connected or the traffic is allowed." ) return False
def dotransform(request, response, config): """ The dotransform function is our transform's entry point. The request object has the following properties: - value: a string containing the value of the input entity. - fields: a dictionary of entity field names and their respective values of the input entity. - params: any additional command-line arguments to be passed to the transform. - entity: the information above is serialized into an Entity object. The entity type is determined by the inputs field in @configure for local transforms. For remote transforms, the entity type is determined by the information in the body of the request. Local transforms suffer from one limitation: if more than one entity type is listed in the inputs field of @configure, the entity type might not be resolvable. Therefore, this should not be referenced in local transforms if there is more than one input entity type defined in @configure. The response object is a container for output entities, UI messages, and exception messages. The config object contains a key-value store of the configuration file. TODO: write your data mining logic below. """ client = get_client(config) prog = 10 progress(prog) debug('Starting RiskIQ passive dns lookup...') value = request.entities[0].value if IP_REGEX.match(value): api_response = client.get_dns_ptr_by_ip(value, rrtype=None) else: api_response = client.get_dns_data_by_name(value, rrtype=None) if not api_response: progress(100) return response dns_data = api_response['records'] a_responses = set() ns_responses = set() mx_responses = set() aaaa_responses = set() cname_responses = set() responses = set() for dns_datum in dns_data: data = dns_datum['data'] if dns_datum.get('rrtype') == 'A': a_responses |= set(data) elif dns_datum.get('rrtype') == 'CNAME': cname_responses |= set(data) elif dns_datum.get('rrtype') == 'NS': ns_responses |= set(data) elif dns_datum.get('rrtype') == 'MX': mx_responses |= set(data) elif dns_datum.get('rrtype') == 'AAAA': aaaa_responses |= set(data) elif dns_datum.get('rrtype') == 'TXT': pass else: responses |= set(data) prog += 40 progress(prog) for rec in a_responses: e = IPv4Address(rec) e.ip = rec response += e prog += 10 progress(prog) """ for rec in aaaa_responses: e = IPv6Address(rec) e.ip = rec response += e prog += 10 progress(prog) """ for _rec in ns_responses: rec = fix_dom(_rec) e = NSRecord(rec) e.fqdn = rec response += e prog += 10 progress(prog) for _rec in mx_responses: rec = fix_dom(_rec) e = MXRecord(rec) e.fqdn = rec response += e prog += 10 progress(prog) for _rec in cname_responses: rec = fix_dom(_rec) e = DNSName(rec) e.fqdn = rec response += e prog += 10 progress(prog) for _rec in responses: rec = fix_dom(_rec) if IP_REGEX.match(rec): e = IPv4Address(rec) e.ip = rec else: e = DNSName(rec) e.fqdn = rec response += e progress(100) return response