def __run_nmap(self): """ Run Nmap against service to retrieve: - Service banner - OS info (os name, os vendor, os family) if possible - Device info (MAC, vendor, device type) if possible Updated in this method: - self.service.banner - self.service.host.os - self.service.host.os_vendor - self.service.host.os_family - self.service.host.mac - self.service.host.vendor - self.service.host.type """ # Run Nmap scan nmap_info = NetUtils.grab_nmap_info(str(self.service.host.ip), self.service.port) # Get original service name as returned by Nmap self.service.name_original = nmap_info['service_name'] # Get banner self.service.banner = NetUtils.clean_nmap_banner(nmap_info['banner']) logger.info('Banner = {banner}'.format(banner=self.service.banner)) # Get OS information if nmap_info['os']: if not self.service.host.os: logger.info('Detected OS = {os}'.format(os=nmap_info['os'])) elif self.service.host.os != nmap_info['os']: logger.info('Detected OS has changed = {os}'.format( os=nmap_info['os'])) self.service.host.os = nmap_info['os'] self.service.host.os_vendor = nmap_info['os_vendor'] self.service.host.os_family = nmap_info['os_family'] # Get device information if nmap_info['mac']: self.service.host.mac = nmap_info['mac'] if nmap_info['vendor']: self.service.host.vendor = nmap_info['vendor'] if nmap_info['type']: self.service.host.type = nmap_info['type']
def smart_check(self, reverse_dns=True, availability_check=True, grab_banner_nmap=False, web_technos_detection=True): """ Check if the target is reachable and update target information :param bool reverse_dns: Set to True to attempt performing reverse DNS lookup when no hostname is specified (only IP) :param bool availability_check: Set to True to check for availability of the target, and also grab headers and HTML title for HTTP services :param bool grab_banner_nmap: Set to True to grab the Nmap banner (for TCP) :param bool web_technos_detection: Set to True to run WebTechnoDetector if target service is HTTP :return: Result of check :rtype: bool """ # If no IP, means that DNS lookup has failed if not self.service.host.ip: return False # Perform reverse DNS lookup if hostname not defined # Note: If lookup fails, it fallbacks to IP if reverse_dns: if self.service.host.hostname == self.service.host.ip: logger.info('Reverse DNS lookup for {ip}...'.format( ip=str(self.service.host.ip))) hostname = NetUtils.reverse_dns_lookup(self.service.host.ip) if hostname != self.service.host.ip: logger.info('{ip} -> {hostname}'.format( ip=self.service.host.ip, hostname=hostname)) else: logger.info('No DNS name found for IP') self.service.host.hostname = hostname # Perform availability check if availability_check: logger.info('Check if service is reachable...') # For HTTP: Check URL availability, grab headers, grab HTML title if self.service.url: is_reachable, status, resp_headers = WebUtils.is_url_reachable( self.service.url) self.service.up = is_reachable if is_reachable: if resp_headers: self.service.http_headers = '\n'.join('{}: {}'.format(key,val) \ for (key,val) in resp_headers.items()) else: self.service.http_headers = '' if not self.service.html_title: self.service.html_title = WebUtils.grab_html_title( self.service.url) # For any other service: Simple port check elif self.service.protocol == Protocol.TCP: self.service.up = NetUtils.is_tcp_port_open( str(self.service.host.ip), self.service.port) else: self.service.up = NetUtils.is_udp_port_open( str(self.service.host.ip), self.service.port) if not self.service.up: return False else: self.service.up = True # consider it as up anyway # Banner grabbing via Nmap (for TCP only) only if there is no banner # already stored in db if grab_banner_nmap \ and self.service.up \ and self.service.protocol == Protocol.TCP \ and not self.service.banner: logger.info( 'Grab banner for [{service}] via Nmap...'.format(service=self)) self.service.banner = NetUtils.clean_nmap_banner( NetUtils.grab_banner_nmap(str(self.service.host.ip), self.service.port)) logger.info('Banner: {banner}'.format(banner=self.service.banner)) # Try to deduce OS from banner if possible if not self.service.host.os: detected_os = NetUtils.os_from_nmap_banner(self.service.banner) if detected_os: self.service.host.os = detected_os logger.info('Detected OS from banner = {os}'.format( os=detected_os)) # Web technologies detection for HTTP if self.service.name == 'http' and web_technos_detection: logger.info('Web technologies detection using Wappalyzer...') detector = WebTechnoDetector(self.service.url) technos = detector.detect() self.service.web_technos = str(technos) detector.print_technos() # Try to deduce OS from detected web technologies if not detected_os: detected_os = detector.get_os() if detected_os: self.service.host.os = detected_os logger.info( 'Detected OS from web technologies = {os}'.format( os=detected_os)) return self.service.up
def parse(self, http_recheck=True, grab_html_title=True): """ Parse the Nmap results :param bool http_recheck: If set to True, TCP ports are re-checked for HTTP(s) :param bool grab_html_title: If set to True, grab title of HTML page (text in <title> tags) and put it as comment for HTTP service. :return: Hosts :rtype: list(Host)|None """ try: nmap_report = NmapParser.parse_fromfile(self.nmap_file) except Exception as e: logger.error('Error when parsing the Nmap file: {0}'.format(e)) return None results = list() for h in nmap_report.hosts: # Get the fingerprinted OS if available os = '' os_vendor = '' os_family = '' device_type = '' if h.os_fingerprinted is True and h.os_match_probabilities( ) is not None: os_matchs = h.os_match_probabilities() os = os_matchs[0].name if len(os_matchs) > 0 else '' if os_matchs[0].osclasses is not None \ and len(os_matchs[0].osclasses) > 0: os_vendor = os_matchs[0].osclasses[0].vendor os_family = os_matchs[0].osclasses[0].osfamily device_type = NetUtils.get_device_type( os, os_family, os_matchs[0].osclasses[0].type) # Create Host object host = Host(ip=h.ipv4, hostname=h.hostnames[0] if h.hostnames else '', os=os, os_vendor=os_vendor, os_family=os_family, mac=h.mac, vendor=h.vendor, type=device_type) logger.info('Parsing host: {ip}{hostname} ...'.format( ip=host.ip, hostname=' (' + host.hostname + ')' if host.hostname else '')) # Loop over open ports for p in h.get_open_ports(): s = h.get_service(p[0], protocol=p[1]) name = NmapResultsParser.nmap_to_joker_service_name(s.service) url = '' comment = '' html_title = '' # Get URL for http services if name == 'http': if 'https' in s.service \ or 'ssl' in s.service \ or s.tunnel in ('ssl', 'tls'): proto = 'https' else: proto = 'http' url = '{proto}://{host}:{port}'.format(proto=proto, host=host.ip, port=s.port) # Recheck for HTTP/HTTPS for services undetermined by Nmap if http_recheck \ and s.protocol == 'tcp' \ and not self.services_config.is_service_supported(name, multi=False): url = WebUtils.is_returning_http_data( host.hostname or host.ip, s.port) if url: logger.success('{url} seems to return HTTP data, marking it ' \ 'as http service'.format(url=url)) name = 'http' # Grab page title for HTTP services if grab_html_title and name == 'http': html_title = WebUtils.grab_html_title(url) # Only keep services supported by Jok3r if not self.services_config.is_service_supported(name, multi=False): logger.info('Service not supported: host {ip} | port ' \ '{port}/{proto} | service {service}'.format( ip = h.ipv4, port=s.port, proto=s.protocol, service=name)) continue else: logger.info('Parsing service: host {ip} | port {port}/{proto} ' \ '| service {service}'.format( ip = h.ipv4, port=s.port, proto=s.protocol, service=name)) # Deduce OS from banner if possible if not os: host.os = NetUtils.os_from_nmap_banner(s.banner) # Clean Nmap banner banner = NetUtils.clean_nmap_banner(s.banner) # Create Service object service = Service(name=name, port=s.port, protocol={ 'tcp': Protocol.TCP, 'udp': Protocol.UDP }.get(s.protocol), url=url, up=True, banner=banner, comment=comment, html_title=html_title) # Already add specific option https=True if possible if name == 'http' and url.startswith('https://'): service.options.append(Option(name='https', value='true')) host.services.append(service) if host.services: results.append(host) return results
def parse(self, http_recheck=True, html_title_grabbing=True, nmap_banner_grabbing=False, web_technos_detection=True): """ Parse the Nmap results :param bool http_recheck: If set to True, TCP ports are re-checked for HTTP(s) :param bool html_title_grabbing: If set to True, grab title of HTML page (text in <title> tags) and put it as comment for HTTP service :param bool nmap_banner_grabbing: If set to True, run Nmap to grab service banner for each service where it is missing (might be useful if imported Nmap results come from a scan run without -sV/-A) :param bool web_technos_detection: If set to True, try to detect web technos for HTTP service :return: Hosts :rtype: list(Host)|None """ try: nmap_report = NmapParser.parse_fromfile(self.nmap_file) except Exception as e: logger.error('Error when parsing the Nmap file: {0}'.format(e)) return None results = list() host_id = 0 for h in nmap_report.hosts: host_id += 1 # Get the fingerprinted OS if available os = '' os_vendor = '' os_family = '' device_type = '' if h.os_fingerprinted is True \ and h.os_match_probabilities() is not None \ and len(h.os_match_probabilities()) > 0: os_matchs = h.os_match_probabilities() if len(os_matchs) > 0: os = os_matchs[0].name if os_matchs[0].osclasses is not None \ and len(os_matchs[0].osclasses) > 0: os_vendor = os_matchs[0].osclasses[0].vendor os_family = os_matchs[0].osclasses[0].osfamily device_type = OSUtils.get_device_type( os, os_family, os_matchs[0].osclasses[0].type) # Create Host object host = Host(ip=h.ipv4, hostname=h.hostnames[0] if h.hostnames else h.ipv4, os=os, os_vendor=os_vendor, os_family=os_family, mac=h.mac, vendor=h.vendor, type=device_type) logger.info('[File {file} | Host {current_host}/{total_host}] ' \ 'Parsing host: {ip}{hostname} ...'.format( file=FileUtils.extract_filename(self.nmap_file), current_host=host_id, total_host=len(nmap_report.hosts), ip=host.ip, hostname=' ('+host.hostname+')' if host.hostname != host.ip else '')) # Loop over open ports port_id = 0 for p in h.get_open_ports(): port_id += 1 s = h.get_service(p[0], protocol=p[1]) name = get_service_name(s.service) url = '' comment = '' html_title = '' # Print current processed service print() logger.info('[File {file} | Host {current_host}/{total_host} | ' \ 'Service {current_svc}/{total_svc}] Parsing service: ' \ 'host {ip} | port {port}/{proto} | service {service} ...'.format( file=FileUtils.extract_filename(self.nmap_file), current_host=host_id, total_host=len(nmap_report.hosts), current_svc=port_id, total_svc=len(h.get_open_ports()), ip=h.ipv4, port=s.port, proto=s.protocol, service=name)) # Get URL for http services if name == 'http': if 'https' in s.service \ or 'ssl' in s.service \ or s.tunnel in ('ssl', 'tls'): proto = 'https' else: proto = 'http' url = '{proto}://{host}:{port}'.format( proto=proto, host=host.hostname, port=s.port) # Recheck for HTTP/HTTPS for services undetermined by Nmap if http_recheck \ and s.protocol == 'tcp' \ and not self.services_config.is_service_supported(name, multi=False): url = WebUtils.is_returning_http_data(host.ip, s.port) if url: logger.success('{url} seems to return HTTP data, marking it ' \ 'as http service'.format(url=url)) name = 'http' # Only keep services supported by Jok3r if not self.services_config.is_service_supported(name, multi=False): logger.warning('Service not supported: host {ip} | port ' \ '{port}/{proto} | service {service}'.format( ip = h.ipv4, port=s.port, proto=s.protocol, service=name)) continue # # Deduce OS from banner if possible # if not os: # host.os = OSUtils.os_from_nmap_banner(s.banner) # if host.os: # host.os_vendor = OSUtils.get_os_vendor(host.os) # host.os_family = OSUtils.get_os_family(host.os) # Clean Nmap banner banner = NetUtils.clean_nmap_banner(s.banner) # Create Service object service = Service( name=name, name_original=s.service, port=s.port, protocol={'tcp': Protocol.TCP,'udp': Protocol.UDP}.get(s.protocol), url=url, up=True, banner=banner, comment=comment, html_title=html_title) host.services.append(service) # Target smart check: # - Nmap banner grabbing if specified by user and banner is missing in # imported results; # - HTML title and HTTP response headers grabbing for HTTP service; # - Web technologies detection for HTTP service, except if disabled by # user; # - Initialize the context of the target via SmartModules, based on the # information already known (i.e. banner, web technologies...) target = Target(service, self.services_config) up = target.smart_check( reverse_dns_lookup=False, # Done by Nmap availability_check=False, # Done by Nmap nmap_banner_grabbing=nmap_banner_grabbing, # Default: False html_title_grabbing=html_title_grabbing, web_technos_detection=web_technos_detection, # Default: True smart_context_initialize=True) if not up: logger.warning('Service not reachable') if host.services: results.append(host) return results