def add_service(self, ip, hostname, port, protocol, service): protocol = { 'tcp': Protocol.TCP, 'udp': Protocol.UDP }.get(protocol, Protocol.TCP) matching_service = self.sqlsess.query(Service).join(Host).join(Mission)\ .filter(Mission.name == self.current_mission)\ .filter(Host.ip == ip)\ .filter(Service.port == int(port))\ .filter(Service.protocol == protocol).first() if protocol == Protocol.TCP: up = NetUtils.is_tcp_port_open(ip, port) else: up = NetUtils.is_udp_port_open(ip, port) if matching_service: logger.warning('Service already present into database') else: if up: logger.info( 'Grabbing banner from {ip}:{port} with Nmap...'.format( ip=ip, port=port)) banner = NetUtils.grab_banner_nmap(ip, port) logger.info('Banner: {}'.format(banner or 'None')) os = NetUtils.os_from_nmap_banner(banner) if os: logger.info('Detected Host OS: {}'.format(os)) else: logger.warning('Port seems to be closed !') # Add service in db (and host if not existing) service = Service(name=service, port=int(port), protocol=protocol, up=up, banner=banner) matching_host = self.sqlsess.query(Host).join(Mission)\ .filter(Mission.name == self.current_mission)\ .filter(Host.ip == ip).first() new_host = Host(ip=ip, hostname=hostname, os=os) if matching_host: matching_host.merge(new_host) self.sqlsess.commit() service.host = matching_host else: mission = self.sqlsess.query(Mission).filter( Mission.name == self.current_mission).first() new_host.mission = mission service.host = new_host self.sqlsess.add(new_host) self.sqlsess.add(service) self.sqlsess.commit() logger.success('Service added')
def smart_check(self, reverse_dns=True, availability_check=True, grab_banner_nmap=False, web_technos_detection=True): """ Check if the target is reachable and update target information :param bool reverse_dns: Set to True to attempt performing reverse DNS lookup when no hostname is specified (only IP) :param bool availability_check: Set to True to check for availability of the target, and also grab headers and HTML title for HTTP services :param bool grab_banner_nmap: Set to True to grab the Nmap banner (for TCP) :param bool web_technos_detection: Set to True to run WebTechnoDetector if target service is HTTP :return: Result of check :rtype: bool """ # If no IP, means that DNS lookup has failed if not self.service.host.ip: return False # Perform reverse DNS lookup if hostname not defined # Note: If lookup fails, it fallbacks to IP if reverse_dns: if self.service.host.hostname == self.service.host.ip: logger.info('Reverse DNS lookup for {ip}...'.format( ip=str(self.service.host.ip))) hostname = NetUtils.reverse_dns_lookup(self.service.host.ip) if hostname != self.service.host.ip: logger.info('{ip} -> {hostname}'.format( ip=self.service.host.ip, hostname=hostname)) else: logger.info('No DNS name found for IP') self.service.host.hostname = hostname # Perform availability check if availability_check: logger.info('Check if service is reachable...') # For HTTP: Check URL availability, grab headers, grab HTML title if self.service.url: is_reachable, status, resp_headers = WebUtils.is_url_reachable( self.service.url) self.service.up = is_reachable if is_reachable: if resp_headers: self.service.http_headers = '\n'.join('{}: {}'.format(key,val) \ for (key,val) in resp_headers.items()) else: self.service.http_headers = '' if not self.service.html_title: self.service.html_title = WebUtils.grab_html_title( self.service.url) # For any other service: Simple port check elif self.service.protocol == Protocol.TCP: self.service.up = NetUtils.is_tcp_port_open( str(self.service.host.ip), self.service.port) else: self.service.up = NetUtils.is_udp_port_open( str(self.service.host.ip), self.service.port) if not self.service.up: return False else: self.service.up = True # consider it as up anyway # Banner grabbing via Nmap (for TCP only) only if there is no banner # already stored in db if grab_banner_nmap \ and self.service.up \ and self.service.protocol == Protocol.TCP \ and not self.service.banner: logger.info( 'Grab banner for [{service}] via Nmap...'.format(service=self)) self.service.banner = NetUtils.clean_nmap_banner( NetUtils.grab_banner_nmap(str(self.service.host.ip), self.service.port)) logger.info('Banner: {banner}'.format(banner=self.service.banner)) # Try to deduce OS from banner if possible if not self.service.host.os: detected_os = NetUtils.os_from_nmap_banner(self.service.banner) if detected_os: self.service.host.os = detected_os logger.info('Detected OS from banner = {os}'.format( os=detected_os)) # Web technologies detection for HTTP if self.service.name == 'http' and web_technos_detection: logger.info('Web technologies detection using Wappalyzer...') detector = WebTechnoDetector(self.service.url) technos = detector.detect() self.service.web_technos = str(technos) detector.print_technos() # Try to deduce OS from detected web technologies if not detected_os: detected_os = detector.get_os() if detected_os: self.service.host.os = detected_os logger.info( 'Detected OS from web technologies = {os}'.format( os=detected_os)) return self.service.up
def add_url(self, url): matching_service = self.sqlsess.query(Service).join(Host).join(Mission)\ .filter(Mission.name == self.current_mission)\ .filter(Service.url == url).first() if matching_service: logger.warning('URL already present into database') else: # Parse URL: Get IP, hostname, port parsed = urlparse(url) if NetUtils.is_valid_ip(parsed.hostname): ip = parsed.hostname hostname = NetUtils.reverse_dns_lookup(parsed.hostname) else: ip = NetUtils.dns_lookup(parsed.hostname) if not ip: logger.error('Host cannot be resolved') return hostname = parsed.hostname port = WebUtils.get_port_from_url(url) # Check URL, grab headers, html title is_reachable, status, resp_headers = WebUtils.is_url_reachable(url) if is_reachable: comment = WebUtils.grab_html_title(url) if resp_headers: http_headers = '\n'.join("{}: {}".format(key, val) for (key, val) in resp_headers.items()) logger.info('HTTP Headers:') print(http_headers) logger.info('Title: {}'.format(comment)) logger.info( 'Grabbing banner from {ip}:{port} with Nmap...'.format( ip=ip, port=port)) banner = NetUtils.grab_banner_nmap(ip, port) logger.info('Banner: {}'.format(banner or 'None')) os = NetUtils.os_from_nmap_banner(banner) if os: logger.info('Detected Host OS: {}'.format(os)) else: comment = 'Not reachable' banner = http_headers = '' logger.warning('URL seems not to be reachable') # Add service in db (and host if not existing) service = Service(name='http', port=port, protocol=Protocol.TCP, url=url, up=is_reachable, http_headers=http_headers, banner=banner, comment=comment) matching_host = self.sqlsess.query(Host).join(Mission)\ .filter(Mission.name == self.current_mission)\ .filter(Host.ip == ip).first() new_host = Host(ip=ip, hostname=hostname, os=os) if matching_host: matching_host.merge(new_host) self.sqlsess.commit() service.host = matching_host else: mission = self.sqlsess.query(Mission).filter( Mission.name == self.current_mission).first() new_host.mission = mission service.host = new_host self.sqlsess.add(new_host) self.sqlsess.add(service) self.sqlsess.commit() logger.success('Service/URL added')
def parse(self, http_recheck=True, grab_html_title=True): """ Parse the Nmap results :param bool http_recheck: If set to True, TCP ports are re-checked for HTTP(s) :param bool grab_html_title: If set to True, grab title of HTML page (text in <title> tags) and put it as comment for HTTP service. :return: Hosts :rtype: list(Host)|None """ try: nmap_report = NmapParser.parse_fromfile(self.nmap_file) except Exception as e: logger.error('Error when parsing the Nmap file: {0}'.format(e)) return None results = list() for h in nmap_report.hosts: # Get the fingerprinted OS if available os = '' os_vendor = '' os_family = '' device_type = '' if h.os_fingerprinted is True and h.os_match_probabilities( ) is not None: os_matchs = h.os_match_probabilities() os = os_matchs[0].name if len(os_matchs) > 0 else '' if os_matchs[0].osclasses is not None \ and len(os_matchs[0].osclasses) > 0: os_vendor = os_matchs[0].osclasses[0].vendor os_family = os_matchs[0].osclasses[0].osfamily device_type = NetUtils.get_device_type( os, os_family, os_matchs[0].osclasses[0].type) # Create Host object host = Host(ip=h.ipv4, hostname=h.hostnames[0] if h.hostnames else '', os=os, os_vendor=os_vendor, os_family=os_family, mac=h.mac, vendor=h.vendor, type=device_type) logger.info('Parsing host: {ip}{hostname} ...'.format( ip=host.ip, hostname=' (' + host.hostname + ')' if host.hostname else '')) # Loop over open ports for p in h.get_open_ports(): s = h.get_service(p[0], protocol=p[1]) name = NmapResultsParser.nmap_to_joker_service_name(s.service) url = '' comment = '' html_title = '' # Get URL for http services if name == 'http': if 'https' in s.service \ or 'ssl' in s.service \ or s.tunnel in ('ssl', 'tls'): proto = 'https' else: proto = 'http' url = '{proto}://{host}:{port}'.format(proto=proto, host=host.ip, port=s.port) # Recheck for HTTP/HTTPS for services undetermined by Nmap if http_recheck \ and s.protocol == 'tcp' \ and not self.services_config.is_service_supported(name, multi=False): url = WebUtils.is_returning_http_data( host.hostname or host.ip, s.port) if url: logger.success('{url} seems to return HTTP data, marking it ' \ 'as http service'.format(url=url)) name = 'http' # Grab page title for HTTP services if grab_html_title and name == 'http': html_title = WebUtils.grab_html_title(url) # Only keep services supported by Jok3r if not self.services_config.is_service_supported(name, multi=False): logger.info('Service not supported: host {ip} | port ' \ '{port}/{proto} | service {service}'.format( ip = h.ipv4, port=s.port, proto=s.protocol, service=name)) continue else: logger.info('Parsing service: host {ip} | port {port}/{proto} ' \ '| service {service}'.format( ip = h.ipv4, port=s.port, proto=s.protocol, service=name)) # Deduce OS from banner if possible if not os: host.os = NetUtils.os_from_nmap_banner(s.banner) # Clean Nmap banner banner = NetUtils.clean_nmap_banner(s.banner) # Create Service object service = Service(name=name, port=s.port, protocol={ 'tcp': Protocol.TCP, 'udp': Protocol.UDP }.get(s.protocol), url=url, up=True, banner=banner, comment=comment, html_title=html_title) # Already add specific option https=True if possible if name == 'http' and url.startswith('https://'): service.options.append(Option(name='https', value='true')) host.services.append(service) if host.services: results.append(host) return results