def __run_nmap(self):
"""
Run Nmap against service to retrieve:
- Service banner
- OS info (os name, os vendor, os family) if possible
- Device info (MAC, vendor, device type) if possible
Updated in this method:
- self.service.banner
- self.service.host.os
- self.service.host.os_vendor
- self.service.host.os_family
- self.service.host.mac
- self.service.host.vendor
- self.service.host.type
"""
# Run Nmap scan
nmap_info = NetUtils.grab_nmap_info(str(self.service.host.ip),
self.service.port)
# Get original service name as returned by Nmap
self.service.name_original = nmap_info['service_name']
# Get banner
self.service.banner = NetUtils.clean_nmap_banner(nmap_info['banner'])
logger.info('Banner = {banner}'.format(banner=self.service.banner))
# Get OS information
if nmap_info['os']:
if not self.service.host.os:
logger.info('Detected OS = {os}'.format(os=nmap_info['os']))
elif self.service.host.os != nmap_info['os']:
logger.info('Detected OS has changed = {os}'.format(
os=nmap_info['os']))
self.service.host.os = nmap_info['os']
self.service.host.os_vendor = nmap_info['os_vendor']
self.service.host.os_family = nmap_info['os_family']
# Get device information
if nmap_info['mac']:
self.service.host.mac = nmap_info['mac']
if nmap_info['vendor']:
self.service.host.vendor = nmap_info['vendor']
if nmap_info['type']:
self.service.host.type = nmap_info['type']
def smart_check(self,
reverse_dns=True,
availability_check=True,
grab_banner_nmap=False,
web_technos_detection=True):
"""
Check if the target is reachable and update target information
:param bool reverse_dns: Set to True to attempt performing reverse DNS lookup
when no hostname is specified (only IP)
:param bool availability_check: Set to True to check for availability of the
target, and also grab headers and HTML title for HTTP services
:param bool grab_banner_nmap: Set to True to grab the Nmap banner (for TCP)
:param bool web_technos_detection: Set to True to run WebTechnoDetector if
target service is HTTP
:return: Result of check
:rtype: bool
"""
# If no IP, means that DNS lookup has failed
if not self.service.host.ip:
return False
# Perform reverse DNS lookup if hostname not defined
# Note: If lookup fails, it fallbacks to IP
if reverse_dns:
if self.service.host.hostname == self.service.host.ip:
logger.info('Reverse DNS lookup for {ip}...'.format(
ip=str(self.service.host.ip)))
hostname = NetUtils.reverse_dns_lookup(self.service.host.ip)
if hostname != self.service.host.ip:
logger.info('{ip} -> {hostname}'.format(
ip=self.service.host.ip, hostname=hostname))
else:
logger.info('No DNS name found for IP')
self.service.host.hostname = hostname
# Perform availability check
if availability_check:
logger.info('Check if service is reachable...')
# For HTTP: Check URL availability, grab headers, grab HTML title
if self.service.url:
is_reachable, status, resp_headers = WebUtils.is_url_reachable(
self.service.url)
self.service.up = is_reachable
if is_reachable:
if resp_headers:
self.service.http_headers = '\n'.join('{}: {}'.format(key,val) \
for (key,val) in resp_headers.items())
else:
self.service.http_headers = ''
if not self.service.html_title:
self.service.html_title = WebUtils.grab_html_title(
self.service.url)
# For any other service: Simple port check
elif self.service.protocol == Protocol.TCP:
self.service.up = NetUtils.is_tcp_port_open(
str(self.service.host.ip), self.service.port)
else:
self.service.up = NetUtils.is_udp_port_open(
str(self.service.host.ip), self.service.port)
if not self.service.up:
return False
else:
self.service.up = True # consider it as up anyway
# Banner grabbing via Nmap (for TCP only) only if there is no banner
# already stored in db
if grab_banner_nmap \
and self.service.up \
and self.service.protocol == Protocol.TCP \
and not self.service.banner:
logger.info(
'Grab banner for [{service}] via Nmap...'.format(service=self))
self.service.banner = NetUtils.clean_nmap_banner(
NetUtils.grab_banner_nmap(str(self.service.host.ip),
self.service.port))
logger.info('Banner: {banner}'.format(banner=self.service.banner))
# Try to deduce OS from banner if possible
if not self.service.host.os:
detected_os = NetUtils.os_from_nmap_banner(self.service.banner)
if detected_os:
self.service.host.os = detected_os
logger.info('Detected OS from banner = {os}'.format(
os=detected_os))
# Web technologies detection for HTTP
if self.service.name == 'http' and web_technos_detection:
logger.info('Web technologies detection using Wappalyzer...')
detector = WebTechnoDetector(self.service.url)
technos = detector.detect()
self.service.web_technos = str(technos)
detector.print_technos()
# Try to deduce OS from detected web technologies
if not detected_os:
detected_os = detector.get_os()
if detected_os:
self.service.host.os = detected_os
logger.info(
'Detected OS from web technologies = {os}'.format(
os=detected_os))
return self.service.up
def parse(self, http_recheck=True, grab_html_title=True):
"""
Parse the Nmap results
:param bool http_recheck: If set to True, TCP ports are re-checked for HTTP(s)
:param bool grab_html_title: If set to True, grab title of HTML page (text in
<title> tags) and put it as comment for HTTP service.
:return: Hosts
:rtype: list(Host)|None
"""
try:
nmap_report = NmapParser.parse_fromfile(self.nmap_file)
except Exception as e:
logger.error('Error when parsing the Nmap file: {0}'.format(e))
return None
results = list()
for h in nmap_report.hosts:
# Get the fingerprinted OS if available
os = ''
os_vendor = ''
os_family = ''
device_type = ''
if h.os_fingerprinted is True and h.os_match_probabilities(
) is not None:
os_matchs = h.os_match_probabilities()
os = os_matchs[0].name if len(os_matchs) > 0 else ''
if os_matchs[0].osclasses is not None \
and len(os_matchs[0].osclasses) > 0:
os_vendor = os_matchs[0].osclasses[0].vendor
os_family = os_matchs[0].osclasses[0].osfamily
device_type = NetUtils.get_device_type(
os, os_family, os_matchs[0].osclasses[0].type)
# Create Host object
host = Host(ip=h.ipv4,
hostname=h.hostnames[0] if h.hostnames else '',
os=os,
os_vendor=os_vendor,
os_family=os_family,
mac=h.mac,
vendor=h.vendor,
type=device_type)
logger.info('Parsing host: {ip}{hostname} ...'.format(
ip=host.ip,
hostname=' (' + host.hostname + ')' if host.hostname else ''))
# Loop over open ports
for p in h.get_open_ports():
s = h.get_service(p[0], protocol=p[1])
name = NmapResultsParser.nmap_to_joker_service_name(s.service)
url = ''
comment = ''
html_title = ''
# Get URL for http services
if name == 'http':
if 'https' in s.service \
or 'ssl' in s.service \
or s.tunnel in ('ssl', 'tls'):
proto = 'https'
else:
proto = 'http'
url = '{proto}://{host}:{port}'.format(proto=proto,
host=host.ip,
port=s.port)
# Recheck for HTTP/HTTPS for services undetermined by Nmap
if http_recheck \
and s.protocol == 'tcp' \
and not self.services_config.is_service_supported(name, multi=False):
url = WebUtils.is_returning_http_data(
host.hostname or host.ip, s.port)
if url:
logger.success('{url} seems to return HTTP data, marking it ' \
'as http service'.format(url=url))
name = 'http'
# Grab page title for HTTP services
if grab_html_title and name == 'http':
html_title = WebUtils.grab_html_title(url)
# Only keep services supported by Jok3r
if not self.services_config.is_service_supported(name,
multi=False):
logger.info('Service not supported: host {ip} | port ' \
'{port}/{proto} | service {service}'.format(
ip = h.ipv4, port=s.port, proto=s.protocol, service=name))
continue
else:
logger.info('Parsing service: host {ip} | port {port}/{proto} ' \
'| service {service}'.format(
ip = h.ipv4, port=s.port, proto=s.protocol, service=name))
# Deduce OS from banner if possible
if not os:
host.os = NetUtils.os_from_nmap_banner(s.banner)
# Clean Nmap banner
banner = NetUtils.clean_nmap_banner(s.banner)
# Create Service object
service = Service(name=name,
port=s.port,
protocol={
'tcp': Protocol.TCP,
'udp': Protocol.UDP
}.get(s.protocol),
url=url,
up=True,
banner=banner,
comment=comment,
html_title=html_title)
# Already add specific option https=True if possible
if name == 'http' and url.startswith('https://'):
service.options.append(Option(name='https', value='true'))
host.services.append(service)
if host.services:
results.append(host)
return results
def parse(self,
http_recheck=True,
html_title_grabbing=True,
nmap_banner_grabbing=False,
web_technos_detection=True):
"""
Parse the Nmap results
:param bool http_recheck: If set to True, TCP ports are re-checked for HTTP(s)
:param bool html_title_grabbing: If set to True, grab title of HTML page (text in
<title> tags) and put it as comment for HTTP service
:param bool nmap_banner_grabbing: If set to True, run Nmap to grab
service banner for each service where it is missing (might be useful if
imported Nmap results come from a scan run without -sV/-A)
:param bool web_technos_detection: If set to True, try to detect web technos
for HTTP service
:return: Hosts
:rtype: list(Host)|None
"""
try:
nmap_report = NmapParser.parse_fromfile(self.nmap_file)
except Exception as e:
logger.error('Error when parsing the Nmap file: {0}'.format(e))
return None
results = list()
host_id = 0
for h in nmap_report.hosts:
host_id += 1
# Get the fingerprinted OS if available
os = ''
os_vendor = ''
os_family = ''
device_type = ''
if h.os_fingerprinted is True \
and h.os_match_probabilities() is not None \
and len(h.os_match_probabilities()) > 0:
os_matchs = h.os_match_probabilities()
if len(os_matchs) > 0:
os = os_matchs[0].name
if os_matchs[0].osclasses is not None \
and len(os_matchs[0].osclasses) > 0:
os_vendor = os_matchs[0].osclasses[0].vendor
os_family = os_matchs[0].osclasses[0].osfamily
device_type = OSUtils.get_device_type(
os,
os_family,
os_matchs[0].osclasses[0].type)
# Create Host object
host = Host(ip=h.ipv4,
hostname=h.hostnames[0] if h.hostnames else h.ipv4,
os=os,
os_vendor=os_vendor,
os_family=os_family,
mac=h.mac,
vendor=h.vendor,
type=device_type)
logger.info('[File {file} | Host {current_host}/{total_host}] ' \
'Parsing host: {ip}{hostname} ...'.format(
file=FileUtils.extract_filename(self.nmap_file),
current_host=host_id,
total_host=len(nmap_report.hosts),
ip=host.ip,
hostname=' ('+host.hostname+')' if host.hostname != host.ip else ''))
# Loop over open ports
port_id = 0
for p in h.get_open_ports():
port_id += 1
s = h.get_service(p[0], protocol=p[1])
name = get_service_name(s.service)
url = ''
comment = ''
html_title = ''
# Print current processed service
print()
logger.info('[File {file} | Host {current_host}/{total_host} | ' \
'Service {current_svc}/{total_svc}] Parsing service: ' \
'host {ip} | port {port}/{proto} | service {service} ...'.format(
file=FileUtils.extract_filename(self.nmap_file),
current_host=host_id,
total_host=len(nmap_report.hosts),
current_svc=port_id,
total_svc=len(h.get_open_ports()),
ip=h.ipv4,
port=s.port,
proto=s.protocol,
service=name))
# Get URL for http services
if name == 'http':
if 'https' in s.service \
or 'ssl' in s.service \
or s.tunnel in ('ssl', 'tls'):
proto = 'https'
else:
proto = 'http'
url = '{proto}://{host}:{port}'.format(
proto=proto, host=host.hostname, port=s.port)
# Recheck for HTTP/HTTPS for services undetermined by Nmap
if http_recheck \
and s.protocol == 'tcp' \
and not self.services_config.is_service_supported(name, multi=False):
url = WebUtils.is_returning_http_data(host.ip, s.port)
if url:
logger.success('{url} seems to return HTTP data, marking it ' \
'as http service'.format(url=url))
name = 'http'
# Only keep services supported by Jok3r
if not self.services_config.is_service_supported(name, multi=False):
logger.warning('Service not supported: host {ip} | port ' \
'{port}/{proto} | service {service}'.format(
ip = h.ipv4, port=s.port, proto=s.protocol, service=name))
continue
# # Deduce OS from banner if possible
# if not os:
# host.os = OSUtils.os_from_nmap_banner(s.banner)
# if host.os:
# host.os_vendor = OSUtils.get_os_vendor(host.os)
# host.os_family = OSUtils.get_os_family(host.os)
# Clean Nmap banner
banner = NetUtils.clean_nmap_banner(s.banner)
# Create Service object
service = Service(
name=name,
name_original=s.service,
port=s.port,
protocol={'tcp': Protocol.TCP,'udp': Protocol.UDP}.get(s.protocol),
url=url,
up=True,
banner=banner,
comment=comment,
html_title=html_title)
host.services.append(service)
# Target smart check:
# - Nmap banner grabbing if specified by user and banner is missing in
# imported results;
# - HTML title and HTTP response headers grabbing for HTTP service;
# - Web technologies detection for HTTP service, except if disabled by
# user;
# - Initialize the context of the target via SmartModules, based on the
# information already known (i.e. banner, web technologies...)
target = Target(service, self.services_config)
up = target.smart_check(
reverse_dns_lookup=False, # Done by Nmap
availability_check=False, # Done by Nmap
nmap_banner_grabbing=nmap_banner_grabbing, # Default: False
html_title_grabbing=html_title_grabbing,
web_technos_detection=web_technos_detection, # Default: True
smart_context_initialize=True)
if not up:
logger.warning('Service not reachable')
if host.services:
results.append(host)
return results
