def smart_check(self, grab_banner_nmap=False):
"""
Check if the target is reachable and update target info
:return: Boolean indicating status
"""
# If no IP, means that DNS lookup has failed
if not self.service.host.ip:
return False
# For HTTP: Check URL availability, grab headers, grab HTML title
if self.service.url:
is_reachable, status, resp_headers = WebUtils.is_url_reachable(self.service.url)
self.service.up = is_reachable
if resp_headers:
self.service.http_headers = '\n'.join("{}: {}".format(key,val) for (key,val) in resp_headers.items())
else:
self.service.http_headers = ''
self.service.comment = WebUtils.grab_html_title(self.service.url)
# For any other service: Simple port check
elif self.service.protocol == Protocol.TCP:
self.service.up = NetUtils.is_tcp_port_open(str(self.service.host.ip), self.service.port)
else:
self.service.up = NetUtils.is_udp_port_open(str(self.service.host.ip), self.service.port)
# Banner grabbing via Nmap (for TCP only) only if there is no banner already stored in db
if grab_banner_nmap and self.service.up and self.service.protocol == Protocol.TCP and not self.service.banner:
self.service.banner = NetUtils.grab_banner_nmap(str(self.service.host.ip), self.service.port)
return self.service.up
def __init_with_url(self):
"""
Initialize with an URL (when targeting HTTP).
This method updates: URL, Hostname, IP, Port
:raises TargetException: Exception raised if DNS lookup fails
"""
self.service.url = WebUtils.add_prefix_http(self.service.url)
self.service.url = WebUtils.remove_ending_slash(self.service.url)
url = urlparse(self.service.url)
if NetUtils.is_valid_ip(url.hostname):
self.service.host.ip = url.hostname
self.service.host.hostname = url.hostname # updated in smart_check
else:
self.service.host.ip = NetUtils.dns_lookup(url.hostname)
if not self.service.host.ip:
raise TargetException('Unable to resolve {}'.format(
url.hostname))
self.service.host.hostname = url.hostname
if not self.service.port:
self.service.port = WebUtils.get_port_from_url(self.service.url)
if not NetUtils.is_valid_port(self.service.port):
raise TargetException('Invalid port number {}'.format(
self.service.port))
def initWithURL(self):
"""
Initialize the target with an URL (for web app as target)
@Returns boolean
"""
# Add http:// if necessary
self.url = WebUtils.addProtocolHttp(self.url)
parsed = urlparse(self.url)
# Parse url
self.proto = 'HTTPS' if parsed.scheme == 'https' else 'HTTP'
self.host = parsed.netloc
if ':' in self.host:
self.host = self.host[:self.host.rfind(':')]
if parsed.port:
self.port = str(parsed.port)
else:
self.port = '443' if self.proto == 'HTTPS' else '80'
# Check if url is reachable and retrieve headers
if not self.no_port_check:
self.is_reachable, self.status, self.resp_headers = WebUtils.checkUrlExists(
self.url)
if not self.is_reachable:
return False
else:
self.is_reachable = True
self.status = ''
self.resp_headers = {}
# DNS lookup to get IP corresponding to host (if several, just take the first one)
self.ip = DnsUtils.dnsLookup(self.host)[0]
return True
def __grab_html_title_and_headers(self):
"""
Grab HTML title and HTTP headers for service HTTP.
This function is also used to check availability of HTTP services.
Updated in this method:
- self.service.up
- self.service.http_headers
- self.service.html_title
"""
if self.service.url:
# For HTTP: Check URL availability
try:
is_reachable, status, resp_headers = WebUtils.is_url_reachable(
self.service.url)
# In case URL is not reachable, we rebuild it using IP and
# give a new try, i.e. :
# http(s)://hostname:port/ -> http(s)://ip:port/
if not is_reachable \
and not self.initialized_with_url \
and self.service.host.hostname != self.service.host.ip:
new_url = WebUtils.replace_hostname_by_ip(
self.service.url, self.service.host.ip,
self.service.port)
print(new_url)
is_reachable, status, resp_headers = WebUtils.is_url_reachable(
new_url)
if is_reachable:
self.service.url = new_url
#print(is_reachable)
self.service.up = is_reachable
except:
self.service.up = False
return
# Grab HTML title and HTTP Headers
if is_reachable:
if resp_headers:
self.service.http_headers = '\n'.join('{}: {}'.format(key,val) \
for (key,val) in resp_headers.items())
else:
self.service.http_headers = ''
if not self.service.html_title:
self.service.html_title = WebUtils.grab_html_title(
self.service.url)
def __init_with_url(self):
"""
"""
self.service.url = WebUtils.add_prefix_http(self.service.url)
url = urlparse(self.service.url)
if NetUtils.is_valid_ip(url.hostname):
self.service.host.ip = url.hostname
self.service.host.hostname = NetUtils.reverse_dns_lookup(url.hostname)
else:
self.service.host.ip = NetUtils.dns_lookup(url.hostname)
self.service.host.hostname = url.hostname
if not self.service.port:
self.service.port = WebUtils.get_port_from_url(self.service.url)
def switch_https(self):
results = self.get_results()
if not results:
logger.error('No matching service')
else:
for r in results:
if r.url:
r.url = WebUtils.switch_http_https(r.url)
self.sqlsess.commit()
logger.success('Switch done')
def switch_https(self):
"""Switch between HTTP and HTTPS on selected services"""
results = self.get_results()
if not results:
logger.error('No matching service')
else:
for r in results:
if r.url:
r.url = WebUtils.switch_http_https(r.url)
self.sqlsess.commit()
logger.success('Switch done')
def spider_link(self, url):
self.logger.info("Spidering url: %s", url)
if not WebUtils.is_valid_spider_url(url):
return
spider = LinkSpider(url)
spider.get_links()
self.links_to_spider = self.links_to_spider + spider.absolute_links
self.links_to_bak_check = self.links_to_bak_check + spider.fileonly_links
def process(args):
output = Output()
logging.basicConfig(format='[%(levelname)s]: %(message)s',
level=logging.INFO)
logger = logging.getLogger("bakspider")
if args.debug:
logger.info('Debug mode is enabled, output will be verbose.')
else:
logger.disabled = True
if not WebUtils.is_valid_target_url(args.url):
output.error(
"The URL you specified is not in the correct format, see examples:"
)
print("\nValid examples:")
output.status("http://www.example.com/")
output.status("http://example.com/")
print("\nInvalid examples:")
output.negative("www.example.com")
output.negative("http://www.example.com")
sys.exit(1)
# Check host is online
if WebUtils.is_200_response(args.url):
output.page_found("{0} -> Beginning scan...".format(args.url), False)
else:
output.error(
"The URL you specified is returning an invalid response code.")
sys.exit(1)
website = SiteScanner(args.url, output, args.threads)
if args.dir:
dir_scan = DirScanner(args.url, args.dir, output)
website.additional_dirs = dir_scan.scan(args.threads)
website.backup_extensions = FileUtils.read_file_into_array(args.bakext)
website.whitelist_extensions = FileUtils.read_file_into_array(args.ext)
website.begin_scan()
def backup_check(self, fileonly_url):
url_ext = WebUtils.get_url_extension(fileonly_url)
if url_ext not in self.whitelist_extensions:
self.logger.info(
"This URL has no extension or it isn't in the whitelist. [{0}]"
.format(fileonly_url))
return
self.logger.info("Searching for backup files: %s", fileonly_url)
check = BackupScanner(fileonly_url, self.backup_extensions,
self.output, self.thread_count)
if self.additional_dirs:
check.begin_scan(self.additional_dirs)
else:
check.begin_scan()
def __init__(self, url, output, thread_count):
self.url = url
self.output = output
self.thread_count = thread_count
self.links_to_spider = []
self.links_to_bak_check = []
self.additional_dirs = []
self.backup_extensions = []
self.whitelist_extensions = []
self.spidered_links = []
self.checked_files = []
self.logger = logging.getLogger("bakspider")
if not WebUtils.site_has_valid_response_codes(url):
output.negative(
"The site is returning invalid response codes for non-existent links..."
)
output.progress(
"Scan time may increase as we will now check for false-positives.",
True)
def __check_args_attack_single_target(self):
"""Check arguments for subcommand Attack > Single target options"""
target = self.args.target_ip_or_url
if not target:
return True
# Target specified is an URL
if target.lower().startswith('http://') or target.lower().startswith(
'https://'):
self.args.target_mode = TargetMode.URL
if self.args.service and self.args.service.lower() != 'http':
logger.warning('URL only supported for HTTP service. ' \
'Automatically switch to HTTP')
elif not self.args.service:
logger.info('URL given as target, targeted service is HTTP')
self.args.service = 'http'
self.args.target_port = WebUtils.get_port_from_url(target)
# Target specified is IP[:PORT] or HOSTNAME[:PORT]
else:
self.args.target_mode = TargetMode.IP # Actually can be either IP or Hostname
self.args.target_port = None
s = target.split(':')
self.args.target_ip_or_url = s[0]
# Extract port
if len(s) == 2:
self.args.target_port = int(s[1])
if not (0 <= self.args.target_port <= 65535):
logger.error('Target port is not valid. Must be in the ' \
'range [0-65535]')
return False
elif len(s) > 2:
logger.error('Incorrect target format. Must be either IP[:PORT] or ' \
'an URL')
return False
# Check or define targeted service and port
if self.args.service:
# Check if service is supported
if not self.settings.services.is_service_supported(
self.args.service, multi=False):
logger.error('Service "{service}" is not supported. ' \
'Check "info --services".'.format(
service=self.args.service.upper()))
return False
# Try to get default port if it is not specified
if not self.args.target_port:
self.args.target_port = self.settings.services.get_default_port(
self.args.service)
if self.args.target_port:
logger.info('Default port for service {service} will be used: ' \
'{port}/{proto}'.format(
service = self.args.service,
port = self.args.target_port,
proto = self.settings.services.get_protocol(
self.args.service)))
else:
logger.error('Target port is not specified and no default port' \
' can be found for the service {service}'.format(
service=self.args.service))
return False
# Try to get default service for provided port if not specified
else:
if not self.args.target_port:
logger.error(
'Target port and/or service must be specified')
return False
else:
self.args.service = self.settings.services.get_service_from_port(
self.args.target_port)
if not self.args.service:
logger.error('Cannot automatically determine the target ' \
'service for port {port}/tcp, use --target IP:PORT ' \
'syntax'.format(port=self.args.target_port))
return False
logger.info('Automatic service detection based on target port: ' \
'{service}'.format(service=self.args.service))
return True
def parse(self, http_recheck=True):
"""
Parse the Shodan results
:param bool http_recheck: If set to True, TCP ports are re-checked for HTTP(s)
:return: Hosts
:rtype: list(Host)|None
"""
results = list()
host_id = 0
for ip in self.ips_list:
host_id += 1
# Lookup the host
query = None
try:
query = self.api.host(ip)
except Exception as e:
logger.error(
"Error when querying shodan for IP {ip}: {exc}".format(
ip=ip, exc=e))
return None
logger.info('Importing Shodan results from https://www.shodan.io/host/' \
'{ip}'.format(ip=ip))
#print(query)
# Get host information
hostname = query["hostnames"][0] if query["hostnames"] else ip
os = query.get(
"os",
'') # Shodan is often missing OS detection in my tests...
os_vendor = ''
os_family = ''
if os:
os_vendor = OSUtils.get_os_vendor(os)
os_family = OSUtils.get_os_family(os)
device_type = ''
services = query["data"]
# Create Host object
host = Host(
ip=ip,
hostname=hostname,
os=os,
os_vendor=os_vendor,
os_family=os_family,
mac='',
vendor='',
type=device_type,
)
logger.info('[Host {current_host}/{total_host}] Parsing host: ' \
'{ip}{hostname} ...'.format(
current_host=host_id,
total_host=len(self.ips_list),
ip=host.ip,
hostname=' ('+host.hostname+')' if host.hostname != host.ip else ''))
# Loop over ports/services
port_id = 0
for service in services:
port_id += 1
module = service["_shodan"]["module"]
name = get_service_name(module)
port = service.get("port", None)
protocol = service.get("transport", None)
url = ''
comment = ''
html_title = ''
http_headers = ''
# Print current processed service
print()
logger.info('[Host {current_host}/{total_host} | ' \
'Service {current_svc}/{total_svc}] Parsing service: ' \
'host {ip} | port {port}/{proto} | service {service} ...'.format(
current_host=host_id,
total_host=len(self.ips_list),
current_svc=port_id,
total_svc=len(services),
ip=host.ip,
port=port,
proto=protocol,
service=name))
# Get banner
product_name = service.get('product', '')
product_version = service.get('version', '')
banner = '{name}{version}'.format(
name=product_name,
version=' {}'.format(product_version)
if product_version else '')
# # Deduce OS from banner if possible
# if not host.os:
# host.os = OSUtils.os_from_nmap_banner(banner)
# if host.os:
# host.os_vendor = OSUtils.get_os_vendor(host.os)
# host.os_family = OSUtils.get_os_family(host.os)
# Get URL for http services
if name == 'http':
if 'https' in module or 'ssl' in module:
proto = 'https'
else:
proto = 'http'
url = "{proto}://{host}:{port}".format(proto=proto,
host=hostname,
port=port)
# Recheck for HTTP/HTTPS for services undetermined by Shodan
if http_recheck \
and protocol == "tcp" \
and not self.services_config.is_service_supported(name, multi=False):
url = WebUtils.is_returning_http_data(ip, port)
if url:
logger.success("{url} seems to return HTTP data, marking it " \
"as http service".format(url=url))
name = "http"
# Get page title and HTTP headers for HTTP services
if "http" in name:
if 'http' in service:
html_title = service['http'].get('title', '')
http_headers = service.get('data', '')
# Only keep services supported by Jok3r
if not self.services_config.is_service_supported(name,
multi=False):
logger.warning("Service not supported: host {ip} | port "
"{port}/{proto} | service {service}".format(
ip=ip,
port=port,
proto=protocol,
service=name))
continue
# Create Service object
if protocol and port:
service = Service(
name=name,
name_original=module,
port=port,
protocol={
"tcp": Protocol.TCP,
"udp": Protocol.UDP
}.get(protocol),
url=url,
up=True,
banner=banner,
comment=comment,
html_title=html_title,
http_headers=http_headers,
)
host.services.append(service)
# Target smart check:
# - Check if service is still reachable (possible that it has been
# shut down since Shodan scan)
# - Perform web technologies detection: We could use the technologies
# returned by Shodan API in host['data'][id]['http']['components'],
# however it does not detect the version if it is possible
# - Initialize the context of the target via SmartModules, based on the
# information already known (i.e. banner, web technologies...)
target = Target(service, self.services_config)
up = target.smart_check(
reverse_dns_lookup=False, # Done by Shodan
availability_check=
True, # Check if service is still reachable
nmap_banner_grabbing=False, # Done by Shodan
html_title_grabbing=False, # Done by Shodan
web_technos_detection=True,
smart_context_initialize=True)
# TODO: Add an option to disable web technos detections by Jok3r
# and only use technos names returned by Shodan (to speed up import
# if needed)
if not up:
logger.warning('Service not reachable')
if host.services:
results.append(host)
return results
def add_url(self,
url,
services_config,
reverse_dns=True,
availability_check=True,
grab_banner_nmap=True,
web_technos_detection=True):
"""
Add a URL into the current mission scope in database.
:param str url: URL to add
:param lib.core.ServicesConfig services_config: Services configuration object
:param bool reverse_dns: If set to True, perform a reverse DNS lookup
:param bool availability_check: If set to True, check if port is open
:param bool grab_banner_nmap: If set to True, run Nmap to grab server banner
:param bool web_technos_detection: If set to True, try to detect web technos
:return: Status
:rtype: bool
"""
matching_service = self.sqlsess.query(Service).join(Host).join(Mission)\
.filter(Mission.name == self.current_mission)\
.filter((Service.url == url) | \
(Service.url == WebUtils.remove_ending_slash(url))).first()
if matching_service:
logger.warning('URL already present into database')
return False
else:
service = Service(name='http', protocol=Protocol.TCP, url=url)
service.host = Host() # Update in target.smart_check()
try:
target = Target(service, services_config)
except Exception as e:
logger.error(e)
return False
up = target.smart_check(reverse_dns, availability_check,
grab_banner_nmap, web_technos_detection)
if up:
matching_host = self.sqlsess.query(Host).join(Mission)\
.filter(Mission.name == self.current_mission)\
.filter(Host.ip == service.host.ip).first()
new_host = Host(ip=service.host.ip,
hostname=service.host.hostname,
os=service.host.os,
os_vendor=service.host.os_vendor,
os_family=service.host.os_family,
mac=service.host.mac,
vendor=service.host.vendor,
type=service.host.type)
if matching_host:
matching_host.merge(new_host)
self.sqlsess.commit()
service.host = matching_host
else:
mission = self.sqlsess.query(Mission)\
.filter(Mission.name == self.current_mission).first()
new_host.mission = mission
service.host = new_host
self.sqlsess.add(new_host)
self.sqlsess.add(service)
self.sqlsess.commit()
logger.success('Service/URL added: {url}'.format(url=url))
return True
else:
logger.error('URL is not reachable, therefore it is not added')
return False
def smart_check(self,
reverse_dns=True,
availability_check=True,
grab_banner_nmap=False,
web_technos_detection=True):
"""
Check if the target is reachable and update target information
:param bool reverse_dns: Set to True to attempt performing reverse DNS lookup
when no hostname is specified (only IP)
:param bool availability_check: Set to True to check for availability of the
target, and also grab headers and HTML title for HTTP services
:param bool grab_banner_nmap: Set to True to grab the Nmap banner (for TCP)
:param bool web_technos_detection: Set to True to run WebTechnoDetector if
target service is HTTP
:return: Result of check
:rtype: bool
"""
# If no IP, means that DNS lookup has failed
if not self.service.host.ip:
return False
# Perform reverse DNS lookup if hostname not defined
# Note: If lookup fails, it fallbacks to IP
if reverse_dns:
if self.service.host.hostname == self.service.host.ip:
logger.info('Reverse DNS lookup for {ip}...'.format(
ip=str(self.service.host.ip)))
hostname = NetUtils.reverse_dns_lookup(self.service.host.ip)
if hostname != self.service.host.ip:
logger.info('{ip} -> {hostname}'.format(
ip=self.service.host.ip, hostname=hostname))
else:
logger.info('No DNS name found for IP')
self.service.host.hostname = hostname
# Perform availability check
if availability_check:
logger.info('Check if service is reachable...')
# For HTTP: Check URL availability, grab headers, grab HTML title
if self.service.url:
is_reachable, status, resp_headers = WebUtils.is_url_reachable(
self.service.url)
self.service.up = is_reachable
if is_reachable:
if resp_headers:
self.service.http_headers = '\n'.join('{}: {}'.format(key,val) \
for (key,val) in resp_headers.items())
else:
self.service.http_headers = ''
if not self.service.html_title:
self.service.html_title = WebUtils.grab_html_title(
self.service.url)
# For any other service: Simple port check
elif self.service.protocol == Protocol.TCP:
self.service.up = NetUtils.is_tcp_port_open(
str(self.service.host.ip), self.service.port)
else:
self.service.up = NetUtils.is_udp_port_open(
str(self.service.host.ip), self.service.port)
if not self.service.up:
return False
else:
self.service.up = True # consider it as up anyway
# Banner grabbing via Nmap (for TCP only) only if there is no banner
# already stored in db
if grab_banner_nmap \
and self.service.up \
and self.service.protocol == Protocol.TCP \
and not self.service.banner:
logger.info(
'Grab banner for [{service}] via Nmap...'.format(service=self))
self.service.banner = NetUtils.clean_nmap_banner(
NetUtils.grab_banner_nmap(str(self.service.host.ip),
self.service.port))
logger.info('Banner: {banner}'.format(banner=self.service.banner))
# Try to deduce OS from banner if possible
if not self.service.host.os:
detected_os = NetUtils.os_from_nmap_banner(self.service.banner)
if detected_os:
self.service.host.os = detected_os
logger.info('Detected OS from banner = {os}'.format(
os=detected_os))
# Web technologies detection for HTTP
if self.service.name == 'http' and web_technos_detection:
logger.info('Web technologies detection using Wappalyzer...')
detector = WebTechnoDetector(self.service.url)
technos = detector.detect()
self.service.web_technos = str(technos)
detector.print_technos()
# Try to deduce OS from detected web technologies
if not detected_os:
detected_os = detector.get_os()
if detected_os:
self.service.host.os = detected_os
logger.info(
'Detected OS from web technologies = {os}'.format(
os=detected_os))
return self.service.up
def parse(self, http_recheck=True, grab_html_title=True):
"""
Parse the Nmap results
:param bool http_recheck: If set to True, TCP ports are re-checked for HTTP(s)
:param bool grab_html_title: If set to True, grab title of HTML page (text in
<title> tags) and put it as comment for HTTP service.
:return: Hosts
:rtype: list(Host)|None
"""
try:
nmap_report = NmapParser.parse_fromfile(self.nmap_file)
except Exception as e:
logger.error('Error when parsing the Nmap file: {0}'.format(e))
return None
results = list()
for h in nmap_report.hosts:
# Get the fingerprinted OS if available
os = ''
os_vendor = ''
os_family = ''
device_type = ''
if h.os_fingerprinted is True and h.os_match_probabilities(
) is not None:
os_matchs = h.os_match_probabilities()
os = os_matchs[0].name if len(os_matchs) > 0 else ''
if os_matchs[0].osclasses is not None \
and len(os_matchs[0].osclasses) > 0:
os_vendor = os_matchs[0].osclasses[0].vendor
os_family = os_matchs[0].osclasses[0].osfamily
device_type = NetUtils.get_device_type(
os, os_family, os_matchs[0].osclasses[0].type)
# Create Host object
host = Host(ip=h.ipv4,
hostname=h.hostnames[0] if h.hostnames else '',
os=os,
os_vendor=os_vendor,
os_family=os_family,
mac=h.mac,
vendor=h.vendor,
type=device_type)
logger.info('Parsing host: {ip}{hostname} ...'.format(
ip=host.ip,
hostname=' (' + host.hostname + ')' if host.hostname else ''))
# Loop over open ports
for p in h.get_open_ports():
s = h.get_service(p[0], protocol=p[1])
name = NmapResultsParser.nmap_to_joker_service_name(s.service)
url = ''
comment = ''
html_title = ''
# Get URL for http services
if name == 'http':
if 'https' in s.service \
or 'ssl' in s.service \
or s.tunnel in ('ssl', 'tls'):
proto = 'https'
else:
proto = 'http'
url = '{proto}://{host}:{port}'.format(proto=proto,
host=host.ip,
port=s.port)
# Recheck for HTTP/HTTPS for services undetermined by Nmap
if http_recheck \
and s.protocol == 'tcp' \
and not self.services_config.is_service_supported(name, multi=False):
url = WebUtils.is_returning_http_data(
host.hostname or host.ip, s.port)
if url:
logger.success('{url} seems to return HTTP data, marking it ' \
'as http service'.format(url=url))
name = 'http'
# Grab page title for HTTP services
if grab_html_title and name == 'http':
html_title = WebUtils.grab_html_title(url)
# Only keep services supported by Jok3r
if not self.services_config.is_service_supported(name,
multi=False):
logger.info('Service not supported: host {ip} | port ' \
'{port}/{proto} | service {service}'.format(
ip = h.ipv4, port=s.port, proto=s.protocol, service=name))
continue
else:
logger.info('Parsing service: host {ip} | port {port}/{proto} ' \
'| service {service}'.format(
ip = h.ipv4, port=s.port, proto=s.protocol, service=name))
# Deduce OS from banner if possible
if not os:
host.os = NetUtils.os_from_nmap_banner(s.banner)
# Clean Nmap banner
banner = NetUtils.clean_nmap_banner(s.banner)
# Create Service object
service = Service(name=name,
port=s.port,
protocol={
'tcp': Protocol.TCP,
'udp': Protocol.UDP
}.get(s.protocol),
url=url,
up=True,
banner=banner,
comment=comment,
html_title=html_title)
# Already add specific option https=True if possible
if name == 'http' and url.startswith('https://'):
service.options.append(Option(name='https', value='true'))
host.services.append(service)
if host.services:
results.append(host)
return results
def do_services(self, args):
"""Services in the current mission scope"""
print()
req = ServicesRequester(self.sqlsess)
req.select_mission(self.current_mission)
# Logical AND is applied between all specified filtering options
filter_ = Filter(FilterOperator.AND)
if args.names:
for n in args.names:
if not self.settings.services.is_service_supported(n, multi=False):
logger.error('Service {name} is not valid/supported'.format(name=n.lower()))
return
filter_.add_condition(Condition(args.names, FilterData.SERVICE_EXACT))
if args.order:
req.order_by(args.order)
if args.hostname:
# OR between submitted hostnames
filter_.add_condition(Condition(args.hostname.split(','), FilterData.HOST))
if args.ip:
# OR between submitted ips/ranges
filter_.add_condition(Condition(args.ip.split(','), FilterData.IP))
if args.port:
# OR between ports/port-ranges
filter_.add_condition(Condition(args.port.split(','), FilterData.PORT))
if args.proto:
filter_.add_condition(Condition(args.proto, FilterData.PROTOCOL))
if args.up:
filter_.add_condition(Condition(args.up, FilterData.UP))
if args.search:
filter_search = Filter(FilterOperator.OR)
filter_search.add_condition(Condition(args.search, FilterData.HOST))
filter_search.add_condition(Condition(args.search, FilterData.BANNER))
filter_search.add_condition(Condition(args.search, FilterData.URL))
filter_search.add_condition(Condition(args.search, FilterData.COMMENT_SERVICE))
filter_.add_condition(filter_search)
try:
req.add_filter(filter_)
except FilterException as e:
logger.error(e)
return
# Operations
if args.add:
host, port, service = args.add
if NetUtils.is_valid_ip(host):
ip = host
hostname = NetUtils.reverse_dns_lookup(ip)
logger.info('Reverse DNS lookup on IP {ip}: {hostname}'.format(ip=ip, hostname=hostname))
else:
ip = NetUtils.dns_lookup(host)
if not ip:
logger.error('Cannot resolve hostname')
return
hostname = host
logger.info('DNS lookup on {hostname}: IP {ip}'.format(hostname=host, ip=ip))
if not NetUtils.is_valid_port(port):
logger.error('Port is invalid, not in range [0-65535]')
elif not self.settings.services.is_service_supported(service, multi=False):
logger.error('Service {name} is not valid/supported'.format(name=service.lower()))
else:
req.add_service(ip, hostname, port, self.settings.services.get_protocol(service), service)
elif args.url:
args.url = WebUtils.add_prefix_http(args.url)
if not WebUtils.is_valid_url(args.url):
logger.error('URL is invalid')
else:
req.add_url(args.url)
elif args.delete:
if not req.filter_applied:
if not Output.prompt_confirm('No filter applied. Are you sure you want to delete ALL services in current mission ?', default=False):
logger.info('Canceled')
return
req.delete()
elif args.comment:
if not req.filter_applied:
if not Output.prompt_confirm('No filter applied. Are you sure you want to edit comment for ALL services in current mission ?', default=False):
logger.info('Canceled')
return
req.edit_comment(args.comment)
elif args.https:
if not req.filter_applied:
if not Output.prompt_confirm('No filter applied. Are you sure you want to apply switch for ALL URLs in current mission ?', default=False):
logger.info('Canceled')
return
req.switch_https()
elif args.addcred:
if not req.filter_applied:
if not Output.prompt_confirm('No filter applied. Are you sure you want to add same creds for ALL services in current mission ?', default=False):
logger.info('Canceled')
return
req.add_cred(args.addcred[0], args.addcred[1], None)
elif args.addcred_http:
if not req.are_only_http_services_selected():
logger.warning('Some non-HTTP services are selected. Use --addcred instead for non-HTTP services')
return
if not self.settings.services.is_valid_authentication_type(args.addcred_http[2]):
logger.warning('Invalid HTTP authentication type')
logger.info('List of supported authentication types: ')
for auth_type in self.settings.services.get_authentication_types('http'):
logger.info('- {type}'.format(type=auth_type))
return
if not req.filter_applied:
if not Output.prompt_confirm('No filter applied. Are you sure you want to add same creds for ALL HTTP services in current mission ?', default=False):
logger.info('Canceled')
return
req.add_cred(args.addcred_http[0], args.addcred_http[1], args.addcred_http[2])
elif args.adduser:
if not req.filter_applied:
if not Output.prompt_confirm('No filter applied. Are you sure you want to add same username for ALL services in current mission ?', default=False):
logger.info('Canceled')
return
req.add_cred(args.adduser[0], None, None)
elif args.adduser_http:
if not req.are_only_http_services_selected():
logger.warning('Some non-HTTP services are selected. Use --adduser instead for non-HTTP services')
return
if not self.settings.services.is_valid_authentication_type(args.adduser_http[1]):
logger.warning('Invalid HTTP authentication type')
logger.info('List of supported authentication types: ')
for auth_type in self.settings.services.get_authentication_types('http'):
logger.info('- {type}'.format(type=auth_type))
return
if not req.filter_applied:
if not Output.prompt_confirm('No filter applied. Are you sure you want to add same username for ALL HTTP services in current mission ?', default=False):
logger.info('Canceled')
return
req.add_cred(args.adduser_http[0], None, args.adduser_http[1])
else:
req.show()
print()
def parse(self,
http_recheck=True,
html_title_grabbing=True,
nmap_banner_grabbing=False,
web_technos_detection=True):
"""
Parse the Nmap results
:param bool http_recheck: If set to True, TCP ports are re-checked for HTTP(s)
:param bool html_title_grabbing: If set to True, grab title of HTML page (text in
<title> tags) and put it as comment for HTTP service
:param bool nmap_banner_grabbing: If set to True, run Nmap to grab
service banner for each service where it is missing (might be useful if
imported Nmap results come from a scan run without -sV/-A)
:param bool web_technos_detection: If set to True, try to detect web technos
for HTTP service
:return: Hosts
:rtype: list(Host)|None
"""
try:
nmap_report = NmapParser.parse_fromfile(self.nmap_file)
except Exception as e:
logger.error('Error when parsing the Nmap file: {0}'.format(e))
return None
results = list()
host_id = 0
for h in nmap_report.hosts:
host_id += 1
# Get the fingerprinted OS if available
os = ''
os_vendor = ''
os_family = ''
device_type = ''
if h.os_fingerprinted is True \
and h.os_match_probabilities() is not None \
and len(h.os_match_probabilities()) > 0:
os_matchs = h.os_match_probabilities()
if len(os_matchs) > 0:
os = os_matchs[0].name
if os_matchs[0].osclasses is not None \
and len(os_matchs[0].osclasses) > 0:
os_vendor = os_matchs[0].osclasses[0].vendor
os_family = os_matchs[0].osclasses[0].osfamily
device_type = OSUtils.get_device_type(
os,
os_family,
os_matchs[0].osclasses[0].type)
# Create Host object
host = Host(ip=h.ipv4,
hostname=h.hostnames[0] if h.hostnames else h.ipv4,
os=os,
os_vendor=os_vendor,
os_family=os_family,
mac=h.mac,
vendor=h.vendor,
type=device_type)
logger.info('[File {file} | Host {current_host}/{total_host}] ' \
'Parsing host: {ip}{hostname} ...'.format(
file=FileUtils.extract_filename(self.nmap_file),
current_host=host_id,
total_host=len(nmap_report.hosts),
ip=host.ip,
hostname=' ('+host.hostname+')' if host.hostname != host.ip else ''))
# Loop over open ports
port_id = 0
for p in h.get_open_ports():
port_id += 1
s = h.get_service(p[0], protocol=p[1])
name = get_service_name(s.service)
url = ''
comment = ''
html_title = ''
# Print current processed service
print()
logger.info('[File {file} | Host {current_host}/{total_host} | ' \
'Service {current_svc}/{total_svc}] Parsing service: ' \
'host {ip} | port {port}/{proto} | service {service} ...'.format(
file=FileUtils.extract_filename(self.nmap_file),
current_host=host_id,
total_host=len(nmap_report.hosts),
current_svc=port_id,
total_svc=len(h.get_open_ports()),
ip=h.ipv4,
port=s.port,
proto=s.protocol,
service=name))
# Get URL for http services
if name == 'http':
if 'https' in s.service \
or 'ssl' in s.service \
or s.tunnel in ('ssl', 'tls'):
proto = 'https'
else:
proto = 'http'
url = '{proto}://{host}:{port}'.format(
proto=proto, host=host.hostname, port=s.port)
# Recheck for HTTP/HTTPS for services undetermined by Nmap
if http_recheck \
and s.protocol == 'tcp' \
and not self.services_config.is_service_supported(name, multi=False):
url = WebUtils.is_returning_http_data(host.ip, s.port)
if url:
logger.success('{url} seems to return HTTP data, marking it ' \
'as http service'.format(url=url))
name = 'http'
# Only keep services supported by Jok3r
if not self.services_config.is_service_supported(name, multi=False):
logger.warning('Service not supported: host {ip} | port ' \
'{port}/{proto} | service {service}'.format(
ip = h.ipv4, port=s.port, proto=s.protocol, service=name))
continue
# # Deduce OS from banner if possible
# if not os:
# host.os = OSUtils.os_from_nmap_banner(s.banner)
# if host.os:
# host.os_vendor = OSUtils.get_os_vendor(host.os)
# host.os_family = OSUtils.get_os_family(host.os)
# Clean Nmap banner
banner = NetUtils.clean_nmap_banner(s.banner)
# Create Service object
service = Service(
name=name,
name_original=s.service,
port=s.port,
protocol={'tcp': Protocol.TCP,'udp': Protocol.UDP}.get(s.protocol),
url=url,
up=True,
banner=banner,
comment=comment,
html_title=html_title)
host.services.append(service)
# Target smart check:
# - Nmap banner grabbing if specified by user and banner is missing in
# imported results;
# - HTML title and HTTP response headers grabbing for HTTP service;
# - Web technologies detection for HTTP service, except if disabled by
# user;
# - Initialize the context of the target via SmartModules, based on the
# information already known (i.e. banner, web technologies...)
target = Target(service, self.services_config)
up = target.smart_check(
reverse_dns_lookup=False, # Done by Nmap
availability_check=False, # Done by Nmap
nmap_banner_grabbing=nmap_banner_grabbing, # Default: False
html_title_grabbing=html_title_grabbing,
web_technos_detection=web_technos_detection, # Default: True
smart_context_initialize=True)
if not up:
logger.warning('Service not reachable')
if host.services:
results.append(host)
return results
def add_url(self, url):
matching_service = self.sqlsess.query(Service).join(Host).join(Mission)\
.filter(Mission.name == self.current_mission)\
.filter(Service.url == url).first()
if matching_service:
logger.warning('URL already present into database')
else:
# Parse URL: Get IP, hostname, port
parsed = urlparse(url)
if NetUtils.is_valid_ip(parsed.hostname):
ip = parsed.hostname
hostname = NetUtils.reverse_dns_lookup(parsed.hostname)
else:
ip = NetUtils.dns_lookup(parsed.hostname)
if not ip:
logger.error('Host cannot be resolved')
return
hostname = parsed.hostname
port = WebUtils.get_port_from_url(url)
# Check URL, grab headers, html title
is_reachable, status, resp_headers = WebUtils.is_url_reachable(url)
if is_reachable:
comment = WebUtils.grab_html_title(url)
if resp_headers:
http_headers = '\n'.join("{}: {}".format(key, val)
for (key,
val) in resp_headers.items())
logger.info('HTTP Headers:')
print(http_headers)
logger.info('Title: {}'.format(comment))
logger.info(
'Grabbing banner from {ip}:{port} with Nmap...'.format(
ip=ip, port=port))
banner = NetUtils.grab_banner_nmap(ip, port)
logger.info('Banner: {}'.format(banner or 'None'))
os = NetUtils.os_from_nmap_banner(banner)
if os:
logger.info('Detected Host OS: {}'.format(os))
else:
comment = 'Not reachable'
banner = http_headers = ''
logger.warning('URL seems not to be reachable')
# Add service in db (and host if not existing)
service = Service(name='http',
port=port,
protocol=Protocol.TCP,
url=url,
up=is_reachable,
http_headers=http_headers,
banner=banner,
comment=comment)
matching_host = self.sqlsess.query(Host).join(Mission)\
.filter(Mission.name == self.current_mission)\
.filter(Host.ip == ip).first()
new_host = Host(ip=ip, hostname=hostname, os=os)
if matching_host:
matching_host.merge(new_host)
self.sqlsess.commit()
service.host = matching_host
else:
mission = self.sqlsess.query(Mission).filter(
Mission.name == self.current_mission).first()
new_host.mission = mission
service.host = new_host
self.sqlsess.add(new_host)
self.sqlsess.add(service)
self.sqlsess.commit()
logger.success('Service/URL added')
def parse(self, http_recheck):
"""
Parse the Nmap results
:param http_recheck: Boolean indicating if tcp open ports must be rechecked for HTTP(s)
:return: List of Host objects or None if fail
"""
try:
nmap_report = NmapParser.parse_fromfile(self.nmap_file)
except Exception as e:
logger.error('Error when parsing the Nmap file: {0}'.format(e))
return None
results = list()
for h in nmap_report.hosts:
# Get the fingerprinted OS if available
os = ''
if h.os_fingerprinted is True and h.os_match_probabilities() is not None:
os_matchs = h.os_match_probabilities()
os = os_matchs[0].name if len(os_matchs) > 0 else ''
# Create Host object
host = Host(ip = h.ipv4,
hostname = h.hostnames[0] if h.hostnames else '',
os = os)
logger.info('Parsing host: {ip}{hostname} ...'.format(
ip=host.ip, hostname=' ('+host.hostname+')' if host.hostname else ''))
for p in h.get_open_ports():
s = h.get_service(p[0], protocol=p[1])
name = NmapResultsParser.convert_nmap_service_to_joker_service_name(s.service)
url = ''
# Get URL for http services
if name == 'http':
url = '{proto}://{host}:{port}'.format(
proto = 'https' if 'https' in s.service or 'ssl' in s.service or s.tunnel in ('ssl', 'tls') else 'http',
host = host.ip,
port = s.port)
# Recheck for HTTP/HTTPS
if http_recheck and s.protocol == 'tcp' and name != 'http':
#Output.print_inline('Checking http(s)://{hostname}:{port} ...'.format(
# hostname=host.hostname or host.ip, port=s.port))
url = WebUtils.is_returning_http_data(host.hostname or host.ip, s.port)
if url:
logger.success('{url} seems to return HTTP data, marking it as http service'.format(url=url))
name = 'http'
# Only keep services supported by Jok3r
if not self.services_config.is_service_supported(name, multi=False):
logger.info('Service not supported: host {ip} | port {port}/{proto} | service {service}'.format(
ip = h.ipv4, port=s.port, proto=s.protocol, service=name))
continue
else:
logger.info('Parsing service: host {ip} | port {port}/{proto} | service {service}'.format(
ip = h.ipv4, port=s.port, proto=s.protocol, service=name))
# Create Service object
service = Service(name = name,
port = s.port,
protocol = {'tcp': Protocol.TCP, 'udp': Protocol.UDP}.get(s.protocol),
url = url,
up = True,
banner = s.banner)
if name == 'http' and url.startswith('https://'):
service.options.append(Option(name='https', value='true'))
host.services.append(service)
if host.services:
results.append(host)
return results