def clean_up(self):
"""
Perform any house-keeping necessary to clean up after the inspector once it has finished
doing its job.
:return: None
"""
for file_path in self.temp_file_paths:
logger.debug(
"Cleaning up file at path %s."
% (file_path,)
)
if FilesystemHelper.does_file_exist(file_path):
FilesystemHelper.delete_file(file_path)
def create_zmap_whitelist_file_for_order(order_uuid=None,
file_path=None,
db_session=None):
"""
Create a Zmap whitelist file for the given order at the given file path.
:param order_uuid: The UUID of the order to create the whitelist file for.
:param file_path: The local file path to write the file to.
:param db_session: A SQLAlchemy session.
:return: None
"""
ranges = get_monitored_network_ranges_for_order(order_uuid=order_uuid,
db_session=db_session)
FilesystemHelper.write_to_file(file_path=file_path, data="\n".join(ranges))
def create_zmap_whitelist_file_for_organization(org_uuid=None,
file_path=None,
db_session=None):
"""
Create a Zmap whitelist file for the given organization at the given file path.
:param org_uuid: The UUID of the organization to create the whitelist file for.
:param file_path: The local file path to write the file to.
:param db_session: A SQLAlchemy session.
:return: None
"""
ranges = get_enabled_network_ranges_for_organization(org_uuid=org_uuid,
db_session=db_session)
whitelist_string = ConversionHelper.network_ranges_to_zmap_list(ranges)
FilesystemHelper.write_to_file(file_path=file_path, data=whitelist_string)
def crawl_web_service(
self,
web_service_uuid=None,
org_uuid=None,
web_service_scan_uuid=None,
):
"""
Crawl the given web service and index the results in Elasticsearch.
:param web_service_uuid: The UUID of the web service to crawl.
:param org_uuid: The UUID of the organization to crawl the web service on behalf of.
:param web_service_scan_uuid: The UUID of the scan that this crawling session is part of.
:return: None
"""
ip_address, port, hostname, use_ssl = get_endpoint_information_for_web_service(
web_service_uuid=web_service_uuid,
db_session=self.db_session,
)
logger.info(
"Now crawling endpoint at %s:%s for scan %s. Organization is %s."
% (ip_address, port, web_service_scan_uuid, org_uuid)
)
runner = CrawlRunner()
results_file_path, results_wrapper = runner.crawl_endpoint_to_file(
ip_address=ip_address,
port=port,
use_ssl=use_ssl,
hostname=hostname,
in_separate_process=True,
)
logger.info(
"Crawling completed for endpoint at %s:%s. Indexing results to Elasticsearch."
% (ip_address, port)
)
bulk_query = BulkElasticsearchQuery()
web_service_scan = WebServiceScan.by_uuid(db_session=self.db_session, uuid=web_service_scan_uuid)
site_url_wrapper = UrlWrapper.from_endpoint(hostname=hostname, port=port, path="/", use_ssl=use_ssl)
for es_model in results_wrapper.iter_es_models(web_service_scan=web_service_scan, site_url=site_url_wrapper):
bulk_query.add_model_for_indexing(model=es_model, index=org_uuid)
logger.info(
"Now updating Elasticsearch via bulk query. Total operations: %s."
% (bulk_query.batch_length,)
)
bulk_query.save()
FilesystemHelper.delete_file(results_file_path)
logger.info(
"Elasticsearch updated with crawling results for endpoint %s:%s and local file deleted."
% (ip_address, port)
)
def from_file(cls, file_path):
"""
Create an instance of this wrapper class based on the contents of a given file.
:param file_path: The file path to read data from.
:return: An instance of this wrapper class wrapping the contents of the given file.
"""
file_contents = FilesystemHelper.get_file_contents(file_path).strip()
return cls(file_contents)
def get_supported_tlds():
"""
Get a list of strings representing the TLDs that are supported for subdomain discovery.
:return: A list of strings representing the TLDs that are supported for subdomain discovery.
"""
file_contents = FilesystemHelper.get_file_contents(
path=config.files_tlds_path)
return [x.strip() for x in file_contents.strip().split("\n")]
def get_resolvers():
"""
Get a list of IP addresses to use as DNS resolvers.
:return: A list of IP addresses to use as DNS resolvers.
"""
contents = FilesystemHelper.get_file_contents(path=config.files_dns_resolvers_path)
to_return = [x.strip() for x in contents.strip().split("\n")]
to_return = filter(lambda x: not x.startswith("#"), to_return)
return random.sample(to_return, len(to_return)/2)
def screenshot_endpoint(
self,
ip_address=None,
port=None,
hostname=None,
use_ssl=False,
use_sni=False,
path="/",
in_separate_process=False,
):
"""
Take a screenshot of the given endpoint, save it to a local temporary file, and return the local
file path.
:param ip_address: The IP address where the web service resides.
:param port: The port where the web service resides.
:param hostname: The hostname to request.
:param use_ssl: Whether or not to use SSL to request the endpoint.
:param use_sni: Whether or not the endpoint uses SNI.
:param path: The path of the resource to screenshot.
:param in_separate_process: Whether or not to take the screenshot in a separate process. This is to
address the incredibly long time that the Selenium webdriver can take when it hangs.
:return: A tuple containing (1) the local file path where the screenshot was saved and (2) whether or not
the screenshot was taken successfully.
"""
logger.debug(
"Now attempting to take a screenshot of the web service at %s:%s (%s). Hostname is %s, SNI support is %s."
% (ip_address, port, "using SSL" if use_ssl else "plain HTTP",
hostname, use_sni))
self.__set_endpoint(
ip_address=ip_address,
port=port,
hostname=hostname,
use_ssl=use_ssl,
use_sni=use_sni,
path=path,
)
self._output_file_path = self.get_temporary_file_path()
if in_separate_process:
process = Process(target=self.__take_screenshot)
try:
process.start()
process.join(config.selenium_screenshot_delay +
config.inspection_screenshot_join_timeout)
except IOError as e:
if e.errno == errno.EINTR:
logger.warning("Interrupted system call error received.")
else:
raise e
finally:
if process.is_alive():
print("PROCESS IS ALIVE - PID IS %s" % (process.pid, ))
os.kill(process.pid, signal.SIGTERM)
else:
self.__take_screenshot()
return self.output_file_path, FilesystemHelper.does_file_exist(
self.output_file_path)
def upload_certificate_to_s3(org_uuid=None, cert_string=None, local_file_path=None):
"""
Upload the given SSL certificate to AWS S3 and return a tuple describing where it was uploaded
to.
:param org_uuid: The UUID of the organization that this SSL certificate is being uploaded on behalf
of.
:param cert_string: A string containing the SSL certificate.
:param local_file_path: A local file path that can be used to write data to.
:return: A tuple containing (1) the bucket where the file was uploaded to and (2) the key it
was uploaded under.
"""
FilesystemHelper.write_to_file(file_path=local_file_path, data=cert_string)
storage_helper = get_storage_helper()
response, key = storage_helper.upload_ssl_certificate(
org_uuid=org_uuid,
local_file_path=local_file_path,
bucket=config.storage_bucket,
)
return config.storage_bucket, key
def __save_endpoint_to_file(self):
"""
Send an HTTP request to the remote endpoint to get the content of the requested resource and
save that content to the local filesystem.
:return: The file path where the resource resides on the local filesystem.
"""
web_inspector = WebServiceInspector(
ip_address=self.ip_address,
port=self.port,
hostname=self.hostname,
use_ssl=self.use_ssl,
)
transaction = web_inspector.get(path=self.path)
temp_file_path = self.get_temporary_file_path(file_ext="html")
FilesystemHelper.write_to_file(
file_path=temp_file_path,
data=transaction.response_content,
write_mode="wb+",
)
return temp_file_path
def get_output_file_name(self):
"""
Get the file name that results should be written to. Note that this will
first check to see if such a file name has already been set and return it
if it has, and second will use a temporary file name that us guaranteed to be
writable if it has not.
:return: The file name that results should be written to.
"""
if self.output_file_path is not None:
return self.output_file_path
else:
return FilesystemHelper.get_temporary_file_path()
def post(self, request, pk=None):
"""
Upload a file containing one domain name per line and associate all of the domains with the referenced
organization.
"""
if "file" not in request.FILES:
raise NotFound("No file found in request body.")
uploaded_file = request.FILES["file"]
temp_path = FilesystemHelper.get_temporary_file_path()
FilesystemHelper.write_to_file(file_path=temp_path,
data=uploaded_file.read(),
write_mode="wb+")
storage_helper = get_storage_helper()
response, key = storage_helper.upload_dns_text_file(
org_uuid=str(self.organization.uuid),
local_file_path=temp_path,
bucket=config.storage_bucket,
)
upload_model = UserUploadModel.from_database_model(
database_model=self.request.user, upload_type="dns_text")
upload_model.set_s3_attributes(bucket=config.storage_bucket,
key=key,
file_type="dns_text")
upload_model.save(self.organization.uuid)
contents = FilesystemHelper.get_file_contents(path=temp_path,
read_mode="rb")
if contents.count("\n") > config.rest_domains_file_cutoff:
process_dns_text_file.delay(
org_uuid=str(self.organization.uuid),
file_key=key,
file_bucket=config.storage_bucket,
)
to_return = DomainsUploadResponse(batch_required=True)
else:
file_wrapper = DomainsTextFileWrapper(contents)
new_domains = 0
skipped_domains = 0
existing_domains = []
for entry in self.organization.domain_names.all().values("name"):
existing_domains.append(entry["name"])
for row in file_wrapper.rows:
if row in existing_domains:
skipped_domains += 1
else:
new_domains += 1
new_domain = DomainName(
name=row,
is_monitored=False,
scanning_enabled=True,
organization=self.organization,
)
new_domain.save()
to_return = DomainsUploadResponse(
new_domains=new_domains,
skipped=skipped_domains,
errored=len(file_wrapper.errored_rows),
batch_required=False,
)
FilesystemHelper.delete_file(temp_path)
return to_return
def crawl_endpoint_to_file(
self,
ip_address=None,
port=None,
hostname=None,
use_ssl=False,
use_sni=False,
start_urls=[],
in_separate_process=True,
):
"""
Start crawling the given endpoint using the given list of URLs and write the results to
a local file.
:param ip_address: The IP address to crawl.
:param port: The port where the application resides.
:param hostname: The hostname to submit alongside all requests to the remote endpoint.
:param use_ssl: Whether or not to use SSL to connect to the remote web service.
:param use_sni: Whether or not to use SNI to connect to the remote web service.
:param start_urls: A list of URLs to start crawling from.
:param in_separate_process: Whether or not to spawn off a separate process for the crawl. This
enables us to call this method multiple times in the same process, as a Twisted reactor can only
be started and stopped once per process.
:return: A tuple containing (1) the string containing the local file path where crawling
results are stored and (2) a ScrapyResultWrapper configured to process the contents of the file.
"""
temp_file_path = FilesystemHelper.get_temporary_file_path()
local_file_path = "%s-%s-%s:%s" % (temp_file_path, self.bot_name, ip_address, port)
spider_kwargs = {
"input_ip_address": ip_address,
"input_start_urls": start_urls,
"input_file_path": local_file_path,
"input_hostname": hostname,
"input_use_ssl": use_ssl,
"input_use_sni": use_sni,
"input_port": port,
}
pipeline_settings = self.__get_local_storage_item_pipeline()
requested_hostname = hostname if hostname is not None else ip_address
settings = self.get_scrapy_settings(item_pipeline=pipeline_settings, hostname=requested_hostname)
crawling_config = {
"spider_kwargs": spider_kwargs,
"settings": settings,
}
if in_separate_process:
process = Process(target=self.__crawl, kwargs=crawling_config)
process.start()
process.join()
process.terminate()
else:
self.__crawl(**crawling_config)
return local_file_path, ScrapyResultWrapper.from_file(local_file_path)
def scan_for_open_tcp_ports(self, ports=None, db_session=None):
"""
Scan the wrapped IP address for open services on the list of ports.
:param ports: A list of integers representing the ports to scan.
:param db_session: A SQLAlchemy session.
:return: A list of the ports found to be open.
"""
nmap_scanner = NmapScanner.from_default_configuration(db_session)
nmap_scanner.add_ports(ports)
nmap_scanner.add_ipv4_address(self.ip_address)
temp_path = FilesystemHelper.get_temporary_file_path()
nmap_scanner.output_file_path = temp_path
nmap_scanner.scan_type = "tcp connect"
nmap_scanner.run()
results_wrapper = nmap_scanner.get_results_parser()
results = results_wrapper.get_results_for_ip_address(self.ip_address)
if not results:
to_return = []
else:
to_return = [x.port_number for x in results.open_ports]
extensions = [".xml", ".gnmap", ".nmap"]
for extension in extensions:
FilesystemHelper.delete_file("%s%s" % (temp_path, extension))
return to_return
def get_dns_record_types_for_scan():
"""
Get a list of tuples containing (1) the DNS record type and (2) whether or not resolved IPs
associated with the record type should be scanned from the DNS record types file.
:return: A list of tuples containing (1) the DNS record type and (2) whether or not resolved IPs
associated with the record type should be scanned from the DNS record types file.
"""
file_contents = FilesystemHelper.get_file_contents(
path=config.files_dns_record_types_path)
file_contents = [x.strip() for x in file_contents.strip().split("\n")]
to_return = []
for line in file_contents:
line_split = line.split(",")
if line_split[1] == "True":
to_return.append((
line_split[0],
True if line_split[2] == "True" else False,
))
return to_return
def scan_endpoints_from_bitsquat_file(self, org_uuid=None, file_path=None):
"""
Perform a custom scan for the web applications found on the endpoints contained within the
given CSV file.
:param org_uuid: The UUID of the organization to associate scan results with.
:param file_path: The local file path to where the file containing the endpoints to scan
resides.
:return: None
"""
contents = FilesystemHelper.get_file_contents(file_path)
contents = [x.strip() for x in contents.strip().split("\n")]
task_sigs = []
for line in contents:
domain, ip_address = [x.strip() for x in line.split(",")]
task_sigs.append(
scan_endpoint_from_bitsquat_file.si(
org_uuid=org_uuid,
domain=domain,
ip_address=ip_address,
))
group(task_sigs).apply_async()
def __create_scan_ports(self):
"""
Get a list of ScanPort objects representing the default ports that should be scanned for
this organization.
:return: A list of ScanPort objects representing the default ports that should be scanned for
this organization.
"""
contents = FilesystemHelper.get_file_contents(
path=config.files_default_scan_ports_path)
contents = [x.strip() for x in contents.strip().split("\n")]
ports = []
for line in contents:
line_split = [x.strip() for x in line.split(",")]
ports.append((int(line_split[0]), line_split[1]))
to_return = []
for port_number, protocol in ports:
to_return.append(
ScanPort.objects.create(
port_number=port_number,
protocol=protocol,
))
return to_return
def get_scanned_networks(self):
"""
Get a list of CIDR ranges that this Zmap scanner is configured to scan.
:return: A list of CIDR ranges that this Zmap scanner is configured to scan.
"""
return FilesystemHelper.get_lines_from_file(self.whitelist_file)
def _prepare_pre_run(self):
FilesystemHelper.touch(self.output_file)