def test_download_url_to_tempfileobj_and_lengths(self):
# We do *not* catch
# 'securesystemslib.exceptions.DownloadLengthMismatchError' in the
# following two calls because the file at 'self.url' contains enough bytes
# to satisfy the smaller number of required bytes requested.
# safe_download() and unsafe_download() will only log a warning when the
# the server-reported length of the file does not match the
# required_length. 'updater.py' *does* verify the hashes of downloaded
# content.
download.safe_download(self.url, self.target_data_length - 4,
self.fetcher).close()
download.unsafe_download(self.url, self.target_data_length - 4,
self.fetcher).close()
# We catch 'tuf.exceptions.DownloadLengthMismatchError' for safe_download()
# because it will not download more bytes than requested (in this case, a
# length greater than the size of the target file).
self.assertRaises(tuf.exceptions.DownloadLengthMismatchError,
download.safe_download, self.url,
self.target_data_length + 1, self.fetcher)
# Calling unsafe_download() with a mismatched length should not raise an
# exception.
download.unsafe_download(self.url, self.target_data_length + 1,
self.fetcher).close()
def test_https_dl_via_smart_http_proxy(self):
"""
Test a length-validating TUF download of a file through a proxy. Use an
HTTP proxy that supports HTTP CONNECT (which essentially causes it to act
as a TCP proxy), and perform an HTTPS connection through with the final
server.
Note that the proxy address is still http://... even though the connection
with the target server is an HTTPS connection. The proxy itself will act as
a TCP proxy via HTTP CONNECT.
"""
self.set_env_value('HTTP_PROXY',
self.http_proxy_addr) # http as intended
self.set_env_value('HTTPS_PROXY',
self.http_proxy_addr) # http as intended
self.set_env_value('REQUESTS_CA_BUNDLE',
os.path.join('ssl_certs', 'ssl_cert.crt'))
# Clear sessions to ensure that the certificate we just specified is used.
# TODO: Confirm necessity of this session clearing and lay out mechanics.
tuf.download._sessions = {}
logger.info('Trying HTTPS download via HTTP proxy: ' + self.url_https)
download.safe_download(self.url_https, self.target_data_length)
download.unsafe_download(self.url_https, self.target_data_length)
def test_download_url_to_tempfileobj_and_lengths(self):
# We do *not* catch 'tuf.DownloadLengthMismatchError' in the following two
# calls because the file at 'self.url' contains enough bytes to satisfy the
# smaller number of required bytes requested. safe_download() and
# unsafe_download() will only log a warning when the the server-reported
# length of the file does not match the required_length. 'updater.py'
# *does* verify the hashes of downloaded content.
download.safe_download(self.url, self.target_data_length - 4)
download.unsafe_download(self.url, self.target_data_length - 4)
# We catch 'tuf.DownloadLengthMismatchError' here because safe_download()
# will not download more bytes than requested.
self.assertRaises(tuf.DownloadLengthMismatchError,
download.safe_download, self.url,
self.target_data_length + 1)
# However, we do *not* catch 'tuf.DownloadLengthMismatchError' in the next
# test condition because unsafe_download() does not enforce the required
# length argument. The length reported and the length downloaded are still
# logged.
temp_fileobj = \
download.unsafe_download(self.url, self.target_data_length + 1)
self.assertEqual(self.target_data, temp_fileobj.read().decode('utf-8'))
self.assertEqual(self.target_data_length, len(temp_fileobj.read()))
temp_fileobj.close_temp_file()
def test_baseline_no_proxy(self):
"""
Test a length-validating TUF download of a file through a proxy. Use an
HTTP proxy, and perform an HTTP connection with the final server.
"""
logger.info('Trying HTTP download with no proxy: ' + self.url)
download.safe_download(self.url, self.target_data_length)
download.unsafe_download(self.url, self.target_data_length)
def test_http_dl_via_smart_http_proxy(self):
"""
Test a length-validating TUF download of a file through a proxy. Use an
HTTP proxy normally, and make an HTTP connection with the final server.
"""
self.set_env_value('HTTP_PROXY', self.http_proxy_addr)
logger.info('Trying HTTP download via HTTP proxy: ' + self.url)
download.safe_download(self.url, self.target_data_length)
download.unsafe_download(self.url, self.target_data_length)
def test_http_dl_via_https_proxy(self):
"""
Test a length-validating TUF download of a file through a proxy. Use an
HTTPS proxy, and perform an HTTP connection with the final server.
"""
self.set_env_value('HTTP_PROXY', self.https_proxy_addr)
self.set_env_value('HTTPS_PROXY', self.https_proxy_addr) # unnecessary
# We're making an HTTPS connection with the proxy. The proxy will make a
# plain HTTP connection to the target server.
self.set_env_value('REQUESTS_CA_BUNDLE',
os.path.join('ssl_certs', 'proxy_ca.crt'))
# Clear sessions to ensure that the certificate we just specified is used.
# TODO: Confirm necessity of this session clearing and lay out mechanics.
tuf.download._sessions = {}
logger.info('Trying HTTP download via HTTPS proxy: ' + self.url_https)
download.safe_download(self.url, self.target_data_length)
download.unsafe_download(self.url, self.target_data_length)
def test_https_connection(self):
# Make a temporary file to be served to the client.
current_directory = os.getcwd()
target_filepath = self.make_temp_data_file(directory=current_directory)
target_data = None
target_data_length = 0
with open(target_filepath, 'r') as target_file_object:
target_data = target_file_object.read()
target_data_length = len(target_data)
# Launch an https server (serves files in the current dir).
port = random.randint(30000, 45000)
command = ['python', 'simple_https_server.py', str(port)]
https_server_process = subprocess.Popen(command,
stderr=subprocess.PIPE)
# NOTE: Following error is raised if delay is not applied:
# <urlopen error [Errno 111] Connection refused>
time.sleep(1)
junk, relative_target_filepath = os.path.split(target_filepath)
https_url = 'https://localhost:' + str(
port) + '/' + relative_target_filepath
# Download the target file using an https connection.
tuf.conf.ssl_certificates = 'ssl_cert.crt'
message = 'Downloading target file from https server: ' + https_url
logger.info(message)
try:
download.safe_download(https_url, target_data_length)
download.unsafe_download(https_url, target_data_length)
finally:
https_server_process
if https_server_process.returncode is None:
message = \
'Server process ' + str(https_server_process.pid) + ' terminated.'
logger.info(message)
self.server_proc.kill()
def test_https_connection(self):
# Make a temporary file to be served to the client.
current_directory = os.getcwd()
target_filepath = self.make_temp_data_file(directory=current_directory)
target_data = None
target_data_length = 0
with open(target_filepath, 'r') as target_file_object:
target_data = target_file_object.read()
target_data_length = len(target_data)
# Launch an https server (serves files in the current dir).
port = random.randint(30000, 45000)
command = ['python', 'simple_https_server.py', str(port)]
https_server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
# NOTE: Following error is raised if delay is not applied:
# <urlopen error [Errno 111] Connection refused>
time.sleep(1)
junk, relative_target_filepath = os.path.split(target_filepath)
https_url = 'https://localhost:' + str(port) + '/' + relative_target_filepath
# Download the target file using an https connection.
tuf.conf.ssl_certificates = 'ssl_cert.crt'
message = 'Downloading target file from https server: ' + https_url
logger.info(message)
try:
download.safe_download(https_url, target_data_length)
download.unsafe_download(https_url, target_data_length)
finally:
https_server_process
if https_server_process.returncode is None:
message = \
'Server process ' + str(https_server_process.pid) + ' terminated.'
logger.info(message)
self.server_proc.kill()
def test_download_url_to_tempfileobj_and_lengths(self):
# NOTE: We catch tuf.BadHashError here because the file, shorter by a byte,
# would not match the expected hashes. We log a warning when we find that
# the server-reported length of the file does not match our
# required_length. We also see that STRICT_REQUIRED_LENGTH does not change
# the outcome of the previous test.
download.safe_download(self.url, self.target_data_length - 1)
download.unsafe_download(self.url, self.target_data_length - 1)
# NOTE: We catch tuf.DownloadLengthMismatchError here because the
# STRICT_REQUIRED_LENGTH, which is True by default, mandates that we must
# download exactly what is required.
self.assertRaises(tuf.DownloadLengthMismatchError, download.safe_download,
self.url, self.target_data_length + 1)
# NOTE: However, we do not catch a tuf.DownloadLengthMismatchError here for
# the same test as the previous one because we have disabled
# STRICT_REQUIRED_LENGTH.
temp_fileobj = download.unsafe_download(self.url,
self.target_data_length + 1)
self.assertEquals(self.target_data, temp_fileobj.read())
self.assertEquals(self.target_data_length, len(temp_fileobj.read()))
temp_fileobj.close_temp_file()
def test_download_url_to_tempfileobj_and_lengths(self):
# We do *not* catch 'tuf.DownloadLengthMismatchError' in the following two
# calls because the file at 'self.url' contains enough bytes to satisfy the
# smaller number of required bytes requested. safe_download() and
# unsafe_download() will only log a warning when the the server-reported
# length of the file does not match the required_length. 'updater.py'
# *does* verify the hashes of downloaded content.
download.safe_download(self.url, self.target_data_length - 4)
download.unsafe_download(self.url, self.target_data_length - 4)
# We catch 'tuf.DownloadLengthMismatchError' here because safe_download()
# will not download more bytes than requested.
self.assertRaises(tuf.DownloadLengthMismatchError, download.safe_download,
self.url, self.target_data_length + 1)
# However, we do *not* catch 'tuf.DownloadLengthMismatchError' in the next
# test condition because unsafe_download() does not enforce the required
# length argument. The length reported and the length downloaded are still
# logged.
temp_fileobj = \
download.unsafe_download(self.url, self.target_data_length + 1)
self.assertEqual(self.target_data, temp_fileobj.read().decode('utf-8'))
self.assertEqual(self.target_data_length, len(temp_fileobj.read()))
temp_fileobj.close_temp_file()
def test_https_connection(self):
"""
Try various HTTPS downloads using trusted and untrusted certificates with
and without the correct hostname listed in the SSL certificate.
"""
# Make a temporary file to be served to the client.
current_directory = os.getcwd()
target_filepath = self.make_temp_data_file(directory=current_directory)
with open(target_filepath, 'r') as target_file_object:
target_data_length = len(target_file_object.read())
# These cert files provide various test cases:
# good: A valid cert from an older generation of test_download.py tests.
# good2: A valid cert made simultaneous to the bad certs below, with the
# same settings otherwise, tested here in case the difference
# between the way the new bad certs and the old good cert were
# generated turns out to matter at some point.
# bad: An otherwise-valid cert with the wrong hostname. The good certs
# list "localhost", but this lists "notmyhostname".
# expired: An otherwise-valid cert but which is expired (no valid dates
# exist, fwiw: startdate > enddate).
good_cert_fname = os.path.join('ssl_certs', 'ssl_cert.crt')
good2_cert_fname = os.path.join('ssl_certs', 'ssl_cert_2.crt')
bad_cert_fname = os.path.join('ssl_certs', 'ssl_cert_wronghost.crt')
expired_cert_fname = os.path.join('ssl_certs', 'ssl_cert_expired.crt')
# Launch four HTTPS servers (serve files in the current dir).
# 1: we expect to operate correctly
# 2: also good; uses a slightly different cert (controls for the cert
# generation method used for the next two, in case it comes to matter)
# 3: run with an HTTPS certificate with an unexpected hostname
# 4: run with an HTTPS certificate that is expired
# Be sure to offset from the port used in setUp to avoid collision.
port1 = str(self.PORT + 1)
port2 = str(self.PORT + 2)
port3 = str(self.PORT + 3)
port4 = str(self.PORT + 4)
good_https_server_proc = popen_python(
['simple_https_server.py', port1, good_cert_fname])
good2_https_server_proc = popen_python(
['simple_https_server.py', port2, good2_cert_fname])
bad_https_server_proc = popen_python(
['simple_https_server.py', port3, bad_cert_fname])
expd_https_server_proc = popen_python(
['simple_https_server.py', port4, expired_cert_fname])
# Provide a delay long enough to allow the four HTTPS servers to start.
# Have encountered errors at 0.2s, 0.5s, and 2s, primarily on AppVeyor.
# Increasing to 4s for this test.
# Expect to see "Connection refused" if this delay is not long enough
# (though other issues could cause that).
time.sleep(3)
relative_target_fpath = os.path.basename(target_filepath)
good_https_url = 'https://localhost:' + port1 + '/' + relative_target_fpath
good2_https_url = good_https_url.replace(':' + port1, ':' + port2)
bad_https_url = good_https_url.replace(':' + port1, ':' + port3)
expired_https_url = good_https_url.replace(':' + port1, ':' + port4)
# Download the target file using an HTTPS connection.
# Use try-finally solely to ensure that the server processes are killed.
try:
# Trust the certfile that happens to use a different hostname than we
# will expect.
os.environ['REQUESTS_CA_BUNDLE'] = bad_cert_fname
# Clear sessions to ensure that the certificate we just specified is used.
# TODO: Confirm necessity of this session clearing and lay out mechanics.
tuf.download._sessions = {}
# Try connecting to the server process with the bad cert while trusting
# the bad cert. Expect failure because even though we trust it, the
# hostname we're connecting to does not match the hostname in the cert.
logger.info('Trying HTTPS download of target file: ' + bad_https_url)
with self.assertRaises(requests.exceptions.SSLError):
download.safe_download(bad_https_url, target_data_length)
with self.assertRaises(requests.exceptions.SSLError):
download.unsafe_download(bad_https_url, target_data_length)
# Try connecting to the server processes with the good certs while not
# trusting the good certs (trusting the bad cert instead). Expect failure
# because even though the server's cert file is otherwise OK, we don't
# trust it.
print('Trying HTTPS download of target file: ' + good_https_url)
with self.assertRaises(requests.exceptions.SSLError):
download.safe_download(good_https_url, target_data_length)
with self.assertRaises(requests.exceptions.SSLError):
download.unsafe_download(good_https_url, target_data_length)
print('Trying HTTPS download of target file: ' + good2_https_url)
with self.assertRaises(requests.exceptions.SSLError):
download.safe_download(good2_https_url, target_data_length)
with self.assertRaises(requests.exceptions.SSLError):
download.unsafe_download(good2_https_url, target_data_length)
# Configure environment to now trust the certfile that is expired.
os.environ['REQUESTS_CA_BUNDLE'] = expired_cert_fname
# Clear sessions to ensure that the certificate we just specified is used.
# TODO: Confirm necessity of this session clearing and lay out mechanics.
tuf.download._sessions = {}
# Try connecting to the server process with the expired cert while
# trusting the expired cert. Expect failure because even though we trust
# it, it is expired.
logger.info('Trying HTTPS download of target file: ' + expired_https_url)
with self.assertRaises(requests.exceptions.SSLError):
download.safe_download(expired_https_url, target_data_length)
with self.assertRaises(requests.exceptions.SSLError):
download.unsafe_download(expired_https_url, target_data_length)
# Try connecting to the server processes with the good certs while
# trusting the appropriate good certs. Expect success.
# TODO: expand testing to switch expected certificates back and forth a
# bit more while clearing / not clearing sessions.
os.environ['REQUESTS_CA_BUNDLE'] = good_cert_fname
# Clear sessions to ensure that the certificate we just specified is used.
# TODO: Confirm necessity of this session clearing and lay out mechanics.
tuf.download._sessions = {}
logger.info('Trying HTTPS download of target file: ' + good_https_url)
download.safe_download(good_https_url, target_data_length)
download.unsafe_download(good_https_url, target_data_length)
os.environ['REQUESTS_CA_BUNDLE'] = good2_cert_fname
# Clear sessions to ensure that the certificate we just specified is used.
# TODO: Confirm necessity of this session clearing and lay out mechanics.
tuf.download._sessions = {}
logger.info('Trying HTTPS download of target file: ' + good2_https_url)
download.safe_download(good2_https_url, target_data_length)
download.unsafe_download(good2_https_url, target_data_length)
finally:
for proc in [
good_https_server_proc,
good2_https_server_proc,
bad_https_server_proc,
expd_https_server_proc]:
if proc.returncode is None:
logger.info('Terminating server process ' + str(proc.pid))
proc.kill()
except Exception,e: raise e #Write the data from the package fetched by TUF to 0install target file while True: try: #Read data 256 bytes at a time. This is an optmization in case of large packages data = doc.read(256) except IncompleteRead as ex: data = ex.partial if not data: break target_file.write(data) target_file.flush() else: #Safe Download Code added here from tuf import download as tufdownload doc = tufdownload.safe_download(url,expected_size) #Should read fixed size bytes at a time but currently bug with tuf.util.TempFile.read(size) ''' while True: try: #Read data 256 bytes at a time. This is an optmization in case of large packages data = doc.read(256) except IncompleteRead as ex: data = ex.partial if not data: break target_file.write(data) target_file.flush() ''' data = doc.read() target_file.write(data) target_file.flush()
def test_https_connection(self):
"""
Try various HTTPS downloads using trusted and untrusted certificates with
and without the correct hostname listed in the SSL certificate.
"""
# Make a temporary file to be served to the client.
current_directory = os.getcwd()
target_filepath = self.make_temp_data_file(directory=current_directory)
with open(target_filepath, 'r') as target_file_object:
target_data_length = len(target_file_object.read())
# These cert files provide various test cases:
# good: A valid cert from an older generation of test_download.py tests.
# good2: A valid cert made simultaneous to the bad certs below, with the
# same settings otherwise, tested here in case the difference
# between the way the new bad certs and the old good cert were
# generated turns out to matter at some point.
# bad: An otherwise-valid cert with the wrong hostname. The good certs
# list "localhost", but this lists "notmyhostname".
# expired: An otherwise-valid cert but which is expired (no valid dates
# exist, fwiw: startdate > enddate).
good_cert_fname = os.path.join('ssl_certs', 'ssl_cert.crt')
good2_cert_fname = os.path.join('ssl_certs', 'ssl_cert_2.crt')
bad_cert_fname = os.path.join('ssl_certs', 'ssl_cert_wronghost.crt')
expired_cert_fname = os.path.join('ssl_certs', 'ssl_cert_expired.crt')
# Launch four HTTPS servers (serve files in the current dir).
# 1: we expect to operate correctly
# 2: also good; uses a slightly different cert (controls for the cert
# generation method used for the next two, in case it comes to matter)
# 3: run with an HTTPS certificate with an unexpected hostname
# 4: run with an HTTPS certificate that is expired
# Be sure to offset from the port used in setUp to avoid collision.
port1 = self.server_process_handler.port + 1
port2 = self.server_process_handler.port + 2
port3 = self.server_process_handler.port + 3
port4 = self.server_process_handler.port + 4
good_https_server_handler = utils.TestServerProcess(
log=logger,
server='simple_https_server.py',
port=port1,
extra_cmd_args=[good_cert_fname])
good2_https_server_handler = utils.TestServerProcess(
log=logger,
server='simple_https_server.py',
port=port2,
extra_cmd_args=[good2_cert_fname])
bad_https_server_handler = utils.TestServerProcess(
log=logger,
server='simple_https_server.py',
port=port3,
extra_cmd_args=[bad_cert_fname])
expd_https_server_handler = utils.TestServerProcess(
log=logger,
server='simple_https_server.py',
port=port4,
extra_cmd_args=[expired_cert_fname])
suffix = '/' + os.path.basename(target_filepath)
good_https_url = 'https://localhost:' + str(port1) + suffix
good2_https_url = 'https://localhost:' + str(port2) + suffix
bad_https_url = 'https://localhost:' + str(port3) + suffix
expired_https_url = 'https://localhost:' + str(port4) + suffix
# Download the target file using an HTTPS connection.
# Use try-finally solely to ensure that the server processes are killed.
try:
# Trust the certfile that happens to use a different hostname than we
# will expect.
os.environ['REQUESTS_CA_BUNDLE'] = bad_cert_fname
# Clear sessions to ensure that the certificate we just specified is used.
# TODO: Confirm necessity of this session clearing and lay out mechanics.
tuf.download._sessions = {}
# Try connecting to the server process with the bad cert while trusting
# the bad cert. Expect failure because even though we trust it, the
# hostname we're connecting to does not match the hostname in the cert.
logger.info('Trying HTTPS download of target file: ' +
bad_https_url)
with warnings.catch_warnings():
# We're ok with a slightly fishy localhost cert
warnings.filterwarnings(
'ignore',
category=urllib3.exceptions.SubjectAltNameWarning)
with self.assertRaises(requests.exceptions.SSLError):
download.safe_download(bad_https_url, target_data_length)
with self.assertRaises(requests.exceptions.SSLError):
download.unsafe_download(bad_https_url, target_data_length)
# Try connecting to the server processes with the good certs while not
# trusting the good certs (trusting the bad cert instead). Expect failure
# because even though the server's cert file is otherwise OK, we don't
# trust it.
logger.info('Trying HTTPS download of target file: ' +
good_https_url)
with self.assertRaises(requests.exceptions.SSLError):
download.safe_download(good_https_url, target_data_length)
with self.assertRaises(requests.exceptions.SSLError):
download.unsafe_download(good_https_url,
target_data_length)
logger.info('Trying HTTPS download of target file: ' +
good2_https_url)
with self.assertRaises(requests.exceptions.SSLError):
download.safe_download(good2_https_url, target_data_length)
with self.assertRaises(requests.exceptions.SSLError):
download.unsafe_download(good2_https_url,
target_data_length)
# Configure environment to now trust the certfile that is expired.
os.environ['REQUESTS_CA_BUNDLE'] = expired_cert_fname
# Clear sessions to ensure that the certificate we just specified is used.
# TODO: Confirm necessity of this session clearing and lay out mechanics.
tuf.download._sessions = {}
# Try connecting to the server process with the expired cert while
# trusting the expired cert. Expect failure because even though we trust
# it, it is expired.
logger.info('Trying HTTPS download of target file: ' +
expired_https_url)
with self.assertRaises(requests.exceptions.SSLError):
download.safe_download(expired_https_url,
target_data_length)
with self.assertRaises(requests.exceptions.SSLError):
download.unsafe_download(expired_https_url,
target_data_length)
# Try connecting to the server processes with the good certs while
# trusting the appropriate good certs. Expect success.
# TODO: expand testing to switch expected certificates back and forth a
# bit more while clearing / not clearing sessions.
os.environ['REQUESTS_CA_BUNDLE'] = good_cert_fname
# Clear sessions to ensure that the certificate we just specified is used.
# TODO: Confirm necessity of this session clearing and lay out mechanics.
tuf.download._sessions = {}
logger.info('Trying HTTPS download of target file: ' +
good_https_url)
download.safe_download(good_https_url,
target_data_length).close()
download.unsafe_download(good_https_url,
target_data_length).close()
os.environ['REQUESTS_CA_BUNDLE'] = good2_cert_fname
# Clear sessions to ensure that the certificate we just specified is used.
# TODO: Confirm necessity of this session clearing and lay out mechanics.
tuf.download._sessions = {}
logger.info('Trying HTTPS download of target file: ' +
good2_https_url)
download.safe_download(good2_https_url,
target_data_length).close()
download.unsafe_download(good2_https_url,
target_data_length).close()
finally:
for proc_handler in [
good_https_server_handler, good2_https_server_handler,
bad_https_server_handler, expd_https_server_handler
]:
# Logs stdout and stderr from the server subprocess and then it
# kills it and closes the temp file used for logging.
proc_handler.clean()