def test_download_succeeded_honors_validate_units_set_true(self, download_failed):
"""
We have a setting that makes download validation optional. This test ensures that
download_succeeded()
honors that setting.
"""
# In this config, we will set validate_units to False, which should make our
# "wrong_checksum" OK
config = importer_mocks.get_basic_config(
**{importer_constants.KEY_FEED: 'http://fake.com/iso_feed/',
importer_constants.KEY_VALIDATE: True})
iso_sync_run = ISOSyncRun(self.sync_conduit, config)
destination = os.path.join(self.temp_dir, 'test.txt')
with open(destination, 'w') as test_file:
test_file.write('Boring test data.')
unit = MagicMock()
unit.storage_path = destination
iso = models.ISO('test.txt', 114, 'wrong checksum', unit)
iso.url = 'http://fake.com'
report = DownloadReport(iso.url, destination, iso)
# Let's fake having downloaded the whole file
iso.bytes_downloaded = iso.size
report.bytes_downloaded = iso.size
iso_sync_run.progress_report._state = SyncProgressReport.STATE_ISOS_IN_PROGRESS
iso_sync_run.download_succeeded(report)
# Because we fail validation, the save_unit step will not be called
self.assertEqual(self.sync_conduit.save_unit.call_count, 0)
# The download should be marked failed
self.assertEqual(download_failed.call_count, 1)
download_failed.assert_called_once_with(report)
def test_download_succeeded_fails_checksum(self, download_failed):
"""
This test verifies that download_succeeded does the right thing if the checksum fails. Note
that we are also implicitly testing that the default behavior is to validate downloads by
not setting it in this test. There are two other tests that verify that setting the boolean
explicitly is honored.
"""
self.config.override_config[importer_constants.KEY_VALIDATE] = True
iso_sync_run = ISOSyncRun(self.sync_conduit, self.config)
destination = os.path.join(self.temp_dir, 'test.txt')
with open(destination, 'w') as test_file:
test_file.write('Boring test data.')
unit = MagicMock()
unit.storage_path = destination
iso = models.ISO('test.txt', 114, 'wrong checksum', unit)
iso.url = 'http://fake.com'
report = DownloadReport(iso.url, destination, iso)
# Let's fake having downloaded the whole file
iso.bytes_downloaded = iso.size
report.bytes_downloaded = iso.size
iso_sync_run.progress_report._state = SyncProgressReport.STATE_ISOS_IN_PROGRESS
iso_sync_run.download_succeeded(report)
# Because we fail validation, the save_unit step will not be called
self.assertEqual(self.sync_conduit.save_unit.call_count, 0)
# The download should be marked failed
self.assertEqual(download_failed.call_count, 1)
download_failed.assert_called_once_with(report)
def test_download_succeeded(self, download_failed):
destination = os.path.join(self.temp_dir, 'test.txt')
with open(destination, 'w') as test_file:
test_file.write(
'Descartes walks into a bar and sits down, the bartender walks up to him and says '
'"You, my '
'man, look like you need a stiff drink." Descartes considers this, and shakes his '
'head "No, '
'I don\'t think-" and ceases to exist.')
unit = MagicMock()
unit.storage_path = destination
iso = models.ISO('test.txt', 217,
'a1552efee6f04012bc7e1f3e02c00c6177b08217cead958c47ec83cb8f97f835',
unit)
iso.url = 'http://fake.com'
report = DownloadReport(iso.url, destination, iso)
# Simulate having downloaded the whole file
iso.bytes_downloaded = iso.size
report.bytes_downloaded = iso.size
self.iso_sync_run.progress_report._state = SyncProgressReport.STATE_ISOS_IN_PROGRESS
self.iso_sync_run.download_succeeded(report)
# The sync conduit should have been called to save the unit
self.sync_conduit.save_unit.assert_any_call(unit)
# The download should not fail
self.assertEqual(download_failed.call_count, 0)
def test_download_failed(self):
report = DownloadReport('', '')
report.error_report['response_code'] = 1234
# test
listener = DownloadListener(None, None)
listener.download_failed(report)
def test_handle_get(self, model, _download, _on_succeeded, responder):
"""
Three catalog entries.
The 1st download fails but succeeds on the 2nd.
The 3rd is not tried.
"""
request = Mock(uri='http://content-world.com/content/bear.rpm')
responder.return_value.__enter__.return_value = responder.return_value
report = DownloadReport('', '')
_download.side_effect = SideEffect(DownloadFailed(report), report,
None)
catalog = [
Mock(url='url-a'),
Mock(url='url-b'),
Mock(url='url-c'), # not tried.
]
model.objects.filter.return_value.order_by.return_value.all.return_value = catalog
model.objects.filter.return_value.order_by.return_value.count.return_value = len(
catalog)
# test
streamer = Streamer(Mock())
streamer._handle_get(request)
# validation
model.objects.filter.assert_called_once_with(path='/content/bear.rpm')
model.objects.filter.return_value.order_by.\
assert_called_once_with('-_id', '-revision')
responder.assert_called_once_with(request)
_on_succeeded.assert_called_once_with(catalog[1], request, report)
self.assertEqual(_download.call_args_list, [
call(request, catalog[0], responder.return_value),
call(request, catalog[1], responder.return_value)
])
def test_download_headers(self):
request = Mock(uri='http://content-world.com/content/bear.rpm',
headers={})
request.setHeader.side_effect = request.headers.__setitem__
report = DownloadReport('', '')
report.headers = {
'A': 1,
'B': 2,
}
# should be ignored.
report.headers.update({k: '' for k in HOP_BY_HOP_HEADERS})
config = Mock(properties={'streamer': {'cache_timeout': 100}})
def get(s, p):
return config.properties[s][p]
config.get.side_effect = get
streamer = Mock(config=config)
# test
listener = DownloadListener(streamer, request)
listener.download_headers(report)
# validation
self.assertEqual(request.headers, {
'Cache-Control': 'public, s-maxage=100, max-age=100',
'A': 1,
'B': 2,
})
def test_handle_get_all_failed(self, model, _download, _on_all_failed,
responder):
"""
Three catalog entries.
All (3) failed.
"""
request = Mock(uri='http://content-world.com/content/bear.rpm')
responder.return_value.__enter__.return_value = responder.return_value
report = DownloadReport('', '')
_download.side_effect = SideEffect(PluginNotFound(), DoesNotExist(),
DownloadFailed(report))
catalog = [
Mock(url='url-a'),
Mock(url='url-b'),
Mock(url='url-c'),
]
model.objects.filter.return_value.order_by.return_value.all.return_value = catalog
model.objects.filter.return_value.order_by.return_value.count.return_value = len(
catalog)
# test
streamer = Streamer(Mock())
streamer._handle_get(request)
# validation
model.objects.filter.assert_called_once_with(path='/content/bear.rpm')
model.objects.filter.return_value.order_by.\
assert_called_once_with('-_id', '-revision')
responder.assert_called_once_with(request)
_on_all_failed.assert_called_once_with(request)
self.assertEqual(_download.call_args_list, [
call(request, catalog[0], responder.return_value),
call(request, catalog[1], responder.return_value),
call(request, catalog[2], responder.return_value)
])
def setUp(self):
self.mock_sync = mock.MagicMock()
# this causes validation to be skipped
self.mock_sync.config.get.return_value = False
self.mock_metadata_files = mock.MagicMock()
self.listener = listener.DRPMListener(self.mock_sync,
self.mock_metadata_files)
self.report = DownloadReport('http://pulpproject.org', '/a/b/c')
def test_get_single_path_failure(self, mock_download_one):
report = DownloadReport('http://pulpproject.org/v1/repositories/pulp/crane/images',
StringIO(''))
report.headers = {}
report.state = report.DOWNLOAD_FAILED
mock_download_one.return_value = report
self.assertRaises(IOError, self.repo._get_single_path, '/v1/repositories/pulp/crane/images')
def download_succeeded(self, request):
"""
Notification that downloading has succeeded for the specified request.
Fields mapped and forwarded to the wrapped listener.
:param request: A download request.
:type request: pulp.server.content.sources.model.Request
"""
report = DownloadReport(request.url, request.destination, request.data)
self.content_listener.download_succeeded(report)
def download_one(request):
"""
Mock the download_one() method to manipulate the path.
"""
self.assertEqual(request.url, 'https://registry.example.com/v2/')
self.assertEqual(type(request.destination), type(StringIO()))
report = DownloadReport(request.url, request.destination)
report.download_succeeded()
report.headers = {'Docker-Distribution-API-Version': 'registry/2.0'}
report.destination.write("")
return report
def download_one(request):
"""
Mock the download_one() method to manipulate the path.
"""
self.assertEqual(request.url, 'https://registry.example.com/v2/pulp/tags/list')
self.assertEqual(type(request.destination), type(StringIO()))
report = DownloadReport(request.url, request.destination)
report.download_succeeded()
report.headers = {}
report.destination.write('{"name": "pulp", "tags": ["best_ever", "latest", "decent"]}')
return report
def download_one(request):
"""
Mock the download_one() method.
"""
self.assertEqual(request.url, 'https://registry.example.com/some/path')
self.assertEqual(type(request.destination), type(StringIO()))
report = DownloadReport(request.url, request.destination)
report.download_succeeded()
report.headers = {'some': 'cool stuff'}
report.destination.write("This is the stuff you've been waiting for.")
return report
def download_one(request):
"""
Mock the download_one() method to manipulate the path.
"""
self.assertEqual(request.url, 'https://registry.example.com/v2/')
self.assertEqual(type(request.destination), type(StringIO()))
report = DownloadReport(request.url, request.destination)
report.download_succeeded()
# The Version header is not present
report.headers = {}
report.destination.write("")
return report
def test__raise_path_error_not_found(self):
"""
For a standard error like 404, the report's error message should be used.
"""
report = DownloadReport('http://foo/bar', '/a/b/c')
report.error_report = {'response_code': httplib.NOT_FOUND}
report.error_msg = 'oops'
with self.assertRaises(IOError) as assertion:
registry.V2Repository._raise_path_error(report)
self.assertEqual(assertion.exception.message, report.error_msg)
def test_failed_reports(self):
self.metadata_files.downloader.download = mock.MagicMock(
spec_set=self.metadata_files.downloader.download)
self.metadata_files.metadata = {
'primary': file_info_factory('primary'),
}
report = DownloadReport('url', '/destination')
report.download_failed()
self.metadata_files.event_listener.failed_reports.append(report)
# Ensure an exception is raised if the download failed
self.assertRaises(IOError, self.metadata_files.download_metadata_files)
def test_get_tags(self, mock_download_one):
body = json.dumps({'latest': 'abc123'})
report = DownloadReport('http://pulpproject.org/v1/repositories/pulp/crane/tags',
StringIO(body))
report.headers = {}
mock_download_one.return_value = report
ret = self.repo._get_single_path('/v1/repositories/pulp/crane/tags')
self.assertEqual(ret, {'latest': 'abc123'})
self.assertEqual(mock_download_one.call_count, 1)
self.assertTrue(isinstance(mock_download_one.call_args[0][0], DownloadRequest))
req = mock_download_one.call_args[0][0]
self.assertEqual(req.url, 'http://pulpproject.org/v1/repositories/pulp/crane/tags')
def test_download_failed_during_iso_download(self, _logger):
self.iso_sync_run.progress_report._state = SyncProgressReport.STATE_ISOS_IN_PROGRESS
url = 'http://www.theonion.com/articles/american-airlines-us-airways-merge-to-form' \
'-worlds,31302/'
iso = models.ISO('test.txt', 217,
'a1552efee6f04012bc7e1f3e02c00c6177b08217cead958c47ec83cb8f97f835')
report = DownloadReport(url, '/fake/destination', iso)
report.error_msg = 'uh oh'
self.iso_sync_run.download_failed(report)
self.assertEqual(_logger.error.call_count, 1)
log_msg = _logger.error.mock_calls[0][1][0]
self.assertTrue('uh oh' in log_msg)
def test_get_with_headers(self, mock_download_one):
body = json.dumps(['abc123'])
report = DownloadReport('http://pulpproject.org/v1/repositories/pulp/crane/images',
StringIO(body))
report.headers = {
self.repo.DOCKER_TOKEN_HEADER: 'token',
self.repo.DOCKER_ENDPOINT_HEADER: 'endpoint',
}
mock_download_one.return_value = report
self.repo._get_single_path('/v1/repositories/pulp/crane/images')
self.assertEqual(self.repo.token, 'token')
self.assertEqual(self.repo.endpoint, 'endpoint')
def download_one(request):
"""
Mock the download_one() method to manipulate the path.
"""
self.assertEqual(request.url,
'https://registry.example.com/v2/pulp/manifests/best_version_ever')
self.assertEqual(type(request.destination), type(StringIO()))
report = DownloadReport(request.url, request.destination)
report.download_succeeded()
schema2 = 'application/vnd.docker.distribution.manifest.v2+json'
report.headers = {'Docker-Distribution-API-Version': 'registry/2.0',
'docker-content-digest': digest,
'content-type': schema2}
report.destination.write(manifest)
return report
def test_retrieve_metadata_with_error(self, mock_downloader_download,
mock_listener_constructor):
# Setup
mock_listener = mock.MagicMock()
report = DownloadReport(None, None)
report.error_msg = 'oops'
mock_listener.failed_reports = [report]
mock_listener_constructor.return_value = mock_listener
# Test
try:
self.downloader.retrieve_metadata(self.mock_progress_report)
self.fail()
except exceptions.FileRetrievalException:
pass
def test_get_images(self, mock_download_one):
body = json.dumps(['abc123'])
report = DownloadReport('http://pulpproject.org/v1/repositories/pulp/crane/images',
StringIO(body))
report.headers = {}
mock_download_one.return_value = report
ret = self.repo._get_single_path('/v1/repositories/pulp/crane/images')
self.assertEqual(ret, ['abc123'])
self.assertEqual(mock_download_one.call_count, 1)
self.assertTrue(isinstance(mock_download_one.call_args[0][0], DownloadRequest))
req = mock_download_one.call_args[0][0]
self.assertEqual(req.url, 'http://pulpproject.org/v1/repositories/pulp/crane/images')
# make sure this header is set, which is required by the docker API in order
# to give us an auth token
self.assertEqual(req.headers[self.repo.DOCKER_TOKEN_HEADER], 'true')
def test_download_failed_during_manifest(self, _logger):
self.iso_sync_run.progress_report._state = SyncProgressReport.STATE_MANIFEST_IN_PROGRESS
url = 'http://www.theonion.com/articles/' + \
'american-airlines-us-airways-merge-to-form-worlds,31302/'
report = DownloadReport(url, '/fake/destination')
report.error_report = {'why': 'because'}
report.error_msg = 'uh oh'
self.iso_sync_run.download_failed(report)
# The manifest_state should be failed
self.assertEqual(self.iso_sync_run.progress_report._state,
SyncProgressReport.STATE_MANIFEST_FAILED)
self.assertEqual(self.iso_sync_run.progress_report.error_message, report.error_report)
self.assertEqual(_logger.error.call_count, 1)
log_msg = _logger.error.mock_calls[0][1][0]
self.assertTrue('uh oh' in log_msg)
def test__raise_path_error_unathorized(self):
"""
Specifically for a 401, a custom error message should be used explaining that the cause
could be either that the client is unauthorized, or that the resource was not found.
Docker hub responds 401 for the not found case, which is why this function exists.
"""
report = DownloadReport('http://foo/bar', '/a/b/c')
report.error_report = {'response_code': httplib.UNAUTHORIZED}
report.error_msg = 'oops'
with self.assertRaises(IOError) as assertion:
registry.V2Repository._raise_path_error(report)
# not worrying about what the exact contents are; just that the function added its
# own message
self.assertNotEqual(assertion.exception.message, report.error_msg)
self.assertTrue(len(assertion.exception.message) > 0)
def test_on_succeeded_pulp_requested(self, _insert_deferred):
entry = Mock(url='url-a')
request = Mock(uri='http://content-world.com/content/bear.rpm')
request.getHeader.side_effect = {
constants.PULP_STREAM_REQUEST_HEADER: True
}.__getitem__
report = DownloadReport('', '')
report.headers = {
'A': 1,
'B': 2,
}
# test
streamer = Streamer(Mock())
streamer._on_succeeded(entry, request, report)
# validation
self.assertFalse(_insert_deferred.called)
def test__get_path_failed(self, mock_download_one, mock_request_token):
"""
Test _get_path() for the case when an IOError is raised by the downloader.
"""
name = 'pulp'
download_config = DownloaderConfig(max_concurrent=25)
registry_url = 'https://registry.example.com'
working_dir = '/a/working/dir'
r = registry.V2Repository(name, download_config, registry_url, working_dir)
report = DownloadReport(registry_url + '/some/path', StringIO())
report.error_report['response_code'] = httplib.UNAUTHORIZED
report.state = DownloadReport.DOWNLOAD_FAILED
report.headers = {}
mock_download_one.return_value = report
# The request will fail because the requested path does not exist
self.assertRaises(IOError, r._get_path, '/some/path')
def test_get_tags_from_endpoint(self, mock_download_one):
body = json.dumps({'latest': 'abc123'})
report = DownloadReport('http://some-endpoint.org/v1/repositories/pulp/crane/tags',
StringIO(body))
report.headers = {}
mock_download_one.return_value = report
self.repo.endpoint = 'some-endpoint.org'
# this lets us test that auth was added to the request
self.repo.token = 'letmein'
ret = self.repo._get_single_path('/v1/repositories/pulp/crane/tags')
self.assertEqual(ret, {'latest': 'abc123'})
self.assertEqual(mock_download_one.call_count, 1)
self.assertTrue(isinstance(mock_download_one.call_args[0][0], DownloadRequest))
req = mock_download_one.call_args[0][0]
self.assertEqual(req.url, 'http://some-endpoint.org/v1/repositories/pulp/crane/tags')
# make sure the authorization was added, which is usually required by an endpoint
self.assertTrue('Authorization' in req.headers)
def test_retrieve_module_missing_module(self, mock_downloader_download,
mock_listener_constructor):
# Setup
mock_listener = mock.MagicMock()
report = DownloadReport(None, None)
report.error_msg = 'oops'
mock_listener.failed_reports = [report]
mock_listener_constructor.return_value = mock_listener
# Test
try:
self.downloader.retrieve_module(self.mock_progress_report,
self.module)
self.fail()
except exceptions.FileRetrievalException:
expected_filename = web._create_download_tmp_dir(self.working_dir)
expected_filename = os.path.join(expected_filename,
self.module.filename())
self.assertFalse(os.path.exists(os.path.join(expected_filename)))
def fake_download(self, requests):
requests = list(requests)
req = requests[0]
try:
# try to write the manifest data
req.destination.write(
'test.iso,f02d5a72cd2d57fa802840a76b44c6c6920a8b8e6b90b20e26c03876275069e0,16\n'
'test2.iso,c7fbc0e821c0871805a99584c6a384533909f68a6bbe9a2a687d28d9f3b10c16,22\n'
'test3.iso,94f7fe923212286855dea858edac1b4a292301045af0ddb275544e5251a50b3c,34'
)
except AttributeError:
# this happens for all requests except the manifest
pass
reports = []
for r in requests:
# pretend everything worked great
report = DownloadReport(r.url, r.destination, r.data)
self.iso_sync_run.download_succeeded(report)
reports.append(report)
def test_retrieve_module_missing_module(self, mock_get_working_dir,
mock_downloader_download,
mock_listener_constructor):
# Setup
self.module.author = 'asdf'
self.module.puppet_standard_filename.return_value = 'puppet-filename.tar.gz'
mock_listener = mock.MagicMock()
report = DownloadReport(None, None)
report.error_msg = 'oops'
mock_listener.failed_reports = [report]
mock_listener_constructor.return_value = mock_listener
# Test
try:
self.downloader.retrieve_module(self.mock_progress_report,
self.module)
self.fail()
except exceptions.FileRetrievalException:
expected_filename = web._create_download_tmp_dir(self.working_dir)
expected_filename = os.path.join(expected_filename,
self.module.filename())
