def test_download_succeeded_fails_checksum(self, download_failed):
"""
This test verifies that download_succeeded does the right thing if the checksum fails. Note
that we are also implicitly testing that the default behavior is to validate downloads by
not setting it in this test. There are two other tests that verify that setting the boolean
explicitly is honored.
"""
self.config.override_config[importer_constants.KEY_VALIDATE] = True
iso_sync_run = ISOSyncRun(self.sync_conduit, self.config)
destination = os.path.join(self.temp_dir, 'test.txt')
with open(destination, 'w') as test_file:
test_file.write('Boring test data.')
unit = MagicMock()
unit.storage_path = destination
iso = models.ISO('test.txt', 114, 'wrong checksum', unit)
iso.url = 'http://fake.com'
report = DownloadReport(iso.url, destination, iso)
# Let's fake having downloaded the whole file
iso.bytes_downloaded = iso.size
report.bytes_downloaded = iso.size
iso_sync_run.progress_report._state = SyncProgressReport.STATE_ISOS_IN_PROGRESS
iso_sync_run.download_succeeded(report)
# Because we fail validation, the save_unit step will not be called
self.assertEqual(self.sync_conduit.save_unit.call_count, 0)
# The download should be marked failed
self.assertEqual(download_failed.call_count, 1)
download_failed.assert_called_once_with(report)
def test_download_failed(self):
report = DownloadReport('', '')
report.error_report['response_code'] = 1234
# test
listener = DownloadListener(None, None)
listener.download_failed(report)
def test_download_succeeded_honors_validate_units_set_true(self, download_failed):
"""
We have a setting that makes download validation optional. This test ensures that
download_succeeded()
honors that setting.
"""
# In this config, we will set validate_units to False, which should make our
# "wrong_checksum" OK
config = importer_mocks.get_basic_config(
**{importer_constants.KEY_FEED: 'http://fake.com/iso_feed/',
importer_constants.KEY_VALIDATE: True})
iso_sync_run = ISOSyncRun(self.sync_conduit, config)
destination = os.path.join(self.temp_dir, 'test.txt')
with open(destination, 'w') as test_file:
test_file.write('Boring test data.')
unit = MagicMock()
unit.storage_path = destination
iso = models.ISO('test.txt', 114, 'wrong checksum', unit)
iso.url = 'http://fake.com'
report = DownloadReport(iso.url, destination, iso)
# Let's fake having downloaded the whole file
iso.bytes_downloaded = iso.size
report.bytes_downloaded = iso.size
iso_sync_run.progress_report._state = SyncProgressReport.STATE_ISOS_IN_PROGRESS
iso_sync_run.download_succeeded(report)
# Because we fail validation, the save_unit step will not be called
self.assertEqual(self.sync_conduit.save_unit.call_count, 0)
# The download should be marked failed
self.assertEqual(download_failed.call_count, 1)
download_failed.assert_called_once_with(report)
def test_download_succeeded(self, download_failed):
destination = os.path.join(self.temp_dir, 'test.txt')
with open(destination, 'w') as test_file:
test_file.write(
'Descartes walks into a bar and sits down, the bartender walks up to him and says '
'"You, my '
'man, look like you need a stiff drink." Descartes considers this, and shakes his '
'head "No, '
'I don\'t think-" and ceases to exist.')
unit = MagicMock()
unit.storage_path = destination
iso = models.ISO('test.txt', 217,
'a1552efee6f04012bc7e1f3e02c00c6177b08217cead958c47ec83cb8f97f835',
unit)
iso.url = 'http://fake.com'
report = DownloadReport(iso.url, destination, iso)
# Simulate having downloaded the whole file
iso.bytes_downloaded = iso.size
report.bytes_downloaded = iso.size
self.iso_sync_run.progress_report._state = SyncProgressReport.STATE_ISOS_IN_PROGRESS
self.iso_sync_run.download_succeeded(report)
# The sync conduit should have been called to save the unit
self.sync_conduit.save_unit.assert_any_call(unit)
# The download should not fail
self.assertEqual(download_failed.call_count, 0)
def test_download_succeeded_honors_validate_units_set_false(self, download_failed):
"""
We have a setting that makes download validation optional. This test ensures that download_succeeded()
honors that setting.
"""
# In this config, we will set validate_units to False, which should make our "wrong_checksum" OK
config = importer_mocks.get_basic_config(**{importer_constants.KEY_FEED: 'http://fake.com/iso_feed/',
importer_constants.KEY_VALIDATE: False})
iso_sync_run = ISOSyncRun(self.sync_conduit, config)
destination = os.path.join(self.temp_dir, 'test.iso')
with open(destination, 'w') as test_iso:
test_iso.write('What happens when you combine a mosquito with a mountain climber? Nothing. You '
'can\'t cross a vector with a scalar.')
unit = MagicMock()
unit.storage_path = destination
iso = models.ISO('test.txt', 114, 'wrong checksum', unit)
iso.url = 'http://fake.com'
report = DownloadReport(iso.url, destination, iso)
# Let's fake having downloaded the whole file
iso.bytes_downloaded = iso.size
report.bytes_downloaded = iso.size
iso_sync_run.progress_report._state = SyncProgressReport.STATE_ISOS_IN_PROGRESS
iso_sync_run.download_succeeded(report)
# The sync conduit should have been called to save the unit
self.sync_conduit.save_unit.assert_any_call(unit)
# The download should not fail
self.assertEqual(download_failed.call_count, 0)
def test_download_headers(self):
request = Mock(uri='http://content-world.com/content/bear.rpm',
headers={})
request.setHeader.side_effect = request.headers.__setitem__
report = DownloadReport('', '')
report.headers = {
'A': 1,
'B': 2,
}
# should be ignored.
report.headers.update({k: '' for k in HOP_BY_HOP_HEADERS})
config = Mock(properties={'streamer': {'cache_timeout': 100}})
def get(s, p):
return config.properties[s][p]
config.get.side_effect = get
streamer = Mock(config=config)
# test
listener = DownloadListener(streamer, request)
listener.download_headers(report)
# validation
self.assertEqual(request.headers, {
'Cache-Control': 'public, s-maxage=100, max-age=100',
'A': 1,
'B': 2,
})
def test_get_single_path_failure(self, mock_download_one):
report = DownloadReport('http://pulpproject.org/v1/repositories/pulp/crane/images',
StringIO(''))
report.headers = {}
report.state = report.DOWNLOAD_FAILED
mock_download_one.return_value = report
self.assertRaises(IOError, self.repo._get_single_path, '/v1/repositories/pulp/crane/images')
def test_get_single_path_failure(self, mock_download_one):
report = DownloadReport('http://pulpproject.org/v1/repositories/pulp/crane/images',
StringIO(''))
report.headers = {}
report.state = report.DOWNLOAD_FAILED
mock_download_one.return_value = report
self.assertRaises(IOError, self.repo._get_single_path, '/v1/repositories/pulp/crane/images')
class TestDownloadReport(unittest.TestCase):
def setUp(self):
self.report = DownloadReport("fakeurl", "fakedestination")
def test_download_connection_error(self):
self.report.download_connection_error()
self.assertEqual(self.report.state, self.report.DOWNLOAD_FAILED)
self.assertEqual(self.report.error_msg, "A connection error occurred")
class TestDownloadReport(unittest.TestCase):
def setUp(self):
self.report = DownloadReport("fakeurl", "fakedestination")
def test_download_connection_error(self):
self.report.download_connection_error()
self.assertEqual(self.report.state, self.report.DOWNLOAD_FAILED)
self.assertEqual(self.report.error_msg, "A connection error occurred")
def download_failed(self, request):
"""
Notification that downloading has failed for the specified request.
Fields mapped and forwarded to the wrapped listener.
:param request: A download request.
:type request: pulp.server.content.sources.model.Request
"""
report = DownloadReport(request.url, request.destination, request.data)
report.error_report['errors'] = request.errors
self.content_listener.download_failed(report)
def download_one(request):
"""
Mock the download_one() method to manipulate the path.
"""
self.assertEqual(request.url, 'https://registry.example.com/v2/pulp/tags/list')
self.assertEqual(type(request.destination), type(StringIO()))
report = DownloadReport(request.url, request.destination)
report.download_succeeded()
report.headers = {}
report.destination.write('{"name": "pulp", "tags": ["best_ever", "latest", "decent"]}')
return report
def download_one(request):
"""
Mock the download_one() method to manipulate the path.
"""
self.assertEqual(request.url, 'https://registry.example.com/v2/pulp/tags/list')
self.assertEqual(type(request.destination), type(StringIO()))
report = DownloadReport(request.url, request.destination)
report.download_succeeded()
report.headers = {}
report.destination.write('{"name": "pulp", "tags": ["best_ever", "latest", "decent"]}')
return report
def download_one(request):
"""
Mock the download_one() method.
"""
self.assertEqual(request.url, 'https://registry.example.com/some/path')
self.assertEqual(type(request.destination), type(StringIO()))
report = DownloadReport(request.url, request.destination)
report.download_succeeded()
report.headers = {'some': 'cool stuff'}
report.destination.write("This is the stuff you've been waiting for.")
return report
def download_one(request):
"""
Mock the download_one() method to manipulate the path.
"""
self.assertEqual(request.url, 'https://registry.example.com/v2/')
self.assertEqual(type(request.destination), type(StringIO()))
report = DownloadReport(request.url, request.destination)
report.download_succeeded()
report.headers = {'Docker-Distribution-API-Version': 'registry/2.0'}
report.destination.write("")
return report
def download_one(request):
"""
Mock the download_one() method.
"""
self.assertEqual(request.url, 'https://registry.example.com/some/path')
self.assertEqual(type(request.destination), type(StringIO()))
report = DownloadReport(request.url, request.destination)
report.download_succeeded()
report.headers = {'some': 'cool stuff'}
report.destination.write("This is the stuff you've been waiting for.")
return report
def download_one(request):
"""
Mock the download_one() method to manipulate the path.
"""
self.assertEqual(request.url, 'https://registry.example.com/v2/')
self.assertEqual(type(request.destination), type(StringIO()))
report = DownloadReport(request.url, request.destination)
report.download_succeeded()
report.headers = {'Docker-Distribution-API-Version': 'registry/2.0'}
report.destination.write("")
return report
def test__raise_path_error_not_found(self):
"""
For a standard error like 404, the report's error message should be used.
"""
report = DownloadReport('http://foo/bar', '/a/b/c')
report.error_report = {'response_code': httplib.NOT_FOUND}
report.error_msg = 'oops'
with self.assertRaises(IOError) as assertion:
registry.V2Repository._raise_path_error(report)
self.assertEqual(assertion.exception.message, report.error_msg)
def download_one(request):
"""
Mock the download_one() method to manipulate the path.
"""
self.assertEqual(request.url, 'https://registry.example.com/v2/')
self.assertEqual(type(request.destination), type(StringIO()))
report = DownloadReport(request.url, request.destination)
report.download_succeeded()
# The Version header is not present
report.headers = {}
report.destination.write("")
return report
def test__raise_path_error_not_found(self):
"""
For a standard error like 404, the report's error message should be used.
"""
report = DownloadReport('http://foo/bar', '/a/b/c')
report.error_report = {'response_code': httplib.NOT_FOUND}
report.error_msg = 'oops'
with self.assertRaises(IOError) as assertion:
registry.V2Repository._raise_path_error(report)
self.assertEqual(assertion.exception.message, report.error_msg)
def download_one(request):
"""
Mock the download_one() method to manipulate the path.
"""
self.assertEqual(request.url, 'https://registry.example.com/v2/')
self.assertEqual(type(request.destination), type(StringIO()))
report = DownloadReport(request.url, request.destination)
report.download_succeeded()
# The Version header is not present
report.headers = {}
report.destination.write("")
return report
def test_failed_reports(self):
self.metadata_files.downloader.download = mock.MagicMock(
spec_set=self.metadata_files.downloader.download)
self.metadata_files.metadata = {
'primary': file_info_factory('primary'),
}
report = DownloadReport('url', '/destination')
report.download_failed()
self.metadata_files.event_listener.failed_reports.append(report)
# Ensure an exception is raised if the download failed
self.assertRaises(IOError, self.metadata_files.download_metadata_files)
def test_download_failed_during_manifest(self):
self.iso_sync_run.progress_report._state = SyncProgressReport.STATE_MANIFEST_IN_PROGRESS
url = 'http://www.theonion.com/articles/' +\
'american-airlines-us-airways-merge-to-form-worlds,31302/'
report = DownloadReport(url, '/fake/destination')
report.error_report = {'why': 'because'}
self.iso_sync_run.download_failed(report)
# The manifest_state should be failed
self.assertEqual(self.iso_sync_run.progress_report._state,
SyncProgressReport.STATE_MANIFEST_FAILED)
self.assertEqual(self.iso_sync_run.progress_report.error_message, report.error_report)
def test_download_failed_during_iso_download(self, _logger):
self.iso_sync_run.progress_report._state = SyncProgressReport.STATE_ISOS_IN_PROGRESS
url = 'http://www.theonion.com/articles/american-airlines-us-airways-merge-to-form' \
'-worlds,31302/'
iso = models.ISO('test.txt', 217,
'a1552efee6f04012bc7e1f3e02c00c6177b08217cead958c47ec83cb8f97f835')
report = DownloadReport(url, '/fake/destination', iso)
report.error_msg = 'uh oh'
self.iso_sync_run.download_failed(report)
self.assertEqual(_logger.error.call_count, 1)
log_msg = _logger.error.mock_calls[0][1][0]
self.assertTrue('uh oh' in log_msg)
def test_retrieve_metadata_with_error(self, mock_downloader_download, mock_listener_constructor):
# Setup
mock_listener = mock.MagicMock()
report = DownloadReport(None, None)
report.error_msg = 'oops'
mock_listener.failed_reports = [report]
mock_listener_constructor.return_value = mock_listener
# Test
try:
self.downloader.retrieve_metadata(self.mock_progress_report)
self.fail()
except exceptions.FileRetrievalException:
pass
def test_get_with_headers(self, mock_download_one):
body = json.dumps(['abc123'])
report = DownloadReport('http://pulpproject.org/v1/repositories/pulp/crane/images',
StringIO(body))
report.headers = {
self.repo.DOCKER_TOKEN_HEADER: 'token',
self.repo.DOCKER_ENDPOINT_HEADER: 'endpoint',
}
mock_download_one.return_value = report
self.repo._get_single_path('/v1/repositories/pulp/crane/images')
self.assertEqual(self.repo.token, 'token')
self.assertEqual(self.repo.endpoint, 'endpoint')
def test_get_tags(self, mock_download_one):
body = json.dumps({'latest': 'abc123'})
report = DownloadReport('http://pulpproject.org/v1/repositories/pulp/crane/tags',
StringIO(body))
report.headers = {}
mock_download_one.return_value = report
ret = self.repo._get_single_path('/v1/repositories/pulp/crane/tags')
self.assertEqual(ret, {'latest': 'abc123'})
self.assertEqual(mock_download_one.call_count, 1)
self.assertTrue(isinstance(mock_download_one.call_args[0][0], DownloadRequest))
req = mock_download_one.call_args[0][0]
self.assertEqual(req.url, 'http://pulpproject.org/v1/repositories/pulp/crane/tags')
def test_failed_reports(self):
self.metadata_files.downloader.download = mock.MagicMock(
spec_set=self.metadata_files.downloader.download
)
self.metadata_files.metadata = {
'primary': file_info_factory('primary'),
}
report = DownloadReport('url', '/destination')
report.download_failed()
self.metadata_files.event_listener.failed_reports.append(report)
# Ensure an exception is raised if the download failed
self.assertRaises(IOError, self.metadata_files.download_metadata_files)
def test_get_with_headers(self, mock_download_one):
body = json.dumps(['abc123'])
report = DownloadReport('http://pulpproject.org/v1/repositories/pulp/crane/images',
StringIO(body))
report.headers = {
self.repo.DOCKER_TOKEN_HEADER: 'token',
self.repo.DOCKER_ENDPOINT_HEADER: 'endpoint',
}
mock_download_one.return_value = report
self.repo._get_single_path('/v1/repositories/pulp/crane/images')
self.assertEqual(self.repo.token, 'token')
self.assertEqual(self.repo.endpoint, 'endpoint')
def test_get_tags(self, mock_download_one):
body = json.dumps({'latest': 'abc123'})
report = DownloadReport('http://pulpproject.org/v1/repositories/pulp/crane/tags',
StringIO(body))
report.headers = {}
mock_download_one.return_value = report
ret = self.repo._get_single_path('/v1/repositories/pulp/crane/tags')
self.assertEqual(ret, {'latest': 'abc123'})
self.assertEqual(mock_download_one.call_count, 1)
self.assertTrue(isinstance(mock_download_one.call_args[0][0], DownloadRequest))
req = mock_download_one.call_args[0][0]
self.assertEqual(req.url, 'http://pulpproject.org/v1/repositories/pulp/crane/tags')
def test_retrieve_metadata_with_error(self, mock_downloader_download,
mock_listener_constructor):
# Setup
mock_listener = mock.MagicMock()
report = DownloadReport(None, None)
report.error_msg = 'oops'
mock_listener.failed_reports = [report]
mock_listener_constructor.return_value = mock_listener
# Test
try:
self.downloader.retrieve_metadata(self.mock_progress_report)
self.fail()
except exceptions.FileRetrievalException:
pass
def download_one(request):
"""
Mock the download_one() method to manipulate the path.
"""
self.assertEqual(request.url,
'https://registry.example.com/v2/pulp/manifests/best_version_ever')
self.assertEqual(type(request.destination), type(StringIO()))
report = DownloadReport(request.url, request.destination)
report.download_succeeded()
schema2 = 'application/vnd.docker.distribution.manifest.v2+json'
report.headers = {'Docker-Distribution-API-Version': 'registry/2.0',
'docker-content-digest': digest,
'content-type': schema2}
report.destination.write(manifest)
return report
def _common_link(self, link_method, request, report=None):
"""
Link files using either a hard link or symbolic link method.
:param link_method: hard link or symbolic link method
:type link_method: callable
:param request: request instance
:type request: nectar.request.DownloadRequest
:param report: report instance for the request
:type report: nectar.report.DownloadReport
:return: report instance
:rtype: nectar.report.DownloadReport
"""
report = report or DownloadReport.from_download_request(request)
report.download_started()
self.fire_download_started(report)
if self.is_canceled:
report.download_cancelled()
return report
try:
if not isinstance(request.destination, basestring):
raise UnlinkableDestination(request.destination)
src_path = self._file_path_from_url(request.url)
link_method(src_path, request.destination)
except Exception, e:
_LOG.exception(e)
report.error_msg = str(e)
report.download_failed()
def test_handle_get_all_failed(self, model, _download, _on_all_failed,
responder):
"""
Three catalog entries.
All (3) failed.
"""
request = Mock(uri='http://content-world.com/content/bear.rpm')
responder.return_value.__enter__.return_value = responder.return_value
report = DownloadReport('', '')
_download.side_effect = SideEffect(PluginNotFound(), DoesNotExist(),
DownloadFailed(report))
catalog = [
Mock(url='url-a'),
Mock(url='url-b'),
Mock(url='url-c'),
]
model.objects.filter.return_value.order_by.return_value.all.return_value = catalog
model.objects.filter.return_value.order_by.return_value.count.return_value = len(
catalog)
# test
streamer = Streamer(Mock())
streamer._handle_get(request)
# validation
model.objects.filter.assert_called_once_with(path='/content/bear.rpm')
model.objects.filter.return_value.order_by.\
assert_called_once_with('-_id', '-revision')
responder.assert_called_once_with(request)
_on_all_failed.assert_called_once_with(request)
self.assertEqual(_download.call_args_list, [
call(request, catalog[0], responder.return_value),
call(request, catalog[1], responder.return_value),
call(request, catalog[2], responder.return_value)
])
def test_handle_get(self, model, _download, _on_succeeded, responder):
"""
Three catalog entries.
The 1st download fails but succeeds on the 2nd.
The 3rd is not tried.
"""
request = Mock(uri='http://content-world.com/content/bear.rpm')
responder.return_value.__enter__.return_value = responder.return_value
report = DownloadReport('', '')
_download.side_effect = SideEffect(DownloadFailed(report), report,
None)
catalog = [
Mock(url='url-a'),
Mock(url='url-b'),
Mock(url='url-c'), # not tried.
]
model.objects.filter.return_value.order_by.return_value.all.return_value = catalog
model.objects.filter.return_value.order_by.return_value.count.return_value = len(
catalog)
# test
streamer = Streamer(Mock())
streamer._handle_get(request)
# validation
model.objects.filter.assert_called_once_with(path='/content/bear.rpm')
model.objects.filter.return_value.order_by.\
assert_called_once_with('-_id', '-revision')
responder.assert_called_once_with(request)
_on_succeeded.assert_called_once_with(catalog[1], request, report)
self.assertEqual(_download.call_args_list, [
call(request, catalog[0], responder.return_value),
call(request, catalog[1], responder.return_value)
])
def test_retrieve_module_missing_module(self, mock_downloader_download, mock_listener_constructor):
# Setup
mock_listener = mock.MagicMock()
report = DownloadReport(None, None)
report.error_msg = 'oops'
mock_listener.failed_reports = [report]
mock_listener_constructor.return_value = mock_listener
# Test
try:
self.downloader.retrieve_module(self.mock_progress_report, self.module)
self.fail()
except exceptions.FileRetrievalException:
expected_filename = web._create_download_tmp_dir(self.working_dir)
expected_filename = os.path.join(expected_filename, self.module.filename())
self.assertFalse(os.path.exists(os.path.join(expected_filename)))
def drain(self):
"""
Read and fail all requests remaining in the queue.
"""
for request in NectarFeed(self):
report = NectarDownloadReport.from_download_request(request)
self.downloader.fire_download_failed(report)
def drain(self):
"""
Read and fail all requests remaining in the queue.
"""
for request in NectarFeed(self):
report = NectarDownloadReport.from_download_request(request)
self.downloader.fire_download_failed(report)
def test__raise_path_error_unathorized(self):
"""
Specifically for a 401, a custom error message should be used explaining that the cause
could be either that the client is unauthorized, or that the resource was not found.
Docker hub responds 401 for the not found case, which is why this function exists.
"""
report = DownloadReport('http://foo/bar', '/a/b/c')
report.error_report = {'response_code': httplib.UNAUTHORIZED}
report.error_msg = 'oops'
with self.assertRaises(IOError) as assertion:
registry.V2Repository._raise_path_error(report)
# not worrying about what the exact contents are; just that the function added its
# own message
self.assertNotEqual(assertion.exception.message, report.error_msg)
self.assertTrue(len(assertion.exception.message) > 0)
def test_retrieve_module_missing_module(self, mock_downloader_download, mock_listener_constructor):
# Setup
self.module.author = 'asdf'
self.module.puppet_standard_filename.return_value = 'puppet-filename.tar.gz'
mock_listener = mock.MagicMock()
report = DownloadReport(None, None)
report.error_msg = 'oops'
mock_listener.failed_reports = [report]
mock_listener_constructor.return_value = mock_listener
# Test
try:
self.downloader.retrieve_module(self.mock_progress_report, self.module)
self.fail()
except exceptions.FileRetrievalException:
expected_filename = web._create_download_tmp_dir(self.working_dir)
expected_filename = os.path.join(expected_filename, self.module.filename())
def test__get_path_failed(self, mock_download_one, mock_request_token):
"""
Test _get_path() for the case when an IOError is raised by the downloader.
"""
name = 'pulp'
download_config = DownloaderConfig(max_concurrent=25)
registry_url = 'https://registry.example.com'
working_dir = '/a/working/dir'
r = registry.V2Repository(name, download_config, registry_url, working_dir)
report = DownloadReport(registry_url + '/some/path', StringIO())
report.error_report['response_code'] = httplib.UNAUTHORIZED
report.state = DownloadReport.DOWNLOAD_FAILED
mock_download_one.return_value = report
# The request will fail because the requested path does not exist
self.assertRaises(IOError, r._get_path, '/some/path')
def test__raise_path_error_unathorized(self):
"""
Specifically for a 401, a custom error message should be used explaining that the cause
could be either that the client is unauthorized, or that the resource was not found.
Docker hub responds 401 for the not found case, which is why this function exists.
"""
report = DownloadReport('http://foo/bar', '/a/b/c')
report.error_report = {'response_code': httplib.UNAUTHORIZED}
report.error_msg = 'oops'
with self.assertRaises(IOError) as assertion:
registry.V2Repository._raise_path_error(report)
# not worrying about what the exact contents are; just that the function added its
# own message
self.assertNotEqual(assertion.exception.message, report.error_msg)
self.assertTrue(len(assertion.exception.message) > 0)
def test_download_failed_during_manifest(self, _logger):
self.iso_sync_run.progress_report._state = SyncProgressReport.STATE_MANIFEST_IN_PROGRESS
url = 'http://www.theonion.com/articles/' + \
'american-airlines-us-airways-merge-to-form-worlds,31302/'
report = DownloadReport(url, '/fake/destination')
report.error_report = {'why': 'because'}
report.error_msg = 'uh oh'
self.iso_sync_run.download_failed(report)
# The manifest_state should be failed
self.assertEqual(self.iso_sync_run.progress_report._state,
SyncProgressReport.STATE_MANIFEST_FAILED)
self.assertEqual(self.iso_sync_run.progress_report.error_message, report.error_report)
self.assertEqual(_logger.error.call_count, 1)
log_msg = _logger.error.mock_calls[0][1][0]
self.assertTrue('uh oh' in log_msg)
def setUp(self):
self.mock_sync = mock.MagicMock()
# this causes validation to be skipped
self.mock_sync.config.get.return_value = False
self.mock_metadata_files = mock.MagicMock()
self.listener = listener.DRPMListener(self.mock_sync,
self.mock_metadata_files)
self.report = DownloadReport('http://pulpproject.org', '/a/b/c')
def test_get_images(self, mock_download_one):
body = json.dumps(['abc123'])
report = DownloadReport('http://pulpproject.org/v1/repositories/pulp/crane/images',
StringIO(body))
report.headers = {}
mock_download_one.return_value = report
ret = self.repo._get_single_path('/v1/repositories/pulp/crane/images')
self.assertEqual(ret, ['abc123'])
self.assertEqual(mock_download_one.call_count, 1)
self.assertTrue(isinstance(mock_download_one.call_args[0][0], DownloadRequest))
req = mock_download_one.call_args[0][0]
self.assertEqual(req.url, 'http://pulpproject.org/v1/repositories/pulp/crane/images')
# make sure this header is set, which is required by the docker API in order
# to give us an auth token
self.assertEqual(req.headers[self.repo.DOCKER_TOKEN_HEADER], 'true')
def test_get_images(self, mock_download_one):
body = json.dumps(['abc123'])
report = DownloadReport('http://pulpproject.org/v1/repositories/pulp/crane/images',
StringIO(body))
report.headers = {}
mock_download_one.return_value = report
ret = self.repo._get_single_path('/v1/repositories/pulp/crane/images')
self.assertEqual(ret, ['abc123'])
self.assertEqual(mock_download_one.call_count, 1)
self.assertTrue(isinstance(mock_download_one.call_args[0][0], DownloadRequest))
req = mock_download_one.call_args[0][0]
self.assertEqual(req.url, 'http://pulpproject.org/v1/repositories/pulp/crane/images')
# make sure this header is set, which is required by the docker API in order
# to give us an auth token
self.assertEqual(req.headers[self.repo.DOCKER_TOKEN_HEADER], 'true')
def test_on_succeeded_pulp_requested(self, _insert_deferred):
entry = Mock(url='url-a')
request = Mock(uri='http://content-world.com/content/bear.rpm')
request.getHeader.side_effect = {
constants.PULP_STREAM_REQUEST_HEADER: True
}.__getitem__
report = DownloadReport('', '')
report.headers = {
'A': 1,
'B': 2,
}
# test
streamer = Streamer(Mock())
streamer._on_succeeded(entry, request, report)
# validation
self.assertFalse(_insert_deferred.called)
def test__get_path_failed(self, mock_download_one, mock_request_token):
"""
Test _get_path() for the case when an IOError is raised by the downloader.
"""
name = 'pulp'
download_config = DownloaderConfig(max_concurrent=25)
registry_url = 'https://registry.example.com'
working_dir = '/a/working/dir'
r = registry.V2Repository(name, download_config, registry_url, working_dir)
report = DownloadReport(registry_url + '/some/path', StringIO())
report.error_report['response_code'] = httplib.UNAUTHORIZED
report.state = DownloadReport.DOWNLOAD_FAILED
report.headers = {}
mock_download_one.return_value = report
# The request will fail because the requested path does not exist
self.assertRaises(IOError, r._get_path, '/some/path')
def _copy(self, request, report=None):
"""
Copy the source file to the destination.
This is the default behavior and most useful for files that live on
different disk partitions or networked file systems.
:param request: request instance
:type request: nectar.request.DownloadRequest
:param report: report instance for the request
:type report: nectar.report.DownloadReport
:return: report instance
:rtype: nectar.report.DownloadReport
"""
report = report or DownloadReport.from_download_request(request)
report.download_started()
src_handle = None
try:
src_path = self._file_path_from_url(request.url)
src_handle = open(src_path, 'rb')
dst_handle = request.initialize_file_handle()
buffer_size = self.buffer_size
self.fire_download_started(report)
last_progress_update = datetime.datetime.now()
while True:
if self.is_canceled or request.canceled:
report.download_canceled()
# NOTE the control flow here will pass through the finally
# block on the way out, but not the else block :D
return report
chunk = src_handle.read(buffer_size)
if not chunk:
break
dst_handle.write(chunk)
report.bytes_downloaded += len(chunk)
now = datetime.datetime.now()
if now - last_progress_update < self.progress_interval:
continue
self.fire_download_progress(report)
last_progress_update = now
except IOError, e:
logger.debug(e)
report.error_msg = str(e)
report.download_failed()
def _copy(self, request, report=None):
"""
Copy the source file to the destination.
This is the default behavior and most useful for files that live on
different disk partitions or networked file systems.
:param request: request instance
:type request: nectar.request.DownloadRequest
:param report: report instance for the request
:type report: nectar.report.DownloadReport
:return: report instance
:rtype: nectar.report.DownloadReport
"""
report = report or DownloadReport.from_download_request(request)
report.download_started()
src_handle = None
try:
src_path = self._file_path_from_url(request.url)
src_handle = open(src_path, 'rb')
dst_handle = request.initialize_file_handle()
buffer_size = self.buffer_size
self.fire_download_started(report)
last_progress_update = datetime.datetime.now()
while True:
if self.is_canceled:
report.download_canceled()
# NOTE the control flow here will pass through the finally
# block on the way out, but not the else block :D
return report
chunk = src_handle.read(buffer_size)
if not chunk:
break
dst_handle.write(chunk)
report.bytes_downloaded += len(chunk)
now = datetime.datetime.now()
if now - last_progress_update < self.progress_interval:
continue
self.fire_download_progress(report)
last_progress_update = now
except Exception, e:
logger.exception(e)
report.error_msg = str(e)
report.download_failed()
def test_retrieve_module_missing_module(self, mock_downloader_download,
mock_listener_constructor):
# Setup
mock_listener = mock.MagicMock()
report = DownloadReport(None, None)
report.error_msg = 'oops'
mock_listener.failed_reports = [report]
mock_listener_constructor.return_value = mock_listener
# Test
try:
self.downloader.retrieve_module(self.mock_progress_report,
self.module)
self.fail()
except exceptions.FileRetrievalException:
expected_filename = web._create_download_tmp_dir(self.working_dir)
expected_filename = os.path.join(expected_filename,
self.module.filename())
self.assertFalse(os.path.exists(os.path.join(expected_filename)))
def test_get_tags_from_endpoint(self, mock_download_one):
body = json.dumps({'latest': 'abc123'})
report = DownloadReport('http://some-endpoint.org/v1/repositories/pulp/crane/tags',
StringIO(body))
report.headers = {}
mock_download_one.return_value = report
self.repo.endpoint = 'some-endpoint.org'
# this lets us test that auth was added to the request
self.repo.token = 'letmein'
ret = self.repo._get_single_path('/v1/repositories/pulp/crane/tags')
self.assertEqual(ret, {'latest': 'abc123'})
self.assertEqual(mock_download_one.call_count, 1)
self.assertTrue(isinstance(mock_download_one.call_args[0][0], DownloadRequest))
req = mock_download_one.call_args[0][0]
self.assertEqual(req.url, 'http://some-endpoint.org/v1/repositories/pulp/crane/tags')
# make sure the authorization was added, which is usually required by an endpoint
self.assertTrue('Authorization' in req.headers)
def test_calls_fetch(self, mock_fetch):
config = DownloaderConfig()
request = DownloadRequest('http://foo', StringIO())
report = DownloadReport.from_download_request(request)
downloader = threaded.HTTPThreadedDownloader(config)
mock_fetch.return_value = report
ret = downloader._download_one(request)
self.assertEqual(mock_fetch.call_count, 1)
self.assertTrue(ret is report)
self.assertTrue(mock_fetch.call_args[0][0] is request)