def test_find_sources(self, fake_manager):
type_id = 'test_1'
unit_key = 1
destination = '/tmp/123'
url = 'http://redhat.com/repository'
primary = PrimarySource(None)
alternatives = dict([(s, ContentSource(s, d)) for s, d in DESCRIPTOR])
fake_manager().find.return_value = CATALOG
# test
request = Request(type_id, unit_key, url, destination)
request.find_sources(primary, alternatives)
# validation
# validate sources sorted by priority with the primary last.
# should only have matched on s-1 and s-3.
request.sources = list(request.sources)
self.assertEqual(len(request.sources), 5)
self.assertEqual(request.sources[0][0].id, 's-3')
self.assertEqual(request.sources[0][1], CATALOG[2][constants.URL])
self.assertEqual(request.sources[1][0].id, 's-3')
self.assertEqual(request.sources[1][1], CATALOG[3][constants.URL])
self.assertEqual(request.sources[2][0].id, 's-1')
self.assertEqual(request.sources[2][1], CATALOG[0][constants.URL])
self.assertEqual(request.sources[3][0].id, 's-1')
self.assertEqual(request.sources[3][1], CATALOG[1][constants.URL])
self.assertEqual(request.sources[4][0].id, primary.id)
self.assertEqual(request.sources[4][1], url)
def test_find_sources(self, fake_manager):
type_id = 'test_1'
unit_key = 1
destination = '/tmp/123'
url = 'http://redhat.com/repository'
primary = PrimarySource(None)
alternatives = dict([(s, ContentSource(s, d)) for s, d in DESCRIPTOR])
fake_manager().find.return_value = CATALOG
# test
request = Request(type_id, unit_key, url, destination)
request.find_sources(primary, alternatives)
# validation
# validate sources sorted by priority with the primary last.
# should only have matched on s-1 and s-3.
self.assertEqual(len(request.sources), 5)
self.assertEqual(request.sources[0][0].id, 's-3')
self.assertEqual(request.sources[0][1], CATALOG[2][constants.URL])
self.assertEqual(request.sources[1][0].id, 's-3')
self.assertEqual(request.sources[1][1], CATALOG[3][constants.URL])
self.assertEqual(request.sources[2][0].id, 's-1')
self.assertEqual(request.sources[2][1], CATALOG[0][constants.URL])
self.assertEqual(request.sources[3][0].id, 's-1')
self.assertEqual(request.sources[3][1], CATALOG[1][constants.URL])
self.assertEqual(request.sources[4][0].id, primary.id)
self.assertEqual(request.sources[4][1], url)
def test_download_to_stream(self):
request_list = []
_dir, cataloged = self.populate_catalog(ORPHANED, 0, 10)
_dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10)
_dir = self.populate_content(PRIMARY, 0, 20)
# unit-world
for n in range(0, 10):
request = Request(
cataloged[n].type_id,
cataloged[n].unit_key,
'file://%s/unit_%d' % (_dir, n),
StringIO())
request_list.append(request)
# primary
for n in range(11, 20):
unit_key = {
'name': 'unit_%d' % n,
'version': '1.0.%d' % n,
'release': '1',
'checksum': str(uuid4())
}
request = Request(
TYPE_ID,
unit_key,
'file://%s/unit_%d' % (_dir, n),
StringIO())
request_list.append(request)
downloader = LocalFileDownloader(DownloaderConfig())
listener = Mock()
container = ContentContainer(path=self.tmp_dir)
container.threaded = False
container.refresh = Mock()
# test
report = container.download(downloader, request_list, listener)
# validation
# unit-world
for i in range(0, 10):
request = request_list[i]
self.assertTrue(request.downloaded)
self.assertEqual(len(request.errors), 0)
fp = request.destination
s = fp.getvalue()
self.assertTrue(UNIT_WORLD in s)
# primary
for i in range(11, len(request_list)):
request = request_list[i]
self.assertTrue(request.downloaded)
self.assertEqual(len(request.errors), 0)
fp = request.destination
s = fp.getvalue()
self.assertTrue(PRIMARY in s)
self.assertEqual(report.total_sources, 2)
self.assertEqual(len(report.downloads), 2)
self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 9)
self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 0)
self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 10)
self.assertEqual(report.downloads[UNIT_WORLD].total_failed, 0)
def test_next_source(self):
sources = [1, 2, 3]
request = Request('', {}, '', '')
request.sources = sources
# test and validation
for i, source in enumerate(request.sources):
self.assertEqual(source, sources[i])
def test_next_source(self):
sources = [1, 2, 3]
request = Request('', {}, '', '')
request.sources = sources
# test and validation
for i, source in enumerate(request.sources):
self.assertEqual(source, sources[i])
def test_download(self, fake_load):
sources = []
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.get_downloader = Mock()
sources.append(s)
fake_load.return_value = sources
request_list = []
for n in range(6):
r = Request('T', {}, 'url-%d' % n, 'path-%d' % n)
r.find_sources = Mock(return_value=sources[n % 3:])
request_list.append(r)
collated = [
{
sources[0]: ['nectar-1'],
sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'],
sources[2]: ['nectar-5', 'nectar-6']
},
{}
]
fake_collated = Mock(side_effect=collated)
fake_listener = Mock()
canceled = FakeEvent()
fake_primary = PrimarySource(Mock())
# test
container = ContentContainer('')
container.refresh = Mock()
container.collated = fake_collated
report = container.download(canceled, fake_primary, request_list, fake_listener)
# validation
container.refresh.assert_called_with(canceled)
for r in request_list:
r.find_sources.assert_called_with(fake_primary, container.sources)
self.assertEqual(report.total_passes, 1)
self.assertEqual(report.total_sources, len(sources))
self.assertEqual(len(report.downloads), 3)
for source in sources:
self.assertEqual(report.downloads[source.id].total_succeeded, 0)
self.assertEqual(report.downloads[source.id].total_failed, 0)
for source in sources:
source.get_downloader.assert_called_with()
downloader = source.get_downloader()
listener = downloader.event_listener
self.assertEqual(listener.cancel_event, canceled)
self.assertEqual(listener.downloader, downloader)
self.assertEqual(listener.listener, fake_listener)
downloader.download.assert_called_with(collated[0][source])
def test_has_source(self):
request = Request('', {}, '', '')
request.sources = [1]
# test and validation
self.assertTrue(request.has_source())
request.index = 1
self.assertFalse(request.has_source())
def test_download(self, fake_load):
sources = []
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.get_downloader = Mock()
sources.append(s)
fake_load.return_value = sources
request_list = []
for n in range(6):
r = Request('T', {}, 'url-%d' % n, 'path-%d' % n)
r.find_sources = Mock(return_value=sources[n % 3:])
request_list.append(r)
collated = [{
sources[0]: ['nectar-1'],
sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'],
sources[2]: ['nectar-5', 'nectar-6']
}, {}]
fake_collated = Mock(side_effect=collated)
fake_listener = Mock()
canceled = FakeEvent()
fake_primary = PrimarySource(Mock())
# test
container = ContentContainer('')
container.refresh = Mock()
container.collated = fake_collated
report = container.download(canceled, fake_primary, request_list,
fake_listener)
# validation
container.refresh.assert_called_with(canceled)
for r in request_list:
r.find_sources.assert_called_with(fake_primary, container.sources)
self.assertEqual(report.total_passes, 1)
self.assertEqual(report.total_sources, len(sources))
self.assertEqual(len(report.downloads), 3)
for source in sources:
self.assertEqual(report.downloads[source.id].total_succeeded, 0)
self.assertEqual(report.downloads[source.id].total_failed, 0)
for source in sources:
source.get_downloader.assert_called_with()
downloader = source.get_downloader()
listener = downloader.event_listener
self.assertEqual(listener.cancel_event, canceled)
self.assertEqual(listener.downloader, downloader)
self.assertEqual(listener.listener, fake_listener)
downloader.download.assert_called_with(collated[0][source])
def test_next_source(self):
request = Request('', {}, '', '')
request.sources = [1, 2, 3]
# test and validation
for n in request.sources:
source = request.next_source()
self.assertEqual(source, n)
self.assertTrue(request.next_source() is None)
def test_next_source(self):
request = Request('', {}, '', '')
request.sources = [1, 2, 3]
# test and validation
for n in request.sources:
source = request.next_source()
self.assertEqual(source, n)
self.assertTrue(request.next_source() is None)
def test_download_canceled_after_collated(self, fake_load):
sources = []
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.get_downloader = Mock()
sources.append(s)
fake_load.return_value = sources
request_list = []
for n in range(6):
r = Request('T', {}, 'url-%d' % n, 'path-%d' % n)
r.find_sources = Mock(return_value=sources[n % 3:])
request_list.append(r)
collated = [
{
sources[0]: ['nectar-1'],
sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'],
sources[2]: ['nectar-5', 'nectar-6']
},
{}
]
fake_collated = Mock(side_effect=collated)
fake_listener = Mock()
canceled = Mock()
canceled.isSet.side_effect = [False, True, True]
fake_primary = PrimarySource(Mock())
# test
container = ContentContainer('')
container.refresh = Mock()
container.collated = fake_collated
report = container.download(canceled, fake_primary, request_list, fake_listener)
# validation
container.refresh.assert_called_with(canceled)
for r in request_list:
r.find_sources.assert_called_with(fake_primary, container.sources)
called = 0
for s in sources:
if s.get_downloader.called:
called += 1
self.assertEqual(called, 1)
self.assertEqual(report.total_passes, 1)
self.assertEqual(report.total_sources, len(sources))
self.assertEqual(len(report.downloads), 1)
self.assertEqual(report.downloads[sources[2].id].total_succeeded, 0)
self.assertEqual(report.downloads[sources[2].id].total_failed, 0)
def test_download_canceled_after_collated(self, fake_load):
sources = []
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.get_downloader = Mock()
sources.append(s)
fake_load.return_value = sources
request_list = []
for n in range(6):
r = Request('T', {}, 'url-%d' % n, 'path-%d' % n)
r.find_sources = Mock(return_value=sources[n % 3:])
request_list.append(r)
collated = [{
sources[0]: ['nectar-1'],
sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'],
sources[2]: ['nectar-5', 'nectar-6']
}, {}]
fake_collated = Mock(side_effect=collated)
fake_listener = Mock()
canceled = Mock()
canceled.isSet.side_effect = [False, True, True]
fake_primary = PrimarySource(Mock())
# test
container = ContentContainer('')
container.refresh = Mock()
container.collated = fake_collated
report = container.download(canceled, fake_primary, request_list,
fake_listener)
# validation
container.refresh.assert_called_with(canceled)
for r in request_list:
r.find_sources.assert_called_with(fake_primary, container.sources)
called = 0
for s in sources:
if s.get_downloader.called:
called += 1
self.assertEqual(called, 1)
self.assertEqual(report.total_passes, 1)
self.assertEqual(report.total_sources, len(sources))
self.assertEqual(len(report.downloads), 1)
self.assertEqual(report.downloads[sources[2].id].total_succeeded, 0)
self.assertEqual(report.downloads[sources[2].id].total_failed, 0)
def test_on_succeeded(self):
request = Request('T1', {'A': 1}, 'http://test', '/tmp/test')
request.data = Mock()
content_listener = Mock()
# test
listener = ContainerListener(content_listener)
listener.on_succeeded(request)
# validation
calls = content_listener.download_succeeded.mock_calls
self.assertEqual(len(calls), 1)
report = calls[0][1][0]
self.assertEqual(report.url, request.url)
self.assertEqual(report.destination, request.destination)
self.assertEqual(report.data, request.data)
def test_download_succeeded(self):
request = Request("T1", {"A": 1}, "http://test", "/tmp/test")
request.data = Mock()
content_listener = Mock()
# test
listener = ContainerListener(content_listener)
listener.download_succeeded(request)
# validation
calls = content_listener.download_succeeded.mock_calls
self.assertEqual(len(calls), 1)
report = calls[0][1][0]
self.assertEqual(report.url, request.url)
self.assertEqual(report.destination, request.destination)
self.assertEqual(report.data, request.data)
def get_requests(self):
"""
Get requests for the units requested to be downloaded.
:return: An iterable of: Request
:rtype: iterable
"""
for unit in self.units:
base_url = unit.base_url or self.base_url
url = self.url_modify(base_url, path_append=unit.download_path)
destination = os.path.join(self.dst_dir, unit.filename)
request = Request(type_id=unit.type_id,
unit_key=unit.unit_key,
url=url,
destination=destination)
request.data = unit
yield request
def get_requests(self):
"""
Get requests for the units requested to be downloaded.
:return: An iterable of: Request
:rtype: iterable
"""
for unit in self.units:
base_url = unit.base_url or self.base_url
url = self.url_modify(base_url, path_append=unit.filename)
destination = os.path.join(self.dst_dir, unit.filename)
request = Request(
type_id=unit.type_id,
unit_key=unit.unit_key,
url=url,
destination=destination)
request.data = unit
yield request
def test_on_failed(self):
request = Request("T1", {"A": 1}, "http://test", "/tmp/test")
request.data = Mock()
request.errors = [1, 2, 3]
content_listener = Mock()
# test
listener = ContainerListener(content_listener)
listener.on_failed(request)
# validation
calls = content_listener.download_failed.mock_calls
self.assertEqual(len(calls), 1)
report = calls[0][1][0]
self.assertEqual(report.url, request.url)
self.assertEqual(report.destination, request.destination)
self.assertEqual(report.data, request.data)
self.assertEqual(report.error_report["errors"], request.errors)
def test_download_failed(self):
request = Request('T1', {'A': 1}, 'http://test', '/tmp/test')
request.data = Mock()
request.errors = [1, 2, 3]
content_listener = Mock()
# test
listener = ContainerListener(content_listener)
listener.download_failed(request)
# validation
calls = content_listener.download_failed.mock_calls
self.assertEqual(len(calls), 1)
report = calls[0][1][0]
self.assertEqual(report.url, request.url)
self.assertEqual(report.destination, request.destination)
self.assertEqual(report.data, request.data)
self.assertEqual(report.error_report['errors'], request.errors)
def get_requests(self):
"""
Get requests for the units requested to be downloaded.
:return: An iterable of: Request
:rtype: iterable
"""
for unit in self.units:
if unit.metadata.get('base_url'):
url = urljoin(unit.metadata.get('base_url'), unit.download_path)
else:
url = urljoin(self.base_url, unit.download_path)
file_name = os.path.basename(unit.relative_path)
destination = os.path.join(self.dst_dir, file_name)
request = Request(
type_id=unit.TYPE,
unit_key=unit.unit_key,
url=url,
destination=destination)
request.data = unit
yield request
def create_request(url, destination, unit, unit_ref):
"""
Create a content container download request that is compatible with the listener.
The destination directory is created as needed.
:param url: The download URL.
:type url: str
:param destination: The absolute path to where the file is to be downloaded.
:type destination: str
:param unit: A published content unit.
:type unit: dict
:param unit_ref: A reference to the unit association.
:type unit_ref: pulp_node.manifest.UnitRef.
:return: A download request.
:rtype: Request
"""
data = {STORAGE_PATH: unit[constants.STORAGE_PATH], UNIT_REF: unit_ref}
dir_path = os.path.dirname(destination)
pathlib.mkdir(dir_path)
request = Request(unit[constants.TYPE_ID], unit[constants.UNIT_KEY],
url, destination)
request.data = data
return request
def create_request(url, destination, unit, unit_ref):
"""
Create a content container download request that is compatible with the listener.
The destination directory is created as needed.
:param url: The download URL.
:type url: str
:param destination: The absolute path to where the file is to be downloaded.
:type destination: str
:param unit: A published content unit.
:type unit: dict
:param unit_ref: A reference to the unit association.
:type unit_ref: pulp_node.manifest.UnitRef.
:return: A download request.
:rtype: Request
"""
data = {
STORAGE_PATH: unit[constants.STORAGE_PATH],
UNIT_REF: unit_ref
}
dir_path = os.path.dirname(destination)
pathlib.mkdir(dir_path)
request = Request(unit[constants.TYPE_ID], unit[constants.UNIT_KEY], url, destination)
request.data = data
return request
def test_has_source(self):
request = Request('', {}, '', '')
request.sources = [1]
# test and validation
self.assertTrue(request.has_source())
request.index = 1
self.assertFalse(request.has_source())
def test_construction(self):
type_id = 'test_1'
unit_key = {'name': 'A', 'version': '1.0'}
destination = '/tmp/123'
url = 'http://redhat.com/repository'
# test
request = Request(type_id, unit_key, url, destination)
# validation
self.assertEqual(request.type_id, type_id)
self.assertEqual(request.unit_key, unit_key)
self.assertEqual(request.url, url)
self.assertEqual(request.destination, destination)
self.assertFalse(request.downloaded)
self.assertEqual(request.sources, [])
self.assertEqual(request.index, 0)
self.assertEqual(request.errors, [])
self.assertEqual(request.data, None)
def __init__(self, *args, **kwargs):
DownloadRequest.__init__(self, *args, **kwargs)
self.url = 'http:/NOWHERE/FAIL_ME_%f' % random.random()
def __init__(self, *args, **kwargs):
DownloadRequest.__init__(self, *args, **kwargs)
self.url = 'http:/NOWHERE/FAIL_ME_%f' % random.random()
def test_download_with_errors(self):
request_list = []
_dir, cataloged = self.populate_catalog(ORPHANED, 0, 10)
_dir, cataloged = self.populate_catalog(UNDERGROUND, 0, 10)
_dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10)
shutil.rmtree(_dir)
_dir = self.populate_content(PRIMARY, 0, 20)
# unit-world
for n in range(0, 10):
request = Request(
cataloged[n].type_id,
cataloged[n].unit_key,
'file://%s/unit_%d' % (_dir, n),
os.path.join(self.downloaded, 'unit_%d' % n))
request_list.append(request)
# primary
for n in range(11, 20):
unit_key = {
'name': 'unit_%d' % n,
'version': '1.0.%d' % n,
'release': '1',
'checksum': str(uuid4())
}
request = Request(
TYPE_ID,
unit_key,
'file://%s/unit_%d' % (_dir, n),
os.path.join(self.downloaded, 'unit_%d' % n))
request_list.append(request)
downloader = LocalFileDownloader(DownloaderConfig())
listener = Mock()
container = ContentContainer(path=self.tmp_dir)
container.refresh = Mock()
# test
report = container.download(downloader, request_list, listener)
# validation
# unit-world
for i in range(0, 10):
request = request_list[i]
self.assertTrue(request.downloaded, msg='URL: %s' % request.url)
self.assertEqual(len(request.errors), 1)
with open(request.destination) as fp:
s = fp.read()
self.assertTrue(UNDERGROUND in s)
# primary
for i in range(11, len(request_list)):
request = request_list[i]
self.assertTrue(request.downloaded, msg='URL: %s' % request.url)
self.assertEqual(len(request.errors), 0)
with open(request.destination) as fp:
s = fp.read()
self.assertTrue(PRIMARY in s)
self.assertEqual(report.total_sources, 2)
self.assertEqual(len(report.downloads), 3)
self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 9)
self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 0)
self.assertEqual(report.downloads[UNDERGROUND].total_succeeded, 10)
self.assertEqual(report.downloads[UNDERGROUND].total_failed, 0)
self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 0)
self.assertEqual(report.downloads[UNIT_WORLD].total_failed, 10)
