def test_refresh_cancel_in_plugin(self, mock_plugin, *unused):
container = ContentContainer(path=self.tmp_dir)
event = CancelEvent(3)
report = container.refresh(event, force=True)
plugin = mock_plugin.return_value[0]
self.assertEqual(plugin.refresh.call_count, 1)
self.assertEqual(len(report), 1)
def test_download_cancelled_in_failed(self, mock_started, mock_cancel):
request_list = []
for n in range(0, 5):
unit_key = {
'name': 'unit_%d' % n,
'version': '1.0.%d' % n,
'release': '1',
'checksum': str(uuid4())
}
request = Request(
TYPE_ID,
unit_key,
'http://unit-city/unit_%d' % n,
os.path.join(self.downloaded, 'unit_%d' % n))
request_list.append(request)
downloader = HTTPThreadedDownloader(DownloaderConfig())
container = ContentContainer(path=self.tmp_dir)
container.refresh = Mock()
event = CancelEvent(2)
report = container.download(event, downloader, request_list)
self.assertTrue(mock_started.called)
self.assertTrue(mock_cancel.called)
self.assertEqual(report.total_passes, 1)
self.assertEqual(report.total_sources, 2)
self.assertEqual(len(report.downloads), 1)
self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 0)
self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 5)
def test_download_fail_completely(self):
request_list = []
_dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10)
shutil.rmtree(_dir)
_dir = self.populate_content(PRIMARY, 0, 20)
# primary
for n in range(0, 10):
unit_key = {
'name': 'unit_%d' % n,
'version': '1.0.%d' % n,
'release': '1',
'checksum': str(uuid4())
}
request = Request(
TYPE_ID,
unit_key,
'http://redhat.com/%s/unit_%d' % (_dir, n),
os.path.join(self.downloaded, 'unit_%d' % n))
request_list.append(request)
downloader = HTTPThreadedDownloader(DownloaderConfig())
listener = MockListener()
container = ContentContainer(path=self.tmp_dir)
container.refresh = Mock()
event = Event()
container.download(event, downloader, request_list, listener)
# primary
for i in range(0, len(request_list)):
request = request_list[i]
self.assertFalse(request.downloaded)
self.assertEqual(len(request.errors), 1)
self.assertEqual(listener.download_started.call_count, len(request_list))
self.assertEqual(listener.download_succeeded.call_count, 0)
self.assertEqual(listener.download_failed.call_count, len(request_list))
def test_refresh_cancel_in_plugin(self, mock_plugin, *unused):
container = ContentContainer(path=self.tmp_dir)
event = CancelEvent(3)
report = container.refresh(event, force=True)
plugin = mock_plugin.return_value[0]
self.assertEqual(plugin.refresh.call_count, 1)
self.assertEqual(len(report), 1)
def test_download_cancelled_in_failed(self, mock_started, mock_cancel):
request_list = []
for n in range(0, 5):
unit_key = {
'name': 'unit_%d' % n,
'version': '1.0.%d' % n,
'release': '1',
'checksum': str(uuid4())
}
request = Request(TYPE_ID, unit_key,
'http://unit-city/unit_%d' % n,
os.path.join(self.downloaded, 'unit_%d' % n))
request_list.append(request)
downloader = HTTPThreadedDownloader(DownloaderConfig())
container = ContentContainer(path=self.tmp_dir)
container.refresh = Mock()
event = CancelEvent(2)
report = container.download(event, downloader, request_list)
self.assertTrue(mock_started.called)
self.assertTrue(mock_cancel.called)
self.assertEqual(report.total_passes, 1)
self.assertEqual(report.total_sources, 2)
self.assertEqual(len(report.downloads), 1)
self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 0)
self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 5)
def test_download_with_errors(self):
request_list = []
_dir, cataloged = self.populate_catalog(ORPHANED, 0, 10)
_dir, cataloged = self.populate_catalog(UNDERGROUND, 0, 10)
_dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10)
shutil.rmtree(_dir)
_dir = self.populate_content(PRIMARY, 0, 20)
# unit-world
for n in range(0, 10):
request = Request(cataloged[n].type_id, cataloged[n].unit_key,
'file://%s/unit_%d' % (_dir, n),
os.path.join(self.downloaded, 'unit_%d' % n))
request_list.append(request)
# primary
for n in range(11, 20):
unit_key = {
'name': 'unit_%d' % n,
'version': '1.0.%d' % n,
'release': '1',
'checksum': str(uuid4())
}
request = Request(TYPE_ID, unit_key,
'file://%s/unit_%d' % (_dir, n),
os.path.join(self.downloaded, 'unit_%d' % n))
request_list.append(request)
downloader = LocalFileDownloader(DownloaderConfig())
listener = Mock()
container = ContentContainer(path=self.tmp_dir)
container.refresh = Mock()
event = Event()
# test
report = container.download(event, downloader, request_list, listener)
# validation
# unit-world
for i in range(0, 10):
request = request_list[i]
self.assertTrue(request.downloaded, msg='URL: %s' % request.url)
self.assertEqual(len(request.errors), 1)
with open(request.destination) as fp:
s = fp.read()
self.assertTrue(UNDERGROUND in s)
# primary
for i in range(11, len(request_list)):
request = request_list[i]
self.assertTrue(request.downloaded, msg='URL: %s' % request.url)
self.assertEqual(len(request.errors), 0)
with open(request.destination) as fp:
s = fp.read()
self.assertTrue(PRIMARY in s)
self.assertEqual(report.total_sources, 2)
self.assertEqual(len(report.downloads), 3)
self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 9)
self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 0)
self.assertEqual(report.downloads[UNDERGROUND].total_succeeded, 10)
self.assertEqual(report.downloads[UNDERGROUND].total_failed, 0)
self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 0)
self.assertEqual(report.downloads[UNIT_WORLD].total_failed, 10)
def test_download(self):
request_list = []
_dir, cataloged = self.populate_catalog(ORPHANED, 0, 10)
_dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10)
_dir = self.populate_content(PRIMARY, 0, 20)
# unit-world
for n in range(0, 10):
request = Request(
cataloged[n].type_id,
cataloged[n].unit_key,
'file://%s/unit_%d' % (_dir, n),
os.path.join(self.downloaded, 'unit_%d' % n))
request_list.append(request)
# primary
for n in range(11, 20):
unit_key = {
'name': 'unit_%d' % n,
'version': '1.0.%d' % n,
'release': '1',
'checksum': str(uuid4())
}
request = Request(
TYPE_ID,
unit_key,
'file://%s/unit_%d' % (_dir, n),
os.path.join(self.downloaded, 'unit_%d' % n))
request_list.append(request)
downloader = LocalFileDownloader(DownloaderConfig())
listener = MockListener()
container = ContentContainer(path=self.tmp_dir)
container.refresh = Mock()
event = Event()
report = container.download(event, downloader, request_list, listener)
# unit-world
for i in range(0, 10):
request = request_list[i]
self.assertTrue(request.downloaded)
self.assertEqual(len(request.errors), 0)
with open(request.destination) as fp:
s = fp.read()
self.assertTrue(UNIT_WORLD in s)
# primary
for i in range(11, len(request_list)):
request = request_list[i]
self.assertTrue(request.downloaded)
self.assertEqual(len(request.errors), 0)
with open(request.destination) as fp:
s = fp.read()
self.assertTrue(PRIMARY in s)
self.assertEqual(listener.download_started.call_count, len(request_list))
self.assertEqual(listener.download_succeeded.call_count, len(request_list))
self.assertEqual(listener.download_failed.call_count, 0)
self.assertEqual(report.total_passes, 1)
self.assertEqual(report.total_sources, 2)
self.assertEqual(len(report.downloads), 2)
self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 9)
self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 0)
self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 10)
self.assertEqual(report.downloads[UNIT_WORLD].total_failed, 0)
def test_download_with_errors(self):
request_list = []
_dir, cataloged = self.populate_catalog(ORPHANED, 0, 10)
_dir, cataloged = self.populate_catalog(UNDERGROUND, 0, 10)
_dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10)
shutil.rmtree(_dir)
_dir = self.populate_content(PRIMARY, 0, 20)
# unit-world
for n in range(0, 10):
request = Request(
cataloged[n].type_id,
cataloged[n].unit_key,
"file://%s/unit_%d" % (_dir, n),
os.path.join(self.downloaded, "unit_%d" % n),
)
request_list.append(request)
# primary
for n in range(11, 20):
unit_key = {"name": "unit_%d" % n, "version": "1.0.%d" % n, "release": "1", "checksum": str(uuid4())}
request = Request(
TYPE_ID, unit_key, "file://%s/unit_%d" % (_dir, n), os.path.join(self.downloaded, "unit_%d" % n)
)
request_list.append(request)
downloader = LocalFileDownloader(DownloaderConfig())
listener = Mock()
container = ContentContainer(path=self.tmp_dir)
container.refresh = Mock()
event = Event()
# test
report = container.download(event, downloader, request_list, listener)
# validation
# unit-world
for i in range(0, 10):
request = request_list[i]
self.assertTrue(request.downloaded, msg="URL: %s" % request.url)
self.assertEqual(len(request.errors), 1)
with open(request.destination) as fp:
s = fp.read()
self.assertTrue(UNDERGROUND in s)
# primary
for i in range(11, len(request_list)):
request = request_list[i]
self.assertTrue(request.downloaded, msg="URL: %s" % request.url)
self.assertEqual(len(request.errors), 0)
with open(request.destination) as fp:
s = fp.read()
self.assertTrue(PRIMARY in s)
self.assertEqual(report.total_sources, 2)
self.assertEqual(len(report.downloads), 3)
self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 9)
self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 0)
self.assertEqual(report.downloads[UNDERGROUND].total_succeeded, 10)
self.assertEqual(report.downloads[UNDERGROUND].total_failed, 0)
self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 0)
self.assertEqual(report.downloads[UNIT_WORLD].total_failed, 10)
def test_refresh_exception(self, mock_refresh):
container = ContentContainer(path=self.tmp_dir)
event = Event()
report = container.refresh(event, force=True)
self.assertEqual(len(report), 2)
for r in report:
self.assertFalse(r.succeeded)
self.assertEqual(r.added_count, 0)
self.assertEqual(r.deleted_count, 0)
self.assertEqual(len(r.errors), 1)
collection = ContentCatalog.get_collection()
self.assertEqual(mock_refresh.call_count, 2)
self.assertEqual(collection.find().count(), 0)
def test_refresh_exception(self, mock_refresh):
container = ContentContainer(path=self.tmp_dir)
event = Event()
report = container.refresh(event, force=True)
self.assertEqual(len(report), 2)
for r in report:
self.assertFalse(r.succeeded)
self.assertEqual(r.added_count, 0)
self.assertEqual(r.deleted_count, 0)
self.assertEqual(len(r.errors), 1)
collection = ContentCatalog.get_collection()
self.assertEqual(mock_refresh.call_count, 2)
self.assertEqual(collection.find().count(), 0)
def test_refresh_failure(self, mock_plugin):
container = ContentContainer(path=self.tmp_dir)
event = Event()
report = container.refresh(event, force=True)
self.assertEqual(len(report), 5)
for r in report:
self.assertFalse(r.succeeded)
self.assertEqual(r.added_count, 0)
self.assertEqual(r.deleted_count, 0)
self.assertEqual(len(r.errors), 1)
plugin = mock_plugin.return_value[0]
collection = ContentCatalog.get_collection()
self.assertEqual(plugin.refresh.call_count, 5)
self.assertEqual(collection.find().count(), 0)
def test_refresh_failure(self, mock_plugin):
container = ContentContainer(path=self.tmp_dir)
event = Event()
report = container.refresh(event, force=True)
self.assertEqual(len(report), 5)
for r in report:
self.assertFalse(r.succeeded)
self.assertEqual(r.added_count, 0)
self.assertEqual(r.deleted_count, 0)
self.assertEqual(len(r.errors), 1)
plugin = mock_plugin.return_value[0]
collection = ContentCatalog.get_collection()
self.assertEqual(plugin.refresh.call_count, 5)
self.assertEqual(collection.find().count(), 0)
def test_download_with_errors(self):
request_list = []
_dir, cataloged = self.populate_catalog(ORPHANED, 0, 1000)
_dir, cataloged = self.populate_catalog(UNDERGROUND, 0, 1000)
_dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 1000)
shutil.rmtree(_dir)
_dir = self.populate_content(PRIMARY, 0, 2000)
# unit-world
for n in range(0, 1000):
request = Request(
cataloged[n].type_id,
cataloged[n].unit_key,
'file://%s/unit_%d' % (_dir, n),
os.path.join(self.downloaded, 'unit_%d' % n))
request_list.append(request)
# primary
for n in range(1001, 2000):
unit_key = {
'name': 'unit_%d' % n,
'version': '1.0.%d' % n,
'release': '1',
'checksum': str(uuid4())
}
request = Request(
TYPE_ID,
unit_key,
'file://%s/unit_%d' % (_dir, n),
os.path.join(self.downloaded, 'unit_%d' % n))
request_list.append(request)
downloader = LocalFileDownloader(DownloaderConfig())
event = Event()
threshold = len(request_list) * 0.10 # cancel after 10% started
listener = TestListener(event, threshold)
container = ContentContainer(path=self.tmp_dir)
container.refresh = Mock()
# test
report = container.download(event, downloader, request_list, listener)
# validation
self.assertEqual(report.total_sources, 2)
self.assertEqual(len(report.downloads), 2)
self.assertTrue(0 < report.downloads[UNDERGROUND].total_succeeded < 500)
self.assertEqual(report.downloads[UNDERGROUND].total_failed, 0)
self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 0)
self.assertTrue(0 < report.downloads[UNIT_WORLD].total_failed < 1000)
def test_download_with_unsupported_url(self):
request_list = []
_dir, cataloged = self.populate_catalog(UNSUPPORTED_PROTOCOL, 0, 10)
_dir = self.populate_content(PRIMARY, 0, 20)
# unit-world
for n in range(0, 10):
request = Request(cataloged[n].type_id, cataloged[n].unit_key,
'file://%s/unit_%d' % (_dir, n),
os.path.join(self.downloaded, 'unit_%d' % n))
request_list.append(request)
# primary
for n in range(11, 20):
unit_key = {
'name': 'unit_%d' % n,
'version': '1.0.%d' % n,
'release': '1',
'checksum': str(uuid4())
}
request = Request(TYPE_ID, unit_key,
'file://%s/unit_%d' % (_dir, n),
os.path.join(self.downloaded, 'unit_%d' % n))
request_list.append(request)
downloader = LocalFileDownloader(DownloaderConfig())
listener = MockListener()
container = ContentContainer(path=self.tmp_dir)
container.refresh = Mock()
event = Event()
report = container.download(event, downloader, request_list, listener)
for i in range(0, len(request_list)):
request = request_list[i]
self.assertTrue(request.downloaded)
self.assertEqual(len(request.errors), 0)
with open(request.destination) as fp:
s = fp.read()
self.assertTrue(PRIMARY in s)
self.assertEqual(listener.download_started.call_count,
len(request_list))
self.assertEqual(listener.download_succeeded.call_count,
len(request_list))
self.assertEqual(listener.download_failed.call_count, 0)
self.assertEqual(report.total_passes, 1)
self.assertEqual(report.total_sources, 2)
self.assertEqual(len(report.downloads), 1)
self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 19)
self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 0)
def test_refresh(self, mock_plugin):
container = ContentContainer(path=self.tmp_dir)
event = Event()
report = container.refresh(event, force=True)
plugin = mock_plugin.return_value[0]
self.assertEqual(plugin.refresh.call_count, 5)
self.assertEqual(len(report), 5)
for r in report:
self.assertTrue(r.succeeded)
self.assertEqual(r.added_count, 100)
self.assertEqual(r.deleted_count, 0)
calls = iter(plugin.refresh.call_args_list)
for source in ContentSource.load_all(self.tmp_dir).values():
for url in source.urls:
args = calls.next()[0]
self.assertTrue(isinstance(args[0], CatalogerConduit))
self.assertEqual(args[1], source.descriptor)
self.assertEqual(args[2], url)
def test_refresh(self, mock_plugin):
container = ContentContainer(path=self.tmp_dir)
event = Event()
report = container.refresh(event, force=True)
plugin = mock_plugin.return_value[0]
self.assertEqual(plugin.refresh.call_count, 5)
self.assertEqual(len(report), 5)
for r in report:
self.assertTrue(r.succeeded)
self.assertEqual(r.added_count, 100)
self.assertEqual(r.deleted_count, 0)
calls = iter(plugin.refresh.call_args_list)
for source in ContentSource.load_all(self.tmp_dir).values():
for url in source.urls:
args = calls.next()[0]
self.assertTrue(isinstance(args[0], CatalogerConduit))
self.assertEqual(args[1], source.descriptor)
self.assertEqual(args[2], url)
def test_download_with_errors(self):
request_list = []
_dir, cataloged = self.populate_catalog(ORPHANED, 0, 1000)
_dir, cataloged = self.populate_catalog(UNDERGROUND, 0, 1000)
_dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 1000)
shutil.rmtree(_dir)
_dir = self.populate_content(PRIMARY, 0, 2000)
# unit-world
for n in range(0, 1000):
request = Request(cataloged[n].type_id, cataloged[n].unit_key,
'file://%s/unit_%d' % (_dir, n),
os.path.join(self.downloaded, 'unit_%d' % n))
request_list.append(request)
# primary
for n in range(1001, 2000):
unit_key = {
'name': 'unit_%d' % n,
'version': '1.0.%d' % n,
'release': '1',
'checksum': str(uuid4())
}
request = Request(TYPE_ID, unit_key,
'file://%s/unit_%d' % (_dir, n),
os.path.join(self.downloaded, 'unit_%d' % n))
request_list.append(request)
downloader = LocalFileDownloader(DownloaderConfig())
event = Event()
threshold = len(request_list) * 0.10 # cancel after 10% started
listener = TestListener(event, threshold)
container = ContentContainer(path=self.tmp_dir)
container.refresh = Mock()
# test
report = container.download(event, downloader, request_list, listener)
# validation
self.assertEqual(report.total_sources, 2)
self.assertEqual(len(report.downloads), 2)
self.assertTrue(
0 < report.downloads[UNDERGROUND].total_succeeded < 500)
self.assertEqual(report.downloads[UNDERGROUND].total_failed, 0)
self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 0)
self.assertTrue(0 < report.downloads[UNIT_WORLD].total_failed < 1000)
def test_download_cancelled_in_started(self, mock_cancel):
request_list = []
_dir = self.populate_content(PRIMARY, 0, 5)
for n in range(0, 5):
unit_key = {
'name': 'unit_%d' % n,
'version': '1.0.%d' % n,
'release': '1',
'checksum': str(uuid4())
}
request = Request(
TYPE_ID,
unit_key,
'file://%s/unit_%d' % (_dir, n),
os.path.join(self.downloaded, 'unit_%d' % n))
request_list.append(request)
downloader = LocalFileDownloader(DownloaderConfig())
container = ContentContainer(path=self.tmp_dir)
container.refresh = Mock()
event = CancelEvent(2)
container.download(event, downloader, request_list)
self.assertTrue(mock_cancel.called)
def test_download_fail_completely(self):
request_list = []
_dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10)
shutil.rmtree(_dir)
_dir = self.populate_content(PRIMARY, 0, 20)
# primary
for n in range(0, 10):
unit_key = {
'name': 'unit_%d' % n,
'version': '1.0.%d' % n,
'release': '1',
'checksum': str(uuid4())
}
request = Request(TYPE_ID, unit_key,
'http://redhat.com/%s/unit_%d' % (_dir, n),
os.path.join(self.downloaded, 'unit_%d' % n))
request_list.append(request)
downloader = HTTPThreadedDownloader(DownloaderConfig())
listener = MockListener()
container = ContentContainer(path=self.tmp_dir)
container.refresh = Mock()
event = Event()
report = container.download(event, downloader, request_list, listener)
# primary
for i in range(0, len(request_list)):
request = request_list[i]
self.assertFalse(request.downloaded)
self.assertEqual(len(request.errors), 1)
self.assertEqual(listener.download_started.call_count,
len(request_list))
self.assertEqual(listener.download_succeeded.call_count, 0)
self.assertEqual(listener.download_failed.call_count,
len(request_list))
self.assertEqual(report.total_passes, 1)
self.assertEqual(report.total_sources, 2)
self.assertEqual(len(report.downloads), 1)
self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 0)
self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 10)
def test_download(self):
request_list = []
_dir, cataloged = self.populate_catalog(ORPHANED, 0, 1000)
_dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 1000)
_dir = self.populate_content(PRIMARY, 0, 2000)
# unit-world
for n in range(0, 1000):
request = Request(
cataloged[n].type_id,
cataloged[n].unit_key,
"file://%s/unit_%d" % (_dir, n),
os.path.join(self.downloaded, "unit_%d" % n),
)
request_list.append(request)
# primary
for n in range(1001, 2000):
unit_key = {"name": "unit_%d" % n, "version": "1.0.%d" % n, "release": "1", "checksum": str(uuid4())}
request = Request(
TYPE_ID, unit_key, "file://%s/unit_%d" % (_dir, n), os.path.join(self.downloaded, "unit_%d" % n)
)
request_list.append(request)
event = Event()
threshold = len(request_list) * 0.80 # cancel after 80% started
downloader = LocalFileDownloader(DownloaderConfig())
listener = TestListener(event, threshold)
container = ContentContainer(path=self.tmp_dir)
container.refresh = Mock()
# test
report = container.download(event, downloader, request_list, listener)
# validation
self.assertEqual(report.total_sources, 2)
self.assertEqual(len(report.downloads), 2)
self.assertTrue(0 < report.downloads[PRIMARY_ID].total_succeeded < 999)
self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 0)
self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 1000)
self.assertEqual(report.downloads[UNIT_WORLD].total_failed, 0)