def test_download(self, fake_load):
sources = []
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.get_downloader = Mock()
sources.append(s)
fake_load.return_value = sources
request_list = []
for n in range(6):
r = Request('T', {}, 'url-%d' % n, 'path-%d' % n)
r.find_sources = Mock(return_value=sources[n % 3:])
request_list.append(r)
collated = [
{
sources[0]: ['nectar-1'],
sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'],
sources[2]: ['nectar-5', 'nectar-6']
},
{}
]
fake_collated = Mock(side_effect=collated)
fake_listener = Mock()
canceled = FakeEvent()
fake_primary = PrimarySource(Mock())
# test
container = ContentContainer('')
container.refresh = Mock()
container.collated = fake_collated
report = container.download(canceled, fake_primary, request_list, fake_listener)
# validation
container.refresh.assert_called_with(canceled)
for r in request_list:
r.find_sources.assert_called_with(fake_primary, container.sources)
self.assertEqual(report.total_passes, 1)
self.assertEqual(report.total_sources, len(sources))
self.assertEqual(len(report.downloads), 3)
for source in sources:
self.assertEqual(report.downloads[source.id].total_succeeded, 0)
self.assertEqual(report.downloads[source.id].total_failed, 0)
for source in sources:
source.get_downloader.assert_called_with()
downloader = source.get_downloader()
listener = downloader.event_listener
self.assertEqual(listener.cancel_event, canceled)
self.assertEqual(listener.downloader, downloader)
self.assertEqual(listener.listener, fake_listener)
downloader.download.assert_called_with(collated[0][source])
def test_download(self, fake_load):
sources = []
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.get_downloader = Mock()
sources.append(s)
fake_load.return_value = sources
request_list = []
for n in range(6):
r = Request('T', {}, 'url-%d' % n, 'path-%d' % n)
r.find_sources = Mock(return_value=sources[n % 3:])
request_list.append(r)
collated = [{
sources[0]: ['nectar-1'],
sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'],
sources[2]: ['nectar-5', 'nectar-6']
}, {}]
fake_collated = Mock(side_effect=collated)
fake_listener = Mock()
canceled = FakeEvent()
fake_primary = PrimarySource(Mock())
# test
container = ContentContainer('')
container.refresh = Mock()
container.collated = fake_collated
report = container.download(canceled, fake_primary, request_list,
fake_listener)
# validation
container.refresh.assert_called_with(canceled)
for r in request_list:
r.find_sources.assert_called_with(fake_primary, container.sources)
self.assertEqual(report.total_passes, 1)
self.assertEqual(report.total_sources, len(sources))
self.assertEqual(len(report.downloads), 3)
for source in sources:
self.assertEqual(report.downloads[source.id].total_succeeded, 0)
self.assertEqual(report.downloads[source.id].total_failed, 0)
for source in sources:
source.get_downloader.assert_called_with()
downloader = source.get_downloader()
listener = downloader.event_listener
self.assertEqual(listener.cancel_event, canceled)
self.assertEqual(listener.downloader, downloader)
self.assertEqual(listener.listener, fake_listener)
downloader.download.assert_called_with(collated[0][source])
def test_collated_all_downloaded(self):
request_list = []
for n in range(10):
r = Mock()
r.downloaded = True
request_list.append(r)
# test
collated = ContentContainer.collated(request_list)
# validation
self.assertEqual(len(collated), 0)
def test_download_canceled_after_collated(self, fake_load):
sources = []
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.get_downloader = Mock()
sources.append(s)
fake_load.return_value = sources
request_list = []
for n in range(6):
r = Request('T', {}, 'url-%d' % n, 'path-%d' % n)
r.find_sources = Mock(return_value=sources[n % 3:])
request_list.append(r)
collated = [
{
sources[0]: ['nectar-1'],
sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'],
sources[2]: ['nectar-5', 'nectar-6']
},
{}
]
fake_collated = Mock(side_effect=collated)
fake_listener = Mock()
canceled = Mock()
canceled.isSet.side_effect = [False, True, True]
fake_primary = PrimarySource(Mock())
# test
container = ContentContainer('')
container.refresh = Mock()
container.collated = fake_collated
report = container.download(canceled, fake_primary, request_list, fake_listener)
# validation
container.refresh.assert_called_with(canceled)
for r in request_list:
r.find_sources.assert_called_with(fake_primary, container.sources)
called = 0
for s in sources:
if s.get_downloader.called:
called += 1
self.assertEqual(called, 1)
self.assertEqual(report.total_passes, 1)
self.assertEqual(report.total_sources, len(sources))
self.assertEqual(len(report.downloads), 1)
self.assertEqual(report.downloads[sources[2].id].total_succeeded, 0)
self.assertEqual(report.downloads[sources[2].id].total_failed, 0)
def test_collated_all_downloaded(self):
request_list = []
for n in range(10):
r = Mock()
r.downloaded = True
request_list.append(r)
# test
collated = ContentContainer.collated(request_list)
# validation
self.assertEqual(len(collated), 0)
def test_download_canceled_after_collated(self, fake_load):
sources = []
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.get_downloader = Mock()
sources.append(s)
fake_load.return_value = sources
request_list = []
for n in range(6):
r = Request('T', {}, 'url-%d' % n, 'path-%d' % n)
r.find_sources = Mock(return_value=sources[n % 3:])
request_list.append(r)
collated = [{
sources[0]: ['nectar-1'],
sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'],
sources[2]: ['nectar-5', 'nectar-6']
}, {}]
fake_collated = Mock(side_effect=collated)
fake_listener = Mock()
canceled = Mock()
canceled.isSet.side_effect = [False, True, True]
fake_primary = PrimarySource(Mock())
# test
container = ContentContainer('')
container.refresh = Mock()
container.collated = fake_collated
report = container.download(canceled, fake_primary, request_list,
fake_listener)
# validation
container.refresh.assert_called_with(canceled)
for r in request_list:
r.find_sources.assert_called_with(fake_primary, container.sources)
called = 0
for s in sources:
if s.get_downloader.called:
called += 1
self.assertEqual(called, 1)
self.assertEqual(report.total_passes, 1)
self.assertEqual(report.total_sources, len(sources))
self.assertEqual(len(report.downloads), 1)
self.assertEqual(report.downloads[sources[2].id].total_succeeded, 0)
self.assertEqual(report.downloads[sources[2].id].total_failed, 0)
def test_collated(self):
source_id = 1
request_list = []
# simulated requests
# 0-8 need to be downloaded and have a next source.
# 6-7 have no next source.
# 9-10 are already downloaded.
for n in range(10):
r = Mock()
r.downloaded = n > 8
r.destination = 'path-%d' % n
if n < 6:
r.next_source.return_value = ('s-%d' % source_id, 'url-%d' % n)
else:
r.next_source.return_value = None
request_list.append(r)
if n % 3 == 0:
source_id += 1
# test
collated = ContentContainer.collated(request_list)
# validation
self.assertEqual(len(collated), 3)
for requests in collated.values():
for r in requests:
self.assertTrue(isinstance(r, DownloadRequest))
self.assertEqual(
[s.__dict__ for s in collated['s-1']],
[
DownloadRequest('url-0', 'path-0', data=request_list[0]).__dict__
])
self.assertEqual(
[s.__dict__ for s in collated['s-2']],
[
DownloadRequest('url-1', 'path-1', data=request_list[1]).__dict__,
DownloadRequest('url-2', 'path-2', data=request_list[2]).__dict__,
DownloadRequest('url-3', 'path-3', data=request_list[3]).__dict__
])
self.assertEqual(
[s.__dict__ for s in collated['s-3']],
[
DownloadRequest('url-4', 'path-4', data=request_list[4]).__dict__,
DownloadRequest('url-5', 'path-5', data=request_list[5]).__dict__,
])
def test_download_canceled_before_collated(self, fake_load):
fake_load.return_value = []
canceled = FakeEvent()
canceled.set()
# test
container = ContentContainer('')
container.refresh = Mock()
container.collated = Mock()
report = container.download(canceled, None, [], None)
container.refresh.assert_called_with(canceled)
self.assertFalse(container.collated.called)
self.assertEqual(report.total_passes, 0)
self.assertEqual(report.total_sources, 0)
self.assertEqual(len(report.downloads), 0)
def test_download_canceled_before_collated(self, fake_load):
fake_load.return_value = []
canceled = FakeEvent()
canceled.set()
# test
container = ContentContainer('')
container.refresh = Mock()
container.collated = Mock()
report = container.download(canceled, None, [], None)
container.refresh.assert_called_with(canceled)
self.assertFalse(container.collated.called)
self.assertEqual(report.total_passes, 0)
self.assertEqual(report.total_sources, 0)
self.assertEqual(len(report.downloads), 0)
def test_collated(self):
source_id = 1
request_list = []
# simulated requests
# 0-8 need to be downloaded and have a next source.
# 6-7 have no next source.
# 9-10 are already downloaded.
for n in range(10):
r = Mock()
r.downloaded = n > 8
r.destination = 'path-%d' % n
if n < 6:
r.next_source.return_value = ('s-%d' % source_id, 'url-%d' % n)
else:
r.next_source.return_value = None
request_list.append(r)
if n % 3 == 0:
source_id += 1
# test
collated = ContentContainer.collated(request_list)
# validation
self.assertEqual(len(collated), 3)
for requests in collated.values():
for r in requests:
self.assertTrue(isinstance(r, DownloadRequest))
self.assertEqual([s.__dict__ for s in collated['s-1']], [
DownloadRequest('url-0', 'path-0', data=request_list[0]).__dict__
])
self.assertEqual([s.__dict__ for s in collated['s-2']], [
DownloadRequest('url-1', 'path-1', data=request_list[1]).__dict__,
DownloadRequest('url-2', 'path-2', data=request_list[2]).__dict__,
DownloadRequest('url-3', 'path-3', data=request_list[3]).__dict__
])
self.assertEqual([s.__dict__ for s in collated['s-3']], [
DownloadRequest('url-4', 'path-4', data=request_list[4]).__dict__,
DownloadRequest('url-5', 'path-5', data=request_list[5]).__dict__,
])
