def test_download(self, fake_load):
sources = []
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.get_downloader = Mock()
sources.append(s)
fake_load.return_value = sources
request_list = []
for n in range(6):
r = Request('T', {}, 'url-%d' % n, 'path-%d' % n)
r.find_sources = Mock(return_value=sources[n % 3:])
request_list.append(r)
collated = [
{
sources[0]: ['nectar-1'],
sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'],
sources[2]: ['nectar-5', 'nectar-6']
},
{}
]
fake_collated = Mock(side_effect=collated)
fake_listener = Mock()
canceled = FakeEvent()
fake_primary = PrimarySource(Mock())
# test
container = ContentContainer('')
container.refresh = Mock()
container.collated = fake_collated
report = container.download(canceled, fake_primary, request_list, fake_listener)
# validation
container.refresh.assert_called_with(canceled)
for r in request_list:
r.find_sources.assert_called_with(fake_primary, container.sources)
self.assertEqual(report.total_passes, 1)
self.assertEqual(report.total_sources, len(sources))
self.assertEqual(len(report.downloads), 3)
for source in sources:
self.assertEqual(report.downloads[source.id].total_succeeded, 0)
self.assertEqual(report.downloads[source.id].total_failed, 0)
for source in sources:
source.get_downloader.assert_called_with()
downloader = source.get_downloader()
listener = downloader.event_listener
self.assertEqual(listener.cancel_event, canceled)
self.assertEqual(listener.downloader, downloader)
self.assertEqual(listener.listener, fake_listener)
downloader.download.assert_called_with(collated[0][source])
def test_download(self, fake_load):
sources = []
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.get_downloader = Mock()
sources.append(s)
fake_load.return_value = sources
request_list = []
for n in range(6):
r = Request('T', {}, 'url-%d' % n, 'path-%d' % n)
r.find_sources = Mock(return_value=sources[n % 3:])
request_list.append(r)
collated = [{
sources[0]: ['nectar-1'],
sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'],
sources[2]: ['nectar-5', 'nectar-6']
}, {}]
fake_collated = Mock(side_effect=collated)
fake_listener = Mock()
canceled = FakeEvent()
fake_primary = PrimarySource(Mock())
# test
container = ContentContainer('')
container.refresh = Mock()
container.collated = fake_collated
report = container.download(canceled, fake_primary, request_list,
fake_listener)
# validation
container.refresh.assert_called_with(canceled)
for r in request_list:
r.find_sources.assert_called_with(fake_primary, container.sources)
self.assertEqual(report.total_passes, 1)
self.assertEqual(report.total_sources, len(sources))
self.assertEqual(len(report.downloads), 3)
for source in sources:
self.assertEqual(report.downloads[source.id].total_succeeded, 0)
self.assertEqual(report.downloads[source.id].total_failed, 0)
for source in sources:
source.get_downloader.assert_called_with()
downloader = source.get_downloader()
listener = downloader.event_listener
self.assertEqual(listener.cancel_event, canceled)
self.assertEqual(listener.downloader, downloader)
self.assertEqual(listener.listener, fake_listener)
downloader.download.assert_called_with(collated[0][source])
def test_download_canceled_after_collated(self, fake_load):
sources = []
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.get_downloader = Mock()
sources.append(s)
fake_load.return_value = sources
request_list = []
for n in range(6):
r = Request('T', {}, 'url-%d' % n, 'path-%d' % n)
r.find_sources = Mock(return_value=sources[n % 3:])
request_list.append(r)
collated = [
{
sources[0]: ['nectar-1'],
sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'],
sources[2]: ['nectar-5', 'nectar-6']
},
{}
]
fake_collated = Mock(side_effect=collated)
fake_listener = Mock()
canceled = Mock()
canceled.isSet.side_effect = [False, True, True]
fake_primary = PrimarySource(Mock())
# test
container = ContentContainer('')
container.refresh = Mock()
container.collated = fake_collated
report = container.download(canceled, fake_primary, request_list, fake_listener)
# validation
container.refresh.assert_called_with(canceled)
for r in request_list:
r.find_sources.assert_called_with(fake_primary, container.sources)
called = 0
for s in sources:
if s.get_downloader.called:
called += 1
self.assertEqual(called, 1)
self.assertEqual(report.total_passes, 1)
self.assertEqual(report.total_sources, len(sources))
self.assertEqual(len(report.downloads), 1)
self.assertEqual(report.downloads[sources[2].id].total_succeeded, 0)
self.assertEqual(report.downloads[sources[2].id].total_failed, 0)
def test_is_valid_bad_descriptor(self, mock_descriptor_is_valid):
source = ContentSource('s-1', {'A': 1})
source.get_downloader = Mock()
source.get_cataloger = Mock()
mock_descriptor_is_valid.side_effect = ValueError()
# Test
valid = source.is_valid()
# validation
self.assertFalse(valid)
def test_is_valid_bad_descriptor(self, mock_descriptor_is_valid):
source = ContentSource('s-1', {'A': 1})
source.get_downloader = Mock()
source.get_cataloger = Mock()
mock_descriptor_is_valid.side_effect = ValueError()
# Test
valid = source.is_valid()
# validation
self.assertFalse(valid)
def test_download_canceled_after_collated(self, fake_load):
sources = []
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.get_downloader = Mock()
sources.append(s)
fake_load.return_value = sources
request_list = []
for n in range(6):
r = Request('T', {}, 'url-%d' % n, 'path-%d' % n)
r.find_sources = Mock(return_value=sources[n % 3:])
request_list.append(r)
collated = [{
sources[0]: ['nectar-1'],
sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'],
sources[2]: ['nectar-5', 'nectar-6']
}, {}]
fake_collated = Mock(side_effect=collated)
fake_listener = Mock()
canceled = Mock()
canceled.isSet.side_effect = [False, True, True]
fake_primary = PrimarySource(Mock())
# test
container = ContentContainer('')
container.refresh = Mock()
container.collated = fake_collated
report = container.download(canceled, fake_primary, request_list,
fake_listener)
# validation
container.refresh.assert_called_with(canceled)
for r in request_list:
r.find_sources.assert_called_with(fake_primary, container.sources)
called = 0
for s in sources:
if s.get_downloader.called:
called += 1
self.assertEqual(called, 1)
self.assertEqual(report.total_passes, 1)
self.assertEqual(report.total_sources, len(sources))
self.assertEqual(len(report.downloads), 1)
self.assertEqual(report.downloads[sources[2].id].total_succeeded, 0)
self.assertEqual(report.downloads[sources[2].id].total_failed, 0)
def test_is_valid(self, mock_descriptor_is_valid):
source = ContentSource('s-1', {'A': 1})
source.get_downloader = Mock()
source.get_cataloger = Mock()
# Test
valid = source.is_valid()
# validation
source.get_cataloger.assert_called_with()
source.get_downloader.assert_called_with()
mock_descriptor_is_valid.assert_called_with(source.id, source.descriptor)
self.assertTrue(valid)
def test_is_valid_no_downloader(self, mock_descriptor_is_valid):
source = ContentSource('s-1', {'A': 1})
source.get_downloader = Mock(side_effect=NotImplementedError())
source.get_cataloger = Mock()
# Test
valid = source.is_valid()
# validation
source.get_cataloger.assert_called_with()
source.get_downloader.assert_called_with()
self.assertFalse(mock_descriptor_is_valid.called)
self.assertFalse(valid)
def test_is_valid_no_downloader(self, mock_descriptor_is_valid):
source = ContentSource('s-1', {'A': 1})
source.get_downloader = Mock(side_effect=NotImplementedError())
source.get_cataloger = Mock()
# Test
valid = source.is_valid()
# validation
source.get_cataloger.assert_called_with()
source.get_downloader.assert_called_with()
self.assertFalse(mock_descriptor_is_valid.called)
self.assertFalse(valid)
def test_is_valid(self, mock_descriptor_is_valid):
source = ContentSource('s-1', {'A': 1})
source.get_downloader = Mock()
source.get_cataloger = Mock()
# Test
valid = source.is_valid()
# validation
source.get_cataloger.assert_called_with()
source.get_downloader.assert_called_with()
mock_descriptor_is_valid.assert_called_with(source.id, source.descriptor)
self.assertTrue(valid)
def test_is_valid_no_plugin(self, mock_descriptor_is_valid):
mock_descriptor_is_valid.return_value = True
source = ContentSource("s-1", {"A": 1})
source.get_downloader = Mock()
source.get_cataloger = Mock(side_effect=NotImplementedError())
# Test
valid = source.is_valid()
# validation
source.get_cataloger.assert_called_with()
self.assertFalse(source.get_downloader.called)
self.assertTrue(mock_descriptor_is_valid.called)
self.assertFalse(valid)
def test_downloader(self):
url = 'http://xyz.com'
fake_conduit = Mock()
fake_cataloger = Mock()
fake_downloader = Mock()
fake_cataloger.get_downloader = Mock(return_value=fake_downloader)
source = ContentSource('s-1', {constants.BASE_URL: url})
source.get_conduit = Mock(return_value=fake_conduit)
source.get_cataloger = Mock(return_value=fake_cataloger)
# test
downloader = source.get_downloader()
# validation
source.get_cataloger.assert_called_with()
fake_cataloger.get_downloader.assert_called_with(fake_conduit, source.descriptor, url)
self.assertEqual(downloader, fake_downloader)
def test_downloader(self):
url = 'http://xyz.com'
fake_conduit = Mock()
fake_cataloger = Mock()
fake_downloader = Mock()
fake_cataloger.get_downloader = Mock(return_value=fake_downloader)
source = ContentSource('s-1', {constants.BASE_URL: url})
source.get_conduit = Mock(return_value=fake_conduit)
source.get_cataloger = Mock(return_value=fake_cataloger)
# test
downloader = source.get_downloader()
# validation
source.get_cataloger.assert_called_with()
fake_cataloger.get_downloader.assert_called_with(fake_conduit, source.descriptor, url)
self.assertEqual(downloader, fake_downloader)
def test_refresh(self, fake_manager, fake_load):
sources = {}
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.refresh = Mock(return_value=[n])
s.get_downloader = Mock()
sources[s.id] = s
fake_manager().has_entries.return_value = False
fake_load.return_value = sources
# test
container = ContentContainer('')
report = container.refresh()
# validation
for s in sources.values():
s.refresh.assert_called_with()
self.assertEqual(sorted(report), [0, 1, 2])
def test_refresh(self, fake_manager, fake_load):
sources = {}
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.refresh = Mock(return_value=[n])
s.get_downloader = Mock()
sources[s.id] = s
fake_manager().has_entries.return_value = False
fake_load.return_value = sources
# test
container = ContentContainer('')
report = container.refresh()
# validation
for s in sources.values():
s.refresh.assert_called_with()
self.assertEqual(sorted(report), [0, 1, 2])
def test_refresh_raised(self, fake_manager, fake_load):
sources = {}
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.refresh = Mock(side_effect=ValueError('must be int'))
s.get_downloader = Mock()
sources[s.id] = s
fake_manager().has_entries.return_value = False
fake_load.return_value = sources
# test
container = ContentContainer('')
report = container.refresh()
# validation
for s in sources.values():
s.refresh.assert_called_with()
for r in report:
r.errors = ['must be int']
def test_refresh_raised(self, fake_manager, fake_load):
sources = {}
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.refresh = Mock(side_effect=ValueError('must be int'))
s.get_downloader = Mock()
sources[s.id] = s
fake_manager().has_entries.return_value = False
fake_load.return_value = sources
# test
container = ContentContainer('')
report = container.refresh()
# validation
for s in sources.values():
s.refresh.assert_called_with()
for r in report:
r.errors = ['must be int']