def test_refresh_raised(self, fake_urls):
url = 'http://xyz.com'
urls = ['url-1', 'url-2']
fake_urls.__get__ = Mock(return_value=urls)
canceled = Mock()
canceled.isSet = Mock(return_value=False)
conduit = Mock()
cataloger = Mock()
cataloger.refresh.side_effect = ValueError('just failed')
source = ContentSource('s-1', {constants.BASE_URL: url})
source.get_conduit = Mock(return_value=conduit)
source.get_cataloger = Mock(return_value=cataloger)
# test
report = source.refresh(canceled)
# validation
self.assertEqual(canceled.isSet.call_count, len(urls))
self.assertEqual(conduit.reset.call_count, len(urls))
self.assertEqual(cataloger.refresh.call_count, len(urls))
n = 0
for _url in source.urls:
cataloger.refresh.assert_any(conduit, source.descriptor, _url)
self.assertEqual(report[n].source_id, source.id)
self.assertEqual(report[n].url, _url)
self.assertFalse(report[n].succeeded)
self.assertEqual(report[n].errors, ['just failed'])
self.assertEqual(report[n].added_count, 0)
self.assertEqual(report[n].deleted_count, 0)
n += 1
def test_enabled(self):
source = ContentSource('s-1', {constants.ENABLED: 'true'})
self.assertTrue(source.enabled)
source.descriptor[constants.ENABLED] = 'false'
self.assertFalse(source.enabled)
source.descriptor = {}
self.assertFalse(source.enabled)
def test_refresh_canceled(self, fake_urls):
url = 'http://xyz.com'
urls = ['url-1', 'url-2']
fake_urls.__get__ = Mock(return_value=urls)
canceled = Mock()
canceled.isSet = Mock(return_value=True)
conduit = Mock()
cataloger = Mock()
source = ContentSource('s-1', {constants.BASE_URL: url})
source.get_conduit = Mock(return_value=conduit)
source.get_cataloger = Mock(return_value=cataloger)
# test
report = source.refresh(canceled)
# validation
self.assertEqual(canceled.isSet.call_count, 1)
self.assertEqual(conduit.reset.call_count, 0)
self.assertEqual(cataloger.refresh.call_count, 0)
self.assertEqual(report, [])
def test_refresh_raised(self, fake_urls):
url = 'http://xyz.com'
urls = ['url-1', 'url-2']
fake_urls.__get__ = Mock(return_value=urls)
canceled = Mock()
canceled.isSet = Mock(return_value=False)
conduit = Mock()
cataloger = Mock()
cataloger.refresh.side_effect = ValueError('just failed')
source = ContentSource('s-1', {constants.BASE_URL: url})
source.get_conduit = Mock(return_value=conduit)
source.get_cataloger = Mock(return_value=cataloger)
# test
report = source.refresh(canceled)
# validation
self.assertEqual(canceled.isSet.call_count, len(urls))
self.assertEqual(conduit.reset.call_count, len(urls))
self.assertEqual(cataloger.refresh.call_count, len(urls))
n = 0
for _url in source.urls:
cataloger.refresh.assert_any(conduit, source.descriptor, _url)
self.assertEqual(report[n].source_id, source.id)
self.assertEqual(report[n].url, _url)
self.assertFalse(report[n].succeeded)
self.assertEqual(report[n].errors, ['just failed'])
self.assertEqual(report[n].added_count, 0)
self.assertEqual(report[n].deleted_count, 0)
n += 1
def test_refresh_canceled(self, fake_urls):
url = 'http://xyz.com'
urls = ['url-1', 'url-2']
fake_urls.__get__ = Mock(return_value=urls)
canceled = Mock()
canceled.isSet = Mock(return_value=True)
conduit = Mock()
cataloger = Mock()
source = ContentSource('s-1', {constants.BASE_URL: url})
source.get_conduit = Mock(return_value=conduit)
source.get_cataloger = Mock(return_value=cataloger)
# test
report = source.refresh(canceled)
# validation
self.assertEqual(canceled.isSet.call_count, 1)
self.assertEqual(conduit.reset.call_count, 0)
self.assertEqual(cataloger.refresh.call_count, 0)
self.assertEqual(report, [])
def test_enabled(self):
source = ContentSource('s-1', {constants.ENABLED: 'true'})
self.assertTrue(source.enabled)
source.descriptor[constants.ENABLED] = 'false'
self.assertFalse(source.enabled)
source.descriptor = {}
self.assertFalse(source.enabled)
def test_sorting(self):
s1 = ContentSource('s-1', {constants.PRIORITY: 2})
s2 = ContentSource('s-2', {constants.PRIORITY: 1})
s3 = ContentSource('s-3', {constants.PRIORITY: 0})
_list = sorted([s1, s2, s3])
self.assertEqual([s.id for s in _list], [s3.id, s2.id, s1.id])
def test_conduit(self):
source = ContentSource('s-1', {constants.EXPIRES: '1h'})
conduit = source.get_conduit()
self.assertEqual(conduit.source_id, source.id)
self.assertEqual(conduit.expires, 3600)
self.assertTrue(isinstance(conduit, CatalogerConduit))
def test_conduit(self):
source = ContentSource('s-1', {constants.EXPIRES: '1h'})
conduit = source.get_conduit()
self.assertEqual(conduit.source_id, source.id)
self.assertEqual(conduit.expires, 3600)
self.assertTrue(isinstance(conduit, CatalogerConduit))
def test_download(self, fake_load):
sources = []
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.get_downloader = Mock()
sources.append(s)
fake_load.return_value = sources
request_list = []
for n in range(6):
r = Request('T', {}, 'url-%d' % n, 'path-%d' % n)
r.find_sources = Mock(return_value=sources[n % 3:])
request_list.append(r)
collated = [
{
sources[0]: ['nectar-1'],
sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'],
sources[2]: ['nectar-5', 'nectar-6']
},
{}
]
fake_collated = Mock(side_effect=collated)
fake_listener = Mock()
canceled = FakeEvent()
fake_primary = PrimarySource(Mock())
# test
container = ContentContainer('')
container.refresh = Mock()
container.collated = fake_collated
report = container.download(canceled, fake_primary, request_list, fake_listener)
# validation
container.refresh.assert_called_with(canceled)
for r in request_list:
r.find_sources.assert_called_with(fake_primary, container.sources)
self.assertEqual(report.total_passes, 1)
self.assertEqual(report.total_sources, len(sources))
self.assertEqual(len(report.downloads), 3)
for source in sources:
self.assertEqual(report.downloads[source.id].total_succeeded, 0)
self.assertEqual(report.downloads[source.id].total_failed, 0)
for source in sources:
source.get_downloader.assert_called_with()
downloader = source.get_downloader()
listener = downloader.event_listener
self.assertEqual(listener.cancel_event, canceled)
self.assertEqual(listener.downloader, downloader)
self.assertEqual(listener.listener, fake_listener)
downloader.download.assert_called_with(collated[0][source])
def test_dict(self):
descriptor = {'A': 1, 'B': 2}
# test
source = ContentSource('s-1', descriptor)
# validation
expected = {}
expected.update(descriptor)
expected[constants.SOURCE_ID] = source.id
self.assertEqual(source.dict(), expected)
def test_download(self, fake_load):
sources = []
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.get_downloader = Mock()
sources.append(s)
fake_load.return_value = sources
request_list = []
for n in range(6):
r = Request('T', {}, 'url-%d' % n, 'path-%d' % n)
r.find_sources = Mock(return_value=sources[n % 3:])
request_list.append(r)
collated = [{
sources[0]: ['nectar-1'],
sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'],
sources[2]: ['nectar-5', 'nectar-6']
}, {}]
fake_collated = Mock(side_effect=collated)
fake_listener = Mock()
canceled = FakeEvent()
fake_primary = PrimarySource(Mock())
# test
container = ContentContainer('')
container.refresh = Mock()
container.collated = fake_collated
report = container.download(canceled, fake_primary, request_list,
fake_listener)
# validation
container.refresh.assert_called_with(canceled)
for r in request_list:
r.find_sources.assert_called_with(fake_primary, container.sources)
self.assertEqual(report.total_passes, 1)
self.assertEqual(report.total_sources, len(sources))
self.assertEqual(len(report.downloads), 3)
for source in sources:
self.assertEqual(report.downloads[source.id].total_succeeded, 0)
self.assertEqual(report.downloads[source.id].total_failed, 0)
for source in sources:
source.get_downloader.assert_called_with()
downloader = source.get_downloader()
listener = downloader.event_listener
self.assertEqual(listener.cancel_event, canceled)
self.assertEqual(listener.downloader, downloader)
self.assertEqual(listener.listener, fake_listener)
downloader.download.assert_called_with(collated[0][source])
def test_dict(self):
descriptor = {'A': 1, 'B': 2}
# test
source = ContentSource('s-1', descriptor)
# validation
expected = {}
expected.update(descriptor)
expected[constants.SOURCE_ID] = source.id
self.assertEqual(source.dict(), expected)
def test_download_canceled_after_collated(self, fake_load):
sources = []
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.get_downloader = Mock()
sources.append(s)
fake_load.return_value = sources
request_list = []
for n in range(6):
r = Request('T', {}, 'url-%d' % n, 'path-%d' % n)
r.find_sources = Mock(return_value=sources[n % 3:])
request_list.append(r)
collated = [
{
sources[0]: ['nectar-1'],
sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'],
sources[2]: ['nectar-5', 'nectar-6']
},
{}
]
fake_collated = Mock(side_effect=collated)
fake_listener = Mock()
canceled = Mock()
canceled.isSet.side_effect = [False, True, True]
fake_primary = PrimarySource(Mock())
# test
container = ContentContainer('')
container.refresh = Mock()
container.collated = fake_collated
report = container.download(canceled, fake_primary, request_list, fake_listener)
# validation
container.refresh.assert_called_with(canceled)
for r in request_list:
r.find_sources.assert_called_with(fake_primary, container.sources)
called = 0
for s in sources:
if s.get_downloader.called:
called += 1
self.assertEqual(called, 1)
self.assertEqual(report.total_passes, 1)
self.assertEqual(report.total_sources, len(sources))
self.assertEqual(len(report.downloads), 1)
self.assertEqual(report.downloads[sources[2].id].total_succeeded, 0)
self.assertEqual(report.downloads[sources[2].id].total_failed, 0)
def test_is_valid_bad_descriptor(self, mock_descriptor_is_valid):
source = ContentSource('s-1', {'A': 1})
source.get_downloader = Mock()
source.get_cataloger = Mock()
mock_descriptor_is_valid.side_effect = ValueError()
# Test
valid = source.is_valid()
# validation
self.assertFalse(valid)
def test_is_valid_bad_descriptor(self, mock_descriptor_is_valid):
source = ContentSource('s-1', {'A': 1})
source.get_downloader = Mock()
source.get_cataloger = Mock()
mock_descriptor_is_valid.side_effect = ValueError()
# Test
valid = source.is_valid()
# validation
self.assertFalse(valid)
def test_cataloger(self, fake_plugins):
plugin = Mock()
fake_plugins.get_cataloger_by_id.return_value = plugin, {}
# test
source = ContentSource('s-1', {constants.TYPE: 1234})
cataloger = source.get_cataloger()
# validation
fake_plugins.get_cataloger_by_id.assert_called_with(1234)
self.assertEqual(plugin, cataloger)
def test_cataloger(self, fake_plugins):
plugin = Mock()
fake_plugins.get_cataloger_by_id.return_value = plugin, {}
# test
source = ContentSource('s-1', {constants.TYPE: 1234})
cataloger = source.get_cataloger()
# validation
fake_plugins.get_cataloger_by_id.assert_called_with(1234)
self.assertEqual(plugin, cataloger)
def test_download_canceled_after_collated(self, fake_load):
sources = []
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.get_downloader = Mock()
sources.append(s)
fake_load.return_value = sources
request_list = []
for n in range(6):
r = Request('T', {}, 'url-%d' % n, 'path-%d' % n)
r.find_sources = Mock(return_value=sources[n % 3:])
request_list.append(r)
collated = [{
sources[0]: ['nectar-1'],
sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'],
sources[2]: ['nectar-5', 'nectar-6']
}, {}]
fake_collated = Mock(side_effect=collated)
fake_listener = Mock()
canceled = Mock()
canceled.isSet.side_effect = [False, True, True]
fake_primary = PrimarySource(Mock())
# test
container = ContentContainer('')
container.refresh = Mock()
container.collated = fake_collated
report = container.download(canceled, fake_primary, request_list,
fake_listener)
# validation
container.refresh.assert_called_with(canceled)
for r in request_list:
r.find_sources.assert_called_with(fake_primary, container.sources)
called = 0
for s in sources:
if s.get_downloader.called:
called += 1
self.assertEqual(called, 1)
self.assertEqual(report.total_passes, 1)
self.assertEqual(report.total_sources, len(sources))
self.assertEqual(len(report.downloads), 1)
self.assertEqual(report.downloads[sources[2].id].total_succeeded, 0)
self.assertEqual(report.downloads[sources[2].id].total_failed, 0)
def test_is_valid_no_downloader(self, mock_descriptor_is_valid):
source = ContentSource('s-1', {'A': 1})
source.get_downloader = Mock(side_effect=NotImplementedError())
source.get_cataloger = Mock()
# Test
valid = source.is_valid()
# validation
source.get_cataloger.assert_called_with()
source.get_downloader.assert_called_with()
self.assertFalse(mock_descriptor_is_valid.called)
self.assertFalse(valid)
def test_is_valid(self, mock_descriptor_is_valid):
source = ContentSource('s-1', {'A': 1})
source.get_downloader = Mock()
source.get_cataloger = Mock()
# Test
valid = source.is_valid()
# validation
source.get_cataloger.assert_called_with()
source.get_downloader.assert_called_with()
mock_descriptor_is_valid.assert_called_with(source.id, source.descriptor)
self.assertTrue(valid)
def test_is_valid_no_downloader(self, mock_descriptor_is_valid):
source = ContentSource('s-1', {'A': 1})
source.get_downloader = Mock(side_effect=NotImplementedError())
source.get_cataloger = Mock()
# Test
valid = source.is_valid()
# validation
source.get_cataloger.assert_called_with()
source.get_downloader.assert_called_with()
self.assertFalse(mock_descriptor_is_valid.called)
self.assertFalse(valid)
def test_is_valid(self, mock_descriptor_is_valid):
source = ContentSource('s-1', {'A': 1})
source.get_downloader = Mock()
source.get_cataloger = Mock()
# Test
valid = source.is_valid()
# validation
source.get_cataloger.assert_called_with()
source.get_downloader.assert_called_with()
mock_descriptor_is_valid.assert_called_with(source.id, source.descriptor)
self.assertTrue(valid)
def test_is_valid_no_plugin(self, mock_descriptor_is_valid):
mock_descriptor_is_valid.return_value = True
source = ContentSource("s-1", {"A": 1})
source.get_downloader = Mock()
source.get_cataloger = Mock(side_effect=NotImplementedError())
# Test
valid = source.is_valid()
# validation
source.get_cataloger.assert_called_with()
self.assertFalse(source.get_downloader.called)
self.assertTrue(mock_descriptor_is_valid.called)
self.assertFalse(valid)
def test_load_all(self, fake_valid, fake_enabled, fake_parser, fake_listdir, fake_isfile):
conf_d = '/fake/conf_d'
files = ['one.conf', 'other']
fake_listdir.return_value = files
fake_valid.side_effect = [
True, # s-0
# s-1 not enabled
True, # s-2
False # s-3
]
fake_isfile.side_effect = [True, False]
fake_enabled.__get__ = Mock(side_effect=[d[1]['enabled'] for d in DESCRIPTOR])
parser = Mock()
parser.items.side_effect = [d[1].items() for d in DESCRIPTOR]
parser.sections.return_value = [d[0] for d in DESCRIPTOR]
fake_parser.return_value = parser
# test
sources = ContentSource.load_all(conf_d)
# validation
fake_listdir.assert_called_with(conf_d)
fake_parser.assert_called_with()
fake_parser().read.assert_called_with(os.path.join(conf_d, files[0]))
self.assertEqual(len(sources), 2)
self.assertTrue(DESCRIPTOR[0][0] in sources)
self.assertTrue(DESCRIPTOR[2][0] in sources)
def __init__(self, path=None):
"""
:param path: The absolute path to a directory containing
content source descriptor files.
:type path: str
"""
self.sources = ContentSource.load_all(path)
def test_load_all(self, fake_valid, fake_enabled, fake_parser, fake_listdir, fake_isfile):
conf_d = '/fake/conf_d'
files = ['one.conf', 'two.conf.swp', 'dir.conf']
fake_listdir.return_value = files
fake_valid.side_effect = [
True, # s-0
# s-1 not enabled
True, # s-2
False # s-3
]
fake_isfile.side_effect = [True, False]
fake_enabled.__get__ = Mock(side_effect=[d[1]['enabled'] for d in DESCRIPTOR])
parser = Mock()
parser.items.side_effect = [d[1].items() for d in DESCRIPTOR]
parser.sections.return_value = [d[0] for d in DESCRIPTOR]
fake_parser.return_value = parser
# test
sources = ContentSource.load_all(conf_d)
# validation
fake_listdir.assert_called_with(conf_d)
fake_parser.assert_called_with()
fake_parser().read.assert_called_with(os.path.join(conf_d, files[0]))
self.assertEqual(len(sources), 2)
self.assertTrue(DESCRIPTOR[0][0] in sources)
self.assertTrue(DESCRIPTOR[2][0] in sources)
def test_load_all(self, fake_valid, fake_enabled, fake_parser,
fake_listdir):
conf_d = '/fake/conf_d'
files = ['one.conf', 'other']
fake_listdir.return_value = files
fake_valid.side_effect = [d[1]['valid'] for d in DESCRIPTOR]
fake_enabled.__get__ = Mock(
side_effect=[d[1]['enabled'] for d in DESCRIPTOR])
parser = Mock()
parser.items.side_effect = [d[1].items() for d in DESCRIPTOR]
parser.sections.return_value = dict(DESCRIPTOR)
fake_parser.return_value = parser
# test
sources = ContentSource.load_all(conf_d)
# validation
fake_listdir.assert_called_with(conf_d)
fake_parser.assert_called_with()
fake_parser().read.assert_called_with(os.path.join(conf_d, files[0]))
self.assertEqual(len(sources), 2)
self.assertTrue(DESCRIPTOR[1][0] in sources)
self.assertTrue(DESCRIPTOR[3][0] in sources)
def __init__(self, path=None):
"""
:param path: The absolute path to a directory containing
content source descriptor files.
:type path: str
"""
self.sources = ContentSource.load_all(path)
def test_find_sources(self, fake_manager):
type_id = 'test_1'
unit_key = 1
destination = '/tmp/123'
url = 'http://redhat.com/repository'
primary = PrimarySource(None)
alternatives = dict([(s, ContentSource(s, d)) for s, d in DESCRIPTOR])
fake_manager().find.return_value = CATALOG
# test
request = Request(type_id, unit_key, url, destination)
request.find_sources(primary, alternatives)
# validation
# validate sources sorted by priority with the primary last.
# should only have matched on s-1 and s-3.
self.assertEqual(len(request.sources), 5)
self.assertEqual(request.sources[0][0].id, 's-3')
self.assertEqual(request.sources[0][1], CATALOG[2][constants.URL])
self.assertEqual(request.sources[1][0].id, 's-3')
self.assertEqual(request.sources[1][1], CATALOG[3][constants.URL])
self.assertEqual(request.sources[2][0].id, 's-1')
self.assertEqual(request.sources[2][1], CATALOG[0][constants.URL])
self.assertEqual(request.sources[3][0].id, 's-1')
self.assertEqual(request.sources[3][1], CATALOG[1][constants.URL])
self.assertEqual(request.sources[4][0].id, primary.id)
self.assertEqual(request.sources[4][1], url)
def test_forced_refresh(self, fake_manager, fake_load):
sources = {}
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.refresh = Mock()
sources[s.id] = s
fake_manager().has_entries.return_value = True
fake_load.return_value = sources
# test
container = ContentContainer('')
container.refresh(force=True)
# validation
for s in sources.values():
s.refresh.assert_called_with()
def test_construction(self):
# test
source = ContentSource(DESCRIPTOR[0][0], DESCRIPTOR[0][1])
# validation
self.assertEqual(source.id, DESCRIPTOR[0][0])
self.assertEqual(source.descriptor, DESCRIPTOR[0][1])
def test_forced_refresh(self, fake_manager, fake_load):
sources = {}
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.refresh = Mock()
sources[s.id] = s
fake_manager().has_entries.return_value = True
fake_load.return_value = sources
# test
container = ContentContainer('')
container.refresh(force=True)
# validation
for s in sources.values():
s.refresh.assert_called_with()
def test_downloader(self):
url = 'http://xyz.com'
fake_conduit = Mock()
fake_cataloger = Mock()
fake_downloader = Mock()
fake_cataloger.get_downloader = Mock(return_value=fake_downloader)
source = ContentSource('s-1', {constants.BASE_URL: url})
source.get_conduit = Mock(return_value=fake_conduit)
source.get_cataloger = Mock(return_value=fake_cataloger)
# test
downloader = source.get_downloader()
# validation
source.get_cataloger.assert_called_with()
fake_cataloger.get_downloader.assert_called_with(fake_conduit, source.descriptor, url)
self.assertEqual(downloader, fake_downloader)
def test_refresh_canceled(self, fake_load):
sources = {}
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.refresh = Mock()
sources[s.id] = s
fake_load.return_value = sources
# test
canceled = FakeEvent()
canceled.set()
container = ContentContainer('')
container.refresh(canceled, force=True)
# validation
for s in sources.values():
self.assertFalse(s.refresh.called)
def test_downloader(self):
url = 'http://xyz.com'
fake_conduit = Mock()
fake_cataloger = Mock()
fake_downloader = Mock()
fake_cataloger.get_downloader = Mock(return_value=fake_downloader)
source = ContentSource('s-1', {constants.BASE_URL: url})
source.get_conduit = Mock(return_value=fake_conduit)
source.get_cataloger = Mock(return_value=fake_cataloger)
# test
downloader = source.get_downloader()
# validation
source.get_cataloger.assert_called_with()
fake_cataloger.get_downloader.assert_called_with(fake_conduit, source.descriptor, url)
self.assertEqual(downloader, fake_downloader)
def test_refresh_canceled(self, fake_load):
sources = {}
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.refresh = Mock()
sources[s.id] = s
fake_load.return_value = sources
# test
canceled = FakeEvent()
canceled.set()
container = ContentContainer('')
container.refresh(canceled, force=True)
# validation
for s in sources.values():
self.assertFalse(s.refresh.called)
def test_urls(self):
sources = ContentSource.load_all(self.tmp_dir)
underground = sources[UNDERGROUND]
urls = underground.urls
self.assertEqual(len(urls), 4)
self.assertEqual(urls[0], 'file:///underground/fedora/18/x86_64/')
self.assertEqual(urls[1], 'file:///underground/fedora/18/i386/')
self.assertEqual(urls[2], 'file:///underground/fedora/19/x86_64/')
self.assertEqual(urls[3], 'file:///underground/fedora/19/i386/')
def __init__(self, path=None, threaded=True):
"""
:param path: The absolute path to a directory containing
content source descriptor files.
:type path: str
:param threaded: Whether or not to use the threaded download method.
:type threaded: bool
"""
self.sources = ContentSource.load_all(path)
self.threaded = threaded
def __init__(self, path=None, threaded=True):
"""
:param path: The absolute path to a directory containing
content source descriptor files.
:type path: str
:param threaded: Whether or not to use the threaded download method.
:type threaded: bool
"""
self.sources = ContentSource.load_all(path)
self.threaded = threaded
def test_refresh(self, fake_manager, fake_load):
sources = {}
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.refresh = Mock(return_value=[n])
s.get_downloader = Mock()
sources[s.id] = s
fake_manager().has_entries.return_value = False
fake_load.return_value = sources
# test
container = ContentContainer('')
report = container.refresh()
# validation
for s in sources.values():
s.refresh.assert_called_with()
self.assertEqual(sorted(report), [0, 1, 2])
def test_refresh(self, fake_manager, fake_load):
sources = {}
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.refresh = Mock(return_value=[n])
s.get_downloader = Mock()
sources[s.id] = s
fake_manager().has_entries.return_value = False
fake_load.return_value = sources
# test
container = ContentContainer('')
report = container.refresh()
# validation
for s in sources.values():
s.refresh.assert_called_with()
self.assertEqual(sorted(report), [0, 1, 2])
def test_refresh_raised(self, fake_manager, fake_load):
sources = {}
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.refresh = Mock(side_effect=ValueError('must be int'))
s.get_downloader = Mock()
sources[s.id] = s
fake_manager().has_entries.return_value = False
fake_load.return_value = sources
# test
container = ContentContainer('')
report = container.refresh()
# validation
for s in sources.values():
s.refresh.assert_called_with()
for r in report:
r.errors = ['must be int']
def test_refresh_raised(self, fake_manager, fake_load):
sources = {}
for n in range(3):
s = ContentSource('s-%d' % n, {})
s.refresh = Mock(side_effect=ValueError('must be int'))
s.get_downloader = Mock()
sources[s.id] = s
fake_manager().has_entries.return_value = False
fake_load.return_value = sources
# test
container = ContentContainer('')
report = container.refresh()
# validation
for s in sources.values():
s.refresh.assert_called_with()
for r in report:
r.errors = ['must be int']
def test_load_all_unreadable(self, mock_warn, mock_read, mock_listdir, mock_isfile):
conf_d = '/mock/conf_d/'
mock_listdir.return_value = ['somefile.conf']
# this return value indicates that the read did not succeed
mock_read.return_value = []
sources = ContentSource.load_all(conf_d)
# make sure no sources are returned, and a message was logged with the path
self.assertEqual(len(sources), 0)
self.assertEqual(mock_warn.call_count, 1)
self.assertEqual(os.path.join(conf_d, 'somefile.conf'), mock_warn.call_args[0][1])
def test_urls_no_paths(self):
base_url = 'http://xyz.com'
source = ContentSource('s-1', {constants.BASE_URL: base_url})
# test
urls = source.urls
# validation
expected = [
'http://xyz.com',
]
self.assertEqual(urls, expected)
def test_refresh(self, fake_urls):
url = 'http://xyz.com'
urls = ['url-1', 'url-2']
fake_urls.__get__ = Mock(return_value=urls)
conduit = Mock()
cataloger = Mock()
cataloger.refresh.side_effect = FakeRefresh()
source = ContentSource('s-1', {constants.BASE_URL: url})
source.get_conduit = Mock(return_value=conduit)
source.get_cataloger = Mock(return_value=cataloger)
# test
report = source.refresh()
# validation
self.assertEqual(conduit.reset.call_count, len(urls))
self.assertEqual(cataloger.refresh.call_count, len(urls))
n = 0
added = 10
deleted = 1
for _url in source.urls:
cataloger.refresh.assert_any(conduit, source.descriptor, _url)
self.assertEqual(report[n].source_id, source.id)
self.assertEqual(report[n].url, _url)
self.assertTrue(report[n].succeeded)
self.assertEqual(report[n].errors, [])
self.assertEqual(report[n].added_count, added)
self.assertEqual(report[n].deleted_count, deleted)
added += 10
deleted += 1
n += 1
def test_urls(self):
base_url = 'http://xyz.com'
paths = 'path1/ path2 path3/ \\\npath4'
source = ContentSource('s-1', {constants.BASE_URL: base_url, constants.PATHS: paths})
# test
urls = source.urls
# validation
expected = [
'http://xyz.com/path1/',
'http://xyz.com/path2/',
'http://xyz.com/path3/',
'http://xyz.com/path4/',
]
self.assertEqual(urls, expected)
def test_refresh(self, mock_plugin):
container = ContentContainer(path=self.tmp_dir)
# test
report = container.refresh(force=True)
# validation
plugin = mock_plugin.return_value[0]
self.assertEqual(plugin.refresh.call_count, 5)
self.assertEqual(len(report), 5)
for r in report:
self.assertTrue(r.succeeded)
self.assertEqual(r.added_count, 100)
self.assertEqual(r.deleted_count, 0)
calls = iter(plugin.refresh.call_args_list)
for source in ContentSource.load_all(self.tmp_dir).values():
for url in source.urls:
args = calls.next()[0]
self.assertTrue(isinstance(args[0], CatalogerConduit))
self.assertEqual(args[1], source.descriptor)
self.assertEqual(args[2], url)
def setUp(self):
self.conduit = mock.MagicMock()
self.step = ContentSourcesRefreshStep(self.conduit)
self.source = ContentSource('foo', {'name': 'foo'})
self.report = RefreshReport('foo', 'http://foo.com')
self.report.succeeded = True
def test_loading(self):
sources = ContentSource.load_all(self.tmp_dir)
self.assertEqual(len(sources), 2)