Esempio n. 1
0
    def test_sorting(self):
        s1 = ContentSource('s-1', {constants.PRIORITY: 2})
        s2 = ContentSource('s-2', {constants.PRIORITY: 1})
        s3 = ContentSource('s-3', {constants.PRIORITY: 0})

        _list = sorted([s1, s2, s3])
        self.assertEqual([s.id for s in _list], [s3.id, s2.id, s1.id])
Esempio n. 2
0
    def test_refresh_raised(self, fake_urls):
        url = 'http://xyz.com'
        urls = ['url-1', 'url-2']
        fake_urls.__get__ = Mock(return_value=urls)

        canceled = Mock()
        canceled.isSet = Mock(return_value=False)
        conduit = Mock()
        cataloger = Mock()
        cataloger.refresh.side_effect = ValueError('just failed')

        source = ContentSource('s-1', {constants.BASE_URL: url})
        source.get_conduit = Mock(return_value=conduit)
        source.get_cataloger = Mock(return_value=cataloger)

        # test

        report = source.refresh(canceled)

        # validation

        self.assertEqual(canceled.isSet.call_count, len(urls))
        self.assertEqual(conduit.reset.call_count, len(urls))
        self.assertEqual(cataloger.refresh.call_count, len(urls))

        n = 0
        for _url in source.urls:
            cataloger.refresh.assert_any(conduit, source.descriptor, _url)
            self.assertEqual(report[n].source_id, source.id)
            self.assertEqual(report[n].url, _url)
            self.assertFalse(report[n].succeeded)
            self.assertEqual(report[n].errors, ['just failed'])
            self.assertEqual(report[n].added_count, 0)
            self.assertEqual(report[n].deleted_count, 0)
            n += 1
Esempio n. 3
0
 def test_enabled(self):
     source = ContentSource('s-1', {constants.ENABLED: 'true'})
     self.assertTrue(source.enabled)
     source.descriptor[constants.ENABLED] = 'false'
     self.assertFalse(source.enabled)
     source.descriptor = {}
     self.assertFalse(source.enabled)
Esempio n. 4
0
    def test_find_sources(self, fake_manager):
        type_id = 'test_1'
        unit_key = 1
        destination = '/tmp/123'
        url = 'http://redhat.com/repository'

        primary = PrimarySource(None)
        alternatives = dict([(s, ContentSource(s, d)) for s, d in DESCRIPTOR])
        fake_manager().find.return_value = CATALOG

        # test

        request = Request(type_id, unit_key, url, destination)
        request.find_sources(primary, alternatives)

        # validation

        # validate sources sorted by priority with the primary last.
        # should only have matched on s-1 and s-3.

        self.assertEqual(len(request.sources), 5)
        self.assertEqual(request.sources[0][0].id, 's-3')
        self.assertEqual(request.sources[0][1], CATALOG[2][constants.URL])
        self.assertEqual(request.sources[1][0].id, 's-3')
        self.assertEqual(request.sources[1][1], CATALOG[3][constants.URL])
        self.assertEqual(request.sources[2][0].id, 's-1')
        self.assertEqual(request.sources[2][1], CATALOG[0][constants.URL])
        self.assertEqual(request.sources[3][0].id, 's-1')
        self.assertEqual(request.sources[3][1], CATALOG[1][constants.URL])
        self.assertEqual(request.sources[4][0].id, primary.id)
        self.assertEqual(request.sources[4][1], url)
Esempio n. 5
0
    def test_refresh_canceled(self, fake_urls):
        url = 'http://xyz.com'
        urls = ['url-1', 'url-2']

        fake_urls.__get__ = Mock(return_value=urls)

        canceled = Mock()
        canceled.isSet = Mock(return_value=True)
        conduit = Mock()
        cataloger = Mock()

        source = ContentSource('s-1', {constants.BASE_URL: url})
        source.get_conduit = Mock(return_value=conduit)
        source.get_cataloger = Mock(return_value=cataloger)

        # test

        report = source.refresh(canceled)

        # validation

        self.assertEqual(canceled.isSet.call_count, 1)
        self.assertEqual(conduit.reset.call_count, 0)
        self.assertEqual(cataloger.refresh.call_count, 0)
        self.assertEqual(report, [])
Esempio n. 6
0
    def test_conduit(self):
        source = ContentSource('s-1', {constants.EXPIRES: '1h'})

        conduit = source.get_conduit()

        self.assertEqual(conduit.source_id, source.id)
        self.assertEqual(conduit.expires, 3600)
        self.assertTrue(isinstance(conduit, CatalogerConduit))
Esempio n. 7
0
    def test_construction(self):
        # test
        source = ContentSource(DESCRIPTOR[0][0], DESCRIPTOR[0][1])

        # validation

        self.assertEqual(source.id, DESCRIPTOR[0][0])
        self.assertEqual(source.descriptor, DESCRIPTOR[0][1])
Esempio n. 8
0
    def test_download(self, fake_load):
        sources = []
        for n in range(3):
            s = ContentSource('s-%d' % n, {})
            s.get_downloader = Mock()
            sources.append(s)

        fake_load.return_value = sources

        request_list = []
        for n in range(6):
            r = Request('T', {}, 'url-%d' % n, 'path-%d' % n)
            r.find_sources = Mock(return_value=sources[n % 3:])
            request_list.append(r)

        collated = [{
            sources[0]: ['nectar-1'],
            sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'],
            sources[2]: ['nectar-5', 'nectar-6']
        }, {}]
        fake_collated = Mock(side_effect=collated)

        fake_listener = Mock()
        canceled = FakeEvent()
        fake_primary = PrimarySource(Mock())

        # test
        container = ContentContainer('')
        container.refresh = Mock()
        container.collated = fake_collated
        report = container.download(canceled, fake_primary, request_list,
                                    fake_listener)

        # validation
        container.refresh.assert_called_with(canceled)

        for r in request_list:
            r.find_sources.assert_called_with(fake_primary, container.sources)

        self.assertEqual(report.total_passes, 1)
        self.assertEqual(report.total_sources, len(sources))
        self.assertEqual(len(report.downloads), 3)
        for source in sources:
            self.assertEqual(report.downloads[source.id].total_succeeded, 0)
            self.assertEqual(report.downloads[source.id].total_failed, 0)

        for source in sources:
            source.get_downloader.assert_called_with()
            downloader = source.get_downloader()
            listener = downloader.event_listener
            self.assertEqual(listener.cancel_event, canceled)
            self.assertEqual(listener.downloader, downloader)
            self.assertEqual(listener.listener, fake_listener)
            downloader.download.assert_called_with(collated[0][source])
Esempio n. 9
0
    def test_dict(self):
        descriptor = {'A': 1, 'B': 2}

        # test
        source = ContentSource('s-1', descriptor)

        # validation
        expected = {}
        expected.update(descriptor)
        expected[constants.SOURCE_ID] = source.id
        self.assertEqual(source.dict(), expected)
Esempio n. 10
0
    def test_is_valid_bad_descriptor(self, mock_descriptor_is_valid):
        source = ContentSource('s-1', {'A': 1})
        source.get_downloader = Mock()
        source.get_cataloger = Mock()
        mock_descriptor_is_valid.side_effect = ValueError()

        # Test

        valid = source.is_valid()

        # validation

        self.assertFalse(valid)
Esempio n. 11
0
    def test_cataloger(self, fake_plugins):
        plugin = Mock()
        fake_plugins.get_cataloger_by_id.return_value = plugin, {}

        # test

        source = ContentSource('s-1', {constants.TYPE: 1234})
        cataloger = source.get_cataloger()

        # validation

        fake_plugins.get_cataloger_by_id.assert_called_with(1234)
        self.assertEqual(plugin, cataloger)
Esempio n. 12
0
    def test_download_canceled_after_collated(self, fake_load):
        sources = []
        for n in range(3):
            s = ContentSource('s-%d' % n, {})
            s.get_downloader = Mock()
            sources.append(s)

        fake_load.return_value = sources

        request_list = []
        for n in range(6):
            r = Request('T', {}, 'url-%d' % n, 'path-%d' % n)
            r.find_sources = Mock(return_value=sources[n % 3:])
            request_list.append(r)

        collated = [{
            sources[0]: ['nectar-1'],
            sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'],
            sources[2]: ['nectar-5', 'nectar-6']
        }, {}]
        fake_collated = Mock(side_effect=collated)

        fake_listener = Mock()
        canceled = Mock()
        canceled.isSet.side_effect = [False, True, True]
        fake_primary = PrimarySource(Mock())

        # test
        container = ContentContainer('')
        container.refresh = Mock()
        container.collated = fake_collated
        report = container.download(canceled, fake_primary, request_list,
                                    fake_listener)

        # validation
        container.refresh.assert_called_with(canceled)

        for r in request_list:
            r.find_sources.assert_called_with(fake_primary, container.sources)

        called = 0
        for s in sources:
            if s.get_downloader.called:
                called += 1

        self.assertEqual(called, 1)
        self.assertEqual(report.total_passes, 1)
        self.assertEqual(report.total_sources, len(sources))
        self.assertEqual(len(report.downloads), 1)
        self.assertEqual(report.downloads[sources[2].id].total_succeeded, 0)
        self.assertEqual(report.downloads[sources[2].id].total_failed, 0)
Esempio n. 13
0
    def test_urls_no_paths(self):
        base_url = 'http://xyz.com'
        source = ContentSource('s-1', {constants.BASE_URL: base_url})

        # test
        urls = source.urls

        # validation

        expected = [
            'http://xyz.com',
        ]

        self.assertEqual(urls, expected)
Esempio n. 14
0
    def test_is_valid_no_downloader(self, mock_descriptor_is_valid):
        source = ContentSource('s-1', {'A': 1})
        source.get_downloader = Mock(side_effect=NotImplementedError())
        source.get_cataloger = Mock()

        # Test

        valid = source.is_valid()

        # validation

        source.get_cataloger.assert_called_with()
        source.get_downloader.assert_called_with()
        self.assertFalse(mock_descriptor_is_valid.called)
        self.assertFalse(valid)
Esempio n. 15
0
    def test_is_valid(self, mock_descriptor_is_valid):
        source = ContentSource('s-1', {'A': 1})
        source.get_downloader = Mock()
        source.get_cataloger = Mock()

        # Test

        valid = source.is_valid()

        # validation

        source.get_cataloger.assert_called_with()
        source.get_downloader.assert_called_with()
        mock_descriptor_is_valid.assert_called_with(source.id, source.descriptor)
        self.assertTrue(valid)
Esempio n. 16
0
    def test_forced_refresh(self, fake_manager, fake_load):
        sources = {}
        for n in range(3):
            s = ContentSource('s-%d' % n, {})
            s.refresh = Mock()
            sources[s.id] = s

        fake_manager().has_entries.return_value = True
        fake_load.return_value = sources

        # test
        container = ContentContainer('')
        container.refresh(force=True)

        # validation
        for s in sources.values():
            s.refresh.assert_called_with()
Esempio n. 17
0
    def test_downloader(self):
        url = 'http://xyz.com'
        fake_conduit = Mock()
        fake_cataloger = Mock()
        fake_downloader = Mock()
        fake_cataloger.get_downloader = Mock(return_value=fake_downloader)

        source = ContentSource('s-1', {constants.BASE_URL: url})
        source.get_conduit = Mock(return_value=fake_conduit)
        source.get_cataloger = Mock(return_value=fake_cataloger)

        # test
        downloader = source.get_downloader()

        # validation
        source.get_cataloger.assert_called_with()
        fake_cataloger.get_downloader.assert_called_with(fake_conduit, source.descriptor, url)
        self.assertEqual(downloader, fake_downloader)
Esempio n. 18
0
    def test_urls(self):
        base_url = 'http://xyz.com'
        paths = 'path1/ path2 path3/ \\\npath4'
        source = ContentSource('s-1', {constants.BASE_URL: base_url, constants.PATHS: paths})

        # test
        urls = source.urls

        # validation

        expected = [
            'http://xyz.com/path1/',
            'http://xyz.com/path2/',
            'http://xyz.com/path3/',
            'http://xyz.com/path4/',
        ]

        self.assertEqual(urls, expected)
Esempio n. 19
0
    def test_refresh_canceled(self, fake_load):
        sources = {}
        for n in range(3):
            s = ContentSource('s-%d' % n, {})
            s.refresh = Mock()
            sources[s.id] = s

        fake_load.return_value = sources

        # test
        canceled = FakeEvent()
        canceled.set()
        container = ContentContainer('')
        container.refresh(canceled, force=True)

        # validation
        for s in sources.values():
            self.assertFalse(s.refresh.called)
Esempio n. 20
0
    def test_refresh(self, fake_manager, fake_load):
        sources = {}
        for n in range(3):
            s = ContentSource('s-%d' % n, {})
            s.refresh = Mock(return_value=[n])
            s.get_downloader = Mock()
            sources[s.id] = s

        fake_manager().has_entries.return_value = False
        fake_load.return_value = sources

        # test
        container = ContentContainer('')
        report = container.refresh()

        # validation
        for s in sources.values():
            s.refresh.assert_called_with()

        self.assertEqual(sorted(report), [0, 1, 2])
Esempio n. 21
0
    def test_refresh_raised(self, fake_manager, fake_load):
        sources = {}
        for n in range(3):
            s = ContentSource('s-%d' % n, {})
            s.refresh = Mock(side_effect=ValueError('must be int'))
            s.get_downloader = Mock()
            sources[s.id] = s

        fake_manager().has_entries.return_value = False
        fake_load.return_value = sources

        # test
        container = ContentContainer('')
        report = container.refresh()

        # validation
        for s in sources.values():
            s.refresh.assert_called_with()

        for r in report:
            r.errors = ['must be int']
Esempio n. 22
0
    def test_refresh(self, fake_urls):
        url = 'http://xyz.com'
        urls = ['url-1', 'url-2']
        fake_urls.__get__ = Mock(return_value=urls)

        conduit = Mock()
        cataloger = Mock()
        cataloger.refresh.side_effect = FakeRefresh()

        source = ContentSource('s-1', {constants.BASE_URL: url})
        source.get_conduit = Mock(return_value=conduit)
        source.get_cataloger = Mock(return_value=cataloger)

        # test

        report = source.refresh()

        # validation

        self.assertEqual(conduit.reset.call_count, len(urls))
        self.assertEqual(cataloger.refresh.call_count, len(urls))

        n = 0
        added = 10
        deleted = 1
        for _url in source.urls:
            cataloger.refresh.assert_any(conduit, source.descriptor, _url)
            self.assertEqual(report[n].source_id, source.id)
            self.assertEqual(report[n].url, _url)
            self.assertTrue(report[n].succeeded)
            self.assertEqual(report[n].errors, [])
            self.assertEqual(report[n].added_count, added)
            self.assertEqual(report[n].deleted_count, deleted)
            added += 10
            deleted += 1
            n += 1
Esempio n. 23
0
 def setUp(self):
     self.conduit = mock.MagicMock()
     self.step = ContentSourcesRefreshStep(self.conduit)
     self.source = ContentSource('foo', {'name': 'foo'})
     self.report = RefreshReport('foo', 'http://foo.com')
     self.report.succeeded = True
Esempio n. 24
0
 def test_max_concurrent(self):
     source = ContentSource('s-1', {constants.MAX_CONCURRENT: 123})
     self.assertEqual(source.max_concurrent, 123)
Esempio n. 25
0
 def test_lt_(self):
     s1 = ContentSource('s-1', {constants.PRIORITY: 0})
     s2 = ContentSource('s-2', {constants.PRIORITY: 1})
     self.assertTrue(s1 < s2)
Esempio n. 26
0
 def test_hash_(self):
     s1 = ContentSource('s-1', {})
     self.assertEqual(hash(s1), hash(s1.id))
Esempio n. 27
0
 def test_neq_(self):
     s1 = ContentSource('s-1', {})
     s2 = ContentSource('s-2', {})
     self.assertTrue(s1 != s2)
Esempio n. 28
0
 def test_priority(self):
     source = ContentSource('s-1', {constants.PRIORITY: 123})
     self.assertEqual(source.priority, 123)
Esempio n. 29
0
 def test_expires(self):
     source = ContentSource('s-1', {constants.EXPIRES: '1h'})
     self.assertEqual(source.expires, 3600)
Esempio n. 30
0
 def test_base_url(self):
     source = ContentSource('s-1', {constants.BASE_URL: 'http://xyz.com'})
     self.assertEqual(source.base_url, 'http://xyz.com')