def find(self, type_id, unit_key): """ Find entries in the content catalog using the specified unit type_id and unit_key. The catalog may contain more than one entry matching the locator for a given content source. In this case, only the newest entry for each source is included in the result set. :param type_id: The unit type ID. :type type_id: str :param unit_key: The unit key. :type unit_key: dict :return: A list of matching entries. :rtype: list """ collection = ContentCatalog.get_collection() locator = ContentCatalog.get_locator(type_id, unit_key) query = { 'locator': locator, 'expiration': { '$gte': ContentCatalog.get_expiration(0) } } newest_by_source = {} for entry in collection.find(query, sort=[('_id', ASCENDING)]): newest_by_source[entry['source_id']] = entry return newest_by_source.values()
def setUp(self): super(ContainerTest, self).setUp() ContentCatalog.get_collection().remove() self.tmp_dir = mkdtemp() self.downloaded = os.path.join(self.tmp_dir, 'downloaded') os.makedirs(self.downloaded) self.add_sources() plugins._create_manager() plugins._MANAGER.catalogers.add_plugin('yum', FakeCataloger, {})
def setUp(self): PulpAsyncServerTests.setUp(self) ContentCatalog.get_collection().remove() self.tmp_dir = mkdtemp() self.downloaded = os.path.join(self.tmp_dir, 'downloaded') os.makedirs(self.downloaded) self.add_sources() MockListener.download_started.reset_mock() MockListener.download_succeeded.reset_mock() MockListener.download_failed.reset_mock() plugins._create_manager() plugins._MANAGER.catalogers.add_plugin('yum', MockCataloger, {})
def test_locator(self): key_1 = {'a': 1, 'b': 2, 'c': 3} key_2 = {'c': 3, 'b': 2, 'a': 1} key_3 = {'c': 1, 'b': 2, 'a': 3} locator_1 = ContentCatalog.get_locator(TYPE_ID, key_1) # eq locator_2 = ContentCatalog.get_locator(TYPE_ID, key_2) # eq locator_3 = ContentCatalog.get_locator(TYPE_ID, key_3) # neq locator_4 = ContentCatalog.get_locator(TYPE_ID[1:], key_1) # neq self.assertTrue(isinstance(locator_1, str)) self.assertTrue(isinstance(locator_2, str)) self.assertEqual(locator_1, locator_2) self.assertNotEqual(locator_1, locator_3) self.assertNotEqual(locator_1, locator_4)
def test_add(self): units = self.units(0, 10) manager = ContentCatalogManager() for unit_key, url in units: manager.add_entry(SOURCE_ID, EXPIRATION, TYPE_ID, unit_key, url) collection = ContentCatalog.get_collection() self.assertEqual(len(units), collection.find().count()) for unit_key, url in units: locator = ContentCatalog.get_locator(TYPE_ID, unit_key) entry = collection.find_one({'locator': locator}) self.assertEqual(entry['type_id'], TYPE_ID) self.assertEqual(entry['unit_key'], unit_key) self.assertEqual(entry['url'], url)
def delete_entry(self, source_id, type_id, unit_key): """ Delete an entry from the content catalog. :param source_id: A content source ID. :type source_id: str :param type_id: The unit type ID. :type type_id: str :param unit_key: The unit key. :type unit_key: dict """ collection = ContentCatalog.get_collection() locator = ContentCatalog.get_locator(type_id, unit_key) query = {'source_id': source_id, 'locator': locator} collection.remove(query, safe=True)
def purge_expired(self, grace_period=GRACE_PERIOD): """ Purge (delete) expired entries from the content catalog belonging to the specified content source by ID. :param grace_period: The grace period in seconds. The grace_period defines how long an entry is permitted to remain in the catalog after it has expired. The default is 1 hour. :type grace_period: int """ collection = ContentCatalog.get_collection() now = ContentCatalog.get_expiration(0) timestamp = now - grace_period query = {'expiration': {'$lt': timestamp}} collection.remove(query, safe=True)
def has_entries(self, source_id): """ Get whether the specified content source has entries in the catalog. :param source_id: A content source ID. :type source_id: str :return: True if has entries. :rtype: bool """ collection = ContentCatalog.get_collection() query = { 'source_id': source_id, 'expiration': {'$gte': ContentCatalog.get_expiration(0)} } cursor = collection.find(query) return cursor.count() > 0
def test_add(self): units = self.units(0, 10) conduit = CatalogerConduit(SOURCE_ID, EXPIRES) for unit_key, url in units: conduit.add_entry(TYPE_ID, unit_key, url) collection = ContentCatalog.get_collection() self.assertEqual(conduit.source_id, SOURCE_ID) self.assertEqual(conduit.expires, EXPIRES) self.assertEqual(len(units), collection.find().count()) self.assertEqual(conduit.added_count, len(units)) self.assertEqual(conduit.deleted_count, 0) for unit_key, url in units: locator = ContentCatalog.get_locator(TYPE_ID, unit_key) entry = collection.find_one({"locator": locator}) self.assertEqual(entry["type_id"], TYPE_ID) self.assertEqual(entry["unit_key"], unit_key) self.assertEqual(entry["url"], url)
def add_entry(self, source_id, expires, type_id, unit_key, url): """ Add an entry to the content catalog. :param source_id: A content source ID. :type source_id: str :param expires: The entry expiration in seconds. :type expires: int :param type_id: The unit type ID. :type type_id: str :param unit_key: The unit key. :type unit_key: dict :param url: The download URL. :type url: str """ collection = ContentCatalog.get_collection() entry = ContentCatalog(source_id, expires, type_id, unit_key, url) collection.insert(entry, safe=True)
def populate_catalog(self, source_id, n_start, n_units, checksum='0xAA'): _dir = self.populate_content(source_id, n_start, n_units) collection = ContentCatalog.get_collection() entry_list = [] for n in range(n_start, n_start + n_units): unit_key = { 'name': 'unit_%d' % n, 'version': '1.0.%d' % n, 'release': '1', 'checksum': checksum } url = 'file://%s/unit_%d' % (_dir, n) entry = ContentCatalog(source_id, EXPIRES, TYPE_ID, unit_key, url) entry_list.append(entry) for entry in entry_list: collection.insert(entry, safe=True) return _dir, entry_list
def test_add(self): units = self.units(0, 10) conduit = CatalogerConduit(SOURCE_ID, EXPIRES) for unit_key, url in units: conduit.add_entry(TYPE_ID, unit_key, url) collection = ContentCatalog.get_collection() self.assertEqual(conduit.source_id, SOURCE_ID) self.assertEqual(conduit.expires, EXPIRES) self.assertEqual(len(units), collection.find().count()) self.assertEqual(conduit.added_count, len(units)) self.assertEqual(conduit.deleted_count, 0) for unit_key, url in units: locator = ContentCatalog.get_locator(TYPE_ID, unit_key) entry = collection.find_one({'locator': locator}) self.assertEqual(entry['type_id'], TYPE_ID) self.assertEqual(entry['unit_key'], unit_key) self.assertEqual(entry['url'], url)
def purge(self, source_id): """ Purge (delete) entries from the content catalog belonging to the specified content source by ID. :param source_id: A content source ID. :type source_id: str """ collection = ContentCatalog.get_collection() query = {'source_id': source_id} collection.remove(query, safe=True)
def test_delete(self): units = self.units(0, 10) conduit = CatalogerConduit(SOURCE_ID, EXPIRES) for unit_key, url in units: conduit.add_entry(TYPE_ID, unit_key, url) collection = ContentCatalog.get_collection() self.assertEqual(len(units), collection.find().count()) unit_key, url = units[5] locator = ContentCatalog.get_locator(TYPE_ID, unit_key) entry = collection.find_one({"locator": locator}) self.assertEqual(entry["type_id"], TYPE_ID) self.assertEqual(entry["unit_key"], unit_key) self.assertEqual(entry["url"], url) conduit.delete_entry(TYPE_ID, unit_key) self.assertEqual(len(units) - 1, collection.find().count()) self.assertEqual(conduit.added_count, len(units)) self.assertEqual(conduit.deleted_count, 1) entry = collection.find_one({"locator": locator}) self.assertTrue(entry is None)
def test_delete(self): units = self.units(0, 10) conduit = CatalogerConduit(SOURCE_ID, EXPIRES) for unit_key, url in units: conduit.add_entry(TYPE_ID, unit_key, url) collection = ContentCatalog.get_collection() self.assertEqual(len(units), collection.find().count()) unit_key, url = units[5] locator = ContentCatalog.get_locator(TYPE_ID, unit_key) entry = collection.find_one({'locator': locator}) self.assertEqual(entry['type_id'], TYPE_ID) self.assertEqual(entry['unit_key'], unit_key) self.assertEqual(entry['url'], url) conduit.delete_entry(TYPE_ID, unit_key) self.assertEqual(len(units) - 1, collection.find().count()) self.assertEqual(conduit.added_count, len(units)) self.assertEqual(conduit.deleted_count, 1) entry = collection.find_one({'locator': locator}) self.assertTrue(entry is None)
def populate_catalog(self, source_id, n_start, n_units, checksum="0xAA"): _dir = self.populate_content(source_id, n_start, n_units) collection = ContentCatalog.get_collection() entry_list = [] for n in range(n_start, n_start + n_units): unit_key = {"name": "unit_%d" % n, "version": "1.0.%d" % n, "release": "1", "checksum": checksum} url = "file://%s/unit_%d" % (_dir, n) entry = ContentCatalog(source_id, EXPIRES, TYPE_ID, unit_key, url) entry_list.append(entry) for entry in entry_list: collection.insert(entry) return _dir, entry_list
def purge_orphans(self, valid_ids): """ Purge orphan entries from the content catalog. Entries are orphaned when the content source to which they belong is no longer loaded. :param valid_ids: The list of valid (loaded) content source IDs. :type valid_ids: list """ collection = ContentCatalog.get_collection() for source_id in collection.distinct('source_id'): if source_id not in valid_ids: self.purge(source_id)
def test_purge_orphans(self): _dir, cataloged = self.populate_catalog(ORPHANED, 0, 10) _dir, cataloged = self.populate_catalog(UNDERGROUND, 0, 10) _dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10) collection = ContentCatalog.get_collection() self.assertEqual(collection.find().count(), 30) container = ContentContainer(path=self.tmp_dir) container.purge_orphans() self.assertEqual(collection.find().count(), 20) self.assertEqual(collection.find({'source_id': ORPHANED}).count(), 0) self.assertEqual(collection.find({'source_id': UNDERGROUND}).count(), 10) self.assertEqual(collection.find({'source_id': UNIT_WORLD}).count(), 10)
def purge(self, source_id): """ Purge (delete) entries from the content catalog belonging to the specified content source by ID. :param source_id: A content source ID. :type source_id: str :return: The number of entries purged. :rtype: int """ collection = ContentCatalog.get_collection() query = {'source_id': source_id} result = collection.remove(query) return result['n']
def test_refresh_exception(self, mock_refresh): container = ContentContainer(path=self.tmp_dir) event = Event() report = container.refresh(event, force=True) self.assertEqual(len(report), 2) for r in report: self.assertFalse(r.succeeded) self.assertEqual(r.added_count, 0) self.assertEqual(r.deleted_count, 0) self.assertEqual(len(r.errors), 1) collection = ContentCatalog.get_collection() self.assertEqual(mock_refresh.call_count, 2) self.assertEqual(collection.find().count(), 0)
def find(self, type_id, unit_key): """ Find entries in the content catalog using the specified unit type_id and unit_key. The catalog may contain more than one entry matching the locator for a given content source. In this case, only the newest entry for each source is included in the result set. :param type_id: The unit type ID. :type type_id: str :param unit_key: The unit key. :type unit_key: dict :return: A list of matching entries. :rtype: list """ collection = ContentCatalog.get_collection() locator = ContentCatalog.get_locator(type_id, unit_key) query = { 'locator': locator, 'expiration': {'$gte': ContentCatalog.get_expiration(0)} } newest_by_source = {} for entry in collection.find(query, sort=[('_id', ASCENDING)]): newest_by_source[entry['source_id']] = entry return newest_by_source.values()
def test_purge(self): source_a = 'A' source_b = 'B' manager = ContentCatalogManager() for unit_key, url in self.units(0, 10): manager.add_entry(source_a, EXPIRATION, TYPE_ID, unit_key, url) for unit_key, url in self.units(0, 10): manager.add_entry(source_b, EXPIRATION, TYPE_ID, unit_key, url) collection = ContentCatalog.get_collection() self.assertEqual(20, collection.find().count()) manager = ContentCatalogManager() manager.purge(source_a) self.assertEqual(collection.find({'source_id': source_a}).count(), 0) self.assertEqual(collection.find({'source_id': source_b}).count(), 10)
def test_refresh_failure(self, mock_plugin): container = ContentContainer(path=self.tmp_dir) event = Event() report = container.refresh(event, force=True) self.assertEqual(len(report), 5) for r in report: self.assertFalse(r.succeeded) self.assertEqual(r.added_count, 0) self.assertEqual(r.deleted_count, 0) self.assertEqual(len(r.errors), 1) plugin = mock_plugin.return_value[0] collection = ContentCatalog.get_collection() self.assertEqual(plugin.refresh.call_count, 5) self.assertEqual(collection.find().count(), 0)
def test_purge_orphans(self): _dir, cataloged = self.populate_catalog(ORPHANED, 0, 10) _dir, cataloged = self.populate_catalog(UNDERGROUND, 0, 10) _dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10) collection = ContentCatalog.get_collection() self.assertEqual(collection.find().count(), 30) container = ContentContainer(path=self.tmp_dir) # test container.purge_orphans() # validation self.assertEqual(collection.find().count(), 20) self.assertEqual(collection.find({'source_id': ORPHANED}).count(), 0) self.assertEqual(collection.find({'source_id': UNDERGROUND}).count(), 10) self.assertEqual(collection.find({'source_id': UNIT_WORLD}).count(), 10)
def test_has_entries(self): source_a = 'A' source_b = 'B' source_c = 'C' manager = ContentCatalogManager() for unit_key, url in self.units(0, 10): manager.add_entry(source_a, EXPIRATION, TYPE_ID, unit_key, url) for unit_key, url in self.units(0, 10): manager.add_entry(source_b, -1, TYPE_ID, unit_key, url) for unit_key, url in self.units(0, 10): manager.add_entry(source_c, EXPIRATION, TYPE_ID, unit_key, url) collection = ContentCatalog.get_collection() self.assertEqual(30, collection.find().count()) manager = ContentCatalogManager() self.assertTrue(manager.has_entries(source_a)) self.assertFalse(manager.has_entries(source_b)) self.assertTrue(manager.has_entries(source_c)) manager.purge(source_c) self.assertTrue(manager.has_entries(source_a)) self.assertFalse(manager.has_entries(source_b)) self.assertFalse(manager.has_entries(source_c))
def tearDown(self): super(ContainerTest, self).tearDown() ContentCatalog.get_collection().remove() shutil.rmtree(self.tmp_dir, ignore_errors=True) plugins.finalize()
def setUp(self): super(TestCatalogManager, self).setUp() ContentCatalog.get_collection().remove()
def tearDown(self): super(TestCatalogerConduit, self).tearDown() ContentCatalog.get_collection().remove()
def tearDown(self): super(TestCatalogManager, self).tearDown() ContentCatalog.get_collection().remove()
def tearDown(self): PulpAsyncServerTests.tearDown(self) ContentCatalog.get_collection().remove()
def tearDown(self): PulpAsyncServerTests.tearDown(self) ContentCatalog.get_collection().remove() shutil.rmtree(self.tmp_dir, ignore_errors=True) plugins.finalize()
def setUp(self): super(TestCatalogerConduit, self).setUp() ContentCatalog.get_collection().remove()
def setUp(self): PulpAsyncServerTests.setUp(self) ContentCatalog.get_collection().remove()