def find(self, type_id, unit_key):
"""
Find entries in the content catalog using the specified unit type_id
and unit_key. The catalog may contain more than one entry matching the
locator for a given content source. In this case, only the newest entry
for each source is included in the result set.
:param type_id: The unit type ID.
:type type_id: str
:param unit_key: The unit key.
:type unit_key: dict
:return: A list of matching entries.
:rtype: list
"""
collection = ContentCatalog.get_collection()
locator = ContentCatalog.get_locator(type_id, unit_key)
query = {
'locator': locator,
'expiration': {
'$gte': ContentCatalog.get_expiration(0)
}
}
newest_by_source = {}
for entry in collection.find(query, sort=[('_id', ASCENDING)]):
newest_by_source[entry['source_id']] = entry
return newest_by_source.values()
def setUp(self):
super(ContainerTest, self).setUp()
ContentCatalog.get_collection().remove()
self.tmp_dir = mkdtemp()
self.downloaded = os.path.join(self.tmp_dir, 'downloaded')
os.makedirs(self.downloaded)
self.add_sources()
plugins._create_manager()
plugins._MANAGER.catalogers.add_plugin('yum', FakeCataloger, {})
def setUp(self):
super(ContainerTest, self).setUp()
ContentCatalog.get_collection().remove()
self.tmp_dir = mkdtemp()
self.downloaded = os.path.join(self.tmp_dir, 'downloaded')
os.makedirs(self.downloaded)
self.add_sources()
plugins._create_manager()
plugins._MANAGER.catalogers.add_plugin('yum', FakeCataloger, {})
def setUp(self):
PulpAsyncServerTests.setUp(self)
ContentCatalog.get_collection().remove()
self.tmp_dir = mkdtemp()
self.downloaded = os.path.join(self.tmp_dir, 'downloaded')
os.makedirs(self.downloaded)
self.add_sources()
MockListener.download_started.reset_mock()
MockListener.download_succeeded.reset_mock()
MockListener.download_failed.reset_mock()
plugins._create_manager()
plugins._MANAGER.catalogers.add_plugin('yum', MockCataloger, {})
def test_locator(self):
key_1 = {'a': 1, 'b': 2, 'c': 3}
key_2 = {'c': 3, 'b': 2, 'a': 1}
key_3 = {'c': 1, 'b': 2, 'a': 3}
locator_1 = ContentCatalog.get_locator(TYPE_ID, key_1) # eq
locator_2 = ContentCatalog.get_locator(TYPE_ID, key_2) # eq
locator_3 = ContentCatalog.get_locator(TYPE_ID, key_3) # neq
locator_4 = ContentCatalog.get_locator(TYPE_ID[1:], key_1) # neq
self.assertTrue(isinstance(locator_1, str))
self.assertTrue(isinstance(locator_2, str))
self.assertEqual(locator_1, locator_2)
self.assertNotEqual(locator_1, locator_3)
self.assertNotEqual(locator_1, locator_4)
def test_add(self):
units = self.units(0, 10)
manager = ContentCatalogManager()
for unit_key, url in units:
manager.add_entry(SOURCE_ID, EXPIRATION, TYPE_ID, unit_key, url)
collection = ContentCatalog.get_collection()
self.assertEqual(len(units), collection.find().count())
for unit_key, url in units:
locator = ContentCatalog.get_locator(TYPE_ID, unit_key)
entry = collection.find_one({'locator': locator})
self.assertEqual(entry['type_id'], TYPE_ID)
self.assertEqual(entry['unit_key'], unit_key)
self.assertEqual(entry['url'], url)
def test_add(self):
units = self.units(0, 10)
manager = ContentCatalogManager()
for unit_key, url in units:
manager.add_entry(SOURCE_ID, EXPIRATION, TYPE_ID, unit_key, url)
collection = ContentCatalog.get_collection()
self.assertEqual(len(units), collection.find().count())
for unit_key, url in units:
locator = ContentCatalog.get_locator(TYPE_ID, unit_key)
entry = collection.find_one({'locator': locator})
self.assertEqual(entry['type_id'], TYPE_ID)
self.assertEqual(entry['unit_key'], unit_key)
self.assertEqual(entry['url'], url)
def test_locator(self):
key_1 = {'a': 1, 'b': 2, 'c': 3}
key_2 = {'c': 3, 'b': 2, 'a': 1}
key_3 = {'c': 1, 'b': 2, 'a': 3}
locator_1 = ContentCatalog.get_locator(TYPE_ID, key_1) # eq
locator_2 = ContentCatalog.get_locator(TYPE_ID, key_2) # eq
locator_3 = ContentCatalog.get_locator(TYPE_ID, key_3) # neq
locator_4 = ContentCatalog.get_locator(TYPE_ID[1:], key_1) # neq
self.assertTrue(isinstance(locator_1, str))
self.assertTrue(isinstance(locator_2, str))
self.assertEqual(locator_1, locator_2)
self.assertNotEqual(locator_1, locator_3)
self.assertNotEqual(locator_1, locator_4)
def delete_entry(self, source_id, type_id, unit_key):
"""
Delete an entry from the content catalog.
:param source_id: A content source ID.
:type source_id: str
:param type_id: The unit type ID.
:type type_id: str
:param unit_key: The unit key.
:type unit_key: dict
"""
collection = ContentCatalog.get_collection()
locator = ContentCatalog.get_locator(type_id, unit_key)
query = {'source_id': source_id, 'locator': locator}
collection.remove(query, safe=True)
def delete_entry(self, source_id, type_id, unit_key):
"""
Delete an entry from the content catalog.
:param source_id: A content source ID.
:type source_id: str
:param type_id: The unit type ID.
:type type_id: str
:param unit_key: The unit key.
:type unit_key: dict
"""
collection = ContentCatalog.get_collection()
locator = ContentCatalog.get_locator(type_id, unit_key)
query = {'source_id': source_id, 'locator': locator}
collection.remove(query, safe=True)
def purge_expired(self, grace_period=GRACE_PERIOD):
"""
Purge (delete) expired entries from the content catalog belonging
to the specified content source by ID.
:param grace_period: The grace period in seconds.
The grace_period defines how long an entry is permitted to remain
in the catalog after it has expired. The default is 1 hour.
:type grace_period: int
"""
collection = ContentCatalog.get_collection()
now = ContentCatalog.get_expiration(0)
timestamp = now - grace_period
query = {'expiration': {'$lt': timestamp}}
collection.remove(query, safe=True)
def purge_expired(self, grace_period=GRACE_PERIOD):
"""
Purge (delete) expired entries from the content catalog belonging
to the specified content source by ID.
:param grace_period: The grace period in seconds.
The grace_period defines how long an entry is permitted to remain
in the catalog after it has expired. The default is 1 hour.
:type grace_period: int
"""
collection = ContentCatalog.get_collection()
now = ContentCatalog.get_expiration(0)
timestamp = now - grace_period
query = {'expiration': {'$lt': timestamp}}
collection.remove(query, safe=True)
def has_entries(self, source_id):
"""
Get whether the specified content source has entries in the catalog.
:param source_id: A content source ID.
:type source_id: str
:return: True if has entries.
:rtype: bool
"""
collection = ContentCatalog.get_collection()
query = {
'source_id': source_id,
'expiration': {'$gte': ContentCatalog.get_expiration(0)}
}
cursor = collection.find(query)
return cursor.count() > 0
def has_entries(self, source_id):
"""
Get whether the specified content source has entries in the catalog.
:param source_id: A content source ID.
:type source_id: str
:return: True if has entries.
:rtype: bool
"""
collection = ContentCatalog.get_collection()
query = {
'source_id': source_id,
'expiration': {'$gte': ContentCatalog.get_expiration(0)}
}
cursor = collection.find(query)
return cursor.count() > 0
def test_add(self):
units = self.units(0, 10)
conduit = CatalogerConduit(SOURCE_ID, EXPIRES)
for unit_key, url in units:
conduit.add_entry(TYPE_ID, unit_key, url)
collection = ContentCatalog.get_collection()
self.assertEqual(conduit.source_id, SOURCE_ID)
self.assertEqual(conduit.expires, EXPIRES)
self.assertEqual(len(units), collection.find().count())
self.assertEqual(conduit.added_count, len(units))
self.assertEqual(conduit.deleted_count, 0)
for unit_key, url in units:
locator = ContentCatalog.get_locator(TYPE_ID, unit_key)
entry = collection.find_one({"locator": locator})
self.assertEqual(entry["type_id"], TYPE_ID)
self.assertEqual(entry["unit_key"], unit_key)
self.assertEqual(entry["url"], url)
def add_entry(self, source_id, expires, type_id, unit_key, url):
"""
Add an entry to the content catalog.
:param source_id: A content source ID.
:type source_id: str
:param expires: The entry expiration in seconds.
:type expires: int
:param type_id: The unit type ID.
:type type_id: str
:param unit_key: The unit key.
:type unit_key: dict
:param url: The download URL.
:type url: str
"""
collection = ContentCatalog.get_collection()
entry = ContentCatalog(source_id, expires, type_id, unit_key, url)
collection.insert(entry, safe=True)
def populate_catalog(self, source_id, n_start, n_units, checksum='0xAA'):
_dir = self.populate_content(source_id, n_start, n_units)
collection = ContentCatalog.get_collection()
entry_list = []
for n in range(n_start, n_start + n_units):
unit_key = {
'name': 'unit_%d' % n,
'version': '1.0.%d' % n,
'release': '1',
'checksum': checksum
}
url = 'file://%s/unit_%d' % (_dir, n)
entry = ContentCatalog(source_id, EXPIRES, TYPE_ID, unit_key, url)
entry_list.append(entry)
for entry in entry_list:
collection.insert(entry, safe=True)
return _dir, entry_list
def test_add(self):
units = self.units(0, 10)
conduit = CatalogerConduit(SOURCE_ID, EXPIRES)
for unit_key, url in units:
conduit.add_entry(TYPE_ID, unit_key, url)
collection = ContentCatalog.get_collection()
self.assertEqual(conduit.source_id, SOURCE_ID)
self.assertEqual(conduit.expires, EXPIRES)
self.assertEqual(len(units), collection.find().count())
self.assertEqual(conduit.added_count, len(units))
self.assertEqual(conduit.deleted_count, 0)
for unit_key, url in units:
locator = ContentCatalog.get_locator(TYPE_ID, unit_key)
entry = collection.find_one({'locator': locator})
self.assertEqual(entry['type_id'], TYPE_ID)
self.assertEqual(entry['unit_key'], unit_key)
self.assertEqual(entry['url'], url)
def purge(self, source_id):
"""
Purge (delete) entries from the content catalog belonging
to the specified content source by ID.
:param source_id: A content source ID.
:type source_id: str
"""
collection = ContentCatalog.get_collection()
query = {'source_id': source_id}
collection.remove(query, safe=True)
def test_delete(self):
units = self.units(0, 10)
conduit = CatalogerConduit(SOURCE_ID, EXPIRES)
for unit_key, url in units:
conduit.add_entry(TYPE_ID, unit_key, url)
collection = ContentCatalog.get_collection()
self.assertEqual(len(units), collection.find().count())
unit_key, url = units[5]
locator = ContentCatalog.get_locator(TYPE_ID, unit_key)
entry = collection.find_one({"locator": locator})
self.assertEqual(entry["type_id"], TYPE_ID)
self.assertEqual(entry["unit_key"], unit_key)
self.assertEqual(entry["url"], url)
conduit.delete_entry(TYPE_ID, unit_key)
self.assertEqual(len(units) - 1, collection.find().count())
self.assertEqual(conduit.added_count, len(units))
self.assertEqual(conduit.deleted_count, 1)
entry = collection.find_one({"locator": locator})
self.assertTrue(entry is None)
def purge(self, source_id):
"""
Purge (delete) entries from the content catalog belonging
to the specified content source by ID.
:param source_id: A content source ID.
:type source_id: str
"""
collection = ContentCatalog.get_collection()
query = {'source_id': source_id}
collection.remove(query, safe=True)
def test_delete(self):
units = self.units(0, 10)
conduit = CatalogerConduit(SOURCE_ID, EXPIRES)
for unit_key, url in units:
conduit.add_entry(TYPE_ID, unit_key, url)
collection = ContentCatalog.get_collection()
self.assertEqual(len(units), collection.find().count())
unit_key, url = units[5]
locator = ContentCatalog.get_locator(TYPE_ID, unit_key)
entry = collection.find_one({'locator': locator})
self.assertEqual(entry['type_id'], TYPE_ID)
self.assertEqual(entry['unit_key'], unit_key)
self.assertEqual(entry['url'], url)
conduit.delete_entry(TYPE_ID, unit_key)
self.assertEqual(len(units) - 1, collection.find().count())
self.assertEqual(conduit.added_count, len(units))
self.assertEqual(conduit.deleted_count, 1)
entry = collection.find_one({'locator': locator})
self.assertTrue(entry is None)
def populate_catalog(self, source_id, n_start, n_units, checksum="0xAA"):
_dir = self.populate_content(source_id, n_start, n_units)
collection = ContentCatalog.get_collection()
entry_list = []
for n in range(n_start, n_start + n_units):
unit_key = {"name": "unit_%d" % n, "version": "1.0.%d" % n, "release": "1", "checksum": checksum}
url = "file://%s/unit_%d" % (_dir, n)
entry = ContentCatalog(source_id, EXPIRES, TYPE_ID, unit_key, url)
entry_list.append(entry)
for entry in entry_list:
collection.insert(entry)
return _dir, entry_list
def purge_orphans(self, valid_ids):
"""
Purge orphan entries from the content catalog.
Entries are orphaned when the content source to which they belong
is no longer loaded.
:param valid_ids: The list of valid (loaded) content source IDs.
:type valid_ids: list
"""
collection = ContentCatalog.get_collection()
for source_id in collection.distinct('source_id'):
if source_id not in valid_ids:
self.purge(source_id)
def test_purge_orphans(self):
_dir, cataloged = self.populate_catalog(ORPHANED, 0, 10)
_dir, cataloged = self.populate_catalog(UNDERGROUND, 0, 10)
_dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10)
collection = ContentCatalog.get_collection()
self.assertEqual(collection.find().count(), 30)
container = ContentContainer(path=self.tmp_dir)
container.purge_orphans()
self.assertEqual(collection.find().count(), 20)
self.assertEqual(collection.find({'source_id': ORPHANED}).count(), 0)
self.assertEqual(collection.find({'source_id': UNDERGROUND}).count(), 10)
self.assertEqual(collection.find({'source_id': UNIT_WORLD}).count(), 10)
def purge_orphans(self, valid_ids):
"""
Purge orphan entries from the content catalog.
Entries are orphaned when the content source to which they belong
is no longer loaded.
:param valid_ids: The list of valid (loaded) content source IDs.
:type valid_ids: list
"""
collection = ContentCatalog.get_collection()
for source_id in collection.distinct('source_id'):
if source_id not in valid_ids:
self.purge(source_id)
def purge(self, source_id):
"""
Purge (delete) entries from the content catalog belonging
to the specified content source by ID.
:param source_id: A content source ID.
:type source_id: str
:return: The number of entries purged.
:rtype: int
"""
collection = ContentCatalog.get_collection()
query = {'source_id': source_id}
result = collection.remove(query)
return result['n']
def test_refresh_exception(self, mock_refresh):
container = ContentContainer(path=self.tmp_dir)
event = Event()
report = container.refresh(event, force=True)
self.assertEqual(len(report), 2)
for r in report:
self.assertFalse(r.succeeded)
self.assertEqual(r.added_count, 0)
self.assertEqual(r.deleted_count, 0)
self.assertEqual(len(r.errors), 1)
collection = ContentCatalog.get_collection()
self.assertEqual(mock_refresh.call_count, 2)
self.assertEqual(collection.find().count(), 0)
def test_refresh_exception(self, mock_refresh):
container = ContentContainer(path=self.tmp_dir)
event = Event()
report = container.refresh(event, force=True)
self.assertEqual(len(report), 2)
for r in report:
self.assertFalse(r.succeeded)
self.assertEqual(r.added_count, 0)
self.assertEqual(r.deleted_count, 0)
self.assertEqual(len(r.errors), 1)
collection = ContentCatalog.get_collection()
self.assertEqual(mock_refresh.call_count, 2)
self.assertEqual(collection.find().count(), 0)
def find(self, type_id, unit_key):
"""
Find entries in the content catalog using the specified unit type_id
and unit_key. The catalog may contain more than one entry matching the
locator for a given content source. In this case, only the newest entry
for each source is included in the result set.
:param type_id: The unit type ID.
:type type_id: str
:param unit_key: The unit key.
:type unit_key: dict
:return: A list of matching entries.
:rtype: list
"""
collection = ContentCatalog.get_collection()
locator = ContentCatalog.get_locator(type_id, unit_key)
query = {
'locator': locator,
'expiration': {'$gte': ContentCatalog.get_expiration(0)}
}
newest_by_source = {}
for entry in collection.find(query, sort=[('_id', ASCENDING)]):
newest_by_source[entry['source_id']] = entry
return newest_by_source.values()
def test_purge(self):
source_a = 'A'
source_b = 'B'
manager = ContentCatalogManager()
for unit_key, url in self.units(0, 10):
manager.add_entry(source_a, EXPIRATION, TYPE_ID, unit_key, url)
for unit_key, url in self.units(0, 10):
manager.add_entry(source_b, EXPIRATION, TYPE_ID, unit_key, url)
collection = ContentCatalog.get_collection()
self.assertEqual(20, collection.find().count())
manager = ContentCatalogManager()
manager.purge(source_a)
self.assertEqual(collection.find({'source_id': source_a}).count(), 0)
self.assertEqual(collection.find({'source_id': source_b}).count(), 10)
def test_refresh_failure(self, mock_plugin):
container = ContentContainer(path=self.tmp_dir)
event = Event()
report = container.refresh(event, force=True)
self.assertEqual(len(report), 5)
for r in report:
self.assertFalse(r.succeeded)
self.assertEqual(r.added_count, 0)
self.assertEqual(r.deleted_count, 0)
self.assertEqual(len(r.errors), 1)
plugin = mock_plugin.return_value[0]
collection = ContentCatalog.get_collection()
self.assertEqual(plugin.refresh.call_count, 5)
self.assertEqual(collection.find().count(), 0)
def test_refresh_failure(self, mock_plugin):
container = ContentContainer(path=self.tmp_dir)
event = Event()
report = container.refresh(event, force=True)
self.assertEqual(len(report), 5)
for r in report:
self.assertFalse(r.succeeded)
self.assertEqual(r.added_count, 0)
self.assertEqual(r.deleted_count, 0)
self.assertEqual(len(r.errors), 1)
plugin = mock_plugin.return_value[0]
collection = ContentCatalog.get_collection()
self.assertEqual(plugin.refresh.call_count, 5)
self.assertEqual(collection.find().count(), 0)
def test_purge(self):
source_a = 'A'
source_b = 'B'
manager = ContentCatalogManager()
for unit_key, url in self.units(0, 10):
manager.add_entry(source_a, EXPIRATION, TYPE_ID, unit_key, url)
for unit_key, url in self.units(0, 10):
manager.add_entry(source_b, EXPIRATION, TYPE_ID, unit_key, url)
collection = ContentCatalog.get_collection()
self.assertEqual(20, collection.find().count())
manager = ContentCatalogManager()
manager.purge(source_a)
self.assertEqual(collection.find({'source_id': source_a}).count(), 0)
self.assertEqual(collection.find({'source_id': source_b}).count(), 10)
def test_purge_orphans(self):
_dir, cataloged = self.populate_catalog(ORPHANED, 0, 10)
_dir, cataloged = self.populate_catalog(UNDERGROUND, 0, 10)
_dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10)
collection = ContentCatalog.get_collection()
self.assertEqual(collection.find().count(), 30)
container = ContentContainer(path=self.tmp_dir)
# test
container.purge_orphans()
# validation
self.assertEqual(collection.find().count(), 20)
self.assertEqual(collection.find({'source_id': ORPHANED}).count(), 0)
self.assertEqual(collection.find({'source_id': UNDERGROUND}).count(), 10)
self.assertEqual(collection.find({'source_id': UNIT_WORLD}).count(), 10)
def populate_catalog(self, source_id, n_start, n_units, checksum='0xAA'):
_dir = self.populate_content(source_id, n_start, n_units)
collection = ContentCatalog.get_collection()
entry_list = []
for n in range(n_start, n_start + n_units):
unit_key = {
'name': 'unit_%d' % n,
'version': '1.0.%d' % n,
'release': '1',
'checksum': checksum
}
url = 'file://%s/unit_%d' % (_dir, n)
entry = ContentCatalog(source_id, EXPIRES, TYPE_ID, unit_key, url)
entry_list.append(entry)
for entry in entry_list:
collection.insert(entry, safe=True)
return _dir, entry_list
def add_entry(self, source_id, expires, type_id, unit_key, url):
"""
Add an entry to the content catalog.
:param source_id: A content source ID.
:type source_id: str
:param expires: The entry expiration in seconds.
:type expires: int
:param type_id: The unit type ID.
:type type_id: str
:param unit_key: The unit key.
:type unit_key: dict
:param url: The download URL.
:type url: str
"""
collection = ContentCatalog.get_collection()
entry = ContentCatalog(source_id, expires, type_id, unit_key, url)
collection.insert(entry, safe=True)
def test_has_entries(self):
source_a = 'A'
source_b = 'B'
source_c = 'C'
manager = ContentCatalogManager()
for unit_key, url in self.units(0, 10):
manager.add_entry(source_a, EXPIRATION, TYPE_ID, unit_key, url)
for unit_key, url in self.units(0, 10):
manager.add_entry(source_b, -1, TYPE_ID, unit_key, url)
for unit_key, url in self.units(0, 10):
manager.add_entry(source_c, EXPIRATION, TYPE_ID, unit_key, url)
collection = ContentCatalog.get_collection()
self.assertEqual(30, collection.find().count())
manager = ContentCatalogManager()
self.assertTrue(manager.has_entries(source_a))
self.assertFalse(manager.has_entries(source_b))
self.assertTrue(manager.has_entries(source_c))
manager.purge(source_c)
self.assertTrue(manager.has_entries(source_a))
self.assertFalse(manager.has_entries(source_b))
self.assertFalse(manager.has_entries(source_c))
def test_has_entries(self):
source_a = 'A'
source_b = 'B'
source_c = 'C'
manager = ContentCatalogManager()
for unit_key, url in self.units(0, 10):
manager.add_entry(source_a, EXPIRATION, TYPE_ID, unit_key, url)
for unit_key, url in self.units(0, 10):
manager.add_entry(source_b, -1, TYPE_ID, unit_key, url)
for unit_key, url in self.units(0, 10):
manager.add_entry(source_c, EXPIRATION, TYPE_ID, unit_key, url)
collection = ContentCatalog.get_collection()
self.assertEqual(30, collection.find().count())
manager = ContentCatalogManager()
self.assertTrue(manager.has_entries(source_a))
self.assertFalse(manager.has_entries(source_b))
self.assertTrue(manager.has_entries(source_c))
manager.purge(source_c)
self.assertTrue(manager.has_entries(source_a))
self.assertFalse(manager.has_entries(source_b))
self.assertFalse(manager.has_entries(source_c))
def tearDown(self):
super(ContainerTest, self).tearDown()
ContentCatalog.get_collection().remove()
shutil.rmtree(self.tmp_dir, ignore_errors=True)
plugins.finalize()
def setUp(self):
super(TestCatalogManager, self).setUp()
ContentCatalog.get_collection().remove()
def tearDown(self):
super(TestCatalogerConduit, self).tearDown()
ContentCatalog.get_collection().remove()
def tearDown(self):
super(TestCatalogManager, self).tearDown()
ContentCatalog.get_collection().remove()
def tearDown(self):
super(ContainerTest, self).tearDown()
ContentCatalog.get_collection().remove()
shutil.rmtree(self.tmp_dir, ignore_errors=True)
plugins.finalize()
def tearDown(self):
PulpAsyncServerTests.tearDown(self)
ContentCatalog.get_collection().remove()
def tearDown(self):
PulpAsyncServerTests.tearDown(self)
ContentCatalog.get_collection().remove()
shutil.rmtree(self.tmp_dir, ignore_errors=True)
plugins.finalize()
def setUp(self):
super(TestCatalogerConduit, self).setUp()
ContentCatalog.get_collection().remove()
def setUp(self):
PulpAsyncServerTests.setUp(self)
ContentCatalog.get_collection().remove()
