def cleanUpProductOnDepots(backend, depotIds, existingProductIdents):
"""
Deletes obsolete information that occurs if either a depot or a \
product is not existing anymore.
:param backend: The backend where the data should be cleaned.
:type backend: OPSI.Backend.Backend
:param depotIds: IDs of the existing depot.
:type depotIds: [str, ]
:param existingProductIdents: Idents of the existing products.
:type existingProductIdents: [str, ]
"""
deleteProductOnDepots = []
for productOnDepot in backend.productOnDepot_getObjects():
productIdent = ";".join([
productOnDepot.productId, productOnDepot.productVersion,
productOnDepot.packageVersion
])
if productOnDepot.depotId not in depotIds:
LOGGER.info(
u"Marking product on depot {poc} for deletion, because "
u"opsiDepot-Server '{depotId}' not found".format(
poc=productOnDepot, depotId=productOnDepot.depotId))
deleteProductOnDepots.append(productOnDepot)
elif productIdent not in existingProductIdents:
LOGGER.info(
u"Marking product on depot {0} with missing product reference "
u"for deletion".format(productOnDepot))
deleteProductOnDepots.append(productOnDepot)
if deleteProductOnDepots:
for productOnDepots in chunk(deleteProductOnDepots, _CHUNK_SIZE):
LOGGER.debug(
u"Deleting products on depots: {0!r}".format(productOnDepots))
backend.productOnDepot_deleteObjects(productOnDepots)
def cleanUpGroups(backend):
"""
This checks if a group has a parent set that does not exist and
removes non-existing parents.
:param backend: The backend where the data should be cleaned.
:type backend: OPSI.Backend.Backend
"""
updatedGroups = []
groups = backend.group_getObjects(type='HostGroup')
groupIds = set(group.id for group in groups)
for group in groups:
if group.getParentGroupId() and group.getParentGroupId(
) not in groupIds:
LOGGER.info(
u"Removing parent group id '{parentGroupId}' from group "
u"'{groupId}' because parent group does not exist".format(
parentGroupId=group.parentGroupId, groupId=group.id))
group.parentGroupId = None
updatedGroups.append(group)
if updatedGroups:
for group in chunk(updatedGroups, _CHUNK_SIZE):
LOGGER.debug(u"Updating groups: {0!r}".format(group))
backend.group_createObjects(group)
def cleanUpProductOnClients(backend):
"""
Delete :py:class:`ProductOnClient` if the client does not exist or \
is either *not_installed* without an action request set.
:param backend: The backend where the data should be cleaned.
:type backend: OPSI.Backend.Backend
"""
deleteProductOnClients = []
clientIds = set(client.id
for client in backend.host_getObjects(type=["OpsiClient"]))
for productOnClient in backend.productOnClient_getObjects():
if productOnClient.clientId not in clientIds:
LOGGER.info(u"Marking productOnClient {0} for deletion, client "
u"doesn't exists".format(productOnClient))
deleteProductOnClients.append(productOnClient)
elif (productOnClient.installationStatus == u'not_installed'
and productOnClient.actionRequest == u'none'):
LOGGER.info(u"Marking productOnClient {0} for "
u"deletion".format(productOnClient))
deleteProductOnClients.append(productOnClient)
if deleteProductOnClients:
for productOnClients in chunk(deleteProductOnClients, _CHUNK_SIZE):
LOGGER.debug(u"Deleting products on clients: {0!r}".format(
productOnClients))
backend.productOnClient_deleteObjects(productOnClients)
deleteProductOnClients = []
productIds = set(product.getId()
for product in backend.product_getObjects())
for productOnClient in backend.productOnClient_getObjects():
if productOnClient.productId not in productIds:
LOGGER.info(u"Marking productOnClient {0} for "
u"deletion".format(productOnClient))
deleteProductOnClients.append(productOnClient)
if deleteProductOnClients:
for productOnClients in chunk(deleteProductOnClients, _CHUNK_SIZE):
LOGGER.debug(u"Deleting products on clients: {0!r}".format(
productOnClients))
backend.productOnClient_deleteObjects(productOnClients)
def testChunkingList(): base = list(range(10)) chunks = chunk(base, size=3) assert (0, 1, 2) == next(chunks) assert (3, 4, 5) == next(chunks) assert (6, 7, 8) == next(chunks) assert (9, ) == next(chunks) with pytest.raises(StopIteration): next(chunks)
def testChunkingGeneratorWithDifferentSize(): def gen(): yield 0 yield 1 yield 2 yield 3 yield 4 yield 5 yield 6 yield 7 yield 8 yield 9 chunks = chunk(gen(), size=5) assert (0, 1, 2, 3, 4) == next(chunks) assert (5, 6, 7, 8, 9) == next(chunks) with pytest.raises(StopIteration): next(chunks)
def testChunkingGenerator(): def gen(): yield 0 yield 1 yield 2 yield 3 yield 4 yield 5 yield 6 yield 7 yield 8 yield 9 chunks = chunk(gen(), size=3) assert (0, 1, 2) == next(chunks) assert (3, 4, 5) == next(chunks) assert (6, 7, 8) == next(chunks) assert (9, ) == next(chunks) with pytest.raises(StopIteration): next(chunks)
def cleanUpConfigStates(backend):
"""
Deletes configStates if the corresponding config is nonexisting.
:param backend: The backend where the data should be cleaned.
:type backend: OPSI.Backend.Backend
"""
deleteConfigStates = []
configIds = set(backend.config_getIdents())
for configState in backend.configState_getObjects():
if configState.configId not in configIds:
LOGGER.info(
u"Marking configState {configState} of non existent config "
u"'{config}' for deletion".format(configState=configState,
config=configState.configId))
deleteConfigStates.append(configState)
if deleteConfigStates:
for configStates in chunk(deleteConfigStates, _CHUNK_SIZE):
LOGGER.debug(u"Deleting config states: {0!r}".format(configStates))
backend.configState_deleteObjects(configStates)
def cleanUpProducts(backend):
"""
This will delete any unreferenced product from the backend.
:param backend: The backend where the data should be cleaned.
:type backend: OPSI.Backend.Backend
"""
productIdents = set()
for productOnDepot in backend.productOnDepot_getObjects():
productIdent = ";".join(
(productOnDepot.productId, productOnDepot.productVersion,
productOnDepot.packageVersion))
productIdents.add(productIdent)
deleteProducts = []
for product in backend.product_getObjects():
if product.getIdent(returnType='unicode') not in productIdents:
LOGGER.info(u"Marking unreferenced product {0} for deletion",
product)
deleteProducts.append(product)
for products in chunk(deleteProducts, _CHUNK_SIZE):
LOGGER.debug(u"Deleting products: {0!r}", products)
backend.product_deleteObjects(products)
def cleanupBackend(backend=None):
"""
Clean up data from your backends.
This method uses different cleanup methods to ensure that no
obsolete data is present in your backend.
:param backend: the backend to check. If ``None`` this will create a \
BackendManager from default paths.
:type backend: OPSI.Backend.Backend
"""
def usesMysqlBackend():
LOGGER.notice(u"Parsing dispatch.conf")
bdc = BackendDispatchConfigFile(
u'/etc/opsi/backendManager/dispatch.conf')
dispatchConfig = bdc.parse()
for entry in dispatchConfig:
(regex, backends) = entry
if not re.search(regex, u'backend_createBase'):
continue
if 'mysql' in backends:
return True
return False
LOGGER.debug("Cleaning backend chunk size: {0}".format(_CHUNK_SIZE))
if backend is None:
backend = BackendManager(
dispatchConfigFile=u'/etc/opsi/backendManager/dispatch.conf',
backendConfigDir=u'/etc/opsi/backends',
extensionConfigDir=u'/etc/opsi/backendManager/extend.d',
depotbackend=False)
try:
if usesMysqlBackend():
LOGGER.notice(
u"Mysql-backend detected. Trying to cleanup mysql-backend first"
)
# ToDo: backendConfigFile should be as dynamic as possible
# What if we have 2 mysql backends set up?
cleanUpMySQL()
except Exception as error:
LOGGER.warning(error)
LOGGER.notice(u"Cleaning up groups")
cleanUpGroups(backend)
LOGGER.notice(u"Cleaning up products")
cleanUpProducts(backend)
LOGGER.debug(u'Getting current depots...')
depotIds = set(depot.id for depot in backend.host_getObjects(
type=["OpsiConfigserver", "OpsiDepotserver"])) # pylint: disable=maybe-no-member
LOGGER.debug(u'Depots are: {0}'.format(depotIds))
LOGGER.debug(u'Getting current products...')
productIdents = set(
product.getIdent(returnType='unicode')
for product in backend.product_getObjects())
LOGGER.debug(u'Product idents are: {0}'.format(productIdents))
LOGGER.notice(u"Cleaning up product on depots")
cleanUpProductOnDepots(backend, depotIds, productIdents)
LOGGER.notice(u"Cleaning up product on clients")
cleanUpProductOnClients(backend)
LOGGER.notice(u"Cleaning up product properties")
productPropertyIdents = set()
deleteProductProperties = []
productPropertiesToCleanup = {}
for productProperty in backend.productProperty_getObjects(): # pylint: disable=maybe-no-member
productIdent = u"%s;%s;%s" % (productProperty.productId,
productProperty.productVersion,
productProperty.packageVersion)
if not productProperty.editable and productProperty.possibleValues:
productPropertyIdent = u"%s;%s" % (productIdent,
productProperty.propertyId)
productPropertiesToCleanup[productPropertyIdent] = productProperty
if productIdent not in productIdents:
LOGGER.info(
u"Marking productProperty %s of non existent product '%s' for deletion"
% (productProperty, productIdent))
deleteProductProperties.append(productProperty)
else:
productPropertyIdent = u'%s;%s' % (productProperty.productId,
productProperty.propertyId)
productPropertyIdents.add(productPropertyIdent)
if deleteProductProperties:
for productProperties in chunk(deleteProductProperties, _CHUNK_SIZE):
LOGGER.debug(u"Deleting product properties: {0!r}".format(
productProperties))
backend.productProperty_deleteObjects(productProperties) # pylint: disable=maybe-no-member
LOGGER.notice(u"Cleaning up product property states")
deleteProductPropertyStates = []
for productPropertyState in backend.productPropertyState_getObjects(): # pylint: disable=maybe-no-member
productPropertyIdent = u'%s;%s' % (productPropertyState.productId,
productPropertyState.propertyId)
if productPropertyIdent not in productPropertyIdents:
LOGGER.info(
u"Marking productPropertyState %s of non existent productProperty '%s' for deletion"
% (productPropertyState, productPropertyIdent))
deleteProductPropertyStates.append(productPropertyState)
if deleteProductPropertyStates:
for productPropertyStates in chunk(deleteProductPropertyStates,
_CHUNK_SIZE):
LOGGER.debug(u"Deleting product property states: {0!r}".format(
productPropertyStates))
backend.productPropertyState_deleteObjects(productPropertyStates) # pylint: disable=maybe-no-member
for depot in backend.host_getObjects(type='OpsiDepotserver'): # pylint: disable=maybe-no-member
objectIds = set(
ClientToDepot['clientId']
for ClientToDepot in backend.configState_getClientToDepotserver(
depotIds=depot.id))
objectIds.add(depot.id)
productOnDepotIdents = {}
for productOnDepot in backend.productOnDepot_getObjects(
depotId=depot.id): # pylint: disable=maybe-no-member
productIdent = u"%s;%s;%s" % (productOnDepot.productId,
productOnDepot.productVersion,
productOnDepot.packageVersion)
productOnDepotIdents[productOnDepot.productId] = productIdent
if not productOnDepotIdents:
continue
deleteProductPropertyStates = []
updateProductPropertyStates = []
for productPropertyState in backend.productPropertyState_getObjects( # pylint: disable=maybe-no-member
objectId=objectIds,
productId=productOnDepotIdents.keys(),
propertyId=[]):
productIdent = productOnDepotIdents.get(
productPropertyState.productId)
if not productIdent:
continue
productPropertyIdent = u"%s;%s" % (productIdent,
productPropertyState.propertyId)
productProperty = productPropertiesToCleanup.get(
productPropertyIdent)
if not productProperty:
continue
changed = False
newValues = []
removeValues = []
changedValues = []
for value in productPropertyState.values:
if value in productProperty.possibleValues:
newValues.append(value)
continue
if productProperty.getType(
) == u'BoolProductProperty' and forceBool(
value) in productProperty.possibleValues:
newValues.append(forceBool(value))
changedValues.append(value)
changed = True
continue
if productProperty.getType() == u'UnicodeProductProperty':
newValue = None
for possibleValue in productProperty.possibleValues:
if forceUnicodeLower(
possibleValue) == forceUnicodeLower(value):
newValue = possibleValue
break
if newValue:
newValues.append(newValue)
changedValues.append(value)
changed = True
continue
removeValues.append(value)
changed = True
if changed:
if not newValues:
LOGGER.info(
u"Marking productPropertyState %s for deletion: no value in possible values (%s)"
% (productPropertyState, removeValues))
deleteProductPropertyStates.append(productPropertyState)
else:
productPropertyState.setValues(newValues)
LOGGER.info(
u"Marking productPropertyState %s for update: values not in possible values: %s, values corrected: %s"
% (productPropertyState, removeValues, changedValues))
updateProductPropertyStates.append(productPropertyState)
if deleteProductPropertyStates:
for productPropertyStates in chunk(deleteProductPropertyStates,
_CHUNK_SIZE):
LOGGER.debug(u"Deleting product property states: {0!r}".format(
productPropertyStates))
backend.productPropertyState_deleteObjects(
productPropertyStates) # pylint: disable=maybe-no-member
del deleteProductPropertyStates
if updateProductPropertyStates:
for productPropertyStates in chunk(updateProductPropertyStates,
_CHUNK_SIZE):
LOGGER.debug(u"Updating product property states: {0!r}".format(
productPropertyStates))
backend.productPropertyState_updateObjects(
productPropertyStates) # pylint: disable=maybe-no-member
del updateProductPropertyStates
LOGGER.notice(u"Cleaning up config states")
cleanUpConfigStates(backend)
LOGGER.notice(u"Cleaning up audit softwares")
cleanUpAuditSoftwares(backend)
LOGGER.notice(u"Cleaning up audit software on clients")
cleanUpAuditSoftwareOnClients(backend)