def Archive(filename, format=None, compression=None, progressSubject=None):
filename = forceFilename(filename)
Class = None
if format:
format = forceUnicodeLower(format)
if format == 'tar':
Class = TarArchive
elif format == 'cpio':
Class = CpioArchive
else:
raise ValueError(u"Unsupported format '%s'" % format)
elif os.path.exists(filename):
fileType = getFileType(filename)
if 'tar archive' in fileType.lower():
Class = TarArchive
elif 'cpio archive' in fileType.lower():
Class = CpioArchive
elif filename.lower().endswith(('tar', 'tar.gz')):
Class = TarArchive
elif filename.lower().endswith(('cpio', 'cpio.gz')):
Class = CpioArchive
if not Class:
raise RuntimeError(u"Failed to guess archive type of '%s'" % filename)
return Class(filename=filename,
compression=compression,
progressSubject=progressSubject)
def __init__(self, filename, compression=None, progressSubject=None):
self._filename = forceFilename(filename)
self._progressSubject = progressSubject
self._compression = None
if compression:
compression = forceUnicodeLower(compression)
if compression not in ('gzip', 'bzip2'):
raise ValueError(u"Compression '%s' not supported" %
compression)
self._compression = compression
elif os.path.exists(self._filename):
fileType = getFileType(self._filename)
if fileType.lower().startswith('gzip compressed data'):
self._compression = u'gzip'
elif fileType.lower().startswith('bzip2 compressed data'):
self._compression = u'bzip2'
else:
self._compression = None
def getSharedConnectionPool(scheme, host, port, **kw):
scheme = forceUnicodeLower(scheme)
host = forceUnicode(host)
port = forceInt(port)
poolKey = u'httplib:%s:%d' % (host, port)
global connectionPools
if poolKey not in connectionPools:
if scheme in ('https', 'webdavs'):
connectionPools[poolKey] = HTTPSConnectionPool(host,
port=port,
**kw)
else:
connectionPools[poolKey] = HTTPConnectionPool(host,
port=port,
**kw)
else:
connectionPools[poolKey].increaseUsageCount()
maxsize = kw.get('maxsize', 0)
if maxsize > connectionPools[poolKey].maxsize:
connectionPools[poolKey].adjustSize(maxsize)
return connectionPools[poolKey]
def getEventConfigs(): # pylint: disable=too-many-locals,too-many-branches,too-many-statements
preconditions = {}
for (section, options) in config.getDict().items():
section = section.lower()
if section.startswith('precondition_'):
preconditionId = section.split('_', 1)[1]
preconditions[preconditionId] = {}
try:
for key in options.keys():
if forceBool(options[key]):
# Only check if value in precondition is true
# false means: do not check state
preconditions[preconditionId][key] = True
logger.info("Precondition '%s' created: %s", preconditionId, preconditions[preconditionId])
except Exception as err: # pylint: disable=broad-except
logger.error("Failed to parse precondition '%s': %s", preconditionId, err)
rawEventConfigs = {}
for (section, options) in config.getDict().items(): # pylint: disable=too-many-nested-blocks
section = section.lower()
if section.startswith('event_'):
eventConfigId = section.split('_', 1)[1]
if not eventConfigId:
logger.error("No event config id defined in section '%s'", section)
continue
rawEventConfigs[eventConfigId] = {
'active': True,
'args': {
'name': eventConfigId.split("{")[0]
},
'super': None,
'precondition': None
}
try:
for key in options.keys():
if key.lower() == 'active':
rawEventConfigs[eventConfigId]['active'] = str(options[key]).lower() not in ('0', 'false', 'off', 'no')
elif key.lower() == 'super':
rawEventConfigs[eventConfigId]['super'] = options[key]
if rawEventConfigs[eventConfigId]['super'].startswith('event_'):
rawEventConfigs[eventConfigId]['super'] = rawEventConfigs[eventConfigId]['super'].split('_', 1)[1]
else:
rawEventConfigs[eventConfigId]['args'][key.lower()] = options[key]
if '{' in eventConfigId:
superEventName, precondition = eventConfigId.split('{', 1)
if not rawEventConfigs[eventConfigId]['super']:
rawEventConfigs[eventConfigId]['super'] = superEventName.strip()
rawEventConfigs[eventConfigId]['precondition'] = precondition.replace('}', '').strip()
except Exception as err: # pylint: disable=broad-except
logger.error("Failed to parse event config '%s': %s", eventConfigId, err)
# Process inheritance
newRawEventConfigs = {}
while rawEventConfigs:
num_configs = len(rawEventConfigs)
for eventConfigId in sorted(list(rawEventConfigs)):
rawEventConfig = rawEventConfigs[eventConfigId]
if rawEventConfig['super']:
if rawEventConfig['super'] in newRawEventConfigs:
super_args = pycopy.deepcopy(newRawEventConfigs[rawEventConfig['super']]['args'])
# Do not overwrite values with emptystring or emptylist (behaves like no value given)
cleaned_args = {
key : value
for key, value in rawEventConfig['args'].items()
if not (
key in ("include_product_group_ids", "exclude_product_group_ids")
and value in ("", [])
)
}
super_args.update(cleaned_args)
rawEventConfig['args'] = super_args
logger.debug("Inheritance for event '%s' processed", eventConfigId)
newRawEventConfigs[eventConfigId] = rawEventConfig
rawEventConfigs.pop(eventConfigId)
elif rawEventConfig['super'] not in rawEventConfigs:
logger.error("Super event '%s' not found", rawEventConfig['super'])
rawEventConfigs.pop(eventConfigId)
else:
logger.debug("Inheritance for event '%s' processed", eventConfigId)
newRawEventConfigs[eventConfigId] = rawEventConfig
rawEventConfigs.pop(eventConfigId)
if num_configs == len(rawEventConfigs):
logger.error("Failed to process event inheritance: %s", rawEventConfigs)
break
rawEventConfigs = newRawEventConfigs
eventConfigs = {}
for (eventConfigId, rawEventConfig) in rawEventConfigs.items(): # pylint: disable=too-many-nested-blocks
try:
if rawEventConfig['args'].get('type', 'template').lower() == 'template':
continue
eventConfigs[eventConfigId] = {
'active': rawEventConfig['active'],
'preconditions': {}
}
if rawEventConfig.get('precondition'):
precondition = preconditions.get(rawEventConfig['precondition'])
if not precondition:
logger.error(
"Precondition '%s' referenced by event config '%s' not found, deactivating event",
precondition, eventConfigId
)
eventConfigs[eventConfigId]['active'] = False
else:
eventConfigs[eventConfigId]['preconditions'] = precondition
for (key, value) in rawEventConfig['args'].items():
try:
if key == 'type':
eventConfigs[eventConfigId]['type'] = value
elif key == 'wql':
eventConfigs[eventConfigId]['wql'] = value
elif key.startswith(('action_message', 'message')):
try:
mLanguage = key.split('[')[1].split(']')[0].strip().lower()
except Exception: # pylint: disable=broad-except
mLanguage = None
if mLanguage:
if mLanguage == getLanguage():
eventConfigs[eventConfigId]['actionMessage'] = value
elif not eventConfigs[eventConfigId].get('actionMessage'):
eventConfigs[eventConfigId]['actionMessage'] = value
elif key.startswith('shutdown_warning_message'):
try:
mLanguage = key.split('[')[1].split(']')[0].strip().lower()
except Exception: # pylint: disable=broad-except
mLanguage = None
if mLanguage:
if mLanguage == getLanguage():
eventConfigs[eventConfigId]['shutdownWarningMessage'] = value
elif not eventConfigs[eventConfigId].get('shutdownWarningMessage'):
eventConfigs[eventConfigId]['shutdownWarningMessage'] = value
elif key.startswith('name'):
try:
mLanguage = key.split('[')[1].split(']')[0].strip().lower()
except Exception: # pylint: disable=broad-except
mLanguage = None
if mLanguage:
if mLanguage == getLanguage():
eventConfigs[eventConfigId]['name'] = value
else:
eventConfigs[eventConfigId]['name'] = value
elif key == 'start_interval':
eventConfigs[eventConfigId]['startInterval'] = int(value)
elif key == 'interval':
eventConfigs[eventConfigId]['interval'] = int(value)
elif key == 'max_repetitions':
eventConfigs[eventConfigId]['maxRepetitions'] = int(value)
elif key == 'activation_delay':
eventConfigs[eventConfigId]['activationDelay'] = int(value)
elif key == 'notification_delay':
eventConfigs[eventConfigId]['notificationDelay'] = int(value)
elif key == 'action_warning_time':
eventConfigs[eventConfigId]['actionWarningTime'] = int(value)
elif key == 'action_user_cancelable':
eventConfigs[eventConfigId]['actionUserCancelable'] = int(value)
elif key == 'shutdown':
eventConfigs[eventConfigId]['shutdown'] = forceBool(value)
elif key == 'reboot':
eventConfigs[eventConfigId]['reboot'] = forceBool(value)
elif key == 'shutdown_warning_time':
eventConfigs[eventConfigId]['shutdownWarningTime'] = int(value)
elif key == 'shutdown_warning_repetition_time':
eventConfigs[eventConfigId]['shutdownWarningRepetitionTime'] = int(value)
elif key == 'shutdown_user_cancelable':
eventConfigs[eventConfigId]['shutdownUserCancelable'] = int(value)
elif key == 'shutdown_user_selectable_time':
eventConfigs[eventConfigId]['shutdownUserSelectableTime'] = forceBool(value)
elif key == 'shutdown_warning_time_after_time_select':
eventConfigs[eventConfigId]['shutdownWarningTimeAfterTimeSelect'] = int(value)
elif key == 'block_login':
eventConfigs[eventConfigId]['blockLogin'] = forceBool(value)
elif key == 'lock_workstation':
eventConfigs[eventConfigId]['lockWorkstation'] = forceBool(value)
elif key == 'logoff_current_user':
eventConfigs[eventConfigId]['logoffCurrentUser'] = forceBool(value)
elif key == 'process_shutdown_requests':
eventConfigs[eventConfigId]['processShutdownRequests'] = forceBool(value)
elif key == 'get_config_from_service':
eventConfigs[eventConfigId]['getConfigFromService'] = forceBool(value)
elif key == 'update_config_file':
eventConfigs[eventConfigId]['updateConfigFile'] = forceBool(value)
elif key == 'write_log_to_service':
eventConfigs[eventConfigId]['writeLogToService'] = forceBool(value)
elif key == 'cache_products':
eventConfigs[eventConfigId]['cacheProducts'] = forceBool(value)
elif key == 'cache_max_bandwidth':
eventConfigs[eventConfigId]['cacheMaxBandwidth'] = int(value)
elif key == 'cache_dynamic_bandwidth':
eventConfigs[eventConfigId]['cacheDynamicBandwidth'] = forceBool(value)
elif key == 'use_cached_products':
eventConfigs[eventConfigId]['useCachedProducts'] = forceBool(value)
elif key == 'sync_config_from_server':
eventConfigs[eventConfigId]['syncConfigFromServer'] = forceBool(value)
elif key == 'sync_config_to_server':
eventConfigs[eventConfigId]['syncConfigToServer'] = forceBool(value)
elif key == 'use_cached_config':
eventConfigs[eventConfigId]['useCachedConfig'] = forceBool(value)
elif key == 'update_action_processor':
eventConfigs[eventConfigId]['updateActionProcessor'] = forceBool(value)
elif key == 'action_type':
eventConfigs[eventConfigId]['actionType'] = forceUnicodeLower(value)
elif key == 'event_notifier_command':
eventConfigs[eventConfigId]['eventNotifierCommand'] = config.replace(forceUnicodeLower(value), escaped=True)
elif key == 'event_notifier_desktop':
eventConfigs[eventConfigId]['eventNotifierDesktop'] = forceUnicodeLower(value)
elif key == 'process_actions':
eventConfigs[eventConfigId]['processActions'] = forceBool(value)
elif key == 'action_notifier_command':
eventConfigs[eventConfigId]['actionNotifierCommand'] = config.replace(forceUnicodeLower(value), escaped=True)
elif key == 'action_notifier_desktop':
eventConfigs[eventConfigId]['actionNotifierDesktop'] = forceUnicodeLower(value)
elif key == 'action_processor_command':
eventConfigs[eventConfigId]['actionProcessorCommand'] = forceUnicodeLower(value)
elif key == 'action_processor_desktop':
eventConfigs[eventConfigId]['actionProcessorDesktop'] = forceUnicodeLower(value)
elif key == 'action_processor_timeout':
eventConfigs[eventConfigId]['actionProcessorTimeout'] = int(value)
elif key == 'trusted_installer_detection':
eventConfigs[eventConfigId]['trustedInstallerDetection'] = forceBool(value)
elif key == 'shutdown_notifier_command':
eventConfigs[eventConfigId]['shutdownNotifierCommand'] = config.replace(forceUnicodeLower(value), escaped=True)
elif key == 'shutdown_notifier_desktop':
eventConfigs[eventConfigId]['shutdownNotifierDesktop'] = forceUnicodeLower(value)
elif key == 'pre_action_processor_command':
eventConfigs[eventConfigId]['preActionProcessorCommand'] = config.replace(forceUnicodeLower(value), escaped=True)
elif key == 'post_action_processor_command':
eventConfigs[eventConfigId]['postActionProcessorCommand'] = config.replace(forceUnicodeLower(value), escaped=True)
elif key == 'post_event_command':
eventConfigs[eventConfigId]['postEventCommand'] = config.replace(forceUnicodeLower(value), escaped=True)
elif key == 'action_processor_productids':
if not isinstance(value, list):
value = [x.strip() for x in value.split(",") if x.strip()]
eventConfigs[eventConfigId]['actionProcessorProductIds'] = forceList(value)
elif key == 'depot_protocol':
eventConfigs[eventConfigId]['depotProtocol'] = forceUnicodeLower(value)
elif key == 'exclude_product_group_ids':
if not isinstance(value, list):
value = [x.strip() for x in value.split(",") if x.strip()]
eventConfigs[eventConfigId]['excludeProductGroupIds'] = forceList(value)
elif key == 'include_product_group_ids':
if not isinstance(value, list):
value = [x.strip() for x in value.split(",") if x.strip()]
eventConfigs[eventConfigId]['includeProductGroupIds'] = forceList(value)
elif key == 'working_window':
eventConfigs[eventConfigId]['workingWindow'] = str(value)
else:
logger.error("Skipping unknown option '%s' in definition of event '%s'", key, eventConfigId)
for section in list(config.getDict()):
if section.startswith("event_") and config.has_option(section, key):
logger.info("Removing config option %s.%s", section, key)
config.del_option(section, key)
except Exception as err: # pylint: disable=broad-except
logger.debug(err, exc_info=True)
logger.error("Failed to set event config argument '%s' to '%s': %s", key, value, err)
logger.info(
"Event config '%s' args:\n%s",
eventConfigId,
pprint.pformat(eventConfigs[eventConfigId], indent=4, width=300, compact=False)
)
except Exception as err: # pylint: disable=broad-except
logger.error(err, exc_info=True)
return eventConfigs
def cleanupBackend(backend=None):
"""
Clean up data from your backends.
This method uses different cleanup methods to ensure that no
obsolete data is present in your backend.
:param backend: the backend to check. If ``None`` this will create a \
BackendManager from default paths.
:type backend: OPSI.Backend.Backend
"""
def usesMysqlBackend():
LOGGER.notice(u"Parsing dispatch.conf")
bdc = BackendDispatchConfigFile(
u'/etc/opsi/backendManager/dispatch.conf')
dispatchConfig = bdc.parse()
for entry in dispatchConfig:
(regex, backends) = entry
if not re.search(regex, u'backend_createBase'):
continue
if 'mysql' in backends:
return True
return False
LOGGER.debug("Cleaning backend chunk size: {0}".format(_CHUNK_SIZE))
if backend is None:
backend = BackendManager(
dispatchConfigFile=u'/etc/opsi/backendManager/dispatch.conf',
backendConfigDir=u'/etc/opsi/backends',
extensionConfigDir=u'/etc/opsi/backendManager/extend.d',
depotbackend=False)
try:
if usesMysqlBackend():
LOGGER.notice(
u"Mysql-backend detected. Trying to cleanup mysql-backend first"
)
# ToDo: backendConfigFile should be as dynamic as possible
# What if we have 2 mysql backends set up?
cleanUpMySQL()
except Exception as error:
LOGGER.warning(error)
LOGGER.notice(u"Cleaning up groups")
cleanUpGroups(backend)
LOGGER.notice(u"Cleaning up products")
cleanUpProducts(backend)
LOGGER.debug(u'Getting current depots...')
depotIds = set(depot.id for depot in backend.host_getObjects(
type=["OpsiConfigserver", "OpsiDepotserver"])) # pylint: disable=maybe-no-member
LOGGER.debug(u'Depots are: {0}'.format(depotIds))
LOGGER.debug(u'Getting current products...')
productIdents = set(
product.getIdent(returnType='unicode')
for product in backend.product_getObjects())
LOGGER.debug(u'Product idents are: {0}'.format(productIdents))
LOGGER.notice(u"Cleaning up product on depots")
cleanUpProductOnDepots(backend, depotIds, productIdents)
LOGGER.notice(u"Cleaning up product on clients")
cleanUpProductOnClients(backend)
LOGGER.notice(u"Cleaning up product properties")
productPropertyIdents = set()
deleteProductProperties = []
productPropertiesToCleanup = {}
for productProperty in backend.productProperty_getObjects(): # pylint: disable=maybe-no-member
productIdent = u"%s;%s;%s" % (productProperty.productId,
productProperty.productVersion,
productProperty.packageVersion)
if not productProperty.editable and productProperty.possibleValues:
productPropertyIdent = u"%s;%s" % (productIdent,
productProperty.propertyId)
productPropertiesToCleanup[productPropertyIdent] = productProperty
if productIdent not in productIdents:
LOGGER.info(
u"Marking productProperty %s of non existent product '%s' for deletion"
% (productProperty, productIdent))
deleteProductProperties.append(productProperty)
else:
productPropertyIdent = u'%s;%s' % (productProperty.productId,
productProperty.propertyId)
productPropertyIdents.add(productPropertyIdent)
if deleteProductProperties:
for productProperties in chunk(deleteProductProperties, _CHUNK_SIZE):
LOGGER.debug(u"Deleting product properties: {0!r}".format(
productProperties))
backend.productProperty_deleteObjects(productProperties) # pylint: disable=maybe-no-member
LOGGER.notice(u"Cleaning up product property states")
deleteProductPropertyStates = []
for productPropertyState in backend.productPropertyState_getObjects(): # pylint: disable=maybe-no-member
productPropertyIdent = u'%s;%s' % (productPropertyState.productId,
productPropertyState.propertyId)
if productPropertyIdent not in productPropertyIdents:
LOGGER.info(
u"Marking productPropertyState %s of non existent productProperty '%s' for deletion"
% (productPropertyState, productPropertyIdent))
deleteProductPropertyStates.append(productPropertyState)
if deleteProductPropertyStates:
for productPropertyStates in chunk(deleteProductPropertyStates,
_CHUNK_SIZE):
LOGGER.debug(u"Deleting product property states: {0!r}".format(
productPropertyStates))
backend.productPropertyState_deleteObjects(productPropertyStates) # pylint: disable=maybe-no-member
for depot in backend.host_getObjects(type='OpsiDepotserver'): # pylint: disable=maybe-no-member
objectIds = set(
ClientToDepot['clientId']
for ClientToDepot in backend.configState_getClientToDepotserver(
depotIds=depot.id))
objectIds.add(depot.id)
productOnDepotIdents = {}
for productOnDepot in backend.productOnDepot_getObjects(
depotId=depot.id): # pylint: disable=maybe-no-member
productIdent = u"%s;%s;%s" % (productOnDepot.productId,
productOnDepot.productVersion,
productOnDepot.packageVersion)
productOnDepotIdents[productOnDepot.productId] = productIdent
if not productOnDepotIdents:
continue
deleteProductPropertyStates = []
updateProductPropertyStates = []
for productPropertyState in backend.productPropertyState_getObjects( # pylint: disable=maybe-no-member
objectId=objectIds,
productId=productOnDepotIdents.keys(),
propertyId=[]):
productIdent = productOnDepotIdents.get(
productPropertyState.productId)
if not productIdent:
continue
productPropertyIdent = u"%s;%s" % (productIdent,
productPropertyState.propertyId)
productProperty = productPropertiesToCleanup.get(
productPropertyIdent)
if not productProperty:
continue
changed = False
newValues = []
removeValues = []
changedValues = []
for value in productPropertyState.values:
if value in productProperty.possibleValues:
newValues.append(value)
continue
if productProperty.getType(
) == u'BoolProductProperty' and forceBool(
value) in productProperty.possibleValues:
newValues.append(forceBool(value))
changedValues.append(value)
changed = True
continue
if productProperty.getType() == u'UnicodeProductProperty':
newValue = None
for possibleValue in productProperty.possibleValues:
if forceUnicodeLower(
possibleValue) == forceUnicodeLower(value):
newValue = possibleValue
break
if newValue:
newValues.append(newValue)
changedValues.append(value)
changed = True
continue
removeValues.append(value)
changed = True
if changed:
if not newValues:
LOGGER.info(
u"Marking productPropertyState %s for deletion: no value in possible values (%s)"
% (productPropertyState, removeValues))
deleteProductPropertyStates.append(productPropertyState)
else:
productPropertyState.setValues(newValues)
LOGGER.info(
u"Marking productPropertyState %s for update: values not in possible values: %s, values corrected: %s"
% (productPropertyState, removeValues, changedValues))
updateProductPropertyStates.append(productPropertyState)
if deleteProductPropertyStates:
for productPropertyStates in chunk(deleteProductPropertyStates,
_CHUNK_SIZE):
LOGGER.debug(u"Deleting product property states: {0!r}".format(
productPropertyStates))
backend.productPropertyState_deleteObjects(
productPropertyStates) # pylint: disable=maybe-no-member
del deleteProductPropertyStates
if updateProductPropertyStates:
for productPropertyStates in chunk(updateProductPropertyStates,
_CHUNK_SIZE):
LOGGER.debug(u"Updating product property states: {0!r}".format(
productPropertyStates))
backend.productPropertyState_updateObjects(
productPropertyStates) # pylint: disable=maybe-no-member
del updateProductPropertyStates
LOGGER.notice(u"Cleaning up config states")
cleanUpConfigStates(backend)
LOGGER.notice(u"Cleaning up audit softwares")
cleanUpAuditSoftwares(backend)
LOGGER.notice(u"Cleaning up audit software on clients")
cleanUpAuditSoftwareOnClients(backend)
def cleanUpProductPropertyStates(backend, productProperties, depotId,
productOnDepot):
productPropertiesToCleanup = {}
for productProperty in productProperties:
if productProperty.editable or not productProperty.possibleValues:
continue
productPropertiesToCleanup[
productProperty.propertyId] = productProperty
if productPropertiesToCleanup:
clientIds = set(clientToDepot['clientId'] for clientToDepot in
backend.configState_getClientToDepotserver(
depotIds=depotId))
if clientIds:
deleteProductPropertyStates = []
updateProductPropertyStates = []
states = backend.productPropertyState_getObjects(
objectId=clientIds,
productId=productOnDepot.getProductId(),
propertyId=list(productPropertiesToCleanup.keys()))
for productPropertyState in states:
changed = False
newValues = []
for value in productPropertyState.values:
productProperty = productPropertiesToCleanup[
productPropertyState.propertyId]
if value in productProperty.possibleValues:
newValues.append(value)
continue
if productProperty.getType(
) == 'BoolProductProperty' and forceBool(
value) in productProperty.possibleValues:
newValues.append(forceBool(value))
changed = True
continue
elif productProperty.getType(
) == 'UnicodeProductProperty':
newValue = None
for possibleValue in productProperty.possibleValues:
if forceUnicodeLower(
possibleValue
) == forceUnicodeLower(value):
newValue = possibleValue
break
if newValue is not None:
newValues.append(newValue)
changed = True
continue
changed = True
if changed:
if not newValues:
logger.debug(
u"Properties changed: marking productPropertyState {0} for deletion",
productPropertyState)
deleteProductPropertyStates.append(
productPropertyState)
else:
productPropertyState.setValues(newValues)
logger.debug(
u"Properties changed: marking productPropertyState {0} for update",
productPropertyState)
updateProductPropertyStates.append(
productPropertyState)
if deleteProductPropertyStates:
backend.productPropertyState_deleteObjects(
deleteProductPropertyStates)
if updateProductPropertyStates:
backend.productPropertyState_updateObjects(
updateProductPropertyStates)