def setUpBackend(self):
self._fileBackendConfig = {}
self._fileTempDir = self._copyOriginalBackendToTemporaryLocation()
self.backend = ExtendedConfigDataBackend(
FileBackend(**self._fileBackendConfig))
self.backend.backend_createBase()
def extendedConfigDataBackend(configDataBackend):
"""
Returns an `OPSI.Backend.ExtendedConfigDataBackend` for testing.
This will return multiple backends but some of these may lead to
skips if required libraries are missing or conditions for the
execution are not met.
"""
yield ExtendedConfigDataBackend(configDataBackend)
def __init__(self,
readBackend,
writeBackend,
newServerId=None,
oldServerId=None,
cleanupFirst=True):
self.__readBackend = readBackend
self.__writeBackend = writeBackend
self._extendedReadBackend = ExtendedConfigDataBackend(
self.__readBackend)
self._extendedWriteBackend = ExtendedConfigDataBackend(
self.__writeBackend)
self.__newServerId = None
self.__oldServerId = None
self.__cleanupFirst = forceBool(cleanupFirst)
self.__strict = False
self.__serverIds = []
self.__depotIds = []
self.__clientIds = []
self.__groupIds = []
self.__productIds = []
if newServerId:
self.__newServerId = forceHostId(newServerId)
if oldServerId:
self.__oldServerId = forceHostId(oldServerId)
self.__overallProgressSubject = ProgressSubject(
id=u'overall_replication',
title=u'Replicating',
end=100,
fireAlways=True)
self.__currentProgressSubject = ProgressSubject(
id=u'current_replication', fireAlways=True)
def getTestBackend(extended=False):
"""
Get a backend for tests.
Each call to this will return a different backend.
If `extended` is True the returned backend will be an
`ExtendedConfigDataBackend`.
"""
from .File import getFileBackend # lazy import
with getFileBackend() as backend:
if extended:
backend = ExtendedConfigDataBackend(backend)
backend.backend_createBase()
try:
yield backend
finally:
backend.backend_deleteBase()
class FileBackendMixin(BackendMixin):
BACKEND_SUBFOLDER = os.path.join('etc', 'opsi')
CONFIG_DIRECTORY = os.path.join('var', 'lib', 'opsi')
CREATES_INVENTORY_HISTORY = False
def setUpBackend(self):
self._fileBackendConfig = {}
self._fileTempDir = self._copyOriginalBackendToTemporaryLocation()
self.backend = ExtendedConfigDataBackend(
FileBackend(**self._fileBackendConfig))
self.backend.backend_createBase()
def _copyOriginalBackendToTemporaryLocation(self):
tempDir = tempfile.mkdtemp()
originalBackendDir = _getOriginalBackendLocation()
shutil.copytree(originalBackendDir,
os.path.join(tempDir, self.BACKEND_SUBFOLDER))
self._setupFileBackend(tempDir)
self._patchDispatchConfig(tempDir)
return tempDir
def _setupFileBackend(self, targetDirectory):
self._patchFileBackend(targetDirectory)
self._createClientTemplateFolders(
os.path.join(targetDirectory, self.CONFIG_DIRECTORY))
def _patchFileBackend(self, backendDirectory):
baseDir = os.path.join(backendDirectory, self.CONFIG_DIRECTORY,
'config')
hostKeyDir = os.path.join(backendDirectory, self.BACKEND_SUBFOLDER,
'pckeys')
currentGroupId = os.getgid()
groupName = grp.getgrgid(currentGroupId)[0]
userName = pwd.getpwuid(os.getuid())[0]
self._fileBackendConfig.update(
dict(basedir=baseDir,
hostKeyFile=hostKeyDir,
fileGroupName=groupName,
fileUserName=userName))
config_file = os.path.join(backendDirectory, self.BACKEND_SUBFOLDER,
'backends', 'file.conf')
with open(config_file, 'w') as config:
new_configuration = """
# -*- coding: utf-8 -*-
module = 'File'
config = {{
"baseDir": u"{basedir}",
"hostKeyFile": u"{keydir}",
"fileGroupName": u"{groupName}",
"fileUserName": u"{userName}",
}}
""".format(basedir=baseDir,
keydir=hostKeyDir,
groupName=groupName,
userName=userName)
config.write(new_configuration)
@classmethod
def _createClientTemplateFolders(cls, targetDirectory):
templateDirectory = os.path.join(targetDirectory, 'config',
'templates')
os.makedirs(templateDirectory)
def _patchDispatchConfig(self, targetDirectory):
configDir = os.path.join(targetDirectory, self.BACKEND_SUBFOLDER,
'backends')
dispatchConfigPath = os.path.join(configDir, 'dispatch.conf')
self._fileBackendConfig['dispatchConfig'] = dispatchConfigPath
with open(dispatchConfigPath, 'w') as dpconf:
dpconf.write("""
.* : file
""")
def tearDownBackend(self):
self.backend.backend_deleteBase()
try:
shutil.rmtree(self._fileTempDir)
except OSError:
pass
del self.backend
def testMultiThreadingBackend(multithreadingBackend, numberOfThreads):
backend = ExtendedConfigDataBackend(multithreadingBackend)
MySQLdb = pytest.importorskip("MySQLdb")
IntegrityError = MySQLdb.IntegrityError
errorConstants = pytest.importorskip("MySQLdb.constants.ER")
DUP_ENTRY = errorConstants.DUP_ENTRY
o2g, _, clients = fillBackendWithObjectToGroups(backend)
class MultiThreadTester(threading.Thread):
def __init__(self, backend, clients, objectToGroups):
threading.Thread.__init__(self)
self.exitCode = 0
self.errorMessage = None
self.backend = backend
self.clients = clients
self.objectToGroups = objectToGroups
def run(self):
self.client1 = clients[0]
self.client2 = clients[1]
self.objectToGroup1 = o2g[0]
self.objectToGroup2 = o2g[1]
try:
print(u"Thread %s started" % self)
time.sleep(1)
self.backend.host_getObjects()
self.backend.host_deleteObjects(self.client1)
self.backend.host_getObjects()
self.backend.host_deleteObjects(self.client2)
self.backend.host_createObjects(self.client2)
self.backend.host_createObjects(self.client1)
self.backend.objectToGroup_createObjects(self.objectToGroup1)
self.backend.objectToGroup_createObjects(self.objectToGroup2)
self.backend.host_getObjects()
self.backend.host_createObjects(self.client1)
self.backend.host_deleteObjects(self.client2)
self.backend.host_createObjects(self.client1)
self.backend.host_getObjects()
except IntegrityError as e:
if e[0] != DUP_ENTRY:
self.errorMessage = e
self.exitCode = 1
except Exception as e:
self.errorMessage = e
self.exitCode = 1
finally:
print(u"Thread %s done" % self)
mtts = [
MultiThreadTester(backend, clients, o2g)
for i in range(numberOfThreads)
]
for mtt in mtts:
mtt.start()
for mtt in mtts:
mtt.join()
client1 = clients[0]
backend.host_createObjects(client1)
while mtts:
mtt = mtts.pop(0)
if not mtt.is_alive():
if mtt.exitCode != 0:
assert 0 == mtt.exitCode, (
u"Multithreading test failed: Exit Code {0}: {1}".format(
mtt.exitCode, mtt.errorMessage))
else:
mtts.append(mtt)
def main():
# TODO: make this a test that is run automatically
#init and reset
fileBackend = ExtendedConfigDataBackend(FileBackend())
fileBackend.backend_deleteBase()
fileBackend.backend_createBase()
mysqlBackend = ExtendedConfigDataBackend(
MySQLBackend(username='******', password='******', database='opsi'))
mysqlBackend.backend_deleteBase()
mysqlBackend.backend_createBase()
def check(one, two):
objectTypesToCheck = ('host', 'config', 'configState', 'product',
'productProperty', 'productDependency',
'productOnDepot', 'productOnClient',
'productPropertyState', 'group', 'objectToGroup')
for objectType in objectTypesToCheck:
oneObjects = eval('%s.%s_getObjects()' % (one, objectType))
twoObjects = eval('%s.%s_getObjects()' % (two, objectType))
oneIdents = [
oneObject.getIdent(returnType='unicode')
for oneObject in oneObjects
]
twoIdents = [
twoObject.getIdent(returnType='unicode')
for twoObject in twoObjects
]
logger.warning(
u"assert length %s\noneIdents: '%s'\ntwoIdents: '%s'" %
(objectType, oneIdents, twoIdents))
assert len(oneIdents) == len(twoIdents)
for oneIdent in oneIdents:
assert any(oneIdent == twoIdent for twoIdent in twoIdents)
#convert fileBackend -> mysqlBackend
mysqlBackend.host_createObjects(fileBackend.host_getObjects())
mysqlBackend.config_createObjects(fileBackend.config_getObjects())
mysqlBackend.configState_createObjects(
fileBackend.configState_getObjects())
mysqlBackend.product_createObjects(fileBackend.product_getObjects())
mysqlBackend.productProperty_createObjects(
fileBackend.productProperty_getObjects())
mysqlBackend.productDependency_createObjects(
fileBackend.productDependency_getObjects())
mysqlBackend.productOnDepot_createObjects(
fileBackend.productOnDepot_getObjects())
mysqlBackend.productOnClient_createObjects(
fileBackend.productOnClient_getObjects())
mysqlBackend.productPropertyState_createObjects(
fileBackend.productPropertyState_getObjects())
mysqlBackend.group_createObjects(fileBackend.group_getObjects())
mysqlBackend.objectToGroup_createObjects(
fileBackend.objectToGroup_getObjects())
check('fileBackend', 'mysqlBackend')
fileBackend.host_deleteObjects(fileBackend.host_getObjects())
fileBackend.config_deleteObjects(fileBackend.config_getObjects())
fileBackend.configState_deleteObjects(fileBackend.configState_getObjects())
fileBackend.product_deleteObjects(fileBackend.product_getObjects())
fileBackend.productProperty_deleteObjects(
fileBackend.productProperty_getObjects())
fileBackend.productDependency_deleteObjects(
fileBackend.productDependency_getObjects())
fileBackend.productOnDepot_deleteObjects(
fileBackend.productOnDepot_getObjects())
fileBackend.productOnClient_deleteObjects(
fileBackend.productOnClient_getObjects())
fileBackend.productPropertyState_deleteObjects(
fileBackend.productPropertyState_getObjects())
fileBackend.group_deleteObjects(fileBackend.group_getObjects())
fileBackend.objectToGroup_deleteObjects(
fileBackend.objectToGroup_getObjects())
objectTypes = ('host', 'config', 'configState', 'product',
'productProperty', 'productDependency', 'productOnDepot',
'productOnClient', 'productPropertyState', 'group',
'objectToGroup')
for objectType in objectTypes:
objects = eval('fileBackend.%s_getObjects()' % objectType)
idents = [obj.getIdent(returnType='unicode') for obj in objects]
logger.warning(u"assert length %s-idents: '%s'" \
% (objectType, idents))
assert len(idents) == 0
#convert mysqlBackend -> btfileBackend
fileBackend.host_createObjects(mysqlBackend.host_getObjects())
fileBackend.config_createObjects(mysqlBackend.config_getObjects())
fileBackend.configState_createObjects(
mysqlBackend.configState_getObjects())
fileBackend.product_createObjects(mysqlBackend.product_getObjects())
fileBackend.productProperty_createObjects(
mysqlBackend.productProperty_getObjects())
fileBackend.productDependency_createObjects(
mysqlBackend.productDependency_getObjects())
fileBackend.productOnDepot_createObjects(
mysqlBackend.productOnDepot_getObjects())
fileBackend.productOnClient_createObjects(
mysqlBackend.productOnClient_getObjects())
fileBackend.productPropertyState_createObjects(
mysqlBackend.productPropertyState_getObjects())
fileBackend.group_createObjects(mysqlBackend.group_getObjects())
fileBackend.objectToGroup_createObjects(
mysqlBackend.objectToGroup_getObjects())
check('mysqlBackend', 'fileBackend')
print("-------------------------------------")
print("- seem to work ... all tests passed -")
print("-------------------------------------")
def cleanableDataBackend(_serverBackend):
"""
Returns an backend that can be cleaned.
"""
yield ExtendedConfigDataBackend(_serverBackend)
def licenseManagentAndAuditBackend(request):
with request.param() as backend:
with _backendBase(backend):
yield ExtendedConfigDataBackend(backend)
def auditDataBackend(request, hardwareAuditConfigPath):
with request.param(
auditHardwareConfigFile=hardwareAuditConfigPath) as backend:
with _backendBase(backend):
yield ExtendedConfigDataBackend(backend)
def licenseManagementBackend(sqlBackendCreationContextManager):
'''Returns a backend that can handle License Management.'''
with sqlBackendCreationContextManager() as backend:
with _backendBase(backend):
yield ExtendedConfigDataBackend(backend)
def __init__(self, **kwargs):
"""
Creating a BackendManager.
If no configuration is given a default config for accessing the
local backend as configured through the files in
/etc/opsi/backendManager/ will be used.
The constructor takes several parameters and these are all optional.
:param username: A username for authentication.
:type username: str
:param password: The corresponding password.
:type password: str
:param backend: A backend to use.
:param dispatchconfig: A pre-definded dispatch config to use.
:param dispatchconfigfile: The configuration file for dispatching.
:type dispatchconfigfile: str
:param backendconfigdir: The location of backend configurations.
:type backendconfigdir: str
:param depotbackend: Allow depot actions?
:type depotbackend: bool
:param hostcontrolbackend: Allow controlling hosts?
:type hostcontrolbackend: bool
:param hostcontrolsafebackend: Allow controlling hosts (safe variant)?
:type hostcontrolsafebackend: bool
:param extensionconfigdir: The directory where backend extensions can be found.
:type extensionconfigdir: str
:param extend: Extend the backends?
:type extend: bool
:param acl: An access control list (ACL) configuration to use.
:type acl: [[str, ]]
:param aclfile: Load the ACL from this file.
:type aclfile: str
"""
self._backend = None
self._backendConfigDir = None
self._options = {}
self._overwrite = True
self._context = self
Backend.__init__(self, **kwargs)
username = None
password = None
dispatch = False
extend = False
extensionConfigDir = None
extensionClass = None
accessControl = False
depotBackend = False
hostControlBackend = False
hostControlSafeBackend = False
startReactor = True
loadBackend = None
if not kwargs:
kwargs = {
"dispatchConfigFile":
u'/etc/opsi/backendManager/dispatch.conf',
"backendConfigDir": u'/etc/opsi/backends',
"extensionConfigDir": u'/etc/opsi/backendManager/extend.d',
"depotBackend": True,
"hostControlBackend": True,
"hostControlSafeBackend": True,
}
logger.debug("No config given, using {0!r}".format(kwargs))
for (option, value) in kwargs.items():
option = option.lower()
if option == 'username':
username = value
elif option == 'password':
password = value
elif option == 'backend':
if isinstance(value, (str, unicode)):
loadBackend = value
else:
self._backend = value
del kwargs[option]
elif option == 'backendconfigdir':
self._backendConfigDir = value
elif option in ('dispatchconfig', 'dispatchconfigfile') and value:
dispatch = True
elif option == 'depotbackend':
depotBackend = forceBool(value)
elif option == 'hostcontrolbackend':
hostControlBackend = forceBool(value)
elif option == 'hostcontrolsafebackend':
hostControlSafeBackend = forceBool(value)
elif option == 'extensionconfigdir' and value:
extensionConfigDir = value
extend = True
elif option == 'extensionclass':
extensionClass = value
extend = True
elif option == 'extend':
extend = forceBool(value)
elif option in ('acl', 'aclfile') and value:
accessControl = True
elif option == 'startreactor' and value is False:
startReactor = False
if loadBackend:
logger.info(u"* BackendManager is loading backend '%s'" %
loadBackend)
self._backend = self.__loadBackend(loadBackend)
# self._backend is now a ConfigDataBackend
if not dispatch and not self._backend:
raise BackendConfigurationError(
u"Neither backend nor dispatch config given")
if dispatch:
logger.info(u"* BackendManager is creating BackendDispatcher")
self._backend = BackendDispatcher(context=self, **kwargs)
# self._backend is now a BackendDispatcher which is a ConfigDataBackend
if extend or depotBackend:
logger.info(
u"* BackendManager is creating ExtendedConfigDataBackend")
# DepotserverBackend/BackendExtender need ExtendedConfigDataBackend backend
self._backend = ExtendedConfigDataBackend(self._backend)
# self._backend is now an ExtendedConfigDataBackend
if depotBackend:
logger.info(u"* BackendManager is creating DepotserverBackend")
self._backend = DepotserverBackend(self._backend)
if hostControlBackend:
logger.info(u"* BackendManager is creating HostControlBackend")
try:
hostControlBackendConfig = self.__loadBackendConfig(
'hostcontrol')['config']
except Exception as backendConfigLoadError:
logger.error(
"Failed to load configuration for HostControlBackend: {}",
backendConfigLoadError)
hostControlBackendConfig = {}
self._backend = HostControlBackend(self._backend,
**hostControlBackendConfig)
if hostControlSafeBackend:
logger.info(u"* BackendManager is creating HostControlSafeBackend")
try:
hostControlSafeBackendConfig = self.__loadBackendConfig(
'hostcontrol')['config']
except Exception as backendConfigLoadError:
logger.error(
"Failed to load configuration for HostControlSafeBackend: {}",
backendConfigLoadError)
hostControlSafeBackendConfig = {}
self._backend = HostControlSafeBackend(
self._backend, **hostControlSafeBackendConfig)
if accessControl:
logger.info(u"* BackendManager is creating BackendAccessControl")
self._backend = BackendAccessControl(backend=self._backend,
**kwargs)
if extensionConfigDir or extensionClass:
logger.info(u"* BackendManager is creating BackendExtender")
self._backend = BackendExtender(self._backend, **kwargs)
self._createInstanceMethods()