def status(self, _bStateful = True, _iIntent = None):
if self._iVerbose: self._INFO('Querying status')
iStatus = KiscRuntime.STATUS_STARTED
if _bStateful:
# Exists ?
# NOTE: look only for a name match, independently from potentially mismatching
# type, VLAN ID, physical device or options
try:
KiscRuntime.shell(['test', '-e', '/sys/class/net/%s' % self._dsConfig['name']], _bTrace = self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
except OSError as e:
if e.filename == 0 and e.errno == 1:
iStatus = KiscRuntime.STATUS_STOPPED
else:
if self._iVerbose: self._ERROR(str(e))
iStatus = KiscRuntime.STATUS_ERROR
# UP ?
if iStatus == KiscRuntime.STATUS_STARTED and _iIntent == KiscRuntime.STATUS_STARTED:
try:
KiscRuntime.shell(['grep', '-Fq', 'up', '/sys/class/net/%s/operstate' % self._dsConfig['name']], _bTrace = self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
except OSError as e:
if self._iVerbose: self._ERROR(str(e))
iStatus = KiscRuntime.STATUS_ERROR
else:
iStatus = self._iStatus
# Done
self._iStatus = iStatus
if self._iVerbose: self._INFO('Status is %s' % KiscRuntime.STATUS_MESSAGE[self._iStatus])
return self._iStatus
def status(self, _bStateful=True, _iIntent=None):
if self._iVerbose: self._INFO('Querying status')
iStatus = KiscRuntime.STATUS_STARTED
if _bStateful:
# Exists ?
# NOTE: look only for a mountpoint match, independently from potentially mismatching
# fstype, device or options
try:
KiscRuntime.shell([
'awk',
'BEGIN{e=22}; {if($2=="%s") {e=0; exit}}; END {exit e}' %
self._dsConfig['mountpoint'], '/proc/mounts'
],
_bTrace=self._iVerbose >=
KiscRuntime.VERBOSE_TRACE)
except OSError as e:
if e.filename == 0 and e.errno == 22:
iStatus = KiscRuntime.STATUS_STOPPED
else:
if self._iVerbose: self._ERROR(str(e))
iStatus = KiscRuntime.STATUS_ERROR
else:
iStatus = self._iStatus
# Done
self._iStatus = iStatus
if self._iVerbose:
self._INFO('Status is %s' %
KiscRuntime.STATUS_MESSAGE[self._iStatus])
return self._iStatus
def stop(self):
if self._iVerbose: self._INFO('Stopping')
lsErrors = list()
# Be idempotent
if self.status(
True,
KiscRuntime.STATUS_STOPPED) == KiscRuntime.STATUS_STOPPED:
if self._iVerbose: self._INFO('Already stopped')
return lsErrors
# Stop the resource
# ... delete address
try:
KiscRuntime.shell(
[
'ip', '-4', 'address', 'delete',
'%s/%s' %
(self._dsConfig['address'], self._dsConfig['mask']), 'dev',
self._dsConfig['device']
],
_bTrace=self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
self._iStatus = KiscRuntime.STATUS_STARTED
if self._iVerbose: self._INFO('Stopped')
except OSError as e:
if self._iVerbose: self._ERROR(str(e))
self._iStatus = KiscRuntime.STATUS_ERROR
lsErrors.append(str(e))
# Done
return lsErrors
def status(self, _bStateful = True, _iIntent = None):
if self._iVerbose: self._INFO('Querying status')
iStatus = KiscRuntime.STATUS_UNKNOWN
if _bStateful:
# Exists ?
try:
KiscRuntime.shell(['invoke-rc.d', '--quiet', self._dsConfig['name'], 'status'], _bTrace = self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
iStatus = KiscRuntime.STATUS_STARTED
except OSError as e:
# REF: http://refspecs.linuxbase.org/LSB_3.1.0/LSB-Core-generic/LSB-Core-generic/iniscrptact.html
if e.filename == 0 and e.errno == 3:
iStatus = KiscRuntime.STATUS_STOPPED
else:
if self._iVerbose: self._ERROR(str(e))
iStatus = KiscRuntime.STATUS_ERROR
else:
iStatus = self._iStatus
# Done
self._iStatus = iStatus
if self._iVerbose: self._INFO('Status is %s' % KiscRuntime.STATUS_MESSAGE[self._iStatus])
return self._iStatus
def stop(self):
if self._iVerbose: self._INFO('Stopping')
lsErrors = list()
# Be idempotent
if self.status(
True,
KiscRuntime.STATUS_STOPPED) == KiscRuntime.STATUS_STOPPED:
if self._iVerbose: self._INFO('Already stopped')
return lsErrors
# Stop the resource
# ... unmount device
try:
KiscRuntime.shell(
['umount', self._dsConfig['mountpoint']],
_bTrace=self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
self._iStatus = KiscRuntime.STATUS_STOPPED
if self._iVerbose: self._INFO('Stopped')
except OSError as e:
if self._iVerbose: self._ERROR(str(e))
self._iStatus = KiscRuntime.STATUS_ERROR
lsErrors.append(str(e))
# Done
return lsErrors
def start(self):
if self._iVerbose: self._INFO('Starting')
lsErrors = list()
# Restart ?
bRestart = KiscRuntime.parseBool(self._dsConfig.get('restart', False))
# Be idempotent
if self.status(True, KiscRuntime.STATUS_STARTED) == KiscRuntime.STATUS_STARTED:
if not bRestart:
if self._iVerbose: self._INFO('Already started')
return lsErrors
else:
if self._iVerbose: self._INFO('Restarting')
else:
bRestart = False
# Start the resource
try:
if bRestart:
KiscRuntime.shell(['systemctl', '-q', 'restart', self._dsConfig['name']], _bTrace = self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
else:
KiscRuntime.shell(['systemctl', '-q', 'start', self._dsConfig['name']], _bTrace = self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
self._iStatus = KiscRuntime.STATUS_STARTED
if self._iVerbose: self._INFO('Started')
except OSError as e:
if self._iVerbose: self._ERROR(str(e))
self._iStatus = KiscRuntime.STATUS_ERROR
lsErrors.append(str(e))
# Done
return lsErrors
def status(self, _bStateful=True, _iIntent=None):
if self._iVerbose: self._INFO('Querying status')
iStatus = KiscRuntime.STATUS_STARTED
if _bStateful:
# Exists ?
# NOTE: look only for an address match, independently from potentially mismatching
# network mask, device or options
try:
KiscRuntime.shell(
[['ip', '-4', 'address', 'show'],
['grep', '-Fq',
'inet %s/' % self._dsConfig['address']]],
_bTrace=self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
except OSError as e:
if e.filename == 0 and e.errno == 1:
iStatus = KiscRuntime.STATUS_STOPPED
else:
if self._iVerbose: self._ERROR(str(e))
iStatus = KiscRuntime.STATUS_ERROR
else:
iStatus = self._iStatus
# Done
self._iStatus = iStatus
if self._iVerbose:
self._INFO('Status is %s' %
KiscRuntime.STATUS_MESSAGE[self._iStatus])
return self._iStatus
def stop(self):
if self._iVerbose: self._INFO('Stopping')
lsErrors = list()
# Be idempotent
if self.status(True, KiscRuntime.STATUS_STOPPED) == KiscRuntime.STATUS_STOPPED:
if self._iVerbose: self._INFO('Already stopped')
return lsErrors
# Stop the resource
# ... DOWN!
try:
KiscRuntime.shell(['ip', 'link', 'set', self._dsConfig['name'], 'down'], _bTrace = self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
except OSError as e:
if self._iVerbose: self._WARNING(str(e))
lsErrors.append(str(e))
# ... delete device
try:
KiscRuntime.shell(['ip', 'link', 'delete', self._dsConfig['name']], _bTrace = self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
self._iStatus = KiscRuntime.STATUS_STOPPED
if self._iVerbose: self._INFO('Stopped')
except OSError as e:
if self._iVerbose: self._ERROR(str(e))
self._iStatus = KiscRuntime.STATUS_ERROR
lsErrors.append(str(e))
# Done
return lsErrors
def __loadResources(self, _sConfigFile, _bBootstrap = False, _bAutostart = False):
"""
Load resources configuration from file
@param str _sConfigFile Configuration file (path)
@param bool _bBootstrap Bootstrap (host startup) configuration
@param bool _bAutostart Automatically start bootstrap resources
@return list Empty if resource is successfully started, (ordered) error messages otherwise
"""
if self._iVerbose: self._DEBUG('Loading resources configuration from file (%s)' % _sConfigFile)
lsErrors = list()
try:
# Load configuration from file
with open(_sConfigFile, 'r') as oFile:
oConfig = configparser.RawConfigParser()
oConfig.optionxform = lambda sOption: sOption # do not lowercase option (key) name
oConfig.read_file(oFile, _sConfigFile)
# Loop through sections/resources (IDs)
for sId in oConfig.sections():
if sId == 'KiSC': continue
dsConfig = {tOption[0]: tOption[1] for tOption in oConfig.items(sId)}
if 'TYPE' not in dsConfig:
lsErrors.append('<%s> [%s] Invalid configuration section; missing "TYPE" parameter' % (_sConfigFile, sId))
continue
if dsConfig['TYPE'] == 'include':
# Include configuration file(s)
bBootstrap_sub = KiscRuntime.parseBool(dsConfig.get('BOOTSTRAP', False))
bAutostart_sub = KiscRuntime.parseBool(dsConfig.get('AUTOSTART', False))
if 'file' in dsConfig:
lsErrors_sub = self.__loadResources(dsConfig['file'], bBootstrap_sub, bAutostart_sub)
if lsErrors_sub:
lsErrors.extend(['<%s> %s' % (_sConfigFile, sError) for sError in lsErrors_sub])
if 'directory' in dsConfig:
import glob
for sFile in glob.glob('%s/%s' % (dsConfig['directory'], dsConfig.get('glob', '*.cfg'))):
lsErrors_sub = self.__loadResources(sFile, bBootstrap_sub, bAutostart_sub)
if lsErrors_sub:
lsErrors.extend(['<%s> %s' % (_sConfigFile, sError) for sError in lsErrors_sub])
else:
if _bBootstrap:
lsErrors_sub = self.__createResource_bootstrap(dsConfig['TYPE'], sId, dsConfig, _bAutostart)
else:
lsErrors_sub = self.__createResource(dsConfig['TYPE'], sId, dsConfig)
if lsErrors_sub:
lsErrors.extend(['<%s> %s' % (_sConfigFile, sError) for sError in lsErrors_sub])
except (OSError, configparser.Error, RuntimeError) as e:
if self._iVerbose: self._ERROR(str(e))
lsErrors.append('<%s> %s' % (_sConfigFile, str(e)))
# Done
return lsErrors
def stop(self):
if self._iVerbose: self._INFO('Stopping')
lsErrors = list()
# Be idempotent
if self.status(
True,
KiscRuntime.STATUS_STOPPED) == KiscRuntime.STATUS_STOPPED:
if self._iVerbose: self._INFO('Already stopped')
return lsErrors
# Stop the resource
try:
# ... timeout
try:
iTimeout = int(self._dsConfig.get('timeout_stop', 15))
except ValueError:
raise RuntimeError('Invalid timeout value (%s)' %
self._dsConfig['timeout_stop'])
# ... stop domain
KiscRuntime.shell(
['virsh', '-q', 'shutdown', self._dsConfig['name']],
_bTrace=self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
# ... wait for domain to stop
while iTimeout >= 0:
iTimeout -= 1
try:
if KiscRuntime.shell(
['virsh', 'domstate', self._dsConfig['name']],
_bTrace=self._iVerbose >=
KiscRuntime.VERBOSE_TRACE).strip() == 'shut off':
break
except OSError as e:
if e.filename == 0:
break
else:
raise e
if iTimeout < 0:
KiscRuntime.shell(
['virsh', '-q', 'destroy', self._dsConfig['name']],
_bTrace=self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
raise RuntimeError('Domain did not stop')
time.sleep(1)
# ... done
self._iStatus = KiscRuntime.STATUS_STOPPED
if self._iVerbose: self._INFO('Stopped')
except OSError as e:
if self._iVerbose: self._ERROR(str(e))
self._iStatus = KiscRuntime.STATUS_ERROR
lsErrors.append(str(e))
# Done
return lsErrors
def start(self):
if self._iVerbose: self._INFO('Starting')
lsErrors = list()
# Be idempotent
if self.status(True, KiscRuntime.STATUS_STARTED) == KiscRuntime.STATUS_STARTED:
if self._iVerbose: self._INFO('Already started')
return lsErrors
# Start the resource
try:
# ... add device
lsCommand = [
'ip', 'link', 'add',
'link', self._dsConfig['device'],
'name', self._dsConfig['name'],
]
for sSetting in ['address', 'mtu', 'txqueuelen', 'numtxqueues', 'numrxqueues']:
if sSetting in self._dsConfig:
lsCommand.extend([sSetting, self._dsConfig[sSetting]])
lsCommand.extend(['type', 'vlan'])
# ... protocol
for sSetting in ['protocol']:
if sSetting in self._dsConfig:
lsCommand.extend([sSetting, self._dsConfig[sSetting]])
# ... VLAN ID
lsCommand.extend(['id', self._dsConfig['vlan']])
# ... options
for sSetting in ['reorder_hdr', 'gvrp', 'mvrp', 'loose_binding']:
if sSetting in self._dsConfig:
lsCommand.extend([sSetting, self._dsConfig[sSetting]])
# ... QoS configuration
for sSetting in ['ingress_qos_map', 'egress_qos_map']:
if sSetting in self._dsConfig:
lsCommand.append(sSetting.replace('_', '-'))
for sMapping in self._dsConfig[sSetting].split(','):
lsCommand.append(sMapping)
KiscRuntime.shell(lsCommand, _bTrace = self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
# ... UP!
KiscRuntime.shell(['ip', 'link', 'set', self._dsConfig['name'], 'up'], _bTrace = self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
self._iStatus = KiscRuntime.STATUS_STARTED
if self._iVerbose: self._INFO('Started')
except OSError as e:
if self._iVerbose: self._ERROR(str(e))
self._iStatus = KiscRuntime.STATUS_ERROR
lsErrors.append(str(e))
# Done
return lsErrors
def resume(self):
if self._iVerbose: self._INFO('Resuming')
lsErrors = list()
# Be idempotent
iStatus = self.status(True, KiscRuntime.STATUS_STARTED)
if iStatus == KiscRuntime.STATUS_STARTED:
if self._iVerbose: self._INFO('Domain is running')
return lsErrors
# Resuming the resource
if iStatus != KiscRuntime.STATUS_SUSPENDED:
raise RuntimeError('Domain not suspended')
try:
# ... timeout
try:
iTimeout = int(self._dsConfig.get('timeout_resume', 5))
except ValueError:
raise RuntimeError('Invalid timeout value (%s)' %
self._dsConfig['timeout_resume'])
# ... resume domain
KiscRuntime.shell(
['virsh', '-q', 'resume', self._dsConfig['name']],
_bTrace=self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
# ... wait for domain to resume
while iTimeout >= 0:
iTimeout -= 1
try:
if KiscRuntime.shell(
['virsh', 'domstate', self._dsConfig['name']],
_bTrace=self._iVerbose >=
KiscRuntime.VERBOSE_TRACE).strip() == 'running':
break
except OSError as e:
if e.filename == 0:
break
else:
raise e
if iTimeout < 0:
raise RuntimeError('Domain did not resume')
time.sleep(1)
# ... done
self._iStatus = KiscRuntime.STATUS_RESUMED
if self._iVerbose: self._INFO('Resumed')
except OSError as e:
if self._iVerbose: self._ERROR(str(e))
self._iStatus = KiscRuntime.STATUS_ERROR
lsErrors.append(str(e))
# Done
return lsErrors
def migrate(self, _oHost):
if self._iVerbose: self._INFO('Migrating')
lsErrors = list()
# Be idempotent
if self.status(
True,
KiscRuntime.STATUS_STARTED) != KiscRuntime.STATUS_STARTED:
if self._iVerbose: self._INFO('Not started')
return lsErrors
# Migrate the resource
try:
# ... remote URI
sRemoteUri = self._dsConfig.get('remote_uri',
'qemu://%{host}/system')
sRemoteUri = sRemoteUri.replace('%{host}', _oHost.id())
sRemoteUri = sRemoteUri.replace('%{hostname}',
_oHost.getHostname())
# ... timeout
try:
iTimeout = int(self._dsConfig.get('timeout_migrate', 60))
except ValueError:
raise RuntimeError('Invalid timeout value (%s)' %
self._dsConfig['timeout_migrate'])
# ... migrate domain
lsCommand = ['virsh', '-q', 'migrate', '--live']
if iTimeout > 0:
lsCommand.extend(
['--timeout',
str(iTimeout), '--timeout-suspend'])
lsCommand.extend([self._dsConfig['name'], sRemoteUri])
KiscRuntime.shell(
lsCommand, _bTrace=self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
# ... done
self._iStatus = KiscRuntime.STATUS_STARTED
if self._iVerbose: self._INFO('Started')
except (OSError, RuntimeError) as e:
if self._iVerbose: self._ERROR(str(e))
self.status(True, KiscRuntime.STATUS_SUSPENDED)
lsErrors.append(str(e))
# Done
return lsErrors
def status(self, _bStateful=True, _iIntent=None):
if self._iVerbose: self._INFO('Querying status')
iStatus = KiscRuntime.STATUS_UNKNOWN
if _bStateful:
# Locate resource on Pacekamer cluster
try:
sOutput = KiscRuntime.shell(
['crm_resource', '-Q', '-r', self._dsConfig['name'], '-W'],
_bTrace=self._iVerbose >=
KiscRuntime.VERBOSE_TRACE).strip()
if sOutput is None or not len(sOutput):
if '$PACEMAKER_NODES' in self._dsConfig:
del self._dsConfig['$PACEMAKER_NODES']
iStatus = KiscRuntime.STATUS_STOPPED
else:
self._dsConfig['$PACEMAKER_NODES'] = sOutput
iStatus = KiscRuntime.STATUS_STARTED
except OSError as e:
if e.filename == 0 and e.errno == 6:
iStatus = KiscRuntime.STATUS_STOPPED
else:
if self._iVerbose: self._ERROR(str(e))
iStatus = KiscRuntime.STATUS_ERROR
else:
iStatus = self._iStatus
# Done
self._iStatus = iStatus
if self._iVerbose:
self._INFO('Status is %s' %
KiscRuntime.STATUS_MESSAGE[self._iStatus])
return self._iStatus
def unregisterHost(self, _oHost):
"""
Unregister the given host resource (as running this resource)
@param KiscResource_cluster_host _oHost Host resource (object) to unregister
@return list Empty if resource is successfully unregistered, (ordered) error messages otherwise
"""
if self._iVerbose: self._INFO('Unregistering host (%s)' % _oHost.id())
lsErrors = list()
# Unregister host
try:
sHost_id = _oHost.id()
# ... registered hosts
lsHosts = KiscRuntime.parseList(self._dsConfig.get('$HOSTS', None))
# ... check registration status
if sHost_id not in lsHosts:
return lsErrors
# ... finalize
lsHosts.remove(sHost_id)
self._dsConfig['$HOSTS'] = ','.join(lsHosts)
if not len(self._dsConfig['$HOSTS']):
del self._dsConfig['$HOSTS']
if self._iVerbose: self._INFO('Host unregistered (%s)' % _oHost.id())
except RuntimeError as e:
if self._iVerbose: self._ERROR(str(e))
lsErrors.append(str(e))
# Done
return lsErrors
def getHostsIDs(self):
"""
Return hosts (IDs) running the resource
@return list Resource running hosts IDs
"""
return KiscRuntime.parseList(self._dsConfig.get('$HOSTS', None))
def resolveFile(self, _sFile_from, _sFile_to, _mHost = None, _mResource = None, _bBootstrap = False, _tPermissions = None):
"""
Copy the given source file to the given destination, resolving cluster variables in their content
This method will substitute cluster variables found in the file; see self.resolveString().
@param str _sFile_from Source file (standard input if None)
@param str _sFile_to Destination file (standard output if None)
@param str _sHost_id Target host ID (substition for '$HOST' magic ID)
@param str _sResource_id Target resource ID (substition for '$SELF' magic ID)
@param bool _bBootstrap Whether to consider bootstrap (host startup) resources
@param tup _tPermissions Permissions tuple (<user>, <group>, <mode>)
@exception OSError On file I/O error
@exception RuntimeError On cluster variables subsitution errors
"""
if self._iVerbose: self._INFO('Caching file: %s > %s' % (_sFile_from, _sFile_to))
# Load source file
if self._iVerbose: self._DEBUG('Reading file (%s)' % _sFile_from)
if _sFile_from is None:
sFile = sys.stdin.read()
else:
with open(_sFile_from, 'r') as oFile:
sFile = oFile.read()
# Resolve variables
sFile = self.resolveString(sFile, _mHost, _mResource, _bBootstrap)
# Save destination file
if self._iVerbose: self._DEBUG('Writing file (%s)' % _sFile_to)
if _sFile_to is None:
sys.stdout.write(sFile)
else:
os.makedirs(os.path.dirname(_sFile_to), exist_ok=True)
iUmask = os.umask(0o077)
oFile = None
try:
oFile = open(_sFile_to, 'w')
oFile.write(sFile)
if _tPermissions is not None:
(mUser, mGroup, mMode) = _tPermissions
KiscRuntime.perms(oFile.fileno(), mUser, mGroup, mMode, _bTrace = self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
finally:
if oFile: oFile.close()
os.umask(iUmask)
def start(self):
lsErrors = list()
# Be idempotent
if self.status(
True,
KiscRuntime.STATUS_STARTED) == KiscRuntime.STATUS_STARTED:
if self._iVerbose: self._INFO('Already started')
return lsErrors
# Start the resource
if self._iVerbose: self._INFO('Starting')
try:
# ... stonith command
lsCommand = [
'stonith', '-s', '-S', '-t', self._dsConfig['device_type']
]
# ... arguments
lsCommand.extend(['-c', self._dsConfig.get('count', '1')])
# ... parameters
if 'parameters' in self._dsConfig:
dsParameters = KiscRuntime.parseDictionary(
self._dsConfig['parameters'], _sAssignmentOperator='=')
for sParameter_name in dsParameters:
lsCommand.append(
'%s=%s' %
(sParameter_name, dsParameters[sParameter_name]))
# ... do it !
KiscRuntime.shell(
lsCommand, _bTrace=self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
self._iStatus = KiscRuntime.STATUS_STARTED
if self._iVerbose: self._INFO('Started')
except (OSError, RuntimeError) as e:
if self._iVerbose: self._ERROR(str(e))
self._iStatus = KiscRuntime.STATUS_ERROR
lsErrors.append(str(e))
# Done
return lsErrors
def _cleanup(self):
if KiscRuntime.parseBool(self._dsConfig.get('cleanup', False)):
if 'constraint_file' in self._dsConfig:
KiscRuntime.shell(
[
'cibadmin', '-o', 'constraints', '-d', '-f', '-A',
'//rsc_location[@rsc=\'%s\']' % self._dsConfig['name']
],
_bTrace=self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
time.sleep(3)
if 'resource_file' in self._dsConfig:
KiscRuntime.shell(
[
'cibadmin', '-o', 'resources', '-D', '-A',
'//primitive[@id=\'%s\'] | //group[@id=\'%s\']' %
(self._dsConfig['name'], self._dsConfig['name'])
],
_bTrace=self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
time.sleep(3)
def start(self):
if self._iVerbose: self._INFO('Starting')
lsErrors = list()
# Be idempotent
if self.status(
True,
KiscRuntime.STATUS_STARTED) == KiscRuntime.STATUS_STARTED:
if self._iVerbose: self._INFO('Already started')
return lsErrors
# Start the resource
try:
# ... add address
lsCommand = [
'ip', '-4', 'address', 'add',
'%s/%s' % (self._dsConfig['address'], self._dsConfig['mask'])
]
# ... options
for sSetting in ['broadcast', 'anycast', 'label', 'scope']:
if sSetting in self._dsConfig:
lsCommand.extend([sSetting, self._dsConfig[sSetting]])
lsCommand.extend(['dev', self._dsConfig['device']])
# ... DO IT!
KiscRuntime.shell(
lsCommand, _bTrace=self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
self._iStatus = KiscRuntime.STATUS_STARTED
if self._iVerbose: self._INFO('Started')
except OSError as e:
if self._iVerbose: self._ERROR(str(e))
self._iStatus = KiscRuntime.STATUS_ERROR
lsErrors.append(str(e))
# Done
return lsErrors
def start(self):
if self._iVerbose: self._INFO('Starting')
lsErrors = list()
# Be idempotent
if self.status(
True,
KiscRuntime.STATUS_STARTED) == KiscRuntime.STATUS_STARTED:
if self._iVerbose: self._INFO('Already started')
return lsErrors
# Start the resource
try:
# ... mkdir ?
if KiscRuntime.parseBool(self._dsConfig.get('mkdir', True)):
os.makedirs(self._dsConfig['mountpoint'], exist_ok=True)
# ... mount device
lsCommand = ['mount', '-t', self._dsConfig['fstype']]
if 'options' in self._dsConfig:
lsCommand.extend(['-o', self._dsConfig['options']])
lsCommand.extend(
[self._dsConfig['device'], self._dsConfig['mountpoint']])
KiscRuntime.shell(
lsCommand, _bTrace=self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
self._iStatus = KiscRuntime.STATUS_STARTED
if self._iVerbose: self._INFO('Started')
except OSError as e:
if self._iVerbose: self._ERROR(str(e))
self._iStatus = KiscRuntime.STATUS_ERROR
lsErrors.append(str(e))
# Done
return lsErrors
def start(self):
if self._iVerbose: self._INFO('Starting')
lsErrors = list()
# Be idempotent
if self.status(True, KiscRuntime.STATUS_STARTED) == KiscRuntime.STATUS_STARTED:
if self._iVerbose: self._INFO('Already started')
return lsErrors
# Start the resource
try:
# ... add device
lsCommand = [
'ip', 'tuntap', 'add',
'dev', self._dsConfig['name'],
'mode', self._dsConfig['mode'],
]
# ... options
for sSetting in ['user', 'group']:
if sSetting in self._dsConfig:
lsCommand.extend([sSetting, self._dsConfig[sSetting]])
# ... DO IT!
KiscRuntime.shell(lsCommand, _bTrace = self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
self._iStatus = KiscRuntime.STATUS_STARTED
if self._iVerbose: self._INFO('Started')
except OSError as e:
if self._iVerbose: self._ERROR(str(e))
self._iStatus = KiscRuntime.STATUS_ERROR
lsErrors.append(str(e))
# Done
return lsErrors
def status(self, _bStateful = True, _iIntent = None):
if self._iVerbose: self._INFO('Querying status')
iStatus = KiscRuntime.STATUS_UNKNOWN
if _bStateful:
# Exists ?
try:
KiscRuntime.shell(['systemctl', '-q', 'is-active', self._dsConfig['name']], _bTrace = self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
iStatus = KiscRuntime.STATUS_STARTED
except OSError as e:
if e.filename == 0:
iStatus = KiscRuntime.STATUS_STOPPED
else:
if self._iVerbose: self._ERROR(str(e))
iStatus = KiscRuntime.STATUS_ERROR
else:
iStatus = self._iStatus
# Done
self._iStatus = iStatus
if self._iVerbose: self._INFO('Status is %s' % KiscRuntime.STATUS_MESSAGE[self._iStatus])
return self._iStatus
def status(self, _bStateful=True, _iIntent=None):
if self._iVerbose: self._INFO('Querying status')
iStatus = KiscRuntime.STATUS_UNKNOWN
if _bStateful:
# Query domain state
try:
sOutput = KiscRuntime.shell(
['virsh', 'domstate', self._dsConfig['name']],
_bTrace=self._iVerbose >=
KiscRuntime.VERBOSE_TRACE).strip()
if sOutput is None or not len(sOutput):
iStatus = KiscRuntime.STATUS_ERROR
elif sOutput == 'shut off':
iStatus = KiscRuntime.STATUS_STOPPED
elif sOutput == 'paused':
iStatus = KiscRuntime.STATUS_SUSPENDED
else:
iStatus = KiscRuntime.STATUS_STARTED
except OSError as e:
if e.filename == 0:
iStatus = KiscRuntime.STATUS_STOPPED
else:
if self._iVerbose: self._ERROR(str(e))
iStatus = KiscRuntime.STATUS_ERROR
else:
iStatus = self._iStatus
# Done
self._iStatus = iStatus
if self._iVerbose:
self._INFO('Status is %s' %
KiscRuntime.STATUS_MESSAGE[self._iStatus])
return self._iStatus
def start(self):
if self._iVerbose: self._INFO('Starting')
lsErrors = list()
# Be idempotent
if self.status(
True,
KiscRuntime.STATUS_STARTED) == KiscRuntime.STATUS_STARTED:
if self._iVerbose: self._INFO('Already started')
return lsErrors
# Start the resource
try:
# ... add device
lsCommand = [
'ip',
'link',
'add',
'name',
self._dsConfig['name'],
]
for sSetting in [
'address', 'mtu', 'txqueuelen', 'numtxqueues',
'numrxqueues'
]:
if sSetting in self._dsConfig:
lsCommand.extend([sSetting, self._dsConfig[sSetting]])
lsCommand.extend(['type', 'bridge'])
KiscRuntime.shell(
lsCommand, _bTrace=self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
# ... options
for sSetting in [
'ageing_time', 'stp_state', 'priority', 'hello_time',
'forward_delay', 'max_age'
]:
if sSetting in self._dsConfig:
KiscRuntime.echo(
self._dsConfig[sSetting],
'/sys/class/net/%s/bridge/%s' %
(self._dsConfig['name'], sSetting),
_bTrace=self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
# ... attach slaves
for sDevice in self._dsConfig['devices'].split(','):
KiscRuntime.shell(
[
'ip', 'link', 'set', sDevice, 'master',
self._dsConfig['name'], 'up'
],
_bTrace=self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
# ... UP!
KiscRuntime.shell(
['ip', 'link', 'set', self._dsConfig['name'], 'up'],
_bTrace=self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
self._iStatus = KiscRuntime.STATUS_STARTED
if self._iVerbose: self._INFO('Started')
except OSError as e:
if self._iVerbose: self._ERROR(str(e))
self._iStatus = KiscRuntime.STATUS_ERROR
lsErrors.append(str(e))
# Done
return lsErrors
def start(self):
lsErrors = list()
# Be idempotent
if self.status(
True,
KiscRuntime.STATUS_STARTED) == KiscRuntime.STATUS_STARTED:
if self._iVerbose: self._INFO('Already started')
return lsErrors
# Start the resource
if self._iVerbose: self._INFO('Starting')
try:
# ... satisfy ?
try:
iSatisfy = int(self._dsConfig['satisfy'])
except KeyError:
iSatisfy = None
except ValueError:
raise RuntimeError('Invalid "satisfy" setting (%s)' %
self._dsConfig['satisfy'])
iSatisfied = 0
# ... ping command
lsCommand = ['ping6', '-q', '-n']
# ... arguments
lsCommand.extend(['-c', self._dsConfig.get('count', '1')])
lsCommand.extend(['-i', self._dsConfig.get('interval', '1')])
lsCommand.extend(['-W', self._dsConfig.get('timeout', '5')])
# ... options
if 'deadline' in self._dsConfig:
lsCommand.extend(['-w', self._dsConfig['deadline']])
if 'interface' in self._dsConfig:
lsCommand.extend(['-I', self._dsConfig['interface']])
if 'mark' in self._dsConfig:
lsCommand.extend(['-m', self._dsConfig['mark']])
if 'flow' in self._dsConfig:
lsCommand.extend(['-F', self._dsConfig['flow']])
# ... loop through IP address(es)
iAddresses = 0
for sAddress in self._dsConfig['address'].split(','):
sAddress = sAddress.strip()
if not len(sAddress): continue
iAddresses += 1
lsCommand_sub = copy.deepcopy(lsCommand)
lsCommand_sub.append(sAddress)
try:
KiscRuntime.shell(
lsCommand_sub,
_bTrace=self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
iSatisfied += 1
except OSError as e:
if e.filename == 0 and e.errno == 1:
# address not reachable
pass
else:
raise e
# ... satisfied ?
if iSatisfy is None:
if iSatisfied < iAddresses:
raise RuntimeError('Ping failed (%d<%d)' %
(iSatisfied, iAddresses))
else:
if iSatisfied < iSatisfy:
raise RuntimeError('Ping failed (%d<%d)' %
(iSatisfied, iSatisfy))
self._iStatus = KiscRuntime.STATUS_STARTED
if self._iVerbose: self._INFO('Started')
except (OSError, RuntimeError) as e:
if self._iVerbose: self._ERROR(str(e))
self._iStatus = KiscRuntime.STATUS_ERROR
lsErrors.append(str(e))
# Done
return lsErrors
def resolveString(self, _sString, _mHost = None, _mResource = None, _bBootstrap = False):
"""
Resolve the cluster variables in the given string
This method will resolve cluster variables defined as '%{<ID>[.<setting>][|<filter>][|...]}'.
The following filter can be applied:
- int, float, strip, lower, upper, dirname, basename
- add(<int|float>), sub(<int|float>), mul(<int|float>), div(<int|float>)
- remove(<str>), replace(<str>,<str>)
@param str _sString Input string
@param str _mHost Target host object or ID (substition for '$HOST' magic ID)
@param str _mResource Target resource object or ID (substition for '$SELF' magic ID)
@param bool _bBootstrap Whether to consider bootstrap (host startup) resources
@return str String with cluster variables resolved
@exception RuntimeError On cluster variables subsitution error
"""
if self._iVerbose: self._DEBUG('Resolving cluster variables string (%s)' % _sString)
# 'HOST' host
if _mHost is not None and type(_mHost) is str:
_mHost = self.getHost(_mHost)
# 'SELF' resource
if _mResource is not None and type(_mResource) is str:
_mResource = self.getResource(_mResource, _bBootstrap)
# Prepare filter regexp
import re
lsRegexpFilters = [
re.compile('(int|float|strip|lower|upper|dirname|basename)'),
re.compile('(add|sub|mul|div)\( *([.0-9]+) *\)'),
re.compile('(remove)\( *\'([^\']+)\' *\)'),
re.compile('(replace)\( *\'([^\']+)\' *, *\'([^\']+)\' *\)'),
]
# Look for cluster variables
for sVariable in set(re.findall('%\{[^\{]*\}', _sString)):
if self._iVerbose: self._DEBUG('Substituting variable (%s)' % sVariable)
# ... split <resource-id>.<setting>
try:
(sResource_id, sSetting) = sVariable[2:-1].split('.')
except ValueError as e:
sResource_id = sVariable[2:-1]
sSetting = 'ID'
# ... apply filters
lsSettingFilters = sSetting.split('|')
if len(lsSettingFilters) > 1:
sSetting = lsSettingFilters[0]
# ... retrieve value
try:
# ... retrieve configuration
if sResource_id == 'KiSC':
dsConfig = self._dsConfig
elif sResource_id == '$HOST':
if _mHost is None:
raise RuntimeError('Target host not specified')
dsConfig = _mHost.config()
elif sResource_id == '$SELF':
if _mResource is None:
raise RuntimeError('Target resource not specified')
dsConfig = _mResource.config()
else:
try:
oResource_sub = self.getResource(sResource_id, False)
except RuntimeError as e:
oResource_sub = self.getResource(sResource_id, True)
dsConfig = oResource_sub.config()
# ... retrieve configuration setting
if sSetting[:8] == 'CONSUMES':
sConsumable_id = sSetting[9:-1]
diConsumables = KiscRuntime.parseDictionary(dsConfig['CONSUMES'], 1, int)
sValue = str(diConsumables[sConsumable_id])
elif sSetting[:11] == 'CONSUMABLES':
sConsumable_id = sSetting[12:-1]
diConsumables = KiscRuntime.parseDictionary(dsConfig['CONSUMABLES'], 1, int)
sValue = str(diConsumables[sConsumable_id])
else:
sValue = dsConfig[sSetting]
except (RuntimeError, KeyError) as e:
raise RuntimeError('Invalid cluster variable; %s (%s)' % (sVariable, str(e)))
if self._iVerbose: self._DEBUG('Variable value is: %s' % sValue)
# ... apply filters
if len(lsSettingFilters) > 1:
mValue = sValue
try:
for sFilter in lsSettingFilters[1:]:
bMatched = False
for sRegexpFilter in lsRegexpFilters:
oMatch = sRegexpFilter.match(sFilter)
if oMatch is None:
continue
if self._iVerbose: self._DEBUG('Applying filter: %s' % sFilter)
bMatched = True
sOperator = oMatch.group(1)
tValue = type(mValue)
if sOperator == 'int':
mValue = int(mValue)
elif sOperator == 'float':
mValue = float(mValue)
elif sOperator == 'strip':
mValue = mValue.strip()
elif sOperator == 'lower':
mValue = mValue.lower()
elif sOperator == 'upper':
mValue = mValue.upper()
elif sOperator == 'dirname':
mValue = os.path.dirname(mValue)
elif sOperator == 'basename':
mValue = os.path.basename(mValue)
elif sOperator == 'add':
if tValue is float:
mValue += float(oMatch.group(2))
else:
mValue += int(oMatch.group(2))
elif sOperator == 'sub':
if tValue is float:
mValue -= float(oMatch.group(2))
else:
mValue -= int(oMatch.group(2))
elif sOperator == 'mul':
if tValue is float:
mValue *= float(oMatch.group(2))
else:
mValue *= int(oMatch.group(2))
elif sOperator == 'div':
if tValue is float:
mValue /= float(oMatch.group(2))
else:
mValue /= int(oMatch.group(2))
elif sOperator == 'remove':
mValue = mValue.replace(oMatch.group(2), '')
elif sOperator == 'replace':
mValue = mValue.replace(oMatch.group(2), oMatch.group(3))
break
if not bMatched:
raise RuntimeError(sFilter)
sValue = str(mValue)
if self._iVerbose: self._DEBUG('Final variable value is: %s' % sValue)
except Exception as e:
raise RuntimeError('Invalid variable filter; %s' % str(e))
# ... substitute value
_sString = _sString.replace(sVariable, sValue)
# Done
if self._iVerbose: self._DEBUG('Resolved cluster variables string (%s)' % _sString)
return _sString
def start(self):
if self._iVerbose: self._INFO('Starting')
lsErrors = list()
# Be idempotent
if self.status(
True,
KiscRuntime.STATUS_STARTED) == KiscRuntime.STATUS_STARTED:
if self._iVerbose: self._INFO('Already started')
return lsErrors
# Start the resource
try:
# ... load bonding driver (but do not create any default device)
KiscRuntime.shell(
['modprobe', 'bonding', 'max_bonds=0'],
_bTrace=self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
# ... add device
lsCommand = [
'ip',
'link',
'add',
'name',
self._dsConfig['name'],
]
for sSetting in [
'address', 'mtu', 'txqueuelen', 'numtxqueues',
'numrxqueues'
]:
if sSetting in self._dsConfig:
lsCommand.extend([sSetting, self._dsConfig[sSetting]])
lsCommand.extend(['type', 'bond', 'mode', self._dsConfig['mode']])
KiscRuntime.shell(
lsCommand, _bTrace=self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
# ... options
for sSetting in [
'miimon', 'updelay', 'downdelay', 'use_carrier',
'arp_interval', 'arp_ip_target', 'arp_all_targets',
'arp_validate', 'primary_reselect', 'all_slaves_active',
'fail_over_mac', 'xmit_hash_policy', 'packets_per_slave',
'tlb_dynamic_lb', 'lacp_rate', 'ad_select', 'num_grat_arp',
'num_unsol_na', 'lp_interval', 'resend_igmp'
]:
if sSetting in self._dsConfig:
KiscRuntime.echo(
self._dsConfig[sSetting],
'/sys/class/net/%s/bonding/%s' %
(self._dsConfig['name'], sSetting),
_bTrace=self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
# ... attach slaves
for sDevice in self._dsConfig['devices'].split(','):
KiscRuntime.shell(
[
'ip', 'link', 'set', sDevice, 'master',
self._dsConfig['name'], 'up'
],
_bTrace=self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
# ... default/active slave
for sSetting in ['active_slave', 'primary']:
if sSetting in self._dsConfig:
KiscRuntime.echo(
self._dsConfig[sSetting],
'/sys/class/net/%s/bonding/%s' %
(self._dsConfig['name'], sSetting),
_bTrace=self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
# ... UP!
KiscRuntime.shell(
['ip', 'link', 'set', self._dsConfig['name'], 'up'],
_bTrace=self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
self._iStatus = KiscRuntime.STATUS_STARTED
if self._iVerbose: self._INFO('Started')
except OSError as e:
if self._iVerbose: self._ERROR(str(e))
self._iStatus = KiscRuntime.STATUS_ERROR
lsErrors.append(str(e))
# Done
return lsErrors
def stop(self, _bForce=False):
"""
Stop the host
@param bool _bForce Forcefully stop the host, ignoring non-critical errors
@return list Empty if host is successfully stopped, (ordered) error messages otherwise
"""
if self._iVerbose: self._INFO('Stopping')
lsErrors = list()
# Stop the host
try:
# ... virtual ?
bVirtual = self._oHost.isVirtual()
# ... localhost ?
sHost_id = self._oHost.id()
sLocalhost_id = self._oClusterConfig.getHostByHostname().id()
if not bVirtual:
if sHost_id != sLocalhost_id:
raise RuntimeError('Cannot stop remote host')
else:
dsConfig = self._oHost.config()
if 'HOSTS' in dsConfig and not self._oClusterConfig.isHostAllowed(
dsConfig['HOSTS'], sLocalhost_id):
raise RuntimeError(
'Local host (%s) not allowed to handle this (virtual) host'
% sLocalhost_id)
# ... runtime status check
bHost_runtime_file = self.existsRuntime()
if not bHost_runtime_file and not _bForce:
raise RuntimeError('Host not started')
# ... load runtime configuration and status from file
if bHost_runtime_file:
self.loadRuntime()
# ... check/stop the host's (regular) resources
if len(self._oHost.getResourcesIDs(_bBootstrap=False)):
if not _bForce:
raise RuntimeError('Resources are running on host')
from KiSC.Cluster.resource import KiscCluster_resource
for sResource_id in reversed(
self._oHost.getResourcesIDs(_bBootstrap=False)):
if not self._oClusterConfig.isHostResource(
sHost_id, sResource_id, _bBootstrap=False):
continue
oClusterResource = KiscCluster_resource(
self._oClusterConfig,
sHost_id,
sResource_id,
_bBootstrap=False)
oClusterResource.VERBOSE(self._iVerbose)
lsErrors_sub = oClusterResource.stop(_bForce)
if lsErrors_sub:
lsErrors.extend(lsErrors_sub)
raise RuntimeError(
'Failed to stop host\'s resource (%s)' %
sResource_id)
# ... stop the host resource
lsErrors_sub = self._oHost.stop()
if lsErrors_sub and not _bForce:
lsErrors.extend(lsErrors_sub)
raise RuntimeError('Failed to stop host resource')
# ... update runtime configuration and status from file
if bHost_runtime_file:
self.saveRuntime()
# ... stop the host's bootstrap resources
if not bVirtual:
from KiSC.Cluster.resource import KiscCluster_resource
for sResource_id in reversed(
self._oHost.getResourcesIDs(_bBootstrap=True)):
if not self._oClusterConfig.isHostResource(
sHost_id, sResource_id, _bBootstrap=True):
continue
oClusterResource = KiscCluster_resource(
self._oClusterConfig,
sHost_id,
sResource_id,
_bBootstrap=True)
oClusterResource.VERBOSE(self._iVerbose)
if not KiscRuntime.parseBool(
oClusterResource.resource().config().get(
'PERSISTENT', False)):
lsErrors_sub = oClusterResource.stop(_bForce)
if lsErrors_sub:
lsErrors.extend(lsErrors_sub)
raise RuntimeError(
'Failed to stop host\'s bootstrap resource (%s)'
% sResource_id)
# ... delete runtime file
if bHost_runtime_file:
os.unlink(self._dsPaths['runtime_file'])
# ... done
if self._iVerbose: self._INFO('Stopped')
except (OSError, RuntimeError) as e:
if self._iVerbose: self._ERROR(str(e))
lsErrors.append(str(e))
# Done
return lsErrors
def start(self):
if self._iVerbose: self._INFO('Starting')
lsErrors = list()
# Be idempotent
if self.status(
True,
KiscRuntime.STATUS_STARTED) == KiscRuntime.STATUS_STARTED:
if self._iVerbose: self._INFO('Already started')
return lsErrors
# Start the resource
try:
# ... mkdir ?
if KiscRuntime.parseBool(self._dsConfig.get('mkdir', True)):
os.makedirs(os.path.dirname(self._dsConfig['destination']),
exist_ok=True)
# ... pre-cache command ?
if 'command_pre' in self._dsConfig:
KiscRuntime.shell(
self._dsConfig['command_pre'].split(' '),
_bTrace=self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
# ... permissions
mUser = self._dsConfig.get('user', None)
mGroup = self._dsConfig.get('group', None)
try:
mMode = int(self._dsConfig['mode'], 8)
except KeyError:
mMode = None
# ... cache file
if 'config_file' in self._dsConfig:
oClusterConfig = KiscCluster_config(
self._dsConfig['config_file'])
sHost_id = oClusterConfig.getHostByHostname().id()
oClusterConfig.resolveFile(self._dsConfig['source'],
self._dsConfig['destination'],
_sHost_id=sHost_id,
_tPermissions=(mUser, mGroup,
mMode))
else:
if self._iVerbose:
self._DEBUG('Reading file (%s)' % self._dsConfig['source'])
with open(self._dsConfig['source'], 'r') as oFile:
sFile = oFile.read()
if self._iVerbose:
self._DEBUG('Writing file (%s)' %
self._dsConfig['destination'])
iUmask = os.umask(0o077)
oFile = None
try:
oFile = open(self._dsConfig['destination'], 'w')
KiscRuntime.perms(
oFile.fileno(),
mUser,
mGroup,
mMode,
_bTrace=self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
oFile.write(sFile)
finally:
if oFile: oFile.close()
os.umask(iUmask)
# ... post-cache command ?
if 'command_post' in self._dsConfig:
KiscRuntime.shell(
self._dsConfig['command_post'].split(' '),
_bTrace=self._iVerbose >= KiscRuntime.VERBOSE_TRACE)
# ... done
self._iStatus = KiscRuntime.STATUS_STARTED
if self._iVerbose: self._INFO('Started')
except (OSError, RuntimeError) as e:
if self._iVerbose: self._ERROR(str(e))
self._iStatus = KiscRuntime.STATUS_ERROR
lsErrors.append(str(e))
# Done
return lsErrors
