def __execCmdInFreiburg(self, cmd):
"""Execute command on Freiburg login node via SSH.
:param cmd:
:return: Tuple: (return_code, std_out, std_err)
"""
frSsh = Ssh(host=self.getConfig(self.configFreiburgServer),
username=self.getConfig(self.configFreiburgUser),
key=self.getConfig(self.configFreiburgKey))
return frSsh.handleSshCall(call=cmd, quiet=True)
def __execCmdInFreiburg(self, cmd):
"""Execute command on Freiburg login node via SSH.
:param cmd:
:return: Tuple: (return_code, std_out, std_err)
"""
frSsh = Ssh(host=self.getConfig(self.configFreiburgServer),
username=self.getConfig(self.configFreiburgUser),
key=self.getConfig(self.configFreiburgKey))
return frSsh.handleSshCall(call=cmd, quiet=True)
def __cancelFreiburgMachines(self, batchJobIds):
"""Cancel batch job (VM) in Freiburg.
:param batchJobIds:
:type batchJobIds: list
:return: [idsRemoved], [idsInvalidated]
"""
# It is also possible to use just one single command with multiple ids, but no machine gets
# cancelled if a single id is invalid! This can happen when the VM fails to boot due to
# network problems.
command = ""
if not isinstance(batchJobIds, (list, tuple)):
batchJobIds = [batchJobIds]
for batchJobId in batchJobIds:
command += "mjobctl -c %s; " % batchJobId
result = self.__execCmdInFreiburg(command)
# catch 0:"successful" and 1:"invalid job id" return codes
# the return code of the first cancellation command is returned here, we can handle them
# both to remove cancelled and invalid machines
idsRemoved = []
idsInvalidated = []
if result[0] <= 1:
Ssh.debugOutput(self.logger, "FR-terminate", result)
idsRemoved += re.findall("\'(\d+)\'", result[1])
idsInvalidated += re.findall("invalid job specified \((\d+)",
result[2])
if len(idsRemoved) > 0:
self.logger.info("Terminated machines (%d): %s" %
(len(idsRemoved), ", ".join(idsRemoved)))
if len(idsInvalidated) > 0:
self.logger.warning(
"Removed invalid machines (%d): %s" %
(len(idsInvalidated), ", ".join(idsInvalidated)))
if (len(idsRemoved) + len(idsInvalidated)) == 0:
self.logger.warning(
"A problem occurred while canceling VMs (RC: %d)\n%s" %
(result[0], result[2]))
else:
self.logger.warning(
"A problem occurred while canceling VMs (RC: %d)\n%s" %
(result[0], result[2]))
return idsRemoved, idsInvalidated
def __cancelFreiburgMachines(self, batchJobIds):
"""Cancel batch job (VM) in Freiburg.
:param batchJobIds:
:type batchJobIds: list
:return: [idsRemoved], [idsInvalidated]
"""
# It is also possible to use just one single command with multiple ids, but no machine gets
# cancelled if a single id is invalid! This can happen when the VM fails to boot due to
# network problems.
command = ""
if not isinstance(batchJobIds, (list, tuple)):
batchJobIds = [batchJobIds]
for batchJobId in batchJobIds:
command += "mjobctl -c %s; " % batchJobId
result = self.__execCmdInFreiburg(command)
# catch 0:"successful" and 1:"invalid job id" return codes
# the return code of the first cancellation command is returned here, we can handle them
# both to remove cancelled and invalid machines
idsRemoved = []
idsInvalidated = []
if result[0] <= 1:
Ssh.debugOutput(self.logger, "FR-terminate", result)
idsRemoved += re.findall("\'(\d+)\'", result[1])
idsInvalidated += re.findall("invalid job specified \((\d+)", result[2])
if len(idsRemoved) > 0:
self.logger.info("Terminated machines (%d): %s" % (len(idsRemoved), ", ".join(idsRemoved)))
if len(idsInvalidated) > 0:
self.logger.warning("Removed invalid machines (%d): %s"
% (len(idsInvalidated), ", ".join(idsInvalidated)))
if (len(idsRemoved) + len(idsInvalidated)) == 0:
self.logger.warning("A problem occurred while canceling VMs (RC: %d)\n%s" % (result[0], result[2]))
else:
self.logger.warning("A problem occurred while canceling VMs (RC: %d)\n%s" % (result[0], result[2]))
return idsRemoved, idsInvalidated
