def getServiceAddCmd(self, user):
version = self.getScidbVersion()
initdBinSrc = os.path.join(self._ctx._installPath, "bin", "scidb.initd")
initdName = self.getInitdBinName(version)
initdBinPath = os.path.join("/etc/init.d", initdName)
configDBDir = self.getConfigDBDir()
installPath = "\\\\/".join(self._ctx._installPath.split("/"))
scidb.printDebug("installPath = %s"%(installPath))
cmdList=["/bin/sed", "s/XXX_INSTALL_PATH_XXX/"+installPath+"/", initdBinSrc,
"|", "/bin/sed", "s/XXX_SERVICE_USER_XXX/"+user+"/",
"|", "/bin/sed", "s/XXX_SERVICE_NAME_XXX/"+initdName+"/",
">", initdBinPath]
cmdList.extend([" && ", "/bin/chmod", "a-rwx,u+rwx,g+rx,o+rx", initdBinPath])
cmdList.extend([" && ", "/bin/mkdir", "-p", configDBDir])
cmdList.extend([" && ", "/bin/chown", user, configDBDir])
#
# On Ubuntu use update-rc.d to manage services
# On CentOS/RedHat use chkconfig to manage services
#
if os.path.exists("/usr/sbin/update-rc.d"):
cmdList.extend([" && ", "/usr/sbin/update-rc.d", initdName, "defaults"])
else:
cmdList.extend([" && ", "/sbin/chkconfig", "--add", initdName])
scidb.printDebug("cmd = %s"%(str(cmdList)))
return cmdList
def getServiceAddCmd(self, user):
version = self.getScidbVersion()
initdBinSrc = os.path.join(self._ctx._installPath, "bin",
"scidb.initd")
initdName = self.getInitdBinName(version)
initdBinPath = os.path.join("/etc/init.d", initdName)
configDBDir = self.getConfigDBDir()
installPath = "\\\\/".join(self._ctx._installPath.split("/"))
scidb.printDebug("installPath = %s" % (installPath))
cmdList = [
"/bin/sed", "s/XXX_INSTALL_PATH_XXX/" + installPath + "/",
initdBinSrc, "|", "/bin/sed",
"s/XXX_SERVICE_USER_XXX/" + user + "/", "|", "/bin/sed",
"s/XXX_SERVICE_NAME_XXX/" + initdName + "/", ">", initdBinPath
]
cmdList.extend(
[" && ", "/bin/chmod", "a-rwx,u+rwx,g+rx,o+rx", initdBinPath])
cmdList.extend([" && ", "/bin/mkdir", "-p", configDBDir])
cmdList.extend([" && ", "/bin/chown", user, configDBDir])
#
# On Ubuntu use update-rc.d to manage services
# On CentOS/RedHat use chkconfig to manage services
#
if os.path.exists("/usr/sbin/update-rc.d"):
cmdList.extend(
[" && ", "/usr/sbin/update-rc.d", initdName, "defaults"])
else:
cmdList.extend([" && ", "/sbin/chkconfig", "--add", initdName])
scidb.printDebug("cmd = %s" % (str(cmdList)))
return cmdList
def runRemote(self, servers, func, opStr, remoteUsers=None, remotePwds=None, conns=None):
"""
Run remote commands generate by a given functor on a list of remote servers
"""
closeSSH = False
if not conns:
closeSSH = True
cons = []
try:
if closeSSH:
if not remoteUsers:
conns = [scidb.sshconnect(srv) for srv in servers]
else:
conns = [scidb.sshconnect(trio[0],username=trio[1], password=trio[2])
for trio in zip(servers,remoteUsers,remotePwds)]
# generate remote commands
cmds = func(servers,conns)
# execute
(ret,out,err) = scidb.parallelRemoteExec(conns, cmds)
map(lambda c: c.close(),conns)
finally:
if closeSSH: scidb.sshCloseNoError(conns)
i=0
scidb.printDebug("parallelRemoteExec out's: %s"%(str(out)))
scidb.printDebug("parallelRemoteExec err's: %s"%(str(err)))
for rc in ret:
if rc != 0:
raise AppError("Failed to %s on server %d, errors: %s" % (
opStr, servers[i][0], str(err)))
i += 1
def getServiceRegisterCmd(self, srv):
configCopyPath = self.getConfigDBFilename(srv)
cmdList = ["/bin/chmod", "a-rwx,u+r", configCopyPath+".tmp"]
cmdList.extend(["&&", "/bin/mv", configCopyPath+".tmp", configCopyPath])
cmdList.extend(["&&", "/bin/chmod", "a-rwx,u+r", configCopyPath])
cmd = " ".join(cmdList)
scidb.printDebug("cmd = %s"%(cmd))
return cmd
def getServiceRegisterCmd(self, srv):
configCopyPath = self.getConfigDBFilename(srv)
cmdList = ["/bin/chmod", "a-rwx,u+r", configCopyPath + ".tmp"]
cmdList.extend(
["&&", "/bin/mv", configCopyPath + ".tmp", configCopyPath])
cmdList.extend(["&&", "/bin/chmod", "a-rwx,u+r", configCopyPath])
cmd = " ".join(cmdList)
scidb.printDebug("cmd = %s" % (cmd))
return cmd
def getScidbVersion(self):
cmdList=[self._ctx._installPath+"/bin/scidb", "--version"]
(ret,out,err) = scidb.executeLocal(cmdList, None,
nocwd=True,
useConnstr=False,
useShell=False,
ignoreError=False,
sout=os.tmpfile())
version = out.strip().split("\n")[0].split(":")[1].split(".")
version = ".".join(version[0:2]).strip()
scidb.printDebug(" version = %s"%(version))
return version
def getScidbVersion(self):
cmdList = [self._ctx._installPath + "/bin/scidb", "--version"]
(ret, out, err) = scidb.executeLocal(cmdList,
None,
nocwd=True,
useConnstr=False,
useShell=False,
ignoreError=False,
sout=os.tmpfile())
version = out.strip().split("\n")[0].split(":")[1].split(".")
version = ".".join(version[0:2]).strip()
scidb.printDebug(" version = %s" % (version))
return version
def getServiceUnregisterCmd(self, srv, force=False):
version = self.getScidbVersion()
configCopyPath = self.getConfigDBFilename(srv)
if force:
sep = ";"
else:
sep = "&&"
cmdList = ["/bin/rm", "-f", configCopyPath + ".tmp"]
cmdList.extend([sep, "/bin/rm", configCopyPath])
if force:
cmdList.extend(["||", "true"])
cmd = " ".join(cmdList)
scidb.printDebug("cmd = %s" % (cmd))
return cmd
def getServiceUnregisterCmd(self, srv, force=False):
version = self.getScidbVersion()
configCopyPath = self.getConfigDBFilename(srv)
if force:
sep=";"
else:
sep="&&"
cmdList=["/bin/rm", "-f", configCopyPath+".tmp"]
cmdList.extend([sep, "/bin/rm", configCopyPath])
if force:
cmdList.extend(["||", "true"])
cmd = " ".join(cmdList)
scidb.printDebug("cmd = %s"%(cmd))
return cmd
def runRemote(self,
servers,
func,
opStr,
remoteUsers=None,
remotePwds=None,
conns=None):
"""
Run remote commands generate by a given functor on a list of remote servers
"""
closeSSH = False
if not conns:
closeSSH = True
cons = []
try:
if closeSSH:
if not remoteUsers:
conns = [scidb.sshconnect(srv) for srv in servers]
else:
conns = [
scidb.sshconnect(trio[0],
username=trio[1],
password=trio[2])
for trio in zip(servers, remoteUsers, remotePwds)
]
# generate remote commands
cmds = func(servers, conns)
# execute
(ret, out, err) = scidb.parallelRemoteExec(conns, cmds)
map(lambda c: c.close(), conns)
finally:
if closeSSH: scidb.sshCloseNoError(conns)
i = 0
scidb.printDebug("parallelRemoteExec out's: %s" % (str(out)))
scidb.printDebug("parallelRemoteExec err's: %s" % (str(err)))
for rc in ret:
if rc != 0:
raise AppError("Failed to %s on server %d, errors: %s" %
(opStr, servers[i][0], str(err)))
i += 1
def depositConfigDBFile(self, sshClient, remoteFile, config):
"""
Deposit a configuration file used by the SciDB service to a remote host
using sftp
"""
sftp = None
sftpFile = None
try:
sftp = sshClient.get_transport().open_sftp_client()
scidb.printDebug("@@@@ sftp remote file is: %s" % remoteFile)
sftpFile = sftp.file(remoteFile, mode='wx') #O_EXCL
config.write(sftpFile)
sftpFile.close()
sftpFile = None
sftp.close()
sftp = None
finally:
if sftpFile: scidb.sshCloseNoError([sftpFile])
if sftp: scidb.sshCloseNoError([sftp])
def depositConfigDBFile(self, sshClient, remoteFile, config):
"""
Deposit a configuration file used by the SciDB service to a remote host
using sftp
"""
sftp = None
sftpFile = None
try:
sftp = sshClient.get_transport().open_sftp_client()
scidb.printDebug("@@@@ sftp remote file is: %s" % remoteFile)
sftpFile = sftp.file(remoteFile, mode='wx') #O_EXCL
config.write(sftpFile)
sftpFile.close()
sftpFile=None
sftp.close()
sftp=None
finally:
if sftpFile: scidb.sshCloseNoError([sftpFile])
if sftp: scidb.sshCloseNoError([sftp])
def getServiceRemoveCmd(self, force=False):
version = self.getScidbVersion()
initdName = self.getInitdBinName(version)
initdBinPath = os.path.join("/etc/init.d", initdName)
if force:
sep = ";"
else:
sep = "&&"
#
# On Ubuntu use update-rc.d to manage services
# On CentOS/RedHat use chkconfig to manage services
#
if os.path.exists("/usr/sbin/update-rc.d"):
cmdList = ["/usr/sbin/update-rc.d", "-f", initdName, "remove"]
else:
cmdList = ["/sbin/chkconfig", "--del", initdName]
cmdList.extend([sep, "/bin/rm", initdBinPath])
if force:
cmdList.extend(["||", "true"])
scidb.printDebug("cmd = %s" % (str(cmdList)))
return cmdList
def getServiceRemoveCmd(self, force=False):
version = self.getScidbVersion()
initdName = self.getInitdBinName(version)
initdBinPath = os.path.join("/etc/init.d", initdName)
if force:
sep=";"
else:
sep="&&"
#
# On Ubuntu use update-rc.d to manage services
# On CentOS/RedHat use chkconfig to manage services
#
if os.path.exists("/usr/sbin/update-rc.d"):
cmdList=[ "/usr/sbin/update-rc.d", "-f", initdName, "remove"]
else:
cmdList=[ "/sbin/chkconfig", "--del", initdName]
cmdList.extend([sep, "/bin/rm", initdBinPath])
if force:
cmdList.extend(["||", "true"])
scidb.printDebug("cmd = %s"%(str(cmdList)))
return cmdList
def unregisterSomeServers(self, deltaSrvs, force):
"""
Unregister the instances on the specified servers from SciDB
using the unregister_instances() operator
@param deltaSrvs the servers whose instances to be unregistered
@param force if set the existence of data directories is not checked
and the absence of the specified instances in SciDB is ignored.
If unset, the above conditions will cause an exception
@throw scidblib.AppError
"""
if not force:
self.checkDirs(deltaSrvs)
coordinator = self._ctx._srvList[0] #coordinator
iqueryPrefix = [self._ctx._installPath + "/bin/iquery"]
if self._ctx._args.auth_file:
iqueryPrefix.extend(['--auth-file', self._ctx._args.auth_file])
iid = coordinator.getServerInstances()[0]
scidb.printDebug("Coordinator srv: %s" % str(coordinator))
scidb.printDebug("Coordinator instance %s" % str(iid))
iqueryPrefix.extend([
"-c",
coordinator.getServerHost(), "-p",
str(self._ctx._basePort + iid)
])
iqueryPrefix.extend(["-o", 'csv'])
cmdList = [i for i in iqueryPrefix]
cmdList.extend([
"-aq",
"\"project(list_instances(), server_id, server_instance_id, instance_id);\""
])
(ret, out, err) = scidb.executeLocal(cmdList,
None,
nocwd=True,
useConnstr=False,
ignoreError=False,
useShell=True,
sout=os.tmpfile())
assert ret == 0, "Unexpected error from: %s " % str(cmdList)
scidb.printDebug("Instances: %s" % out)
lines = out.splitlines()
out = None
parsedIds = {}
def parseLine(line):
ids = line.lstrip().rstrip().split(',')
if len(ids) < 3:
raise AppError(
"Unexpected instance [server_id,server_instance_id,instance_id]: %s "
% (ids))
parsedIds[','.join(ids[0:2])] = int(ids[2])
map(parseLine, lines)
scidb.printDebug("Parsed IDs: %s " % str(parsedIds))
instance_ids = []
for srv in deltaSrvs:
for liid in srv.getServerInstances():
key = ','.join([str(srv.getServerId()), str(liid)])
if key in parsedIds:
instance_ids.append(parsedIds[key])
else:
msg = "Unknown instance: %s " % (scidb.validateInstance(
srv, liid))
if force:
scidb.printWarn(msg)
else:
raise AppError(msg)
ret = 0
if len(instance_ids) > 0:
cmdList = [i for i in iqueryPrefix]
cmdList.extend([
"-naq",
"\"unregister_instances(%s);\"" %
(",".join(map(str, instance_ids)))
])
(ret, out, err) = scidb.executeLocal(cmdList,
None,
nocwd=True,
useConnstr=False,
ignoreError=False,
useShell=True)
elif not force:
raise AppError("No instances to unregister")
return (ret == 0)
def unregisterSomeServers(self, deltaSrvs, force):
"""
Unregister the instances on the specified servers from SciDB
using the unregister_instances() operator
@param deltaSrvs the servers whose instances to be unregistered
@param force if set the existence of data directories is not checked
and the absence of the specified instances in SciDB is ignored.
If unset, the above conditions will cause an exception
@throw scidblib.AppError
"""
if not force:
self.checkDirs(deltaSrvs)
coordinator = self._ctx._srvList[0] #coordinator
iqueryPrefix = [ self._ctx._installPath + "/bin/iquery"]
if self._ctx._args.auth_file:
iqueryPrefix.extend(['--auth-file', self._ctx._args.auth_file])
iid = coordinator.getServerInstances()[0]
scidb.printDebug("Coordinator srv: %s" % str(coordinator))
scidb.printDebug("Coordinator instance %s" % str(iid))
iqueryPrefix.extend ([
"-c"
, coordinator.getServerHost()
, "-p"
, str(self._ctx._basePort + iid)
])
iqueryPrefix.extend([ "-o", 'csv'])
cmdList = [i for i in iqueryPrefix]
cmdList.extend(["-aq", "\"project(list_instances(), server_id, server_instance_id, instance_id);\"" ])
(ret,out,err) = scidb.executeLocal(cmdList,
None,
nocwd=True,
useConnstr=False,
ignoreError=False,
useShell=True,
sout=os.tmpfile())
assert ret==0, "Unexpected error from: %s " % str(cmdList)
scidb.printDebug("Instances: %s" % out)
lines = out.splitlines()
out=None
parsedIds = {}
def parseLine(line):
ids = line.lstrip().rstrip().split(',')
if len(ids) < 3:
raise AppError("Unexpected instance [server_id,server_instance_id,instance_id]: %s " % (ids))
parsedIds[','.join(ids[0:2])] = int(ids[2])
map(parseLine, lines)
scidb.printDebug("Parsed IDs: %s " % str(parsedIds))
instance_ids = []
for srv in deltaSrvs:
for liid in srv.getServerInstances():
key = ','.join([str(srv.getServerId()), str(liid)])
if key in parsedIds:
instance_ids.append(parsedIds[key])
else:
msg = "Unknown instance: %s " % (scidb.validateInstance(srv, liid))
if force:
scidb.printWarn(msg)
else:
raise AppError(msg)
ret = 0
if len(instance_ids) > 0:
cmdList = [i for i in iqueryPrefix]
cmdList.extend(["-naq", "\"unregister_instances(%s);\"" % (",".join(map(str, instance_ids)))])
(ret,out,err) = scidb.executeLocal(cmdList,
None,
nocwd=True,
useConnstr=False,
ignoreError=False,
useShell=True)
elif not force:
raise AppError("No instances to unregister")
return (ret==0)