def qconf_refresh(arguments):
from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper
qcm = QueueConfigMapper()
qcm.lastUpdate = None
qcm.load_data()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--queueName', action='store', dest='queueName', default=None, required=True,
help='the name of queue where harvester is installed')
parser.add_argument('--middleware', action='store', dest='middleware', default='rpc',
help='middleware to access the remote target machine')
options = parser.parse_args()
# get queue
qcm = QueueConfigMapper()
qcm.load_data()
queueConfig = qcm.get_queue(options.queueName)
if queueConfig is None:
print ('ERROR: queue={0} not found in panda_queueconfig.json'.format(options.queueName))
sys.exit(1)
# get middleware
if not hasattr(queueConfig, options.middleware):
print ('ERROR: middleware={0} is not defined for {1} in panda_queueconfig.json'.format(options.middleware,
options.queueName))
sys.exit(1)
middleware = getattr(queueConfig, options.middleware)
# get ssh parameters
sshHost = middleware['remoteHost']
try:
sshPort = middleware['remotePort']
except Exception:
sshPort = 22
sshUserName = middleware['sshUserName']
try:
sshPassword = middleware['sshPassword']
except Exception:
sshPassword = None
privateKey = None
passPhrase = None
if sshPassword is None:
try:
privateKey = middleware['privateKey']
except Exception:
print ("ERROR: set sshPassword or privateKey in middleware={0}".format(options.middleware))
sys.exit(1)
try:
passPhrase = middleware['passPhrase']
except Exception:
passPhrase = None
try:
jumpHost = middleware['jumpHost']
except Exception:
jumpHost = None
try:
jumpPort = middleware['jumpPort']
except Exception:
jumpPort = 22
# ssh
sshTunnelPool.make_tunnel_server(sshHost, sshPort, remote_bind_port=middleware['remoteBindPort'],
num_tunnels=1, ssh_username=sshUserName, ssh_password=sshPassword,
private_key=privateKey, pass_phrase=passPhrase,
jump_host=jumpHost, jump_port=jumpPort
)
ssh = sshTunnelPool.get_tunnel(sshHost, sshPort)[-1]
return ssh
def qconf_refresh(arguments):
from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper
qcm = QueueConfigMapper()
qcm._update_last_reload_time()
qcm.lastUpdate = None
qcm.load_data(refill_table=arguments.refill)
def main():
logging.basicConfig()
parser = argparse.ArgumentParser()
parser.add_argument(
'--remoteDir',
action='store',
dest='remoteDir',
default='harvester',
help=
'directory on the remote target machine where harvester is installed')
parser.add_argument(
'--remoteBuildDir',
action='store',
dest='remoteBuildDir',
default='harvester_build',
help='directory on the remote target machine where harvester is build')
parser.add_argument('--remotePythonSetup',
action='store',
dest='remotePythonSetup',
default='',
help='python setup on remote target machine')
parser.add_argument('--queueName',
action='store',
dest='queueName',
default=None,
required=True,
help='the name of queue where harvester is installed')
parser.add_argument('--middleware',
action='store',
dest='middleware',
default='rpc',
help='middleware to access the remote target machine')
options = parser.parse_args()
# remove ~/ which doesn't work with sftp
options.remoteDir = re.sub('^~/', '', options.remoteDir)
options.remoteBuildDir = re.sub('^~/', '', options.remoteBuildDir)
# get queue
qcm = QueueConfigMapper()
qcm.load_data()
queueConfig = qcm.get_queue(options.queueName)
if queueConfig is None:
print('ERROR: queue={0} not found in panda_queueconfig.json'.format(
options.queueName))
sys.exit(1)
# get middleware
if not hasattr(queueConfig, options.middleware):
print(
'ERROR: middleware={0} is not defined for {1} in panda_queueconfig.json'
.format(options.middleware, options.queueName))
sys.exit(1)
middleware = getattr(queueConfig, options.middleware)
# get ssh parameters
sshHost = middleware['remoteHost']
try:
sshPort = middleware['remotePort']
except Exception:
sshPort = 22
sshUserName = middleware['sshUserName']
try:
sshPassword = middleware['sshPassword']
except Exception:
sshPassword = None
privateKey = None
passPhrase = None
if sshPassword is None:
try:
privateKey = middleware['privateKey']
except Exception:
print("ERROR: set sshPassword or privateKey in middleware={0}".
format(options.middleware))
sys.exit(1)
try:
passPhrase = middleware['passPhrase']
except Exception:
passPhrase = None
try:
jumpHost = middleware['jumpHost']
except Exception:
jumpHost = None
try:
jumpPort = middleware['jumpPort']
except Exception:
jumpPort = 22
# ssh
sshClient = make_ssh_connection(sshHost, sshPort, sshUserName, sshPassword,
passPhrase, privateKey, jumpHost, jumpPort)
# get remote python version
exec_out = sshClient.exec_command(';'.join([
options.remotePythonSetup,
"""python -c 'import sys;print("{0}{1}".format(*(sys.version_info[:2])))' """
]))
remotePythonVer = exec_out[1].read().rstrip()
sshClient.close()
print('remote python version : {0}'.format(remotePythonVer))
# make tmp dir
with TemporaryDirectory() as tmpDir:
harvesterGit = "git+git://github.com/PanDAWMS/panda-harvester.git"
# get all dependencies
print("getting dependencies")
p = subprocess.Popen("pip download -d {0} {1}; rm -rf {0}/*".format(
tmpDir, harvesterGit),
stdout=subprocess.PIPE,
shell=True)
stdout, stderr = p.communicate()
packages = []
for line in stdout.split('\n'):
if line.startswith('Successfully downloaded'):
packages = line.split()[2:]
packages.append(harvesterGit)
packages.append('pip')
packages.remove('pandaharvester')
# download packages
print("pip download to {0}".format(tmpDir))
for package in packages:
print("getting {0}".format(package))
ret = subprocess.call(
"pip download --no-deps --python-version {0} -d {1} {2}".
format(remotePythonVer, tmpDir, package),
shell=True)
if ret != 0:
print("ERROR: failed to download {0}".format(package))
sys.exit(1)
# sftp
sshClient = make_ssh_connection(sshHost, sshPort, sshUserName,
sshPassword, passPhrase, privateKey,
jumpHost, jumpPort)
try:
sshClient.exec_command('rm -rf {0}; mkdir -p {0}'.format(
options.remoteBuildDir))
except Exception:
pass
sftp = sshClient.open_sftp()
for name in os.listdir(tmpDir):
path = os.path.join(tmpDir, name)
if os.path.isdir(path):
continue
remotePath = os.path.join(options.remoteBuildDir, name)
print("copy {0} to {1}".format(name, remotePath))
sftp.put(path, remotePath)
# install
print("install harvester")
buildDir = options.remoteBuildDir
if not buildDir.startswith('/'):
buildDir = '~/' + buildDir
exec_out = sshClient.exec_command(';'.join([
options.remotePythonSetup, 'cd {0}'.format(options.remoteDir),
'pip install pip pandaharvester --no-index --find-links {0}'.
format(buildDir)
]))
print(exec_out[1].read())
print(exec_out[2].read())
sshClient.close()
def main():
logging.basicConfig()
parser = argparse.ArgumentParser()
parser.add_argument('--remoteDir', action='store', dest='remoteDir', default='harvester',
help='directory on the remote target machine where harvester is installed')
parser.add_argument('--remoteBuildDir', action='store', dest='remoteBuildDir', default='harvester_build',
help='directory on the remote target machine where harvester is build')
parser.add_argument('--remotePythonSetup', action='store', dest='remotePythonSetup', default='',
help='python setup on remote target machine')
parser.add_argument('--queueName', action='store', dest='queueName', default=None, required=True,
help='the name of queue where harvester is installed')
parser.add_argument('--middleware', action='store', dest='middleware', default='rpc',
help='middleware to access the remote target machine')
options = parser.parse_args()
# remove ~/ which doesn't work with sftp
options.remoteDir = re.sub('^~/', '', options.remoteDir)
options.remoteBuildDir = re.sub('^~/', '', options.remoteBuildDir)
# get queue
qcm = QueueConfigMapper()
qcm.load_data()
queueConfig = qcm.get_queue(options.queueName)
if queueConfig is None:
print ('ERROR: queue={0} not found in panda_queueconfig.json'.format(options.queueName))
sys.exit(1)
# get middleware
if not hasattr(queueConfig, options.middleware):
print ('ERROR: middleware={0} is not defined for {1} in panda_queueconfig.json'.format(options.middleware,
options.queueName))
sys.exit(1)
middleware = getattr(queueConfig, options.middleware)
# get ssh parameters
sshHost = middleware['remoteHost']
try:
sshPort = middleware['remotePort']
except Exception:
sshPort = 22
sshUserName = middleware['sshUserName']
try:
sshPassword = middleware['sshPassword']
except Exception:
sshPassword = None
privateKey = None
passPhrase = None
if sshPassword is None:
try:
privateKey = middleware['privateKey']
except Exception:
print ("ERROR: set sshPassword or privateKey in middleware={0}".format(options.middleware))
sys.exit(1)
try:
passPhrase = middleware['passPhrase']
except Exception:
passPhrase = None
try:
jumpHost = middleware['jumpHost']
except Exception:
jumpHost = None
try:
jumpPort = middleware['jumpPort']
except Exception:
jumpPort = 22
# ssh
sshClient = make_ssh_connection(sshHost, sshPort, sshUserName, sshPassword, passPhrase, privateKey,
jumpHost, jumpPort)
# get remote python version
exec_out = sshClient.exec_command(
';'.join([options.remotePythonSetup,
"""python -c 'import sys;print("{0}{1}".format(*(sys.version_info[:2])))' """])
)
remotePythonVer = exec_out[1].read().rstrip()
sshClient.close()
print ('remote python version : {0}'.format(remotePythonVer))
# make tmp dir
with TemporaryDirectory() as tmpDir:
harvesterGit = "git+git://github.com/PanDAWMS/panda-harvester.git"
# get all dependencies
print ("getting dependencies")
p = subprocess.Popen("pip download -d {0} {1}; rm -rf {0}/*".format(tmpDir, harvesterGit),
stdout=subprocess.PIPE,
shell=True)
stdout, stderr = p.communicate()
packages = []
for line in stdout.split('\n'):
if line.startswith('Successfully downloaded'):
packages = line.split()[2:]
packages.append(harvesterGit)
packages.append('pip')
packages.remove('pandaharvester')
# download packages
print ("pip download to {0}".format(tmpDir))
for package in packages:
print ("getting {0}".format(package))
ret = subprocess.call("pip download --no-deps --python-version {0} -d {1} {2}".format(remotePythonVer,
tmpDir, package),
shell=True)
if ret != 0:
print ("ERROR: failed to download {0}".format(package))
sys.exit(1)
# sftp
sshClient = make_ssh_connection(sshHost, sshPort, sshUserName, sshPassword, passPhrase, privateKey,
jumpHost, jumpPort)
try:
sshClient.exec_command('rm -rf {0}; mkdir -p {0}'.format(options.remoteBuildDir))
except Exception:
pass
sftp = sshClient.open_sftp()
for name in os.listdir(tmpDir):
path = os.path.join(tmpDir, name)
if os.path.isdir(path):
continue
remotePath = os.path.join(options.remoteBuildDir, name)
print ("copy {0} to {1}".format(name, remotePath))
sftp.put(path, remotePath)
# install
print ("install harvester")
buildDir = options.remoteBuildDir
if not buildDir.startswith('/'):
buildDir = '~/' + buildDir
exec_out = sshClient.exec_command(
';'.join([options.remotePythonSetup,
'cd {0}'.format(options.remoteDir),
'pip install pip pandaharvester --no-index --find-links {0}'.format(buildDir)])
)
print (exec_out[1].read())
print (exec_out[2].read())
sshClient.close()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--queueName',
action='store',
dest='queueName',
default=None,
required=True,
help='the name of queue where harvester is installed')
parser.add_argument('--middleware',
action='store',
dest='middleware',
default='rpc',
help='middleware to access the remote target machine')
options = parser.parse_args()
# get queue
qcm = QueueConfigMapper()
qcm.load_data()
queueConfig = qcm.get_queue(options.queueName)
if queueConfig is None:
print('ERROR: queue={0} not found in panda_queueconfig.json'.format(
options.queueName))
sys.exit(1)
# get middleware
if not hasattr(queueConfig, options.middleware):
print(
'ERROR: middleware={0} is not defined for {1} in panda_queueconfig.json'
.format(options.middleware, options.queueName))
sys.exit(1)
middleware = getattr(queueConfig, options.middleware)
# get ssh parameters
sshHost = middleware['remoteHost']
try:
sshPort = middleware['remotePort']
except Exception:
sshPort = 22
sshUserName = middleware['sshUserName']
try:
sshPassword = middleware['sshPassword']
except Exception:
sshPassword = None
privateKey = None
passPhrase = None
if sshPassword is None:
try:
privateKey = middleware['privateKey']
except Exception:
print("ERROR: set sshPassword or privateKey in middleware={0}".
format(options.middleware))
sys.exit(1)
try:
passPhrase = middleware['passPhrase']
except Exception:
passPhrase = None
try:
jumpHost = middleware['jumpHost']
except Exception:
jumpHost = None
try:
jumpPort = middleware['jumpPort']
except Exception:
jumpPort = 22
# ssh
sshTunnelPool.make_tunnel_server(
sshHost,
sshPort,
remote_bind_port=middleware['remoteBindPort'],
num_tunnels=1,
ssh_username=sshUserName,
ssh_password=sshPassword,
private_key=privateKey,
pass_phrase=passPhrase,
jump_host=jumpHost,
jump_port=jumpPort)
ssh = sshTunnelPool.get_tunnel(sshHost, sshPort)[-1]
return ssh
