def qconf_refresh(arguments): from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper qcm = QueueConfigMapper() qcm.lastUpdate = None qcm.load_data()
def main(): parser = argparse.ArgumentParser() parser.add_argument('--queueName', action='store', dest='queueName', default=None, required=True, help='the name of queue where harvester is installed') parser.add_argument('--middleware', action='store', dest='middleware', default='rpc', help='middleware to access the remote target machine') options = parser.parse_args() # get queue qcm = QueueConfigMapper() qcm.load_data() queueConfig = qcm.get_queue(options.queueName) if queueConfig is None: print ('ERROR: queue={0} not found in panda_queueconfig.json'.format(options.queueName)) sys.exit(1) # get middleware if not hasattr(queueConfig, options.middleware): print ('ERROR: middleware={0} is not defined for {1} in panda_queueconfig.json'.format(options.middleware, options.queueName)) sys.exit(1) middleware = getattr(queueConfig, options.middleware) # get ssh parameters sshHost = middleware['remoteHost'] try: sshPort = middleware['remotePort'] except Exception: sshPort = 22 sshUserName = middleware['sshUserName'] try: sshPassword = middleware['sshPassword'] except Exception: sshPassword = None privateKey = None passPhrase = None if sshPassword is None: try: privateKey = middleware['privateKey'] except Exception: print ("ERROR: set sshPassword or privateKey in middleware={0}".format(options.middleware)) sys.exit(1) try: passPhrase = middleware['passPhrase'] except Exception: passPhrase = None try: jumpHost = middleware['jumpHost'] except Exception: jumpHost = None try: jumpPort = middleware['jumpPort'] except Exception: jumpPort = 22 # ssh sshTunnelPool.make_tunnel_server(sshHost, sshPort, remote_bind_port=middleware['remoteBindPort'], num_tunnels=1, ssh_username=sshUserName, ssh_password=sshPassword, private_key=privateKey, pass_phrase=passPhrase, jump_host=jumpHost, jump_port=jumpPort ) ssh = sshTunnelPool.get_tunnel(sshHost, sshPort)[-1] return ssh
def qconf_refresh(arguments): from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper qcm = QueueConfigMapper() qcm._update_last_reload_time() qcm.lastUpdate = None qcm.load_data(refill_table=arguments.refill)
def main(): logging.basicConfig() parser = argparse.ArgumentParser() parser.add_argument( '--remoteDir', action='store', dest='remoteDir', default='harvester', help= 'directory on the remote target machine where harvester is installed') parser.add_argument( '--remoteBuildDir', action='store', dest='remoteBuildDir', default='harvester_build', help='directory on the remote target machine where harvester is build') parser.add_argument('--remotePythonSetup', action='store', dest='remotePythonSetup', default='', help='python setup on remote target machine') parser.add_argument('--queueName', action='store', dest='queueName', default=None, required=True, help='the name of queue where harvester is installed') parser.add_argument('--middleware', action='store', dest='middleware', default='rpc', help='middleware to access the remote target machine') options = parser.parse_args() # remove ~/ which doesn't work with sftp options.remoteDir = re.sub('^~/', '', options.remoteDir) options.remoteBuildDir = re.sub('^~/', '', options.remoteBuildDir) # get queue qcm = QueueConfigMapper() qcm.load_data() queueConfig = qcm.get_queue(options.queueName) if queueConfig is None: print('ERROR: queue={0} not found in panda_queueconfig.json'.format( options.queueName)) sys.exit(1) # get middleware if not hasattr(queueConfig, options.middleware): print( 'ERROR: middleware={0} is not defined for {1} in panda_queueconfig.json' .format(options.middleware, options.queueName)) sys.exit(1) middleware = getattr(queueConfig, options.middleware) # get ssh parameters sshHost = middleware['remoteHost'] try: sshPort = middleware['remotePort'] except Exception: sshPort = 22 sshUserName = middleware['sshUserName'] try: sshPassword = middleware['sshPassword'] except Exception: sshPassword = None privateKey = None passPhrase = None if sshPassword is None: try: privateKey = middleware['privateKey'] except Exception: print("ERROR: set sshPassword or privateKey in middleware={0}". format(options.middleware)) sys.exit(1) try: passPhrase = middleware['passPhrase'] except Exception: passPhrase = None try: jumpHost = middleware['jumpHost'] except Exception: jumpHost = None try: jumpPort = middleware['jumpPort'] except Exception: jumpPort = 22 # ssh sshClient = make_ssh_connection(sshHost, sshPort, sshUserName, sshPassword, passPhrase, privateKey, jumpHost, jumpPort) # get remote python version exec_out = sshClient.exec_command(';'.join([ options.remotePythonSetup, """python -c 'import sys;print("{0}{1}".format(*(sys.version_info[:2])))' """ ])) remotePythonVer = exec_out[1].read().rstrip() sshClient.close() print('remote python version : {0}'.format(remotePythonVer)) # make tmp dir with TemporaryDirectory() as tmpDir: harvesterGit = "git+git://github.com/PanDAWMS/panda-harvester.git" # get all dependencies print("getting dependencies") p = subprocess.Popen("pip download -d {0} {1}; rm -rf {0}/*".format( tmpDir, harvesterGit), stdout=subprocess.PIPE, shell=True) stdout, stderr = p.communicate() packages = [] for line in stdout.split('\n'): if line.startswith('Successfully downloaded'): packages = line.split()[2:] packages.append(harvesterGit) packages.append('pip') packages.remove('pandaharvester') # download packages print("pip download to {0}".format(tmpDir)) for package in packages: print("getting {0}".format(package)) ret = subprocess.call( "pip download --no-deps --python-version {0} -d {1} {2}". format(remotePythonVer, tmpDir, package), shell=True) if ret != 0: print("ERROR: failed to download {0}".format(package)) sys.exit(1) # sftp sshClient = make_ssh_connection(sshHost, sshPort, sshUserName, sshPassword, passPhrase, privateKey, jumpHost, jumpPort) try: sshClient.exec_command('rm -rf {0}; mkdir -p {0}'.format( options.remoteBuildDir)) except Exception: pass sftp = sshClient.open_sftp() for name in os.listdir(tmpDir): path = os.path.join(tmpDir, name) if os.path.isdir(path): continue remotePath = os.path.join(options.remoteBuildDir, name) print("copy {0} to {1}".format(name, remotePath)) sftp.put(path, remotePath) # install print("install harvester") buildDir = options.remoteBuildDir if not buildDir.startswith('/'): buildDir = '~/' + buildDir exec_out = sshClient.exec_command(';'.join([ options.remotePythonSetup, 'cd {0}'.format(options.remoteDir), 'pip install pip pandaharvester --no-index --find-links {0}'. format(buildDir) ])) print(exec_out[1].read()) print(exec_out[2].read()) sshClient.close()
def main(): logging.basicConfig() parser = argparse.ArgumentParser() parser.add_argument('--remoteDir', action='store', dest='remoteDir', default='harvester', help='directory on the remote target machine where harvester is installed') parser.add_argument('--remoteBuildDir', action='store', dest='remoteBuildDir', default='harvester_build', help='directory on the remote target machine where harvester is build') parser.add_argument('--remotePythonSetup', action='store', dest='remotePythonSetup', default='', help='python setup on remote target machine') parser.add_argument('--queueName', action='store', dest='queueName', default=None, required=True, help='the name of queue where harvester is installed') parser.add_argument('--middleware', action='store', dest='middleware', default='rpc', help='middleware to access the remote target machine') options = parser.parse_args() # remove ~/ which doesn't work with sftp options.remoteDir = re.sub('^~/', '', options.remoteDir) options.remoteBuildDir = re.sub('^~/', '', options.remoteBuildDir) # get queue qcm = QueueConfigMapper() qcm.load_data() queueConfig = qcm.get_queue(options.queueName) if queueConfig is None: print ('ERROR: queue={0} not found in panda_queueconfig.json'.format(options.queueName)) sys.exit(1) # get middleware if not hasattr(queueConfig, options.middleware): print ('ERROR: middleware={0} is not defined for {1} in panda_queueconfig.json'.format(options.middleware, options.queueName)) sys.exit(1) middleware = getattr(queueConfig, options.middleware) # get ssh parameters sshHost = middleware['remoteHost'] try: sshPort = middleware['remotePort'] except Exception: sshPort = 22 sshUserName = middleware['sshUserName'] try: sshPassword = middleware['sshPassword'] except Exception: sshPassword = None privateKey = None passPhrase = None if sshPassword is None: try: privateKey = middleware['privateKey'] except Exception: print ("ERROR: set sshPassword or privateKey in middleware={0}".format(options.middleware)) sys.exit(1) try: passPhrase = middleware['passPhrase'] except Exception: passPhrase = None try: jumpHost = middleware['jumpHost'] except Exception: jumpHost = None try: jumpPort = middleware['jumpPort'] except Exception: jumpPort = 22 # ssh sshClient = make_ssh_connection(sshHost, sshPort, sshUserName, sshPassword, passPhrase, privateKey, jumpHost, jumpPort) # get remote python version exec_out = sshClient.exec_command( ';'.join([options.remotePythonSetup, """python -c 'import sys;print("{0}{1}".format(*(sys.version_info[:2])))' """]) ) remotePythonVer = exec_out[1].read().rstrip() sshClient.close() print ('remote python version : {0}'.format(remotePythonVer)) # make tmp dir with TemporaryDirectory() as tmpDir: harvesterGit = "git+git://github.com/PanDAWMS/panda-harvester.git" # get all dependencies print ("getting dependencies") p = subprocess.Popen("pip download -d {0} {1}; rm -rf {0}/*".format(tmpDir, harvesterGit), stdout=subprocess.PIPE, shell=True) stdout, stderr = p.communicate() packages = [] for line in stdout.split('\n'): if line.startswith('Successfully downloaded'): packages = line.split()[2:] packages.append(harvesterGit) packages.append('pip') packages.remove('pandaharvester') # download packages print ("pip download to {0}".format(tmpDir)) for package in packages: print ("getting {0}".format(package)) ret = subprocess.call("pip download --no-deps --python-version {0} -d {1} {2}".format(remotePythonVer, tmpDir, package), shell=True) if ret != 0: print ("ERROR: failed to download {0}".format(package)) sys.exit(1) # sftp sshClient = make_ssh_connection(sshHost, sshPort, sshUserName, sshPassword, passPhrase, privateKey, jumpHost, jumpPort) try: sshClient.exec_command('rm -rf {0}; mkdir -p {0}'.format(options.remoteBuildDir)) except Exception: pass sftp = sshClient.open_sftp() for name in os.listdir(tmpDir): path = os.path.join(tmpDir, name) if os.path.isdir(path): continue remotePath = os.path.join(options.remoteBuildDir, name) print ("copy {0} to {1}".format(name, remotePath)) sftp.put(path, remotePath) # install print ("install harvester") buildDir = options.remoteBuildDir if not buildDir.startswith('/'): buildDir = '~/' + buildDir exec_out = sshClient.exec_command( ';'.join([options.remotePythonSetup, 'cd {0}'.format(options.remoteDir), 'pip install pip pandaharvester --no-index --find-links {0}'.format(buildDir)]) ) print (exec_out[1].read()) print (exec_out[2].read()) sshClient.close()
def main(): parser = argparse.ArgumentParser() parser.add_argument('--queueName', action='store', dest='queueName', default=None, required=True, help='the name of queue where harvester is installed') parser.add_argument('--middleware', action='store', dest='middleware', default='rpc', help='middleware to access the remote target machine') options = parser.parse_args() # get queue qcm = QueueConfigMapper() qcm.load_data() queueConfig = qcm.get_queue(options.queueName) if queueConfig is None: print('ERROR: queue={0} not found in panda_queueconfig.json'.format( options.queueName)) sys.exit(1) # get middleware if not hasattr(queueConfig, options.middleware): print( 'ERROR: middleware={0} is not defined for {1} in panda_queueconfig.json' .format(options.middleware, options.queueName)) sys.exit(1) middleware = getattr(queueConfig, options.middleware) # get ssh parameters sshHost = middleware['remoteHost'] try: sshPort = middleware['remotePort'] except Exception: sshPort = 22 sshUserName = middleware['sshUserName'] try: sshPassword = middleware['sshPassword'] except Exception: sshPassword = None privateKey = None passPhrase = None if sshPassword is None: try: privateKey = middleware['privateKey'] except Exception: print("ERROR: set sshPassword or privateKey in middleware={0}". format(options.middleware)) sys.exit(1) try: passPhrase = middleware['passPhrase'] except Exception: passPhrase = None try: jumpHost = middleware['jumpHost'] except Exception: jumpHost = None try: jumpPort = middleware['jumpPort'] except Exception: jumpPort = 22 # ssh sshTunnelPool.make_tunnel_server( sshHost, sshPort, remote_bind_port=middleware['remoteBindPort'], num_tunnels=1, ssh_username=sshUserName, ssh_password=sshPassword, private_key=privateKey, pass_phrase=passPhrase, jump_host=jumpHost, jump_port=jumpPort) ssh = sshTunnelPool.get_tunnel(sshHost, sshPort)[-1] return ssh