def setUp(self): nshost, nsport = "localhost", 5000 # 2000+i: ssh forwarder (forwards to localhost:3000+i); username testuser-2001 (number is port number), password is always mmp-secret-key # 3000+i: mupif application server # 4000+i: natport # 5000: nameserver # setup nameserver self.nsloop = multiprocessing.Process( target=Pyro4.naming.startNSloop, kwargs=dict(host=nshost, port=nsport, hmac=self.hkey) ) self.nsloop.start() time.sleep(2) # some time for nameserver to start # setup apps self.apps = ( LocalApp("/dev/null"), ) # CelsianApp("/dev/null"),MicressApp("/dev/null"),MmpraytracerApp("/dev/null") for i, app in enumerate(self.apps): # start "remote" servers (locally) print("Starting", app) PyroUtil.runAppServer( server="localhost", port=3000 + i, nathost="localhost", natport=4000 + i, nshost=nshost, nsport=nsport, nsname=PyroUtil.getNSAppName(self.jobname, app.__class__.__name__), hkey=self.hkey, app=app, )
def setUp(self): nshost, nsport = 'localhost', 5000 # 2000+i: ssh forwarder (forwards to localhost:3000+i); username testuser-2001 (number is port number), password is always mmp-secret-key # 3000+i: mupif application server # 4000+i: natport # 5000: nameserver # setup nameserver self.nsloop = multiprocessing.Process(target=Pyro4.naming.startNSloop, kwargs=dict(host=nshost, port=nsport, hmac=self.hkey)) self.nsloop.start() time.sleep(2) # some time for nameserver to start # setup apps self.apps = LocalApp( "/dev/null" ), # CelsianApp("/dev/null"),MicressApp("/dev/null"),MmpraytracerApp("/dev/null") for i, app in enumerate(self.apps): # start "remote" servers (locally) print('Starting', app) PyroUtil.runAppServer(server='localhost', port=3000 + i, nathost='localhost', natport=4000 + i, nshost=nshost, nsport=nsport, nsname=PyroUtil.getNSAppName( self.jobname, app.__class__.__name__), hkey=self.hkey, app=app)
def testConnect(self): "Test connection through nameserver" ns = PyroUtil.connectNameServer("localhost", 5000, "mmp-secret-key") print("Connected to nameserver") for i, app in enumerate(self.apps): # what is localport?? # tunnel=PyroUtil.sshTunnel(remoteHost='localhost',userName='******'%(2000+i),localPort=2000+i,remotePort=4000+i,sshClient='ssh',options='-oStrictHostKeyChecking=no',sshHost='') # print('Tunnel established') a = PyroUtil.connectApp(ns, PyroUtil.getNSAppName(self.jobname, app.__class__.__name__)) print("Connected to App through Pyro") self.assert_(a) appsig = a.getApplicationSignature() print(appsig) tunnel.terminate()
def testConnect(self): 'Test connection through nameserver' ns=PyroUtil.connectNameServer('localhost',5000,'mmp-secret-key') print('Connected to nameserver') for i,app in enumerate(self.apps): # what is localport?? #tunnel=PyroUtil.sshTunnel(remoteHost='localhost',userName='******'%(2000+i),localPort=2000+i,remotePort=4000+i,sshClient='ssh',options='-oStrictHostKeyChecking=no',sshHost='') #print('Tunnel established') a=PyroUtil.connectApp(ns,PyroUtil.getNSAppName(self.jobname,app.__class__.__name__)) print('Connected to App through Pyro') self.assert_(a) appsig=a.getApplicationSignature() print(appsig) tunnel.terminate()
def main(): # Parse arguments args = parser.parse_args() sys.path.append(os.getcwd()) # Load config conf = args.configFile if conf[-3:] == '.py': conf = conf[:-3] print(conf) tConf = importlib.import_module(conf) # locate nameserver ns = PyroUtil.connectNameServer(nshost=tConf.nshost, nsport=tConf.nsport, hkey=tConf.hkey) # Run a daemon for jobMamager on this machine daemon = PyroUtil.runDaemon(host=tConf.daemonHost, port=tConf.jobManPort, nathost=tConf.nathost, natport=tConf.jobManNatport) # Run job manager on a serverdaemon, ns, appAPIClass, appName, portRange, # jobManWorkDir, serverConfigPath, serverConfigFile, jobMan2CmdPath, # maxJobs=1, jobMancmdCommPort=10000 jobMan = jb.SimpleJobManager2(daemon, ns, appAPIClass=tConf.applicationClass, appName=tConf.jobManName, portRange=tConf.jobManPortsForJobs, jobManWorkDir=tConf.jobManWorkDir, serverConfigPath=tConf.serverConfigPath, serverConfigFile=conf, jobMan2CmdPath=tConf.jobMan2CmdPath, maxJobs=tConf.jobManMaxJobs, jobMancmdCommPort=tConf.jobManSocket) # set up daemon with JobManager uri = daemon.register(jobMan) # register JobManager to nameServer ns.register(tConf.jobManName, uri) logger.debug("Daemon for JobManager runs at " + str(uri)) print(80 * '-') print("Started " + tConf.jobManName) # waits for requests daemon.requestLoop()
def runSingleServerInstance(): """ Run a single instance of the Tracer server. The configuration file given in args must include the following: server, serverPort, serverNathost, serverNatport, nshost, nsport, appName, hkey """ # Parse arguments args = parser.parse_args() sys.path.append(os.getcwd()) # Load config conf = args.configFile if conf[-3:] == ".py": conf = conf[:-3] print(conf) cfg = importlib.import_module(conf) app = MMPRaytracer("localhost") PyroUtil.runAppServer( cfg.server, cfg.serverPort, cfg.serverNathost, cfg.serverNatport, cfg.nshost, cfg.nsport, cfg.appName, cfg.hkey, app=app, )
def runSingleServerInstance(): ''' Run a single instance of the Tracer server. The configuration file given in args must include the following: server, serverPort, serverNathost, serverNatport, nshost, nsport, appName, hkey ''' # Parse arguments args = parser.parse_args() sys.path.append(os.getcwd()) # Load config conf = args.configFile if conf[-3:] == '.py': conf = conf[:-3] print(conf) cfg = importlib.import_module(conf) app = MMPRaytracer('localhost') PyroUtil.runAppServer(cfg.server, cfg.serverPort, cfg.serverNathost, cfg.serverNatport, cfg.nshost, cfg.nsport, cfg.appName, cfg.hkey, app=app)
#nameserver app name appname = 'ctu-server' class local(Application.Application): """ Mupif interface to Local dummy task """ def __init__(self, file): super(local, self).__init__(file) #call basereturn def getApplicationSignature(self): return "CTU-server@" + socket.gethostbyaddr( socket.gethostname())[0] + " version 1.0" #create application app = local("/dev/null") # run the application server appRecord = [item for item in conf.apps if item[0] == appname][0] PyroUtil.runAppServer(server=appRecord[conf.appIndx_ServerName], port=appRecord[conf.appIndx_RemotePort], nathost=conf.nathost, natport=appRecord[conf.appIndx_NATPort], nshost=conf.nshost, nsport=conf.nsport, appName=PyroUtil.getNSAppName(conf.jobname, appname), hkey=conf.hkey, app=app)
def main(): global host global jobmanname nshost = '172.30.0.1' nsport = 9090 hkey = 'mupif-secret-key' jobmanname = None debug = False #nshost=None ssh = False #ssh flag (set to True if ssh tunnel need to be established) log = logging.getLogger() try: opts, args = getopt.getopt(sys.argv[1:], "h:j:p:k:u:n:r:t") # print(opts, args) except getopt.GetoptError as err: # print help information and exit: print(str(err)) # will print something like "option -a not recognized" usage() sys.exit(2) for o, a in opts: # if o in ("-p"): # port = int(a) # elif o in ("-h"): # host = a if o in ("-j"): jobmanname = a elif o in ("-k"): hkey = a elif o in ("-t"): ssh = True elif o in ("-u"): username = a elif o in ("-n"): nshost = a elif o in ("-r"): nsport = int(a) else: assert False, "unhandled option" # print("huhu:"+host+str(port)) if (not jobmanname): usage() sys.exit(2) #locate nameserver ns = PyroUtil.connectNameServer(nshost, nsport, hkey) # locate remote jobManager application, request remote proxy jobManUri = ns.lookup(jobmanname) #get local port of jobmanager (from uri) jobmannatport = int(re.search('(\d+)$', str(jobManUri)).group(0)) host = PyroUtil.getIPfromUri(jobManUri) #extablish secure ssh tunnel connection if ssh: (host, jobmannatport, jobManNatHost, port) = PyroUtil.getNSConnectionInfo(ns, jobmanname) tunnel = PyroUtil.sshTunnel(remoteHost=host, userName=username, localPort=jobmannatport, remotePort=port, sshClient='ssh') jobMan = PyroUtil.connectJobManager(ns, jobmanname, hkey=hkey) curses.wrapper(processor, jobMan) if ssh: tunnel.terminate()
import conf from mupif import Application from mupif import PyroUtil #nameserver app name appname='local' class local(Application.Application): """ Mupif interface to Local dummy task """ def __init__(self, file): super(local, self).__init__(file) #call basereturn def getApplicationSignature(self): return "local@"+ socket.gethostbyaddr(socket.gethostname())[0]+" version 1.0" #create application app = local("/dev/null") # run the application server appRecord = [item for item in conf.apps if item[0] == appname][0] PyroUtil.runAppServer(server=appRecord[conf.appIndx_ServerName], port=appRecord[conf.appIndx_RemotePort], nathost=conf.nathost, natport=appRecord[conf.appIndx_NATPort], nshost=conf.nshost, nsport=conf.nsport, nsname=PyroUtil.getNSAppName(conf.jobname, appname), hkey=conf.hkey, app=app)
elif o in ("-n"): nshost = a elif o in ("-r"): nsport = int(a) else: assert False, "unhandled option" print "huhu:"+host+str(port) jobid_col = 0 port_col = 35 user_col = 41 time_col = 70 #locate nameserver ns = PyroUtil.connectNameServer(nshost, nsport, hkey) # locate remote jobManager application, request remote proxy jobManUri = ns.lookup(jobmanname) #get local port of jabmanager (from uri) jobmannatport = int(re.search('(\d+)$',str(jobManUri)).group(0)) #extablish secure ssh tunnel connection if ssh: tunnel = PyroUtil.sshTunnel(remoteHost=host, userName=username, localPort=jobmannatport, remotePort=port, sshClient='ssh') jobMan = PyroUtil.connectApp(ns, jobmanname) #ssh flag (set to True if ssh tunnel need to be established) ssh = False
import clientConfig as cConf import Pyro4 from mmp_tracer_api import objID from comsol_api import MMPComsolDummy from mupif import PyroUtil, Property, PropertyID, FieldID, ValueType import logging logger = logging.getLogger() import time as timeTime start = timeTime.time() logger.info('Timer started') # locate nameserver ns = PyroUtil.connectNameServer(nshost=cConf.nshost, nsport=cConf.nsport, hkey=cConf.hkey) logger.info('NS connected: %s' % str(ns)) # Tunnels to different machines machine mieTunnel = PyroUtil.sshTunnel(remoteHost=cConf.mieServer, userName=cConf.mieUser, localPort=cConf.mieNatPort, remotePort=cConf.miePort, sshClient=cConf.sshClient, options=cConf.options, sshHost=cConf.sshHost) tracerTunnel = PyroUtil.sshTunnel(remoteHost=cConf.tracerServer, userName=cConf.tracerUser, localPort=cConf.tracerNatPort, remotePort=cConf.tracerPort, sshClient=cConf.sshClient,
import Pyro4 from mmp_tracer_api import objID from comsol_api import MMPComsolDummy from mupif import PyroUtil, Property, PropertyID, FieldID, ValueType import logging logger = logging.getLogger() import time as timeTime start = timeTime.time() logger.info('Timer started') # locate nameserver ns = PyroUtil.connectNameServer(nshost=cConf.nshost, nsport=cConf.nsport, hkey=cConf.hkey) # localize JobManager running on (remote) server and create a tunnel to it # allocate the first application app1 tracerSolverAppRec = PyroUtil.allocateApplicationWithJobManager( ns, cConf.tracerSolverJobManRec, cConf.jobNatPorts.pop(0), cConf.sshClient, cConf.options, cConf.sshHost) mieSolverAppRec = PyroUtil.allocateApplicationWithJobManager( ns, cConf.mieSolverJobManRec, cConf.jobNatPorts.pop(0), cConf.sshClient, cConf.options, cConf.sshHost) comsolAppRec = PyroUtil.allocateApplicationWithJobManager( ns, cConf.comsolSolverJobManRec, cConf.jobNatPorts.pop(0), cConf.sshClient, cConf.options, cConf.sshHost)
def main(): global host global jobmanname host = 'ksm.fsv.cvut.cz' port = 9090 hkey ="" ssh = False#ssh flag (set to tru if ssh tunnel need to be established) logger = logging.getLogger() try: opts, args = getopt.getopt(sys.argv[1:], "h:j:p:k:u:n:r:t") print(opts, args) except getopt.GetoptError as err: # print help information and exit: print(str(err)) # will print something like "option -a not recognized" usage() sys.exit(2) for o, a in opts: if o in ("-p"): port = int(a) elif o in ("-h"): host = a elif o in ("-j"): jobmanname = a elif o in ("-k"): hkey = a elif o in ("-t"): ssh = True elif o in ("-u"): username = a elif o in ("-n"): nshost = a elif o in ("-r"): nsport = int(a) else: assert False, "unhandled option" print("huhu:"+host+str(port)) #locate nameserver ns = PyroUtil.connectNameServer(nshost, nsport, hkey) # locate remote jobManager application, request remote proxy jobManUri = ns.lookup(jobmanname) #get local port of jabmanager (from uri) jobmannatport = int(re.search('(\d+)$',str(jobManUri)).group(0)) #extablish secure ssh tunnel connection if ssh: tunnel = PyroUtil.sshTunnel(remoteHost=host, userName=username, localPort=jobmannatport, remotePort=port, sshClient='ssh') jobMan = PyroUtil.connectApp(ns, jobmanname) #ssh flag (set to True if ssh tunnel need to be established) ssh = False curses.wrapper(processor, jobMan) if ssh: tunnel.terminate()
from mupif import Application from mupif import PyroUtil # import basic definitions -> need to be customized import conf #set application name (used also as an index to apps dictionary defined in conf.py) appname='micress' class micress(Application.Application): """ Mupif interface to micress (microstructure evolution simulation tool) """ def __init__(self, file): super(micress, self).__init__(file) #call base return def getApplicationSignature(self): return "Micress@"+ socket.gethostbyaddr(socket.gethostname())[0]+" version 1.0" #create application app = micress("/dev/null") # run the application server appRecord = conf.apps['micress'] PyroUtil.runAppServer(server=appRecord.serverName, port=appRecord.remotePort, nathost=conf.nathost, natport=appRecord.natPort, nshost=conf.nshost, nsport=conf.nsport, nsname=PyroUtil.getNSAppName(conf.jobname, appname), hkey=conf.hkey, app=app)
appname = 'micress' class micress(Application.Application): """ Mupif interface to micress (microstructure evolution simulation tool) """ def __init__(self, file): super(micress, self).__init__(file) #call base return def getApplicationSignature(self): return "Micress@" + socket.gethostbyaddr( socket.gethostname())[0] + " version 1.0" #create application app = micress("/dev/null") # run the application server appRecord = conf.apps['micress'] PyroUtil.runAppServer(server=appRecord.serverName, port=appRecord.remotePort, nathost=conf.nathost, natport=appRecord.natPort, nshost=conf.nshost, nsport=conf.nsport, appName=PyroUtil.getNSAppName(conf.jobname, appname), hkey=conf.hkey, app=app)
from __future__ import print_function from builtins import str import sys sys.path.append('../../..') import os import conf from mupif import Application from mupif import APIError from mupif import PyroUtil import time as timeTime start = timeTime.time() #locate nameserver ns = PyroUtil.connectNameServer(conf.nshost, 9090, conf.hkey) results=[] tunnel= None for apprecord in conf.apps: starti = timeTime.time() conf.logger.info("Trying to connect to server " + str(apprecord[conf.appIndx_Name])) #Find if we need different ssh server for authentication if apprecord[conf.appIndx_SshHost] == '': sshHost = apprecord[conf.appIndx_ServerName] else: sshHost = apprecord[conf.appIndx_SshHost] try: tunnel = PyroUtil.sshTunnel(remoteHost=apprecord[conf.appIndx_ServerName],
from mupif import WorkflowMonitor from mupif import Util from mupif import PyroUtil import argparse # Read int for mode as number behind '-m' argument: 0-local (default), 1-ssh, 2-VPN mode = argparse.ArgumentParser(parents=[Util.getParentParser()]).parse_args().mode from Config import config cfg = config(mode) Util.changeRootLogger('monitor.log') # bp stuff # cfg.monitorServer='172.30.0.3' # locate nameserver ns = PyroUtil.connectNameServer(nshost=cfg.nshost, nsport=cfg.nsport, hkey=cfg.hkey) monitor = WorkflowMonitor.WorkflowMonitor() print(monitor.getAllMetadata()) PyroUtil.runAppServer( server=cfg.monitorServer, port=cfg.monitorPort, nathost=None, natport=None, nshost=cfg.nshost, nsport=cfg.nsport, appName=cfg.monitorName, hkey=cfg.hkey, app=monitor )
from builtins import str import sys sys.path.append('../../..') import os import conf from mupif import Application from mupif import APIError from mupif import PyroUtil import time as timeTime start = timeTime.time() #locate nameserver ns = PyroUtil.connectNameServer(conf.nshost, 9090, conf.hkey) results = [] tunnel = None for apprecord in conf.apps: starti = timeTime.time() conf.log.info("Trying to connect to server " + str(apprecord[conf.appIndx_Name])) #Find if we need different ssh server for authentication if apprecord[conf.appIndx_SshHost] == '': sshHost = apprecord[conf.appIndx_ServerName] else: sshHost = apprecord[conf.appIndx_SshHost] try: tunnel = PyroUtil.sshTunnel(
import clientConfig as cConf import Pyro4 from mmp_tracer_api import objID from comsol_api import MMPComsolDummy from mupif import PyroUtil, Property, PropertyID, FieldID, ValueType import logging logger = logging.getLogger() import time as timeTime start = timeTime.time() logger.info('Timer started') # locate nameserver ns = PyroUtil.connectNameServer( nshost=cConf.nshost, nsport=cConf.nsport, hkey=cConf.hkey) # localize JobManager running on (remote) server and create a tunnel to it # allocate the first application app1 tracerSolverAppRec = PyroUtil.allocateApplicationWithJobManager( ns, cConf.tracerSolverJobManRec, cConf.jobNatPorts.pop(0), cConf.sshClient, cConf.options, cConf.sshHost) mieSolverAppRec = PyroUtil.allocateApplicationWithJobManager( ns, cConf.mieSolverJobManRec, cConf.jobNatPorts.pop(0), cConf.sshClient, cConf.options, cConf.sshHost) comsolAppRec = PyroUtil.allocateApplicationWithJobManager(ns, cConf.comsolSolverJobManRec, cConf.jobNatPorts.pop(0), cConf.sshClient, cConf.options, cConf.sshHost) mieApp = mieSolverAppRec.getApplication() tracerApp = tracerSolverAppRec.getApplication()
import clientConfig as cConf import Pyro4 from mmp_tracer_api import objID from comsol_api import MMPComsolDummy from mupif import PyroUtil, Property, PropertyID, FieldID, ValueType import logging logger = logging.getLogger() import time as timeTime start = timeTime.time() logger.info('Timer started') # locate nameserver ns = PyroUtil.connectNameServer(nshost=cConf.nshost, nsport=cConf.nsport, hkey=cConf.hkey) logger.info('NS connected: %s' % str(ns)) # Tunnels to different machines machine mieTunnel = PyroUtil.sshTunnel(remoteHost=cConf.mieServer, userName=cConf.mieUser, localPort=cConf.mieNatPort, remotePort=cConf.miePort, sshClient=cConf.sshClient, options=cConf.options, sshHost=cConf.sshHost) tracerTunnel = PyroUtil.sshTunnel(remoteHost=cConf.tracerServer, userName=cConf.tracerUser, localPort=cConf.tracerNatPort,