def setUp(self):
nshost, nsport = "localhost", 5000
# 2000+i: ssh forwarder (forwards to localhost:3000+i); username testuser-2001 (number is port number), password is always mmp-secret-key
# 3000+i: mupif application server
# 4000+i: natport
# 5000: nameserver
# setup nameserver
self.nsloop = multiprocessing.Process(
target=Pyro4.naming.startNSloop, kwargs=dict(host=nshost, port=nsport, hmac=self.hkey)
)
self.nsloop.start()
time.sleep(2) # some time for nameserver to start
# setup apps
self.apps = (
LocalApp("/dev/null"),
) # CelsianApp("/dev/null"),MicressApp("/dev/null"),MmpraytracerApp("/dev/null")
for i, app in enumerate(self.apps):
# start "remote" servers (locally)
print("Starting", app)
PyroUtil.runAppServer(
server="localhost",
port=3000 + i,
nathost="localhost",
natport=4000 + i,
nshost=nshost,
nsport=nsport,
nsname=PyroUtil.getNSAppName(self.jobname, app.__class__.__name__),
hkey=self.hkey,
app=app,
)
def setUp(self):
nshost, nsport = 'localhost', 5000
# 2000+i: ssh forwarder (forwards to localhost:3000+i); username testuser-2001 (number is port number), password is always mmp-secret-key
# 3000+i: mupif application server
# 4000+i: natport
# 5000: nameserver
# setup nameserver
self.nsloop = multiprocessing.Process(target=Pyro4.naming.startNSloop,
kwargs=dict(host=nshost,
port=nsport,
hmac=self.hkey))
self.nsloop.start()
time.sleep(2) # some time for nameserver to start
# setup apps
self.apps = LocalApp(
"/dev/null"
), # CelsianApp("/dev/null"),MicressApp("/dev/null"),MmpraytracerApp("/dev/null")
for i, app in enumerate(self.apps):
# start "remote" servers (locally)
print('Starting', app)
PyroUtil.runAppServer(server='localhost',
port=3000 + i,
nathost='localhost',
natport=4000 + i,
nshost=nshost,
nsport=nsport,
nsname=PyroUtil.getNSAppName(
self.jobname, app.__class__.__name__),
hkey=self.hkey,
app=app)
def testConnect(self):
"Test connection through nameserver"
ns = PyroUtil.connectNameServer("localhost", 5000, "mmp-secret-key")
print("Connected to nameserver")
for i, app in enumerate(self.apps):
# what is localport??
# tunnel=PyroUtil.sshTunnel(remoteHost='localhost',userName='******'%(2000+i),localPort=2000+i,remotePort=4000+i,sshClient='ssh',options='-oStrictHostKeyChecking=no',sshHost='')
# print('Tunnel established')
a = PyroUtil.connectApp(ns, PyroUtil.getNSAppName(self.jobname, app.__class__.__name__))
print("Connected to App through Pyro")
self.assert_(a)
appsig = a.getApplicationSignature()
print(appsig)
tunnel.terminate()
def testConnect(self):
'Test connection through nameserver'
ns=PyroUtil.connectNameServer('localhost',5000,'mmp-secret-key')
print('Connected to nameserver')
for i,app in enumerate(self.apps):
# what is localport??
#tunnel=PyroUtil.sshTunnel(remoteHost='localhost',userName='******'%(2000+i),localPort=2000+i,remotePort=4000+i,sshClient='ssh',options='-oStrictHostKeyChecking=no',sshHost='')
#print('Tunnel established')
a=PyroUtil.connectApp(ns,PyroUtil.getNSAppName(self.jobname,app.__class__.__name__))
print('Connected to App through Pyro')
self.assert_(a)
appsig=a.getApplicationSignature()
print(appsig)
tunnel.terminate()
def main():
# Parse arguments
args = parser.parse_args()
sys.path.append(os.getcwd())
# Load config
conf = args.configFile
if conf[-3:] == '.py':
conf = conf[:-3]
print(conf)
tConf = importlib.import_module(conf)
# locate nameserver
ns = PyroUtil.connectNameServer(nshost=tConf.nshost,
nsport=tConf.nsport,
hkey=tConf.hkey)
# Run a daemon for jobMamager on this machine
daemon = PyroUtil.runDaemon(host=tConf.daemonHost,
port=tConf.jobManPort,
nathost=tConf.nathost,
natport=tConf.jobManNatport)
# Run job manager on a serverdaemon, ns, appAPIClass, appName, portRange,
# jobManWorkDir, serverConfigPath, serverConfigFile, jobMan2CmdPath,
# maxJobs=1, jobMancmdCommPort=10000
jobMan = jb.SimpleJobManager2(daemon,
ns,
appAPIClass=tConf.applicationClass,
appName=tConf.jobManName,
portRange=tConf.jobManPortsForJobs,
jobManWorkDir=tConf.jobManWorkDir,
serverConfigPath=tConf.serverConfigPath,
serverConfigFile=conf,
jobMan2CmdPath=tConf.jobMan2CmdPath,
maxJobs=tConf.jobManMaxJobs,
jobMancmdCommPort=tConf.jobManSocket)
# set up daemon with JobManager
uri = daemon.register(jobMan)
# register JobManager to nameServer
ns.register(tConf.jobManName, uri)
logger.debug("Daemon for JobManager runs at " + str(uri))
print(80 * '-')
print("Started " + tConf.jobManName)
# waits for requests
daemon.requestLoop()
def main():
# Parse arguments
args = parser.parse_args()
sys.path.append(os.getcwd())
# Load config
conf = args.configFile
if conf[-3:] == '.py':
conf = conf[:-3]
print(conf)
tConf = importlib.import_module(conf)
# locate nameserver
ns = PyroUtil.connectNameServer(nshost=tConf.nshost,
nsport=tConf.nsport,
hkey=tConf.hkey)
# Run a daemon for jobMamager on this machine
daemon = PyroUtil.runDaemon(host=tConf.daemonHost,
port=tConf.jobManPort,
nathost=tConf.nathost,
natport=tConf.jobManNatport)
# Run job manager on a serverdaemon, ns, appAPIClass, appName, portRange,
# jobManWorkDir, serverConfigPath, serverConfigFile, jobMan2CmdPath,
# maxJobs=1, jobMancmdCommPort=10000
jobMan = jb.SimpleJobManager2(daemon, ns,
appAPIClass=tConf.applicationClass,
appName=tConf.jobManName,
portRange=tConf.jobManPortsForJobs,
jobManWorkDir=tConf.jobManWorkDir,
serverConfigPath=tConf.serverConfigPath,
serverConfigFile=conf,
jobMan2CmdPath=tConf.jobMan2CmdPath,
maxJobs=tConf.jobManMaxJobs,
jobMancmdCommPort=tConf.jobManSocket)
# set up daemon with JobManager
uri = daemon.register(jobMan)
# register JobManager to nameServer
ns.register(tConf.jobManName, uri)
logger.debug("Daemon for JobManager runs at " + str(uri))
print(80 * '-')
print("Started " + tConf.jobManName)
# waits for requests
daemon.requestLoop()
def runSingleServerInstance():
"""
Run a single instance of the Tracer server.
The configuration file given in args must include the following:
server,
serverPort,
serverNathost,
serverNatport,
nshost,
nsport,
appName,
hkey
"""
# Parse arguments
args = parser.parse_args()
sys.path.append(os.getcwd())
# Load config
conf = args.configFile
if conf[-3:] == ".py":
conf = conf[:-3]
print(conf)
cfg = importlib.import_module(conf)
app = MMPRaytracer("localhost")
PyroUtil.runAppServer(
cfg.server,
cfg.serverPort,
cfg.serverNathost,
cfg.serverNatport,
cfg.nshost,
cfg.nsport,
cfg.appName,
cfg.hkey,
app=app,
)
def runSingleServerInstance():
'''
Run a single instance of the Tracer server.
The configuration file given in args must include the following:
server,
serverPort,
serverNathost,
serverNatport,
nshost,
nsport,
appName,
hkey
'''
# Parse arguments
args = parser.parse_args()
sys.path.append(os.getcwd())
# Load config
conf = args.configFile
if conf[-3:] == '.py':
conf = conf[:-3]
print(conf)
cfg = importlib.import_module(conf)
app = MMPRaytracer('localhost')
PyroUtil.runAppServer(cfg.server,
cfg.serverPort,
cfg.serverNathost,
cfg.serverNatport,
cfg.nshost,
cfg.nsport,
cfg.appName,
cfg.hkey,
app=app)
#nameserver app name
appname = 'ctu-server'
class local(Application.Application):
"""
Mupif interface to Local dummy task
"""
def __init__(self, file):
super(local, self).__init__(file) #call basereturn
def getApplicationSignature(self):
return "CTU-server@" + socket.gethostbyaddr(
socket.gethostname())[0] + " version 1.0"
#create application
app = local("/dev/null")
# run the application server
appRecord = [item for item in conf.apps if item[0] == appname][0]
PyroUtil.runAppServer(server=appRecord[conf.appIndx_ServerName],
port=appRecord[conf.appIndx_RemotePort],
nathost=conf.nathost,
natport=appRecord[conf.appIndx_NATPort],
nshost=conf.nshost,
nsport=conf.nsport,
appName=PyroUtil.getNSAppName(conf.jobname, appname),
hkey=conf.hkey,
app=app)
def main():
global host
global jobmanname
nshost = '172.30.0.1'
nsport = 9090
hkey = 'mupif-secret-key'
jobmanname = None
debug = False
#nshost=None
ssh = False #ssh flag (set to True if ssh tunnel need to be established)
log = logging.getLogger()
try:
opts, args = getopt.getopt(sys.argv[1:], "h:j:p:k:u:n:r:t")
# print(opts, args)
except getopt.GetoptError as err:
# print help information and exit:
print(str(err)) # will print something like "option -a not recognized"
usage()
sys.exit(2)
for o, a in opts:
# if o in ("-p"):
# port = int(a)
# elif o in ("-h"):
# host = a
if o in ("-j"):
jobmanname = a
elif o in ("-k"):
hkey = a
elif o in ("-t"):
ssh = True
elif o in ("-u"):
username = a
elif o in ("-n"):
nshost = a
elif o in ("-r"):
nsport = int(a)
else:
assert False, "unhandled option"
# print("huhu:"+host+str(port))
if (not jobmanname):
usage()
sys.exit(2)
#locate nameserver
ns = PyroUtil.connectNameServer(nshost, nsport, hkey)
# locate remote jobManager application, request remote proxy
jobManUri = ns.lookup(jobmanname)
#get local port of jobmanager (from uri)
jobmannatport = int(re.search('(\d+)$', str(jobManUri)).group(0))
host = PyroUtil.getIPfromUri(jobManUri)
#extablish secure ssh tunnel connection
if ssh:
(host, jobmannatport, jobManNatHost,
port) = PyroUtil.getNSConnectionInfo(ns, jobmanname)
tunnel = PyroUtil.sshTunnel(remoteHost=host,
userName=username,
localPort=jobmannatport,
remotePort=port,
sshClient='ssh')
jobMan = PyroUtil.connectJobManager(ns, jobmanname, hkey=hkey)
curses.wrapper(processor, jobMan)
if ssh:
tunnel.terminate()
import conf
from mupif import Application
from mupif import PyroUtil
#nameserver app name
appname='local'
class local(Application.Application):
"""
Mupif interface to Local dummy task
"""
def __init__(self, file):
super(local, self).__init__(file) #call basereturn
def getApplicationSignature(self):
return "local@"+ socket.gethostbyaddr(socket.gethostname())[0]+" version 1.0"
#create application
app = local("/dev/null")
# run the application server
appRecord = [item for item in conf.apps if item[0] == appname][0]
PyroUtil.runAppServer(server=appRecord[conf.appIndx_ServerName],
port=appRecord[conf.appIndx_RemotePort],
nathost=conf.nathost, natport=appRecord[conf.appIndx_NATPort],
nshost=conf.nshost, nsport=conf.nsport,
nsname=PyroUtil.getNSAppName(conf.jobname, appname), hkey=conf.hkey, app=app)
elif o in ("-n"):
nshost = a
elif o in ("-r"):
nsport = int(a)
else:
assert False, "unhandled option"
print "huhu:"+host+str(port)
jobid_col = 0
port_col = 35
user_col = 41
time_col = 70
#locate nameserver
ns = PyroUtil.connectNameServer(nshost, nsport, hkey)
# locate remote jobManager application, request remote proxy
jobManUri = ns.lookup(jobmanname)
#get local port of jabmanager (from uri)
jobmannatport = int(re.search('(\d+)$',str(jobManUri)).group(0))
#extablish secure ssh tunnel connection
if ssh:
tunnel = PyroUtil.sshTunnel(remoteHost=host, userName=username, localPort=jobmannatport, remotePort=port, sshClient='ssh')
jobMan = PyroUtil.connectApp(ns, jobmanname)
#ssh flag (set to True if ssh tunnel need to be established)
ssh = False
import clientConfig as cConf
import Pyro4
from mmp_tracer_api import objID
from comsol_api import MMPComsolDummy
from mupif import PyroUtil, Property, PropertyID, FieldID, ValueType
import logging
logger = logging.getLogger()
import time as timeTime
start = timeTime.time()
logger.info('Timer started')
# locate nameserver
ns = PyroUtil.connectNameServer(nshost=cConf.nshost,
nsport=cConf.nsport,
hkey=cConf.hkey)
logger.info('NS connected: %s' % str(ns))
# Tunnels to different machines machine
mieTunnel = PyroUtil.sshTunnel(remoteHost=cConf.mieServer,
userName=cConf.mieUser,
localPort=cConf.mieNatPort,
remotePort=cConf.miePort,
sshClient=cConf.sshClient,
options=cConf.options,
sshHost=cConf.sshHost)
tracerTunnel = PyroUtil.sshTunnel(remoteHost=cConf.tracerServer,
userName=cConf.tracerUser,
localPort=cConf.tracerNatPort,
remotePort=cConf.tracerPort,
sshClient=cConf.sshClient,
import Pyro4
from mmp_tracer_api import objID
from comsol_api import MMPComsolDummy
from mupif import PyroUtil, Property, PropertyID, FieldID, ValueType
import logging
logger = logging.getLogger()
import time as timeTime
start = timeTime.time()
logger.info('Timer started')
# locate nameserver
ns = PyroUtil.connectNameServer(nshost=cConf.nshost,
nsport=cConf.nsport,
hkey=cConf.hkey)
# localize JobManager running on (remote) server and create a tunnel to it
# allocate the first application app1
tracerSolverAppRec = PyroUtil.allocateApplicationWithJobManager(
ns, cConf.tracerSolverJobManRec, cConf.jobNatPorts.pop(0), cConf.sshClient,
cConf.options, cConf.sshHost)
mieSolverAppRec = PyroUtil.allocateApplicationWithJobManager(
ns, cConf.mieSolverJobManRec, cConf.jobNatPorts.pop(0), cConf.sshClient,
cConf.options, cConf.sshHost)
comsolAppRec = PyroUtil.allocateApplicationWithJobManager(
ns, cConf.comsolSolverJobManRec, cConf.jobNatPorts.pop(0), cConf.sshClient,
cConf.options, cConf.sshHost)
def main():
global host
global jobmanname
host = 'ksm.fsv.cvut.cz'
port = 9090
hkey =""
ssh = False#ssh flag (set to tru if ssh tunnel need to be established)
logger = logging.getLogger()
try:
opts, args = getopt.getopt(sys.argv[1:], "h:j:p:k:u:n:r:t")
print(opts, args)
except getopt.GetoptError as err:
# print help information and exit:
print(str(err)) # will print something like "option -a not recognized"
usage()
sys.exit(2)
for o, a in opts:
if o in ("-p"):
port = int(a)
elif o in ("-h"):
host = a
elif o in ("-j"):
jobmanname = a
elif o in ("-k"):
hkey = a
elif o in ("-t"):
ssh = True
elif o in ("-u"):
username = a
elif o in ("-n"):
nshost = a
elif o in ("-r"):
nsport = int(a)
else:
assert False, "unhandled option"
print("huhu:"+host+str(port))
#locate nameserver
ns = PyroUtil.connectNameServer(nshost, nsport, hkey)
# locate remote jobManager application, request remote proxy
jobManUri = ns.lookup(jobmanname)
#get local port of jabmanager (from uri)
jobmannatport = int(re.search('(\d+)$',str(jobManUri)).group(0))
#extablish secure ssh tunnel connection
if ssh:
tunnel = PyroUtil.sshTunnel(remoteHost=host, userName=username, localPort=jobmannatport, remotePort=port, sshClient='ssh')
jobMan = PyroUtil.connectApp(ns, jobmanname)
#ssh flag (set to True if ssh tunnel need to be established)
ssh = False
curses.wrapper(processor, jobMan)
if ssh:
tunnel.terminate()
from mupif import Application
from mupif import PyroUtil
# import basic definitions -> need to be customized
import conf
#set application name (used also as an index to apps dictionary defined in conf.py)
appname='micress'
class micress(Application.Application):
"""
Mupif interface to micress (microstructure evolution simulation tool)
"""
def __init__(self, file):
super(micress, self).__init__(file) #call base
return
def getApplicationSignature(self):
return "Micress@"+ socket.gethostbyaddr(socket.gethostname())[0]+" version 1.0"
#create application
app = micress("/dev/null")
# run the application server
appRecord = conf.apps['micress']
PyroUtil.runAppServer(server=appRecord.serverName, port=appRecord.remotePort,
nathost=conf.nathost, natport=appRecord.natPort,
nshost=conf.nshost, nsport=conf.nsport,
nsname=PyroUtil.getNSAppName(conf.jobname, appname), hkey=conf.hkey, app=app)
appname = 'micress'
class micress(Application.Application):
"""
Mupif interface to micress (microstructure evolution simulation tool)
"""
def __init__(self, file):
super(micress, self).__init__(file) #call base
return
def getApplicationSignature(self):
return "Micress@" + socket.gethostbyaddr(
socket.gethostname())[0] + " version 1.0"
#create application
app = micress("/dev/null")
# run the application server
appRecord = conf.apps['micress']
PyroUtil.runAppServer(server=appRecord.serverName,
port=appRecord.remotePort,
nathost=conf.nathost,
natport=appRecord.natPort,
nshost=conf.nshost,
nsport=conf.nsport,
appName=PyroUtil.getNSAppName(conf.jobname, appname),
hkey=conf.hkey,
app=app)
from __future__ import print_function
from builtins import str
import sys
sys.path.append('../../..')
import os
import conf
from mupif import Application
from mupif import APIError
from mupif import PyroUtil
import time as timeTime
start = timeTime.time()
#locate nameserver
ns = PyroUtil.connectNameServer(conf.nshost, 9090, conf.hkey)
results=[]
tunnel= None
for apprecord in conf.apps:
starti = timeTime.time()
conf.logger.info("Trying to connect to server " + str(apprecord[conf.appIndx_Name]))
#Find if we need different ssh server for authentication
if apprecord[conf.appIndx_SshHost] == '':
sshHost = apprecord[conf.appIndx_ServerName]
else:
sshHost = apprecord[conf.appIndx_SshHost]
try:
tunnel = PyroUtil.sshTunnel(remoteHost=apprecord[conf.appIndx_ServerName],
from mupif import WorkflowMonitor
from mupif import Util
from mupif import PyroUtil
import argparse
# Read int for mode as number behind '-m' argument: 0-local (default), 1-ssh, 2-VPN
mode = argparse.ArgumentParser(parents=[Util.getParentParser()]).parse_args().mode
from Config import config
cfg = config(mode)
Util.changeRootLogger('monitor.log')
# bp stuff
# cfg.monitorServer='172.30.0.3'
# locate nameserver
ns = PyroUtil.connectNameServer(nshost=cfg.nshost, nsport=cfg.nsport, hkey=cfg.hkey)
monitor = WorkflowMonitor.WorkflowMonitor()
print(monitor.getAllMetadata())
PyroUtil.runAppServer(
server=cfg.monitorServer,
port=cfg.monitorPort,
nathost=None,
natport=None,
nshost=cfg.nshost,
nsport=cfg.nsport,
appName=cfg.monitorName,
hkey=cfg.hkey,
app=monitor
)
from builtins import str
import sys
sys.path.append('../../..')
import os
import conf
from mupif import Application
from mupif import APIError
from mupif import PyroUtil
import time as timeTime
start = timeTime.time()
#locate nameserver
ns = PyroUtil.connectNameServer(conf.nshost, 9090, conf.hkey)
results = []
tunnel = None
for apprecord in conf.apps:
starti = timeTime.time()
conf.log.info("Trying to connect to server " +
str(apprecord[conf.appIndx_Name]))
#Find if we need different ssh server for authentication
if apprecord[conf.appIndx_SshHost] == '':
sshHost = apprecord[conf.appIndx_ServerName]
else:
sshHost = apprecord[conf.appIndx_SshHost]
try:
tunnel = PyroUtil.sshTunnel(
import clientConfig as cConf
import Pyro4
from mmp_tracer_api import objID
from comsol_api import MMPComsolDummy
from mupif import PyroUtil, Property, PropertyID, FieldID, ValueType
import logging
logger = logging.getLogger()
import time as timeTime
start = timeTime.time()
logger.info('Timer started')
# locate nameserver
ns = PyroUtil.connectNameServer(
nshost=cConf.nshost, nsport=cConf.nsport, hkey=cConf.hkey)
# localize JobManager running on (remote) server and create a tunnel to it
# allocate the first application app1
tracerSolverAppRec = PyroUtil.allocateApplicationWithJobManager(
ns, cConf.tracerSolverJobManRec, cConf.jobNatPorts.pop(0),
cConf.sshClient, cConf.options, cConf.sshHost)
mieSolverAppRec = PyroUtil.allocateApplicationWithJobManager(
ns, cConf.mieSolverJobManRec, cConf.jobNatPorts.pop(0),
cConf.sshClient, cConf.options, cConf.sshHost)
comsolAppRec = PyroUtil.allocateApplicationWithJobManager(ns, cConf.comsolSolverJobManRec, cConf.jobNatPorts.pop(0), cConf.sshClient, cConf.options, cConf.sshHost)
mieApp = mieSolverAppRec.getApplication()
tracerApp = tracerSolverAppRec.getApplication()
import clientConfig as cConf
import Pyro4
from mmp_tracer_api import objID
from comsol_api import MMPComsolDummy
from mupif import PyroUtil, Property, PropertyID, FieldID, ValueType
import logging
logger = logging.getLogger()
import time as timeTime
start = timeTime.time()
logger.info('Timer started')
# locate nameserver
ns = PyroUtil.connectNameServer(nshost=cConf.nshost,
nsport=cConf.nsport,
hkey=cConf.hkey)
logger.info('NS connected: %s' % str(ns))
# Tunnels to different machines machine
mieTunnel = PyroUtil.sshTunnel(remoteHost=cConf.mieServer,
userName=cConf.mieUser,
localPort=cConf.mieNatPort,
remotePort=cConf.miePort,
sshClient=cConf.sshClient,
options=cConf.options,
sshHost=cConf.sshHost)
tracerTunnel = PyroUtil.sshTunnel(remoteHost=cConf.tracerServer,
userName=cConf.tracerUser,
localPort=cConf.tracerNatPort,
