def runDaemon(host, port, nathost, natport):
"""
Runs a daemon without geristering to a name server
:param str(int) host: Host name where daemon runs. This is typically a localhost
:param int port: Port number where daemon will listen (internal port number)
:param str(int) nathost: Hostname of the server as reported by nameserver, for secure ssh tunnel it should be set to 'localhost' (external host name)
:param int natport: Server NAT port, optional (external port)
:return Instance of the running daemon, None if a problem
:rtype Pyro4.Daemon
"""
try:
daemon = Pyro4.Daemon(host=host, port=port, nathost=nathost, natport=natport)
log.info('Pyro4 daemon runs on %s:%d using nathost %s:%d' % (host, port, nathost, natport))
except socket.error as e:
log.debug('Socket port %s:%d seems to be already in use' % (host,port))
daemon = None
raise e
except Exception:
log.exception('Can not run Pyro4 daemon on %s:%d using nathost %s:%d' % (host, port, nathost, natport))
daemon = None
raise
return daemon
def sshTunnel(remoteHost, userName, localPort, remotePort, sshClient='ssh', options='', sshHost='', Reverse=False):
"""
Automatic creation of ssh tunnel, using putty.exe for Windows and ssh for Linux
:param str remoteHost: IP of remote host
:param str userName: User name, if empty, current user name is used
:param int localPort: Local port
:param int remotePort: Remote port
:param str sshClient: Path to executable ssh client (on Windows use double backslashes 'C:\\Program Files\\Putty\putty.exe')
:param str options: Arguments to ssh clinent, e.g. the location of private ssh keys
:param str sshHost: Computer used for tunelling, optional. If empty, equals to remoteHost
:param bool Reverse: True if reverse tunnel to be created (default is False)
:return: Instance of subprocess.Popen running the tunneling command
:rtype: subprocess.Popen
:raises Exception: if creation of a tunnel failed
"""
if sshHost =='':
sshHost = remoteHost
if userName =='':
userName = os.getenv('USER')
direction = 'L'
if Reverse == True:
direction = 'R'
#use direct system command. Paramiko or sshtunnel do not work.
#put ssh public key on a server - interaction with a keyboard for password will not work here (password goes through TTY, not stdin)
if sshClient=='ssh':
cmd = 'ssh -%s %d:%s:%d %s@%s -N %s' % (direction, localPort, remoteHost, remotePort, userName, sshHost, options)
log.debug("Creating ssh tunnel via command: " + cmd)
elif sshClient=='autossh':
cmd = 'autossh -%s %d:%s:%d %s@%s -N %s' % (direction, localPort, remoteHost, remotePort, userName, sshHost, options)
log.debug("Creating autossh tunnel via command: " + cmd)
elif 'putty' in sshClient.lower():
#need to create a public key *.ppk using puttygen. It can be created by importing Linux private key. The path to that key is given as -i option
cmd = '%s -%s %d:%s:%d %s@%s -N %s' % (sshClient, direction, localPort, remoteHost, remotePort, userName, sshHost, options)
log.debug("Creating ssh tunnel via command: " + cmd)
elif sshClient=='manual':
#You need ssh server running, e.g. UNIX-sshd or WIN-freesshd
cmd1 = 'ssh -%s %d:%s:%d %s@%s -N %s' % (direction, localPort, remoteHost, remotePort, userName, sshHost, options)
cmd2 = 'putty.exe -%s %d:%s:%d %s@%s -N %s' % (direction, localPort, remoteHost, remotePort, userName, sshHost, options)
log.info("If ssh tunnel does not exist, do it manually using a command e.g. " + cmd1 + " , or " + cmd2)
return 'manual'
else:
log.error("Unknown ssh client, exiting")
exit(0)
try:
tunnel = subprocess.Popen(cmd.split())
except Exception:
log.exception("Creation of a tunnel failed. Can not execute the command: %s " % cmd)
raise
time.sleep(1.0)
return tunnel
def allocateApplicationWithJobManager (ns, jobManRec, natPort, sshClient='ssh', options='', sshHost=''):
"""
Connect to jobManager described by given jobManRec
:param Pyro4.naming.Nameserver ns: running name server
:param tuple jobManRec: tuple containing (jobManPort, jobManNatport, jobManHostname, jobManUserName, jobManDNSName), see clientConfig.py
:param int natPort: nat port in local computer for ssh tunnel for the application
:param str sshClient: client for ssh tunnel, see :func:`sshTunnel`, default 'ssh'
:param str options: parameters for ssh tunnel, see :func:`sshTunnel`, default ''
:param str sshHost: parameters for ssh tunnel, see :func:`sshTunnel`, default ''
:return: RemoteAppRecord containing application, tunnel to application, tunnel to jobman, jobid
:rtype: RemoteAppRecord
:raises Exception: if allocation of job fails
"""
(jobManPort, jobManNatport, jobManHostname, jobManUserName, jobManName) = jobManRec
log.debug('Trying to connect to JobManager')
(jobMan, tunnelJobMan) = connectJobManager (ns, jobManRec, sshClient, options, sshHost)
if jobMan is None:
e = OSError("Can not connect to JobManager")
log.exception(e)
raise e
else:
log.debug('Connected to JobManager %s using tunnel %s' % (jobMan, tunnelJobMan))
if tunnelJobMan is None:
e = OSError("Can not create a ssh tunnel to JobManager")
log.exception(e)
raise
try:
retRec = jobMan.allocateJob(getUserInfo(), natPort=natPort)
log.info('Allocated job, returned record from jobMan:' + str(retRec))
except Exception:
log.exception("JobManager allocateJob() failed")
raise
#create tunnel to application's daemon running on (remote) server
try:
tunnelApp = sshTunnel(remoteHost=jobManHostname, userName=jobManUserName, localPort=natPort, remotePort=retRec[2], sshClient=sshClient, options=options, sshHost=sshHost)
except Exception:
log.exception("Creating ssh tunnel for application's daemon failed")
raise
else:
log.info("Scenario: Connecting to " + retRec[1] + " " + str(retRec[2]))
#time.sleep(1)
# connect to (remote) application, requests remote proxy
app = connectApp(ns, retRec[1])
if app==None:
tunnelApp.terminate()
return RemoteAppRecord.RemoteAppRecord(app, tunnelApp, jobMan, tunnelJobMan, retRec[1])
def allocateNextApplication (ns, jobManRec, natPort, appRec, sshClient='ssh', options='', sshHost=''):
"""
Allocate next application instance on a running Job Manager and adds it into
existing applicationRecord.
:param Pyro4.naming.Nameserver ns: running name server
:param tuple jobManRec: tuple containing (jobManPort, jobManNatport, jobManHostname, jobManUserName, jobManDNSName), see clientConfig.py
:param int natPort: nat port in local computer for ssh tunnel for the application
:param RemoteAppRecord appRec: existing RemoteAppRecord where a new application will be added
:param str sshClient: client for ssh tunnel, see :func:`sshTunnel`, default 'ssh'
:param str options: parameters for ssh tunnel, see :func:`sshTunnel`, default ''
:param str sshHost: parameters for ssh tunnel, see :func:`sshTunnel`, default ''
:return: None
:raises Exception: if allocation of job fails
:raises Exception: if ssh tunnel to application instance can not be created
"""
(jobManPort, jobManNatport, jobManHostname, jobManUserName, jobManName) = jobManRec
jobMan = connectApp(ns, jobManName)
try:
retRec = jobMan.allocateJob(getUserInfo(), natPort=natPort)
log.info('Allocated job, returned record from jobMan:' + str(retRec))
except Exception:
log.exception("jobMan.allocateJob() failed")
raise
#create tunnel to application's daemon running on (remote) server
try:
tunnelApp = sshTunnel(remoteHost=jobManHostname, userName=jobManUserName, localPort=natPort, remotePort=retRec[2], sshClient=sshClient, options=options, sshHost=sshHost)
except Exception:
log.exception("Creating ssh tunnel for application's daemon failed")
raise
else:
log.info("Scenario: Connecting to " + retRec[1] + " " + str(retRec[2]))
app = connectApp(ns, retRec[1])
if app==None:
tunnelApp.terminate()
appRec.appendNextApplication(app,tunnelApp,retRec[1])
def runAppServer(server, port, nathost, natport, nshost, nsport, nsname, hkey, app, daemon=None):
"""
Runs a simple application server
:param str server: Host name of the server (internal host name)
:param int port: Port number on the server where daemon will listen (internal port number)
:param str nathost: Hostname of the server as reported by nameserver, for secure ssh tunnel it should be set to 'localhost' (external host name)
:param int natport: Server NAT port as reported by nameserver (external port)
:param str nshost: Hostname of the computer running nameserver
:param int nsport: Nameserver port
:param str nsname: Name of registered application
:param str hkey: A password string
:param instance app: Application instance
:param daemon: Reference to already running daemon, if available. Optional parameter.
:raises Exception: if can not run Pyro4 daemon
"""
externalDaemon = False
if not daemon:
try:
daemon = Pyro4.Daemon(host=server, port=port, nathost=nathost, natport=natport)
log.info('Pyro4 daemon runs on %s:%d using nathost %s:%d and hkey %s' % (server, port, nathost, natport, hkey))
except Exception:
log.exception('Can not run Pyro4 daemon on %s:%d using nathost %s:%d and hmac %s' % (server, port, nathost, natport, hkey))
raise
exit(1)
else:
externalDaemon = True
ns = connectNameServer(nshost, nsport, hkey)
#register agent
uri = daemon.register(app)
ns.register(nsname, uri)
app.registerPyro(daemon, ns, uri, externalDaemon=externalDaemon)
log.debug('NameServer %s has registered uri %s' % (nsname, uri) )
log.debug('Running runAppServer: server:%s, port:%d, nathost:%s, natport:%d, nameServer:%s, nameServerPort:%d, applicationName:%s, daemon URI %s' % (server, port, nathost, natport, nshost, nsport, nsname, uri) )
daemon.requestLoop()
