def execute(command, host=None, bg=True, **kwds):
'''execute a command (possibly) on a remote host
Execute a process, and return the launcher. Use 'response' to retrieve the
response from the executed command. Use 'kill' to kill the launcher, and 'pid'
to get the process id for the launcher.
Inputs:
command -- command string to be executed
host -- hostname of execution target [default = None (i.e. run locally)]
bg -- run as background process? [default = True]
'''
#XXX: options, background, stdin can be set w/ kwds (also name, launcher)
bg = bool(bg) # overrides 'background'
if host in [None, '']:
from pathos.connection import Pipe
launcher = Pipe(**kwds)
launcher(command=command, background=bg)
else:
from pathos.secure import Pipe
opt = kwds.pop('options', '-q')
launcher = Pipe(**kwds)
launcher(options=opt, command=command, host=host, background=bg)
pathos.logger().info('executing {%s}', launcher.message)
launcher.launch()
#response = launcher.response()
#launcher.kill()
#return response
return launcher
def copy(source, destination=None, **kwds):
'''copy source to (possibly) remote destination
Execute a copy, and return the copier. Use 'kill' to kill the copier, and
'pid' to get the process id for the copier.
Inputs:
source -- path string of source 'file'
destination -- path string for destination target
'''
#XXX: options, background, stdin can be set w/ kwds (also name, launcher)
if destination is None: destination = os.getcwd()
from pathos.secure import Copier
opt = kwds.pop('options', None)
kwds['background'] = kwds.pop('bg', False) # ignores 'background'
copier = Copier(**kwds)
if ':' in source or ':' in destination:
if opt is None: opt = '-q -r'
copier(options=opt, source=source, destination=destination)
else:
if opt is None: opt = '-r'
copier(launcher='cp', options=opt, source=source, destination=destination)
pathos.logger().info('executing {%s}', copier.message)
copier.launch()
copier.kill()
return copier
def serve(server, host=None, port=None, profile='.bash_profile'):
'''begin serving RPC requests
Inputs:
server -- name of RPC server (i.e. 'ppserver')
host -- hostname on which a server should be launched
port -- port number (on host) that server will accept request at
profile -- file on remote host that instantiates the user's environment
[default = '.bash_profile']
'''
if host is None: #XXX: and...?
profile = ''
else:
profile = 'source %s; ' % profile
file = '~/bin/%s.py' % server #XXX: _should_ be on the $PATH
if port is None: port = randomport(host)
command = "%s -p %s" % (file,port)
rserver = execute(command, host, bg=True)
response = rserver.response()
pathos.logger().info('response = %r', response)
if response in ['', None]: #XXX: other responses allowed (?)
pass
else: #XXX: not really error checking...
pathos.logger().error('invalid response = %r', response)
from time import sleep
delay = 2.0
sleep(delay)
return rserver
def execute(command, host=None, bg=True, **kwds):
'''execute a command (possibly) on a remote host
Execute a process, and return the launcher. Use 'response' to retrieve the
response from the executed command. Use 'kill' to kill the launcher, and 'pid'
to get the process id for the launcher.
Inputs:
command -- command string to be executed
host -- hostname of execution target [default = None (i.e. run locally)]
bg -- run as background process? [default = True]
'''
#XXX: options, background, stdin can be set w/ kwds (also name, launcher)
bg = bool(bg) # overrides 'background'
if host in [None, '']:
from pathos.connection import Pipe
launcher = Pipe(**kwds)
launcher(command=command, background=bg)
else:
from pathos.secure import Pipe
opt = kwds.pop('options', '-q')
launcher = Pipe(**kwds)
launcher(options=opt, command=command, host=host, background=bg)
pathos.logger().info('executing {%s}', launcher.message)
launcher.launch()
#response = launcher.response()
#launcher.kill()
#return response
return launcher
def serve(server, host=None, port=None, profile='.bash_profile'):
'''begin serving RPC requests
Inputs:
server -- name of RPC server (i.e. 'ppserver')
host -- hostname on which a server should be launched
port -- port number (on host) that server will accept request at
profile -- file on remote host that instantiates the user's environment
[default = '.bash_profile']
'''
if host is None: #XXX: and...?
profile = ''
else:
profile = 'source %s; ' % profile
file = '~/bin/%s.py' % server #XXX: _should_ be on the $PATH
if port is None: port = randomport(host)
command = "%s -p %s" % (file,port)
rserver = execute(command, host, bg=True)
response = rserver.response()
pathos.logger().info('response = %r', response)
if response in ['', None]: #XXX: other responses allowed (?)
pass
else: #XXX: not really error checking...
pathos.logger().error('invalid response = %r', response)
from time import sleep
delay = 2.0
sleep(delay)
return rserver
def copy(source, destination=None, **kwds):
'''copy source to (possibly) remote destination
Execute a copy, and return the copier. Use 'kill' to kill the copier, and
'pid' to get the process id for the copier.
Inputs:
source -- path string of source 'file'
destination -- path string for destination target
'''
#XXX: options, background, stdin can be set w/ kwds (also name, launcher)
if destination is None: destination = os.getcwd()
from pathos.secure import Copier
opt = kwds.pop('options', None)
kwds['background'] = kwds.pop('bg', False) # ignores 'background'
copier = Copier(**kwds)
if ':' in source or ':' in destination:
if opt is None: opt = '-q -r'
copier(options=opt, source=source, destination=destination)
else:
if opt is None: opt = '-r'
copier(launcher='cp', options=opt, source=source, destination=destination)
pathos.logger().info('executing {%s}', copier.message)
copier.launch()
copier.kill()
return copier
def onChild(pid, fromparent, toparent):
try:
response = self._server._marshaled_dispatch(data)
self._sendResponse(response)
line = fromparent.readline()
toparent.write("done\n")
toparent.flush()
except:
logger(name="pathos.xmlrpc", level=30).error(print_exc_info())
os._exit(0)
def onChild(pid, fromparent, toparent):
try:
response = self._server._marshaled_dispatch(data)
self._sendResponse(response)
line = _str(fromparent.readline())
toparent.write(_b('done\n'))
toparent.flush()
except:
logger(name='pathos.xmlrpc', level=30).error(print_exc_info())
os._exit(0)
def run_home(self):
"""
Intended for parallelization in parent class (e.g. aggregator); runs a
single MPCCalc home.
:return: None
"""
fh = logging.FileHandler(os.path.join("home_logs", f"{self.name}.log"))
fh.setLevel(logging.WARN)
self.log = pathos.logger(level=logging.INFO,
handler=fh,
name=self.name)
self.redis_client = RedisClient()
self.redis_get_initial_values()
self.cast_redis_timestep()
if self.timestep > 0:
self.redis_get_prev_optimal_vals()
self.get_initial_conditions()
self.solve_type_problem()
self.cleanup_and_finish()
self.redis_write_optimal_vals()
self.log.removeHandler(fh)
def randomport(host=None):
'''select a open port on a (possibly) remote host
Inputs:
host -- hostname on which to select a open port
'''
from pathos.portpicker import randomport
if not host:
return randomport()
from pathos.secure import Pipe
from pathos.portpicker import __file__ as src
# make sure src is a .py file, not .pyc or .pyo
src = src.rstrip('co')
launcher = Pipe() #XXX: use pox.which / which_python?
launcher(command='python', host=host, background=False, stdin=open(src))
pathos.logger().info('executing {python <%s} on %s', src, host)
launcher.launch()
try:
rport = int(launcher.response())
except:
from pathos.secure import TunnelException
raise TunnelException("failure to pick remote port")
# return remote port number
return rport
def randomport(host=None):
'''select a open port on a (possibly) remote host
Inputs:
host -- hostname on which to select a open port
'''
from pathos.portpicker import randomport
if not host:
return randomport()
from pathos.secure import Pipe
from pathos.portpicker import __file__ as src
# make sure src is a .py file, not .pyc or .pyo
src = src.rstrip('co')
launcher = Pipe() #XXX: use pox.which / which_python?
launcher(command='python', host=host, background=False, stdin=open(src))
pathos.logger().info('executing {python <%s} on %s', src, host)
launcher.launch()
try:
rport = int(launcher.response())
except:
from pathos.secure import TunnelException
raise TunnelException("failure to pick remote port")
# return remote port number
return rport
handlers[fd].remove(handler)
if not handlers[fd]:
del handlers[fd]
return
def _cleanup(self):
self._debug.info("cleaning up")
for fd in self._input:
fd.close()
for fd in self._output:
fd.close()
for fd in self._exception:
fd.close()
for handler in self._interrupt:
handler(self)
return
# static members
from pathos import logger
_debug = logger(name="pathos.selector", level=30) # logging.WARN
del logger
# constants
_TIMEOUT = .5
# End of file
class XMLRPCRequestHandler(BaseHTTPRequestHandler):
''' create a XML-RPC request handler '''
_debug = logger(name="pathos.xmlrpc", level=30) # logging.WARN
def do_POST(self):
""" Access point from HTTP handler """
def onParent(pid, fromchild, tochild):
self._server._registerChild(pid, fromchild)
tochild.write(_b('done\n'))
tochild.flush()
def onChild(pid, fromparent, toparent):
try:
response = self._server._marshaled_dispatch(data)
self._sendResponse(response)
line = _str(fromparent.readline())
toparent.write(_b('done\n'))
toparent.flush()
except:
logger(name='pathos.xmlrpc', level=30).error(print_exc_info())
os._exit(0)
try:
data = self.rfile.read(int(self.headers['content-length']))
params, method = client.loads(data)
if method == 'run': #XXX: _str?
return spawn2(onParent, onChild)
else:
response = self._server._marshaled_dispatch(data)
self._sendResponse(response)
return
except:
self._debug.error(print_exc_info())
self.send_response(500)
self.end_headers()
return
def log_message(self, format, *args):
""" Overriding BaseHTTPRequestHandler.log_message() """
self._debug.info(
"%s - - [%s] %s\n" % (self.address_string(),
self.log_date_time_string(), format % args))
def _sendResponse(self, response):
""" Write the XML-RPC response """
response = _b(response)
self.send_response(200)
self.send_header("Content-type", "text/xml")
self.send_header("Content-length", str(len(response)))
self.end_headers()
self.wfile.write(response)
self.wfile.flush()
self.connection.shutdown(1)
def __init__(self, server, socket):
"""
Override BaseHTTPRequestHandler.__init__(): we need to be able
to have (potentially) multiple handler objects at a given time.
Inputs:
server -- server object to handle requests for
socket -- socket connection
"""
## Settings required by BaseHTTPRequestHandler
self.rfile = socket.makefile('rb', -1)
self.wfile = socket.makefile('wb', 0)
self.connection = socket
self.client_address = (server.host, server.port)
self._server = server
class Pipe(object):
"""a popen-based pipe for parallel and distributed computing"""
verbose = True
from pathos import logger
_debug = logger(level=30) # logging.WARN
del logger
def __init__(self, name=None, **kwds):
"""create a popen-pipe
Inputs:
name: a unique identifier (string) for the pipe
command: a command to send [default = 'echo <name>']
background: run in background [default = False]
decode: ensure response is 'ascii' [default = True]
stdin: file-like object to serve as standard input for the remote process
"""
xyz = string.ascii_letters
self.name = ''.join(random.choice(xyz) for i in range(16)) \
if name is None else name
self.background = kwds.pop('background', False)
self.stdin = kwds.pop('stdin', sys.stdin)
self.codec = kwds.pop('decode', 'ascii')
self.message = kwds.pop('command', 'echo %s' % self.name) #' '?
self._response = None
self._pid = 0
self.config(**kwds)
return
def __repr__(self):
return "Pipe('%s')" % self.message
def config(self, **kwds):
'''configure the pipe using given keywords
(Re)configure the pipe for the following inputs:
command: a command to send [default = 'echo <name>']
background: run in background [default = False]
decode: ensure response is 'ascii' [default = True]
stdin: file-like object to serve as standard input for the remote process
'''
if self.message is None:
self.message = 'echo %s' % self.name #' '?
if self.stdin is None:
self.stdin = sys.stdin
if self.codec is None:
self.codec = 'ascii'
for key, value in kwds.items():
if key == 'command':
self.message = value
elif key == 'background':
self.background = value
elif key == 'decode':
self.codec = value
elif key == 'stdin':
self.stdin = value
self._stdout = None
names=['message','background','stdin','codec']
return dict((i,getattr(self, i)) for i in names)
def launch(self):
'''launch a configured command'''
self._response = None
self._execute() # preempt with pox.which(message.split()[0]) ?
return
def _execute(self):
#'''execute by piping the command, & saving the file object'''
from subprocess import Popen, PIPE, STDOUT
#XXX: what if saved list/dict of _stdout instead of just the one?
# could associated name/_pid and _stdout
if self.background: #Spawn a background process
try:
p = Popen(self.message, shell=True,
stdin=self.stdin, stdout=PIPE,
stderr=STDOUT, close_fds=True)
except:
raise PipeException('failure to pipe: %s' % self.message)
self._pid = p.pid #get fileobject pid
self._stdout = p.stdout #save fileobject
else:
try:
p = Popen(self.message, shell=True,
stdin=self.stdin, stdout=PIPE)
except:
raise PipeException('failure to pipe: %s' % self.message)
self._stdout = p.stdout
self._pid = 0 #XXX: MMM --> or -1 ?
return
def response(self):
'''Return the response from the launched process.
Return None if no response was received yet from a background process.
''' #XXX: if PY3, return bytes, decode to ascii, take encoding, or ???
if self._stdout is None:
raise PipeException("'launch' is required after any reconfiguration")
if self.codec is True: codec = 'ascii'
elif self.codec is False: codec = False
elif self.codec is None: codec = False
else: codec = self.codec
if self._response is not None: return _str(self._response, codec)
# when running in foreground _pid is 0 (may change to -1)
if self._pid <= 0:
self._response = self._stdout.read()
return _str(self._response, codec)
# handle response from a background process
def onData(selector, fobj):
if self.verbose: print("handling pipe response")
self._debug.info('on_remote')
self._response = fobj.read()
selector.state = False
return
def onTimeout(selector):
selector.state = False
sel = Selector()
#sel._info.activate()
sel.notifyOnReadReady(self._stdout, onData)
sel.notifyWhenIdle(onTimeout)
sel.watch(2.0)
# reset _response to None to allow capture of a next response
# from a background process
return _str(self._response, codec)
def pid(self):
'''get pipe pid'''
return self._pid
def kill(self):
'''terminate the pipe'''
if self._pid > 0:
if self.verbose: print('Kill pid=%d' % self._pid)
os.kill(self._pid, signal.SIGTERM)
os.waitpid(self._pid, 0)
self._pid = 0
return
# interface
__call__ = config
pass
return
def _cleanup(self):
self._debug.info("cleaning up")
for fd in self._input:
fd.close()
for fd in self._output:
fd.close()
for fd in self._exception:
fd.close()
for handler in self._interrupt:
handler(self)
return
# static members
from pathos import logger
_debug = logger(name="pathos.selector", level=30) # logging.WARN
del logger
# constants
_TIMEOUT = .5
# End of file
class Selector(object):
"""
Selector object for watching and event notification.
"""
def watch(self, timeout=None):
"""dispatch events to the registered hanlders"""
if timeout:
self._timeout = timeout
self._watch()
return
# FIXME:
# leave like this until I understand better the set of exceptions I
# would like to handle. It really is bad to catch all exceptions,
# especially since it hides errors during development
try:
self._watch()
except:
# catch all exceptions
self._cleanup()
# get exception information
import sys
type, value = sys.exc_info()[:2]
# rethrow the exception so the clients can handle it
raise type(value)
return
def notifyOnReadReady(self, fd, handler):
"""add <handler> to the list of routines to call when <fd> is read ready"""
self._input.setdefault(fd, []).append(handler)
return
def notifyOnWriteReady(self, fd, handler):
"""add <handler> to the list of routines to call when <fd> is write ready"""
self._output.setdefault(fd, []).append(handler)
return
def notifyOnException(self, fd, handler):
"""add <handler> to the list of routines to call when <fd> raises an exception"""
self._exception.setdefault(fd, []).append(handler)
return
def notifyOnInterrupt(self, handler):
"""add <handler> to the list of routines to call when a signal arrives"""
self._interrupt.append(handler)
return
def notifyWhenIdle(self, handler):
"""add <handler> to the list of routines to call when a timeout occurs"""
self._idle.append(handler)
return
def __init__(self):
"""
Takes no initial input.
"""
self.state = True
self._timeout = self._TIMEOUT
# the fd activity clients
self._input = {}
self._output = {}
self._exception = {}
# clients to notify when there is nothing else to do
self._idle = []
self._interrupt = []
return
def _watch(self):
import select
while self.state:
self._debug.debug("constructing list of watchers")
iwtd = list(self._input.keys())
owtd = list(self._output.keys())
ewtd = list(self._exception.keys())
self._debug.debug("input: %s" % iwtd)
self._debug.debug("output: %s" % owtd)
self._debug.debug("exception: %s" % ewtd)
self._debug.debug("checking for indefinite block")
if not iwtd and not owtd and not ewtd and not self._idle:
self._debug.info("no registered handlers left; exiting")
return
self._debug.debug("calling select")
try:
reads, writes, excepts = select.select(iwtd, owtd, ewtd,
self._timeout)
except select.error as error: # breaks 2.5 compatibility
# GUESS:
# when a signal is delivered to a signal handler registered
# by the application, the select call is interrupted and
# raises a select.error
errno, msg = error.args
self._debug.info("signal received: %d: %s" % (errno, msg))
continue
self._debug.debug("returned from select")
# dispatch to the idle handlers if this was a timeout
if not reads and not writes and not excepts:
self._debug.info("no activity; dispatching to idle handlers")
for handler in self._idle:
if not handler(self):
self._idle.remove(handler)
else:
# dispatch to the registered handlers
self._debug.info("dispatching to exception handlers")
self._dispatch(self._exception, excepts)
self._debug.info("dispatching to output handlers")
self._dispatch(self._output, writes)
self._debug.info("dispatching to input handlers")
self._dispatch(self._input, reads)
return
def _dispatch(self, handlers, entities):
for fd in entities:
for handler in handlers[fd]:
if not handler(self, fd):
handlers[fd].remove(handler)
if not handlers[fd]:
del handlers[fd]
return
def _cleanup(self):
self._debug.info("cleaning up")
for fd in self._input:
fd.close()
for fd in self._output:
fd.close()
for fd in self._exception:
fd.close()
for handler in self._interrupt:
handler(self)
return
# static members
from pathos import logger
_debug = logger(name="pathos.selector", level=30) # logging.WARN
del logger
# constants
_TIMEOUT = .5
#!/usr/bin/env python
#
# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
# Copyright (c) 1997-2015 California Institute of Technology.
# License: 3-clause BSD. The full license text is available at:
# - http://trac.mystic.cacr.caltech.edu/project/pathos/browser/pathos/LICENSE
"""
example of building a simple xmlrpc server and proxy,
and then demonstrate the handling of a few basic requests
To run: python xmlrpc_server.py
"""
from pathos.xmlrpc import XMLRPCServer
from pathos import logger
logger(level=20, name='pathos.xmlrpc') # logging.INFO
logger(level=20, name='pathos.selector') # logging.INFO
if __name__ == '__main__':
import os, time, xmlrpclib
s = XMLRPCServer('', 0)
print 'port=%d' % s.port
port = s.port
pid = os.fork()
if pid > 0: #parent
def add(x, y): return x + y
s.register_function(add)
s.activate()
# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
# Copyright (c) 1997-2016 California Institute of Technology.
# Copyright (c) 2016-2017 The Uncertainty Quantification Foundation.
# License: 3-clause BSD. The full license text is available at:
# - http://trac.mystic.cacr.caltech.edu/project/pathos/browser/pathos/LICENSE
"""
example of building a simple xmlrpc server and proxy,
and then demonstrate the handling of a few basic requests
To run: python xmlrpc_server.py
To stop: < Ctrl+C >
"""
from pathos.xmlrpc import XMLRPCServer
from pathos import logger
logger(level=20, name='pathos.xmlrpc') # logging.INFO
logger(level=20, name='pathos.selector') # logging.INFO
if __name__ == '__main__':
import os, time
try:
import xmlrpc.client as client
except ImportError:
import xmlrpclib as client
s = XMLRPCServer('', 0)
print('port=%d' % s.port)
port = s.port
pid = os.fork()