def main():
# get data from parent over stdin
data = pickle.load(sys.stdin)
sys.stdin.close()
# set some stuff
_logger.setLevel(data['dist_log_level'])
forking.prepare(data)
# create server for a `HostManager` object
server = managers.Server(HostManager._registry, ('', 0), data['authkey'])
current_process()._server = server
# report server address and number of cpus back to parent
conn = connection.Client(data['parent_address'], authkey=data['authkey'])
conn.send((data['index'], server.address, slot_count))
conn.close()
# set name etc
current_process().set_name('Host-%s:%s' % server.address)
util._run_after_forkers()
# register a cleanup function
def cleanup(directory):
debug('removing directory %s', directory)
shutil.rmtree(directory)
debug('shutting down host manager')
util.Finalize(None, cleanup, args=[data['dir']], exitpriority=0)
# start host manager
debug('remote host manager starting in %s', data['dir'])
server.serve_forever()
def main():
# get data from parent over stdin
data = pickle.load(sys.stdin)
sys.stdin.close()
# set some stuff
_logger.setLevel(data['dist_log_level'])
forking.prepare(data)
# create server for a `HostManager` object
server = managers.Server(HostManager._registry, ('', 0), data['authkey'])
current_process()._server = server
# report server address and number of cpus back to parent
conn = connection.Client(data['parent_address'], authkey=data['authkey'])
conn.send((data['index'], server.address, slot_count))
conn.close()
# set name etc
current_process().set_name('Host-%s:%s' % server.address)
util._run_after_forkers()
# register a cleanup function
def cleanup(directory):
debug('removing directory %s', directory)
shutil.rmtree(directory)
debug('shutting down host manager')
util.Finalize(None, cleanup, args=[data['dir']], exitpriority=0)
# start host manager
debug('remote host manager starting in %s', data['dir'])
server.serve_forever()
def main():
assert is_forking(sys.argv)
fd = int(sys.argv[-1])
from_parent = os.fdopen(fd, 'rb')
current_process()._inheriting = True
preparation_data = load(from_parent)
_forking.prepare(preparation_data)
# Huge hack to make logging before Process.run work.
loglevel = os.environ.get("_MP_FORK_LOGLEVEL_")
logfile = os.environ.get("_MP_FORK_LOGFILE_") or None
format = os.environ.get("_MP_FORK_LOGFORMAT_")
if loglevel:
from multiprocessing import util
import logging
logger = util.get_logger()
logger.setLevel(int(loglevel))
if not logger.handlers:
logger._rudimentary_setup = True
logfile = logfile or sys.__stderr__
if hasattr(logfile, "write"):
handler = logging.StreamHandler(logfile)
else:
handler = logging.FileHandler(logfile)
formatter = logging.Formatter(
format or util.DEFAULT_LOGGING_FORMAT)
handler.setFormatter(formatter)
logger.addHandler(handler)
self = load(from_parent)
current_process()._inheriting = False
exitcode = self._bootstrap()
exit(exitcode)
def main():
assert is_forking(sys.argv)
fd = int(sys.argv[-1])
from_parent = os.fdopen(fd, 'rb')
current_process()._inheriting = True
preparation_data = load(from_parent)
_forking.prepare(preparation_data)
# Huge hack to make logging before Process.run work.
loglevel = os.environ.get("_MP_FORK_LOGLEVEL_")
logfile = os.environ.get("_MP_FORK_LOGFILE_") or None
format = os.environ.get("_MP_FORK_LOGFORMAT_")
if loglevel:
from multiprocessing import util
import logging
logger = util.get_logger()
logger.setLevel(int(loglevel))
if not logger.handlers:
logger._rudimentary_setup = True
logfile = logfile or sys.__stderr__
if hasattr(logfile, "write"):
handler = logging.StreamHandler(logfile)
else:
handler = logging.FileHandler(logfile)
formatter = logging.Formatter(format
or util.DEFAULT_LOGGING_FORMAT)
handler.setFormatter(formatter)
logger.addHandler(handler)
self = load(from_parent)
current_process()._inheriting = False
exitcode = self._bootstrap()
exit(exitcode)
def main(): #pragma no cover
"""
Code which runs a host manager.
Expects configuration data from parent on `stdin`.
Replies with address and optionally public key.
The environment variable ``OPENMDAO_KEEPDIRS`` can be used to avoid
removal of the temporary directory used here.
"""
sys.stdout = open('stdout', 'w')
sys.stderr = open('stderr', 'w')
# util.log_to_stderr(logging.DEBUG)
# Avoid root possibly masking us.
logging.getLogger().setLevel(logging.DEBUG)
import platform
hostname = platform.node()
pid = os.getpid()
ident = '(%s:%d)' % (hostname, pid)
print '%s main startup' % ident
sys.stdout.flush()
# Get data from parent over stdin.
data = cPickle.load(sys.stdin)
sys.stdin.close()
print '%s data received' % ident
authkey = data['authkey']
allow_shell = data['allow_shell']
allowed_users = data['allowed_users']
print '%s using %s authentication' % (ident, keytype(authkey))
if allowed_users is None:
print '%s allowed_users: ANY' % ident
else:
print '%s allowed_users: %s' % (ident, sorted(allowed_users.keys()))
if allow_shell:
print '%s ALLOWING SHELL ACCESS' % ident
sys.stdout.flush()
log_level = data['dist_log_level']
os.environ['OPENMDAO_KEEPDIRS'] = data['keep_dirs']
exc = None
server = None
try:
# Update HostManager registry.
dct = data['registry']
print '%s registry:' % ident
for name in dct.keys():
module = dct[name]
print ' %s: %s' % (name, module)
mod = __import__(module, fromlist=name)
cls = getattr(mod, name)
register(cls, HostManager)
# Set some stuff.
print '%s preparing to fork, log level %d' % (ident, log_level)
sys.stdout.flush()
util.get_logger().setLevel(log_level)
forking.prepare(data)
# Create Server for a HostManager object.
name = '%d[%d]' % (data['index'], pid)
logging.getLogger(name).setLevel(log_level)
server = OpenMDAO_Server(HostManager._registry, (hostname, 0),
authkey,
'pickle',
name=name,
allowed_users=allowed_users,
allowed_hosts=[data['parent_address'][0]])
except Exception as exc:
print '%s caught exception: %s' % (ident, exc)
# Report server address and public key back to parent.
print '%s connecting to parent at %s' % (ident, data['parent_address'])
sys.stdout.flush()
conn = connection.Client(data['parent_address'], authkey=authkey)
if exc:
conn.send((data['index'], None, str(exc)))
else:
conn.send((data['index'], server.address, server.public_key_text))
conn.close()
if exc:
print '%s exiting' % ident
sys.exit(1)
# Set name etc.
current_process()._server = server
current_process()._name = 'Host-%s:%s' % server.address
current_process().authkey = authkey
logging.getLogger(current_process()._name).setLevel(log_level)
util._run_after_forkers()
# Register a cleanup function.
def cleanup(directory):
keep_dirs = int(os.environ.get('OPENMDAO_KEEPDIRS', '0'))
if not keep_dirs and os.path.exists(directory):
print '%s removing directory %s' % (ident, directory)
shutil.rmtree(directory)
print '%s shutting down host manager' % ident
util.Finalize(None, cleanup, args=[data['dir']], exitpriority=0)
# Start host manager.
print '%s remote host manager starting in %s' % (ident, data['dir'])
sys.stdout.flush()
server.serve_forever()
def _bootstrap(self):
forking.prepare({'main_path': self._main_path})
self._target, self._args, self._kwargs = pickle.loads(self._data)
return Process._bootstrap(self)
def main(): #pragma no cover
"""
Code which runs a host manager.
Expects configuration data from parent on `stdin`.
Replies with address and optionally public key.
The environment variable ``OPENMDAO_KEEPDIRS`` can be used to avoid
removal of the temporary directory used here.
"""
sys.stdout = open('stdout', 'w')
sys.stderr = open('stderr', 'w')
# util.log_to_stderr(logging.DEBUG)
# Avoid root possibly masking us.
logging.getLogger().setLevel(logging.DEBUG)
pid = os.getpid()
ident = '(%s:%d)' % (socket.gethostname(), pid)
print '%s main startup' % ident
sys.stdout.flush()
# Get data from parent over stdin.
dump = sys.stdin.read()
sys.stdin.close()
print '%s data received (%s)' % (ident, len(dump))
data = cPickle.loads(base64.b64decode(dump))
hostname = data['hostname']
print '%s using hostname %s' % (ident, hostname)
authkey = data['authkey']
print '%s using %s authentication' % (ident, keytype(authkey))
allowed_users = data['allowed_users']
if allowed_users is None:
print '%s allowed_users: ANY' % ident
else:
print '%s allowed_users: %s' % (ident, sorted(allowed_users.keys()))
allow_shell = data['allow_shell']
if allow_shell:
print '%s ALLOWING SHELL ACCESS' % ident
allow_tunneling = data['allow_tunneling']
print '%s allow_tunneling: %s' % (ident, allow_tunneling)
if allow_tunneling:
hostname = 'localhost'
sys.stdout.flush()
log_level = data['dist_log_level']
os.environ['OPENMDAO_KEEPDIRS'] = data['keep_dirs']
exc = None
server = None
try:
# Update HostManager registry.
dct = data['registry']
print '%s registry:' % ident
for name in dct.keys():
module = dct[name]
print' %s: %s' % (name, module)
mod = __import__(module, fromlist=name)
cls = getattr(mod, name)
register(cls, HostManager)
# Set some stuff.
print '%s preparing to fork, log level %d' % (ident, log_level)
sys.stdout.flush()
util.get_logger().setLevel(log_level)
forking.prepare(data)
# Create Server for a HostManager object.
name = '%d[%d]' % (data['index'], pid)
logging.getLogger(name).setLevel(log_level)
server = OpenMDAO_Server(HostManager._registry, (hostname, 0),
authkey, 'pickle', name=name,
allowed_users=allowed_users,
allowed_hosts=[data['parent_address'][0]],
allow_tunneling=allow_tunneling)
print '%s server listening at %s' % (ident, server.address)
except Exception as exc:
print '%s caught exception: %s' % (ident, exc)
# Report server address and public key back to parent.
print '%s connecting to parent at %s' % (ident, data['parent_address'])
sys.stdout.flush()
for retry in range(10):
try:
conn = connection.Client(data['parent_address'], authkey=authkey)
except socket.error as sock_exc:
print '%s %s' % (ident, sock_exc)
if retry < 9 and (sock_exc.args[0] == errno.ECONNREFUSED or \
sock_exc.args[0] == errno.ENOENT):
print '%s retrying...' % ident
time.sleep(1)
else:
print '%s exiting' % ident
sys.exit(1)
else:
break
if exc:
conn.send((data['index'], None, str(exc)))
else:
conn.send((data['index'], server.address, server.public_key_text))
conn.close()
if exc:
print '%s exiting' % ident
sys.exit(1)
# Set name etc.
current_process()._server = server
current_process()._name = 'Host-%s:%s' % server.address
current_process().authkey = authkey
logging.getLogger(current_process()._name).setLevel(log_level)
util._run_after_forkers()
# Register a cleanup function.
def cleanup(directory):
""" Removes our directory unless OPENMDAO_KEEPDIRS set. """
keep_dirs = int(os.environ.get('OPENMDAO_KEEPDIRS', '0'))
if not keep_dirs and os.path.exists(directory):
print '%s removing directory %s' % (ident, directory)
try:
shutil.rmtree(directory, onerror=onerror)
except WindowsError as exc:
print '%s %s' % (ident, exc)
print '%s shutting down host manager' % ident
util.Finalize(None, cleanup, args=[data['dir']], exitpriority=0)
# Start host manager.
print '%s remote host manager starting in %s' % (ident, data['dir'])
sys.stdout.flush()
server.serve_forever()
def _bootstrap(self):
forking.prepare({'main_path': self._main_path})
self._target, self._args, self._kwargs = pickle.loads(self._data)
return Process._bootstrap(self)
