def test_errors(self):
logging.debug("")
logging.debug("test_errors")
logging.debug("allocate server")
server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
try:
logging.debug("execute bad command")
code = "server.execute_command(dict(remote_command='no-such-command'))"
if sys.platform == "win32":
msg = "WindowsError: [Error 2] The system cannot find the file specified"
else:
msg = "OSError: [Errno 2] No such file or directory"
try:
server.execute_command(dict(remote_command="no-such-command"))
except protocol.RemoteError as exc:
exc_msg = str(exc)
if msg not in exc_msg:
self.fail("%s not in %s" % (msg, exc_msg))
else:
self.fail("Expecting protocol.RemoteError")
logging.debug("open bad file")
msg = "Can\\'t open \\'../../illegal-access\\', not within root"
msg = "RuntimeError: %s" % msg
try:
server.open("../../illegal-access", "r")
except protocol.RemoteError as exc:
exc_msg = str(exc)
if msg not in exc_msg:
self.fail("%s not in %s" % (msg, exc_msg))
else:
self.fail("Expecting protocol.RemoteError")
logging.debug("open missing file")
msg = "[Errno 2] No such file or directory: \\'no-such-file\\'"
msg = "IOError: %s" % msg
try:
server.open("no-such-file", "r")
except protocol.RemoteError as exc:
exc_msg = str(exc)
if msg not in exc_msg:
self.fail("%s not in %s" % (msg, exc_msg))
else:
self.fail("Expecting protocol.RemoteError")
finally:
logging.debug("release")
RAM.release(server)
# Test for exited or never started server.
logging.debug("dead server")
self.proc.terminate()
self.proc = None
time.sleep(2)
hostname = socket.gethostname()
if sys.platform == "win32": # Server doesn't clean up.
root = protocol._server_root(hostname)
mapped_root = os.path.join(_DMZ_ROOT, protocol._map_dir(root))
for name in glob.glob("%s*" % mapped_root):
os.remove(name)
code = "NAS_Allocator(dmz_host=hostname, server_host=hostname)"
assert_raises(
self,
code,
globals(),
locals(),
RuntimeError,
"NAS_Allocator: can't connect: server root 'RJE-%s='" " on '%s' not found" % (hostname, hostname),
)
# Test for missing heartbeat.
logging.debug("no heartbeat")
with open(os.path.join(_DMZ_ROOT, "RJE-%s=" % hostname), "w") as out:
out.write("empty\n")
try:
NAS_Allocator(dmz_host=hostname, server_host=hostname)
except RuntimeError as exc:
msg = "IOError: [Errno 2] No such file or directory:" " 'RJE-%s=heartbeat'\n" % hostname
logging.debug(str(exc))
self.assertTrue(str(exc).endswith(msg))
else:
self.fail("Expected RuntimeError")
# Test for stale heartbeat.
logging.debug("stale heartbeat")
protocol.server_heartbeat(hostname, 1, logging.getLogger())
time.sleep(5)
assert_raises(
self,
code,
globals(),
locals(),
RuntimeError,
"NAS_Allocator: can't connect: server heartbeat" " hasn't been updated in 0:00:0",
)
def test_errors(self):
logging.debug('')
logging.debug('test_errors')
logging.debug('allocate server')
server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
try:
logging.debug('execute bad command')
code = "server.execute_command(dict(remote_command='no-such-command'))"
if sys.platform == 'win32':
msg = "WindowsError('[Error 2] The system cannot find the file specified')"
else:
msg = "OSError('[Errno 2] No such file or directory')"
assert_raises(self, code, globals(), locals(),
protocol.RemoteError, msg)
logging.debug('open bad file')
msg = "Can't open '../../illegal-access', not within root"
msg = 'RuntimeError("%s' % msg
assert_raises(self, "server.open('../../illegal-access', 'r')",
globals(), locals(), protocol.RemoteError, msg)
logging.debug('open missing file')
msg = "[Errno 2] No such file or directory: 'no-such-file'"
msg = 'IOError("%s' % msg
assert_raises(self, "server.open('no-such-file', 'r')",
globals(), locals(), protocol.RemoteError, msg)
finally:
logging.debug('release')
RAM.release(server)
# Test for exited or never started server.
logging.debug('dead server')
self.proc.terminate()
self.proc = None
time.sleep(2)
hostname = socket.gethostname()
if sys.platform == 'win32': # Server doesn't clean up.
root = protocol._server_root(hostname)
mapped_root = os.path.join(_DMZ_ROOT, protocol._map_dir(root))
for name in glob.glob('%s*' % mapped_root):
os.remove(name)
code = 'NAS_Allocator(dmz_host=hostname, server_host=hostname)'
assert_raises(self, code, globals(), locals(), RuntimeError,
"NAS_Allocator: can't connect: server root 'RJE-%s='"
" on '%s' not found" % (hostname, hostname))
# Test for missing heartbeat.
logging.debug('no heartbeat')
with open(os.path.join(_DMZ_ROOT, 'RJE-%s=' % hostname), 'w') as out:
out.write('empty\n')
try:
NAS_Allocator(dmz_host=hostname, server_host=hostname)
except RuntimeError as exc:
msg = "IOError: [Errno 2] No such file or directory:" \
" 'RJE-%s=heartbeat'\n" % hostname
logging.debug(str(exc))
self.assertTrue(str(exc).endswith(msg))
else:
self.fail('Expected RuntimeError')
# Test for stale heartbeat.
logging.debug('stale heartbeat')
protocol.server_heartbeat(hostname, 1, logging.getLogger())
time.sleep(5)
assert_raises(self, code, globals(), locals(), RuntimeError,
"NAS_Allocator: can't connect: server heartbeat"
" hasn't been updated in 0:00:0")
def test_errors(self):
logging.debug('')
logging.debug('test_errors')
logging.debug('allocate server')
server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
try:
logging.debug('execute bad command')
code = "server.execute_command(dict(remote_command='no-such-command'))"
if sys.platform == 'win32':
msg = "WindowsError: [Error 2] The system cannot find the file specified"
else:
msg = "OSError: [Errno 2] No such file or directory"
try:
server.execute_command(dict(remote_command='no-such-command'))
except protocol.RemoteError as exc:
exc_msg = str(exc)
if msg not in exc_msg:
self.fail('%s not in %s' % (msg, exc_msg))
else:
self.fail('Expecting protocol.RemoteError')
logging.debug('open bad file')
msg = "Can\\'t open \\'../../illegal-access\\', not within root"
msg = 'RuntimeError: %s' % msg
try:
server.open('../../illegal-access', 'r')
except protocol.RemoteError as exc:
exc_msg = str(exc)
if msg not in exc_msg:
self.fail('%s not in %s' % (msg, exc_msg))
else:
self.fail('Expecting protocol.RemoteError')
logging.debug('open missing file')
msg = "[Errno 2] No such file or directory: \\'no-such-file\\'"
msg = 'IOError: %s' % msg
try:
server.open('no-such-file', 'r')
except protocol.RemoteError as exc:
exc_msg = str(exc)
if msg not in exc_msg:
self.fail('%s not in %s' % (msg, exc_msg))
else:
self.fail('Expecting protocol.RemoteError')
finally:
logging.debug('release')
RAM.release(server)
# Test for exited or never started server.
logging.debug('dead server')
self.proc.terminate()
self.proc = None
time.sleep(2)
hostname = socket.gethostname()
if sys.platform == 'win32': # Server doesn't clean up.
root = protocol._server_root(hostname)
mapped_root = os.path.join(_DMZ_ROOT, protocol._map_dir(root))
for name in glob.glob('%s*' % mapped_root):
os.remove(name)
code = 'NAS_Allocator(dmz_host=hostname, server_host=hostname)'
assert_raises(self, code, globals(), locals(), RuntimeError,
"NAS_Allocator: can't connect: server root 'RJE-%s='"
" on '%s' not found" % (hostname, hostname))
# Test for missing heartbeat.
logging.debug('no heartbeat')
with open(os.path.join(_DMZ_ROOT, 'RJE-%s=' % hostname), 'w') as out:
out.write('empty\n')
try:
NAS_Allocator(dmz_host=hostname, server_host=hostname)
except RuntimeError as exc:
msg = "IOError: [Errno 2] No such file or directory:" \
" 'RJE-%s=heartbeat'\n" % hostname
logging.debug(str(exc))
self.assertTrue(str(exc).endswith(msg))
else:
self.fail('Expected RuntimeError')
# Test for stale heartbeat.
logging.debug('stale heartbeat')
protocol.server_heartbeat(hostname, 1, logging.getLogger())
time.sleep(5)
assert_raises(self, code, globals(), locals(), RuntimeError,
"NAS_Allocator: can't connect: server heartbeat"
" hasn't been updated in 0:00:0")
def main(): # pragma no cover
"""
Runs the RJE server.
Usage: python rje.py [--allocator=name][--dmz-host=name][--poll-delay=secs][--resources=filename]
--allocator: string
Allocator to provide remote access to. Default ``PBS``.
--dmz-host: string
DMZ file server to use. Default ``dmzfs1``.
--poll-delay: int
Maximum seconds between checks for new client activity. Default 60.
--resources: string
Filename for resource configuration. If not specified then the
default of ``~/.openmdao/resources.cfg`` will be used.
"""
parser = optparse.OptionParser()
parser.add_option('--allocator',
action='store',
type='str',
default='PBS',
help='Allocator to provide remote access to')
parser.add_option('--dmz-host',
action='store',
type='str',
default='dmzfs1',
help='DMZ file server to use')
parser.add_option(
'--poll-delay',
action='store',
type='int',
default=60,
help='Max seconds between checks for new client activity')
parser.add_option('--resources',
action='store',
type='str',
default=None,
help='Filename for resource configuration')
parser.add_option('--ssh',
action='store',
type='str',
default=None,
help='ssh command (used during testing)')
parser.add_option('--scp',
action='store',
type='str',
default=None,
help='scp command (used during testing)')
options, arguments = parser.parse_args()
if arguments:
parser.print_help()
sys.exit(1)
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
# Configure ssh and scp.
if options.ssh:
configure_ssh(options.ssh.split())
if options.scp:
configure_scp(options.scp.split())
# Optionally configure resources.
if options.resources is not None:
RAM.configure(options.resources)
# Get allocator to wrap.
try:
allocator = RAM.get_allocator(options.allocator)
except ValueError:
msg = "Can't find allocator %r" % options.allocator
print msg
logger.error(msg)
sys.exit(1)
dmz_host = options.dmz_host
poll_delay = options.poll_delay
# Initialize DMZ protocol.
server_init(dmz_host, logger)
global _DMZ_HOST
_DMZ_HOST = dmz_host
msg = 'RJE server ready'
print msg
logger.info(msg)
# Setup for cleanup by this process only.
global _RJE_PID
_RJE_PID = os.getpid()
signal.signal(signal.SIGTERM, _sigterm_handler)
# And away we go...
wrappers = {}
try:
delay = 1 # Start with high polling rate.
while True:
conn_info, removed = server_accept(dmz_host, poll_delay, logger)
for client in removed:
wrapper = wrappers.pop(client, None)
if wrapper is not None:
wrapper.shutdown()
if conn_info is None:
server_heartbeat(dmz_host, poll_delay, logger)
delay = min(delay + 1, poll_delay) # Back-off.
time.sleep(delay)
else:
client, connection = conn_info
wrapper = AllocatorWrapper(allocator, client, connection)
handler = threading.Thread(name='%s_handler' % client,
target=wrapper.process_requests)
handler.daemon = True
handler.start()
wrappers[client] = wrapper
delay = 1 # Reset.
except KeyboardInterrupt:
pass
finally:
_cleanup()
sys.exit(0)
def main(): # pragma no cover
"""
Runs the RJE server.
Usage: python rje.py [--allocator=name][--dmz-host=name][--poll-delay=secs][--resources=filename]
--allocator: string
Allocator to provide remote access to. Default ``PBS``.
--dmz-host: string
DMZ file server to use. Default ``dmzfs1``.
--poll-delay: int
Maximum seconds between checks for new client activity. Default 60.
--resources: string
Filename for resource configuration. If not specified then the
default of ``~/.openmdao/resources.cfg`` will be used.
"""
parser = optparse.OptionParser()
parser.add_option('--allocator', action='store', type='str', default='PBS',
help='Allocator to provide remote access to')
parser.add_option('--dmz-host', action='store', type='str', default='dmzfs1',
help='DMZ file server to use')
parser.add_option('--poll-delay', action='store', type='int', default=60,
help='Max seconds between checks for new client activity')
parser.add_option('--resources', action='store', type='str', default=None,
help='Filename for resource configuration')
parser.add_option('--ssh', action='store', type='str', default=None,
help='ssh command (used during testing)')
parser.add_option('--scp', action='store', type='str', default=None,
help='scp command (used during testing)')
options, arguments = parser.parse_args()
if arguments:
parser.print_help()
sys.exit(1)
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
# Configure ssh and scp.
if options.ssh:
configure_ssh(options.ssh.split())
if options.scp:
configure_scp(options.scp.split())
# Optionally configure resources.
if options.resources is not None:
RAM.configure(options.resources)
# Get allocator to wrap.
try:
allocator = RAM.get_allocator(options.allocator)
except ValueError:
msg = "Can't find allocator %r" % options.allocator
print msg
logger.error(msg)
sys.exit(1)
dmz_host = options.dmz_host
poll_delay = options.poll_delay
# Initialize DMZ protocol.
server_init(dmz_host, logger)
global _DMZ_HOST
_DMZ_HOST = dmz_host
msg = 'RJE server ready'
print msg
logger.info(msg)
# Setup for cleanup by this process only.
global _RJE_PID
_RJE_PID = os.getpid()
signal.signal(signal.SIGTERM, _sigterm_handler)
# And away we go...
wrappers = {}
try:
delay = 1 # Start with high polling rate.
while True:
conn_info, removed = server_accept(dmz_host, poll_delay, logger)
for client in removed:
wrapper = wrappers.pop(client, None)
if wrapper is not None:
wrapper.shutdown()
if conn_info is None:
server_heartbeat(dmz_host, poll_delay, logger)
delay = min(delay + 1, poll_delay) # Back-off.
time.sleep(delay)
else:
client, connection = conn_info
wrapper = AllocatorWrapper(allocator, client, connection)
handler = threading.Thread(name='%s_handler' % client,
target=wrapper.process_requests)
handler.daemon = True
handler.start()
wrappers[client] = wrapper
delay = 1 # Reset.
except KeyboardInterrupt:
pass
finally:
_cleanup()
sys.exit(0)