def test_case_eval(self):
logging.debug("")
logging.debug("test_case_eval")
# Run a fake job in style of CaseIteratorDriver.
logging.debug("allocate server")
server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
echo = set_as_top(Echo(1))
egg_info = echo.save_to_egg("EchoTest", "1", need_requirements=False)
egg_filename = egg_info[0]
try:
logging.debug("transfer egg")
filexfer(None, egg_filename, server, egg_filename, "b")
logging.debug("load model")
tlo = server.load_model(egg_filename)
logging.debug("set input")
tlo.set("inp_0", 42)
logging.debug("run")
tlo.run()
logging.debug("get output")
output = tlo.get("out_0")
self.assertEqual(output, 42)
finally:
os.remove(egg_filename)
logging.debug("release")
RAM.release(server)
def test_case_eval(self):
logging.debug('')
logging.debug('test_case_eval')
# Run a fake job in style of CaseIteratorDriver.
logging.debug('allocate server')
server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
echo = set_as_top(Echo(1))
egg_info = echo.save_to_egg('EchoTest', '1', need_requirements=False)
egg_filename = egg_info[0]
try:
logging.debug('transfer egg')
filexfer(None, egg_filename, server, egg_filename, 'b')
logging.debug('load model')
tlo = server.load_model(egg_filename)
logging.debug('set input')
tlo.set('inp_0', 42)
logging.debug('run')
tlo.run()
logging.debug('get output')
output = tlo.get('out_0')
self.assertEqual(output, 42)
finally:
os.remove(egg_filename)
logging.debug('release')
RAM.release(server)
def test_case_eval(self):
logging.debug('')
logging.debug('test_case_eval')
# Run a fake job in style of CaseIteratorDriver.
logging.debug('allocate server')
server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
echo = set_as_top(Echo(1))
egg_info = echo.save_to_egg('EchoTest', '1', need_requirements=False)
egg_filename = egg_info[0]
try:
logging.debug('transfer egg')
filexfer(None, egg_filename, server, egg_filename, 'b')
logging.debug('load model')
tlo = server.load_model(egg_filename)
logging.debug('set input')
tlo.set('inp_0', 42)
logging.debug('run')
tlo.run()
logging.debug('get output')
output = tlo.get('out_0')
self.assertEqual(output, 42)
finally:
os.remove(egg_filename)
logging.debug('release')
RAM.release(server)
def _service_loop(self, name, resource_desc, credentials, reply_q):
""" Each server has an associated thread executing this. """
set_credentials(credentials)
server, server_info = RAM.allocate(resource_desc)
# Just being defensive, this should never happen.
if server is None: # pragma no cover
self._logger.error('Server allocation for %r failed :-(', name)
reply_q.put((name, False, None))
return
else:
# Clear egg re-use indicator.
server_info['egg_file'] = None
self._logger.debug('%r using %r', name, server_info['name'])
if self._logger.level == logging.NOTSET:
# By default avoid lots of protocol messages.
server.set_log_level(logging.DEBUG)
else:
server.set_log_level(self._logger.level)
request_q = Queue.Queue()
try:
with self._server_lock:
sdata = self._servers[name]
sdata.server = server
sdata.info = server_info
sdata.queue = request_q
reply_q.put((name, True, None)) # ACK startup.
while True:
request = request_q.get()
if request is None:
break
try:
result = request[0](request[1])
except Exception as req_exc:
self._logger.error('%r: %s caused %r', name,
request[0], req_exc)
result = None
else:
req_exc = None
reply_q.put((name, result, req_exc))
except Exception as exc: # pragma no cover
# This can easily happen if we take a long time to allocate and
# we get 'cleaned-up' before we get started.
if self._server_lock is not None:
self._logger.error('%r: %r', name, exc)
finally:
self._logger.debug('%r releasing server', name)
RAM.release(server)
reply_q.put((name, True, None)) # ACK shutdown.
def _service_loop(self, name, resource_desc, credentials, reply_q):
""" Each server has an associated thread executing this. """
set_credentials(credentials)
server, server_info = RAM.allocate(resource_desc)
# Just being defensive, this should never happen.
if server is None: # pragma no cover
self._logger.error('Server allocation for %r failed :-(', name)
reply_q.put((name, False, None))
return
else:
# Clear egg re-use indicator.
server_info['egg_file'] = None
self._logger.debug('%r using %r', name, server_info['name'])
if self._logger.level == logging.NOTSET:
# By default avoid lots of protocol messages.
server.set_log_level(logging.DEBUG)
else:
server.set_log_level(self._logger.level)
request_q = Queue.Queue()
try:
with self._server_lock:
sdata = self._servers[name]
sdata.server = server
sdata.info = server_info
sdata.queue = request_q
reply_q.put((name, True, None)) # ACK startup.
while True:
request = request_q.get()
if request is None:
break
try:
result = request[0](request[1])
except Exception as req_exc:
self._logger.error('%r: %s caused %r', name, request[0],
req_exc)
result = None
else:
req_exc = None
reply_q.put((name, result, req_exc))
except Exception as exc: # pragma no cover
# This can easily happen if we take a long time to allocate and
# we get 'cleaned-up' before we get started.
if self._server_lock is not None:
self._logger.error('%r: %r', name, exc)
finally:
self._logger.debug('%r releasing server', name)
RAM.release(server)
reply_q.put((name, True, None)) # ACK shutdown.
def _execute_remote(self):
"""
Allocate a server based on required resources, send inputs,
run command, and retrieve results.
"""
# Allocate server.
self._server, server_info = RAM.allocate(self.resources)
if self._server is None:
self.raise_exception('Server allocation failed :-(', RuntimeError)
return_code = -88888888
error_msg = ''
try:
# Send inputs.
patterns = []
for metadata in self.external_files:
if metadata.get('input', False):
patterns.append(metadata.path)
if patterns:
self._send_inputs(patterns)
else:
self._logger.debug("No input metadata paths")
# Run command.
self._logger.info("executing '%s'...", self.command)
start_time = time.time()
return_code, error_msg = \
self._server.execute_command(self.command, self.stdin,
self.stdout, self.stderr,
self.env_vars, self.poll_delay,
self.timeout)
et = time.time() - start_time
if et >= 60: #pragma no cover
self._logger.info('elapsed time: %f sec.', et)
# Retrieve results.
patterns = []
for metadata in self.external_files:
if metadata.get('output', False):
patterns.append(metadata.path)
if patterns:
self._retrieve_results(patterns)
else:
self._logger.debug("No output metadata paths")
finally:
RAM.release(self._server)
self._server = None
return (return_code, error_msg)
def test_errors(self):
logging.debug("")
logging.debug("test_errors")
logging.debug("allocate server")
server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
try:
logging.debug("execute bad command")
code = "server.execute_command(dict(remote_command='no-such-command'))"
if sys.platform == "win32":
msg = "WindowsError: [Error 2] The system cannot find the file specified"
else:
msg = "OSError: [Errno 2] No such file or directory"
try:
server.execute_command(dict(remote_command="no-such-command"))
except protocol.RemoteError as exc:
exc_msg = str(exc)
if msg not in exc_msg:
self.fail("%s not in %s" % (msg, exc_msg))
else:
self.fail("Expecting protocol.RemoteError")
logging.debug("open bad file")
msg = "Can\\'t open \\'../../illegal-access\\', not within root"
msg = "RuntimeError: %s" % msg
try:
server.open("../../illegal-access", "r")
except protocol.RemoteError as exc:
exc_msg = str(exc)
if msg not in exc_msg:
self.fail("%s not in %s" % (msg, exc_msg))
else:
self.fail("Expecting protocol.RemoteError")
logging.debug("open missing file")
msg = "[Errno 2] No such file or directory: \\'no-such-file\\'"
msg = "IOError: %s" % msg
try:
server.open("no-such-file", "r")
except protocol.RemoteError as exc:
exc_msg = str(exc)
if msg not in exc_msg:
self.fail("%s not in %s" % (msg, exc_msg))
else:
self.fail("Expecting protocol.RemoteError")
finally:
logging.debug("release")
RAM.release(server)
# Test for exited or never started server.
logging.debug("dead server")
self.proc.terminate()
self.proc = None
time.sleep(2)
hostname = socket.gethostname()
if sys.platform == "win32": # Server doesn't clean up.
root = protocol._server_root(hostname)
mapped_root = os.path.join(_DMZ_ROOT, protocol._map_dir(root))
for name in glob.glob("%s*" % mapped_root):
os.remove(name)
code = "NAS_Allocator(dmz_host=hostname, server_host=hostname)"
assert_raises(
self,
code,
globals(),
locals(),
RuntimeError,
"NAS_Allocator: can't connect: server root 'RJE-%s='" " on '%s' not found" % (hostname, hostname),
)
# Test for missing heartbeat.
logging.debug("no heartbeat")
with open(os.path.join(_DMZ_ROOT, "RJE-%s=" % hostname), "w") as out:
out.write("empty\n")
try:
NAS_Allocator(dmz_host=hostname, server_host=hostname)
except RuntimeError as exc:
msg = "IOError: [Errno 2] No such file or directory:" " 'RJE-%s=heartbeat'\n" % hostname
logging.debug(str(exc))
self.assertTrue(str(exc).endswith(msg))
else:
self.fail("Expected RuntimeError")
# Test for stale heartbeat.
logging.debug("stale heartbeat")
protocol.server_heartbeat(hostname, 1, logging.getLogger())
time.sleep(5)
assert_raises(
self,
code,
globals(),
locals(),
RuntimeError,
"NAS_Allocator: can't connect: server heartbeat" " hasn't been updated in 0:00:0",
)
def test_extcode(self):
logging.debug("")
logging.debug("test_extcode")
# Run a fake job in style of ExternalCode component.
logging.debug("allocate server")
server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
try:
with open("junk.dat", "w") as out:
out.write("just some junk")
filename = "inputs.zip"
logging.debug("pack inputs")
pfiles, pbytes = pack_zipfile(("junk.dat",), filename, logging.getLogger())
os.remove("junk.dat")
logging.debug("transfer inputs")
filexfer(None, filename, server, filename, "b")
logging.debug("unpack inputs")
ufiles, ubytes = server.unpack_zipfile(filename)
logging.debug("remove inputs")
os.remove(filename)
server.remove(filename)
logging.debug("execute command")
if sys.platform == "win32":
remote_command = "cmd"
args = ("/c", "echo", "Hello", "World!")
else:
remote_command = "echo"
args = ("Hello", "World!")
return_code, error_msg = server.execute_command(
dict(job_name="Testing", remote_command=remote_command, args=args, output_path="echo.out")
)
logging.debug("pack outputs")
filename = "outputs.zip"
pfiles, pbytes = server.pack_zipfile(("echo.out", "junk.dat"), filename)
logging.debug("transfer outputs")
filexfer(server, filename, None, filename, "b")
logging.debug("unpack outputs")
ufiles, ubytes = unpack_zipfile(filename)
logging.debug("remove outputs")
os.remove(filename)
server.remove(filename)
finally:
logging.debug("release")
RAM.release(server)
self.assertEqual(return_code, 0)
self.assertEqual(error_msg, "")
self.assertTrue(os.path.exists("echo.out"))
with open("echo.out", "rU") as out:
data = out.read()
os.remove("echo.out")
self.assertEqual(data, "Hello World!\n")
self.assertTrue(os.path.exists("junk.dat"))
with open("junk.dat", "rU") as out:
data = out.read()
os.remove("junk.dat")
self.assertEqual(data, "just some junk")
def test_errors(self):
logging.debug('')
logging.debug('test_errors')
logging.debug('allocate server')
server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
try:
logging.debug('execute bad command')
code = "server.execute_command(dict(remote_command='no-such-command'))"
if sys.platform == 'win32':
msg = "WindowsError('[Error 2] The system cannot find the file specified')"
else:
msg = "OSError('[Errno 2] No such file or directory')"
assert_raises(self, code, globals(), locals(),
protocol.RemoteError, msg)
logging.debug('open bad file')
msg = "Can't open '../../illegal-access', not within root"
msg = 'RuntimeError("%s' % msg
assert_raises(self, "server.open('../../illegal-access', 'r')",
globals(), locals(), protocol.RemoteError, msg)
logging.debug('open missing file')
msg = "[Errno 2] No such file or directory: 'no-such-file'"
msg = 'IOError("%s' % msg
assert_raises(self, "server.open('no-such-file', 'r')",
globals(), locals(), protocol.RemoteError, msg)
finally:
logging.debug('release')
RAM.release(server)
# Test for exited or never started server.
logging.debug('dead server')
self.proc.terminate()
self.proc = None
time.sleep(2)
hostname = socket.gethostname()
if sys.platform == 'win32': # Server doesn't clean up.
root = protocol._server_root(hostname)
mapped_root = os.path.join(_DMZ_ROOT, protocol._map_dir(root))
for name in glob.glob('%s*' % mapped_root):
os.remove(name)
code = 'NAS_Allocator(dmz_host=hostname, server_host=hostname)'
assert_raises(self, code, globals(), locals(), RuntimeError,
"NAS_Allocator: can't connect: server root 'RJE-%s='"
" on '%s' not found" % (hostname, hostname))
# Test for missing heartbeat.
logging.debug('no heartbeat')
with open(os.path.join(_DMZ_ROOT, 'RJE-%s=' % hostname), 'w') as out:
out.write('empty\n')
try:
NAS_Allocator(dmz_host=hostname, server_host=hostname)
except RuntimeError as exc:
msg = "IOError: [Errno 2] No such file or directory:" \
" 'RJE-%s=heartbeat'\n" % hostname
logging.debug(str(exc))
self.assertTrue(str(exc).endswith(msg))
else:
self.fail('Expected RuntimeError')
# Test for stale heartbeat.
logging.debug('stale heartbeat')
protocol.server_heartbeat(hostname, 1, logging.getLogger())
time.sleep(5)
assert_raises(self, code, globals(), locals(), RuntimeError,
"NAS_Allocator: can't connect: server heartbeat"
" hasn't been updated in 0:00:0")
def test_extcode(self):
logging.debug('')
logging.debug('test_extcode')
# Run a fake job in style of ExternalCode component.
logging.debug('allocate server')
server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
try:
with open('junk.dat', 'w') as out:
out.write('just some junk')
filename = 'inputs.zip'
logging.debug('pack inputs')
pfiles, pbytes = pack_zipfile(('junk.dat',), filename,
logging.getLogger())
os.remove('junk.dat')
logging.debug('transfer inputs')
filexfer(None, filename, server, filename, 'b')
logging.debug('unpack inputs')
ufiles, ubytes = server.unpack_zipfile(filename)
logging.debug('remove inputs')
os.remove(filename)
server.remove(filename)
logging.debug('execute command')
if sys.platform == 'win32':
remote_command = 'cmd'
args = ('/c', 'echo', 'Hello', 'World!')
else:
remote_command = 'echo'
args = ('Hello', 'World!')
return_code, error_msg = \
server.execute_command(dict(job_name='Testing',
remote_command=remote_command,
args=args,
output_path='echo.out'))
logging.debug('pack outputs')
filename = 'outputs.zip'
pfiles, pbytes = server.pack_zipfile(('echo.out', 'junk.dat'),
filename)
logging.debug('transfer outputs')
filexfer(server, filename, None, filename, 'b')
logging.debug('unpack outputs')
ufiles, ubytes = unpack_zipfile(filename)
logging.debug('remove outputs')
os.remove(filename)
server.remove(filename)
finally:
logging.debug('release')
RAM.release(server)
self.assertEqual(return_code, 0)
self.assertEqual(error_msg, '')
self.assertTrue(os.path.exists('echo.out'))
with open('echo.out', 'rU') as out:
data = out.read()
os.remove('echo.out')
self.assertEqual(data, 'Hello World!\n')
self.assertTrue(os.path.exists('junk.dat'))
with open('junk.dat', 'rU') as out:
data = out.read()
os.remove('junk.dat')
self.assertEqual(data, 'just some junk')
def test_remote(self):
logging.debug('')
logging.debug('test_remote')
# Start remote server.
server_dir = 'Factory'
if os.path.exists(server_dir):
shutil.rmtree(server_dir, onerror=onerror)
os.mkdir(server_dir)
os.chdir(server_dir)
try:
server, server_cfg = start_server()
cfg = read_server_config(server_cfg)
factory = None
try:
factory = connect(cfg['address'], cfg['port'],
pubkey=cfg['key'])
prefix = RAM._make_prefix(factory.host)
remote = '%s_LocalHost' % prefix
# Show no remotes currently in RAM.
allocator_names = \
[allocator.name for allocator in RAM.list_allocators()]
logging.debug('%s', allocator_names)
self.assertFalse(remote in allocator_names)
# Add remote server's allocator.
RAM.add_remotes(factory)
allocator_names = \
[allocator.name for allocator in RAM.list_allocators()]
logging.debug('%s', allocator_names)
self.assertTrue(remote in allocator_names)
self.assertFalse(RAM.get_allocator(remote) is RAM.list_allocators()[0])
self.assertTrue(RAM.get_allocator(remote) is RAM.list_allocators()[1])
# Max servers.
max_servers = RAM.max_servers(dict(allocator=remote))
self.assertTrue(max_servers >= 0) # Avoid host load issues.
remote_alloc = RAM.get_allocator(remote)
max_servers, info = \
remote_alloc.max_servers(dict(localhost=True))
self.assertEqual(max_servers, 0)
self.assertEqual(info, dict(localhost='requested local host'))
max_servers, info = \
remote_alloc.max_servers(dict(allocator='LocalHost'))
self.assertEqual(max_servers, 0)
self.assertEqual(info, dict(allocator='wrong allocator'))
estimate, info = \
remote_alloc.time_estimate(dict(allocator='LocalHost'))
self.assertEqual(estimate, -2)
self.assertEqual(info, dict(allocator='wrong allocator'))
# Allocate, release.
remote_server, info = RAM.allocate(dict(allocator=remote))
RAM.release(remote_server)
# Remove remote allocators.
allocator_names = \
[allocator.name for allocator in RAM.list_allocators()]
for name in allocator_names:
if name.startswith(prefix):
RAM.remove_allocator(name)
allocator_names = \
[allocator.name for allocator in RAM.list_allocators()]
logging.debug('%s', allocator_names)
self.assertFalse(remote in allocator_names)
finally:
if factory is not None:
factory.cleanup()
server.terminate(timeout=10)
finally:
os.chdir('..')
shutil.rmtree(server_dir, onerror=onerror)
# Access local RAM in manner it would be accessed in the server.
self.assertEqual(RAM._get_instance().get_total_allocators(), 1)
self.assertTrue(RAM._get_instance().get_allocator_proxy(0)
is RAM.list_allocators()[0])
def test_remote(self):
logging.debug('')
logging.debug('test_remote')
# Start remote server.
server_dir = 'Factory'
if os.path.exists(server_dir):
shutil.rmtree(server_dir, onerror=onerror)
os.mkdir(server_dir)
os.chdir(server_dir)
try:
server, server_cfg = start_server()
cfg = read_server_config(server_cfg)
factory = None
try:
factory = connect(cfg['address'],
cfg['port'],
pubkey=cfg['key'])
prefix = RAM._make_prefix(factory.host)
remote = '%s_LocalHost' % prefix
# Show no remotes currently in RAM.
allocator_names = \
[allocator.name for allocator in RAM.list_allocators()]
logging.debug('%s', allocator_names)
self.assertFalse(remote in allocator_names)
# Add remote server's allocator.
RAM.add_remotes(factory)
allocator_names = \
[allocator.name for allocator in RAM.list_allocators()]
logging.debug('%s', allocator_names)
self.assertTrue(remote in allocator_names)
self.assertFalse(
RAM.get_allocator(remote) is RAM.list_allocators()[0])
self.assertTrue(
RAM.get_allocator(remote) is RAM.list_allocators()[1])
# Max servers.
max_servers = RAM.max_servers(dict(allocator=remote))
self.assertTrue(max_servers >= 0) # Avoid host load issues.
remote_alloc = RAM.get_allocator(remote)
max_servers, info = \
remote_alloc.max_servers(dict(localhost=True))
self.assertEqual(max_servers, 0)
self.assertEqual(info, dict(localhost='requested local host'))
max_servers, info = \
remote_alloc.max_servers(dict(allocator='LocalHost'))
self.assertEqual(max_servers, 0)
self.assertEqual(info, dict(allocator='wrong allocator'))
estimate, info = \
remote_alloc.time_estimate(dict(allocator='LocalHost'))
self.assertEqual(estimate, -2)
self.assertEqual(info, dict(allocator='wrong allocator'))
# Allocate, release.
remote_server, info = RAM.allocate(dict(allocator=remote))
RAM.release(remote_server)
# Remove remote allocators.
allocator_names = \
[allocator.name for allocator in RAM.list_allocators()]
for name in allocator_names:
if name.startswith(prefix):
RAM.remove_allocator(name)
allocator_names = \
[allocator.name for allocator in RAM.list_allocators()]
logging.debug('%s', allocator_names)
self.assertFalse(remote in allocator_names)
finally:
if factory is not None:
factory.cleanup()
server.terminate(timeout=10)
finally:
os.chdir('..')
shutil.rmtree(server_dir, onerror=onerror)
# Access local RAM in manner it would be accessed in the server.
self.assertEqual(RAM._get_instance().get_total_allocators(), 1)
self.assertTrue(RAM._get_instance().get_allocator_proxy(0) is
RAM.list_allocators()[0])
def _execute_remote(self):
"""
Allocate a server based on required resources, send inputs,
run command, and retrieve results.
"""
# Allocate server.
self._server, server_info = RAM.allocate(self.resources)
if self._server is None:
self.raise_exception('Server allocation failed :-(', RuntimeError)
return_code = -88888888
error_msg = ''
try:
# Create resource description for command.
rdesc = self.resources.copy()
rdesc['job_name'] = self.get_pathname()
rdesc['remote_command'] = self.command[0]
if len(self.command) > 1:
rdesc['args'] = self.command[1:]
if self.env_vars:
rdesc['job_environment'] = self.env_vars
if self.stdin:
rdesc['input_path'] = self.stdin
if self.stdout:
rdesc['output_path'] = self.stdout
if self.stderr:
if self.stderr == self.STDOUT:
rdesc['join_files'] = True
else:
rdesc['error_path'] = self.stderr
if self.timeout:
rdesc['hard_run_duration_limit'] = self.timeout
# Send inputs.
patterns = []
textfiles = []
for metadata in self.external_files:
if metadata.get('input', False):
patterns.append(metadata.path)
if not metadata.binary:
textfiles.append(metadata.path)
if patterns:
self._send_inputs(patterns, textfiles)
else:
self._logger.debug('No input metadata paths')
# Run command.
self._logger.info('executing %s...', self.command)
start_time = time.time()
return_code, error_msg = \
self._server.execute_command(rdesc)
et = time.time() - start_time
if et >= 60: #pragma no cover
self._logger.info('elapsed time: %.1f sec.', et)
# Retrieve results.
patterns = []
textfiles = []
for metadata in self.external_files:
if metadata.get('output', False):
patterns.append(metadata.path)
if not metadata.binary:
textfiles.append(metadata.path)
if patterns:
self._retrieve_results(patterns, textfiles)
else:
self._logger.debug('No output metadata paths')
finally:
RAM.release(self._server)
self._server = None
return (return_code, error_msg)
def test_errors(self):
logging.debug('')
logging.debug('test_errors')
logging.debug('allocate server')
server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
try:
logging.debug('execute bad command')
code = "server.execute_command(dict(remote_command='no-such-command'))"
if sys.platform == 'win32':
msg = "WindowsError: [Error 2] The system cannot find the file specified"
else:
msg = "OSError: [Errno 2] No such file or directory"
try:
server.execute_command(dict(remote_command='no-such-command'))
except protocol.RemoteError as exc:
exc_msg = str(exc)
if msg not in exc_msg:
self.fail('%s not in %s' % (msg, exc_msg))
else:
self.fail('Expecting protocol.RemoteError')
logging.debug('open bad file')
msg = "Can\\'t open \\'../../illegal-access\\', not within root"
msg = 'RuntimeError: %s' % msg
try:
server.open('../../illegal-access', 'r')
except protocol.RemoteError as exc:
exc_msg = str(exc)
if msg not in exc_msg:
self.fail('%s not in %s' % (msg, exc_msg))
else:
self.fail('Expecting protocol.RemoteError')
logging.debug('open missing file')
msg = "[Errno 2] No such file or directory: \\'no-such-file\\'"
msg = 'IOError: %s' % msg
try:
server.open('no-such-file', 'r')
except protocol.RemoteError as exc:
exc_msg = str(exc)
if msg not in exc_msg:
self.fail('%s not in %s' % (msg, exc_msg))
else:
self.fail('Expecting protocol.RemoteError')
finally:
logging.debug('release')
RAM.release(server)
# Test for exited or never started server.
logging.debug('dead server')
self.proc.terminate()
self.proc = None
time.sleep(2)
hostname = socket.gethostname()
if sys.platform == 'win32': # Server doesn't clean up.
root = protocol._server_root(hostname)
mapped_root = os.path.join(_DMZ_ROOT, protocol._map_dir(root))
for name in glob.glob('%s*' % mapped_root):
os.remove(name)
code = 'NAS_Allocator(dmz_host=hostname, server_host=hostname)'
assert_raises(self, code, globals(), locals(), RuntimeError,
"NAS_Allocator: can't connect: server root 'RJE-%s='"
" on '%s' not found" % (hostname, hostname))
# Test for missing heartbeat.
logging.debug('no heartbeat')
with open(os.path.join(_DMZ_ROOT, 'RJE-%s=' % hostname), 'w') as out:
out.write('empty\n')
try:
NAS_Allocator(dmz_host=hostname, server_host=hostname)
except RuntimeError as exc:
msg = "IOError: [Errno 2] No such file or directory:" \
" 'RJE-%s=heartbeat'\n" % hostname
logging.debug(str(exc))
self.assertTrue(str(exc).endswith(msg))
else:
self.fail('Expected RuntimeError')
# Test for stale heartbeat.
logging.debug('stale heartbeat')
protocol.server_heartbeat(hostname, 1, logging.getLogger())
time.sleep(5)
assert_raises(self, code, globals(), locals(), RuntimeError,
"NAS_Allocator: can't connect: server heartbeat"
" hasn't been updated in 0:00:0")
def test_extcode(self):
logging.debug('')
logging.debug('test_extcode')
# Run a fake job in style of ExternalCode component.
logging.debug('allocate server')
server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
try:
with open('junk.dat', 'w') as out:
out.write('just some junk')
filename = 'inputs.zip'
logging.debug('pack inputs')
pfiles, pbytes = pack_zipfile(('junk.dat',), filename,
logging.getLogger())
os.remove('junk.dat')
logging.debug('transfer inputs')
filexfer(None, filename, server, filename, 'b')
logging.debug('unpack inputs')
ufiles, ubytes = server.unpack_zipfile(filename)
logging.debug('remove inputs')
os.remove(filename)
server.remove(filename)
logging.debug('execute command')
if sys.platform == 'win32':
remote_command = 'cmd'
args = ('/c', 'echo', 'Hello', 'World!')
else:
remote_command = 'echo'
args = ('Hello', 'World!')
return_code, error_msg = \
server.execute_command(dict(job_name='Testing',
remote_command=remote_command,
args=args,
output_path='echo.out'))
logging.debug('pack outputs')
filename = 'outputs.zip'
pfiles, pbytes = server.pack_zipfile(('echo.out', 'junk.dat'),
filename)
logging.debug('transfer outputs')
filexfer(server, filename, None, filename, 'b')
logging.debug('unpack outputs')
ufiles, ubytes = unpack_zipfile(filename)
logging.debug('remove outputs')
os.remove(filename)
server.remove(filename)
finally:
logging.debug('release')
RAM.release(server)
self.assertEqual(return_code, 0)
self.assertEqual(error_msg, '')
self.assertTrue(os.path.exists('echo.out'))
with open('echo.out', 'rU') as out:
data = out.read()
os.remove('echo.out')
self.assertEqual(data, 'Hello World!\n')
self.assertTrue(os.path.exists('junk.dat'))
with open('junk.dat', 'rU') as out:
data = out.read()
os.remove('junk.dat')
self.assertEqual(data, 'just some junk')
def _execute_remote(self):
"""
Allocate a server based on required resources, send inputs,
run command, and retrieve results.
"""
rdesc = self.resources.copy()
# Allocate server.
self._server, server_info = RAM.allocate(rdesc)
if self._server is None:
self.raise_exception('Server allocation failed :-(', RuntimeError)
if self._logger.level == logging.NOTSET:
# By default avoid lots of protocol messages.
self._server.set_log_level(logging.DEBUG)
else:
self._server.set_log_level(self._logger.level)
return_code = -88888888
error_msg = ''
try:
# Create resource description for command.
rdesc['job_name'] = self.get_pathname()
rdesc['remote_command'] = self.command[0]
if len(self.command) > 1:
rdesc['args'] = self.command[1:]
if self.env_vars:
rdesc['job_environment'] = self.env_vars
if not self.stdin:
self.raise_exception('Remote execution requires stdin of'
' DEV_NULL or filename, got %r'
% self.stdin, ValueError)
if self.stdin != self.DEV_NULL:
rdesc['input_path'] = self.stdin
if self.stdout:
rdesc['output_path'] = self.stdout
else:
rdesc['output_path'] = '%s.stdout' % self.command[0]
if self.stderr:
if self.stderr == self.STDOUT:
rdesc['join_files'] = True
else:
rdesc['error_path'] = self.stderr
else:
rdesc['error_path'] = '%s.stderr' % self.command[0]
if self.timeout:
if 'resource_limits' in rdesc:
limits = rdesc['resource_limits'].copy()
else:
limits = {}
limits['wallclock_time'] = self.timeout
rdesc['resource_limits'] = limits
# Send inputs.
patterns = []
textfiles = []
for metadata in self.external_files:
if metadata.get('input', False):
patterns.append(metadata.path)
if not metadata.binary:
textfiles.append(metadata.path)
for pathname, obj in self.items(iotype='in', recurse=True):
if isinstance(obj, FileRef):
local_path = self.get_metadata(pathname, 'local_path')
if local_path:
patterns.append(local_path)
if not obj.binary:
textfiles.append(local_path)
if self.stdin and self.stdin != self.DEV_NULL:
patterns.append(self.stdin)
textfiles.append(self.stdin)
if patterns:
self._send_inputs(patterns, textfiles)
else:
self._logger.debug('No input files')
# Run command.
self._logger.info('executing %s...', self.command)
start_time = time.time()
return_code, error_msg = \
self._server.execute_command(rdesc)
et = time.time() - start_time
if et >= 60: #pragma no cover
self._logger.info('elapsed time: %.1f sec.', et)
# Retrieve results.
patterns = []
textfiles = []
for metadata in self.external_files:
if metadata.get('output', False):
patterns.append(metadata.path)
if not metadata.binary:
textfiles.append(metadata.path)
for pathname, obj in self.items(iotype='out', recurse=True):
if isinstance(obj, FileRef):
patterns.append(obj.path)
if not obj.binary:
textfiles.append(obj.path)
patterns.append(rdesc['output_path'])
textfiles.append(rdesc['output_path'])
if self.stderr != self.STDOUT:
patterns.append(rdesc['error_path'])
textfiles.append(rdesc['error_path'])
self._retrieve_results(patterns, textfiles)
# Echo stdout if not redirected.
if not self.stdout:
name = rdesc['output_path']
if os.path.exists(name):
with open(name, 'rU') as inp:
sys.stdout.write(inp.read())
os.remove(name)
else:
sys.stdout.write('\n[No stdout available]\n')
# Echo stderr if not redirected.
if not self.stderr:
name = rdesc['error_path']
if os.path.exists(name):
with open(name, 'rU') as inp:
sys.stderr.write(inp.read())
os.remove(name)
else:
sys.stdout.write('\n[No stderr available]\n')
finally:
RAM.release(self._server)
self._server = None
return (return_code, error_msg)
def _execute_remote(self):
"""
Allocate a server based on required resources, send inputs,
run command, and retrieve results.
"""
# Allocate server.
self._server, server_info = RAM.allocate(self.resources)
if self._server is None:
self.raise_exception('Server allocation failed :-(', RuntimeError)
return_code = -88888888
error_msg = ''
try:
# Create resource description for command.
rdesc = self.resources.copy()
rdesc['job_name'] = self.get_pathname()
rdesc['remote_command'] = self.command[0]
if len(self.command) > 1:
rdesc['args'] = self.command[1:]
if self.env_vars:
rdesc['job_environment'] = self.env_vars
if self.stdin:
rdesc['input_path'] = self.stdin
if self.stdout:
rdesc['output_path'] = self.stdout
if self.stderr:
if self.stderr == self.STDOUT:
rdesc['join_files'] = True
else:
rdesc['error_path'] = self.stderr
if self.timeout:
rdesc['hard_run_duration_limit'] = self.timeout
# Send inputs.
patterns = []
textfiles = []
for metadata in self.external_files:
if metadata.get('input', False):
patterns.append(metadata.path)
if not metadata.binary:
textfiles.append(metadata.path)
if patterns:
self._send_inputs(patterns, textfiles)
else:
self._logger.debug('No input metadata paths')
# Run command.
self._logger.info('executing %s...', self.command)
start_time = time.time()
return_code, error_msg = \
self._server.execute_command(rdesc)
et = time.time() - start_time
if et >= 60: #pragma no cover
self._logger.info('elapsed time: %.1f sec.', et)
# Retrieve results.
patterns = []
textfiles = []
for metadata in self.external_files:
if metadata.get('output', False):
patterns.append(metadata.path)
if not metadata.binary:
textfiles.append(metadata.path)
if patterns:
self._retrieve_results(patterns, textfiles)
else:
self._logger.debug('No output metadata paths')
finally:
RAM.release(self._server)
self._server = None
return (return_code, error_msg)
