def _retrieve_results(self, patterns, textfiles):
""" Retrieves result files matching `patterns`. """
self._logger.info('retrieving results...')
start_time = time.time()
filename = 'outputs.zip'
pfiles, pbytes = self._server.pack_zipfile(patterns, filename)
filexfer(self._server, filename, None, filename, 'b')
# Valid, but empty, file causes unpack_zipfile() problems.
try:
if os.path.getsize(filename) > 0:
ufiles, ubytes = unpack_zipfile(filename, logger=self._logger,
textfiles=textfiles)
else:
ufiles, ubytes = 0, 0
finally:
os.remove(filename)
self._server.remove(filename)
# Difficult to force file transfer error.
if ufiles != pfiles or ubytes != pbytes: #pragma no cover
msg = 'Results xfer error: %d:%d vs. %d:%d' \
% (ufiles, ubytes, pfiles, pbytes)
self.raise_exception(msg, RuntimeError)
et = time.time() - start_time
if et >= 60: #pragma no cover
self._logger.info('elapsed time: %f sec.', et)
def _remote_load_model(self, server):
""" Load model into remote server. """
egg_file = self._server_info[server].get('egg_file', None)
if egg_file is None or egg_file is not self._egg_file:
# Only transfer if changed.
try:
filexfer(None, self._egg_file,
self._servers[server], self._egg_file, 'b')
# Difficult to force model file transfer error.
except Exception as exc: #pragma nocover
self._logger.error('server %r filexfer of %r failed: %r',
server, self._egg_file, exc)
self._top_levels[server] = None
self._exceptions[server] = TracedError(exc, traceback.format_exc())
return
else:
self._server_info[server]['egg_file'] = self._egg_file
try:
tlo = self._servers[server].load_model(self._egg_file)
# Difficult to force load error.
except Exception as exc: #pragma nocover
self._logger.error('server.load_model of %r failed: %r',
self._egg_file, exc)
self._top_levels[server] = None
self._exceptions[server] = TracedError(exc, traceback.format_exc())
else:
self._top_levels[server] = tlo
def _remote_load_model(self, server):
""" Load model into remote server. """
egg_file = server.info.get('egg_file', None)
if egg_file is None or egg_file is not self._egg_file:
# Only transfer if changed.
try:
filexfer(None, self._egg_file, server.server, self._egg_file,
'b')
# Difficult to force model file transfer error.
except Exception as exc: # pragma nocover
self._logger.error('server %r filexfer of %r failed: %r',
server.name, self._egg_file, exc)
server.top = None
server.exception = sys.exc_info()
return
else:
server.info['egg_file'] = self._egg_file
try:
tlo = server.server.load_model(self._egg_file)
# Difficult to force load error.
except Exception as exc: # pragma nocover
self._logger.error('server.load_model of %r failed: %r',
self._egg_file, exc)
server.top = None
server.exception = sys.exc_info()
else:
server.top = tlo
def test_case_eval(self):
logging.debug("")
logging.debug("test_case_eval")
# Run a fake job in style of CaseIteratorDriver.
logging.debug("allocate server")
server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
echo = set_as_top(Echo(1))
egg_info = echo.save_to_egg("EchoTest", "1", need_requirements=False)
egg_filename = egg_info[0]
try:
logging.debug("transfer egg")
filexfer(None, egg_filename, server, egg_filename, "b")
logging.debug("load model")
tlo = server.load_model(egg_filename)
logging.debug("set input")
tlo.set("inp_0", 42)
logging.debug("run")
tlo.run()
logging.debug("get output")
output = tlo.get("out_0")
self.assertEqual(output, 42)
finally:
os.remove(egg_filename)
logging.debug("release")
RAM.release(server)
def _remote_load_model(self, server):
""" Load model into remote server. """
egg_file = server.info.get('egg_file', None)
if egg_file is None or egg_file is not self._egg_file:
# Only transfer if changed.
try:
filexfer(None, self._egg_file,
server.server, self._egg_file, 'b')
# Difficult to force model file transfer error.
except Exception as exc: # pragma nocover
self._logger.error('server %r filexfer of %r failed: %r',
server.name, self._egg_file, exc)
server.top = None
server.exception = sys.exc_info()
return
else:
server.info['egg_file'] = self._egg_file
try:
tlo = server.server.load_model(self._egg_file)
# Difficult to force load error.
except Exception as exc: # pragma nocover
self._logger.error('server.load_model of %r failed: %r',
self._egg_file, exc)
server.top = None
server.exception = sys.exc_info()
else:
server.top = tlo
def test_case_eval(self):
logging.debug('')
logging.debug('test_case_eval')
# Run a fake job in style of CaseIteratorDriver.
logging.debug('allocate server')
server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
echo = set_as_top(Echo(1))
egg_info = echo.save_to_egg('EchoTest', '1', need_requirements=False)
egg_filename = egg_info[0]
try:
logging.debug('transfer egg')
filexfer(None, egg_filename, server, egg_filename, 'b')
logging.debug('load model')
tlo = server.load_model(egg_filename)
logging.debug('set input')
tlo.set('inp_0', 42)
logging.debug('run')
tlo.run()
logging.debug('get output')
output = tlo.get('out_0')
self.assertEqual(output, 42)
finally:
os.remove(egg_filename)
logging.debug('release')
RAM.release(server)
def test_case_eval(self):
logging.debug('')
logging.debug('test_case_eval')
# Run a fake job in style of CaseIteratorDriver.
logging.debug('allocate server')
server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
echo = set_as_top(Echo(1))
egg_info = echo.save_to_egg('EchoTest', '1', need_requirements=False)
egg_filename = egg_info[0]
try:
logging.debug('transfer egg')
filexfer(None, egg_filename, server, egg_filename, 'b')
logging.debug('load model')
tlo = server.load_model(egg_filename)
logging.debug('set input')
tlo.set('inp_0', 42)
logging.debug('run')
tlo.run()
logging.debug('get output')
output = tlo.get('out_0')
self.assertEqual(output, 42)
finally:
os.remove(egg_filename)
logging.debug('release')
RAM.release(server)
def putfile(self, filename, binary):
"""
Copy `filename` to remote file server.
filename: string
Name of file to send.
binary: boolean
If True, file in binary.
"""
fd, path = tempfile.mkstemp()
try:
os.close(fd)
mode = "b" if binary else ""
filexfer(self._delegate, filename, None, path, mode, False)
self._conn.send_file(path, filename)
finally:
try:
os.remove(path)
except Exception as exc: # pragma no cover
self._logger.warning("Can't remove temporary file: %s", exc)
def putfile(self, filename, binary):
"""
Copy `filename` to remote file server.
filename: string
Name of file to send.
binary: boolean
If True, file in binary.
"""
fd, path = tempfile.mkstemp()
try:
os.close(fd)
mode = 'b' if binary else ''
filexfer(self._delegate, filename, None, path, mode, False)
self._conn.send_file(path, filename)
finally:
try:
os.remove(path)
except Exception as exc: # pragma no cover
self._logger.warning("Can't remove temporary file: %s", exc)
def _retrieve_results(self, patterns):
""" Retrieves result files matching `patterns`. """
self._logger.info('retrieving results...')
start_time = time.time()
filename = 'outputs.zip'
pfiles, pbytes = self._server.pack_zipfile(tuple(patterns), filename)
try:
filexfer(self._server, filename, None, filename, 'b')
ufiles, ubytes = unpack_zipfile(filename, self._logger)
finally:
os.remove(filename)
# Difficult to force file transfer error.
if ufiles != pfiles or ubytes != pbytes: #pragma no cover
msg = 'Results xfer error: %d:%d vs. %d:%d' \
% (ufiles, ubytes, pfiles, pbytes)
self.raise_exception(msg, RuntimeError)
et = time.time() - start_time
if et >= 60: #pragma no cover
self._logger.info('elapsed time: %f sec.', et)
def _send_inputs(self, patterns):
""" Sends input files matching `patterns`. """
self._logger.info('sending inputs...')
start_time = time.time()
filename = 'inputs.zip'
pfiles, pbytes = pack_zipfile(patterns, filename, self._logger)
try:
filexfer(None, filename, self._server, filename, 'b')
ufiles, ubytes = self._server.unpack_zipfile(filename)
finally:
os.remove(filename)
# Difficult to force file transfer error.
if ufiles != pfiles or ubytes != pbytes: #pragma no cover
msg = 'Inputs xfer error: %d:%d vs. %d:%d' \
% (ufiles, ubytes, pfiles, pbytes)
self.raise_exception(msg, RuntimeError)
et = time.time() - start_time
if et >= 60: #pragma no cover
self._logger.info('elapsed time: %f sec.', et)
def getfile(self, filename, binary):
"""
Copy `filename` from remote file server.
filename: string
Name of file to receive.
binary: boolean
If True, file in binary.
"""
fd, path = tempfile.mkstemp()
try:
os.close(fd)
self._conn.recv_file(filename, path)
mode = 'b' if binary else ''
filexfer(None, path, self._delegate, filename, mode)
self._conn.remove_files((filename,))
finally:
try:
os.remove(path)
except Exception as exc: # pragma no cover
self._logger.warning("Can't remove temporary file: %s", exc)
def _send_inputs(self, patterns, textfiles):
""" Sends input files matching `patterns`. """
self._logger.info('sending inputs...')
start_time = time.time()
filename = 'inputs.zip'
pfiles, pbytes = pack_zipfile(patterns, filename, self._logger)
try:
filexfer(None, filename, self._server, filename, 'b')
ufiles, ubytes = self._server.unpack_zipfile(filename,
textfiles=textfiles)
finally:
os.remove(filename)
# Difficult to force file transfer error.
if ufiles != pfiles or ubytes != pbytes: #pragma no cover
msg = 'Inputs xfer error: %d:%d vs. %d:%d' \
% (ufiles, ubytes, pfiles, pbytes)
self.raise_exception(msg, RuntimeError)
et = time.time() - start_time
if et >= 60: #pragma no cover
self._logger.info('elapsed time: %f sec.', et)
def test_extcode(self):
logging.debug("")
logging.debug("test_extcode")
# Run a fake job in style of ExternalCode component.
logging.debug("allocate server")
server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
try:
with open("junk.dat", "w") as out:
out.write("just some junk")
filename = "inputs.zip"
logging.debug("pack inputs")
pfiles, pbytes = pack_zipfile(("junk.dat",), filename, logging.getLogger())
os.remove("junk.dat")
logging.debug("transfer inputs")
filexfer(None, filename, server, filename, "b")
logging.debug("unpack inputs")
ufiles, ubytes = server.unpack_zipfile(filename)
logging.debug("remove inputs")
os.remove(filename)
server.remove(filename)
logging.debug("execute command")
if sys.platform == "win32":
remote_command = "cmd"
args = ("/c", "echo", "Hello", "World!")
else:
remote_command = "echo"
args = ("Hello", "World!")
return_code, error_msg = server.execute_command(
dict(job_name="Testing", remote_command=remote_command, args=args, output_path="echo.out")
)
logging.debug("pack outputs")
filename = "outputs.zip"
pfiles, pbytes = server.pack_zipfile(("echo.out", "junk.dat"), filename)
logging.debug("transfer outputs")
filexfer(server, filename, None, filename, "b")
logging.debug("unpack outputs")
ufiles, ubytes = unpack_zipfile(filename)
logging.debug("remove outputs")
os.remove(filename)
server.remove(filename)
finally:
logging.debug("release")
RAM.release(server)
self.assertEqual(return_code, 0)
self.assertEqual(error_msg, "")
self.assertTrue(os.path.exists("echo.out"))
with open("echo.out", "rU") as out:
data = out.read()
os.remove("echo.out")
self.assertEqual(data, "Hello World!\n")
self.assertTrue(os.path.exists("junk.dat"))
with open("junk.dat", "rU") as out:
data = out.read()
os.remove("junk.dat")
self.assertEqual(data, "just some junk")
def test_extcode(self):
logging.debug('')
logging.debug('test_extcode')
# Run a fake job in style of ExternalCode component.
logging.debug('allocate server')
server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
try:
with open('junk.dat', 'w') as out:
out.write('just some junk')
filename = 'inputs.zip'
logging.debug('pack inputs')
pfiles, pbytes = pack_zipfile(('junk.dat',), filename,
logging.getLogger())
os.remove('junk.dat')
logging.debug('transfer inputs')
filexfer(None, filename, server, filename, 'b')
logging.debug('unpack inputs')
ufiles, ubytes = server.unpack_zipfile(filename)
logging.debug('remove inputs')
os.remove(filename)
server.remove(filename)
logging.debug('execute command')
if sys.platform == 'win32':
remote_command = 'cmd'
args = ('/c', 'echo', 'Hello', 'World!')
else:
remote_command = 'echo'
args = ('Hello', 'World!')
return_code, error_msg = \
server.execute_command(dict(job_name='Testing',
remote_command=remote_command,
args=args,
output_path='echo.out'))
logging.debug('pack outputs')
filename = 'outputs.zip'
pfiles, pbytes = server.pack_zipfile(('echo.out', 'junk.dat'),
filename)
logging.debug('transfer outputs')
filexfer(server, filename, None, filename, 'b')
logging.debug('unpack outputs')
ufiles, ubytes = unpack_zipfile(filename)
logging.debug('remove outputs')
os.remove(filename)
server.remove(filename)
finally:
logging.debug('release')
RAM.release(server)
self.assertEqual(return_code, 0)
self.assertEqual(error_msg, '')
self.assertTrue(os.path.exists('echo.out'))
with open('echo.out', 'rU') as out:
data = out.read()
os.remove('echo.out')
self.assertEqual(data, 'Hello World!\n')
self.assertTrue(os.path.exists('junk.dat'))
with open('junk.dat', 'rU') as out:
data = out.read()
os.remove('junk.dat')
self.assertEqual(data, 'just some junk')
def test_extcode(self):
logging.debug('')
logging.debug('test_extcode')
# Run a fake job in style of ExternalCode component.
logging.debug('allocate server')
server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
try:
with open('junk.dat', 'w') as out:
out.write('just some junk')
filename = 'inputs.zip'
logging.debug('pack inputs')
pfiles, pbytes = pack_zipfile(('junk.dat',), filename,
logging.getLogger())
os.remove('junk.dat')
logging.debug('transfer inputs')
filexfer(None, filename, server, filename, 'b')
logging.debug('unpack inputs')
ufiles, ubytes = server.unpack_zipfile(filename)
logging.debug('remove inputs')
os.remove(filename)
server.remove(filename)
logging.debug('execute command')
if sys.platform == 'win32':
remote_command = 'cmd'
args = ('/c', 'echo', 'Hello', 'World!')
else:
remote_command = 'echo'
args = ('Hello', 'World!')
return_code, error_msg = \
server.execute_command(dict(job_name='Testing',
remote_command=remote_command,
args=args,
output_path='echo.out'))
logging.debug('pack outputs')
filename = 'outputs.zip'
pfiles, pbytes = server.pack_zipfile(('echo.out', 'junk.dat'),
filename)
logging.debug('transfer outputs')
filexfer(server, filename, None, filename, 'b')
logging.debug('unpack outputs')
ufiles, ubytes = unpack_zipfile(filename)
logging.debug('remove outputs')
os.remove(filename)
server.remove(filename)
finally:
logging.debug('release')
RAM.release(server)
self.assertEqual(return_code, 0)
self.assertEqual(error_msg, '')
self.assertTrue(os.path.exists('echo.out'))
with open('echo.out', 'rU') as out:
data = out.read()
os.remove('echo.out')
self.assertEqual(data, 'Hello World!\n')
self.assertTrue(os.path.exists('junk.dat'))
with open('junk.dat', 'rU') as out:
data = out.read()
os.remove('junk.dat')
self.assertEqual(data, 'just some junk')