def _retrieve_results(self, patterns, textfiles):
""" Retrieves result files matching `patterns`. """
self._logger.info('retrieving results...')
start_time = time.time()
filename = 'outputs.zip'
pfiles, pbytes = self._server.pack_zipfile(patterns, filename)
filexfer(self._server, filename, None, filename, 'b')
# Valid, but empty, file causes unpack_zipfile() problems.
try:
if os.path.getsize(filename) > 0:
ufiles, ubytes = unpack_zipfile(filename, logger=self._logger,
textfiles=textfiles)
else:
ufiles, ubytes = 0, 0
finally:
os.remove(filename)
self._server.remove(filename)
# Difficult to force file transfer error.
if ufiles != pfiles or ubytes != pbytes: #pragma no cover
msg = 'Results xfer error: %d:%d vs. %d:%d' \
% (ufiles, ubytes, pfiles, pbytes)
self.raise_exception(msg, RuntimeError)
et = time.time() - start_time
if et >= 60: #pragma no cover
self._logger.info('elapsed time: %f sec.', et)
def unpack_zipfile(self, filename):
"""
Unpack ZipFile `filename` if `filename` is legal.
filename: string
Name of ZipFile to unpack.
"""
self._logger.debug('unpack_zipfile %r', filename)
self._check_path(filename, 'unpack_zipfile')
return unpack_zipfile(filename, self._logger)
def unpack_zipfile(self, filename, textfiles=None):
"""
Unpack ZipFile `filename` if `filename` is legal.
filename: string
Name of ZipFile to unpack.
textfiles: list
List of :mod:`fnmatch` style patterns specifying which upnapcked
files are text files possibly needing newline translation. If not
supplied, the first 4KB of each is scanned for a zero byte. If not
found then the file is assumed to be a text file.
"""
self._logger.debug('unpack_zipfile %r', filename)
self._check_path(filename, 'unpack_zipfile')
return unpack_zipfile(filename, self._logger, textfiles)
def unpack_zipfile(self, filename, textfiles=None):
"""
Unpack ZipFile `filename` if `filename` is legal.
filename: string
Name of ZipFile to unpack.
textfiles: list
List of :mod:`fnmatch` style patterns specifying which unpacked
files are text files possibly needing newline translation. If not
supplied, the first 4KB of each is scanned for a zero byte. If none
is found, then the file is assumed to be a text file.
"""
self._logger.debug('unpack_zipfile %r', filename)
self._check_path(filename, 'unpack_zipfile')
return unpack_zipfile(filename, self._logger, textfiles)
def _retrieve_results(self, patterns):
""" Retrieves result files matching `patterns`. """
self._logger.info('retrieving results...')
start_time = time.time()
filename = 'outputs.zip'
pfiles, pbytes = self._server.pack_zipfile(tuple(patterns), filename)
try:
filexfer(self._server, filename, None, filename, 'b')
ufiles, ubytes = unpack_zipfile(filename, self._logger)
finally:
os.remove(filename)
# Difficult to force file transfer error.
if ufiles != pfiles or ubytes != pbytes: #pragma no cover
msg = 'Results xfer error: %d:%d vs. %d:%d' \
% (ufiles, ubytes, pfiles, pbytes)
self.raise_exception(msg, RuntimeError)
et = time.time() - start_time
if et >= 60: #pragma no cover
self._logger.info('elapsed time: %f sec.', et)
def test_extcode(self):
logging.debug("")
logging.debug("test_extcode")
# Run a fake job in style of ExternalCode component.
logging.debug("allocate server")
server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
try:
with open("junk.dat", "w") as out:
out.write("just some junk")
filename = "inputs.zip"
logging.debug("pack inputs")
pfiles, pbytes = pack_zipfile(("junk.dat",), filename, logging.getLogger())
os.remove("junk.dat")
logging.debug("transfer inputs")
filexfer(None, filename, server, filename, "b")
logging.debug("unpack inputs")
ufiles, ubytes = server.unpack_zipfile(filename)
logging.debug("remove inputs")
os.remove(filename)
server.remove(filename)
logging.debug("execute command")
if sys.platform == "win32":
remote_command = "cmd"
args = ("/c", "echo", "Hello", "World!")
else:
remote_command = "echo"
args = ("Hello", "World!")
return_code, error_msg = server.execute_command(
dict(job_name="Testing", remote_command=remote_command, args=args, output_path="echo.out")
)
logging.debug("pack outputs")
filename = "outputs.zip"
pfiles, pbytes = server.pack_zipfile(("echo.out", "junk.dat"), filename)
logging.debug("transfer outputs")
filexfer(server, filename, None, filename, "b")
logging.debug("unpack outputs")
ufiles, ubytes = unpack_zipfile(filename)
logging.debug("remove outputs")
os.remove(filename)
server.remove(filename)
finally:
logging.debug("release")
RAM.release(server)
self.assertEqual(return_code, 0)
self.assertEqual(error_msg, "")
self.assertTrue(os.path.exists("echo.out"))
with open("echo.out", "rU") as out:
data = out.read()
os.remove("echo.out")
self.assertEqual(data, "Hello World!\n")
self.assertTrue(os.path.exists("junk.dat"))
with open("junk.dat", "rU") as out:
data = out.read()
os.remove("junk.dat")
self.assertEqual(data, "just some junk")
def test_extcode(self):
logging.debug('')
logging.debug('test_extcode')
# Run a fake job in style of ExternalCode component.
logging.debug('allocate server')
server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
try:
with open('junk.dat', 'w') as out:
out.write('just some junk')
filename = 'inputs.zip'
logging.debug('pack inputs')
pfiles, pbytes = pack_zipfile(('junk.dat',), filename,
logging.getLogger())
os.remove('junk.dat')
logging.debug('transfer inputs')
filexfer(None, filename, server, filename, 'b')
logging.debug('unpack inputs')
ufiles, ubytes = server.unpack_zipfile(filename)
logging.debug('remove inputs')
os.remove(filename)
server.remove(filename)
logging.debug('execute command')
if sys.platform == 'win32':
remote_command = 'cmd'
args = ('/c', 'echo', 'Hello', 'World!')
else:
remote_command = 'echo'
args = ('Hello', 'World!')
return_code, error_msg = \
server.execute_command(dict(job_name='Testing',
remote_command=remote_command,
args=args,
output_path='echo.out'))
logging.debug('pack outputs')
filename = 'outputs.zip'
pfiles, pbytes = server.pack_zipfile(('echo.out', 'junk.dat'),
filename)
logging.debug('transfer outputs')
filexfer(server, filename, None, filename, 'b')
logging.debug('unpack outputs')
ufiles, ubytes = unpack_zipfile(filename)
logging.debug('remove outputs')
os.remove(filename)
server.remove(filename)
finally:
logging.debug('release')
RAM.release(server)
self.assertEqual(return_code, 0)
self.assertEqual(error_msg, '')
self.assertTrue(os.path.exists('echo.out'))
with open('echo.out', 'rU') as out:
data = out.read()
os.remove('echo.out')
self.assertEqual(data, 'Hello World!\n')
self.assertTrue(os.path.exists('junk.dat'))
with open('junk.dat', 'rU') as out:
data = out.read()
os.remove('junk.dat')
self.assertEqual(data, 'just some junk')
def test_extcode(self):
logging.debug('')
logging.debug('test_extcode')
# Run a fake job in style of ExternalCode component.
logging.debug('allocate server')
server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
try:
with open('junk.dat', 'w') as out:
out.write('just some junk')
filename = 'inputs.zip'
logging.debug('pack inputs')
pfiles, pbytes = pack_zipfile(('junk.dat',), filename,
logging.getLogger())
os.remove('junk.dat')
logging.debug('transfer inputs')
filexfer(None, filename, server, filename, 'b')
logging.debug('unpack inputs')
ufiles, ubytes = server.unpack_zipfile(filename)
logging.debug('remove inputs')
os.remove(filename)
server.remove(filename)
logging.debug('execute command')
if sys.platform == 'win32':
remote_command = 'cmd'
args = ('/c', 'echo', 'Hello', 'World!')
else:
remote_command = 'echo'
args = ('Hello', 'World!')
return_code, error_msg = \
server.execute_command(dict(job_name='Testing',
remote_command=remote_command,
args=args,
output_path='echo.out'))
logging.debug('pack outputs')
filename = 'outputs.zip'
pfiles, pbytes = server.pack_zipfile(('echo.out', 'junk.dat'),
filename)
logging.debug('transfer outputs')
filexfer(server, filename, None, filename, 'b')
logging.debug('unpack outputs')
ufiles, ubytes = unpack_zipfile(filename)
logging.debug('remove outputs')
os.remove(filename)
server.remove(filename)
finally:
logging.debug('release')
RAM.release(server)
self.assertEqual(return_code, 0)
self.assertEqual(error_msg, '')
self.assertTrue(os.path.exists('echo.out'))
with open('echo.out', 'rU') as out:
data = out.read()
os.remove('echo.out')
self.assertEqual(data, 'Hello World!\n')
self.assertTrue(os.path.exists('junk.dat'))
with open('junk.dat', 'rU') as out:
data = out.read()
os.remove('junk.dat')
self.assertEqual(data, 'just some junk')