def _retrieve_results(self, patterns, textfiles): """ Retrieves result files matching `patterns`. """ self._logger.info('retrieving results...') start_time = time.time() filename = 'outputs.zip' pfiles, pbytes = self._server.pack_zipfile(patterns, filename) filexfer(self._server, filename, None, filename, 'b') # Valid, but empty, file causes unpack_zipfile() problems. try: if os.path.getsize(filename) > 0: ufiles, ubytes = unpack_zipfile(filename, logger=self._logger, textfiles=textfiles) else: ufiles, ubytes = 0, 0 finally: os.remove(filename) self._server.remove(filename) # Difficult to force file transfer error. if ufiles != pfiles or ubytes != pbytes: #pragma no cover msg = 'Results xfer error: %d:%d vs. %d:%d' \ % (ufiles, ubytes, pfiles, pbytes) self.raise_exception(msg, RuntimeError) et = time.time() - start_time if et >= 60: #pragma no cover self._logger.info('elapsed time: %f sec.', et)
def _remote_load_model(self, server): """ Load model into remote server. """ egg_file = self._server_info[server].get('egg_file', None) if egg_file is None or egg_file is not self._egg_file: # Only transfer if changed. try: filexfer(None, self._egg_file, self._servers[server], self._egg_file, 'b') # Difficult to force model file transfer error. except Exception as exc: #pragma nocover self._logger.error('server %r filexfer of %r failed: %r', server, self._egg_file, exc) self._top_levels[server] = None self._exceptions[server] = TracedError(exc, traceback.format_exc()) return else: self._server_info[server]['egg_file'] = self._egg_file try: tlo = self._servers[server].load_model(self._egg_file) # Difficult to force load error. except Exception as exc: #pragma nocover self._logger.error('server.load_model of %r failed: %r', self._egg_file, exc) self._top_levels[server] = None self._exceptions[server] = TracedError(exc, traceback.format_exc()) else: self._top_levels[server] = tlo
def _remote_load_model(self, server): """ Load model into remote server. """ egg_file = server.info.get('egg_file', None) if egg_file is None or egg_file is not self._egg_file: # Only transfer if changed. try: filexfer(None, self._egg_file, server.server, self._egg_file, 'b') # Difficult to force model file transfer error. except Exception as exc: # pragma nocover self._logger.error('server %r filexfer of %r failed: %r', server.name, self._egg_file, exc) server.top = None server.exception = sys.exc_info() return else: server.info['egg_file'] = self._egg_file try: tlo = server.server.load_model(self._egg_file) # Difficult to force load error. except Exception as exc: # pragma nocover self._logger.error('server.load_model of %r failed: %r', self._egg_file, exc) server.top = None server.exception = sys.exc_info() else: server.top = tlo
def test_case_eval(self): logging.debug("") logging.debug("test_case_eval") # Run a fake job in style of CaseIteratorDriver. logging.debug("allocate server") server, server_info = RAM.allocate(dict(allocator=self.allocator.name)) echo = set_as_top(Echo(1)) egg_info = echo.save_to_egg("EchoTest", "1", need_requirements=False) egg_filename = egg_info[0] try: logging.debug("transfer egg") filexfer(None, egg_filename, server, egg_filename, "b") logging.debug("load model") tlo = server.load_model(egg_filename) logging.debug("set input") tlo.set("inp_0", 42) logging.debug("run") tlo.run() logging.debug("get output") output = tlo.get("out_0") self.assertEqual(output, 42) finally: os.remove(egg_filename) logging.debug("release") RAM.release(server)
def test_case_eval(self): logging.debug('') logging.debug('test_case_eval') # Run a fake job in style of CaseIteratorDriver. logging.debug('allocate server') server, server_info = RAM.allocate(dict(allocator=self.allocator.name)) echo = set_as_top(Echo(1)) egg_info = echo.save_to_egg('EchoTest', '1', need_requirements=False) egg_filename = egg_info[0] try: logging.debug('transfer egg') filexfer(None, egg_filename, server, egg_filename, 'b') logging.debug('load model') tlo = server.load_model(egg_filename) logging.debug('set input') tlo.set('inp_0', 42) logging.debug('run') tlo.run() logging.debug('get output') output = tlo.get('out_0') self.assertEqual(output, 42) finally: os.remove(egg_filename) logging.debug('release') RAM.release(server)
def putfile(self, filename, binary): """ Copy `filename` to remote file server. filename: string Name of file to send. binary: boolean If True, file in binary. """ fd, path = tempfile.mkstemp() try: os.close(fd) mode = "b" if binary else "" filexfer(self._delegate, filename, None, path, mode, False) self._conn.send_file(path, filename) finally: try: os.remove(path) except Exception as exc: # pragma no cover self._logger.warning("Can't remove temporary file: %s", exc)
def putfile(self, filename, binary): """ Copy `filename` to remote file server. filename: string Name of file to send. binary: boolean If True, file in binary. """ fd, path = tempfile.mkstemp() try: os.close(fd) mode = 'b' if binary else '' filexfer(self._delegate, filename, None, path, mode, False) self._conn.send_file(path, filename) finally: try: os.remove(path) except Exception as exc: # pragma no cover self._logger.warning("Can't remove temporary file: %s", exc)
def _retrieve_results(self, patterns): """ Retrieves result files matching `patterns`. """ self._logger.info('retrieving results...') start_time = time.time() filename = 'outputs.zip' pfiles, pbytes = self._server.pack_zipfile(tuple(patterns), filename) try: filexfer(self._server, filename, None, filename, 'b') ufiles, ubytes = unpack_zipfile(filename, self._logger) finally: os.remove(filename) # Difficult to force file transfer error. if ufiles != pfiles or ubytes != pbytes: #pragma no cover msg = 'Results xfer error: %d:%d vs. %d:%d' \ % (ufiles, ubytes, pfiles, pbytes) self.raise_exception(msg, RuntimeError) et = time.time() - start_time if et >= 60: #pragma no cover self._logger.info('elapsed time: %f sec.', et)
def _send_inputs(self, patterns): """ Sends input files matching `patterns`. """ self._logger.info('sending inputs...') start_time = time.time() filename = 'inputs.zip' pfiles, pbytes = pack_zipfile(patterns, filename, self._logger) try: filexfer(None, filename, self._server, filename, 'b') ufiles, ubytes = self._server.unpack_zipfile(filename) finally: os.remove(filename) # Difficult to force file transfer error. if ufiles != pfiles or ubytes != pbytes: #pragma no cover msg = 'Inputs xfer error: %d:%d vs. %d:%d' \ % (ufiles, ubytes, pfiles, pbytes) self.raise_exception(msg, RuntimeError) et = time.time() - start_time if et >= 60: #pragma no cover self._logger.info('elapsed time: %f sec.', et)
def getfile(self, filename, binary): """ Copy `filename` from remote file server. filename: string Name of file to receive. binary: boolean If True, file in binary. """ fd, path = tempfile.mkstemp() try: os.close(fd) self._conn.recv_file(filename, path) mode = 'b' if binary else '' filexfer(None, path, self._delegate, filename, mode) self._conn.remove_files((filename,)) finally: try: os.remove(path) except Exception as exc: # pragma no cover self._logger.warning("Can't remove temporary file: %s", exc)
def _send_inputs(self, patterns, textfiles): """ Sends input files matching `patterns`. """ self._logger.info('sending inputs...') start_time = time.time() filename = 'inputs.zip' pfiles, pbytes = pack_zipfile(patterns, filename, self._logger) try: filexfer(None, filename, self._server, filename, 'b') ufiles, ubytes = self._server.unpack_zipfile(filename, textfiles=textfiles) finally: os.remove(filename) # Difficult to force file transfer error. if ufiles != pfiles or ubytes != pbytes: #pragma no cover msg = 'Inputs xfer error: %d:%d vs. %d:%d' \ % (ufiles, ubytes, pfiles, pbytes) self.raise_exception(msg, RuntimeError) et = time.time() - start_time if et >= 60: #pragma no cover self._logger.info('elapsed time: %f sec.', et)
def test_extcode(self): logging.debug("") logging.debug("test_extcode") # Run a fake job in style of ExternalCode component. logging.debug("allocate server") server, server_info = RAM.allocate(dict(allocator=self.allocator.name)) try: with open("junk.dat", "w") as out: out.write("just some junk") filename = "inputs.zip" logging.debug("pack inputs") pfiles, pbytes = pack_zipfile(("junk.dat",), filename, logging.getLogger()) os.remove("junk.dat") logging.debug("transfer inputs") filexfer(None, filename, server, filename, "b") logging.debug("unpack inputs") ufiles, ubytes = server.unpack_zipfile(filename) logging.debug("remove inputs") os.remove(filename) server.remove(filename) logging.debug("execute command") if sys.platform == "win32": remote_command = "cmd" args = ("/c", "echo", "Hello", "World!") else: remote_command = "echo" args = ("Hello", "World!") return_code, error_msg = server.execute_command( dict(job_name="Testing", remote_command=remote_command, args=args, output_path="echo.out") ) logging.debug("pack outputs") filename = "outputs.zip" pfiles, pbytes = server.pack_zipfile(("echo.out", "junk.dat"), filename) logging.debug("transfer outputs") filexfer(server, filename, None, filename, "b") logging.debug("unpack outputs") ufiles, ubytes = unpack_zipfile(filename) logging.debug("remove outputs") os.remove(filename) server.remove(filename) finally: logging.debug("release") RAM.release(server) self.assertEqual(return_code, 0) self.assertEqual(error_msg, "") self.assertTrue(os.path.exists("echo.out")) with open("echo.out", "rU") as out: data = out.read() os.remove("echo.out") self.assertEqual(data, "Hello World!\n") self.assertTrue(os.path.exists("junk.dat")) with open("junk.dat", "rU") as out: data = out.read() os.remove("junk.dat") self.assertEqual(data, "just some junk")
def test_extcode(self): logging.debug('') logging.debug('test_extcode') # Run a fake job in style of ExternalCode component. logging.debug('allocate server') server, server_info = RAM.allocate(dict(allocator=self.allocator.name)) try: with open('junk.dat', 'w') as out: out.write('just some junk') filename = 'inputs.zip' logging.debug('pack inputs') pfiles, pbytes = pack_zipfile(('junk.dat',), filename, logging.getLogger()) os.remove('junk.dat') logging.debug('transfer inputs') filexfer(None, filename, server, filename, 'b') logging.debug('unpack inputs') ufiles, ubytes = server.unpack_zipfile(filename) logging.debug('remove inputs') os.remove(filename) server.remove(filename) logging.debug('execute command') if sys.platform == 'win32': remote_command = 'cmd' args = ('/c', 'echo', 'Hello', 'World!') else: remote_command = 'echo' args = ('Hello', 'World!') return_code, error_msg = \ server.execute_command(dict(job_name='Testing', remote_command=remote_command, args=args, output_path='echo.out')) logging.debug('pack outputs') filename = 'outputs.zip' pfiles, pbytes = server.pack_zipfile(('echo.out', 'junk.dat'), filename) logging.debug('transfer outputs') filexfer(server, filename, None, filename, 'b') logging.debug('unpack outputs') ufiles, ubytes = unpack_zipfile(filename) logging.debug('remove outputs') os.remove(filename) server.remove(filename) finally: logging.debug('release') RAM.release(server) self.assertEqual(return_code, 0) self.assertEqual(error_msg, '') self.assertTrue(os.path.exists('echo.out')) with open('echo.out', 'rU') as out: data = out.read() os.remove('echo.out') self.assertEqual(data, 'Hello World!\n') self.assertTrue(os.path.exists('junk.dat')) with open('junk.dat', 'rU') as out: data = out.read() os.remove('junk.dat') self.assertEqual(data, 'just some junk')