예제 #1
0
    def _retrieve_results(self, patterns, textfiles):
        """ Retrieves result files matching `patterns`. """
        self._logger.info('retrieving results...')
        start_time = time.time()

        filename = 'outputs.zip'
        pfiles, pbytes = self._server.pack_zipfile(patterns, filename)
        filexfer(self._server, filename, None, filename, 'b')

        # Valid, but empty, file causes unpack_zipfile() problems.
        try:
            if os.path.getsize(filename) > 0:
                ufiles, ubytes = unpack_zipfile(filename, logger=self._logger,
                                                textfiles=textfiles)
            else:
                ufiles, ubytes = 0, 0
        finally:
            os.remove(filename)
            self._server.remove(filename)

        # Difficult to force file transfer error.
        if ufiles != pfiles or ubytes != pbytes:  #pragma no cover
            msg = 'Results xfer error: %d:%d vs. %d:%d' \
                  % (ufiles, ubytes, pfiles, pbytes)
            self.raise_exception(msg, RuntimeError)

        et = time.time() - start_time
        if et >= 60:  #pragma no cover
            self._logger.info('elapsed time: %f sec.', et)
 def _remote_load_model(self, server):
     """ Load model into remote server. """
     egg_file = self._server_info[server].get('egg_file', None)
     if egg_file is None or egg_file is not self._egg_file:
         # Only transfer if changed.
         try:
             filexfer(None, self._egg_file,
                      self._servers[server], self._egg_file, 'b')
         # Difficult to force model file transfer error.
         except Exception as exc:  #pragma nocover
             self._logger.error('server %r filexfer of %r failed: %r',
                                server, self._egg_file, exc)
             self._top_levels[server] = None
             self._exceptions[server] = TracedError(exc, traceback.format_exc())
             return
         else:
             self._server_info[server]['egg_file'] = self._egg_file
     try:
         tlo = self._servers[server].load_model(self._egg_file)
     # Difficult to force load error.
     except Exception as exc:  #pragma nocover
         self._logger.error('server.load_model of %r failed: %r',
                            self._egg_file, exc)
         self._top_levels[server] = None
         self._exceptions[server] = TracedError(exc, traceback.format_exc())
     else:
         self._top_levels[server] = tlo
 def _remote_load_model(self, server):
     """ Load model into remote server. """
     egg_file = server.info.get('egg_file', None)
     if egg_file is None or egg_file is not self._egg_file:
         # Only transfer if changed.
         try:
             filexfer(None, self._egg_file, server.server, self._egg_file,
                      'b')
         # Difficult to force model file transfer error.
         except Exception as exc:  # pragma nocover
             self._logger.error('server %r filexfer of %r failed: %r',
                                server.name, self._egg_file, exc)
             server.top = None
             server.exception = sys.exc_info()
             return
         else:
             server.info['egg_file'] = self._egg_file
     try:
         tlo = server.server.load_model(self._egg_file)
     # Difficult to force load error.
     except Exception as exc:  # pragma nocover
         self._logger.error('server.load_model of %r failed: %r',
                            self._egg_file, exc)
         server.top = None
         server.exception = sys.exc_info()
     else:
         server.top = tlo
    def test_case_eval(self):
        logging.debug("")
        logging.debug("test_case_eval")

        # Run a fake job in style of CaseIteratorDriver.
        logging.debug("allocate server")
        server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
        echo = set_as_top(Echo(1))
        egg_info = echo.save_to_egg("EchoTest", "1", need_requirements=False)
        egg_filename = egg_info[0]
        try:
            logging.debug("transfer egg")
            filexfer(None, egg_filename, server, egg_filename, "b")

            logging.debug("load model")
            tlo = server.load_model(egg_filename)

            logging.debug("set input")
            tlo.set("inp_0", 42)

            logging.debug("run")
            tlo.run()

            logging.debug("get output")
            output = tlo.get("out_0")
            self.assertEqual(output, 42)
        finally:
            os.remove(egg_filename)
            logging.debug("release")
            RAM.release(server)
 def _remote_load_model(self, server):
     """ Load model into remote server. """
     egg_file = server.info.get('egg_file', None)
     if egg_file is None or egg_file is not self._egg_file:
         # Only transfer if changed.
         try:
             filexfer(None, self._egg_file,
                      server.server, self._egg_file, 'b')
         # Difficult to force model file transfer error.
         except Exception as exc:  # pragma nocover
             self._logger.error('server %r filexfer of %r failed: %r',
                                server.name, self._egg_file, exc)
             server.top = None
             server.exception = sys.exc_info()
             return
         else:
             server.info['egg_file'] = self._egg_file
     try:
         tlo = server.server.load_model(self._egg_file)
     # Difficult to force load error.
     except Exception as exc:  # pragma nocover
         self._logger.error('server.load_model of %r failed: %r',
                            self._egg_file, exc)
         server.top = None
         server.exception = sys.exc_info()
     else:
         server.top = tlo
예제 #6
0
    def test_case_eval(self):
        logging.debug('')
        logging.debug('test_case_eval')

        # Run a fake job in style of CaseIteratorDriver.
        logging.debug('allocate server')
        server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
        echo = set_as_top(Echo(1))
        egg_info = echo.save_to_egg('EchoTest', '1', need_requirements=False)
        egg_filename = egg_info[0]
        try:
            logging.debug('transfer egg')
            filexfer(None, egg_filename, server, egg_filename, 'b')

            logging.debug('load model')
            tlo = server.load_model(egg_filename)

            logging.debug('set input')
            tlo.set('inp_0', 42)

            logging.debug('run')
            tlo.run()

            logging.debug('get output')
            output = tlo.get('out_0')
            self.assertEqual(output, 42)
        finally:
            os.remove(egg_filename)
            logging.debug('release')
            RAM.release(server)
예제 #7
0
    def test_case_eval(self):
        logging.debug('')
        logging.debug('test_case_eval')

        # Run a fake job in style of CaseIteratorDriver.
        logging.debug('allocate server')
        server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
        echo = set_as_top(Echo(1))
        egg_info = echo.save_to_egg('EchoTest', '1', need_requirements=False)
        egg_filename = egg_info[0]
        try:
            logging.debug('transfer egg')
            filexfer(None, egg_filename, server, egg_filename, 'b')

            logging.debug('load model')
            tlo = server.load_model(egg_filename)

            logging.debug('set input')
            tlo.set('inp_0', 42)

            logging.debug('run')
            tlo.run()

            logging.debug('get output')
            output = tlo.get('out_0')
            self.assertEqual(output, 42)
        finally:
            os.remove(egg_filename)
            logging.debug('release')
            RAM.release(server)
예제 #8
0
    def putfile(self, filename, binary):
        """
        Copy `filename` to remote file server.

        filename: string
            Name of file to send.

        binary: boolean
            If True, file in binary.
        """
        fd, path = tempfile.mkstemp()
        try:
            os.close(fd)
            mode = "b" if binary else ""
            filexfer(self._delegate, filename, None, path, mode, False)
            self._conn.send_file(path, filename)
        finally:
            try:
                os.remove(path)
            except Exception as exc:  # pragma no cover
                self._logger.warning("Can't remove temporary file: %s", exc)
예제 #9
0
    def putfile(self, filename, binary):
        """
        Copy `filename` to remote file server.

        filename: string
            Name of file to send.

        binary: boolean
            If True, file in binary.
        """
        fd, path = tempfile.mkstemp()
        try:
            os.close(fd)
            mode = 'b' if binary else ''
            filexfer(self._delegate, filename, None, path, mode, False)
            self._conn.send_file(path, filename)
        finally:
            try:
                os.remove(path)
            except Exception as exc:  # pragma no cover
                self._logger.warning("Can't remove temporary file: %s", exc)
예제 #10
0
    def _retrieve_results(self, patterns):
        """ Retrieves result files matching `patterns`. """
        self._logger.info('retrieving results...')
        start_time = time.time()

        filename = 'outputs.zip'
        pfiles, pbytes = self._server.pack_zipfile(tuple(patterns), filename)
        try:
            filexfer(self._server, filename, None, filename, 'b')
            ufiles, ubytes = unpack_zipfile(filename, self._logger)
        finally:
            os.remove(filename)

        # Difficult to force file transfer error.
        if ufiles != pfiles or ubytes != pbytes:  #pragma no cover
            msg = 'Results xfer error: %d:%d vs. %d:%d' \
                  % (ufiles, ubytes, pfiles, pbytes)
            self.raise_exception(msg, RuntimeError)

        et = time.time() - start_time
        if et >= 60:  #pragma no cover
            self._logger.info('elapsed time: %f sec.', et)
예제 #11
0
    def _send_inputs(self, patterns):
        """ Sends input files matching `patterns`. """
        self._logger.info('sending inputs...')
        start_time = time.time()

        filename = 'inputs.zip'
        pfiles, pbytes = pack_zipfile(patterns, filename, self._logger)
        try:
            filexfer(None, filename, self._server, filename, 'b')
            ufiles, ubytes = self._server.unpack_zipfile(filename)
        finally:
            os.remove(filename)

        # Difficult to force file transfer error.
        if ufiles != pfiles or ubytes != pbytes:  #pragma no cover
            msg = 'Inputs xfer error: %d:%d vs. %d:%d' \
                  % (ufiles, ubytes, pfiles, pbytes)
            self.raise_exception(msg, RuntimeError)

        et = time.time() - start_time
        if et >= 60:  #pragma no cover
            self._logger.info('elapsed time: %f sec.', et)
예제 #12
0
    def getfile(self, filename, binary):
        """
        Copy `filename` from remote file server.

        filename: string
            Name of file to receive.

        binary: boolean
            If True, file in binary.
        """
        fd, path = tempfile.mkstemp()
        try:
            os.close(fd)
            self._conn.recv_file(filename, path)
            mode = 'b' if binary else ''
            filexfer(None, path, self._delegate, filename, mode)
            self._conn.remove_files((filename,))
        finally:
            try:
                os.remove(path)
            except Exception as exc:  # pragma no cover
                self._logger.warning("Can't remove temporary file: %s", exc)
    def _send_inputs(self, patterns, textfiles):
        """ Sends input files matching `patterns`. """
        self._logger.info('sending inputs...')
        start_time = time.time()

        filename = 'inputs.zip'
        pfiles, pbytes = pack_zipfile(patterns, filename, self._logger)
        try:
            filexfer(None, filename, self._server, filename, 'b')
            ufiles, ubytes = self._server.unpack_zipfile(filename,
                                                         textfiles=textfiles)
        finally:
            os.remove(filename)

        # Difficult to force file transfer error.
        if ufiles != pfiles or ubytes != pbytes:  #pragma no cover
            msg = 'Inputs xfer error: %d:%d vs. %d:%d' \
                  % (ufiles, ubytes, pfiles, pbytes)
            self.raise_exception(msg, RuntimeError)

        et = time.time() - start_time
        if et >= 60:  #pragma no cover
            self._logger.info('elapsed time: %f sec.', et)
예제 #14
0
    def test_extcode(self):
        logging.debug("")
        logging.debug("test_extcode")

        # Run a fake job in style of ExternalCode component.
        logging.debug("allocate server")
        server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
        try:
            with open("junk.dat", "w") as out:
                out.write("just some junk")
            filename = "inputs.zip"

            logging.debug("pack inputs")
            pfiles, pbytes = pack_zipfile(("junk.dat",), filename, logging.getLogger())
            os.remove("junk.dat")

            logging.debug("transfer inputs")
            filexfer(None, filename, server, filename, "b")

            logging.debug("unpack inputs")
            ufiles, ubytes = server.unpack_zipfile(filename)

            logging.debug("remove inputs")
            os.remove(filename)
            server.remove(filename)

            logging.debug("execute command")
            if sys.platform == "win32":
                remote_command = "cmd"
                args = ("/c", "echo", "Hello", "World!")
            else:
                remote_command = "echo"
                args = ("Hello", "World!")
            return_code, error_msg = server.execute_command(
                dict(job_name="Testing", remote_command=remote_command, args=args, output_path="echo.out")
            )

            logging.debug("pack outputs")
            filename = "outputs.zip"
            pfiles, pbytes = server.pack_zipfile(("echo.out", "junk.dat"), filename)
            logging.debug("transfer outputs")
            filexfer(server, filename, None, filename, "b")

            logging.debug("unpack outputs")
            ufiles, ubytes = unpack_zipfile(filename)

            logging.debug("remove outputs")
            os.remove(filename)
            server.remove(filename)

        finally:
            logging.debug("release")
            RAM.release(server)

        self.assertEqual(return_code, 0)
        self.assertEqual(error_msg, "")

        self.assertTrue(os.path.exists("echo.out"))
        with open("echo.out", "rU") as out:
            data = out.read()
        os.remove("echo.out")
        self.assertEqual(data, "Hello World!\n")

        self.assertTrue(os.path.exists("junk.dat"))
        with open("junk.dat", "rU") as out:
            data = out.read()
        os.remove("junk.dat")
        self.assertEqual(data, "just some junk")
예제 #15
0
    def test_extcode(self):
        logging.debug('')
        logging.debug('test_extcode')

        # Run a fake job in style of ExternalCode component.
        logging.debug('allocate server')
        server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
        try:
            with open('junk.dat', 'w') as out:
                out.write('just some junk')
            filename = 'inputs.zip'

            logging.debug('pack inputs')
            pfiles, pbytes = pack_zipfile(('junk.dat',), filename,
                                          logging.getLogger())
            os.remove('junk.dat')

            logging.debug('transfer inputs')
            filexfer(None, filename, server, filename, 'b')

            logging.debug('unpack inputs')
            ufiles, ubytes = server.unpack_zipfile(filename)

            logging.debug('remove inputs')
            os.remove(filename)
            server.remove(filename)

            logging.debug('execute command')
            if sys.platform == 'win32':
                remote_command = 'cmd'
                args = ('/c', 'echo', 'Hello', 'World!')
            else:
                remote_command = 'echo'
                args = ('Hello', 'World!')
            return_code, error_msg = \
                server.execute_command(dict(job_name='Testing',
                                            remote_command=remote_command,
                                            args=args,
                                            output_path='echo.out'))

            logging.debug('pack outputs')
            filename = 'outputs.zip'
            pfiles, pbytes = server.pack_zipfile(('echo.out', 'junk.dat'),
                                                 filename)
            logging.debug('transfer outputs')
            filexfer(server, filename, None, filename, 'b')

            logging.debug('unpack outputs')
            ufiles, ubytes = unpack_zipfile(filename)

            logging.debug('remove outputs')
            os.remove(filename)
            server.remove(filename)

        finally:
            logging.debug('release')
            RAM.release(server)

        self.assertEqual(return_code, 0)
        self.assertEqual(error_msg, '')

        self.assertTrue(os.path.exists('echo.out'))
        with open('echo.out', 'rU') as out:
            data = out.read()
        os.remove('echo.out')
        self.assertEqual(data, 'Hello World!\n')

        self.assertTrue(os.path.exists('junk.dat'))
        with open('junk.dat', 'rU') as out:
            data = out.read()
        os.remove('junk.dat')
        self.assertEqual(data, 'just some junk')
예제 #16
0
    def test_extcode(self):
        logging.debug('')
        logging.debug('test_extcode')

        # Run a fake job in style of ExternalCode component.
        logging.debug('allocate server')
        server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
        try:
            with open('junk.dat', 'w') as out:
                out.write('just some junk')
            filename = 'inputs.zip'

            logging.debug('pack inputs')
            pfiles, pbytes = pack_zipfile(('junk.dat',), filename,
                                          logging.getLogger())
            os.remove('junk.dat')

            logging.debug('transfer inputs')
            filexfer(None, filename, server, filename, 'b')

            logging.debug('unpack inputs')
            ufiles, ubytes = server.unpack_zipfile(filename)

            logging.debug('remove inputs')
            os.remove(filename)
            server.remove(filename)

            logging.debug('execute command')
            if sys.platform == 'win32':
                remote_command = 'cmd'
                args = ('/c', 'echo', 'Hello', 'World!')
            else:
                remote_command = 'echo'
                args = ('Hello', 'World!')
            return_code, error_msg = \
                server.execute_command(dict(job_name='Testing',
                                            remote_command=remote_command,
                                            args=args,
                                            output_path='echo.out'))

            logging.debug('pack outputs')
            filename = 'outputs.zip'
            pfiles, pbytes = server.pack_zipfile(('echo.out', 'junk.dat'),
                                                 filename)
            logging.debug('transfer outputs')
            filexfer(server, filename, None, filename, 'b')

            logging.debug('unpack outputs')
            ufiles, ubytes = unpack_zipfile(filename)

            logging.debug('remove outputs')
            os.remove(filename)
            server.remove(filename)

        finally:
            logging.debug('release')
            RAM.release(server)

        self.assertEqual(return_code, 0)
        self.assertEqual(error_msg, '')

        self.assertTrue(os.path.exists('echo.out'))
        with open('echo.out', 'rU') as out:
            data = out.read()
        os.remove('echo.out')
        self.assertEqual(data, 'Hello World!\n')

        self.assertTrue(os.path.exists('junk.dat'))
        with open('junk.dat', 'rU') as out:
            data = out.read()
        os.remove('junk.dat')
        self.assertEqual(data, 'just some junk')