def _retrieve_results(self, patterns, textfiles):
        """ Retrieves result files matching `patterns`. """
        self._logger.info('retrieving results...')
        start_time = time.time()

        filename = 'outputs.zip'
        pfiles, pbytes = self._server.pack_zipfile(patterns, filename)
        filexfer(self._server, filename, None, filename, 'b')

        # Valid, but empty, file causes unpack_zipfile() problems.
        try:
            if os.path.getsize(filename) > 0:
                ufiles, ubytes = unpack_zipfile(filename, logger=self._logger,
                                                textfiles=textfiles)
            else:
                ufiles, ubytes = 0, 0
        finally:
            os.remove(filename)
            self._server.remove(filename)

        # Difficult to force file transfer error.
        if ufiles != pfiles or ubytes != pbytes:  #pragma no cover
            msg = 'Results xfer error: %d:%d vs. %d:%d' \
                  % (ufiles, ubytes, pfiles, pbytes)
            self.raise_exception(msg, RuntimeError)

        et = time.time() - start_time
        if et >= 60:  #pragma no cover
            self._logger.info('elapsed time: %f sec.', et)
    def unpack_zipfile(self, filename):
        """
        Unpack ZipFile `filename` if `filename` is legal.

        filename: string
            Name of ZipFile to unpack.
        """
        self._logger.debug('unpack_zipfile %r', filename)
        self._check_path(filename, 'unpack_zipfile')
        return unpack_zipfile(filename, self._logger)
    def unpack_zipfile(self, filename, textfiles=None):
        """
        Unpack ZipFile `filename` if `filename` is legal.

        filename: string
            Name of ZipFile to unpack.

        textfiles: list
            List of :mod:`fnmatch` style patterns specifying which upnapcked
            files are text files possibly needing newline translation. If not
            supplied, the first 4KB of each is scanned for a zero byte. If not
            found then the file is assumed to be a text file.
        """
        self._logger.debug('unpack_zipfile %r', filename)
        self._check_path(filename, 'unpack_zipfile')
        return unpack_zipfile(filename, self._logger, textfiles)
    def unpack_zipfile(self, filename, textfiles=None):
        """
        Unpack ZipFile `filename` if `filename` is legal.

        filename: string
            Name of ZipFile to unpack.

        textfiles: list
            List of :mod:`fnmatch` style patterns specifying which unpacked
            files are text files possibly needing newline translation. If not
            supplied, the first 4KB of each is scanned for a zero byte. If none
            is found, then the file is assumed to be a text file.
        """
        self._logger.debug('unpack_zipfile %r', filename)
        self._check_path(filename, 'unpack_zipfile')
        return unpack_zipfile(filename, self._logger, textfiles)
    def _retrieve_results(self, patterns):
        """ Retrieves result files matching `patterns`. """
        self._logger.info('retrieving results...')
        start_time = time.time()

        filename = 'outputs.zip'
        pfiles, pbytes = self._server.pack_zipfile(tuple(patterns), filename)
        try:
            filexfer(self._server, filename, None, filename, 'b')
            ufiles, ubytes = unpack_zipfile(filename, self._logger)
        finally:
            os.remove(filename)

        # Difficult to force file transfer error.
        if ufiles != pfiles or ubytes != pbytes:  #pragma no cover
            msg = 'Results xfer error: %d:%d vs. %d:%d' \
                  % (ufiles, ubytes, pfiles, pbytes)
            self.raise_exception(msg, RuntimeError)

        et = time.time() - start_time
        if et >= 60:  #pragma no cover
            self._logger.info('elapsed time: %f sec.', et)
    def test_extcode(self):
        logging.debug("")
        logging.debug("test_extcode")

        # Run a fake job in style of ExternalCode component.
        logging.debug("allocate server")
        server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
        try:
            with open("junk.dat", "w") as out:
                out.write("just some junk")
            filename = "inputs.zip"

            logging.debug("pack inputs")
            pfiles, pbytes = pack_zipfile(("junk.dat",), filename, logging.getLogger())
            os.remove("junk.dat")

            logging.debug("transfer inputs")
            filexfer(None, filename, server, filename, "b")

            logging.debug("unpack inputs")
            ufiles, ubytes = server.unpack_zipfile(filename)

            logging.debug("remove inputs")
            os.remove(filename)
            server.remove(filename)

            logging.debug("execute command")
            if sys.platform == "win32":
                remote_command = "cmd"
                args = ("/c", "echo", "Hello", "World!")
            else:
                remote_command = "echo"
                args = ("Hello", "World!")
            return_code, error_msg = server.execute_command(
                dict(job_name="Testing", remote_command=remote_command, args=args, output_path="echo.out")
            )

            logging.debug("pack outputs")
            filename = "outputs.zip"
            pfiles, pbytes = server.pack_zipfile(("echo.out", "junk.dat"), filename)
            logging.debug("transfer outputs")
            filexfer(server, filename, None, filename, "b")

            logging.debug("unpack outputs")
            ufiles, ubytes = unpack_zipfile(filename)

            logging.debug("remove outputs")
            os.remove(filename)
            server.remove(filename)

        finally:
            logging.debug("release")
            RAM.release(server)

        self.assertEqual(return_code, 0)
        self.assertEqual(error_msg, "")

        self.assertTrue(os.path.exists("echo.out"))
        with open("echo.out", "rU") as out:
            data = out.read()
        os.remove("echo.out")
        self.assertEqual(data, "Hello World!\n")

        self.assertTrue(os.path.exists("junk.dat"))
        with open("junk.dat", "rU") as out:
            data = out.read()
        os.remove("junk.dat")
        self.assertEqual(data, "just some junk")
示例#7
0
    def test_extcode(self):
        logging.debug('')
        logging.debug('test_extcode')

        # Run a fake job in style of ExternalCode component.
        logging.debug('allocate server')
        server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
        try:
            with open('junk.dat', 'w') as out:
                out.write('just some junk')
            filename = 'inputs.zip'

            logging.debug('pack inputs')
            pfiles, pbytes = pack_zipfile(('junk.dat',), filename,
                                          logging.getLogger())
            os.remove('junk.dat')

            logging.debug('transfer inputs')
            filexfer(None, filename, server, filename, 'b')

            logging.debug('unpack inputs')
            ufiles, ubytes = server.unpack_zipfile(filename)

            logging.debug('remove inputs')
            os.remove(filename)
            server.remove(filename)

            logging.debug('execute command')
            if sys.platform == 'win32':
                remote_command = 'cmd'
                args = ('/c', 'echo', 'Hello', 'World!')
            else:
                remote_command = 'echo'
                args = ('Hello', 'World!')
            return_code, error_msg = \
                server.execute_command(dict(job_name='Testing',
                                            remote_command=remote_command,
                                            args=args,
                                            output_path='echo.out'))

            logging.debug('pack outputs')
            filename = 'outputs.zip'
            pfiles, pbytes = server.pack_zipfile(('echo.out', 'junk.dat'),
                                                 filename)
            logging.debug('transfer outputs')
            filexfer(server, filename, None, filename, 'b')

            logging.debug('unpack outputs')
            ufiles, ubytes = unpack_zipfile(filename)

            logging.debug('remove outputs')
            os.remove(filename)
            server.remove(filename)

        finally:
            logging.debug('release')
            RAM.release(server)

        self.assertEqual(return_code, 0)
        self.assertEqual(error_msg, '')

        self.assertTrue(os.path.exists('echo.out'))
        with open('echo.out', 'rU') as out:
            data = out.read()
        os.remove('echo.out')
        self.assertEqual(data, 'Hello World!\n')

        self.assertTrue(os.path.exists('junk.dat'))
        with open('junk.dat', 'rU') as out:
            data = out.read()
        os.remove('junk.dat')
        self.assertEqual(data, 'just some junk')
示例#8
0
    def test_extcode(self):
        logging.debug('')
        logging.debug('test_extcode')

        # Run a fake job in style of ExternalCode component.
        logging.debug('allocate server')
        server, server_info = RAM.allocate(dict(allocator=self.allocator.name))
        try:
            with open('junk.dat', 'w') as out:
                out.write('just some junk')
            filename = 'inputs.zip'

            logging.debug('pack inputs')
            pfiles, pbytes = pack_zipfile(('junk.dat',), filename,
                                          logging.getLogger())
            os.remove('junk.dat')

            logging.debug('transfer inputs')
            filexfer(None, filename, server, filename, 'b')

            logging.debug('unpack inputs')
            ufiles, ubytes = server.unpack_zipfile(filename)

            logging.debug('remove inputs')
            os.remove(filename)
            server.remove(filename)

            logging.debug('execute command')
            if sys.platform == 'win32':
                remote_command = 'cmd'
                args = ('/c', 'echo', 'Hello', 'World!')
            else:
                remote_command = 'echo'
                args = ('Hello', 'World!')
            return_code, error_msg = \
                server.execute_command(dict(job_name='Testing',
                                            remote_command=remote_command,
                                            args=args,
                                            output_path='echo.out'))

            logging.debug('pack outputs')
            filename = 'outputs.zip'
            pfiles, pbytes = server.pack_zipfile(('echo.out', 'junk.dat'),
                                                 filename)
            logging.debug('transfer outputs')
            filexfer(server, filename, None, filename, 'b')

            logging.debug('unpack outputs')
            ufiles, ubytes = unpack_zipfile(filename)

            logging.debug('remove outputs')
            os.remove(filename)
            server.remove(filename)

        finally:
            logging.debug('release')
            RAM.release(server)

        self.assertEqual(return_code, 0)
        self.assertEqual(error_msg, '')

        self.assertTrue(os.path.exists('echo.out'))
        with open('echo.out', 'rU') as out:
            data = out.read()
        os.remove('echo.out')
        self.assertEqual(data, 'Hello World!\n')

        self.assertTrue(os.path.exists('junk.dat'))
        with open('junk.dat', 'rU') as out:
            data = out.read()
        os.remove('junk.dat')
        self.assertEqual(data, 'just some junk')