def test_iostream(self):
        def line_iterator():
            yield "line1\n"
            yield b"line2\n"

        outstream = io.StringIO()
        s = IOStream(line_iterator(), outstream)

        s.start()
        s.join()
    def test_invalid_unicode(self):
        def line_iterator():
            yield b"line1\n"
            yield b"error\x99\n"
            yield b"line3\n"

        outstream = io.StringIO()
        s = IOStream(line_iterator(), outstream)

        s.start()
        s.join()

        self.assertEqual(outstream.getvalue(), "line1\nerror\ufffd\nline3\n")
示例#3
0
class BufferedPopen(Popen):
    """
    Open a process and Buffer the output to *any* IO object (like `io.BytesIO`)
    """
    def __init__(self, args, stdout=None, iotimeout=None, timeout=None, **kwargs):
        self._output = stdout

        Popen.__init__(self, args, stdout=PIPE, stderr=STDOUT,
                       **kwargs)

        self._iostream = IOStream(self.stdout, self._output, iotimeout, timeout, self.timeout_callback)
        self._iostream.start()

    def wait(self):
        """Wait for child process to terminate.  Returns returncode
        attribute.
        
        If timeout is given, the process will be killed after timeout seconds if it is not finished 
        """
        returncode = Popen.wait(self)

#         self._finished_event.set()
        log.debug("returncode", returncode)

        if self._iostream.is_alive():
            log.debug("self._io_thread.join()")
            self._iostream.join()

        return returncode

    def timeout_callback(self, reason='iotimeout'):

        log.debug("timeout_callback")

        if reason == 'iotimeout':
            self._output.write("\nTimeout: No output from program for %s seconds\n" % self._iostream.iotimeout)
            self._output.write("\nTimeout: If you require a longer timeout you "
                      "may set the 'iotimeout' variable in your .binstar.yml file\n")
            self._output.write("[Terminating]\n")
        elif reason == 'timeout':
            self._output.write("\nTimeout: build exceeded maximum build time of %s seconds\n" % self._iostream.timeout)
            self._output.write("[Terminating]\n")
        else:
            self._output.write("\nTerminate: User requested build to be terminated\n")
            self._output.write("[Terminating]\n")

        self.kill_tree()

    def kill_tree(self):
        'Kill all processes and child processes'
        try:
            log.warning("Kill Tree parent pid:%s" % self.pid)
            parent = psutil.Process(self.pid)
        except psutil.NoSuchProcess:
            log.warning("Parent pid %s is already dead" % self.pid)
            # Already dead
            return

        children = parent.children(recursive=True)

        self.kill()
        for child in children:
            if child.is_running():
                log.warning(" - Kill child pid %s" % child.pid)
                child.kill()
示例#4
0
class BufferedPopen(Popen):
    """
    Open a process and Buffer the output to *any* IO object (like `io.BytesIO`)
    """
    def __init__(self,
                 args,
                 stdout=None,
                 iotimeout=None,
                 timeout=None,
                 **kwargs):
        self._output = stdout

        Popen.__init__(self, args, stdout=PIPE, stderr=STDOUT, **kwargs)

        self._iostream = IOStream(self.stdout, self._output, iotimeout,
                                  timeout, self.timeout_callback)
        self._iostream.start()

    def wait(self):
        """Wait for child process to terminate.  Returns returncode
        attribute.
        
        If timeout is given, the process will be killed after timeout seconds if it is not finished 
        """
        #
        #         if timeout:
        #             self.kill_after(timeout)

        returncode = Popen.wait(self)

        #         self._finished_event.set()
        log.debug("returncode", returncode)

        if self._iostream.is_alive():
            log.debug("self._io_thread.join()")
            self._iostream.join()

        return returncode

    def timeout_callback(self, reason='iotimeout'):
        self.kill_tree()

        log.debug("timeout_callback")

        if reason == 'iotimeout':
            self._output.write(
                "\nTimeout: No output from program for %s seconds\n" %
                self._iostream.iotimeout)
            self._output.write(
                "\nTimeout: If you require a longer timeout you "
                "may set the 'iotimeout' variable in your .binstar.yml file\n")
            self._output.write("[Terminating]\n")
        elif reason == 'timeout':
            self._output.write(
                "\nTimeout: build exceeded maximum build time of %s seconds\n"
                % self._iostream.timeout)
            self._output.write("[Terminating]\n")
        else:
            self._output.write(
                "\nTerminate: User requested build to be terminated\n")
            self._output.write("[Terminating]\n")

    def kill_tree(self):
        'Kill all processes and child processes'
        try:
            log.warn("Kill Tree parent pid:%s" % self.pid)
            parent = psutil.Process(self.pid)
        except psutil.NoSuchProcess:
            log.warn("Parent pid %s is already dead" % self.pid)
            # Already dead
            return

        children = parent.get_children(recursive=True)

        self.kill()
        for child in children:
            if child.is_running():
                log.warn(" - Kill child pid %s" % child.pid)
                child.kill()
示例#5
0
    def run(self,
            build_data,
            script_filename,
            build_log,
            timeout,
            iotimeout,
            api_token=None,
            git_oauth_token=None,
            build_filename=None,
            instructions=None):
        """
        """
        cli = self.client
        image = self.args.image
        container_script_filename = '/%s' % basename(script_filename)

        volumes = [
            container_script_filename,
        ]
        binds = {
            abspath(script_filename): {
                'bind': container_script_filename,
                'ro': False
            }
        }

        args = ["bash", container_script_filename, '--api-token', api_token]

        if git_oauth_token:
            args.extend(['--git-oauth-token', git_oauth_token])

        elif build_filename:
            container_build_filename = '/%s' % basename(build_filename)
            volumes.append(container_build_filename)
            binds[build_filename] = {
                'bind': container_build_filename,
                'ro': False
            }
            args.extend(['--build-tarball', container_build_filename])

        log.info("Running command: (iotimeout=%s)" % iotimeout)
        if self.args.allow_user_images:
            if instructions and instructions.get('docker_image'):
                image = instructions['docker_image']
                if ':' in image:
                    repository, tag = image.rsplit(':', 1)
                else:
                    repository, tag = image, None

                build_log.write('Docker: Pull %s\n' % image)
                for line in cli.pull(repository, tag=tag, stream=True):
                    msg = json.loads(line)
                    if msg.get('status') == 'Downloading':
                        build_log.write('.')
                    elif msg.get('status'):
                        build_log.write(msg.get('status', '') + '\n')
                    else:
                        build_log.write(line + '\n')

        else:
            if instructions and instructions.get('docker_image'):
                build_log.write(
                    "WARNING: User specified images are not allowed on this build worker\n"
                )
                build_log.write("Using default docker image\n")

        command = " ".join(args)
        log.info(command)
        build_log.write("Docker Image: %s\n" % image)
        log.info("Volumes: %r" % volumes)

        build_log.write("Docker: Create container\n")
        cont = cli.create_container(image, command=command, volumes=volumes)

        build_log.write("Docker: Attach output\n")
        stream = cli.attach(cont, stream=True, stdout=True, stderr=True)

        def timeout_callback(reason='iotimeout'):

            cli.kill(cont)

            if reason == 'iotimeout':
                build_log.write(
                    "\nTimeout: No output from program for %s seconds\n" %
                    iotimeout)
                build_log.write(
                    "\nTimeout: If you require a longer timeout you "
                    "may set the 'iotimeout' variable in your .binstar.yml file\n"
                )
                self._output.write("[Terminating]\n")
            elif reason == 'timeout':
                build_log.write(
                    "\nTimeout: build exceeded maximum build time of %s seconds\n"
                    % timeout)
                build_log.write("[Terminating]\n")
            else:
                build_log.write(
                    "\nTerminate: User requested build to be terminated\n")
                build_log.write("[Terminating]\n")

        ios = IOStream(stream, build_log, iotimeout, timeout, timeout_callback)

        build_log.write("Docker: Start\n")

        ios.start()

        log.info("Binds: %r" % binds)

        cli.start(cont, binds=binds)

        exit_code = cli.wait(cont)

        ios.join()

        log.info("Remove Container: %r" % cont)
        cli.remove_container(cont, v=True)

        return exit_code
示例#6
0
    def run(self, build_data, script_filename, build_log, timeout, iotimeout,
            api_token=None, git_oauth_token=None, build_filename=None, instructions=None):
        """
        """
        cli = self.client
        image = self.args.image
        container_script_filename = '/%s' % basename(script_filename)

        volumes = [container_script_filename,
                   ]
        binds = {abspath(script_filename): {'bind': container_script_filename, 'ro': False}}

        args = ["bash", container_script_filename, '--api-token', api_token]

        if git_oauth_token:
            args.extend(['--git-oauth-token', git_oauth_token])

        elif build_filename:
            container_build_filename = '/%s' % basename(build_filename)
            volumes.append(container_build_filename)
            binds[build_filename] = {'bind': container_build_filename, 'ro': False}
            args.extend(['--build-tarball', container_build_filename])

        log.info("Running command: (iotimeout=%s)" % iotimeout)
        if self.args.allow_user_images:
            if instructions and instructions.get('docker_image'):
                image = instructions['docker_image']
                if ':' in image:
                    repository, tag = image.rsplit(':', 1)
                else:
                    repository, tag = image, None

                build_log.write('Docker: Pull %s\n' % image)
                for line in cli.pull(repository, tag=tag, stream=True):
                    msg = json.loads(line)
                    if msg.get('status') == 'Downloading':
                        build_log.write('.')
                    elif msg.get('status'):
                        build_log.write(msg.get('status', '') + '\n')
                    else:
                        build_log.write(line + '\n')

        else:
            if instructions and instructions.get('docker_image'):
                build_log.write("WARNING: User specified images are not allowed on this build worker\n")
                build_log.write("Using default docker image\n")

        command = " ".join(args)
        log.info(command)
        build_log.write("Docker Image: %s\n" % image)
        log.info("Volumes: %r" % volumes)

        build_log.write("Docker: Create container\n")
        cont = cli.create_container(image, command=command, volumes=volumes)

        build_log.write("Docker: Attach output\n")
        stream = cli.attach(cont, stream=True, stdout=True, stderr=True)

        def timeout_callback(reason='iotimeout'):

            cli.kill(cont)

            if reason == 'iotimeout':
                build_log.write("\nTimeout: No output from program for %s seconds\n" % iotimeout)
                build_log.write("\nTimeout: If you require a longer timeout you "
                          "may set the 'iotimeout' variable in your .binstar.yml file\n")
                self._output.write("[Terminating]\n")
            elif reason == 'timeout':
                build_log.write("\nTimeout: build exceeded maximum build time of %s seconds\n" % timeout)
                build_log.write("[Terminating]\n")
            else:
                build_log.write("\nTerminate: User requested build to be terminated\n")
                build_log.write("[Terminating]\n")


        ios = IOStream(stream, build_log, iotimeout, timeout, timeout_callback)

        build_log.write("Docker: Start\n")

        ios.start()

        log.info("Binds: %r" % binds)

        cli.start(cont, binds=binds)

        exit_code = cli.wait(cont)

        ios.join()

        log.info("Remove Container: %r" % cont)
        cli.remove_container(cont, v=True)

        return exit_code