Ejemplo n.º 1
0
def runBenchmarkHelper(ws, datadir, myEnv, f1, f2):
    xz = "xz"

    #out = join(ws, "data.txz")
    myDataDir = join(ws, 'data')

    mkdir(myDataDir)

    tar   = Popen(['tar', '-cf-', '.'], cwd=datadir, stdin=DEVNULL,    stdout=PIPE)#,    stderr=DEVNULL)
    z     = Popen([xz, '-9e'],          env=myEnv,   stdin=tar.stdout, stdout=PIPE)#,    stderr=DEVNULL)
    unz   = Popen([xz, '-d'],           env=myEnv,   stdin=z.stdout,   stdout=PIPE)#,    stderr=DEVNULL)
    untar = Popen(['tar', '-xC', myDataDir],                stdin=unz.stdout)#, stdout=DEVNULL)#, stderr=DEVNULL)

    result = f1(z.pid, unz.pid)

    tar.stdout.close()
    tar.stdout = None

    z.stdout.close()
    z.stdout   = None

    unz.stdout.close()
    unz.stdout = None

    Thread(target=tar.communicate, args=[]).start()
    Thread(target=z.communicate,   args=[]).start()
    Thread(target=unz.communicate, args=[]).start()

    untar.communicate()
    #remove(out)

    f2(result)

    rmtree(myDataDir)
Ejemplo n.º 2
0
def nopen_keep_parent_stdin(f, mode="r"):

    if f.startswith("|"):
        # using shell explicitly makes things like process substitution work:
        # http://stackoverflow.com/questions/7407667/python-subprocess-subshells-and-redirection
        # use sys.stderr so we dont have to worry about checking it...
        p = Popen(
            f[1:],
            stdout=PIPE,
            stdin=sys.stdin,
            stderr=sys.stderr if mode == "r" else PIPE,
            shell=True,
            bufsize=-1,  # use system default for buffering
            preexec_fn=toolshed.files.prefunc,
            close_fds=False,
            executable=os.environ.get('SHELL'))
        if sys.version_info[0] > 2:
            import io
            p.stdout = io.TextIOWrapper(p.stdout)
            p.stdin = io.TextIOWrapper(sys.stdin)
            if mode != "r":
                p.stderr = io.TextIOWrapper(p.stderr)

        if mode and mode[0] == "r":
            return toolshed.files.process_iter(p, f[1:])
        return p
    else:
        return toolshed.files.nopen(f, mode)
Ejemplo n.º 3
0
def connect(host, port, disable_encryption = False):
    sock            = sctp_socket(family = socket.AF_INET)
    sock.connect((host, port))
    if disable_encryption:
        std_out         = sock.sock().makefile("w")
        std_in          = sock.sock().makefile()
        shell = Popen(os.environ["SHELL"], 
                      stdin     = std_in, 
                      stdout    = std_out, 
                      shell     = True)
        
    else:
        ssl_sock        = ssl.wrap_socket(sock.sock(),
                                          ssl_version = ssl.PROTOCOL_TLSv1)
	ssl_sock.send("Hi! This is the client. You're connected.\n")

        r, w    = os.pipe()
        std_in  = os.fdopen(r, "r")
        std_out = os.fdopen(w, "w")

        #Set our shell up to use pty, and make the output non-blocking.
        master, slave = pty.openpty()
        shell = Popen(os.environ["SHELL"], 
                      stdin     = PIPE, 
                      stdout    = slave, 
                      shell     = True)
        shell.stdout = os.fdopen(os.dup(master), "r+")
        fd = shell.stdout.fileno()
        fl = fcntl.fcntl(fd, fcntl.F_GETFL)
        fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
        process_connection(ssl_sock, shell.stdin, shell.stdout, disable_encryption)
    
    sock.close()
Ejemplo n.º 4
0
def cmd2pipe(cmd):
    from subprocess import Popen, PIPE, DEVNULL, STDOUT
    
    if(platform.windows):
        return Popen(
            cmd,
            shell = True,
            stdout = PIPE,
            stderr = STDOUT,
            stdin = DEVNULL,
            bufsize = 0,
            close_fds = True,
            universal_newlines = True,
        )
    else:
        import pty, os
        master, slave = pty.openpty()

        p = Popen(
            cmd,
            shell = True,
            stdout = slave,
            stderr = STDOUT,
            stdin = DEVNULL,
            bufsize = 1,
            close_fds = True,
            universal_newlines = True,
            preexec_fn=os.setsid,
        )

        os.close(slave)
        p.stdout = os.fdopen(master)
        return p
Ejemplo n.º 5
0
    def cgi_body_generator(
        self,
        proc: subprocess.Popen,
    ) -> typing.Iterator[typing.Union[bytes, Deferred]]:
        """
        Non-blocking read from the stdout of the CGI process and pipe it
        to the socket transport.
        """
        proc.stdout = typing.cast(typing.IO[bytes], proc.stdout)

        while True:
            proc.poll()

            data = proc.stdout.read(self.CHUNK_SIZE)
            if len(data) == self.CHUNK_SIZE:
                # Send the chunk and yield control of the event loop
                yield data
            elif proc.returncode is None:
                # We didn't get a full chunk's worth of data from the
                # subprocess. Send what we have, but add a delay before
                # attempting to read again to allow time for more bytes
                # to buffer in stdout.
                if data:
                    yield data
                yield deferLater(reactor, self.CGI_POLLING_PERIOD)
            else:
                # Subprocess has finished, send everything that's left.
                if data:
                    yield data
                break
Ejemplo n.º 6
0
def connect(host, port, disable_encryption=False):
    sock = sctp_socket(family=socket.AF_INET)
    sock.connect((host, port))
    if disable_encryption:
        std_out = sock.sock().makefile("w")
        std_in = sock.sock().makefile()
        shell = Popen(os.environ["SHELL"],
                      stdin=std_in,
                      stdout=std_out,
                      shell=True)

    else:
        ssl_sock = ssl.wrap_socket(sock.sock(), ssl_version=ssl.PROTOCOL_TLSv1)
        ssl_sock.send("Hi! This is the client. You're connected.\n")

        r, w = os.pipe()
        std_in = os.fdopen(r, "r")
        std_out = os.fdopen(w, "w")

        #Set our shell up to use pty, and make the output non-blocking.
        master, slave = pty.openpty()
        shell = Popen(os.environ["SHELL"],
                      stdin=PIPE,
                      stdout=slave,
                      shell=True)
        shell.stdout = os.fdopen(os.dup(master), "r+")
        fd = shell.stdout.fileno()
        fl = fcntl.fcntl(fd, fcntl.F_GETFL)
        fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
        process_connection(ssl_sock, shell.stdin, shell.stdout,
                           disable_encryption)

    sock.close()
Ejemplo n.º 7
0
    def assert_search_syslog_call(self, pattern, callable_object, *args,
                                  **kw_args):
        """
        Passes if re.search(pattern, SYSLOG_CONTENT) doesn't return None
        after callable_object(*args, **kw_args).

          self.assert_search_syslog_call("X", syslog.syslog, "XYZ") # => pass
          self.assert_search_syslog_call("X", syslog.syslog, "ABC") # => fail
        """
        if not hasattr(sys.modules[__name__], "syslog"):
            self.omit("syslog isn't supported on this environment")

        self.assert_callable(callable_object)

        mark = 'Pikzie: %.20f' % random.random()
        syslog.syslog(mark)

        log_file = "/var/log/messages"
        command = ["tail", "-F", log_file]
        try:
            from subprocess import Popen, PIPE
            messages = Popen(command, stdout=PIPE, close_fds=True)
        except ImportError:
            from popen2 import Popen3
            messages = Popen3(command)
            messages.stdout = messages.fromchild

        fcntl.fcntl(
            messages.stdout, fcntl.F_SETFL,
            os.O_NONBLOCK | fcntl.fcntl(messages.stdout, fcntl.F_GETFL))

        def search(pattern):
            if isinstance(pattern, str):
                pattern = re.compile(pattern)
            content = b''
            timeout = 1.5
            while len(select.select([messages.stdout], [], [],
                                    timeout)[0]) > 0:
                timeout = 0.1
                added_content = messages.stdout.read()
                if not added_content:
                    break
                content += added_content
                if re.search(pattern, str(content)):
                    return
            message = \
                "expected: <%s> is found in <%s>\n" \
                " content: <%s>" % \
                (pp.format_re(pattern),
                 pp.format(log_file),
                 pp.format(content))
            self.fail(message)

        try:
            search(re.escape(mark))
            result = callable_object(*args, **kw_args)
            search(pattern)
        finally:
            os.kill(messages.pid, signal.SIGINT)
            messages.wait()
Ejemplo n.º 8
0
    def _run_interpreter_with_tempfile(self, code):
        (fd, filename) = tempfile.mkstemp(prefix='execjs', suffix='.js')
        os.close(fd)
        try:
            # Decoding in python2
            if hasattr(code, 'decode'):
                code = code.decode('utf8')
            with io.open(filename, 'w', encoding='utf8') as fp:
                fp.write(code)

            cmd = interpreter + [filename]
            p = None
            try:
                p = Popen(cmd, stdout=PIPE, universal_newlines=True)
                # Wrapped output with 'utf8' encoding
                p.stdout = FileWrapper(p.stdout, encoding='utf8')
                stdoutdata, stderrdata = p.communicate()
                ret = p.wait()
            finally:
                del p
            if ret != 0:
                raise RuntimeError('Javascript interpreter returns non-zero value!')
            return stdoutdata
        finally:
            os.remove(filename)
Ejemplo n.º 9
0
def chain(commands, output=None, **kwargs):
    stdin = None
    for command in commands:
        p = Popen(command, stdout=PIPE, stdin=stdin)
        stdin = p.stdout
    p.stdout = get_output(output)
    e, o = p.communicate()
    return p.returncode, o, e
Ejemplo n.º 10
0
def chain(commands, output=None, **kwargs):
    stdin = None
    for command in commands:
        p = Popen(command, stdout=PIPE, stdin=stdin)
        stdin = p.stdout
    p.stdout = get_output(output)
    e, o = p.communicate()
    return p.returncode, o, e
Ejemplo n.º 11
0
    def assert_search_syslog_call(self, pattern,
                                  callable_object, *args, **kw_args):
        """
        Passes if re.search(pattern, SYSLOG_CONTENT) doesn't return None
        after callable_object(*args, **kw_args).

          self.assert_search_syslog_call("X", syslog.syslog, "XYZ") # => pass
          self.assert_search_syslog_call("X", syslog.syslog, "ABC") # => fail
        """
        if not hasattr(sys.modules[__name__], "syslog"):
            self.omit("syslog isn't supported on this environment")

        self.assert_callable(callable_object)

        mark = 'Pikzie: %.20f' % random.random()
        syslog.syslog(mark)

        log_file = "/var/log/messages"
        command = ["tail", "-F", log_file]
        try:
            from subprocess import Popen, PIPE
            messages = Popen(command, stdout=PIPE, close_fds=True)
        except ImportError:
            from popen2 import Popen3
            messages = Popen3(command)
            messages.stdout = messages.fromchild

        fcntl.fcntl(messages.stdout, fcntl.F_SETFL,
                    os.O_NONBLOCK | fcntl.fcntl(messages.stdout, fcntl.F_GETFL))

        def search(pattern):
            if isinstance(pattern, str):
                pattern = re.compile(pattern)
            content = b''
            timeout = 1.5
            while len(select.select([messages.stdout], [], [], timeout)[0]) > 0:
                timeout = 0.1
                added_content = messages.stdout.read()
                if not added_content:
                    break
                content += added_content
                if re.search(pattern, str(content)):
                    return
            message = \
                "expected: <%s> is found in <%s>\n" \
                " content: <%s>" % \
                (pp.format_re(pattern),
                 pp.format(log_file),
                 pp.format(content))
            self.fail(message)

        try:
            search(re.escape(mark))
            result = callable_object(*args, **kw_args)
            search(pattern)
        finally:
            os.kill(messages.pid, signal.SIGINT)
            messages.wait()
Ejemplo n.º 12
0
def proc_run(cmd: str,
             break_lines: list = [],
             shell: bool = False,
             input_encode: str = 'utf-8',
             output_decode: str = 'utf-8'):
    def terminated_read(stdout: codecs.StreamReader, terminators: str) -> str:
        buf = []
        while stdout.readable():
            r = stdout.read(1)
            # print(r)
            buf.append(r)
            if r in terminators:
                break
        return "".join(buf)

    proc = Popen(shlex.split(cmd),
                 shell=shell,
                 stdin=PIPE,
                 stdout=PIPE,
                 stderr=STDOUT)
    try:
        try:
            proc.stdout = codecs.getreader(output_decode)(proc.stdout)
        except LookupError:
            raise LookupError

        while True:
            line = terminated_read(proc.stdout, "\n?")
            print(line.rstrip())
            if not line:
                break
            elif line.rstrip() in break_lines:
                proc.stdin.write(input().encode(input_encode))
                proc.stdin.close()
    except TimeoutExpired:
        print('Time out. Kill the process')
    finally:
        proc.kill()
Ejemplo n.º 13
0
def nopen_keep_parent_stdin(f, mode="r"):

    if f.startswith("|"):
        # using shell explicitly makes things like process substitution work:
        # http://stackoverflow.com/questions/7407667/python-subprocess-subshells-and-redirection
        # use sys.stderr so we dont have to worry about checking it...
        p = Popen(f[1:], stdout=PIPE, stdin=sys.stdin,
                  stderr=sys.stderr if mode == "r" else PIPE,
                  shell=True, bufsize=-1, # use system default for buffering
                  preexec_fn=toolshed.files.prefunc,
                  close_fds=False, executable=os.environ.get('SHELL'))
        if sys.version_info[0] > 2:
            import io
            p.stdout = io.TextIOWrapper(p.stdout)
            p.stdin = io.TextIOWrapper(sys.stdin)
            if mode != "r":
                p.stderr = io.TextIOWrapper(p.stderr)

        if mode and mode[0] == "r":
            return toolshed.files.process_iter(p, f[1:])
        return p
    else:
        return toolshed.files.nopen(f,mode)
Ejemplo n.º 14
0
    def shell(self):
        thr_id = threading.get_ident()
        p = self._shell.get(thr_id)
        if p:
            return p
        scmd = self._prep_scmd("bash -l", ssh=self._ssh)
        p = Popen(scmd,
                  shell=True,
                  stdin=PIPE,
                  stdout=PIPE,
                  stderr=STDOUT,
                  preexec_fn=os.setpgrp,
                  executable="/bin/bash")
        p.stdin = p.stdin.detach()  # don't use buffer
        p.stdout = p.stdout.detach()
        os.set_blocking(p.stdout.fileno(), False)
        if p.stderr:
            p.stderr = p.stderr.detach()
            os.set_blocking(p.stderr.fileno(), False)
        p.stdin.write(b'echo \xFF\n')
        while True:
            b = p.stdout.read()
            if b == b'\xFF\n':
                break
            if p.poll() is not None:
                return None
        self._shell[thr_id] = p
        p.read = p.stdout.read

        def p_write(p, data):
            off = 0
            while off < len(data):
                wlen = p.stdin.write(data[off:])
                off += wlen

        p.write = p_write.__get__(p, p.__class__)
        return p
Ejemplo n.º 15
0
def nopen(f, mode="r"):
    r"""
    open a file that's gzipped or return stdin for '-'
    if f is a number, the result of nopen(sys.argv[f]) is returned.

    >>> nopen('-') == sys.stdin, nopen('-', 'w') == sys.stdout
    (True, True)

    >>> nopen(sys.argv[0])
    <...file...>

    # expands user and vars ($HOME)
    >>> nopen("~/.bashrc").name == nopen("$HOME/.bashrc").name
    True

    # an already open file.
    >>> nopen(open(sys.argv[0]))
    <...file...>

    >>> nopen(0)
    <...file...>

    Or provide nicer access to Popen.stdout
    >>> files = list(nopen("|ls"))
    >>> assert 'setup.py\n' in files or b'setup.py\n' in files, files
    """
    if isinstance(f, int_types):
        return nopen(sys.argv[f], mode)

    if not isinstance(f, basestring):
        return f
    if f.startswith("|"):
        # using shell explicitly makes things like process substitution work:
        # http://stackoverflow.com/questions/7407667/python-subprocess-subshells-and-redirection
        # use sys.stderr so we dont have to worry about checking it...
        p = Popen(f[1:], stdout=PIPE, stdin=PIPE,
                  stderr=sys.stderr if mode == "r" else PIPE,
                  shell=True, bufsize=-1, # use system default for buffering
                  close_fds=False, executable=os.environ.get('SHELL'))
        if sys.version_info[0] > 2:
            import io
            p.stdout = io.TextIOWrapper(p.stdout)
            p.stdin = io.TextIOWrapper(p.stdin)
            if mode != "r":
                p.stderr = io.TextIOWrapper(p.stderr)

        if mode and mode[0] == "r":
            return process_iter(p, f[1:])
        return p

    if f.startswith(("http://", "https://", "ftp://")):
        fh = urlopen(f)
        if f.endswith(".gz"):
            return ungzipper(fh)
        if sys.version_info[0] < 3:
            return fh
        import io
        return io.TextIOWrapper(fh)
    f = op.expanduser(op.expandvars(f))
    if f.endswith((".gz", ".Z", ".z")):
        fh = gzip.open(f, mode)
        if sys.version_info[0] < 3:
            return fh
        import io
        return io.TextIOWrapper(fh)
    elif f.endswith((".bz", ".bz2", ".bzip2")):
        fh = bz2.BZ2File(f, mode)
        if sys.version_info[0] < 3:
            return fh
        import io
        return io.TextIOWrapper(fh)

    return {"r": sys.stdin, "w": sys.stdout}[mode[0]] if f == "-" \
         else open(f, mode)
Ejemplo n.º 16
0
def main():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        description="""Create token LM for input manifest and tokenizer.""",
    )
    parser.add_argument(
        "--manifest",
        required=True,
        type=str,
        help="Comma separated list of manifest files",
    )
    parser.add_argument(
        "--tokenizer_dir",
        required=True,
        type=str,
        help=
        "The directory path to the tokenizer vocabulary + additional metadata",
    )
    parser.add_argument(
        "--tokenizer_type",
        required=True,
        type=str,
        choices=["bpe", "wpe"],
        help="The type of the tokenizer. Currently supports `bpe` and `wpe`",
    )
    parser.add_argument(
        "--lm_builder",
        default="chain-est-phone-lm",
        type=str,
        help=
        ("The path or name of an LM builder. Supported builders: chain-est-phone-lm "
         "and scripts/asr_language_modeling/ngram_lm/make_phone_lm.py"),
    )
    parser.add_argument(
        "--ngram_order",
        type=int,
        default=2,
        choices=[2, 3, 4, 5],
        help="Order of n-gram to use",
    )
    parser.add_argument(
        "--output_file",
        required=True,
        type=str,
        help="The path to store the token LM",
    )
    parser.add_argument(
        "--do_lowercase",
        action="store_true",
        help="Whether to apply lower case conversion on the text",
    )
    args = parser.parse_args()

    is_chain_builder = Path(args.lm_builder).stem == "chain-est-phone-lm"
    """ TOKENIZER SETUP """
    logging.info(
        f"Loading {args.tokenizer_type} tokenizer from '{args.tokenizer_dir}' ..."
    )
    if args.tokenizer_type == "bpe":
        # This is a BPE Tokenizer
        model_path = os.path.join(args.tokenizer_dir, "tokenizer.model")

        # Update special tokens
        tokenizer = tokenizers.SentencePieceTokenizer(model_path=model_path)
    else:
        # This is a WPE Tokenizer
        vocab_path = os.path.join(args.tokenizer_dir, "vocab.txt")
        tokenizer = tokenizers.AutoTokenizer(
            pretrained_model_name="bert-base-cased", vocab_file=vocab_path)

    logging.info(
        f"Tokenizer {tokenizer.__class__.__name__} loaded with {tokenizer.vocab_size} tokens"
    )
    """ DATA PROCESSING """
    if "," in args.manifest:
        manifests = args.manifest.split(",")
    else:
        manifests = [args.manifest]

    offset = 1  # tokens in token LM cannot be 0
    tok_text_list = []
    num_lines = 0
    for manifest in manifests:
        logging.info(f"Processing manifest : {manifest} ...")
        with open(manifest, "r") as in_reader:
            for line in in_reader:
                item = json.loads(line)
                text = item["text"]
                if args.do_lowercase:
                    text = text.lower()
                tok_text = " ".join(
                    [str(i + offset) for i in tokenizer.text_to_ids(text)])
                if is_chain_builder:
                    tok_text = f"line_{num_lines} " + tok_text
                tok_text_list.append(tok_text)
                num_lines += 1

    tok_texts = "\n".join(tok_text_list)
    del tok_text_list
    logging.info(
        "Finished processing all manifests ! Number of sentences : {}".format(
            num_lines))
    """ LM BUILDING """
    logging.info(f"Calling {args.lm_builder} ...")
    if is_chain_builder:
        pipe_args = [
            args.lm_builder,
            f"--ngram-order={args.ngram_order}",
            f"--no-prune-ngram-order={args.ngram_order}",
            "ark:-",
            "-",
        ]
        p1 = Popen(pipe_args, stdin=PIPE, stdout=PIPE, text=True)
        p2 = Popen(["fstprint"], stdin=p1.stdout, stdout=PIPE, text=True)
        p1.stdout.close()
        p1.stdout = None
        Thread(target=p1.communicate, args=[tok_texts]).start()
        out, err = p2.communicate()
    else:
        pipe_args = [
            args.lm_builder,
            f"--ngram-order={args.ngram_order}",
            f"--no-backoff-ngram-order={args.ngram_order}",
            "--phone-disambig-symbol=-11",
        ]
        p1 = Popen(pipe_args, stdout=PIPE, stdin=PIPE, text=True)
        out, err = p1.communicate(tok_texts)

    logging.info(f"LM is built, writing to {args.output_file} ...")
    with open(args.output_file, "w", encoding="utf-8") as f:
        f.write(out)
    logging.info(f"Done writing to '{args.output_file}'.")
Ejemplo n.º 17
0
def nopen(f, mode="r"):
    r"""
    open a file that's gzipped or return stdin for '-'
    if f is a number, the result of nopen(sys.argv[f]) is returned.

    >>> nopen('-') == sys.stdin, nopen('-', 'w') == sys.stdout
    (True, True)

    >>> nopen(sys.argv[0])
    <...file...>

    # expands user and vars ($HOME)
    >>> nopen("~/.bashrc").name == nopen("$HOME/.bashrc").name
    True

    # an already open file.
    >>> nopen(open(sys.argv[0]))
    <...file...>

    >>> nopen(0)
    <...file...>

    Or provide nicer access to Popen.stdout
    >>> files = list(nopen("|ls"))
    >>> assert 'setup.py\n' in files or b'setup.py\n' in files, files
    """
    if isinstance(f, int_types):
        return nopen(sys.argv[f], mode)

    if not isinstance(f, basestring):
        return f
    if f.startswith("|"):
        # using shell explicitly makes things like process substitution work:
        # http://stackoverflow.com/questions/7407667/python-subprocess-subshells-and-redirection
        # use sys.stderr so we dont have to worry about checking it...
        p = Popen(
            f[1:],
            stdout=PIPE,
            stdin=PIPE,
            stderr=sys.stderr if mode == "r" else PIPE,
            shell=True,
            bufsize=-1,  # use system default for buffering
            close_fds=False,
            executable=os.environ.get('SHELL'))
        if sys.version_info[0] > 2:
            import io
            p.stdout = io.TextIOWrapper(p.stdout)
            p.stdin = io.TextIOWrapper(p.stdin)
            if mode != "r":
                p.stderr = io.TextIOWrapper(p.stderr)

        if mode and mode[0] == "r":
            return process_iter(p, f[1:])
        return p

    if f.startswith(("http://", "https://", "ftp://")):
        fh = urlopen(f)
        if f.endswith(".gz"):
            return ungzipper(fh)
        if sys.version_info[0] < 3:
            return fh
        import io
        return io.TextIOWrapper(fh)
    f = op.expanduser(op.expandvars(f))
    if f.endswith((".gz", ".Z", ".z")):
        fh = gzip.open(f, mode)
        if sys.version_info[0] < 3:
            return fh
        import io
        return io.TextIOWrapper(fh)
    elif f.endswith((".bz", ".bz2", ".bzip2")):
        fh = bz2.BZ2File(f, mode)
        if sys.version_info[0] < 3:
            return fh
        import io
        return io.TextIOWrapper(fh)

    return {"r": sys.stdin, "w": sys.stdout}[mode[0]] if f == "-" \
         else open(f, mode)
Ejemplo n.º 18
0
Archivo: waiter.py Proyecto: o11c/mypy
 def __call__(self) -> Popen:
     tf = tempfile.TemporaryFile()
     rv = Popen(self.args, cwd=self.cwd, env=self.env, stdout=tf, stderr=STDOUT)
     rv.stdout = tf
     return rv
Ejemplo n.º 19
0
def run_subproc(cmds, captured=True):
    """Runs a subprocess, in its many forms. This takes a list of 'commands,'
    which may be a list of command line arguments or a string, representing
    a special connecting character.  For example::

        $ ls | grep wakka

    is represented by the following cmds::

        [['ls'], '|', ['grep', 'wakka']]

    Lastly, the captured argument affects only the last real command.
    """
    global ENV
    last_stdout = PIPE if captured else None
    background = False
    if cmds[-1] == '&':
        background = True
        cmds = cmds[:-1]
    write_target = None
    if len(cmds) >= 3 and cmds[-2] in WRITER_MODES:
        write_target = cmds[-1][0]
        write_mode = WRITER_MODES[cmds[-2]]
        cmds = cmds[:-2]
        if write_target is not None:
            try:
                last_stdout = open(write_target, write_mode)
            except FileNotFoundError:
                e = 'xonsh: {0}: no such file or directory'
                raise XonshError(e.format(write_target))
        else:
            last_stdout = PIPE
    last_cmd = cmds[-1]
    prev = None
    procs = []
    prev_proc = None
    for cmd in cmds:
        if isinstance(cmd, string_types):
            prev = cmd
            continue
        stdout = last_stdout if cmd is last_cmd else PIPE
        uninew = cmd is last_cmd
        alias = builtins.aliases.get(cmd[0], None)
        if _is_runnable_name(cmd[0]):
            try:
                aliased_cmd = get_script_subproc_command(cmd[0], cmd[1:])
            except PermissionError:
                e = 'xonsh: subprocess mode: permission denied: {0}'
                raise XonshError(e.format(cmd[0]))
        elif alias is None:
            aliased_cmd = cmd
        elif callable(alias):
            aliased_cmd = alias
        else:
            aliased_cmd = alias + cmd[1:]
        # compute stdin for subprocess
        if prev_proc is None:
            stdin = None
        else:
            stdin = prev_proc.stdout
        if callable(aliased_cmd):
            prev_is_proxy = True
            numargs = len(inspect.signature(aliased_cmd).parameters)
            if numargs == 2:
                cls = SimpleProcProxy
            elif numargs == 4:
                cls = ProcProxy
            else:
                e = 'Expected callable with 2 or 4 arguments, not {}'
                raise XonshError(e.format(numargs))
            proc = cls(aliased_cmd, cmd[1:],
                       stdin, stdout, None,
                       universal_newlines=uninew)
        else:
            prev_is_proxy = False
            subproc_kwargs = {}
            if os.name == 'posix':
                subproc_kwargs['preexec_fn'] = _subproc_pre
            try:
                proc = Popen(aliased_cmd,
                             universal_newlines=uninew,
                             env=ENV.detype(),
                             stdin=stdin,
                             stdout=PIPE,
                             stderr=PIPE, **subproc_kwargs)
                proc_out, proc_err = proc.communicate()
                sys.stdout.write(proc_out)
                sys.stderr.write(proc_err)
                proc.stdout = sys.stdout 
                proc.stderr = sys.stderr
            except PermissionError:
                e = 'xonsh: subprocess mode: permission denied: {0}'
                raise XonshError(e.format(aliased_cmd[0]))
            except FileNotFoundError:
                cmd = aliased_cmd[0]
                e = 'xonsh: subprocess mode: command not found: {0}'.format(cmd)
                e += '\n' + suggest_commands(cmd, ENV, builtins.aliases)
                raise XonshError(e)
        procs.append(proc)
        prev = None
        prev_proc = proc
    for proc in procs[:-1]:
        try:
            proc.stdout.close()
        except OSError:
            pass
    if not prev_is_proxy:
        add_job({
            'cmds': cmds,
            'pids': [i.pid for i in procs],
            'obj': prev_proc,
            'bg': background
        })
    if background:
        return
    if prev_is_proxy:
        prev_proc.wait()
    wait_for_active_job()
    if write_target is None:
        # get output
        output = ''
        if prev_proc.stdout not in (None, sys.stdout):
            output = prev_proc.stdout.read()
        if captured:
            return output
    elif last_stdout not in (PIPE, None, sys.stdout):
        last_stdout.close()
Ejemplo n.º 20
0
def ssh(host,
        cmd='',
        input=None,
        interactive=False,
        wait=True,
        port=22,
        **kwargs):
    """a utility that mimics Popen() and run() but under `ssh` command

    Examples
    --------
    # interactive
    >>> p = ssh('node1', interactive=True)
    >>> p.stdin.write(b'hostname\n')
    9
    >>> p.stdout.read()
    b'node1\n'
    >>> p.stdin.write(b'pwd\n')
    4
    >>> p.stdout.read()
    b'/root\n'
    >>>

    # one shot
    >>> p = ssh('node1', cmd='hostname')
    >>> p.wait()
    0
    >>> p.stdout.read()
    b'node1\n'

    # one shot with input
    >>> p = ssh('node1', cmd='bash', input='echo $HOSTNAME')
    >>> p.wait()
    0
    >>> p.stdout.read()
    b'node1\n'

    Returns
    -------
    obj(Popen): a Popen object for the ssh process
    """
    _kwargs = dict(shell=True,
                   stdout=PIPE,
                   stdin=PIPE,
                   stderr=STDOUT,
                   executable="/bin/bash")
    _kwargs.update(kwargs)
    p = Popen("ssh -T {} -p {} {}".format(host, port, cmd), **_kwargs)
    p.stdin = p.stdin.detach()  # don't use buffer
    p.stdout = p.stdout.detach()
    os.set_blocking(p.stdout.fileno(), False)
    if p.stderr:
        p.stderr = p.stderr.detach()
        os.set_blocking(p.stderr.fileno(), False)
    if input:
        p.stdin.write(BYTES(input))
    if not interactive:
        p.stdin.close()
        if wait:
            p.wait()
    return p
Ejemplo n.º 21
0
    def execute(self,
                command: Union[str, Sequence[Any]],
                istream: Union[None, BinaryIO] = None,
                with_extended_output: bool = False,
                with_exceptions: bool = True,
                as_process: bool = False,
                output_stream: Union[None, BinaryIO] = None,
                stdout_as_string: bool = True,
                kill_after_timeout: Union[None, int] = None,
                with_stdout: bool = True,
                universal_newlines: bool = False,
                shell: Union[None, bool] = None,
                env: Union[None, Mapping[str, str]] = None,
                max_chunk_size: int = io.DEFAULT_BUFFER_SIZE,
                **subprocess_kwargs: Any
                ) -> Union[str, bytes, Tuple[int, Union[str, bytes], str], AutoInterrupt]:
        """Handles executing the command on the shell and consumes and returns
        the returned information (stdout)

        :param command:
            The command argument list to execute.
            It should be a string, or a sequence of program arguments. The
            program to execute is the first item in the args sequence or string.

        :param istream:
            Standard input filehandle passed to subprocess.Popen.

        :param with_extended_output:
            Whether to return a (status, stdout, stderr) tuple.

        :param with_exceptions:
            Whether to raise an exception when git returns a non-zero status.

        :param as_process:
            Whether to return the created process instance directly from which
            streams can be read on demand. This will render with_extended_output and
            with_exceptions ineffective - the caller will have
            to deal with the details himself.
            It is important to note that the process will be placed into an AutoInterrupt
            wrapper that will interrupt the process once it goes out of scope. If you
            use the command in iterators, you should pass the whole process instance
            instead of a single stream.

        :param output_stream:
            If set to a file-like object, data produced by the git command will be
            output to the given stream directly.
            This feature only has any effect if as_process is False. Processes will
            always be created with a pipe due to issues with subprocess.
            This merely is a workaround as data will be copied from the
            output pipe to the given output stream directly.
            Judging from the implementation, you shouldn't use this flag !

        :param stdout_as_string:
            if False, the commands standard output will be bytes. Otherwise, it will be
            decoded into a string using the default encoding (usually utf-8).
            The latter can fail, if the output contains binary data.

        :param env:
            A dictionary of environment variables to be passed to `subprocess.Popen`.

        :param max_chunk_size:
            Maximum number of bytes in one chunk of data passed to the output_stream in
            one invocation of write() method. If the given number is not positive then
            the default value is used.

        :param subprocess_kwargs:
            Keyword arguments to be passed to subprocess.Popen. Please note that
            some of the valid kwargs are already set by this method, the ones you
            specify may not be the same ones.

        :param with_stdout: If True, default True, we open stdout on the created process
        :param universal_newlines:
            if True, pipes will be opened as text, and lines are split at
            all known line endings.
        :param shell:
            Whether to invoke commands through a shell (see `Popen(..., shell=True)`).
            It overrides :attr:`USE_SHELL` if it is not `None`.
        :param kill_after_timeout:
            To specify a timeout in seconds for the git command, after which the process
            should be killed. This will have no effect if as_process is set to True. It is
            set to None by default and will let the process run until the timeout is
            explicitly specified. This feature is not supported on Windows. It's also worth
            noting that kill_after_timeout uses SIGKILL, which can have negative side
            effects on a repository. For example, stale locks in case of git gc could
            render the repository incapable of accepting changes until the lock is manually
            removed.

        :return:
            * str(output) if extended_output = False (Default)
            * tuple(int(status), str(stdout), str(stderr)) if extended_output = True

            if output_stream is True, the stdout value will be your output stream:
            * output_stream if extended_output = False
            * tuple(int(status), output_stream, str(stderr)) if extended_output = True

            Note git is executed with LC_MESSAGES="C" to ensure consistent
            output regardless of system language.

        :raise GitCommandError:

        :note:
           If you add additional keyword arguments to the signature of this method,
           you must update the execute_kwargs tuple housed in this module."""
        # Remove password for the command if present
        redacted_command = remove_password_if_present(command)
        if self.GIT_PYTHON_TRACE and (self.GIT_PYTHON_TRACE != 'full' or as_process):
            log.info(' '.join(redacted_command))

        # Allow the user to have the command executed in their working dir.
        cwd = self._working_dir or os.getcwd()

        # Start the process
        inline_env = env
        env = os.environ.copy()
        # Attempt to force all output to plain ascii english, which is what some parsing code
        # may expect.
        # According to stackoverflow (http://goo.gl/l74GC8), we are setting LANGUAGE as well
        # just to be sure.
        env["LANGUAGE"] = "C"
        env["LC_ALL"] = "C"
        env.update(self._environment)
        if inline_env is not None:
            env.update(inline_env)

        if is_win:
            cmd_not_found_exception = OSError
            if kill_after_timeout:
                raise GitCommandError(redacted_command, '"kill_after_timeout" feature is not supported on Windows.')
        else:
            if sys.version_info[0] > 2:
                cmd_not_found_exception = FileNotFoundError  # NOQA # exists, flake8 unknown @UndefinedVariable
            else:
                cmd_not_found_exception = OSError
        # end handle

        stdout_sink = (PIPE
                       if with_stdout
                       else getattr(subprocess, 'DEVNULL', None) or open(os.devnull, 'wb'))
        istream_ok = "None"
        if istream:
            istream_ok = "<valid stream>"
        log.debug("Popen(%s, cwd=%s, universal_newlines=%s, shell=%s, istream=%s)",
                  redacted_command, cwd, universal_newlines, shell, istream_ok)
        try:
            proc = Popen(command,
                         env=env,
                         cwd=cwd,
                         bufsize=-1,
                         stdin=istream,
                         stderr=PIPE,
                         stdout=stdout_sink,
                         shell=shell is not None and shell or self.USE_SHELL,
                         close_fds=is_posix,  # unsupported on windows
                         universal_newlines=universal_newlines,
                         creationflags=PROC_CREATIONFLAGS,
                         **subprocess_kwargs
                         )

        except cmd_not_found_exception as err:
            raise GitCommandNotFound(redacted_command, err) from err
        else:
            proc = cast(Popen, proc)
            proc.stdout = cast(BinaryIO, proc.stdout)
            proc.stderr = cast(BinaryIO, proc.stderr)

        if as_process:
            return self.AutoInterrupt(proc, command)

        def _kill_process(pid: int) -> None:
            """ Callback method to kill a process. """
            p = Popen(['ps', '--ppid', str(pid)], stdout=PIPE,
                      creationflags=PROC_CREATIONFLAGS)
            child_pids = []
            if p.stdout is not None:
                for line in p.stdout:
                    if len(line.split()) > 0:
                        local_pid = (line.split())[0]
                        if local_pid.isdigit():
                            child_pids.append(int(local_pid))
            try:
                # Windows does not have SIGKILL, so use SIGTERM instead
                sig = getattr(signal, 'SIGKILL', signal.SIGTERM)
                os.kill(pid, sig)
                for child_pid in child_pids:
                    try:
                        os.kill(child_pid, sig)
                    except OSError:
                        pass
                kill_check.set()    # tell the main routine that the process was killed
            except OSError:
                # It is possible that the process gets completed in the duration after timeout
                # happens and before we try to kill the process.
                pass
            return
        # end

        if kill_after_timeout:
            kill_check = threading.Event()
            watchdog = threading.Timer(kill_after_timeout, _kill_process, args=(proc.pid,))

        # Wait for the process to return
        status = 0
        stdout_value = b''  # type: Union[str, bytes]
        stderr_value = b''  # type: Union[str, bytes]
        newline = "\n" if universal_newlines else b"\n"
        try:
            if output_stream is None:
                if kill_after_timeout:
                    watchdog.start()
                stdout_value, stderr_value = proc.communicate()
                if kill_after_timeout:
                    watchdog.cancel()
                    if kill_check.is_set():
                        stderr_value = ('Timeout: the command "%s" did not complete in %d '
                                        'secs.' % (" ".join(redacted_command), kill_after_timeout))
                        if not universal_newlines:
                            stderr_value = stderr_value.encode(defenc)
                # strip trailing "\n"
                if stdout_value.endswith(newline):  # type: ignore
                    stdout_value = stdout_value[:-1]
                if stderr_value.endswith(newline):  # type: ignore
                    stderr_value = stderr_value[:-1]

                status = proc.returncode
            else:
                max_chunk_size = max_chunk_size if max_chunk_size and max_chunk_size > 0 else io.DEFAULT_BUFFER_SIZE
                stream_copy(proc.stdout, output_stream, max_chunk_size)
                stdout_value = proc.stdout.read()
                stderr_value = proc.stderr.read()
                # strip trailing "\n"
                if stderr_value.endswith(newline):   # type: ignore
                    stderr_value = stderr_value[:-1]
                status = proc.wait()
            # END stdout handling
        finally:
            proc.stdout.close()
            proc.stderr.close()

        if self.GIT_PYTHON_TRACE == 'full':
            cmdstr = " ".join(redacted_command)

            def as_text(stdout_value):
                return not output_stream and safe_decode(stdout_value) or '<OUTPUT_STREAM>'
            # end

            if stderr_value:
                log.info("%s -> %d; stdout: '%s'; stderr: '%s'",
                         cmdstr, status, as_text(stdout_value), safe_decode(stderr_value))
            elif stdout_value:
                log.info("%s -> %d; stdout: '%s'", cmdstr, status, as_text(stdout_value))
            else:
                log.info("%s -> %d", cmdstr, status)
        # END handle debug printing

        if with_exceptions and status != 0:
            raise GitCommandError(redacted_command, status, stderr_value, stdout_value)

        if isinstance(stdout_value, bytes) and stdout_as_string:  # could also be output_stream
            stdout_value = safe_decode(stdout_value)

        # Allow access to the command's status code
        if with_extended_output:
            return (status, stdout_value, safe_decode(stderr_value))
        else:
            return stdout_value