def main(): import argparse # Helper error handling routines def perror(message): sys.stderr.write(message) sys.stderr.write('\n') def error(message, filename=None, location=None): if location: args = (filename,) + location + (message,) perror("%s:%d:%d: error: %s" % args) elif filename: perror("%s: error: %s" % (filename, message)) else: perror("error: %s" % message) sys.exit(1) # Parse the arguments and options parser = argparse.ArgumentParser(prog='python -m tokenize') parser.add_argument(dest='filename', nargs='?', metavar='filename.py', help='the file to tokenize; defaults to stdin') parser.add_argument('-e', '--exact', dest='exact', action='store_true', help='display token names using the exact type') args = parser.parse_args() try: # Tokenize the input if args.filename: filename = args.filename with _builtin_open(filename, 'rb') as f: tokens = list(tokenize(f.readline)) else: filename = "<stdin>" tokens = _tokenize(sys.stdin.readline, None) # Output the tokenization for token in tokens: token_type = token.type if args.exact: token_type = token.exact_type token_range = "%d,%d-%d,%d:" % (token.start + token.end) print("%-20s%-15s%-15r" % (token_range, tok_name[token_type], token.string)) except IndentationError as err: line, column = err.args[1][1:3] error(err.args[0], filename, (line, column)) except TokenError as err: line, column = err.args[1] error(err.args[0], filename, (line, column)) except SyntaxError as err: error(err, filename) except OSError as err: error(err) except KeyboardInterrupt: print("interrupted\n") except Exception as err: perror("unexpected error: %s" % err) raise
def __init__(self, filename, mode="r", *, compresslevel=9): """Open a bzip2-compressed file. If filename is a str, bytes, or PathLike object, it gives the name of the file to be opened. Otherwise, it should be a file object, which will be used to read or write the compressed data. mode can be 'r' for reading (default), 'w' for (over)writing, 'x' for creating exclusively, or 'a' for appending. These can equivalently be given as 'rb', 'wb', 'xb', and 'ab'. If mode is 'w', 'x' or 'a', compresslevel can be a number between 1 and 9 specifying the level of compression: 1 produces the least compression, and 9 (default) produces the most compression. If mode is 'r', the input file may be the concatenation of multiple compressed streams. """ self._fp = None self._closefp = False self._mode = _MODE_CLOSED if not (1 <= compresslevel <= 9): raise ValueError("compresslevel must be between 1 and 9") if mode in ("", "r", "rb"): mode = "rb" mode_code = _MODE_READ elif mode in ("w", "wb"): mode = "wb" mode_code = _MODE_WRITE self._compressor = BZ2Compressor(compresslevel) elif mode in ("x", "xb"): mode = "xb" mode_code = _MODE_WRITE self._compressor = BZ2Compressor(compresslevel) elif mode in ("a", "ab"): mode = "ab" mode_code = _MODE_WRITE self._compressor = BZ2Compressor(compresslevel) else: raise ValueError("Invalid mode: %r" % (mode,)) if isinstance(filename, (str, bytes, os.PathLike)): self._fp = _builtin_open(filename, mode) self._closefp = True self._mode = mode_code elif hasattr(filename, "read") or hasattr(filename, "write"): self._fp = filename self._mode = mode_code else: raise TypeError("filename must be a str, bytes, file or PathLike object") if self._mode == _MODE_READ: raw = _compression.DecompressReader(self._fp, BZ2Decompressor, trailing_error=OSError) self._buffer = io.BufferedReader(raw) else: self._pos = 0
def open(filename): """Open a file in read only mode using the encoding detected by detect_encoding(). """ buffer = _builtin_open(filename, 'rb') encoding, lines = detect_encoding(buffer.readline) buffer.seek(0) text = TextIOWrapper(buffer, encoding, line_buffering=True) text.mode = 'r' return text
def open(filename): """Open a file in read only mode using the encoding detected by detect_encoding(). """ buffer = _builtin_open(filename, "rb") try: encoding, lines = detect_encoding(buffer.readline) buffer.seek(0) text = TextIOWrapper(buffer, encoding, line_buffering=True) text.mode = "r" return text except: buffer.close() raise
def open(filename): """Open a file in read only mode using the encoding detected by detect_encoding(). """ # TRUFFLE TODO: revert-me buffer = _builtin_open(filename, 'rb', buffering=0) # buffer = _builtin_open(filename, 'rb') try: encoding, lines = detect_encoding(buffer.readline) buffer.seek(0) return buffer # TRUFFLE TODO: remove-me text = TextIOWrapper(buffer, encoding, line_buffering=True) text.mode = 'r' return text except: buffer.close() raise
def __init__(self, filename, mode="r", buffering=None, compresslevel=9): """Open a bzip2-compressed file. If filename is a str or bytes object, it gives the name of the file to be opened. Otherwise, it should be a file object, which will be used to read or write the compressed data. mode can be 'r' for reading (default), 'w' for (over)writing, 'x' for creating exclusively, or 'a' for appending. These can equivalently be given as 'rb', 'wb', 'xb', and 'ab'. buffering is ignored. Its use is deprecated. If mode is 'w', 'x' or 'a', compresslevel can be a number between 1 and 9 specifying the level of compression: 1 produces the least compression, and 9 (default) produces the most compression. If mode is 'r', the input file may be the concatenation of multiple compressed streams. """ # This lock must be recursive, so that BufferedIOBase's # writelines() does not deadlock. self._lock = RLock() self._fp = None self._closefp = False self._mode = _MODE_CLOSED if buffering is not None: warnings.warn("Use of 'buffering' argument is deprecated", DeprecationWarning) if not (1 <= compresslevel <= 9): raise ValueError("compresslevel must be between 1 and 9") if mode in ("", "r", "rb"): mode = "rb" mode_code = _MODE_READ elif mode in ("w", "wb"): mode = "wb" mode_code = _MODE_WRITE self._compressor = BZ2Compressor(compresslevel) elif mode in ("x", "xb"): mode = "xb" mode_code = _MODE_WRITE self._compressor = BZ2Compressor(compresslevel) elif mode in ("a", "ab"): mode = "ab" mode_code = _MODE_WRITE self._compressor = BZ2Compressor(compresslevel) else: raise ValueError("Invalid mode: %r" % (mode,)) if isinstance(filename, (str, bytes)): self._fp = _builtin_open(filename, mode) self._closefp = True self._mode = mode_code elif hasattr(filename, "read") or hasattr(filename, "write"): self._fp = filename self._mode = mode_code else: raise TypeError("filename must be a str or bytes object, or a file") if self._mode == _MODE_READ: raw = _compression.DecompressReader(self._fp, BZ2Decompressor, trailing_error=OSError) self._buffer = io.BufferedReader(raw) else: self._pos = 0
def __init__(self, filename, mode="r", buffering=None, compresslevel=9): """Open a bzip2-compressed file. If filename is a str or bytes object, it gives the name of the file to be opened. Otherwise, it should be a file object, which will be used to read or write the compressed data. mode can be 'r' for reading (default), 'w' for (over)writing, 'x' for creating exclusively, or 'a' for appending. These can equivalently be given as 'rb', 'wb', 'xb', and 'ab'. buffering is ignored. Its use is deprecated. If mode is 'w', 'x' or 'a', compresslevel can be a number between 1 and 9 specifying the level of compression: 1 produces the least compression, and 9 (default) produces the most compression. If mode is 'r', the input file may be the concatenation of multiple compressed streams. """ # This lock must be recursive, so that BufferedIOBase's # readline(), readlines() and writelines() don't deadlock. self._lock = RLock() self._fp = None self._closefp = False self._mode = _MODE_CLOSED self._pos = 0 self._size = -1 if buffering is not None: warnings.warn("Use of 'buffering' argument is deprecated", DeprecationWarning) if not (1 <= compresslevel <= 9): raise ValueError("compresslevel must be between 1 and 9") if mode in ("", "r", "rb"): mode = "rb" mode_code = _MODE_READ self._decompressor = BZ2Decompressor() self._buffer = b"" self._buffer_offset = 0 elif mode in ("w", "wb"): mode = "wb" mode_code = _MODE_WRITE self._compressor = BZ2Compressor(compresslevel) elif mode in ("x", "xb"): mode = "xb" mode_code = _MODE_WRITE self._compressor = BZ2Compressor(compresslevel) elif mode in ("a", "ab"): mode = "ab" mode_code = _MODE_WRITE self._compressor = BZ2Compressor(compresslevel) else: raise ValueError("Invalid mode: %r" % (mode, )) if isinstance(filename, (str, bytes)): self._fp = _builtin_open(filename, mode) self._closefp = True self._mode = mode_code elif hasattr(filename, "read") or hasattr(filename, "write"): self._fp = filename self._mode = mode_code else: raise TypeError( "filename must be a str or bytes object, or a file")
def __init__(self, filename, mode='r', buffering=None, compresslevel=9): """Open a bzip2-compressed file. If filename is a str, bytes, or PathLike object, it gives the name of the file to be opened. Otherwise, it should be a file object, which will be used to read or write the compressed data. mode can be 'r' for reading (default), 'w' for (over)writing, 'x' for creating exclusively, or 'a' for appending. These can equivalently be given as 'rb', 'wb', 'xb', and 'ab'. buffering is ignored. Its use is deprecated. If mode is 'w', 'x' or 'a', compresslevel can be a number between 1 and 9 specifying the level of compression: 1 produces the least compression, and 9 (default) produces the most compression. If mode is 'r', the input file may be the concatenation of multiple compressed streams. """ self._lock = RLock() self._fp = None self._closefp = False self._mode = _MODE_CLOSED if buffering is not None: warnings.warn("Use of 'buffering' argument is deprecated", DeprecationWarning) if not 1 <= compresslevel <= 9: raise ValueError('compresslevel must be between 1 and 9') if mode in ('', 'r', 'rb'): mode = 'rb' mode_code = _MODE_READ elif mode in ('w', 'wb'): mode = 'wb' mode_code = _MODE_WRITE self._compressor = BZ2Compressor(compresslevel) elif mode in ('x', 'xb'): mode = 'xb' mode_code = _MODE_WRITE self._compressor = BZ2Compressor(compresslevel) elif mode in ('a', 'ab'): mode = 'ab' mode_code = _MODE_WRITE self._compressor = BZ2Compressor(compresslevel) else: raise ValueError('Invalid mode: %r' % (mode, )) if isinstance(filename, (str, bytes, os.PathLike)): self._fp = _builtin_open(filename, mode) self._closefp = True self._mode = mode_code elif hasattr(filename, 'read') or hasattr(filename, 'write'): self._fp = filename self._mode = mode_code else: raise TypeError( 'filename must be a str, bytes, file or PathLike object') if self._mode == _MODE_READ: raw = _compression.DecompressReader((self._fp), BZ2Decompressor, trailing_error=OSError) self._buffer = io.BufferedReader(raw) else: self._pos = 0
def main(): import argparse def perror(message): print(message, file=(sys.stderr)) def error(message, filename=None, location=None): if location: args = (filename, ) + location + (message, ) perror('%s:%d:%d: error: %s' % args) elif filename: perror('%s: error: %s' % (filename, message)) else: perror('error: %s' % message) sys.exit(1) parser = argparse.ArgumentParser(prog='python -m tokenize') parser.add_argument(dest='filename', nargs='?', metavar='filename.py', help='the file to tokenize; defaults to stdin') parser.add_argument('-e', '--exact', dest='exact', action='store_true', help='display token names using the exact type') args = parser.parse_args() try: if args.filename: filename = args.filename with _builtin_open(filename, 'rb') as (f): tokens = list(tokenize(f.readline)) else: filename = '<stdin>' tokens = _tokenize(sys.stdin.readline, None) for token in tokens: token_type = token.type if args.exact: token_type = token.exact_type token_range = '%d,%d-%d,%d:' % (token.start + token.end) print('%-20s%-15s%-15r' % (token_range, tok_name[token_type], token.string)) except IndentationError as err: try: line, column = err.args[1][1:3] error(err.args[0], filename, (line, column)) finally: err = None del err except TokenError as err: try: line, column = err.args[1] error(err.args[0], filename, (line, column)) finally: err = None del err except SyntaxError as err: try: error(err, filename) finally: err = None del err except OSError as err: try: error(err) finally: err = None del err except KeyboardInterrupt: print('interrupted\n') except Exception as err: try: perror('unexpected error: %s' % err) raise finally: err = None del err
sys.exit(1) # Parse the arguments and options parser = argparse.ArgumentParser(prog='python -m tokenize') parser.add_argument(dest='filename', nargs='?', metavar='filename.py', help='the file to tokenize; defaults to stdin') parser.add_argument('-e', '--exact', dest='exact', action='store_true', help='display token names using the exact type') args = parser.parse_args() try: # Tokenize the input if args.filename: filename = args.filename with _builtin_open(filename, 'rb') as f: tokens = list(tokenize(f.readline)) else: filename = "<stdin>" tokens = _tokenize(sys.stdin.readline, None) # Output the tokenization for token in tokens: token_type = token.type if args.exact: token_type = token.exact_type token_range = "%d,%d-%d,%d:" % (token.start + token.end) print("%-20s%-15s%-15r" % (token_range, tok_name[token_type], token.string)) except IndentationError as err: line, column = err.args[1][1:3]