def read_py_url(url, errors='replace', skip_encoding_cookie=True): """Read a Python file from a URL, using the encoding declared inside the file. Parameters ---------- url : str The URL from which to fetch the file. errors : str How to handle decoding errors in the file. Options are the same as for bytes.decode(), but here 'replace' is the default. skip_encoding_cookie : bool If True (the default), and the encoding declaration is found in the first two lines, that line will be excluded from the output - compiling a unicode string with an encoding declaration is a SyntaxError in Python 2. Returns ------- A unicode string containing the contents of the file. """ response = urllib.request.urlopen(url) buffer = io.BytesIO(response.read()) encoding, lines = detect_encoding(buffer.readline) buffer.seek(0) text = TextIOWrapper(buffer, encoding, errors=errors, line_buffering=True) text.mode = 'r' if skip_encoding_cookie: return "".join(strip_encoding_cookie(text)) else: return text.read()
def open_source(fname): buffer = open(fname, 'rb') encoding, _ = detect_encoding(buffer.readline) buffer.seek(0) text = TextIOWrapper(buffer, encoding, line_buffering=True) text.mode = 'r' return text
def open(filename): buffer = builtins.open(filename, 'rb') (encoding, lines) = detect_encoding(buffer.readline) buffer.seek(0) text = TextIOWrapper(buffer, encoding, line_buffering=True) text.mode = 'r' return text
def read_py_url(url, errors='replace', skip_encoding_cookie=True): """Read a Python file from a URL, using the encoding declared inside the file. Parameters ---------- url : str The URL from which to fetch the file. errors : str How to handle decoding errors in the file. Options are the same as for bytes.decode(), but here 'replace' is the default. skip_encoding_cookie : bool If True (the default), and the encoding declaration is found in the first two lines, that line will be excluded from the output - compiling a unicode string with an encoding declaration is a SyntaxError in Python 2. Returns ------- A unicode string containing the contents of the file. """ response = urllib.urlopen(url) buffer = io.BytesIO(response.read()) encoding, lines = detect_encoding(buffer.readline) buffer.seek(0) text = TextIOWrapper(buffer, encoding, errors=errors, line_buffering=True) text.mode = 'r' if skip_encoding_cookie: return "".join(strip_encoding_cookie(text)) else: return text.read()
def source_to_unicode(txt, errors='replace', skip_encoding_cookie=True): """Converts a bytes string with python source code to unicode. Unicode strings are passed through unchanged. Byte strings are checked for the python source file encoding cookie to determine encoding. txt can be either a bytes buffer or a string containing the source code. """ if isinstance(txt, unicode_type): return txt if isinstance(txt, bytes): buffer = BytesIO(txt) else: buffer = txt try: encoding, _ = detect_encoding(buffer.readline) except SyntaxError: encoding = "ascii" buffer.seek(0) text = TextIOWrapper(buffer, encoding, errors=errors, line_buffering=True) text.mode = 'r' if skip_encoding_cookie: return u"".join(strip_encoding_cookie(text)) else: return text.read()
def source_to_unicode(txt, errors='replace', skip_encoding_cookie=True): """Converts a bytes string with python source code to unicode. Unicode strings are passed through unchanged. Byte strings are checked for the python source file encoding cookie to determine encoding. txt can be either a bytes buffer or a string containing the source code. """ if isinstance(txt, unicode): return txt if isinstance(txt, bytes): buffer = BytesIO(txt) else: buffer = txt try: encoding, _ = detect_encoding(buffer.readline) except SyntaxError: encoding = "ascii" buffer.seek(0) text = TextIOWrapper(buffer, encoding, errors=errors, line_buffering=True) text.mode = 'r' if skip_encoding_cookie: return u"".join(strip_encoding_cookie(text)) else: return text.read()
def open(filename): """Open a file in read only mode using the encoding detected by detect_encoding(). """ buffer = io.open(filename, 'rb') # Tweaked to use io.open for Python 2 encoding, lines = detect_encoding(buffer.readline) buffer.seek(0) text = TextIOWrapper(buffer, encoding, line_buffering=True) text.mode = 'r' return text
def open(filename): """Open a file in read only mode using the encoding detected by detect_encoding(). """ buffer = builtins.open(filename, 'rb') encoding, lines = detect_encoding(buffer.readline) buffer.seek(0) text = TextIOWrapper(buffer, encoding, line_buffering=True) text.mode = 'r' return text
def __init_io(self, buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None): """Initializes the underlying raw IO and buffered IO """ # The following code is modified based on python io.open() modes = set(self.mode) if modes - set("axrwb+t") or len(self.mode) > len(modes): raise ValueError("invalid mode: %r" % self.mode) creating = "x" in modes reading = "r" in modes writing = "w" in modes appending = "a" in modes updating = "+" in modes text = "t" in modes binary = "b" in modes self.__validate_args(text, binary, creating, reading, writing, appending, encoding, errors, newline, buffering) self.raw_io = self.__open_raw_io(closefd, opener) # Track the opened IO opened_io = self.raw_io try: line_buffering = False if buffering == 1 or buffering < 0 and self.raw_io.isatty(): buffering = -1 line_buffering = True opened_io = self.__init_buffer_io(buffering, binary, updating, creating, reading, writing, appending) # Use TextIOWrapper for text mode if binary: return opened_io text_io = TextIOWrapper(opened_io, encoding, errors, newline, line_buffering) opened_io = text_io text_io.mode = self.mode return opened_io except Exception as ex: # Close the opened IO if there is an error if opened_io: logger.debug("Closing opened_io...") opened_io.close() raise ex
def _tokopen(filename): """Open a file in read only mode using the encoding detected by detect_encoding(). """ buffer = _builtin_open(filename, 'rb') try: encoding, lines = detect_encoding(buffer.readline) buffer.seek(0) text = TextIOWrapper(buffer, encoding, line_buffering=True) text.mode = 'r' return text except: buffer.close() raise
def _open(filename): """Open a file in read only mode using the encoding detected by detect_encoding(). """ buffer = open(filename, "rb") try: encoding = File.detect_encoding(filename, buffer.readline) buffer.seek(0) text = TextIOWrapper(buffer, encoding, line_buffering=True, newline="") text.mode = "r" # type: ignore return text except Exception: buffer.close() raise
def srcopen(filename): buff = open(filename, 'rb') try: encoding, lines = detect_encoding(buff.readline) buff.seek(0) if encoding == 'utf-8': if next(iter(buff.read(1)), -1) == 0xEF: encoding = 'utf-8-sig' buff.seek(0) stream = TextIOWrapper(buff, encoding, line_buffering=True) stream.mode = 'r' return stream except: buff.close() raise
def open(filename): """Open a file in read only mode using the encoding detected by detect_encoding(). """ # TRUFFLE TODO: revert-me buffer = _builtin_open(filename, 'rb', buffering=0) # buffer = _builtin_open(filename, 'rb') try: encoding, lines = detect_encoding(buffer.readline) buffer.seek(0) return buffer # TRUFFLE TODO: remove-me text = TextIOWrapper(buffer, encoding, line_buffering=True) text.mode = 'r' return text except: buffer.close() raise
def read_source_lines(filename): buffer = _builtin_open(filename, 'rb') try: encoding, lines, cookie_present = detect_encoding_ex(buffer.readline) buffer.seek(0) text = TextIOWrapper(buffer, encoding, line_buffering=True) text.mode = 'r' except: buffer.close() raise with text: if cookie_present: for i in lines: yield text.readline().replace("coding", "Coding") # so compile() won't complain about encoding declatation in a Unicode string # see 2.7/Python/ast.c:228 for line in text: yield line
def _open_function(filename, encoding=None): '''Open a file in read only mode using the encoding detected by detect_encoding(). ''' # Note: Python 3 uses builtins.open here.. buffer = _io_open_function(filename, 'rb') try: encoding, lines, found = detect_encoding(buffer.readline) # Note: Python 3's tokenize does buffer seek(0), but that # leaves the encoding cookie in the file and ast.parse # does not like Unicode text with an encoding cookie. # If the encoding was not found we seek to the start anyway if found: buffer.seek(sum(len(line) for line in lines)) else: buffer.seek(0) text = TextIOWrapper(buffer, encoding, line_buffering=True) text.mode = 'r' return text except: buffer.close() raise
def translate_newlines(self, mode, *text_args, **text_kwargs): wrapper = TextIOWrapper(self._io, *text_args, **text_kwargs) if mode: wrapper.mode = mode self.io = wrapper self._translate = True
def wrap_text(bf, mode, encoding, errors, newline): if "t" in mode: tf = TextIOWrapper(bf, encoding, errors, newline) tf.mode = mode return tf return bf