def read_py_url(url, errors='replace', skip_encoding_cookie=True):
"""Read a Python file from a URL, using the encoding declared inside the file.
Parameters
----------
url : str
The URL from which to fetch the file.
errors : str
How to handle decoding errors in the file. Options are the same as for
bytes.decode(), but here 'replace' is the default.
skip_encoding_cookie : bool
If True (the default), and the encoding declaration is found in the first
two lines, that line will be excluded from the output - compiling a
unicode string with an encoding declaration is a SyntaxError in Python 2.
Returns
-------
A unicode string containing the contents of the file.
"""
response = urllib.request.urlopen(url)
buffer = io.BytesIO(response.read())
encoding, lines = detect_encoding(buffer.readline)
buffer.seek(0)
text = TextIOWrapper(buffer, encoding, errors=errors, line_buffering=True)
text.mode = 'r'
if skip_encoding_cookie:
return "".join(strip_encoding_cookie(text))
else:
return text.read()
def open_source(fname):
buffer = open(fname, 'rb')
encoding, _ = detect_encoding(buffer.readline)
buffer.seek(0)
text = TextIOWrapper(buffer, encoding, line_buffering=True)
text.mode = 'r'
return text
def open(filename):
buffer = builtins.open(filename, 'rb')
(encoding, lines) = detect_encoding(buffer.readline)
buffer.seek(0)
text = TextIOWrapper(buffer, encoding, line_buffering=True)
text.mode = 'r'
return text
def read_py_url(url, errors='replace', skip_encoding_cookie=True):
"""Read a Python file from a URL, using the encoding declared inside the file.
Parameters
----------
url : str
The URL from which to fetch the file.
errors : str
How to handle decoding errors in the file. Options are the same as for
bytes.decode(), but here 'replace' is the default.
skip_encoding_cookie : bool
If True (the default), and the encoding declaration is found in the first
two lines, that line will be excluded from the output - compiling a
unicode string with an encoding declaration is a SyntaxError in Python 2.
Returns
-------
A unicode string containing the contents of the file.
"""
response = urllib.urlopen(url)
buffer = io.BytesIO(response.read())
encoding, lines = detect_encoding(buffer.readline)
buffer.seek(0)
text = TextIOWrapper(buffer, encoding, errors=errors, line_buffering=True)
text.mode = 'r'
if skip_encoding_cookie:
return "".join(strip_encoding_cookie(text))
else:
return text.read()
def source_to_unicode(txt, errors='replace', skip_encoding_cookie=True):
"""Converts a bytes string with python source code to unicode.
Unicode strings are passed through unchanged. Byte strings are checked
for the python source file encoding cookie to determine encoding.
txt can be either a bytes buffer or a string containing the source
code.
"""
if isinstance(txt, unicode_type):
return txt
if isinstance(txt, bytes):
buffer = BytesIO(txt)
else:
buffer = txt
try:
encoding, _ = detect_encoding(buffer.readline)
except SyntaxError:
encoding = "ascii"
buffer.seek(0)
text = TextIOWrapper(buffer, encoding, errors=errors, line_buffering=True)
text.mode = 'r'
if skip_encoding_cookie:
return u"".join(strip_encoding_cookie(text))
else:
return text.read()
def open_source(fname):
buffer = open(fname, 'rb')
encoding, _ = detect_encoding(buffer.readline)
buffer.seek(0)
text = TextIOWrapper(buffer, encoding, line_buffering=True)
text.mode = 'r'
return text
def source_to_unicode(txt, errors='replace', skip_encoding_cookie=True):
"""Converts a bytes string with python source code to unicode.
Unicode strings are passed through unchanged. Byte strings are checked
for the python source file encoding cookie to determine encoding.
txt can be either a bytes buffer or a string containing the source
code.
"""
if isinstance(txt, unicode):
return txt
if isinstance(txt, bytes):
buffer = BytesIO(txt)
else:
buffer = txt
try:
encoding, _ = detect_encoding(buffer.readline)
except SyntaxError:
encoding = "ascii"
buffer.seek(0)
text = TextIOWrapper(buffer, encoding, errors=errors, line_buffering=True)
text.mode = 'r'
if skip_encoding_cookie:
return u"".join(strip_encoding_cookie(text))
else:
return text.read()
def open(filename):
"""Open a file in read only mode using the encoding detected by
detect_encoding().
"""
buffer = io.open(filename, 'rb') # Tweaked to use io.open for Python 2
encoding, lines = detect_encoding(buffer.readline)
buffer.seek(0)
text = TextIOWrapper(buffer, encoding, line_buffering=True)
text.mode = 'r'
return text
def open(filename):
"""Open a file in read only mode using the encoding detected by
detect_encoding().
"""
buffer = io.open(filename, 'rb') # Tweaked to use io.open for Python 2
encoding, lines = detect_encoding(buffer.readline)
buffer.seek(0)
text = TextIOWrapper(buffer, encoding, line_buffering=True)
text.mode = 'r'
return text
def open(filename):
"""Open a file in read only mode using the encoding detected by
detect_encoding().
"""
buffer = builtins.open(filename, 'rb')
encoding, lines = detect_encoding(buffer.readline)
buffer.seek(0)
text = TextIOWrapper(buffer, encoding, line_buffering=True)
text.mode = 'r'
return text
def open(filename):
"""Open a file in read only mode using the encoding detected by
detect_encoding().
"""
buffer = builtins.open(filename, 'rb')
encoding, lines = detect_encoding(buffer.readline)
buffer.seek(0)
text = TextIOWrapper(buffer, encoding, line_buffering=True)
text.mode = 'r'
return text
def __init_io(self,
buffering=-1,
encoding=None,
errors=None,
newline=None,
closefd=True,
opener=None):
"""Initializes the underlying raw IO and buffered IO
"""
# The following code is modified based on python io.open()
modes = set(self.mode)
if modes - set("axrwb+t") or len(self.mode) > len(modes):
raise ValueError("invalid mode: %r" % self.mode)
creating = "x" in modes
reading = "r" in modes
writing = "w" in modes
appending = "a" in modes
updating = "+" in modes
text = "t" in modes
binary = "b" in modes
self.__validate_args(text, binary, creating, reading, writing,
appending, encoding, errors, newline, buffering)
self.raw_io = self.__open_raw_io(closefd, opener)
# Track the opened IO
opened_io = self.raw_io
try:
line_buffering = False
if buffering == 1 or buffering < 0 and self.raw_io.isatty():
buffering = -1
line_buffering = True
opened_io = self.__init_buffer_io(buffering, binary, updating,
creating, reading, writing,
appending)
# Use TextIOWrapper for text mode
if binary:
return opened_io
text_io = TextIOWrapper(opened_io, encoding, errors, newline,
line_buffering)
opened_io = text_io
text_io.mode = self.mode
return opened_io
except Exception as ex:
# Close the opened IO if there is an error
if opened_io:
logger.debug("Closing opened_io...")
opened_io.close()
raise ex
def _tokopen(filename):
"""Open a file in read only mode using the encoding detected by
detect_encoding().
"""
buffer = _builtin_open(filename, 'rb')
try:
encoding, lines = detect_encoding(buffer.readline)
buffer.seek(0)
text = TextIOWrapper(buffer, encoding, line_buffering=True)
text.mode = 'r'
return text
except:
buffer.close()
raise
def _open(filename):
"""Open a file in read only mode using the encoding detected by
detect_encoding().
"""
buffer = open(filename, "rb")
try:
encoding = File.detect_encoding(filename, buffer.readline)
buffer.seek(0)
text = TextIOWrapper(buffer, encoding, line_buffering=True, newline="")
text.mode = "r" # type: ignore
return text
except Exception:
buffer.close()
raise
def _tokopen(filename):
"""Open a file in read only mode using the encoding detected by
detect_encoding().
"""
buffer = _builtin_open(filename, 'rb')
try:
encoding, lines = detect_encoding(buffer.readline)
buffer.seek(0)
text = TextIOWrapper(buffer, encoding, line_buffering=True)
text.mode = 'r'
return text
except:
buffer.close()
raise
def srcopen(filename):
buff = open(filename, 'rb')
try:
encoding, lines = detect_encoding(buff.readline)
buff.seek(0)
if encoding == 'utf-8':
if next(iter(buff.read(1)), -1) == 0xEF:
encoding = 'utf-8-sig'
buff.seek(0)
stream = TextIOWrapper(buff, encoding, line_buffering=True)
stream.mode = 'r'
return stream
except:
buff.close()
raise
def srcopen(filename):
buff = open(filename, 'rb')
try:
encoding, lines = detect_encoding(buff.readline)
buff.seek(0)
if encoding == 'utf-8':
if next(iter(buff.read(1)), -1) == 0xEF:
encoding = 'utf-8-sig'
buff.seek(0)
stream = TextIOWrapper(buff, encoding, line_buffering=True)
stream.mode = 'r'
return stream
except:
buff.close()
raise
def open(filename):
"""Open a file in read only mode using the encoding detected by
detect_encoding().
"""
# TRUFFLE TODO: revert-me
buffer = _builtin_open(filename, 'rb', buffering=0)
# buffer = _builtin_open(filename, 'rb')
try:
encoding, lines = detect_encoding(buffer.readline)
buffer.seek(0)
return buffer # TRUFFLE TODO: remove-me
text = TextIOWrapper(buffer, encoding, line_buffering=True)
text.mode = 'r'
return text
except:
buffer.close()
raise
def read_source_lines(filename):
buffer = _builtin_open(filename, 'rb')
try:
encoding, lines, cookie_present = detect_encoding_ex(buffer.readline)
buffer.seek(0)
text = TextIOWrapper(buffer, encoding, line_buffering=True)
text.mode = 'r'
except:
buffer.close()
raise
with text:
if cookie_present:
for i in lines:
yield text.readline().replace("coding", "Coding")
# so compile() won't complain about encoding declatation in a Unicode string
# see 2.7/Python/ast.c:228
for line in text:
yield line
def read_source_lines(filename):
buffer = _builtin_open(filename, 'rb')
try:
encoding, lines, cookie_present = detect_encoding_ex(buffer.readline)
buffer.seek(0)
text = TextIOWrapper(buffer, encoding, line_buffering=True)
text.mode = 'r'
except:
buffer.close()
raise
with text:
if cookie_present:
for i in lines:
yield text.readline().replace("coding", "Coding")
# so compile() won't complain about encoding declatation in a Unicode string
# see 2.7/Python/ast.c:228
for line in text:
yield line
def _open_function(filename, encoding=None):
'''Open a file in read only mode using the encoding detected by
detect_encoding().
'''
# Note: Python 3 uses builtins.open here..
buffer = _io_open_function(filename, 'rb')
try:
encoding, lines, found = detect_encoding(buffer.readline)
# Note: Python 3's tokenize does buffer seek(0), but that
# leaves the encoding cookie in the file and ast.parse
# does not like Unicode text with an encoding cookie.
# If the encoding was not found we seek to the start anyway
if found:
buffer.seek(sum(len(line) for line in lines))
else:
buffer.seek(0)
text = TextIOWrapper(buffer, encoding, line_buffering=True)
text.mode = 'r'
return text
except:
buffer.close()
raise
def _open_function(filename, encoding=None):
'''Open a file in read only mode using the encoding detected by
detect_encoding().
'''
# Note: Python 3 uses builtins.open here..
buffer = _io_open_function(filename, 'rb')
try:
encoding, lines, found = detect_encoding(buffer.readline)
# Note: Python 3's tokenize does buffer seek(0), but that
# leaves the encoding cookie in the file and ast.parse
# does not like Unicode text with an encoding cookie.
# If the encoding was not found we seek to the start anyway
if found:
buffer.seek(sum(len(line) for line in lines))
else:
buffer.seek(0)
text = TextIOWrapper(buffer, encoding, line_buffering=True)
text.mode = 'r'
return text
except:
buffer.close()
raise
def translate_newlines(self, mode, *text_args, **text_kwargs):
wrapper = TextIOWrapper(self._io, *text_args, **text_kwargs)
if mode:
wrapper.mode = mode
self.io = wrapper
self._translate = True
def translate_newlines(self, mode, *text_args, **text_kwargs):
wrapper = TextIOWrapper(self._io, *text_args, **text_kwargs)
if mode:
wrapper.mode = mode
self.io = wrapper
self._translate = True
def wrap_text(bf, mode, encoding, errors, newline): if "t" in mode: tf = TextIOWrapper(bf, encoding, errors, newline) tf.mode = mode return tf return bf