def testWriteOverflow(self):
writer = MockRawIO()
bufio = io.BufferedWriter(writer, 8)
bufio.write(b"abc")
bufio.write(b"defghijkl")
self.assertEquals(b"abcdefghijkl", writer._write_stack[0])
def testWrite(self):
# Write to the buffered IO but don't overflow the buffer.
writer = MockRawIO()
bufio = io.BufferedWriter(writer, 8)
bufio.write(b"abc")
self.assertFalse(writer._write_stack)
def __init__(
self, flush_callback: Callable[[str], None], name: Optional[str] = None
):
# we just need to set internal buffer's name as
# it will automatically buble up to each buffer
internal_buffer = _CallbackBuffer(flush_callback, name=name)
buffer = io.BufferedWriter(internal_buffer)
super().__init__(buffer, line_buffering=True) # type: ignore
def _get_text_stderr(buffer_stream):
text_stream = StreamWrapper(
io.BufferedWriter(_WindowsConsoleWriter(STDERR_HANDLE)),
"utf-16-le",
"strict",
line_buffering=True,
)
return ConsoleStream(text_stream, buffer_stream)
def _get_text_stdout(buffer_stream):
text_stream = _NonClosingTextIOWrapper(
io.BufferedWriter(_WindowsConsoleWriter(STDOUT_HANDLE)),
"utf-16-le",
"strict",
line_buffering=True,
)
return ConsoleStream(text_stream, buffer_stream)
def _create(self,
path,
mime_type='application/octet-stream',
compression_type=CompressionTypes.AUTO):
stream = io.BufferedWriter(filesystemio.UploaderStream(
HdfsUploader(self._hdfs_client, path)),
buffer_size=_DEFAULT_BUFFER_SIZE)
return self._add_compression(stream, path, mime_type, compression_type)
def testFlush(self):
writer = MockRawIO()
bufio = io.BufferedWriter(writer, 8)
bufio.write(b"abc")
bufio.flush()
self.assertEquals(b"abc", writer._write_stack[0])
def open(self, mode="rb", encoding="ascii", buffering=1024, size=None,
block_transfer=False):
"""Open the data stream as a file like object.
:param str mode:
========= ==========================================================
Character Meaning
--------- ----------------------------------------------------------
'r' open for reading (default)
'w' open for writing
'b' binary mode (default)
't' text mode
========= ==========================================================
:param str encoding:
The str name of the encoding used to decode or encode the file.
This will only be used in text mode.
:param int buffering:
An optional integer used to set the buffering policy. Pass 0 to
switch buffering off (only allowed in binary mode), 1 to select line
buffering (only usable in text mode), and an integer > 1 to indicate
the size in bytes of a fixed-size chunk buffer.
:param int size:
Size of data to that will be transmitted.
:param bool block_transfer:
If block transfer should be used.
:returns:
A file like object.
"""
buffer_size = buffering if buffering > 1 else io.DEFAULT_BUFFER_SIZE
if "r" in mode:
if block_transfer:
raw_stream = BlockUploadStream(
self.sdo_node, self.od.index, self.od.subindex)
else:
raw_stream = ReadableStream(
self.sdo_node, self.od.index, self.od.subindex)
if buffering:
buffered_stream = io.BufferedReader(raw_stream, buffer_size=buffer_size)
else:
return raw_stream
if "w" in mode:
if block_transfer:
raw_stream = BlockDownloadStream(
self.sdo_node, self.od.index, self.od.subindex, size)
else:
raw_stream = WritableStream(
self.sdo_node, self.od.index, self.od.subindex, size)
if buffering:
buffered_stream = io.BufferedWriter(raw_stream, buffer_size=buffer_size)
else:
return raw_stream
if "b" not in mode:
# Text mode
line_buffering = buffering == 1
return io.TextIOWrapper(buffered_stream, encoding,
line_buffering=line_buffering)
return buffered_stream
def __init__(
self, raw: IO,
buffer_size: Optional[int] = io.DEFAULT_BUFFER_SIZE,
*, loop: Loop = asyncio.get_event_loop()) -> None:
self._loop = loop
if isinstance(raw, (AsyncFileIO, AsyncTextIOWrapper)):
raw = raw._hidden
self._hidden = io.BufferedWriter(raw, buffer_size)
TRunner.lq.append(loop)
def FileIO(file_name, method, chunksize=CHUNK_SIZE):
if xbmc.getCondVisibility('System.Platform.Android'):
file_obj = io.FileIO(file_name, method)
if method.startswith('r'):
return io.BufferedReader(file_obj, buffer_size=chunksize)
else:
return io.BufferedWriter(file_obj, buffer_size=chunksize)
else:
return open(file_name, method, chunksize)
def decrypt_in_mm(cls, enc_fd, cipher):
zip_fd = io.BytesIO()
buf_w = io.BufferedWriter(zip_fd)
buf_r = io.BufferedReader(enc_fd)
cipher.decrypt_fileobj(buf_r, buf_w)
buf_w.flush()
buf_w.detach()
zip_fd.seek(0)
return zip_fd
def __enter__(self):
self.open()
if self.progress:
self.progress.__enter__()
if self.bufferSize:
self.bufferedWriter = io.BufferedWriter(self, buffer_size=self.bufferSize)
return self.bufferedWriter
else:
return self
def convert_and_filter_topk(args):
""" Convert to lowercase, count word occurrences and save top-k words to a file """
counter = Counter()
data_lower = os.path.join(args.output_dir, "lower.txt.gz")
print("\nConverting to lowercase and counting word occurrences ...")
with io.TextIOWrapper(io.BufferedWriter(gzip.open(data_lower, "w+")),
encoding="utf-8") as file_out:
# Open the input file either from input.txt or input.txt.gz
_, file_extension = os.path.splitext(args.input_txt)
if file_extension == ".gz":
file_in = io.TextIOWrapper(io.BufferedReader(
gzip.open(args.input_txt)),
encoding="utf-8")
else:
file_in = open(args.input_txt, encoding="utf-8")
for line in file_in:
line_lower = line.lower().translate(
str.maketrans('', '', string.punctuation))
counter.update(line_lower.split())
file_out.write(line_lower)
file_in.close()
# Save top-k words
print("\nSaving top {} words ...".format(args.top_k))
top_counter = counter.most_common(args.top_k)
vocab_str = "\n".join(word for word, count in top_counter)
vocab_path = "vocab-{}.txt".format(args.top_k)
vocab_path = os.path.join(args.output_dir, vocab_path)
with open(vocab_path, "w+") as file:
file.write(vocab_str)
print("\nCalculating word statistics ...")
total_words = sum(counter.values())
print(" Your text file has {} words in total".format(total_words))
print(" It has {} unique words".format(len(counter)))
top_words_sum = sum(count for word, count in top_counter)
word_fraction = (top_words_sum / total_words) * 100
print(" Your top-{} words are {:.4f} percent of all words".format(
args.top_k, word_fraction))
print(' Your most common word "{}" occurred {} times'.format(
*top_counter[0]))
last_word, last_count = top_counter[-1]
print(' The least common word in your top-k is "{}" with {} times'.format(
last_word, last_count))
for i, (w, c) in enumerate(reversed(top_counter)):
if c > last_count:
print(' The first word with {} occurrences is "{}" at place {}'.
format(c, w,
len(top_counter) - 1 - i))
break
return data_lower, vocab_str
def connect1(self, host, port):
try:
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.connect((host, port))
self.out = io.BufferedWriter(socket.SocketIO(self.socket, 'w'))
return True
except socket.error as e:
LOGGER.error('Connection failed: %s', e)
return False
def get_output_objects(self):
output_objects = {}
output_path = self.input_file
output_suffix = '.CGmap' if not self.bedgraph_output else '.bg'
if self.output_prefix:
output_path = self.output_prefix
if self.text_output:
output_objects['CGmap'] = open(f'{output_path}{output_suffix}',
'w')
if self.ATCGmap:
output_objects['ATCGmap'] = open(f'{output_path}.ATCGmap', 'w')
else:
output_objects['CGmap'] = io.BufferedWriter(
gzip.open(f'{output_path}{output_suffix}.gz', 'wb'))
if self.ATCGmap:
output_objects['ATCGmap'] = io.BufferedWriter(
gzip.open(f'{output_path}.ATCGmap.gz', 'wb'))
return output_objects
def test_write_to_cp437_output(self):
"""Check writing to a cp437 output (e.g. Windows console)."""
raw = io.BytesIO()
output = io.TextIOWrapper(io.BufferedWriter(raw),
encoding='cp437') # Windows console.
massedit.edit_files(['tests.py'], expressions=['line[:10]'],
output=output)
actual = raw.getvalue()
self.assertIsNotNone(actual)
def open_buffered_stream_writer(
uri: str,
buffer_size: int = io.DEFAULT_BUFFER_SIZE) -> io.BufferedWriter:
try:
output_stream = open_output_stream(uri)
return io.BufferedWriter(output_stream, buffer_size=buffer_size)
except Exception as e:
output_stream.close()
raise e
def initialize_upload(youtube,
file,
mimetype,
title,
description,
keywords=None,
category=22,
privacyStatus='private'):
tags = None
if keywords:
tags = keywords.split(",")
body = dict(snippet=dict(title=title,
description=description,
tags=tags,
categoryId=category),
status=dict(privacyStatus=privacyStatus))
fh = io.BytesIO()
writer = io.BufferedWriter(fh)
# try:
# os.mkfifo('my_fifo')
# except FileExistsError:
# pass
# fh = open('my_fifo', 'w', os.O_WRONLY|os.O_NONBLOCK)
downloader = MediaIoBaseDownload(writer, file, chunksize=1024 * 1024)
#o = os.open('my_fifo', os.O_RDONLY | os.O_NONBLOCK)
reader = io.BufferedReader(fh)
# Call the API's videos.insert method to create and upload the video.
insert_request = youtube.videos().insert(
part=",".join(body.keys()),
body=body,
# The chunksize parameter specifies the size of each chunk of data, in
# bytes, that will be uploaded at a time. Set a higher value for
# reliable connections as fewer chunks lead to faster uploads. Set a lower
# value for better recovery on less reliable connections.
#
# Setting "chunksize" equal to -1 in the code below means that the entire
# file will be uploaded in a single HTTP request. (If the upload fails,
# it will still be retried where it left off.) This is usually a best
# practice, but if you're using Python older than 2.6 or if you're
# running on App Engine, you should set the chunksize to something like
# 1024 * 1024 (1 megabyte).
media_body=MediaIoBaseUpload(reader,
chunksize=1024 * 1024,
resumable=True,
mimetype=mimetype))
if "id" in insert_request:
print("Video id", insert_request["id"])
else:
print("id not in insert_request")
resumable_upload(insert_request, downloader, writer)
def makefile(self, mode="r", buffering=None, **_3to2kwargs):
"""makefile(...) -> an I/O stream connected to the socket
The arguments are as for io.open() after the filename,
except the only mode characters supported are 'r', 'w' and 'b'.
The semantics are similar too. (XXX refactor to share code?)
"""
if "newline" in _3to2kwargs:
newline = _3to2kwargs["newline"]
del _3to2kwargs["newline"]
else:
newline = None
if "errors" in _3to2kwargs:
errors = _3to2kwargs["errors"]
del _3to2kwargs["errors"]
else:
errors = None
if "encoding" in _3to2kwargs:
encoding = _3to2kwargs["encoding"]
del _3to2kwargs["encoding"]
else:
encoding = None
for c in mode:
if c not in ("r", "w", "b"):
raise ValueError("invalid mode %r (only r, w, b allowed)")
writing = "w" in mode
reading = "r" in mode or not writing
assert reading or writing
binary = "b" in mode
rawmode = ""
if reading:
rawmode += "r"
if writing:
rawmode += "w"
raw = SocketIO(self, rawmode)
self._io_refs += 1
if buffering is None:
buffering = -1
if buffering < 0:
buffering = io.DEFAULT_BUFFER_SIZE
if buffering == 0:
if not binary:
raise ValueError("unbuffered streams must be binary")
return raw
if reading and writing:
buffer = io.BufferedRWPair(raw, raw, buffering)
elif reading:
buffer = io.BufferedReader(raw, buffering)
else:
assert writing
buffer = io.BufferedWriter(raw, buffering)
if binary:
return buffer
text = io.TextIOWrapper(buffer, encoding, errors, newline)
text.mode = mode
return text
def w_test_nonblock_pipe_write(self, bufsize):
import _io as io
class NonBlockingPipe(io._BufferedIOBase):
"write() returns None when buffer is full"
def __init__(self, buffersize=4096):
self.buffersize = buffersize
self.buffer = b''
def readable(self): return True
def writable(self): return True
def write(self, data):
available = self.buffersize - len(self.buffer)
if available <= 0:
return None
self.buffer += data[:available]
return min(len(data), available)
def read(self, size=-1):
if not self.buffer:
return None
if size == -1:
size = len(self.buffer)
data = self.buffer[:size]
self.buffer = self.buffer[size:]
return data
sent = []
received = []
pipe = NonBlockingPipe()
rf = io.BufferedReader(pipe, bufsize)
wf = io.BufferedWriter(pipe, bufsize)
for N in 9999, 7574:
try:
i = 0
while True:
msg = chr(i % 26 + 97) * N
sent.append(msg)
wf.write(msg)
i += 1
except io.BlockingIOError as e:
sent[-1] = sent[-1][:e.characters_written]
received.append(rf.read())
msg = b'BLOCKED'
wf.write(msg)
sent.append(msg)
while True:
try:
wf.flush()
break
except io.BlockingIOError as e:
received.append(rf.read())
received += iter(rf.read, None)
rf.close()
wf.close()
sent, received = b''.join(sent), b''.join(received)
assert sent == received
def test_tag_data_to_int():
bio = io.BytesIO()
buf = io.BufferedWriter(bio)
data = {'tag_name': 'zCenter', 'value': -3200, 'type': 'Int'}
nbt_data = nbt.write_tag(buf, data)
output = '03 00 07 7A 43 65 6E 74 65 72 FF FF F3 80'
expected_output = bytes.fromhex(output)
assert nbt_data == expected_output, "int data is not equal"
def test_tag_data_to_bytes():
bio = io.BytesIO()
buf = io.BufferedWriter(bio)
data = {'tag_name': 'unlimitedTracking', 'value': 0, 'type': 'Byte'}
nbt_data = nbt.write_tag(buf, data)
output = '01 00 11 75 6E 6C 69 6D 69 74 65 64 54 72 61 63 6B 69 6E 67 00'
expected_output = bytes.fromhex(output)
assert nbt_data == expected_output, "byte data is not equal"
def test_tag_data_to_double():
bio = io.BytesIO()
buf = io.BufferedWriter(bio)
data = {'tag_name': 'BorderCenterZ', 'value': 0, 'type': 'Double'}
nbt_data = nbt.write_tag(buf, data)
output = '06 00 0D 42 6F 72 64 65 72 43 65 6E 74 65 72 5A 00 00 00 00 00 00 00 00'
expected_output = bytes.fromhex(output)
assert nbt_data == expected_output, "double data is not equal"
def test_buffered_writer(gfile):
cancel = Gio.Cancellable.new()
with gopen(gfile, "r+b", cancellable=cancel) as f:
with open(f.name, "wb") as h:
h.write(b"foo")
with io.BufferedWriter(f) as b:
b.write(b"x")
b.flush()
f.seek(0)
assert f.read() == b"xoo"
def __init__(self):
with self.id_lock:
self.id = MockSocketBase.id_next
MockSocketBase.id_next += 1
self.blocking = True
self.send_pipe = Pipe(timeout=10, name="MockSocket.send[%i]" % self.id)
self.recv_pipe = Pipe(timeout=10, name="MockSocket.recv[%i]" % self.id)
self.send_rbuffer = io.BufferedReader(self.send_pipe)
self.recv_wbuffer = io.BufferedWriter(self.recv_pipe)
self.io_error = None
def __init__(self, fname, **kwargs):
self.fname = fname
self.fobj = gzip.open(
self.fname, mode='wb',
compresslevel=6) if self.fname.endswith('.gz') else open(
self.fname, 'wb')
self.f = io.BufferedWriter(self.fobj)
for key, value in kwargs.items():
setattr(self, key, value)
def _buffered_write_file(fobj):
"""Return a buffered version of a write file object."""
if PY27 and bz2 is not None and isinstance(fobj, bz2.BZ2File):
# Python 2.7 doesn't work with BZ2File through a buffer: no attribute
# 'writable'.
# BZ2File doesn't implement the file object context manager in python 2
# so we wrap the fileobj using `closing`.
return closing(fobj)
else:
return io.BufferedWriter(fobj, buffer_size=_IO_BUFFER_SIZE)
def testLineBuffering(self):
r = io.BytesIO()
b = io.BufferedWriter(r, 1000)
t = io.TextIOWrapper(b, newline="\n", line_buffering=True)
t.write(u"X")
self.assertEquals(r.getvalue(), b"") # No flush happened
t.write(u"Y\nZ")
self.assertEquals(r.getvalue(), b"XY\nZ") # All got flushed
t.write(u"A\rB")
self.assertEquals(r.getvalue(), b"XY\nZA\rB")
def test_sif(self):
"""Test 'Subtitle Intermediate Format' loaders and dumpers."""
# Make a test subtitle with two lines
subtitle = Subtitle()
subtitle.add_unit(
SubtitleUnit(start=15, end=30, lines=['First line with \u0161']))
subtitle.add_unit(
SubtitleUnit(start=65,
end=89,
position={
'x': 50,
'y': 30,
},
lines=[
'Another, but a two liner \u010d',
'Yes, I said two liner! \u017e'
]))
subtitle[0][0].special = True
subtitle[0][0].complex = {'a': {'4': 2}, 'b': [1, 2, 3]}
subtitle[1][1].test = 'test'
# Write it
tmpfd, tmp = tempfile.mkstemp()
tmpfd2, tmp2 = tempfile.mkstemp()
subtitle.save(io.BufferedWriter(io.FileIO(tmpfd, mode='w')))
subtitle.save(io.BufferedWriter(io.FileIO(tmpfd2, mode='w')),
human_time=False)
# Load it and test
assert Subtitle.from_file(tmp) == subtitle
assert Subtitle.from_file(tmp2) == subtitle
# Remove temp files
os.unlink(tmp)
os.unlink(tmp2)
# And some minor things
# repr - just checking for exceptions
repr(subtitle)
for u in subtitle:
repr(u)
for l in u:
repr(l)
def _get_writer(file_or_filename, encoding):
# returns text write method and release all resources after using
try:
write = file_or_filename.write
except AttributeError:
# file_or_filename is a file name
f = open(
file_or_filename,
"w",
encoding="utf-8" if encoding == "unicode" else encoding,
errors="xmlcharrefreplace",
)
with f:
yield f.write
else:
# file_or_filename is a file-like object
# encoding determines if it is a text or binary writer
if encoding == "unicode":
# use a text writer as is
yield write
else:
# wrap a binary writer with TextIOWrapper
detach_buffer = False
if isinstance(file_or_filename, io.BufferedIOBase):
buf = file_or_filename
elif isinstance(file_or_filename, io.RawIOBase):
buf = io.BufferedWriter(file_or_filename)
detach_buffer = True
else:
# This is to handle passed objects that aren't in the
# IOBase hierarchy, but just have a write method
buf = io.BufferedIOBase()
buf.writable = lambda: True
buf.write = write
try:
# TextIOWrapper uses this methods to determine
# if BOM (for UTF-16, etc) should be added
buf.seekable = file_or_filename.seekable
buf.tell = file_or_filename.tell
except AttributeError:
pass
wrapper = io.TextIOWrapper(
buf,
encoding=encoding,
errors="xmlcharrefreplace",
newline="\n",
)
try:
yield wrapper.write
finally:
# Keep the original file open when the TextIOWrapper and
# the BufferedWriter are destroyed
wrapper.detach()
if detach_buffer:
buf.detach()
