def testWriteOverflow(self): writer = MockRawIO() bufio = io.BufferedWriter(writer, 8) bufio.write(b"abc") bufio.write(b"defghijkl") self.assertEquals(b"abcdefghijkl", writer._write_stack[0])
def testWrite(self): # Write to the buffered IO but don't overflow the buffer. writer = MockRawIO() bufio = io.BufferedWriter(writer, 8) bufio.write(b"abc") self.assertFalse(writer._write_stack)
def __init__( self, flush_callback: Callable[[str], None], name: Optional[str] = None ): # we just need to set internal buffer's name as # it will automatically buble up to each buffer internal_buffer = _CallbackBuffer(flush_callback, name=name) buffer = io.BufferedWriter(internal_buffer) super().__init__(buffer, line_buffering=True) # type: ignore
def _get_text_stderr(buffer_stream): text_stream = StreamWrapper( io.BufferedWriter(_WindowsConsoleWriter(STDERR_HANDLE)), "utf-16-le", "strict", line_buffering=True, ) return ConsoleStream(text_stream, buffer_stream)
def _get_text_stdout(buffer_stream): text_stream = _NonClosingTextIOWrapper( io.BufferedWriter(_WindowsConsoleWriter(STDOUT_HANDLE)), "utf-16-le", "strict", line_buffering=True, ) return ConsoleStream(text_stream, buffer_stream)
def _create(self, path, mime_type='application/octet-stream', compression_type=CompressionTypes.AUTO): stream = io.BufferedWriter(filesystemio.UploaderStream( HdfsUploader(self._hdfs_client, path)), buffer_size=_DEFAULT_BUFFER_SIZE) return self._add_compression(stream, path, mime_type, compression_type)
def testFlush(self): writer = MockRawIO() bufio = io.BufferedWriter(writer, 8) bufio.write(b"abc") bufio.flush() self.assertEquals(b"abc", writer._write_stack[0])
def open(self, mode="rb", encoding="ascii", buffering=1024, size=None, block_transfer=False): """Open the data stream as a file like object. :param str mode: ========= ========================================================== Character Meaning --------- ---------------------------------------------------------- 'r' open for reading (default) 'w' open for writing 'b' binary mode (default) 't' text mode ========= ========================================================== :param str encoding: The str name of the encoding used to decode or encode the file. This will only be used in text mode. :param int buffering: An optional integer used to set the buffering policy. Pass 0 to switch buffering off (only allowed in binary mode), 1 to select line buffering (only usable in text mode), and an integer > 1 to indicate the size in bytes of a fixed-size chunk buffer. :param int size: Size of data to that will be transmitted. :param bool block_transfer: If block transfer should be used. :returns: A file like object. """ buffer_size = buffering if buffering > 1 else io.DEFAULT_BUFFER_SIZE if "r" in mode: if block_transfer: raw_stream = BlockUploadStream( self.sdo_node, self.od.index, self.od.subindex) else: raw_stream = ReadableStream( self.sdo_node, self.od.index, self.od.subindex) if buffering: buffered_stream = io.BufferedReader(raw_stream, buffer_size=buffer_size) else: return raw_stream if "w" in mode: if block_transfer: raw_stream = BlockDownloadStream( self.sdo_node, self.od.index, self.od.subindex, size) else: raw_stream = WritableStream( self.sdo_node, self.od.index, self.od.subindex, size) if buffering: buffered_stream = io.BufferedWriter(raw_stream, buffer_size=buffer_size) else: return raw_stream if "b" not in mode: # Text mode line_buffering = buffering == 1 return io.TextIOWrapper(buffered_stream, encoding, line_buffering=line_buffering) return buffered_stream
def __init__( self, raw: IO, buffer_size: Optional[int] = io.DEFAULT_BUFFER_SIZE, *, loop: Loop = asyncio.get_event_loop()) -> None: self._loop = loop if isinstance(raw, (AsyncFileIO, AsyncTextIOWrapper)): raw = raw._hidden self._hidden = io.BufferedWriter(raw, buffer_size) TRunner.lq.append(loop)
def FileIO(file_name, method, chunksize=CHUNK_SIZE): if xbmc.getCondVisibility('System.Platform.Android'): file_obj = io.FileIO(file_name, method) if method.startswith('r'): return io.BufferedReader(file_obj, buffer_size=chunksize) else: return io.BufferedWriter(file_obj, buffer_size=chunksize) else: return open(file_name, method, chunksize)
def decrypt_in_mm(cls, enc_fd, cipher): zip_fd = io.BytesIO() buf_w = io.BufferedWriter(zip_fd) buf_r = io.BufferedReader(enc_fd) cipher.decrypt_fileobj(buf_r, buf_w) buf_w.flush() buf_w.detach() zip_fd.seek(0) return zip_fd
def __enter__(self): self.open() if self.progress: self.progress.__enter__() if self.bufferSize: self.bufferedWriter = io.BufferedWriter(self, buffer_size=self.bufferSize) return self.bufferedWriter else: return self
def convert_and_filter_topk(args): """ Convert to lowercase, count word occurrences and save top-k words to a file """ counter = Counter() data_lower = os.path.join(args.output_dir, "lower.txt.gz") print("\nConverting to lowercase and counting word occurrences ...") with io.TextIOWrapper(io.BufferedWriter(gzip.open(data_lower, "w+")), encoding="utf-8") as file_out: # Open the input file either from input.txt or input.txt.gz _, file_extension = os.path.splitext(args.input_txt) if file_extension == ".gz": file_in = io.TextIOWrapper(io.BufferedReader( gzip.open(args.input_txt)), encoding="utf-8") else: file_in = open(args.input_txt, encoding="utf-8") for line in file_in: line_lower = line.lower().translate( str.maketrans('', '', string.punctuation)) counter.update(line_lower.split()) file_out.write(line_lower) file_in.close() # Save top-k words print("\nSaving top {} words ...".format(args.top_k)) top_counter = counter.most_common(args.top_k) vocab_str = "\n".join(word for word, count in top_counter) vocab_path = "vocab-{}.txt".format(args.top_k) vocab_path = os.path.join(args.output_dir, vocab_path) with open(vocab_path, "w+") as file: file.write(vocab_str) print("\nCalculating word statistics ...") total_words = sum(counter.values()) print(" Your text file has {} words in total".format(total_words)) print(" It has {} unique words".format(len(counter))) top_words_sum = sum(count for word, count in top_counter) word_fraction = (top_words_sum / total_words) * 100 print(" Your top-{} words are {:.4f} percent of all words".format( args.top_k, word_fraction)) print(' Your most common word "{}" occurred {} times'.format( *top_counter[0])) last_word, last_count = top_counter[-1] print(' The least common word in your top-k is "{}" with {} times'.format( last_word, last_count)) for i, (w, c) in enumerate(reversed(top_counter)): if c > last_count: print(' The first word with {} occurrences is "{}" at place {}'. format(c, w, len(top_counter) - 1 - i)) break return data_lower, vocab_str
def connect1(self, host, port): try: self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.socket.connect((host, port)) self.out = io.BufferedWriter(socket.SocketIO(self.socket, 'w')) return True except socket.error as e: LOGGER.error('Connection failed: %s', e) return False
def get_output_objects(self): output_objects = {} output_path = self.input_file output_suffix = '.CGmap' if not self.bedgraph_output else '.bg' if self.output_prefix: output_path = self.output_prefix if self.text_output: output_objects['CGmap'] = open(f'{output_path}{output_suffix}', 'w') if self.ATCGmap: output_objects['ATCGmap'] = open(f'{output_path}.ATCGmap', 'w') else: output_objects['CGmap'] = io.BufferedWriter( gzip.open(f'{output_path}{output_suffix}.gz', 'wb')) if self.ATCGmap: output_objects['ATCGmap'] = io.BufferedWriter( gzip.open(f'{output_path}.ATCGmap.gz', 'wb')) return output_objects
def test_write_to_cp437_output(self): """Check writing to a cp437 output (e.g. Windows console).""" raw = io.BytesIO() output = io.TextIOWrapper(io.BufferedWriter(raw), encoding='cp437') # Windows console. massedit.edit_files(['tests.py'], expressions=['line[:10]'], output=output) actual = raw.getvalue() self.assertIsNotNone(actual)
def open_buffered_stream_writer( uri: str, buffer_size: int = io.DEFAULT_BUFFER_SIZE) -> io.BufferedWriter: try: output_stream = open_output_stream(uri) return io.BufferedWriter(output_stream, buffer_size=buffer_size) except Exception as e: output_stream.close() raise e
def initialize_upload(youtube, file, mimetype, title, description, keywords=None, category=22, privacyStatus='private'): tags = None if keywords: tags = keywords.split(",") body = dict(snippet=dict(title=title, description=description, tags=tags, categoryId=category), status=dict(privacyStatus=privacyStatus)) fh = io.BytesIO() writer = io.BufferedWriter(fh) # try: # os.mkfifo('my_fifo') # except FileExistsError: # pass # fh = open('my_fifo', 'w', os.O_WRONLY|os.O_NONBLOCK) downloader = MediaIoBaseDownload(writer, file, chunksize=1024 * 1024) #o = os.open('my_fifo', os.O_RDONLY | os.O_NONBLOCK) reader = io.BufferedReader(fh) # Call the API's videos.insert method to create and upload the video. insert_request = youtube.videos().insert( part=",".join(body.keys()), body=body, # The chunksize parameter specifies the size of each chunk of data, in # bytes, that will be uploaded at a time. Set a higher value for # reliable connections as fewer chunks lead to faster uploads. Set a lower # value for better recovery on less reliable connections. # # Setting "chunksize" equal to -1 in the code below means that the entire # file will be uploaded in a single HTTP request. (If the upload fails, # it will still be retried where it left off.) This is usually a best # practice, but if you're using Python older than 2.6 or if you're # running on App Engine, you should set the chunksize to something like # 1024 * 1024 (1 megabyte). media_body=MediaIoBaseUpload(reader, chunksize=1024 * 1024, resumable=True, mimetype=mimetype)) if "id" in insert_request: print("Video id", insert_request["id"]) else: print("id not in insert_request") resumable_upload(insert_request, downloader, writer)
def makefile(self, mode="r", buffering=None, **_3to2kwargs): """makefile(...) -> an I/O stream connected to the socket The arguments are as for io.open() after the filename, except the only mode characters supported are 'r', 'w' and 'b'. The semantics are similar too. (XXX refactor to share code?) """ if "newline" in _3to2kwargs: newline = _3to2kwargs["newline"] del _3to2kwargs["newline"] else: newline = None if "errors" in _3to2kwargs: errors = _3to2kwargs["errors"] del _3to2kwargs["errors"] else: errors = None if "encoding" in _3to2kwargs: encoding = _3to2kwargs["encoding"] del _3to2kwargs["encoding"] else: encoding = None for c in mode: if c not in ("r", "w", "b"): raise ValueError("invalid mode %r (only r, w, b allowed)") writing = "w" in mode reading = "r" in mode or not writing assert reading or writing binary = "b" in mode rawmode = "" if reading: rawmode += "r" if writing: rawmode += "w" raw = SocketIO(self, rawmode) self._io_refs += 1 if buffering is None: buffering = -1 if buffering < 0: buffering = io.DEFAULT_BUFFER_SIZE if buffering == 0: if not binary: raise ValueError("unbuffered streams must be binary") return raw if reading and writing: buffer = io.BufferedRWPair(raw, raw, buffering) elif reading: buffer = io.BufferedReader(raw, buffering) else: assert writing buffer = io.BufferedWriter(raw, buffering) if binary: return buffer text = io.TextIOWrapper(buffer, encoding, errors, newline) text.mode = mode return text
def w_test_nonblock_pipe_write(self, bufsize): import _io as io class NonBlockingPipe(io._BufferedIOBase): "write() returns None when buffer is full" def __init__(self, buffersize=4096): self.buffersize = buffersize self.buffer = b'' def readable(self): return True def writable(self): return True def write(self, data): available = self.buffersize - len(self.buffer) if available <= 0: return None self.buffer += data[:available] return min(len(data), available) def read(self, size=-1): if not self.buffer: return None if size == -1: size = len(self.buffer) data = self.buffer[:size] self.buffer = self.buffer[size:] return data sent = [] received = [] pipe = NonBlockingPipe() rf = io.BufferedReader(pipe, bufsize) wf = io.BufferedWriter(pipe, bufsize) for N in 9999, 7574: try: i = 0 while True: msg = chr(i % 26 + 97) * N sent.append(msg) wf.write(msg) i += 1 except io.BlockingIOError as e: sent[-1] = sent[-1][:e.characters_written] received.append(rf.read()) msg = b'BLOCKED' wf.write(msg) sent.append(msg) while True: try: wf.flush() break except io.BlockingIOError as e: received.append(rf.read()) received += iter(rf.read, None) rf.close() wf.close() sent, received = b''.join(sent), b''.join(received) assert sent == received
def test_tag_data_to_int(): bio = io.BytesIO() buf = io.BufferedWriter(bio) data = {'tag_name': 'zCenter', 'value': -3200, 'type': 'Int'} nbt_data = nbt.write_tag(buf, data) output = '03 00 07 7A 43 65 6E 74 65 72 FF FF F3 80' expected_output = bytes.fromhex(output) assert nbt_data == expected_output, "int data is not equal"
def test_tag_data_to_bytes(): bio = io.BytesIO() buf = io.BufferedWriter(bio) data = {'tag_name': 'unlimitedTracking', 'value': 0, 'type': 'Byte'} nbt_data = nbt.write_tag(buf, data) output = '01 00 11 75 6E 6C 69 6D 69 74 65 64 54 72 61 63 6B 69 6E 67 00' expected_output = bytes.fromhex(output) assert nbt_data == expected_output, "byte data is not equal"
def test_tag_data_to_double(): bio = io.BytesIO() buf = io.BufferedWriter(bio) data = {'tag_name': 'BorderCenterZ', 'value': 0, 'type': 'Double'} nbt_data = nbt.write_tag(buf, data) output = '06 00 0D 42 6F 72 64 65 72 43 65 6E 74 65 72 5A 00 00 00 00 00 00 00 00' expected_output = bytes.fromhex(output) assert nbt_data == expected_output, "double data is not equal"
def test_buffered_writer(gfile): cancel = Gio.Cancellable.new() with gopen(gfile, "r+b", cancellable=cancel) as f: with open(f.name, "wb") as h: h.write(b"foo") with io.BufferedWriter(f) as b: b.write(b"x") b.flush() f.seek(0) assert f.read() == b"xoo"
def __init__(self): with self.id_lock: self.id = MockSocketBase.id_next MockSocketBase.id_next += 1 self.blocking = True self.send_pipe = Pipe(timeout=10, name="MockSocket.send[%i]" % self.id) self.recv_pipe = Pipe(timeout=10, name="MockSocket.recv[%i]" % self.id) self.send_rbuffer = io.BufferedReader(self.send_pipe) self.recv_wbuffer = io.BufferedWriter(self.recv_pipe) self.io_error = None
def __init__(self, fname, **kwargs): self.fname = fname self.fobj = gzip.open( self.fname, mode='wb', compresslevel=6) if self.fname.endswith('.gz') else open( self.fname, 'wb') self.f = io.BufferedWriter(self.fobj) for key, value in kwargs.items(): setattr(self, key, value)
def _buffered_write_file(fobj): """Return a buffered version of a write file object.""" if PY27 and bz2 is not None and isinstance(fobj, bz2.BZ2File): # Python 2.7 doesn't work with BZ2File through a buffer: no attribute # 'writable'. # BZ2File doesn't implement the file object context manager in python 2 # so we wrap the fileobj using `closing`. return closing(fobj) else: return io.BufferedWriter(fobj, buffer_size=_IO_BUFFER_SIZE)
def testLineBuffering(self): r = io.BytesIO() b = io.BufferedWriter(r, 1000) t = io.TextIOWrapper(b, newline="\n", line_buffering=True) t.write(u"X") self.assertEquals(r.getvalue(), b"") # No flush happened t.write(u"Y\nZ") self.assertEquals(r.getvalue(), b"XY\nZ") # All got flushed t.write(u"A\rB") self.assertEquals(r.getvalue(), b"XY\nZA\rB")
def test_sif(self): """Test 'Subtitle Intermediate Format' loaders and dumpers.""" # Make a test subtitle with two lines subtitle = Subtitle() subtitle.add_unit( SubtitleUnit(start=15, end=30, lines=['First line with \u0161'])) subtitle.add_unit( SubtitleUnit(start=65, end=89, position={ 'x': 50, 'y': 30, }, lines=[ 'Another, but a two liner \u010d', 'Yes, I said two liner! \u017e' ])) subtitle[0][0].special = True subtitle[0][0].complex = {'a': {'4': 2}, 'b': [1, 2, 3]} subtitle[1][1].test = 'test' # Write it tmpfd, tmp = tempfile.mkstemp() tmpfd2, tmp2 = tempfile.mkstemp() subtitle.save(io.BufferedWriter(io.FileIO(tmpfd, mode='w'))) subtitle.save(io.BufferedWriter(io.FileIO(tmpfd2, mode='w')), human_time=False) # Load it and test assert Subtitle.from_file(tmp) == subtitle assert Subtitle.from_file(tmp2) == subtitle # Remove temp files os.unlink(tmp) os.unlink(tmp2) # And some minor things # repr - just checking for exceptions repr(subtitle) for u in subtitle: repr(u) for l in u: repr(l)
def _get_writer(file_or_filename, encoding): # returns text write method and release all resources after using try: write = file_or_filename.write except AttributeError: # file_or_filename is a file name f = open( file_or_filename, "w", encoding="utf-8" if encoding == "unicode" else encoding, errors="xmlcharrefreplace", ) with f: yield f.write else: # file_or_filename is a file-like object # encoding determines if it is a text or binary writer if encoding == "unicode": # use a text writer as is yield write else: # wrap a binary writer with TextIOWrapper detach_buffer = False if isinstance(file_or_filename, io.BufferedIOBase): buf = file_or_filename elif isinstance(file_or_filename, io.RawIOBase): buf = io.BufferedWriter(file_or_filename) detach_buffer = True else: # This is to handle passed objects that aren't in the # IOBase hierarchy, but just have a write method buf = io.BufferedIOBase() buf.writable = lambda: True buf.write = write try: # TextIOWrapper uses this methods to determine # if BOM (for UTF-16, etc) should be added buf.seekable = file_or_filename.seekable buf.tell = file_or_filename.tell except AttributeError: pass wrapper = io.TextIOWrapper( buf, encoding=encoding, errors="xmlcharrefreplace", newline="\n", ) try: yield wrapper.write finally: # Keep the original file open when the TextIOWrapper and # the BufferedWriter are destroyed wrapper.detach() if detach_buffer: buf.detach()