def readchunk(self): """Reads a chunk at a time. If the current position is within a chunk the remainder of the chunk is returned. """ received = len(self.__buffer) chunk_data = EMPTY chunk_size = int(self.chunk_size) if received > 0: chunk_data = self.__buffer elif self.__position < int(self.length): chunk_number = int((received + self.__position) / chunk_size) chunk = self.__chunks.find_one( { "files_id": self._id, "n": chunk_number }, session=self._session) if not chunk: raise CorruptGridFile("no chunk #%d" % chunk_number) chunk_data = chunk["data"][self.__position % chunk_size:] if not chunk_data: raise CorruptGridFile("truncated chunk") self.__position += len(chunk_data) self.__buffer = EMPTY return chunk_data
def next(self): try: chunk = self._next_with_retry() except StopIteration: if self._next_chunk >= self._num_chunks: raise raise CorruptGridFile("no chunk #%d" % self._next_chunk) if chunk["n"] != self._next_chunk: self.close() raise CorruptGridFile( "Missing chunk: expected chunk #%d but found " "chunk with n=%d" % (self._next_chunk, chunk["n"])) if chunk["n"] >= self._num_chunks: # According to spec, ignore extra chunks if they are empty. if len(chunk["data"]): self.close() raise CorruptGridFile( "Extra chunk found: expected %d chunks but found " "chunk with n=%d" % (self._num_chunks, chunk["n"])) expected_length = self.expected_chunk_length(chunk["n"]) if len(chunk["data"]) != expected_length: self.close() raise CorruptGridFile( "truncated chunk #%d: expected chunk length to be %d but " "found chunk with length %d" % (chunk["n"], expected_length, len(chunk["data"]))) self._next_chunk += 1 return chunk
async def readchunk(self): """Reads a chunk at a time. If the current position is within a chunk the remainder of the chunk is returned. """ received = len(self.__buffer) chunk_data = EMPTY chunk_size = int(self.chunk_size) if received > 0: chunk_data = self.__buffer elif self.__position < int(self.length): chunk_number = int((received + self.__position) / chunk_size) chunk = await self.__chunks.find_one({ 'files_id': self._id, 'n': chunk_number }) if not chunk: raise CorruptGridFile('no chunk #%d' % chunk_number) chunk_data = chunk['data'][self.__position % chunk_size:] if not chunk_data: raise CorruptGridFile('truncated chunk') self.__position += len(chunk_data) self.__buffer = EMPTY return chunk_data
def readchunk(self): """Reads a chunk at a time. If the current position is within a chunk the remainder of the chunk is returned. """ received = len(self.__buffer) chunk_data = EMPTY chunk_size = int(self.chunk_size) if received > 0: chunk_data = self.__buffer elif self.__position < int(self.length): chunk_number = int((received + self.__position) / chunk_size) if self.__chunk_iter is None: self.__chunk_iter = _GridOutChunkIterator( self, self.__chunks, self._session, chunk_number) chunk = self.__chunk_iter.next() chunk_data = chunk["data"][self.__position % chunk_size:] if not chunk_data: raise CorruptGridFile("truncated chunk") self.__position += len(chunk_data) self.__buffer = EMPTY return chunk_data
def readchunk(self): """Reads a chunk at a time. If the current position is within a chunk the remainder of the chunk is returned. """ size = int(self.length) - self.__position received = len(self.__buffer) chunk_data = EMPTY if received > 0: chunk_data = self.__buffer elif received < size: chunk_number = int((received + self.__position) / self.chunk_size) chunk = self.__chunks.find_one({ "files_id": self._id, "n": chunk_number }) if not chunk: raise CorruptGridFile("no chunk #%d" % chunk_number) if received: chunk_data = chunk["data"] else: chunk_data = chunk["data"][self.__position % self.chunk_size:] self.__position += len(chunk_data) self.__buffer = EMPTY return chunk_data
def next(self): if self.__current_chunk >= self.__max_chunk: raise StopIteration chunk = self.__chunks.find_one({"files_id": self.__id, "n": self.__current_chunk}) if not chunk: raise CorruptGridFile("no chunk #%d" % self.__current_chunk) self.__current_chunk += 1 return binary_type(chunk["data"])
async def __anext__(self): if self.__current_chunk >= self.__max_chunk: raise StopAsyncIteration chunk = await self.__chunks.find_one({ 'files_id': self.__id, 'n': self.__current_chunk }) if not chunk: raise CorruptGridFile('no chunk #%d' % self.__current_chunk) self.__current_chunk += 1 return bytes(chunk['data'])
def read(self, size=-1): """Read at most `size` bytes from the file (less if there isn't enough data). The bytes are returned as an instance of :class:`str` (:class:`bytes` in python 3). If `size` is negative or omitted all data is read. :Parameters: - `size` (optional): the number of bytes to read """ self._ensure_file() if size == 0: return EMPTY remainder = int(self.length) - self.__position if size < 0 or size > remainder: size = remainder received = 0 data = StringIO() while received < size: chunk_data = self.readchunk() received += len(chunk_data) data.write(chunk_data) # Detect extra chunks. max_chunk_n = math.ceil(self.length / float(self.chunk_size)) chunk = self.__chunks.find_one( { "files_id": self._id, "n": { "$gte": max_chunk_n } }, session=self._session) # According to spec, ignore extra chunks if they are empty. if chunk is not None and len(chunk['data']): raise CorruptGridFile( "Extra chunk found: expected %i chunks but found " "chunk with n=%i" % (max_chunk_n, chunk['n'])) self.__position -= received - size # Return 'size' bytes and store the rest. data.seek(size) self.__buffer = data.read() data.seek(0) return data.read(size)
def read(self, size=-1): """Read at most `size` bytes from the file (less if there isn't enough data). The bytes are returned as an instance of :class:`str` (:class:`bytes` in python 3). If `size` is negative or omitted all data is read. :Parameters: - `size` (optional): the number of bytes to read """ if size == 0: return "" remainder = int(self.length) - self.__position if size < 0 or size > remainder: size = remainder received = len(self.__buffer) chunk_number = int((received + self.__position) / self.chunk_size) chunks = [] while received < size: chunk = self.__chunks.find_one({ "files_id": self._id, "n": chunk_number }) if not chunk: raise CorruptGridFile("no chunk #%d" % chunk_number) if received: chunk_data = chunk["data"] else: chunk_data = chunk["data"][self.__position % self.chunk_size:] received += len(chunk_data) chunks.append(chunk_data) chunk_number += 1 data = EMPTY.join([self.__buffer] + chunks) self.__position += size to_return = data[:size] self.__buffer = data[size:] return to_return
def read(self, size=-1): """Read at most `size` bytes from the file (less if there isn't enough data). The bytes are returned as an instance of :class:`str`. If `size` is negative or omitted all data is read. :Parameters: - `size` (optional): the number of bytes to read """ if size == 0: return "" remainder = int(self.length) - self.__position if size < 0 or size > remainder: size = remainder data = self.__buffer chunk_number = (len(data) + self.__position) / self.chunk_size while len(data) < size: chunk = self.__chunks.find_one({ "files_id": self._id, "n": chunk_number }) if not chunk: raise CorruptGridFile("no chunk #%d" % chunk_number) if not data: data += chunk["data"][self.__position % self.chunk_size:] else: data += chunk["data"] chunk_number += 1 self.__position += size to_return = data[:size] self.__buffer = data[size:] return to_return