def readchunk(self):
"""Reads a chunk at a time. If the current position is within a
chunk the remainder of the chunk is returned.
"""
received = len(self.__buffer)
chunk_data = EMPTY
chunk_size = int(self.chunk_size)
if received > 0:
chunk_data = self.__buffer
elif self.__position < int(self.length):
chunk_number = int((received + self.__position) / chunk_size)
chunk = self.__chunks.find_one(
{
"files_id": self._id,
"n": chunk_number
},
session=self._session)
if not chunk:
raise CorruptGridFile("no chunk #%d" % chunk_number)
chunk_data = chunk["data"][self.__position % chunk_size:]
if not chunk_data:
raise CorruptGridFile("truncated chunk")
self.__position += len(chunk_data)
self.__buffer = EMPTY
return chunk_data
def next(self):
try:
chunk = self._next_with_retry()
except StopIteration:
if self._next_chunk >= self._num_chunks:
raise
raise CorruptGridFile("no chunk #%d" % self._next_chunk)
if chunk["n"] != self._next_chunk:
self.close()
raise CorruptGridFile(
"Missing chunk: expected chunk #%d but found "
"chunk with n=%d" % (self._next_chunk, chunk["n"]))
if chunk["n"] >= self._num_chunks:
# According to spec, ignore extra chunks if they are empty.
if len(chunk["data"]):
self.close()
raise CorruptGridFile(
"Extra chunk found: expected %d chunks but found "
"chunk with n=%d" % (self._num_chunks, chunk["n"]))
expected_length = self.expected_chunk_length(chunk["n"])
if len(chunk["data"]) != expected_length:
self.close()
raise CorruptGridFile(
"truncated chunk #%d: expected chunk length to be %d but "
"found chunk with length %d" %
(chunk["n"], expected_length, len(chunk["data"])))
self._next_chunk += 1
return chunk
async def readchunk(self):
"""Reads a chunk at a time. If the current position is within a
chunk the remainder of the chunk is returned.
"""
received = len(self.__buffer)
chunk_data = EMPTY
chunk_size = int(self.chunk_size)
if received > 0:
chunk_data = self.__buffer
elif self.__position < int(self.length):
chunk_number = int((received + self.__position) / chunk_size)
chunk = await self.__chunks.find_one({
'files_id': self._id,
'n': chunk_number
})
if not chunk:
raise CorruptGridFile('no chunk #%d' % chunk_number)
chunk_data = chunk['data'][self.__position % chunk_size:]
if not chunk_data:
raise CorruptGridFile('truncated chunk')
self.__position += len(chunk_data)
self.__buffer = EMPTY
return chunk_data
def readchunk(self):
"""Reads a chunk at a time. If the current position is within a
chunk the remainder of the chunk is returned.
"""
received = len(self.__buffer)
chunk_data = EMPTY
chunk_size = int(self.chunk_size)
if received > 0:
chunk_data = self.__buffer
elif self.__position < int(self.length):
chunk_number = int((received + self.__position) / chunk_size)
if self.__chunk_iter is None:
self.__chunk_iter = _GridOutChunkIterator(
self, self.__chunks, self._session, chunk_number)
chunk = self.__chunk_iter.next()
chunk_data = chunk["data"][self.__position % chunk_size:]
if not chunk_data:
raise CorruptGridFile("truncated chunk")
self.__position += len(chunk_data)
self.__buffer = EMPTY
return chunk_data
def readchunk(self):
"""Reads a chunk at a time. If the current position is within a
chunk the remainder of the chunk is returned.
"""
size = int(self.length) - self.__position
received = len(self.__buffer)
chunk_data = EMPTY
if received > 0:
chunk_data = self.__buffer
elif received < size:
chunk_number = int((received + self.__position) / self.chunk_size)
chunk = self.__chunks.find_one({
"files_id": self._id,
"n": chunk_number
})
if not chunk:
raise CorruptGridFile("no chunk #%d" % chunk_number)
if received:
chunk_data = chunk["data"]
else:
chunk_data = chunk["data"][self.__position % self.chunk_size:]
self.__position += len(chunk_data)
self.__buffer = EMPTY
return chunk_data
def next(self):
if self.__current_chunk >= self.__max_chunk:
raise StopIteration
chunk = self.__chunks.find_one({"files_id": self.__id,
"n": self.__current_chunk})
if not chunk:
raise CorruptGridFile("no chunk #%d" % self.__current_chunk)
self.__current_chunk += 1
return binary_type(chunk["data"])
async def __anext__(self):
if self.__current_chunk >= self.__max_chunk:
raise StopAsyncIteration
chunk = await self.__chunks.find_one({
'files_id': self.__id,
'n': self.__current_chunk
})
if not chunk:
raise CorruptGridFile('no chunk #%d' % self.__current_chunk)
self.__current_chunk += 1
return bytes(chunk['data'])
def read(self, size=-1):
"""Read at most `size` bytes from the file (less if there
isn't enough data).
The bytes are returned as an instance of :class:`str` (:class:`bytes`
in python 3). If `size` is negative or omitted all data is read.
:Parameters:
- `size` (optional): the number of bytes to read
"""
self._ensure_file()
if size == 0:
return EMPTY
remainder = int(self.length) - self.__position
if size < 0 or size > remainder:
size = remainder
received = 0
data = StringIO()
while received < size:
chunk_data = self.readchunk()
received += len(chunk_data)
data.write(chunk_data)
# Detect extra chunks.
max_chunk_n = math.ceil(self.length / float(self.chunk_size))
chunk = self.__chunks.find_one(
{
"files_id": self._id,
"n": {
"$gte": max_chunk_n
}
},
session=self._session)
# According to spec, ignore extra chunks if they are empty.
if chunk is not None and len(chunk['data']):
raise CorruptGridFile(
"Extra chunk found: expected %i chunks but found "
"chunk with n=%i" % (max_chunk_n, chunk['n']))
self.__position -= received - size
# Return 'size' bytes and store the rest.
data.seek(size)
self.__buffer = data.read()
data.seek(0)
return data.read(size)
def read(self, size=-1):
"""Read at most `size` bytes from the file (less if there
isn't enough data).
The bytes are returned as an instance of :class:`str` (:class:`bytes`
in python 3). If `size` is negative or omitted all data is read.
:Parameters:
- `size` (optional): the number of bytes to read
"""
if size == 0:
return ""
remainder = int(self.length) - self.__position
if size < 0 or size > remainder:
size = remainder
received = len(self.__buffer)
chunk_number = int((received + self.__position) / self.chunk_size)
chunks = []
while received < size:
chunk = self.__chunks.find_one({
"files_id": self._id,
"n": chunk_number
})
if not chunk:
raise CorruptGridFile("no chunk #%d" % chunk_number)
if received:
chunk_data = chunk["data"]
else:
chunk_data = chunk["data"][self.__position % self.chunk_size:]
received += len(chunk_data)
chunks.append(chunk_data)
chunk_number += 1
data = EMPTY.join([self.__buffer] + chunks)
self.__position += size
to_return = data[:size]
self.__buffer = data[size:]
return to_return
def read(self, size=-1):
"""Read at most `size` bytes from the file (less if there
isn't enough data).
The bytes are returned as an instance of :class:`str`. If
`size` is negative or omitted all data is read.
:Parameters:
- `size` (optional): the number of bytes to read
"""
if size == 0:
return ""
remainder = int(self.length) - self.__position
if size < 0 or size > remainder:
size = remainder
data = self.__buffer
chunk_number = (len(data) + self.__position) / self.chunk_size
while len(data) < size:
chunk = self.__chunks.find_one({
"files_id": self._id,
"n": chunk_number
})
if not chunk:
raise CorruptGridFile("no chunk #%d" % chunk_number)
if not data:
data += chunk["data"][self.__position % self.chunk_size:]
else:
data += chunk["data"]
chunk_number += 1
self.__position += size
to_return = data[:size]
self.__buffer = data[size:]
return to_return
