def __init__(self, _uuid):
super(OpenedStagedAjaxFile, self).__init__()
self.__uuid = _uuid
self.__chunks = list(
StagedFile.objects.filter(file_id=self.__uuid).all())
self.__chunks.sort(key=lambda x: x.start_byte)
self.__chunk_map = IntervalMap()
for chunk in self.__chunks:
self.__chunk_map.append_interval(
chunk.end_byte - chunk.start_byte + 1, chunk)
self.__file_pointer = 0
self.__current_chunk = None
def __init__(self, _uuid):
super().__init__()
self._uuid = _uuid
self._chunks = list(
StagedFile.objects.filter(file_id=self._uuid).all()
)
self._chunks.sort(key=lambda x: x.start_byte)
self._chunk_map = IntervalMap()
for chunk in self._chunks:
self._chunk_map.append_interval(
chunk.end_byte - chunk.start_byte + 1, chunk
)
self._file_pointer = 0
self._current_chunk = None
def test_interval_map_indexing_and_length():
im = IntervalMap()
im.append_interval(10, 1)
im.append_interval(10, 2)
assert ([1] * 10 + [2] * 10) == list(im)
assert len(im) == 20
assert im.len == 20
with pytest.raises(IndexError):
im[-1]
with pytest.raises(IndexError):
im[2000]
with pytest.raises(TypeError):
im[0.1]
# Test if we can handle HUUUGE intervals
o = object()
im.append_interval(1 * 10 ** 32, o)
# Ans aldo check that no value-copy issues are ocurring
assert im[1 * 10 ** 32] is o
assert im.len == 1 * 10 ** 32 + 20
with pytest.raises(IndexError):
im[1 * 10 ** 32 + 20]
class OpenedStagedAjaxFile(BufferedIOBase):
"""
This class behaves like a file handle for a :class:`StagedAjaxFile`.
The file handle is strictly read-only. Under the hood, this class
reconstructs the contingent file from the file chunks that have been
uploaded.
"""
# TODO: This really should be an instance of BufferedIOBase and follow the
# specifications there.
def __init__(self, _uuid):
super(OpenedStagedAjaxFile, self).__init__()
self.__uuid = _uuid
self.__chunks = list(
StagedFile.objects.filter(file_id=self.__uuid).all())
self.__chunks.sort(key=lambda x: x.start_byte)
self.__chunk_map = IntervalMap()
for chunk in self.__chunks:
self.__chunk_map.append_interval(
chunk.end_byte - chunk.start_byte + 1, chunk)
self.__file_pointer = 0
self.__current_chunk = None
@property
def closed(self):
return self.__chunks is None
@property
def size(self):
if self.closed:
return None
else:
return len(self.__chunk_map)
def readable(self, *args, **kwargs):
return True
def writable(self, *args, **kwargs):
return False
def seekable(self, *args, **kwargs):
return True
def readinto(self, buffer):
read_bytes = self.read(len(buffer))
buffer[:len(read_bytes)] = read_bytes
return len(read_bytes)
def read(self, size=-1):
if size < 0:
size = None
if self.closed:
raise ValueError('file closed')
if self.size <= self.__file_pointer:
return b""
if self.__file_pointer < 0:
raise IOError('invalid file pointer position')
if size is None:
size = self.size - self.__file_pointer
result = b""
while len(result) < size:
if self.__file_pointer >= len(self.__chunk_map):
break
this_chunk = self.__chunk_map[self.__file_pointer]
if this_chunk is not self.__current_chunk:
# we need to switch to a new chunk
if self.__current_chunk is not None:
self.__current_chunk.file.close()
self.__current_chunk = None
this_chunk.file.open('rb')
this_chunk.file.seek(self.__file_pointer -
this_chunk.start_byte)
self.__current_chunk = this_chunk
read_size = min(
size - len(result),
self.__current_chunk.end_byte + 1 - self.__file_pointer,
)
result += self.__current_chunk.file.read(read_size)
self.__file_pointer += read_size
return result
def read1(self, size=-1):
return self.read(size=size)
def readinto1(self, buffer):
return self.readinto(buffer)
def seek(self, offset, from_what=0):
if self.closed:
raise ValueError('file closed')
new_pointer = None
if from_what == 0:
new_pointer = offset
elif from_what == 1:
new_pointer = self.__file_pointer + offset
elif from_what == 2:
new_pointer = self.size + offset
if new_pointer < 0:
raise IOError('invalid file pointer')
self.__file_pointer = new_pointer
if self.__file_pointer < self.__chunk_map.len:
if self.__chunk_map[self.__file_pointer] is self.__current_chunk:
self.__current_chunk.file.seek(self.__file_pointer -
self.__current_chunk.start_byte)
return self.__file_pointer
def tell(self, *args, **kwargs):
if self.closed:
raise ValueError('file closed')
return self.__file_pointer
def close(self):
if not self.closed:
self.__chunks = None
if self.__current_chunk is not None:
self.__current_chunk.file.close()
self.__current_chunk = None
def test_invalid_types():
im = IntervalMap()
im.append_interval(10, "a")
im.append_interval(5, "b")
im.append_interval(3, "c")
with pytest.raises(TypeError):
im["wrong index"]
with pytest.raises(IndexError):
im[-1]
with pytest.raises(IndexError):
im[1000000]
with pytest.raises(TypeError):
im.get_offset("wrong index")
with pytest.raises(IndexError):
im.get_offset(-1)
with pytest.raises(IndexError):
im.get_offset(1000000)
def test_interval_map_get_offset():
im = IntervalMap()
im.append_interval(10, "a")
im.append_interval(5, "b")
im.append_interval(3, "c")
assert im.get_offset(0) == 0
assert im.get_offset(4) == 0
assert im.get_offset(9) == 0
assert im.get_offset(10) == 10
assert im.get_offset(14) == 10
assert im.get_offset(15) == 15
assert im.get_offset(16) == 15
with pytest.raises(IndexError):
im.get_offset(18)
with pytest.raises(IndexError):
im.get_offset(-1)
with pytest.raises(TypeError):
im.get_offset(-1.0)
class OpenedStagedAjaxFile(BufferedIOBase):
"""
This class behaves like a file handle for a :class:`StagedAjaxFile`.
The file handle is strictly read-only. Under the hood, this class
reconstructs the contingent file from the file chunks that have been
uploaded.
"""
def __init__(self, _uuid):
super().__init__()
self._uuid = _uuid
self._chunks = list(
StagedFile.objects.filter(file_id=self._uuid).all()
)
self._chunks.sort(key=lambda x: x.start_byte)
self._chunk_map = IntervalMap()
for chunk in self._chunks:
self._chunk_map.append_interval(
chunk.end_byte - chunk.start_byte + 1, chunk
)
self._file_pointer = 0
self._current_chunk = None
@property
def closed(self):
return self._chunks is None
@property
def size(self):
if self.closed:
return None
else:
return len(self._chunk_map)
def readable(self, *args, **kwargs):
return True
def writable(self, *args, **kwargs):
return False
def seekable(self, *args, **kwargs):
return True
def readinto(self, buffer):
read_bytes = self.read(len(buffer))
buffer[: len(read_bytes)] = read_bytes
return len(read_bytes)
def read(self, size=-1):
if size < 0:
size = None
if self.closed:
raise ValueError("file closed")
if self.size <= self._file_pointer:
return b""
if self._file_pointer < 0:
raise IOError("invalid file pointer position")
if size is None:
size = self.size - self._file_pointer
result = b""
while len(result) < size:
if self._file_pointer >= len(self._chunk_map):
break
this_chunk = self._chunk_map[self._file_pointer]
if this_chunk is not self._current_chunk:
# we need to switch to a new chunk
if self._current_chunk is not None:
self._current_chunk.file.close()
self._current_chunk = None
this_chunk.file.open("rb")
this_chunk.file.seek(
self._file_pointer - this_chunk.start_byte
)
self._current_chunk = this_chunk
read_size = min(
size - len(result),
self._current_chunk.end_byte + 1 - self._file_pointer,
)
result += self._current_chunk.file.read(read_size)
self._file_pointer += read_size
return result
def read1(self, size=-1):
return self.read(size=size)
def readinto1(self, buffer):
return self.readinto(buffer)
def seek(self, offset, from_what=0):
if self.closed:
raise ValueError("file closed")
new_pointer = None
if from_what == 0:
new_pointer = offset
elif from_what == 1:
new_pointer = self._file_pointer + offset
elif from_what == 2:
new_pointer = self.size + offset
if new_pointer < 0:
raise IOError("invalid file pointer")
self._file_pointer = new_pointer
if self._file_pointer < self._chunk_map.len:
if self._chunk_map[self._file_pointer] is self._current_chunk:
self._current_chunk.file.seek(
self._file_pointer - self._current_chunk.start_byte
)
return self._file_pointer
def tell(self, *args, **kwargs):
if self.closed:
raise ValueError("file closed")
return self._file_pointer
def close(self):
if not self.closed:
self._chunks = None
if self._current_chunk is not None:
self._current_chunk.file.close()
self._current_chunk = None