def __init__(self, _uuid): super(OpenedStagedAjaxFile, self).__init__() self.__uuid = _uuid self.__chunks = list( StagedFile.objects.filter(file_id=self.__uuid).all()) self.__chunks.sort(key=lambda x: x.start_byte) self.__chunk_map = IntervalMap() for chunk in self.__chunks: self.__chunk_map.append_interval( chunk.end_byte - chunk.start_byte + 1, chunk) self.__file_pointer = 0 self.__current_chunk = None
def __init__(self, _uuid): super().__init__() self._uuid = _uuid self._chunks = list( StagedFile.objects.filter(file_id=self._uuid).all() ) self._chunks.sort(key=lambda x: x.start_byte) self._chunk_map = IntervalMap() for chunk in self._chunks: self._chunk_map.append_interval( chunk.end_byte - chunk.start_byte + 1, chunk ) self._file_pointer = 0 self._current_chunk = None
def test_interval_map_indexing_and_length(): im = IntervalMap() im.append_interval(10, 1) im.append_interval(10, 2) assert ([1] * 10 + [2] * 10) == list(im) assert len(im) == 20 assert im.len == 20 with pytest.raises(IndexError): im[-1] with pytest.raises(IndexError): im[2000] with pytest.raises(TypeError): im[0.1] # Test if we can handle HUUUGE intervals o = object() im.append_interval(1 * 10 ** 32, o) # Ans aldo check that no value-copy issues are ocurring assert im[1 * 10 ** 32] is o assert im.len == 1 * 10 ** 32 + 20 with pytest.raises(IndexError): im[1 * 10 ** 32 + 20]
class OpenedStagedAjaxFile(BufferedIOBase): """ This class behaves like a file handle for a :class:`StagedAjaxFile`. The file handle is strictly read-only. Under the hood, this class reconstructs the contingent file from the file chunks that have been uploaded. """ # TODO: This really should be an instance of BufferedIOBase and follow the # specifications there. def __init__(self, _uuid): super(OpenedStagedAjaxFile, self).__init__() self.__uuid = _uuid self.__chunks = list( StagedFile.objects.filter(file_id=self.__uuid).all()) self.__chunks.sort(key=lambda x: x.start_byte) self.__chunk_map = IntervalMap() for chunk in self.__chunks: self.__chunk_map.append_interval( chunk.end_byte - chunk.start_byte + 1, chunk) self.__file_pointer = 0 self.__current_chunk = None @property def closed(self): return self.__chunks is None @property def size(self): if self.closed: return None else: return len(self.__chunk_map) def readable(self, *args, **kwargs): return True def writable(self, *args, **kwargs): return False def seekable(self, *args, **kwargs): return True def readinto(self, buffer): read_bytes = self.read(len(buffer)) buffer[:len(read_bytes)] = read_bytes return len(read_bytes) def read(self, size=-1): if size < 0: size = None if self.closed: raise ValueError('file closed') if self.size <= self.__file_pointer: return b"" if self.__file_pointer < 0: raise IOError('invalid file pointer position') if size is None: size = self.size - self.__file_pointer result = b"" while len(result) < size: if self.__file_pointer >= len(self.__chunk_map): break this_chunk = self.__chunk_map[self.__file_pointer] if this_chunk is not self.__current_chunk: # we need to switch to a new chunk if self.__current_chunk is not None: self.__current_chunk.file.close() self.__current_chunk = None this_chunk.file.open('rb') this_chunk.file.seek(self.__file_pointer - this_chunk.start_byte) self.__current_chunk = this_chunk read_size = min( size - len(result), self.__current_chunk.end_byte + 1 - self.__file_pointer, ) result += self.__current_chunk.file.read(read_size) self.__file_pointer += read_size return result def read1(self, size=-1): return self.read(size=size) def readinto1(self, buffer): return self.readinto(buffer) def seek(self, offset, from_what=0): if self.closed: raise ValueError('file closed') new_pointer = None if from_what == 0: new_pointer = offset elif from_what == 1: new_pointer = self.__file_pointer + offset elif from_what == 2: new_pointer = self.size + offset if new_pointer < 0: raise IOError('invalid file pointer') self.__file_pointer = new_pointer if self.__file_pointer < self.__chunk_map.len: if self.__chunk_map[self.__file_pointer] is self.__current_chunk: self.__current_chunk.file.seek(self.__file_pointer - self.__current_chunk.start_byte) return self.__file_pointer def tell(self, *args, **kwargs): if self.closed: raise ValueError('file closed') return self.__file_pointer def close(self): if not self.closed: self.__chunks = None if self.__current_chunk is not None: self.__current_chunk.file.close() self.__current_chunk = None
def test_invalid_types(): im = IntervalMap() im.append_interval(10, "a") im.append_interval(5, "b") im.append_interval(3, "c") with pytest.raises(TypeError): im["wrong index"] with pytest.raises(IndexError): im[-1] with pytest.raises(IndexError): im[1000000] with pytest.raises(TypeError): im.get_offset("wrong index") with pytest.raises(IndexError): im.get_offset(-1) with pytest.raises(IndexError): im.get_offset(1000000)
def test_interval_map_get_offset(): im = IntervalMap() im.append_interval(10, "a") im.append_interval(5, "b") im.append_interval(3, "c") assert im.get_offset(0) == 0 assert im.get_offset(4) == 0 assert im.get_offset(9) == 0 assert im.get_offset(10) == 10 assert im.get_offset(14) == 10 assert im.get_offset(15) == 15 assert im.get_offset(16) == 15 with pytest.raises(IndexError): im.get_offset(18) with pytest.raises(IndexError): im.get_offset(-1) with pytest.raises(TypeError): im.get_offset(-1.0)
class OpenedStagedAjaxFile(BufferedIOBase): """ This class behaves like a file handle for a :class:`StagedAjaxFile`. The file handle is strictly read-only. Under the hood, this class reconstructs the contingent file from the file chunks that have been uploaded. """ def __init__(self, _uuid): super().__init__() self._uuid = _uuid self._chunks = list( StagedFile.objects.filter(file_id=self._uuid).all() ) self._chunks.sort(key=lambda x: x.start_byte) self._chunk_map = IntervalMap() for chunk in self._chunks: self._chunk_map.append_interval( chunk.end_byte - chunk.start_byte + 1, chunk ) self._file_pointer = 0 self._current_chunk = None @property def closed(self): return self._chunks is None @property def size(self): if self.closed: return None else: return len(self._chunk_map) def readable(self, *args, **kwargs): return True def writable(self, *args, **kwargs): return False def seekable(self, *args, **kwargs): return True def readinto(self, buffer): read_bytes = self.read(len(buffer)) buffer[: len(read_bytes)] = read_bytes return len(read_bytes) def read(self, size=-1): if size < 0: size = None if self.closed: raise ValueError("file closed") if self.size <= self._file_pointer: return b"" if self._file_pointer < 0: raise IOError("invalid file pointer position") if size is None: size = self.size - self._file_pointer result = b"" while len(result) < size: if self._file_pointer >= len(self._chunk_map): break this_chunk = self._chunk_map[self._file_pointer] if this_chunk is not self._current_chunk: # we need to switch to a new chunk if self._current_chunk is not None: self._current_chunk.file.close() self._current_chunk = None this_chunk.file.open("rb") this_chunk.file.seek( self._file_pointer - this_chunk.start_byte ) self._current_chunk = this_chunk read_size = min( size - len(result), self._current_chunk.end_byte + 1 - self._file_pointer, ) result += self._current_chunk.file.read(read_size) self._file_pointer += read_size return result def read1(self, size=-1): return self.read(size=size) def readinto1(self, buffer): return self.readinto(buffer) def seek(self, offset, from_what=0): if self.closed: raise ValueError("file closed") new_pointer = None if from_what == 0: new_pointer = offset elif from_what == 1: new_pointer = self._file_pointer + offset elif from_what == 2: new_pointer = self.size + offset if new_pointer < 0: raise IOError("invalid file pointer") self._file_pointer = new_pointer if self._file_pointer < self._chunk_map.len: if self._chunk_map[self._file_pointer] is self._current_chunk: self._current_chunk.file.seek( self._file_pointer - self._current_chunk.start_byte ) return self._file_pointer def tell(self, *args, **kwargs): if self.closed: raise ValueError("file closed") return self._file_pointer def close(self): if not self.closed: self._chunks = None if self._current_chunk is not None: self._current_chunk.file.close() self._current_chunk = None