def seek_helper(self, offset, whence): """ Helper function for use by implementations of seek(). Calculates the new cursor position from offset, whence, and self.tell(). If size is given, it works for whence=os.SEEK_END; otherwise, UnsupportedOperation is raised. """ if whence == os.SEEK_SET: target = offset elif whence == os.SEEK_CUR: target = offset + self.tell() elif whence == os.SEEK_END: size = self.get_size() if size < 0: raise UnsupportedOperation( "can only seek relative to file start or cursor") target = offset + size else: raise UnsupportedOperation("unsupported seek mode") if target < 0: raise ValueError("can not seek to a negative file position") return target
def __getattribute__(self, name): attr = super().__getattribute__(name) if attr: if 'read' in name and 'r' not in self._mode: raise UnsupportedOperation('not readable') elif 'write' in name and 'w' not in self._mode and 'a' not in self._mode: raise UnsupportedOperation('not writable') return attr
def drop_group(self: DatabaseRegistrar, data_group: str) -> None: with self._db_adapter: if not self.has_group(data_group): raise MissingGroupError(data_group) if data_group == PROTECTED_GROUP: raise UnsupportedOperation( "Cannot delete protected data group") if self._group_contains_protected_tables(data_group): raise UnsupportedOperation( "Cannot delete group that contains protected table") for token in self.list_group_tables(data_group): self.drop_table(token)
def write_genotypes(self, genotypes): """Write genotypes to binary file. Args: genotypes (numpy.ndarray): The genotypes to write in the BED file. """ if self._mode != "w": raise UnsupportedOperation("not available in 'r' mode") # Initializing the number of samples if required if self._nb_values is None: self._nb_values = len(genotypes) # Checking the expected number of samples if self._nb_values != len(genotypes): raise ValueError("{:,d} samples expected, got {:,d}".format( self._nb_values, len(genotypes), )) # Writing to file byte_array = [ g[0] | (g[1] << 2) | (g[2] << 4) | (g[3] << 6) for g in self._grouper((_byte_recode[geno] for geno in genotypes), 4) ] self._bed.write(bytearray(byte_array))
def seek(self, offset, whence=SEEK_SET): if self.closed: raise ValueError('I/O operation on closed file.') if whence not in seek_whence.keys(): raise ValueError(f'whence {whence} not supported') if not isinstance(offset, int): raise TypeError(f'offset must be int, not {type(offset).__name__}') pos = self.position if whence == SEEK_SET: pos = offset elif whence == SEEK_CUR: pos += offset elif whence == SEEK_END: if self.size == None: raise UnsupportedOperation( 'end-relative seek not supported for this URL') pos = self.size + offset if pos < 0: raise ValueError(f'resulting offset {pos} < 0') self._log( f'seek from {self.position} + ({offset}, {seek_whence[whence]}) -> {pos}' ) self.position = pos return self.position
def seek(self, pos: int, whence: int = SEEK_SET) -> int: """Change stream position. Change the stream position to byte offset pos. Argument pos is interpreted relative to the position indicated by whence. Values for whence are ints: * 0 -- start of stream (the default); offset should be zero or positive * 1 -- current stream position; offset may be negative * 2 -- end of stream; offset should be negative Return an int indicating the new absolute position. """ self._check_not_closed() if not self.seekable(): # just in case seekable is overridden raise UnsupportedOperation("seek") if whence == SEEK_SET: pass elif whence == SEEK_CUR: pos += self._pos elif whence == SEEK_END: pos += self._length else: raise ValueError("unsupported whence value") if pos >= 0: self._pos = pos return self._pos raise ValueError("invalid seek position")
def write(self, data: bytes) -> int: """Write data, passed as a bytes object. Returns the number of bytes written, which is always the length of the input data in bytes. """ self._check_not_closed() if not self.writable(): raise UnsupportedOperation("write") written_bytes = len(data) padding_size = self._pos - self._length if padding_size < 0: raise ValueError("write is only supported from EOF") if padding_size > 0: null_bytes = memoryview(bytearray(DEFAULT_BUFFER_SIZE)) self._pos = self._length data = memoryview(data) while padding_size or data: self._write_start() if padding_size > 0: # pad with null bytes, not counted in written_bytes padding = null_bytes[:padding_size] written_len = self._write( padding) # do not stop if nothing was written padding_size -= written_len else: written_len = self._write( data) # do not stop if nothing was written data = data[written_len:] self._pos += written_len self._length = max(self._length, self._pos) return written_bytes
def readinto(self, b): """ Read bytes into a pre-allocated, writable bytes-like object b, and return the number of bytes read. Args: b (bytes-like object): buffer. Returns: int: number of bytes read """ if not self._readable: raise UnsupportedOperation("read") size = len(b) with self._seek_lock: start = self._seek end = start + size self._seek = end with handle_os_exceptions(): read_data = self._read_range(start, end) read_size = len(read_data) if read_size: memoryview(b)[:read_size] = read_data if read_size != size: with self._seek_lock: self._seek = start + read_size return read_size
def drop_table(self: DatabaseRegistrar, data_token: DataToken) -> None: with self._db_adapter: if not self.has_table(data_token): raise MissingTableError(data_token) if self._is_table_protected(data_token): raise UnsupportedOperation( "Cannot delete from protected data group") self._db_adapter.drop_group_table(data_token) store_type, store_version = self._table_store_type_version( data_token) store_type_version_count = sum( (TableReference.store_type == store_type) & (TableReference.store_version == store_version)) version_count = self._db_adapter.query(TableReference.data_token, store_type_version_count) if version_count == 1: self._drop_store_type(store_type, store_version) criteria = (TableReference.table_name == data_token.table_name) & ( TableReference.data_group == data_token.data_group) self._db_adapter.delete(TableReference.data_token, criteria) if len(self.list_group_tables(data_token.data_group)) == 0: self._db_adapter.drop_group(data_token.data_group) store_indices = (IndexReference.store_type == store_type) & ( TableReference.store_version == store_version) self._db_adapter.delete(IndexReference.data_token, store_indices)
def seek(self, offset, whence=SEEK_SET): """ Change the stream position to the given byte offset. Args: offset: Offset is interpreted relative to the position indicated by whence. whence: The default value for whence is SEEK_SET. Values are: SEEK_SET or 0 – start of the stream (the default); offset should be zero or positive SEEK_CUR or 1 – current stream position; offset may be negative SEEK_END or 2 – end of the stream; offset is usually negative Returns: int: The new absolute position. """ if not self._seekable: raise UnsupportedOperation("seek") with self._seek_lock: self.raw.seek(offset, whence) self._seek = seek = self.raw._seek self._preload_range() return seek
def seek(self, offset, whence=SEEK_SET): """ Change the stream position to the given byte offset. Args: offset (int): Offset is interpreted relative to the position indicated by whence. whence (int): The default value for whence is SEEK_SET. Values for whence are: SEEK_SET or 0 – start of the stream (the default); offset should be zero or positive SEEK_CUR or 1 – current stream position; offset may be negative SEEK_END or 2 – end of the stream; offset is usually negative Returns: int: The new absolute position. """ if not self._seekable: raise UnsupportedOperation('seek') # Flush before moving position self.flush() return self._update_seek(offset, whence)
def write(self, *args, **kwargs): """ This streams doesn't support writing. :raises UnsupportedOperation: """ raise UnsupportedOperation("This stream doesn't support write")
def get_unk_toks_indices(sentence, tokenizer, old_style_tokenizer): tokenizer_class = repr(old_style_tokenizer.__class__) if "models.bert" in tokenizer_class: recovery_method = recover_bert_unk_alignment elif "models.roberta" in tokenizer_class: recovery_method = recover_roberta_unk_alignment else: raise UnsupportedOperation( f"Unknown tokenizer class {tokenizer_class}") # when we get to printing, the only guarantee we have is that the outputted # sentence has the same number of wordpieces as the inputted sentence, so # we need to use the wordpiece-based indices chunks = tokenizer.tokenize(sentence) unk_chunk_indices = [ i for i, chunk in enumerate(chunks) if chunk == tokenizer.unk_token ] unknown_strings = recovery_method(old_style_tokenizer, sentence) # make sure no funky tokenization going on here if len(unk_chunk_indices) != len(unknown_strings): import pdb pdb.set_trace() mapping = { idx: string for idx, string in zip(unk_chunk_indices, unknown_strings) } return mapping
def write(self, b): """ Write the given bytes-like object, b, to the underlying raw stream, and return the number of bytes written. Args: b (bytes-like object): Bytes to write. Returns: int: The number of bytes written. """ if not self._writable: raise UnsupportedOperation('write') # This function write data in a buffer # "flush()" need to be called to really write content on # Cloud Storage size = len(b) with self._seek_lock: start = self._seek end = start + size self._seek = end buffer = self._write_buffer if end <= len(buffer): buffer = memoryview(buffer) buffer[start:end] = b return size
def seek(self, offset, whence=SEEK_SET): """ Change the stream position to the given byte offset. Args: offset (int): Offset is interpreted relative to the position indicated by whence. whence (int): The default value for whence is SEEK_SET. Values for whence are: SEEK_SET or 0 – start of the stream (the default); offset should be zero or positive SEEK_CUR or 1 – current stream position; offset may be negative SEEK_END or 2 – end of the stream; offset is usually negative Returns: int: The new absolute position. """ if not self._seekable: raise UnsupportedOperation('seek') seek = self._update_seek(offset, whence) # If seek move out of file, add padding until new seek position. if self._writable: size = len(self._write_buffer) if seek > size: self._write_buffer[seek:size] = b'\0' * (seek - size) return seek
def write(self, b): """ Write the given bytes-like object, b, to the underlying raw stream, and return the number of bytes written. Args: b (bytes-like object): Bytes to write. Returns: int: The number of bytes written. """ if not self._writable: raise UnsupportedOperation("write") size = len(b) with self._seek_lock: start = self._seek end = start + size self._seek = end buffer = self._write_buffer if end <= len(buffer): buffer = memoryview(buffer) buffer[start:end] = b return size
def _position(self): self._log( '_position', 'position=%d, stream_position=%d' % (self.position, self.stream_position)) if self.r: self.r.close() headers = {'Accept-Encoding': 'identity'} headers.update({'Range': 'bytes=%d-' % self.position} if self.position != 0 else {}) headers.update(self.headers) self.r = self.session.get(self.url, auth=self.auth, stream=True, headers=headers) if self.position != 0 and self.r.status_code != 206: raise UnsupportedOperation('could not reposition for this URL') if self.position == 0 and 'Content-Length' in self.r.headers: self.size = int(self.r.headers['Content-Length']) self.accept_ranges = 'bytes' in self.r.headers.get('Accept-Ranges', '') self.r.raw.decode_content = True self.stream_position = self.position
def __init__(self): # Defaults self.tty = False self.curses = None if not stderr: return try: if not stderr.buffer.isatty(): raise UnsupportedOperation() except (AttributeError, UnsupportedOperation): return self.tty = True try: import curses except ImportError: return self.write(str()) self.flush() termstr = os.getenv("TERM", "") fd = stderr.buffer.fileno() try: curses.setupterm(termstr, fd) except curses.error as err: warn(err) self.curses = curses
def get_variant(self, name): """Gets the values for a given variant. Args: name (str): The name of the variant. Returns: list: A list containing all the value for a given variant. The list has more than one item if there are duplicated variants. """ if self._mode != "r": raise UnsupportedOperation("not available in 'w' mode") # Fetching the variant self._bgen_index.execute( "SELECT file_start_position FROM Variant WHERE rsid = ?", (name, ) ) # Fetching all the seek positions seek_positions = [_[0] for _ in self._bgen_index.fetchall()] # Constructing the results results = [] for seek_pos in seek_positions: self._bgen.seek(seek_pos) results.append(self._read_current_variant()) if not results: raise ValueError("{}: name not found".format(name)) return results
def append_column(self, column_name: str, column_data: Column) -> DataStore._Builder[T]: if len(self._row_data) > 0: raise UnsupportedOperation( "Cannot insert column when row data present") self._column_data[str(column_name)] = column_data return self
def contents(self) -> 'Frame': ''' Return a :obj:`Frame` indicating name, dtype, shape, and bytes, of Archive components. ''' if self._writeable: raise UnsupportedOperation('Open with mode "r" to get contents.') from static_frame.core.frame import Frame def gen() -> tp.Iterator[tp.Tuple[tp.Any, ...]]: # metadata is in labels; sort by ext,ension first to put at top for name in sorted(self._archive.labels, key=lambda fn: tuple(reversed(fn.split('.')))): if name == self._archive.FILE_META: yield (name, self._archive.size_metadata()) + ('', '', '') else: header = self._archive.read_array_header(name) yield (name, self._archive.size_array(name)) + header f = Frame.from_records( gen(), columns=('name', 'size', 'dtype', 'fortran', 'shape'), name=str(self._archive._archive), ) return f.set_index('name', drop=True) #type: ignore
def seek(self, offset, whence=SEEK_SET): """ Change the stream position to the given byte offset. Args: offset: Offset is interpreted relative to the position indicated by whence. whence: The default value for whence is SEEK_SET. Values for whence are: SEEK_SET or 0 – start of the stream (the default); offset should be zero or positive SEEK_CUR or 1 – current stream position; offset may be negative SEEK_END or 2 – end of the stream; offset is usually negative Returns: int: The new absolute position. """ if not self._seekable: raise UnsupportedOperation('seek') # Only read mode is seekable with self._seek_lock: # Set seek using raw method and # sync buffered seek with raw seek self.raw.seek(offset, whence) self._seek = seek = self.raw._seek # Preload starting from current seek self._preload_range() return seek
def open_w(self, parts): for path in self.candidate_paths(parts): if path.writable(): return path.open_w() raise UnsupportedOperation("not writable: " + b'/'.join(parts).decode(errors='replace'))
def prepare_copy(self, src_url, dst_url): src_scheme, src_path = splitscheme(src_url) dst_scheme, dst_path = splitscheme(dst_url) if src_scheme == self.scheme and dst_scheme == 'file://': zip_path, path_in_zip = self._split(src_path) dst_ospath = as_human_readable(dst_url) return [Extract(self._fs, zip_path, path_in_zip, dst_ospath)] elif src_scheme == 'file://' and dst_scheme == self.scheme: zip_path, path_in_zip = self._split(dst_path) src_ospath = as_human_readable(src_url) return [ AddToArchive(self, self._fs, src_ospath, zip_path, path_in_zip) ] elif src_scheme == dst_scheme: # Guaranteed by fman's file system implementation: assert src_scheme == self.scheme src_zip_path, path_in_src_zip = self._split(src_path) dst_zip_path, path_in_dst_zip = self._split(dst_path) return [ CopyBetweenArchives(self, self._fs, src_zip_path, path_in_src_zip, dst_zip_path, path_in_dst_zip) ] else: raise UnsupportedOperation()
def open(self, mode="r"): """ Opens the file at this path; returns a file-like object. """ dmode = mode.replace("b", "") if dmode == "r": handle = self.fsobj.open_r(self.parts) elif dmode == "w": handle = self.fsobj.open_w(self.parts) elif dmode in ("r+", "rw"): handle = self.fsobj.open_rw(self.parts) elif dmode == "a": handle = self.fsobj.open_a(self.parts) elif dmode in ("a+", "ar"): handle = self.fsobj.open_ar(self.parts) else: raise UnsupportedOperation("unsupported open mode: " + mode) if "b" in mode: return handle return TextIOWrapper(handle)
def open_w(self, parts): _, open_w, _, _ = self.get_fileentry(parts) if open_w is None: raise UnsupportedOperation("not writable: " + b"/".join(parts).decode( errors='replace'))
def iter_geno_marker(self, markers, return_index=False): """Iterates over genotypes for a list of markers. Args: markers (list): The list of markers to iterate onto. return_index (bool): Wether to return the marker's index or not. Returns: tuple: The name of the marker as a string, and its genotypes as a :py:class:`numpy.ndarray` (additive format). """ if self._mode != "r": raise UnsupportedOperation("not available in 'w' mode") # If string, we change to list if isinstance(markers, str): markers = [markers] # Iterating over all markers if return_index: for marker in markers: geno, seek = self.get_geno_marker(marker, return_index=True) yield marker, geno, seek else: for marker in markers: yield marker, self.get_geno_marker(marker)
def get_geno_marker(self, marker, return_index=False): """Gets the genotypes for a given marker. Args: marker (str): The name of the marker. return_index (bool): Wether to return the marker's index or not. Returns: numpy.ndarray: The genotypes of the marker (additive format). """ if self._mode != "r": raise UnsupportedOperation("not available in 'w' mode") # Check if the marker exists if marker not in self._bim.index: raise ValueError("{}: marker not in BIM".format(marker)) # Seeking to the correct position seek_index = self._bim.loc[marker, "i"] self.seek(seek_index) if return_index: return self._read_current_marker(), seek_index return self._read_current_marker()
def seek(self, offset, whence=os.SEEK_SET): """ Unsupported because this is a FIFO. """ del offset, whence # unused raise UnsupportedOperation("unseekable stream")
def test_3(): """ Unauthorized objects are not taken into account """ global fun_3_obj assert dds.eval(fun_3_f1) is None fun_3_obj = UnsupportedOperation("b") assert dds.eval(fun_3_f1) is None assert fun_3_counter.value == 1, fun_3_counter.value