def seek_helper(self, offset, whence):
"""
Helper function for use by implementations of seek().
Calculates the new cursor position from offset, whence,
and self.tell().
If size is given, it works for whence=os.SEEK_END;
otherwise, UnsupportedOperation is raised.
"""
if whence == os.SEEK_SET:
target = offset
elif whence == os.SEEK_CUR:
target = offset + self.tell()
elif whence == os.SEEK_END:
size = self.get_size()
if size < 0:
raise UnsupportedOperation(
"can only seek relative to file start or cursor")
target = offset + size
else:
raise UnsupportedOperation("unsupported seek mode")
if target < 0:
raise ValueError("can not seek to a negative file position")
return target
def __getattribute__(self, name):
attr = super().__getattribute__(name)
if attr:
if 'read' in name and 'r' not in self._mode:
raise UnsupportedOperation('not readable')
elif 'write' in name and 'w' not in self._mode and 'a' not in self._mode:
raise UnsupportedOperation('not writable')
return attr
def drop_group(self: DatabaseRegistrar, data_group: str) -> None:
with self._db_adapter:
if not self.has_group(data_group):
raise MissingGroupError(data_group)
if data_group == PROTECTED_GROUP:
raise UnsupportedOperation(
"Cannot delete protected data group")
if self._group_contains_protected_tables(data_group):
raise UnsupportedOperation(
"Cannot delete group that contains protected table")
for token in self.list_group_tables(data_group):
self.drop_table(token)
def write_genotypes(self, genotypes):
"""Write genotypes to binary file.
Args:
genotypes (numpy.ndarray): The genotypes to write in the BED file.
"""
if self._mode != "w":
raise UnsupportedOperation("not available in 'r' mode")
# Initializing the number of samples if required
if self._nb_values is None:
self._nb_values = len(genotypes)
# Checking the expected number of samples
if self._nb_values != len(genotypes):
raise ValueError("{:,d} samples expected, got {:,d}".format(
self._nb_values,
len(genotypes),
))
# Writing to file
byte_array = [
g[0] | (g[1] << 2) | (g[2] << 4) | (g[3] << 6)
for g in self._grouper((_byte_recode[geno]
for geno in genotypes), 4)
]
self._bed.write(bytearray(byte_array))
def seek(self, offset, whence=SEEK_SET):
if self.closed:
raise ValueError('I/O operation on closed file.')
if whence not in seek_whence.keys():
raise ValueError(f'whence {whence} not supported')
if not isinstance(offset, int):
raise TypeError(f'offset must be int, not {type(offset).__name__}')
pos = self.position
if whence == SEEK_SET:
pos = offset
elif whence == SEEK_CUR:
pos += offset
elif whence == SEEK_END:
if self.size == None:
raise UnsupportedOperation(
'end-relative seek not supported for this URL')
pos = self.size + offset
if pos < 0:
raise ValueError(f'resulting offset {pos} < 0')
self._log(
f'seek from {self.position} + ({offset}, {seek_whence[whence]}) -> {pos}'
)
self.position = pos
return self.position
def seek(self, pos: int, whence: int = SEEK_SET) -> int:
"""Change stream position.
Change the stream position to byte offset pos. Argument pos is
interpreted relative to the position indicated by whence. Values
for whence are ints:
* 0 -- start of stream (the default); offset should be zero or positive
* 1 -- current stream position; offset may be negative
* 2 -- end of stream; offset should be negative
Return an int indicating the new absolute position.
"""
self._check_not_closed()
if not self.seekable(): # just in case seekable is overridden
raise UnsupportedOperation("seek")
if whence == SEEK_SET:
pass
elif whence == SEEK_CUR:
pos += self._pos
elif whence == SEEK_END:
pos += self._length
else:
raise ValueError("unsupported whence value")
if pos >= 0:
self._pos = pos
return self._pos
raise ValueError("invalid seek position")
def write(self, data: bytes) -> int:
"""Write data, passed as a bytes object.
Returns the number of bytes written, which is always the length
of the input data in bytes.
"""
self._check_not_closed()
if not self.writable():
raise UnsupportedOperation("write")
written_bytes = len(data)
padding_size = self._pos - self._length
if padding_size < 0:
raise ValueError("write is only supported from EOF")
if padding_size > 0:
null_bytes = memoryview(bytearray(DEFAULT_BUFFER_SIZE))
self._pos = self._length
data = memoryview(data)
while padding_size or data:
self._write_start()
if padding_size > 0:
# pad with null bytes, not counted in written_bytes
padding = null_bytes[:padding_size]
written_len = self._write(
padding) # do not stop if nothing was written
padding_size -= written_len
else:
written_len = self._write(
data) # do not stop if nothing was written
data = data[written_len:]
self._pos += written_len
self._length = max(self._length, self._pos)
return written_bytes
def readinto(self, b):
"""
Read bytes into a pre-allocated, writable bytes-like object b, and return the
number of bytes read.
Args:
b (bytes-like object): buffer.
Returns:
int: number of bytes read
"""
if not self._readable:
raise UnsupportedOperation("read")
size = len(b)
with self._seek_lock:
start = self._seek
end = start + size
self._seek = end
with handle_os_exceptions():
read_data = self._read_range(start, end)
read_size = len(read_data)
if read_size:
memoryview(b)[:read_size] = read_data
if read_size != size:
with self._seek_lock:
self._seek = start + read_size
return read_size
def drop_table(self: DatabaseRegistrar, data_token: DataToken) -> None:
with self._db_adapter:
if not self.has_table(data_token):
raise MissingTableError(data_token)
if self._is_table_protected(data_token):
raise UnsupportedOperation(
"Cannot delete from protected data group")
self._db_adapter.drop_group_table(data_token)
store_type, store_version = self._table_store_type_version(
data_token)
store_type_version_count = sum(
(TableReference.store_type == store_type)
& (TableReference.store_version == store_version))
version_count = self._db_adapter.query(TableReference.data_token,
store_type_version_count)
if version_count == 1:
self._drop_store_type(store_type, store_version)
criteria = (TableReference.table_name == data_token.table_name) & (
TableReference.data_group == data_token.data_group)
self._db_adapter.delete(TableReference.data_token, criteria)
if len(self.list_group_tables(data_token.data_group)) == 0:
self._db_adapter.drop_group(data_token.data_group)
store_indices = (IndexReference.store_type == store_type) & (
TableReference.store_version == store_version)
self._db_adapter.delete(IndexReference.data_token, store_indices)
def seek(self, offset, whence=SEEK_SET):
"""
Change the stream position to the given byte offset.
Args:
offset: Offset is interpreted relative to the position indicated by
whence.
whence: The default value for whence is SEEK_SET. Values are:
SEEK_SET or 0 – start of the stream (the default);
offset should be zero or positive
SEEK_CUR or 1 – current stream position; offset may be negative
SEEK_END or 2 – end of the stream; offset is usually negative
Returns:
int: The new absolute position.
"""
if not self._seekable:
raise UnsupportedOperation("seek")
with self._seek_lock:
self.raw.seek(offset, whence)
self._seek = seek = self.raw._seek
self._preload_range()
return seek
def seek(self, offset, whence=SEEK_SET):
"""
Change the stream position to the given byte offset.
Args:
offset (int): Offset is interpreted relative to the position
indicated by whence.
whence (int): The default value for whence is SEEK_SET.
Values for whence are:
SEEK_SET or 0 – start of the stream (the default);
offset should be zero or positive
SEEK_CUR or 1 – current stream position;
offset may be negative
SEEK_END or 2 – end of the stream;
offset is usually negative
Returns:
int: The new absolute position.
"""
if not self._seekable:
raise UnsupportedOperation('seek')
# Flush before moving position
self.flush()
return self._update_seek(offset, whence)
def write(self, *args, **kwargs):
"""
This streams doesn't support writing.
:raises UnsupportedOperation:
"""
raise UnsupportedOperation("This stream doesn't support write")
def get_unk_toks_indices(sentence, tokenizer, old_style_tokenizer):
tokenizer_class = repr(old_style_tokenizer.__class__)
if "models.bert" in tokenizer_class:
recovery_method = recover_bert_unk_alignment
elif "models.roberta" in tokenizer_class:
recovery_method = recover_roberta_unk_alignment
else:
raise UnsupportedOperation(
f"Unknown tokenizer class {tokenizer_class}")
# when we get to printing, the only guarantee we have is that the outputted
# sentence has the same number of wordpieces as the inputted sentence, so
# we need to use the wordpiece-based indices
chunks = tokenizer.tokenize(sentence)
unk_chunk_indices = [
i for i, chunk in enumerate(chunks) if chunk == tokenizer.unk_token
]
unknown_strings = recovery_method(old_style_tokenizer, sentence)
# make sure no funky tokenization going on here
if len(unk_chunk_indices) != len(unknown_strings):
import pdb
pdb.set_trace()
mapping = {
idx: string
for idx, string in zip(unk_chunk_indices, unknown_strings)
}
return mapping
def write(self, b):
"""
Write the given bytes-like object, b, to the underlying raw stream,
and return the number of bytes written.
Args:
b (bytes-like object): Bytes to write.
Returns:
int: The number of bytes written.
"""
if not self._writable:
raise UnsupportedOperation('write')
# This function write data in a buffer
# "flush()" need to be called to really write content on
# Cloud Storage
size = len(b)
with self._seek_lock:
start = self._seek
end = start + size
self._seek = end
buffer = self._write_buffer
if end <= len(buffer):
buffer = memoryview(buffer)
buffer[start:end] = b
return size
def seek(self, offset, whence=SEEK_SET):
"""
Change the stream position to the given byte offset.
Args:
offset (int): Offset is interpreted relative to the position
indicated by whence.
whence (int): The default value for whence is SEEK_SET.
Values for whence are:
SEEK_SET or 0 – start of the stream (the default);
offset should be zero or positive
SEEK_CUR or 1 – current stream position;
offset may be negative
SEEK_END or 2 – end of the stream;
offset is usually negative
Returns:
int: The new absolute position.
"""
if not self._seekable:
raise UnsupportedOperation('seek')
seek = self._update_seek(offset, whence)
# If seek move out of file, add padding until new seek position.
if self._writable:
size = len(self._write_buffer)
if seek > size:
self._write_buffer[seek:size] = b'\0' * (seek - size)
return seek
def write(self, b):
"""
Write the given bytes-like object, b, to the underlying raw stream, and return
the number of bytes written.
Args:
b (bytes-like object): Bytes to write.
Returns:
int: The number of bytes written.
"""
if not self._writable:
raise UnsupportedOperation("write")
size = len(b)
with self._seek_lock:
start = self._seek
end = start + size
self._seek = end
buffer = self._write_buffer
if end <= len(buffer):
buffer = memoryview(buffer)
buffer[start:end] = b
return size
def _position(self):
self._log(
'_position', 'position=%d, stream_position=%d' %
(self.position, self.stream_position))
if self.r:
self.r.close()
headers = {'Accept-Encoding': 'identity'}
headers.update({'Range': 'bytes=%d-' %
self.position} if self.position != 0 else {})
headers.update(self.headers)
self.r = self.session.get(self.url,
auth=self.auth,
stream=True,
headers=headers)
if self.position != 0 and self.r.status_code != 206:
raise UnsupportedOperation('could not reposition for this URL')
if self.position == 0 and 'Content-Length' in self.r.headers:
self.size = int(self.r.headers['Content-Length'])
self.accept_ranges = 'bytes' in self.r.headers.get('Accept-Ranges', '')
self.r.raw.decode_content = True
self.stream_position = self.position
def __init__(self):
# Defaults
self.tty = False
self.curses = None
if not stderr:
return
try:
if not stderr.buffer.isatty():
raise UnsupportedOperation()
except (AttributeError, UnsupportedOperation):
return
self.tty = True
try:
import curses
except ImportError:
return
self.write(str())
self.flush()
termstr = os.getenv("TERM", "")
fd = stderr.buffer.fileno()
try:
curses.setupterm(termstr, fd)
except curses.error as err:
warn(err)
self.curses = curses
def get_variant(self, name):
"""Gets the values for a given variant.
Args:
name (str): The name of the variant.
Returns:
list: A list containing all the value for a given variant. The list
has more than one item if there are duplicated variants.
"""
if self._mode != "r":
raise UnsupportedOperation("not available in 'w' mode")
# Fetching the variant
self._bgen_index.execute(
"SELECT file_start_position FROM Variant WHERE rsid = ?",
(name, )
)
# Fetching all the seek positions
seek_positions = [_[0] for _ in self._bgen_index.fetchall()]
# Constructing the results
results = []
for seek_pos in seek_positions:
self._bgen.seek(seek_pos)
results.append(self._read_current_variant())
if not results:
raise ValueError("{}: name not found".format(name))
return results
def append_column(self, column_name: str,
column_data: Column) -> DataStore._Builder[T]:
if len(self._row_data) > 0:
raise UnsupportedOperation(
"Cannot insert column when row data present")
self._column_data[str(column_name)] = column_data
return self
def contents(self) -> 'Frame':
'''
Return a :obj:`Frame` indicating name, dtype, shape, and bytes, of Archive components.
'''
if self._writeable:
raise UnsupportedOperation('Open with mode "r" to get contents.')
from static_frame.core.frame import Frame
def gen() -> tp.Iterator[tp.Tuple[tp.Any, ...]]:
# metadata is in labels; sort by ext,ension first to put at top
for name in sorted(self._archive.labels,
key=lambda fn: tuple(reversed(fn.split('.')))):
if name == self._archive.FILE_META:
yield (name, self._archive.size_metadata()) + ('', '', '')
else:
header = self._archive.read_array_header(name)
yield (name, self._archive.size_array(name)) + header
f = Frame.from_records(
gen(),
columns=('name', 'size', 'dtype', 'fortran', 'shape'),
name=str(self._archive._archive),
)
return f.set_index('name', drop=True) #type: ignore
def seek(self, offset, whence=SEEK_SET):
"""
Change the stream position to the given byte offset.
Args:
offset: Offset is interpreted relative to the position indicated by
whence.
whence: The default value for whence is SEEK_SET.
Values for whence are:
SEEK_SET or 0 – start of the stream (the default);
offset should be zero or positive
SEEK_CUR or 1 – current stream position;
offset may be negative
SEEK_END or 2 – end of the stream;
offset is usually negative
Returns:
int: The new absolute position.
"""
if not self._seekable:
raise UnsupportedOperation('seek')
# Only read mode is seekable
with self._seek_lock:
# Set seek using raw method and
# sync buffered seek with raw seek
self.raw.seek(offset, whence)
self._seek = seek = self.raw._seek
# Preload starting from current seek
self._preload_range()
return seek
def open_w(self, parts):
for path in self.candidate_paths(parts):
if path.writable():
return path.open_w()
raise UnsupportedOperation("not writable: " +
b'/'.join(parts).decode(errors='replace'))
def prepare_copy(self, src_url, dst_url):
src_scheme, src_path = splitscheme(src_url)
dst_scheme, dst_path = splitscheme(dst_url)
if src_scheme == self.scheme and dst_scheme == 'file://':
zip_path, path_in_zip = self._split(src_path)
dst_ospath = as_human_readable(dst_url)
return [Extract(self._fs, zip_path, path_in_zip, dst_ospath)]
elif src_scheme == 'file://' and dst_scheme == self.scheme:
zip_path, path_in_zip = self._split(dst_path)
src_ospath = as_human_readable(src_url)
return [
AddToArchive(self, self._fs, src_ospath, zip_path, path_in_zip)
]
elif src_scheme == dst_scheme:
# Guaranteed by fman's file system implementation:
assert src_scheme == self.scheme
src_zip_path, path_in_src_zip = self._split(src_path)
dst_zip_path, path_in_dst_zip = self._split(dst_path)
return [
CopyBetweenArchives(self, self._fs, src_zip_path,
path_in_src_zip, dst_zip_path,
path_in_dst_zip)
]
else:
raise UnsupportedOperation()
def open(self, mode="r"):
""" Opens the file at this path; returns a file-like object. """
dmode = mode.replace("b", "")
if dmode == "r":
handle = self.fsobj.open_r(self.parts)
elif dmode == "w":
handle = self.fsobj.open_w(self.parts)
elif dmode in ("r+", "rw"):
handle = self.fsobj.open_rw(self.parts)
elif dmode == "a":
handle = self.fsobj.open_a(self.parts)
elif dmode in ("a+", "ar"):
handle = self.fsobj.open_ar(self.parts)
else:
raise UnsupportedOperation("unsupported open mode: " + mode)
if "b" in mode:
return handle
return TextIOWrapper(handle)
def open_w(self, parts):
_, open_w, _, _ = self.get_fileentry(parts)
if open_w is None:
raise UnsupportedOperation("not writable: " +
b"/".join(parts).decode(
errors='replace'))
def iter_geno_marker(self, markers, return_index=False):
"""Iterates over genotypes for a list of markers.
Args:
markers (list): The list of markers to iterate onto.
return_index (bool): Wether to return the marker's index or not.
Returns:
tuple: The name of the marker as a string, and its genotypes as a
:py:class:`numpy.ndarray` (additive format).
"""
if self._mode != "r":
raise UnsupportedOperation("not available in 'w' mode")
# If string, we change to list
if isinstance(markers, str):
markers = [markers]
# Iterating over all markers
if return_index:
for marker in markers:
geno, seek = self.get_geno_marker(marker, return_index=True)
yield marker, geno, seek
else:
for marker in markers:
yield marker, self.get_geno_marker(marker)
def get_geno_marker(self, marker, return_index=False):
"""Gets the genotypes for a given marker.
Args:
marker (str): The name of the marker.
return_index (bool): Wether to return the marker's index or not.
Returns:
numpy.ndarray: The genotypes of the marker (additive format).
"""
if self._mode != "r":
raise UnsupportedOperation("not available in 'w' mode")
# Check if the marker exists
if marker not in self._bim.index:
raise ValueError("{}: marker not in BIM".format(marker))
# Seeking to the correct position
seek_index = self._bim.loc[marker, "i"]
self.seek(seek_index)
if return_index:
return self._read_current_marker(), seek_index
return self._read_current_marker()
def seek(self, offset, whence=os.SEEK_SET):
"""
Unsupported because this is a FIFO.
"""
del offset, whence # unused
raise UnsupportedOperation("unseekable stream")
def test_3():
""" Unauthorized objects are not taken into account """
global fun_3_obj
assert dds.eval(fun_3_f1) is None
fun_3_obj = UnsupportedOperation("b")
assert dds.eval(fun_3_f1) is None
assert fun_3_counter.value == 1, fun_3_counter.value
