def load_bytes(uri: str,
start: int,
end: int,
write_to_stdout=False,
p2p: bool = True,
from_node: Union[str, None] = None,
from_channel: Union[str, None] = None) -> Union[bytes, None]:
ret = ka.load_bytes(uri,
start=start,
end=end,
write_to_stdout=write_to_stdout)
if ret is not None:
return ret
if not p2p:
return
protocol, algorithm, hash0, additional_path, query = _parse_kachery_uri(
uri)
if query.get('manifest'):
manifest = load_object(f'sha1://{query["manifest"][0]}')
if manifest is None:
print('Unable to load manifest')
return None
assert manifest[
'sha1'] == hash0, 'Manifest sha1 does not match expected.'
data_chunks = []
chunks_to_load = []
for ch in manifest['chunks']:
if start < ch['end'] and end > ch['start']:
chunks_to_load.append(ch)
for ii, ch in enumerate(chunks_to_load):
if len(chunks_to_load) > 1:
print(
f'load_bytes: Loading chunk {ii + 1} of {len(chunks_to_load)}'
)
a = load_bytes(
uri=
f'sha1://{ch["sha1"]}?chunkOf={hash0}~{ch["start"]}~{ch["end"]}',
start=max(0, start - ch['start']),
end=min(ch['end'] - ch['start'], end - ch['start']))
if a is None:
print('Unable to load bytes from chunk')
return None
data_chunks.append(a)
return b''.join(data_chunks)
path = load_file(uri=uri, from_node=from_node, from_channel=from_channel)
if path is None:
print('Unable to load file.')
return None
return ka.load_bytes(uri,
start=start,
end=end,
write_to_stdout=write_to_stdout)
def _read_header(path, verbose=True):
info0 = ka.get_file_info(path)
if info0 is None:
raise Exception(f'Unable to find file: {path}')
bytes0 = ka.load_bytes(path, start=0, end=min(200, info0['size']))
if bytes0 is None:
ka.set_config(fr='default_readonly')
print(ka.get_file_info(path))
raise Exception('Unable to load header bytes from {}'.format(path))
f = io.BytesIO(bytes0)
try:
dt_code = _read_int32(f)
_ = _read_int32(f) # num bytes per entry
num_dims = _read_int32(f)
uses64bitdims = False
if (num_dims < 0):
uses64bitdims = True
num_dims = -num_dims
if (num_dims < 1) or (num_dims >
6): # allow single dimension as of 12/6/17
if verbose:
print("Invalid number of dimensions: {}".format(num_dims))
f.close()
return None
dims = []
dimprod = 1
if uses64bitdims:
for _ in range(0, num_dims):
tmp0 = _read_int64(f)
dimprod = dimprod * tmp0
dims.append(tmp0)
else:
for _ in range(0, num_dims):
tmp0 = _read_int32(f)
dimprod = dimprod * tmp0
dims.append(tmp0)
dt = _dt_from_dt_code(dt_code)
if dt is None:
if verbose:
print("Invalid data type code: {}".format(dt_code))
f.close()
return None
H = MdaHeader(dt, dims)
if (uses64bitdims):
H.uses64bitdims = True
H.header_size = 3 * 4 + H.num_dims * 8
f.close()
return H
except Exception as e: # catch *all* exceptions
if verbose:
print(e)
f.close()
return None
def _read_chunk_1d(self, i, N):
start_byte = self._header.header_size + self._header.num_bytes_per_entry * i
end_byte = start_byte + self._header.num_bytes_per_entry * N
try:
bytes0 = ka.load_bytes(self._path,
start=int(start_byte),
end=int(end_byte))
except:
info0 = ka.get_file_info(self._path)
if info0 is None:
print(
f'Problem reading bytes {start_byte}-{end_byte} from file {self._path} (no info)'
)
else:
print(
f'Problem reading bytes {start_byte}-{end_byte} from file {self._path} of size {info0["size"]}'
)
raise
return np.frombuffer(bytes0, dtype=self._header.dt, count=N)
def _read_chunk_1d(self, i, N):
start_byte = self._header.header_size + self._header.num_bytes_per_entry * i
end_byte = start_byte + self._header.num_bytes_per_entry * N
bytes0 = ka.load_bytes(self._path, start=int(start_byte), end=int(end_byte))
return np.frombuffer(bytes0, dtype=self._header.dt, count=N)