def load_bytes(uri: str, start: int, end: int, write_to_stdout=False, p2p: bool = True, from_node: Union[str, None] = None, from_channel: Union[str, None] = None) -> Union[bytes, None]: ret = ka.load_bytes(uri, start=start, end=end, write_to_stdout=write_to_stdout) if ret is not None: return ret if not p2p: return protocol, algorithm, hash0, additional_path, query = _parse_kachery_uri( uri) if query.get('manifest'): manifest = load_object(f'sha1://{query["manifest"][0]}') if manifest is None: print('Unable to load manifest') return None assert manifest[ 'sha1'] == hash0, 'Manifest sha1 does not match expected.' data_chunks = [] chunks_to_load = [] for ch in manifest['chunks']: if start < ch['end'] and end > ch['start']: chunks_to_load.append(ch) for ii, ch in enumerate(chunks_to_load): if len(chunks_to_load) > 1: print( f'load_bytes: Loading chunk {ii + 1} of {len(chunks_to_load)}' ) a = load_bytes( uri= f'sha1://{ch["sha1"]}?chunkOf={hash0}~{ch["start"]}~{ch["end"]}', start=max(0, start - ch['start']), end=min(ch['end'] - ch['start'], end - ch['start'])) if a is None: print('Unable to load bytes from chunk') return None data_chunks.append(a) return b''.join(data_chunks) path = load_file(uri=uri, from_node=from_node, from_channel=from_channel) if path is None: print('Unable to load file.') return None return ka.load_bytes(uri, start=start, end=end, write_to_stdout=write_to_stdout)
def _read_header(path, verbose=True): info0 = ka.get_file_info(path) if info0 is None: raise Exception(f'Unable to find file: {path}') bytes0 = ka.load_bytes(path, start=0, end=min(200, info0['size'])) if bytes0 is None: ka.set_config(fr='default_readonly') print(ka.get_file_info(path)) raise Exception('Unable to load header bytes from {}'.format(path)) f = io.BytesIO(bytes0) try: dt_code = _read_int32(f) _ = _read_int32(f) # num bytes per entry num_dims = _read_int32(f) uses64bitdims = False if (num_dims < 0): uses64bitdims = True num_dims = -num_dims if (num_dims < 1) or (num_dims > 6): # allow single dimension as of 12/6/17 if verbose: print("Invalid number of dimensions: {}".format(num_dims)) f.close() return None dims = [] dimprod = 1 if uses64bitdims: for _ in range(0, num_dims): tmp0 = _read_int64(f) dimprod = dimprod * tmp0 dims.append(tmp0) else: for _ in range(0, num_dims): tmp0 = _read_int32(f) dimprod = dimprod * tmp0 dims.append(tmp0) dt = _dt_from_dt_code(dt_code) if dt is None: if verbose: print("Invalid data type code: {}".format(dt_code)) f.close() return None H = MdaHeader(dt, dims) if (uses64bitdims): H.uses64bitdims = True H.header_size = 3 * 4 + H.num_dims * 8 f.close() return H except Exception as e: # catch *all* exceptions if verbose: print(e) f.close() return None
def _read_chunk_1d(self, i, N): start_byte = self._header.header_size + self._header.num_bytes_per_entry * i end_byte = start_byte + self._header.num_bytes_per_entry * N try: bytes0 = ka.load_bytes(self._path, start=int(start_byte), end=int(end_byte)) except: info0 = ka.get_file_info(self._path) if info0 is None: print( f'Problem reading bytes {start_byte}-{end_byte} from file {self._path} (no info)' ) else: print( f'Problem reading bytes {start_byte}-{end_byte} from file {self._path} of size {info0["size"]}' ) raise return np.frombuffer(bytes0, dtype=self._header.dt, count=N)
def _read_chunk_1d(self, i, N): start_byte = self._header.header_size + self._header.num_bytes_per_entry * i end_byte = start_byte + self._header.num_bytes_per_entry * N bytes0 = ka.load_bytes(self._path, start=int(start_byte), end=int(end_byte)) return np.frombuffer(bytes0, dtype=self._header.dt, count=N)