def consolidate_metadata(self, metadata_key='.zmetadata'): ''' Wrapper over zarr.consolidate_metadata to pass chunk store when opening the zarr store ''' # same as zarr.consolidate_metadata(self.store, metadata_key) call, # only with key.endswith('.zchunkstore') in is_zarr_key, and passing chunk store def is_zarr_key(key): return (key.endswith('.zchunkstore') or key.endswith('.zarray') or key.endswith('.zgroup') or key.endswith('.zattrs')) out = { 'zarr_consolidated_format': 1, 'metadata': { key: json_loads(self.store[key]) for key in self.store if is_zarr_key(key) } } self.store[metadata_key] = json_dumps(out) meta_store = ConsolidatedMetadataStore(self.store, metadata_key=metadata_key) store_mode_cons = 'r' if self.store_mode == 'r' else 'r+' self.zgroup = zarr.open(store=meta_store, mode=store_mode_cons, chunk_store=self.zgroup.chunk_store, path=self.store_path) return self.zgroup
def parse_metadata(cls, s: Union[MappingType, str]) -> MappingType[str, Any]: # Here we allow that a store may return an already-parsed metadata object, # or a string of JSON that we will parse here. We allow for an already-parsed # object to accommodate a consolidated metadata store, where all the metadata for # all groups and arrays will already have been parsed from JSON. if isinstance(s, Mapping): # assume metadata has already been parsed into a mapping object meta = s else: # assume metadata needs to be parsed as JSON meta = json_loads(s) return meta
def __init__(self, store: StoreLike, metadata_key=meta_root + "consolidated/.zmetadata"): self.store = StoreV3._ensure_store(store) # retrieve consolidated metadata meta = json_loads(self.store[metadata_key]) # check format of consolidated metadata consolidated_format = meta.get('zarr_consolidated_format', None) if consolidated_format != 1: raise MetadataError( 'unsupported zarr consolidated metadata format: %s' % consolidated_format) # decode metadata self.meta_store: Store = KVStoreV3(meta["metadata"])
def generate_fixtures(): # Empty fixture create_simple_array(os.path.join(FIXTURES_FOLDER, "empty.zarr"), dtype="<i4") # little endian for codec in (None, "gzip", "zlib"): path = os.path.join( FIXTURES_FOLDER, "simple{}_LE.zarr".format(f"_{codec}" if codec else "") ) create_simple_array( store=path, dtype="<i4", compression=codec, write_chunks=True, ) # big endian for codec in (None, "gzip", "zlib"): path = os.path.join( FIXTURES_FOLDER, "simple{}_BE.zarr".format(f"_{codec}" if codec else "") ) create_simple_array( store=path, dtype=">i4", compression=codec, write_chunks=True, ) # nested # TODO: Use latest zarr-python once https://github.com/zarr-developers/zarr-python/pull/716 is merged store = zarr.storage.FSStore( os.path.join(FIXTURES_FOLDER, "simple_nested.zarr"), key_separator="/", auto_mkdir=True, ) create_simple_array( store=store, dtype=">i4", compression="blosc", write_chunks=True ) # Manually add dimension separator to array meta meta = json_loads(store[".zarray"]) meta["dimension_separator"] = "/" store[".zarray"] = json_dumps(meta)
def _ensure_dict(self, obj): if isinstance(obj, bytes): return json_loads(obj) else: return obj