def test_create(self): """ Create new soft link by assignment """ g = self.f.create_group('new') sl = SoftLink('/new') self.f['alias'] = sl g2 = self.f['alias'] self.assertEqual(g, g2)
def _set_link(self, path, name, obj): """ Set link name is the symlink path, obj is the obj we want to link to """ # pylint: disable=protected-access if isinstance(obj, BaseGroupWrapper): self._file[str(HDF5Path(path, name))] = SoftLink(str(obj._path)) else: raise TypeError("Not a valid definition of {}".format(name))
def test_copy_soft_links(self): self.f1['bar'] = [1, 2, 3] foo = self.f1.create_group('foo') foo['baz'] = SoftLink('/bar') self.f1.copy(foo, 'qux', expand_soft=True) self.f2.copy(foo, 'foo', expand_soft=True) del self.f1['bar'] self.assertIsInstance(self.f1['qux'], Group) self.assertArrayEqual(self.f1['qux/baz'], np.array([1, 2, 3])) self.assertIsInstance(self.f2['/foo'], Group) self.assertArrayEqual(self.f2['foo/baz'], np.array([1, 2, 3]))
def write_link(self, **kwargs): parent, builder = getargs('parent', 'builder', kwargs) name = builder.name target_builder = builder.builder path = self.__get_path(target_builder) # source will indicate target_builder's location if parent.file.filename == target_builder.source: link_obj = SoftLink(path) elif target_builder.source is not None: link_obj = ExternalLink(target_builder.source, path) else: msg = 'cannot create external link to %s' % path raise ValueError(msg) parent[name] = link_obj return link_obj
def test_get_link_class(self): """ Get link classes """ default = object() sl = SoftLink('/mongoose') el = ExternalLink('somewhere.hdf5', 'mongoose') self.f.create_group('hard') self.f['soft'] = sl self.f['external'] = el out_hl = self.f.get('hard', default, getlink=True, getclass=True) out_sl = self.f.get('soft', default, getlink=True, getclass=True) out_el = self.f.get('external', default, getlink=True, getclass=True) self.assertEqual(out_hl, HardLink) self.assertEqual(out_sl, SoftLink) self.assertEqual(out_el, ExternalLink)
def test_get_link(self): """ Get link values """ sl = SoftLink('/mongoose') el = ExternalLink('somewhere.hdf5', 'mongoose') self.f.create_group('hard') self.f['soft'] = sl self.f['external'] = el out_hl = self.f.get('hard', getlink=True) out_sl = self.f.get('soft', getlink=True) out_el = self.f.get('external', getlink=True) #TODO: redo with SoftLink/ExternalLink built-in equality self.assertIsInstance(out_hl, HardLink) self.assertIsInstance(out_sl, SoftLink) self.assertEqual(out_sl._path, sl._path) self.assertIsInstance(out_el, ExternalLink) self.assertEqual(out_el._path, el._path) self.assertEqual(out_el._filename, el._filename)
def write_link(self, **kwargs): parent, builder = getargs('parent', 'builder', kwargs) if builder.written: return None name = builder.name target_builder = builder.builder path = self.__get_path(target_builder) # source will indicate target_builder's location if parent.file.filename == target_builder.source: link_obj = SoftLink(path) elif target_builder.source is not None: target_filename = os.path.abspath(target_builder.source) parent_filename = os.path.abspath(parent.file.filename) relative_path = os.path.relpath(target_filename, os.path.dirname(parent_filename)) link_obj = ExternalLink(relative_path, path) else: msg = 'cannot create external link to %s' % path raise ValueError(msg) parent[name] = link_obj builder.written = True return link_obj
def test_exc(self): """ Opening dangling soft link results in KeyError """ self.f['alias'] = SoftLink('new') with self.assertRaises(KeyError): self.f['alias']
def test_srepr(self): """ SoftLink path repr """ sl = SoftLink('/foo') self.assertIsInstance(repr(sl), str)
def test_spath(self): """ SoftLink path attribute """ sl = SoftLink('/foo') self.assertEqual(sl.path, '/foo')
def write_dataset(self, **kwargs): """ Write a dataset to HDF5 The function uses other dataset-dependent write functions, e.g, __scalar_fill__, __list_fill__ and __chunked_iter_fill__ to write the data. """ parent, builder = getargs('parent', 'builder', kwargs) name = builder.name data = builder.data attributes = builder.attributes dtype = builder.dtype dset = None link = None if isinstance(data, str): dset = self.__scalar_fill__(parent, name, data) elif isinstance(data, DataChunkIterator): dset = self.__chunked_iter_fill__(parent, name, data) elif isinstance(data, Dataset): data_filename = os.path.abspath(data.file.filename) parent_filename = os.path.abspath(parent.file.filename) if data_filename != parent_filename: link = ExternalLink( os.path.relpath(data_filename, os.path.dirname(parent_filename)), data.name) else: link = SoftLink(data.name) parent[name] = link elif isinstance(data, Builder): _dtype = self.__dtypes[dtype] if dtype == 'region': def _filler(): ref = self.__get_ref(data, builder.region) dset = parent.create_dataset(name, data=ref, shape=None, dtype=_dtype) self.set_attributes(dset, attributes) self.__queue_ref(_filler) else: def _filler(): ref = self.__get_ref(data) dset = parent.create_dataset(name, data=ref, shape=None, dtype=_dtype) self.set_attributes(dset, attributes) self.__queue_ref(_filler) return elif isinstance(data, Iterable) and not self.isinstance_inmemory_array(data): dset = self.__chunked_iter_fill__( parent, name, DataChunkIterator(data=data, buffer_size=100)) elif hasattr(data, '__len__'): dset = self.__list_fill__(parent, name, data, dtype_spec=dtype) else: dset = self.__scalar_fill__(parent, name, data, dtype=dtype) if link is None: self.set_attributes(dset, attributes) return dset
def write_dataset(self, **kwargs): """ Write a dataset to HDF5 The function uses other dataset-dependent write functions, e.g, __scalar_fill__, __list_fill__ and __chunked_iter_fill__ to write the data. """ parent, builder = getargs('parent', 'builder', kwargs) name = builder.name data = builder.data options = dict() if isinstance(data, H5DataIO): options['compression'] = 'gzip' if data.compress else None data = data.data attributes = builder.attributes options['dtype'] = builder.dtype dset = None link = None if isinstance(options['dtype'], list): # do some stuff to figure out what data is a reference refs = list() for i, dts in enumerate(options['dtype']): if self.__is_ref(dts): refs.append(i) if len(refs) > 0: _dtype = self.__resolve_dtype__(options['dtype'], data) dset = parent.require_dataset(name, shape=(len(data),), dtype=_dtype) @self.__queue_ref def _filler(): ret = list() for item in data: new_item = list(item) for i in refs: new_item[i] = self.__get_ref(item[i]) ret.append(tuple(new_item)) dset = parent[name] dset[:] = ret self.set_attributes(dset, attributes) return else: dset = self.__list_fill__(parent, name, data, options) elif self.__is_ref(options['dtype']): _dtype = self.__dtypes[options['dtype']] if isinstance(data, RegionBuilder): dset = parent.require_dataset(name, shape=(), dtype=_dtype) @self.__queue_ref def _filler(): ref = self.__get_ref(data.builder, data.region) dset = parent[name] dset[()] = ref self.set_attributes(dset, attributes) elif isinstance(data, ReferenceBuilder): dset = parent.require_dataset(name, dtype=_dtype, shape=()) @self.__queue_ref def _filler(): ref = self.__get_ref(data.builder) dset = parent[name] dset[()] = ref self.set_attributes(dset, attributes) else: if options['dtype'] == 'region': dset = parent.require_dataset(name, dtype=_dtype, shape=(len(data),)) @self.__queue_ref def _filler(): refs = list() for item in data: refs.append(self.__get_ref(item.builder, item.region)) dset = parent[name] dset[()] = refs self.set_attributes(dset, attributes) else: dset = parent.require_dataset(name, shape=(len(data),), dtype=_dtype) @self.__queue_ref def _filler(): refs = list() for item in data: refs.append(self.__get_ref(item.builder)) dset = parent[name] self.set_attributes(dset, attributes) return else: if isinstance(data, str): dset = self.__scalar_fill__(parent, name, data, options) elif isinstance(data, DataChunkIterator): dset = self.__chunked_iter_fill__(parent, name, data, options) elif isinstance(data, Dataset): data_filename = os.path.abspath(data.file.filename) parent_filename = os.path.abspath(parent.file.filename) if data_filename != parent_filename: link = ExternalLink(os.path.relpath(data_filename, os.path.dirname(parent_filename)), data.name) else: link = SoftLink(data.name) parent[name] = link elif isinstance(data, Iterable) and not self.isinstance_inmemory_array(data): dset = self.__chunked_iter_fill__(parent, name, DataChunkIterator(data=data, buffer_size=100), options) elif hasattr(data, '__len__'): dset = self.__list_fill__(parent, name, data, options) else: dset = self.__scalar_fill__(parent, name, data, options) if link is None: self.set_attributes(dset, attributes) return dset
def write_dataset(self, **kwargs): """ Write a dataset to HDF5 The function uses other dataset-dependent write functions, e.g, __scalar_fill__, __list_fill__ and __chunked_iter_fill__ to write the data. """ parent, builder, link_data = getargs('parent', 'builder', 'link_data', kwargs) if builder.written: print('%s already written to %s' % (self.__get_path(builder), builder.source)) return None name = builder.name data = builder.data options = dict() # dict with additional if isinstance(data, H5DataIO): options['io_settings'] = data.io_settings link_data = data.link_data data = data.data else: options['io_settings'] = {} attributes = builder.attributes options['dtype'] = builder.dtype dset = None link = None # The user provided an existing h5py dataset as input and asked to create a link to the dataset if isinstance(data, Dataset): # Create a Soft/External link to the dataset if link_data: data_filename = os.path.abspath(data.file.filename) parent_filename = os.path.abspath(parent.file.filename) if data_filename != parent_filename: link = ExternalLink( os.path.relpath(data_filename, os.path.dirname(parent_filename)), data.name) else: link = SoftLink(data.name) parent[name] = link # Copy the dataset else: parent.copy(source=data, dest=parent, name=name, expand_soft=False, expand_external=False, expand_refs=False, without_attrs=True) dset = parent[name] # Write a compound dataset, i.e, a dataset with compound data type elif isinstance(options['dtype'], list): # do some stuff to figure out what data is a reference refs = list() for i, dts in enumerate(options['dtype']): if self.__is_ref(dts): refs.append(i) # If one ore more of the parts of the compound data type are references then we need to deal with those if len(refs) > 0: _dtype = self.__resolve_dtype__(options['dtype'], data) dset = parent.require_dataset(name, shape=(len(data), ), dtype=_dtype, **options['io_settings']) builder.written = True @self.__queue_ref def _filler(): ret = list() for item in data: new_item = list(item) for i in refs: new_item[i] = self.__get_ref(item[i]) ret.append(tuple(new_item)) dset = parent[name] dset[:] = ret self.set_attributes(dset, attributes) return # If the compound data type contains only regular data (i.e., no references) then we can write it as usual else: dset = self.__list_fill__(parent, name, data, options) # Write a dataset containing references, i.e., a region or object reference. # NOTE: we can ignore options['io_settings'] for scalar data elif self.__is_ref(options['dtype']): _dtype = self.__dtypes[options['dtype']] # Write a scalar data region reference dataset if isinstance(data, RegionBuilder): dset = parent.require_dataset(name, shape=(), dtype=_dtype) builder.written = True @self.__queue_ref def _filler(): ref = self.__get_ref(data.builder, data.region) dset = parent[name] dset[()] = ref self.set_attributes(dset, attributes) # Write a scalar object reference dataset elif isinstance(data, ReferenceBuilder): dset = parent.require_dataset(name, dtype=_dtype, shape=()) builder.written = True @self.__queue_ref def _filler(): ref = self.__get_ref(data.builder) dset = parent[name] dset[()] = ref self.set_attributes(dset, attributes) # Write an array dataset of references else: # Write a array of region references if options['dtype'] == 'region': dset = parent.require_dataset(name, dtype=_dtype, shape=(len(data), ), **options['io_settings']) builder.written = True @self.__queue_ref def _filler(): refs = list() for item in data: refs.append( self.__get_ref(item.builder, item.region)) dset = parent[name] dset[()] = refs self.set_attributes(dset, attributes) # Write array of object references else: dset = parent.require_dataset(name, shape=(len(data), ), dtype=_dtype, **options['io_settings']) builder.written = True @self.__queue_ref def _filler(): refs = list() for item in data: refs.append(self.__get_ref(item.builder)) dset = parent[name] self.set_attributes(dset, attributes) return # write a "regular" dataset else: # Write a scalar dataset containing a single string if isinstance(data, (text_type, binary_type)): dset = self.__scalar_fill__(parent, name, data, options) # Iterative write of a data chunk iterator elif isinstance(data, AbstractDataChunkIterator): dset = self.__chunked_iter_fill__(parent, name, data, options) # Write a regular in memory array (e.g., numpy array, list etc.) elif hasattr(data, '__len__'): dset = self.__list_fill__(parent, name, data, options) # Write a regular scalar dataset else: dset = self.__scalar_fill__(parent, name, data, options) # Create the attributes on the dataset only if we are the primary and not just a Soft/External link if link is None: self.set_attributes(dset, attributes) # Validate the attributes on the linked dataset elif len(attributes) > 0: pass builder.written = True return
def test_h5py_soft_link(self, empty_registry): registries = RegistryContainer(empty_registry) soft_link = SoftLink("/example") assert soft_link == registries.dump(soft_link)
def test_h5py_soft_link(self, h5py_file_with_group): soft_link = SoftLink("example") registries = new_registry_list() registries.to_file(h5py_file_with_group, "alias", soft_link) assert h5py_file_with_group.get("alias", getlink=True).path == "example"