def test_copy_array_exists_array(self):
source = self.source
dest = self.new_dest()
# copy array, dest array in the way
dest.create_dataset('baz', shape=(10,))
# raise
with pytest.raises(CopyError):
# should raise by default
copy(source['foo/bar/baz'], dest)
assert (10,) == dest['baz'].shape
with pytest.raises(CopyError):
copy(source['foo/bar/baz'], dest, if_exists='raise')
assert (10,) == dest['baz'].shape
# skip
copy(source['foo/bar/baz'], dest, if_exists='skip')
assert (10,) == dest['baz'].shape
# replace
copy(source['foo/bar/baz'], dest, if_exists='replace')
check_copied_array(source['foo/bar/baz'], dest['baz'])
# invalid option
with pytest.raises(ValueError):
copy(source['foo/bar/baz'], dest, if_exists='foobar')
def test_copy_group(self):
source = self.source
dest = self.new_dest()
# copy group, default options
copy(source['foo'], dest)
check_copied_group(source['foo'], dest['foo'])
def test_copy_bad_dest(self):
source = self.source
# try to copy to an array, dest must be a group
dest = self.new_dest().create_dataset('eggs', shape=(100,))
with pytest.raises(ValueError):
copy(source['foo/bar/baz'], dest)
def test_copy_group_shallow(self):
source = self.source
dest = self.new_dest()
# copy group, shallow
copy(source, dest, name='eggs', shallow=True)
check_copied_group(source, dest['eggs'], shallow=True)
def test_copy_group_options(self):
source = self.source
dest = self.new_dest()
# copy group, non-default options
copy(source['foo'], dest, name='qux', without_attrs=True)
assert 'foo' not in dest
check_copied_group(source['foo'], dest['qux'], without_attrs=True)
def test_copy_group_exists_group(self):
source = self.source
dest = self.new_dest()
# copy group, dest groups exist
dest.create_group('foo/bar')
copy(source['foo'], dest)
check_copied_group(source['foo'], dest['foo'])
def test_copy_array_name(self):
source = self.source
dest = self.new_dest()
# copy array with name
copy(source['foo/bar/baz'], dest, name='qux')
assert 'baz' not in dest
check_copied_array(source['foo/bar/baz'], dest['qux'])
def test_copy_array(self):
source = self.source
dest = self.new_dest()
# copy array with default options
copy(source['foo/bar/baz'], dest)
check_copied_array(source['foo/bar/baz'], dest['baz'])
copy(source['spam'], dest)
check_copied_array(source['spam'], dest['spam'])
def test_copy_group_no_name(self):
source = self.source
dest = self.new_dest()
with pytest.raises(TypeError):
# need a name if copy root
copy(source, dest)
copy(source, dest, name='root')
check_copied_group(source, dest['root'])
def test_copy_array_create_options(self):
source = self.source
dest = self.new_dest()
# copy array, provide creation options
compressor = Zlib(9)
create_kws = dict(chunks=(10,))
if self.dest_h5py:
create_kws.update(compression='gzip', compression_opts=9,
shuffle=True, fletcher32=True, fillvalue=42)
else:
create_kws.update(compressor=compressor, fill_value=42, order='F',
filters=[Adler32()])
copy(source['foo/bar/baz'], dest, without_attrs=True, **create_kws)
check_copied_array(source['foo/bar/baz'], dest['baz'],
without_attrs=True, expect_props=create_kws)
def test_copy_group_dry_run(self):
source = self.source
dest = self.new_dest()
# dry run, empty destination
n_copied, n_skipped, n_bytes_copied = \
copy(source['foo'], dest, dry_run=True, return_stats=True)
assert 0 == len(dest)
assert 3 == n_copied
assert 0 == n_skipped
assert 0 == n_bytes_copied
# dry run, array exists in destination
baz = np.arange(100, 200)
dest.create_dataset('foo/bar/baz', data=baz)
assert not np.all(source['foo/bar/baz'][:] == dest['foo/bar/baz'][:])
assert 1 == len(dest)
# raise
with pytest.raises(CopyError):
copy(source['foo'], dest, dry_run=True)
assert 1 == len(dest)
# skip
n_copied, n_skipped, n_bytes_copied = \
copy(source['foo'], dest, dry_run=True, if_exists='skip',
return_stats=True)
assert 1 == len(dest)
assert 2 == n_copied
assert 1 == n_skipped
assert 0 == n_bytes_copied
assert_array_equal(baz, dest['foo/bar/baz'])
# replace
n_copied, n_skipped, n_bytes_copied = \
copy(source['foo'], dest, dry_run=True, if_exists='replace',
return_stats=True)
assert 1 == len(dest)
assert 3 == n_copied
assert 0 == n_skipped
assert 0 == n_bytes_copied
assert_array_equal(baz, dest['foo/bar/baz'])
def test_copy_array_exists_array(self, source, dest):
# copy array, dest array in the way
dest.create_dataset('baz', shape=(10,))
# raise
with pytest.raises(CopyError):
# should raise by default
copy(source['foo/bar/baz'], dest)
assert (10,) == dest['baz'].shape
with pytest.raises(CopyError):
copy(source['foo/bar/baz'], dest, if_exists='raise')
assert (10,) == dest['baz'].shape
# skip
copy(source['foo/bar/baz'], dest, if_exists='skip')
assert (10,) == dest['baz'].shape
# replace
copy(source['foo/bar/baz'], dest, if_exists='replace')
check_copied_array(source['foo/bar/baz'], dest['baz'])
# invalid option
with pytest.raises(ValueError):
copy(source['foo/bar/baz'], dest, if_exists='foobar')
def test_copy_group_no_name(self, source, dest):
if source.__module__.startswith('h5py'):
with pytest.raises(TypeError):
copy(source, dest)
else:
# For v3, dest.name will be inferred from source.name
copy(source, dest)
check_copied_group(source, dest[source.name.lstrip('/')])
copy(source, dest, name='root')
check_copied_group(source, dest['root'])
def test_logging(self):
source = self.source
dest = self.new_dest()
# callable log
copy(source['foo'], dest, dry_run=True, log=print)
# file name
fn = tempfile.mktemp()
atexit.register(os.remove, fn)
copy(source['foo'], dest, dry_run=True, log=fn)
# file
with tempfile.TemporaryFile(mode='w') as f:
copy(source['foo'], dest, dry_run=True, log=f)
# bad option
with pytest.raises(TypeError):
copy(source['foo'], dest, dry_run=True, log=True)
def test_copy_group_exists_array(self, source, dest):
# copy group, dest array in the way
dest.create_dataset('foo/bar', shape=(10, ))
# raise
with pytest.raises(CopyError):
copy(source['foo'], dest)
assert dest['foo/bar'].shape == (10, )
with pytest.raises(CopyError):
copy(source['foo'], dest, if_exists='raise')
assert dest['foo/bar'].shape == (10, )
# skip
copy(source['foo'], dest, if_exists='skip')
assert dest['foo/bar'].shape == (10, )
# replace
copy(source['foo'], dest, if_exists='replace')
check_copied_group(source['foo'], dest['foo'])
def test_copy_array_exists_group(self, source, dest):
# copy array, dest group in the way
dest.create_group('baz')
# raise
with pytest.raises(CopyError):
copy(source['foo/bar/baz'], dest)
assert not hasattr(dest['baz'], 'shape')
with pytest.raises(CopyError):
copy(source['foo/bar/baz'], dest, if_exists='raise')
assert not hasattr(dest['baz'], 'shape')
# skip
copy(source['foo/bar/baz'], dest, if_exists='skip')
assert not hasattr(dest['baz'], 'shape')
# replace
copy(source['foo/bar/baz'], dest, if_exists='replace')
check_copied_array(source['foo/bar/baz'], dest['baz'])
def test_logging(self):
source = self.source
dest = self.new_dest()
# callable log
copy(source['foo'], dest, dry_run=True, log=print)
# file name
fn = tempfile.mktemp()
atexit.register(os.remove, fn)
copy(source['foo'], dest, dry_run=True, log=fn)
# file
with tempfile.TemporaryFile(mode='w') as f:
copy(source['foo'], dest, dry_run=True, log=f)
# bad option
with pytest.raises(TypeError):
copy(source['foo'], dest, dry_run=True, log=True)
def test_copy_array_exists_group(self):
source = self.source
dest = self.new_dest()
# copy array, dest group in the way
dest.create_group('baz')
# raise
with pytest.raises(CopyError):
copy(source['foo/bar/baz'], dest)
assert not hasattr(dest['baz'], 'shape')
with pytest.raises(CopyError):
copy(source['foo/bar/baz'], dest, if_exists='raise')
assert not hasattr(dest['baz'], 'shape')
# skip
copy(source['foo/bar/baz'], dest, if_exists='skip')
assert not hasattr(dest['baz'], 'shape')
# replace
copy(source['foo/bar/baz'], dest, if_exists='replace')
check_copied_array(source['foo/bar/baz'], dest['baz'])
def test_copy_group_exists_array(self):
source = self.source
dest = self.new_dest()
# copy group, dest array in the way
dest.create_dataset('foo/bar', shape=(10,))
# raise
with pytest.raises(CopyError):
copy(source['foo'], dest)
assert dest['foo/bar'].shape == (10,)
with pytest.raises(CopyError):
copy(source['foo'], dest, if_exists='raise')
assert dest['foo/bar'].shape == (10,)
# skip
copy(source['foo'], dest, if_exists='skip')
assert dest['foo/bar'].shape == (10,)
# replace
copy(source['foo'], dest, if_exists='replace')
check_copied_group(source['foo'], dest['foo'])
def test_copy_array_skip_initialized(self):
source = self.source
dest = self.new_dest()
dest.create_dataset('baz', shape=(100, ), chunks=(10, ), dtype='i8')
assert not np.all(source['foo/bar/baz'][:] == dest['baz'][:])
if self.dest_h5py:
with pytest.raises(ValueError):
# not available with copy to h5py
copy(source['foo/bar/baz'], dest, if_exists='skip_initialized')
else:
# copy array, dest array exists but not yet initialized
copy(source['foo/bar/baz'], dest, if_exists='skip_initialized')
check_copied_array(source['foo/bar/baz'], dest['baz'])
# copy array, dest array exists and initialized, will be skipped
dest['baz'][:] = np.arange(100, 200)
copy(source['foo/bar/baz'], dest, if_exists='skip_initialized')
assert_array_equal(np.arange(100, 200), dest['baz'][:])
assert not np.all(source['foo/bar/baz'][:] == dest['baz'][:])
def test_copy_array_skip_initialized(self):
source = self.source
dest = self.new_dest()
dest.create_dataset('baz', shape=(100,), chunks=(10,), dtype='i8')
assert not np.all(source['foo/bar/baz'][:] == dest['baz'][:])
if self.dest_h5py:
with pytest.raises(ValueError):
# not available with copy to h5py
copy(source['foo/bar/baz'], dest, if_exists='skip_initialized')
else:
# copy array, dest array exists but not yet initialized
copy(source['foo/bar/baz'], dest, if_exists='skip_initialized')
check_copied_array(source['foo/bar/baz'], dest['baz'])
# copy array, dest array exists and initialized, will be skipped
dest['baz'][:] = np.arange(100, 200)
copy(source['foo/bar/baz'], dest, if_exists='skip_initialized')
assert_array_equal(np.arange(100, 200), dest['baz'][:])
assert not np.all(source['foo/bar/baz'][:] == dest['baz'][:])
def test_copy_group_exists_group(self, source, dest):
# copy group, dest groups exist
dest.create_group('foo/bar')
copy(source['foo'], dest)
check_copied_group(source['foo'], dest['foo'])
def test_copy_group_shallow(self, source, dest):
# copy group, shallow
copy(source, dest, name='eggs', shallow=True)
check_copied_group(source, dest['eggs'], shallow=True)
def test_copy_group_options(self, source, dest):
# copy group, non-default options
copy(source['foo'], dest, name='qux', without_attrs=True)
assert 'foo' not in dest
check_copied_group(source['foo'], dest['qux'], without_attrs=True)
def test_copy_group(self, source, dest):
# copy group, default options
copy(source['foo'], dest)
check_copied_group(source['foo'], dest['foo'])
def test_copy_array_name(self, source, dest):
# copy array with name
copy(source['foo/bar/baz'], dest, name='qux')
assert 'baz' not in dest
check_copied_array(source['foo/bar/baz'], dest['qux'])
def test_copy_bad_dest(self, source, dest):
# try to copy to an array, dest must be a group
dest = dest.create_dataset('eggs', shape=(100, ))
with pytest.raises(ValueError):
copy(source['foo/bar/baz'], dest)
def test_copy_array(self, source, dest):
# copy array with default options
copy(source['foo/bar/baz'], dest)
check_copied_array(source['foo/bar/baz'], dest['baz'])
copy(source['spam'], dest)
check_copied_array(source['spam'], dest['spam'])