def _test_missing_chunks(self, shape, chunk_overrides=None):
# Put fake dataset into chunk store
store = NpyFileChunkStore(self.tempdir)
prefix = 'cb2'
data, chunk_info = put_fake_dataset(store, prefix, shape, chunk_overrides)
# Delete some random chunks in each array of the dataset
missing_chunks = {}
rs = random.Random(4)
for array, info in chunk_info.items():
array_name = store.join(prefix, array)
slices = da.core.slices_from_chunks(info['chunks'])
culled_slices = rs.sample(slices, len(slices) // 10 + 1)
missing_chunks[array] = culled_slices
for culled_slice in culled_slices:
chunk_name, shape = store.chunk_metadata(array_name, culled_slice)
os.remove(os.path.join(store.path, chunk_name) + '.npy')
vfw = ChunkStoreVisFlagsWeights(store, chunk_info, None)
assert_equal(vfw.store, store)
assert_equal(vfw.vis_prefix, prefix)
# Check that (only) missing chunks have been replaced by zeros
vis = data['correlator_data']
for culled_slice in missing_chunks['correlator_data']:
vis[culled_slice] = 0.
assert_array_equal(vfw.vis, vis)
weights = data['weights'] * data['weights_channel'][..., np.newaxis]
for culled_slice in missing_chunks['weights'] + missing_chunks['weights_channel']:
weights[culled_slice] = 0.
assert_array_equal(vfw.weights, weights)
# Check that (only) missing chunks have been flagged as 'data lost'
flags = data['flags']
for culled_slice in missing_chunks['flags']:
flags[culled_slice] = 0
for culled_slice in itertools.chain(*missing_chunks.values()):
flags[culled_slice] |= DATA_LOST
assert_array_equal(vfw.flags, flags)
def test_missing_chunks(self):
# Put fake dataset into chunk store
store = NpyFileChunkStore(self.tempdir)
base_name = 'cb2'
shape = (10, 64, 30)
data, chunk_info = put_fake_dataset(store, base_name, shape)
# Delete a random chunk in each array of the dataset
missing_chunks = {}
rs = random.Random(4)
for array, info in chunk_info.items():
array_name = store.join(base_name, array)
slices = da.core.slices_from_chunks(info['chunks'])
culled_slice = rs.choice(slices)
missing_chunks[array] = culled_slice
chunk_name, shape = store.chunk_metadata(array_name, culled_slice)
os.remove(os.path.join(store.path, chunk_name) + '.npy')
vfw = ChunkStoreVisFlagsWeights(store, base_name, chunk_info)
# Check that (only) missing chunks have been replaced by zeros
vis = data['correlator_data']
vis[missing_chunks['correlator_data']] = 0.
assert_array_equal(vfw.vis, vis)
weights = data['weights'] * data['weights_channel'][..., np.newaxis]
weights[missing_chunks['weights']] = 0.
weights[missing_chunks['weights_channel']] = 0.
assert_array_equal(vfw.weights, weights)
# Check that (only) missing chunks have been flagged as 'data lost'
flags = data['flags']
flags[missing_chunks['flags']] = 0.
flags[missing_chunks['correlator_data']] |= 8
flags[missing_chunks['weights']] |= 8
flags[missing_chunks['weights_channel']] |= 8
flags[missing_chunks['flags']] |= 8
assert_array_equal(vfw.flags, flags)