def test_strip_hdf_all(self):
'''
Do not keep any parameters.
'''
strip_hdf(self.hdf_path, [], self.out_path)
with h5py.File(self.out_path, 'r') as hdf_file:
self.assertEqual(hdf_file['series'].keys(), [])
def test_strip_hdf_all(self):
'''
Do not keep any parameters.
'''
strip_hdf(self.hdf_path, [], self.out_path)
with h5py.File(self.out_path, 'r') as hdf_file:
self.assertEqual(hdf_file['series'].keys(), [])
def test_strip_hdf_dme_wow(self):
'''
Does not test that datasets and attributes are maintained, see
test_strip_hdf_ivv.
'''
params_to_keep = ['DME', 'WOW']
strip_hdf(self.hdf_path, params_to_keep, self.out_path)
with h5py.File(self.out_path, 'r') as hdf_file:
self.assertEqual(hdf_file['series'].keys(), params_to_keep)
def test_strip_hdf_dme_wow(self):
'''
Does not test that datasets and attributes are maintained, see
test_strip_hdf_ivv.
'''
params_to_keep = ['DME', 'WOW']
strip_hdf(self.hdf_path, params_to_keep, self.out_path)
with h5py.File(self.out_path, 'r') as hdf_file:
self.assertEqual(hdf_file['series'].keys(), params_to_keep)
def test_strip_hdf_ivv(self):
params_to_keep = ['IVV']
strip_hdf(self.hdf_path, params_to_keep, self.out_path)
with h5py.File(self.out_path, 'r') as hdf_file:
self.assertEqual(list(hdf_file['series'].keys()), params_to_keep)
# Ensure datasets are unchanged.
self.assertTrue(all(hdf_file['series']['IVV']['data'][:] == self.ivv_data))
self.assertTrue(all(hdf_file['series']['IVV']['mask'][:] == self.ivv_mask))
# Ensure attributes are unchanged.
self.assertEqual(hdf_file['series']['IVV'].attrs['supf_offset'],
self.ivv_supf_offset)
self.assertEqual(hdf_file['series']['IVV'].attrs['frequency'],
self.ivv_frequency)
def test_strip_hdf_ivv(self):
params_to_keep = ['IVV']
strip_hdf(self.hdf_path, params_to_keep, self.out_path)
with h5py.File(self.out_path, 'r') as hdf_file:
self.assertEqual(list(hdf_file['series'].keys()), params_to_keep)
# Ensure datasets are unchanged.
self.assertTrue(all(hdf_file['series']['IVV']['data'][:] == self.ivv_data))
self.assertTrue(all(hdf_file['series']['IVV']['mask'][:] == self.ivv_mask))
# Ensure attributes are unchanged.
self.assertEqual(hdf_file['series']['IVV'].attrs['supf_offset'],
self.ivv_supf_offset)
self.assertEqual(hdf_file['series']['IVV'].attrs['frequency'],
self.ivv_frequency)
def derived_trimmer(hdf_path, node_names, dest):
'''
Trims an HDF file of parameters which are not dependencies of nodes in
node_names.
:param hdf_path: file path of hdf file.
:type hdf_path: str
:param node_names: A list of Node names which are required.
:type node_names: list of str
:param dest: destination path for trimmed output file
:type dest: str
:return: parameters in stripped hdf file
:rtype: [str]
'''
params = []
with hdf_file(hdf_path) as hdf:
derived_nodes = get_derived_nodes(settings.NODE_MODULES)
node_mgr = NodeManager(
datetime.now(), hdf.duration, hdf.valid_param_names(), [], [],
derived_nodes, {}, {})
_graph = graph_nodes(node_mgr)
for node_name in node_names:
deps = dependencies3(_graph, node_name, node_mgr)
params.extend(filter(lambda d: d in node_mgr.hdf_keys, deps))
return strip_hdf(hdf_path, params, dest)
def derived_trimmer(hdf_path, node_names, dest):
'''
Trims an HDF file of parameters which are not dependencies of nodes in
node_names.
:param hdf_path: file path of hdf file.
:type hdf_path: str
:param node_names: A list of Node names which are required.
:type node_names: list of str
:param dest: destination path for trimmed output file
:type dest: str
:return: parameters in stripped hdf file
:rtype: [str]
'''
with hdf_file(hdf_path) as hdf:
duration = hdf.duration
raw_nodes = hdf.valid_param_names()
derived_nodes = get_derived_nodes(settings.NODE_MODULES)
node_mgr = NodeManager({}, duration, raw_nodes, derived_nodes.keys(), [],
derived_nodes, {}, {})
order, graph = dependency_order(node_mgr)
node_names = set(node_names)
queue = deque(i for i in order if i in node_names)
params = {i for i in node_mgr.hdf_keys if i in node_names}
visited = set()
while queue:
current = queue.popleft()
visited.add(current)
for name in graph[current]:
if graph.nodes[name]['node_type'] == 'HDFNode':
params.add(name)
elif name not in visited:
queue.append(name)
return strip_hdf(hdf_path, params, dest)
