def test_strip_hdf_all(self): ''' Do not keep any parameters. ''' strip_hdf(self.hdf_path, [], self.out_path) with h5py.File(self.out_path, 'r') as hdf_file: self.assertEqual(hdf_file['series'].keys(), [])
def test_strip_hdf_dme_wow(self): ''' Does not test that datasets and attributes are maintained, see test_strip_hdf_ivv. ''' params_to_keep = ['DME', 'WOW'] strip_hdf(self.hdf_path, params_to_keep, self.out_path) with h5py.File(self.out_path, 'r') as hdf_file: self.assertEqual(hdf_file['series'].keys(), params_to_keep)
def test_strip_hdf_ivv(self): params_to_keep = ['IVV'] strip_hdf(self.hdf_path, params_to_keep, self.out_path) with h5py.File(self.out_path, 'r') as hdf_file: self.assertEqual(list(hdf_file['series'].keys()), params_to_keep) # Ensure datasets are unchanged. self.assertTrue(all(hdf_file['series']['IVV']['data'][:] == self.ivv_data)) self.assertTrue(all(hdf_file['series']['IVV']['mask'][:] == self.ivv_mask)) # Ensure attributes are unchanged. self.assertEqual(hdf_file['series']['IVV'].attrs['supf_offset'], self.ivv_supf_offset) self.assertEqual(hdf_file['series']['IVV'].attrs['frequency'], self.ivv_frequency)
def derived_trimmer(hdf_path, node_names, dest): ''' Trims an HDF file of parameters which are not dependencies of nodes in node_names. :param hdf_path: file path of hdf file. :type hdf_path: str :param node_names: A list of Node names which are required. :type node_names: list of str :param dest: destination path for trimmed output file :type dest: str :return: parameters in stripped hdf file :rtype: [str] ''' params = [] with hdf_file(hdf_path) as hdf: derived_nodes = get_derived_nodes(settings.NODE_MODULES) node_mgr = NodeManager( datetime.now(), hdf.duration, hdf.valid_param_names(), [], [], derived_nodes, {}, {}) _graph = graph_nodes(node_mgr) for node_name in node_names: deps = dependencies3(_graph, node_name, node_mgr) params.extend(filter(lambda d: d in node_mgr.hdf_keys, deps)) return strip_hdf(hdf_path, params, dest)
def derived_trimmer(hdf_path, node_names, dest): ''' Trims an HDF file of parameters which are not dependencies of nodes in node_names. :param hdf_path: file path of hdf file. :type hdf_path: str :param node_names: A list of Node names which are required. :type node_names: list of str :param dest: destination path for trimmed output file :type dest: str :return: parameters in stripped hdf file :rtype: [str] ''' with hdf_file(hdf_path) as hdf: duration = hdf.duration raw_nodes = hdf.valid_param_names() derived_nodes = get_derived_nodes(settings.NODE_MODULES) node_mgr = NodeManager({}, duration, raw_nodes, derived_nodes.keys(), [], derived_nodes, {}, {}) order, graph = dependency_order(node_mgr) node_names = set(node_names) queue = deque(i for i in order if i in node_names) params = {i for i in node_mgr.hdf_keys if i in node_names} visited = set() while queue: current = queue.popleft() visited.add(current) for name in graph[current]: if graph.nodes[name]['node_type'] == 'HDFNode': params.add(name) elif name not in visited: queue.append(name) return strip_hdf(hdf_path, params, dest)