def __init__(self, data_dir, tag): ''' Args: data_dir (str) : Data directory for the classifications data tag (str) : Identifying tag for the data to load. ''' # Open the file classifications_filepath = os.path.join( data_dir, 'classifications_{}.hdf5'.format(tag)) with h5py.File(classifications_filepath, 'r') as f: # Store the data self.data = {} for key in f.keys(): if key != 'parameters': self.data[key] = f[key][...] # Store the data attributes self.data_attrs = {} for key in f.attrs.keys(): self.data_attrs[key] = utilities.check_and_decode_bytes( f.attrs[key]) # Store the parameters self.parameters = {} param_grp = f['parameters'] for key in param_grp.attrs.keys(): self.parameters[key] = utilities.check_and_decode_bytes( param_grp.attrs[key])
def test_select_ids_jug(self): kwargs = copy.copy(default_kwargs) # Modifications to the output dirs to account for shifting up one level kwargs['out_dir'] = self.out_dir kwargs['snapshot_kwargs'][ 'sdir'] = './linefinder/tests/data/stars_included_test_data' kwargs['snapshot_kwargs'][ 'halo_data_dir'] = './linefinder/tests/data/ahf_test_data' data_filters = { 'radial_cut': { 'data_key': 'Rf', 'data_min': 0., 'data_max': 1., }, } id_selector = select.IDSelector(**kwargs) id_selector.select_ids(data_filters) # Run jug version os.system("jug execute ./linefinder/tests/select_jugfile.py &") os.system("jug execute ./linefinder/tests/select_jugfile.py") files = [] for filepath in self.filepaths: files.append(h5py.File(filepath, 'r')) for key in ['target_ids', 'target_child_ids']: npt.assert_allclose(files[0][key][...], files[1][key][...]) for key in files[0]['parameters'].attrs.keys(): # These shouldn't match if key == 'tag': continue try: self.assertEqual( utilities.check_and_decode_bytes( files[0]['parameters'].attrs[key], ), utilities.check_and_decode_bytes( files[1]['parameters'].attrs[key], ), ) # In case it's an array except ValueError: npt.assert_allclose( files[0]['parameters'].attrs[key], files[1]['parameters'].attrs[key], )
def __init__(self, data_dir, tag): ''' Args: data_dir (str) : Data directory for the classified data tag (str) : Identifying tag for the data to load. ''' # Open the file ids_filepath = os.path.join(data_dir, 'ids_{}.hdf5'.format(tag)) with h5py.File(ids_filepath, 'r') as f: # Store the data self.data = {} for key in f.keys(): if key != 'parameters': self.data[key] = f[key][...] # Store the data attributes self.data_attrs = {} for key in f.attrs.keys(): self.data_attrs[key] = f.attrs[key] # Store the parameters self.parameters = {} param_grp = f['parameters'] for key in param_grp.attrs.keys(): self.parameters[key] = utilities.check_and_decode_bytes( param_grp.attrs[key]) # Store the parameters self.snapshot_parameters = {} try: snap_param_grp = f['parameters/snapshot_parameters'] for key in snap_param_grp.attrs.keys(): self.snapshot_parameters[ key] = utilities.check_and_decode_bytes( snap_param_grp.attrs[key]) except KeyError: print('No snapshot parameters stored. Not loading.') # Store the used data filters self.data_filters = {} try: filters_grp = f['parameters/data_filters'] for filters_subset in filters_grp.keys(): subgroup = filters_grp[filters_subset] self.data_filters[filters_subset] = {} for key in subgroup.attrs.keys(): self.data_filters[filters_subset][key] = \ subgroup.attrs[key] except KeyError: print("Failed to load data_filters. Using older data?")
def load_data_into_ptrack(filename, store_parameters=False): filepath = os.path.join(self.out_dir, filename) f = h5py.File(filepath, 'r') # Store the particle track data in a dictionary for key in f.keys(): if key != 'parameters': self.ptrack[key] = f[key][...] # Store the ptrack attributes for key in f.attrs.keys(): self.ptrack_attrs[key] = f.attrs[key] if store_parameters: default_attrs_to_replace = [ 'halo_data_dir', 'mtree_halos_index', 'halo_file_tag', ] for attr in default_attrs_to_replace: if getattr(self, attr) is default: attr_value = utilities.check_and_decode_bytes( f['parameters'].attrs[attr]) setattr(self, attr, attr_value) for parameter_key in [ 'galaxy_cut', 'length_scale', ]: self.ptrack_attrs[ parameter_key ] = \ f['parameters'].attrs[parameter_key] f.close()
def replace_default_attr(attr_name): attr = getattr(self, attr_name) if attr is default: try: if attr_name == 'sdir': attr = utilities.check_and_decode_bytes( f['parameters/snapshot_parameters']. attrs[attr_name]) else: attr = utilities.check_and_decode_bytes( f['parameters'].attrs[attr_name]) setattr(self, attr_name, attr) except KeyError: raise LookupError( 'Cannot fallback to default parameters because ' + \ '{} does not include default parameters.'.format( id_filename, ) )
def __init__(self, data_dir, tag, ahf_index=None, *args, **kwargs): ''' Args: data_dir (str) : Data directory for the classified data tag (str) : Identifying tag for the data to load. ahf_index (str or int) : Index to use for AHF data. ''' # Open the file ptracks_filepath = os.path.join(data_dir, 'ptracks_{}.hdf5'.format(tag)) with h5py.File(ptracks_filepath, 'r') as f: # Store the data self.data = {} for key in f.keys(): if key != 'parameters': self.data[key] = f[key][...] # Store the data attributes self.data_attrs = {} for key in f.attrs.keys(): self.data_attrs[key] = utilities.check_and_decode_bytes( f.attrs[key]) # Store the parameters self.parameters = {} param_grp = f['parameters'] for key in param_grp.attrs.keys(): self.parameters[key] = utilities.check_and_decode_bytes( param_grp.attrs[key]) # Reorganize data to match with formatting in TimeData self.data['P'] = np.rollaxis(self.data['P'], 2) self.data['V'] = np.rollaxis(self.data['V'], 2) super(PTracks, self).__init__(data_dir=data_dir, snum=self.data['snum'], ahf_index=ahf_index, *args, **kwargs)
def identify_duplicate_ids(self): '''Get all IDs that have duplicates at the latest snapshot. This draws data from the same sample as the original. Returns: duplicate_ids (set) : A set of all IDs that have duplicates at snum_end. ''' assert 'target_child_ids' not in self.f.keys( ), "Identifying duplicate IDs does not work with new ID system." if self.p_types is None: self.p_types = self.f['parameters'].attrs['p_types'] if self.snapshot_kwargs is None: self.snapshot_kwargs = dict( self.f['parameters/snapshot_parameters'].attrs) duplicate_ids = set() id_sets = [] for ptype in self.p_types: # Check for bytes data and decode for key, item in copy.deepcopy(self.snapshot_kwargs).items(): self.snapshot_kwargs[key] = \ utilities.check_and_decode_bytes( item ) self.snapshot_kwargs['snum'] = self.reference_snum_for_duplicates self.snapshot_kwargs['ptype'] = ptype p_data = particle_data.ParticleData(**self.snapshot_kwargs) duplicate_ids = duplicate_ids | set(p_data.find_duplicate_ids()) # Get the IDs out, so that we can find duplicates where the particles are of different types id_sets.append(set(p_data.get_data('ID'))) duplicate_ids = duplicate_ids | set.intersection(*id_sets) return duplicate_ids