def load_lib(self, libname='', driver=None): """Read a CKC library which has been pre-convolved to be close to your resolution. This library should be stored as an HDF5 file, with the datasets ``wavelengths``, ``parameters`` and ``spectra``. These are ndarrays of shape (nwave,), (nmodels,), and (nmodels, nwave) respecitvely. The ``parameters`` array is a structured array. Spectra with no fluxes > 1e-32 are removed from the library if the librarty is kept in memory. """ import h5py f = h5py.File(libname, "r", driver=driver) self._wave = np.array(f['wavelengths']) self._libparams = np.array(f['parameters']) if self._in_memory: self._spectra = np.array(f['spectra']) f.close() # Filter library so that only existing spectra are included maxf = np.max(self._spectra, axis=1) good = maxf > 1e-32 self._libparams = self._libparams[good] self._spectra = self._spectra[good, :] else: self._spectra = f['spectra'] if self.logify_Z: from numpy.lib import recfunctions as rfn self._libparams['Z'] = np.log10(self._libparams['Z']) rfn.rename_fields(self._libparams, {'Z': 'logZ'})
def read_halos_Rockstar_SO(*args, **kwargs): halos = read_halos_Rockstar(*args, **kwargs) halos = rfn.rename_fields(halos, {'m':'m_rockstar', 'alt_m':'alt_m_rockstar', 'N':'N_rockstar', 'alt_N':'alt_N_rockstar'}) halos = rfn.rename_fields(halos, {'m_SO':'m', 'alt_m_SO':'alt_m', 'N_SO':'N', 'alt_N_SO':'alt_N'}) return halos
def load_data(self): """Loads the data, which is described by the dataset. Note: This does not call the ``prepare_data`` method! It only loads the data as the method names says. Returns ------- data : numpy record ndarray A numpy record ndarray holding the monte-carlo data. """ pathfilenames = self.pathfilenames if isinstance(pathfilenames, str): pathfilenames = [pathfilenames] pathfilename = pathfilenames[0] assert_file_exists(pathfilename) data = np.load(pathfilename) for i in range(1, len(pathfilenames)): pathfilename = pathfilenames[i] assert_file_exists(pathfilename) data = np.append(data, np.load(pathfilename)) data = np_rfn.rename_fields(data, self._mc_field_name_renaming_dict) return data
def __init__(self, query=None, table=None): # Either query or table, but not both (query xor table). if (query and table is not None) or\ (not query and table is None): raise Exception('Either the query or the table parameter is ' 'required. But not both.') if table is not None: # Use table if it was provided. self.table = table else: # Otherwise use the query to create the table. self.table = SDSS.query_sql(query) self.catalog = np.array(self.table) self.catalog = rename_fields(self.catalog, {'objID': 'id'}) # Calculate B and V like the VizieR data. # Use Robert Lupton's derived equations found here: # http://www.sdss3.org/dr8/algorithms/sdssUBVRITransform.php g = self.catalog['g'] r = self.catalog['r'] B = g + 0.3130 * (g - r) + 0.2271 # sigma = 0.0107 V = g - 0.5784 * (g - r) - 0.0038 # sigma = 0.0054 self.catalog = append_fields(self.catalog, 'B', B) self.catalog = append_fields(self.catalog, 'V', V)
def load_and_prepare_data(pathfilenames): """Loads the data file(s), renames fields and applies diffuse dataset cuts. Parameters ---------- pathfilenames : str | sequence of str The file name(s), including path(s), of the monte-carlo data file(s). Returns ------- data : numpy record ndarray Loaded and prepared monte-carlo data. """ if isinstance(pathfilenames, basestring): pathfilenames = [pathfilenames] pathfilename = pathfilenames[0] assert_file_exists(pathfilename) data = np.load(pathfilename) for i in range(1, len(pathfilenames)): pathfilename = pathfilenames[i] assert_file_exists(pathfilename) data = np.append(data, np.load(pathfilename)) # Rename fields based on MC_keys dictionary. data = np_rfn.rename_fields(data, CFG['MC_keys']) # Apply diffuse dataset cuts. data = diffuse_cuts(data) return data
def get_price(self, order_book_id, start, end): """ :param order_book_id: e.g. 000002.XSHE :param start: 20160101 :param end: 20160201 :returns: :rtype: numpy.rec.array """ # start = get_date_from_int(start) # end = get_date_from_int(end) # bar_count = (end - start).days # TODO: this is slow, make it run faster bar_count = 1000 origin_bars = bars = self.history_bars(order_book_id, bar_count, "1d") dtype = copy.deepcopy(bars.dtype) names = list(dtype.names) names[0] = "date" dtype.names = names bars = rfn.rename_fields(bars, {"datetime": "date"}) bars["date"] = origin_bars["datetime"] / 1000000 return bars
def load_labels(fname='cocotools/coords/standard_labels.csv', usecols=None): """Load the standard labels file; return a record array with the atlas. """ # Converters for the fields we're interested in: strings without whitespace # and floats cleanstr = lambda x: str(x).strip() conv = {0:cleanstr, 1:cleanstr, 2:cleanstr, 3:cleanstr, 4:float, 5:float, 6:float, 7:cleanstr, 8:cleanstr, 9:int} # Exclude comments column by default. usecols = range(7)+[8] if usecols is None else usecols with open(fname) as f: # Read the first line for the names all_names = [n.strip() for n in f.next().split(',')] # genfromtxt is buggy, it gets confused with commas inside strings # use the stdlib csv reader and rebuild lines with | as separator, # which genfromtxt can then use data = ['|'.join(x) for x in list(csv.reader(f))] # Older versions of numpy need an actual filehandle, not an arbitrary # iterable. with tempfile.TemporaryFile() as fdata: fdata.write('\n'.join(data)) fdata.seek(0) # Now, let genfromtxt iterate over the rest of the file araw = np.genfromtxt(fdata, delimiter='|', usecols=usecols, converters=conv) # Rename the fields so the dtype has more useful names than f0, f1, etc. # Also make it a recarray for more convenient use further down names = [all_names[i] for i in usecols] renamer = dict(zip(araw.dtype.names, names)) atlas = rfn.rename_fields(araw, renamer).view(np.recarray) # Flip y axis to have frontal areas on the left in axial projections #atlas.y *= -1 return atlas
def load_as_recarr(self, filename, fields=None, formatmarker='# format:'): """Warning: if there's missing value, it will be filled with default value. See: https://docs.scipy.org/doc/numpy/user/basics.rec.html numpy.lib.recfunctions.merge_arrays() -1 for integers -1.0 for floating point numbers '-' for characters '-1' for strings True for boolean values """ cols = self.load(filename, fields=fields, formatmarker=formatmarker) _fields, cols_data = cols.keys(), cols.values( ) # None fields is updated during self.load cols_type = [ self.name2fmter[field_name]._type_ for field_name in _fields ] #for name,x,t in zip(fields, cols_data, cols_type): # print '***', name,x,t cols_nparr = [ np.array(x, dtype=t) for name, x, t in zip(_fields, cols_data, cols_type) ] unnamed_recarr = rfn.merge_arrays(cols_nparr, flatten=True, usemask=False).view(np.recarray) return rfn.rename_fields( unnamed_recarr, dict(zip(unnamed_recarr.dtype.names, _fields)))
def filter_on_fields(to_filter, for_filter, filter_fields, filter_fields_2=None, return_selection=False): """ Returns entries of to_filter whose combination of the filter_fields values are present in for_filter. filter_fields_2: names of filter_fields in for_filter (if different than in to_filter) If return_selection, will instead """ a = np.array(fields_view(to_filter, filter_fields)) if filter_fields_2 is None: filter_fields_2 = filter_fields b = np.array(fields_view(for_filter, filter_fields_2)) # Rename the fields, if needed # If only one field is selected, this won't be needed (and would return None instead of working) if not isinstance(filter_fields, str) and len(filter_fields) > 1: b = recfunctions.rename_fields( b, dict(zip(filter_fields_2, filter_fields))) selection = np.in1d(a, b) if return_selection: return selection else: return to_filter[selection]
def show(self, header, newdata): # print("{0} new, {1} stations".format(header['num_sources'][0], header['num_stations'][0])) if newdata.shape[0] > 0: newdata = rename_fields(newdata, {'t':'time'}) newdata['time'] -= self._t_offset self._dataq.append(newdata) self.send("B4D_LMAnewsources_live")
def load_catalogue(path): sources = np.load(path) # Maintain backwards-compatibility maps = [ ("ra", "ra_rad"), ("dec", "dec_rad"), ("Relative Injection Weight", "injection_weight_modifier"), ("Distance (Mpc)", "distance_mpc"), ("Name", "source_name"), ("Ref Time (MJD)", "ref_time_mjd"), ("Start Time (MJD)", "start_time_mjd"), ("End Time (MJD)", "end_time_mjd"), ] for (old_key, new_key) in maps: if old_key in sources.dtype.names: sources = rename_fields(sources, {old_key: new_key}) if "base_weight" not in sources.dtype.names: base_weight = np.ones(len(sources)) sources = append_fields(sources, "base_weight", base_weight, usemask=False, dtypes=[float]) # Check that ra and dec are really in radians! if max(sources["ra_rad"]) > 2.0 * np.pi: raise Exception("Sources have Right Ascension values greater than 2 " "pi. Are you sure you're not using degrees rather " "than radians?") if max(abs(sources["dec_rad"])) > np.pi / 2.0: raise Exception("Sources have Declination values exceeding " "+/- pi/2. Are you sure you're not using degrees " "rather than radians?") # Check that all sources have a unique name if len(set(sources["source_name"])) < len(sources["source_name"]): raise Exception("Some sources in catalogue do not have unique " "names. Please assign unique names to each source.") # Rescale 'base_weight' # sources["base_weight"] /= np.mean(sources["base_weight"]) # Order sources sources = np.sort(sources, order="distance_mpc") return sources
def format_data_keys(exp, mc): exp = np_rfn.rename_fields(exp, { 'logE': 'log_energy', 'angErr': 'ang_err' }) mc = np_rfn.rename_fields( mc, { 'trueAzi': 'true_azi', 'trueZen': 'true_zen', 'logE': 'log_energy', 'angErr': 'ang_err', 'trueE': 'true_energy', 'trueRa': 'true_ra', 'trueDec': 'true_dec', 'ow': 'mcweight' }) return (exp, mc)
def test_Osiris_Dev_Hdf5_ParticleFile_is_valid_backend( make_prt_file: Callable[[str, np.ndarray, Optional[str]], Path]): data = unstructured_to_structured(np.random.random((10, 4))) data = rename_fields(data, {"f0": "q"}) prt_path = make_prt_file("osiris_dev_particles_hdf5", data) assert Osiris_Dev_Hdf5_ParticleFile.is_valid_backend(prt_path)
def write_gfas(filename, data, indir=None, nside=None, survey="?", gaiaepoch=None): """Write a catalogue of Guide/Focus/Alignment targets. Parameters ---------- filename : :class:`str` Output file name. data : :class:`~numpy.ndarray` Array of GFAs to write to file. indir : :class:`str`, optional, defaults to None. Name of input Legacy Survey Data Release directory, write to header of output file if passed (and if not None). nside: :class:`int`, defaults to None. If passed, add a column to the GFAs array popluated with HEALPixels at resolution `nside`. survey : :class:`str`, optional, defaults to "?" Written to output file header as the keyword `SURVEY`. gaiaepoch: :class:`float`, defaults to None Gaia proper motion reference epoch. If not None, write to header of output file. If None, default to an epoch of 2015.5. """ # ADM rename 'TYPE' to 'MORPHTYPE'. data = rfn.rename_fields(data, {'TYPE': 'MORPHTYPE'}) # ADM create header to include versions, etc. hdr = fitsio.FITSHDR() depend.setdep(hdr, 'desitarget', desitarget_version) depend.setdep(hdr, 'desitarget-git', gitversion()) if indir is not None: depend.setdep(hdr, 'input-data-release', indir) # ADM note that if 'dr' is not in the indir DR # ADM directory structure, garbage will # ADM be rewritten gracefully in the header. drstring = 'dr'+indir.split('dr')[-1][0] depend.setdep(hdr, 'photcat', drstring) # ADM add HEALPix column, if requested by input. if nside is not None: theta, phi = np.radians(90-data["DEC"]), np.radians(data["RA"]) hppix = hp.ang2pix(nside, theta, phi, nest=True) data = rfn.append_fields(data, 'HPXPIXEL', hppix, usemask=False) hdr['HPXNSIDE'] = nside hdr['HPXNEST'] = True # ADM add the type of survey (main, or commissioning "cmx") to the header. hdr["SURVEY"] = survey # ADM add the Gaia reference epoch, or pass 2015.5 if not included. hdr['REFEPOCH'] = {'name': 'REFEPOCH', 'value': 2015.5, 'comment': "Gaia Proper Motion Reference Epoch"} if gaiaepoch is not None: hdr['REFEPOCH'] = gaiaepoch fitsio.write(filename, data, extname='GFA_TARGETS', header=hdr, clobber=True)
def test_rename_fields(self): # Test rename fields a = np.array([(1, (2, [3.0, 30.])), (4, (5, [6.0, 60.]))], dtype=[('a', int), ('b', [('ba', float), ('bb', (float, 2))])]) test = rename_fields(a, {'a': 'A', 'bb': 'BB'}) newdtype = [('A', int), ('b', [('ba', float), ('BB', (float, 2))])] control = a.view(newdtype) assert_equal(test.dtype, newdtype) assert_equal(test, control)
def test_rename_fields(self): # Test rename fields a = np.array( [(1, (2, [3.0, 30.0])), (4, (5, [6.0, 60.0]))], dtype=[("a", int), ("b", [("ba", float), ("bb", (float, 2))])], ) test = rename_fields(a, {"a": "A", "bb": "BB"}) newdtype = [("A", int), ("b", [("ba", float), ("BB", (float, 2))])] control = a.view(newdtype) assert_equal(test.dtype, newdtype) assert_equal(test, control)
def read_photometry_file(header_path): """Read simulated light-curves from file Files are expected in pairs of a header file (`*HEAD.fits`) with target meta data and a photometry file (`*PHOT.fits`) with simulated light-curves. Args: header_path (str): Path of the header file Returns: - An array of data from the header file - An array of data from the photometry file """ # Map of column names to names that snmachine/sncosmo will recognize column_name_mapping = { 'MJD': 'mjd', 'FLT': 'filter', 'FIELD': 'field', 'PHOTFLAG': 'photflag', 'PHOTPROB': 'photprob', 'FLUXCAL': 'flux', 'FLUXCALERR': 'flux_error', 'PSF_SIG1': 'psf_sig1', 'SKY_SIG': 'sky_sig', 'ZEROPT': 'zp', 'SIM_MAGOBS': 'sim_magobs' } # Data types to use when reading in the photometry file dtypes = [ ('MJD', '>f8'), ('FLT', 'U10'), ('FIELD', '|S12'), ('PHOTFLAG', '>i4'), ('PHOTPROB', '>f4'), ('FLUXCAL', '>f4'), ('FLUXCALERR', '>f4'), ('PSF_SIG1', '>f4'), ('SKY_SIG', '>f4'), ('ZEROPT', '>f4'), ('SIM_MAGOBS', '>f4') ] with fits.open(header_path) as header_hdulist: meta_data = header_hdulist[1].data phot_file_path = header_path.replace('HEAD', 'PHOT') with fits.open(phot_file_path) as photometry_hdulist: # Typecasting to an array avoids astropy bugs/performance issues phot_data = np.array(photometry_hdulist[1].data, dtype=dtypes) # Rename columns to sncosmo friendly format phot_data = rfn.rename_fields(phot_data, column_name_mapping) # Rename filters to sncosmo friendly format lowercase = np.char.strip(np.char.lower(phot_data['filter'])) phot_data['filter'] = np.char.add('lsst', lowercase) return meta_data, phot_data
def updateNames(self, rename: dict) -> None: datas = self.field("laserdata") for i in range(len(datas)): remove = [ name for name in datas[i].dtype.names if name not in rename ] datas[i] = rfn.drop_fields(datas[i], remove, usemask=False) datas[i] = rfn.rename_fields(datas[i], rename) self.setField("laserdata", datas) self.setElidedNames(datas[0].dtype.names)
def grl_loader(season): if isinstance(season.grl_path, list): grl = np.sort(np.array( np.concatenate([np.load(x) for x in season.grl_path])), order="run") else: grl = np.load(season.grl_path) # Check if bad runs are found in GRL try: if np.sum(~grl["good_i3"]) == 0: pass else: logger.error("Trying to load", season) logger.error("The following runs are included:") logger.error(grl[~grl["good_i3"]]) raise Exception("Runs marked as 'bad' are found in Good Run List") except ValueError: logger.warning( "No field called 'good_i3' found in GoodRunList. " "Cannot check if all runs in GoodRunList are actually good.") if "length" not in grl.dtype.names: if "livetime" in grl.dtype.names: grl = rename_fields(grl, {"livetime": "length"}) else: raise Exception("No recognised Livetime field found in " "GoodRunList. (Searched for 'livetime' and " "'length')") # Check if there are events in runs not found in GRL exp_data = season.get_exp_data() if "run" in exp_data.dtype.names: bad_runs = [x for x in set(exp_data["run"]) if x not in grl["run"]] if len(bad_runs) > 0: raise Exception( "Trying to use GoodRunList, but events in data have " "runs that are not included on this GoodRunList. \n" "Please check to make sure both the GoodRunList, " "and the event selection, are correct. \n" + "The following runs are affected: \n" + str(bad_runs)) # Sometimes, inexplicable, the runs come in random orders rather than # ascending order. This deals with that. grl = np.sort(grl, order="run") del exp_data return grl
def write_fom_targets(targets, FoM, desi_target, bgs_target, mws_target): """Return new targets array with added/renamed columns including ELG Figure of Merit Args: targets: numpy structured array of targets FoM: Figure of Merit calculated by apply_XD_globalerror desi_target: 1D array of target selection bit flags bgs_target: 1D array of target selection bit flags mws_target: 1D array of target selection bit flags Returns: New targets structured array with those changes Notes: Finalize target list by: * renaming OBJID -> BRICK_OBJID (it is only unique within a brick) * Adding new columns: - TARGETID: unique ID across all bricks - FoM: ELG XD Figure of Merit - DESI_TARGET: target selection flags - MWS_TARGET: target selection flags - BGS_TARGET: target selection flags """ ntargets = len(targets) assert ntargets == len(FoM) assert ntargets == len(desi_target) assert ntargets == len(bgs_target) assert ntargets == len(mws_target) #- OBJID in tractor files is only unique within the brick; rename and #- create a new unique TARGETID targets = rfn.rename_fields(targets, {'OBJID': 'BRICK_OBJID'}) targetid = targets['BRICKID'].astype( np.int64) * 1000000 + targets['BRICK_OBJID'] #- Add new columns: TARGETID, TARGETFLAG, NUMOBS targets = rfn.append_fields( targets, ['TARGETID', 'DESI_TARGET', 'BGS_TARGET', 'MWS_TARGET', 'FOM'], [targetid, desi_target, bgs_target, mws_target, FoM], usemask=False) io.write_targets('FoM.fits', targets, qso_selection='irrelevant', sandboxcuts=True) print('{} targets written to {}'.format(len(targets), 'FoM.fits')) return
def read_nest_output_device_data_from_ascii_to_dict(filepath): """This function reads data from a NEST recording device ascii file into an events dictionary Arguments: - filepath: absolute or relative path to the file (string) Returns: the events dictionary of the recorded data """ recarray = rename_fields( np.genfromtxt(filepath, names=True, skip_header=2), { "sender": "senders", "time_ms": "times" }) return {name: ensure_list(recarray[name]) for name in recarray.dtype.names}
def load_array(self, d, file): import time t0 = time.time() if self.params['has_sheared'] & (file == 'shapefile'): d['flags_1p'] = 'flags_select_1p' d['flags_1m'] = 'flags_select_1m' d['flags_2p'] = 'flags_select_2p' d['flags_2m'] = 'flags_select_2m' if self.params['pdf_type'] == 'pdf': keys = [ key for key in d.keys() if (d[key] is not None) & (key is not 'pzstack') ] else: keys = [key for key in d.keys() if (d[key] is not None)] if 'objid' in keys: dtypes = [('objid', 'i8')] else: raise ValueError('missing object id in ' + file) dtypes += [(key, 'f8') for key in keys if (key is not 'objid')] if self.params['pdf_type'] == 'pdf': dtypes += [('pzstack_' + str(i), 'f8') for i in range(len(self.params['pdf_z']))] fits = fio.FITS(self.params[file])[-1] array = fits.read(columns=[d[key] for key in keys]) array = rename_fields(array, {v: k for k, v in d.iteritems()}) if ('weight' not in array.dtype.names) & (file == 'shapefile'): array = append_fields(array, 'weight', np.ones(len(array)), usemask=False) if self.params['pdf_type'] == 'pdf': for i in range(len(self.params['pdf_z'])): array['pzstack' + str(i)] = fits.read(columns=d['pzstack'] + str(i)) if np.any(np.diff(array['objid']) < 1): raise ValueError('misordered or duplicate ids in ' + file) return array
def test_Osiris_Dev_Hdf5_ParticleFile_properties( make_prt_file: Callable[[str, np.ndarray, Optional[str]], Path]): data = unstructured_to_structured(np.random.random((10, 4))) data = rename_fields(data, {"f0": "q"}) prt_path = make_prt_file("osiris_dev_particles_hdf5", data, name="some particles") backend = Osiris_Dev_Hdf5_ParticleFile(prt_path) assert backend.name == "osiris_dev_particles_hdf5" assert backend.location == prt_path assert backend.dataset_name == "some_particles" assert backend.dataset_label == "some particles" assert backend.quantity_names == ["f1", "f2", "f3", "q"] assert backend.quantity_labels == [ "f1 label", "f2 label", "f3 label", "q label", ] assert backend.quantity_units == [ "f1 unit", "f2 unit", "f3 unit", "q unit", ] assert backend.shape == (10, ) assert backend.dtype == np.dtype([ ("f1", float), ("f2", float), ("f3", float), ("q", float), ]) # taken function 'make_osiris_444_particles_hdf' assert backend.iteration == 12345 assert np.isclose(backend.time_step, -321.9) assert backend.time_unit == "time unit" # check reading of data for indexing in (np.s_[0], np.s_[-1], np.s_[:], np.s_[3:7], np.s_[4:1]): expected_data = data[indexing] np.testing.assert_array_equal(backend.get_data((indexing, )), expected_data)
def parse_data(): logger = logging.Logger("default_logger") logger.setLevel("DEBUG") with fits.open(os.path.join(alertstack_data_dir, "table-4LAC-DR2-h.fits")) as hdul: # table_4LAC cat = hdul[1].data cat = np.sort(cat, order="Energy_Flux100")[::-1]#Energy_Flux100 logging.info("Selecting blazars from 4FGL catalogue") blazar_class = ["bll", "BLL", "fsrq", "FSRQ", "bcu", "BCU"] logging.info("Using all sources from class {0}".format(blazar_class)) # mask = np.array([x["CLASS"] in blazar_class for x in cat]) blazars = np.array(cat[mask]) cut_e = -11.6 mask_e = np.array([x["Energy_Flux100"]>10**cut_e for x in blazars]) blazars = np.array(blazars[mask_e]) maps = [ ("RAJ2000", "ra_rad"), ("DEJ2000", "dec_rad"), ] for (old_key, new_key) in maps: blazars = rename_fields(blazars, {old_key: new_key}) mask_GP = [is_outside_GP(blazars['ra_rad'][i],blazars['dec_rad'][i]) for i in range(len(blazars))] blazars = blazars[mask_GP] #################### #fluxes = blazars['Energy_Flux100'].byteswap().newbyteorder() #blazar_df = pd.DataFrame({'flux': fluxes}) #blazar_df['bins'] = pd.cut(blazar_df['flux'],50) #blazar_df2 = blazar_df.groupby('bins').agg({'flux': sum, 'bins': 'count'}).rename(columns = {'bins': 'count', 'flux':'background'}).reset_index() #blazar_df2['flux2'] = (len(blazars)/sum(blazar_df2['background']))*blazar_df2['background'] #blazar_df2['s/b'] = blazar_df2['flux2']/(blazar_df2['count']) #blazar_df = blazar_df.merge(blazar_df2,how='left',on='bins') #blazars = np.lib.recfunctions.append_fields(blazars, 's/b', blazar_df['s/b'].values) logging.info("Found {0} sources in total".format(len(blazars))) return blazars
def load_labels(fname='cocotools/coords/standard_labels.csv', usecols=None): """Load the standard labels file; return a record array with the atlas. """ # Converters for the fields we're interested in: strings without whitespace # and floats cleanstr = lambda x: str(x).strip() conv = { 0: cleanstr, 1: cleanstr, 2: cleanstr, 3: cleanstr, 4: float, 5: float, 6: float, 7: cleanstr, 8: cleanstr, 9: int } # Exclude comments column by default. usecols = range(7) + [8] if usecols is None else usecols with open(fname) as f: # Read the first line for the names all_names = [n.strip() for n in f.next().split(',')] # genfromtxt is buggy, it gets confused with commas inside strings # use the stdlib csv reader and rebuild lines with | as separator, # which genfromtxt can then use data = ['|'.join(x) for x in list(csv.reader(f))] # Older versions of numpy need an actual filehandle, not an arbitrary # iterable. with tempfile.TemporaryFile() as fdata: fdata.write('\n'.join(data)) fdata.seek(0) # Now, let genfromtxt iterate over the rest of the file araw = np.genfromtxt(fdata, delimiter='|', usecols=usecols, converters=conv) # Rename the fields so the dtype has more useful names than f0, f1, etc. # Also make it a recarray for more convenient use further down names = [all_names[i] for i in usecols] renamer = dict(zip(araw.dtype.names, names)) atlas = rfn.rename_fields(araw, renamer).view(np.recarray) # Flip y axis to have frontal areas on the left in axial projections #atlas.y *= -1 return atlas
def filter_on_fields(to_filter, for_filter, filter_fields, filter_fields_2=None, return_selection=False): """Returns entries of to_filter whose combination of the filter_fields values are present in for_filter. filter_fields_2: names of filter_fields in for_filter (if different than in to_filter) If return_selection, will instead return boolean selection array for to_filter """ a = np.array(fields_view(to_filter, filter_fields)) if filter_fields_2 is None: filter_fields_2 = filter_fields b = np.array(fields_view(for_filter, filter_fields_2)) # Rename the fields, if needed # If only one field is selected, this won't be needed (and would return None instead of working) if not isinstance(filter_fields, str) and len(filter_fields) > 1: b = recfunctions.rename_fields(b, dict(zip(filter_fields_2, filter_fields))) selection = np.in1d(a, b) if return_selection: return selection else: return to_filter[selection]
def test_complex_dataset(self): """ Tests for a dataset containing recorded data for a multiple configurations. """ # define multiple configurations for one dataset self.mod.knobs.n_configs = 3 # -- dset with 1 shotnum ---- self.mod.knobs.sn_size = 1 self.assertInRangeSN() self.assertOutRangeSN() # -- typical dset with sequential shot numbers ---- self.mod.knobs.sn_size = 100 self.assertInRangeSN() self.assertOutRangeSN() # -- dset with non-sequential shot numbers ---- self.mod.knobs.sn_size = 100 data = self.cgroup["Run time list"][...] sn_arr = np.append( np.arange(5, 25, dtype=np.uint32), np.append(np.arange(51, 111, dtype=np.uint32), np.arange(150, 170, dtype=np.uint32)), ) data["Shot number"][0::3] = sn_arr data["Shot number"][1::3] = sn_arr data["Shot number"][2::3] = sn_arr del self.cgroup["Run time list"] self.cgroup.create_dataset("Run time list", data=data) self.assertInRangeSN() self.assertOutRangeSN() # -- dset without a configuration fields ---- self.mod.knobs.sn_size = 50 data = self.cgroup["Run time list"][...] data = rfn.rename_fields(data, {"Configuration name": "oops"}) del self.cgroup["Run time list"] self.cgroup.create_dataset("Run time list", data=data) cdset = self.cgroup["Run time list"] with self.assertRaises(ValueError): build_shotnum_dset_relation(np.empty(5, dtype=np.uint32), cdset, "Shot number", self.map, "config01")
def rename(array: np.ndarray, original: Union[list, str], new: Union[list, str]) -> np.ndarray: """ Rename the structured array. :param array: np.ndarray(structured), array that houses the columns/fields to be renamed. :param original: list or str, the original column/s to be renamed. :param new: list or str, the new column/s names. :return : np.ndarray(structured), the renamed structured array. """ if (type(original) == str): original = [original] new = [new] mapping = {} for ori, ne in zip(original, new): mapping[ori] = ne return rfn.rename_fields(array, mapping)
def make_atlas(kind=pht00, usecols=None): """Load the standard labels files returning a record array with the atlas. """ # Converters for the fields we're interested in: strings without whitespace # and floats fname = kind['fname'] if fname.endswith('txt'): from matplotlib import mlab return mlab.csv2rec(fname, delimiter='\t') conv = kind['converters'] usecols = kind['cols'] if usecols is None else usecols with open(fname) as f: # Read the first line for the names all_names = [n.strip() for n in f.next().split(',')] # genfromtxt is buggy, it gets confused with commas inside strings # use the stdlib csv reader and rebuild lines with | as separator, # which genfromtxt can then use data = ['|'.join(x) for x in list(csv.reader(f))] # Older versions of numpy need an actual filehandle, not an arbitrary # iterable. with tempfile.TemporaryFile() as fdata: fdata.write('\n'.join(data)) fdata.seek(0) # Now, let genfromtxt iterate over the rest of the file araw = np.genfromtxt(fdata, delimiter='|', usecols=usecols, converters=conv) # Rename the fields so the dtype has more useful names than f0, f1, etc. # Also make it a recarray for more convenient use further down names = [all_names[i] for i in usecols] renamer = dict(zip(araw.dtype.names, names)) atlas = rfn.rename_fields(araw, renamer).view(np.recarray) # Flip y axis to have frontal areas on the left in axial projections #atlas.y *= -1 return atlas
def test_add_start_and_length_modifications(self): """test add_raw_start_and_raw_length_to_events and add_start_and_length_to_events methods""" # get input data sampling_freq = self.dna_handle.sample_rate start_time = self.dna_handle.raw_attributes['start_time'] event_table = self.dna_handle.get_basecall_data() self.assertTrue(check_event_table_time(event_table), "Invalid initial start times") # add raw fields event_table = add_raw_start_and_raw_length_to_events( event_table, start_time, sampling_freq) raw_starts = event_table['raw_start'].tolist() raw_lengths = event_table['raw_length'].tolist() # save old fields event_table = rename_fields(event_table, { 'start': 'original_start', 'length': 'original_length' }) # add non-raw fields event_table = add_start_and_length_to_events(event_table, start_time, sampling_freq) self.assertTrue( check_event_table_time(event_table, min_difference=1.0 / sampling_freq), "Invalid modified start times") # get fields original_starts = event_table['original_start'].tolist() original_lengths = event_table['original_length'].tolist() starts = event_table['start'].tolist() lengths = event_table['length'].tolist() # compare elementwise places = int(math.log10(sampling_freq)) + 1 for original_start, start in zip(original_starts, starts): self.assertAlmostEqual(original_start, start, places=places) for original_length, length in zip(original_lengths, lengths): self.assertAlmostEqual(original_length, length, places=places)
def finalize(targets, desi_target, bgs_target, mws_target, sky=0): """Return new targets array with added/renamed columns Args: targets: numpy structured array of targets kwargs: colname=array of columns to add desi_target: 1D array of target selection bit flags bgs_target: 1D array of target selection bit flags mws_target: 1D array of target selection bit flags sky: Pass 1 to indicate these are blank sky targets Returns new targets structured array with those changes Finalize target list by: * renaming OBJID -> BRICK_OBJID (it is only unique within a brick) * Adding new columns: - TARGETID: unique ID across all bricks - LVM_TARGET: target selection flags - MWS_TARGET: target selection flags - BGS_TARGET: target selection flags """ ntargets = len(targets) assert ntargets == len(desi_target) assert ntargets == len(bgs_target) assert ntargets == len(mws_target) #- OBJID in tractor files is only unique within the brick; rename and #- create a new unique TARGETID targets = rfn.rename_fields(targets, {'OBJID':'BRICK_OBJID'}) targetid = encode_targetid(objid=targets['BRICK_OBJID'], brickid=targets['BRICKID'], release=targets['RELEASE'], sky=sky) #- Add new columns: TARGETID, TARGETFLAG, NUMOBS targets = rfn.append_fields(targets, ['TARGETID', 'LVM_TARGET', 'BGS_TARGET', 'MWS_TARGET'], [targetid, desi_target, bgs_target, mws_target], usemask=False) return targets
def GetCOSMOS23(maxdist=10.0): morph = esutil.io.read('cosmos_morph_cassata_1.1.fits') morph = rec.rename_fields(morph, {'RA':'ra', 'DEC':'dec'}) cuts = GetFiducialCuts() cut = XYCuts(morph['MAG_AUTO_ACS'], np.log10(morph['R_HALF']*0.03), cuts) morph = morph[cut] m1 = np.zeros( (len(morph),2) ) m1[:,0] = morph['ra'] m1[:,1] = morph['dec'] np.savetxt('morph-pre-mask.txt', m1) #enrique = np.loadtxt('zCOSMOS-rndnb104Ia20.004-22.525.023.024.0zCOSMOS-Bext3.dat') #e = np.zeros( len(enrique), dtype=[('ra',np.float32), ('dec',np.float32)]) #e['ra'] = enrique[:,2] #e['dec'] = enrique[:,1] enrique = np.loadtxt('zCOSMOS-mask.dat') e = np.zeros( len(enrique), dtype=[('ra',np.float32), ('dec',np.float32)]) e['ra'] = enrique[:,1] e['dec'] = enrique[:,0] masked_morph = ApplyMaskRandoms(morph, e, cat_ra='ra', cat_dec='dec', rand_ra='ra', rand_dec='dec', maxdist=maxdist) GeoCuts = GetGeometryCuts() mcut = XYCuts(masked_morph['ra'], masked_morph['dec'], GeoCuts) masked_morph = masked_morph[mcut] m2 = np.zeros( (len(masked_morph),2) ) m2[:,0] = masked_morph['ra'] m2[:,1] = masked_morph['dec'] np.savetxt('morph-post-mask.txt', m2) ecut = XYCuts(e['ra'], e['dec'], GeoCuts) e = e[ecut] u = UniformRandom(GeoCuts) u = ApplyMaskRandoms(u, e, cat_ra='ra', cat_dec='dec', rand_ra='ra', rand_dec='dec', maxdist=maxdist) return masked_morph, e, u
def parse_data(): logger = logging.Logger("default_logger") logger.setLevel("DEBUG") with fits.open(os.path.join(alertstack_data_dir, "table_4LAC.fits")) as hdul: cat = hdul[1].data cat = np.sort(cat, order="Flux1000")[::-1] logging.info("Selecting blazars from 4FGL catalogue") blazar_class = ["bll", "BLL", "fsrq", "FSRQ", "bcu", "BCU"] logging.info("Using all sources from class {0}".format(blazar_class)) # mask = np.array([x["CLASS"] in blazar_class for x in cat]) blazars = np.array(cat[mask]) cut_e = -11.6 mask_e = np.array([x["Energy_Flux100"] > 10**cut_e for x in blazars]) blazars = np.array(blazars[mask_e]) maps = [ ("RAJ2000", "ra_rad"), ("DEJ2000", "dec_rad"), ] for (old_key, new_key) in maps: blazars = rename_fields(blazars, {old_key: new_key}) mask_GP = [ is_outside_GP(blazars['ra_rad'][i], blazars['dec_rad'][i]) for i in range(len(blazars)) ] blazars = blazars[mask_GP] logging.info("Found {0} sources in total".format(len(blazars))) return blazars
def finalize(targets, desi_target, bgs_target, mws_target): """Return new targets array with added/renamed columns Args: targets: numpy structured array of targets kwargs: colname=array of columns to add desi_target: 1D array of target selection bit flags bgs_target: 1D array of target selection bit flags mws_target: 1D array of target selection bit flags Returns new targets structured array with those changes Finalize target list by: * renaming OBJID -> BRICK_OBJID (it is only unique within a brick) * Adding new columns: - TARGETID: unique ID across all bricks - DESI_TARGET: target selection flags - MWS_TARGET: target selection flags - BGS_TARGET: target selection flags """ ntargets = len(targets) assert ntargets == len(desi_target) assert ntargets == len(bgs_target) assert ntargets == len(mws_target) #- OBJID in tractor files is only unique within the brick; rename and #- create a new unique TARGETID targets = rfn.rename_fields(targets, {'OBJID':'BRICK_OBJID'}) targetid = targets['BRICKID'].astype(np.int64)*1000000 + targets['BRICK_OBJID'] #- Add new columns: TARGETID, TARGETFLAG, NUMOBS targets = rfn.append_fields(targets, ['TARGETID', 'DESI_TARGET', 'BGS_TARGET', 'MWS_TARGET'], [targetid, desi_target, bgs_target, mws_target], usemask=False) return targets
bdir = '/gpfs01/astro/workarea/esuchyta/git-repos/BalrogDirs/2015-Nov/BalrogDB/%s'%(name) dfile = os.path.join(bdir, '%s-des.fits'%(name)) outdir = os.path.join(ddir, name) if MPI.COMM_WORLD.Get_rank()==0: if not os.path.exists(outdir): os.makedirs(outdir) if not os.path.exists(zzfile): if MPI.COMM_WORLD.Get_rank()==0: z = fitsio.read(zfile, ext=-1) z = np.sort(z, order='COADD_OBJECTS_ID') names = {} for n in z.dtype.names: names[n] = n.lower() z = rec.rename_fields(z, names) f = fitsio.FITS(zzfile, 'rw') f.write(z) if MPI.COMM_WORLD.Get_rank()==0: size = fitsio.read_header(zzfile, ext=-1)['NAXIS2'] z = fitsio.read(zzfile, ext=-1, columns=['COADD_OBJECTS_ID']) num = size/inc print num starts = np.arange(num)*inc ends = starts + inc mod = size%inc if mod != 0: starts = np.append(starts, ends[-1]) ends = np.append(ends, ends[-1]+mod)
def get_triggers(channel, etg, segments, cache=None, snr=None, frange=None, columns=None, raw=False, **kwargs): """Get triggers for the given channel """ # get table from etg try: Table = TABLE[etg.lower()] except KeyError as e: e.args = ('Unknown ETG %r, cannot map to LIGO_LW Table class' % etg,) raise tablename = strip_table_name(Table.tableName) # get default columns for this table if columns is None: for key in COLUMNS: if issubclass(Table, key): columns = COLUMNS[key][:] break if 'channel' in columns: columns.pop('channel') # find triggers if cache is None: cache = find_trigger_files(channel, etg, segments, **kwargs) # read cache trigs = lsctables.New(Table, columns=columns) cache = cache.unique() cache.sort(key=lambda x: x.segment[0]) for segment in segments: if len(cache.sieve(segment=segment)): if tablename.endswith('_inspiral'): filt = lambda t: float(t.get_end()) in segment else: filt = lambda t: float(t.get_peak()) in segment trigs.extend(Table.read(cache.sieve(segment=segment), filt=filt)) # format table as numpy.recarray recarray = trigs.to_recarray(columns=columns) # filter if snr is not None: recarray = recarray[recarray['snr'] >= snr] if tablename.endswith('_burst') and frange is not None: recarray = recarray[ (recarray['peak_frequency'] >= frange[0]) & (recarray['peak_frequency'] < frange[1])] # return basic table if 'raw' if raw: return recarray # otherwise spend the rest of this function converting functions to # something useful for the hveto core analysis addfields = {} dropfields = [] # append channel to all events columns.append('channel') addfields['channel'] = numpy.repeat(channel, recarray.shape[0]) # rename frequency column if tablename.endswith('_burst'): recarray = recfunctions.rename_fields( recarray, {'peak_frequency': 'frequency'}) idx = columns.index('peak_frequency') columns.pop(idx) columns.insert(idx, 'frequency') # map time to its own column if tablename.endswith('_inspiral'): tcols = ['end_time', 'end_time_ns'] elif tablename.endswith('_burst'): tcols = ['peak_time', 'peak_time_ns'] else: tcols = None if tcols: times = recarray[tcols[0]] + recarray[tcols[1]] * 1e-9 addfields['time'] = times dropfields.extend(tcols) columns = ['time'] + columns[2:] # add and remove fields as required if addfields: names, data = zip(*addfields.items()) recarray = recfunctions.rec_append_fields(recarray, names, data) recarray = recfunctions.rec_drop_fields(recarray, dropfields) return recarray[columns]
radec = s2dec('01 10 16.2','-02 18 50') q0107b = rf.rec_append_fields(q0107b,'RA' ,radec[0]*np.ones(len(q0107b))) q0107b = rf.rec_append_fields(q0107b,'DEC',radec[1]*np.ones(len(q0107b))) radec = s2dec('01 10 14.52','-02 16 57.5') q0107c = rf.rec_append_fields(q0107c,'RA' ,radec[0]*np.ones(len(q0107c))) q0107c = rf.rec_append_fields(q0107c,'DEC',radec[1]*np.ones(len(q0107c))) radec = s2dec('10 22 18.99','01 32 18.8') q1022 = rf.rec_append_fields(q1022,'RA' ,radec[0]*np.ones(len(q1022))) q1022 = rf.rec_append_fields(q1022,'DEC',radec[1]*np.ones(len(q1022))) #read galaxies VVDSF10 = readtxt('/home/ntejos/catalogs/VVDS/VVDS_F10.cat',readnames=True) VVDSF14 = readtxt('/home/ntejos/catalogs/VVDS/VVDS_F14.cat',readnames=True) VVDSF22 = readtxt('/home/ntejos/catalogs/VVDS/VVDS_F22.cat',readnames=True) VVDSF10 = rf.rename_fields(VVDSF10,{'Z':'ZGAL','MAG_AUTO_I':'MAG','ALPHA_J2000':'RA','DELTA_J2000':'DEC'}) VVDSF14 = rf.rename_fields(VVDSF14,{'Z':'ZGAL','MAG_AUTO_I':'MAG','ALPHA_J2000':'RA','DELTA_J2000':'DEC'}) VVDSF22 = rf.rename_fields(VVDSF22,{'Z':'ZGAL','MAG_AUTO_I':'MAG','ALPHA_J2000':'RA','DELTA_J2000':'DEC'}) names = 'RA,DEC,ZGAL,MAG' usecols = [0,1,2,3] VIMOSQ1 = readtxt('/home/ntejos/catalogs/Q0107/VIMOS.txt',names=names,usecols=usecols,comment='#') DEIMOSQ1 = readtxt('/home/ntejos/catalogs/Q0107/DEIMOS.txt',names=names,usecols=usecols,comment='#') CFHTQ1 = readtxt('/home/ntejos/catalogs/Q0107/CFHT.txt',names=names,usecols=usecols,comment='#') GMOSQ1 = readtxt('/home/ntejos/catalogs/Q0107/GMOS.txt',names=names,usecols=usecols,comment='#') VIMOSF1005 = readtxt('/home/ntejos/catalogs/J1005/VIMOS.txt',names=names,usecols=usecols,comment='#') VIMOSF1022 = readtxt('/home/ntejos/catalogs/J1022/VIMOS.txt',names=names,usecols=usecols,comment='#') #clean galaxy catalog galaxies=[VVDSF10,VVDSF14,VVDSF22,VIMOSF1005,VIMOSF1022]
def GetCOSMOS21(maxdist=10.0, usemorph=False, jfile=None, hfile=None, usepz=False): if hfile is not None: morph = esutil.io.read(hfile) morph = rec.rename_fields(morph, {'RA':'ra', 'DEC':'dec'}) elif jfile is not None: morph = esutil.io.read(jfile) print len(morph) elif usemorph: morph = esutil.io.read('cosmos_morph_cassata_1.1.fits') morph = rec.rename_fields(morph, {'RA':'ra', 'DEC':'dec'}) elif usepz: morph = esutil.io.read('cosmos_zphot_mag25.fits') morph = rec.rename_fields(morph, {'imag':'i_mag', 'rmag':'r_mag'}) cut = (morph['auto_flag'] > -1) else: morph = esutil.io.read('cosmos_phot_20060103.fits') morph = rec.rename_fields(morph, {'RA':'ra', 'DEC':'dec'}) #cut = (morph['star']==0) & (morph['auto_flag']==1) & (morph['i_mask']==0) #cut = (morph['star']==0) & (morph['auto_flag']==1) & (morph['i_mask']==0) & (morph['blend_mask']==0) #cut = (morph['star']==0) & (morph['auto_flag']==1) & (morph['blend_mask']==0) & (morph['i_mask']==0) & (morph['z_mask']==0) & (morph['V_mask']==0) & (morph['B_mask']==0) cut = (morph['blend_mask'] == 0) & (morph['star'] == 0) & (morph['auto_flag'] > -1) #cut = (morph['star']==0) & (morph['i_mask']==0) morph = morph[cut] ''' fig, ax = plt.subplots(1,1) ax.scatter(morph['ra'], morph['dec'], s=0.2, lw=0) plt.show() sys.exit() ''' ''' cuts = GetBrightCuts() #cut = XYCuts(morph['MAG_AUTO_ACS'], np.log10(morph['R_HALF']*0.03), cuts) cut = XYCuts(morph['i_mag_auto'], morph['i_mag_auto'], cuts) morph = morph[cut] ''' enrique = np.loadtxt('zCOSMOS-mask.dat') e = np.zeros( len(enrique), dtype=[('ra',np.float32), ('dec',np.float32)]) e['ra'] = enrique[:,1] e['dec'] = enrique[:,0] """ masked_morph = ApplyMaskRandoms(morph, e, cat_ra='ra', cat_dec='dec', rand_ra='ra', rand_dec='dec', maxdist=maxdist) if jfile is not None: GeoCuts = GetEGCuts() mcut = XYCuts(masked_morph['ra'], masked_morph['dec'], GeoCuts) masked_morph = masked_morph[mcut] r = RectArea() u = UniformRandom(r) u = ApplyMaskRandoms(u, e, cat_ra='ra', cat_dec='dec', rand_ra='ra', rand_dec='dec', maxdist=maxdist) elif usemorph: GeoCuts = GetGeometryCuts() mcut = XYCuts(masked_morph['ra'], masked_morph['dec'], GeoCuts) masked_morph = masked_morph[mcut] ecut = XYCuts(e['ra'], e['dec'], GeoCuts) e = e[ecut] u = UniformRandom(GeoCuts) u = ApplyMaskRandoms(u, e, cat_ra='ra', cat_dec='dec', rand_ra='ra', rand_dec='dec', maxdist=maxdist) else: r = RectArea() u = UniformRandom(r) u = ApplyMaskRandoms(u, e, cat_ra='ra', cat_dec='dec', rand_ra='ra', rand_dec='dec', maxdist=maxdist) return masked_morph, e, u #return masked_morph, e """ if hfile is not None: GeoCuts = GetEGCuts() elif jfile is not None: GeoCuts = GetEGCuts() elif usemorph: GeoCuts = GetGeometryCuts() else: GeoCuts = GetEGCuts() mcut = XYCuts(morph['ra'], morph['dec'], GeoCuts) morph = morph[mcut] u = UniformRandom(GeoCuts) u = ApplyMaskRandoms(u, e, cat_ra='ra', cat_dec='dec', rand_ra='ra', rand_dec='dec', maxdist=maxdist) morph = ApplyMaskRandoms(morph, e, cat_ra='ra', cat_dec='dec', rand_ra='ra', rand_dec='dec', maxdist=maxdist) return morph, e, u