def download_gaia_hip() -> None: """Download HIP data from the Gaia archive.""" hip_file = GAIA_DIR/'gaiadr2_hip-result.csv' if not proceed_checkfile(hip_file): return conesearch_file = GAIA_DIR/'hip2conesearch.zip' if proceed_checkfile(conesearch_file): download_file(conesearch_file, CONESEARCH_URL) # the gaiadr2.hipparcos2_best_neighbour table misses a large number of HIP stars that are # actually present, so use the mapping from Kervella et al. (2019) "Binarity of Hipparcos # stars from Gaia pm anomaly" instead. with open_cds_tarfile(VIZIER_DIR/'hipgpma.tar.gz') as tf: hip_map = unique(tf.read_gzip('hipgpma.dat', ['HIP', 'GDR2'])) with ZipFile(conesearch_file, 'r') as csz: with csz.open('Hipparcos2GaiaDR2coneSearch.csv', 'r') as f: cone_map = io_ascii.read( f, format='csv', names=['HIP', 'GDR2', 'dist'], include_names=['HIP', 'GDR2'], ) cone_map = unique(cone_map) hip_map = join(hip_map, cone_map, join_type='outer', keys='HIP', table_names=['pm', 'cone']) hip_map['GDR2'] = hip_map['GDR2_pm'].filled(hip_map['GDR2_cone']) hip_map.remove_columns(['GDR2_pm', 'GDR2_cone']) hip_map.rename_column('HIP', 'original_ext_source_id') hip_map.rename_column('GDR2', 'source_id') download_gaia_data('hip_id', hip_map, hip_file)
def load_tyc_hd() -> Table: """Load the Tycho-HD cross index.""" print('Loading TYC-HD cross index') with tarfile.open(os.path.join('vizier', 'tyc2hd.tar.gz'), 'r:gz') as tf: with tf.extractfile('./ReadMe') as readme: col_names = ['TYC1', 'TYC2', 'TYC3', 'HD'] reader = io_ascii.get_reader(io_ascii.Cds, readme=readme, include_names=col_names) reader.data.table_name = 'tyc2_hd.dat' with tf.extractfile('./tyc2_hd.dat.gz') as gzf, gzip.open(gzf, 'rb') as f: data = reader.read(f) parse_tyc_cols(data) err_del = np.array(TYC_HD_ERRATA['delete'] + [a[1] for a in TYC_HD_ERRATA['add']]) data = data[np.logical_not(np.isin(data['HD'], err_del))] err_add = Table(np.array(TYC_HD_ERRATA['add']), names=['TYC', 'HD'], dtype=[np.int64, np.int64]) data = vstack([data, err_add], join_type='exact') data = unique(data.group_by('HD'), keys='TYC') data = unique(data.group_by('TYC'), keys='HD') return data
def _append(self, frame, unique=True): vstack_t = at.vstack([self._t, frame._t]) vstack_mods_t = at.vstack([self._mods_t, frame._mods_t]) if unique: self._t = at.unique(vstack_t, keys=['z0', 'z']) self._mods_t = at.unique(vstack_mods_t, keys=['z0']) #print(self._mods_t['z0', 'id']) return 0
def configmap(infilelist, confitemlist, debug='False'): """general purpose mapper of observing configurations Parameters ---------- infilelist: list List of filenames confitemlist: list List of header keywords which define a configuration Returns ------- obs_i: numpy 1d array observation number (unique object, config) for each file config_i: numpy 1d array config number (unique config) for each file obstab: astropy Table object name and config number for each observation configtab: astropy Table config items for each config """ # create the observation log obsdict = obslog(infilelist) images = len(infilelist) # make table of unique configurations confdatlisti = [] for i in range(images): confdatlist = [] for item in confitemlist: confdatlist.append(obsdict[item][i]) confdatlisti.append(confdatlist) dtypelist = map(type, confdatlist) configtab = Table(np.array(confdatlisti), names=confitemlist, dtype=dtypelist) config_i = np.array([np.where(configtab[i]==unique(configtab)) \ for i in range(images)]).flatten() configtab = unique(configtab) # make table of unique observations obsdatlisti = [] for i in range(images): object = obsdict['OBJECT'][i].replace(' ', '') obsdatlisti.append([object, config_i[i]]) obstab = Table(np.array(obsdatlisti), names=['object', 'config'], dtype=[str, int]) obs_i = np.array([np.where(obstab[i]==unique(obstab)) \ for i in range(images)]).flatten() obstab = unique(obstab) return obs_i, config_i, obstab, configtab
def find_best_comp(self, result_file_path=None, best_comparison_star=None): result_file = Table.read(result_file_path, format='ascii.commented_header') # read comparison star list # and check manual assigned comp star if best_comparison_star is None: result_unique_by_cat = table.unique(result_file, keys='nomad1') else: result_unique_by_cat = table.unique( result_file[(result_file['nomad1'] == best_comparison_star)], keys='nomad1') std_list = [] t_c_list = [] # calculates diff_mag for all target objects and comp. stars for star in result_unique_by_cat['nomad1']: frame_results = result_file[(result_file['nomad1'] == star)] # diff phot. frame_results[ 't-c'] = frame_results['magt_i'] - frame_results['magc_i'] # error propagation frame_results['t-c-err'] = np.sqrt( np.power(frame_results['magt_i_err'], 2) + np.power(frame_results['magc_i_err'], 2)) # extracting usefull columns t_c_list.append(frame_results['ast_num', 'nomad1', 'jd', 't-c', 't-c-err']) # calculating all t-c stars STD then adding list std_list.append(np.std(frame_results['t-c'])) # calculating all STD's mean and its index number in the list mean_idx = (np.abs(np.asanyarray(std_list) - np.mean(std_list))).argmin() # choosing STD with min, mean and max stars diff_stats = { 'min': [std_list.index(min(std_list)), min(std_list)], 'mean': [mean_idx, np.mean(std_list)], 'max': [std_list.index(max(std_list)), max(std_list)] } # getting these diff mags and their other columns results = { 'with_min_comp': t_c_list[diff_stats['min'][0]], 'with_mean_comp': t_c_list[diff_stats['mean'][0]], 'with_max_comp': t_c_list[diff_stats['max'][0]] } return results
def download_galaxies_without_cuts(galaxies, nsa, joint_loc, png_dir, fits_dir): """[summary] Args: galaxies ([type]): [description] nsa ([type]): [description] joint_loc ([type]): save location for new joint catalog """ # let's redownload all, without filtering - make temp joint catalog galaxies_with_nsa_maybe_duplicate, _ = matching_utils.match_galaxies_to_catalog_table( galaxies=galaxies, catalog=nsa, galaxy_suffix='', catalog_suffix='_nsa') # if duplicate match to NSA catalog, pick the first galaxies_with_nsa = table.unique(galaxies_with_nsa_maybe_duplicate, keys='iauname', keep='first') logging.warning( 'Dropped {} galaxies that matched to the same NSA entry'.format( len(galaxies_with_nsa_maybe_duplicate) - len(galaxies_with_nsa))) assert len(table.unique(galaxies_with_nsa, keys='iauname')) == len(galaxies_with_nsa) logging.info('In NSA: {}'.format(len(galaxies_with_nsa))) in_decals_bricks = get_joint_nsa_decals_catalog.create_joint_catalog( nsa=galaxies_with_nsa, bricks=bricks, data_release='5') # dont apply selection cuts assert len(table.unique(in_decals_bricks, keys='iauname')) == len(in_decals_bricks) for dir in [png_dir, fits_dir]: if not os.path.isdir(dir): os.mkdir(dir) joint_catalog = download_images_threaded.download_images_multithreaded( in_decals_bricks, '5', fits_dir, png_dir, overwrite_fits=False, overwrite_png=False) logging.info('Downloaded {} galaxies without cuts'.format(joint_catalog)) image_download_stats(joint_catalog) joint_catalog['iauname'] = list( map(lambda x: str(x), joint_catalog['iauname'])) # avoid dtype problems joint_catalog.write( joint_loc, overwrite=False) # not allowed to overwrite, for safety logging.info('Written new joint catalog to {} for uploader'.format( joint_catalog_loc))
def configmap(infilelist,confitemlist,debug='False'): """general purpose mapper of observing configurations Parameters ---------- infilelist: list List of filenames confitemlist: list List of header keywords which define a configuration Returns ------- obs_i: numpy 1d array observation number (unique object, config) for each file config_i: numpy 1d array config number (unique config) for each file obstab: astropy Table object name and config number for each observation configtab: astropy Table config items for each config """ # create the observation log obsdict = obslog(infilelist) images = len(infilelist) # make table of unique configurations confdatlisti = [] for i in range(images): confdatlist = [] for item in confitemlist: confdatlist.append(obsdict[item][i]) confdatlisti.append(confdatlist) dtypelist = map(type,confdatlist) configtab = Table(np.array(confdatlisti),names=confitemlist,dtype=dtypelist) config_i = np.array([np.where(configtab[i]==unique(configtab)) \ for i in range(images)]).flatten() configtab = unique(configtab) # make table of unique observations obsdatlisti = [] for i in range(images): object = obsdict['OBJECT'][i].replace(' ','') obsdatlisti.append([object, config_i[i]]) obstab = Table(np.array(obsdatlisti),names=['object','config'],dtype=[str,int]) obs_i = np.array([np.where(obstab[i]==unique(obstab)) \ for i in range(images)]).flatten() obstab = unique(obstab) return obs_i,config_i,obstab,configtab
def load_sao() -> Table: """Loads the SAO catalog.""" print("Loading SAO") with (VIZIER_DIR/'sao.readme').open('r') as readme: reader = WorkaroundCDSReader('sao.dat', ['SAO', 'HD'], [np.int64, np.int64], readme) with gzip.open(VIZIER_DIR/'sao.dat.gz', 'rt', encoding='ascii') as f: data = reader.read(f) data = unique(data.group_by('SAO'), keys=['HD']) data = unique(data.group_by('HD'), keys=['SAO']) return data
def __add__(self, other): if not isinstance(other, type(self)): raise TypeError( f'Products must of same type not {type(self)} and {type(other)}' ) other.control[ 'index'] = other.control['index'] + self.control['index'].max() + 1 other.data['control_index'] = other.data[ 'control_index'] + self.control['index'].max() + 1 control = vstack((self.control, other.control)) control = unique(control, ['scet_coarse', 'scet_fine', 'seq_count']) orig_indices = control['index'] new_index = range(len(control)) control['index'] = new_index data = vstack((self.data, other.data)) data = data[np.nonzero( orig_indices[:, None] == data['control_index'].data)[1]] data['control_index'] = range(len(data)) if np.abs([((len(data['data'][i]) / 2) - (control['data_len'][i] + 7)) for i in range(len(data))]).sum() > 0: logger.error('Expected and actual data length do not match') return type(self)(control, data)
def rand_loop_through__sep(f, length): sep_list = [] while len(sep_list) < length: righta = np.random.uniform(-180, 180) declin = np.random.uniform(-90, 90) pos = coords.SkyCoord(ra=righta * u.degree, dec=declin * u.degree) galatic_pos = pos.galactic xid = SDSS.query_region(galatic_pos, spectro=False, radius=2.5 * u.arcmin, photoobj_fields=['ra', 'dec']) if xid is None: continue xid = unique(xid, keys=['ra', 'dec']) righta_list = xid["ra"] declin_list = xid["dec"] sep = pos.separation( coords.SkyCoord(ra=righta_list * u.degree, dec=declin_list * u.degree)) sep_list = np.concatenate((sep_list, sep)) print(len(sep_list)) return sep_list
def get_vi(): cols = ['TARGETID', 'best_z', 'best_quality', 'best_spectype'] aa = atable.Table.read( '/global/cfs/cdirs/desi/sv/vi/TruthTables/Andes/BGS/Truth_table_Andes_reinspection_BGS_66003_20200315_v1.csv' ) for x in ['z', 'quality', 'spectype']: aa.rename_column('best {}'.format(x, x), 'best_{}'.format(x, x)) aa = aa[cols] bb = atable.Table.read( '/global/cfs/cdirs/desi/sv/vi/TruthTables/Blanc/BGS/desi-vi_BGS_tile80613_nightdeep_merged_all_210202.csv' )[cols] cc = atable.Table.read( '/global/cfs/cdirs/desi/sv/vi/TruthTables/Cascades/BGS/desi-vi_SV_cascades_combination_BGS_all_210521.csv' )[cols] vi = atable.vstack([aa, bb, cc]) vi = vi[vi['best_quality'] >= 2.5] vi.sort('best_quality') # 4 targets with repeated VI. vi = atable.unique(vi, keys='TARGETID', keep='last') vi.rename_column('best_z', 'VI_Z') vi.rename_column('best_quality', 'VI_Q') vi.rename_column('best_spectype', 'VI_SPECTYPE') return vi
def _load_curve_data(self): """ Use the configuration data for a given curve to configure the astropy.io.ascii fixed-width reader and load the data. Returns ------- data: dict-like Data columns read from extinction data file. """ curve_config = self.laws[self.law] filename = os.path.join(default_refdata_directory, "extinction", "curves", curve_config['filename']) data = ascii.read(filename, Reader=ascii.FixedWidth, data_start=curve_config['data_start'], names=curve_config['names'], col_starts=curve_config['col_starts'], col_ends=curve_config['col_ends'], guess=False) # make sure the data is sorted in order of increasing wavelength. 'data' is an astropy.table.Table object so has # built-in method to do this. data.sort(['wave']) # some of the files have duplicate rows so use astropy.table.unique() to weed them out, too uniq_data = unique(data, keys='wave') return uniq_data
def load_catalog(catalogs,sigs): filenames = catalogs.split(",") sigs = np.array(sigs.split(","),dtype=float) cnt = 0 for ii, (filename, sig) in enumerate(zip(filenames,sigs)): data_tmp = Table.read(filename, format='fits') idx = np.where(data_tmp["sig"] > sig)[0] data_tmp = data_tmp[idx] if cnt == 0: data = copy.copy(data_tmp) else: data = vstack([data,data_tmp]) cnt = cnt + 1 sig = data["sig"] idx = (1+np.arange(len(sig)))/len(sig+1) #sigsort = np.ceil(3.0*idx[np.argsort(sig)]) sigsort = 3.0*idx[np.argsort(sig)] data["sigsort"] = sigsort data.sort("sigsort") data.reverse() ras, decs, sigs = data["ra"], data["dec"], data["sigsort"] objnames = [] for ra, dec in zip(ras, decs): objname = "%.3f_%.3f" % (ra, dec) objnames.append(objname) data["objname"] = objnames data = unique(data, keys=['objname'], keep='first') data.sort("ra") return data
def drop_lines(dataset, drop_fractions): #inserting all supernovae into a single table for supernova in dataset.data.keys(): new_sna=dataset.data[supernova].copy() new_col=Table.Column(name='supernova', data=np.repeat(supernova, len(dat.data[supernova]))) new_sna.add_column(new_col, index=0) if supernova==dataset.data.keys()[0]: data_table=new_sna else: data_table=vstack([data_table, new_sna], metadata_conflicts='silent') #in each band, determine the number of lines to drop, and drop them for band in unique(data_table, keys='filter')['filter']: band_indices=np.where(data_table['filter']==band) num_drops=int(np.floor(len(data_table[band_indices])*drop_fractions[band])) #print 'Dropping '+str(num_drops)+' lines of '+str(len(data_table))+' ...' if num_drops > 0: lines_to_drop=np.random.choice(band_indices[0], num_drops, replace=False) else: lines_to_drop=[] data_table.remove_rows(lines_to_drop) #make copy of the original dataset, place rows from data_table in there new_dataset=copy.deepcopy(dataset) types=dataset.get_types() new_types=new_dataset.get_types() for supernova in dataset.data.keys(): new_dataset.data[supernova]=data_table[data_table['supernova']==supernova] new_dataset.data[supernova].remove_column('supernova') return new_dataset
def read(): table_south = read_data_as_table("hip/hip_south.vot") table_mag = ascii.read("hip/hip_mag.txt", format="fixed_width") t = table.vstack((table_mag, table_south)) t = table.unique(t, "hip") read_data_cols(t) return t
def doGaiaCat(inputFile, outputFile): table = Table.read(inputFile) table2 = table.filled(0) table3 = unique(table2) magJ, magH, magK = 99.000, 99.000, 99.000 JD = 15.0 * 365.25 + 2451545 catFile = open(outputFile, "w") for row in table3: catFile.write(" ".ljust(64)) catFile.write(("%.3f" % row[12]).rjust(6) ) catFile.write(" ".ljust(7)) catFile.write(" " + ("%.3f" % magJ).rjust(6)) catFile.write(" " + ("%.3f" % magH).rjust(6)) catFile.write(" " + ("%.3f" % magK).rjust(6)) catFile.write(" ".rjust(35)) catFile.write(" " + ("%.3f" % (row[5]/1000.0)).rjust(7)) catFile.write(" " + ("%.3f" % (row[7]/1000.0)).rjust(7)) catFile.write(" " + ("%.3f" % (row[6]/1000.0)).rjust(7)) catFile.write(" " + ("%.3f" % (row[8]/1000.0)).rjust(7)) catFile.write(" ".rjust(71)) catFile.write(" " + ("%.9f" % (row[0]/15.0)).rjust(13)) catFile.write(" " + ("%.9f" % row[2]).rjust(13)) catFile.write(" ".ljust(24)) catFile.write(("%.8f" % JD).rjust(16)) catFile.write(" ".ljust(119)) catFile.write(" " + ("%.3f" % (row[1]/1000.0)).rjust(6)) catFile.write(" " + ("%.3f" % (row[3]/1000.0)).rjust(6)) catFile.write("\n") catFile.close()
def join(self, cat, name=None): """ Returns a new ``Catalogue`` joining the current catalogue with 'cat'. Both catalogue must be consistent: same coordinates, positional errors and magnitudes, if they are included. If the original catalogues have areas defined through MOCs, the final area is the union of their MOCs, otherwise the area of the current catalogue is used. If the original catalogues have common sources, repeated entries will be remove from the final catalogue. """ if name is None: name = self.name join_cat_data = vstack([self.save(), cat.save()]) join_cat_data = unique(join_cat_data) try: area = self.moc.union(cat.moc) except: area = self.area mag_cols = None if self.mags is not None: mag_cols = self.mags.colnames join_cat = Catalogue(join_cat_data, poserr_cols=self.poserr.components.colnames, poserr_type=self.poserr.errtype, area=area, name=self.name, mag_cols=mag_cols) return join_cat
def load_sao() -> Table: """Load the SAO-TYC2 cross match.""" print('Loading SAO-TYC2 cross match') xmatch_files = [ 'sao_tyc2_xmatch.csv', 'sao_tyc2_suppl1_xmatch.csv', 'sao_tyc2_suppl2_xmatch.csv', ] data = vstack( [ io_ascii.read( XMATCH_DIR/f, include_names=['SAO', 'TYC1', 'TYC2', 'TYC3', 'angDist', 'delFlag'], format='csv', converters={'delFlag': [io_ascii.convert_numpy(np.str)]}, ) for f in xmatch_files ], join_type='exact', ) data = data[data['delFlag'].mask] data.remove_column('delFlag') parse_tyc_cols(data) data = unique(data.group_by(['TYC', 'angDist']), keys=['TYC']) data.remove_column('angDist') data.add_index('TYC') return data
def search_chandra_database(self): u = "https://cxcfps.cfa.harvard.edu/cgi-bin/cda/footprint/get_vo_table.pl?pos={0},{1}&size={2}" url = u.format(self.ra, self.dec, self.radius) url += "&inst=ACIS-I,ACIS-S" url += "&grating=NONE" r = requests.get(url) if r.status_code != 200: return f = io.BytesIO(r.text.encode()) votable = parse(f) tbdata = votable.get_first_table().to_table() if len(tbdata) > 0: table = unique(tbdata, keys='ObsId') self.total_exp = np.sum(table['Exposure']) self.obsids = table['ObsId'] self.targnames = table['target_name'] self.n_obsid = len(table) self.targ_ras = table['RA'] self.targ_decs = table['Dec'] self.exp_times = table['Exposure'] self.obs_dates = table['obs_date'] self.jpegs = table['preview_uri'] self.images = table['full_uri']
def align(self,tabref, taba): tabajoin = join(tabref,taba,keys=['time'],join_type='left') cols = tabajoin.columns csel = [] for c in cols: if '_2' in c and 'index' not in c and 'band' not in c: csel.append(c) csel.append('time') csel.append('band_1') tabanew = Table(tabajoin[csel]) tabanew['flux_2'].fill_value=0.0 tabanew = tabanew.filled() for vv in csel: if '_2' in vv or '_1' in vv: tabanew[vv].name= '_'.join(vv.split('_')[:-1]) return unique(tabanew, keys=['time','band'])
def _load_curve_data(self): """ Use the configuration data for a given curve to configure the astropy.io.ascii fixed-width reader and load the data. Returns ------- data: dict-like Data columns read from extinction data file. """ curve_config = self.laws[self.law] filename = os.path.join(default_refdata_directory, "extinction", "curves", curve_config['filename']) data = ascii.read( filename, Reader=ascii.FixedWidth, data_start=curve_config['data_start'], names=curve_config['names'], col_starts=curve_config['col_starts'], col_ends=curve_config['col_ends'], guess=False ) # make sure the data is sorted in order of increasing wavelength. 'data' is an astropy.table.Table object so has # built-in method to do this. data.sort(['wave']) # some of the files have duplicate rows so use astropy.table.unique() to weed them out, too uniq_data = unique(data, keys='wave') return uniq_data
def what_Tims_stars_are_missing_in_my_dataset(): usco = Table.read('usco_res/usco_run_subset.fit') ucl = Table.read('ucl_res/ucl_run_subset.fit') lcc = Table.read('lcc_res/lcc_run_subset.fit') tim = vstack([usco, ucl, lcc]) tim = unique(tim, keys='source_id') #tim = join(usco, ucl, keys='source_id') #tim = join(tim, lcc, keys='source_id') print len(tim), len(usco), len(ucl), len(lcc) d = Table.read('data_table_cartesian_with_bg_ols.fits') print len(d) # source_id sd = set(d['source_id']) st = set(tim['source_id']) print 'Stars in Tims set and not in mine', len(st.difference(sd)) for x in st.difference(sd): print x ct = set(tim.colnames) cd = set(d.colnames) print 'Colnames in mine and not in Tims set', cd.difference(ct) print 'Colnames in Tims and not in my set', ct.difference(cd)
def create_MPA_catalog(): galCl_table = read_fits_table(fnCl) galLi_table = read_fits_table(fnLi) SFR_table = read_fits_table(fnSFR) sSFR_table = read_fits_table(fnsSFR) S_M_table = create_mass_catalog() SDSS_table = Table([ galCl_table['objID'], galCl_table['I_CLASS'], galLi_table['OIII_5007_FLUX'], galLi_table['NII_6584_FLUX'], galLi_table['SII_6717_FLUX'], galLi_table['OI_6300_FLUX'], galLi_table['H_ALPHA_FLUX'], galLi_table['H_BETA_FLUX'], SFR_table['AVG'], sSFR_table['AVG'], SFR_table['FLAG'], sSFR_table['FLAG'] ], names=('objID', 'class', 'OIII', 'NII', 'SII', 'OI', 'H_a', 'H_b', 'SFR avg', 'sSFR avg', 'SFR flag', 'sSFR flag'), dtype=(np.int64, np.int32, np.float32, np.float32, np.float32, np.float32, np.float32, np.float32, np.float32, np.float32, np.int16, np.int16)) SDSS_table = unique(SDSS_table, keys='objID') shared = ['objID'] t = join(S_M_table, SDSS_table, keys=shared) t.rename_column('_RA', 'RA') t.rename_column('_DE', 'DE') return t
def why_galaxies_not_included(galaxies, nsa, bricks): logging.info('Original galaxies: {}'.format(len(galaxies))) in_nsa_maybe_duplicate, not_in_nsa = matching_utils.match_galaxies_to_catalog_table( galaxies=galaxies, catalog=nsa, galaxy_suffix='', catalog_suffix='_nsa') not_in_nsa_save_loc = 'galaxies_not_in_nsa.csv' not_in_nsa.to_pandas().to_csv(not_in_nsa_save_loc) logging.info('{} galaxies not in NSA listed in {}'.format( len(not_in_nsa), not_in_nsa_save_loc)) # Are they duplicates? in_nsa = table.unique(in_nsa_maybe_duplicate, keep='first', keys='sdss_id') logging.info( 'Duplicate NSA cross-matches, selecting {} first matches only'.format( len(in_nsa_maybe_duplicate) - len(in_nsa))) # Are they in the NSA? logging.info('In NSA 1_0_0: {}'.format(len(in_nsa))) # Do they pass the selection cuts? good_petrotheta = selection_cuts.apply_selection_cuts(in_nsa) logging.info('Good petrotheta: {}'.format(len(good_petrotheta))) # Are they in decals? joint_catalog = get_joint_nsa_decals_catalog.create_joint_catalog( in_nsa, bricks, '5') # dont apply selection cuts logging.info('In DECALS bricks: {}'.format(len(joint_catalog))) # Are they successfully downloaded? fits_dir = download_decals_settings.fits_dir png_dir = download_decals_settings.png_dir set_download_directory(joint_catalog, fits_dir, png_dir) joint_catalog = download_images_threaded.check_images_are_downloaded( joint_catalog, n_processes=1) image_download_stats(joint_catalog) return joint_catalog
def query_tmass(source_names): Simbad.reset_votable_fields() Simbad.add_votable_fields('typed_id') # keep search term in result table Simbad.add_votable_fields('flux(J)') Simbad.add_votable_fields('flux_error(J)') Simbad.add_votable_fields('flux_bibcode(J)') Simbad.add_votable_fields('flux(H)') Simbad.add_votable_fields('flux_error(H)') Simbad.add_votable_fields('flux_bibcode(H)') Simbad.add_votable_fields('flux(K)') Simbad.add_votable_fields('flux_error(K)') Simbad.add_votable_fields('flux_bibcode(K)') print('2MASS query started') result_table = Simbad.query_objects(source_names) print('2MASS query complete') # find indexes which contain 2MASS results ind = result_table['FLUX_BIBCODE_J'] == '2003yCat.2246....0C' tmass_phot = result_table['TYPED_ID', 'FLUX_J', 'FLUX_ERROR_J', 'FLUX_H', 'FLUX_ERROR_H', 'FLUX_K', 'FLUX_ERROR_K'][ind] tmass_phot_unique = unique(tmass_phot, keys='TYPED_ID', keep='first') print(len(tmass_phot), len(tmass_phot_unique)) return tmass_phot_unique
def mkfullran(tile, goodloc, pdict, randir): ranf = randir + 'fba-0' + str(tile) + '.fits' f1 = fitsio.read(ranf) f2 = fitsio.read(ranf, ext=2) f3 = fitsio.read(ranf, ext=3) goodranw = np.isin(f3['LOCATION'], goodloc) goodranid = np.unique(f3[goodranw]['TARGETID']) t2 = Table.read(ranf, hdu=2) tj = Table() tj['TARGETID'] = f3[goodranw]['TARGETID'] tj['LOCATION'] = f3[goodranw]['LOCATION'] tj['FIBER'] = f3[goodranw]['FIBER'] tj = unique(tj, keys=['TARGETID']) t2.remove_columns(['PRIORITY', 'OBSCONDITIONS', 'SUBPRIORITY']) rant = join(tj, t2, keys=['TARGETID'], join_type='left') #now match back to randoms with all columns tall = Table.read(randir + 'tilenofa-' + str(tile) + '.fits') tall.remove_columns([ 'NUMOBS_MORE', 'PRIORITY', 'OBSCONDITIONS', 'SUBPRIORITY', 'NUMOBS_INIT' ]) ranall = join(rant, tall, keys=['TARGETID'], join_type='left') print('number of randoms:') print(len(ranall)) ranall['PRIORITY'] = np.vectorize(pdict.__getitem__)(ranall['LOCATION']) return ranall
def __add__(self, other): """ Combine two products stacking data along columns and removing duplicated data using time as the primary key. Parameters ---------- other : A subclass of stix_parser.products.quicklook.Product Returns ------- A subclass of stix_parser.products.quicklook.Product The combined data product """ if not isinstance(other, type(self)): raise TypeError( f'Products must of same type not {type(self)} and {type(other)}' ) # TODO reindex and update data control_index other.control[ 'index'] = other.control['index'] + self.control['index'].max() + 1 control = vstack((self.control, other.control)) # control = unique(control, keys=['scet_coarse', 'scet_fine']) # control = control.group_by(['scet_coarse', 'scet_fine']) other.data['control_index'] = other.data[ 'control_index'] + self.control['index'].max() + 1 data = vstack((self.data, other.data)) data = unique(data, keys='time') # data = data.group_by('time') unique_control_inds = np.unique(data['control_index']) control = control[np.isin(control['index'], unique_control_inds)] return type(self)(control, data)
def match_stars(skycat, srccat, in_wcs, max_sep=1.5 * u.deg): """ skycat : `~astropy.table.Table` Skycam catalog as produced by load_skycam_catalog() with Alt and Az columns added for the appropriate time. srccat : `~astropy.table.Table` Source catalog as produced by make_catalog(). in_wcs : `~astropy.wcs.WCS` WCS used to create Alt/Az for skycat. max_sep : `~astropy.units.Quantity` (default: 1 degree) Separation criterium for valid matching. """ pred_az, pred_alt = in_wcs.all_pix2world(srccat['xcentroid'], srccat['ycentroid'], 0) pred_coord = SkyCoord(ra=pred_az * u.deg, dec=pred_alt * u.deg) act_coord = SkyCoord(ra=skycat['Az'], dec=skycat['Alt']) idx, d2d, d3d = pred_coord.match_to_catalog_sky(act_coord, nthneighbor=1) sep_constraint = d2d < max_sep matches = srccat[sep_constraint] cat_matches = skycat[idx[sep_constraint]] matched_cat = hstack([cat_matches, matches]) matched_cat.sort('obs_mag') # If we get multiple matches, keep the brightest one. May be wrong, but at least consistent matched_cat = unique(matched_cat, keys='Star Name', keep='first') return matched_cat
def field_stars_init(self,dwarfs_or_giants): from astropy.table import unique # data cleaning steps (Temperature, RV scatter, surface gravity) self.data = unique(self.data,keys='APOGEE_ID') self.data = self.data[self.data['TEFF']>3600] self.data = self.data[self.data['VSCATTER']<1] self.data = self.data[self.data['LOGG']<3] # remove stars with 'bad' quality flags self.data = self.data[np.logical_and.reduce((self.data['STARFLAG'] & 2**17 == 0, self.data['STARFLAG'] & 2**2 == 0,\ self.data['STARFLAG'] & 2**3 == 0, self.data['ASPCAPFLAG'] & 2**19 == 0,\ self.data['ASPCAPFLAG'] & 2**20 == 0,self.data['ASPCAPFLAG'] & 2**23 == 0))] # select stars with good signal to noise ratio self.data = self.data[self.data['SNR']>50] self.data=self.data[np.logical_and(self.data['TEFF']>-9000,self.data['M_H']>-9000)] # initialize the SNR and Teff bins self.data['SNR_bin']=np.nan self.data['TEFF_MH_bin']=np.nan # select dwarves/red giants based on the property 'CLASS' in the file if dwarfs_or_giants == 'dwarfs': cond=np.logical_or.reduce((['Fd' in i for i in self.data['ASPCAP_CLASS']],\ ['GKd' in i for i in self.data['ASPCAP_CLASS']],\ ['Md' in i for i in self.data['ASPCAP_CLASS']])) elif dwarfs_or_giants == 'giants': cond=np.logical_or(['Mg' in i for i in self.data['ASPCAP_CLASS']],\ ['GKg' in i for i in self.data['ASPCAP_CLASS']]) else: print("Please enter 'dwarfs' or 'giants'") self.data = self.data[cond]
def update_local_data(year: int, timeout: bool = None): """Download data from SuomiNet for a given year and update the master table Args: year: The year to update data for timeout: Optional seconds to wait while connecting to SuomiNet Returns: A boolean representing whether any data was downloaded """ # Determine what years to download if year > datetime.now().year: raise ValueError( 'Cannot download data for years greater than the current year.') # Get any local data that has already been downloaded local_data = _get_local_data() # Download new data from SuomiNet new_data = _download_data_for_year(year, timeout) stacked_tables = vstack([local_data, new_data]) if stacked_tables: updated_data = unique(stacked_tables, keys=['date'], keep='last') # Update local files updated_data.write(settings._pwv_measured_path, overwrite=True) return True else: return False
def _set_identifiers(self): """make a useful table of identifiers of lines covered by this model""" n = deepcopy(self._table['numerator']) n.name = 'ID' d = deepcopy(self._table['denominator']) d.name = 'ID' t1 = Table([self._table['title'], n], copy=True) # discard the summed fluxes as user would input them individually for id in ['OI_145+CII_158', 'OI_63+CII_158']: a = np.where(t1['ID'] == id)[0] for z in a: t1.remove_row(z) # now remove denominator from title (everything from / onwards) for i in range(len(t1['title'])): t1['title'][i] = t1['title'][i][0:t1['title'][i].index('/')] t2 = Table([self._table['title'], d], copy=True) # remove numermator from title (everything before and including /) for i in range(len(t2['title'])): t2['title'][i] = t2['title'][i][t2['title'][i].index('/') + 1:] t = vstack([t1, t2]) t = unique(t, keys=['ID'], keep='first', silent=True) t['title'].unit = None t['ID'].unit = None t.rename_column('title', 'canonical name') self._identifiers = t
def parse_spectra(data: Table) -> Table: """Parse the spectral types into the celestia.Sci format.""" print('Parsing spectral types') data['SpType'] = data['SpType'].filled('') sptypes = unique(data['SpType', ]) sptypes['CelSpec'] = parse_spectrum_vec(sptypes['SpType']) return join(data, sptypes)
def get_planets_table(): """ Get the joined planets table from the NASA Exoplanet Archive and the Exoplanet Orbit Database. Returns ------- table : `~astropy.table.Table` Table of exoplanet properties """ if not os.path.exists(planet_table_path): raise ValueError("You must run salter.cache.cache_planets_table first " "before you can run get_joined_table") table = ascii.read(planet_table_path, format='csv') # Toss out multis first_kois_only = np.array( [koi.endswith('01') for koi in table['kepoi_name']]) table = table[first_kois_only] table.add_index('kepid') # Ensure only unique results unique_table = unique(table, keys='kepid') unique_table.add_index('kepid') return unique_table
def find_best_comp(self, result_file_path=None, best_comparison_star=None): result_file = Table.read(result_file_path, format='ascii.commented_header') # read comparison star list # and check manual assigned comp star if best_comparison_star is None: result_unique_by_cat = table.unique(result_file, keys='nomad1') else: result_unique_by_cat = table.unique( result_file[(result_file['nomad1'] == best_comparison_star)], keys='nomad1') std_list = [] t_c_list = [] # calculates diff_mag for all target objects and comp. stars for star in result_unique_by_cat['nomad1']: frame_results = result_file[(result_file['nomad1'] == star)] # diff phot. frame_results['t-c'] = frame_results['magt_i'] - frame_results['magc_i'] # error propagation frame_results['t-c-err'] = np.sqrt( np.power(frame_results['magt_i_err'], 2) + np.power(frame_results['magc_i_err'], 2)) # extracting usefull columns t_c_list.append(frame_results['ast_num', 'nomad1', 'jd', 't-c', 't-c-err']) # calculating all t-c stars STD then adding list std_list.append(np.std(frame_results['t-c'])) # calculating all STD's mean and its index number in the list mean_idx = (np.abs(np.asanyarray(std_list) - np.mean(std_list))).argmin() # choosing STD with min, mean and max stars diff_stats = {'min': [std_list.index(min(std_list)), min(std_list)], 'mean': [mean_idx, np.mean(std_list)], 'max': [std_list.index(max(std_list)), max(std_list)] } # getting these diff mags and their other columns results = {'with_min_comp': t_c_list[diff_stats['min'][0]], 'with_mean_comp': t_c_list[diff_stats['mean'][0]], 'with_max_comp': t_c_list[diff_stats['max'][0]] } return results
def test_group_mixins(): """ Test grouping a table with mixin columns """ # Setup mixins idx = np.arange(4) x = np.array([3., 1., 2., 1.]) q = x * u.m lon = coordinates.Longitude(x * u.deg) lat = coordinates.Latitude(x * u.deg) # For Time do J2000.0 + few * 0.1 ns (this requires > 64 bit precision) tm = time.Time(2000, format='jyear') + time.TimeDelta(x * 1e-10, format='sec') sc = coordinates.SkyCoord(ra=lon, dec=lat) aw = table_helpers.ArrayWrapper(x) nd = np.array([(3, 'c'), (1, 'a'), (2, 'b'), (1, 'a')], dtype='<i4,|S1').view(NdarrayMixin) qt = QTable([idx, x, q, lon, lat, tm, sc, aw, nd], names=['idx', 'x', 'q', 'lon', 'lat', 'tm', 'sc', 'aw', 'nd']) # Test group_by with each supported mixin type mixin_keys = ['x', 'q', 'lon', 'lat', 'tm', 'sc', 'aw', 'nd'] for key in mixin_keys: qtg = qt.group_by(key) # Test that it got the sort order correct assert np.all(qtg['idx'] == [1, 3, 2, 0]) # Test that the groups are right # Note: skip testing SkyCoord column because that doesn't have equality for name in ['x', 'q', 'lon', 'lat', 'tm', 'aw', 'nd']: assert np.all(qt[name][[1, 3]] == qtg.groups[0][name]) assert np.all(qt[name][[2]] == qtg.groups[1][name]) assert np.all(qt[name][[0]] == qtg.groups[2][name]) # Test that unique also works with mixins since most of the work is # done with group_by(). This is using *every* mixin as key. uqt = unique(qt, keys=mixin_keys) assert len(uqt) == 3 assert np.all(uqt['idx'] == [1, 2, 0]) assert np.all(uqt['x'] == [1., 2., 3.]) # Column group_by() with mixins idxg = qt['idx'].group_by(qt[mixin_keys]) assert np.all(idxg == [1, 3, 2, 0])
def merge_cat(rootname,rastr='ra',decstr='dec'): g=glob.glob(rootname+'/*.vo') tlist=[] for f in g: try: t=Table.read(f) except: print 'Error reading table',f raise t2=Table() t2['ra']=t[rastr] t2['dec']=t[decstr] tlist.append(t2) t=vstack(tlist) t2=unique(t,keys=['ra','dec']) t2.write(rootname+'.fits',overwrite=True) return t2
i=0 j=0 for filename in os.listdir(os.getcwd()): start = filename.find('tile_') + 5 end = filename.find('.fits', start) tile_id =filename[start:end].lstrip("0") tile_data1 = fits.open(filename)[1].data tile_data2 = fits.open(filename)[2].data add_rows_all1 = Table((tile_data1['TARGETID'],), names=('TARGETID',)) add_rows_all2 = Table((tile_data2['POTENTIALTARGETID'],),names=('TARGETID',)) del tile_data1, tile_data2 add_rows1 = unique(add_rows_all1,keys='TARGETID') add_rows1['OBS'] = np.ones(len(add_rows1)) add_rows2 = unique(add_rows_all2,keys='TARGETID') add_rows2['OBS'] = np.zeros(len(add_rows2)) add_rows1['TILEID']= np.ones(len(add_rows1))*int(tile_id) add_rows2['TILEID']= np.ones(len(add_rows2))*int(tile_id) tot_targs = (vstack([add_rows1,add_rows2])) del add_rows1, add_rows2 unique_targs = unique(tot_targs, keys = 'TARGETID') if i==0: tot_unique_targsL100 = unique_targs else:
def create_tables(self, uv_type, directory, radius, print_progress = False): ''' Perform photometry on a directory of .fits files and create a list of two tables. The first table contains the redshift, exposure time, luminosity, and surface brightness for various supernova, along with the associated error values. The second table is a log outlining any files that do not contain a supernova or are missing checkfiles. If print_progress is set equal to true, the path each fits file will be printed before performing photometry on along with the number of remaining files. Args: uv_type (str) : Specifies which type of uv to create a table for. Use either "NUV" or "FUV". directory (str) : A directory containing .fits files radius (float): Radius of desired photometry aperture in kpc print_progress (bool) : Whether or not to print the file path of each fits file Returns: results (list): [Data table (Table), Log table (Table)] ''' #Make sure we have redshift and coordinates of each supernova if self.cord_dict.keys() != self.red_dict.keys(): raise ValueError('''Keys in coordinate and redshift dictionaries (self.cord_dict, self.red_dict) do not match''') label = uv_type + " " + str(radius) + "kpc " #Define the tables that will be returned by the function log = Table(names = ["File Path", "Issue"], dtype = [object, object]) out = Table(names = ["sn", "Redshift", "Redshift Error", label + "Exposure Time", "Flux", "Flux Error", label + "Luminosity", label + "Luminosity Error", label + "Surface Brightness", label + "Surface Brightness Error"], dtype = ("S70", "float64", "float64", "float64", "float64", "float64", "float64", "float64", "float64", "float64")) out["Redshift"].unit = u.dimensionless_unscaled out["Redshift Error"].unit = u.dimensionless_unscaled out[label + "Exposure Time"].unit = u.s out["Flux"].unit = u.erg / u.s / u.Angstrom / u.kpc / u.kpc / u.cm / u.cm / np.pi out["Flux Error"].unit = u.erg / u.s / u.Angstrom / u.kpc / u.kpc / u.cm / u.cm / np.pi out[label + "Luminosity"].unit = u.erg / u.s / u.Angstrom / u.kpc / u.kpc out[label + "Luminosity Error"].unit = u.erg / u.s / u.Angstrom / u.kpc / u.kpc out[label + "Surface Brightness"].unit = u.erg / u.s / u.Angstrom / u.arcsec / u.arcsec out[label + "Surface Brightness Error"].unit = u.erg / u.s / u.Angstrom / u.arcsec / u.arcsec #Set parameters that are specific to NUV or FUV observations if "N" in uv_type.upper(): file_key = "nd-int" #A string distinguing galex file types flux_conv = 2.06 * 1e-16 #A conversion factor from counts per second to flux elif "F" in uv_type.upper(): file_key = "fd-int" flux_conv = 1.40 * 1e-15 #Create a list of files to perform photometry on file_list = [] for path, subdirs, files in os.walk(directory): for name in files: if file_key in name and len(name.split(".")) < 3: file_list.append(os.path.join(path, name)) count = len(file_list) #Perform photometry on each .fits file for fits_file in file_list: if print_progress == True: print(count, ":", fits_file, flush = True) count -= 1 p = self.photometry(fits_file, radius) for elt in p: if elt[0] == "error": log.add_row([elt[1], elt[2]]) if print_progress == True: print("error", elt[2], "\n", flush = True) else: #We calculate the values to be entered in the table redshift = float(self.red_dict[elt[0]]) peculiar_redshift = np.sqrt((1 + (300 / 299792.458)) / (1 - (300 / 299792.458))) - 1 redshift_err = np.sqrt((redshift / 1000)**2 + (peculiar_redshift)**2) arcmin = cosmo.kpc_comoving_per_arcmin(redshift).value**2 #kpc^2 per arcmin^2 arcmin_err = self.conv_error(redshift, redshift_err) photom = elt[2] #The photometry value photom_err = elt[3] flux = flux_conv * photom #convert cps to flux using the conversion factor flux_err = self.flux_error(photom, photom_err, flux_conv) ldist = cosmo.luminosity_distance(redshift).cgs.value #Luminosity Distance (cm) ldist_err = self.lum_dist_error(redshift, redshift_err) lum = flux * 4 * np.pi * (ldist**2) #luminosity = flux*4*pi*r^2 lum_err = self.luminosity_error(flux, flux_err, ldist, ldist_err) sbrightness = lum * arcmin / 3600 sbrightness_err = self.surf_brightness_error(lum, lum_err, arcmin, arcmin_err) out.add_row([elt[0], redshift, redshift_err, elt[1], flux, flux_err, lum, lum_err, sbrightness, sbrightness_err]) out.sort(label + "Surface Brightness Error") out_unique = unique(out, keys = "sn") out_unique.sort("sn") return([out_unique, log])
def __init__(self,**kwargs): minWLin = kwargs.get("minWLin",605.2) maxWLin = kwargs.get("maxWLin",605.8) minlinWL = kwargs.get("minlinWL",None) maxlinWL = kwargs.get("maxlinWL",None) IDin = kwargs.get("IDin",1) starttime = kwargs.get("starttime",None) walltime = kwargs.get("walltime",None) initlines = kwargs.get("initlines",None) injectlines = kwargs.get("injectlines",None) outputfile = kwargs.get("outputfile",None) outputdir = kwargs.get("outputdir",None) arcscale = kwargs.get("arcscale",None) previousball = kwargs.get("previousball",None) # change the following lines to be inputs when you init the class self.minWL = minWLin self.maxWL = maxWLin if minlinWL == None: self.minLINWL = self.minWL self.maxLINWL = self.maxWL else: self.minLINWL = minlinWL self.maxLINWL = maxlinWL self.ID = IDin self.numstars = 2 self.previousball = previousball self.IDlist = [int(10000000*x)+self.ID for x in range(1,self.numstars+1,1)] # Define synthesis wavelength range self.wavebuffer = 0.15 self.waverange = [self.minWL-self.wavebuffer,self.maxWL+self.wavebuffer] # setting cut for line selection self.condst = [{'LP':'RESID','OP':np.less,'LV':0.99}] # set output file name self.outputfile = outputfile self.outputdir = outputdir if arcscale == None: self.arcscale = 1.0 else: self.arcscale = arcscale # define a general start time so that the code can stop short of the wall time and save everything if starttime == None: self.starttime = time.time() else: self.starttime = starttime # starttime in seconds if walltime == None: # an impossibly large walltime self.walltime = self.starttime + 300000 else: self.walltime = walltime # walltime in seconds print('Pro: {0} --> Total possible run time = {1} seconds'.format(self.ID,self.walltime-self.starttime)) print('Pro: {0} --> Full Wavelength Range = {1:9.5f}-{2:9.5f} ({3:9.5f}) nm'.format( self.ID,self.minWL-self.wavebuffer,self.maxWL+self.wavebuffer,self.maxWL-self.minWL+(2.0*self.wavebuffer))) # set up some dictionaries for passing objects self.fmdict = {} origsyndict = {} # run synthe to get all needed lines for ID_i,star_i in zip(self.IDlist,['Sun','Arcturus']): print('Pro: {0} --> Running original synthe on full master line list for {1}'.format(self.ID,star_i)) # initialize the class fm_i = FALmod.FALmod(ID=ID_i,waverange=self.waverange,starpars=star_i) # run SYNTHE using the master line list to grab all important lines spec_i,ll_i = fm_i.runsynthe(timeit=False,linelist='readmaster') self.fmdict[ID_i] = fm_i origsyndict[ID_i] = [Table(spec_i),ll_i] # Assemble working line list (union of ll_i from last for-loop) print('Pro: {0} --> Assemble working line list'.format(self.ID)) # stack the tables fmll = vstack([origsyndict[ID_i][1] for ID_i in self.IDlist]) # fmll['FILTERBOOL'] = np.zeros(len(fmll['WL']),dtype=int) # sort tables on all collumns, include RESID as that way the stronger line will be listed first and will be set as a fit parameter tabpars = ['WL','GFLOG', 'CODE', 'E', 'XJ', 'LABEL', 'EP', 'XJP', 'LABELP', 'GR', 'GS', 'GW', 'WAVENO', 'REF', 'NBLO', 'NBUP', 'ISO1', 'X1', 'ISO2', 'X2', 'OTHER'] fmll.sort(tabpars+['RESID']) fmll = unique(fmll,tabpars) # check to see if working in H-band where we don't incldue a background model if ((self.minWL > 1300.0) & (self.maxWL < 2300.0)): print("Pro: {0} --> Not using a background model for weak lines".format(self.ID)) else: # remove lines with RESID > 0.9999 as these are included in the static background model fmll = fmll[fmll['RESID'] < 0.9999] print("Pro: {0} --> Using a background model for weak lines".format(self.ID)) # # inject fake lines # fmll = self.injectfake(fmll.copy()) if initlines != None: # inject previous parameters print('Pro: {0} --> Injecting Previous Line Parameters'.format(self.ID)) if type(initlines) == type(''): fmll = self.injectprev(fmll.copy(),presetll=initlines) elif type(initlines) == type([]): for initlines_i in initlines: fmll = self.injectprev(fmll.copy(),presetll=initlines_i) else: print("Pro: {0} --> WARNING!!! Did not understand previous line parameter file".format(self.ID)) # set it into self self.fmll = fmll # remove all predicted lines with depths < 0.99 plinesind = np.array((self.fmll['LABEL'] == ' ') & (self.fmll['LABELP'] == ' ') & (self.fmll['RESID'] < 0.99),dtype=bool) self.fmll = self.fmll[~plinesind] # change all DWL back to zero, hack for pervious solar-only fit self.fmll['DWL'] = np.zeros_like(self.fmll['DWL']) # now run each stellar spectrum again and archive the results print('Pro: {0} --> Archiving the results into working directories'.format(self.ID)) for ID_i,star_i in zip(self.IDlist,['Sun','Arcturus']): if star_i == 'Arcturus': print('Pro: {0} --> Changing C12/C13 abundance fraction for Arcturus'.format(self.ID)) C12C13 = 7.4 fmll_i = self.fmll.copy() for ll_ii in fmll_i: if ll_ii['ISO1'] == 12: ll_ii['GFLOG'] = ll_ii['GFLOG']+0.005-np.log10(1.0+(1.0/C12C13)) elif ll_ii['ISO1'] == 13: ll_ii['GFLOG'] = ll_ii['GFLOG']+1.955-np.log10(1.0+C12C13) else: pass else: fmll_i = self.fmll.copy() _spec,_ll = self.fmdict[ID_i].runsynthe(timeit=False,linelist=fmll_i,archive=True) # reset orgll to fmll because we don't want to use the synthe parsed ll self.fmdict[ID_i].orgll = fmll_i # run function to select which lines are modeled (self.parr,self.psig,self.pflag,self.Tarr) = FALlinesel.linesel(self.fmll,self.condst,self.minLINWL,self.maxLINWL) # calculate number of lines with free paramters and their wavelengths fitlineind = [] fitlinewl = [] for ii,tt in enumerate(self.Tarr): if not all(tt == -1): fitlineind.append(ii) fitlinewl.append(float(self.fmll['WL'][ii])) print("Pro: {0} --> Total number of lines considered in WL segment = {1}".format(self.ID,len(self.fmll))) print("Pro: {0} --> Number of lines that are free in WL segment = {1}".format(self.ID,len(fitlinewl))) # print("Pro: {0} --> Index in line list of the modeled lines...".format(self.ID),fitlineind) # print("Pro: {0} --> WL of modeled lines...".format(self.ID),fitlinewl) # number of dimensions self.ndim = len(self.parr) print("Pro: {0} --> Number of Free Line Parameters...".format(self.ID),self.ndim) print("Pro: {0} --> Fitting Transmission Spectrum (scaling and velocity)".format(self.ID)) self.ndim = self.ndim + 2 print("Pro: {0} --> Fitting Arcturus Spectrum scaling and velocity".format(self.ID)) self.ndim = self.ndim + 2 # get observed data, transmission spectrum, and background model (self.solobswave,self.solobsflux,self.arcobswave,self.arcobsflux,self.transflux,self.bg_sol_flux,self.bg_arc_flux) = self.getspecdata() # initialize output files self.initoutput() # # now that the output files have been written... # # for arcturus, remove troublesome pixels in observed spectrum (flux < 0.001 & flux > 0.99) # prunearc = (self.arcobsflux >= 0.001) & (self.arcobsflux <= 0.99) # self.arcobsflux = self.arcobsflux[np.array(prunearc)] # self.arcobswave = self.arcobswave[np.array(prunearc)] self.arcobsflux = self.arcobsflux*self.arcscale print("Pro: {0} --> Number of Pixels in Obs Sol ...".format(self.ID),len(self.solobswave)) print("Pro: {0} --> Number of Pixels in Obs Arc ...".format(self.ID),len(self.arcobswave)) print("Pro: {0} --> Using an initial scaling of {1} for Obs Arc ...".format(self.ID,self.arcscale)) print("Pro: {0} --> Finished Setup".format(self.ID))
def lc_plot_general(self, result_file_path=None, xcol='jd', ycol='magt_i', errcol='magt_i_err', mark_color="blue", bar_color="red"): """ Plot light curve of photometry result. @param result_file_path: Result file path @type result_file_path: path @param xcol: X-axis data for plotting @type xcol: array @param ycol: Y-axis data for plotting @type ycol: array @param errcol: Error bar data for plotting @type errcol: array @param mark_color: Marker color @type mark_color: str @param bar_color: Bar marker color @type bar_color: str @return: str """ print("Plotting asteroid's LC...") fn = os.path.basename(result_file_path).split('.')[0] result_file = Table.read(result_file_path, format='ascii.commented_header') result_unique_by_keys = table.unique(result_file, keys='jd') rcParams['figure.figsize'] = [10., 8.] figlc = plt.figure(1) gs = gridspec.GridSpec(2, 1, height_ratios=[6, 2]) # Two subplots, the axes array is 1-d axlc1 = figlc.add_subplot(gs[0]) axlc2 = figlc.add_subplot(gs[1]) axlc1.set_title(fn) filtered_data = sigma_clip(result_unique_by_keys[ycol], sigma=3, iters=10, stdfunc=mad_std) axlc1.errorbar( result_unique_by_keys[xcol][np.logical_not(filtered_data.mask)], result_unique_by_keys[ycol][np.logical_not(filtered_data.mask)], yerr=result_unique_by_keys[errcol][np.logical_not( filtered_data.mask)], fmt='o', ecolor=bar_color, color=mark_color, capsize=5, elinewidth=2) axlc1.invert_yaxis() axlc2.set_xlabel("JD", fontsize=12) axlc1.set_ylabel("Magnitude (R - INST)", fontsize=12) axlc2.set_ylabel("STD", fontsize=12) fit = np.polyfit( result_unique_by_keys[xcol][np.logical_not(filtered_data.mask)], result_unique_by_keys[errcol][np.logical_not(filtered_data.mask)], 1) fit_fn = np.poly1d(fit) axlc2.plot( result_unique_by_keys[xcol][np.logical_not(filtered_data.mask)], result_unique_by_keys[errcol][np.logical_not(filtered_data.mask)], 'yo', result_unique_by_keys[xcol][np.logical_not(filtered_data.mask)], fit_fn(result_unique_by_keys[xcol][np.logical_not( filtered_data.mask)]), '--k') axlc1.grid(True) axlc2.grid(True) axlc1.legend(loc=2, numpoints=1) figlc.savefig("{0}/{1}_jd_vs_magi_lc.pdf".format(os.getcwd(), fn))
def time_unique_float(self): unique(self.table, keys='float')
def time_unique_int(self): unique(self.table, keys='int')
def time_unique_str(self): unique(self.table, keys='str')
def lc_plot_std_mag(self, result_file_path=None, xcol='magc_i', ycol='star_Rmag', errcol='magc_i_err', mark_color="blue", bar_color="red"): print("Plotting asteroid's LC...") # Fixing random state for reproducibility np.random.seed(19680801) fn = os.path.basename(result_file_path).split('.')[0] # Two subplots, the axes array is 1-d # Plotting settings rcParams['figure.figsize'] = [10., 8.] lc = plt.figure(1) lc_ast_std = plt.figure() gs = gridspec.GridSpec(2, 1, height_ratios=[6, 2]) # magi vs catalogue lc1 = lc.add_subplot(gs[0]) lc1.set_title(fn) lc1.grid(True) lc1.set_ylabel("Magnitude (R - NOMAD1)", fontsize=12) lc1.invert_yaxis() # magi vs STD lc2 = lc.add_subplot(gs[1]) lc2.set_title(fn) lc2.grid(True) lc2.set_xlabel("Magnitude (Inst)", fontsize=12) lc2.set_ylabel("$STD$", fontsize=12) # magt vs estimated mag lc3 = lc_ast_std.add_subplot(gs[0]) lc3.set_title(fn) lc3.legend(loc=2, numpoints=1) lc3.grid(True) lc3.invert_yaxis() lc3.set_xlabel("$JD$", fontsize=12) lc3.set_ylabel("Magnitude (R - Estimated from NOMAD1)", fontsize=12) # Plotting settings result_file = Table.read(result_file_path, format='ascii.commented_header') # result_unique_by_keys = table.unique(result_file, keys='nomad1') result_unique_by_jd = table.unique(result_file, keys='jd') magt_std_list = [] for jd in result_unique_by_jd['jd']: frame_results = result_file[(result_file['jd'] == jd)] # for reject outliers filtered_frame_results = sigma_clip(frame_results['magt_i'], sigma=3, iters=10, stdfunc=mad_std) # use only not rejected data (because umask used) filtered_f_umask = np.logical_not(filtered_frame_results.mask) # magci vs catalogue with error bar lc1.errorbar( frame_results[xcol][filtered_f_umask], frame_results[ycol][filtered_f_umask], yerr=frame_results[errcol][filtered_f_umask], fmt='o', ecolor=bar_color, color=mark_color, capsize=5, elinewidth=2) # magci vs catalogue fit calculation fit = np.polyfit( frame_results[xcol][filtered_f_umask], frame_results[ycol][filtered_f_umask], 1) fit_fn = np.poly1d(fit) magt_to_std = fit_fn(frame_results['magt_i'][filtered_f_umask]) magt_std_list.append([jd, magt_to_std[0], frame_results['magt_i_err'][0]]) # magci vs catalogue fit plot lc1.plot( frame_results[xcol][filtered_f_umask], fit_fn(frame_results[xcol][filtered_f_umask]), '--k') # magi vs catalogue error fit calc. fit = np.polyfit( frame_results[xcol][filtered_f_umask], frame_results[errcol][filtered_f_umask], 1) fit_fn = np.poly1d(fit) # magi vs STD fit plot lc2.plot( frame_results[xcol][filtered_f_umask], frame_results[errcol][filtered_f_umask], 'yo', frame_results[xcol][filtered_f_umask], fit_fn(frame_results[xcol][filtered_f_umask]), '--k') # jd vs magt_std jd_vs_magt = np.asanyarray(magt_std_list) filtered_jd_vs_magt = sigma_clip(jd_vs_magt[:, 1], sigma=3, iters=10, stdfunc=mad_std) # use only not rejected data (because umask used) filtered_f_umask = np.logical_not(filtered_jd_vs_magt.mask) # jd vs magt plotting with error bars lc3.errorbar( jd_vs_magt[:, 0][filtered_f_umask], jd_vs_magt[:, 1][filtered_f_umask], yerr=jd_vs_magt[:, 2][filtered_f_umask], fmt='o', ecolor=bar_color, color=mark_color, capsize=5, elinewidth=2, label='{0} - R (Estimated)'.format(fn)) lc3.legend(loc=2, numpoints=1) lc_ast_std.savefig("{0}/{1}_jd_vs_mag_std_lc.pdf".format(os.getcwd(), fn))