def fix_wcs(obj, cat, sr, header=None, use_header_wcs=False, maxmatch=1, order=6, fix=True): '''Get a refined WCS solution based on cross-matching of objects with catalogue on the sphere. Uses external 'fit-wcs' binary from Astrometry.Net suite''' if header is not None: width,height = header['NAXIS1'],header['NAXIS2'] if use_header_wcs: wcs = WCS(header) if wcs: obj['ra'],obj['dec'] = wcs.all_pix2world(obj['x'], obj['y'], 0) else: width,height = int(np.max(obj['x'])),int(np.max(obj['y'])) h = htm.HTM(10) oidx,cidx,dist = h.match(obj['ra'], obj['dec'], cat['ra'], cat['dec'], sr, maxmatch=1) dir = tempfile.mkdtemp(prefix='astrometry') wcs = None binname = None for path in ['.', '/usr/local', '/opt/local']: if os.path.isfile(posixpath.join(path, 'astrometry', 'bin', 'fit-wcs')): binname = posixpath.join(path, 'astrometry', 'bin', 'fit-wcs') break if binname: columns = [fits.Column(name='FIELD_X', format='1D', array=obj['x'][oidx] + 1), fits.Column(name='FIELD_Y', format='1D', array=obj['y'][oidx] + 1), fits.Column(name='INDEX_RA', format='1D', array=cat['ra'][cidx]), fits.Column(name='INDEX_DEC', format='1D', array=cat['dec'][cidx])] tbhdu = fits.BinTableHDU.from_columns(columns) filename = posixpath.join(dir, 'list.fits') wcsname = posixpath.join(dir, 'list.wcs') tbhdu.writeto(filename, overwrite=True) os.system("%s -c %s -o %s -W %d -H %d -C -s %d" % (binname, filename, wcsname, width, height, order)) if os.path.isfile(wcsname): header = fits.getheader(wcsname) wcs = WCS(header) if fix and wcs: obj['ra'],obj['dec'] = wcs.all_pix2world(obj['x'], obj['y'], 0) else: print("Astrometry.Net binary not found") #print order shutil.rmtree(dir) return wcs
def match(cat_1, cat_2, column_1, column_2, column_3, column_4,error): h = htm.HTM(depth=10) m1, m2, d12 = h.match(np.array(cat_1[column_1]), np.array(cat_1[column_3]), np.array(cat_2[column_2]), np.array(cat_2[column_4]), error, maxmatch=1) submatched = cat_1[m1] manmatched = cat_2[m2] matched = hstack([submatched, manmatched]) return matched
def match(cat_1, cat_2, column_1, column_2, column_3, column_4, error): 'Function to make the matching with two tables, using the esutil' h = htm.HTM(depth=10) m1, m2, d12 = h.match(np.array(cat_1[column_1]), np.array(cat_1[column_3]), np.array(cat_2[column_2]), np.array(cat_2[column_4]), error, maxmatch=1) submatched = cat_1[m1] manmatched = cat_2[m2] matched = hstack([submatched, manmatched]) return matched
def match(c1, c2, sr=None, maxmatch=0): h = htm.HTM(10) if sr is None: sr = np.hypot(c1['sr'], c2['sr']) / 3600 cat1 = catslist.get(c1['name']) cat2 = catslist.get(c2['name']) ra1 = c1['table'][cat1.get('ra', 'RAJ2000')] dec1 = c1['table'][cat1.get('dec', 'DEJ2000')] ra2 = c2['table'][cat2.get('ra', 'RAJ2000')] dec2 = c2['table'][cat2.get('dec', 'DEJ2000')] return h.match(ra1, dec1, ra2, dec2, sr, maxmatch=maxmatch)
from esutil import htm # field 26 (2014:04) -10:00:28.800 +02:12:36.00 # field 40 (2014:10) -10:20:28.800 -06:31:12.00 # field 42 (2014:09) -10:21:52.800 -04:57:00.00 indexer = htm.HTM(depth=8) ras = [ 15. * (10. + (0. / 60.) + (28.800 / 3600.)), 15. * (10. + (20. / 60.) + (28.800 / 3600.)), 15. * (10. + (21. / 60.) + (52.800 / 3600.)) ] decs = [ +(2 + (12. / 60.) + (36. / 3600.)), -(6 + (31. / 60.) + (12.00 / 3600.)), -(4. + (57. / 60.)) ] shards = [] for ra, dec in zip(ras, decs): shards += list(indexer.intersect(ra, dec, 2., True)) print "mkdir ./gaia_HiTS_2015" print "cp -v /datasets/refcats/htm/gaia_DR1_v1/config.py ./gaia_HiTS_2015" print "cp -v /datasets/refcats/htm/gaia_DR1_v1/master_schema.fits ./gaia_HiTS_2015" for shard in shards: print "cp -v gaia_DR1_v1/%i.fits ./gaia_HiTS_2015/" % shard
def add_gaia(data): """ Add GAIA data to input structure, with 2MASS match and coordinate match to (cross-matched) GAIA reference file """ # get the GAIA data from both matches gaia_twomass, gaia_posn = getdata(data) # add new columns tab = Table(data) in_names = ('source_id', 'parallax', 'parallax_error', 'pmra', 'pmra_error', 'pmdec', 'pmdec_error', 'phot_g_mean_mag', 'phot_bp_mean_mag', 'phot_rp_mean_mag', 'a_g_val', 'e_bp_min_rp_val', 'radial_velocity', 'radial_velocity_error', 'r_est', 'r_lo', 'r_hi') dtypes = ('i8', 'f8', 'f8', 'f8', 'f8', 'f8', 'f8', 'f4', 'f4', 'f4', 'f4', 'f4', 'f8', 'f8', 'f8', 'f8', 'f8') out_names = [] for name in in_names: out_names.append(('gaia_' + name).upper()) # initialize newcols = Table(np.zeros([len(tab), len(out_names)]) - 9999., names=out_names, dtype=dtypes) # for source_id, default to 0, not -9999. newcols['GAIA_SOURCE_ID'] = 0 # rename targetting proper motions to avoid confusion! try: tab.rename_column('PMRA', 'TARG_PMRA') except: pass try: tab.rename_column('PMDEC', 'TARG_PMDEC') except: pass try: tab.rename_column('PM_SRC', 'TARG_PM_SRC') except: pass # add unpopulated columns tab.add_columns(newcols.columns.values()) # remove dups in GAIA twomass in favor of brightest print('number in GAIA-2MASS xmatch catalog: ', len(gaia_twomass), len(set(gaia_twomass['original_ext_source_id']))) ind = [] for tid in set(gaia_twomass['original_ext_source_id']): j = np.where(gaia_twomass['original_ext_source_id'] == tid)[0] if len(j) > 1: ii = np.argsort(gaia_twomass['phot_rp_mean_mag'][j]) ind.append(j[ii[0]]) print('duplicate 2MASS: ', gaia_twomass['phot_rp_mean_mag'][j[ii]]) else: ind.append(j) # read gaia 2MASS matched file, match by 2MASS ID, and populate while True: # loop for matches since we may have repeats and want them all matched j = np.where(tab['GAIA_SOURCE_ID'] == 0)[0] print('Number missing gaia_source_id: ', len(j)) if len(j) == 0: break m1, m2 = match.match( np.core.defchararray.replace(tab['APOGEE_ID'][j], b'2M', b''), gaia_twomass['original_ext_source_id']) print('Number matched by 2MASS: ', len(m1)) if len(m1) == 0: break for inname, outname in zip(in_names, out_names): tab[outname][j[m1]] = gaia_twomass[inname][m2] j = np.where(tab['GAIA_SOURCE_ID'] > 0)[0] print('number of unique APOGEE_ID matches: ', len(set(tab['APOGEE_ID'][j]))) j = np.where(tab['GAIA_SOURCE_ID'] == 0)[0] print('missing sources after 2MASS matches: ', len(j)) h = htm.HTM() # now do a positional match, take the brightest object within 3 arcsec (which is the max from the GAIA crossmatch) maxrad = 3. / 3600. m1, m2, rad = h.match(tab['RA'][j], tab['DEC'][j], gaia_posn['ra'], gaia_posn['dec'], maxrad, maxmatch=10) for m in set(m1): jj = np.where(m1 == m)[0] ii = np.argsort(gaia_posn['phot_rp_mean_mag'][m2[jj]]) for inname, outname in zip(in_names, out_names): tab[outname][j[m]] = gaia_posn[inname][m2[jj[ii[0]]]] j = np.where(tab['GAIA_SOURCE_ID'] == 0)[0] print('missing sources after second match: ', len(j)) # replace NaNs for name in out_names: bd = np.where(np.isnan(tab[name]))[0] tab[name][bd] = -9999. return tab
def get_zero_point_simple(obj, cat, sr0=50.0, filter='Clear', maglim=None, spatial=False): zero = {} h = htm.HTM(10) m = h.match(obj['ra'], obj['dec'], cat['ra'], cat['dec'], sr0 * 1.0 / 3600, maxmatch=0) a = [0, 0, 0] sd = 0.0 oidx = m[0] cidx = m[1] dist = m[2] * 3600 maxdist = 3.0 * np.std(dist) print "Max distance = %g arcsec" % maxdist zero['maxdist'] = maxdist if filter == 'B': mag_cat = cat['b'][cidx] elif filter == 'V': mag_cat = cat['v'][cidx] elif filter == 'R': mag_cat = cat['r'][cidx] elif filter == 'U': mag_cat = cat['u'][cidx] else: mag_cat = cat['v'][cidx] mag_instr = obj['mag'][oidx] x = obj['x'][oidx] y = obj['y'][oidx] zero['mag_cat'] = mag_cat zero['mag_instr'] = mag_instr zero['dist'] = dist zero['x'] = x zero['y'] = y #weights = 10**(-0.4*mag_cat) weights = np.repeat(1.0, len(mag_cat)) idx = (dist < 3.0 * np.std(dist)) if maglim: idx = np.logical_and(idx, mag_cat < maglim) mag0 = 0 for iter in range(3): print len(dist[idx]) if spatial and len(dist[idx]) < 5: break elif len(dist[idx]) < 2: break sigma = np.std(mag_instr[idx] - mag_cat[idx]) mag0 = np.mean(mag_instr[idx] - mag_cat[idx]) #idx = np.logical_and(idx, np.abs(mag_instr - mag_v - mag0) < 3.0*sigma) if spatial: X = np.vstack([np.ones(len(mag_instr)), x * x, x, x * y, y, y * y]).T else: X = np.vstack([np.ones(len(mag_instr))]).T Y = mag_cat - mag_instr C2 = sm.WLS(Y[idx], X[idx], weights=weights[idx]).fit().params if spatial: mag0 = C2[0] + C2[1] * x * x + C2[2] * x + C2[3] * x * y + C2[ 4] * y + C2[5] * y * y else: mag0 = np.repeat(C2[0], len(x)) sigma2 = np.std(mag0[idx] + mag_instr[idx] - mag_cat[idx]) print C2, sigma2 idx = np.logical_and(idx, np.abs(mag0 + mag_instr - mag_cat) < 3.0 * sigma2) zero['C2'] = C2 zero['mag0s'] = mag0 zero['sigma'] = sigma2 zero['sigma0'] = np.std(mag0 + mag_instr - mag_cat) zero['idx'] = idx if spatial: x = obj['x'] y = obj['y'] mag0 = C2[0] + C2[1] * x * x + C2[2] * x + C2[3] * x * y + C2[ 4] * y + C2[5] * y * y else: mag0 = C2[0] zero['mag0'] = mag0 zero['mag'] = obj['mag'] + mag0 return zero
def get_zero_point(obj, cat, sr0=50.0, maglim=None): zero = {} h = htm.HTM(10) m = h.match(obj['ra'], obj['dec'], cat['ra'], cat['dec'], sr0 * 1.0 / 3600, maxmatch=0) a = [0, 0, 0] sd = 0.0 oidx = m[0] cidx = m[1] dist = m[2] * 3600 maxdist = 3.0 * np.std(dist) print "Max distance = %g arcsec" % maxdist zero['maxdist'] = maxdist mag_b = cat['b'][cidx] mag_v = cat['v'][cidx] mag_r = cat['r'][cidx] mag_bv = mag_b - mag_v mag_vr = mag_v - mag_r mag_instr = obj['mag'][oidx] x = obj['x'][oidx] y = obj['y'][oidx] zero['mag_b'] = mag_b zero['mag_v'] = mag_v zero['mag_r'] = mag_r zero['mag_instr'] = mag_instr weights = 10**(-0.4 * mag_v) idx = (dist < 3.0 * np.std(dist)) if maglim: idx = np.logical_and(idx, mag_v < 8) for iter in range(3): print len(dist[idx]) sigma = np.std(mag_instr[idx] - mag_v[idx]) mag0 = np.mean(mag_instr[idx] - mag_v[idx]) #idx = np.logical_and(idx, np.abs(mag_instr - mag_v - mag0) < 3.0*sigma) #plt.clf(); plt.scatter(mag_v[idx], mag_instr[idx]) #X = np.vstack([np.ones(len(mag_instr)), x*x, x, x*y, y, y*y, mag_bv, mag_bv**2, mag_vr, mag_vr**2]).T #X = np.vstack([np.ones(len(mag_instr)), x*x, x, x*y, y, y*y]).T X = np.vstack([np.ones(len(mag_instr))]).T Y = mag_v - mag_instr C2 = sm.WLS(Y[idx], X[idx], weights=weights[idx]).fit().params print sm.WLS(Y[idx], X[idx], weights=weights[idx]).fit().summary() #mag0 = C2[0] + C2[1]*x*x + C2[2]*x + C2[3]*x*y + C2[4]*y + C2[5]*y*y + C2[6]*mag_bv + C2[7]*mag_bv**2 + C2[8]*mag_vr + C2[9]*mag_vr**2 #mag0 = C2[0] + C2[1]*x*x + C2[2]*x + C2[3]*x*y + C2[4]*y + C2[5]*y*y mag0 = np.repeat(C2[0], len(x)) sigma2 = np.std(mag0[idx] + mag_instr[idx] - mag_v[idx]) print C2, sigma2 idx = np.logical_and(idx, np.abs(mag0 + mag_instr - mag_v) < 3.0 * sigma2) # plt.clf() # plt.scatter(mag_instr, mag0 + mag_instr - mag_v, s=1) # plt.scatter(mag_instr[idx], mag0[idx] + mag_instr[idx] - mag_v[idx], marker='.', c='red', s=3) pass zero['mag0'] = C2 return zero
def stack(infile='mastar-goodspec-v2_7_1-trunk', apogee='r13/l33/allStar-r13-l33-58932', out='mastar_apogee_stack', cm=False): """ Create list of MaStar objects with matching APOGEE spectra and stack multiple observations into a single Add in model spectra for warmer stars """ # read MaStar file and filter mastar = fits.open(infile + '.fits')[1].data #gd = clean(mastar,plot=False) #mastar=mastar[gd] # read APOGEE file and filter apogee = fits.open(os.environ['APOGEE_ASPCAP'] + '/' + apogee + '.fits')[1].data gd = apselect.select(apogee, badval=['STAR_BAD']) apogee = apogee[gd] # get matches within 3 arcsec h = htm.HTM() maxrad = 3. / 3600. m1, m2, rad = h.match(apogee['RA'], apogee['DEC'], mastar['OBJRA'], mastar['OBJDEC'], maxrad, maxmatch=10) # other stars to remove, e.g., based on bad modeling in initial model bdstars = [] """ bdstars=['3-15080992','3-105812093','7-12951805', '3-142131061','60-36093244935984954','3-126079790', '60-36091793237038712','3-126306258','3-24068365', '3-23593772','60-36093244935984954', '60-36091793237038712', '3-140608451', '7-14409160', '3-114437013', '7-27039482', '3-15080992', '3-148139562', '3-143029659', '3-103574358', '3-139462126', '3-136745748', '3-142131061', '3-142325470', '3-137769951', '3-141476933', '3-23120750', '60-3615860471750191104', '7-17756858', '3-49507807', '3-36341647', '3-22166243', '7-9789478', '3-124777940', '60-676895989836309632', '3-141873923', '3-140806405', '3-21696715', '7-7407046', '3-126309023', '3-125275478'], """ allbd = [] ngd = 0 nlo = 0 lo = [] spec = [] param = [] fparam = [] name = [] stars = set(mastar['MANGAID']) nrot = 5 vsinis = 1. + np.arange(nrot) * np.log10(2.5) deltav = (np.log(mastar['WAVE'][0, 1]) - np.log(mastar['WAVE'][0, 0])) * 3.e5 for star in stars: if star in bdstars: continue j = np.where(mastar['MANGAID'] == star)[0] print(star, len(j)) # create weighted average spectrum num = np.sum(mastar['FLUX'][j, :] * mastar['IVAR'][j, :], axis=0) den = np.sum(mastar['IVAR'][j, :], axis=0) avg = num / den err = 1 / den sn = avg / err # where S/N<2, set flux to 0 faint = np.where(scipy.signal.medfilt(sn, 21) < 2.)[0] avg[faint] = 0. if (sn[200:-200].min() < 10): #print(star,apogee['PARAM'][j[0],:]) lo.append(j[0]) nlo += 1 # only take spectra with no bad pixels (neglecting first and last 10) bd = np.where(np.isinf(err[10:-10]))[0] if len(bd) < 1: allbd.extend(bd.tolist()) spec.append(avg) par = apogee['FPARAM'][j[0], :] if cm: fparam.append([par[0], par[1], par[3], par[6], par[7], par[4]]) else: fparam.append([par[0], par[1], par[3], par[6], par[7]]) par = apogee['PARAM'][j[0], :] elem = apogee['X_M'][j[0], :] if cm: param.append([par[0], par[1], par[3], elem[5], par[7], par[4]]) else: param.append([par[0], par[1], par[3], elem[5], par[7]]) name.append(star) ngd += 1 # if a slow rotator, broaden to accommodate different vsini if 10.**par[7] < 10.: for irot, vsini in enumerate(vsinis[1:]): print(vsini) rot = rotate(deltav, 10.**vsini) conv = np.convolve(avg, rot, mode='same') spec.append(conv) if cm: fparam.append( [par[0], par[1], par[3], par[6], vsini, par[4]]) else: fparam.append([par[0], par[1], par[3], par[6], vsini]) if cm: param.append( [par[0], par[1], par[3], elem[5], vsini, par[4]]) else: param.append([par[0], par[1], par[3], elem[5], vsini]) name.append('{:s}_{:.1f}'.format(star, vsini)) param = np.array(param) fparam = np.array(fparam) print('Number of good stars: ', ngd) print('Number of bad spectra: ', len(allbd), len(set(allbd))) # plots fig, ax = plots.multi(2, 2) plots.plotc(ax[0, 0], param[:, 0], param[:, 1], param[:, 2], xr=[8000, 3000], yr=[6, -1], zr=[-2, 0.5], xt='Teff', yt='logg', zt='[M/H]', colorbar=True) ax[0, 0].text(0.1, 0.9, '[M/H]', transform=ax[0, 0].transAxes) plots.plotc(ax[1, 0], param[:, 0], param[:, 1], param[:, 3], xr=[8000, 3000], yr=[6, -1], zr=[-0.5, 0.5], xt='Teff', yt='logg', zt='[alpha/M]', colorbar=True) ax[1, 0].text(0.1, 0.9, '[alpha/M]', transform=ax[1, 0].transAxes) plots.plotc(ax[0, 1], param[:, 2], param[:, 3], param[:, 0], zr=[3500, 6000], xr=[-2.5, 1], yr=[-0.5, 0.5], zt='Teff', xt='[M/H]', yt='[alpha/M]', colorbar=True) ax[1, 0].text(0.1, 0.9, '[alpha/M]', transform=ax[1, 0].transAxes) if cm: plots.plotc(ax[1, 1], param[:, 0], param[:, 1], param[:, 5], xr=[8000, 3000], yr=[6, -1], zr=[-0.5, 0.5], xt='Teff', yt='logg', zt='[C/M]', colorbar=True) ax[1, 1].text(0.1, 0.9, '[C/M]', transform=ax[1, 1].transAxes) #plots.plotc(ax[0,1],fparam[:,0],fparam[:,1],fparam[:,2],xr=[8000,3000],yr=[6,-1],zr=[-2,0.5],xt='uncal Teff',yt='uncal logg',zt='uncal [M/H]') #plots.plotc(ax[1,1],fparam[:,0],fparam[:,1],fparam[:,3],xr=[8000,3000],yr=[6,-1],zr=[-0.5,0.5],xt='uncal Teff',yt='uncal logg',zt='uncal [alpha/H]') fig.tight_layout() fig.savefig(out + '.png') #output FITS file tab = Table() tab.add_column(Column(name='TEFF', data=param[:, 0])) tab.add_column(Column(name='LOGG', data=param[:, 1])) tab.add_column(Column(name='[M/H]', data=param[:, 2])) tab.add_column(Column(name='[alpha/M]', data=param[:, 3])) tab.add_column(Column(name='LOG(VSINI)', data=param[:, 4])) tab.add_column(Column(name='MANGAID', data=np.array(name))) if cm: tab.add_column(Column(name='[C/M]', data=param[:, 5])) tab.add_column(Column(name='SPEC', data=np.array(spec))) tab.meta['LABELS'] = ['TEFF', 'LOGG', '[M/H]', '[alpha/M]', 'LOG(VSINI)'] tab.write(out + '.fits', overwrite=True) # add in model spectra a = [] b = [] c = [] d = [] e = [] f = [] v = [] for file in ['f_nsc2', 'f_nsc3', 'f_nsc4', 'f_nsc5']: synth = fits.open(file + '_rot.fits')[0] for irot, vsini in enumerate(spectra.fits2vector(synth.header, 5)): for imh, mh in enumerate(spectra.fits2vector(synth.header, 4)): if mh < -1: continue for iteff, teff in enumerate( spectra.fits2vector(synth.header, 3)): if teff < 6000: continue for ilogg, logg in enumerate( spectra.fits2vector(synth.header, 2)): if mh >= -2.5: a.append(teff) b.append(logg) c.append(mh) d.append(0.) v.append(vsini) e.append( file + '_{:.0f}_{:.1f}_{:.1f}'.format(teff, logg, mh)) f.append(synth.data[irot, imh, iteff, ilogg, :] / np.median(synth.data[irot, imh, iteff, ilogg, :])) synthtab = Table() synthtab.add_column(Column(name='TEFF', data=np.array(a))) synthtab.add_column(Column(name='LOGG', data=np.array(b))) synthtab.add_column(Column(name='[M/H]', data=np.array(c))) synthtab.add_column(Column(name='[alpha/M]', data=np.array(d))) synthtab.add_column(Column(name='LOG(VSINI)', data=np.array(v))) synthtab.add_column(Column(name='MANGAID', data=np.array(e))) if cm: synthtab.add_column(Column(name='[C/M]', data=np.array(d))) synthtab.add_column(Column(name='SPEC', data=np.array(f))) vstack([tab, synthtab]).write(out + '_synth.fits', overwrite=True)
def find_transients_in_frame(filename, refine=True, favor2=None, workdir=None, figure=None, show=False, verbose=False, plot=True): transients = [] favor2 = favor2 or Favor2() ra0, dec0, sr0 = get_frame_center(filename=filename) print "Frame %s at RA=%g Dec=%g with SR=%g" % (filename, ra0, dec0, sr0) # Dir to hold all temporary files basedir = workdir or tempfile.mkdtemp(prefix='survey') filtered_name = posixpath.join(basedir, "filtered.fits") print "Cleaning the frame" cleaned_name = posixpath.join(basedir, "clean.fits") filter_cosmics_in_file(filename, cleaned_name) header = pyfits.getheader(cleaned_name, -1) time = datetime.datetime.strptime(header['TIME'], '%d.%m.%Y %H:%M:%S.%f') print "Pre-processing the frame" os.system("nebuliser %s noconf %s 3 3 --takeout_sky" % (cleaned_name, filtered_name)) if refine: print "Refining the WCS" matched_name = posixpath.join(basedir, "image.fits") if not favor2.blind_match_file(filtered_name, outfile=matched_name): print "Failure refining the WCS" return transients else: matched_name = filtered_name print "Extracting the objects" obj = ss.get_objects(filename=matched_name, options={ 'DETECT_MINAREA': 3.0, 'DETECT_THRESH': 1.5, 'BACKPHOTO_TYPE': 'GLOBAL', 'DEBLEND_NTHRESH': 64, 'DEBLEND_MINCONT': 0.001, 'FILTER': 'Y', 'BACK_TYPE': 'MANUAL', 'BACK_VALUE': 0.0, 'BACK_SIZE': 64, 'BACK_FILTERSIZE': 3 }, fwhm=2, aper_fwhm=2) print "Querying the catalogues" cat = favor2.get_stars(ra0, dec0, sr0, catalog='twomass', limit=1000000) catn = favor2.get_stars(ra0, dec0, sr0, catalog='nomad', limit=1000000) catt = favor2.get_stars(ra0, dec0, sr0, catalog='tycho2', limit=1000000) cattb = favor2.get_stars(ra0, dec0, sr0, catalog='tycho2', limit=1000000, extra='vt < 7') print "%d objects, %d 2MASS stars, %d NOMAD stars, %d bright Tycho2 stars" % ( len(obj['ra']), len(cat['ra']), len(catn['ra']), len(cattb['ra'])) print "Matching the objects" wcs = pywcs.WCS(header) pixscale = wcs.pixel_scale_matrix.max( ) * 3600 # estimate of pixel scale, in arcsec/pix size = 2.0 * obj['A_WORLD'] * pixscale size = np.maximum(size, np.repeat(40.0, len(obj['ra']))) h = htm.HTM(10) m = h.match(obj['ra'], obj['dec'], cat['ra'], cat['dec'], size / 3600, maxmatch=0) mn = h.match(obj['ra'], obj['dec'], catn['ra'], catn['dec'], size / 3600, maxmatch=0) mt = h.match(obj['ra'], obj['dec'], catt['ra'], catt['dec'], size / 3600, maxmatch=0) if len(cattb['ra']): mtb = h.match(obj['ra'], obj['dec'], cattb['ra'], cattb['dec'], (size + 100.0) / 3600, maxmatch=0) uidx = np.setdiff1d(np.arange(len(obj['ra'])), m[0]) print "%d objects not matched with 2MASS" % len(uidx) uidx = np.setdiff1d(uidx, mn[0]) print "%d objects not matched with NOMAD" % len(uidx) uidx = np.setdiff1d(uidx, mt[0]) print "%d objects not matched with Tycho2" % len(uidx) if len(cattb['ra']): uidx = np.setdiff1d(uidx, mtb[0]) print "%d objects not matched with brightest Tycho2 stars" % len(uidx) if len(uidx) < 100: uidx = filter_indices(uidx, obj, time) print "%d objects after line filtering and GSC 2.3.2 matching" % len( uidx) # Get the image and project the catalogues onto it image = pyfits.getdata(filtered_name, -1) cat_x, cat_y = wcs.all_world2pix(cat['ra'], cat['dec'], 0) catn_x, catn_y = wcs.all_world2pix(catn['ra'], catn['dec'], 0) catt_x, catt_y = wcs.all_world2pix(catt['ra'], catt['dec'], 0) if len(cattb['ra']): cattb_x, cattb_y = wcs.all_world2pix(cattb['ra'], cattb['dec'], 0) if len(uidx) > 100: print "Too many unmatched objects (%d), something is wrong!" % len( uidx) else: # Let's iterate over the transients for i in uidx: transient = { 'x': float(obj['x'][i]), 'y': float(obj['y'][i]), 'flux': float(obj['FLUX_AUTO'][i]), 'flux_err': float(obj['FLUXERR_AUTO'][i]), 'flags': int(obj['FLAGS'][i]), 'a': float(obj['A_WORLD'][i]), 'b': float(obj['B_WORLD'][i]), 'theta': float(obj['THETA_WORLD'][i]), 'ra': float(obj['ra'][i]), 'dec': float(obj['dec'][i]), 'filename': filename, 'time': time, 'channel_id': header['CHANNEL ID'] } transient['simbad'] = None transient['mpc'] = None try: res = Simbad.query_region( SkyCoord(obj['ra'][i], obj['dec'][i], unit='deg'), "30 seconds") if res is not None: transient['simbad'] = ", ".join( [r for r in res['MAIN_ID']]) except: pass try: # Minor planets mpc = get_skybot(obj['ra'][i], obj['dec'][i], 30.0 / 3600, time, objFilter=110) # Comets if mpc is None: mpc = get_skybot(obj['ra'][i], obj['dec'][i], 300.0 / 3600, time, objFilter=1) if mpc is not None: transient['mpc'] = ", ".join([ "%s %s %s (Mv=%s)" % (mpc['class'][ii], mpc['num'][ii], mpc['name'][ii], mpc['Mv'][ii]) for ii in xrange(len(mpc['name'])) ]) except: pass if verbose: #print transient print "x = %g y = %g" % (obj['x'][i], obj['y'][i]) print "RA/Dec = %g %g = %s %s" % ( obj['ra'][i], obj['dec'][i], str(ephem.hours(obj['ra'][i] * np.pi / 180)).replace( ":", " "), str(ephem.degrees( obj['dec'][i] * np.pi / 180)).replace(":", " ")) time0 = time.replace(hour=0, minute=0, second=0, microsecond=0) print "Time = %s" % time print "Time = %s + %g" % (time0, (time - time0).total_seconds() / 24 / 3600) if transient['simbad']: print "SIMBAD: " + transient['simbad'] print if plot: icrop, x0, y0 = crop_image(image, obj['x'][i], obj['y'][i], 40) limits = np.percentile(icrop, [0.5, 99.5]) if not figure: size = 400 figure = Figure(facecolor='white', dpi=72, figsize=(size / 72, size / 72), tight_layout=True) figure.clear() ax = figure.add_axes([0, 0, 1, 1]) ax.axis('off') im = ax.imshow(icrop, origin="bottom", interpolation="nearest", cmap="hot", vmin=limits[0], vmax=limits[1]) #figure.colorbar(im) ax.autoscale(False) ax.scatter(obj['x'] - x0, obj['y'] - y0, marker='o', color='green', s=400, facecolors='None', lw=3) ax.scatter(cat_x - x0, cat_y - y0, marker='o', color='blue', s=100, facecolors='None', lw=2) ax.scatter(catt_x - x0, catt_y - y0, marker='x', s=100, color='blue', facecolors='None', lw=2) #ellipse = Ellipse(xy=(obj['x'][i]-x0, obj['y'][i]-y0), width=2.0*obj['A_WORLD'][i], height=2.0*obj['B_WORLD'][i], angle=obj['THETA_WORLD'][i], edgecolor='b', fc='None', axes=ax) #ax.add_patch(ellipse) if show: figure.show(warn=False) else: canvas = FigureCanvas(figure) sio = StringIO.StringIO() canvas.print_png(sio, bbox_inches='tight') transient[ 'preview'] = "data:image/png;base64," + base64.b64encode( sio.getvalue()) pass else: transient['preview'] = None transients.append(transient) # cleanup if not workdir: shutil.rmtree(basedir) print "Done" return transients
def match_objects(obj, cat, sr, fname='V', order=4, bg_order=None, cmag=None, color_order=None, thresh=5.0, clim=None, sn=10, mag_idx=0): x0,y0,width,height = np.mean(obj['x']), np.mean(obj['y']), np.max(obj['x'])-np.min(obj['x']), np.max(obj['y'])-np.min(obj['y']) # Match stars h = htm.HTM(10) m = h.match(obj['ra'],obj['dec'], cat['ra'],cat['dec'], sr, maxmatch=0) oidx = m[0] cidx = m[1] dist = m[2] if fname == 'B': cmag,cmagerr = cat['B'], cat['Berr'] elif fname == 'V': cmag,cmagerr = cat['V'], cat['Verr'] elif fname == 'R' or fname == 'N': cmag,cmagerr = cat['R'], cat['Rerr'] elif fname == 'I': cmag,cmagerr = cat['I'], cat['Ierr'] elif fname == 'g': cmag,cmagerr = cat['g'], cat['gerr'] elif fname == 'r': cmag,cmagerr = cat['r'], cat['rerr'] elif fname == 'i': cmag,cmagerr = cat['i'], cat['ierr'] elif fname == 'z': cmag,cmagerr = cat['z'], cat['zerr'] color = cat['B'] - cat['V'] if cmag is not None: cmag = cmag[cidx] cmagerr = cmagerr[cidx] color = color[cidx] else: print('Unsupported filter:', fname) return None # Optimal fit limit, as mean mag where S/N = 10 idx = (1.0/obj['magerr'][oidx] > 5) & (1.0/obj['magerr'][oidx] < 15) if np.sum(idx) > 10: X = make_series(1.0, obj['x'][oidx], obj['y'][oidx], order=order) X = np.vstack(X).T Y = cmag weights = 1/obj['magerr'][oidx]**2 Cmag_lim = sm.WLS(Y[idx], X[idx], weights=weights[idx]).fit() cmag_lim = np.sum(X*Cmag_lim.params, axis=1) else: # print('Not enough matches with SN~10:', np.sum(idx)) Cmag_lim = None cmag_lim = 99.0*np.ones_like(cmag) # User provided upper fit limit if clim is not None: idx = cmag < clim oidx,cidx,dist,cmag,cmagerr,cmag_lim,color = [_[idx] for _ in [oidx,cidx,dist,cmag,cmagerr,cmag_lim,color]] x,y = obj['x'][oidx],obj['y'][oidx] oflags = obj['flags'][oidx] omag,omagerr = obj['mag'][oidx],obj['magerr'][oidx] if len(obj['mag'].shape) > 1: omag,omagerr = omag[:,mag_idx],omagerr[:,mag_idx] x = (x - x0)*2/width y = (y - y0)*2/height tmagerr = np.hypot(cmagerr, omagerr) tmagerr = np.hypot(tmagerr, 0.02) delta_mag = cmag - omag weights = 1.0/tmagerr**2 X = make_series(1.0, x, y, order=order) if bg_order is not None and bg_order >= 0: X += make_series(-2.5/np.log(10)/10**(-0.4*omag), x, y, order=bg_order) if color_order is not None and color_order >= 0: X += make_series(color, x, y, order=color_order) X = np.vstack(X).T Y = delta_mag # X /= tmagerr[:, None] # Y /= tmagerr idx0 = (oflags == 0) & (cmag < cmag_lim) if sn and sn>0: idx0 &= (omagerr < 1/sn) idx = idx0.copy() # Do the fitting for iter in range(3): if len(X[idx]) < 3: print("Fit failed - %d objects" % len(X[idx])) return None C = sm.WLS(Y[idx], X[idx], weights=weights[idx]).fit() # C = sm.RLM(Y[idx], X[idx]).fit() YY = np.sum(X*C.params, axis=1) idx = idx0.copy() if thresh and thresh > 0: idx &= (np.abs((Y-YY)/tmagerr) < thresh) # Apply the fit for all objects x = (obj['x'] - x0)*2/width y = (obj['y'] - y0)*2/height if len(obj['mag'].shape) > 1: mag = obj['mag'][:,mag_idx] magerr = obj['magerr'][:,mag_idx] else: mag = obj['mag'] magerr = obj['magerr'] # Magnitude estimation for every position X = make_series(1.0, x, y, order=order) if bg_order is not None and bg_order >= 0: X += make_series(-2.5/np.log(10)/10**(-0.4*obj['mag']), x, y, order=bg_order) X = np.vstack(X).T YY1 = np.sum(X*C.params[0:X.shape[1]], axis=1) if len(obj['mag'].shape) > 1: mag = obj['mag'][:,mag_idx] + YY1 else: mag = obj['mag'] + YY1 # Additive flux component for every position if bg_order is not None and bg_order >= 0: Xbg = make_series(1, x, y, order=bg_order) Xbg = np.vstack(Xbg).T delta_flux = np.sum(Xbg*C.params[X.shape[1]-Xbg.shape[1]:X.shape[1]], axis=1) else: delta_flux = np.zeros_like(mag) # Color term for every position if color_order is not None and color_order >= 0: Xc = make_series(1, x, y, order=color_order) Xc = np.vstack(Xc).T Cbv = np.sum(Xc*C.params[X.shape[1]:], axis=1) else: Cbv = np.zeros_like(mag) # Approx magnitude limit (S/N=10) at every position if Cmag_lim is not None: Xlim = make_series(1.0, x, y, order=order) Xlim = np.vstack(Xlim).T cmag_lim = np.sum(Xlim*Cmag_lim.params, axis=1) else: cmag_lim = 99.0*np.ones_like(x) # Simple analysis of proximity to "good" points mx,my = obj['x'][oidx][idx], obj['y'][oidx][idx] kdo = cKDTree(np.array([obj['x'], obj['y']]).T) kdm = cKDTree(np.array([mx, my]).T) mr0 = np.sqrt(width*height/np.sum(idx)) m = kdm.query_ball_tree(kdm, 5.0*mr0) dists = [] for i,ii in enumerate(m): if len(ii) > 1: d1 = [np.hypot(mx[i]-mx[_], my[i]-my[_]) for _ in ii] d1 = np.sort(d1) dists.append(d1[1]) mr1 = np.median(dists) m = kdo.query_ball_tree(kdm, 5.0*mr1) midx = np.array([len(_)>1 for _ in m]) return { # Matching indices and a distance in degrees 'cidx':cidx, 'oidx':oidx, 'dist':dist, # Pixel and sky coordinates of matched stars, as well as their flags 'x':obj['x'][oidx], 'y':obj['y'][oidx], 'flags':oflags, 'ra':obj['ra'][oidx], 'dec':obj['dec'][oidx], # All catalogue magnitudes of matched stars 'cB':cat['B'][cidx], 'cV':cat['V'][cidx], 'cR':cat['R'][cidx], 'cI':cat['I'][cidx], 'cBerr':cat['Berr'][cidx], 'cVerr':cat['Verr'][cidx], 'cRerr':cat['Rerr'][cidx], 'cIerr':cat['Ierr'][cidx], # Catalogue magnitudes of matched stars in proper filter 'cmag':cmag, 'cmagerr':cmagerr, 'tmagerr':tmagerr, # Model zero point for all objects, and their corrected magnitudes 'mag0':YY1, 'mag':mag, 'Cbv':Cbv, 'delta_flux': delta_flux, 'mag_idx': mag_idx, 'Y':Y, 'YY':YY, 'cmag_lim':cmag_lim, # Subset of matched stars used in the fits 'idx':idx, # Subset of all objects at 'good' distances from matched ones 'midx':midx, 'mr0':mr0, 'mr1':mr1}
def add_gaia(data, gaia_1='gaia_2mass_xmatch.fits.gz', gaia_2='gaia_posn_xmatch.fits.gz'): """ Add GAIA data to allStar file, with coordinate match to (cross-matched) GAIA reference file """ tab = Table(data) in_names = ('source_id', 'parallax', 'parallax_error', 'pmra', 'pmra_error', 'pmdec', 'pmdec_error', 'phot_g_mean_mag', 'phot_bp_mean_mag', 'phot_rp_mean_mag', 'a_g_val', 'e_bp_min_rp_val', 'radial_velocity', 'radial_velocity_error', 'r_est', 'r_lo', 'r_hi') dtypes = ('i8', 'f8', 'f8', 'f8', 'f8', 'f8', 'f8', 'f4', 'f4', 'f4', 'f4', 'f4', 'f8', 'f8', 'f8', 'f8', 'f8') out_names = [] for name in in_names: out_names.append(('gaia_' + name).upper()) newcols = Table(np.zeros([len(tab), len(out_names)]) - 9999., names=out_names, dtype=dtypes) # for source_id, default to 0, not -9999. newcols['GAIA_SOURCE_ID'] = 0 # get rid of targetting proper motions to avoid confusion! #tab.remove_columns(['PMRA','PMDEC','PM_SRC']) # change to rename tab.rename_column('PMRA', 'TARG_PMRA') tab.rename_column('PMDEC', 'TARG_PMDEC') tab.rename_column('PM_SRC', 'TARG_PM_SRC') # add unpopulated columns tab.add_columns(newcols.columns.values()) # read gaia 2MASS matched file, match by 2MASS ID, and populate gaia = fits.open(gaia_1)[1].data print('number in GAIA-2MASS xmatch catalog: ', len(gaia), len(set(gaia['original_ext_source_id']))) while True: # loop for matches since we have repeats and want them all matched j = np.where(tab['GAIA_SOURCE_ID'] == 0)[0] print('Number missing gaia_source_id: ', len(j)) m1, m2 = match.match( np.core.defchararray.replace(tab['APOGEE_ID'][j], '2M', ''), gaia['original_ext_source_id']) print('Number matched by 2MASS: ', len(m1)) if len(m1) == 0: break for inname, outname in zip(in_names, out_names): tab[outname][j[m1]] = gaia[inname][m2] #if len(m1) < 100 : # for i in m1 : print(tab['APOGEE_ID'][j[m1]]) j = np.where(tab['GAIA_SOURCE_ID'] > 0)[0] print('number of unique APOGEE_ID matches: ', len(set(tab['APOGEE_ID'][j]))) j = np.where(tab['GAIA_SOURCE_ID'] == 0)[0] print('missing sources after 2MASS matches: ', len(j)) gaia = fits.open(gaia_2)[1].data h = htm.HTM() # now do a positional match, take the brightest object within 3 arcsec (which is the max from the GAIA crossmatch) maxrad = 3. / 3600. m1, m2, rad = h.match(tab['RA'][j], tab['DEC'][j], gaia['RA'], gaia['DEC'], maxrad, maxmatch=10) for m in set(m1): jj = np.where(m1 == m)[0] ii = np.argsort(gaia['phot_rp_mean_mag'][m2[jj]]) #print(tab['RA'][j[m]],tab['DEC'][j[m]],tab['TARGET_ID'][j[m]]) #print(gaia['RA'][m2[jj]],gaia['DEC'][m2[jj]],gaia['PHOT_RP_MEAN_MAG'][m2[jj]]) #print(ii) for inname, outname in zip(in_names, out_names): tab[outname][j[m]] = gaia[inname][m2[jj[ii[0]]]] j = np.where(tab['GAIA_SOURCE_ID'] == 0)[0] print('missing sources after second match: ', len(j)) # replace NaNs for name in out_names: bd = np.where(np.isnan(tab[name]))[0] tab[name][bd] = -9999. return tab
def htm_matching(cat_A, cat_B, coord_A_names, coord_B_names, radius, exclude_multiple=0, n_neigh=1,depth=10): """ Performs matching between two catalogs with the esutil HTM module. This pipeline matches two catalogs in coordinates specified as input, excluding unmatched objects in A and finding the neighbors in catalog B to each of catalog A objects. It then returns cat_A and cat_B reordered so that the neighbors are in the same line. For objects in A with more than one nearby neighbor, entries are repeated. There are two additional options concerning cases where there's multiple matching. The first (default is 0) excludes multiple matching cases (i.e. cases where the matching is not unique in both senses). The second option (default is 1), selects the number of neighbors to be kept. The value 0 will keep all the neighbors within the given radius. For more information on the details of the procedure, refer to the documentation of the esutil library in http://code.google.com/p/esutil/. Input: - cat_A hdudata : FITS Table data catalog - cat_B hdudata : FITS Table data catalog - coord_A_names [str,str] : ra_A and dec_A column names - coord_B_names [str,str] : ra_B and dec_B column names - radius float : Radius to perform matching - exclude_multiple int : Keeps only the neighbor in the case of single matching if not 0. If 0, gets nearest of multiple neighbors. Overrides n_neigh choice by definition. - n_neigh int : Number of neighbors to keep for each cat_A entry. 0 keeps all within radius. Default is 1. - depth int : Depth of the HTM triangular levels. Ranges from 1 to 12. Default is 10. Output: - cat_A_matched hdudata : FITS Table data catalog of matched objs - cat_B_matched hdudata : FITS Table data catalog of matched objs --- """ # Create HTM triangles t0 = time() print "Performing matching between catalogs..." h = htm.HTM(depth) cat_A_ra = cat_A.field(coord_A_names[0]) cat_A_dec = cat_A.field(coord_A_names[1]) cat_B_ra = cat_B.field(coord_B_names[0]) cat_B_dec = cat_B.field(coord_B_names[1]) print "Number of objects in catalog A: " + str(len(cat_A_ra)) print "Number of objects in catalog B: " + str(len(cat_B_ra)) # Exclude objects with multiple matching if exclude_multiple: print "Excluding objects in catalog A that have multiple matches..." m1,m2,d12 = h.match(cat_A_ra, cat_A_dec, cat_B_ra, cat_B_dec, radius, maxmatch = 0) mltp = (np.arange(len(m2)) < len(m2)) for i in range(len(m2)): if (m2[i] == m2).sum() > 1: mltp = ~(m2[i] == m2)&mltp else: pass m1 = m1[mltp] m2 = m2[mltp] d12 = d12[mltp] else: m1,m2,d12 = h.match(cat_A_ra, cat_A_dec, cat_B_ra, cat_B_dec, radius, maxmatch = n_neigh) # Apply matching to catalogs cat_A_matched = cat_A[m1] cat_B_matched = cat_B[m2] print "Number of objects in final matched catalogs: " + str(len(m2)) print "Total matching time: " + str(time()-t0) +" sec" return cat_A_matched, cat_B_matched
def process_file(filename, favor2=None, verbose=False, replace=False, dbname=None, dbhost=None, photodir='photometry'): #### Some parameters aper = 2.0 bkgann = None order = 4 bg_order = 4 color_order = 2 sn = 5 if not posixpath.exists(filename): return None # Rough but fast checking of whether the file is already processed if not replace and posixpath.exists(photodir + '/' + filename.split('/')[-2] + '/' + posixpath.splitext(posixpath.split(filename)[-1])[0] + '.cat'): return #### Preparation header = fits.getheader(filename, -1) if header['TYPE'] not in ['survey', 'imaging', 'widefield', 'Swift', 'Fermi', 'test']: return channel = header.get('CHANNEL ID') fname = header.get('FILTER', 'unknown') time = parse_time(header['TIME']) shutter = header.get('SHUTTER', -1) if fname not in ['Clear']: return if fname == 'Clear': effective_fname = 'V' else: effective_fname = fname night = get_night(time) dirname = '%s/%s' % (photodir, night) basename = posixpath.splitext(posixpath.split(filename)[-1])[0] basename = dirname + '/' + basename catname = basename + '.cat' if not replace and posixpath.exists(catname): return if verbose: print(filename, channel, night, fname, effective_fname) image = fits.getdata(filename, -1).astype(np.double) if favor2 is None: favor2 = Favor2(dbname=options.db, dbhost=options.dbhost) #### Basic calibration darkname = favor2.find_image('masterdark', header=header, debug=False) flatname = favor2.find_image('masterflat', header=header, debug=False) if darkname: dark = fits.getdata(darkname) else: dark = None if flatname: flat = fits.getdata(flatname) else: flat = None if dark is None or flat is None: survey.save_objects(catname, None) return image,header = calibrate.calibrate(image, header, dark=dark) # Check whether the calibration failed if 'SATURATE' not in header: print('Calibration failed for', filename) survey.save_objects(catname, None) return #### Basic masking mask = image > 0.9*header['SATURATE'] fmask = ~np.isfinite(flat) | (flat < 0.5) dmask = dark > 10.0*mad_std(dark) + np.nanmedian(dark) if header.get('BLEMISHCORRECTION', 1): # We have to mask blemished pixels blemish = fits.getdata('calibrations/blemish_shutter_%d_channel_%d.fits' % (header['SHUTTER'], header['CHANNEL ID'])) if verbose: print(100*np.sum(blemish>0)/blemish.shape[0]/blemish.shape[1], '% pixels blemished') dmask |= blemish > 0 image[~fmask] *= np.median(flat[~fmask])/flat[~fmask] #### WCS wcs = WCS(header) pixscale = np.hypot(wcs.pixel_scale_matrix[0,0], wcs.pixel_scale_matrix[0,1]) gain = 0.67 if header.get('SHUTTER') == 0 else 1.9 #### Background mask mask_bg = np.zeros_like(mask) mask_segm = np.zeros_like(mask) # bg2 = sep.Background(image, mask=mask|mask_bg, bw=64, bh=64) # for _ in xrange(3): # bg1 = sep.Background(image, mask=mask|mask_bg, bw=256, bh=256) # ibg = bg2.back() - bg1.back() # tmp = np.abs(ibg - np.median(ibg)) > 5.0*mad_std(ibg) # mask_bg |= survey.dilate(tmp, np.ones([50, 50])) # mask_bg = survey.dilate(tmp, np.ones([50, 50])) # Large objects?.. bg = sep.Background(image, mask=mask|dmask|fmask|mask_bg|mask_segm, bw=128, bh=128) image1 = image - bg.back() obj0,segm = sep.extract(image1, err=bg.rms(), thresh=10, minarea=10, mask=mask|dmask|fmask|mask_bg, filter_kernel=None, clean=False, segmentation_map=True) mask_segm = np.isin(segm, [_+1 for _,npix in enumerate(obj0['npix']) if npix > 500]) mask_segm = survey.dilate(mask_segm, np.ones([20, 20])) if np.sum(mask_bg|mask_segm|mask|fmask|dmask)/mask_bg.shape[0]/mask_bg.shape[1] > 0.4: print(100*np.sum(mask_bg|mask_segm|mask|fmask|dmask)/mask_bg.shape[0]/mask_bg.shape[1], '% of image masked, skipping', filename) survey.save_objects(catname, None) return elif verbose: print(100*np.sum(mask_bg|mask_segm|mask|fmask|dmask)/mask_bg.shape[0]/mask_bg.shape[1], '% of image masked') # Frame footprint at +10 pixels from the edge ra,dec = wcs.all_pix2world([10, 10, image.shape[1]-10, image.shape[1]-10], [10, image.shape[0]-10, image.shape[0]-10, 10], 0) footprint = "(" + ",".join(["(%g,%g)" % (_,__) for _,__ in zip(ra, dec)]) + ")" #### Catalogue ra0,dec0,sr0 = survey.get_frame_center(header=header) cat = favor2.get_stars(ra0, dec0, sr0, catalog='gaia', extra=['g<14', 'q3c_poly_query(ra, dec, \'%s\'::polygon)' % footprint], limit=1000000) if verbose: print(len(cat['ra']), 'star positions from Gaia down to g=%.1f mag' % np.max(cat['g'])) ## Detection of blended and not really needed stars in the catalogue h = htm.HTM(10) m = h.match(cat['ra'], cat['dec'], cat['ra'], cat['dec'], 2.0*aper*pixscale, maxmatch=0) m = [_[m[2]>1e-5] for _ in m] blended = np.zeros_like(cat['ra'], dtype=np.bool) notneeded = np.zeros_like(cat['ra'], dtype=np.bool) for i1,i2,dist in zip(*m): if dist*3600 > 0.5*aper*pixscale: if cat['g'][i1] - cat['g'][i2] < 3: blended[i1] = True blended[i2] = True else: # i1 is fainter by more than 3 mag notneeded[i1] = True if dist*3600 < 0.5*aper*pixscale: if cat['g'][i1] > cat['g'][i2]: notneeded[i1] = True cat,blended = [_[~notneeded] for _ in cat,blended] #### Background subtraction bg = sep.Background(image, mask=mask|dmask|fmask|mask_bg|mask_segm, bw=128, bh=128) # bg = sep.Background(image, mask=mask|dmask|fmask|mask_bg|mask_segm, bw=32, bh=32) image1 = image - bg.back() #### Detection of all objects on the frame obj0,segm = sep.extract(image1, err=bg.rms(), thresh=2, minarea=3, mask=mask|dmask|fmask|mask_bg|mask_segm, filter_kernel=None, clean=False, segmentation_map=True) obj0 = obj0[(obj0['x'] > 10) & (obj0['y'] > 10) & (obj0['x'] < image.shape[1]-10) & (obj0['y'] < image.shape[0]-10)] obj0 = obj0[obj0['flag'] <= 1] # We keep only normal and blended oblects fields = ['ra', 'dec', 'fluxerr', 'mag', 'magerr', 'flags', 'cat'] obj0 = np.lib.recfunctions.append_fields(obj0, fields, [np.zeros_like(obj0['x'], dtype=np.int if _ in ['flags', 'cat'] else np.double) for _ in fields], usemask=False) obj0['ra'],obj0['dec'] = wcs.all_pix2world(obj0['x'], obj0['y'], 0) obj0['flags'] = obj0['flag'] if verbose: print(len(obj0['x']), 'objects detected on the frame') ## Filter out objects not coincident with catalogue positions h = htm.HTM(10) m = h.match(obj0['ra'], obj0['dec'], cat['ra'], cat['dec'], aper*pixscale) nidx = np.isin(np.arange(len(obj0['ra'])), m[0], invert=True) obj0 = obj0[nidx] if verbose: print(len(obj0['x']), 'are outside catalogue apertures') # Catalogue stars xc,yc = wcs.all_world2pix(cat['ra'], cat['dec'], 0) obj = {'x':xc, 'y':yc, 'ra':cat['ra'], 'dec':cat['dec']} obj['flags'] = np.zeros_like(xc, dtype=np.int) obj['flags'][blended] |= FLAG_BLENDED obj['cat'] = np.ones_like(xc, dtype=np.int) for _ in ['mag', 'magerr', 'flux', 'fluxerr']: obj[_] = np.zeros_like(xc) # Merge detected objects for _ in ['x', 'y', 'ra', 'dec', 'flags', 'mag', 'magerr', 'flux', 'fluxerr', 'cat']: obj[_] = np.concatenate((obj[_], obj0[_])) if verbose: print(len(obj['x']), 'objects for photometry') # Simple aperture photometry obj['flux'],obj['fluxerr'],flag = sep.sum_circle(image1, obj['x'], obj['y'], aper, err=bg.rms(), gain=gain, mask=mask|dmask|fmask|mask_bg|mask_segm, bkgann=bkgann) obj['flags'] |= flag # Normalize flags obj['flags'][obj['flags'] & sep.APER_TRUNC] |= FLAG_TRUNCATED obj['flags'][obj['flags'] & sep.APER_ALLMASKED] |= FLAG_MASKED obj['flags'] &= FLAG_NORMAL | FLAG_BLENDED | FLAG_TRUNCATED | FLAG_MASKED | FLAG_NO_BACKGROUND | FLAG_BAD_CALIBRATION area,_,_ = sep.sum_circle(np.ones_like(image1), obj['x'], obj['y'], aper, err=bg.rms(), gain=gain, mask=mask|dmask|fmask|mask_bg|mask_segm, bkgann=bkgann) # Simple local background estimation bgflux,bgfluxerr,bgflag = sep.sum_circann(image1, obj['x'], obj['y'], 10, 15, err=bg.rms(), gain=gain, mask=mask|dmask|fmask|mask_bg|mask_segm|(segm>0)) bgarea,_,_ = sep.sum_circann(np.ones_like(image1), obj['x'], obj['y'], 10, 15, err=bg.rms(), gain=gain, mask=mask|dmask|fmask|mask_bg|mask_segm|(segm>0)) bgidx = np.isfinite(bgarea) & np.isfinite(area) bgidx[bgidx] &= (bgarea[bgidx] > 10) & (area[bgidx] > 1) obj['flux'][bgidx] -= bgflux[bgidx]*area[bgidx]/bgarea[bgidx] obj['flags'][~bgidx] |= FLAG_NO_BACKGROUND # No local background obj['deltabgflux'] = np.zeros_like(obj['x']) obj['deltabgflux'][bgidx] = bgflux[bgidx]*area[bgidx]/bgarea[bgidx] fidx = np.isfinite(obj['flux']) & np.isfinite(obj['fluxerr']) fidx[fidx] &= (obj['flux'][fidx] > 0) obj['mag'][fidx] = -2.5*np.log10(obj['flux'][fidx]) obj['magerr'][fidx] = 2.5/np.log(10)*obj['fluxerr'][fidx]/obj['flux'][fidx] fidx[fidx] &= (obj['magerr'][fidx] > 0) fidx[fidx] &= 1/obj['magerr'][fidx] > sn for _ in obj.keys(): if hasattr(obj[_], '__len__'): obj[_] = obj[_][fidx] obj['aper'] = aper if verbose: print(len(obj['x']), 'objects with S/N >', sn) if len(obj['x']) < 1000: print('Only', len(obj['x']), 'objects on the frame, skipping', filename) survey.save_objects(catname, None) return obj['fwhm'] = 2.0*sep.flux_radius(image1, obj['x'], obj['y'], 2.0*aper*np.ones_like(obj['x']), 0.5, mask=mask|dmask|fmask|mask_bg|mask_segm)[0] #### Check FWHM of all objects and select only 'good' ones idx = obj['flags'] == 0 idx &= obj['magerr'] < 1/20 fwhm0 = survey.fit_2d(obj['x'][idx], obj['y'][idx], obj['fwhm'][idx], obj['x'], obj['y'], weights=1/obj['magerr'][idx]) fwhm_idx = np.abs(obj['fwhm'] - fwhm0 - np.median((obj['fwhm'] - fwhm0)[idx])) < 3.0*mad_std((obj['fwhm'] - fwhm0)[idx]) obj['flags'][~fwhm_idx] |= FLAG_BLENDED #### Catalogue matching idx = obj['flags'] m = htm.HTM(10).match(obj['ra'], obj['dec'], cat['ra'], cat['dec'], 1e-5) fidx = np.in1d(np.arange(len(cat['ra'])), m[1]) # Stars that got successfully measured and not blended cidx = (cat['good'] == 1) & (cat['var'] == 0) cidx &= np.isfinite(cat['B']) & np.isfinite(cat['V']) # & np.isfinite(cat['lum']) cidx[cidx] &= ((cat['B'] - cat['V'])[cidx] > -0.5) & ((cat['B'] - cat['V'])[cidx] < 2.0) # cidx[cidx] &= (cat['lum'][cidx] > 0.3) & (cat['lum'][cidx] < 30) if np.sum(cidx & fidx & (cat['multi_70'] == 0)) > 2000: cidx &= (cat['multi_70'] == 0) obj['cat_multi'] = 70 elif np.sum(cidx & fidx & (cat['multi_45'] == 0)) > 1000: cidx &= (cat['multi_45'] == 0) obj['cat_multi'] = 45 else: cidx &= (cat['multi_30'] == 0) obj['cat_multi'] = 30 if verbose: print(np.sum(obj['flags'] == 0), 'objects without flags') print('Amount of good stars:', np.sum(cidx & fidx & (cat['multi_70'] == 0)), np.sum(cidx & fidx & (cat['multi_45'] == 0)), np.sum(cidx & fidx & (cat['multi_30'] == 0))) print('Using %d arcsec avoidance radius' % obj['cat_multi']) # We match with very small SR to only account for manually placed apertures if verbose: print('Trying full fit:', len(obj['x']), 'objects,', np.sum(cidx), 'stars') match = Match(width=image.shape[1], height=image.shape[0]) prev_ngoodstars = len(obj['x']) for iter in range(10): if not match.match(obj=obj, cat=cat[cidx], sr=1e-5, filter_name='V', order=order, bg_order=bg_order, color_order=color_order, verbose=False) or match.ngoodstars < 500: if verbose: print(match.ngoodstars, 'good matches, matching failed for', filename) survey.save_objects(catname, None) return if verbose: print(match.ngoodstars, 'good matches, std =', match.std) if match.ngoodstars == prev_ngoodstars: if verbose: print('Converged on iteration', iter) break prev_ngoodstars = match.ngoodstars # Match good objects with stars oidx = obj['flags'] == 0 oidx1,cidx1,dist1 = htm.HTM(10).match(obj['ra'][oidx], obj['dec'][oidx], cat['ra'][cidx], cat['dec'][cidx], 1e-5) x = obj['x'][oidx][oidx1] y = obj['y'][oidx][oidx1] cbv = match.color_term[oidx][oidx1] cbv2 = match.color_term2[oidx][oidx1] cbv3 = match.color_term3[oidx][oidx1] bv = (cat['B'] - cat['V'])[cidx][cidx1] cmag = cat[match.cat_filter_name][cidx][cidx1] mag = match.mag[oidx][oidx1] + bv*cbv + bv**2*cbv2 + bv**3*cbv3 magerr = np.hypot(obj['magerr'][oidx][oidx1], 0.02) dmag = mag-cmag ndmag = ((mag-cmag)/magerr) idx = cmag < match.mag_limit[oidx][oidx1] x,y,cbv,cbv2,cbv3,bv,cmag,mag,magerr,dmag,ndmag = [_[idx] for _ in [x,y,cbv,cbv2,cbv3,bv,cmag,mag,magerr,dmag,ndmag]] # Match all objects with good objects xy = np.array([x,y]).T xy0 = np.array([obj['x'], obj['y']]).T kd = cKDTree(xy) dist,m = kd.query(xy0, 101) dist = dist[:,1:] m = m[:,1:] vchi2 = mad_std(ndmag[m]**2, axis=1) # Mark regions of too sparse or too noisy matches as bad obj['flags'][vchi2 > 5] |= FLAG_BAD_CALIBRATION # obj['flags'][vchi2 > np.median(vchi2) + 5.0*mad_std(vchi2)] |= FLAG_BAD_CALIBRATION obj['flags'][dist[:,10] > np.median(dist[:,10]) + 10.0*mad_std(dist[:,10])] |= FLAG_BAD_CALIBRATION match.good_idx = (obj['flags'] & FLAG_BAD_CALIBRATION) == 0 if verbose: print(np.sum(match.good_idx), 'of', len(match.good_idx), 'stars are good') #### Store objects to file try: os.makedirs(dirname) except: pass obj['mag_limit'] = match.mag_limit obj['color_term'] = match.color_term obj['color_term2'] = match.color_term2 obj['color_term3'] = match.color_term3 obj['filename'] = filename obj['night'] = night obj['channel'] = channel obj['filter'] = fname obj['cat_filter'] = match.cat_filter_name obj['time'] = time obj['mag_id'] = match.mag_id obj['good_idx'] = match.good_idx obj['calib_mag'] = match.mag obj['calib_magerr'] = match.magerr obj['std'] = match.std obj['nstars'] = match.ngoodstars survey.save_objects(catname, obj, header=header)
def calibrate_objects(obj, cat, sr=15.0 / 3600, smooth=False, correct_brightness=False, smoothr=100, smoothmin=20, retval='std'): h = htm.HTM(10) m = h.match(obj['ra'], obj['dec'], cat['ra'], cat['dec'], sr, maxmatch=0) oidx = m[0] cidx = m[1] dist = m[2] * 3600 if len(oidx) < 300: return None x, y = obj['x'][oidx], obj['y'][oidx] x = (x - 2560 / 2) * 2 / 2560 y = (y - 2160 / 2) * 2 / 2160 omag = obj['mag'][oidx] omagerr = obj['magerr'][oidx] oflux = obj['flux'][oidx] ofluxerr = obj['fluxerr'][oidx] oflags = obj['flags'][oidx] ochisq = obj['chisq'][oidx] omag0 = (omag - np.min(omag)) / (np.max(omag) - np.min(omag)) try: cb = cat['g'][cidx] cv = cat['r'][cidx] cr = cat['i'][cidx] cmagerr = np.hypot(cat['gerr'][cidx], cat['rerr'][cidx], cat['ierr'][cidx]) except: cb = cat['b'][cidx] cv = cat['v'][cidx] cr = cat['r'][cidx] try: cmagerr = np.hypot(cat['berr'][cidx], cat['verr'][cidx], cat['rerr'][cidx]) except: cmagerr = np.hypot(cat['ebt'][cidx], cat['evt'][cidx]) cmag = cv tmagerr = np.hypot(cmagerr, omagerr) delta_mag = cmag - omag weights = 1.0 / tmagerr X = [ np.ones(len(delta_mag)), x, y, x * x, x * y, y * y, x * x * x, x * x * y, x * y * y, y * y * y, x * x * x * x, x * x * x * y, x * x * y * y, x * y * y * y, y * y * y * y, x * x * x * x * x, x * x * x * x * y, x * x * x * y * y, x * x * y * y * y, x * y * y * y * y, y * y * y * y * y, x * x * x * x * x * x, x * x * x * x * x * y, x * x * x * x * y * y, x * x * x * y * y * y, x * x * y * y * y * y, x * y * y * y * y * y, y * y * y * y * y * y ] #X += [omag0, omag0**2, omag0**3, omag0**4, omag0**5, omag0**6] X += [ cb - cv, (cb - cv) * x, (cb - cv) * y, (cb - cv) * x * x, (cb - cv) * x * y, (cb - cv) * y * y, (cb - cv) * x * x * x, (cb - cv) * x * x * y, (cb - cv) * x * y * y, (cb - cv) * y * y * y ] X += [ cv - cr, (cv - cr) * x, (cv - cr) * y, (cv - cr) * x * x, (cv - cr) * x * y, (cv - cr) * y * y, (cv - cr) * x * x * x, (cv - cr) * x * x * y, (cv - cr) * x * y * y, (cv - cr) * y * y * y ] X = np.vstack(X).T Y = delta_mag idx = (oflags == 0) & (ochisq < 2.0) for iter in range(5): if len(X[idx]) < 100: print "Fit failed - %d objects" % len(X[idx]) return None C = sm.WLS(Y[idx], X[idx], weights=weights[idx]).fit() YY = np.sum(X * C.params, axis=1) idx = (np.abs(Y - YY) < 2.0 * np.std(Y - YY)) & (oflags == 0) & (ochisq < 2.0) print np.std((Y - YY)[idx]), np.std(Y - YY), '-', np.std( ((Y - YY) / tmagerr)), np.std( (((Y - YY) / tmagerr))[idx]), '-', len(idx[idx]) mag0 = YY mag = omag + mag0 x = (obj['x'] - 2560 / 2) * 2 / 2560 y = (obj['y'] - 2160 / 2) * 2 / 2160 omag0 = (obj['mag'] - np.min(omag)) / (np.max(omag) - np.min(omag)) obj['mag0'] = C.params[0] * np.ones_like( obj['mag'] ) + C.params[1] * x + C.params[2] * y + C.params[3] * x * x + C.params[ 4] * x * y + C.params[5] * y * y + C.params[6] * x * x * x + C.params[ 7] * x * x * y + C.params[8] * x * y * y + C.params[9] * y * y * y + C.params[ 10] * x * x * x * x + C.params[11] * x * x * x * y + C.params[ 12] * x * x * y * y + C.params[13] * x * y * y * y + C.params[ 14] * y * y * y * y + C.params[15] * x * x * x * x * x + C.params[ 16] * x * x * x * x * y + C.params[17] * x * x * x * y * y + C.params[ 18] * x * x * y * y * y + C.params[19] * x * y * y * y * y + C.params[ 20] * y * y * y * y * y + C.params[ 21] * x * x * x * x * x * x + C.params[ 22] * x * x * x * x * x * y + C.params[ 23] * x * x * x * x * y * y + C.params[ 24] * x * x * x * y * y * y + C.params[ 25] * x * x * y * y * y * y + C.params[ 26] * x * y * y * y * y * y + C.params[ 27] * y * y * y * y * y * y #obj['mag0'] += C.params[28]*omag0 + C.params[29]*omag0**2 + C.params[30]*omag0**3 + C.params[31]*omag0**4 + C.params[32]*omag0**5 + C.params[33]*omag0**6 obj['cmag'] = obj['mag'] + obj['mag0'] obj['Cbv'] = np.ones_like(obj['mag']) * C.params[-20] + x * C.params[ -19] + y * C.params[-18] + x * x * C.params[-17] + x * y * C.params[ -16] + y * y * C.params[-15] + x * x * x * C.params[ -14] + x * x * y * C.params[-13] + x * y * y * C.params[ -12] + y * y * y * C.params[-11] obj['Cvr'] = np.ones_like(obj['mag']) * C.params[-10] + x * C.params[ -9] + y * C.params[-8] + x * x * C.params[-7] + x * y * C.params[ -6] + y * y * C.params[-5] + x * x * x * C.params[ -4] + x * x * y * C.params[-3] + x * y * y * C.params[ -2] + y * y * y * C.params[-1] # Correct uncorrected brightness-dependent variations obj['bcorr'] = np.zeros_like(obj['cmag']) if correct_brightness: step = 0.5 omag0 = omag + np.median(cmag - omag) for vmin in np.arange(5, 16, step): idx0 = (omag0[idx] >= vmin) & (omag0[idx] < vmin + step) if len(omag0[idx][idx0] > 20): omag1 = obj['mag'] + np.median(cmag - omag) idx1 = (omag1 >= vmin) & (omag1 < vmin + step) obj['bcorr'][idx1] = np.average( (Y - YY)[idx][idx0], weights=1.0 / tmagerr[idx][idx0]) # Local smoothing of residuals if smooth: kd0 = cKDTree(np.array([obj['x'][oidx][idx], obj['y'][oidx][idx]]).T) kd = cKDTree(np.array([obj['x'], obj['y']]).T) m = kd.query_ball_tree(kd0, smoothr) nm = np.array(([len(_) for _ in m])) corr = np.array([ np.average((Y - YY)[idx][_], weights=weights[idx][_]) if len(_) > smoothmin else 0 for _ in m ]) icorr = nm > smoothmin obj['corr'] = corr obj['corrected'] = icorr obj['ncorr'] = nm else: obj['corr'] = np.zeros_like(obj['mag']) obj['corrected'] = np.ones_like(obj['mag'], dtype=np.bool) obj['ncorr'] = np.zeros_like(obj['mag']) if retval == 'std': return np.std(Y - YY) elif retval == 'map': return Y elif retval == 'fit': return YY elif retval == 'diff': return Y - YY elif retval == 'all': return Y, YY, obj['corr'], oidx
def match(self, obj=None, cat=None, sr=5./3600, verbose=False, predict=True, ra=None, dec=None, x=None, y=None, mag=None, magerr=None, flags=None, filter_name='V', order=4, bg_order=None, color_order=None, hard_mag_limit=99, mag_id=0, magerr0=0.02, sn=None, thresh=5.0, mask=None, good_flags=0x0): """Match a set of points with catalogue""" self.success = False self.ngoodstars = 0 self.order = order self.bg_order = bg_order self.color_order = color_order self.mag_id = mag_id self.filter_name = filter_name if filter_name in ['B', 'V', 'R', 'I', 'g', 'r', 'i', 'z']: # Generic names cmag,cmagerr = cat[filter_name], cat[filter_name + 'err'] self.cat_filter_name = filter_name elif filter_name == 'Clear': # Mini-MegaTORTORA cmag,cmagerr = cat['V'], cat['Verr'] self.cat_filter_name = 'V' elif filter_name == 'N': # FRAMs cmag,cmagerr = cat['R'], cat['Rerr'] self.cat_filter_name = 'R' else: if verbose: print('Unsupported filter name: %s' % filter_name) return False # TODO: make it configurable?.. color = cat['B'] - cat['V'] self.cat_color_name = 'B - V' # Objects to match if obj is not None: ra = obj['ra'] dec = obj['dec'] x = obj['x'] y = obj['y'] mag = obj['mag'] magerr = obj['magerr'] flags = obj['flags'] else: if ra is None or dec is None or x is None or y is None or mag is None: raise ValueError('Data for matching are missing') if magerr is None: magerr = np.ones_like(mag)*np.std(mag) if flags is None: flags = np.zeros_like(ra, dtype=np.int) if self.width is None or self.height is None: self.x0,self.y0,self.width,self.height = np.mean(x), np.mean(y), np.max(x)-np.min(x), np.max(y) - np.min(y) # Match stars h = htm.HTM(10) oidx,cidx,dist = h.match(ra, dec, cat['ra'],cat['dec'], sr, maxmatch=0) if verbose: print(len(oidx), 'matches between', len(ra), 'objects and', len(cat['ra']), 'stars, sr = %.1f arcsec' % (3600.0*sr)) self.oidx,self.cidx,self.dist = oidx, cidx, dist self.cmag = cmag[cidx] self.cmagerr = cmagerr[cidx] self.color = color[cidx] self.ox,self.oy = x[oidx], y[oidx] self.oflags = flags[oidx] self.omag,self.omagerr = mag[oidx], magerr[oidx] if len(self.omag.shape) > 1: # If we are given a multi-aperture magnitude column self.omag,self.omagerr = self.omag[:, mag_id], self.omagerr[:, mag_id] # Scaled spatial coordinates for fitting sx = (self.ox - self.x0)*2/self.width sy = (self.oy - self.y0)*2/self.height # Optimal magnitude cutoff for fitting, as a mean mag where S/N = 10 idx = (1.0/self.omagerr > 5) & (1.0/self.omagerr < 15) if np.sum(idx) > 10: X = make_series(1.0, sx, sy, order=order) X = np.vstack(X).T Y = self.cmag self.C_mag_limit = sm.RLM(Y[idx], X[idx]).fit() mag_limit = np.sum(X*self.C_mag_limit.params, axis=1) else: if verbose: print('Not enough matches with SN~10:', np.sum(idx)) self.C_mag_limit = None mag_limit = 99.0*np.ones_like(cmag) self.zero = self.cmag - self.omag # We will build a model for this variable self.zeroerr = np.hypot(self.omagerr, self.cmagerr) self.zeroerr = np.hypot(self.zeroerr, magerr0) self.weights = 1.0/self.zeroerr**2 X = make_series(1.0, sx, sy, order=self.order) if self.bg_order is not None: X += make_series(-2.5/np.log(10)/10**(-0.4*self.omag), sx, sy, order=self.bg_order) if self.color_order is not None: X += make_series(self.color, sx, sy, order=self.color_order) X += make_series(self.color**2, sx, sy, order=self.color_order) X += make_series(self.color**3, sx, sy, order=self.color_order) X = np.vstack(X).T self.idx0 = ((self.oflags & (~good_flags)) == 0) & (self.cmag < hard_mag_limit) & (self.cmag < mag_limit) if mask is not None: # Exclude masked objects self.idx0 &= ~mask if sn is not None: self.idx0 &= (self.omagerr < 1.0/sn) # Actual fitting self.idx = self.idx0.copy() for iter in range(3): if np.sum(self.idx) < 3: if verbose: print("Fit failed - %d objects" % np.sum(self.idx)) return False self.C = sm.WLS(self.zero[self.idx], X[self.idx], weights=self.weights[self.idx]).fit() # self.C = sm.RLM(self.zero[self.idx], X[self.idx]).fit() self.zero_model = np.sum(X*self.C.params, axis=1) self.idx = self.idx0.copy() if thresh and thresh > 0: self.idx &= (np.abs((self.zero - self.zero_model)/self.zeroerr) < thresh) self.std = np.std((self.zero - self.zero_model)[self.idx]) self.ngoodstars = np.sum(self.idx) self.success = True if verbose: print('Fit finished:', self.ngoodstars, 'stars, rms', self.std) if predict: self.predict(obj=obj, x=x, y=y, mag=mag, magerr=magerr, mag_id=mag_id, verbose=verbose) return True
def clustmember(data,cluster,logg=[-1,3.8],te=[3800,5500],rv=True,pm=True,dist=True,raw=False,firstgen=False,firstpos=True, ratag='RA',dectag='DEC',rvtag='VHELIO',idtag='APOGEE_ID',btag='J',rtag='K', plot=False,hard=None) : clust=clustdata() ic = np.where( np.core.defchararray.strip(clust.name) == cluster)[0] if len(ic) == 0 : print('no cluster found: ',cluster) return [] print('cluster: ', cluster) ic=ic[0] # adjust ra for wraparound if needed ra=copy.copy(data[ratag]) if clust[ic].ra > 300 : j=np.where(data[ratag] < 180)[0] ra[j]+=360 if clust[ic].ra < 60 : j=np.where(data[ratag] > 180)[0] ra[j]-=360 # select by location relative to cluster jc=np.where((np.abs(ra-clust[ic].ra)*np.cos(clust[ic].dec*np.pi/180.) < clust[ic].rad/60.) & (np.abs(data[dectag]-clust[ic].dec) < clust[ic].rad/60.))[0] if len(jc) > 0 : j=np.where( ((ra[jc]-clust[ic].ra)*np.cos(clust[ic].dec*np.pi/180.))**2+ (data[jc][dectag]-clust[ic].dec)**2 < (clust[ic].rad/60.)**2)[0] jc=jc[j] else : jc=[] print('{:d} stars after location criterion'.format(len(jc))) if len(jc) == 0 : return jc if plot : jf=np.where((np.abs(ra-clust[ic].ra)*np.cos(clust[ic].dec*np.pi/180.) < 1.5) & (np.abs(data[dectag]-clust[ic].dec) < 1.5))[0] fig,ax=plots.multi(1,1) fig.suptitle('{:s} Radius: {:4.2f} arcmin Ngood: {:d}'.format(cluster,clust[ic].rad,len(jc))) plots.plotp(ax,ra[jf],data[dectag][jf],color='k',size=20,draw=False,xt='RA',yt='DEC') plots.plotp(ax,ra[jc],data[dectag][jc],color='g',size=20,draw=False) circle = plt.Circle((clust[ic].ra,clust[ic].dec), clust[ic].rad/60., color='g', fill=False) ax.add_artist(circle) if hard is not None : print(hard+'/'+clust[ic].name+'_pos.png') fig.savefig(hard+'/'+clust[ic].name+'_pos.png') plt.close() else : pdb.set_trace() # RV criterion try : vhelio = data[rvtag] except : vhelio = data['VHELIO_AVG'] j=np.where(np.abs(vhelio[jc]-clust[ic].rv) < clust[ic].drv)[0] if len(j) > 0 : if rv: jc=jc[j] else : jc=[] print('{:d} stars after RV criterion'.format(len(jc))) if plot : ax.cla() try : if len(jf) > 0 : ax.hist(vhelio[jf],color='k',bins=np.arange(clust[ic].rv-100,clust[ic].rv+100,1.),histtype='step') if len(jc) > 0 : ax.hist(vhelio[jc],color='r',bins=np.arange(clust[ic].rv-100,clust[ic].rv+100,1.),histtype='step') if len(j) > 0 : ax.hist(vhelio[jc[j]],color='g',bins=np.arange(clust[ic].rv-100,clust[ic].rv+100,1.),histtype='step',linewidth=3) except : pass ymax=ax.get_ylim()[1] ax.plot([clust[ic].rv,clust[ic].rv],[0,ymax],color='g') ax.plot([clust[ic].rv+clust[ic].drv,clust[ic].rv+clust[ic].drv],[0,ymax],color='r',ls=':') ax.plot([clust[ic].rv-clust[ic].drv,clust[ic].rv-clust[ic].drv],[0,ymax],color='r',ls=':') fig.suptitle('{:s} RV: {:4.2f} +/- {:4.2f} Ngood: {:d}'.format(cluster,clust[ic].rv,clust[ic].drv,len(j))) ax.set_xlabel('RV') if hard is not None : fig.savefig(hard+'/'+clust[ic].name+'_rv.png') plt.close() else : plt.draw() pdb.set_trace() if len(jc) <= 1 : return jc # proper motion criterion if dist or pm : gaia = True else : gaia = False if gaia : job=Gaia.launch_job_async("SELECT xm.original_ext_source_id, gaia.ra, gaia.dec,"+ "gaia.pmra, gaia.pmra_error, gaia.pmdec, gaia.pmdec_error, gaia.parallax, gaia.parallax_error "+ "FROM gaiadr2.gaia_source AS gaia, gaiadr2.tmass_best_neighbour AS xm "+ "WHERE gaia.source_id = xm.source_id AND "+ "CONTAINS(POINT('ICRS',gaia.ra,gaia.dec),CIRCLE('ICRS',{:12.6f},{:12.6f},{:12.6f}))=1;".format( clust[ic].ra,clust[ic].dec,clust[ic].rad/60.)) # convert to velocities (note mas and kpc cancel out factors of 1000) and get median gaia=job.get_results() h=htm.HTM() maxrad=3/3600. i1,i2,rad = h.match(data[ratag][jc],data[dectag][jc],gaia['ra'],gaia['dec'],maxrad,maxmatch=1) #i1, i2 = match.match(np.core.defchararray.replace(data[idtag][jc],'2M',''),gaia['original_ext_source_id']) vra=4.74*gaia['pmra']*clust[ic].dist vra_err=4.74*gaia['pmra_error']*clust[ic].dist vdec=4.74*gaia['pmdec']*clust[ic].dist vdec_err=4.74*gaia['pmdec_error']*clust[ic].dist med_vra=np.median(vra[i2]) med_vdec=np.median(vdec[i2]) j=np.where((vra[i2]-med_vra)**2+(vdec[i2]-med_vdec)**2 < 2*clust[ic].drv**2+vra_err[i2]**2+vdec_err[i2]**2)[0] if len(j) > 0 : if pm: #jc=jc[i1[j]] # allow for the possibility of multiple instances of a given star in input list jnew=[] for jjj in j : iii= np.where(data[idtag][jc] == data[idtag][jc[i1[jjj]]])[0] jnew.extend(iii) jc=jc[list(set(jnew))] else : jc=[] print('{:d} stars after PM criterion'.format(len(jc))) if plot : ax.cla() plots.plotp(ax,vra,vdec,color='k', xr=[med_vra-150,med_vra+150],xt='PMRA (km/sec at cluster dist)', yr=[med_vdec-150,med_vdec+150],yt='PMDEC (km/sec at cluster dist)') plots.plotp(ax,vra[i2],vdec[i2],color='r',size=30) plots.plotp(ax,vra[i2[j]],vdec[i2[j]],color='g',size=30) fig.suptitle('{:s} PM (km/s): {:4.2f} +/- {:4.2f} {:4.2f} +/ {:4.2f} Ngood: {:d}'.format( cluster,med_vra,clust[ic].drv, med_vdec,clust[ic].drv,len(jc))) if hard is not None : fig.savefig(hard+'/'+clust[ic].name+'_pm.png') plt.close() else : pdb.set_trace() if len(jc) <= 1 : return jc # parallaxes #gaia=job.get_results() #i1, i2 = match.match(np.core.defchararray.replace(data[idtag][jc],'2M',''),gaia['original_ext_source_id']) i1,i2,rad = h.match(data[ratag][jc],data[dectag][jc],gaia['ra'],gaia['dec'],maxrad,maxmatch=1) par=gaia['parallax'] par_error=gaia['parallax_error'] gd=np.where(np.isfinite(par[i2]))[0] med_par=np.median(par[i2[gd]]) med_par_error=np.median(par_error[i2[gd]]) j=np.where(np.isfinite(par[i2]) & (np.abs(par[i2]-med_par) < 3*med_par_error))[0] if len(j) > 0 : if dist: #jc=jc[i1[j]] # allow for the possibility of multiple instances of a given star in input list jnew=[] for jjj in j : iii= np.where(data['APOGEE_ID'][jc] == data['APOGEE_ID'][jc[i1[jjj]]])[0] jnew.extend(iii) jc=jc[jnew] else : jc=[] if plot : ax.cla() ax.hist(par,color='k',bins=np.arange(par.min(),par.max(),0.01),histtype='step',range=(0,2)) ax.hist(par[i2],color='r',bins=np.arange(par.min(),par.max(),0.01),histtype='step',range=(0,2)) ax.hist(par[i2[j]],color='g',bins=np.arange(par.min(),par.max(),0.01),histtype='step',range=(0,2)) ax.set_xlabel('Parallax') fig.suptitle('{:s} Parallax : {:4.2f} +/- {:4.2f} Ngood: {:d}'.format(cluster,med_par, 3*med_par_error,len(j))) if hard is not None : fig.savefig(hard+'/'+clust[ic].name+'_parallax.png') plt.close() else : plt.draw() pdb.set_trace() if len(j) > 0 : if dist: #jc=jc[i1[j]] # allow for the possibility of multiple instances of a given star in input list jnew=[] for jjj in j : iii= np.where(data[idtag][jc] == data[idtag][jc[i1[jjj]]])[0] jnew.extend(iii) jc=jc[list(set(jnew))] else : jc=[] print('{:d} stars after parallax criterion'.format(len(jc))) if len(jc) <= 1 : return jc # parameters criteria if raw : param='FPARAM' else : param='PARAM' try : j = np.where((data[param][jc,1] >= logg[0]) & (data[param][jc,1] <= logg[1]) & (data[param][jc,0] >= te[0]) & (data[param][jc,0] <= te[1]) )[0] if len(j) > 0 : jc=jc[j] else : jc=[] except: pass print('{:d} stars after parameters criterion'.format(len(jc))) if len(jc) <= 1 : return jc # Remove badstars if plot : ax.cla() plots.plotp(ax,data[btag][jf]-data[rtag][jf],data[rtag][jf],color='k',size=20,xr=[-0.5,1.5],yr=[17,6], facecolors='none',linewidth=1,draw=False,xt=btag+'-'+rtag,yt=rtag) plots.plotp(ax,data[btag][jc]-data[rtag][jc],data[rtag][jc],color='g',size=30,xr=[-0.5,1.5],draw=False,yr=[17,6]) badstars = open(os.environ['APOGEE_DIR']+'/data/calib/badcal.dat') bad = [] for line in badstars : bad.append(line.split()[0]) jc = [x for x in jc if data[x][idtag] not in bad] print('{:d} stars after badstars rejection'.format(len(jc))) if len(jc) <= 1 : return jc # remove non firstgen GC stars if requested if firstgen : gcstars = ascii.read(os.environ['APOGEE_DIR']+'/data/calib/gc_szabolcs.dat') if firstpos : gd=np.where(gcstars['pop'] == 1)[0] jc = [x for x in jc if data[x][idtag] in gcstars['id'][gd]] else : bd=np.where(gcstars['pop'] != 1)[0] jc = [x for x in jc if data[x][idtag] not in gcstars['id'][bd]] print('{:d} stars after firstgen rejection'.format(len(jc))) if plot : plots.plotp(ax,data[btag][jc]-data[rtag][jc],data[rtag][jc],color='b',size=30,draw=False) if hard is not None : fig.savefig(hard+'/'+clust[ic].name+'_cmd.png') plt.close() else : plt.draw() pdb.set_trace() ax.cla() plots.plotp(ax,data[param][jf,0],data[param][jf,1],size=30,draw=False,xt='Teff',yt='logg',xr=[7000,3000],yr=[6,-1]) plots.plotp(ax,data[param][jc,0],data[param][jc,1],color='b',size=30,draw=False) if hard is not None : fig.savefig(hard+'/'+clust[ic].name+'_kiel.png') plt.close() else : plt.draw() pdb.set_trace() return jc
def match_objects(obj, cat, sr, fname='V', order=4, thresh=5.0, clim=None, sn=10, mag_idx=0): x0,y0,width,height = np.mean(obj['x']), np.mean(obj['y']), np.max(obj['x'])-np.min(obj['x']), np.max(obj['y'])-np.min(obj['y']) # Match stars h = htm.HTM(10) m = h.match(obj['ra'],obj['dec'], cat['ra'],cat['dec'], sr, maxmatch=0) # m = h.match(obj['ra'],obj['dec'], cat['ra'],cat['dec'], 15.0/3600, maxmatch=0) oidx = m[0] cidx = m[1] dist = m[2] if fname == 'B': cmag,cmagerr = cat['B'], cat['Berr'] elif fname == 'V': cmag,cmagerr = cat['V'], cat['Verr'] elif fname == 'R' or fname == 'N': cmag,cmagerr = cat['R'], cat['Rerr'] elif fname == 'I': cmag,cmagerr = cat['I'], cat['Ierr'] elif fname == 'z': cmag,cmagerr = cat['z'], cat['zerr'] if cmag is not None: cmag = cmag[cidx] cmagerr = cmagerr[cidx] else: print('Unsupported filter:', fname) return None if clim is not None: idx = cmag < clim oidx,cidx,dist,cmag,cmagerr = [_[idx] for _ in [oidx,cidx,dist,cmag,cmagerr]] x,y = obj['x'][oidx],obj['y'][oidx] oflags = obj['flags'][oidx] omag,omagerr = obj['mag'][oidx],obj['magerr'][oidx] if len(obj['mag'].shape) > 1: omag,omagerr = omag[:,mag_idx],omagerr[:,mag_idx] x = (x - x0)*2/width y = (y - y0)*2/height tmagerr = np.hypot(cmagerr, omagerr) delta_mag = cmag - omag weights = 1.0/tmagerr**2 X = make_series(1.0, x, y, order=order) X = np.vstack(X).T Y = delta_mag idx0 = (oflags == 0) if sn and sn>0: idx0 &= (omagerr < 1/sn) idx = idx0.copy() for iter in range(3): if len(X[idx]) < 3: print("Fit failed - %d objects" % len(X[idx])) return None C = sm.WLS(Y[idx], X[idx], weights=weights[idx]).fit() YY = np.sum(X*C.params,axis=1) idx = idx0.copy() if thresh and thresh > 0: idx &= (np.abs((Y-YY)/tmagerr) < thresh) x = (obj['x'] - x0)*2/width y = (obj['y'] - y0)*2/height X = make_series(1.0, x, y, order=order) X = np.vstack(X).T YY1 = np.sum(X*C.params,axis=1) if len(obj['mag'].shape) > 1: mag = obj['mag'][:,mag_idx] + YY1 else: mag = obj['mag'] + YY1 # Simple analysis of proximity to "good" points mx,my = obj['x'][oidx][idx], obj['y'][oidx][idx] kdo = cKDTree(np.array([obj['x'], obj['y']]).T) kdm = cKDTree(np.array([mx, my]).T) mr0 = np.sqrt(width*height/np.sum(idx)) m = kdm.query_ball_tree(kdm, 5.0*mr0) dists = [] for i,ii in enumerate(m): if len(ii) > 1: d1 = [np.hypot(mx[i]-mx[_], my[i]-my[_]) for _ in ii] d1 = np.sort(d1) dists.append(d1[1]) mr1 = np.median(dists) m = kdo.query_ball_tree(kdm, 5.0*mr1) midx = np.array([len(_)>1 for _ in m]) return { # Matching indices and a distance in degrees 'cidx':cidx, 'oidx':oidx, 'dist':dist, # Pixel and sky coordinates of matched stars, as well as their flags 'x':obj['x'][oidx], 'y':obj['y'][oidx], 'flags':oflags, 'ra':obj['ra'][oidx], 'dec':obj['dec'][oidx], # All catalogue magnitudes of matched stars 'cB':cat['B'][cidx], 'cV':cat['V'][cidx], 'cR':cat['R'][cidx], 'cI':cat['I'][cidx], 'cBerr':cat['Berr'][cidx], 'cVerr':cat['Verr'][cidx], 'cRerr':cat['Rerr'][cidx], 'cIerr':cat['Ierr'][cidx], # Catalogue magnitudes of matched stars in proper filter 'cmag':cmag, 'cmagerr':cmagerr, 'tmagerr':tmagerr, # Model zero point for all objects, and their corrected magnitudes 'mag0':YY1, 'mag':mag, 'mag_idx': mag_idx, 'Y':Y, 'YY':YY, # Subset of matched stars used in the fits 'idx':idx, # Subset of all objects at 'good' distances from matched ones 'midx':midx, 'mr0':mr0, 'mr1':mr1}