def fix_wcs(obj, cat, sr, header=None, use_header_wcs=False, maxmatch=1, order=6, fix=True):
'''Get a refined WCS solution based on cross-matching of objects with catalogue on the sphere.
Uses external 'fit-wcs' binary from Astrometry.Net suite'''
if header is not None:
width,height = header['NAXIS1'],header['NAXIS2']
if use_header_wcs:
wcs = WCS(header)
if wcs:
obj['ra'],obj['dec'] = wcs.all_pix2world(obj['x'], obj['y'], 0)
else:
width,height = int(np.max(obj['x'])),int(np.max(obj['y']))
h = htm.HTM(10)
oidx,cidx,dist = h.match(obj['ra'], obj['dec'], cat['ra'], cat['dec'], sr, maxmatch=1)
dir = tempfile.mkdtemp(prefix='astrometry')
wcs = None
binname = None
for path in ['.', '/usr/local', '/opt/local']:
if os.path.isfile(posixpath.join(path, 'astrometry', 'bin', 'fit-wcs')):
binname = posixpath.join(path, 'astrometry', 'bin', 'fit-wcs')
break
if binname:
columns = [fits.Column(name='FIELD_X', format='1D', array=obj['x'][oidx] + 1),
fits.Column(name='FIELD_Y', format='1D', array=obj['y'][oidx] + 1),
fits.Column(name='INDEX_RA', format='1D', array=cat['ra'][cidx]),
fits.Column(name='INDEX_DEC', format='1D', array=cat['dec'][cidx])]
tbhdu = fits.BinTableHDU.from_columns(columns)
filename = posixpath.join(dir, 'list.fits')
wcsname = posixpath.join(dir, 'list.wcs')
tbhdu.writeto(filename, overwrite=True)
os.system("%s -c %s -o %s -W %d -H %d -C -s %d" % (binname, filename, wcsname, width, height, order))
if os.path.isfile(wcsname):
header = fits.getheader(wcsname)
wcs = WCS(header)
if fix and wcs:
obj['ra'],obj['dec'] = wcs.all_pix2world(obj['x'], obj['y'], 0)
else:
print("Astrometry.Net binary not found")
#print order
shutil.rmtree(dir)
return wcs
def match(cat_1, cat_2, column_1, column_2, column_3, column_4,error):
h = htm.HTM(depth=10)
m1, m2, d12 = h.match(np.array(cat_1[column_1]),
np.array(cat_1[column_3]),
np.array(cat_2[column_2]),
np.array(cat_2[column_4]),
error, maxmatch=1)
submatched = cat_1[m1]
manmatched = cat_2[m2]
matched = hstack([submatched, manmatched])
return matched
def match(cat_1, cat_2, column_1, column_2, column_3, column_4, error):
'Function to make the matching with two tables, using the esutil'
h = htm.HTM(depth=10)
m1, m2, d12 = h.match(np.array(cat_1[column_1]),
np.array(cat_1[column_3]),
np.array(cat_2[column_2]),
np.array(cat_2[column_4]),
error,
maxmatch=1)
submatched = cat_1[m1]
manmatched = cat_2[m2]
matched = hstack([submatched, manmatched])
return matched
def match(c1, c2, sr=None, maxmatch=0):
h = htm.HTM(10)
if sr is None:
sr = np.hypot(c1['sr'], c2['sr']) / 3600
cat1 = catslist.get(c1['name'])
cat2 = catslist.get(c2['name'])
ra1 = c1['table'][cat1.get('ra', 'RAJ2000')]
dec1 = c1['table'][cat1.get('dec', 'DEJ2000')]
ra2 = c2['table'][cat2.get('ra', 'RAJ2000')]
dec2 = c2['table'][cat2.get('dec', 'DEJ2000')]
return h.match(ra1, dec1, ra2, dec2, sr, maxmatch=maxmatch)
from esutil import htm
# field 26 (2014:04) -10:00:28.800 +02:12:36.00
# field 40 (2014:10) -10:20:28.800 -06:31:12.00
# field 42 (2014:09) -10:21:52.800 -04:57:00.00
indexer = htm.HTM(depth=8)
ras = [
15. * (10. + (0. / 60.) + (28.800 / 3600.)),
15. * (10. + (20. / 60.) + (28.800 / 3600.)),
15. * (10. + (21. / 60.) + (52.800 / 3600.))
]
decs = [
+(2 + (12. / 60.) + (36. / 3600.)), -(6 + (31. / 60.) + (12.00 / 3600.)),
-(4. + (57. / 60.))
]
shards = []
for ra, dec in zip(ras, decs):
shards += list(indexer.intersect(ra, dec, 2., True))
print "mkdir ./gaia_HiTS_2015"
print "cp -v /datasets/refcats/htm/gaia_DR1_v1/config.py ./gaia_HiTS_2015"
print "cp -v /datasets/refcats/htm/gaia_DR1_v1/master_schema.fits ./gaia_HiTS_2015"
for shard in shards:
print "cp -v gaia_DR1_v1/%i.fits ./gaia_HiTS_2015/" % shard
def add_gaia(data):
""" Add GAIA data to input structure, with 2MASS match and coordinate match to (cross-matched) GAIA reference file
"""
# get the GAIA data from both matches
gaia_twomass, gaia_posn = getdata(data)
# add new columns
tab = Table(data)
in_names = ('source_id', 'parallax', 'parallax_error', 'pmra',
'pmra_error', 'pmdec', 'pmdec_error', 'phot_g_mean_mag',
'phot_bp_mean_mag', 'phot_rp_mean_mag', 'a_g_val',
'e_bp_min_rp_val', 'radial_velocity', 'radial_velocity_error',
'r_est', 'r_lo', 'r_hi')
dtypes = ('i8', 'f8', 'f8', 'f8', 'f8', 'f8', 'f8', 'f4', 'f4', 'f4', 'f4',
'f4', 'f8', 'f8', 'f8', 'f8', 'f8')
out_names = []
for name in in_names:
out_names.append(('gaia_' + name).upper())
# initialize
newcols = Table(np.zeros([len(tab), len(out_names)]) - 9999.,
names=out_names,
dtype=dtypes)
# for source_id, default to 0, not -9999.
newcols['GAIA_SOURCE_ID'] = 0
# rename targetting proper motions to avoid confusion!
try:
tab.rename_column('PMRA', 'TARG_PMRA')
except:
pass
try:
tab.rename_column('PMDEC', 'TARG_PMDEC')
except:
pass
try:
tab.rename_column('PM_SRC', 'TARG_PM_SRC')
except:
pass
# add unpopulated columns
tab.add_columns(newcols.columns.values())
# remove dups in GAIA twomass in favor of brightest
print('number in GAIA-2MASS xmatch catalog: ', len(gaia_twomass),
len(set(gaia_twomass['original_ext_source_id'])))
ind = []
for tid in set(gaia_twomass['original_ext_source_id']):
j = np.where(gaia_twomass['original_ext_source_id'] == tid)[0]
if len(j) > 1:
ii = np.argsort(gaia_twomass['phot_rp_mean_mag'][j])
ind.append(j[ii[0]])
print('duplicate 2MASS: ', gaia_twomass['phot_rp_mean_mag'][j[ii]])
else:
ind.append(j)
# read gaia 2MASS matched file, match by 2MASS ID, and populate
while True:
# loop for matches since we may have repeats and want them all matched
j = np.where(tab['GAIA_SOURCE_ID'] == 0)[0]
print('Number missing gaia_source_id: ', len(j))
if len(j) == 0: break
m1, m2 = match.match(
np.core.defchararray.replace(tab['APOGEE_ID'][j], b'2M', b''),
gaia_twomass['original_ext_source_id'])
print('Number matched by 2MASS: ', len(m1))
if len(m1) == 0: break
for inname, outname in zip(in_names, out_names):
tab[outname][j[m1]] = gaia_twomass[inname][m2]
j = np.where(tab['GAIA_SOURCE_ID'] > 0)[0]
print('number of unique APOGEE_ID matches: ',
len(set(tab['APOGEE_ID'][j])))
j = np.where(tab['GAIA_SOURCE_ID'] == 0)[0]
print('missing sources after 2MASS matches: ', len(j))
h = htm.HTM()
# now do a positional match, take the brightest object within 3 arcsec (which is the max from the GAIA crossmatch)
maxrad = 3. / 3600.
m1, m2, rad = h.match(tab['RA'][j],
tab['DEC'][j],
gaia_posn['ra'],
gaia_posn['dec'],
maxrad,
maxmatch=10)
for m in set(m1):
jj = np.where(m1 == m)[0]
ii = np.argsort(gaia_posn['phot_rp_mean_mag'][m2[jj]])
for inname, outname in zip(in_names, out_names):
tab[outname][j[m]] = gaia_posn[inname][m2[jj[ii[0]]]]
j = np.where(tab['GAIA_SOURCE_ID'] == 0)[0]
print('missing sources after second match: ', len(j))
# replace NaNs
for name in out_names:
bd = np.where(np.isnan(tab[name]))[0]
tab[name][bd] = -9999.
return tab
def get_zero_point_simple(obj,
cat,
sr0=50.0,
filter='Clear',
maglim=None,
spatial=False):
zero = {}
h = htm.HTM(10)
m = h.match(obj['ra'],
obj['dec'],
cat['ra'],
cat['dec'],
sr0 * 1.0 / 3600,
maxmatch=0)
a = [0, 0, 0]
sd = 0.0
oidx = m[0]
cidx = m[1]
dist = m[2] * 3600
maxdist = 3.0 * np.std(dist)
print "Max distance = %g arcsec" % maxdist
zero['maxdist'] = maxdist
if filter == 'B':
mag_cat = cat['b'][cidx]
elif filter == 'V':
mag_cat = cat['v'][cidx]
elif filter == 'R':
mag_cat = cat['r'][cidx]
elif filter == 'U':
mag_cat = cat['u'][cidx]
else:
mag_cat = cat['v'][cidx]
mag_instr = obj['mag'][oidx]
x = obj['x'][oidx]
y = obj['y'][oidx]
zero['mag_cat'] = mag_cat
zero['mag_instr'] = mag_instr
zero['dist'] = dist
zero['x'] = x
zero['y'] = y
#weights = 10**(-0.4*mag_cat)
weights = np.repeat(1.0, len(mag_cat))
idx = (dist < 3.0 * np.std(dist))
if maglim:
idx = np.logical_and(idx, mag_cat < maglim)
mag0 = 0
for iter in range(3):
print len(dist[idx])
if spatial and len(dist[idx]) < 5:
break
elif len(dist[idx]) < 2:
break
sigma = np.std(mag_instr[idx] - mag_cat[idx])
mag0 = np.mean(mag_instr[idx] - mag_cat[idx])
#idx = np.logical_and(idx, np.abs(mag_instr - mag_v - mag0) < 3.0*sigma)
if spatial:
X = np.vstack([np.ones(len(mag_instr)), x * x, x, x * y, y,
y * y]).T
else:
X = np.vstack([np.ones(len(mag_instr))]).T
Y = mag_cat - mag_instr
C2 = sm.WLS(Y[idx], X[idx], weights=weights[idx]).fit().params
if spatial:
mag0 = C2[0] + C2[1] * x * x + C2[2] * x + C2[3] * x * y + C2[
4] * y + C2[5] * y * y
else:
mag0 = np.repeat(C2[0], len(x))
sigma2 = np.std(mag0[idx] + mag_instr[idx] - mag_cat[idx])
print C2, sigma2
idx = np.logical_and(idx,
np.abs(mag0 + mag_instr - mag_cat) < 3.0 * sigma2)
zero['C2'] = C2
zero['mag0s'] = mag0
zero['sigma'] = sigma2
zero['sigma0'] = np.std(mag0 + mag_instr - mag_cat)
zero['idx'] = idx
if spatial:
x = obj['x']
y = obj['y']
mag0 = C2[0] + C2[1] * x * x + C2[2] * x + C2[3] * x * y + C2[
4] * y + C2[5] * y * y
else:
mag0 = C2[0]
zero['mag0'] = mag0
zero['mag'] = obj['mag'] + mag0
return zero
def get_zero_point(obj, cat, sr0=50.0, maglim=None):
zero = {}
h = htm.HTM(10)
m = h.match(obj['ra'],
obj['dec'],
cat['ra'],
cat['dec'],
sr0 * 1.0 / 3600,
maxmatch=0)
a = [0, 0, 0]
sd = 0.0
oidx = m[0]
cidx = m[1]
dist = m[2] * 3600
maxdist = 3.0 * np.std(dist)
print "Max distance = %g arcsec" % maxdist
zero['maxdist'] = maxdist
mag_b = cat['b'][cidx]
mag_v = cat['v'][cidx]
mag_r = cat['r'][cidx]
mag_bv = mag_b - mag_v
mag_vr = mag_v - mag_r
mag_instr = obj['mag'][oidx]
x = obj['x'][oidx]
y = obj['y'][oidx]
zero['mag_b'] = mag_b
zero['mag_v'] = mag_v
zero['mag_r'] = mag_r
zero['mag_instr'] = mag_instr
weights = 10**(-0.4 * mag_v)
idx = (dist < 3.0 * np.std(dist))
if maglim:
idx = np.logical_and(idx, mag_v < 8)
for iter in range(3):
print len(dist[idx])
sigma = np.std(mag_instr[idx] - mag_v[idx])
mag0 = np.mean(mag_instr[idx] - mag_v[idx])
#idx = np.logical_and(idx, np.abs(mag_instr - mag_v - mag0) < 3.0*sigma)
#plt.clf(); plt.scatter(mag_v[idx], mag_instr[idx])
#X = np.vstack([np.ones(len(mag_instr)), x*x, x, x*y, y, y*y, mag_bv, mag_bv**2, mag_vr, mag_vr**2]).T
#X = np.vstack([np.ones(len(mag_instr)), x*x, x, x*y, y, y*y]).T
X = np.vstack([np.ones(len(mag_instr))]).T
Y = mag_v - mag_instr
C2 = sm.WLS(Y[idx], X[idx], weights=weights[idx]).fit().params
print sm.WLS(Y[idx], X[idx], weights=weights[idx]).fit().summary()
#mag0 = C2[0] + C2[1]*x*x + C2[2]*x + C2[3]*x*y + C2[4]*y + C2[5]*y*y + C2[6]*mag_bv + C2[7]*mag_bv**2 + C2[8]*mag_vr + C2[9]*mag_vr**2
#mag0 = C2[0] + C2[1]*x*x + C2[2]*x + C2[3]*x*y + C2[4]*y + C2[5]*y*y
mag0 = np.repeat(C2[0], len(x))
sigma2 = np.std(mag0[idx] + mag_instr[idx] - mag_v[idx])
print C2, sigma2
idx = np.logical_and(idx,
np.abs(mag0 + mag_instr - mag_v) < 3.0 * sigma2)
# plt.clf()
# plt.scatter(mag_instr, mag0 + mag_instr - mag_v, s=1)
# plt.scatter(mag_instr[idx], mag0[idx] + mag_instr[idx] - mag_v[idx], marker='.', c='red', s=3)
pass
zero['mag0'] = C2
return zero
def stack(infile='mastar-goodspec-v2_7_1-trunk',
apogee='r13/l33/allStar-r13-l33-58932',
out='mastar_apogee_stack',
cm=False):
""" Create list of MaStar objects with matching APOGEE spectra
and stack multiple observations into a single
Add in model spectra for warmer stars
"""
# read MaStar file and filter
mastar = fits.open(infile + '.fits')[1].data
#gd = clean(mastar,plot=False)
#mastar=mastar[gd]
# read APOGEE file and filter
apogee = fits.open(os.environ['APOGEE_ASPCAP'] + '/' + apogee +
'.fits')[1].data
gd = apselect.select(apogee, badval=['STAR_BAD'])
apogee = apogee[gd]
# get matches within 3 arcsec
h = htm.HTM()
maxrad = 3. / 3600.
m1, m2, rad = h.match(apogee['RA'],
apogee['DEC'],
mastar['OBJRA'],
mastar['OBJDEC'],
maxrad,
maxmatch=10)
# other stars to remove, e.g., based on bad modeling in initial model
bdstars = []
"""
bdstars=['3-15080992','3-105812093','7-12951805',
'3-142131061','60-36093244935984954','3-126079790',
'60-36091793237038712','3-126306258','3-24068365',
'3-23593772','60-36093244935984954',
'60-36091793237038712',
'3-140608451', '7-14409160', '3-114437013', '7-27039482',
'3-15080992', '3-148139562', '3-143029659', '3-103574358',
'3-139462126', '3-136745748', '3-142131061',
'3-142325470', '3-137769951', '3-141476933', '3-23120750',
'60-3615860471750191104', '7-17756858', '3-49507807',
'3-36341647', '3-22166243', '7-9789478', '3-124777940',
'60-676895989836309632', '3-141873923', '3-140806405',
'3-21696715', '7-7407046', '3-126309023', '3-125275478'],
"""
allbd = []
ngd = 0
nlo = 0
lo = []
spec = []
param = []
fparam = []
name = []
stars = set(mastar['MANGAID'])
nrot = 5
vsinis = 1. + np.arange(nrot) * np.log10(2.5)
deltav = (np.log(mastar['WAVE'][0, 1]) -
np.log(mastar['WAVE'][0, 0])) * 3.e5
for star in stars:
if star in bdstars: continue
j = np.where(mastar['MANGAID'] == star)[0]
print(star, len(j))
# create weighted average spectrum
num = np.sum(mastar['FLUX'][j, :] * mastar['IVAR'][j, :], axis=0)
den = np.sum(mastar['IVAR'][j, :], axis=0)
avg = num / den
err = 1 / den
sn = avg / err
# where S/N<2, set flux to 0
faint = np.where(scipy.signal.medfilt(sn, 21) < 2.)[0]
avg[faint] = 0.
if (sn[200:-200].min() < 10):
#print(star,apogee['PARAM'][j[0],:])
lo.append(j[0])
nlo += 1
# only take spectra with no bad pixels (neglecting first and last 10)
bd = np.where(np.isinf(err[10:-10]))[0]
if len(bd) < 1:
allbd.extend(bd.tolist())
spec.append(avg)
par = apogee['FPARAM'][j[0], :]
if cm:
fparam.append([par[0], par[1], par[3], par[6], par[7], par[4]])
else:
fparam.append([par[0], par[1], par[3], par[6], par[7]])
par = apogee['PARAM'][j[0], :]
elem = apogee['X_M'][j[0], :]
if cm:
param.append([par[0], par[1], par[3], elem[5], par[7], par[4]])
else:
param.append([par[0], par[1], par[3], elem[5], par[7]])
name.append(star)
ngd += 1
# if a slow rotator, broaden to accommodate different vsini
if 10.**par[7] < 10.:
for irot, vsini in enumerate(vsinis[1:]):
print(vsini)
rot = rotate(deltav, 10.**vsini)
conv = np.convolve(avg, rot, mode='same')
spec.append(conv)
if cm:
fparam.append(
[par[0], par[1], par[3], par[6], vsini, par[4]])
else:
fparam.append([par[0], par[1], par[3], par[6], vsini])
if cm:
param.append(
[par[0], par[1], par[3], elem[5], vsini, par[4]])
else:
param.append([par[0], par[1], par[3], elem[5], vsini])
name.append('{:s}_{:.1f}'.format(star, vsini))
param = np.array(param)
fparam = np.array(fparam)
print('Number of good stars: ', ngd)
print('Number of bad spectra: ', len(allbd), len(set(allbd)))
# plots
fig, ax = plots.multi(2, 2)
plots.plotc(ax[0, 0],
param[:, 0],
param[:, 1],
param[:, 2],
xr=[8000, 3000],
yr=[6, -1],
zr=[-2, 0.5],
xt='Teff',
yt='logg',
zt='[M/H]',
colorbar=True)
ax[0, 0].text(0.1, 0.9, '[M/H]', transform=ax[0, 0].transAxes)
plots.plotc(ax[1, 0],
param[:, 0],
param[:, 1],
param[:, 3],
xr=[8000, 3000],
yr=[6, -1],
zr=[-0.5, 0.5],
xt='Teff',
yt='logg',
zt='[alpha/M]',
colorbar=True)
ax[1, 0].text(0.1, 0.9, '[alpha/M]', transform=ax[1, 0].transAxes)
plots.plotc(ax[0, 1],
param[:, 2],
param[:, 3],
param[:, 0],
zr=[3500, 6000],
xr=[-2.5, 1],
yr=[-0.5, 0.5],
zt='Teff',
xt='[M/H]',
yt='[alpha/M]',
colorbar=True)
ax[1, 0].text(0.1, 0.9, '[alpha/M]', transform=ax[1, 0].transAxes)
if cm:
plots.plotc(ax[1, 1],
param[:, 0],
param[:, 1],
param[:, 5],
xr=[8000, 3000],
yr=[6, -1],
zr=[-0.5, 0.5],
xt='Teff',
yt='logg',
zt='[C/M]',
colorbar=True)
ax[1, 1].text(0.1, 0.9, '[C/M]', transform=ax[1, 1].transAxes)
#plots.plotc(ax[0,1],fparam[:,0],fparam[:,1],fparam[:,2],xr=[8000,3000],yr=[6,-1],zr=[-2,0.5],xt='uncal Teff',yt='uncal logg',zt='uncal [M/H]')
#plots.plotc(ax[1,1],fparam[:,0],fparam[:,1],fparam[:,3],xr=[8000,3000],yr=[6,-1],zr=[-0.5,0.5],xt='uncal Teff',yt='uncal logg',zt='uncal [alpha/H]')
fig.tight_layout()
fig.savefig(out + '.png')
#output FITS file
tab = Table()
tab.add_column(Column(name='TEFF', data=param[:, 0]))
tab.add_column(Column(name='LOGG', data=param[:, 1]))
tab.add_column(Column(name='[M/H]', data=param[:, 2]))
tab.add_column(Column(name='[alpha/M]', data=param[:, 3]))
tab.add_column(Column(name='LOG(VSINI)', data=param[:, 4]))
tab.add_column(Column(name='MANGAID', data=np.array(name)))
if cm: tab.add_column(Column(name='[C/M]', data=param[:, 5]))
tab.add_column(Column(name='SPEC', data=np.array(spec)))
tab.meta['LABELS'] = ['TEFF', 'LOGG', '[M/H]', '[alpha/M]', 'LOG(VSINI)']
tab.write(out + '.fits', overwrite=True)
# add in model spectra
a = []
b = []
c = []
d = []
e = []
f = []
v = []
for file in ['f_nsc2', 'f_nsc3', 'f_nsc4', 'f_nsc5']:
synth = fits.open(file + '_rot.fits')[0]
for irot, vsini in enumerate(spectra.fits2vector(synth.header, 5)):
for imh, mh in enumerate(spectra.fits2vector(synth.header, 4)):
if mh < -1: continue
for iteff, teff in enumerate(
spectra.fits2vector(synth.header, 3)):
if teff < 6000: continue
for ilogg, logg in enumerate(
spectra.fits2vector(synth.header, 2)):
if mh >= -2.5:
a.append(teff)
b.append(logg)
c.append(mh)
d.append(0.)
v.append(vsini)
e.append(
file +
'_{:.0f}_{:.1f}_{:.1f}'.format(teff, logg, mh))
f.append(synth.data[irot, imh, iteff, ilogg, :] /
np.median(synth.data[irot, imh, iteff,
ilogg, :]))
synthtab = Table()
synthtab.add_column(Column(name='TEFF', data=np.array(a)))
synthtab.add_column(Column(name='LOGG', data=np.array(b)))
synthtab.add_column(Column(name='[M/H]', data=np.array(c)))
synthtab.add_column(Column(name='[alpha/M]', data=np.array(d)))
synthtab.add_column(Column(name='LOG(VSINI)', data=np.array(v)))
synthtab.add_column(Column(name='MANGAID', data=np.array(e)))
if cm: synthtab.add_column(Column(name='[C/M]', data=np.array(d)))
synthtab.add_column(Column(name='SPEC', data=np.array(f)))
vstack([tab, synthtab]).write(out + '_synth.fits', overwrite=True)
def find_transients_in_frame(filename,
refine=True,
favor2=None,
workdir=None,
figure=None,
show=False,
verbose=False,
plot=True):
transients = []
favor2 = favor2 or Favor2()
ra0, dec0, sr0 = get_frame_center(filename=filename)
print "Frame %s at RA=%g Dec=%g with SR=%g" % (filename, ra0, dec0, sr0)
# Dir to hold all temporary files
basedir = workdir or tempfile.mkdtemp(prefix='survey')
filtered_name = posixpath.join(basedir, "filtered.fits")
print "Cleaning the frame"
cleaned_name = posixpath.join(basedir, "clean.fits")
filter_cosmics_in_file(filename, cleaned_name)
header = pyfits.getheader(cleaned_name, -1)
time = datetime.datetime.strptime(header['TIME'], '%d.%m.%Y %H:%M:%S.%f')
print "Pre-processing the frame"
os.system("nebuliser %s noconf %s 3 3 --takeout_sky" %
(cleaned_name, filtered_name))
if refine:
print "Refining the WCS"
matched_name = posixpath.join(basedir, "image.fits")
if not favor2.blind_match_file(filtered_name, outfile=matched_name):
print "Failure refining the WCS"
return transients
else:
matched_name = filtered_name
print "Extracting the objects"
obj = ss.get_objects(filename=matched_name,
options={
'DETECT_MINAREA': 3.0,
'DETECT_THRESH': 1.5,
'BACKPHOTO_TYPE': 'GLOBAL',
'DEBLEND_NTHRESH': 64,
'DEBLEND_MINCONT': 0.001,
'FILTER': 'Y',
'BACK_TYPE': 'MANUAL',
'BACK_VALUE': 0.0,
'BACK_SIZE': 64,
'BACK_FILTERSIZE': 3
},
fwhm=2,
aper_fwhm=2)
print "Querying the catalogues"
cat = favor2.get_stars(ra0, dec0, sr0, catalog='twomass', limit=1000000)
catn = favor2.get_stars(ra0, dec0, sr0, catalog='nomad', limit=1000000)
catt = favor2.get_stars(ra0, dec0, sr0, catalog='tycho2', limit=1000000)
cattb = favor2.get_stars(ra0,
dec0,
sr0,
catalog='tycho2',
limit=1000000,
extra='vt < 7')
print "%d objects, %d 2MASS stars, %d NOMAD stars, %d bright Tycho2 stars" % (
len(obj['ra']), len(cat['ra']), len(catn['ra']), len(cattb['ra']))
print "Matching the objects"
wcs = pywcs.WCS(header)
pixscale = wcs.pixel_scale_matrix.max(
) * 3600 # estimate of pixel scale, in arcsec/pix
size = 2.0 * obj['A_WORLD'] * pixscale
size = np.maximum(size, np.repeat(40.0, len(obj['ra'])))
h = htm.HTM(10)
m = h.match(obj['ra'],
obj['dec'],
cat['ra'],
cat['dec'],
size / 3600,
maxmatch=0)
mn = h.match(obj['ra'],
obj['dec'],
catn['ra'],
catn['dec'],
size / 3600,
maxmatch=0)
mt = h.match(obj['ra'],
obj['dec'],
catt['ra'],
catt['dec'],
size / 3600,
maxmatch=0)
if len(cattb['ra']):
mtb = h.match(obj['ra'],
obj['dec'],
cattb['ra'],
cattb['dec'], (size + 100.0) / 3600,
maxmatch=0)
uidx = np.setdiff1d(np.arange(len(obj['ra'])), m[0])
print "%d objects not matched with 2MASS" % len(uidx)
uidx = np.setdiff1d(uidx, mn[0])
print "%d objects not matched with NOMAD" % len(uidx)
uidx = np.setdiff1d(uidx, mt[0])
print "%d objects not matched with Tycho2" % len(uidx)
if len(cattb['ra']):
uidx = np.setdiff1d(uidx, mtb[0])
print "%d objects not matched with brightest Tycho2 stars" % len(uidx)
if len(uidx) < 100:
uidx = filter_indices(uidx, obj, time)
print "%d objects after line filtering and GSC 2.3.2 matching" % len(
uidx)
# Get the image and project the catalogues onto it
image = pyfits.getdata(filtered_name, -1)
cat_x, cat_y = wcs.all_world2pix(cat['ra'], cat['dec'], 0)
catn_x, catn_y = wcs.all_world2pix(catn['ra'], catn['dec'], 0)
catt_x, catt_y = wcs.all_world2pix(catt['ra'], catt['dec'], 0)
if len(cattb['ra']):
cattb_x, cattb_y = wcs.all_world2pix(cattb['ra'], cattb['dec'], 0)
if len(uidx) > 100:
print "Too many unmatched objects (%d), something is wrong!" % len(
uidx)
else:
# Let's iterate over the transients
for i in uidx:
transient = {
'x': float(obj['x'][i]),
'y': float(obj['y'][i]),
'flux': float(obj['FLUX_AUTO'][i]),
'flux_err': float(obj['FLUXERR_AUTO'][i]),
'flags': int(obj['FLAGS'][i]),
'a': float(obj['A_WORLD'][i]),
'b': float(obj['B_WORLD'][i]),
'theta': float(obj['THETA_WORLD'][i]),
'ra': float(obj['ra'][i]),
'dec': float(obj['dec'][i]),
'filename': filename,
'time': time,
'channel_id': header['CHANNEL ID']
}
transient['simbad'] = None
transient['mpc'] = None
try:
res = Simbad.query_region(
SkyCoord(obj['ra'][i], obj['dec'][i], unit='deg'),
"30 seconds")
if res is not None:
transient['simbad'] = ", ".join(
[r for r in res['MAIN_ID']])
except:
pass
try:
# Minor planets
mpc = get_skybot(obj['ra'][i],
obj['dec'][i],
30.0 / 3600,
time,
objFilter=110)
# Comets
if mpc is None:
mpc = get_skybot(obj['ra'][i],
obj['dec'][i],
300.0 / 3600,
time,
objFilter=1)
if mpc is not None:
transient['mpc'] = ", ".join([
"%s %s %s (Mv=%s)" % (mpc['class'][ii], mpc['num'][ii],
mpc['name'][ii], mpc['Mv'][ii])
for ii in xrange(len(mpc['name']))
])
except:
pass
if verbose:
#print transient
print "x = %g y = %g" % (obj['x'][i], obj['y'][i])
print "RA/Dec = %g %g = %s %s" % (
obj['ra'][i], obj['dec'][i],
str(ephem.hours(obj['ra'][i] * np.pi / 180)).replace(
":",
" "), str(ephem.degrees(
obj['dec'][i] * np.pi / 180)).replace(":", " "))
time0 = time.replace(hour=0, minute=0, second=0, microsecond=0)
print "Time = %s" % time
print "Time = %s + %g" % (time0,
(time - time0).total_seconds() / 24 /
3600)
if transient['simbad']:
print "SIMBAD: " + transient['simbad']
print
if plot:
icrop, x0, y0 = crop_image(image, obj['x'][i], obj['y'][i], 40)
limits = np.percentile(icrop, [0.5, 99.5])
if not figure:
size = 400
figure = Figure(facecolor='white',
dpi=72,
figsize=(size / 72, size / 72),
tight_layout=True)
figure.clear()
ax = figure.add_axes([0, 0, 1, 1])
ax.axis('off')
im = ax.imshow(icrop,
origin="bottom",
interpolation="nearest",
cmap="hot",
vmin=limits[0],
vmax=limits[1])
#figure.colorbar(im)
ax.autoscale(False)
ax.scatter(obj['x'] - x0,
obj['y'] - y0,
marker='o',
color='green',
s=400,
facecolors='None',
lw=3)
ax.scatter(cat_x - x0,
cat_y - y0,
marker='o',
color='blue',
s=100,
facecolors='None',
lw=2)
ax.scatter(catt_x - x0,
catt_y - y0,
marker='x',
s=100,
color='blue',
facecolors='None',
lw=2)
#ellipse = Ellipse(xy=(obj['x'][i]-x0, obj['y'][i]-y0), width=2.0*obj['A_WORLD'][i], height=2.0*obj['B_WORLD'][i], angle=obj['THETA_WORLD'][i], edgecolor='b', fc='None', axes=ax)
#ax.add_patch(ellipse)
if show:
figure.show(warn=False)
else:
canvas = FigureCanvas(figure)
sio = StringIO.StringIO()
canvas.print_png(sio, bbox_inches='tight')
transient[
'preview'] = "data:image/png;base64," + base64.b64encode(
sio.getvalue())
pass
else:
transient['preview'] = None
transients.append(transient)
# cleanup
if not workdir:
shutil.rmtree(basedir)
print "Done"
return transients
def match_objects(obj, cat, sr, fname='V', order=4, bg_order=None, cmag=None, color_order=None, thresh=5.0, clim=None, sn=10, mag_idx=0):
x0,y0,width,height = np.mean(obj['x']), np.mean(obj['y']), np.max(obj['x'])-np.min(obj['x']), np.max(obj['y'])-np.min(obj['y'])
# Match stars
h = htm.HTM(10)
m = h.match(obj['ra'],obj['dec'], cat['ra'],cat['dec'], sr, maxmatch=0)
oidx = m[0]
cidx = m[1]
dist = m[2]
if fname == 'B':
cmag,cmagerr = cat['B'], cat['Berr']
elif fname == 'V':
cmag,cmagerr = cat['V'], cat['Verr']
elif fname == 'R' or fname == 'N':
cmag,cmagerr = cat['R'], cat['Rerr']
elif fname == 'I':
cmag,cmagerr = cat['I'], cat['Ierr']
elif fname == 'g':
cmag,cmagerr = cat['g'], cat['gerr']
elif fname == 'r':
cmag,cmagerr = cat['r'], cat['rerr']
elif fname == 'i':
cmag,cmagerr = cat['i'], cat['ierr']
elif fname == 'z':
cmag,cmagerr = cat['z'], cat['zerr']
color = cat['B'] - cat['V']
if cmag is not None:
cmag = cmag[cidx]
cmagerr = cmagerr[cidx]
color = color[cidx]
else:
print('Unsupported filter:', fname)
return None
# Optimal fit limit, as mean mag where S/N = 10
idx = (1.0/obj['magerr'][oidx] > 5) & (1.0/obj['magerr'][oidx] < 15)
if np.sum(idx) > 10:
X = make_series(1.0, obj['x'][oidx], obj['y'][oidx], order=order)
X = np.vstack(X).T
Y = cmag
weights = 1/obj['magerr'][oidx]**2
Cmag_lim = sm.WLS(Y[idx], X[idx], weights=weights[idx]).fit()
cmag_lim = np.sum(X*Cmag_lim.params, axis=1)
else:
# print('Not enough matches with SN~10:', np.sum(idx))
Cmag_lim = None
cmag_lim = 99.0*np.ones_like(cmag)
# User provided upper fit limit
if clim is not None:
idx = cmag < clim
oidx,cidx,dist,cmag,cmagerr,cmag_lim,color = [_[idx] for _ in [oidx,cidx,dist,cmag,cmagerr,cmag_lim,color]]
x,y = obj['x'][oidx],obj['y'][oidx]
oflags = obj['flags'][oidx]
omag,omagerr = obj['mag'][oidx],obj['magerr'][oidx]
if len(obj['mag'].shape) > 1:
omag,omagerr = omag[:,mag_idx],omagerr[:,mag_idx]
x = (x - x0)*2/width
y = (y - y0)*2/height
tmagerr = np.hypot(cmagerr, omagerr)
tmagerr = np.hypot(tmagerr, 0.02)
delta_mag = cmag - omag
weights = 1.0/tmagerr**2
X = make_series(1.0, x, y, order=order)
if bg_order is not None and bg_order >= 0:
X += make_series(-2.5/np.log(10)/10**(-0.4*omag), x, y, order=bg_order)
if color_order is not None and color_order >= 0:
X += make_series(color, x, y, order=color_order)
X = np.vstack(X).T
Y = delta_mag
# X /= tmagerr[:, None]
# Y /= tmagerr
idx0 = (oflags == 0) & (cmag < cmag_lim)
if sn and sn>0:
idx0 &= (omagerr < 1/sn)
idx = idx0.copy()
# Do the fitting
for iter in range(3):
if len(X[idx]) < 3:
print("Fit failed - %d objects" % len(X[idx]))
return None
C = sm.WLS(Y[idx], X[idx], weights=weights[idx]).fit()
# C = sm.RLM(Y[idx], X[idx]).fit()
YY = np.sum(X*C.params, axis=1)
idx = idx0.copy()
if thresh and thresh > 0:
idx &= (np.abs((Y-YY)/tmagerr) < thresh)
# Apply the fit for all objects
x = (obj['x'] - x0)*2/width
y = (obj['y'] - y0)*2/height
if len(obj['mag'].shape) > 1:
mag = obj['mag'][:,mag_idx]
magerr = obj['magerr'][:,mag_idx]
else:
mag = obj['mag']
magerr = obj['magerr']
# Magnitude estimation for every position
X = make_series(1.0, x, y, order=order)
if bg_order is not None and bg_order >= 0:
X += make_series(-2.5/np.log(10)/10**(-0.4*obj['mag']), x, y, order=bg_order)
X = np.vstack(X).T
YY1 = np.sum(X*C.params[0:X.shape[1]], axis=1)
if len(obj['mag'].shape) > 1:
mag = obj['mag'][:,mag_idx] + YY1
else:
mag = obj['mag'] + YY1
# Additive flux component for every position
if bg_order is not None and bg_order >= 0:
Xbg = make_series(1, x, y, order=bg_order)
Xbg = np.vstack(Xbg).T
delta_flux = np.sum(Xbg*C.params[X.shape[1]-Xbg.shape[1]:X.shape[1]], axis=1)
else:
delta_flux = np.zeros_like(mag)
# Color term for every position
if color_order is not None and color_order >= 0:
Xc = make_series(1, x, y, order=color_order)
Xc = np.vstack(Xc).T
Cbv = np.sum(Xc*C.params[X.shape[1]:], axis=1)
else:
Cbv = np.zeros_like(mag)
# Approx magnitude limit (S/N=10) at every position
if Cmag_lim is not None:
Xlim = make_series(1.0, x, y, order=order)
Xlim = np.vstack(Xlim).T
cmag_lim = np.sum(Xlim*Cmag_lim.params, axis=1)
else:
cmag_lim = 99.0*np.ones_like(x)
# Simple analysis of proximity to "good" points
mx,my = obj['x'][oidx][idx], obj['y'][oidx][idx]
kdo = cKDTree(np.array([obj['x'], obj['y']]).T)
kdm = cKDTree(np.array([mx, my]).T)
mr0 = np.sqrt(width*height/np.sum(idx))
m = kdm.query_ball_tree(kdm, 5.0*mr0)
dists = []
for i,ii in enumerate(m):
if len(ii) > 1:
d1 = [np.hypot(mx[i]-mx[_], my[i]-my[_]) for _ in ii]
d1 = np.sort(d1)
dists.append(d1[1])
mr1 = np.median(dists)
m = kdo.query_ball_tree(kdm, 5.0*mr1)
midx = np.array([len(_)>1 for _ in m])
return {
# Matching indices and a distance in degrees
'cidx':cidx, 'oidx':oidx, 'dist':dist,
# Pixel and sky coordinates of matched stars, as well as their flags
'x':obj['x'][oidx], 'y':obj['y'][oidx], 'flags':oflags,
'ra':obj['ra'][oidx], 'dec':obj['dec'][oidx],
# All catalogue magnitudes of matched stars
'cB':cat['B'][cidx], 'cV':cat['V'][cidx], 'cR':cat['R'][cidx], 'cI':cat['I'][cidx],
'cBerr':cat['Berr'][cidx], 'cVerr':cat['Verr'][cidx], 'cRerr':cat['Rerr'][cidx], 'cIerr':cat['Ierr'][cidx],
# Catalogue magnitudes of matched stars in proper filter
'cmag':cmag, 'cmagerr':cmagerr, 'tmagerr':tmagerr,
# Model zero point for all objects, and their corrected magnitudes
'mag0':YY1, 'mag':mag,
'Cbv':Cbv, 'delta_flux': delta_flux,
'mag_idx': mag_idx,
'Y':Y, 'YY':YY,
'cmag_lim':cmag_lim,
# Subset of matched stars used in the fits
'idx':idx,
# Subset of all objects at 'good' distances from matched ones
'midx':midx, 'mr0':mr0, 'mr1':mr1}
def add_gaia(data,
gaia_1='gaia_2mass_xmatch.fits.gz',
gaia_2='gaia_posn_xmatch.fits.gz'):
""" Add GAIA data to allStar file, with coordinate match to (cross-matched) GAIA reference file
"""
tab = Table(data)
in_names = ('source_id', 'parallax', 'parallax_error', 'pmra',
'pmra_error', 'pmdec', 'pmdec_error', 'phot_g_mean_mag',
'phot_bp_mean_mag', 'phot_rp_mean_mag', 'a_g_val',
'e_bp_min_rp_val', 'radial_velocity', 'radial_velocity_error',
'r_est', 'r_lo', 'r_hi')
dtypes = ('i8', 'f8', 'f8', 'f8', 'f8', 'f8', 'f8', 'f4', 'f4', 'f4', 'f4',
'f4', 'f8', 'f8', 'f8', 'f8', 'f8')
out_names = []
for name in in_names:
out_names.append(('gaia_' + name).upper())
newcols = Table(np.zeros([len(tab), len(out_names)]) - 9999.,
names=out_names,
dtype=dtypes)
# for source_id, default to 0, not -9999.
newcols['GAIA_SOURCE_ID'] = 0
# get rid of targetting proper motions to avoid confusion!
#tab.remove_columns(['PMRA','PMDEC','PM_SRC'])
# change to rename
tab.rename_column('PMRA', 'TARG_PMRA')
tab.rename_column('PMDEC', 'TARG_PMDEC')
tab.rename_column('PM_SRC', 'TARG_PM_SRC')
# add unpopulated columns
tab.add_columns(newcols.columns.values())
# read gaia 2MASS matched file, match by 2MASS ID, and populate
gaia = fits.open(gaia_1)[1].data
print('number in GAIA-2MASS xmatch catalog: ', len(gaia),
len(set(gaia['original_ext_source_id'])))
while True:
# loop for matches since we have repeats and want them all matched
j = np.where(tab['GAIA_SOURCE_ID'] == 0)[0]
print('Number missing gaia_source_id: ', len(j))
m1, m2 = match.match(
np.core.defchararray.replace(tab['APOGEE_ID'][j], '2M', ''),
gaia['original_ext_source_id'])
print('Number matched by 2MASS: ', len(m1))
if len(m1) == 0: break
for inname, outname in zip(in_names, out_names):
tab[outname][j[m1]] = gaia[inname][m2]
#if len(m1) < 100 :
# for i in m1 : print(tab['APOGEE_ID'][j[m1]])
j = np.where(tab['GAIA_SOURCE_ID'] > 0)[0]
print('number of unique APOGEE_ID matches: ',
len(set(tab['APOGEE_ID'][j])))
j = np.where(tab['GAIA_SOURCE_ID'] == 0)[0]
print('missing sources after 2MASS matches: ', len(j))
gaia = fits.open(gaia_2)[1].data
h = htm.HTM()
# now do a positional match, take the brightest object within 3 arcsec (which is the max from the GAIA crossmatch)
maxrad = 3. / 3600.
m1, m2, rad = h.match(tab['RA'][j],
tab['DEC'][j],
gaia['RA'],
gaia['DEC'],
maxrad,
maxmatch=10)
for m in set(m1):
jj = np.where(m1 == m)[0]
ii = np.argsort(gaia['phot_rp_mean_mag'][m2[jj]])
#print(tab['RA'][j[m]],tab['DEC'][j[m]],tab['TARGET_ID'][j[m]])
#print(gaia['RA'][m2[jj]],gaia['DEC'][m2[jj]],gaia['PHOT_RP_MEAN_MAG'][m2[jj]])
#print(ii)
for inname, outname in zip(in_names, out_names):
tab[outname][j[m]] = gaia[inname][m2[jj[ii[0]]]]
j = np.where(tab['GAIA_SOURCE_ID'] == 0)[0]
print('missing sources after second match: ', len(j))
# replace NaNs
for name in out_names:
bd = np.where(np.isnan(tab[name]))[0]
tab[name][bd] = -9999.
return tab
def htm_matching(cat_A, cat_B, coord_A_names, coord_B_names, radius,
exclude_multiple=0, n_neigh=1,depth=10):
"""
Performs matching between two catalogs with the esutil HTM module.
This pipeline matches two catalogs in coordinates specified as input,
excluding unmatched objects in A and finding the neighbors in catalog B
to each of catalog A objects. It then returns cat_A and cat_B reordered
so that the neighbors are in the same line. For objects in A with more
than one nearby neighbor, entries are repeated.
There are two additional options concerning cases where there's multiple
matching. The first (default is 0) excludes multiple matching cases
(i.e. cases where the matching is not unique in both senses). The second
option (default is 1), selects the number of neighbors to be kept. The
value 0 will keep all the neighbors within the given radius.
For more information on the details of the procedure, refer to the
documentation of the esutil library in http://code.google.com/p/esutil/.
Input:
- cat_A hdudata : FITS Table data catalog
- cat_B hdudata : FITS Table data catalog
- coord_A_names [str,str] : ra_A and dec_A column names
- coord_B_names [str,str] : ra_B and dec_B column names
- radius float : Radius to perform matching
- exclude_multiple int : Keeps only the neighbor in the case of
single matching if not 0. If 0,
gets nearest of multiple neighbors.
Overrides n_neigh choice by definition.
- n_neigh int : Number of neighbors to keep for each
cat_A entry. 0 keeps all within radius.
Default is 1.
- depth int : Depth of the HTM triangular levels.
Ranges from 1 to 12. Default is 10.
Output:
- cat_A_matched hdudata : FITS Table data catalog of matched objs
- cat_B_matched hdudata : FITS Table data catalog of matched objs
---
"""
# Create HTM triangles
t0 = time()
print "Performing matching between catalogs..."
h = htm.HTM(depth)
cat_A_ra = cat_A.field(coord_A_names[0])
cat_A_dec = cat_A.field(coord_A_names[1])
cat_B_ra = cat_B.field(coord_B_names[0])
cat_B_dec = cat_B.field(coord_B_names[1])
print "Number of objects in catalog A: " + str(len(cat_A_ra))
print "Number of objects in catalog B: " + str(len(cat_B_ra))
# Exclude objects with multiple matching
if exclude_multiple:
print "Excluding objects in catalog A that have multiple matches..."
m1,m2,d12 = h.match(cat_A_ra, cat_A_dec, cat_B_ra, cat_B_dec, radius,
maxmatch = 0)
mltp = (np.arange(len(m2)) < len(m2))
for i in range(len(m2)):
if (m2[i] == m2).sum() > 1:
mltp = ~(m2[i] == m2)&mltp
else: pass
m1 = m1[mltp]
m2 = m2[mltp]
d12 = d12[mltp]
else:
m1,m2,d12 = h.match(cat_A_ra, cat_A_dec, cat_B_ra, cat_B_dec, radius,
maxmatch = n_neigh)
# Apply matching to catalogs
cat_A_matched = cat_A[m1]
cat_B_matched = cat_B[m2]
print "Number of objects in final matched catalogs: " + str(len(m2))
print "Total matching time: " + str(time()-t0) +" sec"
return cat_A_matched, cat_B_matched
def process_file(filename, favor2=None, verbose=False, replace=False, dbname=None, dbhost=None, photodir='photometry'):
#### Some parameters
aper = 2.0
bkgann = None
order = 4
bg_order = 4
color_order = 2
sn = 5
if not posixpath.exists(filename):
return None
# Rough but fast checking of whether the file is already processed
if not replace and posixpath.exists(photodir + '/' + filename.split('/')[-2] + '/' + posixpath.splitext(posixpath.split(filename)[-1])[0] + '.cat'):
return
#### Preparation
header = fits.getheader(filename, -1)
if header['TYPE'] not in ['survey', 'imaging', 'widefield', 'Swift', 'Fermi', 'test']:
return
channel = header.get('CHANNEL ID')
fname = header.get('FILTER', 'unknown')
time = parse_time(header['TIME'])
shutter = header.get('SHUTTER', -1)
if fname not in ['Clear']:
return
if fname == 'Clear':
effective_fname = 'V'
else:
effective_fname = fname
night = get_night(time)
dirname = '%s/%s' % (photodir, night)
basename = posixpath.splitext(posixpath.split(filename)[-1])[0]
basename = dirname + '/' + basename
catname = basename + '.cat'
if not replace and posixpath.exists(catname):
return
if verbose:
print(filename, channel, night, fname, effective_fname)
image = fits.getdata(filename, -1).astype(np.double)
if favor2 is None:
favor2 = Favor2(dbname=options.db, dbhost=options.dbhost)
#### Basic calibration
darkname = favor2.find_image('masterdark', header=header, debug=False)
flatname = favor2.find_image('masterflat', header=header, debug=False)
if darkname:
dark = fits.getdata(darkname)
else:
dark = None
if flatname:
flat = fits.getdata(flatname)
else:
flat = None
if dark is None or flat is None:
survey.save_objects(catname, None)
return
image,header = calibrate.calibrate(image, header, dark=dark)
# Check whether the calibration failed
if 'SATURATE' not in header:
print('Calibration failed for', filename)
survey.save_objects(catname, None)
return
#### Basic masking
mask = image > 0.9*header['SATURATE']
fmask = ~np.isfinite(flat) | (flat < 0.5)
dmask = dark > 10.0*mad_std(dark) + np.nanmedian(dark)
if header.get('BLEMISHCORRECTION', 1):
# We have to mask blemished pixels
blemish = fits.getdata('calibrations/blemish_shutter_%d_channel_%d.fits' % (header['SHUTTER'], header['CHANNEL ID']))
if verbose:
print(100*np.sum(blemish>0)/blemish.shape[0]/blemish.shape[1], '% pixels blemished')
dmask |= blemish > 0
image[~fmask] *= np.median(flat[~fmask])/flat[~fmask]
#### WCS
wcs = WCS(header)
pixscale = np.hypot(wcs.pixel_scale_matrix[0,0], wcs.pixel_scale_matrix[0,1])
gain = 0.67 if header.get('SHUTTER') == 0 else 1.9
#### Background mask
mask_bg = np.zeros_like(mask)
mask_segm = np.zeros_like(mask)
# bg2 = sep.Background(image, mask=mask|mask_bg, bw=64, bh=64)
# for _ in xrange(3):
# bg1 = sep.Background(image, mask=mask|mask_bg, bw=256, bh=256)
# ibg = bg2.back() - bg1.back()
# tmp = np.abs(ibg - np.median(ibg)) > 5.0*mad_std(ibg)
# mask_bg |= survey.dilate(tmp, np.ones([50, 50]))
# mask_bg = survey.dilate(tmp, np.ones([50, 50]))
# Large objects?..
bg = sep.Background(image, mask=mask|dmask|fmask|mask_bg|mask_segm, bw=128, bh=128)
image1 = image - bg.back()
obj0,segm = sep.extract(image1, err=bg.rms(), thresh=10, minarea=10, mask=mask|dmask|fmask|mask_bg, filter_kernel=None, clean=False, segmentation_map=True)
mask_segm = np.isin(segm, [_+1 for _,npix in enumerate(obj0['npix']) if npix > 500])
mask_segm = survey.dilate(mask_segm, np.ones([20, 20]))
if np.sum(mask_bg|mask_segm|mask|fmask|dmask)/mask_bg.shape[0]/mask_bg.shape[1] > 0.4:
print(100*np.sum(mask_bg|mask_segm|mask|fmask|dmask)/mask_bg.shape[0]/mask_bg.shape[1], '% of image masked, skipping', filename)
survey.save_objects(catname, None)
return
elif verbose:
print(100*np.sum(mask_bg|mask_segm|mask|fmask|dmask)/mask_bg.shape[0]/mask_bg.shape[1], '% of image masked')
# Frame footprint at +10 pixels from the edge
ra,dec = wcs.all_pix2world([10, 10, image.shape[1]-10, image.shape[1]-10], [10, image.shape[0]-10, image.shape[0]-10, 10], 0)
footprint = "(" + ",".join(["(%g,%g)" % (_,__) for _,__ in zip(ra, dec)]) + ")"
#### Catalogue
ra0,dec0,sr0 = survey.get_frame_center(header=header)
cat = favor2.get_stars(ra0, dec0, sr0, catalog='gaia', extra=['g<14', 'q3c_poly_query(ra, dec, \'%s\'::polygon)' % footprint], limit=1000000)
if verbose:
print(len(cat['ra']), 'star positions from Gaia down to g=%.1f mag' % np.max(cat['g']))
## Detection of blended and not really needed stars in the catalogue
h = htm.HTM(10)
m = h.match(cat['ra'], cat['dec'], cat['ra'], cat['dec'], 2.0*aper*pixscale, maxmatch=0)
m = [_[m[2]>1e-5] for _ in m]
blended = np.zeros_like(cat['ra'], dtype=np.bool)
notneeded = np.zeros_like(cat['ra'], dtype=np.bool)
for i1,i2,dist in zip(*m):
if dist*3600 > 0.5*aper*pixscale:
if cat['g'][i1] - cat['g'][i2] < 3:
blended[i1] = True
blended[i2] = True
else:
# i1 is fainter by more than 3 mag
notneeded[i1] = True
if dist*3600 < 0.5*aper*pixscale:
if cat['g'][i1] > cat['g'][i2]:
notneeded[i1] = True
cat,blended = [_[~notneeded] for _ in cat,blended]
#### Background subtraction
bg = sep.Background(image, mask=mask|dmask|fmask|mask_bg|mask_segm, bw=128, bh=128)
# bg = sep.Background(image, mask=mask|dmask|fmask|mask_bg|mask_segm, bw=32, bh=32)
image1 = image - bg.back()
#### Detection of all objects on the frame
obj0,segm = sep.extract(image1, err=bg.rms(), thresh=2, minarea=3, mask=mask|dmask|fmask|mask_bg|mask_segm, filter_kernel=None, clean=False, segmentation_map=True)
obj0 = obj0[(obj0['x'] > 10) & (obj0['y'] > 10) & (obj0['x'] < image.shape[1]-10) & (obj0['y'] < image.shape[0]-10)]
obj0 = obj0[obj0['flag'] <= 1] # We keep only normal and blended oblects
fields = ['ra', 'dec', 'fluxerr', 'mag', 'magerr', 'flags', 'cat']
obj0 = np.lib.recfunctions.append_fields(obj0, fields, [np.zeros_like(obj0['x'], dtype=np.int if _ in ['flags', 'cat'] else np.double) for _ in fields], usemask=False)
obj0['ra'],obj0['dec'] = wcs.all_pix2world(obj0['x'], obj0['y'], 0)
obj0['flags'] = obj0['flag']
if verbose:
print(len(obj0['x']), 'objects detected on the frame')
## Filter out objects not coincident with catalogue positions
h = htm.HTM(10)
m = h.match(obj0['ra'], obj0['dec'], cat['ra'], cat['dec'], aper*pixscale)
nidx = np.isin(np.arange(len(obj0['ra'])), m[0], invert=True)
obj0 = obj0[nidx]
if verbose:
print(len(obj0['x']), 'are outside catalogue apertures')
# Catalogue stars
xc,yc = wcs.all_world2pix(cat['ra'], cat['dec'], 0)
obj = {'x':xc, 'y':yc, 'ra':cat['ra'], 'dec':cat['dec']}
obj['flags'] = np.zeros_like(xc, dtype=np.int)
obj['flags'][blended] |= FLAG_BLENDED
obj['cat'] = np.ones_like(xc, dtype=np.int)
for _ in ['mag', 'magerr', 'flux', 'fluxerr']:
obj[_] = np.zeros_like(xc)
# Merge detected objects
for _ in ['x', 'y', 'ra', 'dec', 'flags', 'mag', 'magerr', 'flux', 'fluxerr', 'cat']:
obj[_] = np.concatenate((obj[_], obj0[_]))
if verbose:
print(len(obj['x']), 'objects for photometry')
# Simple aperture photometry
obj['flux'],obj['fluxerr'],flag = sep.sum_circle(image1, obj['x'], obj['y'], aper, err=bg.rms(), gain=gain, mask=mask|dmask|fmask|mask_bg|mask_segm, bkgann=bkgann)
obj['flags'] |= flag
# Normalize flags
obj['flags'][obj['flags'] & sep.APER_TRUNC] |= FLAG_TRUNCATED
obj['flags'][obj['flags'] & sep.APER_ALLMASKED] |= FLAG_MASKED
obj['flags'] &= FLAG_NORMAL | FLAG_BLENDED | FLAG_TRUNCATED | FLAG_MASKED | FLAG_NO_BACKGROUND | FLAG_BAD_CALIBRATION
area,_,_ = sep.sum_circle(np.ones_like(image1), obj['x'], obj['y'], aper, err=bg.rms(), gain=gain, mask=mask|dmask|fmask|mask_bg|mask_segm, bkgann=bkgann)
# Simple local background estimation
bgflux,bgfluxerr,bgflag = sep.sum_circann(image1, obj['x'], obj['y'], 10, 15, err=bg.rms(), gain=gain, mask=mask|dmask|fmask|mask_bg|mask_segm|(segm>0))
bgarea,_,_ = sep.sum_circann(np.ones_like(image1), obj['x'], obj['y'], 10, 15, err=bg.rms(), gain=gain, mask=mask|dmask|fmask|mask_bg|mask_segm|(segm>0))
bgidx = np.isfinite(bgarea) & np.isfinite(area)
bgidx[bgidx] &= (bgarea[bgidx] > 10) & (area[bgidx] > 1)
obj['flux'][bgidx] -= bgflux[bgidx]*area[bgidx]/bgarea[bgidx]
obj['flags'][~bgidx] |= FLAG_NO_BACKGROUND # No local background
obj['deltabgflux'] = np.zeros_like(obj['x'])
obj['deltabgflux'][bgidx] = bgflux[bgidx]*area[bgidx]/bgarea[bgidx]
fidx = np.isfinite(obj['flux']) & np.isfinite(obj['fluxerr'])
fidx[fidx] &= (obj['flux'][fidx] > 0)
obj['mag'][fidx] = -2.5*np.log10(obj['flux'][fidx])
obj['magerr'][fidx] = 2.5/np.log(10)*obj['fluxerr'][fidx]/obj['flux'][fidx]
fidx[fidx] &= (obj['magerr'][fidx] > 0)
fidx[fidx] &= 1/obj['magerr'][fidx] > sn
for _ in obj.keys():
if hasattr(obj[_], '__len__'):
obj[_] = obj[_][fidx]
obj['aper'] = aper
if verbose:
print(len(obj['x']), 'objects with S/N >', sn)
if len(obj['x']) < 1000:
print('Only', len(obj['x']), 'objects on the frame, skipping', filename)
survey.save_objects(catname, None)
return
obj['fwhm'] = 2.0*sep.flux_radius(image1, obj['x'], obj['y'], 2.0*aper*np.ones_like(obj['x']), 0.5, mask=mask|dmask|fmask|mask_bg|mask_segm)[0]
#### Check FWHM of all objects and select only 'good' ones
idx = obj['flags'] == 0
idx &= obj['magerr'] < 1/20
fwhm0 = survey.fit_2d(obj['x'][idx], obj['y'][idx], obj['fwhm'][idx], obj['x'], obj['y'], weights=1/obj['magerr'][idx])
fwhm_idx = np.abs(obj['fwhm'] - fwhm0 - np.median((obj['fwhm'] - fwhm0)[idx])) < 3.0*mad_std((obj['fwhm'] - fwhm0)[idx])
obj['flags'][~fwhm_idx] |= FLAG_BLENDED
#### Catalogue matching
idx = obj['flags']
m = htm.HTM(10).match(obj['ra'], obj['dec'], cat['ra'], cat['dec'], 1e-5)
fidx = np.in1d(np.arange(len(cat['ra'])), m[1]) # Stars that got successfully measured and not blended
cidx = (cat['good'] == 1) & (cat['var'] == 0)
cidx &= np.isfinite(cat['B']) & np.isfinite(cat['V']) # & np.isfinite(cat['lum'])
cidx[cidx] &= ((cat['B'] - cat['V'])[cidx] > -0.5) & ((cat['B'] - cat['V'])[cidx] < 2.0)
# cidx[cidx] &= (cat['lum'][cidx] > 0.3) & (cat['lum'][cidx] < 30)
if np.sum(cidx & fidx & (cat['multi_70'] == 0)) > 2000:
cidx &= (cat['multi_70'] == 0)
obj['cat_multi'] = 70
elif np.sum(cidx & fidx & (cat['multi_45'] == 0)) > 1000:
cidx &= (cat['multi_45'] == 0)
obj['cat_multi'] = 45
else:
cidx &= (cat['multi_30'] == 0)
obj['cat_multi'] = 30
if verbose:
print(np.sum(obj['flags'] == 0), 'objects without flags')
print('Amount of good stars:',
np.sum(cidx & fidx & (cat['multi_70'] == 0)),
np.sum(cidx & fidx & (cat['multi_45'] == 0)),
np.sum(cidx & fidx & (cat['multi_30'] == 0)))
print('Using %d arcsec avoidance radius' % obj['cat_multi'])
# We match with very small SR to only account for manually placed apertures
if verbose:
print('Trying full fit:', len(obj['x']), 'objects,', np.sum(cidx), 'stars')
match = Match(width=image.shape[1], height=image.shape[0])
prev_ngoodstars = len(obj['x'])
for iter in range(10):
if not match.match(obj=obj, cat=cat[cidx], sr=1e-5, filter_name='V', order=order, bg_order=bg_order, color_order=color_order, verbose=False) or match.ngoodstars < 500:
if verbose:
print(match.ngoodstars, 'good matches, matching failed for', filename)
survey.save_objects(catname, None)
return
if verbose:
print(match.ngoodstars, 'good matches, std =', match.std)
if match.ngoodstars == prev_ngoodstars:
if verbose:
print('Converged on iteration', iter)
break
prev_ngoodstars = match.ngoodstars
# Match good objects with stars
oidx = obj['flags'] == 0
oidx1,cidx1,dist1 = htm.HTM(10).match(obj['ra'][oidx], obj['dec'][oidx], cat['ra'][cidx], cat['dec'][cidx], 1e-5)
x = obj['x'][oidx][oidx1]
y = obj['y'][oidx][oidx1]
cbv = match.color_term[oidx][oidx1]
cbv2 = match.color_term2[oidx][oidx1]
cbv3 = match.color_term3[oidx][oidx1]
bv = (cat['B'] - cat['V'])[cidx][cidx1]
cmag = cat[match.cat_filter_name][cidx][cidx1]
mag = match.mag[oidx][oidx1] + bv*cbv + bv**2*cbv2 + bv**3*cbv3
magerr = np.hypot(obj['magerr'][oidx][oidx1], 0.02)
dmag = mag-cmag
ndmag = ((mag-cmag)/magerr)
idx = cmag < match.mag_limit[oidx][oidx1]
x,y,cbv,cbv2,cbv3,bv,cmag,mag,magerr,dmag,ndmag = [_[idx] for _ in [x,y,cbv,cbv2,cbv3,bv,cmag,mag,magerr,dmag,ndmag]]
# Match all objects with good objects
xy = np.array([x,y]).T
xy0 = np.array([obj['x'], obj['y']]).T
kd = cKDTree(xy)
dist,m = kd.query(xy0, 101)
dist = dist[:,1:]
m = m[:,1:]
vchi2 = mad_std(ndmag[m]**2, axis=1)
# Mark regions of too sparse or too noisy matches as bad
obj['flags'][vchi2 > 5] |= FLAG_BAD_CALIBRATION
# obj['flags'][vchi2 > np.median(vchi2) + 5.0*mad_std(vchi2)] |= FLAG_BAD_CALIBRATION
obj['flags'][dist[:,10] > np.median(dist[:,10]) + 10.0*mad_std(dist[:,10])] |= FLAG_BAD_CALIBRATION
match.good_idx = (obj['flags'] & FLAG_BAD_CALIBRATION) == 0
if verbose:
print(np.sum(match.good_idx), 'of', len(match.good_idx), 'stars are good')
#### Store objects to file
try:
os.makedirs(dirname)
except:
pass
obj['mag_limit'] = match.mag_limit
obj['color_term'] = match.color_term
obj['color_term2'] = match.color_term2
obj['color_term3'] = match.color_term3
obj['filename'] = filename
obj['night'] = night
obj['channel'] = channel
obj['filter'] = fname
obj['cat_filter'] = match.cat_filter_name
obj['time'] = time
obj['mag_id'] = match.mag_id
obj['good_idx'] = match.good_idx
obj['calib_mag'] = match.mag
obj['calib_magerr'] = match.magerr
obj['std'] = match.std
obj['nstars'] = match.ngoodstars
survey.save_objects(catname, obj, header=header)
def calibrate_objects(obj,
cat,
sr=15.0 / 3600,
smooth=False,
correct_brightness=False,
smoothr=100,
smoothmin=20,
retval='std'):
h = htm.HTM(10)
m = h.match(obj['ra'], obj['dec'], cat['ra'], cat['dec'], sr, maxmatch=0)
oidx = m[0]
cidx = m[1]
dist = m[2] * 3600
if len(oidx) < 300:
return None
x, y = obj['x'][oidx], obj['y'][oidx]
x = (x - 2560 / 2) * 2 / 2560
y = (y - 2160 / 2) * 2 / 2160
omag = obj['mag'][oidx]
omagerr = obj['magerr'][oidx]
oflux = obj['flux'][oidx]
ofluxerr = obj['fluxerr'][oidx]
oflags = obj['flags'][oidx]
ochisq = obj['chisq'][oidx]
omag0 = (omag - np.min(omag)) / (np.max(omag) - np.min(omag))
try:
cb = cat['g'][cidx]
cv = cat['r'][cidx]
cr = cat['i'][cidx]
cmagerr = np.hypot(cat['gerr'][cidx], cat['rerr'][cidx],
cat['ierr'][cidx])
except:
cb = cat['b'][cidx]
cv = cat['v'][cidx]
cr = cat['r'][cidx]
try:
cmagerr = np.hypot(cat['berr'][cidx], cat['verr'][cidx],
cat['rerr'][cidx])
except:
cmagerr = np.hypot(cat['ebt'][cidx], cat['evt'][cidx])
cmag = cv
tmagerr = np.hypot(cmagerr, omagerr)
delta_mag = cmag - omag
weights = 1.0 / tmagerr
X = [
np.ones(len(delta_mag)), x, y, x * x, x * y, y * y, x * x * x,
x * x * y, x * y * y, y * y * y, x * x * x * x, x * x * x * y,
x * x * y * y, x * y * y * y, y * y * y * y, x * x * x * x * x,
x * x * x * x * y, x * x * x * y * y, x * x * y * y * y,
x * y * y * y * y, y * y * y * y * y, x * x * x * x * x * x,
x * x * x * x * x * y, x * x * x * x * y * y, x * x * x * y * y * y,
x * x * y * y * y * y, x * y * y * y * y * y, y * y * y * y * y * y
]
#X += [omag0, omag0**2, omag0**3, omag0**4, omag0**5, omag0**6]
X += [
cb - cv, (cb - cv) * x,
(cb - cv) * y, (cb - cv) * x * x, (cb - cv) * x * y, (cb - cv) * y * y,
(cb - cv) * x * x * x, (cb - cv) * x * x * y, (cb - cv) * x * y * y,
(cb - cv) * y * y * y
]
X += [
cv - cr, (cv - cr) * x,
(cv - cr) * y, (cv - cr) * x * x, (cv - cr) * x * y, (cv - cr) * y * y,
(cv - cr) * x * x * x, (cv - cr) * x * x * y, (cv - cr) * x * y * y,
(cv - cr) * y * y * y
]
X = np.vstack(X).T
Y = delta_mag
idx = (oflags == 0) & (ochisq < 2.0)
for iter in range(5):
if len(X[idx]) < 100:
print "Fit failed - %d objects" % len(X[idx])
return None
C = sm.WLS(Y[idx], X[idx], weights=weights[idx]).fit()
YY = np.sum(X * C.params, axis=1)
idx = (np.abs(Y - YY) < 2.0 * np.std(Y - YY)) & (oflags
== 0) & (ochisq < 2.0)
print np.std((Y - YY)[idx]), np.std(Y - YY), '-', np.std(
((Y - YY) / tmagerr)), np.std(
(((Y - YY) / tmagerr))[idx]), '-', len(idx[idx])
mag0 = YY
mag = omag + mag0
x = (obj['x'] - 2560 / 2) * 2 / 2560
y = (obj['y'] - 2160 / 2) * 2 / 2160
omag0 = (obj['mag'] - np.min(omag)) / (np.max(omag) - np.min(omag))
obj['mag0'] = C.params[0] * np.ones_like(
obj['mag']
) + C.params[1] * x + C.params[2] * y + C.params[3] * x * x + C.params[
4] * x * y + C.params[5] * y * y + C.params[6] * x * x * x + C.params[
7] * x * x * y + C.params[8] * x * y * y + C.params[9] * y * y * y + C.params[
10] * x * x * x * x + C.params[11] * x * x * x * y + C.params[
12] * x * x * y * y + C.params[13] * x * y * y * y + C.params[
14] * y * y * y * y + C.params[15] * x * x * x * x * x + C.params[
16] * x * x * x * x * y + C.params[17] * x * x * x * y * y + C.params[
18] * x * x * y * y * y + C.params[19] * x * y * y * y * y + C.params[
20] * y * y * y * y * y + C.params[
21] * x * x * x * x * x * x + C.params[
22] * x * x * x * x * x * y + C.params[
23] * x * x * x * x * y * y + C.params[
24] * x * x * x * y * y * y + C.params[
25] * x * x * y * y * y * y + C.params[
26] * x * y * y * y * y * y + C.params[
27] * y * y * y * y * y * y
#obj['mag0'] += C.params[28]*omag0 + C.params[29]*omag0**2 + C.params[30]*omag0**3 + C.params[31]*omag0**4 + C.params[32]*omag0**5 + C.params[33]*omag0**6
obj['cmag'] = obj['mag'] + obj['mag0']
obj['Cbv'] = np.ones_like(obj['mag']) * C.params[-20] + x * C.params[
-19] + y * C.params[-18] + x * x * C.params[-17] + x * y * C.params[
-16] + y * y * C.params[-15] + x * x * x * C.params[
-14] + x * x * y * C.params[-13] + x * y * y * C.params[
-12] + y * y * y * C.params[-11]
obj['Cvr'] = np.ones_like(obj['mag']) * C.params[-10] + x * C.params[
-9] + y * C.params[-8] + x * x * C.params[-7] + x * y * C.params[
-6] + y * y * C.params[-5] + x * x * x * C.params[
-4] + x * x * y * C.params[-3] + x * y * y * C.params[
-2] + y * y * y * C.params[-1]
# Correct uncorrected brightness-dependent variations
obj['bcorr'] = np.zeros_like(obj['cmag'])
if correct_brightness:
step = 0.5
omag0 = omag + np.median(cmag - omag)
for vmin in np.arange(5, 16, step):
idx0 = (omag0[idx] >= vmin) & (omag0[idx] < vmin + step)
if len(omag0[idx][idx0] > 20):
omag1 = obj['mag'] + np.median(cmag - omag)
idx1 = (omag1 >= vmin) & (omag1 < vmin + step)
obj['bcorr'][idx1] = np.average(
(Y - YY)[idx][idx0], weights=1.0 / tmagerr[idx][idx0])
# Local smoothing of residuals
if smooth:
kd0 = cKDTree(np.array([obj['x'][oidx][idx], obj['y'][oidx][idx]]).T)
kd = cKDTree(np.array([obj['x'], obj['y']]).T)
m = kd.query_ball_tree(kd0, smoothr)
nm = np.array(([len(_) for _ in m]))
corr = np.array([
np.average((Y - YY)[idx][_], weights=weights[idx][_])
if len(_) > smoothmin else 0 for _ in m
])
icorr = nm > smoothmin
obj['corr'] = corr
obj['corrected'] = icorr
obj['ncorr'] = nm
else:
obj['corr'] = np.zeros_like(obj['mag'])
obj['corrected'] = np.ones_like(obj['mag'], dtype=np.bool)
obj['ncorr'] = np.zeros_like(obj['mag'])
if retval == 'std':
return np.std(Y - YY)
elif retval == 'map':
return Y
elif retval == 'fit':
return YY
elif retval == 'diff':
return Y - YY
elif retval == 'all':
return Y, YY, obj['corr'], oidx
def match(self, obj=None, cat=None, sr=5./3600, verbose=False, predict=True,
ra=None, dec=None, x=None, y=None, mag=None, magerr=None, flags=None,
filter_name='V', order=4, bg_order=None, color_order=None,
hard_mag_limit=99, mag_id=0, magerr0=0.02, sn=None, thresh=5.0,
mask=None, good_flags=0x0):
"""Match a set of points with catalogue"""
self.success = False
self.ngoodstars = 0
self.order = order
self.bg_order = bg_order
self.color_order = color_order
self.mag_id = mag_id
self.filter_name = filter_name
if filter_name in ['B', 'V', 'R', 'I', 'g', 'r', 'i', 'z']:
# Generic names
cmag,cmagerr = cat[filter_name], cat[filter_name + 'err']
self.cat_filter_name = filter_name
elif filter_name == 'Clear':
# Mini-MegaTORTORA
cmag,cmagerr = cat['V'], cat['Verr']
self.cat_filter_name = 'V'
elif filter_name == 'N':
# FRAMs
cmag,cmagerr = cat['R'], cat['Rerr']
self.cat_filter_name = 'R'
else:
if verbose:
print('Unsupported filter name: %s' % filter_name)
return False
# TODO: make it configurable?..
color = cat['B'] - cat['V']
self.cat_color_name = 'B - V'
# Objects to match
if obj is not None:
ra = obj['ra']
dec = obj['dec']
x = obj['x']
y = obj['y']
mag = obj['mag']
magerr = obj['magerr']
flags = obj['flags']
else:
if ra is None or dec is None or x is None or y is None or mag is None:
raise ValueError('Data for matching are missing')
if magerr is None:
magerr = np.ones_like(mag)*np.std(mag)
if flags is None:
flags = np.zeros_like(ra, dtype=np.int)
if self.width is None or self.height is None:
self.x0,self.y0,self.width,self.height = np.mean(x), np.mean(y), np.max(x)-np.min(x), np.max(y) - np.min(y)
# Match stars
h = htm.HTM(10)
oidx,cidx,dist = h.match(ra, dec, cat['ra'],cat['dec'], sr, maxmatch=0)
if verbose:
print(len(oidx), 'matches between', len(ra), 'objects and', len(cat['ra']), 'stars, sr = %.1f arcsec' % (3600.0*sr))
self.oidx,self.cidx,self.dist = oidx, cidx, dist
self.cmag = cmag[cidx]
self.cmagerr = cmagerr[cidx]
self.color = color[cidx]
self.ox,self.oy = x[oidx], y[oidx]
self.oflags = flags[oidx]
self.omag,self.omagerr = mag[oidx], magerr[oidx]
if len(self.omag.shape) > 1:
# If we are given a multi-aperture magnitude column
self.omag,self.omagerr = self.omag[:, mag_id], self.omagerr[:, mag_id]
# Scaled spatial coordinates for fitting
sx = (self.ox - self.x0)*2/self.width
sy = (self.oy - self.y0)*2/self.height
# Optimal magnitude cutoff for fitting, as a mean mag where S/N = 10
idx = (1.0/self.omagerr > 5) & (1.0/self.omagerr < 15)
if np.sum(idx) > 10:
X = make_series(1.0, sx, sy, order=order)
X = np.vstack(X).T
Y = self.cmag
self.C_mag_limit = sm.RLM(Y[idx], X[idx]).fit()
mag_limit = np.sum(X*self.C_mag_limit.params, axis=1)
else:
if verbose:
print('Not enough matches with SN~10:', np.sum(idx))
self.C_mag_limit = None
mag_limit = 99.0*np.ones_like(cmag)
self.zero = self.cmag - self.omag # We will build a model for this variable
self.zeroerr = np.hypot(self.omagerr, self.cmagerr)
self.zeroerr = np.hypot(self.zeroerr, magerr0)
self.weights = 1.0/self.zeroerr**2
X = make_series(1.0, sx, sy, order=self.order)
if self.bg_order is not None:
X += make_series(-2.5/np.log(10)/10**(-0.4*self.omag), sx, sy, order=self.bg_order)
if self.color_order is not None:
X += make_series(self.color, sx, sy, order=self.color_order)
X += make_series(self.color**2, sx, sy, order=self.color_order)
X += make_series(self.color**3, sx, sy, order=self.color_order)
X = np.vstack(X).T
self.idx0 = ((self.oflags & (~good_flags)) == 0) & (self.cmag < hard_mag_limit) & (self.cmag < mag_limit)
if mask is not None:
# Exclude masked objects
self.idx0 &= ~mask
if sn is not None:
self.idx0 &= (self.omagerr < 1.0/sn)
# Actual fitting
self.idx = self.idx0.copy()
for iter in range(3):
if np.sum(self.idx) < 3:
if verbose:
print("Fit failed - %d objects" % np.sum(self.idx))
return False
self.C = sm.WLS(self.zero[self.idx], X[self.idx], weights=self.weights[self.idx]).fit()
# self.C = sm.RLM(self.zero[self.idx], X[self.idx]).fit()
self.zero_model = np.sum(X*self.C.params, axis=1)
self.idx = self.idx0.copy()
if thresh and thresh > 0:
self.idx &= (np.abs((self.zero - self.zero_model)/self.zeroerr) < thresh)
self.std = np.std((self.zero - self.zero_model)[self.idx])
self.ngoodstars = np.sum(self.idx)
self.success = True
if verbose:
print('Fit finished:', self.ngoodstars, 'stars, rms', self.std)
if predict:
self.predict(obj=obj, x=x, y=y, mag=mag, magerr=magerr, mag_id=mag_id, verbose=verbose)
return True
def clustmember(data,cluster,logg=[-1,3.8],te=[3800,5500],rv=True,pm=True,dist=True,raw=False,firstgen=False,firstpos=True,
ratag='RA',dectag='DEC',rvtag='VHELIO',idtag='APOGEE_ID',btag='J',rtag='K',
plot=False,hard=None) :
clust=clustdata()
ic = np.where( np.core.defchararray.strip(clust.name) == cluster)[0]
if len(ic) == 0 :
print('no cluster found: ',cluster)
return []
print('cluster: ', cluster)
ic=ic[0]
# adjust ra for wraparound if needed
ra=copy.copy(data[ratag])
if clust[ic].ra > 300 :
j=np.where(data[ratag] < 180)[0]
ra[j]+=360
if clust[ic].ra < 60 :
j=np.where(data[ratag] > 180)[0]
ra[j]-=360
# select by location relative to cluster
jc=np.where((np.abs(ra-clust[ic].ra)*np.cos(clust[ic].dec*np.pi/180.) < clust[ic].rad/60.) &
(np.abs(data[dectag]-clust[ic].dec) < clust[ic].rad/60.))[0]
if len(jc) > 0 :
j=np.where( ((ra[jc]-clust[ic].ra)*np.cos(clust[ic].dec*np.pi/180.))**2+
(data[jc][dectag]-clust[ic].dec)**2 < (clust[ic].rad/60.)**2)[0]
jc=jc[j]
else :
jc=[]
print('{:d} stars after location criterion'.format(len(jc)))
if len(jc) == 0 : return jc
if plot :
jf=np.where((np.abs(ra-clust[ic].ra)*np.cos(clust[ic].dec*np.pi/180.) < 1.5) &
(np.abs(data[dectag]-clust[ic].dec) < 1.5))[0]
fig,ax=plots.multi(1,1)
fig.suptitle('{:s} Radius: {:4.2f} arcmin Ngood: {:d}'.format(cluster,clust[ic].rad,len(jc)))
plots.plotp(ax,ra[jf],data[dectag][jf],color='k',size=20,draw=False,xt='RA',yt='DEC')
plots.plotp(ax,ra[jc],data[dectag][jc],color='g',size=20,draw=False)
circle = plt.Circle((clust[ic].ra,clust[ic].dec), clust[ic].rad/60., color='g', fill=False)
ax.add_artist(circle)
if hard is not None :
print(hard+'/'+clust[ic].name+'_pos.png')
fig.savefig(hard+'/'+clust[ic].name+'_pos.png')
plt.close()
else :
pdb.set_trace()
# RV criterion
try :
vhelio = data[rvtag]
except :
vhelio = data['VHELIO_AVG']
j=np.where(np.abs(vhelio[jc]-clust[ic].rv) < clust[ic].drv)[0]
if len(j) > 0 :
if rv: jc=jc[j]
else :
jc=[]
print('{:d} stars after RV criterion'.format(len(jc)))
if plot :
ax.cla()
try :
if len(jf) > 0 : ax.hist(vhelio[jf],color='k',bins=np.arange(clust[ic].rv-100,clust[ic].rv+100,1.),histtype='step')
if len(jc) > 0 : ax.hist(vhelio[jc],color='r',bins=np.arange(clust[ic].rv-100,clust[ic].rv+100,1.),histtype='step')
if len(j) > 0 : ax.hist(vhelio[jc[j]],color='g',bins=np.arange(clust[ic].rv-100,clust[ic].rv+100,1.),histtype='step',linewidth=3)
except : pass
ymax=ax.get_ylim()[1]
ax.plot([clust[ic].rv,clust[ic].rv],[0,ymax],color='g')
ax.plot([clust[ic].rv+clust[ic].drv,clust[ic].rv+clust[ic].drv],[0,ymax],color='r',ls=':')
ax.plot([clust[ic].rv-clust[ic].drv,clust[ic].rv-clust[ic].drv],[0,ymax],color='r',ls=':')
fig.suptitle('{:s} RV: {:4.2f} +/- {:4.2f} Ngood: {:d}'.format(cluster,clust[ic].rv,clust[ic].drv,len(j)))
ax.set_xlabel('RV')
if hard is not None :
fig.savefig(hard+'/'+clust[ic].name+'_rv.png')
plt.close()
else :
plt.draw()
pdb.set_trace()
if len(jc) <= 1 : return jc
# proper motion criterion
if dist or pm : gaia = True
else : gaia = False
if gaia :
job=Gaia.launch_job_async("SELECT xm.original_ext_source_id, gaia.ra, gaia.dec,"+
"gaia.pmra, gaia.pmra_error, gaia.pmdec, gaia.pmdec_error, gaia.parallax, gaia.parallax_error "+
"FROM gaiadr2.gaia_source AS gaia, gaiadr2.tmass_best_neighbour AS xm "+
"WHERE gaia.source_id = xm.source_id AND "+
"CONTAINS(POINT('ICRS',gaia.ra,gaia.dec),CIRCLE('ICRS',{:12.6f},{:12.6f},{:12.6f}))=1;".format(
clust[ic].ra,clust[ic].dec,clust[ic].rad/60.))
# convert to velocities (note mas and kpc cancel out factors of 1000) and get median
gaia=job.get_results()
h=htm.HTM()
maxrad=3/3600.
i1,i2,rad = h.match(data[ratag][jc],data[dectag][jc],gaia['ra'],gaia['dec'],maxrad,maxmatch=1)
#i1, i2 = match.match(np.core.defchararray.replace(data[idtag][jc],'2M',''),gaia['original_ext_source_id'])
vra=4.74*gaia['pmra']*clust[ic].dist
vra_err=4.74*gaia['pmra_error']*clust[ic].dist
vdec=4.74*gaia['pmdec']*clust[ic].dist
vdec_err=4.74*gaia['pmdec_error']*clust[ic].dist
med_vra=np.median(vra[i2])
med_vdec=np.median(vdec[i2])
j=np.where((vra[i2]-med_vra)**2+(vdec[i2]-med_vdec)**2 < 2*clust[ic].drv**2+vra_err[i2]**2+vdec_err[i2]**2)[0]
if len(j) > 0 :
if pm:
#jc=jc[i1[j]]
# allow for the possibility of multiple instances of a given star in input list
jnew=[]
for jjj in j :
iii= np.where(data[idtag][jc] == data[idtag][jc[i1[jjj]]])[0]
jnew.extend(iii)
jc=jc[list(set(jnew))]
else :
jc=[]
print('{:d} stars after PM criterion'.format(len(jc)))
if plot :
ax.cla()
plots.plotp(ax,vra,vdec,color='k',
xr=[med_vra-150,med_vra+150],xt='PMRA (km/sec at cluster dist)',
yr=[med_vdec-150,med_vdec+150],yt='PMDEC (km/sec at cluster dist)')
plots.plotp(ax,vra[i2],vdec[i2],color='r',size=30)
plots.plotp(ax,vra[i2[j]],vdec[i2[j]],color='g',size=30)
fig.suptitle('{:s} PM (km/s): {:4.2f} +/- {:4.2f} {:4.2f} +/ {:4.2f} Ngood: {:d}'.format(
cluster,med_vra,clust[ic].drv, med_vdec,clust[ic].drv,len(jc)))
if hard is not None :
fig.savefig(hard+'/'+clust[ic].name+'_pm.png')
plt.close()
else :
pdb.set_trace()
if len(jc) <= 1 : return jc
# parallaxes
#gaia=job.get_results()
#i1, i2 = match.match(np.core.defchararray.replace(data[idtag][jc],'2M',''),gaia['original_ext_source_id'])
i1,i2,rad = h.match(data[ratag][jc],data[dectag][jc],gaia['ra'],gaia['dec'],maxrad,maxmatch=1)
par=gaia['parallax']
par_error=gaia['parallax_error']
gd=np.where(np.isfinite(par[i2]))[0]
med_par=np.median(par[i2[gd]])
med_par_error=np.median(par_error[i2[gd]])
j=np.where(np.isfinite(par[i2]) & (np.abs(par[i2]-med_par) < 3*med_par_error))[0]
if len(j) > 0 :
if dist:
#jc=jc[i1[j]]
# allow for the possibility of multiple instances of a given star in input list
jnew=[]
for jjj in j :
iii= np.where(data['APOGEE_ID'][jc] == data['APOGEE_ID'][jc[i1[jjj]]])[0]
jnew.extend(iii)
jc=jc[jnew]
else :
jc=[]
if plot :
ax.cla()
ax.hist(par,color='k',bins=np.arange(par.min(),par.max(),0.01),histtype='step',range=(0,2))
ax.hist(par[i2],color='r',bins=np.arange(par.min(),par.max(),0.01),histtype='step',range=(0,2))
ax.hist(par[i2[j]],color='g',bins=np.arange(par.min(),par.max(),0.01),histtype='step',range=(0,2))
ax.set_xlabel('Parallax')
fig.suptitle('{:s} Parallax : {:4.2f} +/- {:4.2f} Ngood: {:d}'.format(cluster,med_par, 3*med_par_error,len(j)))
if hard is not None :
fig.savefig(hard+'/'+clust[ic].name+'_parallax.png')
plt.close()
else :
plt.draw()
pdb.set_trace()
if len(j) > 0 :
if dist:
#jc=jc[i1[j]]
# allow for the possibility of multiple instances of a given star in input list
jnew=[]
for jjj in j :
iii= np.where(data[idtag][jc] == data[idtag][jc[i1[jjj]]])[0]
jnew.extend(iii)
jc=jc[list(set(jnew))]
else :
jc=[]
print('{:d} stars after parallax criterion'.format(len(jc)))
if len(jc) <= 1 : return jc
# parameters criteria
if raw :
param='FPARAM'
else :
param='PARAM'
try :
j = np.where((data[param][jc,1] >= logg[0]) & (data[param][jc,1] <= logg[1]) &
(data[param][jc,0] >= te[0]) & (data[param][jc,0] <= te[1]) )[0]
if len(j) > 0 :
jc=jc[j]
else :
jc=[]
except: pass
print('{:d} stars after parameters criterion'.format(len(jc)))
if len(jc) <= 1 : return jc
# Remove badstars
if plot :
ax.cla()
plots.plotp(ax,data[btag][jf]-data[rtag][jf],data[rtag][jf],color='k',size=20,xr=[-0.5,1.5],yr=[17,6],
facecolors='none',linewidth=1,draw=False,xt=btag+'-'+rtag,yt=rtag)
plots.plotp(ax,data[btag][jc]-data[rtag][jc],data[rtag][jc],color='g',size=30,xr=[-0.5,1.5],draw=False,yr=[17,6])
badstars = open(os.environ['APOGEE_DIR']+'/data/calib/badcal.dat')
bad = []
for line in badstars :
bad.append(line.split()[0])
jc = [x for x in jc if data[x][idtag] not in bad]
print('{:d} stars after badstars rejection'.format(len(jc)))
if len(jc) <= 1 : return jc
# remove non firstgen GC stars if requested
if firstgen :
gcstars = ascii.read(os.environ['APOGEE_DIR']+'/data/calib/gc_szabolcs.dat')
if firstpos :
gd=np.where(gcstars['pop'] == 1)[0]
jc = [x for x in jc if data[x][idtag] in gcstars['id'][gd]]
else :
bd=np.where(gcstars['pop'] != 1)[0]
jc = [x for x in jc if data[x][idtag] not in gcstars['id'][bd]]
print('{:d} stars after firstgen rejection'.format(len(jc)))
if plot :
plots.plotp(ax,data[btag][jc]-data[rtag][jc],data[rtag][jc],color='b',size=30,draw=False)
if hard is not None :
fig.savefig(hard+'/'+clust[ic].name+'_cmd.png')
plt.close()
else :
plt.draw()
pdb.set_trace()
ax.cla()
plots.plotp(ax,data[param][jf,0],data[param][jf,1],size=30,draw=False,xt='Teff',yt='logg',xr=[7000,3000],yr=[6,-1])
plots.plotp(ax,data[param][jc,0],data[param][jc,1],color='b',size=30,draw=False)
if hard is not None :
fig.savefig(hard+'/'+clust[ic].name+'_kiel.png')
plt.close()
else :
plt.draw()
pdb.set_trace()
return jc
def match_objects(obj, cat, sr, fname='V', order=4, thresh=5.0, clim=None, sn=10, mag_idx=0):
x0,y0,width,height = np.mean(obj['x']), np.mean(obj['y']), np.max(obj['x'])-np.min(obj['x']), np.max(obj['y'])-np.min(obj['y'])
# Match stars
h = htm.HTM(10)
m = h.match(obj['ra'],obj['dec'], cat['ra'],cat['dec'], sr, maxmatch=0)
# m = h.match(obj['ra'],obj['dec'], cat['ra'],cat['dec'], 15.0/3600, maxmatch=0)
oidx = m[0]
cidx = m[1]
dist = m[2]
if fname == 'B':
cmag,cmagerr = cat['B'], cat['Berr']
elif fname == 'V':
cmag,cmagerr = cat['V'], cat['Verr']
elif fname == 'R' or fname == 'N':
cmag,cmagerr = cat['R'], cat['Rerr']
elif fname == 'I':
cmag,cmagerr = cat['I'], cat['Ierr']
elif fname == 'z':
cmag,cmagerr = cat['z'], cat['zerr']
if cmag is not None:
cmag = cmag[cidx]
cmagerr = cmagerr[cidx]
else:
print('Unsupported filter:', fname)
return None
if clim is not None:
idx = cmag < clim
oidx,cidx,dist,cmag,cmagerr = [_[idx] for _ in [oidx,cidx,dist,cmag,cmagerr]]
x,y = obj['x'][oidx],obj['y'][oidx]
oflags = obj['flags'][oidx]
omag,omagerr = obj['mag'][oidx],obj['magerr'][oidx]
if len(obj['mag'].shape) > 1:
omag,omagerr = omag[:,mag_idx],omagerr[:,mag_idx]
x = (x - x0)*2/width
y = (y - y0)*2/height
tmagerr = np.hypot(cmagerr, omagerr)
delta_mag = cmag - omag
weights = 1.0/tmagerr**2
X = make_series(1.0, x, y, order=order)
X = np.vstack(X).T
Y = delta_mag
idx0 = (oflags == 0)
if sn and sn>0:
idx0 &= (omagerr < 1/sn)
idx = idx0.copy()
for iter in range(3):
if len(X[idx]) < 3:
print("Fit failed - %d objects" % len(X[idx]))
return None
C = sm.WLS(Y[idx], X[idx], weights=weights[idx]).fit()
YY = np.sum(X*C.params,axis=1)
idx = idx0.copy()
if thresh and thresh > 0:
idx &= (np.abs((Y-YY)/tmagerr) < thresh)
x = (obj['x'] - x0)*2/width
y = (obj['y'] - y0)*2/height
X = make_series(1.0, x, y, order=order)
X = np.vstack(X).T
YY1 = np.sum(X*C.params,axis=1)
if len(obj['mag'].shape) > 1:
mag = obj['mag'][:,mag_idx] + YY1
else:
mag = obj['mag'] + YY1
# Simple analysis of proximity to "good" points
mx,my = obj['x'][oidx][idx], obj['y'][oidx][idx]
kdo = cKDTree(np.array([obj['x'], obj['y']]).T)
kdm = cKDTree(np.array([mx, my]).T)
mr0 = np.sqrt(width*height/np.sum(idx))
m = kdm.query_ball_tree(kdm, 5.0*mr0)
dists = []
for i,ii in enumerate(m):
if len(ii) > 1:
d1 = [np.hypot(mx[i]-mx[_], my[i]-my[_]) for _ in ii]
d1 = np.sort(d1)
dists.append(d1[1])
mr1 = np.median(dists)
m = kdo.query_ball_tree(kdm, 5.0*mr1)
midx = np.array([len(_)>1 for _ in m])
return {
# Matching indices and a distance in degrees
'cidx':cidx, 'oidx':oidx, 'dist':dist,
# Pixel and sky coordinates of matched stars, as well as their flags
'x':obj['x'][oidx], 'y':obj['y'][oidx], 'flags':oflags,
'ra':obj['ra'][oidx], 'dec':obj['dec'][oidx],
# All catalogue magnitudes of matched stars
'cB':cat['B'][cidx], 'cV':cat['V'][cidx], 'cR':cat['R'][cidx], 'cI':cat['I'][cidx],
'cBerr':cat['Berr'][cidx], 'cVerr':cat['Verr'][cidx], 'cRerr':cat['Rerr'][cidx], 'cIerr':cat['Ierr'][cidx],
# Catalogue magnitudes of matched stars in proper filter
'cmag':cmag, 'cmagerr':cmagerr, 'tmagerr':tmagerr,
# Model zero point for all objects, and their corrected magnitudes
'mag0':YY1, 'mag':mag,
'mag_idx': mag_idx,
'Y':Y, 'YY':YY,
# Subset of matched stars used in the fits
'idx':idx,
# Subset of all objects at 'good' distances from matched ones
'midx':midx, 'mr0':mr0, 'mr1':mr1}
