def process_KMT_patch(site,
coords=None,
quality_max=1.25,
mag_err_max=0.3,
seeing_max=7.0,
sky_max=10000,
name_pattern_has_site=True,
date_header='MIDHJD',
parameters=None,
q_sigma_threshold=1.0,
locate_date_range=None,
locate_half_width=None):
params = DIA.DS.Parameters()
params.gain = 1.5
params.readnoise = 15.0
params.pixel_max = 57000
params.datekey = date_header
params.use_GPU = True
params.n_parallel = 1
params.pdeg = 1
params.sdeg = 1
params.bdeg = 1
params.reference_seeing_factor = 1.01
params.reference_min_seeing = 1.0
params.loc_data = 'RAW'
params.fwhm_mult = 10
if parameters is not None:
for par, value in parameters.iteritems():
try:
exec('params.' + par + ' = ' + str(value))
except NameError:
exec('params.' + par + ' = \'' + str(value) + '\'')
lightcurve_header = ' Date Delta_Flux Err_Delta_Flux Mag Err_Mag Q FWHM Roundness Sky Signal'
min_ref_images = {'I': 20, 'V': 3}
median_mag = {}
for band in ['I', 'V']:
prefix = site + band
params.loc_output = prefix
params.ref_include_file = prefix + '-ref.list'
if not (os.path.exists(params.ref_include_file)):
params.ref_include_file = False
if band == 'V':
params.star_file = site + 'I/ref.mags'
params.star_reference_image = site + 'I/ref.fits'
params.registration_image = site + 'I/ref.fits'
if name_pattern_has_site:
params.name_pattern = site + '*' + band + '*.fits'
else:
params.name_pattern = '*' + band + '*.fits'
params.min_ref_images = min_ref_images[band]
if not (os.path.exists(params.loc_output)):
DIA.imsub_all_fits(params)
if not (os.path.exists(params.loc_output + '/calibration.png')):
cal.calibrate(params.loc_output)
if coords is not None:
if band == 'I':
x0, y0 = coords
print 'Starting photometry for', prefix, 'at', (x0, y0)
dates, seeing, roundness, bgnd, signal, flux, dflux, quality, x0, y0 = \
CF.photom_variable_star(x0,y0,params,save_stamps=True,patch_half_width=20,locate_date_range=locate_date_range,
locate_half_width=locate_half_width)
print 'Converged to', (x0, y0)
else:
Imags = np.loadtxt(site + 'I/ref.mags')
Vmags = np.loadtxt(site + 'V/ref.mags')
x0 += np.median(Vmags[:, 1] - Imags[:, 1])
y0 += np.median(Vmags[:, 2] - Imags[:, 2])
print 'Starting photometry for', prefix, 'at', (x0, y0)
dates, seeing, roundness, bgnd, signal, flux, dflux, quality, x0, y0 = \
CF.photom_variable_star(x0,y0,params,save_stamps=True,patch_half_width=20,converge=False)
print 'Converged to', (x0, y0)
print 'Photometry for', site, band, 'at', x0, y0
refmags = np.loadtxt(params.loc_output + '/ref.mags.calibrated')
refflux = np.loadtxt(params.loc_output + '/ref.flux.calibrated')
star_dist2 = (refmags[:, 1] - x0)**2 + (refmags[:, 2] - y0)**2
star_num = np.argmin(star_dist2)
print 'x0 y0:', x0, y0
print 'Nearest star', star_num, 'located at', refmags[
star_num, 1], refmags[star_num, 2]
print 'Reference flux', refflux[star_num, :]
print 'Reference mag', refmags[star_num, :]
mag = 25 - 2.5 * np.log10(refflux[star_num, 0] + flux)
mag_err = 25 - 2.5 * np.log10(refflux[star_num, 0] + flux -
dflux) - mag
np.savetxt(prefix+'-lightcurve.dat',np.vstack((dates,flux,dflux,mag,mag_err,quality,seeing,roundness,bgnd,signal)).T, \
fmt='%12.5f %12.4f %12.4f %7.4f % 7.4f %6.2f %6.2f %5.2f %10.2f %8.2f', \
header=lightcurve_header)
q = np.where((quality < quality_max) & (mag_err < mag_err_max)
& (seeing < seeing_max) & (bgnd < sky_max))[0]
median_mag[band] = np.nanmedian(mag[q])
np.savetxt(prefix+'-lightcurve-filtered.dat',np.vstack((dates[q],flux[q],dflux[q], \
mag[q],mag_err[q],quality[q],seeing[q],roundness[q],bgnd[q],signal[q])).T, \
fmt='%12.5f %12.4f %12.4f %7.4f % 7.4f %6.2f %6.2f %5.2f %10.2f %8.2f', \
header=lightcurve_header)
cal.plot_lightcurve(prefix + '-lightcurve.dat',
plotfile=prefix + '-lightcurve.png')
cal.plot_lightcurve(prefix + '-lightcurve-filtered.dat',
plotfile=prefix + '-lightcurve-filtered.png')
if site in ['A', 'SSOre']:
return
RC = cal.makeCMD(site + 'I', site + 'V')
VI, VI_err = cal.source_colour(site + 'I-lightcurve-filtered.dat',
site + 'V-lightcurve-filtered.dat',
plotfile=site + '-source-colour.png')
cal.makeCMD(site + 'I',
site + 'V',
plot_density=False,
IV=(median_mag['I'], median_mag['V']),
RC=RC,
source_colour=(VI, VI_err))
def process_KMT_patch(site,coords=None,quality_max=1.25,mag_err_max=0.3,seeing_max=7.0,sky_max=10000,signal_min=125,name_pattern_has_site=True,date_header='MIDHJD',
parameters=None,q_sigma_threshold=1.0,locate_date_range=None,locate_half_width=None,loc_data='RAW',RC_limit=16.0):
params = DIA.DS.Parameters()
params.gain = 1.5
params.readnoise = 15.0
params.pixel_max = 57000
params.datekey = date_header
params.use_GPU = True
params.n_parallel = 1
params.pdeg = 1
params.sdeg = 1
params.bdeg = 1
params.reference_seeing_factor = 1.01
params.reference_min_seeing = 1.0
params.loc_data = loc_data
params.fwhm_mult = 10
params.iterations = 5
params.bleed_mask_multiplier_above = 0
params.bleed_mask_multiplier_below = 2.0
if parameters is not None:
for par, value in parameters.iteritems():
try:
exec('params.'+par+' = '+str(value))
except NameError:
exec('params.'+par+' = \''+str(value)+'\'')
lightcurve_header=' Date Delta_Flux Err_Delta_Flux Mag Err_Mag Q FWHM Roundness Sky Signal'
min_ref_images = {'I':20,'V':3}
median_mag = {}
median_mag_err = {}
if os.path.exists(site+'-reg.fits'):
params.registration_image = site+'-reg.fits'
for band in ['I','V']:
prefix = site+band
params.loc_output = prefix
params.ref_include_file = prefix+'-ref.list'
if not(os.path.exists(params.ref_include_file)):
params.ref_include_file = False
photometric_zeropoint = 28.0
if band == 'V':
params.star_file = site+'I/ref.mags'
params.star_reference_image = site+'I/ref.fits'
params.registration_image = site+'I/ref.fits'
photometric_zeropoint = 28.65
if name_pattern_has_site:
params.name_pattern = site+'*'+band+'*.fits'
else:
params.name_pattern = '*'+band+'*.fits'
params.min_ref_images = min_ref_images[band]
print params.loc_output
print params.loc_data
print params.name_pattern
if not(os.path.exists(params.loc_output)):
# Read the header of the first image to determine which direction the
# detector has been read out.
all_files = os.listdir(params.loc_data)
hdr = fits.getheader(params.loc_data+'/'+all_files[0])
try:
if hdr['ampname'][0] in ['K','N']:
params.bleed_mask_multiplier_above = 0
params.bleed_mask_multiplier_below = 2.0
else:
params.bleed_mask_multiplier_above = 2.0
params.bleed_mask_multiplier_below = 0
except:
pass
print 'starting imsub'
DIA.imsub_all_fits(params)
if not(os.path.exists(params.loc_output+'/calibration.png')):
cal.calibrate(params.loc_output,ZP=photometric_zeropoint)
if coords is not None:
if band == 'I':
x0, y0 = coords
print 'Starting photometry for',prefix,'at', (x0,y0)
dates, seeing, roundness, bgnd, signal, flux, dflux, quality, x0, y0 = \
CF.photom_variable_star(x0,y0,params,save_stamps=True,patch_half_width=20,locate_date_range=locate_date_range,
locate_half_width=locate_half_width)
print 'Converged to', (x0,y0)
else:
Imags = np.loadtxt(site+'I/ref.mags')
Vmags = np.loadtxt(site+'V/ref.mags')
x0 += np.median(Vmags[:,1]-Imags[:,1])
y0 += np.median(Vmags[:,2]-Imags[:,2])
print 'Starting photometry for',prefix,'at', (x0,y0)
dates, seeing, roundness, bgnd, signal, flux, dflux, quality, x0, y0 = \
CF.photom_variable_star(x0,y0,params,save_stamps=True,patch_half_width=20,converge=False)
print 'Converged to', (x0,y0)
print 'Photometry for', site, band, 'at', x0, y0
refmags = np.loadtxt(params.loc_output+'/ref.mags.calibrated')
refflux = np.loadtxt(params.loc_output+'/ref.flux.calibrated')
star_dist2 = (refmags[:,1]-x0)**2 + (refmags[:,2]-y0)**2
star_num = np.argmin(star_dist2)
print 'x0 y0:', x0, y0
print 'Nearest star', star_num, 'located at', refmags[star_num,1], refmags[star_num,2]
print 'Reference flux', refflux[star_num,:]
print 'Reference mag', refmags[star_num,:]
mag = photometric_zeropoint - 2.5*np.log10(refflux[star_num,0] + flux)
mag_err = photometric_zeropoint - 2.5*np.log10(refflux[star_num,0] + flux - dflux) - mag
np.savetxt(prefix+'-lightcurve.dat',np.vstack((dates,flux,dflux,mag,mag_err,quality,seeing,roundness,bgnd,signal)).T, \
fmt='%12.5f %12.4f %12.4f %7.4f % 7.4f %6.2f %6.2f %5.2f %10.2f %8.2f', \
header=lightcurve_header)
if band == 'V':
signal_min = 0.0
q = np.where( (quality < quality_max) & (mag_err < mag_err_max) & (seeing < seeing_max) & (bgnd < sky_max) & (signal > signal_min) & \
(np.abs(flux) > 1.0e-6) )[0]
for i in range(3):
ldf = np.log10(dflux[q])
q = q[ldf > np.mean(ldf) - 4*np.std(ldf)]
median_mag[band] = np.nanmedian(mag[q])
np.savetxt(prefix+'-lightcurve-filtered.dat',np.vstack((dates[q],flux[q],dflux[q], \
mag[q],mag_err[q],quality[q],seeing[q],roundness[q],bgnd[q],signal[q])).T, \
fmt='%12.5f %12.4f %12.4f %7.4f % 7.4f %6.2f %6.2f %5.2f %10.2f %8.2f', \
header=lightcurve_header)
cal.plot_lightcurve(prefix+'-lightcurve.dat',plotfile=prefix+'-lightcurve.png')
cal.plot_lightcurve(prefix+'-lightcurve-filtered.dat',plotfile=prefix+'-lightcurve-filtered.png')
if site in ['A','SSOre']:
return
#RC = cal.makeCMD(site+'I',site+'V',RC_limit=16.0)
VI, VI_err = cal.source_colour(site+'I-lightcurve-filtered.dat',site+'V-lightcurve-filtered.dat',plotfile=site+'-source-colour.png',VIoffset=0.65)
cal.makeCMD(site+'I',site+'V',plot_density=True,IV=(median_mag['I'],median_mag['V']),source_colour=(VI[0],VI_err[0]),RC_limit=RC_limit)
def do_photometry_variables(files,
params,
position,
extname='vflux',
psf_file=None,
reference_image='ref.fits',
iterations=10):
#
# Get image size
#
dtarget = params.loc_output+os.path.sep+'d_'+ \
os.path.basename(files[0])
d, h = read_fits_file(dtarget)
imsize = d.shape
#
# Check that the PSF exists
#
if not (psf_file):
psf_file = params.loc_output + os.path.sep + 'psf.fits'
if not (os.path.exists(psf_file)):
print 'Error: PSF file', psf_file, 'not found'
sys.exit(1)
#position[0] += 2.0
#
# Read and store difference images
#
nfiles = len(files)
xpos = int(np.floor(position[0] + 0.5) - 16)
ypos = int(np.floor(position[1] + 0.5) - 16)
dd = np.zeros([nfiles, 32, 32])
vv = np.zeros([nfiles, 32, 32])
nn = np.zeros([nfiles, 32, 32])
good = np.zeros(nfiles, dtype=bool)
slice = (xpos, xpos + 32, ypos, ypos + 32)
print 'slice', slice
for j, f in enumerate(files):
print 'Reading', f
target = f
dtarget = params.loc_output+os.path.sep+'d_'+ \
os.path.basename(target)
ntarget = params.loc_output+os.path.sep+'n_'+ \
os.path.basename(target)
ztarget = params.loc_output+os.path.sep+'z_'+ \
os.path.basename(target)
ktable = params.loc_output+os.path.sep+'k_'+ \
os.path.basename(target)
if os.path.exists(dtarget) and os.path.exists(ntarget) and \
os.path.exists(ktable):
norm, h = read_fits_file(ntarget, slice=slice)
diff, h = read_fits_file(dtarget, slice=slice)
mask, h = read_fits_file(ztarget, slice=slice)
kernelIndex, extendedBasis, c, params = \
read_kernel_table(ktable,params)
diff[abs(diff) > 1.2 * params.pixel_max] = np.nan
inv_var = (norm / diff)**2 + (1 - mask)
diff[np.isnan(diff)] = 0.0
diff = undo_photometric_scale(diff,
c,
params.pdeg,
size=imsize,
position=(xpos, ypos))
dd[j, :, :] = diff
nn[j, :, :] = norm
vv[j, :, :] = inv_var
ngood = 0
good_threshold = 6.0
while (ngood < 12) and (good_threshold > 3.0):
ngood = 0
print 'good_threshold =', good_threshold
for j, f in enumerate(files):
snorm = np.std(nn[j, :, :])
print 'j,snorm, max = ', j, snorm, np.max(
np.abs(nn[j, 14:17, 14:17]))
if not (np.isinf(vv[j, :, :]).any() or np.isnan(vv[j, :, :]).any()
or np.isnan(dd[j, :, :]).any()) and (np.nanstd(
nn[j, :, :]) < params.diff_std_threshold) and (np.max(
np.abs(nn[j, 14:17,
14:17])) > good_threshold * snorm):
good[j] = True
ngood += 1
good_threshold *= 0.9
if ngood < 6:
print 'Convergence failed'
return position, np.zeros(nfiles), np.zeros(nfiles)
#
# Iterate
#
position0 = position.copy()
position1 = position.copy()
position = np.array([
position[0] - int(np.floor(position[0] + 0.5)) + 16,
position[1] - int(np.floor(position[1] + 0.5)) + 16
])
for i in range(iterations):
x1 = 0.0
x2 = 0.0
y1 = 0.0
y2 = 0.0
flux = np.zeros(nfiles)
dflux = np.zeros(nfiles)
#
# Process difference images
#
for j, f in enumerate(files):
print i, j, f
ktable = params.loc_output+os.path.sep+'k_'+ \
os.path.basename(f)
if os.path.exists(ktable) and good[j]:
kernelIndex, extendedBasis, c, params = \
read_kernel_table(ktable,params)
kernelRadius = np.max(kernelIndex[:, 0]) + 1
if np.sum(extendedBasis) > 0:
kernelRadius += 1
flux[j], dflux[j], sjx1, sjx2, sjy1, sjy2 = \
ci.photom_variable_star(dd[j,:,:],vv[j,:,:],position,
position0,psf_file,c,kernelIndex,
extendedBasis,kernelRadius,
params)
#print sjx1/sjx2, sjy1/sjy2, sjx1,sjx2,sjy1,sjy2
#if good[j] & (abs(sjx2)>10) & (abs(sjy2)>10) & (sjx1/sjx2 < 4) & (sjy1/sjy2 < 4):
if (sjx1 / sjx2 < 2) & (sjy1 / sjy2 < 2):
x1 += sjx1
x2 += sjx2
y1 += sjy1
y2 += sjy2
if (abs(x2) > 1.e-6) and (abs(y2) > 1.e-6):
dx = -x1 / x2
dy = -y1 / y2
else:
dx = 0.0
dy = 0.0
print x1, x2, dx
print y1, y2, dy
if (abs(dx) > 5.0) | (abs(dy) > 5.0) | np.isnan(dx) | np.isnan(dy):
break
x0 = position0[0] + dx
y0 = position0[1] + dy
x = position[0] + dx
y = position[1] + dy
print position, ' + ', dx, ',', dy, ' = ', x, y
print position0, ' + ', dx, ',', dy, ' = ', x0, y0
position0[0] = x0
position0[1] = y0
position[0] = x
position[1] = y
if (position1[0] - position0[0])**2 + (position1[1] -
position0[1])**2 > 25.0:
position0 = position1
print 'Convergence failed'
return position0, flux, dflux
params.reference_min_seeing = 0.5
params.reference_seeing_factor = 10.0
params.use_stamps = False
#params.nstamps = 100
params.name_pattern = '*.fits'
params.datekey = 'MJD'
params.loc_data = os.path.join('/root/input_images', input_folder)
params.loc_output = os.path.join('/root/output_dir', output_folder)
# WD physical position
x0, y0 = 268.349, 265.786
prefix = input_folder
locate_date_range = (-1e10, 1e10)
dates, seeing, roundness, bgnd, signal, flux, dflux, quality, x0, y0 = \
CF.photom_variable_star(x0,y0,params,save_stamps=True,patch_half_width=20,locate_half_width=3,locate_date_range=locate_date_range)
refmags = np.loadtxt(params.loc_output + '/ref.mags.calibrated')
refflux = np.loadtxt(params.loc_output + '/ref.flux.calibrated')
star_dist2 = (refmags[:, 1] - x0)**2 + (refmags[:, 2] - y0)**2
star_num = np.argmin(star_dist2)
print 'x0 y0:', x0, y0
print 'Nearest star', star_num, 'located at', refmags[star_num,
1], refmags[star_num, 2]
print 'Reference flux', refflux[star_num, :]
print 'Reference mag', refmags[star_num, :]
mag = 25 - 2.5 * np.log10(refflux[star_num, 0] + flux)
mag_err = 25 - 2.5 * np.log10(refflux[star_num, 0] + flux - dflux) - mag