def process_KMT_patch(site, coords=None, quality_max=1.25, mag_err_max=0.3, seeing_max=7.0, sky_max=10000, name_pattern_has_site=True, date_header='MIDHJD', parameters=None, q_sigma_threshold=1.0, locate_date_range=None, locate_half_width=None): params = DIA.DS.Parameters() params.gain = 1.5 params.readnoise = 15.0 params.pixel_max = 57000 params.datekey = date_header params.use_GPU = True params.n_parallel = 1 params.pdeg = 1 params.sdeg = 1 params.bdeg = 1 params.reference_seeing_factor = 1.01 params.reference_min_seeing = 1.0 params.loc_data = 'RAW' params.fwhm_mult = 10 if parameters is not None: for par, value in parameters.iteritems(): try: exec('params.' + par + ' = ' + str(value)) except NameError: exec('params.' + par + ' = \'' + str(value) + '\'') lightcurve_header = ' Date Delta_Flux Err_Delta_Flux Mag Err_Mag Q FWHM Roundness Sky Signal' min_ref_images = {'I': 20, 'V': 3} median_mag = {} for band in ['I', 'V']: prefix = site + band params.loc_output = prefix params.ref_include_file = prefix + '-ref.list' if not (os.path.exists(params.ref_include_file)): params.ref_include_file = False if band == 'V': params.star_file = site + 'I/ref.mags' params.star_reference_image = site + 'I/ref.fits' params.registration_image = site + 'I/ref.fits' if name_pattern_has_site: params.name_pattern = site + '*' + band + '*.fits' else: params.name_pattern = '*' + band + '*.fits' params.min_ref_images = min_ref_images[band] if not (os.path.exists(params.loc_output)): DIA.imsub_all_fits(params) if not (os.path.exists(params.loc_output + '/calibration.png')): cal.calibrate(params.loc_output) if coords is not None: if band == 'I': x0, y0 = coords print 'Starting photometry for', prefix, 'at', (x0, y0) dates, seeing, roundness, bgnd, signal, flux, dflux, quality, x0, y0 = \ CF.photom_variable_star(x0,y0,params,save_stamps=True,patch_half_width=20,locate_date_range=locate_date_range, locate_half_width=locate_half_width) print 'Converged to', (x0, y0) else: Imags = np.loadtxt(site + 'I/ref.mags') Vmags = np.loadtxt(site + 'V/ref.mags') x0 += np.median(Vmags[:, 1] - Imags[:, 1]) y0 += np.median(Vmags[:, 2] - Imags[:, 2]) print 'Starting photometry for', prefix, 'at', (x0, y0) dates, seeing, roundness, bgnd, signal, flux, dflux, quality, x0, y0 = \ CF.photom_variable_star(x0,y0,params,save_stamps=True,patch_half_width=20,converge=False) print 'Converged to', (x0, y0) print 'Photometry for', site, band, 'at', x0, y0 refmags = np.loadtxt(params.loc_output + '/ref.mags.calibrated') refflux = np.loadtxt(params.loc_output + '/ref.flux.calibrated') star_dist2 = (refmags[:, 1] - x0)**2 + (refmags[:, 2] - y0)**2 star_num = np.argmin(star_dist2) print 'x0 y0:', x0, y0 print 'Nearest star', star_num, 'located at', refmags[ star_num, 1], refmags[star_num, 2] print 'Reference flux', refflux[star_num, :] print 'Reference mag', refmags[star_num, :] mag = 25 - 2.5 * np.log10(refflux[star_num, 0] + flux) mag_err = 25 - 2.5 * np.log10(refflux[star_num, 0] + flux - dflux) - mag np.savetxt(prefix+'-lightcurve.dat',np.vstack((dates,flux,dflux,mag,mag_err,quality,seeing,roundness,bgnd,signal)).T, \ fmt='%12.5f %12.4f %12.4f %7.4f % 7.4f %6.2f %6.2f %5.2f %10.2f %8.2f', \ header=lightcurve_header) q = np.where((quality < quality_max) & (mag_err < mag_err_max) & (seeing < seeing_max) & (bgnd < sky_max))[0] median_mag[band] = np.nanmedian(mag[q]) np.savetxt(prefix+'-lightcurve-filtered.dat',np.vstack((dates[q],flux[q],dflux[q], \ mag[q],mag_err[q],quality[q],seeing[q],roundness[q],bgnd[q],signal[q])).T, \ fmt='%12.5f %12.4f %12.4f %7.4f % 7.4f %6.2f %6.2f %5.2f %10.2f %8.2f', \ header=lightcurve_header) cal.plot_lightcurve(prefix + '-lightcurve.dat', plotfile=prefix + '-lightcurve.png') cal.plot_lightcurve(prefix + '-lightcurve-filtered.dat', plotfile=prefix + '-lightcurve-filtered.png') if site in ['A', 'SSOre']: return RC = cal.makeCMD(site + 'I', site + 'V') VI, VI_err = cal.source_colour(site + 'I-lightcurve-filtered.dat', site + 'V-lightcurve-filtered.dat', plotfile=site + '-source-colour.png') cal.makeCMD(site + 'I', site + 'V', plot_density=False, IV=(median_mag['I'], median_mag['V']), RC=RC, source_colour=(VI, VI_err))
def process_KMT_patch(site,coords=None,quality_max=1.25,mag_err_max=0.3,seeing_max=7.0,sky_max=10000,signal_min=125,name_pattern_has_site=True,date_header='MIDHJD', parameters=None,q_sigma_threshold=1.0,locate_date_range=None,locate_half_width=None,loc_data='RAW',RC_limit=16.0): params = DIA.DS.Parameters() params.gain = 1.5 params.readnoise = 15.0 params.pixel_max = 57000 params.datekey = date_header params.use_GPU = True params.n_parallel = 1 params.pdeg = 1 params.sdeg = 1 params.bdeg = 1 params.reference_seeing_factor = 1.01 params.reference_min_seeing = 1.0 params.loc_data = loc_data params.fwhm_mult = 10 params.iterations = 5 params.bleed_mask_multiplier_above = 0 params.bleed_mask_multiplier_below = 2.0 if parameters is not None: for par, value in parameters.iteritems(): try: exec('params.'+par+' = '+str(value)) except NameError: exec('params.'+par+' = \''+str(value)+'\'') lightcurve_header=' Date Delta_Flux Err_Delta_Flux Mag Err_Mag Q FWHM Roundness Sky Signal' min_ref_images = {'I':20,'V':3} median_mag = {} median_mag_err = {} if os.path.exists(site+'-reg.fits'): params.registration_image = site+'-reg.fits' for band in ['I','V']: prefix = site+band params.loc_output = prefix params.ref_include_file = prefix+'-ref.list' if not(os.path.exists(params.ref_include_file)): params.ref_include_file = False photometric_zeropoint = 28.0 if band == 'V': params.star_file = site+'I/ref.mags' params.star_reference_image = site+'I/ref.fits' params.registration_image = site+'I/ref.fits' photometric_zeropoint = 28.65 if name_pattern_has_site: params.name_pattern = site+'*'+band+'*.fits' else: params.name_pattern = '*'+band+'*.fits' params.min_ref_images = min_ref_images[band] print params.loc_output print params.loc_data print params.name_pattern if not(os.path.exists(params.loc_output)): # Read the header of the first image to determine which direction the # detector has been read out. all_files = os.listdir(params.loc_data) hdr = fits.getheader(params.loc_data+'/'+all_files[0]) try: if hdr['ampname'][0] in ['K','N']: params.bleed_mask_multiplier_above = 0 params.bleed_mask_multiplier_below = 2.0 else: params.bleed_mask_multiplier_above = 2.0 params.bleed_mask_multiplier_below = 0 except: pass print 'starting imsub' DIA.imsub_all_fits(params) if not(os.path.exists(params.loc_output+'/calibration.png')): cal.calibrate(params.loc_output,ZP=photometric_zeropoint) if coords is not None: if band == 'I': x0, y0 = coords print 'Starting photometry for',prefix,'at', (x0,y0) dates, seeing, roundness, bgnd, signal, flux, dflux, quality, x0, y0 = \ CF.photom_variable_star(x0,y0,params,save_stamps=True,patch_half_width=20,locate_date_range=locate_date_range, locate_half_width=locate_half_width) print 'Converged to', (x0,y0) else: Imags = np.loadtxt(site+'I/ref.mags') Vmags = np.loadtxt(site+'V/ref.mags') x0 += np.median(Vmags[:,1]-Imags[:,1]) y0 += np.median(Vmags[:,2]-Imags[:,2]) print 'Starting photometry for',prefix,'at', (x0,y0) dates, seeing, roundness, bgnd, signal, flux, dflux, quality, x0, y0 = \ CF.photom_variable_star(x0,y0,params,save_stamps=True,patch_half_width=20,converge=False) print 'Converged to', (x0,y0) print 'Photometry for', site, band, 'at', x0, y0 refmags = np.loadtxt(params.loc_output+'/ref.mags.calibrated') refflux = np.loadtxt(params.loc_output+'/ref.flux.calibrated') star_dist2 = (refmags[:,1]-x0)**2 + (refmags[:,2]-y0)**2 star_num = np.argmin(star_dist2) print 'x0 y0:', x0, y0 print 'Nearest star', star_num, 'located at', refmags[star_num,1], refmags[star_num,2] print 'Reference flux', refflux[star_num,:] print 'Reference mag', refmags[star_num,:] mag = photometric_zeropoint - 2.5*np.log10(refflux[star_num,0] + flux) mag_err = photometric_zeropoint - 2.5*np.log10(refflux[star_num,0] + flux - dflux) - mag np.savetxt(prefix+'-lightcurve.dat',np.vstack((dates,flux,dflux,mag,mag_err,quality,seeing,roundness,bgnd,signal)).T, \ fmt='%12.5f %12.4f %12.4f %7.4f % 7.4f %6.2f %6.2f %5.2f %10.2f %8.2f', \ header=lightcurve_header) if band == 'V': signal_min = 0.0 q = np.where( (quality < quality_max) & (mag_err < mag_err_max) & (seeing < seeing_max) & (bgnd < sky_max) & (signal > signal_min) & \ (np.abs(flux) > 1.0e-6) )[0] for i in range(3): ldf = np.log10(dflux[q]) q = q[ldf > np.mean(ldf) - 4*np.std(ldf)] median_mag[band] = np.nanmedian(mag[q]) np.savetxt(prefix+'-lightcurve-filtered.dat',np.vstack((dates[q],flux[q],dflux[q], \ mag[q],mag_err[q],quality[q],seeing[q],roundness[q],bgnd[q],signal[q])).T, \ fmt='%12.5f %12.4f %12.4f %7.4f % 7.4f %6.2f %6.2f %5.2f %10.2f %8.2f', \ header=lightcurve_header) cal.plot_lightcurve(prefix+'-lightcurve.dat',plotfile=prefix+'-lightcurve.png') cal.plot_lightcurve(prefix+'-lightcurve-filtered.dat',plotfile=prefix+'-lightcurve-filtered.png') if site in ['A','SSOre']: return #RC = cal.makeCMD(site+'I',site+'V',RC_limit=16.0) VI, VI_err = cal.source_colour(site+'I-lightcurve-filtered.dat',site+'V-lightcurve-filtered.dat',plotfile=site+'-source-colour.png',VIoffset=0.65) cal.makeCMD(site+'I',site+'V',plot_density=True,IV=(median_mag['I'],median_mag['V']),source_colour=(VI[0],VI_err[0]),RC_limit=RC_limit)
def do_photometry_variables(files, params, position, extname='vflux', psf_file=None, reference_image='ref.fits', iterations=10): # # Get image size # dtarget = params.loc_output+os.path.sep+'d_'+ \ os.path.basename(files[0]) d, h = read_fits_file(dtarget) imsize = d.shape # # Check that the PSF exists # if not (psf_file): psf_file = params.loc_output + os.path.sep + 'psf.fits' if not (os.path.exists(psf_file)): print 'Error: PSF file', psf_file, 'not found' sys.exit(1) #position[0] += 2.0 # # Read and store difference images # nfiles = len(files) xpos = int(np.floor(position[0] + 0.5) - 16) ypos = int(np.floor(position[1] + 0.5) - 16) dd = np.zeros([nfiles, 32, 32]) vv = np.zeros([nfiles, 32, 32]) nn = np.zeros([nfiles, 32, 32]) good = np.zeros(nfiles, dtype=bool) slice = (xpos, xpos + 32, ypos, ypos + 32) print 'slice', slice for j, f in enumerate(files): print 'Reading', f target = f dtarget = params.loc_output+os.path.sep+'d_'+ \ os.path.basename(target) ntarget = params.loc_output+os.path.sep+'n_'+ \ os.path.basename(target) ztarget = params.loc_output+os.path.sep+'z_'+ \ os.path.basename(target) ktable = params.loc_output+os.path.sep+'k_'+ \ os.path.basename(target) if os.path.exists(dtarget) and os.path.exists(ntarget) and \ os.path.exists(ktable): norm, h = read_fits_file(ntarget, slice=slice) diff, h = read_fits_file(dtarget, slice=slice) mask, h = read_fits_file(ztarget, slice=slice) kernelIndex, extendedBasis, c, params = \ read_kernel_table(ktable,params) diff[abs(diff) > 1.2 * params.pixel_max] = np.nan inv_var = (norm / diff)**2 + (1 - mask) diff[np.isnan(diff)] = 0.0 diff = undo_photometric_scale(diff, c, params.pdeg, size=imsize, position=(xpos, ypos)) dd[j, :, :] = diff nn[j, :, :] = norm vv[j, :, :] = inv_var ngood = 0 good_threshold = 6.0 while (ngood < 12) and (good_threshold > 3.0): ngood = 0 print 'good_threshold =', good_threshold for j, f in enumerate(files): snorm = np.std(nn[j, :, :]) print 'j,snorm, max = ', j, snorm, np.max( np.abs(nn[j, 14:17, 14:17])) if not (np.isinf(vv[j, :, :]).any() or np.isnan(vv[j, :, :]).any() or np.isnan(dd[j, :, :]).any()) and (np.nanstd( nn[j, :, :]) < params.diff_std_threshold) and (np.max( np.abs(nn[j, 14:17, 14:17])) > good_threshold * snorm): good[j] = True ngood += 1 good_threshold *= 0.9 if ngood < 6: print 'Convergence failed' return position, np.zeros(nfiles), np.zeros(nfiles) # # Iterate # position0 = position.copy() position1 = position.copy() position = np.array([ position[0] - int(np.floor(position[0] + 0.5)) + 16, position[1] - int(np.floor(position[1] + 0.5)) + 16 ]) for i in range(iterations): x1 = 0.0 x2 = 0.0 y1 = 0.0 y2 = 0.0 flux = np.zeros(nfiles) dflux = np.zeros(nfiles) # # Process difference images # for j, f in enumerate(files): print i, j, f ktable = params.loc_output+os.path.sep+'k_'+ \ os.path.basename(f) if os.path.exists(ktable) and good[j]: kernelIndex, extendedBasis, c, params = \ read_kernel_table(ktable,params) kernelRadius = np.max(kernelIndex[:, 0]) + 1 if np.sum(extendedBasis) > 0: kernelRadius += 1 flux[j], dflux[j], sjx1, sjx2, sjy1, sjy2 = \ ci.photom_variable_star(dd[j,:,:],vv[j,:,:],position, position0,psf_file,c,kernelIndex, extendedBasis,kernelRadius, params) #print sjx1/sjx2, sjy1/sjy2, sjx1,sjx2,sjy1,sjy2 #if good[j] & (abs(sjx2)>10) & (abs(sjy2)>10) & (sjx1/sjx2 < 4) & (sjy1/sjy2 < 4): if (sjx1 / sjx2 < 2) & (sjy1 / sjy2 < 2): x1 += sjx1 x2 += sjx2 y1 += sjy1 y2 += sjy2 if (abs(x2) > 1.e-6) and (abs(y2) > 1.e-6): dx = -x1 / x2 dy = -y1 / y2 else: dx = 0.0 dy = 0.0 print x1, x2, dx print y1, y2, dy if (abs(dx) > 5.0) | (abs(dy) > 5.0) | np.isnan(dx) | np.isnan(dy): break x0 = position0[0] + dx y0 = position0[1] + dy x = position[0] + dx y = position[1] + dy print position, ' + ', dx, ',', dy, ' = ', x, y print position0, ' + ', dx, ',', dy, ' = ', x0, y0 position0[0] = x0 position0[1] = y0 position[0] = x position[1] = y if (position1[0] - position0[0])**2 + (position1[1] - position0[1])**2 > 25.0: position0 = position1 print 'Convergence failed' return position0, flux, dflux
params.reference_min_seeing = 0.5 params.reference_seeing_factor = 10.0 params.use_stamps = False #params.nstamps = 100 params.name_pattern = '*.fits' params.datekey = 'MJD' params.loc_data = os.path.join('/root/input_images', input_folder) params.loc_output = os.path.join('/root/output_dir', output_folder) # WD physical position x0, y0 = 268.349, 265.786 prefix = input_folder locate_date_range = (-1e10, 1e10) dates, seeing, roundness, bgnd, signal, flux, dflux, quality, x0, y0 = \ CF.photom_variable_star(x0,y0,params,save_stamps=True,patch_half_width=20,locate_half_width=3,locate_date_range=locate_date_range) refmags = np.loadtxt(params.loc_output + '/ref.mags.calibrated') refflux = np.loadtxt(params.loc_output + '/ref.flux.calibrated') star_dist2 = (refmags[:, 1] - x0)**2 + (refmags[:, 2] - y0)**2 star_num = np.argmin(star_dist2) print 'x0 y0:', x0, y0 print 'Nearest star', star_num, 'located at', refmags[star_num, 1], refmags[star_num, 2] print 'Reference flux', refflux[star_num, :] print 'Reference mag', refmags[star_num, :] mag = 25 - 2.5 * np.log10(refflux[star_num, 0] + flux) mag_err = 25 - 2.5 * np.log10(refflux[star_num, 0] + flux - dflux) - mag