def make_meas_missing(exposure): """ Make meas_missing.fits for exposures.""" t00 = time.time() hostname = socket.gethostname() host = hostname.split('.')[0] version = 'v3' # Load the exposures table print('Loading exposure table') expstr = fits.getdata( '/net/dl2/dnidever/nsc/instcal/' + version + '/lists/nsc_v3_exposure_table.fits.gz', 1) # Make sure it's a list if type(exposure) is str: exposure = [exposure] # Match exposures to exposure catalog ind1, ind2 = dln.match(expstr['EXPOSURE'], exposure) nmatch = len(ind1) print(str(nmatch) + ' exposures found in exposure table') # Loop over files for i in range(nmatch): exp1 = expstr['EXPOSURE'][ind1[i]] instcode = expstr['INSTRUMENT'][ind1[i]] dateobs = expstr['DATEOBS'][ind1[i]] night = dateobs[0:4] + dateobs[5:7] + dateobs[8:10] measfile = '/net/dl2/dnidever/nsc/instcal/v3/' + instcode + '/' + night + '/' + exp1 + '/' + exp1 + '_meas.fits.gz' if os.path.exists(measfile): meas = fits.getdata(measfile, 1) objectid = np.char.array(meas['OBJECTID']).strip() miss, = np.where(objectid == '') if len(miss) > 0: print( str(i + 1) + ' ' + exp1 + ' ' + str(len(miss)) + ' missing') missfile = '/net/dl2/dnidever/nsc/instcal/v3/' + instcode + '/' + night + '/' + exp1 + '/' + exp1 + '_meas_missing.fits' print(' Writing missing catalog to ' + missfile) Table(meas[miss]).write(missfile, overwrite=True) else: print(str(i + 1) + ' ' + exp1 + ' none missing') else: print(measfile + ' NOT FOUND')
def make_4panelplot(obj, outdir): matplotlib.use('Agg') #params = {'tex.usetex': True} #plt.rcParams.update(params) #plt.rc(usetex = True) idv = obj['id'] #gather all data used dataselect = "select mjd,ra,dec,mag_auto,raerr,decerr,filter,fwhm,exposure from nsc_dr2.meas where objectid='" + idv + "'" meas = qc.query(sql=dataselect, fmt='table', profile='db01') #obj = qc.query(sql="select id,pmra,pmdec from nsc_dr2.object where id='"+ idv +"'",fmt='table',profile='db01') #obj = qc.query(sql="select id,pmra,pmdec from nsc_dr2.object where id='"+ idv +"'",fmt='table') # Make cut on FWHM # maybe only use values for 0.5*fwhm_chip to 1.5*fwhm_chip sql = "select chip.* from nsc_dr2.chip as chip join nsc_dr2.meas as meas on chip.exposure=meas.exposure and chip.ccdnum=meas.ccdnum" sql += " where meas.objectid='" + idv + "'" chip = qc.query(sql=sql, fmt='table') ind3, ind4 = dln.match(chip['exposure'], meas['exposure']) si = np.argsort(ind4) # sort by input meas catalog ind3 = ind3[si] ind4 = ind4[si] chip = chip[ind3] meas = meas[ind4] gdfwhm, = np.where((meas['fwhm'] > 0.2 * chip['fwhm']) & (meas['fwhm'] < 2.0 * chip['fwhm'])) if len(gdfwhm) == 0: print('All measurements have bad FWHM values') return if len(gdfwhm) < len(meas): print('Removing ' + str(len(meas) - len(gdfwhm)) + ' measurements with bad FWHM values') meas = meas[gdfwhm] plt.subplots(2, 2, figsize=(12, 8)) plt.subplots_adjust(hspace=.4, wspace=.4) plt.suptitle(idv) meas["mjd"] -= min(meas["mjd"]) mjd = (meas["mjd"]) cenra = np.mean(meas["ra"]) cendec = np.mean(meas["dec"]) mag = meas["mag_auto"] filters = meas["filter"] #colors = ["b", "g", "r", "c", "y"] colordict = { 'u': 'c', 'g': 'b', 'r': 'g', 'i': 'y', 'z': 'orange', 'Y': 'r', 'VR': 'purple' } ra = meas["ra"] raerr = meas['raerr'] dra = ra - cenra dra *= 3600 * np.cos(np.deg2rad(cendec)) dec = meas["dec"] decerr = meas['decerr'] ddec = dec - cendec ddec *= 3600 t = meas["mjd"] - np.mean(meas["mjd"]) pmra = obj["pmra"] / 1000 / 365.2425 # mas/yr -> as/day pmdec = obj["pmdec"] / 1000 / 365.2425 size = 15 # filter some goodind = np.where(np.logical_and(abs(ddec) < 500, abs(dra) < 500)) ddec = ddec[goodind] dra = dra[goodind] raerr = raerr[goodind] decerr = decerr[goodind] mjd = mjd[goodind] filters = filters[goodind] mag = mag[goodind] meandra = np.mean(dra) meanddec = np.mean(ddec) # Unique filters # put them in this order [u,g,r,i,z,Y,VR] ufilter = [] for filt in ['u', 'g', 'r', 'i', 'z', 'Y', 'VR']: if filt in filters: ufilter.append(filt) # ra dec plot plt.subplot(2, 2, 1) plt.errorbar(dra, ddec, raerr, decerr, fmt='none', ecolor='lightgray', elinewidth=1, capsize=0, alpha=0.5, zorder=0) plt.scatter(dra, ddec, c=mjd, s=size, zorder=1) diffra = max(dra) - min(dra) diffdec = max(ddec) - min(ddec) plt.xlim(min(dra) - diffra / 4, max(dra) + diffra / 4) plt.ylim(min(ddec) - diffdec / 4, max(ddec) + diffdec / 4) m, b = np.polyfit(dra, ddec, 1) plt.plot(dra, m * dra + b, c="k") plt.colorbar(label=r'$\Delta$ MJD') plt.xlabel("$\Delta$ RA (arcsec)") plt.ylabel("$\Delta$ DEC (arcsec)") # ra mjd plot plt.subplot(2, 2, 2) plt.errorbar(mjd, dra, yerr=raerr, fmt='none', ecolor='lightgray', elinewidth=1, capsize=0, alpha=0.5, zorder=0) count = 0 for fil in ufilter: filind = np.where(filters == fil) #plt.scatter(mjd[filind], dra[filind], c = colors[count], label = fil, s = size, zorder=1) plt.scatter(mjd[filind], dra[filind], c=colordict[fil], label=fil, s=size, zorder=1) count += 1 plt.legend() diffmjd = max(mjd) - min(mjd) plt.xlim(min(mjd) - diffmjd / 4, max(mjd) + diffmjd / 4) plt.ylim(min(dra) - diffra / 4, max(dra) + diffra / 4) m, b = np.polyfit(mjd, dra, 1) plt.plot(mjd, mjd * pmra + b, c="k") plt.xlabel(r'$\Delta$ MJD (days)') plt.ylabel(r'$\Delta$ RA (arcsec)') # dec mjd plot plt.subplot(2, 2, 4) plt.errorbar(mjd, ddec, yerr=decerr, fmt='none', ecolor='lightgray', elinewidth=1, capsize=0, alpha=0.5, zorder=0) count = 0 for fil in ufilter: filind = np.where(filters == fil) plt.scatter(mjd[filind], ddec[filind], c=colordict[fil], label=fil, s=size, zorder=1) count += 1 plt.legend() plt.xlim(min(mjd) - diffmjd / 4, max(mjd) + diffmjd / 4) plt.ylim(min(ddec) - diffdec / 4, max(ddec) + diffdec / 4) m, b = np.polyfit(mjd, ddec, 1) plt.plot(mjd, mjd * pmdec + b, c="k") plt.xlabel(r'$\Delta$ MJD (days)') plt.ylabel(r'$\Delta$ DEC (arcsec)') #magtime plt.subplot(2, 2, 3) count = 0 for fil in ufilter: filind = np.where(filters == fil) plt.scatter(mjd[filind], mag[filind], c=colordict[fil], label=fil, s=size) count += 1 plt.legend() diffmag = max(mag) - min(mag) plt.xlim(min(mjd) - diffmjd / 4, max(mjd) + diffmjd / 4) plt.ylim(min(mag) - diffmag / 4, max(mag) + diffmag / 4) plt.xlabel(r'$\Delta$ MJD (days)') plt.ylabel("Magnitude") outfile = outdir + idv + '.png' #outfile = outdir+idv+'_4panel.png' print('Saving figure to ' + outfile) plt.savefig(outfile, bbox_inches='tight')
def exposure_update(exposure, redo=False): """ Update the measurement table using the broken up measid/objectid lists.""" t00 = time.time() hostname = socket.gethostname() host = hostname.split('.')[0] iddir = '/data0/dnidever/nsc/instcal/v3/idstr/' version = 'v3' # Load the exposures table print('Loading exposure table') expcat = fits.getdata( '/net/dl2/dnidever/nsc/instcal/' + version + '/lists/nsc_v3_exposure_table.fits.gz', 1) # Make sure it's a list if type(exposure) is str: exposure = [exposure] # Match exposures to exposure catalog eind1, eind2 = dln.match(expcat['EXPOSURE'], exposure) nmatch = len(eind1) print( str(nmatch) + ' matches for ' + str(len(exposure)) + ' input exposures') if len(eind1) == 0: print('No exposures matched to exposure table') sys.exit() print('Updating measid for ' + str(len(exposure)) + ' exposures') # Loop over files for i in range(len(exposure)): t0 = time.time() exp = expcat['EXPOSURE'][eind1[i]] print(str(i + 1) + ' ' + exp) instcode = expcat['INSTRUMENT'][eind1[i]] dateobs = expcat['DATEOBS'][eind1[i]] night = dateobs[0:4] + dateobs[5:7] + dateobs[8:10] expdir = '/net/dl2/dnidever/nsc/instcal/' + version + '/' + instcode + '/' + night + '/' + exp edir = iddir + instcode + '/' + night + '/' + exp + '/' # local directory for ID files #outdir = edir outdir = expdir # Check that the directory exists if os.path.exists(expdir) is False: print(expdir + ' NOT FOUND') continue # Check output file measfile = outdir + '/' + exp + '_meas.fits' if (os.path.exists(measfile + '.gz')) & (redo is False): print(measfile + '.gz already exists. Skipping') continue # Log file #------------------ # format is EXPOSURE_measure_update.DATETIME.log ltime = time.localtime() # time.struct_time(tm_year=2019, tm_mon=7, tm_mday=22, tm_hour=0, tm_min=30, tm_sec=20, tm_wday=0, tm_yday=203, tm_isdst=1) smonth = str(ltime[1]) if ltime[1] < 10: smonth = '0' + smonth sday = str(ltime[2]) if ltime[2] < 10: sday = '0' + sday syear = str(ltime[0])[2:] shour = str(ltime[3]) if ltime[3] < 10: shour = '0' + shour sminute = str(ltime[4]) if ltime[4] < 10: sminute = '0' + sminute ssecond = str(int(ltime[5])) if ltime[5] < 10: ssecond = '0' + ssecond logtime = smonth + sday + syear + shour + sminute + ssecond logfile = outdir + '/' + exp + '_measure_update.' + logtime + '.log' if os.path.exists(logfile): os.remove(logfile) # Set up logging to screen and logfile logFormatter = logging.Formatter( "%(asctime)s [%(levelname)-5.5s] %(message)s") if logging.getLogger().hasHandlers() is True: rootLogger.handlers = [] # remove all handlers rootLogger = logging.getLogger() fileHandler = logging.FileHandler(logfile) fileHandler.setFormatter(logFormatter) rootLogger.addHandler(fileHandler) consoleHandler = logging.StreamHandler() consoleHandler.setFormatter(logFormatter) rootLogger.addHandler(consoleHandler) rootLogger.setLevel(logging.NOTSET) rootLogger.info( 'Adding objectID for measurement catalogs for exposure = ' + exp) rootLogger.info("expdir = " + expdir) rootLogger.info("host = " + host) rootLogger.info(" ") # Load the exposure and metadata files metafile = expdir + '/' + exp + '_meta.fits' meta = Table.read(metafile, 1) nmeta = len(meta) chstr = Table.read(metafile, 2) rootLogger.info('KLUDGE!!! Changing /dl1 filenames to /dl2 filenames') cols = ['EXPDIR', 'FILENAME', 'MEASFILE'] for c in cols: f = np.char.array(chstr[c]).decode() f = np.char.array(f).replace('/dl1/users/dnidever/', '/dl2/dnidever/') chstr[c] = f nchips = len(chstr) # Get "good" chips, astrometrically calibrated astokay = np.zeros(nchips, bool) for k in range(nchips): # Check that this chip was astrometrically calibrated # and falls in to HEALPix region # Also check for issues with my astrometric corrections if (chstr['NGAIAMATCH'][k] == 0) | (np.max(np.abs(chstr['RACOEF'][k])) > 1) | (np.max( np.abs(chstr['DECCOEF'][k])) > 1): astokay[k] = False else: astokay[k] = True #gdch,ngdch,bdch,nbdch = dln.where(chstr['NGAIAMATCH']>0,comp=True) gdch, ngdch, bdch, nbdch = dln.where(astokay == True, comp=True) if nbdch > 0: rootLogger.info( str(nbdch) + ' chips were not astrometrically calibrated') measdtype = np.dtype([('MEASID', 'S50'), ('OBJECTID', 'S50'), ('EXPOSURE', 'S50'), ('CCDNUM', '>i2'), ('FILTER', 'S2'), ('MJD', '>f8'), ('X', '>f4'), ('Y', '>f4'), ('RA', '>f8'), ('RAERR', '>f4'), ('DEC', '>f8'), ('DECERR', '>f4'), ('MAG_AUTO', '>f4'), ('MAGERR_AUTO', '>f4'), ('MAG_APER1', '>f4'), ('MAGERR_APER1', '>f4'), ('MAG_APER2', '>f4'), ('MAGERR_APER2', '>f4'), ('MAG_APER4', '>f4'), ('MAGERR_APER4', '>f4'), ('MAG_APER8', '>f4'), ('MAGERR_APER8', '>f4'), ('KRON_RADIUS', '>f4'), ('ASEMI', '>f4'), ('ASEMIERR', '>f4'), ('BSEMI', '>f4'), ('BSEMIERR', '>f4'), ('THETA', '>f4'), ('THETAERR', '>f4'), ('FWHM', '>f4'), ('FLAGS', '>i2'), ('CLASS_STAR', '>f4')]) # Load and concatenate the meas catalogs chstr[ 'MEAS_INDEX'] = -1 # keep track of where each chip catalog starts count = 0 meas = Table( data=np.zeros(int(np.sum(chstr['NMEAS'][gdch])), dtype=measdtype)) rootLogger.info( 'Loading and concatenating the chip measurement catalogs') for j in range(ngdch): jch = gdch[j] chfile = chstr['MEASFILE'][jch].strip() if chfile == '': continue #print(str(j+1)+' Loading '+chfile) meas1 = Table.read(chfile, 1) # load chip meas catalog nmeas1 = len(meas1) meas[count:count + nmeas1] = meas1 chstr['MEAS_INDEX'][jch] = count count += nmeas1 measid = np.char.array(meas['MEASID']).strip().decode() nmeas = len(meas) rootLogger.info(str(nmeas) + ' measurements') # Look for the id files allfiles = glob(edir + exp + '__*.npy') # check for duplicates, single and split into high-res healpix idstr files # always use the split ones base = [os.path.splitext(os.path.basename(f))[0] for f in allfiles] hfile = [f.split('__')[-1] for f in base] hh = [f.split('_')[0] for f in hfile] # the healpix portion hindex = dln.create_index(hh) files = [] for j in range(len(hindex['value'])): hpix1 = hindex['value'][j] hind = hindex['index'][hindex['lo'][j]:hindex['hi'][j] + 1] files1 = np.array(allfiles)[hind] # duplicates, use the split/hires ones if hindex['num'][j] > 1: gd = dln.grep(files1, str(hpix1) + '_n', index=True) if len(gd) == 0: raise ValueError( 'Something is wrong with the idstr files, duplicates') files += list(files1[gd]) else: files += list(files1) nfiles = len(files) rootLogger.info(str(nfiles) + ' ID files to load') # Loop over ID files and load them up df = np.dtype([('measid', np.str, 50), ('objectid', np.str, 50)]) idcat = np.zeros(10000, dtype=df) count = 0 for k in range(nfiles): idcat1 = np.load(files[k]) nidcat1 = len(idcat1) # Add more elements if count + nidcat1 > len(idcat): idcat = dln.add_elements(idcat, np.maximum(100000, nidcat1)) # Stuff in the data idcat[count:count + nidcat1] = idcat1 count += nidcat1 # Trim extra elements if len(idcat) > count: idcat = idcat[0:count] rootLogger.info('IDs for ' + str(len(idcat)) + ' measurements') # Match up with measid idcat_measid = np.char.array(idcat['measid']).strip() if isinstance(idcat_measid[0], bytes): idcat_measid = idcat_measid.decode() ind1, ind2 = dln.match(idcat_measid, measid) nmatch = len(ind1) rootLogger.info('Matches for ' + str(nmatch) + ' measurements') if nmatch > 0: meas['OBJECTID'][ind2] = idcat['objectid'][ind1] if (len(ind1) > len(measid)) | (len(idcat) > len(meas)): rootLogger.info('There are ' + str(len(idcat) - len(meas)) + ' duplicates!!') # Checking for missing objectid ind, nind = dln.where( np.char.array(meas['OBJECTID']).strip().decode() == '') # There can be missing/orphaned measurements at healpix boundaries in crowded # regions when the DBSCAN eps is different. But there should be very few of these. # At this point, let's allow this to pass if nind > 0: rootLogger.info('WARNING: ' + str(nind) + ' measurements are missing OBJECTIDs') #if ((nmeas>=20000) & (nind>20)) | ((nmeas<20000) & (nind>3)): # rootLogger.info('More missing OBJECTIDs than currently allowed.') # hpix = hp.ang2pix(128,meas['RA'][ind],meas['DEC'][ind],lonlat=True) # hindex = dln.create_index(hpix) # out = [] # for i in range(len(hindex['value'])): # out.append(str(hindex['value'][i])+' ('+str(hindex['num'][i])+')') # rootLogger.info('healpix of missing measurements: '+', '.join(out)) # outtxt = [str(nind)+' missing IDs','healpix of missing measurements: '+', '.join(out)] # dln.writelines(outdir+'/'+exp+'_meas.ERROR',outtxt) # continue # Output the updated measurement catalog # Writing a single FITS file is much faster than many small ones # could put it in /data0 but db01 won't be able to access that rootLogger.info('Writing final measurement catalog to ' + measfile) meas.write(measfile, overwrite=True) if os.path.exists(measfile + '.gz'): os.remove(measfile + '.gz') ret = subprocess.call(['gzip', measfile]) # compress final catalog # Update the meta file as well, need to update the /dl2 filenames metafile = outdir + '/' + exp + '_meta.fits' rootLogger.info('Updating meta file ' + metafile) meta.write(metafile, overwrite=True) hdulist = fits.open(metafile) hdu = fits.table_to_hdu(chstr) hdulist.append(hdu) hdulist.writeto(metafile, overwrite=True) hdulist.close() # Create a file saying that the files were updated okay. #dln.writelines(expdir+'/'+exp+'_meas.updated','') dln.writelines(outdir + '/' + exp + '_meas.updated', '') # Remove meas.ERROR, if it exists if os.path.exists(outdir + '/' + exp + '_meas.ERROR'): os.remove(outdir + '/' + exp + '_meas.ERROR') rootLogger.info('dt = ' + str(time.time() - t0) + ' sec.') print('dt = %6.1f sec.' % (time.time() - t00))
def measurement_info(pix): t0 = time.time() hostname = socket.gethostname() host = hostname.split('.')[0] # Get version number from exposure directory #lo = expdir.find('nsc/instcal/') #dum = expdir[lo+12:] #version = dum[0:dum.find('/')] version = 'v3' cmbdir = '/net/dl2/dnidever/nsc/instcal/' + version + '/' #edir = '/net/dl1/users/dnidever/nsc/instcal/'+version+'/' #nside = 128 #expstr = fits.getdata('/net/dl2/dnidever/nsc/instcal/'+version+'/lists/nsc_'+version+'_exposures.fits.gz',1) # too much many columns, just need full path and base metadb = '/net/dl2/dnidever/nsc/instcal/' + version + '/lists/nsc_meta.db' data = querydb(metadb, 'exposure', 'expdir') data = [a[0] for a in data] expdir = np.char.array(data) expdir = expdir.rstrip('/') base = [os.path.basename(e) for e in expdir] base = np.char.array(base) # If we put the output files in a PIX_idstr/ subdirectory then I wouldn't need to # know all of this exposure path information dbfile = cmbdir + 'combine/' + str( int(pix) // 1000) + '/' + str(pix) + '_idstr.db' print(dbfile) # Deal with sub-pixels!! # Get the row count db = sqlite3.connect(dbfile, detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES) cur = db.cursor() cur.execute('select count(rowid) from idstr') data = cur.fetchall() db.close() nrows = data[0][0] print(str(nrows) + ' rows') print('Loading the data') idstr = readidstrdb(dbfile) # Need to do this in chunks if there are too many rows # Get unique exposures exposure = np.char.array(idstr['exposure']) expindex = dln.create_index(exposure) nexp = len(expindex['value']) print(str(nexp) + ' exposures') # Get absolute paths ind1, ind2 = dln.match(base, expindex['value']) expdirs = np.zeros(nexp, (np.str, 200)) expdirs[ind2] = expdir[ind1] # Convert /dl1 to /dl2 expdirs = np.char.array(expdirs).replace('/dl1/users/dnidever/', '/dl2/dnidever/') # Loop through the exposures and write out their information for e in range(nexp): exposure1 = expindex['value'][e] eind = expindex['index'][expindex['lo'][e]:expindex['hi'][e] + 1] idstr1 = idstr[eind] nidstr1 = len(idstr1) # Just need measid,objectid, and only the width that we need mlen = np.max([len(m) for m in idstr1['measid']]) olen = np.max([len(o) for o in idstr1['objectid']]) dt = np.dtype([('measid', np.str, mlen), ('objectid', np.str, olen)]) new = np.zeros(nidstr1, dtype=dt) new['measid'] = idstr1['measid'] new['objectid'] = idstr1['objectid'] print(str(e + 1) + ' ' + exposure1 + ' ' + str(nidstr1)) # Put these files in expdir/idstr/ subdirectory!! # Write it out outfile = expdirs[e] + '/' + exposure1 + '_objectid_list.fits' #outfile = expdirs[e]+'/'+exposure1+'_objectid_list.npy' print(' Writing ' + outfile) #if os.path.exists(outfile): os.remove(outfile) #np.save(outfile,new) # not any faster Table(new).write(outfile, overwrite=True) print('dt = ' + str(time.time() - t0) + ' sec.') import pdb pdb.set_trace() # Check if output file already exists #base = os.path.basename(expdir) ## Log file ##------------------ ## format is nsc_combine_main.DATETIME.log #ltime = time.localtime() ## time.struct_time(tm_year=2019, tm_mon=7, tm_mday=22, tm_hour=0, tm_min=30, tm_sec=20, tm_wday=0, tm_yday=203, tm_isdst=1) #smonth = str(ltime[1]) #if ltime[1]<10: smonth = '0'+smonth #sday = str(ltime[2]) #if ltime[2]<10: sday = '0'+sday #syear = str(ltime[0])[2:] #shour = str(ltime[3]) #if ltime[3]<10: shour='0'+shour #sminute = str(ltime[4]) #if ltime[4]<10: sminute='0'+sminute #ssecond = str(int(ltime[5])) #if ltime[5]<10: ssecond='0'+ssecond #logtime = smonth+sday+syear+shour+sminute+ssecond #logfile = expdir+'/'+base+'_measure_update.'+logtime+'.log' #if os.path.exists(logfile): os.remove(logfile) ## Set up logging to screen and logfile #logFormatter = logging.Formatter("%(asctime)s [%(levelname)-5.5s] %(message)s") #rootLogger = logging.getLogger() #fileHandler = logging.FileHandler(logfile) #fileHandler.setFormatter(logFormatter) #rootLogger.addHandler(fileHandler) #consoleHandler = logging.StreamHandler() #consoleHandler.setFormatter(logFormatter) #rootLogger.addHandler(consoleHandler) #rootLogger.setLevel(logging.NOTSET) #rootLogger.info('Adding objectID for measurement catalogs for exposure = '+base) #rootLogger.info("expdir = "+expdir) #rootLogger.info("host = "+host) #rootLogger.info(" ") # Load the exposure and metadata files metafile = expdir + '/' + base + '_meta.fits' meta = Table.read(metafile, 1) nmeta = len(meta) chstr = Table.read(metafile, 2) rootLogger.info('KLUDGE!!! Changing /dl1 filenames to /dl2 filenames') cols = ['EXPDIR', 'FILENAME', 'MEASFILE'] for c in cols: f = np.char.array(chstr[c]).decode() f = np.char.array(f).replace('/dl1/users/dnidever/', '/dl2/dnidever/') chstr[c] = f nchips = len(chstr) measdtype = np.dtype([('MEASID', 'S50'), ('OBJECTID', 'S50'), ('EXPOSURE', 'S50'), ('CCDNUM', '>i2'), ('FILTER', 'S2'), ('MJD', '>f8'), ('X', '>f4'), ('Y', '>f4'), ('RA', '>f8'), ('RAERR', '>f4'), ('DEC', '>f8'), ('DECERR', '>f4'), ('MAG_AUTO', '>f4'), ('MAGERR_AUTO', '>f4'), ('MAG_APER1', '>f4'), ('MAGERR_APER1', '>f4'), ('MAG_APER2', '>f4'), ('MAGERR_APER2', '>f4'), ('MAG_APER4', '>f4'), ('MAGERR_APER4', '>f4'), ('MAG_APER8', '>f4'), ('MAGERR_APER8', '>f4'), ('KRON_RADIUS', '>f4'), ('ASEMI', '>f4'), ('ASEMIERR', '>f4'), ('BSEMI', '>f4'), ('BSEMIERR', '>f4'), ('THETA', '>f4'), ('THETAERR', '>f4'), ('FWHM', '>f4'), ('FLAGS', '>i2'), ('CLASS_STAR', '>f4')]) # Load and concatenate the meas catalogs chstr['MEAS_INDEX'] = 0 # keep track of where each chip catalog starts count = 0 meas = Table(data=np.zeros(int(np.sum(chstr['NMEAS'])), dtype=measdtype)) rootLogger.info('Loading and concatenating the chip measurement catalogs') for i in range(nchips): meas1 = Table.read(chstr['MEASFILE'][i].strip(), 1) # load chip meas catalog nmeas1 = len(meas1) meas[count:count + nmeas1] = meas1 chstr['MEAS_INDEX'][i] = count count += nmeas1 measid = np.char.array(meas['MEASID']).strip().decode() nmeas = len(meas) rootLogger.info(str(nmeas) + ' measurements') # Get the OBJECTID from the combined healpix file IDSTR structure # remove any sources that weren't used # Figure out which healpix this figure overlaps pix = hp.ang2pix(nside, meas['RA'], meas['DEC'], lonlat=True) upix = np.unique(pix) npix = len(upix) rootLogger.info(str(npix) + ' HEALPix to query') # Loop over the HEALPix pixels ntotmatch = 0 idstr_dtype = np.dtype([('measid', np.str, 200), ('objectid', np.str, 200), ('pix', int)]) idstr = np.zeros(nmeas, dtype=idstr_dtype) cnt = 0 for i in range(npix): fitsfile = cmbdir + 'combine/' + str(int(upix[i]) // 1000) + '/' + str( upix[i]) + '.fits.gz' dbfile = cmbdir + 'combine/' + str(int(upix[i]) // 1000) + '/' + str( upix[i]) + '_idstr.db' if os.path.exists(dbfile): # Read meas id information from idstr database for this expoure #data = querydb(dbfile,table='idstr',cols='measid,objectid',where="exposure=='"+base+"'") idstr1 = readidstrdb(dbfile, where="exposure=='" + base + "'") nidstr1 = len(idstr1) if nidstr1 > 0: idstr['measid'][cnt:cnt + nidstr1] = idstr1['measid'] idstr['objectid'][cnt:cnt + nidstr1] = idstr1['objectid'] idstr['pix'][cnt:cnt + nidstr1] = upix[i] cnt += nidstr1 rootLogger.info( str(i + 1) + ' ' + str(upix[i]) + ' ' + str(nidstr1)) #nmatch = 0 #if nidstr>0: # idstr_measid = np.char.array(idstr['measid']).strip() # idstr_objectid = np.char.array(idstr['objectid']).strip() # #ind1,ind2 = dln.match(idstr_measid,measid) # nmatch = len(ind1) # if nmatch>0: # meas['OBJECTID'][ind2] = idstr_objectid[ind1] # ntotmatch += nmatch #rootLogger.info(str(i+1)+' '+str(upix[i])+' '+str(nmatch)) else: rootLogger.info( str(i + 1) + ' ' + dbfile + ' NOT FOUND. Checking for high-resolution database files.') # Check if there are high-resolution healpix idstr databases hidbfiles = glob(cmbdir + 'combine/' + str(int(upix[i]) // 1000) + '/' + str(upix[i]) + '_n*_*_idstr.db') nhidbfiles = len(hidbfiles) if os.path.exists(fitsfile) & (nhidbfiles > 0): rootLogger.info('Found high-resolution HEALPix IDSTR files') for j in range(nhidbfiles): dbfile1 = hidbfiles[j] dbbase1 = os.path.basename(dbfile1) idstr1 = readidstrdb(dbfile1, where="exposure=='" + base + "'") nidstr1 = len(idstr1) if nidstr1 > 0: idstr['measid'][cnt:cnt + nidstr1] = idstr1['measid'] idstr['objectid'][cnt:cnt + nidstr1] = idstr1['objectid'] idstr['pix'][cnt:cnt + nidstr1] = upix[i] cnt += nidstr1 rootLogger.info(' ' + str(j + 1) + ' ' + dbbase1 + ' ' + str(upix[i]) + ' ' + str(nidstr1)) #idstr_measid = np.char.array(idstr['measid']).strip() #idstr_objectid = np.char.array(idstr['objectid']).strip() #ind1,ind2 = dln.match(idstr_measid,measid) #nmatch = len(ind1) #if nmatch>0: # meas['OBJECTID'][ind2] = idstr_objectid[ind1] # ntotmatch += nmatch #rootLogger.info(' '+str(j+1)+' '+dbbase1+' '+str(upix[i])+' '+str(nmatch)) # Trim any leftover elements of IDSTR if cnt < nmeas: idstr = idstr[0:cnt] # Now match them all up rootLogger.info('Matching the measurements') idstr_measid = np.char.array(idstr['measid']).strip() idstr_objectid = np.char.array(idstr['objectid']).strip() ind1, ind2 = dln.match(idstr_measid, measid) nmatch = len(ind1) if nmatch > 0: meas['OBJECTID'][ind2] = idstr_objectid[ind1] # Only keep sources with an objectid ind, nind = dln.where( np.char.array(meas['OBJECTID']).strip().decode() == '') # There can be missing/orphaned measurements at healpix boundaries in crowded # regions when the DBSCAN eps is different. But there should be very few of these. # At this point, let's allow this to pass if nind > 0: rootLogger.info('WARNING: ' + str(nind) + ' measurements are missing OBJECTIDs') if ((nmeas >= 20000) & (nind > 20)) | ((nmeas < 20000) & (nind > 3)): rootLogger.info('More missing OBJECTIDs than currently allowed.') raise ValueError('More missing OBJECTIDs than currently allowed.') # Output the updated catalogs #rootLogger.info('Updating measurement catalogs') #for i in range(nchips): # measfile1 = chstr['MEASFILE'][i].strip() # lo = chstr['MEAS_INDEX'][i] # hi = lo+chstr['NMEAS'][i] # meas1 = meas[lo:hi] # meta1 = Table.read(measfile1,2) # load the meta extensions # # 'KLUDGE!!! Changing /dl1 filenames to /dl2 filenames') # cols = ['EXPDIR','FILENAME','MEASFILE'] # for c in cols: # f = np.char.array(meta1[c]).decode() # f = np.char.array(f).replace('/dl1/users/dnidever/','/dl2/dnidever/') # meta1[c] = f # # Copy as a backup # if os.path.exists(measfile1+'.bak'): os.remove(measfile1+'.bak') # dum = shutil.move(measfile1,measfile1+'.bak') # # Write new catalog # #meas1.write(measfile1,overwrite=True) # first, measurement table # # append other fits binary tabl # #hdulist = fits.open(measfile1) # rootLogger.info('Writing '+measfile1) # hdulist = fits.HDUList() # hdulist.append(fits.table_to_hdu(meas1)) # first, meas catalog # hdulist.append(fits.table_to_hdu(meta1)) # second, meta # hdulist.writeto(measfile1,overwrite=True) # hdulist.close() # # Create a file saying that the file was successfully updated. # dln.writelines(measfile1+'.updated','') # # Delete backups # if os.path.exists(measfile1+'.bak'): os.remove(measfile1+'.bak') measfile = expdir + '/' + base + '_meas.fits' meas.write(measfile, overwrite=True) if os.path.exists(measfile + '.gz'): os.remove(measfile + '.gz') ret = subprocess.call(['gzip', measfile]) # compress final catalog # Update the meta file as well, need to the /dl2 filenames rootLogger.info('Updating meta file') meta.write(metafile, overwrite=True) hdulist = fits.open(metafile) hdu = fits.table_to_hdu(chstr) hdulist.append(hdu) hdulist.writeto(metafile, overwrite=True) hdulist.close() # Create a file saying that the files were updated okay. dln.writelines(expdir + '/' + base + '_meas.updated', '') rootLogger.info('dt = ' + str(time.time() - t0) + ' sec.')
def breakup_idstr(dbfile): """ Break-up idstr file into separate measid/objectid lists per exposure on /data0.""" t00 = time.time() outdir = '/data0/dnidever/nsc/instcal/v3/idstr/' # Load the exposures table expcat = fits.getdata('/net/dl2/dnidever/nsc/instcal/v3/lists/nsc_v3_exposure_table.fits.gz',1) # Make sure it's a list if type(dbfile) is str: dbfile=[dbfile] print('Breaking up '+str(len(dbfile))+' database files') # Loop over files for i,dbfile1 in enumerate(dbfile): print(str(i+1)+' '+dbfile1) if os.path.exists(dbfile1): t0 = time.time() dbbase1 = os.path.basename(dbfile1)[0:-9] # remove _idstr.db ending # Get existing index names for this database d = sqlite3.connect(dbfile1, detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES) cur = d.cursor() cmd = 'select measid,exposure,objectid from idstr' t1 = time.time() data = cur.execute(cmd).fetchall() print(' '+str(len(data))+' rows read in %5.1f sec. ' % (time.time()-t1)) # Break up data into lists measid,exposure,objectid = list(zip(*data)) measid = np.array(measid) objectid = np.array(objectid) exposure = np.array(exposure) eindex = dln.create_index(exposure) # Match exposures to exposure catalog ind1,ind2 = dln.match(expcat['EXPOSURE'],eindex['value']) # Loop over exposures and write output files nexp = len(eindex['value']) print(' '+str(nexp)+' exposures') measid_maxlen = np.max(dln.strlen(measid)) objectid_maxlen = np.max(dln.strlen(objectid)) df = np.dtype([('measid',np.str,measid_maxlen+1),('objectid',np.str,objectid_maxlen+1)]) # Loop over the exposures and write out the files for k in range(nexp): if nexp>100: if k % 100 == 0: print(' '+str(k+1)) ind = eindex['index'][eindex['lo'][k]:eindex['hi'][k]+1] cat = np.zeros(len(ind),dtype=df) cat['measid'] = measid[ind] cat['objectid'] = objectid[ind] instcode = expcat['INSTRUMENT'][ind1[k]] dateobs = expcat['DATEOBS'][ind1[k]] night = dateobs[0:4]+dateobs[5:7]+dateobs[8:10] if os.path.exists(outdir+instcode+'/'+night+'/'+eindex['value'][k]) is False: # Sometimes this crashes because another process is making the directory at the same time try: os.makedirs(outdir+instcode+'/'+night+'/'+eindex['value'][k]) except: pass outfile = outdir+instcode+'/'+night+'/'+eindex['value'][k]+'/'+eindex['value'][k]+'__'+dbbase1+'.npy' np.save(outfile,cat) print(' dt = %6.1f sec. ' % (time.time()-t0)) else: print(' '+dbfile1+' NOT FOUND') print('dt = %6.1f sec.' % (time.time()-t00))
def meascutout(meas, obj, size=10, outdir='./', domask=True): """ Input the measurements and create cutouts. """ expstr = fits.getdata( '/net/dl2/dnidever/nsc/instcal/v3/lists/nsc_v3_exposures.fits.gz', 1) #expstr = fits.getdata('/net/dl2/dnidever/nsc/instcal/v3/lists/nsc_v3_exposure.fits.gz',1) decam = Table.read('/home/dnidever/projects/delvered/data/decam.txt', format='ascii') objid = obj['id'][0] # Sort by MJD si = np.argsort(meas['mjd']) meas = meas[si] # Make cut on FWHM # maybe only use values for 0.5*fwhm_chip to 1.5*fwhm_chip sql = "select chip.* from nsc_dr2.chip as chip join nsc_dr2.meas as meas on chip.exposure=meas.exposure and chip.ccdnum=meas.ccdnum" sql += " where meas.objectid='" + objid + "'" chip = qc.query(sql=sql, fmt='table') ind3, ind4 = dln.match(chip['exposure'], meas['exposure']) si = np.argsort(ind4) # sort by input meas catalog ind3 = ind3[si] ind4 = ind4[si] chip = chip[ind3] meas = meas[ind4] gdfwhm, = np.where((meas['fwhm'] > 0.2 * chip['fwhm']) & (meas['fwhm'] < 2.0 * chip['fwhm'])) if len(gdfwhm) == 0: print('All measurements have bad FWHM values') return if len(gdfwhm) < len(meas): print('Removing ' + str(len(meas) - len(gdfwhm)) + ' measurements with bad FWHM values') meas = meas[gdfwhm] ind1, ind2 = dln.match(expstr['base'], meas['exposure']) nind = len(ind1) if nind == 0: print('No matches') return # Sort by input meas catalog si = np.argsort(ind2) ind1 = ind1[si] ind2 = ind2[si] # Create the reference WCS wref = WCS(naxis=2) pixscale = 0.26 # DECam, "/pix npix = round(size / pixscale) if npix % 2 == 0: # must be odd npix += 1 hpix = npix // 2 # center of image wref.wcs.ctype = ['RA---TAN', 'DEC--TAN'] wref.wcs.crval = [obj['ra'][0], obj['dec'][0]] wref.wcs.crpix = [npix // 2, npix // 2] wref.wcs.cd = np.array([[pixscale / 3600.0, 0.0], [0.0, pixscale / 3600]]) wref.array_shape = (npix, npix) refheader = wref.to_header() refheader['NAXIS'] = 2 refheader['NAXIS1'] = npix refheader['NAXIS2'] = npix # Load the data instrument = expstr['instrument'][ind1] plver = expstr['plver'][ind1] fluxfile = expstr['file'][ind1] fluxfile = fluxfile.replace('/net/mss1/', '/mss1/') # for thing/hulk maskfile = expstr['maskfile'][ind1] maskfile = maskfile.replace('/net/mss1/', '/mss1/') # for thing/hulk ccdnum = meas['ccdnum'][ind2] figfiles = [] xmeas = [] ymeas = [] cutimarr = np.zeros((npix, npix, nind), float) for i in range(nind): #for i in range(3): instcode = instrument[i] plver1 = plver[i] try: if instrument[i] == 'c4d': dind, = np.where(decam['CCDNUM'] == ccdnum[i]) extname = decam['NAME'][dind[0]] im, head = getfitsext(fluxfile[i], extname, header=True) mim, mhead = getfitsext(maskfile[i], extname, header=True) #im,head = fits.getdata(fluxfile[i],header=True,extname=extname) #mim,mhead = fits.getdata(maskfile[i],header=True,extname=extname) else: im, head = fits.getdata(fluxfile[i], ccdnum[i], header=True) mim, mhead = fits.getdata(maskfile[i], ccdnum[i], header=True) except: print('error') import pdb pdb.set_trace() # Turn the mask from integer to bitmask if ((instcode == 'c4d') & (plver1 >= 'V3.5.0')) | (instcode == 'k4m') | (instcode == 'ksb'): omim = mim.copy() mim *= 0 nonzero = (omim > 0) mim[nonzero] = 2**((omim - 1)[nonzero]) # This takes about 1 sec # Fix the DECam Pre-V3.5.0 masks if (instcode == 'c4d') & (plver1 < 'V3.5.0'): omim = mim.copy() mim *= 0 # re-initialize mim += (np.bitwise_and(omim, 1) == 1) * 1 # bad pixels mim += (np.bitwise_and(omim, 2) == 2) * 4 # saturated mim += (np.bitwise_and(omim, 4) == 4) * 32 # interpolated mim += (np.bitwise_and(omim, 16) == 16) * 16 # cosmic ray mim += (np.bitwise_and(omim, 64) == 64) * 8 # bleed trail # Get chip-level information exposure = os.path.basename(fluxfile[i])[0:-8] # remove fits.fz chres = qc.query(sql="select * from nsc_dr2.chip where exposure='" + exposure + "' and ccdnum=" + str(ccdnum[i]), fmt='table') w = WCS(head) # RA/DEC correction for the object lon = obj['ra'][0] - chres['ra'][0] lat = obj['dec'][0] - chres['dec'][0] racorr = chres['ra_coef1'][0] + chres['ra_coef2'][0] * lon + chres[ 'ra_coef3'][0] * lon * lat + chres['ra_coef4'][0] * lat deccorr = chres['dec_coef1'][0] + chres['dec_coef2'][0] * lon + chres[ 'dec_coef3'][0] * lon * lat + chres['dec_coef4'][0] * lat # apply these offsets to the header WCS CRVAL #w.wcs.crval += [racorr,deccorr] #head['CRVAL1'] += racorr #head['CRVAL2'] += deccorr print(racorr, deccorr) # Object X/Y position xobj, yobj = w.all_world2pix(obj['ra'], obj['dec'], 0) # Get the cutout xcen = meas['x'][ind2[i]] - 1 # convert to 0-indexes ycen = meas['y'][ind2[i]] - 1 smim = dln.gsmooth(im, 2) # use the object coords for centering #cutim,xr,yr = cutout(smim,xobj,yobj,size) # Mask the bad pixels if domask == True: badmask = (mim > 0) im[badmask] = np.nanmedian(im[~badmask]) else: badmask = (im < 0) # Create a common TAN WCS that each image gets interpoled onto!!! #hdu1 = fits.open(fluxfile[i],extname=extname) smim1 = dln.gsmooth(im, 1.5) hdu = fits.PrimaryHDU(smim1, head) cutim, footprint = reproject_interp(hdu, refheader, order='bicubic') # biquadratic cutim[footprint == 0] = np.nanmedian( im[~badmask]) # set out-of-bounds to background #xr = [0,npix-1] #yr = [0,npix-1] xr = [-hpix * pixscale, hpix * pixscale] yr = [-hpix * pixscale, hpix * pixscale] # exposure_ccdnum, filter, MJD, delta_MJD, mag print( str(i + 1) + ' ' + meas['exposure'][ind2[i]] + ' ' + str(ccdnum[i]) + ' ' + str(meas['x'][ind2[i]]) + ' ' + str(meas['y'][ind2[i]]) + ' ' + str(meas['mag_auto'][ind2[i]])) #figdir = '/net/dl2/dnidever/nsc/instcal/v3/hpm2/cutouts/' figfile = outdir figfile += '%s_%04d_%s_%02d.jpg' % (str( obj['id'][0]), i + 1, meas['exposure'][ind2[i]], ccdnum[i]) figfiles.append(figfile) matplotlib.use('Agg') plt.rc('font', size=15) plt.rc('axes', titlesize=20) plt.rc('axes', labelsize=20) plt.rc('xtick', labelsize=20) plt.rc('ytick', labelsize=20) #plt.rcParams.update({'font.size': 15}) #plt.rcParams.update({'axes.size': 20}) #plt.rcParams.update({'xtick.size': 20}) #plt.rcParams.update({'ytick.size': 20}) if os.path.exists(figfile): os.remove(figfile) fig = plt.gcf() # get current graphics window fig.clf() # clear gskw = dict(width_ratios=[30, 1]) fig, ax = plt.subplots(ncols=2, nrows=1, gridspec_kw=gskw) figsize = 8.0 #6.0 figheight = 8.0 figwidth = 9.0 #ax = fig.subplots() # projection=wcs #fig.set_figheight(figsize*0.8) fig.set_figheight(figheight) fig.set_figwidth(figwidth) med = np.nanmedian(smim) sig = dln.mad(smim) bigim, xr2, yr2 = cutout(smim, xcen, ycen, 151, missing=med) lmed = np.nanmedian(bigim) # Get the flux of the object and scale each image to the same height #meas.mag_aper1 = cat1.mag_aper[0] + 2.5*alog10(exptime) + chstr[i].zpterm #cmag = mag_auto + 2.5*alog10(exptime) + zpterm instmag = meas['mag_auto'][ind2[i]] - 2.5 * np.log10( chres['exptime'][0]) - chres['zpterm'][0] #mag = -2.5*log(flux)+25.0 instflux = 10**((25.0 - instmag) / 2.5) print('flux = ' + str(instflux)) # Get height of object # flux of 2D Gaussian is ~2*pi*height*sigma^2 pixscale1 = np.max(np.abs(w.wcs.cd)) * 3600 fwhm = chres['fwhm'][0] / pixscale1 instheight = instflux / (2 * 3.14 * (fwhm / 2.35)**2) print('height = ' + str(instheight)) # Scale the images to the flux level of the first image cutim -= lmed if i == 0: instflux0 = instflux.copy() instheight0 = instheight.copy() else: scale = instflux0 / instflux #scale = instheight0/instheight cutim *= scale print('scaling image by ' + str(scale)) #vmin = lmed-8*sig # 3*sig #vmax = lmed+12*sig # 5*sig if i == 0: vmin = -8 * sig # 3*sig #vmax = 12*sig # 5*sig vmax = 0.5 * instheight # 0.5 vmin0 = vmin vmax0 = vmax else: vmin = vmin0 vmax = vmax0 print('vmin = ' + str(vmin)) print('vmax = ' + str(vmax)) cutimarr[:, :, i] = cutim.copy() ax[0].imshow(cutim, origin='lower', aspect='auto', interpolation='none', extent=(xr[0], xr[1], yr[0], yr[1]), vmin=vmin, vmax=vmax, cmap='viridis') # viridis, Greys, jet #plt.imshow(cutim,origin='lower',aspect='auto',interpolation='none', # vmin=vmin,vmax=vmax,cmap='viridis') # viridis, Greys, jet #plt.colorbar() # show one vertical, one horizontal line pointing to the center but offset # then a small dot on the meas position # 13, 8 ax[0].plot(np.array([0, 0]), np.array([-0.066 * npix, 0.066 * npix]) * pixscale, c='white', alpha=0.7) ax[0].plot(np.array([-0.066 * npix, 0.066 * npix]) * pixscale, np.array([0, 0]), c='white', alpha=0.7) # Meas X/Y position xmeas1, ymeas1 = wref.all_world2pix(meas['ra'][ind2[i]], meas['dec'][ind2[i]], 0) xmeas.append(xmeas1) ymeas.append(ymeas1) ax[0].scatter([(xmeas1 - hpix) * pixscale], [(ymeas1 - hpix) * pixscale], c='r', marker='+', s=20) #plt.scatter([xmeas],[ymeas],c='r',marker='+',s=100) #plt.scatter([xcen],[ycen],c='r',marker='+',s=100) # Object X/Y position #xobj,yobj = w.all_world2pix(obj['ra'],obj['dec'],0) xobj, yobj = wref.all_world2pix(obj['ra'], obj['dec'], 0) #plt.scatter(xobj,yobj,marker='o',s=200,facecolors='none',edgecolors='y',linewidth=3) #plt.scatter(xobj,yobj,c='y',marker='+',s=100) #leg = ax.legend(loc='upper left', frameon=False) ax[0].set_xlabel(r'$\Delta$ RA (arcsec)') ax[0].set_ylabel(r'$\Delta$ DEC (arcsec)') ax[0].set_xlim((xr[1], xr[0])) # sky right ax[0].set_ylim(yr) #plt.xlabel('X') #plt.ylabel('Y') #plt.xlim(xr) #plt.ylim(yr) #ax.annotate(r'S/N=%5.1f',xy=(np.mean(xr), yr[0]+dln.valrange(yr)*0.05),ha='center') co = 'white' #'lightgray' # blue ax[0].annotate('%s %02d %s %6.1f ' % (meas['exposure'][ind2[i]], ccdnum[i], meas['filter'][ind2[i]], expstr['exptime'][ind1[i]]), xy=(np.mean(xr), yr[0] + dln.valrange(yr) * 0.05), ha='center', color=co) ax[0].annotate( '%10.2f $\Delta$t=%7.2f ' % (meas['mjd'][ind2[i]], meas['mjd'][ind2[i]] - np.min(meas['mjd'])), xy=(xr[1] - dln.valrange(xr) * 0.05, yr[1] - dln.valrange(yr) * 0.05), ha='left', color=co) # xy=(xr[0]+dln.valrange(xr)*0.05, yr[1]-dln.valrange(yr)*0.05),ha='left',color=co) ax[0].annotate('%s = %5.2f +/- %4.2f' % (meas['filter'][ind2[i]], meas['mag_auto'][ind2[i]], meas['magerr_auto'][ind2[i]]), xy=(xr[0] + dln.valrange(xr) * 0.05, yr[1] - dln.valrange(yr) * 0.05), ha='right', color=co) # xy=(xr[1]-dln.valrange(xr)*0.05, yr[1]-dln.valrange(yr)*0.05),ha='right',color=co) # Progress bar frameratio = (i + 1) / float(nind) timeratio = (meas['mjd'][ind2[i]] - np.min(meas['mjd'])) / dln.valrange(meas['mjd']) #ratio = frameratio ratio = timeratio print('ratio = ' + str(100 * ratio)) barim = np.zeros((100, 100), int) ind = dln.limit(int(round(ratio * 100)), 1, 99) barim[:, 0:ind] = 1 ax[1].imshow(barim.T, origin='lower', aspect='auto', cmap='Greys') ax[1].set_xlabel('%7.1f \n days' % (meas['mjd'][ind2[i]] - np.min(meas['mjd']))) #ax[1].set_xlabel('%d/%d' % (i+1,nind)) ax[1].set_title('%d/%d' % (i + 1, nind)) ax[1].axes.xaxis.set_ticks([]) #ax[1].axes.xaxis.set_visible(False) ax[1].axes.yaxis.set_visible(False) #ax[1].axis('off') right_side = ax[1].spines['right'] right_side.set_visible(False) left_side = ax[1].spines['left'] left_side.set_visible(False) top_side = ax[1].spines['top'] top_side.set_visible(False) plt.savefig(figfile) print('Cutout written to ' + figfile) #import pdb; pdb.set_trace() avgim = np.sum(cutimarr, axis=2) / nind avgim *= instheight0 / np.max(avgim) medim = np.median(cutimarr, axis=2) # Make a single blank file at the end so you know it looped figfile = outdir figfile += '%s_%04d_%s.jpg' % (str(obj['id'][0]), i + 2, 'path') figfiles.append(figfile) matplotlib.use('Agg') if os.path.exists(figfile): os.remove(figfile) fig = plt.gcf() # get current graphics window fig.clf() # clear gskw = dict(width_ratios=[30, 1]) fig, ax = plt.subplots(ncols=2, nrows=1, gridspec_kw=gskw) fig.set_figheight(figheight) fig.set_figwidth(figwidth) ax[0].imshow(avgim, origin='lower', aspect='auto', interpolation='none', extent=(xr[0], xr[1], yr[0], yr[1]), vmin=vmin, vmax=vmax, cmap='viridis') # viridis, Greys, jet ax[0].plot(np.array([0, 0]), np.array([-0.066 * npix, 0.066 * npix]) * pixscale, c='white', alpha=0.7, zorder=1) ax[0].plot(np.array([-0.066 * npix, 0.066 * npix]) * pixscale, np.array([0, 0]), c='white', alpha=0.7, zorder=1) xmeas = np.array(xmeas) ymeas = np.array(ymeas) ax[0].plot((xmeas - hpix) * pixscale, (ymeas - hpix) * pixscale, c='r') #plt.scatter((xmeas-hpix)*pixscale,(ymeas-hpix)*pixscale,c='r',marker='+',s=30) ax[0].set_xlabel(r'$\Delta$ RA (arcsec)') ax[0].set_ylabel(r'$\Delta$ DEC (arcsec)') ax[0].set_xlim((xr[1], xr[0])) # sky -right ax[0].set_ylim(yr) ax[1].axis('off') plt.savefig(figfile) # Make four copies for j in np.arange(2, 11): #pathfile = figfile.replace('path1','path'+str(j)) pathfile = figfile.replace('%04d' % (i + 2), '%04d' % (i + 1 + j)) if os.path.exists(pathfile): os.remove(pathfile) shutil.copyfile(figfile, pathfile) figfiles.append(pathfile) # Make the animated gif animfile = outdir + str(objid) + '_cutouts.gif' if os.path.exists(animfile): os.remove(animfile) # put list of files in a separate file listfile = outdir + str(objid) + '_cutouts.lst' if os.path.exists(listfile): os.remove(listfile) dln.writelines(listfile, figfiles) delay = dln.scale(nind, [20, 1000], [20, 1]) delay = int(np.round(dln.limit(delay, 1, 20))) print('delay = ' + str(delay)) print('Creating animated gif ' + animfile) #ret = subprocess.run('convert -delay 100 '+figdir+str(objid)+'_*.jpg '+animfile,shell=True) #ret = subprocess.run('convert -delay 20 '+' '.join(figfiles)+' '+animfile,shell=True) ret = subprocess.run('convert @' + listfile + ' -delay ' + str(delay) + ' ' + animfile, shell=True) #import pdb; pdb.set_trace() dln.remove(figfiles)
def modelmag(tab, instfilt, dec, eqnfile): """ This calculates the model magnitudes for the NSC catalog given a catalog with the appropriate information Parameters ---------- tab : table Catalog of sources with appropriate magnitude columns. instfilt : str Short instrument and filter name, e.g. 'c4d-g'. dec : float The declination of the exposure. eqnfile : str File with the model magnitude equations. Returns ------- model_mag : numpy array An [Nsource,3] array with model magnitudes, errors and color. Example ------- model_mag = modelmag(cat,'c4d-g',-50.0,'modelmag_equations.txt') By D. Nidever Feb 2019 Translated to Python by D. Nidever, April 2022 """ # This calculates the model magnitude for stars given the # the magnitudes in reference catalogs # NUV - Galex NUV magnitude # GMAG - Gaia G magnitude # JMAG - 2MASS J magnitude # KMAG - 2MASS Ks magnitude # APASS_GMAG - APASS g magnitue # APASS_RMAG - APASS r magnitude # EBV - E(B-V) reddening ntab = len(tab) colnames = tab.colnames for n in tab.colnames: tab[n].name = n.upper() tabcols = np.char.array(tab.colnames) ## Load the model magnitude equation information ## band, dec range, color equation, color min/max range, quality cuts, model mag equation if os.path.exists(eqnfile) == False: raise ValueError(eqnfile + ' NOT FOUND') eqnstr = Table.read(eqnfile, format='ascii') for c in eqnstr.colnames: eqnstr[c].name = c.lower() neqn = len(eqnstr) ## Get COLOR and DEC ranges eqnstr['colorlim'] = np.zeros((len(eqnstr), 2), float) eqnstr['declim'] = np.zeros((len(eqnstr), 2), float) for i in range(len(eqnstr)): # Color range cr = np.char.array(eqnstr['colorange'][i].split(',')) cr = cr.replace('[', '').replace(']', '') eqnstr['colorlim'][i] = np.array(cr).astype(float) # DEC range dr = np.char.array(eqnstr['decrange'][i].split(',')) dr = dr.replace('[', '').replace(']', '') eqnstr['declim'][i] = np.array(dr).astype(float) ## Get the band for this INSTRUMENT-FILTER and DEC. gd, = np.where((np.char.array(eqnstr['instrument']) + '-' + np.char.array(eqnstr['band']) == instfilt) & (dec >= eqnstr['declim'][:, 0]) & (dec <= eqnstr['declim'][:, 1])) if len(gd) == 0: raise ValueError('No model magnitude equation for INSTRUMENT-FILTER=' + instfilt + ' and DEC=%.2f' % dec) if len(gd) > 1: print('Found multiple magnitude equation for INSTRUMENT-FILTER=' + instfilt + ' and DEC=%.2f. Using the first one' % dec) gd = gd[0] eqnstr1 = eqnstr[gd] eqnstr1 = dict(zip(eqnstr1.colnames, eqnstr1[0])) # convert to dictionary ## No parentheses allowed if eqnstr1['coloreqn'].find('(') != -1 or eqnstr1['coloreqn'].find(')') != -1 or \ eqnstr1['modelmageqn'].find('(') != -1 or eqnstr1['modelmageqn'].find(')') != -1: raise ValueError( 'No parentheses allowed in the model magnitude equations') ## Are we using color? if eqnstr1['colorlim'][0] < -10 and eqnstr1['colorlim'][ 1] > 10 and eqnstr1['modelmageqn'].upper().find('COLOR') == -1: usecolor = False else: usecolor = True ## Get all columns that we need coloreqn = eqnstr1['coloreqn'] qualitycuts = eqnstr1['qualitycuts'] modelmageqn = eqnstr1['modelmageqn'] coloreqn_cols = re.split('[-+*^]', coloreqn) modelmageqn_cols = re.split('[-+*^]', modelmageqn) if usecolor: cols = np.char.array(coloreqn_cols + modelmageqn_cols).upper() else: cols = np.char.array(modelmageqn_cols).upper() ## Remove numbers and "COLOR" isnumeric = np.array([dln.isnumber(c) for c in cols]) bd, = np.where(isnumeric | (cols.upper() == 'COLOR') | (cols == '??')) if len(bd) > 0: if len(bd) < len(cols): cols = np.delete(cols, bd) else: cols = None ncols = len(cols) ## No columns left if ncols == 0: raise ValueError('No columns to use.') ## Only unique columns cols = np.unique(cols) ncols = len(cols) ind1, ind2 = dln.match(tabcols, cols) ntagmatch = len(ind1) if ntagmatch < ncols: leftind = np.arange(ncols) if ntagmatch > 0: leftind = np.delete(leftind, ind2) print('Needed columns missing. ' + ' '.join(cols[leftind])) ## Make the color ## replace the columns by TAB[GD].COLUMN if usecolor: coloreqn_cols = np.char.array(re.split('[-+*^]', coloreqn)).upper() coloreqn_cols = np.unique(coloreqn_cols) # unique ones bd, = np.where(np.array([dln.isnumber(c) for c in coloreqn_cols])) ## Remove numbers if len(bd) > 0: coloreqn_cols = np.delete(coloreqn_cols, bd) colcmd = coloreqn.upper() for i in range(len(coloreqn_cols)): colcmd = colcmd.replace(coloreqn_cols[i], "tab['" + coloreqn_cols[i] + "']") color = eval(colcmd) color = np.array(color) else: color = np.zeros(ntab, float) ## Make quality cuts magcolsind, = np.where((dln.find(cols, 'MAG$') > -1) & (cols.find('^E_') == -1) | (cols == 'NUV')) ## make sure all magnitudes are good (<50) and finite goodmask = np.ones(ntab, bool) for i in range(len(magcolsind)): magind, = np.where(tabcols.upper() == cols[magcolsind[i]].upper()) if len(magind) == 0: print(cols[magcolsind[i]].upper() + ' column NOT found') return [] goodmask &= ((tab[tabcols[magind[0]]] < 50) & (tab[tabcols[magind[0]]] > 0) & np.isfinite(tab[tabcols[magind[0]]])) ## input quality cuts ## replace <=, <, >, >=, =, &, | qualitycuts = qualitycuts.replace('<=', ' <= ') qualitycuts = qualitycuts.replace('>=', ' >= ') qualitycuts = qualitycuts.replace('>', ' > ') qualitycuts = qualitycuts.replace('<', ' < ') qualitycuts = qualitycuts.replace('=', ' == ') qualitycuts = qualitycuts.replace('&', ' & ') qualitycuts = qualitycuts.replace('|', ' | ') ## fix column names qualitycuts_cols = qualitycuts.split() for i in range(len(qualitycuts_cols)): col = qualitycuts_cols[i] colind, = np.where(tabcols == col.upper()) if len(colind) > 0: qualitycuts = qualitycuts.replace( tabcols[colind[0]], "tab['" + tabcols[colind[0]] + "']") goodmask &= eval(qualitycuts) ## Apply the color range if usecolor: goodmask &= ((color >= eqnstr1['colorlim'][0]) & (color <= eqnstr1['colorlim'][1])) ## Get the sources that pass all cuts gd, = np.where(goodmask == True) if len(gd) == 0: print('No good sources left') return [] # Make the model magnitude ## replace the columns by TAB[GD].COLUMN modelmageqn_cols = np.char.array(re.split('[-+*^]', modelmageqn)).upper() bd, = np.where( np.array([dln.isnumber(c) for c in modelmageqn_cols]) | (modelmageqn_cols.upper() == 'COLOR')) ## Remove numbers and "COLOR" if len(bd) > 0: modelmageqn_cols = np.delete(modelmageqn_cols, bd) modelmageqn_cols = np.unique(modelmageqn_cols) # unique ones magcmd = modelmageqn.upper() for i in range(len(modelmageqn_cols)): magcmd = magcmd.replace(modelmageqn_cols[i], "tab['" + modelmageqn_cols[i] + "'][gd]") magcmd = magcmd.replace('COLOR', 'COLOR[gd]') modelmag_gd = eval(magcmd) modelmag = np.zeros(ntab, float) + 99.99 modelmag[gd] = modelmag_gd ## Make the error structure ## Each magnitude has an E_MAG error except for PS and Gaia GMAG ## If we are using PS or GMAG then add the errors for the adderrcols = [] psmagind, = np.where((dln.find(cols, '^PS_') > -1) & (dln.find(cols, 'MAG$') > -1)) if len(psmagind) > 0: adderrcols += list('E_' + cols[psmagind]) nadderrcols = len(adderrcols) ## Making error structure errcolind = np.where(dln.find(tabcols, '^E_') > -1) errcols = tabcols[errcolind] errdt = [] for i in range(len(errcols)): errdt += [(errcols[i], float)] if nadderrcols > 0: for i in range(nadderrcols): errdt += [(adderrcols[i], float)] err = np.zeros(ntab, dtype=np.dtype(errdt)) err = Table(err) for c in err.colnames: err[c] = 0.001 for n in err.colnames: if c in tab.colnames: err[n] = tab[n] ## leave the PS errors at 0.001 ## convert NAN or 99.99 to 9.99 to be consistent for c in err.colnames: bd = np.where((err[c] > 10.0) | (np.isfinite(err[c]) == False)) if len(bd) > 0: err[c][bd] = 9.99 ## Calculate the color errors ## get the columns if usecolor: colorerr_cols = np.char.array(re.split('[-+*^]', coloreqn)).upper() colorerr_cols = np.unique(colorerr_cols) # unique ones bd, = np.where( np.array([dln.isnumber(c) for c in colorerr_cols]) | (colorerr_cols.upper() == 'EBV')) ## Remove numbers and "EBV" if len(bd) > 0: colorerr_cols = np.delete(colorerr_cols, bd) ## use - and + signs to break apart the components that need to be squared coloreqn_terms = np.char.array(re.split('[-+]', coloreqn)).upper() ## remove any terms that don't have a COLORERR_COLS in them okay = np.zeros(len(coloreqn_terms), bool) for i in range(len(coloreqn_terms)): for j in range(len(colorerr_cols)): okay[i] |= (coloreqn_terms[i].find(colorerr_cols[j]) > -1) bd, = np.where(okay == False) if len(bd) > 0: coloreqn_terms = np.delete(coloreqn_terms, bd) ## Now create the equation, add in quadrature colorerrcmd = 'np.sqrt( ' + '+'.join('(' + coloreqn_terms.upper() + ')**2') + ' )' #colorerrcmd = colorerrcmd.upper() for i in range(len(colorerr_cols)): colorerrcmd = colorerrcmd.replace( colorerr_cols[i], "err['E_" + colorerr_cols[i] + "'][gd]") colorerr_gd = eval(colorerrcmd) colorerr = np.zeros(ntab, float) + 9.99 colorerr[gd] = colorerr_gd else: colorerr = np.zeros(ntab, float) ## The modelmag errors ## get the columns modelmagerr_cols = np.char.array(re.split('[-+*^]', modelmageqn)).upper() modelmagerr_cols = np.unique(modelmagerr_cols) # unique ones bd, = np.where( np.array([dln.isnumber(c) for c in modelmagerr_cols]) | (modelmagerr_cols.upper() == 'EBV')) ## Remove numbers and "EBV" if len(bd) > 0: modelmagerr_cols = np.delete(modelmagerr_cols, bd) ## use - and + signs to break apart the components that need to be squared modelmageqn_terms = np.char.array(re.split('[-+]', modelmageqn)).upper() ## remove any terms that don't have a COLORERR_COLS in them okay = np.zeros(len(modelmageqn_terms), bool) for i in range(len(modelmageqn_terms)): for j in range(len(modelmagerr_cols)): okay[i] |= (modelmageqn_terms[i].find(modelmagerr_cols[j]) > -1) bd, = np.where(okay == False) if len(bd) > 0: modelmageqn_terms = np.delete(modelmageqn_terms, bd) ## Now create the equation, add in quadrature modelmagerrcmd = 'np.sqrt( ' + '+'.join('(' + modelmageqn_terms + ')**2') + ' )' for i in range(len(modelmageqn_cols)): modelmagerrcmd = modelmagerrcmd.replace( modelmageqn_cols[i], "err['E_" + modelmageqn_cols[i].upper() + "'][gd]") modelmagerrcmd = modelmagerrcmd.replace('COLOR', 'COLORERR[gd]') modelmagerr_gd = eval(modelmagerrcmd) modelmagerr = np.zeros(ntab, float) + 99.90 modelmagerr[gd] = modelmagerr_gd ## combine modelmag and modelmagerr ## Combine mags and errors mags = np.zeros((len(tab), 3), float) mags[:, 0] = modelmag mags[:, 1] = modelmagerr mags[:, 2] = color # Change back to original column names for n in colnames: tab[n.upper()].name = n return mags
def measurement_update(expdir): t0 = time.time() hostname = socket.gethostname() host = hostname.split('.')[0] # Get version number from exposure directory lo = expdir.find('nsc/instcal/') dum = expdir[lo + 12:] version = dum[0:dum.find('/')] cmbdir = '/net/dl2/dnidever/nsc/instcal/' + version + '/' edir = '/net/dl1/users/dnidever/nsc/instcal/' + version + '/' nside = 128 # Check if output file already exists base = os.path.basename(expdir) # Log file #------------------ # format is nsc_combine_main.DATETIME.log ltime = time.localtime() # time.struct_time(tm_year=2019, tm_mon=7, tm_mday=22, tm_hour=0, tm_min=30, tm_sec=20, tm_wday=0, tm_yday=203, tm_isdst=1) smonth = str(ltime[1]) if ltime[1] < 10: smonth = '0' + smonth sday = str(ltime[2]) if ltime[2] < 10: sday = '0' + sday syear = str(ltime[0])[2:] shour = str(ltime[3]) if ltime[3] < 10: shour = '0' + shour sminute = str(ltime[4]) if ltime[4] < 10: sminute = '0' + sminute ssecond = str(int(ltime[5])) if ltime[5] < 10: ssecond = '0' + ssecond logtime = smonth + sday + syear + shour + sminute + ssecond logfile = expdir + '/' + base + '_measure_update.' + logtime + '.log' if os.path.exists(logfile): os.remove(logfile) # Set up logging to screen and logfile logFormatter = logging.Formatter( "%(asctime)s [%(levelname)-5.5s] %(message)s") rootLogger = logging.getLogger() fileHandler = logging.FileHandler(logfile) fileHandler.setFormatter(logFormatter) rootLogger.addHandler(fileHandler) consoleHandler = logging.StreamHandler() consoleHandler.setFormatter(logFormatter) rootLogger.addHandler(consoleHandler) rootLogger.setLevel(logging.NOTSET) rootLogger.info( 'Adding objectID for measurement catalogs for exposure = ' + base) rootLogger.info("expdir = " + expdir) rootLogger.info("host = " + host) rootLogger.info(" ") # Load the exposure and metadata files metafile = expdir + '/' + base + '_meta.fits' meta = Table.read(metafile, 1) nmeta = len(meta) chstr = Table.read(metafile, 2) rootLogger.info('KLUDGE!!! Changing /dl1 filenames to /dl2 filenames') cols = ['EXPDIR', 'FILENAME', 'MEASFILE'] for c in cols: f = np.char.array(chstr[c]).decode() f = np.char.array(f).replace('/dl1/users/dnidever/', '/dl2/dnidever/') chstr[c] = f nchips = len(chstr) measdtype = np.dtype([('MEASID', 'S50'), ('OBJECTID', 'S50'), ('EXPOSURE', 'S50'), ('CCDNUM', '>i2'), ('FILTER', 'S2'), ('MJD', '>f8'), ('X', '>f4'), ('Y', '>f4'), ('RA', '>f8'), ('RAERR', '>f4'), ('DEC', '>f8'), ('DECERR', '>f4'), ('MAG_AUTO', '>f4'), ('MAGERR_AUTO', '>f4'), ('MAG_APER1', '>f4'), ('MAGERR_APER1', '>f4'), ('MAG_APER2', '>f4'), ('MAGERR_APER2', '>f4'), ('MAG_APER4', '>f4'), ('MAGERR_APER4', '>f4'), ('MAG_APER8', '>f4'), ('MAGERR_APER8', '>f4'), ('KRON_RADIUS', '>f4'), ('ASEMI', '>f4'), ('ASEMIERR', '>f4'), ('BSEMI', '>f4'), ('BSEMIERR', '>f4'), ('THETA', '>f4'), ('THETAERR', '>f4'), ('FWHM', '>f4'), ('FLAGS', '>i2'), ('CLASS_STAR', '>f4')]) # Load and concatenate the meas catalogs chstr['MEAS_INDEX'] = 0 # keep track of where each chip catalog starts count = 0 meas = Table(data=np.zeros(int(np.sum(chstr['NMEAS'])), dtype=measdtype)) rootLogger.info('Loading and concatenating the chip measurement catalogs') for i in range(nchips): meas1 = Table.read(chstr['MEASFILE'][i].strip(), 1) # load chip meas catalog nmeas1 = len(meas1) meas[count:count + nmeas1] = meas1 chstr['MEAS_INDEX'][i] = count count += nmeas1 measid = np.char.array(meas['MEASID']).strip().decode() nmeas = len(meas) rootLogger.info(str(nmeas) + ' measurements') # Get the OBJECTID from the combined healpix file IDSTR structure # remove any sources that weren't used # Figure out which healpix this figure overlaps pix = hp.ang2pix(nside, meas['RA'], meas['DEC'], lonlat=True) upix = np.unique(pix) npix = len(upix) rootLogger.info(str(npix) + ' HEALPix to query') # Loop over the HEALPix pixels ntotmatch = 0 idstr_dtype = np.dtype([('measid', np.str, 200), ('objectid', np.str, 200), ('pix', int)]) idstr = np.zeros(nmeas, dtype=idstr_dtype) cnt = 0 for i in range(npix): fitsfile = cmbdir + 'combine/' + str(int(upix[i]) // 1000) + '/' + str( upix[i]) + '.fits.gz' dbfile = cmbdir + 'combine/' + str(int(upix[i]) // 1000) + '/' + str( upix[i]) + '_idstr.db' if os.path.exists(dbfile): # Read meas id information from idstr database for this expoure idstr1 = readidstrdb(dbfile, where="exposure=='" + base + "'") nidstr1 = len(idstr1) if nidstr1 > 0: idstr['measid'][cnt:cnt + nidstr1] = idstr1['measid'] idstr['objectid'][cnt:cnt + nidstr1] = idstr1['objectid'] idstr['pix'][cnt:cnt + nidstr1] = upix[i] cnt += nidstr1 rootLogger.info( str(i + 1) + ' ' + str(upix[i]) + ' ' + str(nidstr1)) else: rootLogger.info( str(i + 1) + ' ' + dbfile + ' NOT FOUND. Checking for high-resolution database files.') # Check if there are high-resolution healpix idstr databases hidbfiles = glob(cmbdir + 'combine/' + str(int(upix[i]) // 1000) + '/' + str(upix[i]) + '_n*_*_idstr.db') nhidbfiles = len(hidbfiles) if os.path.exists(fitsfile) & (nhidbfiles > 0): rootLogger.info('Found high-resolution HEALPix IDSTR files') for j in range(nhidbfiles): dbfile1 = hidbfiles[j] dbbase1 = os.path.basename(dbfile1) idstr1 = readidstrdb(dbfile1, where="exposure=='" + base + "'") nidstr1 = len(idstr1) if nidstr1 > 0: idstr['measid'][cnt:cnt + nidstr1] = idstr1['measid'] idstr['objectid'][cnt:cnt + nidstr1] = idstr1['objectid'] idstr['pix'][cnt:cnt + nidstr1] = upix[i] cnt += nidstr1 rootLogger.info(' ' + str(j + 1) + ' ' + dbbase1 + ' ' + str(upix[i]) + ' ' + str(nidstr1)) # Trim any leftover elements of IDSTR if cnt < nmeas: idstr = idstr[0:cnt] # Now match them all up rootLogger.info('Matching the measurements') idstr_measid = np.char.array(idstr['measid']).strip() idstr_objectid = np.char.array(idstr['objectid']).strip() ind1, ind2 = dln.match(idstr_measid, measid) nmatch = len(ind1) if nmatch > 0: meas['OBJECTID'][ind2] = idstr_objectid[ind1] # Only keep sources with an objectid ind, nind = dln.where( np.char.array(meas['OBJECTID']).strip().decode() == '') # There can be missing/orphaned measurements at healpix boundaries in crowded # regions when the DBSCAN eps is different. But there should be very few of these. # At this point, let's allow this to pass if nind > 0: rootLogger.info('WARNING: ' + str(nind) + ' measurements are missing OBJECTIDs') if ((nmeas >= 20000) & (nind > 20)) | ((nmeas < 20000) & (nind > 3)): rootLogger.info('More missing OBJECTIDs than currently allowed.') raise ValueError('More missing OBJECTIDs than currently allowed.') # Output the updated catalogs #rootLogger.info('Updating measurement catalogs') #for i in range(nchips): # measfile1 = chstr['MEASFILE'][i].strip() # lo = chstr['MEAS_INDEX'][i] # hi = lo+chstr['NMEAS'][i] # meas1 = meas[lo:hi] # meta1 = Table.read(measfile1,2) # load the meta extensions # # 'KLUDGE!!! Changing /dl1 filenames to /dl2 filenames') # cols = ['EXPDIR','FILENAME','MEASFILE'] # for c in cols: # f = np.char.array(meta1[c]).decode() # f = np.char.array(f).replace('/dl1/users/dnidever/','/dl2/dnidever/') # meta1[c] = f # # Copy as a backup # if os.path.exists(measfile1+'.bak'): os.remove(measfile1+'.bak') # dum = shutil.move(measfile1,measfile1+'.bak') # # Write new catalog # #meas1.write(measfile1,overwrite=True) # first, measurement table # # append other fits binary tabl # #hdulist = fits.open(measfile1) # rootLogger.info('Writing '+measfile1) # hdulist = fits.HDUList() # hdulist.append(fits.table_to_hdu(meas1)) # first, meas catalog # hdulist.append(fits.table_to_hdu(meta1)) # second, meta # hdulist.writeto(measfile1,overwrite=True) # hdulist.close() # # Create a file saying that the file was successfully updated. # dln.writelines(measfile1+'.updated','') # # Delete backups # if os.path.exists(measfile1+'.bak'): os.remove(measfile1+'.bak') # Output the updated measurement catalog # Writing a single FITS file is much faster than many small ones measfile = expdir + '/' + base + '_meas.fits' meas.write(measfile, overwrite=True) if os.path.exists(measfile + '.gz'): os.remove(measfile + '.gz') ret = subprocess.call(['gzip', measfile]) # compress final catalog # Update the meta file as well, need to the /dl2 filenames rootLogger.info('Updating meta file') meta.write(metafile, overwrite=True) hdulist = fits.open(metafile) hdu = fits.table_to_hdu(chstr) hdulist.append(hdu) hdulist.writeto(metafile, overwrite=True) hdulist.close() # Create a file saying that the files were updated okay. dln.writelines(expdir + '/' + base + '_meas.updated', '') rootLogger.info('dt = ' + str(time.time() - t0) + ' sec.')
def meascutout(meas, obj, size=10, outdir='./'): """ Input the measurements and create cutouts. """ expstr = fits.getdata( '/net/dl2/dnidever/nsc/instcal/v3/lists/nsc_v3_exposures.fits.gz', 1) #expstr = fits.getdata('/net/dl2/dnidever/nsc/instcal/v3/lists/nsc_v3_exposure.fits.gz',1) decam = Table.read('/home/dnidever/projects/delvered/data/decam.txt', format='ascii') objid = obj['id'][0] # Sort by MJD si = np.argsort(meas['mjd']) meas = meas[si] ind1, ind2 = dln.match(expstr['base'], meas['exposure']) nind = len(ind1) if nind == 0: print('No matches') return # Sort by input meas catalog si = np.argsort(ind2) ind1 = ind1[si] ind2 = ind2[si] # Create the reference WCS wref = WCS(naxis=2) pixscale = 0.26 # DECam, "/pix npix = round(size / pixscale) if npix % 2 == 0: # must be odd npix += 1 hpix = npix // 2 # center of image wref.wcs.ctype = ['RA---TAN', 'DEC--TAN'] wref.wcs.crval = [obj['ra'][0], obj['dec'][0]] wref.wcs.crpix = [npix // 2, npix // 2] wref.wcs.cd = np.array([[pixscale / 3600.0, 0.0], [0.0, pixscale / 3600]]) wref.array_shape = (npix, npix) refheader = wref.to_header() refheader['NAXIS'] = 2 refheader['NAXIS1'] = npix refheader['NAXIS2'] = npix # Load the data instrument = expstr['instrument'][ind1] fluxfile = expstr['file'][ind1] fluxfile = fluxfile.replace('/net/mss1/', '/mss1/') # for thing/hulk maskfile = expstr['maskfile'][ind1] maskfile = maskfile.replace('/net/mss1/', '/mss1/') # for thing/hulk ccdnum = meas['ccdnum'][ind2] figfiles = [] for i in range(nind): try: if instrument[i] == 'c4d': dind, = np.where(decam['CCDNUM'] == ccdnum[i]) extname = decam['NAME'][dind[0]] im, head = getfitsext(fluxfile[i], extname, header=True) mim, mhead = getfitsext(maskfile[i], extname, header=True) #im,head = fits.getdata(fluxfile[i],header=True,extname=extname) #mim,mhead = fits.getdata(maskfile[i],header=True,extname=extname) else: im, head = fits.getdata(fluxfile[i], ccdnum[i], header=True) mim, mhead = fits.getdata(maskfile[i], ccdnum[i], header=True) except: print('error') import pdb pdb.set_trace() # Get chip-level information exposure = os.path.basename(fluxfile[i])[0:-8] # remove fits.fz chres = qc.query(sql="select * from nsc_dr2.chip where exposure='" + exposure + "' and ccdnum=" + str(ccdnum[i]), fmt='table') w = WCS(head) # RA/DEC correction for the object lon = obj['ra'][0] - chres['ra'][0] lat = obj['dec'][0] - chres['dec'][0] racorr = chres['ra_coef1'][0] + chres['ra_coef2'][0] * lon + chres[ 'ra_coef3'][0] * lon * lat + chres['ra_coef4'][0] * lat deccorr = chres['dec_coef1'][0] + chres['dec_coef2'][0] * lon + chres[ 'dec_coef3'][0] * lon * lat + chres['dec_coef4'][0] * lat # apply these offsets to the header WCS CRVAL w.wcs.crval += [racorr, deccorr] head['CRVAL1'] += racorr head['CRVAL2'] += deccorr # Object X/Y position xobj, yobj = w.all_world2pix(obj['ra'], obj['dec'], 0) # Get the cutout xcen = meas['x'][ind2[i]] - 1 # convert to 0-indexes ycen = meas['y'][ind2[i]] - 1 smim = dln.gsmooth(im, 2) # use the object coords for centering #cutim,xr,yr = cutout(smim,xobj,yobj,size) # Mask the bad pixels badmask = (mim > 0) im[badmask] = np.nanmedian(im[~badmask]) # Create a common TAN WCS that each image gets interpoled onto!!! #hdu1 = fits.open(fluxfile[i],extname=extname) smim1 = dln.gsmooth(im, 1.5) hdu = fits.PrimaryHDU(smim1, head) cutim, footprint = reproject_interp(hdu, refheader, order='bicubic') # biquadratic cutim[footprint == 0] = np.nanmedian( im[~badmask]) # set out-of-bounds to background #xr = [0,npix-1] #yr = [0,npix-1] xr = [-hpix * pixscale, hpix * pixscale] yr = [-hpix * pixscale, hpix * pixscale] # exposure_ccdnum, filter, MJD, delta_MJD, mag print( str(i + 1) + ' ' + meas['exposure'][ind2[i]] + ' ' + str(ccdnum[i]) + ' ' + str(meas['x'][ind2[i]]) + ' ' + str(meas['y'][ind2[i]]) + ' ' + str(meas['mag_auto'][ind2[i]])) #figdir = '/net/dl2/dnidever/nsc/instcal/v3/hpm2/cutouts/' figfile = outdir figfile += '%s_%04d_%s_%02d.jpg' % (str( obj['id'][0]), i + 1, meas['exposure'][ind2[i]], ccdnum[i]) figfiles.append(figfile) matplotlib.use('Agg') plt.rcParams.update({'font.size': 11}) if os.path.exists(figfile): os.remove(figfile) fig = plt.gcf() # get current graphics window fig.clf() # clear figsize = 8.0 #6.0 ax = fig.subplots() # projection=wcs #fig.set_figheight(figsize*0.8) fig.set_figheight(figsize) fig.set_figwidth(figsize) med = np.nanmedian(smim) sig = dln.mad(smim) bigim, xr2, yr2 = cutout(smim, xcen, ycen, 151, missing=med) lmed = np.nanmedian(bigim) # Get the flux of the object and scale each image to the same height #meas.mag_aper1 = cat1.mag_aper[0] + 2.5*alog10(exptime) + chstr[i].zpterm #cmag = mag_auto + 2.5*alog10(exptime) + zpterm instmag = meas['mag_auto'][ind2[i]] - 2.5 * np.log10( chres['exptime'][0]) - chres['zpterm'][0] #mag = -2.5*log(flux)+25.0 instflux = 10**((25.0 - instmag) / 2.5) print('flux = ' + str(instflux)) # Get height of object # flux of 2D Gaussian is ~2*pi*height*sigma^2 pixscale1 = np.max(np.abs(w.wcs.cd)) * 3600 fwhm = chres['fwhm'][0] / pixscale1 instheight = instflux / (2 * 3.14 * (fwhm / 2.35)**2) print('height = ' + str(instheight)) # Scale the images to the flux level of the first image cutim -= lmed if i == 0: instflux0 = instflux.copy() instheight0 = instheight.copy() else: scale = instflux0 / instflux #scale = instheight0/instheight cutim *= scale print('scaling image by ' + str(scale)) #vmin = lmed-8*sig # 3*sig #vmax = lmed+12*sig # 5*sig if i == 0: vmin = -8 * sig # 3*sig #vmax = 12*sig # 5*sig vmax = 0.5 * instheight # 0.5 vmin0 = vmin vmax0 = vmax else: vmin = vmin0 vmax = vmax0 print('vmin = ' + str(vmin)) print('vmax = ' + str(vmax)) plt.imshow(cutim, origin='lower', aspect='auto', interpolation='none', extent=(xr[0], xr[1], yr[0], yr[1]), vmin=vmin, vmax=vmax, cmap='viridis') # viridis, Greys, jet #plt.imshow(cutim,origin='lower',aspect='auto',interpolation='none', # vmin=vmin,vmax=vmax,cmap='viridis') # viridis, Greys, jet #plt.colorbar() # show one vertical, one horizontal line pointing to the center but offset # then a small dot on the meas position # 13, 8 plt.plot(np.array([0, 0]), np.array([-0.066 * npix, 0.066 * npix]) * pixscale, c='white', alpha=0.7) plt.plot(np.array([-0.066 * npix, 0.066 * npix]) * pixscale, np.array([0, 0]), c='white', alpha=0.7) # Meas X/Y position xmeas, ymeas = wref.all_world2pix(meas['ra'][ind2[i]], meas['dec'][ind2[i]], 0) plt.scatter([(xmeas - hpix) * pixscale], [(ymeas - hpix) * pixscale], c='r', marker='+', s=20) #plt.scatter([xmeas],[ymeas],c='r',marker='+',s=100) #plt.scatter([xcen],[ycen],c='r',marker='+',s=100) # Object X/Y position #xobj,yobj = w.all_world2pix(obj['ra'],obj['dec'],0) xobj, yobj = wref.all_world2pix(obj['ra'], obj['dec'], 0) #plt.scatter(xobj,yobj,marker='o',s=200,facecolors='none',edgecolors='y',linewidth=3) #plt.scatter(xobj,yobj,c='y',marker='+',s=100) #leg = ax.legend(loc='upper left', frameon=False) plt.xlabel(r'$\Delta$ RA (arcsec)') plt.ylabel(r'$\Delta$ DEC (arcsec)') #plt.xlabel('X') #plt.ylabel('Y') #plt.xlim(xr) #plt.ylim(yr) #ax.annotate(r'S/N=%5.1f',xy=(np.mean(xr), yr[0]+dln.valrange(yr)*0.05),ha='center') co = 'white' #'lightgray' # blue ax.annotate('%s %02d %s %6.1f ' % (meas['exposure'][ind2[i]], ccdnum[i], meas['filter'][ind2[i]], expstr['exptime'][ind1[i]]), xy=(np.mean(xr), yr[0] + dln.valrange(yr) * 0.05), ha='center', color=co) ax.annotate( '%10.2f %10.2f ' % (meas['mjd'][ind2[i]], meas['mjd'][ind2[i]] - np.min(meas['mjd'])), xy=(xr[0] + dln.valrange(xr) * 0.05, yr[1] - dln.valrange(yr) * 0.05), ha='left', color=co) ax.annotate('%s = %5.2f +/- %4.2f' % (meas['filter'][ind2[i]], meas['mag_auto'][ind2[i]], meas['magerr_auto'][ind2[i]]), xy=(xr[1] - dln.valrange(xr) * 0.05, yr[1] - dln.valrange(yr) * 0.05), ha='right', color=co) plt.savefig(figfile) print('Cutout written to ' + figfile) #import pdb; pdb.set_trace() # Make a single blank file at the end so you know it looped figfile = outdir figfile += '%s_%04d_%s.jpg' % (str(obj['id'][0]), i + 2, 'blank') figfiles.append(figfile) matplotlib.use('Agg') if os.path.exists(figfile): os.remove(figfile) fig = plt.gcf() # get current graphics window fig.clf() # clear figsize = 8.0 #6.0 fig.set_figheight(figsize) fig.set_figwidth(figsize) plt.savefig(figfile) print(figfile) # Make the animated gif animfile = outdir + str(objid) + '_cutouts.gif' print('Creating animated gif ' + animfile) if os.path.exists(animfile): os.remove(animfile) #ret = subprocess.run('convert -delay 100 '+figdir+str(objid)+'_*.jpg '+animfile,shell=True) ret = subprocess.run('convert -delay 20 ' + ' '.join(figfiles) + ' ' + animfile, shell=True) #import pdb; pdb.set_trace() dln.remove(figfiles)
if len(totrim) > 0: # Trim objects idstr = np.delete(idstr, totrim) #idstr = utils.remove_indices(idstr,totrim) # Update IDSTR.objectindex old_idstr_objectindex = idstr['objectindex'] idstr['objectindex'] = newobjindex[old_idstr_objectindex] # Create final summary structure from ALLMETA # get exposures that are in IDSTR # sometimes EXPNUM numbers have the leading 0s removed # and sometimes not, so turn to LONG to match dum, uiexpnum = np.unique(idstr['expnum'].astype(int), return_index=True) uexpnum = idstr[uiexpnum]['expnum'].astype(int) nuexpnum = len(uexpnum) ind1, ind2 = utils.match(allmeta['expnum'].astype(int), uexpnum) nmatch = len(ind1) sumstr = Table(allmeta[ind1]) col_nobj = Column(name='nobjects', dtype=np.int, length=len(sumstr)) col_healpix = Column(name='healpix', dtype=np.int, length=len(sumstr)) sumstr.add_columns([col_nobj, col_healpix]) sumstr['nobjects'] = 0 sumstr['healpix'] = pix # get number of objects per exposure expnum = idstr['expnum'].astype(int) siexp = np.argsort(expnum) expnum = expnum[siexp] if nuexpnum > 1: brklo, = np.where(expnum != np.roll(expnum, 1)) nbrk = len(brklo) brkhi = np.hstack((brklo[1:nbrk], len(expnum)))
def fix_pms(objectid): """ Correct the proper motions in the healpix object catalog.""" t00 = time.time() hostname = socket.gethostname() host = hostname.split('.')[0] version = 'v3' radeg = np.float64(180.00) / np.pi meas = qc.query(sql="select * from nsc_dr2.meas where objectid='"+objectid+"'",fmt='table') nmeas = len(meas) print(' '+str(nmeas)) mnra = np.median(meas['ra'].data) mndec = np.median(meas['dec'].data) lim = 20.0 # 50.0 gd, = np.where( (np.abs(meas['ra'].data-mnra)/np.cos(np.deg2rad(mndec))*3600 < lim) & (np.abs(meas['dec'].data-mndec)*3600 < lim)) ngd = len(gd) nbd = nmeas-ngd print('bad measurements '+str(nbd)) #if nbd==0: # return None meas = meas[gd] # Make cut on FWHM # maybe only use values for 0.5*fwhm_chip to 1.5*fwhm_chip sql = "select chip.* from nsc_dr2.chip as chip join nsc_dr2.meas as meas on chip.exposure=meas.exposure and chip.ccdnum=meas.ccdnum" sql += " where meas.objectid='"+objectid+"'" chip = qc.query(sql=sql,fmt='table') ind3,ind4 = dln.match(chip['exposure'],meas['exposure']) si = np.argsort(ind4) # sort by input meas catalog ind3 = ind3[si] ind4 = ind4[si] chip = chip[ind3] meas = meas[ind4] gdfwhm, = np.where((meas['fwhm'] > 0.2*chip['fwhm']) & (meas['fwhm'] < 2.0*chip['fwhm'])) if len(gdfwhm)==0: print('All measurements have bad FWHM values') return if len(gdfwhm) < len(meas): print('Removing '+str(len(meas)-len(gdfwhm))+' measurements with bad FWHM values') meas = meas[gdfwhm] raerr = np.array(meas['raerr']*1e3,np.float64) # milli arcsec ra = np.array(meas['ra'],np.float64) ra -= np.mean(ra) ra *= 3600*1e3 * np.cos(mndec/radeg) # convert to true angle, milli arcsec t = np.array(meas['mjd'].copy()) t -= np.mean(t) t /= 365.2425 # convert to year # Calculate robust slope try: #pmra, pmraerr = dln.robust_slope(t,ra,raerr,reweight=True) # LADfit pmra_ladcoef, absdev = dln.ladfit(t,ra) pmra_lad = pmra_ladcoef[1] # Run RANSAC ransac = linear_model.RANSACRegressor() ransac.fit(t.reshape(-1,1), ra) inlier_mask = ransac.inlier_mask_ outlier_mask = np.logical_not(inlier_mask) gdmask = inlier_mask pmra_ransac = ransac.estimator_.coef_[0] print(' ransac '+str(np.sum(inlier_mask))+' inliers '+str(np.sum(outlier_mask))+' outliers') # Robust, weighted linear with with INLIERS #pmra_coef, pmra_coeferr = dln.poly_fit(t[gdmask],ra[gdmask],1,sigma=raerr[gdmask],robust=True,error=True) #pmra_coef, pmra_coeferr = dln.poly_fit(t,ra,1,sigma=raerr,robust=True,error=True) #pmra = pmra_coef[0] #pmraerr = pmra_coeferr[0] #radiff = ra-dln.poly(t,pmra_coef) radiff = ra-t*pmra_lad radiff -= np.median(radiff) rasig = dln.mad(radiff) # Reject outliers gdsig = (np.abs(radiff) < 2.5*rasig) | (np.abs(radiff) < 2.5*raerr) print(' '+str(nmeas-np.sum(gdsig))+' 2.5*sigma clip outliers rejected') #if np.sum(gdsig) < nmeas: pmra_coef, pmra_coeferr = dln.poly_fit(t[gdsig],ra[gdsig],1,sigma=raerr[gdsig],robust=True,error=True) pmra = pmra_coef[0] pmraerr = pmra_coeferr[0] rasig = dln.mad(ra-dln.poly(t,pmra_coef)) except: print('problem') #import pdb; pdb.set_trace() return np.append(np.zeros(10,float)+np.nan, np.zeros(2,int)) decerr = np.array(meas['decerr']*1e3,np.float64) # milli arcsec dec = np.array(meas['dec'],np.float64) dec -= np.mean(dec) dec *= 3600*1e3 # convert to milli arcsec # Calculate robust slope try: #pmdec, pmdecerr = dln.robust_slope(t,dec,decerr,reweight=True) # LADfit pmdec_ladcoef, absdev = dln.ladfit(t,dec) pmdec_lad = pmdec_ladcoef[1] # Run RANSAC ransac = linear_model.RANSACRegressor() ransac.fit(t.reshape(-1,1), dec) inlier_mask = ransac.inlier_mask_ outlier_mask = np.logical_not(inlier_mask) gdmask = inlier_mask pmdec_ransac = ransac.estimator_.coef_[0] print(' ransac '+str(np.sum(inlier_mask))+' inliers '+str(np.sum(outlier_mask))+' outliers') # Robust, weighted linear with with INLIERS #pmdec_coef, pmdec_coeferr = dln.poly_fit(t[gdmask],dec[gdmask],1,sigma=decerr[gdmask],robust=True,error=True) #pmdec_coef, pmdec_coeferr = dln.poly_fit(t,dec,1,sigma=decerr,robust=True,error=True) #pmdec = pmdec_coef[0] #pmdecerr = pmdec_coeferr[0] #decdiff = dec-dln.poly(t,pmdec_coef) decdiff = dec-t*pmdec_lad decdiff -= np.median(decdiff) decsig = dln.mad(decdiff) # Reject outliers gdsig = (np.abs(decdiff) < 2.5*decsig) | (np.abs(decdiff) < 2.5*decerr) print(' '+str(nmeas-np.sum(gdsig))+' 2.5*sigma clip outliers rejected') #if np.sum(gdsig) < nmeas: pmdec_coef, pmdec_coeferr = dln.poly_fit(t[gdsig],dec[gdsig],1,sigma=decerr[gdsig],robust=True,error=True) pmdec = pmdec_coef[0] pmdecerr = pmdec_coeferr[0] decsig = dln.mad(dec-dln.poly(t,pmdec_coef)) except: print('problem') #import pdb; pdb.set_trace() return np.append(np.zeros(10,float)+np.nan, np.zeros(2,int)) deltamjd = np.max(meas['mjd'])-np.min(meas['mjd']) out = np.array([pmra,pmraerr,pmra_ransac,pmra_lad,rasig,pmdec,pmdecerr,pmdec_ransac,pmdec_lad,decsig,nmeas,deltamjd]) #print(out[[0,2,3]]) #print(out[[5,7,8]]) #import pdb; pdb.set_trace() return out
def getpsf(psf,image,cat,fitradius=None,lookup=False,lorder=0,method='qr',subnei=False, allcat=None,maxiter=10,minpercdiff=1.0,reject=False,maxrejiter=3,verbose=False): """ Fit PSF model to stars in an image with outlier rejection of badly-fit stars. Parameters ---------- psf : PSF object PSF object with initial parameters to use. image : CCDData object Image to use to fit PSF model to stars. cat : table Catalog with initial amp/x/y values for the stars to use to fit the PSF. fitradius : float, table The fitting radius. If none is input then the initial PSF FWHM will be used. lookup : boolean, optional Use an empirical lookup table. Default is False. lorder : int, optional The order of the spatial variations (0=constant, 1=linear). Default is 0. method : str, optional Method to use for solving the non-linear least squares problem: "qr", "svd", "cholesky", and "curve_fit". Default is "qr". subnei : boolean, optional Subtract stars neighboring the PSF stars. Default is False. allcat : table, optional Catalog of all objects in the image. This is needed for bad PSF star rejection. maxiter : int, optional Maximum number of iterations to allow. Only for methods "qr", "svd", and "cholesky". Default is 10. minpercdiff : float, optional Minimum percent change in the parameters to allow until the solution is considered converged and the iteration loop is stopped. Only for methods "qr" and "svd". Default is 1.0. reject : boolean, optional Reject PSF stars with high RMS values. Default is False. maxrejiter : int, boolean Maximum number of PSF star rejection iterations. Default is 3. verbose : boolean, optional Verbose output. Returns ------- newpsf : PSF object New PSF object with the best-fit model parameters. pars : numpy array Array of best-fit model parameters perror : numpy array Uncertainties in "pars". psfcat : table Table of best-fitting amp/xcen/ycen values for the PSF stars. Example ------- newpsf,pars,perror,psfcat = getpsf(psf,image,cat) """ t0 = time.time() print = utils.getprintfunc() # Get print function to be used locally, allows for easy logging # Fitting radius if fitradius is None: if type(psf)==models.PSFPenny: fitradius = psf.fwhm()*1.5 else: fitradius = psf.fwhm() # subnei but no allcat input if subnei and allcat is None: raise ValueError('allcat is needed for PSF neighbor star subtraction') if 'id' not in cat.colnames: cat['id'] = np.arange(len(cat))+1 psfcat = cat.copy() # Initializing output PSF star catalog dt = np.dtype([('id',int),('amp',float),('x',float),('y',float),('npix',int),('rms',float), ('chisq',float),('ixmin',int),('ixmax',int),('iymin',int),('iymax',int),('reject',int)]) outcat = np.zeros(len(cat),dtype=dt) outcat = Table(outcat) for n in ['id','x','y']: outcat[n] = cat[n] # Remove stars that are too close to the edge ny,nx = image.shape bd = (psfcat['x']<fitradius) | (psfcat['x']>(nx-1-fitradius)) | \ (psfcat['y']<fitradius) | (psfcat['y']>(ny-1-fitradius)) nbd = np.sum(bd) if nbd > 0: if verbose: print('Removing '+str(nbd)+' stars near the edge') psfcat = psfcat[~bd] # Generate an empirical image of the stars # and fit a model to it to get initial estimates if type(psf)!=models.PSFEmpirical: cube = starcube(psfcat,image,npix=psf.npix,fillvalue=np.nan) epsf,nbadstar,rms = mkempirical(cube,order=0) epsfim = CCDData(epsf,error=epsf.copy()*0+1,mask=~np.isfinite(epsf)) pars,perror,mparams = psf.fit(epsfim,pars=[1.0,psf.npix/2,psf.npix//2],allpars=True) initpar = mparams.copy() curpsf = psf.copy() curpsf.params = initpar if verbose: print('Initial estimate from empirical PSF fit = '+str(mparams)) else: curpsf = psf.copy() initpar = psf.params.copy() # Outlier rejection iterations nrejiter = 0 flag = 0 nrejstar = 100 fitrad = fitradius useimage = image.copy() while (flag==0): if verbose: print('--- Iteration '+str(nrejiter+1)+' ---') # Update the fitting radius if nrejiter>0: fitrad = curpsf.fwhm() if verbose: print(' Fitting radius = %5.3f' % (fitrad)) # Reject outliers if reject and nrejiter>0: medrms = np.median(pcat['rms']) sigrms = dln.mad(pcat['rms'].data) gd, = np.where(pcat['rms'] < medrms+3*sigrms) nrejstar = len(psfcat)-len(gd) if verbose: print(' RMS = %6.4f +/- %6.4f' % (medrms,sigrms)) print(' Threshold RMS = '+str(medrms+3*sigrms)) print(' Rejecting '+str(nrejstar)+' stars') if nrejstar>0: psfcat = psfcat[gd] # Subtract neighbors if nrejiter>0 and subnei: if verbose: print('Subtracting neighbors') # Find the neighbors in allcat # Fit the neighbors and PSF stars # Subtract neighbors from the image useimage = image.copy() # start with original image useimage = subtractnei(useimage,allcat,cat,curpsf) # Fitting the PSF to the stars #----------------------------- newpsf,pars,perror,pcat,pf = fitpsf(curpsf,useimage,psfcat,fitradius=fitrad,method=method, maxiter=maxiter,minpercdiff=minpercdiff,verbose=verbose) # Add information into the output catalog ind1,ind2 = dln.match(outcat['id'],pcat['id']) outcat['reject'] = 1 for n in pcat.columns: outcat[n][ind1] = pcat[n][ind2] outcat['reject'][ind1] = 0 # Compare PSF parameters if type(newpsf)!=models.PSFEmpirical: pardiff = newpsf.params-curpsf.params else: pardiff = newpsf._data-curpsf._data sumpardiff = np.sum(np.abs(pardiff)) curpsf = newpsf.copy() # Stopping criteria if reject is False or sumpardiff<0.05 or nrejiter>=maxrejiter or nrejstar==0: flag=1 if subnei is True and nrejiter==0: flag=0 # iterate at least once with neighbor subtraction nrejiter += 1 # Generate an empirical look-up table of corrections if lookup: if verbose: print('Making empirical lookup table with order='+str(lorder)) pf.mklookup(lorder) # Fit the stars again and get new RMS values xdata = np.arange(pf.ntotpix) out = pf.model(xdata,*pf.psf.params) newpsf = pf.psf.copy() # Update information in the output catalog ind1,ind2 = dln.match(outcat['id'],pcat['id']) outcat['reject'] = 1 outcat['reject'][ind1] = 0 outcat['amp'][ind1] = pf.staramp[ind2] outcat['x'][ind1] = pf.starxcen[ind2] outcat['y'][ind1] = pf.starycen[ind2] outcat['rms'][ind1] = pf.starrms[ind2] outcat['chisq'][ind1] = pf.starchisq[ind2] if verbose: print('Median RMS: '+str(np.median(pf.starrms))) if verbose: print('dt = %.2f sec' % (time.time()-t0)) return newpsf, pars, perror, outcat
# Restore the calibration summary file temp = fits.getdata(basedir+'lists/nsc_instcal_calibrate.fits',1) schema = dict(temp.dtype.fields) schema['chipindx'] = (int,0) schema['ngoodchipwcs'] = (int,0) schema['wcscal'] = (np.str,50) schema['telstat'] = (np.str,50) dt = np.dtype(schema) calstr = np.zeros(len(temp),dtype=dt) calstr['chipindx'] = -1 for n in temp.dtype.names: calstr[n]=temp[n] # Add WCSCAL and TELSTAT information coords = fits.getdata(basedir+'lists/allcoords.fits',1) fluxfile = calstr['file'] fluxfile = fluxfile.replace('/net','') ind1,ind2 = dln.match(fluxfile,coords['file']) calstr['wcscal'][ind1] = coords['wcscal'][ind2] # Failed (3153), Poor (14), Successful (308190) calstr['telstat'][ind1] = coords['telstat'][ind2] # NAN (68188), Not (1222), Track (241826), UNKNOWN (116), Unknown (5) # the 2054 failed exposures did not match b/c no fluxfile info # Only want exposures with successful SE processing gd,ncalstr = dln.where(calstr['success']==1) calstr = calstr[gd] si = np.argsort(calstr['expdir']) calstr = calstr[si] chstr = fits.getdata(basedir+'lists/nsc_instcal_calibrate.fits',2) nchstr = len(chstr) # Get indices for CHSTR siexp = np.argsort(chstr['expdir']) chstr = chstr[siexp] expdir = chstr['expdir'] brklo,nbrk = dln.where(expdir != np.roll(expdir,1))
def fix_pms(pix): """ Correct the proper motions in the healpix object catalog.""" t00 = time.time() hostname = socket.gethostname() host = hostname.split('.')[0] version = 'v3' nside = 128 radeg = np.float64(180.00) / np.pi hdir = '/net/dl2/dnidever/nsc/instcal/'+version+'/combine/'+str(int(pix)//1000)+'/' objfile = hdir+str(pix)+'.fits.gz' outfile = hdir+str(pix)+'_pmcorr.fits' print('Correcting proper motions for '+str(pix)) # Check that the object file exists if os.path.exists(objfile) is False: print(objfile+' NOT FOUND') return # Check fixed file if os.path.exists(outfile+'.gz') == True: print(str(pix)+' already fixed') return # Load the object file #meta = fits.getdata(objfile,1) #obj = fits.getdata(objfile,2) meta = Table.read(objfile,1) obj = Table.read(objfile,2) nobj = len(obj) print(str(nobj)+' objects with '+str(np.sum(obj['ndet']))+' measurements') #print('KLUDGE!!! MAKING COPY OF OBJ!!!') #orig = obj.copy() #v = psutil.virtual_memory() #process = psutil.Process(os.getpid()) #print('%6.1f Percent of memory used. %6.1f GB available. Process is using %6.2f GB of memory.' % (v.percent,v.available/1e9,process.memory_info()[0]/1e9)) # Break up into subregions totmeas = np.sum(obj['ndet']) nsub,bestind = dln.closest([1,4,16,64],int(np.ceil(totmeas/500000))) hinside = [128,256,512,1024][bestind] vecbound = hp.boundaries(nside,int(pix)) allpix = hp.query_polygon(hinside,np.transpose(vecbound)) allra,alldec = hp.pix2ang(hinside,allpix,lonlat=True) print(str(nsub)+' sub regions') # Get the objects within this subpixel objpix = hp.ang2pix(hinside,obj['ra'],obj['dec'],lonlat=True) ndet = np.zeros(nobj,int) #allpmra_old = np.zeros(nobj,float) #allpmdec_old = np.zeros(nobj,float) #allpmra_linefit = np.zeros(nobj,float) # Loop over subpixels for i in range(nsub): pix1 = allpix[i] print(str(i+1)+' '+str(pix1)) # Get the measurements meas = get_meas(pix1,nside=hinside) nmeas = len(meas) if nmeas==0: print('No measurements in this subregion') continue #v = psutil.virtual_memory() #process = psutil.Process(os.getpid()) #print('%6.1f Percent of memory used. %6.1f GB available. Process is using %6.2f GB of memory.' % (v.percent,v.available/1e9,process.memory_info()[0]/1e9)) # Get the objects within this subpixel objind, = np.where(objpix==pix1) obj1 = obj[objind] nobj1 = len(obj1) print(' '+str(nobj1)+' objects in this subregion') idindex = dln.create_index(meas['objectid']) ## Not all matched #if len(idindex['value']) != nobj: # print('Number of unique OBJECTIDs in object and meas catalogs do not match') # return ind1,ind2 = dln.match(obj1['objectid'],idindex['value']) # Not all matched if len(ind1) != nobj1: print(str(len(obj1))+' objects in this sub healpix but only measurements for '+str(len(ind1))) #print('Some objects are missing measurements') #return # Ensure they are arrays ind1 = np.atleast_1d(ind1) ind2 = np.atleast_1d(ind2) # sort by object index si = np.argsort(ind1) if len(ind1)>1: ind1 = ind1[si] ind2 = ind2[si] # Loop over ndet1 = np.zeros(nobj1,int) #allpmra_old1 = np.zeros(nobj1,float) #allpmdec_old1 = np.zeros(nobj1,float) #allpmra_linefit1 = np.zeros(nobj1,float) for j in range(len(ind1)): if (j % 1000)==0: print(' '+str(j)) k = ind1[j] # object index # Calculate the proper motions mind = idindex['index'][idindex['lo'][ind2[j]]:idindex['hi'][ind2[j]]+1] cat1 = meas[mind] ncat1 = len(cat1) ndet1[k] = ncat1 if ncat1>1: raerr = np.array(cat1['raerr']*1e3,np.float64) # milli arcsec ra = np.array(cat1['ra'],np.float64) ra -= np.mean(ra) ra *= 3600*1e3 * np.cos(obj1['dec'][k]/radeg) # convert to true angle, milli arcsec t = cat1['mjd'].copy() t -= np.mean(t) t /= 365.2425 # convert to year # Calculate robust slope try: pmra, pmraerr = dln.robust_slope(t,ra,raerr,reweight=True) #pmra_old, pmraerr_old = dln.robust_slope_old(t,ra,raerr,reweight=True) #pmra_linefit = dln.poly_fit(t,ra,2,robust=True,sigma=raerr,initpar=pmra) except: print('problem') import pdb; pdb.set_trace() obj1['pmra'][k] = pmra # mas/yr obj1['pmraerr'][k] = pmraerr # mas/yr #allpmra_old1[k] = pmra_old #allpmra_linefit1[k] = pmra_linefit decerr = np.array(cat1['decerr']*1e3,np.float64) # milli arcsec dec = np.array(cat1['dec'],np.float64) dec -= np.mean(dec) dec *= 3600*1e3 # convert to milli arcsec # Calculate robust slope try: pmdec, pmdecerr = dln.robust_slope(t,dec,decerr,reweight=True) #pmdec_old, pmdecerr_old = dln.robust_slope_old(t,dec,decerr,reweight=True) except: print('problem') import pdb; pdb.set_trace() obj1['pmdec'][k] = pmdec # mas/yr obj1['pmdecerr'][k] = pmdecerr # mas/yr #allpmdec_old1[k] = pmdec_old # Stuff subregion object back into big one obj[objind] = obj1 ndet[objind] = ndet1 #allpmra_old[objind] = allpmra_old1 #allpmdec_old[objind] = allpmdec_old1 #allpmra_linefit[objind] = allpmra_linefit1 #import pdb; pdb.set_trace() #np.save(hdir+str(pix)+'_pmraold.npy',allpmra_old) #np.save(hdir+str(pix)+'_pmdecold.npy',allpmdec_old) #np.save(hdir+str(pix)+'_pmralinefit.npy',allpmra_linefit) #import pdb; pdb.set_trace() # Save the new version of obj # Write the output file print('Writing combined catalog to '+outfile) if os.path.exists(outfile): os.remove(outfile) #Table(meta).write(outfile) # first, summary table meta.write(outfile) # first, summary table # append other fits binary tables hdulist = fits.open(outfile) #hdu = fits.table_to_hdu(Table(obj)) # second, catalog hdu = fits.table_to_hdu(obj) # second, catalog hdulist.append(hdu) hdulist.writeto(outfile,overwrite=True) hdulist.close() if os.path.exists(outfile+'.gz'): os.remove(outfile+'.gz') ret = subprocess.call(['gzip',outfile]) # compress final catalog print('dt = %6.1f sec.' % (time.time()-t00))
def apercorr(psf,image,objects,psfobj,verbose=False): """ Calculate aperture correction. Parameters ---------- psf : PSF object The best-fitting PSF model. image : string or CCDData object The input image to fit. This can be the filename or CCDData object. objects : table The output table of best-fit PSF values for all of the sources. psfobj : table The table of PSF objects. verbose : boolean, optional Verbose output to the screen. Default is False. Returns ------- objects : table The output table with an "apcorr" column inserted and the aperture correction applied to "psfmag". apcor : float The aperture correction in mag. cgrow : numpy array The cumulative aperture correction array. Example ------- apcor = apercorr(psf,image,objects,psfobj) """ # Get model of all stars except the PSF stars ind1,ind2 = dln.match(objects['id'],psfobj['id']) left = np.delete(np.arange(len(objects)),ind1) neiobj = objects[left] neimodel = image.copy() neimodel.data *= 0 neimodel.error[:] = 1 neimodelim = psf.add(neimodel,neiobj) neimodel.data = neimodelim # Subtract everything except the PSF stars from the image resid = image.copy() if image.mask is not None: resid.data[~resid.mask] -= neimodel.data[~resid.mask] else: resid.data -= modelnei.data residim = np.maximum(resid.data-resid.sky,0) resid.data = residim resid.sky[:] = 0.0 # Do aperture photometry with lots of apertures on the PSF # stars # rk = (20/3.)**(1/11.) * rk-1 for k=2,..,12 rseeing = psf.fwhm()*0.5 apers = np.cumprod(np.hstack((3.0,np.ones(11,float)*(20/3.)**(1/11.)))) #apers = np.array([3.0,3.7965,4.8046,6.0803,7.6947,9.7377,12.3232, # 15.5952,19.7360,24.9762,31.6077,40.0000]) apercat = aperphot(resid,psfobj,apers) # Fit curve of growth # use magnitude differences between successive apertures. apars, agrow, derr = fitgrowth(apercat,apers,rseeing=psf.fwhm()*0.5) # Get magnitude difference errors nstars = len(apercat) napers = len(apers) derr = np.zeros((nstars,napers-1),float) for i in range(len(apers)-1): err1 = apercat['magerr_aper'+str(i+1)] err2 = apercat['magerr_aper'+str(i+2)] derr[:,i] = np.sqrt(err1**2+err2**2) wt = 1/derr**2 # THE CURVE TURNS OVER AT LARGE RADIUS!!!!??? # It shouldn't EVER do that. # Calculate empirical growth curve egrow,egrowerr = empgrowth(apercat,apers) # Get "adopted" growth curve by taking the weighted average # of the analytical and empirical growth curves # with the empirical weighted higher at small r and # the analytical weighted higher at large r gwt = np.mean(wt,axis=0) # mean weights over the stars adopgrow = (egrow*gwt + agrow*(1/(0.1*agrow))**2) / (gwt+(1/(0.1*agrow))**2) adopgrowerr = 1 / (gwt+(1/(0.1*agrow))**2) # Adopted cumulative growth curve # sum from the outside in, with an outer tail given by # extrapolation of the analytic model to 2*outer aperture cadopgrow = np.cumsum(adopgrow[::-1])[::-1] # add extrapolation from rlast t=o2*rlast tail = diffprofile([2*apers[-1],apers[-1]],*apars) cadopgrow += tail cadopgrow = np.hstack((cadopgrow,tail)) # add value for outer aperture cadopgrowerr = np.hstack((adopgrowerr,0.0)) # Calculate "total" magnitude for the PSF stars totmag,toterr = totphot(apercat,apers,cadopgrow,cadopgrowerr) # Calculate median offset between total and PSF magnitude # psf - total ind1,ind2 = dln.match(objects['id'],psfobj['id']) diffmag = objects['psfmag'][ind1] - totmag[ind2] apcor = np.median(diffmag) # positive value # Apply aperture correction to the data # add apcorr column and keep initial mags in instmag objects['apcorr'] = apcor objects['inst_psfmag'] = objects['psfmag'] objects['psfmag'] -= apcor # make brighter if verbose: print('Aperture correction = %.3f mag' % apcor) return objects, apcor, cadopgrow
def get_missingids(exposure): """ Get the number of missing IDs from the log files.""" t00 = time.time() hostname = socket.gethostname() host = hostname.split('.')[0] iddir = '/data0/dnidever/nsc/instcal/v3/idstr/' version = 'v3' # Load the exposures table print('Loading exposure table') expcat = fits.getdata( '/net/dl2/dnidever/nsc/instcal/' + version + '/lists/nsc_v3_exposure_table.fits.gz', 1) # Make sure it's a list if type(exposure) is str: exposure = [exposure] # Match exposures to exposure catalog eind1, eind2 = dln.match(expcat['EXPOSURE'], exposure) nexp = len(exposure) outstr = np.zeros( nexp, np.dtype([('exposure', (np.str, 100)), ('mtime', np.float64), ('nmeas', int), ('nids', int), ('nmatches', int), ('nduplicates', int), ('nmissing', int)])) #outstr['exposure'] = exposure outstr['mtime'] = -1 outstr['nmeas'] = -1 outstr['nmatches'] = -1 outstr['nids'] = -1 outstr['nduplicates'] = -1 outstr['nmissing'] = -1 # Loop over files for i in range(nexp): t0 = time.time() exp = expcat['EXPOSURE'][eind1[i]] print(str(i + 1) + ' ' + exp) outstr['exposure'][i] = exp instcode = expcat['INSTRUMENT'][eind1[i]] dateobs = expcat['DATEOBS'][eind1[i]] night = dateobs[0:4] + dateobs[5:7] + dateobs[8:10] expdir = '/net/dl2/dnidever/nsc/instcal/' + version + '/' + instcode + '/' + night + '/' + exp logfile = glob(expdir + '/' + exp + '_measure_update.????????????.log') nlogfile = len(logfile) # No logfile if nlogfile == 0: print('No logfile') continue # more than 1 logfile, get the latest one if nlogfile > 1: mtime = [os.path.getmtime(f) for f in logfile] si = np.argsort(mtime)[::-1] logfile = logfile[si[0]] else: logfile = logfile[0] outstr['mtime'][i] = os.path.getmtime(logfile) # Read in logfile lines = dln.readlines(logfile) # Number of measurements measind = dln.grep(lines, 'measurements', index=True) if len(measind) > 0: line1 = lines[measind[0]] lo = line1.find(']') hi = line1.find('measurements') nmeas = int(line1[lo + 3:hi - 1]) outstr['nmeas'][i] = nmeas else: nmeas = -1 # Number in idcat, "IDs for XX measurements" idsind = dln.grep(lines, 'IDs for', index=True) if len(idsind) > 0: line1 = lines[idsind[0]] lo = line1.find('for') hi = line1.find('measurements') nids = int(line1[lo + 4:hi - 1]) outstr['nids'][i] = nids else: nids = -1 # Number of matches matchind = dln.grep(lines, 'Matches for', index=True) if len(matchind) > 0: line1 = lines[matchind[0]] lo = line1.find('for') hi = line1.find('measurements') nmatches = int(line1[lo + 4:hi - 1]) outstr['nmatches'][i] = nmatches else: nmatches = -1 # Check for duplicates #if (len(measind)>0) & (len(matchind)>0): # ndup = np.maximum(nmatches-nmeas,0) if (len(measind) > 0) & (len(idsind) > 0): ndup = np.maximum(nids - nmeas, 0) outstr['nduplicates'][i] = ndup else: ndup = -1 # Number of missing IDs badind = dln.grep(lines, 'WARNING:', index=True) nbadind = len(badind) # Some missing objectids if nbadind > 0: badline = lines[badind[0]] lo = badline.find('WARNING:') hi = badline.find('measurements') nmissing = int(badline[lo + 9:hi - 1]) outstr['nmissing'][i] = nmissing else: nmissing = 0 outstr['nmissing'][i] = nmissing print(' Nmeas=' + str(nmeas) + ' Nids=' + str(nids) + ' Nmatches=' + str(nmatches) + ' Nduplicates=' + str(ndup) + ' Nmissing=' + str(nmissing)) return outstr