def translate_idl_mjd5_script(scriptfile): """ Translate an IDL MJD5.pro script file to yaml. It returns a list of strings that can be written to a file. Parameters ---------- scriptfile : str Name of MJD5.pro script file. Returns ------- flines : numpy char array The lines of the script file translated to yaml. Examples -------- flines = mkplan.translate_idl_mjd5_script('apo25m_59085.pro') Example file, top part of apo25m_59085.pro apsetver,telescope='apo25m' mjd=59085 plate=11950 psfid=35230030 fluxid=35230030 ims=[35230018,35230019,35230020,35230021,35230022,35230023,35230024,35230025,35230026,35230027,35230028,35230029] mkplan,ims,plate,mjd,psfid,fluxid,vers=vers ;these are not sky frames plate = 12767 psfid=35230015 fluxid=35230015 ims=[35230011,35230012,35230013,35230014] mkplan,ims,plate,mjd,psfid,fluxid,vers=vers;,/sky plate=12673 psfid=35230037 fluxid=35230037 ims=[35230033,35230034,35230035,35230036] mkplan,ims,plate,mjd,psfid,fluxid,vers=vers By D.Nidever, Oct 2020 """ # Check that the file exists if os.path.exists(scriptfile) == False: raise ValueError(scriptfile + " NOT FOUND") # Load the file lines = dln.readlines(scriptfile) lines = np.char.array(lines) # Fix continuation lines lines = fixidlcontinuation(lines) # Remove comments lines = removeidlcomments(lines) # Get telescope from apserver line ind, nind = dln.where(lines.strip().lower().find('apsetver') == 0) telescope = None if nind == 0: print('No APSERVER line found') if scriptfile.lower().find('apo25m') > -1: telescope = 'apo25m' if scriptfile.lower().find('lco25m') > -1: telescope = 'lcoo25m' if telescope is None: raise ValueError('Cannot find TELESCOPE') else: setverline = lines[ind[0]] telescope = setverline[setverline.lower().find('telescope=') + 10:] telescope = telescope.replace("'", "") telescopeline = "telescope: " + telescope # Get MJD ind, nind = dln.where(lines.strip().lower().find('mjd=') == 0) if nind == 0: raise ValueError('No MJD line found') mjdline = lines[ind[0]] mjd = int(mjdline[mjdline.find('=') + 1:]) mjdline = 'mjd: ' + str(mjd) # Get day number ind, nind = dln.where(lines.lower().find('day=') > -1) if nind > 0: dayline = lines[ind[0]].lower() # day=getnum(mjd)*10000 if dayline.lower().find('getnum') > -1: dayline = dayline.replace('getnum(mjd)', '(mjd-55562)') day = int(eval(dayline[dayline.find('=') + 1:])) else: day = None # Remove apvers, mjd and day line gd, ngd = dln.where((lines.strip('').lower().startswith('day=') == False) & (lines.strip('').lower().find('apsetver') == -1) & (lines.strip('').lower().startswith('mjd=') == False)) lines = lines[gd] # Deal with IDL code using day, indgen(), etc. lines = replaceidlcode(lines, mjd, day=day) # Initalize final lines flines = ['---'] # start of yaml file # Loop over mkplan blocks # mkplan command is at the end of the block ind, nind = dln.where(lines.lower().find('mkplan') != -1) for i in range(nind): if i == 0: lo = 0 else: lo = ind[i - 1] + 1 lines1 = lines[lo:ind[i] + 1] nlines1 = len(lines1) # Add TELESCOPE line flines.append("- " + telescopeline) # Add MJD line flines.append(" " + mjdline) # Assume all lines in this block except for mkplan are key: value pairs kvlines = lines1[0:-1] for kvl in kvlines: if kvl.strip() != '': lo = kvl.find('=') key = kvl[0:lo].strip() val = kvl[lo + 1:].strip() flines.append(" " + key + ": " + val) # Deal with mkplan lines planline = lines1[-1] # Trim off the first bit that's always the same, "mkplan,ims,plate,mjd,psfid,fluxid," planline = planline[planline.lower().find('fluxid') + 7:] # Remove vers=vers if it's there if planline.lower().find('vers=vers') == 0: planline = planline[9:] # Deal with keywords if planline != '': if planline[0] == ',': planline = planline[1:] # Add lines for sky, dark, cal if planline.lower().find('/sky') > -1: flines.append(' sky: True') planline = removeidlkeyword(planline, '/sky') # Trim off /sky if planline.lower().find('/dark') > -1: flines.append(' dark: True') planline = removeidlkeyword(planline, '/dark') # Trim off /dark if planline.lower().find('/cal') > -1: flines.append(' cal: True') planline = removeidlkeyword(planline, '/cal') # Trim off /cal # Deal with remaining arguments if planline != '': # Return leftover line as a dictionary import pdb pdb.set_trace() exec("args=args2dict(" + planline + ")") # Loop over keys and add them for k in args.keys(): val = args[k] if (type(val) is int) | (type(val) is str): flines.append(" " + k + ": " + str(val)) else: flines.append(" " + k + ": " + str(list(val))) # End of yaml file flines.append('...') return flines
#hlist = dbutils.query(listfile,'hlist',cols='FILE') #nlist = len(hlist) #healstr = fits.getdata(listfile,1) #expdir = [os.path.dirname(f) for f in healstr['FILE']] #expdir = np.unique(expdir) liststr = fits.getdata(listfile,1) expdir = liststr['EXPDIR'] # change /dl1/users/dnidever/ to /dl2/dnidever/ expdir = expdir.replace('/dl1/users/dnidever/','/dl2/dnidever/') # Trim trailing / #expdir = expdir.rstrip('/') # this doesn't work for some reason expdir = np.char.array([a.rstrip('/') for a in expdir]) nexpdir = len(expdir) else: rootLogger.info("Reading list from "+inplistfile) expdir = dln.readlines(inplistfile) nexpdir = len(expdir) # Create the commands allexpdir = expdir.copy() allcmd = dln.strjoin("/home/dnidever/projects/noaosourcecatalog/python/nsc_instcal_measure_update.py ",allexpdir) alldirs = np.zeros(nexpdir,(np.str,200)) alldirs[:] = tmpdir nallcmd = len(allcmd) # Check what's been done already check = False #if not redo: if check: rootLogger.info("Checking if any have already been done") exists = np.zeros(dln.size(allexpdir),bool)+False
# pars, cov = parallax.fit(meas) # parallax.plotfit(meas,pars, cov) #except: # print("This field threw an error ", idv) # Main command-line program if __name__ == "__main__": parser = ArgumentParser(description='Make diagnostic HPM plots') parser.add_argument('objid', type=str, nargs='*', help='Object IDs') args = parser.parse_args() objid = args.objid nobj = dln.size(objid) if nobj == 1: if os.path.exists(objid[0]): listfile = objid[0] print('Reading ' + listfile) objid = dln.readlines(listfile) nobj = dln.size(objid) if type(objid) is not list: objid = [objid] for i in range(nobj): print(str(i + 1) + ' ' + objid[i]) try: make_4panelplot(objid[i]) except Exception as e: print('Failed on ' + objid[i] + ' ' + str(e)) traceback.print_exc()
print(' Writing missing catalog to ' + missfile) Table(meas[miss]).write(missfile, overwrite=True) else: print(str(i + 1) + ' ' + exp1 + ' none missing') else: print(measfile + ' NOT FOUND') if __name__ == "__main__": parser = ArgumentParser( description='Combine NSC data for one healpix region.') parser.add_argument('exposure', type=str, nargs=1, help='exposure') args = parser.parse_args() version = 'v3' hostname = socket.gethostname() host = hostname.split('.')[0] exposure = args.exposure[0] # Input is a list if exposure[0] == '@': listfile = exposure[1:] if os.path.exists(listfile): exposure = dln.readlines(listfile) else: print(listfile + ' NOT FOUND') sys.exit() make_meas_missing(exposure)
def getdata(refcat, minra, redo=False, silent=False, logger=None): """ Get reference catalog information from DL database Parameters ---------- cenra : float Central RA for the search. cendec : float Central DEC for the search. radius : float Search radius in degrees. refcat : table Reference catalog name (e.g. 2MASS, Gaia, etc.) version : str Version of NSC reduction.. saveref : bool, optional Save the output to SAVEFILE or a default filename. Default is False. savefile : str, optional The file to save to or search for existing catalog. silent : bool, optional Don't print anything to the screen. logger : logging object, optional Logging object to use for printing messages. Returns ------- ref : astropy table Search results from the reference catalog. Example ------- cat = getrefcat(cenra,cendec,radius,refcat,file=file,saveref=saveref) By D. Nidever Sep 2017 Translated to Python by D. Nidever, April 2022 """ count = 0 t0 = time.time() if logger is None: logger = dln.basiclogger() # Check that we have psql installed out = subprocess.check_output(['which', 'psql'], shell=False) if type(out) is bytes: out = out.decode() out = out.strip() if dln.size(out) > 1: out = out[0] if os.path.exists(out) == 0: raise ValueError('No PSQL found on this sytem.') # Temporary directory # /tmp is often small and get get fille dup dldir, mssdir, localdir = utils.rootdirs() # FLIP THIS AROUND, INPUT SHOULD BE THE "EASY" VERSION!!! refname = str(refcat).upper() if refname == 'II/312/AIS': refname = 'GALEX' elif refname == '2MASS-PSC': refname = 'TMASS' elif refname == '2MASS': refname = 'TMASS' elif refname == 'GAIA/GAIA': refname = 'GAIA' elif refname == 'Skymapper': refname = 'SKYMAPPER' elif refname == 'GLIMPSE': catname = 'II/293/glimpse' elif refname == 'SAGE': catname = 'II/305/archive' elif refname == 'ATLASREFCAT2': refname = 'ATLAS' ra0 = float(minra) ra1 = minra + 1.0 outdir = '/net/dl1/users/dnidever/nsc/refcatalogs/' + refname + '/' if os.path.exists(outdir) == False: os.makedirs(outdir) savefile = outdir + 'ref_%.6f_%.6f_%s.fits' % (ra0, ra1, refname) if os.path.exists(os.path.abspath(os.path.dirname(savefile))) == False: os.makedirs(os.path.abspath(os.path.dirname(savefile))) if silent == False: logger.info('Querying %s: %.6f <= RA < %.6f' % (refname, ra0, ra1)) # Loading previously loaded file if os.path.exists(savefile) and redo == False: logger.info(savefile + ' already exists and redo==False') return None # Do the Query #-------------- else: # Use DataLab database search #---------------------------- if refname in [ 'TMASS', 'GAIA', 'GAIADR2', 'GAIAEDR3', 'PS', 'SKYMAPPER', 'SKYMAPPERDR2', 'ALLWISE', 'ATLAS' ]: if refname == 'TMASS': tablename = 'twomass.psc' cols = 'designation,ra as raj2000,dec as dej2000,j_m as jmag,j_cmsig as e_jmag,h_m as hmag,h_cmsig as e_hmag,k_m as kmag,k_cmsig as e_kmag,ph_qual as qflg' ##server = 'gp04.datalab.noao.edu' #server = 'gp01.datalab.noirlab.edu' ##server = 'dldb1.sdm.noao.edu' server = 'db02.datalab.noirlab.edu' user = '******' racol = 'ra' deccol = 'dec' if refname == 'GAIA': tablename = 'gaia_dr1.gaia_source' cols = 'source_id as source,ra as ra_icrs,ra_error as e_ra_icrs,dec as de_icrs,dec_error as e_de_icrs,' cols += 'phot_g_mean_flux as fg,phot_g_mean_flux_error as e_fg,phot_g_mean_mag as gmag' #server = 'gp04.datalab.noirlab.edu' ##server = 'gp01.datalab.noao.edu' ##server = 'dldb1.sdm.noao.edu' server = 'db02.datalab.noirlab.edu' user = '******' if refname == 'GAIADR2': tablename = 'gaia_dr2.gaia_source' cols = 'source_id as source,ra,ra_error,dec,dec_error,pmra,pmra_error,pmdec,pmdec_error,phot_g_mean_flux as fg,phot_g_mean_flux_error as e_fg,' cols += 'phot_g_mean_mag as gmag,phot_bp_mean_mag as bp,phot_bp_mean_flux as fbp,phot_bp_mean_flux_error as e_fbp,' cols += 'phot_rp_mean_mag as rp,phot_rp_mean_flux as frp,phot_rp_mean_flux_error as e_frp' #server = 'gp04.datalab.noirlab.edu' ##server = 'gp01.datalab.noao.edu' server = 'db02.datalab.noirlab.edu' user = '******' if refname == 'GAIAEDR3': tablename = 'gaia_edr3.gaia_source' cols = 'source_id as source,ra,ra_error,dec,dec_error,pmra,pmra_error,pmdec,pmdec_error,phot_g_mean_flux as fg,phot_g_mean_flux_error as e_fg,' cols += 'phot_g_mean_mag as gmag,phot_bp_mean_mag as bp,phot_bp_mean_flux as fbp,phot_bp_mean_flux_error as e_fbp,' cols += 'phot_rp_mean_mag as rp,phot_rp_mean_flux as frp,phot_rp_mean_flux_error as e_frp' #server = 'gp04.datalab.noirlab.edu' ##server = 'gp01.datalab.noao.edu' server = 'db02.datalab.noirlab.edu' user = '******' if refname == 'PS': #tablename = 'cp_calib.ps1' tablename = 'public.ps1' cols = 'ra, dec, g as gmag, r as rmag, i as imag, z as zmag, y as ymag' ##server = 'gp02.datalab.noirlab.edu' #server = 'gp01.datalab.noirlab.edu' server = 'db02.datalab.noirlab.edu' user = '******' if refname == 'SKYMAPPER': tablename = 'skymapper_dr1.master' cols = 'raj2000 as ra, dej2000 as dec, u_psf as sm_umag, e_u_psf as e_sm_umag, g_psf as sm_gmag, e_g_psf as e_sm_gmag, r_psf as sm_rmag,' cols += 'e_r_psf as e_sm_rmag, i_psf as sm_imag,e_i_psf as e_sm_imag, z_psf as sm_zmag, e_z_psf as e_sm_zmag' #server = 'gp04.datalab.noirlab.edu' ##server = 'gp01.datalab.noao.edu' server = 'db02.datalab.noirlab.edu' user = '******' racol = 'raj2000' deccol = 'dej2000' if refname == 'SKYMAPPERDR2': tablename = 'skymapper_dr2.master' cols = 'raj2000 as ra, dej2000 as dec, u_psf as sm_umag, e_u_psf as e_sm_umag, g_psf as sm_gmag, e_g_psf as e_sm_gmag, r_psf as sm_rmag,' cols += 'e_r_psf as e_sm_rmag, i_psf as sm_imag,e_i_psf as e_sm_imag, z_psf as sm_zmag, e_z_psf as e_sm_zmag' #server = 'gp04.datalab.noirlab.edu' ##server = 'gp01.datalab.noao.edu' server = 'db02.datalab.noirlab.edu' user = '******' racol = 'raj2000' deccol = 'dej2000' if refname == 'ALLWISE': tablename = 'allwise.source' #cols = 'ra, dec, w1mdef as w1mag, w1sigmdef as e_w1mag, w2mdef as w2mag, w2sigmdef as e_w2mag' cols = 'ra, dec, w1mpro as w1mag, w1sigmpro as e_w1mag, w2mpro as w2mag, w2sigmpro as e_w2mag' #server = 'gp04.datalab.noao.edu' #server = 'gp01.datalab.noirlab.edu' server = 'db02.datalab.noirlab.edu' user = '******' if refname == 'ATLAS': tablename = 'atlasrefcat2' cols = 'objid,ra,dec,plx as parallax,dplx as parallax_error,pmra,dpmra as pmra_error,pmdec,dpmdec as pmdec_error,gaia,dgaia as gaiaerr,' cols += 'bp,dbp as bperr,rp,drp as rperr,teff,agaia,dupvar,ag,rp1,r1,r10,g as gmag,dg as gerr,gchi,gcontrib,' cols += 'r as rmag, dr as rerr,rchi,rcontrib,i as imag,di as ierr,ichi,icontrib,z as zmag,dz as zerr,zchi,zcontrib,nstat,' cols += 'j as jmag,dj as jerr,h as hmag,dh as herr,k as kmag,dk as kerr' server = 'gp10.datalab.noirlab.edu' user = '******' # Use Postgres command with q3c cone search refcattemp = savefile.replace('.fits', '.txt') cmd = "psql -h " + server + " -U " + user + " -d tapdb -w --pset footer -c 'SELECT " + cols + " FROM " + tablename if refname == 'SKYMAPPER' or refname == 'SKYMAPPERDR2': cmd += " WHERE raj2000 >= %.6f and raj2000 < %.6f'" % (ra0, ra1) else: cmd += " WHERE ra >= %.6f and ra < %.6f'" % (ra0, ra1) cmd += " > " + refcattemp dln.remove(refcattemp, allow=True) dln.remove(savefile, allow=True) out = subprocess.check_output(cmd, shell=True) # Check for empty query tlines = dln.readlines(refcattemp, nreadline=4) if len(tlines) < 4: if silent == False: logger.info('No Results') return [] # Load ASCII file and create the FITS file ref = ascii.read(refcattemp, data_start=3, delimiter='|') #ref = importascii(refcattemp,/header,delim='|',skipline=2,/silent) dln.remove(refcattemp, allow=True) # Fix 0.0 mags/errs in ATLAS if refname == 'ATLAS': magcols = [ 'gaia', 'bp', 'rp', 'gmag', 'rmag', 'imag', 'zmag', 'jmag', 'hmag', 'kmag' ] errcols = [ 'gaiaerr', 'bperr', 'rperr', 'gerr', 'rerr', 'ierr', 'zerr', 'jerr', 'herr', 'kerr' ] cols = ref.colnames # Set mags with 0.0 to 99.99 for j in range(len(magcols)): if magcols[j] in ref.colnames: bdmag = (ref[magcols[j]] <= 0.0) if np.sum(bdmag) > 0: ref[magcols[j]][bdmag] = 99.99 # Set errors with 0.0 to 9.99 for j in range(len(errcols)): if errcols[j] in ref.colnames: bderr = (ref[errcols[j]] <= 0.0) if np.sum(bderr) > 0: ref[errcols[j]][bderr] = 9.99 # Use astroquery vizier #---------------- # for low density with 2MASS/GAIA and always for GALEX and APASS else: # Use QUERYVIZIER for GALEX (python code has problems) #if refname == 'II/312/ais' or refname == 'GALEX': # if refcat eq 'APASS' then cfa=0 else cfa=1 ; cfa doesn't have APASS #cfa = 1 # problems with CDS VizieR and cfa has APASS now #if refcat == 'SAGE': # cfa = 0 if refname.upper() == 'GALEX': cols = [ 'RAJ2000', 'DEJ2000', 'FUVmag', 'e_FUVmag', 'NUVmag', 'e_NUVmag' ] catname = 'II/335/galex_ais' elif refname.upper() == 'GLIMPSE': # Only includes GLIMPSE I,II,3D cols = [ 'RAJ2000', 'DEJ2000', '_2MASS', '_3.6mag', 'e_3.6mag', '_4.5mag', 'e_4.5mag' ] catname = 'II/293/glimpse' elif refname.upper() == 'SAGE': cols = [ 'RAJ2000', 'DEJ2000', '__3.6_', 'e__3.6_', '__4.5_', 'e__4.5_' ] catname = 'II/305/catalog' Vizier.ROW_LIMIT = -1 Vizier.TIMEOUT = 1000000 Vizier.columns = cols result = Vizier.query_constraints(catalog=catname, RA='>=' + str(ra0) + ' & <' + str(ra1)) # Check for failure if len(result) == 0: if silent == False: logger.info('Failure or No Results') return [] ref = result[0] ref.meta['description'] = ref.meta['description'][0:50] #ref = QUERYVIZIER(refname,[cenra,cendec],radius*60,cfa=cfa,timeout=600,/silent) # Fix/homogenize the GAIA tags if refname == 'GAIA': nref = len(ref) orig = ref.copy() dt = [('source', int), ('ra_icrs', float), ('e_ra_icrs', float), ('de_icrs', float), ('e_de_icrs', float), ('fg', float), ('e_fg', float), ('gmag', float)] ref = np.zeros(nref, dtype=np.dtype(dt)) ref = Table(ref) for n in orig.colnames: ref[n] = orig[n] ref['fg'] = orig['_fg_'] ref['e_fg'] = orig['e__fg_'] ref['gmag'] = orig['_gmag_'] del orig # Fix/homogenize the 2MASS tags elif refname == 'TMASS': nref = len(ref) orig = ref.copy() dt = [('designation', (np.str, 50)), ('raj2000', float), ('dej2000', float), ('jmag', float), ('e_jmag', float), ('hmag', float), ('e_hmag', float), ('kmag', float), ('e_kmag', float), ('qflg', (np.str, 20))] ref = np.zeros(nref, dtype=np.dtype(dt)) for n in orig.colnames: ref[n] = orig[n] ref['designation'] = orig['_2mass'] del orig # Fix NANs in ALLWISE elif refname == 'ALLWISE': bd, = np.where(np.isfinite(ref['_3_6_']) == False) if len(bd) > 0: ref['_3_6_'][bd] = 99.99 ref['e__3_6_'][bd] = 9.99 bd, = np.where(np.isfinite(ref['_4_5_']) == False) if len(bd) > 0: ref['_4_5_'][bd] = 99.99 ref['e__4_5_'][bd] = 9.99 # Convert all mags and errmags to float32 for n in ref.colnames: if n.find('mag') > -1: ref[n] = ref[n].astype(np.float32) if n.find('e_') > -1 and n.find('mag') > -1: ref[n] = ref[n].astype(np.float32) # Lowercase column names for n in ref.colnames: ref[n].name = n.lower() # Convert raj2000/dej2000 to ra/dec if 'raj2000' in ref.colnames: ref['raj2000'].name = 'ra' if 'dej2000' in ref.colnames: ref['dej2000'].name = 'dec' # Save the file logger.info('Saving catalog to file ' + savefile) ref.write(savefile, overwrite=True) if silent == False: logger.info('%d sources found dt=%.1f sec.' % (len(ref), time.time() - t0)) return ref
def image_reproject(im, head, outhead, wtim=None, tmproot='.', verbose=False): """ Reproject image onto new projection with interpolation.""" if isinstance(head, WCS): wcs = head else: wcs = WCS(head) if isinstance(outhead, WCS): outwcs = outhead else: outwcs = WCS(outhead) # Make temporary directory and CD to it tmpdir = os.path.abspath(tempfile.mkdtemp(prefix="swrp", dir=tmproot)) origdir = os.path.abspath(os.curdir) os.chdir(tmpdir) # Write image to temporary file imfile = 'image.fits' fits.PrimaryHDU(im, head).writeto(imfile, overwrite=True) if wtim is not None: wtfile = 'wt.fits' fits.PrimaryHDU(wtim, head).writeto(wtfile, overwrite=True) # Use swarp to do the resampling # Create configuration file # fields to modify: IMAGEOUT_NAME, WEIGHTOUT_NAME, WEIGHT_IMAGE, CENTER, PIXEL_SCALE, IMAGE_SIZE, GAIN? fil = os.path.abspath(__file__) codedir = os.path.dirname(os.path.dirname(fil)) paramdir = codedir + '/params/' shutil.copyfile(paramdir + "swarp.config", tmpdir + "/swarp.config") configfile = "swarp.config" clines = dln.readlines(configfile) imoutfile = 'image.out.fits' if wtim is not None: wtoutfile = 'wt.out.fits' # IMAGEOUT_NAME ind = dln.grep(clines, '^IMAGEOUT_NAME ', index=True)[0] clines[ ind] = 'IMAGEOUT_NAME ' + imoutfile + ' # Output filename' # Weight parameteres if wtim is not None: # WEIGHTOUT_NAME ind = dln.grep(clines, '^WEIGHTOUT_NAME ', index=True)[0] clines[ ind] = 'WEIGHTOUT_NAME ' + wtoutfile + ' # Output weight-map filename' # WEIGHT_TYPE ind = dln.grep(clines, '^WEIGHT_TYPE ', index=True)[0] clines[ ind] = 'WEIGHT_TYPE MAP_WEIGHT # BACKGROUND,MAP_RMS,MAP_VARIANCE' # WEIGHT_IMAGE ind = dln.grep(clines, '^WEIGHT_IMAGE ', index=True)[0] clines[ ind] = 'WEIGHT_IMAGE ' + wtfile + ' # Weightmap filename if suffix not used' else: # Remove WEIGHT keywords ind = dln.grep(clines, '^WEIGHT_TYPE ', index=True)[0] clines[ ind] = 'WEIGHT_TYPE NONE # BACKGROUND,MAP_RMS,MAP_VARIANCE' #ind = dln.grep(clines,'^WEIGHT',index=True) #clines = dln.remove_indices(clines,ind) # CENTER ind = dln.grep(clines, '^CENTER ', index=True)[0] clines[ ind] = 'CENTER %9.6f, %9.6f # Coordinates of the image center' % ( outwcs.wcs.crval[0], outwcs.wcs.crval[1]) # PIXEL_SCALE onx, ony = outwcs.array_shape r, d = outwcs.all_pix2world([onx // 2, onx // 2], [ony // 2, ony // 2 + 1], 0) pixscale = (d[1] - d[0]) * 3600 ind = dln.grep(clines, '^PIXEL_SCALE ', index=True)[0] clines[ ind] = 'PIXEL_SCALE %6.3f # Pixel scale' % pixscale # IMAGE_SIZE ind = dln.grep(clines, '^IMAGE_SIZE ', index=True)[0] clines[ ind] = 'IMAGE_SIZE %d,%d # Image size (0 = AUTOMATIC)' % outwcs.array_shape # GAIN?? #ind = dln.grep(clines,'^GAIN_KEYWORD ',index=True)[0] #clines[ind] = 'GAIN_KEYWORD GAIN # FITS keyword for effect. gain (e-/ADU)' # Write the updated configuration file dln.writelines(configfile, clines, overwrite=True) # Run swarp if verbose is False: slogfile = "swarp.log" sf = open(slogfile, 'w') retcode = subprocess.call(["swarp", imfile, "-c", configfile], stdout=sf, stderr=subprocess.STDOUT, shell=False) sf.close() slines = dln.readlines(slogfile) else: retcode = subprocess.call(["swarp", imfile, "-c", configfile], shell=False) # Load the output file oim, ohead = fits.getdata(imoutfile, header=True) out = (oim, ohead) if wtim is not None: owtim, owthead = fits.getdata(wtoutfile, header=True) out = (oim, ohead, owtim, owthead) # Delete temporary directory and files?? tmpfiles = glob.glob('*') for f in tmpfiles: os.remove(f) os.rmdir(tmpdir) # Go back to the original direcotry os.chdir(origdir) return out
help='Output directory') parser.add_argument('--clobber', action='store_true', help='Overwrite existing files') args = parser.parse_args() objid = args.objid nobj = len(args.objid) fov = float(dln.first_el(args.fov)) outdir = dln.first_el(args.outdir) clobber = args.clobber if outdir == '': outdir = '/net/dl2/dnidever/nsc/instcal/v3/hpm2/cutouts2/' if len(args.objid) == 1: if os.path.exists(objid[0]): objid = dln.readlines(objid) nobj = dln.size(objid) else: nobj = len(args.objid) if type(objid) is not list: objid = [objid] for i in range(nobj): print(str(i + 1) + ' ' + objid[i]) animfile = outdir + str(objid) + '_cutouts.gif' if os.path.exists(animfile) and ~clobber: print(animfile + ' already exists and clobber is not set') continue try:
os.makedirs(outdir+instcode+'/'+night+'/'+eindex['value'][k]) except: pass outfile = outdir+instcode+'/'+night+'/'+eindex['value'][k]+'/'+eindex['value'][k]+'__'+dbbase1+'.npy' np.save(outfile,cat) print(' dt = %6.1f sec. ' % (time.time()-t0)) else: print(' '+dbfile1+' NOT FOUND') print('dt = %6.1f sec.' % (time.time()-t00)) if __name__ == "__main__": parser = ArgumentParser(description='Break up idstr into separate lists per exposure.') parser.add_argument('dbfile', type=str, nargs=1, help='Database filename') args = parser.parse_args() hostname = socket.gethostname() host = hostname.split('.')[0] dbfile = args.dbfile[0] # Input is a list if dbfile[0]=='@': listfile = dbfile[1:] if os.path.exists(listfile): dbfile = dln.readlines(listfile) else: print(listfile+' NOT FOUND') sys.exit() breakup_idstr(dbfile)
return out if __name__ == "__main__": parser = ArgumentParser(description='Update measid in exposure.') parser.add_argument('measfiles', type=str, nargs=1, help='meas filenames') parser.add_argument('outfile', type=str, nargs=1, help='output file') args = parser.parse_args() hostname = socket.gethostname() host = hostname.split('.')[0] measfiles = args.measfiles[0] outfile = args.outfile[0] # Input is a list if measfiles[0] == '@': listfile = measfiles[1:] if os.path.exists(listfile): measfiles = dln.readlines(listfile) else: print(listfile + ' NOT FOUND') sys.exit() # Check the measurement files out = check_meas_objectid(measfiles) # Write to output file print('Writing to ' + outfile) if os.path.exists(outfile): os.remove(outfile) Table(out).write(outfile, overwrite=True)
def readcal(calfile): """ This reads all of the information from a master calibration index and returns it in a dictionary where each calibration type has a structured arrays that can be accessed by the calibration name (e.g. 'dark'). """ if os.path.exists(calfile) == False: raise ValueError(calfile+' NOT FOUND') lines = dln.readlines(calfile) lines = np.char.array(lines) # Get rid of comment and blank lines gd,ngd = dln.where(( lines.find('#') != 0) & (lines=='')==False ) if ngd==0: raise ValueError('No good calibration lines') lines = lines[gd] # Initialize calibration dictionary caldict = OrderedDict() dtdict = OrderedDict() # -- Darks -- # mjd1, mjd2, name, frames # dark 55600 56860 12910009 12910009-12910037 # dark 56861 99999 15640003 15640003-15640021 dtdict['dark'] = np.dtype([('mjd1',int),('mjd2',int),('name',np.str,50),('frames',np.str,100)]) # -- Flats -- # mjd1, mjd2, name, frames, nrep, dithered # flat 99999 55761 01380106 1380106-1380134 1 1 # flat 99999 99999 02410013 2410013-2410022 1 0 dtdict['flat'] = np.dtype([('mjd1',int),('mjd2',int),('name',np.str,50),('frames',np.str,100), ('nrep',int),('dithered',int)]) # -- Sparse -- # mjd1, mjd2, name, frames, darkframes, dmax, maxread # sparse 55600 55761 01590015 1590015-1590024 0 21 30,30,20 # sparse 55797 99999 02410059 2410059-2410068 2410058,2410069 21 30,30,20 dtdict['sparse'] = np.dtype([('mjd1',int),('mjd2',int),('name',np.str,50),('frames',np.str,100), ('darkframes',np.str,100),('dmax',int),('maxread',np.str,100)]) # -- Fiber -- # mjd1, mjd2, name # fiber 55600 55761 01970078 # fiber 55797 56860 02410024 dtdict['fiber'] = np.dtype([('mjd1',int),('mjd2',int),('name',np.str,50)]) # -- Badfiber -- # mjd1, mjd2, frames # badfiber 55600 57008 0 # badfiber 57009 57177 195 dtdict['badfiber'] = np.dtype([('mjd1',int),('mjd2',int),('frames',np.str,100)]) # -- Fixfiber -- # mjd1, mjd2, name # fixfiber 56764 56773 1 # fixfiber 58038 58046 2 dtdict['fixfiber'] = np.dtype([('mjd1',int),('mjd2',int),('name',np.str,50)]) # -- Wave -- # mjd1, mjd2, name, frames, psfid # wave 55699 55699 01370096 1370096,1370099 1370098 # wave 55700 55700 01380079 1380079 1380081 dtdict['wave'] = np.dtype([('mjd1',int),('mjd2',int),('name',np.str,50),('frames',np.str,100), ('psfid',int)]) # -- Multiwave -- # mjd1, mjd2, name, frames # multiwave 55800 56130 2380000 02390007,02390008,02500007 # multiwave 56130 56512 5680000 05870007,05870008,05870018,05870019 dtdict['multiwave'] = np.dtype([('mjd1',int),('mjd2',int),('name',np.str,50),('frames',np.str,500)]) # -- LSF -- # mjd1, mjd2, name, frames, psfid # lsf 55800 56130 03430016 03430016 03430020 # lsf 56130 56512 07510018 07510018 07510022 dtdict['lsf'] = np.dtype([('mjd1',int),('mjd2',int),('name',np.str,50),('frames',np.str,100), ('psfid',int)]) # -- Det -- # mjd1, mjd2, name, linid # det 99999 99999 55640 0 # det 55600 56860 11870003 11870003 dtdict['det'] = np.dtype([('mjd1',int),('mjd2',int),('name',np.str,50),('linid',int)]) # -- BPM -- # mjd1, mjd2, name, darkid, flatid # bpm 99999 99999 05560001 5560001 4750009 # bpm 55600 56860 12910009 12910009 4750009 dtdict['bpm'] = np.dtype([('mjd1',int),('mjd2',int),('name',np.str,50),('darkid',int), ('flatid',int)]) # -- Littrow -- # mjd1, mjd2, name, psfid # littrow 55600 56860 06670109 6670109 # littrow 56861 99999 13400052 13400052 dtdict['littrow'] = np.dtype([('mjd1',int),('mjd2',int),('name',np.str,50),('psfid',int)]) # -- Persist -- # mjd1, mjd2, name, darkid, flatid, thresh # persist 55600 56860 04680019 4680019 4680018 0.03 # persist 56861 99999 13400061 13400061 13400060 0.03 dtdict['persist'] = np.dtype([('mjd1',int),('mjd2',int),('name',np.str,50),('darkid',int), ('flatid',int),('thresh',float)]) # -- Persistmodel -- # mjd1, mjd2, name # persistmodel 55600 56860 57184 # persistmodel 56861 99999 0 dtdict['persistmodel'] = np.dtype([('mjd1',int),('mjd2',int),('name',np.str,50)]) # -- Response -- # mjd1, mjd2, name, fluxid, psfid, temp # response 55600 99999 0 0 0 0 dtdict['response'] = np.dtype([('mjd1',int),('mjd2',int),('name',np.str,50),('fluxid',int), ('psfid',int),('temp',float)]) # Readnoise # frame1, frame2 # rn 1380094 1380095 # rn 1380102 1380103 #dtdict['rn'] = np.dtype([('frame1',int),('frame2',int)]) # Gain # frame1, frame2 #dtdict['gain'] = np.dtype([('frame1',int),('frame2',int)]) # READNOISE and GAIN lines are NOT used # Load the data for caltype in dtdict.keys(): cat = loadcaltype(lines,caltype,dtdict[caltype]) caldict[caltype.strip()] = cat return caldict
print('dt = %6.1f sec.' % (time.time()-t00)) if __name__ == "__main__": parser = ArgumentParser(description='Fix pms in healpix object catalogs.') parser.add_argument('pix', type=str, nargs=1, help='HEALPix') args = parser.parse_args() hostname = socket.gethostname() host = hostname.split('.')[0] pix = args.pix[0] # Input is a list if pix[0]=='@': listfile = pix[1:] if os.path.exists(listfile): pix = dln.readlines(listfile) else: print(listfile+' NOT FOUND') sys.exit() # Fix the pms in healpix object catalogs if type(pix) is not list: pix=[pix] npix = len(pix) print('Correcting PMs for '+str(npix)+' HEALPix') for i in range(len(pix)): fix_pms(pix[i])
def get_missingids(exposure): """ Get the number of missing IDs from the log files.""" t00 = time.time() hostname = socket.gethostname() host = hostname.split('.')[0] iddir = '/data0/dnidever/nsc/instcal/v3/idstr/' version = 'v3' # Load the exposures table print('Loading exposure table') expcat = fits.getdata( '/net/dl2/dnidever/nsc/instcal/' + version + '/lists/nsc_v3_exposure_table.fits.gz', 1) # Make sure it's a list if type(exposure) is str: exposure = [exposure] # Match exposures to exposure catalog eind1, eind2 = dln.match(expcat['EXPOSURE'], exposure) nexp = len(exposure) outstr = np.zeros( nexp, np.dtype([('exposure', (np.str, 100)), ('mtime', np.float64), ('nmeas', int), ('nids', int), ('nmatches', int), ('nduplicates', int), ('nmissing', int)])) #outstr['exposure'] = exposure outstr['mtime'] = -1 outstr['nmeas'] = -1 outstr['nmatches'] = -1 outstr['nids'] = -1 outstr['nduplicates'] = -1 outstr['nmissing'] = -1 # Loop over files for i in range(nexp): t0 = time.time() exp = expcat['EXPOSURE'][eind1[i]] print(str(i + 1) + ' ' + exp) outstr['exposure'][i] = exp instcode = expcat['INSTRUMENT'][eind1[i]] dateobs = expcat['DATEOBS'][eind1[i]] night = dateobs[0:4] + dateobs[5:7] + dateobs[8:10] expdir = '/net/dl2/dnidever/nsc/instcal/' + version + '/' + instcode + '/' + night + '/' + exp logfile = glob(expdir + '/' + exp + '_measure_update.????????????.log') nlogfile = len(logfile) # No logfile if nlogfile == 0: print('No logfile') continue # more than 1 logfile, get the latest one if nlogfile > 1: mtime = [os.path.getmtime(f) for f in logfile] si = np.argsort(mtime)[::-1] logfile = logfile[si[0]] else: logfile = logfile[0] outstr['mtime'][i] = os.path.getmtime(logfile) # Read in logfile lines = dln.readlines(logfile) # Number of measurements measind = dln.grep(lines, 'measurements', index=True) if len(measind) > 0: line1 = lines[measind[0]] lo = line1.find(']') hi = line1.find('measurements') nmeas = int(line1[lo + 3:hi - 1]) outstr['nmeas'][i] = nmeas else: nmeas = -1 # Number in idcat, "IDs for XX measurements" idsind = dln.grep(lines, 'IDs for', index=True) if len(idsind) > 0: line1 = lines[idsind[0]] lo = line1.find('for') hi = line1.find('measurements') nids = int(line1[lo + 4:hi - 1]) outstr['nids'][i] = nids else: nids = -1 # Number of matches matchind = dln.grep(lines, 'Matches for', index=True) if len(matchind) > 0: line1 = lines[matchind[0]] lo = line1.find('for') hi = line1.find('measurements') nmatches = int(line1[lo + 4:hi - 1]) outstr['nmatches'][i] = nmatches else: nmatches = -1 # Check for duplicates #if (len(measind)>0) & (len(matchind)>0): # ndup = np.maximum(nmatches-nmeas,0) if (len(measind) > 0) & (len(idsind) > 0): ndup = np.maximum(nids - nmeas, 0) outstr['nduplicates'][i] = ndup else: ndup = -1 # Number of missing IDs badind = dln.grep(lines, 'WARNING:', index=True) nbadind = len(badind) # Some missing objectids if nbadind > 0: badline = lines[badind[0]] lo = badline.find('WARNING:') hi = badline.find('measurements') nmissing = int(badline[lo + 9:hi - 1]) outstr['nmissing'][i] = nmissing else: nmissing = 0 outstr['nmissing'][i] = nmissing print(' Nmeas=' + str(nmeas) + ' Nids=' + str(nids) + ' Nmatches=' + str(nmatches) + ' Nduplicates=' + str(ndup) + ' Nmissing=' + str(nmissing)) return outstr
if inplistfile is None: if not os.path.exists(listfile): rootLogger.info( listfile + ' NOT FOUND. Run nsc_instcal_combine_qacuts.pro/py') sys.exit() rootLogger.info("Reading list from " + listfile) healstr = fits.getdata(listfile, 1) index = fits.getdata(listfile, 2) upix = index['pix'] npix = len(index) else: rootLogger.info("Reading list from " + inplistfile) upix = dln.readlines(inplistfile) upix = np.array(upix, int) npix = len(upix) # Copy to local directory for faster reading speed if os.path.exists(localdir + 'dnidever/nsc/instcal/' + version + '/' + os.path.basename(listfile)): os.remove(localdir + 'dnidever/nsc/instcal/' + version + '/' + os.path.basename(listfile)) shutil.copyfile( listfile, localdir + 'dnidever/nsc/instcal/' + version + '/' + os.path.basename(listfile)) #file_copy,listfile,localdir+'dnidever/nsc/instcal/'+version+'/',/over # Create the commands allpix = upix.copy()