def translate_idl_mjd5_script(scriptfile):
"""
Translate an IDL MJD5.pro script file to yaml. It returns a list of strings
that can be written to a file.
Parameters
----------
scriptfile : str
Name of MJD5.pro script file.
Returns
-------
flines : numpy char array
The lines of the script file translated to yaml.
Examples
--------
flines = mkplan.translate_idl_mjd5_script('apo25m_59085.pro')
Example file, top part of apo25m_59085.pro
apsetver,telescope='apo25m'
mjd=59085
plate=11950
psfid=35230030
fluxid=35230030
ims=[35230018,35230019,35230020,35230021,35230022,35230023,35230024,35230025,35230026,35230027,35230028,35230029]
mkplan,ims,plate,mjd,psfid,fluxid,vers=vers
;these are not sky frames
plate = 12767
psfid=35230015
fluxid=35230015
ims=[35230011,35230012,35230013,35230014]
mkplan,ims,plate,mjd,psfid,fluxid,vers=vers;,/sky
plate=12673
psfid=35230037
fluxid=35230037
ims=[35230033,35230034,35230035,35230036]
mkplan,ims,plate,mjd,psfid,fluxid,vers=vers
By D.Nidever, Oct 2020
"""
# Check that the file exists
if os.path.exists(scriptfile) == False:
raise ValueError(scriptfile + " NOT FOUND")
# Load the file
lines = dln.readlines(scriptfile)
lines = np.char.array(lines)
# Fix continuation lines
lines = fixidlcontinuation(lines)
# Remove comments
lines = removeidlcomments(lines)
# Get telescope from apserver line
ind, nind = dln.where(lines.strip().lower().find('apsetver') == 0)
telescope = None
if nind == 0:
print('No APSERVER line found')
if scriptfile.lower().find('apo25m') > -1: telescope = 'apo25m'
if scriptfile.lower().find('lco25m') > -1: telescope = 'lcoo25m'
if telescope is None:
raise ValueError('Cannot find TELESCOPE')
else:
setverline = lines[ind[0]]
telescope = setverline[setverline.lower().find('telescope=') + 10:]
telescope = telescope.replace("'", "")
telescopeline = "telescope: " + telescope
# Get MJD
ind, nind = dln.where(lines.strip().lower().find('mjd=') == 0)
if nind == 0:
raise ValueError('No MJD line found')
mjdline = lines[ind[0]]
mjd = int(mjdline[mjdline.find('=') + 1:])
mjdline = 'mjd: ' + str(mjd)
# Get day number
ind, nind = dln.where(lines.lower().find('day=') > -1)
if nind > 0:
dayline = lines[ind[0]].lower()
# day=getnum(mjd)*10000
if dayline.lower().find('getnum') > -1:
dayline = dayline.replace('getnum(mjd)', '(mjd-55562)')
day = int(eval(dayline[dayline.find('=') + 1:]))
else:
day = None
# Remove apvers, mjd and day line
gd, ngd = dln.where((lines.strip('').lower().startswith('day=') == False)
& (lines.strip('').lower().find('apsetver') == -1) &
(lines.strip('').lower().startswith('mjd=') == False))
lines = lines[gd]
# Deal with IDL code using day, indgen(), etc.
lines = replaceidlcode(lines, mjd, day=day)
# Initalize final lines
flines = ['---'] # start of yaml file
# Loop over mkplan blocks
# mkplan command is at the end of the block
ind, nind = dln.where(lines.lower().find('mkplan') != -1)
for i in range(nind):
if i == 0:
lo = 0
else:
lo = ind[i - 1] + 1
lines1 = lines[lo:ind[i] + 1]
nlines1 = len(lines1)
# Add TELESCOPE line
flines.append("- " + telescopeline)
# Add MJD line
flines.append(" " + mjdline)
# Assume all lines in this block except for mkplan are key: value pairs
kvlines = lines1[0:-1]
for kvl in kvlines:
if kvl.strip() != '':
lo = kvl.find('=')
key = kvl[0:lo].strip()
val = kvl[lo + 1:].strip()
flines.append(" " + key + ": " + val)
# Deal with mkplan lines
planline = lines1[-1]
# Trim off the first bit that's always the same, "mkplan,ims,plate,mjd,psfid,fluxid,"
planline = planline[planline.lower().find('fluxid') + 7:]
# Remove vers=vers if it's there
if planline.lower().find('vers=vers') == 0:
planline = planline[9:]
# Deal with keywords
if planline != '':
if planline[0] == ',':
planline = planline[1:]
# Add lines for sky, dark, cal
if planline.lower().find('/sky') > -1:
flines.append(' sky: True')
planline = removeidlkeyword(planline, '/sky') # Trim off /sky
if planline.lower().find('/dark') > -1:
flines.append(' dark: True')
planline = removeidlkeyword(planline,
'/dark') # Trim off /dark
if planline.lower().find('/cal') > -1:
flines.append(' cal: True')
planline = removeidlkeyword(planline, '/cal') # Trim off /cal
# Deal with remaining arguments
if planline != '':
# Return leftover line as a dictionary
import pdb
pdb.set_trace()
exec("args=args2dict(" + planline + ")")
# Loop over keys and add them
for k in args.keys():
val = args[k]
if (type(val) is int) | (type(val) is str):
flines.append(" " + k + ": " + str(val))
else:
flines.append(" " + k + ": " + str(list(val)))
# End of yaml file
flines.append('...')
return flines
#hlist = dbutils.query(listfile,'hlist',cols='FILE')
#nlist = len(hlist)
#healstr = fits.getdata(listfile,1)
#expdir = [os.path.dirname(f) for f in healstr['FILE']]
#expdir = np.unique(expdir)
liststr = fits.getdata(listfile,1)
expdir = liststr['EXPDIR']
# change /dl1/users/dnidever/ to /dl2/dnidever/
expdir = expdir.replace('/dl1/users/dnidever/','/dl2/dnidever/')
# Trim trailing /
#expdir = expdir.rstrip('/') # this doesn't work for some reason
expdir = np.char.array([a.rstrip('/') for a in expdir])
nexpdir = len(expdir)
else:
rootLogger.info("Reading list from "+inplistfile)
expdir = dln.readlines(inplistfile)
nexpdir = len(expdir)
# Create the commands
allexpdir = expdir.copy()
allcmd = dln.strjoin("/home/dnidever/projects/noaosourcecatalog/python/nsc_instcal_measure_update.py ",allexpdir)
alldirs = np.zeros(nexpdir,(np.str,200))
alldirs[:] = tmpdir
nallcmd = len(allcmd)
# Check what's been done already
check = False
#if not redo:
if check:
rootLogger.info("Checking if any have already been done")
exists = np.zeros(dln.size(allexpdir),bool)+False
# pars, cov = parallax.fit(meas)
# parallax.plotfit(meas,pars, cov)
#except:
# print("This field threw an error ", idv)
# Main command-line program
if __name__ == "__main__":
parser = ArgumentParser(description='Make diagnostic HPM plots')
parser.add_argument('objid', type=str, nargs='*', help='Object IDs')
args = parser.parse_args()
objid = args.objid
nobj = dln.size(objid)
if nobj == 1:
if os.path.exists(objid[0]):
listfile = objid[0]
print('Reading ' + listfile)
objid = dln.readlines(listfile)
nobj = dln.size(objid)
if type(objid) is not list: objid = [objid]
for i in range(nobj):
print(str(i + 1) + ' ' + objid[i])
try:
make_4panelplot(objid[i])
except Exception as e:
print('Failed on ' + objid[i] + ' ' + str(e))
traceback.print_exc()
print(' Writing missing catalog to ' + missfile)
Table(meas[miss]).write(missfile, overwrite=True)
else:
print(str(i + 1) + ' ' + exp1 + ' none missing')
else:
print(measfile + ' NOT FOUND')
if __name__ == "__main__":
parser = ArgumentParser(
description='Combine NSC data for one healpix region.')
parser.add_argument('exposure', type=str, nargs=1, help='exposure')
args = parser.parse_args()
version = 'v3'
hostname = socket.gethostname()
host = hostname.split('.')[0]
exposure = args.exposure[0]
# Input is a list
if exposure[0] == '@':
listfile = exposure[1:]
if os.path.exists(listfile):
exposure = dln.readlines(listfile)
else:
print(listfile + ' NOT FOUND')
sys.exit()
make_meas_missing(exposure)
def getdata(refcat, minra, redo=False, silent=False, logger=None):
"""
Get reference catalog information from DL database
Parameters
----------
cenra : float
Central RA for the search.
cendec : float
Central DEC for the search.
radius : float
Search radius in degrees.
refcat : table
Reference catalog name (e.g. 2MASS, Gaia, etc.)
version : str
Version of NSC reduction..
saveref : bool, optional
Save the output to SAVEFILE or a default filename. Default is False.
savefile : str, optional
The file to save to or search for existing catalog.
silent : bool, optional
Don't print anything to the screen.
logger : logging object, optional
Logging object to use for printing messages.
Returns
-------
ref : astropy table
Search results from the reference catalog.
Example
-------
cat = getrefcat(cenra,cendec,radius,refcat,file=file,saveref=saveref)
By D. Nidever Sep 2017
Translated to Python by D. Nidever, April 2022
"""
count = 0
t0 = time.time()
if logger is None:
logger = dln.basiclogger()
# Check that we have psql installed
out = subprocess.check_output(['which', 'psql'], shell=False)
if type(out) is bytes:
out = out.decode()
out = out.strip()
if dln.size(out) > 1:
out = out[0]
if os.path.exists(out) == 0:
raise ValueError('No PSQL found on this sytem.')
# Temporary directory
# /tmp is often small and get get fille dup
dldir, mssdir, localdir = utils.rootdirs()
# FLIP THIS AROUND, INPUT SHOULD BE THE "EASY" VERSION!!!
refname = str(refcat).upper()
if refname == 'II/312/AIS':
refname = 'GALEX'
elif refname == '2MASS-PSC':
refname = 'TMASS'
elif refname == '2MASS':
refname = 'TMASS'
elif refname == 'GAIA/GAIA':
refname = 'GAIA'
elif refname == 'Skymapper':
refname = 'SKYMAPPER'
elif refname == 'GLIMPSE':
catname = 'II/293/glimpse'
elif refname == 'SAGE':
catname = 'II/305/archive'
elif refname == 'ATLASREFCAT2':
refname = 'ATLAS'
ra0 = float(minra)
ra1 = minra + 1.0
outdir = '/net/dl1/users/dnidever/nsc/refcatalogs/' + refname + '/'
if os.path.exists(outdir) == False:
os.makedirs(outdir)
savefile = outdir + 'ref_%.6f_%.6f_%s.fits' % (ra0, ra1, refname)
if os.path.exists(os.path.abspath(os.path.dirname(savefile))) == False:
os.makedirs(os.path.abspath(os.path.dirname(savefile)))
if silent == False:
logger.info('Querying %s: %.6f <= RA < %.6f' % (refname, ra0, ra1))
# Loading previously loaded file
if os.path.exists(savefile) and redo == False:
logger.info(savefile + ' already exists and redo==False')
return None
# Do the Query
#--------------
else:
# Use DataLab database search
#----------------------------
if refname in [
'TMASS', 'GAIA', 'GAIADR2', 'GAIAEDR3', 'PS', 'SKYMAPPER',
'SKYMAPPERDR2', 'ALLWISE', 'ATLAS'
]:
if refname == 'TMASS':
tablename = 'twomass.psc'
cols = 'designation,ra as raj2000,dec as dej2000,j_m as jmag,j_cmsig as e_jmag,h_m as hmag,h_cmsig as e_hmag,k_m as kmag,k_cmsig as e_kmag,ph_qual as qflg'
##server = 'gp04.datalab.noao.edu'
#server = 'gp01.datalab.noirlab.edu'
##server = 'dldb1.sdm.noao.edu'
server = 'db02.datalab.noirlab.edu'
user = '******'
racol = 'ra'
deccol = 'dec'
if refname == 'GAIA':
tablename = 'gaia_dr1.gaia_source'
cols = 'source_id as source,ra as ra_icrs,ra_error as e_ra_icrs,dec as de_icrs,dec_error as e_de_icrs,'
cols += 'phot_g_mean_flux as fg,phot_g_mean_flux_error as e_fg,phot_g_mean_mag as gmag'
#server = 'gp04.datalab.noirlab.edu'
##server = 'gp01.datalab.noao.edu'
##server = 'dldb1.sdm.noao.edu'
server = 'db02.datalab.noirlab.edu'
user = '******'
if refname == 'GAIADR2':
tablename = 'gaia_dr2.gaia_source'
cols = 'source_id as source,ra,ra_error,dec,dec_error,pmra,pmra_error,pmdec,pmdec_error,phot_g_mean_flux as fg,phot_g_mean_flux_error as e_fg,'
cols += 'phot_g_mean_mag as gmag,phot_bp_mean_mag as bp,phot_bp_mean_flux as fbp,phot_bp_mean_flux_error as e_fbp,'
cols += 'phot_rp_mean_mag as rp,phot_rp_mean_flux as frp,phot_rp_mean_flux_error as e_frp'
#server = 'gp04.datalab.noirlab.edu'
##server = 'gp01.datalab.noao.edu'
server = 'db02.datalab.noirlab.edu'
user = '******'
if refname == 'GAIAEDR3':
tablename = 'gaia_edr3.gaia_source'
cols = 'source_id as source,ra,ra_error,dec,dec_error,pmra,pmra_error,pmdec,pmdec_error,phot_g_mean_flux as fg,phot_g_mean_flux_error as e_fg,'
cols += 'phot_g_mean_mag as gmag,phot_bp_mean_mag as bp,phot_bp_mean_flux as fbp,phot_bp_mean_flux_error as e_fbp,'
cols += 'phot_rp_mean_mag as rp,phot_rp_mean_flux as frp,phot_rp_mean_flux_error as e_frp'
#server = 'gp04.datalab.noirlab.edu'
##server = 'gp01.datalab.noao.edu'
server = 'db02.datalab.noirlab.edu'
user = '******'
if refname == 'PS':
#tablename = 'cp_calib.ps1'
tablename = 'public.ps1'
cols = 'ra, dec, g as gmag, r as rmag, i as imag, z as zmag, y as ymag'
##server = 'gp02.datalab.noirlab.edu'
#server = 'gp01.datalab.noirlab.edu'
server = 'db02.datalab.noirlab.edu'
user = '******'
if refname == 'SKYMAPPER':
tablename = 'skymapper_dr1.master'
cols = 'raj2000 as ra, dej2000 as dec, u_psf as sm_umag, e_u_psf as e_sm_umag, g_psf as sm_gmag, e_g_psf as e_sm_gmag, r_psf as sm_rmag,'
cols += 'e_r_psf as e_sm_rmag, i_psf as sm_imag,e_i_psf as e_sm_imag, z_psf as sm_zmag, e_z_psf as e_sm_zmag'
#server = 'gp04.datalab.noirlab.edu'
##server = 'gp01.datalab.noao.edu'
server = 'db02.datalab.noirlab.edu'
user = '******'
racol = 'raj2000'
deccol = 'dej2000'
if refname == 'SKYMAPPERDR2':
tablename = 'skymapper_dr2.master'
cols = 'raj2000 as ra, dej2000 as dec, u_psf as sm_umag, e_u_psf as e_sm_umag, g_psf as sm_gmag, e_g_psf as e_sm_gmag, r_psf as sm_rmag,'
cols += 'e_r_psf as e_sm_rmag, i_psf as sm_imag,e_i_psf as e_sm_imag, z_psf as sm_zmag, e_z_psf as e_sm_zmag'
#server = 'gp04.datalab.noirlab.edu'
##server = 'gp01.datalab.noao.edu'
server = 'db02.datalab.noirlab.edu'
user = '******'
racol = 'raj2000'
deccol = 'dej2000'
if refname == 'ALLWISE':
tablename = 'allwise.source'
#cols = 'ra, dec, w1mdef as w1mag, w1sigmdef as e_w1mag, w2mdef as w2mag, w2sigmdef as e_w2mag'
cols = 'ra, dec, w1mpro as w1mag, w1sigmpro as e_w1mag, w2mpro as w2mag, w2sigmpro as e_w2mag'
#server = 'gp04.datalab.noao.edu'
#server = 'gp01.datalab.noirlab.edu'
server = 'db02.datalab.noirlab.edu'
user = '******'
if refname == 'ATLAS':
tablename = 'atlasrefcat2'
cols = 'objid,ra,dec,plx as parallax,dplx as parallax_error,pmra,dpmra as pmra_error,pmdec,dpmdec as pmdec_error,gaia,dgaia as gaiaerr,'
cols += 'bp,dbp as bperr,rp,drp as rperr,teff,agaia,dupvar,ag,rp1,r1,r10,g as gmag,dg as gerr,gchi,gcontrib,'
cols += 'r as rmag, dr as rerr,rchi,rcontrib,i as imag,di as ierr,ichi,icontrib,z as zmag,dz as zerr,zchi,zcontrib,nstat,'
cols += 'j as jmag,dj as jerr,h as hmag,dh as herr,k as kmag,dk as kerr'
server = 'gp10.datalab.noirlab.edu'
user = '******'
# Use Postgres command with q3c cone search
refcattemp = savefile.replace('.fits', '.txt')
cmd = "psql -h " + server + " -U " + user + " -d tapdb -w --pset footer -c 'SELECT " + cols + " FROM " + tablename
if refname == 'SKYMAPPER' or refname == 'SKYMAPPERDR2':
cmd += " WHERE raj2000 >= %.6f and raj2000 < %.6f'" % (ra0,
ra1)
else:
cmd += " WHERE ra >= %.6f and ra < %.6f'" % (ra0, ra1)
cmd += " > " + refcattemp
dln.remove(refcattemp, allow=True)
dln.remove(savefile, allow=True)
out = subprocess.check_output(cmd, shell=True)
# Check for empty query
tlines = dln.readlines(refcattemp, nreadline=4)
if len(tlines) < 4:
if silent == False:
logger.info('No Results')
return []
# Load ASCII file and create the FITS file
ref = ascii.read(refcattemp, data_start=3, delimiter='|')
#ref = importascii(refcattemp,/header,delim='|',skipline=2,/silent)
dln.remove(refcattemp, allow=True)
# Fix 0.0 mags/errs in ATLAS
if refname == 'ATLAS':
magcols = [
'gaia', 'bp', 'rp', 'gmag', 'rmag', 'imag', 'zmag', 'jmag',
'hmag', 'kmag'
]
errcols = [
'gaiaerr', 'bperr', 'rperr', 'gerr', 'rerr', 'ierr',
'zerr', 'jerr', 'herr', 'kerr'
]
cols = ref.colnames
# Set mags with 0.0 to 99.99
for j in range(len(magcols)):
if magcols[j] in ref.colnames:
bdmag = (ref[magcols[j]] <= 0.0)
if np.sum(bdmag) > 0:
ref[magcols[j]][bdmag] = 99.99
# Set errors with 0.0 to 9.99
for j in range(len(errcols)):
if errcols[j] in ref.colnames:
bderr = (ref[errcols[j]] <= 0.0)
if np.sum(bderr) > 0:
ref[errcols[j]][bderr] = 9.99
# Use astroquery vizier
#----------------
# for low density with 2MASS/GAIA and always for GALEX and APASS
else:
# Use QUERYVIZIER for GALEX (python code has problems)
#if refname == 'II/312/ais' or refname == 'GALEX':
# if refcat eq 'APASS' then cfa=0 else cfa=1 ; cfa doesn't have APASS
#cfa = 1 # problems with CDS VizieR and cfa has APASS now
#if refcat == 'SAGE':
# cfa = 0
if refname.upper() == 'GALEX':
cols = [
'RAJ2000', 'DEJ2000', 'FUVmag', 'e_FUVmag', 'NUVmag',
'e_NUVmag'
]
catname = 'II/335/galex_ais'
elif refname.upper() == 'GLIMPSE':
# Only includes GLIMPSE I,II,3D
cols = [
'RAJ2000', 'DEJ2000', '_2MASS', '_3.6mag', 'e_3.6mag',
'_4.5mag', 'e_4.5mag'
]
catname = 'II/293/glimpse'
elif refname.upper() == 'SAGE':
cols = [
'RAJ2000', 'DEJ2000', '__3.6_', 'e__3.6_', '__4.5_',
'e__4.5_'
]
catname = 'II/305/catalog'
Vizier.ROW_LIMIT = -1
Vizier.TIMEOUT = 1000000
Vizier.columns = cols
result = Vizier.query_constraints(catalog=catname,
RA='>=' + str(ra0) + ' & <' +
str(ra1))
# Check for failure
if len(result) == 0:
if silent == False:
logger.info('Failure or No Results')
return []
ref = result[0]
ref.meta['description'] = ref.meta['description'][0:50]
#ref = QUERYVIZIER(refname,[cenra,cendec],radius*60,cfa=cfa,timeout=600,/silent)
# Fix/homogenize the GAIA tags
if refname == 'GAIA':
nref = len(ref)
orig = ref.copy()
dt = [('source', int), ('ra_icrs', float),
('e_ra_icrs', float), ('de_icrs', float),
('e_de_icrs', float), ('fg', float), ('e_fg', float),
('gmag', float)]
ref = np.zeros(nref, dtype=np.dtype(dt))
ref = Table(ref)
for n in orig.colnames:
ref[n] = orig[n]
ref['fg'] = orig['_fg_']
ref['e_fg'] = orig['e__fg_']
ref['gmag'] = orig['_gmag_']
del orig
# Fix/homogenize the 2MASS tags
elif refname == 'TMASS':
nref = len(ref)
orig = ref.copy()
dt = [('designation', (np.str, 50)), ('raj2000', float),
('dej2000', float), ('jmag', float), ('e_jmag', float),
('hmag', float), ('e_hmag', float), ('kmag', float),
('e_kmag', float), ('qflg', (np.str, 20))]
ref = np.zeros(nref, dtype=np.dtype(dt))
for n in orig.colnames:
ref[n] = orig[n]
ref['designation'] = orig['_2mass']
del orig
# Fix NANs in ALLWISE
elif refname == 'ALLWISE':
bd, = np.where(np.isfinite(ref['_3_6_']) == False)
if len(bd) > 0:
ref['_3_6_'][bd] = 99.99
ref['e__3_6_'][bd] = 9.99
bd, = np.where(np.isfinite(ref['_4_5_']) == False)
if len(bd) > 0:
ref['_4_5_'][bd] = 99.99
ref['e__4_5_'][bd] = 9.99
# Convert all mags and errmags to float32
for n in ref.colnames:
if n.find('mag') > -1:
ref[n] = ref[n].astype(np.float32)
if n.find('e_') > -1 and n.find('mag') > -1:
ref[n] = ref[n].astype(np.float32)
# Lowercase column names
for n in ref.colnames:
ref[n].name = n.lower()
# Convert raj2000/dej2000 to ra/dec
if 'raj2000' in ref.colnames:
ref['raj2000'].name = 'ra'
if 'dej2000' in ref.colnames:
ref['dej2000'].name = 'dec'
# Save the file
logger.info('Saving catalog to file ' + savefile)
ref.write(savefile, overwrite=True)
if silent == False:
logger.info('%d sources found dt=%.1f sec.' %
(len(ref), time.time() - t0))
return ref
def image_reproject(im, head, outhead, wtim=None, tmproot='.', verbose=False):
""" Reproject image onto new projection with interpolation."""
if isinstance(head, WCS):
wcs = head
else:
wcs = WCS(head)
if isinstance(outhead, WCS):
outwcs = outhead
else:
outwcs = WCS(outhead)
# Make temporary directory and CD to it
tmpdir = os.path.abspath(tempfile.mkdtemp(prefix="swrp", dir=tmproot))
origdir = os.path.abspath(os.curdir)
os.chdir(tmpdir)
# Write image to temporary file
imfile = 'image.fits'
fits.PrimaryHDU(im, head).writeto(imfile, overwrite=True)
if wtim is not None:
wtfile = 'wt.fits'
fits.PrimaryHDU(wtim, head).writeto(wtfile, overwrite=True)
# Use swarp to do the resampling
# Create configuration file
# fields to modify: IMAGEOUT_NAME, WEIGHTOUT_NAME, WEIGHT_IMAGE, CENTER, PIXEL_SCALE, IMAGE_SIZE, GAIN?
fil = os.path.abspath(__file__)
codedir = os.path.dirname(os.path.dirname(fil))
paramdir = codedir + '/params/'
shutil.copyfile(paramdir + "swarp.config", tmpdir + "/swarp.config")
configfile = "swarp.config"
clines = dln.readlines(configfile)
imoutfile = 'image.out.fits'
if wtim is not None:
wtoutfile = 'wt.out.fits'
# IMAGEOUT_NAME
ind = dln.grep(clines, '^IMAGEOUT_NAME ', index=True)[0]
clines[
ind] = 'IMAGEOUT_NAME ' + imoutfile + ' # Output filename'
# Weight parameteres
if wtim is not None:
# WEIGHTOUT_NAME
ind = dln.grep(clines, '^WEIGHTOUT_NAME ', index=True)[0]
clines[
ind] = 'WEIGHTOUT_NAME ' + wtoutfile + ' # Output weight-map filename'
# WEIGHT_TYPE
ind = dln.grep(clines, '^WEIGHT_TYPE ', index=True)[0]
clines[
ind] = 'WEIGHT_TYPE MAP_WEIGHT # BACKGROUND,MAP_RMS,MAP_VARIANCE'
# WEIGHT_IMAGE
ind = dln.grep(clines, '^WEIGHT_IMAGE ', index=True)[0]
clines[
ind] = 'WEIGHT_IMAGE ' + wtfile + ' # Weightmap filename if suffix not used'
else:
# Remove WEIGHT keywords
ind = dln.grep(clines, '^WEIGHT_TYPE ', index=True)[0]
clines[
ind] = 'WEIGHT_TYPE NONE # BACKGROUND,MAP_RMS,MAP_VARIANCE'
#ind = dln.grep(clines,'^WEIGHT',index=True)
#clines = dln.remove_indices(clines,ind)
# CENTER
ind = dln.grep(clines, '^CENTER ', index=True)[0]
clines[
ind] = 'CENTER %9.6f, %9.6f # Coordinates of the image center' % (
outwcs.wcs.crval[0], outwcs.wcs.crval[1])
# PIXEL_SCALE
onx, ony = outwcs.array_shape
r, d = outwcs.all_pix2world([onx // 2, onx // 2], [ony // 2, ony // 2 + 1],
0)
pixscale = (d[1] - d[0]) * 3600
ind = dln.grep(clines, '^PIXEL_SCALE ', index=True)[0]
clines[
ind] = 'PIXEL_SCALE %6.3f # Pixel scale' % pixscale
# IMAGE_SIZE
ind = dln.grep(clines, '^IMAGE_SIZE ', index=True)[0]
clines[
ind] = 'IMAGE_SIZE %d,%d # Image size (0 = AUTOMATIC)' % outwcs.array_shape
# GAIN??
#ind = dln.grep(clines,'^GAIN_KEYWORD ',index=True)[0]
#clines[ind] = 'GAIN_KEYWORD GAIN # FITS keyword for effect. gain (e-/ADU)'
# Write the updated configuration file
dln.writelines(configfile, clines, overwrite=True)
# Run swarp
if verbose is False:
slogfile = "swarp.log"
sf = open(slogfile, 'w')
retcode = subprocess.call(["swarp", imfile, "-c", configfile],
stdout=sf,
stderr=subprocess.STDOUT,
shell=False)
sf.close()
slines = dln.readlines(slogfile)
else:
retcode = subprocess.call(["swarp", imfile, "-c", configfile],
shell=False)
# Load the output file
oim, ohead = fits.getdata(imoutfile, header=True)
out = (oim, ohead)
if wtim is not None:
owtim, owthead = fits.getdata(wtoutfile, header=True)
out = (oim, ohead, owtim, owthead)
# Delete temporary directory and files??
tmpfiles = glob.glob('*')
for f in tmpfiles:
os.remove(f)
os.rmdir(tmpdir)
# Go back to the original direcotry
os.chdir(origdir)
return out
help='Output directory')
parser.add_argument('--clobber',
action='store_true',
help='Overwrite existing files')
args = parser.parse_args()
objid = args.objid
nobj = len(args.objid)
fov = float(dln.first_el(args.fov))
outdir = dln.first_el(args.outdir)
clobber = args.clobber
if outdir == '':
outdir = '/net/dl2/dnidever/nsc/instcal/v3/hpm2/cutouts2/'
if len(args.objid) == 1:
if os.path.exists(objid[0]):
objid = dln.readlines(objid)
nobj = dln.size(objid)
else:
nobj = len(args.objid)
if type(objid) is not list: objid = [objid]
for i in range(nobj):
print(str(i + 1) + ' ' + objid[i])
animfile = outdir + str(objid) + '_cutouts.gif'
if os.path.exists(animfile) and ~clobber:
print(animfile + ' already exists and clobber is not set')
continue
try:
os.makedirs(outdir+instcode+'/'+night+'/'+eindex['value'][k])
except:
pass
outfile = outdir+instcode+'/'+night+'/'+eindex['value'][k]+'/'+eindex['value'][k]+'__'+dbbase1+'.npy'
np.save(outfile,cat)
print(' dt = %6.1f sec. ' % (time.time()-t0))
else:
print(' '+dbfile1+' NOT FOUND')
print('dt = %6.1f sec.' % (time.time()-t00))
if __name__ == "__main__":
parser = ArgumentParser(description='Break up idstr into separate lists per exposure.')
parser.add_argument('dbfile', type=str, nargs=1, help='Database filename')
args = parser.parse_args()
hostname = socket.gethostname()
host = hostname.split('.')[0]
dbfile = args.dbfile[0]
# Input is a list
if dbfile[0]=='@':
listfile = dbfile[1:]
if os.path.exists(listfile):
dbfile = dln.readlines(listfile)
else:
print(listfile+' NOT FOUND')
sys.exit()
breakup_idstr(dbfile)
return out
if __name__ == "__main__":
parser = ArgumentParser(description='Update measid in exposure.')
parser.add_argument('measfiles', type=str, nargs=1, help='meas filenames')
parser.add_argument('outfile', type=str, nargs=1, help='output file')
args = parser.parse_args()
hostname = socket.gethostname()
host = hostname.split('.')[0]
measfiles = args.measfiles[0]
outfile = args.outfile[0]
# Input is a list
if measfiles[0] == '@':
listfile = measfiles[1:]
if os.path.exists(listfile):
measfiles = dln.readlines(listfile)
else:
print(listfile + ' NOT FOUND')
sys.exit()
# Check the measurement files
out = check_meas_objectid(measfiles)
# Write to output file
print('Writing to ' + outfile)
if os.path.exists(outfile): os.remove(outfile)
Table(out).write(outfile, overwrite=True)
def readcal(calfile):
"""
This reads all of the information from a master calibration index and returns
it in a dictionary where each calibration type has a structured arrays that
can be accessed by the calibration name (e.g. 'dark').
"""
if os.path.exists(calfile) == False:
raise ValueError(calfile+' NOT FOUND')
lines = dln.readlines(calfile)
lines = np.char.array(lines)
# Get rid of comment and blank lines
gd,ngd = dln.where(( lines.find('#') != 0) & (lines=='')==False )
if ngd==0:
raise ValueError('No good calibration lines')
lines = lines[gd]
# Initialize calibration dictionary
caldict = OrderedDict()
dtdict = OrderedDict()
# -- Darks --
# mjd1, mjd2, name, frames
# dark 55600 56860 12910009 12910009-12910037
# dark 56861 99999 15640003 15640003-15640021
dtdict['dark'] = np.dtype([('mjd1',int),('mjd2',int),('name',np.str,50),('frames',np.str,100)])
# -- Flats --
# mjd1, mjd2, name, frames, nrep, dithered
# flat 99999 55761 01380106 1380106-1380134 1 1
# flat 99999 99999 02410013 2410013-2410022 1 0
dtdict['flat'] = np.dtype([('mjd1',int),('mjd2',int),('name',np.str,50),('frames',np.str,100),
('nrep',int),('dithered',int)])
# -- Sparse --
# mjd1, mjd2, name, frames, darkframes, dmax, maxread
# sparse 55600 55761 01590015 1590015-1590024 0 21 30,30,20
# sparse 55797 99999 02410059 2410059-2410068 2410058,2410069 21 30,30,20
dtdict['sparse'] = np.dtype([('mjd1',int),('mjd2',int),('name',np.str,50),('frames',np.str,100),
('darkframes',np.str,100),('dmax',int),('maxread',np.str,100)])
# -- Fiber --
# mjd1, mjd2, name
# fiber 55600 55761 01970078
# fiber 55797 56860 02410024
dtdict['fiber'] = np.dtype([('mjd1',int),('mjd2',int),('name',np.str,50)])
# -- Badfiber --
# mjd1, mjd2, frames
# badfiber 55600 57008 0
# badfiber 57009 57177 195
dtdict['badfiber'] = np.dtype([('mjd1',int),('mjd2',int),('frames',np.str,100)])
# -- Fixfiber --
# mjd1, mjd2, name
# fixfiber 56764 56773 1
# fixfiber 58038 58046 2
dtdict['fixfiber'] = np.dtype([('mjd1',int),('mjd2',int),('name',np.str,50)])
# -- Wave --
# mjd1, mjd2, name, frames, psfid
# wave 55699 55699 01370096 1370096,1370099 1370098
# wave 55700 55700 01380079 1380079 1380081
dtdict['wave'] = np.dtype([('mjd1',int),('mjd2',int),('name',np.str,50),('frames',np.str,100),
('psfid',int)])
# -- Multiwave --
# mjd1, mjd2, name, frames
# multiwave 55800 56130 2380000 02390007,02390008,02500007
# multiwave 56130 56512 5680000 05870007,05870008,05870018,05870019
dtdict['multiwave'] = np.dtype([('mjd1',int),('mjd2',int),('name',np.str,50),('frames',np.str,500)])
# -- LSF --
# mjd1, mjd2, name, frames, psfid
# lsf 55800 56130 03430016 03430016 03430020
# lsf 56130 56512 07510018 07510018 07510022
dtdict['lsf'] = np.dtype([('mjd1',int),('mjd2',int),('name',np.str,50),('frames',np.str,100),
('psfid',int)])
# -- Det --
# mjd1, mjd2, name, linid
# det 99999 99999 55640 0
# det 55600 56860 11870003 11870003
dtdict['det'] = np.dtype([('mjd1',int),('mjd2',int),('name',np.str,50),('linid',int)])
# -- BPM --
# mjd1, mjd2, name, darkid, flatid
# bpm 99999 99999 05560001 5560001 4750009
# bpm 55600 56860 12910009 12910009 4750009
dtdict['bpm'] = np.dtype([('mjd1',int),('mjd2',int),('name',np.str,50),('darkid',int),
('flatid',int)])
# -- Littrow --
# mjd1, mjd2, name, psfid
# littrow 55600 56860 06670109 6670109
# littrow 56861 99999 13400052 13400052
dtdict['littrow'] = np.dtype([('mjd1',int),('mjd2',int),('name',np.str,50),('psfid',int)])
# -- Persist --
# mjd1, mjd2, name, darkid, flatid, thresh
# persist 55600 56860 04680019 4680019 4680018 0.03
# persist 56861 99999 13400061 13400061 13400060 0.03
dtdict['persist'] = np.dtype([('mjd1',int),('mjd2',int),('name',np.str,50),('darkid',int),
('flatid',int),('thresh',float)])
# -- Persistmodel --
# mjd1, mjd2, name
# persistmodel 55600 56860 57184
# persistmodel 56861 99999 0
dtdict['persistmodel'] = np.dtype([('mjd1',int),('mjd2',int),('name',np.str,50)])
# -- Response --
# mjd1, mjd2, name, fluxid, psfid, temp
# response 55600 99999 0 0 0 0
dtdict['response'] = np.dtype([('mjd1',int),('mjd2',int),('name',np.str,50),('fluxid',int),
('psfid',int),('temp',float)])
# Readnoise
# frame1, frame2
# rn 1380094 1380095
# rn 1380102 1380103
#dtdict['rn'] = np.dtype([('frame1',int),('frame2',int)])
# Gain
# frame1, frame2
#dtdict['gain'] = np.dtype([('frame1',int),('frame2',int)])
# READNOISE and GAIN lines are NOT used
# Load the data
for caltype in dtdict.keys():
cat = loadcaltype(lines,caltype,dtdict[caltype])
caldict[caltype.strip()] = cat
return caldict
print('dt = %6.1f sec.' % (time.time()-t00))
if __name__ == "__main__":
parser = ArgumentParser(description='Fix pms in healpix object catalogs.')
parser.add_argument('pix', type=str, nargs=1, help='HEALPix')
args = parser.parse_args()
hostname = socket.gethostname()
host = hostname.split('.')[0]
pix = args.pix[0]
# Input is a list
if pix[0]=='@':
listfile = pix[1:]
if os.path.exists(listfile):
pix = dln.readlines(listfile)
else:
print(listfile+' NOT FOUND')
sys.exit()
# Fix the pms in healpix object catalogs
if type(pix) is not list: pix=[pix]
npix = len(pix)
print('Correcting PMs for '+str(npix)+' HEALPix')
for i in range(len(pix)):
fix_pms(pix[i])
def get_missingids(exposure):
""" Get the number of missing IDs from the log files."""
t00 = time.time()
hostname = socket.gethostname()
host = hostname.split('.')[0]
iddir = '/data0/dnidever/nsc/instcal/v3/idstr/'
version = 'v3'
# Load the exposures table
print('Loading exposure table')
expcat = fits.getdata(
'/net/dl2/dnidever/nsc/instcal/' + version +
'/lists/nsc_v3_exposure_table.fits.gz', 1)
# Make sure it's a list
if type(exposure) is str: exposure = [exposure]
# Match exposures to exposure catalog
eind1, eind2 = dln.match(expcat['EXPOSURE'], exposure)
nexp = len(exposure)
outstr = np.zeros(
nexp,
np.dtype([('exposure', (np.str, 100)), ('mtime', np.float64),
('nmeas', int), ('nids', int), ('nmatches', int),
('nduplicates', int), ('nmissing', int)]))
#outstr['exposure'] = exposure
outstr['mtime'] = -1
outstr['nmeas'] = -1
outstr['nmatches'] = -1
outstr['nids'] = -1
outstr['nduplicates'] = -1
outstr['nmissing'] = -1
# Loop over files
for i in range(nexp):
t0 = time.time()
exp = expcat['EXPOSURE'][eind1[i]]
print(str(i + 1) + ' ' + exp)
outstr['exposure'][i] = exp
instcode = expcat['INSTRUMENT'][eind1[i]]
dateobs = expcat['DATEOBS'][eind1[i]]
night = dateobs[0:4] + dateobs[5:7] + dateobs[8:10]
expdir = '/net/dl2/dnidever/nsc/instcal/' + version + '/' + instcode + '/' + night + '/' + exp
logfile = glob(expdir + '/' + exp + '_measure_update.????????????.log')
nlogfile = len(logfile)
# No logfile
if nlogfile == 0:
print('No logfile')
continue
# more than 1 logfile, get the latest one
if nlogfile > 1:
mtime = [os.path.getmtime(f) for f in logfile]
si = np.argsort(mtime)[::-1]
logfile = logfile[si[0]]
else:
logfile = logfile[0]
outstr['mtime'][i] = os.path.getmtime(logfile)
# Read in logfile
lines = dln.readlines(logfile)
# Number of measurements
measind = dln.grep(lines, 'measurements', index=True)
if len(measind) > 0:
line1 = lines[measind[0]]
lo = line1.find(']')
hi = line1.find('measurements')
nmeas = int(line1[lo + 3:hi - 1])
outstr['nmeas'][i] = nmeas
else:
nmeas = -1
# Number in idcat, "IDs for XX measurements"
idsind = dln.grep(lines, 'IDs for', index=True)
if len(idsind) > 0:
line1 = lines[idsind[0]]
lo = line1.find('for')
hi = line1.find('measurements')
nids = int(line1[lo + 4:hi - 1])
outstr['nids'][i] = nids
else:
nids = -1
# Number of matches
matchind = dln.grep(lines, 'Matches for', index=True)
if len(matchind) > 0:
line1 = lines[matchind[0]]
lo = line1.find('for')
hi = line1.find('measurements')
nmatches = int(line1[lo + 4:hi - 1])
outstr['nmatches'][i] = nmatches
else:
nmatches = -1
# Check for duplicates
#if (len(measind)>0) & (len(matchind)>0):
# ndup = np.maximum(nmatches-nmeas,0)
if (len(measind) > 0) & (len(idsind) > 0):
ndup = np.maximum(nids - nmeas, 0)
outstr['nduplicates'][i] = ndup
else:
ndup = -1
# Number of missing IDs
badind = dln.grep(lines, 'WARNING:', index=True)
nbadind = len(badind)
# Some missing objectids
if nbadind > 0:
badline = lines[badind[0]]
lo = badline.find('WARNING:')
hi = badline.find('measurements')
nmissing = int(badline[lo + 9:hi - 1])
outstr['nmissing'][i] = nmissing
else:
nmissing = 0
outstr['nmissing'][i] = nmissing
print(' Nmeas=' + str(nmeas) + ' Nids=' + str(nids) + ' Nmatches=' +
str(nmatches) + ' Nduplicates=' + str(ndup) + ' Nmissing=' +
str(nmissing))
return outstr
if inplistfile is None:
if not os.path.exists(listfile):
rootLogger.info(
listfile +
' NOT FOUND. Run nsc_instcal_combine_qacuts.pro/py')
sys.exit()
rootLogger.info("Reading list from " + listfile)
healstr = fits.getdata(listfile, 1)
index = fits.getdata(listfile, 2)
upix = index['pix']
npix = len(index)
else:
rootLogger.info("Reading list from " + inplistfile)
upix = dln.readlines(inplistfile)
upix = np.array(upix, int)
npix = len(upix)
# Copy to local directory for faster reading speed
if os.path.exists(localdir + 'dnidever/nsc/instcal/' + version + '/' +
os.path.basename(listfile)):
os.remove(localdir + 'dnidever/nsc/instcal/' + version + '/' +
os.path.basename(listfile))
shutil.copyfile(
listfile, localdir + 'dnidever/nsc/instcal/' + version + '/' +
os.path.basename(listfile))
#file_copy,listfile,localdir+'dnidever/nsc/instcal/'+version+'/',/over
# Create the commands
allpix = upix.copy()
