def queryCoordSimbad(raw_coord, search_radius):
#Import(s)
import numpy as np
from astropy import coordinates as coord
from astropy import units as u
from astroquery.simbad import Simbad
from astropy.coordinates.sky_coordinate import SkyCoord
#Action
c = SkyCoord(raw_coord, unit=(u.hourangle, u.deg))
c = c.to_string('hmsdms')
result_table = Simbad.query_region(coord.SkyCoord(c, frame='icrs'),
radius=('0d0m' + str(search_radius) +
's'))
names_col = result_table['MAIN_ID']
id = str(names_col[0])[1:]
return id
def reduce_exposure(rawfiles=None,
draw=None,
dpro=None,
expid=None,
sof=None,
temprofolder=None,
rawfolder=None,
logfile=None,
outname='VISIR_OBS',
outfolder='.',
overwrite=False,
maxshift=0.5,
extract=True,
searcharea='chopthrow',
box=None,
statcalfolder=None,
crossrefim=None,
chopsubmeth='averchop',
AA_pos=None,
refpos=None,
alignmethod='fastgauss',
findbeams=True,
verbose=False,
ditsoftaver=1,
sky_xrange=None,
plot=False,
insmode=None,
sky_yrange=None,
debug=False,
instrument=None,
sourceext='unknown',
searchsmooth=0.2,
sigmaclip=[3, 1]):
"""
main reduction routine: reduce a give exposure by combining the nods,
perform alignment of DITs/chops in case of burst/half-cycle data and
extract source images
PARAMETERS:
- rawfiles: (optional) list of raw fits files belonging to the exposure to reduce
- draw: (optional) data structure containing all the raw file information as result of the routine reduce_indi_raw_files
- dpro: (optional) data structure containg all the exposure information as a product of the routine group_files_into_observations
- temprofolder: (optional) folder containing the temporary products, i.e., individually reduced files. If 'None', the output folder is taken
- rawfolder: (optional) folder containing the raw fits files to be reduced
- outname: (default='VISIR_OBS') name prefix for the logfile
- outfolder: (default='.') folder to write output into
- overwrite: (default=False) overwrite existing data?
- maxshift: (default=0.5) maximum allowed discrepany between expected and found beam
position in a chop/nod pattern in arcsec
- searchsmooth: (default=0.2) sigma for Gaussian smoothing for beam detection in arcsecc
- extract: (default=True) Try detect a source in the combined nod image and extract subimage
- searcharea: (default='chopthrow') specify the area to search for the sub beam as string in arcsec (e.g., "2 arcsec")
- box: (optional) by default the subimage will have the size of the chopthrow
- chopsubmeth: (default='averchop') method for the subtraction of the chops in burst mode. By default the average of the exposures in the offset position of the corresponding pair is used.
- AA_pos: (optional) specify position of the beam in chop/nod position AA for burst alignment
- refpos: (optional) instead of searching for the beam position, provide one directly for the alignment
- findbeams: (default: True) Should the code try to find the beam positions or just extract at the computed position?
- alignmethod: (default='fastgauss') specify the fitting algorithm for the frame alignment in burst mode data
- verbose = False
- sourceext: (default ='unknown') expected extent of the source ('compact', 'extended', 'unknown')
NOT FULLY IMPLEMENTED:
- call as a stand-alone routine, i.e., without providing draw and dpro
This would require implementation extracting files from the sof.txt as
well as reduction of the raw files with reduce_indi_raws
TO BE ADDED LATER:
- correct treatment of HR and HRX SPC
- correct treatment of SPC ACQ data
- proper modification of the headers of the products
- de-rotation for classical imaging with pupil tracking (burst should work)
"""
funname = "REDUCE_EXPOSURE"
if debug:
verbose = True
noe = 0 # number of error counter
now = 0 # number of warnings
# --- flag whether only fits header or also table should be updated with
# new Gaussian fit paramaters (will be set false after reducing a burst)
onlyhead = False
if temprofolder is None:
temprofolder = outfolder
# --- create output folders
hcycfolder = temprofolder + '/hcycles'
bsumfolder = hcycfolder + '/blindsums'
nodfolder = temprofolder + '/nods'
if not os.path.exists(hcycfolder):
os.makedirs(hcycfolder)
if not os.path.exists(bsumfolder):
os.makedirs(bsumfolder)
if not os.path.exists(nodfolder):
os.makedirs(nodfolder)
# --- test whether output folder exists
if not os.path.exists(outfolder):
os.makedirs(outfolder)
if logfile is None:
logfile = outfolder + '/' + outname + ".log"
mode = 'w'
else:
mode = 'a'
_print_log_info(funname + ": Reduction of " + outname, logfile, mode=mode)
# --- read some important exposure parameters:
# --- if dpro is provided that is easy
if dpro is not None:
if instrument is None:
instrument = dpro["instrument"][expid]
if insmode is None:
insmode = dpro["tempname"][expid].split("_")[0]
targname = dpro["targname"][expid]
datatype = dpro["datatype"][expid]
tempname = dpro["tempname"][expid]
setup = dpro["setup"][expid]
pfov = dpro["pfov"][expid]
chopthrow = dpro["chopthrow"][expid]
chopangle = dpro["chopangle"][expid]
noddir = dpro["noddir"][expid]
else:
targname = None
datatype = None
tempname = None
setup = None
pfov = None
chopthrow = None
noddir = None
# --- if the table with the raw files is provided get them from there
if expid is not None and draw is not None:
rid = np.where(draw['EXPID'] == expid)[0]
nf = len(rid)
filenames = draw['filename'][rid]
# --- abort reduction if we have any corrput files
if 'CANNOT' in draw['INSTRUME'][rid]:
msg = funname + ": ERROR: This observations contains corrupt files! Aborting..."
_print_log_info(msg, logfile)
return (now, noe + 1)
# # --- in addition we can get some
# if tempname is None:
# tempname = draw['TPL_ID'][rid[0]]
# if insmode is None:
# insmode = draw['insmode'][rid[0]
# #fram_format = draw['FRAM_FORMAT'][rid[0]]
# --- or in case a sof file in esorex style is provided
elif sof is not None:
fsof = open(sof)
# --- implement extraction of the fits files here
# rawfiles =
# --- if the raw files provided
if rawfiles is not None:
nf = len(rawfiles)
rid = np.arange(0, nf)
filenames = rawfiles.split('/')[-1]
if rawfolder is None:
rawfolder = rawfiles.split('/')[0:-1]
# --- check that we actually have some files
if nf == 0:
msg = funname + ": ERROR: No corresponding raw files found! Aborting..."
_print_log_info(msg, logfile)
return (now, noe + 1)
# --- TO BE IMPLEMENTED: check if blindsums are available or whether this needs to be done
# ....
# --- get the first head
f0 = bsumfolder + '/' + filenames[0].replace(".fits", "_blindsum.fits")
print(f0)
hdu = fits.open(f0)
head0 = hdu[0].header
hdu.close()
# --- in case we still don't have them, get some key parameters
if targname is None:
targname = _fits_get_info(head0, "target")
if instrument is None:
instrument = _fits_get_info(head0, "instrument")
if insmode is None:
insmode = _fits_get_info(head0, "insmode")
if tempname is None:
tempname = _fits_get_info(head0, "TPL ID")
if datatype is None:
datatype = _fits_get_info(head0, "datatype")
if pfov is None:
pfov = _fits_get_info(head0, "pfov")
if setup is None:
setup = _fits_get_info(head0, "setup")
if chopthrow is None:
chopthrow = _fits_get_info(head0, "CHOP THROW")
if chopangle is None:
chopangle = _fits_get_info(head0, "CHOP ANGLE")
if noddir is None:
noddir = _fits_get_info(head0, "CHOPNOD DIR")
# --- compute the maximum field of view from half of
if not box:
box = int(np.floor(chopthrow / pfov))
# --- check if target is a calibrator
if insmode != "SPC":
if "cal" in tempname:
should_be_cal = True
silent = False
else:
should_be_cal = False
silent = True
# --- first just check if the target is found in the calibrator table
# even if it was observed with a science template
try:
flux = _get_std_flux(head0,
logfile=logfile,
instrument=instrument,
silent=silent)
except:
flux = -1
# --- if a flux was found update the table and header
if flux != -1:
head0["HIERARCH ESO QC JYVAL"] = (
flux, "STD flux density in filter used [Jy]")
if dpro is not None:
dpro['STDflux_Jy'][expid] = flux
# --- if it was not found but the observation was with a cal template
# raise en error
elif should_be_cal:
msg = (
funname +
": WARNING: target not found in flux reference table but should be calibrator"
)
_print_log_info(msg, logfile)
now += 1
flux = -1
else:
flux = -1
msg = " - Target name: " + str(targname) + "\n"
if flux > 0:
msg += " - STD flux [Jy]: " + "{:.2f}".format(flux) + "\n"
msg += (" - Instrument: " + str(instrument) + "\n" +
" - Instrument mode: " + str(insmode) + "\n" +
" - Template name: " + str(tempname) + "\n" + " - PFOV [as]: " +
str(pfov) + "\n" + " - Setup: " + str(setup) + "\n" +
" - Data type: " + str(datatype) + "\n" + " - Nod direction: " +
str(noddir) + "\n" + " - Chop throw [as]: " + str(chopthrow) +
"\n" + " - Chop angle [de]: " + str(chopangle) + "\n" +
" - Max box size [px]: " + str(box))
_print_log_info(msg, logfile, logtime=False)
# === 1 Go over all files of the exposure and create nodding pairs
msg = funname + ": Creating blind nodding pairs..."
_print_log_info(msg, logfile, empty=1)
nodims = []
heads = []
outnames = []
ima = None
imb = None
heada = None
# headb = None
for i in range(nf):
msg = (" - i, File: " + str(i) + ", " + filenames[i])
_print_log_info(msg, logfile)
fsingred = (bsumfolder + '/' +
filenames[i].replace(".fits", "_blindsum.fits"))
# --- read in the data
hdu = fits.open(fsingred)
head = hdu[0].header
im = hdu[0].data
hdu.close()
# --- nod pos A or B?
if draw['NODPOS'][rid[i]] == 'A':
ima = im
heada = head
else:
imb = im
# print(draw['NODPOS'][rid[i]], ima is not None, imb is not None)
# === 1.1 combine files to nodding pairs
if (i > 0) & (ima is not None) & (imb is not None):
# --- build the nod subtracted image
imc = ima - imb
ind = int(np.floor(i / 2))
fout = (nodfolder + '/' + outname + '_nod' +
"{:02.0f}".format(ind) + '.fits')
if overwrite or not os.path.isfile(fout):
fits.writeto(fout, imc, heada, overwrite=True)
pout = fout.replace(".fits", "_per1.png")
_simple_image_plot(imc, pout, percentile=1, pwidth=6)
pout = fout.replace(".fits", "_minmax.png")
_simple_image_plot(imc, pout, percentile=0.0001, pwidth=6)
msg = (" - Output written: " + fout)
_print_log_info(msg, logfile)
# --- collect for further reduction
nodims.append(imc)
outnames.append(outname)
heads.append(heada)
ima = None
imb = None
heada = None
msg = funname + ": Number of nodding pairs reduced: " + str(len(nodims))
_print_log_info(msg, logfile)
# === 2 Combination of the nodding pairs
# if len(heads) == 0: # --- no idea why this was here???
# continue
msg = funname + ": Merging blind nodding pairs..."
_print_log_info(msg, logfile, empty=1)
nodims = np.array(nodims)
nnods = len(nodims)
# --- check that we have at least one valid nodding pair
if nnods == 0:
msg = (funname +
": ERROR: No nodding pairs could be created! Aborting...")
_print_log_info(msg, logfile)
return (now, noe + 1)
# --- 2.1 do jitter correction
if 'HIERARCH ESO SEQ JITTER WIDTH' in head0:
if head0['HIERARCH ESO SEQ JITTER WIDTH'] > 0:
for i in range(nnods):
# joff = V.compute_jitter(head=heads[iddd[m]])
# print(" - Nod no / jitter offs.: ",m,joff)
nodims[i] = _undo_jitter(im=nodims[i], head=heads[i])
# msg = funname + ": 1. simple combination"
# _print_log_info(msg, logfile)
totim = np.nanmean(nodims, axis=0)
fout = (outfolder + '/' + outnames[0] + '_all_fullframe.fits')
# --- if no specfic range is supplied to measure the sky use the full
# illuminated area of the VISIR detector
if sky_xrange is None:
sky_xrange = _vp.max_illum_xrange
if sky_yrange is None:
sky_yrange = _vp.max_illum_yrange
# --- background estimation:
# bgim = _subtract_source(totim[sky_yrange[0]:sky_yrange[1],
# sky_xrange[0]:sky_xrange[1]])
bgim = sigma_clip(totim[sky_yrange[0]:sky_yrange[1],
sky_xrange[0]:sky_xrange[1]],
sigma=3,
maxiters=3,
masked=False)
dpro["BGmed"][expid] = np.nanmedian(bgim)
dpro["BGstd"][expid] = np.nanstd(bgim)
msg = (" - BG median [ADU/DIT]: " + str(dpro["BGmed"][expid]) + "\n" +
" - BG STDDEV [ADU/DIT]: " + str(dpro["BGstd"][expid]))
_print_log_info(msg, logfile, logtime=False)
if overwrite or not os.path.isfile(fout):
fits.writeto(fout, totim, head0, overwrite=True)
pout = fout.replace(".fits", "_per1.png")
_simple_image_plot(totim, pout, percentile=1, pwidth=6)
pout = fout.replace(".fits", "_minmax.png")
_simple_image_plot(totim, pout, percentile=0.0001, pwidth=6)
msg = (" - Output written: " + fout)
_print_log_info(msg, logfile)
# === 3 BURST & CYCSUM data ===
if datatype == "halfcyc" or datatype == "burst":
msg = (funname + ": Burst/halfcyc data encountered...")
_print_log_info(msg, logfile, empty=1)
burstfolder = hcycfolder + '/' + alignmethod
if not os.path.exists(burstfolder):
os.makedirs(burstfolder)
try:
noe = reduce_burst_exp(logfile, filenames, rawfolder, burstfolder,
ditsoftaver, overwrite, noddir, bsumfolder,
draw, rid, noe, box, AA_pos, alignmethod,
chopsubmeth, refpos, verbose, searchsmooth,
debug, plot, crossrefim, outfolder,
outnames, head0, dpro, pfov, expid)
except:
e = sys.exc_info()
msg = (funname + ": ERROR: Burst reduction failed!")
noe = noe + 1
_print_log_info(msg, logfile)
if dpro is not None:
dpro['noerr'][expid] = dpro['noerr'][expid] + 1
onlyhead = True
# === 4 Automatic source extraction for imaging
bfound = "N/A"
if extract and insmode != 'SPC':
# ---- WARNING: de-rotation for classical imaging with pupil tracking still to be implemented!
# --- first find the source positions
if findbeams:
msg = funname + ": Trying to detect beams in combined image..."
_print_log_info(msg, logfile, empty=2)
try:
bfound, nowarn, beampos, fitparams = _find_beam_pos(
im=totim,
head=head0,
searcharea=searcharea,
fitbox=0.5 * box,
nodpos='both',
verbose=verbose,
sourceext=sourceext,
AA_pos=AA_pos,
plot=plot,
tol=maxshift / pfov,
instrument=instrument,
insmode=insmode,
logfile=logfile,
chopthrow=chopthrow,
noddir=noddir,
filt=setup,
pfov=pfov,
searchsmooth=searchsmooth,
sigmaclip=sigmaclip)
now += nowarn
except:
e = sys.exc_info()
msg = (funname +
": WARNING: Beam Position could not be found: \n" +
str(e[1]) + ' ' + str(traceback.print_tb(e[2])) +
"\nContinue assuming the positions...")
_print_log_info(msg, logfile)
bfound = "fail"
msg = funname + ": Result of beam search: " + bfound
_print_log_info(msg, logfile)
if bfound != "fail":
# --- retrieve the coordinates of the found position
wcs = WCS(head0)
bra, bdec = wcs.wcs_pix2world(beampos[0, 1], beampos[0, 0], 0)
bcoord = SkyCoord(ra=bra,
dec=bdec,
unit=(u.deg, u.deg),
frame='icrs')
tara = _fits_get_info(head0, keys="RA")
tadec = _fits_get_info(head0, keys="DEC")
angdist = _angular_distance(bra, bdec, tara, tadec)
nbeams = len(beampos)
bg = fitparams[0]
amp = fitparams[1]
fwhm = np.mean(fitparams[4:6])
axrat = np.max(fitparams[4:6]) / np.min(fitparams[4:6])
total = 0.25 * (np.pi * fitparams[1] * fitparams[4] *
fitparams[5]) / np.log(2.0)
angle = fitparams[6]
msg = (funname + ":\n" + " - Expected source position: " +
dpro["RA_hms"][expid] + " " + dpro["DEC_dms"][expid] +
"\n" + " - Found source params:\n" +
" - Position x,y [px]: " +
'{:.1f}'.format(beampos[0, 1]) + "," +
'{:.1f}'.format(beampos[0, 0]) + "\n" +
" - Position [wcs]: " + bcoord.to_string('hmsdms') +
"\n" + " - Angular distance [as]: " +
'{:.2f}'.format(angdist) + "\n" + " - BG: " +
str(bg) + "\n" + " - Amplitude: " + str(amp) +
"\n" + " - Total: " + str(total) + "\n" +
" - Aver. FWHM [px]: " + str(fwhm) + "\n" +
" - Aver. FWHM (min 0.3) [as]: " +
str(fwhm * pfov) + "\n" + " - Maj./min axis: " +
str(axrat) + "\n" + " - Angle: " + str(angle))
_print_log_info(msg, logfile)
else:
now += 1
else:
bfound = "not tried"
# === If bright enough, try Fine-centered addition
# --- go over all nodding pairs and extract all the sub-images
if bfound == "first attempt" or bfound == "global fit":
msg = funname + ": Trying to detect and extract beams from individual nods..."
_print_log_info(msg, logfile, empty=2)
suffix = 'fine'
es = extract_beams(nodims,
beampos,
box,
noddir,
head0,
logfile,
verbose,
debug,
outfolder,
outnames[0],
suffix,
nodalign=True,
updatedb=True,
dpro=dpro,
pfov=pfov,
fitparams=fitparams,
maxshift=maxshift,
sigmaclip=sigmaclip,
expid=expid,
onlyhead=onlyhead)
else:
es = 1
# ================= Fixed extraction, if necessary
# --- if a beam was detected do an extraction
if findbeams and bfound != "fail" and es > 0:
# --- first do some tests on fitted beams
s = np.shape(nodims[0])
use = np.full(nbeams, True)
for k in range(nbeams):
# --- first test that at least one of the beams is on the detector
if ((beampos[k, 0] >= s[0]) | (beampos[k, 1] >= s[1])
| (beampos[k, 0] <= 0) | (beampos[k, 1] <= 0)):
use[k] = False
# --- make sure that at least one beam is on the detector
if np.sum(use) == 0:
msg = (
" - WARNING: None of fitted beams on detector! Using reference position..."
)
_print_log_info(msg, logfile)
now += 1
bfound = "fail"
else:
msg = (
funname +
": Extracting beams from individual nods at found fixed position ..."
)
_print_log_info(msg, logfile, empty=1)
# --- label fits file depending on beam recovery mode/success:
if bfound == "first attempt" or bfound == "global fit" or bfound == "global smooth":
suffix = 'fixadet'
elif bfound == "only one found":
suffix = 'fix1det'
es = extract_beams(nodims,
beampos,
box,
noddir,
head0,
logfile,
verbose,
debug,
outfolder,
outnames[0],
suffix,
updatedb=True,
dpro=dpro,
pfov=pfov,
sigmaclip=sigmaclip,
expid=expid,
onlyhead=onlyhead)
noe += es
# --- if none or only 1 beam was, or we did not search, estimate their
# position
if not findbeams or bfound == "fail" or bfound == "only one found":
# --- compute the expected beam positions
msg = (funname +
": Extraction using reference positions for the beams...")
_print_log_info(msg, logfile)
beampos = _calc_beampos(head=head0, verbose=verbose)
nbeams = len(beampos)
# fwhm = 7
# bg = np.nanmedian(totim)
# amp = np.nanmax(gaussian_filter(totim, sigma=3))
# --- Modify the calculated positions in case the user provided the beam
# position of chop A nod A
if AA_pos is not None:
xdif = AA_pos[1] - beampos[0, 1]
ydif = AA_pos[0] - beampos[0, 0]
beampos[:, 1] = beampos[:, 1] + xdif
beampos[:, 0] = beampos[:, 0] + ydif
suffix = 'fixest'
es = extract_beams(nodims,
beampos,
box,
noddir,
head0,
logfile,
verbose,
debug,
outfolder,
outnames[0],
suffix,
updatedb=False,
dpro=dpro,
pfov=pfov,
sigmaclip=sigmaclip,
expid=expid,
onlyhead=onlyhead)
noe += es
if bfound == "first attempt" or bfound == "global fit" or bfound == "global smooth":
dpro['all_beams_det'][expid] = "T"
else:
dpro['all_beams_det'][expid] = "F"
msg = (funname + ": Number of warnings for exposure: " + str(now))
_print_log_info(msg, logfile, empty=1, screen=False)
msg = (funname + ": Number of errors for exposure: " + str(noe))
_print_log_info(msg, logfile, screen=False)
return (now, noe)
def group_raws_to_obs(ftabraw, ftablog, ftabpro, maxgap=5,
logfile=None, overwrite=False):
"""
Look into the raw file table and identify individual observations/exposures
and print out a new table containing the information of the latter
PARAMETERS:
- maxgap: (default 5) provide the maximum time gap between two files in
min
"""
funname = "GROUP_RAWS_TO_OBS"
if logfile is None:
logfile = ftabraw.replace("_raw.csv", ".log")
if not os.path.isfile(logfile):
mode = 'w'
else:
mode = 'a'
_print_log_info('\n\nNew execution of ' + funname + '\n',
logfile, mode=mode)
# --- column names for the product table and their data format
cols = OrderedDict({
"expid": "{0:d}",
"targname": '{:s}',
"RA_deg": '{:.6f}',
"DEC_deg": '{:.5f}',
"RA_hms": '{:s}',
"DEC_dms": '{:s}',
"progid": '{:s}',
"obsid": "{0:d}",
"dateobs": '{:s}',
"mjd": '{:.8f}',
"tempno": "{0:d}",
"instrument": '{:s}',
"insmode": '{:s}',
"tempname": '{:s}',
"datatype": '{:s}',
"setup": '{:s}',
"pfov": '{:.4f}',
"dit": '{:.4f}',
"nodtime": '{:.0f}',
"noddir": '{:s}',
"jitter": '{:.1f}',
"chopthrow": '{:.1f}',
"chopangle": '{:.1f}',
"chopfreq": '{:.2f}',
"expnof": "{0:d}",
"nof": "{0:d}",
"exptime": '{:.0f}',
"grade": '{:s}',
"posang": '{:.1f}',
"absrot_mean": '{:.2f}',
"absrot_start": '{:.2f}',
"absrot_end": '{:.2f}',
"parang_mean": '{:.2f}',
"parang_start": '{:.2f}',
"parang_end": '{:.2f}',
"alt_mean": '{:.2f}',
"alt_start": '{:.2f}',
"alt_end": '{:.2f}',
"az_mean": '{:.2f}',
"az_start": '{:.2f}',
"az_end": '{:.2f}',
"airm_mean": '{:.3f}',
"airm_start": '{:.3f}',
"airm_end": '{:.3f}',
"pwv_median": '{:.2f}',
"pwv_stddev": '{:.2f}',
"skytemp_median": '{:.0f}',
"skytemp_stddev": '{:.0f}',
"pressure_median": '{:.2f}',
"pressure_stddev": '{:.2f}',
"humidity_median": '{:.1f}',
"humidity_stddev": '{:.1f}',
"temperature_median": '{:.2f}',
"temperature_stddev": '{:.2f}',
"m1temp_median": '{:.2f}',
"m1temp_stddev": '{:.2f}',
"winddir_median": '{:.0f}',
"winddir_stddev": '{:.1f}',
"windspeed_median": '{:.2f}',
"windspeed_stddev": '{:.2f}',
"cohertime_median": '{:.4f}',
"cohertime_stddev": '{:.4f}',
"asmfwhm_median": '{:.2f}',
"asmfwhm_stddev": '{:.2f}',
"iafwhm_median": '{:.2f}',
"iafwhm_stddev": '{:.2f}',
"iafwhmlo_median": '{:.2f}',
"iafwhmlo_stddev": '{:.2f}',
"chopamed_median": '{:.0f}',
"chopamed_stddev": '{:.2f}',
"chopastd_median": '{:.1f}',
"chopastd_stddev": '{:.2f}',
"chopbmed_median": '{:.0f}',
"chopbmed_stddev": '{:1f}',
"chopbstd_median": '{:.1f}',
"chopbstd_stddev": '{:.2f}',
"cdifmed_median": '{:.2f}',
"cdifmed_stddev": '{:.2f}',
"cdifstd_median": '{:.2f}',
"cdifstd_stddev": '{:.2f}'
})
# --- read the raw file
draw = ascii.read(ftabraw, header_start=0, delimiter=',',
guess=False)
# --- make all columns to MaskedColumns
draw = Table(draw, masked=True, copy=False) # Convert to masked table
# --- throw out entries with ERRORS
ids = [j for j, s in enumerate(draw['INSTRUME']) if 'CANNOT' not in s]
nbad = len(draw) - len(ids)
if nbad > 0:
msg = (funname + ": WARNING: Number of raw files with errors: "
+ str(nbad)
+ " --> Excluding them...")
_print_log_info(msg, logfile)
draw = draw[ids]
# --- data table integrity check
dupl = _duplicates_in_column(table=draw, colname='MJD-OBS')
if dupl is not None:
msg = (funname + ": ERROR: Duplicated files in raw table! MJDs: "
+ ', '.join([str(e) for e in dupl])
+ '. Exiting...')
_print_log_info(msg, logfile)
raise TypeError(msg)
dg = ascii.read(ftablog, header_start=0, delimiter=',',
guess=False)
# --- make sure that the raw files are sorted by date time (some might
# have been added later)
id = np.argsort(draw['MJD-OBS'])
draw = draw[id]
# --- throw out incomplete entries (e.g., non-nodded through slit images)
id = np.where(np.array(draw['nodexptime'], dtype=float) > 0)[0]
draw = draw[id]
n = len(draw)
# --- group the raw files into exposures
expid = np.zeros(n, dtype=int)
counter = 0
nof = 1
for i in np.arange(1, n):
if (draw['OBS_ID'][i-1] != draw['OBS_ID'][i]):
counter = counter + 1
elif (draw['TPLNO'][i-1] != draw['TPLNO'][i]):
counter = counter + 1
elif (draw['TPL_EXPNO'][i-1] >= draw['TPL_EXPNO'][i]):
counter = counter + 1
elif (draw['TPL_EXPNO'][i-1] == draw['TPL_NEXP'][i-1]):
counter = counter + 1
elif (draw['MJD-OBS'][i] - draw['MJD-OBS'][i-1] > maxgap/60.0/24.0):
counter = counter + 1
expid[i] = counter
# --- get the start MJDs of each exposure as unique identifier
nexp = counter + 1
mjds = np.zeros(nexp)
for i in range(nexp):
id = np.where(expid == i)[0]
mjds[i] = draw['MJD-OBS'][id[0]]
# --- make and backup copy just in case
if os.path.isfile(ftabpro):
shutil.copy(ftabpro, ftabpro.replace(".csv", "_backup.csv"))
# --- in case the file exists update and append information to dpro file
if os.path.isfile(ftabpro) and not overwrite:
new = False
dpro = ascii.read(ftabpro)
# --- change all string columns to object type so that their string
# length becomes variable
for col in dpro.itercols():
if col.dtype.kind in 'SU':
dpro.replace_column(col.name, col.astype('object'))
# --- make all columns to MaskedColumns
dpro = Table(dpro, masked=True, copy=False) # Convert to masked table
# --- if the file not exists generate the empty table structure
else:
new = True
# --- generate the output table
dpro = Table()
for c in cols.keys():
if 'd' in cols[c]:
dt = int
elif 's' in cols[c]:
dt = object
elif 'f' in cols[c]:
dt = float
dpro.add_column(MaskedColumn(np.ma.masked_all(nexp),
name=c, dtype=dt))
# --- loop over all the exposures by MJD:
# for i in tqdm(range(nexp)):
for i in range(nexp):
if not new:
# --- check if already an entry in the table for that exposure
id = np.where(np.isclose(dpro["mjd"], mjds[i], atol=1e-7,
rtol=0))[0]
# --- otherwise add a row at the end of the table
if len(id) == 0:
dpro.add_row()
id = len(dpro)-1
# --- make sure that the columns have fill values of the right type
for c in dpro.colnames:
if dpro[c].dtype == "object":
dpro[c][id] = ""
else:
id = id[0]
# --- for new table just take the i-th row
else:
id = i
# --- now gather all files belonging to that exposure
ids = np.where(expid == i)[0]
j = ids[0]
#print(i, mjds[i], id, dpro["mjd"][id], j)
# print(i,id,j)
nof = len(ids)
dpro["nof"][id] = nof
dpro["expid"][id] = i
if nof == 1 and "MoveToSlit" not in draw['TPL_ID'][j]:
msg = ("WARNING: Only one raw file found for exposure. "
+ "Exp ID: " + str(i)
+ "; DATE-OBS: " + draw['DATE-OBS'][j]
+ "; TPL_ID: " + draw['TPL_ID'][j]
)
_print_log_info(msg, logfile)
# print(i,j, draw['CYCSUM'][j])
if "Burst" in draw['TPL_ID'][j]:
dpro["datatype"][id] = 'burst'
elif (draw['CYCSUM'][j] == 'False') | (draw['CYCSUM'][j] == 'F'):
dpro["datatype"][id] = 'halfcyc'
else:
dpro["datatype"][id] = 'cycsum'
dpro["dateobs"][id] = draw['DATE-OBS'][j][0:19]
# --- fetch the grade and add it
idg = [g for g, s in enumerate(dg['filename'])
if dpro["dateobs"][id] in s]
if idg:
dpro["grade"][id] = dg['grade'][idg[0]]
else:
dpro["grade"][id] = "N"
# --- fill in the other values <--- would be better to get that from
# fits_get_info in the future! For now hard-coded FILT3 for ISAAC
if draw['INSTRUME'][j] == "VISIR":
if draw['insmode'][j] == 'IMG':
dpro["setup"][id] = str(draw['INS_FILT1_NAME'][j])
elif draw['insmode'][j] == 'ACQ-SPC-SPC':
dpro["setup"][id] = str(draw['INS_FILT2_NAME'][j])
elif draw['insmode'][j] == 'ACQ-IMG-SPC':
dpro["setup"][id] = str(draw['INS_FILT1_NAME'][j])
elif draw['insmode'][j] == 'SPC':
dpro["setup"][id] = str(draw['INS_SLIT1_WID'][j])
else:
print(draw['insmode'][j])
raise ValueError()
elif draw['INSTRUME'][j] == "ISAAC":
if draw['insmode'][j] == 'IMG':
if draw['INS_FILT3_NAME'][j] != "open":
dpro["setup"][id] = str(draw['INS_FILT3_NAME'][j])
else:
dpro["setup"][id] = str(draw['INS_FILT4_NAME'][j])
elif draw['insmode'][j] == 'SPC':
dpro["setup"][id] = str(draw['INS_FILT3_NAME'][j])
# --- if pupil tracking was used add that info to the mode
if draw['ALTAZTRACK'][j] == 'True':
dpro["insmode"][id] = str(draw['insmode'][j]) + "-PT"
else:
dpro["insmode"][id] = str(draw['insmode'][j])
try:
dpro["exptime"][id] = float(draw['nodexptime'][j])*nof
except:
dpro["exptime"][id] = np.ma.masked
dpro["targname"][id] = draw['TARG_NAME'][j]
dpro["RA_deg"][id] = draw['RA'][j]
dpro["DEC_deg"][id] = draw['DEC'][j]
coord = SkyCoord(ra=draw['RA'][j], dec=draw['DEC'][j], unit=(u.deg, u.deg), frame='icrs')
dpro["RA_hms"][id] = coord.to_string('hmsdms').split()[0]
dpro["DEC_dms"][id] = coord.to_string('hmsdms').split()[1]
dpro["progid"][id] = draw['PROG_ID'][j]
dpro["obsid"][id] = draw['OBS_ID'][j]
dpro["mjd"][id] = mjds[i]
dpro["instrument"][id] = draw['INSTRUME'][j]
dpro["tempno"][id] = draw['TPLNO'][j]
dpro["tempname"][id] = draw['TPL_ID'][j]
dpro["expnof"][id] = draw['TPL_NEXP'][j]
dpro["pfov"][id] = draw['PFOV'][j]
if not draw['SEQ1_DIT'].mask[j]:
dpro["dit"][id] = draw['SEQ1_DIT'][j]
# else:
# dpro["dit"][id] = -1
# print(draw['CHOPNOD_DIR'][j], type(draw['CHOPNOD_DIR'][j]))
if not draw['CHOPNOD_DIR'].mask[j]:
dpro["noddir"][id] = draw['CHOPNOD_DIR'][j]
else:
dpro["noddir"][id] = "None?"
if not draw['JITTER_WIDTH'].mask[j]:
dpro["jitter"][id] = draw['JITTER_WIDTH'][j]
else:
dpro["jitter"][id] = 0
if not draw['CHOP_THROW'].mask[j]:
dpro["chopthrow"][id] = draw['CHOP_THROW'][j]
# else:
# dpro["chopthrow"][id] = -1
if not draw['CHOP_POSANG'].mask[j]:
dpro["chopangle"][id] = draw['CHOP_POSANG'][j]
# else:
# dpro["chopthrow"][id] = -1
if not draw['CHOP_FREQ'].mask[j]:
dpro["chopfreq"][id] = draw['CHOP_FREQ'][j]
# else:
# dpro["chopfreq"][id] = -1
# --- actual time between two nods
# print(i, id,j, nof)
if nof > 1:
dpro["nodtime"][id] = (draw['MJD-OBS'][j+nof-1]
- draw['MJD-OBS'][j]) / (nof -1) * 24 * 3600
# --- now the changing telescope parameters
if not any(draw['POSANG'].mask[j:j+nof]):
dpro["posang"][id] = np.ma.mean(draw['POSANG'][j:j+nof])
fill_values(draw, dpro, 'ABSROT', 'absrot', id, j, nof,
ktype='tel', deg=True)
fill_values(draw, dpro, 'PARANG', 'parang', id, j, nof,
ktype='tel')
fill_values(draw, dpro, 'TEL_ALT', 'alt', id, j, nof, ktype='tel')
fill_values(draw, dpro, 'TEL_AZ', 'az', id, j, nof, ktype='tel', deg=True)
fill_values(draw, dpro, 'TEL_AIRM', 'airm', id, j, nof, ktype='tel')
# --- now the ambient parameters
fill_values(draw, dpro, 'IWV', 'pwv', id, j, nof,
ktype='ambi', lowlim=0, uplim=10)
fill_values(draw, dpro, 'IRSKY_TEMP', 'skytemp', id, j, nof,
ktype='ambi', lowlim=-150, uplim=-35)
fill_values(draw, dpro, 'AMBI_PRES', 'pressure', id, j, nof,
ktype='ambi')
fill_values(draw, dpro, 'RHUM', 'humidity', id, j, nof,
ktype='ambi')
fill_values(draw, dpro, 'AMBI_TEMP', 'temperature', id, j, nof,
ktype='ambi')
fill_values(draw, dpro, 'M1_TEMP', 'm1temp', id, j, nof,
ktype='ambi')
fill_values(draw, dpro, 'WINDDIR', 'winddir', id, j, nof,
ktype='ambi', deg=True)
fill_values(draw, dpro, 'WINDSP', 'windspeed', id, j, nof,
ktype='ambi')
fill_values(draw, dpro, 'TAU0', 'cohertime', id, j, nof,
ktype='ambi')
fill_values(draw, dpro, 'TEL_AMBI_FWHM', 'asmfwhm', id, j, nof,
ktype='ambi', lowlim=0.1, uplim=3)
fill_values(draw, dpro, 'IA_FWHM', 'iafwhm', id, j, nof, ktype='ambi',
lowlim=0.1, uplim=3)
fill_values(draw, dpro, 'IA_FWHMLINOBS', 'iafwhmlo', id, j, nof,
lowlim=0.1, uplim=3, ktype='ambi')
# --- now the background parameters
fill_values(draw, dpro, 'chopamed', 'chopamed', id, j, nof, ktype='bg')
fill_values(draw, dpro, 'chopastd', 'chopastd', id, j, nof, ktype='bg')
fill_values(draw, dpro, 'chopbmed', 'chopbmed', id, j, nof, ktype='bg')
fill_values(draw, dpro, 'chopbstd', 'chopbstd', id, j, nof, ktype='bg')
fill_values(draw, dpro, 'cdifmed', 'cdifmed', id, j, nof, ktype='bg')
fill_values(draw, dpro, 'cdifstd', 'cdifstd', id, j, nof, ktype='bg')
# --- check is the PWV STD was larger during a file than in between
if not any(draw['IWVSTD'].mask[j:j+nof]):
iwvstd = np.ma.mean(draw['IWVSTD'][j:j+nof])
if dpro['pwv_stddev'][id] is not None:
if (iwvstd > dpro['pwv_stddev'][id]) & (iwvstd > 0):
dpro['pwv_stddev'][id] = iwvstd
else:
dpro['pwv_stddev'][id] = iwvstd
# --- clean some bad values:
if dpro['exptime'][id] == 0:
dpro['exptime'][id] = None
# --- write the finished table sorted by MJD
dpro.sort('mjd')
# --- write out results
# print(len(dpro), ftabpro)
# masked = check_masked(dpro)
# if len(masked) > 0:
# msg = (funname + ": WARNING: DPRO contains masked values: " + str(masked))
# if logfile is not None:
# _print_log_info(msg, logfile)
# # raise ValueError(msg)
# nones = check_none(dpro)
# if len(nones) > 0:
# msg = (funname + ": WARNING: DPRO contains None's: " + str(nones))
# if logfile is not None:
# _print_log_info(msg, logfile)
# # raise ValueError(msg)
# # return(dpro)
dpro.write(ftabpro, delimiter=',', format='ascii',
fill_values=[(ascii.masked, '')], formats=cols,
overwrite=True)
# --- update the draw table with the expid column
if 'EXPID' in draw.colnames:
draw['EXPID'] = expid
else:
newCol = Column(expid, name='EXPID')
draw.add_column(newCol)
draw.write(ftabraw, delimiter=',', format='ascii',
fill_values=[(ascii.masked, '')], overwrite=True)
msg = (" - Number of separate exposures found: " + str(counter+1))
_print_log_info(msg, logfile)
return(dpro)
def extract_beams(nodims,
beampos,
box,
noddir,
head0,
logfile,
verbose,
debug,
outfolder,
outbasename,
suffix,
nodalign=False,
updatedb=False,
dpro=None,
fitparams=None,
pfov=None,
expid=None,
onlyhead=False,
maxshift=None,
sigmaclip=None):
# === 4.1 Blind addition
# --- go over all nodding pairs and extract all the sub-images
# first blindly
funname = "EXTRACT_BEAMS"
# nowarn = 0
noerr = 0
subims = []
# titles = []
nbeams = len(beampos[:, 0])
nnods = len(nodims)
beamposstr = ""
for b in range(nbeams):
beamposstr = (beamposstr + "{:3.0f}".format(beampos[b, 0]) + ", " +
"{:3.0f}".format(beampos[b, 1]) + " | ")
msg = (" - No of beams: " + str(nbeams) + "\n" + " - Beampos: " +
beamposstr + "\n")
_print_log_info(msg, logfile)
# --- exclude beams that are not on the detector
s = np.shape(nodims[0])
use = np.full(nbeams, True)
beamsign = np.zeros(nbeams)
for k in range(nbeams):
beamsign[k] = (-1)**(np.ceil(0.5 * k))
if ((beampos[k, 0] >= s[0]) | (beampos[k, 1] >= s[1])
| (beampos[k, 0] <= 0) | (beampos[k, 1] <= 0)):
msg = (" - Beampos not on frame: " + str(beampos[k, 0]) + ', ' +
str(beampos[k, 1]) + ". Exclude...")
_print_log_info(msg, logfile)
use[k] = False
beampos = beampos[use, :]
beamsign = beamsign[use]
nbeams = len(beampos)
# --- make sure that at least one beam is on the detector
if nbeams == 0:
msg = (" - Error: No beam on detector: " + str(beampos[k, 0]) + ', ' +
str(beampos[k, 1]) + ". Exiting...")
_print_log_info(msg, logfile)
return (noerr + 1)
# --- for the case of fine-centred extraction
if nodalign:
# --- get the expected beam parameters from the fit
guessbg = fitparams[0]
guessamp = np.abs(fitparams[1])
guessFWHM = np.mean(fitparams[4:6])
minamp = 0.3 * guessamp
minFWHM = 0.5 * guessFWHM
maxFWHM = 2 * guessFWHM
maxshift /= pfov # maxshift in px
fitbox = int(np.max([6 * guessFWHM, 0.25 * box]))
nexp = nnods * nbeams
nfound = 0
for j in tqdm(range(nnods)):
for k in range(nbeams):
try:
params, _, _ = _find_source(nodims[j] * beamsign[k],
guesspos=beampos[k, :],
searchbox=box,
fitbox=fitbox,
method='mpfit',
guessbg=guessbg,
guessamp=guessamp,
guessFWHM=guessFWHM,
minamp=minamp,
minFWHM=minFWHM,
maxFWHM=maxFWHM)
except:
if verbose:
msg = (
" - WARNING: Could not find source. Continuing...")
_print_log_info(msg, logfile)
# if dpro is not None:
# dpro['nowarn'][expid] = dpro['nowarn'][expid] + 1
continue
# print('params: ', params)
# print('params[2:4]', params[2:4])
# print('beamsign: ', beamsign)
# print('box: ', box)
# print('i,j: ',i,j)
cim = _crop_image(nodims[j] * beamsign[k],
box=box,
cenpos=params[2:4],
silent=True)
dist = beampos[k, :] - params[2:4]
if verbose:
msg = (" - Nod: " + str(j) + " Beam: " + str(k) +
" Detected shift: " + str(dist))
_print_log_info(msg, logfile)
bg_t = params[0]
amp_t = params[1]
fwhm_t = np.mean(params[4:6])
axrat_t = np.max(params[4:6]) / np.min(params[4:6])
total_t = 0.25 * (np.pi * params[1] * params[4] *
params[5]) / np.log(2.0)
angle_t = params[6]
msg = (" - Found source params:\n" + " - BG: " +
str(bg_t) + "\n" + " - Amplitude: " +
str(amp_t) + "\n" + " - Total: " +
str(total_t) + "\n" + " - Aver. FWHM [px]: " +
str(fwhm_t) + "\n" + " - Aver. FWHM [as]: " +
str(fwhm_t * pfov) + "\n" +
" - Maj./min axis: " + str(axrat_t) + "\n" +
" - Angle: " + str(angle_t))
_print_log_info(msg, logfile)
if np.sqrt(dist[0]**2 + dist[1]**2) > maxshift:
if verbose:
msg = (" - WARNING: shift too large! Exclude frame")
_print_log_info(msg, logfile)
# if dpro is not None:
# dpro['nowarn'][expid] = dpro['nowarn'][expid] + 1
# noe = noe+1
continue
# --- all double beams have to be halved but added twice
if (noddir == "PARALLEL") & (beamsign[k] > 0):
subims.append(0.5 * cim)
subims.append(0.5 * cim)
else:
subims.append(cim)
nfound += 1
# titles.append("Nod: " + str(j) + " Beam: " + str(k))
msg = (funname + ": Number of detected beams/expected: " +
str(nfound) + "/" + str(nexp))
_print_log_info(msg, logfile)
# --- if object not detected in individual nods give warning
if nfound == 0:
msg = (
funname +
": No source detected in the individual nods! Source probably too faint..."
)
_print_log_info(msg, logfile)
return (noerr + 1)
# -- also if <50% of the beams were found abort reject the fine
# tuned extraction and proceed with bling
elif nfound < 0.5 * nexp:
msg = (
funname +
": <50% of the beams detected in individual nods! Source probably too faint... "
)
_print_log_info(msg, logfile)
return (noerr + 1)
# --- fixed extraction
else:
for j in tqdm(range(nnods)):
for k in range(nbeams):
cim = _crop_image(nodims[j] * beamsign[k],
box=box,
cenpos=beampos[k, :],
silent=True)
# --- all double beams have to be halved but added twice
if (noddir == "PARALLEL") & (beamsign[k] > 0):
subims.append(0.5 * cim)
subims.append(0.5 * cim)
else:
subims.append(cim)
# --- update the WCS in the fits header
newhead = update_wcs(head0, beampos[0, 1], beampos[0, 0], box)
# --- write out
if debug:
fout = (outfolder + '/' + outbasename + '_all_extr_cube.fits')
fits.writeto(fout, subims, newhead, overwrite=True)
# fout = fout.replace(".fits", "_log.png")
# I.make_gallery(ims=subims, outname=fout, pfovs=pfov, log=True,
# papercol=2, ncols=nbeams, cmap='gnuplot2', titles=titles,
# inv=False, permin=40, permax=99.9, titcols='white')
#
# fout = fout.replace("_log.png", "_lin.png")
# I.make_gallery(ims=subims, outname=fout, pfovs=pfov, log=False,
# papercol=2, ncols=nbeams, cmap='gnuplot2', titles=titles,
# inv=False, permin=40, permax=99.9, titcols='white')
# --- average all the sub-images
totim = np.nanmean(subims, axis=0)
# --- optional sigma clipping for the beam search
if sigmaclip is not None:
nrepl, totim = _replace_hotpix(totim,
sigmathres=sigmaclip[0],
niters=sigmaclip[1],
verbose=verbose)
msg = (funname + ": Number of replaced (hot) pixels: " + str(nrepl))
_print_log_info(msg, logfile)
if updatedb:
# --- position of the found source
tara = _fits_get_info(head0, keys="RA")
tadec = _fits_get_info(head0, keys="DEC")
wcs = WCS(head0)
ra, dec = wcs.wcs_pix2world(beampos[0, 1], beampos[0, 0], 0)
coord = SkyCoord(ra=ra, dec=dec, unit=(u.deg, u.deg), frame='icrs')
dpro['GF_x_px'][expid] = beampos[0, 1]
dpro['GF_y_px'][expid] = beampos[0, 0]
dpro['GF_RA_hms'][expid] = coord.to_string('hmsdms').split()[0]
dpro['GF_DEC_dms'][expid] = coord.to_string('hmsdms').split()[1]
dpro['GF_angdist_as'][expid] = _angular_distance(ra, dec, tara, tadec)
try:
update_base_params(totim,
newhead,
dpro,
pfov,
expid,
onlyhead=onlyhead)
except:
e = sys.exc_info()
msg = (funname + ": ERROR: Failed to update base parameters: \n" +
str(e[1]) + '' + str(traceback.print_tb(e[2])))
_print_log_info(msg, logfile)
noerr = noerr + 1
fout = (outfolder + '/' + outbasename + '_all_extr-' + suffix + '.fits')
fits.writeto(fout, totim, newhead, overwrite=True)
pout = fout.replace(".fits", "_log.png")
_simple_image_plot(totim, pout, scale="log", pfov=pfov, cenax=True)
pout = fout.replace(".fits", "_lin.png")
# --- if the source is probably faint, use lower colour thresholds
if suffix == "fixest" or suffix == "fix1det":
percentile = 1
else:
percentile = None
_simple_image_plot(totim,
pout,
pfov=pfov,
cenax=True,
percentile=percentile)
# --- to be uncommented once the make_gallery routine has been changed
# try:
# fout = fout.replace(".fits", "_log.png")
# I.make_gallery(ims=[totim], outname=fout, pfovs=pfov, log=True,
# papercol=1, ncols=1, cmap='gnuplot2',
# inv=False, permin=40, permax=99.9, latex=False)
# fout = fout.replace("_log.png", "_lin.png")
# I.make_gallery(ims=[totim], outname=fout, pfovs=pfov, log=False,
# papercol=1, ncols=1, cmap='gnuplot2',
# inv=False, permin=40, permax=99.9, latex=False)
# except:
# e = sys.exc_info()
# msg = (funname + ": ERROR: Gallery plots failed to be created: \n"
# + str(e[1]) + '' + str(traceback.print_tb(e[2])))
# _print_log_info(msg, logfile)
# noe = noe + 1
msg = " - Output written: " + fout
_print_log_info(msg, logfile)
return (noerr)