def psf_reduce(args=None):
"""``psf_reduce [source] rfile (run first twait tmax | flist) log lplot implot
(ccd nx msub xlo xhi ylo yhi iset (ilo ihi | plo phi))``
Performs PSF photometry on a sequence of multi-CCD images, plotting
lightcurves as images come in. It performs PSF photometry on specific
targets, defined in an aperture file using |psf_setaper| or |setaper|.
Aperture repositioning is identical to |reduce|, and the PSF used in photometry
is set by fitting well separated reference stars. For the best results, a
suitable strategy is to only reduce data from a small window near the target.
Set well separated bright stars as reference apertures and link fainter, or
blended objects to the references.
|psf_reduce| can source data from both the ULTRACAM and HiPERCAM servers, from
local 'raw' ULTRACAM and HiPERCAM files (i.e. .xml + .dat for ULTRACAM, 3D
FITS files for HiPERCAM) and from lists of HiPERCAM '.hcm' files. If you
have data from a different instrument you should convert into the
FITS-based hcm format.
|psf_reduce| is primarily configured from a file with extension ".red". This
contains a series of directives, e.g. to say how to re-position the apertures.
An initial reduce file is best generated with the script |genred| after you
have created an aperture file. This contains lots of help on what to do.
A |psf_reduce| run can be terminated at any point with ctrl-C without doing
any harm. You may often want to do this at the start in order to adjust
parameters of the reduce file.
Parameters:
source : string [hidden]
Data source, five options:
| 'hs': HiPERCAM server
| 'hl': local HiPERCAM FITS file
| 'us': ULTRACAM server
| 'ul': local ULTRACAM .xml/.dat files
| 'hf': list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab' or
converted from foreign data formats.
rfile : string
the "reduce" file, i.e. ASCII text file suitable for reading by
ConfigParser. Best seen by example as it has many parts. Generate
a starter file with |genred|.
run : string [if source ends 's' or 'l']
run number to access, e.g. 'run034'
first : int [if source ends 's' or 'l']
exposure number to start from. 1 = first frame; set = 0 to
always try to get the most recent frame (if it has changed)
last : int [if source ends 's' or 'l', hidden]
last frame to reduce. 0 to just continue until the end. This is
not prompted for by default and must be set explicitly. It
defaults to 0 if not set. Its purpose is to allow accurate
profiling tests.
twait : float [if source ends 's'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [if source ends 's'; hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
flist : string [if source ends 'f']
name of file list
log : string
log file for the results
tkeep : float
maximum number of minutes of data to store in internal buffers, 0
for the lot. When large numbers of frames are stored, performance
can be slowed (although I am not entirely clear why) in which case
it makes sense to lose the earlier points (without affecting the
saving to disk). This parameter also gives operation similar to that
of "max_xrange" parameter in the ULTRACAM pipeline whereby just
the last few minutes are shown.
lplot : bool
flag to indicate you want to plot the light curve. Saves time not
to especially in high-speed runs.
implot : bool
flag to indicate you want to plot images.
ccd : string [if implot]
CCD(s) to plot, '0' for all, '1 3' to plot '1' and '3' only, etc.
nx : int [if implot]
number of panels across to display.
msub : bool [if implot]
subtract the median from each window before scaling for the
image display or not. This happens after any bias subtraction.
xlo : float [if implot]
left-hand X-limit for plot
xhi : float [if implot]
right-hand X-limit for plot (can actually be < xlo)
ylo : float [if implot]
lower Y-limit for plot
yhi : float [if implot]
upper Y-limit for plot (can be < ylo)
iset : string [if implot]
determines how the intensities are determined. There are three
options: 'a' for automatic simply scales from the minimum to the
maximum value found on a per CCD basis. 'd' for direct just takes
two numbers from the user. 'p' for percentile dtermines levels
based upon percentiles determined from the entire CCD on a per CCD
basis.
ilo : float [if implot and iset='d']
lower intensity level
ihi : float [if implot and iset='d']
upper intensity level
plo : float [if implot and iset='p']
lower percentile level
phi : float [if implot and iset='p']
upper percentile level
"""
command, args = utils.script_args(args)
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("source", Cline.GLOBAL, Cline.HIDE)
cl.register("rfile", Cline.GLOBAL, Cline.PROMPT)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("last", Cline.LOCAL, Cline.HIDE)
cl.register("twait", Cline.LOCAL, Cline.HIDE)
cl.register("tmax", Cline.LOCAL, Cline.HIDE)
cl.register("flist", Cline.LOCAL, Cline.PROMPT)
cl.register("log", Cline.GLOBAL, Cline.PROMPT)
cl.register("tkeep", Cline.GLOBAL, Cline.PROMPT)
cl.register("lplot", Cline.LOCAL, Cline.PROMPT)
cl.register("implot", Cline.LOCAL, Cline.PROMPT)
cl.register("ccd", Cline.LOCAL, Cline.PROMPT)
cl.register("nx", Cline.LOCAL, Cline.PROMPT)
cl.register("msub", Cline.GLOBAL, Cline.PROMPT)
cl.register("iset", Cline.GLOBAL, Cline.PROMPT)
cl.register("ilo", Cline.GLOBAL, Cline.PROMPT)
cl.register("ihi", Cline.GLOBAL, Cline.PROMPT)
cl.register("plo", Cline.GLOBAL, Cline.PROMPT)
cl.register("phi", Cline.LOCAL, Cline.PROMPT)
cl.register("xlo", Cline.GLOBAL, Cline.PROMPT)
cl.register("xhi", Cline.GLOBAL, Cline.PROMPT)
cl.register("ylo", Cline.GLOBAL, Cline.PROMPT)
cl.register("yhi", Cline.GLOBAL, Cline.PROMPT)
# get inputs
source = cl.get_value(
"source",
"data source [hs, hl, us, ul, hf]",
"hl",
lvals=("hs", "hl", "us", "ul", "hf"),
)
# set some flags
server_or_local = source.endswith("s") or source.endswith("l")
# the reduce file
rfilen = cl.get_value("rfile", "reduce file",
cline.Fname("reduce.red", hcam.RED))
try:
rfile = Rfile.read(rfilen)
except hcam.HipercamError as err:
# abort on failure to read as there are many ways to get reduce
# files wrong
print(err, file=sys.stderr)
exit(1)
if server_or_local:
resource = cl.get_value("run", "run name", "run005")
first = cl.get_value("first", "first frame to reduce", 1, 0)
cl.set_default("last", 0)
last = cl.get_value("last", "last frame to reduce", 0, 0)
if last and last < first:
print("Cannot set last < first unless last == 0")
print("*** psf_reduce aborted")
exit(1)
twait = cl.get_value("twait",
"time to wait for a new frame [secs]", 1.0,
0.0)
tmx = cl.get_value("tmax",
"maximum time to wait for a new frame [secs]",
10.0, 0.0)
else:
resource = cl.get_value("flist", "file list",
cline.Fname("files.lis", hcam.LIST))
first = 1
last = 0
log = cl.get_value(
"log",
"name of log file to store results",
cline.Fname("reduce.log", hcam.LOG, cline.Fname.NEW),
)
tkeep = cl.get_value(
"tkeep",
"number of minute of data to"
" keep in internal buffers (0 for all)",
0.0,
0.0,
)
lplot = cl.get_value("lplot", "do you want to plot light curves?",
True)
implot = cl.get_value("implot", "do you want to plot images?", True)
if implot:
# define the panel grid. first get the labels and maximum
# dimensions
ccdinf = spooler.get_ccd_pars(source, resource)
nxdef = cl.get_default("nx", 3)
if len(ccdinf) > 1:
ccd = cl.get_value("ccd", "CCD(s) to plot [0 for all]", "0")
if ccd == "0":
ccds = list(ccdinf.keys())
else:
ccds = ccd.split()
if len(ccds) > 1:
nxdef = min(len(ccds), nxdef)
cl.set_default("nx", nxdef)
nx = cl.get_value("nx", "number of panels in X", 3, 1)
else:
nx = 1
else:
nx = 1
ccds = list(ccdinf.keys())
# define the display intensities
msub = cl.get_value("msub", "subtract median from each window?",
True)
iset = cl.get_value(
"iset",
"set intensity a(utomatically),"
" d(irectly) or with p(ercentiles)?",
"a",
lvals=["a", "d", "p"],
)
plo, phi = 5, 95
ilo, ihi = 0, 1000
if iset == "d":
ilo = cl.get_value("ilo", "lower intensity limit", 0.0)
ihi = cl.get_value("ihi", "upper intensity limit", 1000.0)
elif iset == "p":
plo = cl.get_value("plo", "lower intensity limit percentile",
5.0, 0.0, 100.0)
phi = cl.get_value("phi", "upper intensity limit percentile",
95.0, 0.0, 100.0)
# region to plot
for i, cnam in enumerate(ccds):
nxtot, nytot, nxpad, nypad = ccdinf[cnam]
if i == 0:
xmin, xmax = float(-nxpad), float(nxtot + nxpad + 1)
ymin, ymax = float(-nypad), float(nytot + nypad + 1)
else:
xmin = min(xmin, float(-nxpad))
xmax = max(xmax, float(nxtot + nxpad + 1))
ymin = min(ymin, float(-nypad))
ymax = max(ymax, float(nytot + nypad + 1))
xlo = cl.get_value("xlo",
"left-hand X value",
xmin,
xmin,
xmax,
enforce=False)
xhi = cl.get_value("xhi",
"right-hand X value",
xmax,
xmin,
xmax,
enforce=False)
ylo = cl.get_value("ylo",
"lower Y value",
ymin,
ymin,
ymax,
enforce=False)
yhi = cl.get_value("yhi",
"upper Y value",
ymax,
ymin,
ymax,
enforce=False)
else:
xlo, xhi, ylo, yhi = None, None, None, None
# save list of parameter values for writing to the reduction file
plist = cl.list()
################################################################
#
# all the inputs have now been obtained. Get on with doing stuff
if implot:
plot_lims = (xlo, xhi, ylo, yhi)
else:
plot_lims = None
imdev, lcdev, spanel, tpanel, xpanel, ypanel, lpanel = setup_plots(
rfile, ccds, nx, plot_lims, implot, lplot)
# a couple of initialisations
total_time = 0 # time waiting for new frame
# dictionary of dictionaries for looking up the window associated with a
# given aperture, i.e. mccdwins[cnam][apnam] give the name of the Window.
mccdwins = {}
if lplot:
lbuffer, xbuffer, ybuffer, tbuffer, sbuffer = setup_plot_buffers(rfile)
else:
lbuffer, xbuffer, ybuffer, tbuffer, sbuffer = None, None, None, None, None
############################################
#
# open the log file and write headers
#
with LogWriter(log, rfile, hipercam_version, plist) as logfile:
ncpu = rfile["general"]["ncpu"]
if ncpu > 1:
pool = multiprocessing.Pool(processes=ncpu)
else:
pool = None
# whether settings have been initialised
initialised = False
# containers for the processed and raw MCCD groups
# and their frame numbers
pccds, mccds, nframes = [], [], []
##############################################
#
# Finally, start winding through the frames
#
with spooler.data_source(source, resource, first, full=False) as spool:
# 'spool' is an iterable source of MCCDs
for nf, mccd in enumerate(spool):
if server_or_local:
# Handle the waiting game ...
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmx, total_time)
if give_up:
# Giving up, but need to handle any partially filled
# frame group
if len(mccds):
# finish processing remaining frames. This step
# will only occur if we have at least once passed
# to later stages during which read and gain will
# be set up
results = processor(pccds, mccds, nframes)
# write out results to the log file
alerts = logfile.write_results(results)
# print out any accumulated alert messages
if len(alerts):
print("\n".join(alerts))
update_plots(
results,
rfile,
implot,
lplot,
imdev,
lcdev,
pccd,
ccds,
msub,
nx,
iset,
plo,
phi,
ilo,
ihi,
xlo,
xhi,
ylo,
yhi,
lpanel,
xpanel,
ypanel,
tpanel,
spanel,
tkeep,
lbuffer,
xbuffer,
ybuffer,
tbuffer,
sbuffer,
)
print("psf_reduce stopped")
break
elif try_again:
continue
# indicate progress
if "NFRAME" in mccd.head:
nframe = mccd.head["NFRAME"]
else:
nframe = nf + 1
if last and nframe > last:
if len(mccds):
# finish processing remaining frames
results = processor(pccds, mccds, nframes)
# write out results to the log file
alerts = logfile.write_results(results)
# print out any accumulated alert messages
if len(alerts):
print("\n".join(alerts))
update_plots(
results,
rfile,
implot,
lplot,
imdev,
lcdev,
pccd,
ccds,
msub,
nx,
iset,
plo,
phi,
ilo,
ihi,
xlo,
xhi,
ylo,
yhi,
lpanel,
xpanel,
ypanel,
tpanel,
spanel,
tkeep,
lbuffer,
xbuffer,
ybuffer,
tbuffer,
sbuffer,
)
print("\nHave reduced up to the last frame set.")
print("psf_reduce finished")
break
print(
"Frame {:d}: {:s} [{:s}]".format(
nframe,
mccd.head["TIMSTAMP"],
"OK" if mccd.head.get("GOODTIME", True) else "NOK",
),
end="" if implot else "\n",
)
if not initialised:
# This is the first frame which allows us to make
# some checks and initialisations.
read, gain, ok = initial_checks(mccd, rfile)
# Define the CCD processor function object
processor = ProcessCCDs(rfile, read, gain, ccdproc, pool)
# set flag to show we are set
if not ok:
break
initialised = True
# De-bias the data. Retain a copy of the raw data as 'mccd'
# in order to judge saturation. Processed data called 'pccd'
if rfile.bias is not None:
# subtract bias
pccd = mccd - rfile.bias
else:
# no bias subtraction
pccd = mccd.copy()
if rfile.flat is not None:
# apply flat field to processed frame
pccd /= rfile.flat
# Acummulate frames into processing groups for faster
# parallelisation
pccds.append(pccd)
mccds.append(mccd)
nframes.append(nframe)
if len(pccds) == rfile["general"]["ngroup"]:
# parallel processing. This should usually be the first
# points at which it takes place
results = processor(pccds, mccds, nframes)
# write out results to the log file
alerts = logfile.write_results(results)
# print out any accumulated alert messages
if len(alerts):
print("\n".join(alerts))
update_plots(
results,
rfile,
implot,
lplot,
imdev,
lcdev,
pccds[-1],
ccds,
msub,
nx,
iset,
plo,
phi,
ilo,
ihi,
xlo,
xhi,
ylo,
yhi,
lpanel,
xpanel,
ypanel,
tpanel,
spanel,
tkeep,
lbuffer,
xbuffer,
ybuffer,
tbuffer,
sbuffer,
)
# Reset the frame buffers
pccds, mccds, nframes = [], [], []
def rtplot(args=None):
"""``rtplot [source device width height] (run first (trim [ncol nrow])
twait tmax | flist) [pause plotall] ccd (nx) bias flat defect
setup [hurl] msub iset (ilo ihi | plo phi) xlo xhi ylo yhi (profit
[fdevice fwidth fheight method beta fwhm fwhm_min shbox smooth
splot fhbox hmin read gain thresh])``
Plots a sequence of images as a movie in near 'real time', hence
'rt'. Designed to be used to look at images coming in while at the
telescope, 'rtplot' comes with many options, a large number of which are
hidden by default, and many of which are only prompted if other arguments
are set correctly. If you want to see them all, invoke as 'rtplot prompt'.
This is worth doing once to know rtplot's capabilities.
rtplot can source data from both the ULTRACAM and HiPERCAM servers, from
local 'raw' ULTRACAM and HiPERCAM files (i.e. .xml + .dat for ULTRACAM, 3D
FITS files for HiPERCAM) and from lists of HiPERCAM '.hcm' files.
rtplot optionally allows the selection of targets to be fitted with
gaussian or moffat profiles, and, if successful, will plot circles of 2x
the measured FWHM in green over the selected targets. This option only
works if a single CCD is being plotted.
Parameters:
source : string [hidden]
Data source, five options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : ULTRACAM server
| 'ul' : local ULTRACAM .xml/.dat files
| 'hf' : list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab' or
converted from foreign data formats.
device : string [hidden]
Plot device. PGPLOT is used so this should be a PGPLOT-style name,
e.g. '/xs', '1/xs' etc. At the moment only ones ending /xs are
supported.
width : float [hidden]
plot width (inches). Set = 0 to let the program choose.
height : float [hidden]
plot height (inches). Set = 0 to let the program choose. BOTH width
AND height must be non-zero to have any effect
run : string [if source ends 's' or 'l']
run number to access, e.g. 'run034'
flist : string [if source ends 'f']
name of file list
first : int [if source ends 's' or 'l']
exposure number to start from. 1 = first frame; set = 0 to always
try to get the most recent frame (if it has changed). For data
from the |hiper| server, a negative number tries to get a frame not
quite at the end. i.e. -10 will try to get 10 from the last
frame. This is mainly to sidestep a difficult bug with the
acquisition system.
trim : bool [if source starts with 'u']
True to trim columns and/or rows off the edges of windows nearest
the readout. This is particularly for ULTRACAM windowed data where
the first few rows and columns can contain bad data.
ncol : int [if trim, hidden]
Number of columns to remove (on left of left-hand window, and right
of right-hand windows)
nrow : int [if trim, hidden]
Number of rows to remove (bottom of windows)
twait : float [if source ends 's' or 'l'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [if source ends 's' or 'l'; hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
pause : float [hidden]
seconds to pause between frames (defaults to 0)
plotall : bool [hidden]
plot all frames regardless of status (i.e. including blank frames
when nskips are enabled (defaults to False). The profile fitting
will still be disabled for bad frames.
ccd : string
CCD(s) to plot, '0' for all, '1 3' to plot '1' and '3' only, etc.
nx : int [if more than 1 CCD]
number of panels across to display.
bias : string
Name of bias frame to subtract, 'none' to ignore.
flat : string
Name of flat field to divide by, 'none' to ignore. Should normally
only be used in conjunction with a bias, although it does allow you
to specify a flat even if you haven't specified a bias.
defect : string
Name of defect file, 'none' to ignore.
setup : bool
True/yes to access the current windows from hdriver. Useful
during observing when seeting up windows, but not normally
otherwise. Next argument (hidden) is the URL to get to
hdriver. Once setup, you should probably turn this off to
avoid overloading hdriver, especially if in drift mode as
it makes a request for the windows for every frame.
hurl : string [if setup; hidden]
URL needed to access window setting from hdriver. The internal
server in hdriver must be switched on using rtplot_server_on
in the hdriver config file.
msub : bool
subtract the median from each window before scaling for the
image display or not. This happens after any bias subtraction.
iset : string [single character]
determines how the intensities are determined. There are three
options: 'a' for automatic simply scales from the minimum to the
maximum value found on a per CCD basis. 'd' for direct just takes
two numbers from the user. 'p' for percentile dtermines levels
based upon percentiles determined from the entire CCD on a per CCD
basis.
ilo : float [if iset='d']
lower intensity level
ihi : float [if iset='d']
upper intensity level
plo : float [if iset='p']
lower percentile level
phi : float [if iset='p']
upper percentile level
xlo : float
left-hand X-limit for plot
xhi : float
right-hand X-limit for plot (can actually be < xlo)
ylo : float
lower Y-limit for plot
yhi : float
upper Y-limit for plot (can be < ylo)
profit : bool [if plotting a single CCD only]
carry out profile fits or not. If you say yes, then on the first
plot, you will have the option to pick objects with a cursor. The
program will then attempt to track these from frame to frame, and
fit their profile. You may need to adjust 'first' to see anything.
The parameters used for profile fits are hidden and you may want to
invoke the command with 'prompt' the first time you try profile
fitting.
fdevice : string [if profit; hidden]
plot device for profile fits, PGPLOT-style name.
e.g. '/xs', '2/xs' etc.
fwidth : float [if profit; hidden]
fit plot width (inches). Set = 0 to let the program choose.
fheight : float [if profit; hidden]
fit plot height (inches). Set = 0 to let the program choose.
BOTH fwidth AND fheight must be non-zero to have any effect
method : string [if profit; hidden]
this defines the profile fitting method, either a gaussian or a
moffat profile. The latter is usually best.
beta : float [if profit and method == 'm'; hidden]
default Moffat exponent
fwhm : float [if profit; hidden]
default FWHM, unbinned pixels.
fwhm_min : float [if profit; hidden]
minimum FWHM to allow, unbinned pixels.
shbox : float [if profit; hidden]
half width of box for searching for a star, unbinned pixels. The
brightest target in a region +/- shbox around an intial position
will be found. 'shbox' should be large enough to allow for likely
changes in position from frame to frame, but try to keep it as
small as you can to avoid jumping to different targets and to
reduce the chances of interference by cosmic rays.
smooth : float [if profit; hidden]
FWHM for gaussian smoothing, binned pixels. The initial position
for fitting is determined by finding the maximum flux in a smoothed
version of the image in a box of width +/- shbox around the starter
position. Typically should be comparable to the stellar width. Its
main purpose is to combat cosmi rays which tend only to occupy a
single pixel.
splot : bool [if profit; hidden]
Controls whether an outline of the search box and a target number
is plotted (in red) or not.
fhbox : float [if profit; hidden]
half width of box for profile fit, unbinned pixels. The fit box is
centred on the position located by the initial search. It should
normally be > ~2x the expected FWHM.
hmin : float [if profit; hidden]
height threshold to accept a fit. If the height is below this
value, the position will not be updated. This is to help in cloudy
conditions.
read : float [if profit; hidden]
readout noise, RMS ADU, for assigning uncertainties
gain : float [if profit; hidden]
gain, ADU/count, for assigning uncertainties
thresh : float [if profit; hidden]
sigma rejection threshold for fits
"""
command, args = utils.script_args(args)
# get the inputs
with Cline('HIPERCAM_ENV', '.hipercam', command, args) as cl:
# register parameters
cl.register('source', Cline.GLOBAL, Cline.HIDE)
cl.register('device', Cline.LOCAL, Cline.HIDE)
cl.register('width', Cline.LOCAL, Cline.HIDE)
cl.register('height', Cline.LOCAL, Cline.HIDE)
cl.register('run', Cline.GLOBAL, Cline.PROMPT)
cl.register('first', Cline.LOCAL, Cline.PROMPT)
cl.register('trim', Cline.GLOBAL, Cline.PROMPT)
cl.register('ncol', Cline.GLOBAL, Cline.HIDE)
cl.register('nrow', Cline.GLOBAL, Cline.HIDE)
cl.register('twait', Cline.LOCAL, Cline.HIDE)
cl.register('tmax', Cline.LOCAL, Cline.HIDE)
cl.register('flist', Cline.LOCAL, Cline.PROMPT)
cl.register('ccd', Cline.LOCAL, Cline.PROMPT)
cl.register('pause', Cline.LOCAL, Cline.HIDE)
cl.register('plotall', Cline.LOCAL, Cline.HIDE)
cl.register('nx', Cline.LOCAL, Cline.PROMPT)
cl.register('bias', Cline.GLOBAL, Cline.PROMPT)
cl.register('flat', Cline.GLOBAL, Cline.PROMPT)
cl.register('defect', Cline.GLOBAL, Cline.PROMPT)
cl.register('setup', Cline.GLOBAL, Cline.PROMPT)
cl.register('hurl', Cline.GLOBAL, Cline.HIDE)
cl.register('msub', Cline.GLOBAL, Cline.PROMPT)
cl.register('iset', Cline.GLOBAL, Cline.PROMPT)
cl.register('ilo', Cline.GLOBAL, Cline.PROMPT)
cl.register('ihi', Cline.GLOBAL, Cline.PROMPT)
cl.register('plo', Cline.GLOBAL, Cline.PROMPT)
cl.register('phi', Cline.LOCAL, Cline.PROMPT)
cl.register('xlo', Cline.GLOBAL, Cline.PROMPT)
cl.register('xhi', Cline.GLOBAL, Cline.PROMPT)
cl.register('ylo', Cline.GLOBAL, Cline.PROMPT)
cl.register('yhi', Cline.GLOBAL, Cline.PROMPT)
cl.register('profit', Cline.LOCAL, Cline.PROMPT)
cl.register('fdevice', Cline.LOCAL, Cline.HIDE)
cl.register('fwidth', Cline.LOCAL, Cline.HIDE)
cl.register('fheight', Cline.LOCAL, Cline.HIDE)
cl.register('method', Cline.LOCAL, Cline.HIDE)
cl.register('beta', Cline.LOCAL, Cline.HIDE)
cl.register('fwhm', Cline.LOCAL, Cline.HIDE)
cl.register('fwhm_min', Cline.LOCAL, Cline.HIDE)
cl.register('shbox', Cline.LOCAL, Cline.HIDE)
cl.register('smooth', Cline.LOCAL, Cline.HIDE)
cl.register('splot', Cline.LOCAL, Cline.HIDE)
cl.register('fhbox', Cline.LOCAL, Cline.HIDE)
cl.register('hmin', Cline.LOCAL, Cline.HIDE)
cl.register('read', Cline.LOCAL, Cline.HIDE)
cl.register('gain', Cline.LOCAL, Cline.HIDE)
cl.register('thresh', Cline.LOCAL, Cline.HIDE)
# get inputs
source = cl.get_value('source', 'data source [hs, hl, us, ul, hf]',
'hl', lvals=('hs','hl','us','ul','hf'))
# set some flags
server_or_local = source.endswith('s') or source.endswith('l')
# plot device stuff
device = cl.get_value('device', 'plot device', '1/xs')
width = cl.get_value('width', 'plot width (inches)', 0.)
height = cl.get_value('height', 'plot height (inches)', 0.)
if server_or_local:
resource = cl.get_value('run', 'run name', 'run005')
if source == 'hs':
first = cl.get_value('first', 'first frame to plot', 1)
else:
first = cl.get_value('first', 'first frame to plot', 1, 0)
if source.startswith('u'):
trim = cl.get_value(
'trim',
'do you want to trim edges of windows? (ULTRACAM only)',
True
)
if trim:
ncol = cl.get_value(
'ncol', 'number of columns to trim from windows', 0)
nrow = cl.get_value(
'nrow', 'number of rows to trim from windows', 0)
else:
trim = False
twait = cl.get_value(
'twait', 'time to wait for a new frame [secs]', 1., 0.)
tmax = cl.get_value(
'tmax', 'maximum time to wait for a new frame [secs]', 10., 0.)
else:
resource = cl.get_value(
'flist', 'file list', cline.Fname('files.lis',hcam.LIST)
)
first = 1
trim = False
# define the panel grid. first get the labels and maximum dimensions
ccdinf = spooler.get_ccd_pars(source, resource)
try:
nxdef = cl.get_default('nx')
except:
nxdef = 3
if len(ccdinf) > 1:
ccd = cl.get_value('ccd', 'CCD(s) to plot [0 for all]', '0')
if ccd == '0':
ccds = list(ccdinf.keys())
else:
ccds = ccd.split()
check = set(ccdinf.keys())
if not set(ccds) <= check:
raise hcam.HipercamError(
'At least one invalid CCD label supplied'
)
if len(ccds) > 1:
nxdef = min(len(ccds), nxdef)
cl.set_default('nx', nxdef)
nx = cl.get_value('nx', 'number of panels in X', 3, 1)
else:
nx = 1
else:
nx = 1
ccds = list(ccdinf.keys())
cl.set_default('pause', 0.)
pause = cl.get_value(
'pause', 'time delay to add between'
' frame plots [secs]', 0., 0.
)
cl.set_default('plotall', False)
plotall = cl.get_value('plotall', 'plot all frames,'
' regardless of status?', False)
# bias frame (if any)
bias = cl.get_value(
'bias', "bias frame ['none' to ignore]",
cline.Fname('bias', hcam.HCAM), ignore='none'
)
if bias is not None:
# read the bias frame
bias = hcam.MCCD.read(bias)
fprompt = "flat frame ['none' to ignore]"
else:
fprompt = "flat frame ['none' is normal choice with no bias]"
# flat (if any)
flat = cl.get_value(
'flat', fprompt,
cline.Fname('flat', hcam.HCAM), ignore='none'
)
if flat is not None:
# read the flat frame
flat = hcam.MCCD.read(flat)
# defect file (if any)
dfct = cl.get_value(
'defect', "defect file ['none' to ignore]",
cline.Fname('defect', hcam.DFCT), ignore='none'
)
if dfct is not None:
# read the defect frame
dfct = defect.MccdDefect.read(dfct)
# Get windows from hdriver
setup = cl.get_value(
'setup', 'display current hdriver window settings', False
)
if setup:
hurl = cl.get_value(
'hurl', 'URL for hdriver windows', 'http://192.168.1.2:5100'
)
# define the display intensities
msub = cl.get_value('msub', 'subtract median from each window?', True)
iset = cl.get_value(
'iset', 'set intensity a(utomatically),'
' d(irectly) or with p(ercentiles)?',
'a', lvals=['a','d','p'])
iset = iset.lower()
plo, phi = 5, 95
ilo, ihi = 0, 1000
if iset == 'd':
ilo = cl.get_value('ilo', 'lower intensity limit', 0.)
ihi = cl.get_value('ihi', 'upper intensity limit', 1000.)
elif iset == 'p':
plo = cl.get_value('plo', 'lower intensity limit percentile',
5., 0., 100.)
phi = cl.get_value('phi', 'upper intensity limit percentile',
95., 0., 100.)
# region to plot
for i, cnam in enumerate(ccds):
nxtot, nytot, nxpad, nypad = ccdinf[cnam]
if i == 0:
xmin, xmax = float(-nxpad), float(nxtot + nxpad + 1)
ymin, ymax = float(-nypad), float(nytot + nypad + 1)
else:
xmin = min(xmin, float(-nxpad))
xmax = max(xmax, float(nxtot + nxpad + 1))
ymin = min(ymin, float(-nypad))
ymax = max(ymax, float(nytot + nypad + 1))
xlo = cl.get_value('xlo', 'left-hand X value', xmin, xmin, xmax)
xhi = cl.get_value('xhi', 'right-hand X value', xmax, xmin, xmax)
ylo = cl.get_value('ylo', 'lower Y value', ymin, ymin, ymax)
yhi = cl.get_value('yhi', 'upper Y value', ymax, ymin, ymax)
# profile fitting if just one CCD chosen
if len(ccds) == 1:
# many parameters for profile fits, although most are not plotted
# by default
profit = cl.get_value('profit', 'do you want profile fits?', False)
if profit:
fdevice = cl.get_value('fdevice', 'plot device for fits',
'2/xs')
fwidth = cl.get_value('fwidth', 'fit plot width (inches)', 0.)
fheight = cl.get_value(
'fheight', 'fit plot height (inches)', 0.)
method = cl.get_value(
'method', 'fit method g(aussian) or m(offat)',
'm', lvals=['g','m'])
if method == 'm':
beta = cl.get_value(
'beta', 'initial exponent for Moffat fits', 5., 0.5)
else:
beta = 0.
fwhm_min = cl.get_value(
'fwhm_min', 'minimum FWHM to allow [unbinned pixels]',
1.5, 0.01
)
fwhm = cl.get_value(
'fwhm', 'initial FWHM [unbinned pixels] for profile fits',
6., fwhm_min
)
shbox = cl.get_value(
'shbox', 'half width of box for initial location'
' of target [unbinned pixels]', 11., 2.)
smooth = cl.get_value(
'smooth', 'FWHM for smoothing for initial object'
' detection [binned pixels]', 6.)
splot = cl.get_value(
'splot', 'plot outline of search box?', True)
fhbox = cl.get_value(
'fhbox', 'half width of box for profile fit'
' [unbinned pixels]', 21., 3.)
hmin = cl.get_value(
'hmin', 'minimum peak height to accept the fit', 50.)
read = cl.get_value('read', 'readout noise, RMS ADU', 3.)
gain = cl.get_value('gain', 'gain, ADU/e-', 1.)
thresh = cl.get_value('thresh', 'number of RMS to reject at', 4.)
else:
profit = False
################################################################
#
# all the inputs have now been obtained. Get on with doing stuff
# open image plot device
imdev = hcam.pgp.Device(device)
if width > 0 and height > 0:
pgpap(width,height/width)
# set up panels and axes
nccd = len(ccds)
ny = nccd // nx if nccd % nx == 0 else nccd // nx + 1
# slice up viewport
pgsubp(nx,ny)
# plot axes, labels, titles. Happens once only
for cnam in ccds:
pgsci(hcam.pgp.Params['axis.ci'])
pgsch(hcam.pgp.Params['axis.number.ch'])
pgenv(xlo, xhi, ylo, yhi, 1, 0)
pglab('X','Y','CCD {:s}'.format(cnam))
# initialisations. 'last_ok' is used to store the last OK frames of each
# CCD for retrieval when coping with skipped data.
total_time = 0 # time waiting for new frame
fpos = [] # list of target positions to fit
fframe = True # waiting for first valid frame with profit
# plot images
with spooler.data_source(source, resource, first, full=False) as spool:
# 'spool' is an iterable source of MCCDs
n = 0
for mccd in spool:
if server_or_local:
# Handle the waiting game ...
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmax, total_time
)
if give_up:
print('rtplot stopped')
break
elif try_again:
continue
# Trim the frames: ULTRACAM windowed data has bad columns
# and rows on the sides of windows closest to the readout
# which can badly affect reduction. This option strips
# them.
if trim:
hcam.ccd.trim_ultracam(mccd, ncol, nrow)
# indicate progress
tstamp = Time(mccd.head['TIMSTAMP'], format='isot', precision=3)
print(
'{:d}, utc = {:s}, '.format(
mccd.head['NFRAME'], tstamp.iso), end=''
)
# accumulate errors
emessages = []
if n == 0:
if bias is not None:
# crop the bias on the first frame only
bias = bias.crop(mccd)
if flat is not None:
# crop the flat on the first frame only
flat = flat.crop(mccd)
if setup:
# Get windows from hdriver. Fair bit of error checking
# needed. 'got_windows' indicates if anything useful
# found, 'hwindows' is a list of (llx,lly,nx,ny) tuples
# if somthing is found.
try:
r = requests.get(
'http://192.168.1.2:5100',timeout=0.2
)
if r.text.strip() == 'No valid data available':
emessages.append(
'** bad return from hdriver = {:s}'.format(r.text.strip())
)
got_windows = False
elif r.text.strip() == 'fullframe':
# to help Stu out a bit, effectively just ignore this one
got_windows = False
else:
# OK, got something
lines = r.text.split('\r\n')
xbinh,ybinh,nwinh = lines[0].split()
xbinh,ybinh,nwinh = int(xbinh),int(ybinh),int(nwinh)
hwindows = []
for line in lines[1:nwinh+1]:
llxh,llyh,nxh,nyh = line.split()
hwindows.append(
(int(llxh),int(llyh),int(nxh),int(nyh))
)
if nwinh != len(hwindows):
emessages.append(
('** expected {:d} windows from'
' hdriver but got {:d}').format(nwinh,len(hwindows))
)
got_windows = False
got_windows = True
except (requests.exceptions.ConnectionError, socket.timeout,
requests.exceptions.Timeout) as err:
emessages.append(' ** hdriver error: {!r}'.format(err))
got_windows = False
else:
got_windows = False
# display the CCDs chosen
message = ''
pgbbuf()
for nc, cnam in enumerate(ccds):
ccd = mccd[cnam]
if plotall or ccd.is_data():
# this should be data as opposed to a blank frame
# between data frames that occur with nskip > 0
# subtract the bias
if bias is not None:
ccd -= bias[cnam]
# divide out the flat
if flat is not None:
ccd /= flat[cnam]
if msub:
# subtract median from each window
for wind in ccd.values():
wind -= wind.median()
# set to the correct panel and then plot CCD
ix = (nc % nx) + 1
iy = nc // nx + 1
pgpanl(ix,iy)
vmin, vmax = hcam.pgp.pCcd(
ccd,iset,plo,phi,ilo,ihi,
xlo=xlo, xhi=xhi, ylo=ylo, yhi=yhi
)
if got_windows:
# plot the current hdriver windows
pgsci(hcam.CNAMS['yellow'])
pgsls(2)
for llxh,llyh,nxh,nyh in hwindows:
pgrect(llxh-0.5,llxh+nxh-0.5,llyh-0.5,llyh+nyh-0.5)
if dfct is not None and cnam in dfct:
# plot defects
hcam.pgp.pCcdDefect(dfct[cnam])
# accumulate string of image scalings
if nc:
message += ', ccd {:s}: {:.1f}, {:.1f}, exp: {:.4f}'.format(
cnam,vmin,vmax,mccd.head['EXPTIME']
)
else:
message += 'ccd {:s}: {:.1f}, {:.1f}, exp: {:.4f}'.format(
cnam,vmin,vmax,mccd.head['EXPTIME']
)
pgebuf()
# end of CCD display loop
print(message)
for emessage in emessages:
print(emessage)
if ccd.is_data() and profit and fframe:
fframe = False
# cursor selection of targets after first plot, if profit
# accumulate list of starter positions
print('Please select targets for profile fitting. You can select as many as you like.')
x, y, reply = (xlo+xhi)/2, (ylo+yhi)/2, ''
ntarg = 0
pgsci(2)
pgslw(2)
while reply != 'Q':
print("Place cursor on fit target. Any key to register, 'q' to quit")
x, y, reply = pgcurs(x, y)
if reply == 'q':
break
else:
# check that the position is inside a window
wnam = ccd.inside(x, y, 2)
if wnam is not None:
# store the position, Window label, target number,
# box size fwhm, beta
ntarg += 1
fpos.append(Fpar(x,y,wnam,ntarg,shbox,fwhm,beta))
# report information, overplot search box
print(
('Target {:d} selected at {:.1f},'
'{:.1f} in window {:s}').format(ntarg,x,y,wnam)
)
if splot:
fpos[-1].plot()
if len(fpos):
print(len(fpos),'targets selected')
# if some targets were selected, open the fit plot device
fdev = hcam.pgp.Device(fdevice)
if fwidth > 0 and fheight > 0:
pgpap(fwidth,fheight/fwidth)
if ccd.is_data():
# carry out fits. Nothing happens if fpos is empty
for fpar in fpos:
# switch to the image plot
imdev.select()
# plot search box
if splot:
fpar.plot()
try:
# extract search box from the CCD. 'fpar' is updated later
# if the fit is successful to reflect the new position
swind = fpar.swind(ccd)
# carry out initial search
x,y,peak = swind.search(smooth, fpar.x, fpar.y, hmin, False)
# now for a more refined fit. First extract fit Window
fwind = ccd[fpar.wnam].window(
x-fhbox, x+fhbox, y-fhbox, y+fhbox
)
# crude estimate of sky background
sky = np.percentile(fwind.data, 50)
# refine the Aperture position by fitting the profile
(sky, height, x, y, fwhm, beta), epars, \
(wfit, X, Y, sigma, chisq, nok,
nrej, npar, message) = hcam.fitting.combFit(
fwind, method, sky, peak-sky,
x, y, fpar.fwhm, fwhm_min, False,
fpar.beta, read, gain, thresh
)
print('Targ {:d}: {:s}'.format(fpar.ntarg,message))
if peak > hmin and ccd[fpar.wnam].distance(x,y) > 1:
# update some initial parameters for next time
if method == 'g':
fpar.x, fpar.y, fpar.fwhm = x, y, fwhm
elif method == 'm':
fpar.x, fpar.y, fpar.fwhm, \
fpar.beta = x, y, fwhm, beta
# plot values versus radial distance
ok = sigma > 0
R = np.sqrt((X-x)**2+(Y-y)**2)
fdev.select()
vmin = min(sky, sky+height, fwind.min())
vmax = max(sky, sky+height, fwind.max())
extent = vmax-vmin
pgeras()
pgvstd()
pgswin(0,R.max(),vmin-0.05*extent,vmax+0.05*extent)
pgsci(4)
pgbox('bcnst',0,0,'bcnst',0,0)
pgsci(2)
pglab('Radial distance [unbinned pixels]','Counts','')
pgsci(1)
pgpt(R[ok].flat, fwind.data[ok].flat, 1)
if nrej:
pgsci(2)
pgpt(R[~ok].flat, fwind.data[~ok].flat, 5)
# line fit
pgsci(3)
r = np.linspace(0,R.max(),400)
if method == 'g':
alpha = 4*np.log(2.)/fwhm**2
f = sky+height*np.exp(-alpha*r**2)
elif method == 'm':
alpha = 4*(2**(1/beta)-1)/fwhm**2
f = sky+height/(1+alpha*r**2)**beta
pgline(r,f)
# back to the image to plot a circle of radius FWHM
imdev.select()
pgsci(3)
pgcirc(x,y,fwhm)
else:
print(' *** below detection threshold; position & FWHM will not updated')
pgsci(2)
# plot location on image as a cross
pgpt1(x, y, 5)
except hcam.HipercamError as err:
print(' >> Targ {:d}: fit failed ***: {!s}'.format(
fpar.ntarg, err)
)
pgsci(2)
if pause > 0.:
# pause between frames
time.sleep(pause)
# update the frame number
n += 1
def makefringe(args=None):
"""``makefringe [source] (run first last [twait tmax] | flist) bias
dark flat fpair ([nhalf]) ccd fwhm [clobber] output``
Averages a set of images to make a frame for defringing (referred
to elsewhere as a "fringe map").
At long wavelengths, CCDs suffer what is known as "fringing",
which in terms of structure looks something like the coloured
patterns you see when there is a thin layer of oil on water. Both
patterns are caused by interference. In the case of CCDs this is
caused by sky emission lines and is variable in strength, and
additive in nature, but seems mostly fairly fixed in position.
De-fringing in |hiper| is implemented according to the method
suggested by Snodgrass & Carry (2013Msngr.152...14S). The idea is
to create a set of pairs of points marking the peak and troughs of
fringes. The differences in intensity between these point pairs in
the data to be corrected is compared with the differences for the
same pairs in a reference "fringe map" by taking their
ratios. Many pairs are used so that the median can be taken to
eliminate ratios affected by celestial targets or cosmic rays. The
median ratio is used to scale the reference data and thereby
subtract the fringes. ``makefringe`` is used to make the reference
fringe map. It works as follows: given a single run or a list of
files, it reads them all in, optionally debiases, dark-subtracts
and flat-fields them, calculates a median count level of each one
which is subtracted from each CCD individually. The pixel-by-pixel
median of all frames is then calculated. It is also possible (and
a good idea) to apply the fringe pair ratio measurements to scale
each frame before combining them, which allows frames of similar
pattern but varying amplitude to be combined. Finally, the output
can be smoothed because fringes are usually a medium- to
large-scale pattern.
Parameters:
source : string [hidden]
Data source, five options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : ULTRACAM server
| 'ul' : local ULTRACAM .xml/.dat files
| 'hf' : list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab' or
converted from foreign data formats.
run : string [if source ends 's' or 'l']
run number to access, e.g. 'run034'
first : int [if source ends 's' or 'l']
exposure number to start from. 1 = first frame ('0' is
not supported).
last : int [if source ends 's' or 'l']
last exposure number must be >= first or 0 for the whole lot.
twait : float [if source ends 's' or 'l'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [if source ends 's' or 'l'; hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
flist : string [if source ends 'f']
name of file list. Assumed that bias, flat-fielding etc have been applied to
these already.
bias : str
Name of bias frame to subtract, 'none' to ignore.
dark : str
Name of dark frame to subtract, 'none' to ignore.
flat : str
Name of flat field, 'none' to ignore.
fpair : str
FringePair file (see setfringe), or 'none' to ignore. If specified
if will be used to scale the frames prior to combining.
nhalf : int [if fpair is not 'none']
When calculating the differences for fringe measurement,
a region extending +/-nhalf binned pixels will be used when
measuring the amplitudes. Basically helps the stats.
ccd : str
CCD(s) to process, '0' for all, '1 3' for '1' and '3' only, etc.
fwhm : float
FWHM (unbinned pixels) of gaussian smoothing to apply to the output.
0 to ignore. Should be << smallest scale between fringes. Probably
no more than 4.
clobber : bool [hidden]
clobber any pre-existing output files
output : str
output file. Set by default to match the last part of "run"
(but it will have a different extension so they won't
clash). The default is
.. Note::
This routine writes the files returned by 'grab' to
automatically generated files, typically in .hipercam/tmp, to
avoid polluting the working directory. These are removed at
the end, but may not be if you ctrl-C. You should check
.hipercam/tmp for redundant files every so often
"""
command, args = utils.script_args(args)
# get the inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("source", Cline.GLOBAL, Cline.HIDE)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("last", Cline.LOCAL, Cline.PROMPT)
cl.register("twait", Cline.LOCAL, Cline.HIDE)
cl.register("tmax", Cline.LOCAL, Cline.HIDE)
cl.register("flist", Cline.LOCAL, Cline.PROMPT)
cl.register("bias", Cline.LOCAL, Cline.PROMPT)
cl.register("dark", Cline.LOCAL, Cline.PROMPT)
cl.register("flat", Cline.LOCAL, Cline.PROMPT)
cl.register("fpair", Cline.LOCAL, Cline.PROMPT)
cl.register("nhalf", Cline.LOCAL, Cline.PROMPT)
cl.register("ccd", Cline.LOCAL, Cline.PROMPT)
cl.register("fwhm", Cline.LOCAL, Cline.PROMPT)
cl.register("clobber", Cline.LOCAL, Cline.HIDE)
cl.register("output", Cline.LOCAL, Cline.PROMPT)
# get inputs
source = cl.get_value(
"source",
"data source [hs, hl, us, ul, hf]",
"hl",
lvals=("hs", "hl", "us", "ul", "hf"),
)
# set a flag
server_or_local = source.endswith("s") or source.endswith("l")
if server_or_local:
resource = cl.get_value("run", "run name", "run005")
root = os.path.basename(resource)
cl.set_default('output', cline.Fname(root, hcam.HCAM))
first = cl.get_value("first", "first frame to average", 1, 1)
last = cl.get_value("last", "last frame to average (0 for all)",
first, 0)
twait = cl.get_value("twait",
"time to wait for a new frame [secs]", 1.0,
0.0)
tmax = cl.get_value("tmax",
"maximum time to wait for a new frame [secs]",
10.0, 0.0)
else:
resource = cl.get_value("flist", "file list",
cline.Fname("files.lis", hcam.LIST))
first = 1
# bias frame (if any)
bias = cl.get_value(
"bias",
"bias frame ['none' to ignore]",
cline.Fname("bias", hcam.HCAM),
ignore="none",
)
# dark frame (if any)
dark = cl.get_value(
"dark",
"dark frame ['none' to ignore]",
cline.Fname("dark", hcam.HCAM),
ignore="none",
)
# flat field (if any)
flat = cl.get_value(
"flat",
"flat field frame ['none' to ignore]",
cline.Fname("flat", hcam.HCAM),
ignore="none",
)
fpair = cl.get_value("fpair",
"fringe pair file",
cline.Fname("fringe", hcam.FRNG),
ignore="none")
if fpair is not None:
nhalf = cl.get_value("nhalf",
"half-size of fringe measurement region", 2,
0)
ccdinf = spooler.get_ccd_pars(source, resource)
if len(ccdinf) > 1:
ccd = cl.get_value("ccd", "CCD(s) to process [0 for all]", "0")
if ccd == "0":
ccds = list(ccdinf.keys())
else:
ccds = ccd.split()
else:
ccds = list(ccdinf.keys())
fwhm = cl.get_value(
"fwhm", "FWHM of gaussian smoothing to apply"
" to final result [unbinned pixels, 0 to ignore]", 4., 0.)
clobber = cl.get_value("clobber",
"clobber any pre-existing files on output",
False)
output = cl.get_value(
"output",
"median output fringe frame",
cline.Fname("hcam", hcam.HCAM,
cline.Fname.NEW if clobber else cline.Fname.NOCLOBBER),
)
# inputs done with.
if server_or_local or bias is not None or dark is not None or flat is not None:
print("\nCalling 'grab' ...")
args = [None, "prompt", source, "yes", resource]
if server_or_local:
args += [str(first), str(last), str(twait), str(tmax)]
args += [
"no",
"none" if bias is None else bias,
"none" if dark is None else dark,
"none" if flat is None else flat,
"none",
"f32",
]
resource = hcam.scripts.grab(args)
# at this point 'resource' is a list of files, no matter the input
# method.
fnames = []
with CleanUp(resource, fnames, server_or_local or bias is not None
or dark is not None) as cleanup:
# Read all the files to determine mean levels (after bias
# subtraction) save the bias-subtracted, flat-fielded,
# mean-level subtracted and normalised results to temporary
# files
print("Reading all files in to determine their mean levels")
bframe, fframe, dframe, fpairobj = None, None, None, None
medians = {}
for cnam in ccds:
medians[cnam] = {}
# We might have a load of temporaries from grab, but we are about to
# make some more to save the bias-subtracted normalised versions.
tdir = utils.temp_dir()
mtype = []
fref = {}
with spooler.HcamListSpool(resource) as spool:
for mccd in spool:
if fpair is not None:
# prepare fringepairs
if fpairobj is None:
fpairobj = fringe.MccdFringePair.read(fpair)
fpairobj = fpairobj.crop(mccd, nhalf)
# here we determine the median levels, subtract them
# then (optionally) normalise by the ratio of fringe
# amplitudes
# generate the name to save to automatically
fd, fname = tempfile.mkstemp(suffix=hcam.HCAM, dir=tdir)
for cnam in ccds:
# its unlikely that fringe frames would be taken
# with skips, but you never know. Eliminate them
# from consideration now.
ccd = mccd[cnam]
if ccd.is_data():
cmedian = mccd[cnam].median()
medians[cnam][fname] = cmedian
mccd[cnam] -= cmedian
if cnam not in fref:
# save reference
fref[cnam] = mccd[cnam]
elif fpair is not None:
fscale = fpairobj[cnam].scale(
mccd[cnam], fref[cnam], nhalf)
mccd[cnam] *= fscale
print(f'{cnam}, fscale = {fscale:.3f}')
# write to disk, save the name, close the filehandle
fnames.append(fname)
mccd.write(fname)
os.close(fd)
# a bit of progress info
print(f"Saved {', '.join(mtype)} frame to {fname}")
# now we go through CCD by CCD, using the first as a template
# for the window names in which we will also store the results.
template = hcam.MCCD.read(fnames[0])
# Now process each file CCD by CCD to reduce the memory
# footprint
for cnam in ccds:
# Read files into memory, insisting that they
# all have the same set of CCDs
print(f"\nLoading all CCDs labelled '{cnam}'")
accds = []
with spooler.HcamListSpool(fnames, cnam) as spool:
mean = None
for ccd in spool:
if ccd.is_data():
accds.append(ccd)
if len(accds) == 0:
raise hcam.HipercamError(
f"Found no valid examples of CCD {cnam}"
f" in {fnames}")
else:
print(f"Loaded {len(accds)} CCDs")
# Median combine
for wnam, wind in template[cnam].items():
# build list of all data arrays
arrs = [ccd[wnam].data for ccd in accds]
# convert to 3D numpy array
arr3d = np.stack(arrs)
wind.data = np.median(arr3d, axis=0)
# Add history
template[cnam].head.add_history(
f"Median combine of {len(accds)} images")
print("Computed and stored their pixel-by-pixel median")
# Remove any CCDs not included to avoid impression of having done
# something to them
dcnams = []
for cnam in template.keys():
if cnam not in ccds:
dcnams.append(cnam)
for cnam in dcnams:
del template[cnam]
if fwhm > 0:
# apply gaussian smoothing
print(f'Smoothing with FWHM = {fwhm}')
sigma = fwhm / np.sqrt(8 * np.log(2))
kern = Gaussian2DKernel(sigma)
for ccd in template.values():
for wind in ccd.values():
wind.data = convolve_fft(wind.data, kern, "fill",
wind.median())
# write out
template.write(output, clobber)
print(f"\nFinal result written to {output}")
print('makefringe finished')
def aligntool(args=None):
"""
Measures alignment of HiperCAM CCDs w.r.t a reference image of the blue CCD.
Arguments::
ref : (string)
name of an MCCD file to use as reference , as produced
by e.g. 'grab'
rccd : (string)
CCD to use as reference
thresh : (float)
threshold for object detection, in multiples of background RMS
source : (string) [hidden]
Data source, five options::
'hs' : HiPERCAM server
'hl' : local HiPERCAM FITS file
'us' : ULTRACAM server
'ul' : local ULTRACAM .xml/.dat files
'hf' : list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab' or
converted from foreign data formats.
device : (string) [hidden]
Plot device. PGPLOT is used so this should be a PGPLOT-style name,
e.g. '/xs', '1/xs' etc. At the moment only ones ending /xs are
supported.
width : (float) [hidden]
plot width (inches). Set = 0 to let the program choose.
height : (float) [hidden]
plot height (inches). Set = 0 to let the program choose. BOTH width
AND height must be non-zero to have any effect
run : (string) [if source == 's' or 'l']
run number to access, e.g. 'run034'
flist : (string) [if source == 'f']
name of file list
first : (int) [if source='s' or 'l']
exposure number to start from. 1 = first frame; set = 0 to
always try to get the most recent frame (if it has changed)
twait : (float) [if source == 's'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : (float) [if source == 's'; hidden]
maximum time to wait between attempts to find a new exposure, seconds.
pause : (float) [hidden]
seconds to pause between frames (defaults to 0)
ccd : (string)
CCD to measure aligment of.
bias : (string)
Name of bias frame to subtract, 'none' to ignore.
msub : (bool)
subtract the median from each window before scaling for the
image display or not. This happens after any bias subtraction.
iset : (string) [single character]
determines how the intensities are determined. There are three
options: 'a' for automatic simply scales from the minimum to the
maximum value found on a per CCD basis. 'd' for direct just takes
two numbers from the user. 'p' for percentile dtermines levels
based upon percentiles determined from the entire CCD on a per CCD
basis.
ilo : (float) [if iset='d']
lower intensity level
ihi : (float) [if iset='d']
upper intensity level
plo : (float) [if iset='p']
lower percentile level
phi : (float) [if iset='p']
upper percentile level
small_spots : (bool) [hidden]
use the small spots for alignment instead of the big spots
"""
command, args = utils.script_args(args)
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("ref", Cline.LOCAL, Cline.PROMPT)
cl.register("thresh", Cline.LOCAL, Cline.PROMPT)
cl.register("small_spots", Cline.LOCAL, Cline.HIDE)
cl.register("source", Cline.LOCAL, Cline.HIDE)
cl.register("device", Cline.LOCAL, Cline.HIDE)
cl.register("width", Cline.LOCAL, Cline.HIDE)
cl.register("height", Cline.LOCAL, Cline.HIDE)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("twait", Cline.LOCAL, Cline.HIDE)
cl.register("tmax", Cline.LOCAL, Cline.HIDE)
cl.register("flist", Cline.LOCAL, Cline.PROMPT)
cl.register("ccd", Cline.LOCAL, Cline.PROMPT)
cl.register("rccd", Cline.LOCAL, Cline.PROMPT)
cl.register("pause", Cline.LOCAL, Cline.HIDE)
cl.register("bias", Cline.GLOBAL, Cline.PROMPT)
cl.register("msub", Cline.GLOBAL, Cline.PROMPT)
cl.register("iset", Cline.GLOBAL, Cline.PROMPT)
cl.register("ilo", Cline.GLOBAL, Cline.PROMPT)
cl.register("ihi", Cline.GLOBAL, Cline.PROMPT)
cl.register("plo", Cline.GLOBAL, Cline.PROMPT)
cl.register("phi", Cline.LOCAL, Cline.PROMPT)
cl.register("xlo", Cline.GLOBAL, Cline.PROMPT)
cl.register("xhi", Cline.GLOBAL, Cline.PROMPT)
cl.register("ylo", Cline.GLOBAL, Cline.PROMPT)
cl.register("yhi", Cline.GLOBAL, Cline.PROMPT)
# get inputs
reference = cl.get_value("ref", "reference frame to align to",
cline.Fname("hcam", hcam.HCAM))
ref_mccd = hcam.MCCD.read(reference)
thresh = cl.get_value("thresh", "detection threshold (sigma)", 5.0)
small_spots = cl.get_value("small_spots",
"use small spots for alignment", False)
# get inputs
source = cl.get_value(
"source",
"data source [hs, hl, us, ul, hf]",
"hl",
lvals=("hs", "hl", "us", "ul", "hf"),
)
# set some flags
server_or_local = source.endswith("s") or source.endswith("l")
inst = source[0]
# plot device stuff
device = cl.get_value("device", "plot device", "1/xs")
width = cl.get_value("width", "plot width (inches)", 0.0)
height = cl.get_value("height", "plot height (inches)", 0.0)
if server_or_local:
resource = cl.get_value("run", "run name", "run005")
first = cl.get_value("first", "first frame to plot", 1, 1)
twait = cl.get_value("twait",
"time to wait for a new frame [secs]", 1.0,
0.0)
tmax = cl.get_value("tmax",
"maximum time to wait for a new frame [secs]",
10.0, 0.0)
else:
# set inst = 'h' as only lists of HiPERCAM files are supported
inst = "h"
resource = cl.get_value("flist", "file list",
cline.Fname("files.lis", hcam.LIST))
first = 1
flist = source.endswith("f")
server = source.endswith("s")
if inst == "u":
instrument = "ULTRA"
elif inst == "h":
instrument = "HIPER"
# get the labels and maximum dimensions and padding factors
ccdinf = spooler.get_ccd_pars(source, resource)
if len(ccdinf) > 1:
ccdnam = cl.get_value("ccd", "CCD to plot alignment of", "1")
rccdnam = cl.get_value("rccd", "CCD to use as reference", "5")
cl.set_default("pause", 0.0)
pause = cl.get_value("pause",
"time delay to add between frame plots [secs]",
0.0, 0.0)
# bias frame (if any)
bias = cl.get_value(
"bias",
"bias frame ['none' to ignore]",
cline.Fname("bias", hcam.HCAM),
ignore="none",
)
if bias is not None:
# read the bias frame
bias = hcam.MCCD.read(bias)
msub = cl.get_value("msub", "subtract median from each window?", True)
iset = cl.get_value(
"iset",
"set intensity a(utomatically),"
" d(irectly) or with p(ercentiles)?",
"a",
lvals=["a", "A", "d", "D", "p", "P"],
)
iset = iset.lower()
plo, phi = 5, 95
ilo, ihi = 0, 1000
if iset == "d":
ilo = cl.get_value("ilo", "lower intensity limit", 0.0)
ihi = cl.get_value("ihi", "upper intensity limit", 1000.0)
elif iset == "p":
plo = cl.get_value("plo", "lower intensity limit percentile", 5.0,
0.0, 100.0)
phi = cl.get_value("phi", "upper intensity limit percentile", 95.0,
0.0, 100.0)
# region to plot
nxtot, nytot, nxpad, nypad = ccdinf[ccdnam]
xmin, xmax = float(-nxpad), float(nxtot + nxpad + 1)
ymin, ymax = float(-nypad), float(nytot + nypad + 1)
xlo = cl.get_value("xlo", "left-hand X value", xmin, xmin, xmax)
xhi = cl.get_value("xhi", "right-hand X value", xmax, xmin, xmax)
ylo = cl.get_value("ylo", "lower Y value", ymin, ymin, ymax)
yhi = cl.get_value("yhi", "upper Y value", ymax, ymin, ymax)
# arguments defined, let's do something!
# open image plot device
imdev = hcam.pgp.Device(device)
if width > 0 and height > 0:
pgpap(width, height / width)
pgsubp(1, 2)
for i in range(2):
pgsci(hcam.pgp.Params["axis.ci"])
pgsch(hcam.pgp.Params["axis.number.ch"])
pgenv(xlo, xhi, ylo, yhi, 1, 0)
pglab("X", "Y", ("reference", "data")[i])
total_time = 0 # time waiting for new frame
with spooler.data_source(source, resource, first) as spool:
# 'spool' is an iterable source of MCCDs
for n, mccd in enumerate(spool):
if server_or_local:
# Handle the waiting game ...
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmax, total_time)
if give_up:
print("alignment tool stopped")
break
elif try_again:
continue
# indicate progress
print(
"Frame {:d}, time = {:s}; ".format(mccd.head["NFRAME"],
mccd.head["TIMSTAMP"]),
end="",
)
if n == 0 and bias is not None:
# crop the bias on the first frame only
bias = bias.crop(mccd)
ccd = mccd[ccdnam]
ref_ccd = ref_mccd[rccdnam]
if bias is not None:
ccd -= bias[ccdnam]
if msub:
for wind in ccd.values():
wind -= wind.median()
for wind in ref_ccd.values():
wind -= wind.median()
# plot images
message = ""
for i in range(2):
this_ccd = (ref_ccd, ccd)[i]
name = ("ref: ", "data: ")[i]
pgpanl(1, i + 1)
vmin, vmax = hcam.pgp.pCcd(this_ccd, iset, plo, phi, ilo, ihi)
message += " {:s}: {:.2f} to {:.2f}".format(name, vmin, vmax)
print(message)
# time for measurement of spots!
try:
xpos, ypos, fwhm, flux = measureSpots(mccd, ccdnam, thresh)
lx, ly, lf, lp, bx, by, bf, bp = separateBigSmallSpots(
xpos, ypos, fwhm, flux)
if not small_spots:
# use big spots
x, y, f, flux = bx, by, bf, bp
else:
x, y, f, flux = lx, ly, lf, lp
except Exception as err:
print("Failed to find spots in frame: ", end=" ")
print(str(err))
continue
# also measure spots in reference image
try:
xpos, ypos, fwhm, rflux = measureSpots(ref_mccd, rccdnam,
thresh)
lx, ly, lf, lp, bx, by, bf, bp = separateBigSmallSpots(
xpos, ypos, fwhm, rflux)
if not small_spots:
# use big spots
xref, yref, fref = bx, by, bf
else:
xref, yref, fref = lx, ly, lf
except Exception as err:
print("Failed to find spots in reference: ", end=" ")
print(str(err))
continue
plt.clf()
displayFWHM(x, y, xref, yref, f, flux, fix_fwhm_scale=False)
plt.draw()
plt.pause(0.05)
plt.ioff()
plt.show()
def makeflat(args=None):
"""``makeflat [source] (run first last [twait tmax] | flist) ngroup bias
dark ccd [clobber] output``
Averages a set of images to make a flat field.
Typically flat-fields for HiPERCAM and ULTRA(CAM|SPEC) are taken with a
strongly time-variable twilight sky as the Sun sets or rises. A typical
flat field run may start out bright, or even saturated, but by the end be
only a few thousand counts above bias. Moreover, there are very often
stars visible in the images, so we usually take them while offsetting the
telescope in a spiral pattern. The challenge is to combine these images
while rejecting the stars and saturated frames and giving due weight to
the better exposed images. This moreover has to be done for each CCD which
vary significantly in sensitivity.
'makeflat' does this as follows: given an input list of files (or
optionally a single run), it reads them all in, debiases them
(optionally), and calculates the mean count level in each CCD,
normalises by the mean and writes out the results to temporary
files. For each CCD it then sorts the files by their (original)
mean level, and for those that lie between defined limits it takes
the median of the mean-mormalised frames in groups of defined
size. Thus, say one had 75 OK images, then these would be divided
into 10 groups, the first 9 having 7 frames, the last having
16. The median average of each of these would be taken. In each
case the mean levels would be adjusted to be the same before
taking the average to overcome the problem of taking a median of a
time-variable sky. The assumption is that while the level may
vary, the pattern of the image does not. It is up to the user to
check that this is correct. Each of the medians is adjusted to
have a mean equal to the sum of the means of the input
frames. Finally the normal average of all of these median frames
is taken and the mean level of the final output normalised to
1. The first step, taking the median in groups is designed to
remove the stars assuming that the telescope was spiralled. The
size of the groups ('ngroup' below is a crucial parameter in
whether this works). A good strategy is to run makeflat for a
succession of ever larger 'ngroup' and then to divide the results
into each other to see if stars are visible.
The final step, the average of the medians with adjusted mean
levels, is to ensure that the flats are combined in a way that
reflects the level of signal that they have, i.e. to avoid giving
equal weights to the median of a series of flats with 20,000 counts
per pixel and another series with 1,000 counts per pixel. This
somewhat complex procedure is implemented through a series of
temporary files which are written and read as the script runs, but
deleted at its end. This allows very large numbers to be combined
as long as there is enough memory to load 'ngroup' CCDs
simultaneously, which should usually be fine.
Parameters:
source : str [hidden]
Data source, five options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : ULTRACAM server
| 'ul' : local ULTRACAM .xml/.dat files
| 'hf' : list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab' or
converted from foreign data formats.
run : str [if source ends 's' or 'l']
run number to access, e.g. 'run034'
flist : str [if source ends 'f']
name of file list
first : int [if source ends 's' or 'l']
exposure number to start from. 1 = first frame ('0' is
not supported).
last : int [if source ends 's' or 'l']
last exposure number must be >= first or 0 for the whole lot.
twait : float [if source ends 's' or 'l'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [if source ends 's' or 'l'; hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
ngroup : int
the number of frames. Probably should be at least 5, preferably
more. Experiment to see its effect.
bias : string
Name of bias frame to subtract, 'none' to ignore.
dark : str
Name of dark frame to subtract, 'none' to ignore. Note that
it is assumed all CCDs have the same exposure time when making
a dark correction.
ccd : str
CCD(s) to process, '0' for all, '1 3' for '1' and '3' only, etc.
Would almost always expect this to be set = '0'.
lower : list of floats
Lower limits to the mean count level for a flat to be included. The
count level is determined after bias subtraction. Should be the
same number as the selected CCDs, and will be assumed to be in the
same order. Use this to elminate frames that are of so low a level
that the accuracy of the bias subtraction could be a worry.
Suggested hipercam values: 3000 for each CCD. Enter values separated
by spaces.
upper : list of floats
Upper limits to the mean count level for a flat to be included. The
count level is determined *after* bias subtraction. Should be the
same number as the selected CCDs, and will be assumed to be in the
same order. Use this to eliminate saturated, peppered or non-linear
frames. Suggested hipercam values: 58000, 58000, 58000, 40000 and
40000 for CCDs 1, 2, 3, 4 and 5. Enter values separated by spaces.
ULTRACAM values 49000, 29000, 27000 for CCDs 1, 2 and 3.
clobber : bool [hidden]
clobber any pre-existing output files
output : str
output file
"""
command, args = utils.script_args(args)
# get the inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("source", Cline.GLOBAL, Cline.HIDE)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("last", Cline.LOCAL, Cline.PROMPT)
cl.register("twait", Cline.LOCAL, Cline.HIDE)
cl.register("tmax", Cline.LOCAL, Cline.HIDE)
cl.register("flist", Cline.LOCAL, Cline.PROMPT)
cl.register("ngroup", Cline.LOCAL, Cline.PROMPT)
cl.register("bias", Cline.LOCAL, Cline.PROMPT)
cl.register("dark", Cline.LOCAL, Cline.PROMPT)
cl.register("ccd", Cline.LOCAL, Cline.PROMPT)
cl.register("lower", Cline.LOCAL, Cline.PROMPT)
cl.register("upper", Cline.LOCAL, Cline.PROMPT)
cl.register("clobber", Cline.LOCAL, Cline.HIDE)
cl.register("output", Cline.LOCAL, Cline.PROMPT)
# get inputs
source = cl.get_value(
"source",
"data source [hs, hl, us, ul, hf]",
"hl",
lvals=("hs", "hl", "us", "ul", "hf"),
)
# set a flag
server_or_local = source.endswith("s") or source.endswith("l")
if server_or_local:
resource = cl.get_value("run", "run name", "run005")
first = cl.get_value("first", "first frame to average", 1, 1)
last = cl.get_value("last", "last frame to average (0 for all)",
first, 0)
twait = cl.get_value("twait",
"time to wait for a new frame [secs]", 1.0,
0.0)
tmax = cl.get_value("tmax",
"maximum time to wait for a new frame [secs]",
10.0, 0.0)
else:
resource = cl.get_value("flist", "file list",
cline.Fname("files.lis", hcam.LIST))
first = 1
ngroup = cl.get_value("ngroup",
"number of frames per median average group", 3,
1)
# bias frame (if any)
bias = cl.get_value(
"bias",
"bias frame ['none' to ignore]",
cline.Fname("bias", hcam.HCAM),
ignore="none",
)
# dark frame (if any)
dark = cl.get_value(
"dark",
"dark frame ['none' to ignore]",
cline.Fname("dark", hcam.HCAM),
ignore="none",
)
ccdinf = spooler.get_ccd_pars(source, resource)
if len(ccdinf) > 1:
ccd = cl.get_value("ccd", "CCD(s) to process [0 for all]", "0")
if ccd == "0":
ccds = list(ccdinf.keys())
else:
ccds = ccd.split()
else:
ccds = list(ccdinf.keys())
# need to check that the default has the right number of items, if not
# overr-ride it
lowers = cl.get_default("lower")
if len(lowers) != len(ccds):
cl.set_default("lower", len(ccds) * (5000, ))
lowers = cl.get_value(
"lower",
"lower limits on mean count level for included flats, 1 per CCD",
len(ccds) * (5000, ),
)
uppers = cl.get_default("upper")
if len(uppers) != len(ccds):
cl.set_default("upper", len(ccds) * (50000, ))
uppers = cl.get_value(
"upper",
"lower limits on mean count level for included flats, 1 per CCD",
len(ccds) * (50000, ),
)
clobber = cl.get_value("clobber",
"clobber any pre-existing files on output",
False)
output = cl.get_value(
"output",
"output average",
cline.Fname("hcam", hcam.HCAM,
cline.Fname.NEW if clobber else cline.Fname.NOCLOBBER),
)
# inputs done with.
try:
# big try / except section here to trap ctrl-C to allow the temporary
# files to be deleted. First make a directory for the temporary files
if server_or_local:
print("\nCalling 'grab' ...")
args = [
None,
"prompt",
source,
resource,
"yes",
str(first),
str(last),
"no",
str(twait),
str(tmax),
"none",
"f32",
]
resource = hcam.scripts.grab(args)
# at this point 'resource' is a list of files, no matter the input
# method.
# Read all the files to determine mean levels (after bias subtraction)
# save the bias-subtracted, mean-level normalised results to temporary
# files
print("Reading all files in to determine their mean levels")
bframe, dframe = None, None
means = {}
for cnam in ccds:
means[cnam] = {}
# We might have a load of temporaries from grab, but we are about to
# make some more to save the bias-subtracted normalised versions.
tdir = os.path.join(tempfile.gettempdir(),
"hipercam-{:s}".format(getpass.getuser()))
os.makedirs(tdir, exist_ok=True)
fnames = []
with spooler.HcamListSpool(resource) as spool:
for mccd in spool:
if bias is not None:
# bias subtraction
if bframe is None:
bframe = hcam.MCCD.read(bias)
bframe = bframe.crop(mccd)
mccd -= bframe
bexpose = bframe.head.get("EXPTIME", 0.0)
else:
bexpose = 0.0
if dark is not None:
# dark subtraction
if dframe is None:
dframe = hcam.MCCD.read(dark)
dframe = dframe.crop(mccd)
# Factor to scale the dark frame by before
# subtracting from flat. Assumes that all
# frames have same exposure time.
scale = (mccd.head["EXPTIME"] -
bexpose) / dframe.head["EXPTIME"]
# make dark correction
mccd -= scale * dframe
# here we determine the mean levels, store them
# then normalise the CCDs by them and save the files
# to disk
# generate the name to save to automatically
fd, fname = tempfile.mkstemp(suffix=hcam.HCAM, dir=tdir)
for cnam in ccds:
# its unlikely that flats would be taken with skips, but
# you never know. Eliminate them from consideration now.
ccd = mccd[cnam]
if ccd.is_data():
cmean = mccd[cnam].mean()
means[cnam][fname] = cmean
mccd[cnam] /= cmean
# write the disk, save the name, close the filehandle
mccd.write(fname)
fnames.append(fname)
os.close(fd)
# a bit of progress info
if bias is not None:
print("Saved debiassed, normalised"
" flat to {:s}".format(fname))
else:
print("Saved normalised flat to {:s}".format(fname))
# now we go through CCD by CCD, using the first as a template
# for the window names in which we will also store the results.
template = hcam.MCCD.read(fnames[0])
for cnam, lower, upper in zip(ccds, lowers, uppers):
tccd = template[cnam]
# get the keys (filenames) and corresponding mean values
mkeys = np.array(list(means[cnam].keys()))
mvals = np.array(list(means[cnam].values()))
# chop down to acceptable ones
ok = (mvals > lower) & (mvals < upper)
mkeys = mkeys[ok]
mvals = mvals[ok]
# some more progress info
print("Found {:d} frames for CCD {:s}".format(len(mkeys), cnam))
if len(mkeys) == 0:
print((".. cannot average 0 frames;"
" will skip CCD {:s}").format(cnam))
continue
elif len(mkeys) < ngroup:
print(("WARNING: fewer than ngroup = {:d} frames"
" found. Output for CCD {:s} could be poor").format(
ngroup, cnam))
nchunk = len(mkeys) // ngroup
if nchunk == 0:
nchunk = 1
# sort by mean value
isort = mvals.argsort()
mvals = mvals[isort]
mkeys = mkeys[isort]
# wsum used to sum all the eight factors to allow overall
# normalisation at the end of the loop
wsum = 0.0
for n in range(nchunk):
# loop through in chunks of ngroup at a time with a
# potentially larger group to sweep up the end ones.
n1 = ngroup * n
n2 = n1 + ngroup
if n == nchunk:
n2 = len(mkeys)
# load the CCDs of this group
ccdgroup = []
with spooler.HcamListSpool(list(mkeys[n1:n2]), cnam) as spool:
for ccd in spool:
ccdgroup.append(ccd)
# take median of the group to get rid of jumping
# stars. 'weight' used to weight the results when summing the
# results together. this stage is like the 'n' option of
# 'combine' except we have already cut out any junk frames and
# we have normalised the remainder
weight = mvals[n1:n2].sum()
wsum += weight
for wnam, wind in tccd.items():
# go through each window, building a list of all data
# arrays
arrs = [ccd[wnam].data for ccd in ccdgroup]
arr3d = np.stack(arrs)
# at this point, arr3d is a 3D array, with the first
# dimension (axis=0) running over the images. We take the
# median over this axis. The first time through we put
# this straight into the output Window. afterwards we add
# it in (with the appropriate weight)
if n == 0:
wind.data = weight * np.median(arr3d, axis=0)
else:
wind.data += weight * np.median(arr3d, axis=0)
# Normalise the final result to a mean = 1.
tccd /= wsum
# Add some history
tccd.head.add_history(
("result of makeflat on {:d}"
" frames, ngroup = {:d}").format(len(mkeys), ngroup))
# Remove any CCDs not included to avoid impression of having done
# something to them
for cnam in template:
if cnam not in ccds:
del template[cnam]
# write out
template.write(output, clobber)
print("\nFinal result written to {:s}".format(output))
except KeyboardInterrupt:
print("\nmakeflat aborted")
if server_or_local:
# grab has created a load of temporaries, including the file list
# 'resource'
with open(resource) as fin:
for fname in fin:
os.remove(fname.strip())
os.remove(resource)
# and another load were created during the later running of the script
for fname in fnames:
os.remove(fname)
print("temporary files have been removed.")
def ncal(args=None):
"""``ncal [source] (run first last [twait tmax] | flist) trim ([ncol
nrow]) (ccd) bias dark flat xybox read gain grain``
Calibrates noise characteristics of CCDs by plotting estimator
of RMS vs signal level from a series of frames. The estimate
is the mean of the absolute difference between each pixel
and the mean of its 8 near-neighbours. This is very local and
fairly robust. Assuming gaussian noise, the RMS is sqrt(4*Pi/9)
times this value, and this is what is plotted as the RMS by this
routine. `ncal` is best to applied to a series of frames with
a large dynamic range, ideally starting from bias-like frames
to well exposed sky flats. A long flat-field run going to low
levels, or a run into twilight at the end of the night could be
good places to start.
Parameters:
source : str [hidden]
Data source, five options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : ULTRACAM server
| 'ul' : local ULTRACAM .xml/.dat files
| 'hf' : list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab'
or converted from foreign data formats. The standard
start-off default for ``source`` can be set using the
environment variable HIPERCAM_DEFAULT_SOURCE. e.g. in bash
:code:`export HIPERCAM_DEFAULT_SOURCE="us"` would ensure it
always started with the ULTRACAM server by default. If
unspecified, it defaults to 'hl'.
run : str [if source ends 's' or 'l']
run number to access, e.g. 'run034'
flist : str [if source ends 'f']
name of file list
first : int [if source ends 's' or 'l']
exposure number to start from. 1 = first frame; set = 0 to always
try to get the most recent frame (if it has changed). For data
from the |hiper| server, a negative number tries to get a frame not
quite at the end. i.e. -10 will try to get 10 from the last
frame. This is mainly to sidestep a difficult bug with the
acquisition system.
last : int [if source ends 's' or 'l']
Last frame to access, 0 for the lot
twait : float [if source ends 's' or 'l'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [if source ends 's' or 'l'; hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
trim : bool [if source starts with 'u']
True to trim columns and/or rows off the edges of windows nearest
the readout which can sometimes contain bad data.
ncol : int [if trim, hidden]
Number of columns to remove (on left of left-hand window, and right
of right-hand windows)
nrow : int [if trim, hidden]
Number of rows to remove (bottom of windows)
ccd : str
The CCD to analyse.
bias : str
Name of bias frame to subtract, 'none' to ignore.
dark : str
Name of dark frame to subtract, 'none' to ignore
flat : str
Name of flat field to divide by, 'none' to ignore. Should normally
only be used in conjunction with a bias, although it does allow you
to specify a flat even if you haven't specified a bias.
xybox : int
the stats will be taken over boxes of xybox-squared pixels
to keep the number of points and scatter under control.
read : float
readout noise, RMS ADU, for overplotting a model
gain : float
gain, e-/count, for overploting a model
grain : float
fractional RMS variation due to flat-field variations,
if you didn't include a flat field.
"""
command, args = utils.script_args(args)
# get the inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("source", Cline.GLOBAL, Cline.HIDE)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("last", Cline.LOCAL, Cline.PROMPT)
cl.register("trim", Cline.GLOBAL, Cline.PROMPT)
cl.register("ncol", Cline.GLOBAL, Cline.HIDE)
cl.register("nrow", Cline.GLOBAL, Cline.HIDE)
cl.register("twait", Cline.LOCAL, Cline.HIDE)
cl.register("tmax", Cline.LOCAL, Cline.HIDE)
cl.register("flist", Cline.LOCAL, Cline.PROMPT)
cl.register("ccd", Cline.LOCAL, Cline.PROMPT)
cl.register("bias", Cline.GLOBAL, Cline.PROMPT)
cl.register("dark", Cline.GLOBAL, Cline.PROMPT)
cl.register("flat", Cline.GLOBAL, Cline.PROMPT)
cl.register("xybox", Cline.LOCAL, Cline.PROMPT)
cl.register("read", Cline.LOCAL, Cline.PROMPT)
cl.register("gain", Cline.LOCAL, Cline.PROMPT)
cl.register("grain", Cline.LOCAL, Cline.PROMPT)
# get inputs
default_source = os.environ.get('HIPERCAM_DEFAULT_SOURCE', 'hl')
source = cl.get_value(
"source",
"data source [hs, hl, us, ul, hf]",
default_source,
lvals=("hs", "hl", "us", "ul", "hf"),
)
# set some flags
server_or_local = source.endswith("s") or source.endswith("l")
if server_or_local:
resource = cl.get_value("run", "run name", "run005")
if source == "hs":
first = cl.get_value("first", "first frame to plot", 1)
else:
first = cl.get_value("first", "first frame to plot", 1, 0)
last = cl.get_value("last", "last frame to grab", 0)
if last < first and last != 0:
sys.stderr.write("last must be >= first or 0")
sys.exit(1)
twait = cl.get_value("twait",
"time to wait for a new frame [secs]", 1.0,
0.0)
tmax = cl.get_value("tmax",
"maximum time to wait for a new frame [secs]",
10.0, 0.0)
else:
resource = cl.get_value("flist", "file list",
cline.Fname("files.lis", hcam.LIST))
first = 1
trim = cl.get_value("trim", "do you want to trim edges of windows?",
True)
if trim:
ncol = cl.get_value("ncol",
"number of columns to trim from windows", 0)
nrow = cl.get_value("nrow", "number of rows to trim from windows",
0)
else:
ncol, nrow = None, None
# define the panel grid. first get the labels and maximum dimensions
ccdinf = spooler.get_ccd_pars(source, resource)
if len(ccdinf) > 1:
cnam = cl.get_value("ccd",
"CCD to analyse",
"1",
lvals=list(ccdinf.keys()))
else:
cnam = ccdinf.keys()[0]
# bias frame (if any)
bias = cl.get_value(
"bias",
"bias frame ['none' to ignore]",
cline.Fname("bias", hcam.HCAM),
ignore="none",
)
if bias is not None:
# read the bias frame
bias = hcam.MCCD.read(bias)
fprompt = "flat frame ['none' to ignore]"
else:
fprompt = "flat frame ['none' is normal choice with no bias]"
# dark (if any)
dark = cl.get_value("dark",
"dark frame to subtract ['none' to ignore]",
cline.Fname("dark", hcam.HCAM),
ignore="none")
if dark is not None:
# read the dark frame
dark = hcam.MCCD.read(dark)
# flat (if any)
flat = cl.get_value("flat",
fprompt,
cline.Fname("flat", hcam.HCAM),
ignore="none")
if flat is not None:
# read the flat frame
flat = hcam.MCCD.read(flat)
xybox = cl.get_value("xybox",
"box size for averaging results [binned pixels]",
11, 1)
read = cl.get_value("read", "readout noise, RMS ADU", 4.0, 0.0)
gain = cl.get_value("gain", "gain, ADU/e-", 1.0, 0.001)
grain = cl.get_value("grain", "flat field graininess", 0.01, 0.)
######################################
# Phew. We finally have all the inputs
# Now on with the analysis
total_time, nframe = 0, 0
with spooler.data_source(source, resource, first, full=False) as spool:
for mccd in spool:
if server_or_local:
# Handle the waiting game ... some awkward stuff
# involving updating on a cycle faster than twait to
# make the plots more responsive, if twait is long.
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmax, total_time)
if give_up:
print("ncal stopped")
break
elif try_again:
continue
# Trim the frames: ULTRACAM windowed data has bad columns
# and rows on the sides of windows closest to the readout
# which can badly affect reduction. This option strips
# them.
if trim:
hcam.ccd.trim_ultracam(mccd, ncol, nrow)
# indicate progress
tstamp = Time(mccd.head["TIMSTAMP"], format="isot", precision=3)
nfrm = mccd.head.get("NFRAME", nframe + 1)
print(
f'{nfrm}, utc= {tstamp.iso} ({"ok" if mccd.head.get("GOODTIME", True) else "nok"}), '
)
if nframe == 0:
# get the bias, dark, flat
# into shape first time through
if bias is not None:
# crop the bias on the first frame only
bias = bias.crop(mccd)
bexpose = bias.head.get("EXPTIME", 0.0)
else:
bexpose = 0.
if dark is not None:
# crop the dark on the first frame only
dark = dark.crop(mccd)
if flat is not None:
# crop the flat on the first frame only
flat = flat.crop(mccd)
# extract the CCD of interest
ccd = mccd[cnam]
if ccd.is_data():
# "is_data" indicates genuine data as opposed to junk
# that results from nskip > 0.
# subtract the bias
if bias is not None:
ccd -= bias[cnam]
# subtract the dark
if dark is not None:
dexpose = dark.head["EXPTIME"]
cexpose = ccd.head["EXPTIME"]
scale = (cexpose - bexpose) / dexpose
ccd -= scale * dark[cnam]
# divide out the flat
if flat is not None:
ccd /= flat[cnam]
# at this point we have the data in the right state to
# start processing.
for wind in ccd.values():
data = wind.data
ny, nx = data.shape
if nx - 2 >= xybox and ny - 2 >= xybox:
means, stds = procdata(wind.data, xybox)
plt.loglog(means, stds, ',b')
# update the frame number
nframe += 1
if server_or_local and last and nframe > last:
break
count = 10**np.linspace(0, 5, 200)
sigma = np.sqrt(read**2 + count / gain + (grain * count)**2)
plt.plot(count, sigma, 'r', lw=2)
plt.xlim(1, 100000)
plt.ylim(1, 1000)
plt.xlabel('Mean count level')
plt.ylabel('RMS [counts]')
plt.title(f'RMS vs level, CCD {cnam}')
plt.show()
def filtid(args=None):
"""``filtid [source] (run first last [twait tmax] |
flist) trim ([ncol nrow]) ccdref maxlev plot``
Computes the ratio of the mean values in each CCD versus those of a reference CCD.
This is supposed to be applied to flat fields. The hope is that this might be
characteristic of the filter combination. Obviously requires > 1 CCD, i.e. not
ULTRASPEC.
Parameters:
source : str [hidden]
Data source, five options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : ULTRACAM server
| 'ul' : local ULTRACAM .xml/.dat files
| 'hf' : list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab'
or converted from foreign data formats. The standard
start-off default for ``source'' can be set using the
environment variable HIPERCAM_DEFAULT_SOURCE. e.g. in bash
:code:`export HIPERCAM_DEFAULT_SOURCE="us"` would ensure it
always started with the ULTRACAM server by default. If
unspecified, it defaults to 'hl'.
run : str [if source ends 's' or 'l']
run number to access, e.g. 'run034'
flist : str [if source ends 'f']
name of file list
first : int [if source ends 's' or 'l']
exposure number to start from. 1 = first frame; set = 0 to always
try to get the most recent frame (if it has changed). For data
from the |hiper| server, a negative number tries to get a frame not
quite at the end. i.e. -10 will try to get 10 from the last
frame. This is mainly to sidestep a difficult bug with the
acquisition system.
last : int [if source ends 's' or 'l']
last frame to reduce. 0 to just continue until the end.
twait : float [if source ends 's' or 'l'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [if source ends 's' or 'l'; hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
trim : bool [if source starts with 'u']
True to trim columns and/or rows off the edges of windows nearest
the readout which can sometimes contain bad data.
ncol : int [if trim, hidden]
Number of columns to remove (on left of left-hand window, and right
of right-hand windows)
nrow : int [if trim, hidden]
Number of rows to remove (bottom of windows)
ccdref : str
The reference CCD (usually choose the g-band one)
bias : str
bias frame to subtract (required)
lower : list of floats
Lower limits to the mean count level for a flat to be
included (after bias subtraction). Should be the same
number as the number of CCDs, and will be assumed to be in
the same order. Separate with spaces. Prevents low exposure
data from being included.
upper : list of floats
Upper limits to the mean count level for a flat to be
included. Should be the same number as the selected CCDs,
and will be assumed to be in the same order. Use to
eliminate saturated, peppered or non-linear
frames. Suggested hipercam values: 58000, 58000, 58000,
40000 and 40000 for CCDs 1, 2, 3, 4 and 5. Enter values
separated by spaces. ULTRACAM values 49000, 29000, 27000
for CCDs 1, 2 and 3.
plot: bool
Plot the fit or not.
.. Note::
This is currently adapted specifically for ULTRACAM data
"""
command, args = utils.script_args(args)
# get the inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("source", Cline.GLOBAL, Cline.HIDE)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("last", Cline.LOCAL, Cline.PROMPT)
cl.register("trim", Cline.GLOBAL, Cline.PROMPT)
cl.register("ncol", Cline.GLOBAL, Cline.HIDE)
cl.register("nrow", Cline.GLOBAL, Cline.HIDE)
cl.register("twait", Cline.LOCAL, Cline.HIDE)
cl.register("tmax", Cline.LOCAL, Cline.HIDE)
cl.register("flist", Cline.LOCAL, Cline.PROMPT)
cl.register("ccdref", Cline.LOCAL, Cline.PROMPT)
cl.register("bias", Cline.LOCAL, Cline.PROMPT)
cl.register("lower", Cline.LOCAL, Cline.PROMPT)
cl.register("upper", Cline.LOCAL, Cline.PROMPT)
cl.register("plot", Cline.LOCAL, Cline.PROMPT)
# get inputs
default_source = os.environ.get('HIPERCAM_DEFAULT_SOURCE', 'hl')
source = cl.get_value(
"source",
"data source [hs, hl, us, ul, hf]",
default_source,
lvals=("hs", "hl", "us", "ul", "hf"),
)
# set some flags
server_or_local = source.endswith("s") or source.endswith("l")
if server_or_local:
resource = cl.get_value("run", "run name", "run005")
first = cl.get_value("first", "first frame to consider", 1, 1)
last = cl.get_value("last", "last frame to consider", 0, 0)
if last and last < first + 2:
raise hcam.HipercamError(
"Must consider at least 2 exposures, and preferably more")
twait = cl.get_value("twait",
"time to wait for a new frame [secs]", 1.0,
0.0)
tmax = cl.get_value("tmax",
"maximum time to wait for a new frame [secs]",
10.0, 0.0)
else:
resource = cl.get_value("flist", "file list",
cline.Fname("files.lis", hcam.LIST))
first = 1
trim = cl.get_value("trim", "do you want to trim edges of windows?",
True)
if trim:
ncol = cl.get_value("ncol",
"number of columns to trim from windows", 0)
nrow = cl.get_value("nrow", "number of rows to trim from windows",
0)
# define the panel grid. first get the labels and maximum dimensions
ccdinf = spooler.get_ccd_pars(source, resource)
if len(ccdinf) == 1:
raise hcam.HipercamError(
"Only one CCD; this routine only works for > 1 CCD")
ccds = list(ccdinf.keys())
ccdref = cl.get_value("ccdref",
"the reference CCD for the ratios",
ccds[1],
lvals=ccds)
bias = cl.get_value("bias", "bias frame",
cline.Fname("bias", hcam.HCAM))
# read the bias frame
bias = hcam.MCCD.read(bias)
if len(bias) != len(ccds):
raise ValueError('bias has an incompatible number of CCDs')
# need to check that the default has the right number of items, if not,
# over-ride it
lowers = cl.get_default("lower")
if lowers is not None and len(lowers) != len(ccds):
cl.set_default("lower", len(ccds) * (5000, ))
lowers = cl.get_value(
"lower",
"lower limits on mean count level for included flats, 1 per CCD",
len(ccds) * (5000, ),
)
lowers = {k: v for k, v in zip(ccds, lowers)}
uppers = cl.get_default("upper")
if uppers is not None and len(uppers) != len(ccds):
cl.set_default("upper", len(ccds) * (50000, ))
uppers = cl.get_value(
"upper",
"lower limits on mean count level for included flats, 1 per CCD",
len(ccds) * (50000, ),
)
uppers = {k: v for k, v in zip(ccds, uppers)}
plot = cl.get_value("plot", "plot the results?", True)
################################################################
#
# all the inputs have now been obtained. Get on with doing stuff
total_time = 0.
xdata, ydata = {}, {}
# only want to look at CCDs other than ccdref
ccds.remove(ccdref)
# access images
with spooler.data_source(source, resource, first, full=False) as spool:
# 'spool' is an iterable source of MCCDs
nframe = 0
for mccd in spool:
if server_or_local:
# Handle the waiting game ...
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmax, total_time)
if give_up:
print("rtplot stopped")
break
elif try_again:
continue
# Trim the frames: ULTRACAM windowed data has bad columns
# and rows on the sides of windows closest to the readout
# which can badly affect reduction. This option strips
# them.
if trim:
hcam.ccd.trim_ultracam(mccd, ncol, nrow)
if nframe == 0:
# crop the bias on the first frame only
bias = bias.crop(mccd)
# bias subtraction
mccd -= bias
# indicate progress
tstamp = Time(mccd.head["TIMSTAMP"], format="isot", precision=3)
print(f"{mccd.head.get('NFRAME',nframe+1)}, utc= {tstamp.iso} " +
f"({'ok' if mccd.head.get('GOODTIME', True) else 'nok'})")
# add in data if it is in range
xd = mccd[ccdref].mean()
if xd > lowers[ccdref] and xd < uppers[ccdref]:
for cnam in ccds:
yd = mccd[cnam].mean()
if yd > lowers[cnam] and yd < uppers[cnam]:
if cnam in xdata:
xdata[cnam].append(xd)
ydata[cnam].append(yd)
else:
xdata[cnam] = [xd]
ydata[cnam] = [yd]
if last and nframe == last:
break
# update the frame number
nframe += 1
print()
invalid = True
for cnam in ccds:
if cnam in xdata:
xdata[cnam] = np.array(xdata[cnam])
ydata[cnam] = np.array(ydata[cnam])
print(
f'{cnam}-vs-{ccdref}: found {len(xdata[cnam])} valid points ranging from'
+
f' {xdata[cnam].min():.1f} to {xdata[cnam].max():.1f} (CCD {ccdref}), '
+
f' {ydata[cnam].min():.1f} to {ydata[cnam].max():.1f} (CCD {cnam})'
)
invalid = False
if invalid:
raise hcam.HipercamError(f"No valid points found at all")
print()
if plot:
fig, axes = plt.subplots(len(ccds), 1, sharex=True)
for cnam, ax in zip(ccds, axes):
if cnam in xdata:
xds = xdata[cnam]
ratios = ydata[cnam] / xds
mratio = ratios.mean()
ax.plot(xds, ratios, '.g')
ax.plot([xds.min(), xds.max()], [mratio, mratio], 'r--')
ax.set_ylabel(f'Ratio CCD {cnam} / CCD {ccdref}')
if cnam == ccds[-1]:
ax.set_xlabel(f'CCD = {ccdref} counts/pixel')
print(f'{cnam}-vs-{ccdref} = {mratio:.4f}')
plt.show()
else:
for cnam in ccds:
if cnam in xdata:
ratios = ydata[cnam] / xdata[cnam]
mratio = ratios.mean()
print(f'{cnam}-vs-{ccdref} = {mratio:.4f}')
def reduce(args=None):
"""``reduce [source] rfile (run first last twait tmax | flist) trim
([ncol nrow]) log lplot implot (ccd nx msub xlo xhi ylo yhi iset
(ilo ihi | plo phi))``
Reduces a sequence of multi-CCD images, plotting lightcurves as images
come in. It can extract with either simple aperture photometry or Tim
Naylor's optimal photometry, on specific targets defined in an aperture
file using |setaper|.
reduce can source data from both the ULTRACAM and HiPERCAM servers, from
local 'raw' ULTRACAM and HiPERCAM files (i.e. .xml + .dat for ULTRACAM, 3D
FITS files for HiPERCAM) and from lists of HiPERCAM '.hcm' files. If you
have data from a different instrument you should convert into the
FITS-based hcm format.
reduce is primarily configured from a file with extension ".red". This
contains a series of directives, e.g. to say how to re-position and
re-size the apertures. An initial reduce file is best generated with
the script |genred| after you have created an aperture file. This contains
lots of help on what to do.
A reduce run can be terminated at any point with ctrl-C without doing
any harm. You may often want to do this at the start in order to adjust
parameters of the reduce file.
Parameters:
source : str [hidden]
Data source, five options:
| 'hs': HiPERCAM server
| 'hl': local HiPERCAM FITS file
| 'us': ULTRACAM server
| 'ul': local ULTRACAM .xml/.dat files
| 'hf': list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab' or
converted from foreign data formats.
run : str [if source ends 's' or 'l']
run number to access, e.g. 'run034' or a file list. If a run,
then reduce and log below will be set to have the same name by
default.
first : int [if source ends 's' or 'l']
first frame to reduce. 1 = first frame; set = 0 to always try to
get the most recent frame (if it has changed).
last : int [if source ends 's' or 'l', hidden]
last frame to reduce. 0 to just continue until the end. This is
not prompted for by default and must be set explicitly. It
defaults to 0 if not set. Its purpose is to allow accurate
profiling tests.
twait : float [if source ends 's'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [if source ends 's'; hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
flist : string [if source ends 'f']
name of file list
trim : bool
True to trim columns and/or rows off the edges of windows nearest
the readout. Particularly useful with ULTRACAM windowed data where
the first few rows and columns can contain bad data.
ncol : int [if trim, hidden]
Number of columns to remove (on left of left-hand window, and right
of right-hand windows)
nrow : int [if trim, hidden]
Number of rows to remove (bottom of windows)
rfile : str
the "reduce" file, i.e. ASCII text file suitable for reading by
ConfigParser. Best seen by example as it has many parts. If you
are reducing a run, this will be set to have the same root name by
default (but a different extension to avoid name clashes).
log : str
log file for the results. If you are reducing a run, this
will be set to have the same root name by default (but a
different extension to avoid name clashes)
tkeep : float
maximum number of minutes of data to store in internal buffers, 0
for the lot. When large numbers of frames are stored, performance
can be slowed (although I am not entirely clear why) in which case
it makes sense to lose the earlier points (without affecting the
saving to disk). This parameter also gives operation similar to that
of "max_xrange" parameter in the ULTRACAM pipeline whereby just
the last few minutes are shown.
lplot : bool
flag to indicate you want to plot the light curve. Saves time not
to especially in high-speed runs.
implot : bool
flag to indicate you want to plot images.
ccd : string [if implot]
CCD(s) to plot, '0' for all, '1 3' to plot '1' and '3' only, etc.
nx : int [if implot]
number of panels across to display.
msub : bool [if implot]
subtract the median from each window before scaling for the
image display or not. This happens after any bias subtraction.
xlo : float [if implot]
left-hand X-limit for plot
xhi : float [if implot]
right-hand X-limit for plot (can actually be < xlo)
ylo : float [if implot]
lower Y-limit for plot
yhi : float [if implot]
upper Y-limit for plot (can be < ylo)
iset : string [if implot]
determines how the intensities are determined. There are three
options: 'a' for automatic simply scales from the minimum to the
maximum value found on a per CCD basis. 'd' for direct just takes
two numbers from the user. 'p' for percentile dtermines levels
based upon percentiles determined from the entire CCD on a per CCD
basis.
ilo : float [if implot and iset='d']
lower intensity level
ihi : float [if implot and iset='d']
upper intensity level
plo : float [if implot and iset='p']
lower percentile level
phi : float [if implot and iset='p']
upper percentile level
.. Warning::
The transmission plot generated with reduce is not reliable in the
case of optimal photometry since it is highly correlated with the
seeing. If you are worried about the transmission during observing,
you should always use normal aperture photometry.
"""
command, args = utils.script_args(args)
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("source", Cline.GLOBAL, Cline.HIDE)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("last", Cline.LOCAL, Cline.HIDE)
cl.register("twait", Cline.LOCAL, Cline.HIDE)
cl.register("tmax", Cline.LOCAL, Cline.HIDE)
cl.register("flist", Cline.LOCAL, Cline.PROMPT)
cl.register("trim", Cline.GLOBAL, Cline.PROMPT)
cl.register("ncol", Cline.GLOBAL, Cline.HIDE)
cl.register("nrow", Cline.GLOBAL, Cline.HIDE)
cl.register("rfile", Cline.GLOBAL, Cline.PROMPT)
cl.register("log", Cline.GLOBAL, Cline.PROMPT)
cl.register("tkeep", Cline.GLOBAL, Cline.PROMPT)
cl.register("lplot", Cline.LOCAL, Cline.PROMPT)
cl.register("implot", Cline.LOCAL, Cline.PROMPT)
cl.register("ccd", Cline.LOCAL, Cline.PROMPT)
cl.register("nx", Cline.LOCAL, Cline.PROMPT)
cl.register("msub", Cline.GLOBAL, Cline.PROMPT)
cl.register("iset", Cline.GLOBAL, Cline.PROMPT)
cl.register("ilo", Cline.GLOBAL, Cline.PROMPT)
cl.register("ihi", Cline.GLOBAL, Cline.PROMPT)
cl.register("plo", Cline.GLOBAL, Cline.PROMPT)
cl.register("phi", Cline.LOCAL, Cline.PROMPT)
cl.register("xlo", Cline.GLOBAL, Cline.PROMPT)
cl.register("xhi", Cline.GLOBAL, Cline.PROMPT)
cl.register("ylo", Cline.GLOBAL, Cline.PROMPT)
cl.register("yhi", Cline.GLOBAL, Cline.PROMPT)
# get inputs
source = cl.get_value(
"source",
"data source [hs, hl, us, ul, hf]",
"hl",
lvals=("hs", "hl", "us", "ul", "hf"),
)
# set some flags
server_or_local = source.endswith("s") or source.endswith("l")
if server_or_local:
resource = cl.get_value("run", "run name", "run005")
first = cl.get_value("first", "first frame to reduce", 1, 0)
cl.set_default("last", 0)
last = cl.get_value("last", "last frame to reduce", 0, 0)
if last and last < first:
print("Cannot set last < first unless last == 0")
print("*** reduce aborted")
exit(1)
twait = cl.get_value("twait",
"time to wait for a new frame [secs]", 1.0,
0.0)
tmx = cl.get_value("tmax",
"maximum time to wait for a new frame [secs]",
10.0, 0.0)
# keep the reduce name and log in sync to save errors
root = os.path.basename(resource)
cl.set_default('rfile', cline.Fname(root, hcam.RED))
cl.set_default('log', cline.Fname(root, hcam.LOG, cline.Fname.NEW))
else:
resource = cl.get_value("flist", "file list",
cline.Fname("files.lis", hcam.LIST))
first = 1
last = 0
trim = cl.get_value("trim", "do you want to trim edges of windows?",
True)
if trim:
ncol = cl.get_value("ncol",
"number of columns to trim from windows", 0)
nrow = cl.get_value("nrow", "number of rows to trim from windows",
0)
# the reduce file
rfilen = cl.get_value("rfile", "reduce file",
cline.Fname("reduce.red", hcam.RED))
try:
rfile = Rfile.read(rfilen)
except hcam.HipercamError as err:
# abort on failure to read as there are many ways to get reduce
# files wrong
print(err, file=sys.stderr)
print("*** reduce aborted")
exit(1)
log = cl.get_value(
"log",
"name of log file to store results",
cline.Fname("reduce.log", hcam.LOG, cline.Fname.NEW),
)
tkeep = cl.get_value(
"tkeep",
"number of minute of data to"
" keep in internal buffers (0 for all)",
0.0,
0.0,
)
lplot = cl.get_value("lplot", "do you want to plot light curves?",
True)
implot = cl.get_value("implot", "do you want to plot images?", True)
if implot:
# define the panel grid. first get the labels and maximum
# dimensions
ccdinf = spooler.get_ccd_pars(source, resource)
try:
nxdef = cl.get_default("nx")
except KeyError:
nxdef = 3
if len(ccdinf) > 1:
ccd = cl.get_value("ccd", "CCD(s) to plot [0 for all]", "0")
if ccd == "0":
ccds = list(ccdinf.keys())
else:
ccds = ccd.split()
if len(ccds) > 1:
nxdef = min(len(ccds), nxdef)
cl.set_default("nx", nxdef)
nx = cl.get_value("nx", "number of panels in X", 3, 1)
else:
nx = 1
else:
nx = 1
ccds = list(ccdinf.keys())
# define the display intensities
msub = cl.get_value("msub", "subtract median from each window?",
True)
iset = cl.get_value(
"iset",
"set intensity a(utomatically),"
" d(irectly) or with p(ercentiles)?",
"a",
lvals=["a", "d", "p"],
)
plo, phi = 5, 95
ilo, ihi = 0, 1000
if iset == "d":
ilo = cl.get_value("ilo", "lower intensity limit", 0.0)
ihi = cl.get_value("ihi", "upper intensity limit", 1000.0)
elif iset == "p":
plo = cl.get_value("plo", "lower intensity limit percentile",
5.0, 0.0, 100.0)
phi = cl.get_value("phi", "upper intensity limit percentile",
95.0, 0.0, 100.0)
# region to plot
for i, cnam in enumerate(ccds):
nxtot, nytot, nxpad, nypad = ccdinf[cnam]
if i == 0:
xmin, xmax = float(-nxpad), float(nxtot + nxpad + 1)
ymin, ymax = float(-nypad), float(nytot + nypad + 1)
else:
xmin = min(xmin, float(-nxpad))
xmax = max(xmax, float(nxtot + nxpad + 1))
ymin = min(ymin, float(-nypad))
ymax = max(ymax, float(nytot + nypad + 1))
xlo = cl.get_value("xlo", "left-hand X value", xmin, xmin, xmax)
xhi = cl.get_value("xhi", "right-hand X value", xmax, xmin, xmax)
ylo = cl.get_value("ylo", "lower Y value", ymin, ymin, ymax)
yhi = cl.get_value("yhi", "upper Y value", ymax, ymin, ymax)
else:
ccds, nx, msub, iset = None, None, None, None
ilo, ihi, plo, phi = None, None, None, None
xlo, xhi, ylo, yhi = None, None, None, None
# save list of parameter values for writing to the reduction file
plist = cl.list()
################################################################
#
# all the inputs have now been obtained. Get on with doing stuff
if implot:
plot_lims = (xlo, xhi, ylo, yhi)
else:
plot_lims = None
imdev, lcdev, spanel, tpanel, xpanel, ypanel, lpanel = setup_plots(
rfile, ccds, nx, plot_lims, implot, lplot)
# a couple of initialisations
total_time = 0 # time waiting for new frame
if lplot:
lbuffer, xbuffer, ybuffer, tbuffer, sbuffer = setup_plot_buffers(rfile)
else:
lbuffer, xbuffer, ybuffer, tbuffer, sbuffer = None, None, None, None, None
############################################
#
# open the log file and write headers
#
with LogWriter(log, rfile, hipercam_version, plist) as logfile:
ncpu = rfile["general"]["ncpu"]
if ncpu > 1:
pool = multiprocessing.Pool(processes=ncpu)
else:
pool = None
# whether some parameters have been initialised
initialised = False
# containers for the processed and raw MCCD groups
# and their frame numbers
pccds, mccds, nframes = [], [], []
##############################################
#
# Finally, start winding through the frames
#
with spooler.data_source(source, resource, first, full=False) as spool:
# 'spool' is an iterable source of MCCDs
for nf, mccd in enumerate(spool):
if server_or_local:
# Handle the waiting game ...
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmx, total_time)
if give_up:
# Giving up, but need to handle any partially filled
# frame group
if len(mccds):
# finish processing remaining frames. This step
# will only occur if we have at least once passed
# to later stages during which read and gain will
# be set up
results = processor(pccds, mccds, nframes)
# write out results to the log file
alerts = logfile.write_results(results)
# print out any accumulated alert messages
if len(alerts):
print("\n".join(alerts))
update_plots(
results,
rfile,
implot,
lplot,
imdev,
lcdev,
pccd,
ccds,
msub,
nx,
iset,
plo,
phi,
ilo,
ihi,
xlo,
xhi,
ylo,
yhi,
lpanel,
xpanel,
ypanel,
tpanel,
spanel,
tkeep,
lbuffer,
xbuffer,
ybuffer,
tbuffer,
sbuffer,
)
mccds = []
print("reduce finished")
break
elif try_again:
continue
# Trim the frames: ULTRACAM windowed data has bad
# columns and rows on the sides of windows closest to
# the readout which can badly affect reduction. This
# option strips them.
if trim:
hcam.ccd.trim_ultracam(mccd, ncol, nrow)
# indicate progress
if "NFRAME" in mccd.head:
nframe = mccd.head["NFRAME"]
else:
nframe = nf + 1
if source != "hf" and last and nframe > last:
# finite last frame number
if len(mccds):
# finish processing remaining frames
results = processor(pccds, mccds, nframes)
# write out results to the log file
alerts = logfile.write_results(results)
# print out any accumulated alert messages
if len(alerts):
print("\n".join(alerts))
update_plots(
results,
rfile,
implot,
lplot,
imdev,
lcdev,
pccd,
ccds,
msub,
nx,
iset,
plo,
phi,
ilo,
ihi,
xlo,
xhi,
ylo,
yhi,
lpanel,
xpanel,
ypanel,
tpanel,
spanel,
tkeep,
lbuffer,
xbuffer,
ybuffer,
tbuffer,
sbuffer,
)
mccds = []
print("\nHave reduced up to the last frame set.")
print("reduce finished")
break
print(
"Frame {:d}: {:s} ({:s})".format(
nframe,
mccd.head["TIMSTAMP"],
"ok" if mccd.head.get("GOODTIME", True) else "nok",
),
end="" if implot else "\n",
)
if not initialised:
# This is the first frame which allows us to make
# some checks and initialisations.
read, gain, ok = initial_checks(mccd, rfile)
# Define the CCD processor function object
processor = ProcessCCDs(rfile, read, gain, ccdproc, pool)
# set flag to show we are set
if not ok:
break
initialised = True
# De-bias the data. Retain a copy of the raw data as 'mccd'
# in order to judge saturation. Processed data called 'pccd'
if rfile.bias is not None:
# subtract bias
pccd = mccd - rfile.bias
bexpose = rfile.bias.head.get("EXPTIME", 0.0)
else:
# no bias subtraction
pccd = mccd.copy()
bexpose = 0.0
if rfile.dark is not None:
# subtract dark, CCD by CCD
dexpose = rfile.dark.head["EXPTIME"]
for cnam in pccd:
ccd = pccd[cnam]
cexpose = ccd.head["EXPTIME"]
scale = (cexpose - bexpose) / dexpose
ccd -= scale * rfile.dark[cnam]
if rfile.flat is not None:
# apply flat field to processed frame
pccd /= rfile.flat
if rfile["focal_mask"]["demask"]:
# attempt to correct for poorly placed frame
# transfer mask causing a step illumination in the
# y-direction. Loop through all windows of all
# CCDs. Also include a stage where we average in
# the Y direction to try to eliminate high pixels.
dthresh = rfile["focal_mask"]["dthresh"]
for cnam, ccd in pccd.items():
for wnam, wind in ccd.items():
# form mean in Y direction, then try to
# mask out high pixels
ymean = np.mean(wind.data, 0)
xmask = ymean == ymean
while 1:
# rejection cycle, rejecting
# overly positive pixels
ave = ymean[xmask].mean()
rms = ymean[xmask].std()
diff = ymean - ave
diff[~xmask] = 0
imax = np.argmax(diff)
if diff[imax] > dthresh * rms:
xmask[imax] = False
else:
break
# form median in X direction
xmedian = np.median(wind.data[:, xmask], 1)
# subtract it's median to avoid removing
# general background
xmedian -= np.median(xmedian)
# now subtract from 2D image using
# broadcasting rules
wind.data -= xmedian.reshape((len(xmedian), 1))
# Acummulate frames into processing groups for faster
# parallelisation
pccds.append(pccd)
mccds.append(mccd)
nframes.append(nframe)
if len(pccds) == rfile["general"]["ngroup"]:
# parallel processing. This should usually be the first
# points at which it takes place
results = processor(pccds, mccds, nframes)
# write out results to the log file
alerts = logfile.write_results(results)
# print out any accumulated alert messages
if len(alerts):
print("\n".join(alerts))
update_plots(
results,
rfile,
implot,
lplot,
imdev,
lcdev,
pccds[-1],
ccds,
msub,
nx,
iset,
plo,
phi,
ilo,
ihi,
xlo,
xhi,
ylo,
yhi,
lpanel,
xpanel,
ypanel,
tpanel,
spanel,
tkeep,
lbuffer,
xbuffer,
ybuffer,
tbuffer,
sbuffer,
)
# Reset the frame buffers
pccds, mccds, nframes = [], [], []
if len(mccds):
# out of loop now. Finish processing any remaining
# frames.
results = processor(pccds, mccds, nframes)
# write out results to the log file
alerts = logfile.write_results(results)
# print out any accumulated alert messages
if len(alerts):
print("\n".join(alerts))
update_plots(
results,
rfile,
implot,
lplot,
imdev,
lcdev,
pccd,
ccds,
msub,
nx,
iset,
plo,
phi,
ilo,
ihi,
xlo,
xhi,
ylo,
yhi,
lpanel,
xpanel,
ypanel,
tpanel,
spanel,
tkeep,
lbuffer,
xbuffer,
ybuffer,
tbuffer,
sbuffer,
)
print("reduce finished")
def reduce(args=None):
"""``reduce [source] rfile (run first last (trim [ncol nrow]) twait
tmax | flist) log lplot implot (ccd nx msub xlo xhi ylo yhi iset
(ilo ihi | plo phi))``
Reduces a sequence of multi-CCD images, plotting lightcurves as images
come in. It can extract with either simple aperture photometry or Tim
Naylor's optimal photometry, on specific targets defined in an aperture
file using |setaper|.
reduce can source data from both the ULTRACAM and HiPERCAM servers, from
local 'raw' ULTRACAM and HiPERCAM files (i.e. .xml + .dat for ULTRACAM, 3D
FITS files for HiPERCAM) and from lists of HiPERCAM '.hcm' files. If you
have data from a different instrument you should convert into the
FITS-based hcm format.
reduce is primarily configured from a file with extension ".red". This
contains a series of directives, e.g. to say how to re-position and
re-size the apertures. An initial reduce file is best generated with
the script |genred| after you have created an aperture file. This contains
lots of help on what to do.
A reduce run can be terminated at any point with ctrl-C without doing
any harm. You may often want to do this at the start in order to adjust
parameters of the reduce file.
Parameters:
source : string [hidden]
Data source, five options:
| 'hs': HiPERCAM server
| 'hl': local HiPERCAM FITS file
| 'us': ULTRACAM server
| 'ul': local ULTRACAM .xml/.dat files
| 'hf': list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab' or
converted from foreign data formats.
rfile : string
the "reduce" file, i.e. ASCII text file suitable for reading by
ConfigParser. Best seen by example as it has many parts.
run : string [if source ends 's' or 'l']
run number to access, e.g. 'run034'
first : int [if source ends 's' or 'l']
first frame to reduce. 1 = first frame; set = 0 to always try to
get the most recent frame (if it has changed).
last : int [if source ends 's' or 'l', hidden]
last frame to reduce. 0 to just continue until the end. This is
not prompted for by default and must be set explicitly. It
defaults to 0 if not set. Its purpose is to allow accurate
profiling tests.
trim : bool [if source starts with 'u']
True to trim columns and/or rows off the edges of windows nearest
the readout. This is particularly for ULTRACAM windowed data where
the first few rows and columns can contain bad data.
ncol : int [if trim]
Number of columns to remove (on left of left-hand window, and right
of right-hand windows)
nrow : int [if trim]
Number of rows to remove (bottom of windows)
twait : float [if source ends 's'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [if source ends 's'; hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
flist : string [if source ends 'f']
name of file list
log : string
log file for the results
tkeep : float
maximum number of minutes of data to store in internal buffers, 0
for the lot. When large numbers of frames are stored, performance
can be slowed (although I am not entirely clear why) in which case
it makes sense to lose the earlier points (without affecting the
saving to disk). This parameter also gives operation similar to that
of "max_xrange" parameter in the ULTRACAM pipeline whereby just
the last few minutes are shown.
lplot : bool
flag to indicate you want to plot the light curve. Saves time not
to especially in high-speed runs.
implot : bool
flag to indicate you want to plot images.
ccd : string [if implot]
CCD(s) to plot, '0' for all, '1 3' to plot '1' and '3' only, etc.
nx : int [if implot]
number of panels across to display.
msub : bool [if implot]
subtract the median from each window before scaling for the
image display or not. This happens after any bias subtraction.
xlo : float [if implot]
left-hand X-limit for plot
xhi : float [if implot]
right-hand X-limit for plot (can actually be < xlo)
ylo : float [if implot]
lower Y-limit for plot
yhi : float [if implot]
upper Y-limit for plot (can be < ylo)
iset : string [if implot]
determines how the intensities are determined. There are three
options: 'a' for automatic simply scales from the minimum to the
maximum value found on a per CCD basis. 'd' for direct just takes
two numbers from the user. 'p' for percentile dtermines levels
based upon percentiles determined from the entire CCD on a per CCD
basis.
ilo : float [if implot and iset='d']
lower intensity level
ihi : float [if implot and iset='d']
upper intensity level
plo : float [if implot and iset='p']
lower percentile level
phi : float [if implot and iset='p']
upper percentile level
.. Warning::
The transmission plot generated with reduce is not reliable in the
case of optimal photometry since it is highly correlated with the
seeing. If you are worried about the transmission during observing,
you should always use normal aperture photometry.
"""
command, args = utils.script_args(args)
with Cline('HIPERCAM_ENV', '.hipercam', command, args) as cl:
# register parameters
cl.register('source', Cline.GLOBAL, Cline.HIDE)
cl.register('rfile', Cline.GLOBAL, Cline.PROMPT)
cl.register('run', Cline.GLOBAL, Cline.PROMPT)
cl.register('first', Cline.LOCAL, Cline.PROMPT)
cl.register('last', Cline.LOCAL, Cline.HIDE)
cl.register('trim', Cline.GLOBAL, Cline.PROMPT)
cl.register('ncol', Cline.GLOBAL, Cline.HIDE)
cl.register('nrow', Cline.GLOBAL, Cline.HIDE)
cl.register('twait', Cline.LOCAL, Cline.HIDE)
cl.register('tmax', Cline.LOCAL, Cline.HIDE)
cl.register('flist', Cline.LOCAL, Cline.PROMPT)
cl.register('log', Cline.GLOBAL, Cline.PROMPT)
cl.register('tkeep', Cline.GLOBAL, Cline.PROMPT)
cl.register('lplot', Cline.LOCAL, Cline.PROMPT)
cl.register('implot', Cline.LOCAL, Cline.PROMPT)
cl.register('ccd', Cline.LOCAL, Cline.PROMPT)
cl.register('nx', Cline.LOCAL, Cline.PROMPT)
cl.register('msub', Cline.GLOBAL, Cline.PROMPT)
cl.register('iset', Cline.GLOBAL, Cline.PROMPT)
cl.register('ilo', Cline.GLOBAL, Cline.PROMPT)
cl.register('ihi', Cline.GLOBAL, Cline.PROMPT)
cl.register('plo', Cline.GLOBAL, Cline.PROMPT)
cl.register('phi', Cline.LOCAL, Cline.PROMPT)
cl.register('xlo', Cline.GLOBAL, Cline.PROMPT)
cl.register('xhi', Cline.GLOBAL, Cline.PROMPT)
cl.register('ylo', Cline.GLOBAL, Cline.PROMPT)
cl.register('yhi', Cline.GLOBAL, Cline.PROMPT)
# get inputs
source = cl.get_value('source',
'data source [hs, hl, us, ul, hf]',
'hl',
lvals=('hs', 'hl', 'us', 'ul', 'hf'))
# set some flags
server_or_local = source.endswith('s') or source.endswith('l')
# the reduce file
rfilen = cl.get_value('rfile', 'reduce file',
cline.Fname('reduce.red', hcam.RED))
try:
rfile = Rfile.read(rfilen)
except hcam.HipercamError as err:
# abort on failure to read as there are many ways to get reduce
# files wrong
print(err, file=sys.stderr)
print('*** reduce aborted')
exit(1)
if server_or_local:
resource = cl.get_value('run', 'run name', 'run005')
first = cl.get_value('first', 'first frame to reduce', 1, 0)
cl.set_default('last', 0)
last = cl.get_value('last', 'last frame to reduce', 0, 0)
if last and last < first:
print('Cannot set last < first unless last == 0')
print('*** reduce aborted')
exit(1)
if source.startswith('u'):
trim = cl.get_value(
'trim',
'do you want to trim edges of windows? (ULTRACAM only)',
True)
if trim:
ncol = cl.get_value(
'ncol', 'number of columns to trim from windows', 0)
nrow = cl.get_value('nrow',
'number of rows to trim from windows',
0)
else:
trim = False
twait = cl.get_value('twait',
'time to wait for a new frame [secs]', 1., 0.)
tmx = cl.get_value('tmax',
'maximum time to wait for a new frame [secs]',
10., 0.)
else:
resource = cl.get_value('flist', 'file list',
cline.Fname('files.lis', hcam.LIST))
first = 1
last = 0
trim = False
log = cl.get_value(
'log', 'name of log file to store results',
cline.Fname('reduce.log', hcam.LOG, cline.Fname.NEW))
tkeep = cl.get_value(
'tkeep', 'number of minute of data to'
' keep in internal buffers (0 for all)', 0., 0.)
lplot = cl.get_value('lplot', 'do you want to plot light curves?',
True)
implot = cl.get_value('implot', 'do you want to plot images?', True)
if implot:
# define the panel grid. first get the labels and maximum
# dimensions
ccdinf = spooler.get_ccd_pars(source, resource)
try:
nxdef = cl.get_default('nx')
except KeyError:
nxdef = 3
if len(ccdinf) > 1:
ccd = cl.get_value('ccd', 'CCD(s) to plot [0 for all]', '0')
if ccd == '0':
ccds = list(ccdinf.keys())
else:
ccds = ccd.split()
if len(ccds) > 1:
nxdef = min(len(ccds), nxdef)
cl.set_default('nx', nxdef)
nx = cl.get_value('nx', 'number of panels in X', 3, 1)
else:
nx = 1
else:
nx = 1
ccds = list(ccdinf.keys())
# define the display intensities
msub = cl.get_value('msub', 'subtract median from each window?',
True)
iset = cl.get_value('iset', 'set intensity a(utomatically),'
' d(irectly) or with p(ercentiles)?',
'a',
lvals=['a', 'd', 'p'])
plo, phi = 5, 95
ilo, ihi = 0, 1000
if iset == 'd':
ilo = cl.get_value('ilo', 'lower intensity limit', 0.)
ihi = cl.get_value('ihi', 'upper intensity limit', 1000.)
elif iset == 'p':
plo = cl.get_value('plo', 'lower intensity limit percentile',
5., 0., 100.)
phi = cl.get_value('phi', 'upper intensity limit percentile',
95., 0., 100.)
# region to plot
for i, cnam in enumerate(ccds):
nxtot, nytot, nxpad, nypad = ccdinf[cnam]
if i == 0:
xmin, xmax = float(-nxpad), float(nxtot + nxpad + 1)
ymin, ymax = float(-nypad), float(nytot + nypad + 1)
else:
xmin = min(xmin, float(-nxpad))
xmax = max(xmax, float(nxtot + nxpad + 1))
ymin = min(ymin, float(-nypad))
ymax = max(ymax, float(nytot + nypad + 1))
xlo = cl.get_value('xlo', 'left-hand X value', xmin, xmin, xmax)
xhi = cl.get_value('xhi', 'right-hand X value', xmax, xmin, xmax)
ylo = cl.get_value('ylo', 'lower Y value', ymin, ymin, ymax)
yhi = cl.get_value('yhi', 'upper Y value', ymax, ymin, ymax)
else:
ccds, nx, msub, iset = None, None, None, None
ilo, ihi, plo, phi = None, None, None, None
xlo, xhi, ylo, yhi = None, None, None, None
# save list of parameter values for writing to the reduction file
plist = cl.list()
################################################################
#
# all the inputs have now been obtained. Get on with doing stuff
if implot:
plot_lims = (xlo, xhi, ylo, yhi)
else:
plot_lims = None
imdev, lcdev, spanel, tpanel, xpanel, ypanel, lpanel = setup_plots(
rfile, ccds, nx, plot_lims, implot, lplot)
# a couple of initialisations
total_time = 0 # time waiting for new frame
if lplot:
lbuffer, xbuffer, ybuffer, tbuffer, sbuffer = setup_plot_buffers(rfile)
else:
lbuffer, xbuffer, ybuffer, tbuffer, sbuffer = None, None, None, None, None
############################################
#
# open the log file and write headers
#
with LogWriter(log, rfile, hipercam_version, plist) as logfile:
ncpu = rfile['general']['ncpu']
if ncpu > 1:
pool = multiprocessing.Pool(processes=ncpu)
else:
pool = None
# whether a tzero has been set
tzset = False
# containers for the processed and raw MCCD groups
# and their frame numbers
pccds, mccds, nframes = [], [], []
##############################################
#
# Finally, start winding through the frames
#
with spooler.data_source(source, resource, first, full=False) as spool:
# 'spool' is an iterable source of MCCDs
for nf, mccd in enumerate(spool):
if server_or_local:
# Handle the waiting game ...
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmx, total_time)
if give_up:
# Giving up, but need to handle any partially filled
# frame group
if len(mccds):
# finish processing remaining frames. This step
# will only occur if we have at least once passed
# to later stages during which read and gain will
# be set up
results = processor(pccds, mccds, nframes)
# write out results to the log file
alerts = logfile.write_results(results)
# print out any accumulated alert messages
if len(alerts):
print('\n'.join(alerts))
update_plots(results, rfile, implot, lplot, imdev,
lcdev, pccd, ccds, msub, nx, iset,
plo, phi, ilo, ihi, xlo, xhi, ylo,
yhi, tzero, lpanel, xpanel, ypanel,
tpanel, spanel, tkeep, lbuffer,
xbuffer, ybuffer, tbuffer, sbuffer)
mccds = []
print('reduce finished')
break
elif try_again:
continue
# Trim the frames: ULTRACAM windowed data has bad
# columns and rows on the sides of windows closest to
# the readout which can badly affect reduction. This
# option strips them.
if trim:
hcam.ccd.trim_ultracam(mccd, ncol, nrow)
# indicate progress
if 'NFRAME' in mccd.head:
nframe = mccd.head['NFRAME']
else:
nframe = nf + 1
if source != 'hf' and last and nframe > last:
# finite last frame number
if len(mccds):
# finish processing remaining frames
results = processor(pccds, mccds, nframes)
# write out results to the log file
alerts = logfile.write_results(results)
# print out any accumulated alert messages
if len(alerts):
print('\n'.join(alerts))
update_plots(results, rfile, implot, lplot, imdev,
lcdev, pccd, ccds, msub, nx, iset, plo,
phi, ilo, ihi, xlo, xhi, ylo, yhi, tzero,
lpanel, xpanel, ypanel, tpanel, spanel,
tkeep, lbuffer, xbuffer, ybuffer, tbuffer,
sbuffer)
mccds = []
print('\nHave reduced up to the last frame set.')
print('reduce finished')
break
print('Frame {:d}: {:s} [{:s}]'.format(
nframe, mccd.head['TIMSTAMP'],
'OK' if mccd.head.get('GOODTIME', True) else 'NOK'),
end='' if implot else '\n')
if not tzset:
# This is the first frame which allows us to make
# some checks and initialisations.
tzero, read, gain, ok = initial_checks(mccd, rfile)
# Define the CCD processor function object
processor = ProcessCCDs(rfile, read, gain, ccdproc, pool)
# set flag to show we are set
if not ok:
break
tzset = True
# De-bias the data. Retain a copy of the raw data as 'mccd'
# in order to judge saturation. Processed data called 'pccd'
if rfile.bias is not None:
# subtract bias
pccd = mccd - rfile.bias
bexpose = rfile.bias.head.get('EXPTIME', 0.)
else:
# no bias subtraction
pccd = mccd.copy()
bexpose = 0.
if rfile.dark is not None:
# subtract dark, CCD by CCD
dexpose = rfile.dark.head['EXPTIME']
for cnam in pccd:
ccd = pccd[cnam]
cexpose = ccd.head['EXPTIME']
scale = (cexpose - bexpose) / dexpose
ccd -= scale * rfile.dark[cnam]
if rfile.flat is not None:
# apply flat field to processed frame
pccd /= rfile.flat
# Acummulate frames into processing groups for faster
# parallelisation
pccds.append(pccd)
mccds.append(mccd)
nframes.append(nframe)
if len(pccds) == rfile['general']['ngroup']:
# parallel processing. This should usually be the first
# points at which it takes place
results = processor(pccds, mccds, nframes)
# write out results to the log file
alerts = logfile.write_results(results)
# print out any accumulated alert messages
if len(alerts):
print('\n'.join(alerts))
update_plots(results, rfile, implot, lplot, imdev, lcdev,
pccds[-1], ccds, msub, nx, iset, plo, phi,
ilo, ihi, xlo, xhi, ylo, yhi, tzero, lpanel,
xpanel, ypanel, tpanel, spanel, tkeep,
lbuffer, xbuffer, ybuffer, tbuffer, sbuffer)
# Reset the frame buffers
pccds, mccds, nframes = [], [], []
if len(mccds):
# out of loop now. Finish processing any remaining
# frames.
results = processor(pccds, mccds, nframes)
# write out results to the log file
alerts = logfile.write_results(results)
# print out any accumulated alert messages
if len(alerts):
print('\n'.join(alerts))
update_plots(results, rfile, implot, lplot, imdev, lcdev, pccd,
ccds, msub, nx, iset, plo, phi, ilo, ihi, xlo, xhi,
ylo, yhi, tzero, lpanel, xpanel, ypanel, tpanel,
spanel, tkeep, lbuffer, xbuffer, ybuffer, tbuffer,
sbuffer)
print('reduce finished')
def ftargets(args=None):
"""``ftargets [source device width height] (run first [twait tmax] |
flist) trim ([ncol nrow]) (ccd (nx)) [pause] thresh fwhm minpix
output bias flat msub iset (ilo ihi | plo phi) xlo xhi ylo yhi``
This script carries out the following steps for each of a series
of images:
(1) detects the sources,
(2) identifies isolated targets suited to profile fits,
(3) fits 2D Moffat profiles to these,
(4) Saves results to disk.
The profile fits are carried out because `sep` does not return
anything that can be used reliably for a FWHM.
Several parameters depends on the object detection threshold
retuned by the source detection. This is referred to as
`threshold`. The source detection is carried out using `sep` which
runs according to the usual source extractor algorithm of Bertin.
The script plots the frames, with ellipses at 3*a,3*b indicated in
red, green boxes indicating the range of pixels identified by
`sep`, and blue boxes marking the targets selected for FWHM
fitting (the boxes indicate the fit region).
Parameters:
source : string [hidden]
Data source, five options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : ULTRACAM server
| 'ul' : local ULTRACAM .xml/.dat files
| 'hf' : list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab' or
converted from foreign data formats.
device : string [hidden]
Plot device. PGPLOT is used so this should be a PGPLOT-style name,
e.g. '/xs', '1/xs' etc. At the moment only ones ending /xs are
supported.
width : float [hidden]
plot width (inches). Set = 0 to let the program choose.
height : float [hidden]
plot height (inches). Set = 0 to let the program choose. BOTH width
AND height must be non-zero to have any effect
run : string [if source ends 's' or 'l']
run number to access, e.g. 'run034'
flist : string [if source ends 'f']
name of file list
first : int [if source ends 's' or 'l']
exposure number to start from. 1 = first frame; set = 0 to always
try to get the most recent frame (if it has changed). For data
from the |hiper| server, a negative number tries to get a frame not
quite at the end. i.e. -10 will try to get 10 from the last
frame. This is mainly to sidestep a difficult bug with the
acquisition system.
twait : float [if source ends 's' or 'l'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [if source ends 's' or 'l'; hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
trim : bool [if source starts with 'u']
True to trim columns and/or rows off the edges of windows nearest
the readout which can sometimes contain bad data.
ncol : int [if trim, hidden]
Number of columns to remove (on left of left-hand window, and right
of right-hand windows)
nrow : int [if trim, hidden]
Number of rows to remove (bottom of windows)
ccd : string
CCD(s) to plot, '0' for all, '1 3' to plot '1' and '3' only, etc.
nx : int [if more than 1 CCD]
number of panels across to display.
pause : float [hidden]
seconds to pause between frames (defaults to 0)
thresh : float
threshold (mutiple of RMS) to use for object detection. Typical
values 2.5 to 4. The higher it is, the fewer objects will be located,
but the fewer false detections will be made.
fwhm : float
FWHM to use for smoothing during object detection. Should be
comparable to the seeing.
minpix : int
Minimum number of pixels above threshold before convolution to count
as a detection. Useful in getting rid of cosmics and high dark count
pixels.
rmin : float
Closest distance of any other detected object for an attempt
to be made to fit the FWHM of an object [unbinned pixels].
pmin : float
Minimum peak height for an attempt to be made to fit the
FWHM of an object. This should be a multiple of the object
detection threshold (returned by `sep` for each object).
pmax : float
Maximum peak height for an attempt to be made to fit the
FWHM of an object. Use to exclude saturated targets
[counts]
emax : float
Maximum elongation (major/minor axis ratio = a/b), > 1. Use
to reduce very non-stellar profiles.
nmax : int
Maximum number of FWHMs to measure. Will take the brightest first,
judging by the flux.
bias : string
Name of bias frame to subtract, 'none' to ignore.
flat : string
Name of flat field to divide by, 'none' to ignore. Should normally
only be used in conjunction with a bias, although it does allow you
to specify a flat even if you haven't specified a bias.
output: string
Name of file for storage of results. Will be a fits file, with
results saved to the HDU 1 as a table.
iset : string [single character]
determines how the intensities are determined. There are three
options: 'a' for automatic simply scales from the minimum to the
maximum value found on a per CCD basis. 'd' for direct just takes
two numbers from the user. 'p' for percentile dtermines levels
based upon percentiles determined from the entire CCD on a per CCD
basis.
ilo : float [if iset='d']
lower intensity level
ihi : float [if iset='d']
upper intensity level
plo : float [if iset='p']
lower percentile level
phi : float [if iset='p']
upper percentile level
xlo : float
left-hand X-limit for plot
xhi : float
right-hand X-limit for plot (can actually be < xlo)
ylo : float
lower Y-limit for plot
yhi : float
upper Y-limit for plot (can be < ylo)
"""
command, args = utils.script_args(args)
# get the inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("source", Cline.GLOBAL, Cline.HIDE)
cl.register("device", Cline.LOCAL, Cline.HIDE)
cl.register("width", Cline.LOCAL, Cline.HIDE)
cl.register("height", Cline.LOCAL, Cline.HIDE)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("trim", Cline.GLOBAL, Cline.PROMPT)
cl.register("ncol", Cline.GLOBAL, Cline.HIDE)
cl.register("nrow", Cline.GLOBAL, Cline.HIDE)
cl.register("twait", Cline.LOCAL, Cline.HIDE)
cl.register("tmax", Cline.LOCAL, Cline.HIDE)
cl.register("flist", Cline.LOCAL, Cline.PROMPT)
cl.register("ccd", Cline.LOCAL, Cline.PROMPT)
cl.register("nx", Cline.LOCAL, Cline.PROMPT)
cl.register("pause", Cline.LOCAL, Cline.HIDE)
cl.register("thresh", Cline.LOCAL, Cline.PROMPT)
cl.register("fwhm", Cline.LOCAL, Cline.PROMPT)
cl.register("minpix", Cline.LOCAL, Cline.PROMPT)
cl.register("rmin", Cline.LOCAL, Cline.PROMPT)
cl.register("pmin", Cline.LOCAL, Cline.PROMPT)
cl.register("pmax", Cline.LOCAL, Cline.PROMPT)
cl.register("emax", Cline.LOCAL, Cline.PROMPT)
cl.register("nmax", Cline.LOCAL, Cline.PROMPT)
cl.register("gain", Cline.LOCAL, Cline.PROMPT)
cl.register("rej", Cline.LOCAL, Cline.PROMPT)
cl.register("bias", Cline.GLOBAL, Cline.PROMPT)
cl.register("flat", Cline.GLOBAL, Cline.PROMPT)
cl.register("output", Cline.LOCAL, Cline.PROMPT)
cl.register("iset", Cline.GLOBAL, Cline.PROMPT)
cl.register("ilo", Cline.GLOBAL, Cline.PROMPT)
cl.register("ihi", Cline.GLOBAL, Cline.PROMPT)
cl.register("plo", Cline.GLOBAL, Cline.PROMPT)
cl.register("phi", Cline.LOCAL, Cline.PROMPT)
cl.register("xlo", Cline.GLOBAL, Cline.PROMPT)
cl.register("xhi", Cline.GLOBAL, Cline.PROMPT)
cl.register("ylo", Cline.GLOBAL, Cline.PROMPT)
cl.register("yhi", Cline.GLOBAL, Cline.PROMPT)
# get inputs
source = cl.get_value(
"source",
"data source [hs, hl, us, ul, hf]",
"hl",
lvals=("hs", "hl", "us", "ul", "hf"),
)
# set some flags
server_or_local = source.endswith("s") or source.endswith("l")
# plot device stuff
device = cl.get_value("device", "plot device", "1/xs")
width = cl.get_value("width", "plot width (inches)", 0.0)
height = cl.get_value("height", "plot height (inches)", 0.0)
if server_or_local:
resource = cl.get_value("run", "run name", "run005")
if source == "hs":
first = cl.get_value("first", "first frame to plot", 1)
else:
first = cl.get_value("first", "first frame to plot", 1, 0)
twait = cl.get_value(
"twait", "time to wait for a new frame [secs]", 1.0, 0.0
)
tmax = cl.get_value(
"tmax", "maximum time to wait for a new frame [secs]", 10.0, 0.0
)
else:
resource = cl.get_value(
"flist", "file list", cline.Fname("files.lis", hcam.LIST)
)
first = 1
trim = cl.get_value("trim", "do you want to trim edges of windows?", True)
if trim:
ncol = cl.get_value("ncol", "number of columns to trim from windows", 0)
nrow = cl.get_value("nrow", "number of rows to trim from windows", 0)
# define the panel grid. first get the labels and maximum dimensions
ccdinf = spooler.get_ccd_pars(source, resource)
try:
nxdef = cl.get_default("nx")
except:
nxdef = 3
if len(ccdinf) > 1:
ccd = cl.get_value("ccd", "CCD(s) to plot [0 for all]", "0")
if ccd == "0":
ccds = list(ccdinf.keys())
else:
ccds = ccd.split()
check = set(ccdinf.keys())
if not set(ccds) <= check:
raise hcam.HipercamError("At least one invalid CCD label supplied")
if len(ccds) > 1:
nxdef = min(len(ccds), nxdef)
cl.set_default("nx", nxdef)
nx = cl.get_value("nx", "number of panels in X", 3, 1)
else:
nx = 1
else:
nx = 1
ccds = list(ccdinf.keys())
cl.set_default("pause", 0.0)
pause = cl.get_value(
"pause", "time delay to add between" " frame plots [secs]", 0.0, 0.0
)
thresh = cl.get_value("thresh", "source detection threshold [RMS]", 3.0)
fwhm = cl.get_value("fwhm", "FWHM for source detection [binned pixels]", 4.0)
minpix = cl.get_value("minpix", "minimum number of pixels above threshold", 3)
rmin = cl.get_value(
"rmin", "nearest neighbour for FWHM fits [unbinned pixels]", 20.0
)
pmin = cl.get_value(
"pmin", "minimum peak value for profile fits [multiple of threshold]", 5.0
)
pmax = cl.get_value(
"pmax", "maximum peak value for profile fits [counts]", 60000.0
)
emax = cl.get_value(
"emax", "maximum elongation (a/b) for profile fits", 1.2, 1.0
)
nmax = cl.get_value("nmax", "maximum number of profile fits", 10, 1)
gain = cl.get_value("gain", "CCD gain [electrons/ADU]", 1.0, 0.0)
rej = cl.get_value(
"rej", "rejection threshold for profile fits [RMS]", 6.0, 2.0
)
# bias frame (if any)
bias = cl.get_value(
"bias",
"bias frame ['none' to ignore]",
cline.Fname("bias", hcam.HCAM),
ignore="none",
)
if bias is not None:
# read the bias frame
bias = hcam.MCCD.read(bias)
fprompt = "flat frame ['none' to ignore]"
else:
fprompt = "flat frame ['none' is normal choice with no bias]"
# flat (if any)
flat = cl.get_value(
"flat", fprompt, cline.Fname("flat", hcam.HCAM), ignore="none"
)
if flat is not None:
# read the flat frame
flat = hcam.MCCD.read(flat)
output = cl.get_value(
"output",
"output file for results",
cline.Fname("sources", hcam.SEP, cline.Fname.NEW),
)
iset = cl.get_value(
"iset",
"set intensity a(utomatically)," " d(irectly) or with p(ercentiles)?",
"a",
lvals=["a", "d", "p"],
)
iset = iset.lower()
plo, phi = 5, 95
ilo, ihi = 0, 1000
if iset == "d":
ilo = cl.get_value("ilo", "lower intensity limit", 0.0)
ihi = cl.get_value("ihi", "upper intensity limit", 1000.0)
elif iset == "p":
plo = cl.get_value(
"plo", "lower intensity limit percentile", 5.0, 0.0, 100.0
)
phi = cl.get_value(
"phi", "upper intensity limit percentile", 95.0, 0.0, 100.0
)
# region to plot
for i, cnam in enumerate(ccds):
nxtot, nytot, nxpad, nypad = ccdinf[cnam]
if i == 0:
xmin, xmax = float(-nxpad), float(nxtot + nxpad + 1)
ymin, ymax = float(-nypad), float(nytot + nypad + 1)
else:
xmin = min(xmin, float(-nxpad))
xmax = max(xmax, float(nxtot + nxpad + 1))
ymin = min(ymin, float(-nypad))
ymax = max(ymax, float(nytot + nypad + 1))
xlo = cl.get_value("xlo", "left-hand X value", xmin, xmin, xmax)
xhi = cl.get_value("xhi", "right-hand X value", xmax, xmin, xmax)
ylo = cl.get_value("ylo", "lower Y value", ymin, ymin, ymax)
yhi = cl.get_value("yhi", "upper Y value", ymax, ymin, ymax)
################################################################
#
# all the inputs have now been obtained. Get on with doing stuff
# open image plot device
imdev = hcam.pgp.Device(device)
if width > 0 and height > 0:
pgpap(width, height / width)
# set up panels and axes
nccd = len(ccds)
ny = nccd // nx if nccd % nx == 0 else nccd // nx + 1
# slice up viewport
pgsubp(nx, ny)
# plot axes, labels, titles. Happens once only
for cnam in ccds:
pgsci(hcam.pgp.Params["axis.ci"])
pgsch(hcam.pgp.Params["axis.number.ch"])
pgenv(xlo, xhi, ylo, yhi, 1, 0)
pglab("X", "Y", "CCD {:s}".format(cnam))
# initialisations. 'last_ok' is used to store the last OK frames of each
# CCD for retrieval when coping with skipped data.
total_time = 0 # time waiting for new frame
nhdu = len(ccds) * [0]
thetas = np.linspace(0, 2 * np.pi, 100)
# values of various parameters
fwhm_min, beta, beta_min, beta_max, readout, max_nfev = (
2.0,
4.0,
1.5,
10.0,
5.5,
100,
)
# number of failed fits
nfail = 0
# open the output file for results
with fitsio.FITS(output, "rw", clobber=True) as fout:
# plot images
with spooler.data_source(source, resource, first, full=False) as spool:
# 'spool' is an iterable source of MCCDs
n = 0
for nf, mccd in enumerate(spool):
if server_or_local:
# Handle the waiting game ...
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmax, total_time
)
if give_up:
print("ftargets stopped")
break
elif try_again:
continue
# Trim the frames: ULTRACAM windowed data has bad columns
# and rows on the sides of windows closest to the readout
# which can badly affect reduction. This option strips
# them.
if trim:
hcam.ccd.trim_ultracam(mccd, ncol, nrow)
# indicate progress
tstamp = Time(mccd.head["TIMSTAMP"], format="isot", precision=3)
print(
"{:d}, utc= {:s} ({:s}), ".format(
mccd.head["NFRAME"],
tstamp.iso,
"ok" if mccd.head.get("GOODTIME", True) else "nok",
),
end="",
)
# accumulate errors
emessages = []
if n == 0:
if bias is not None:
# crop the bias on the first frame only
bias = bias.crop(mccd)
if flat is not None:
# crop the flat on the first frame only
flat = flat.crop(mccd)
# compute maximum length of window name strings
lsmax = 0
for ccd in mccd.values():
for wnam in ccd:
lsmax = max(lsmax, len(wnam))
# display the CCDs chosen
message = ""
pgbbuf()
for nc, cnam in enumerate(ccds):
ccd = mccd[cnam]
if ccd.is_data():
# this should be data as opposed to a blank frame
# between data frames that occur with nskip > 0
# subtract the bias
if bias is not None:
ccd -= bias[cnam]
# divide out the flat
if flat is not None:
ccd /= flat[cnam]
# Where the fancy stuff happens ...
# estimate sky background, look for stars
objs, dofwhms = [], []
nobj = 0
for wnam in ccd:
try:
# chop window, find objects
wind = ccd[wnam].window(xlo, xhi, ylo, yhi)
wind.data = wind.data.astype("float")
objects, bkg = findStars(wind, thresh, fwhm, True)
# subtract background from frame for display
# purposes.
bkg.subfrom(wind.data)
ccd[wnam] = wind
# remove targets with too few pixels
objects = objects[objects["tnpix"] >= minpix]
# run nearest neighbour search on all
# objects, but select only a subset
# with right count levels for FWHM
# measurement, and which are not too
# elongated
results = isolated(objects["x"], objects["y"], rmin)
peaks = objects["peak"]
ok = (
(peaks < pmax)
& (peaks > pmin * objects["thresh"])
& (objects["a"] < emax * objects["b"])
)
dfwhms = np.array(
[
i
for i in range(len(results))
if ok[i] and len(results[i]) == 1
], dtype=int
)
# pick the brightest. 'dfwhms' are the
# indices of the selected targets for
# FWHM measurement
if len(dfwhms):
fluxes = objects["flux"][dfwhms]
isort = np.argsort(fluxes)[::-1]
dfwhms = dfwhms[isort[:nmax]]
# buffer for storing the FWHMs, including NaNs
# for the ones thet are skipped
fwhms = np.zeros_like(peaks, dtype=np.float32)
betas = np.zeros_like(peaks, dtype=np.float32)
nfevs = np.zeros_like(peaks, dtype=np.int32)
for i, (x, y, peak, fwhm) in enumerate(
zip(
objects["x"],
objects["y"],
peaks,
objects["fwhm"],
)
):
# fit FWHMs of selected targets
if i in dfwhms:
try:
# extract fit Window
fwind = wind.window(
x - rmin, x + rmin, y - rmin, y + rmin
)
# fit profile
ofwhm = fwhm
obeta = beta
(
(sky, height, x, y, fwhm, beta),
epars,
(
wfit,
X,
Y,
sigma,
chisq,
nok,
nrej,
npar,
nfev,
),
) = hcam.fitting.fitMoffat(
fwind,
None,
peak,
x,
y,
fwhm,
2.0,
False,
beta,
beta_max,
False,
readout,
gain,
rej,
1,
max_nfev,
)
fwhms[i] = fwhm
betas[i] = beta
nfevs[i] = nfev
# keep value of beta for next round under control
beta = min(beta_max, max(beta_min, beta))
except hcam.HipercamError as err:
emessages.append(
" >> Targ {:d}: fit failed ***: {!s}".format(
i, err
)
)
fwhms[i] = np.nan
betas[i] = np.nan
nfevs[i] = 0
nfail += 1
else:
# skip this one
fwhms[i] = np.nan
betas[i] = np.nan
nfevs[i] = 0
# tack on frame number & window name
frames = (nf + first) * np.ones(
len(objects), dtype=np.int32
)
wnams = np.array(
len(objects) * [wnam], dtype="U{:d}".format(lsmax)
)
objects = append_fields(
objects,
("ffwhm", "beta", "nfev", "nframe", "wnam"),
(fwhms, betas, nfevs, frames, wnams),
)
# save the objects and the
objs.append(objects)
dofwhms.append(dfwhms + nobj)
print(dfwhms,nobj,dofwhms)
nobj += len(objects)
except hcam.HipercamError:
# window may have no overlap with xlo, xhi
# ylo, yhi
pass
if len(objs):
# Plot targets found
# concatenate results of all Windows
objs = np.concatenate(objs)
dofwhms = np.concatenate(dofwhms)
# set to the correct panel and then plot CCD
ix = (nc % nx) + 1
iy = nc // nx + 1
pgpanl(ix, iy)
vmin, vmax = hcam.pgp.pCcd(
ccd,
iset,
plo,
phi,
ilo,
ihi,
xlo=xlo,
xhi=xhi,
ylo=ylo,
yhi=yhi,
)
pgsci(core.CNAMS["red"])
As, Bs, Thetas, Xs, Ys = (
objs["a"],
objs["b"],
objs["theta"],
objs["x"],
objs["y"],
)
for a, b, theta0, x, y in zip(As, Bs, Thetas, Xs, Ys):
xs = x + 3 * a * np.cos(thetas + theta0)
ys = y + 3 * b * np.sin(thetas + theta0)
pgline(xs, ys)
pgsci(core.CNAMS["green"])
for xmin, xmax, ymin, ymax in zip(
objs["xmin"], objs["xmax"], objs["ymin"], objs["ymax"]
):
xs = [xmin, xmax, xmax, xmin, xmin]
ys = [ymin, ymin, ymax, ymax, ymin]
pgline(xs, ys)
pgsci(core.CNAMS["blue"])
if len(dofwhms):
print(dofwhms)
for x, y in zip(objs["x"][dofwhms], objs["y"][dofwhms]):
xs = [x - rmin, x + rmin, x + rmin, x - rmin, x - rmin]
ys = [y - rmin, y - rmin, y + rmin, y + rmin, y - rmin]
pgline(xs, ys)
# remove some less useful fields to save a
# bit more space prior to saving to disk
objs = remove_field_names(
objs,
(
"x2",
"y2",
"xy",
"cxx",
"cxy",
"cyy",
"hfd",
"cflux",
"cpeak",
"xcpeak",
"ycpeak",
"xmin",
"xmax",
"ymin",
"ymax",
),
)
# save to disk
if nhdu[nc]:
fout[nhdu[nc]].append(objs)
else:
fout.write(objs)
nhdu[nc] = nc + 1
# accumulate string of image scalings
if nc:
message += ", ccd {:s}: {:.1f}, {:.1f}, exp: {:.4f}".format(
cnam, vmin, vmax, mccd.head["EXPTIME"]
)
else:
message += "ccd {:s}: {:.1f}, {:.1f}, exp: {:.4f}".format(
cnam, vmin, vmax, mccd.head["EXPTIME"]
)
pgebuf()
# end of CCD display loop
print(message)
for emessage in emessages:
print(emessage)
if pause > 0.0:
# pause between frames
time.sleep(pause)
# update the frame number
n += 1
print("Number of failed fits =", nfail)
def makemovie(args=None):
"""``makemovie [source] (run first last | flist) trim ([ncol nrow])
(ccd (nx)) bias flat defect log (targ comp ymin ymax yscales
yoffset location fraction lpad) cmap width height dstore ndigit fext msub iset
(ilo ihi | plo phi) xlo xhi ylo yhi [dpi (style (ms lw))]``
``makemovie`` is for generating stills to combine into a movie for
presentations. It can optionally also read a log file from the run
to display an evolving light curve. There are lots of fiddly
parameters mostly to do with the plot positioning, so try it out
on a small number of frames first before going mad.
Parameters:
source : str [hidden]
Data source, five options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : ULTRACAM server
| 'ul' : local ULTRACAM .xml/.dat files
| 'hf' : list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab'
or converted from foreign data formats. The standard
start-off default for ``source`` can be set using the
environment variable HIPERCAM_DEFAULT_SOURCE. e.g. in bash
:code:`export HIPERCAM_DEFAULT_SOURCE="us"` would ensure it
always started with the ULTRACAM server by default. If
unspecified, it defaults to 'hl'.
run : str [if source ends 's' or 'l']
run number to access, e.g. 'run034'
flist : str [if source ends 'f']
name of file list
first : int [if source ends 's' or 'l']
exposure number to start from. 1 = first frame.
last : int [if source ends 's' or 'l']
last exposure
trim : bool [if source starts with 'u']
True to trim columns and/or rows off the edges of windows nearest
the readout which can sometimes contain bad data.
ncol : int [if trim, hidden]
Number of columns to remove (on left of left-hand window, and right
of right-hand windows)
nrow : int [if trim, hidden]
Number of rows to remove (bottom of windows)
ccd : str
CCD(s) to plot, '0' for all, '1 3' to plot '1' and '3' only, etc. If
you plot more than one, then a legend is added to any light curve panel
to distinguish the light curves.
nx : int [if more than 1 CCD]
number of panels across to display for the image display.
bias : str
Name of bias frame to subtract, 'none' to ignore.
dark : str
Name of dark frame to subtract, 'none' to ignore.
flat : str
Name of flat field to divide by, 'none' to ignore. Should normally
only be used in conjunction with a bias, although it does allow you
to specify a flat even if you haven't specified a bias.
defect : str
Name of defect file, 'none' to ignore.
fmap : str
Name of fringe map (see e.g. `makefringe`), 'none' to ignore.
fpair : str [if fringe is not 'none']
Name of fringe pair file (see e.g. `setfringe`). Required if
a fringe map has been specified.
nhalf : int [if fringe is not 'none', hidden]
When calculating the differences for fringe measurement,
a region extending +/-nhalf binned pixels will be used when
measuring the amplitudes. Basically helps the stats.
rmin : float [if fringe is not 'none', hidden]
Minimum individual ratio to accept prior to calculating the overall
median in order to reduce the effect of outliers. Although all ratios
should be positive, you might want to set this a little below zero
to allow for some statistical fluctuation.
rmax : float [if fringe is not 'none', hidden]
Maximum individual ratio to accept prior to calculating the overall
median in order to reduce the effect of outliers. Probably typically
< 1 if fringe map was created from longer exposure data.
log : str
Name of reduce log file for light curve plot, 'none' to ignore
targ : str [if log defined]
Target aperture
comp : str [if log defined]
Comparison aperture
ymin : float [if log defined]
Minimum Y value for light curve plot
ymax : float [if log defined]
Maximum Y value for light curve plot
ynorm : list(float) [if log defined]
Normalisation factors, one per CCD for light curve plot
yoffset : list(float) [if log defined]
Offsets, one per CCD for light curve plot
location : str [if log defined]
Offsets, one per CCD for light curve plot
fraction : float [if log defined]
Fraction of figure to occupy, by height if location is South,
by width if it is East
lpad : tuple(float) [if log defined]
padding on left, bottom, right and top of light curve plot as fraction
of allocated width and height
cmap : str
The matplotlib colour map to use. "Greys" gives the usual greyscale.
"none" will give whatever the current default is. Many other choices:
"viridis", "jet", "hot", "Oranges", etc. Enter an invalid one and
the program will fail but return a huge list of possibles in the
process.
width : float
plot width in inches.
height : float
plot height in inches.
dstore : str
root directory for plot files. Will get names like dstore/run003_001.png.
ndigit : int
number of digits in frame counter, i.e. the '001' of the previous section.
fext : str
file extension 'png', 'jpeg' for images generated
msub : bool
subtract the median from each window before scaling for the
image display or not. This happens after any bias subtraction.
iset : str [single character]
determines how the intensities are determined. There are three
options: 'a' for automatic simply scales from the minimum to the
maximum value found on a per CCD basis. 'd' for direct just takes
two numbers from the user. 'p' for percentile dtermines levels
based upon percentiles determined from the entire CCD on a per CCD
basis.
ilo : list(float) [if iset='d']
lower intensity level, one per image
ihi : list(float) [if iset='d']
upper intensity level, one per image
plo : float [if iset='p']
lower percentile level
phi : float [if iset='p']
upper percentile level
xlo : float
left-hand X-limit for plot, initially at least since it is
possible to re-size. For iset='p' these limits also set the
region of the frame over which the percentil will be
calculated. You will usually want yhi-ylo ~ xhi-xlo in
magnitude because the aspect ratio is preserved.
xhi : float
right-hand X-limit for plot (can be < xlo to invert the display)
ylo : float
lower Y-limit for plot
yhi : float
upper Y-limit for plot (can be < ylo)
dpi : int [hidden]
dots per inch of output. Default 72. Allows control over font size versus image size,
in combination with width and height.
style : str [hidden, if log defined]
style for light curves 'dots', 'line', 'both'. The line will be grey for the 'both' option.
ms : float [hidden, if log defined and style==dots or both]
markersize. Controls dot size which is useful when fiddling with dpi
lw : float [hidden, if log defined and style==line or both]
line width
"""
command, args = utils.script_args(args)
# get the inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("source", Cline.GLOBAL, Cline.HIDE)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("last", Cline.LOCAL, Cline.PROMPT)
cl.register("trim", Cline.GLOBAL, Cline.PROMPT)
cl.register("ncol", Cline.GLOBAL, Cline.HIDE)
cl.register("nrow", Cline.GLOBAL, Cline.HIDE)
cl.register("flist", Cline.LOCAL, Cline.PROMPT)
cl.register("ccd", Cline.LOCAL, Cline.PROMPT)
cl.register("nx", Cline.LOCAL, Cline.PROMPT)
cl.register("bias", Cline.GLOBAL, Cline.PROMPT)
cl.register("dark", Cline.GLOBAL, Cline.PROMPT)
cl.register("flat", Cline.GLOBAL, Cline.PROMPT)
cl.register("fmap", Cline.GLOBAL, Cline.PROMPT)
cl.register("fpair", Cline.GLOBAL, Cline.PROMPT)
cl.register("nhalf", Cline.GLOBAL, Cline.HIDE)
cl.register("rmin", Cline.GLOBAL, Cline.HIDE)
cl.register("rmax", Cline.GLOBAL, Cline.HIDE)
cl.register("defect", Cline.GLOBAL, Cline.PROMPT)
cl.register("log", Cline.LOCAL, Cline.PROMPT)
cl.register("targ", Cline.LOCAL, Cline.PROMPT)
cl.register("comp", Cline.LOCAL, Cline.PROMPT)
cl.register("ymin", Cline.LOCAL, Cline.PROMPT)
cl.register("ymax", Cline.LOCAL, Cline.PROMPT)
cl.register("ynorm", Cline.LOCAL, Cline.PROMPT)
cl.register("yoffset", Cline.LOCAL, Cline.PROMPT)
cl.register("location", Cline.LOCAL, Cline.PROMPT)
cl.register("fraction", Cline.LOCAL, Cline.PROMPT)
cl.register("lpad", Cline.LOCAL, Cline.PROMPT)
cl.register("cmap", Cline.LOCAL, Cline.PROMPT)
cl.register("width", Cline.LOCAL, Cline.PROMPT)
cl.register("height", Cline.LOCAL, Cline.PROMPT)
cl.register("dstore", Cline.LOCAL, Cline.PROMPT)
cl.register("ndigit", Cline.LOCAL, Cline.PROMPT)
cl.register("fext", Cline.LOCAL, Cline.PROMPT)
cl.register("msub", Cline.GLOBAL, Cline.PROMPT)
cl.register("iset", Cline.GLOBAL, Cline.PROMPT)
cl.register("ilo", Cline.LOCAL, Cline.PROMPT)
cl.register("ihi", Cline.LOCAL, Cline.PROMPT)
cl.register("plo", Cline.GLOBAL, Cline.PROMPT)
cl.register("phi", Cline.LOCAL, Cline.PROMPT)
cl.register("xlo", Cline.GLOBAL, Cline.PROMPT)
cl.register("xhi", Cline.GLOBAL, Cline.PROMPT)
cl.register("ylo", Cline.GLOBAL, Cline.PROMPT)
cl.register("yhi", Cline.GLOBAL, Cline.PROMPT)
cl.register("dpi", Cline.LOCAL, Cline.HIDE)
cl.register("style", Cline.LOCAL, Cline.HIDE)
cl.register("ms", Cline.LOCAL, Cline.HIDE)
cl.register("lw", Cline.LOCAL, Cline.HIDE)
# get inputs
default_source = os.environ.get('HIPERCAM_DEFAULT_SOURCE', 'hl')
source = cl.get_value(
"source",
"data source [hs, hl, us, ul, hf]",
default_source,
lvals=("hs", "hl", "us", "ul", "hf"),
)
# set some flags
server_or_local = source.endswith("s") or source.endswith("l")
if server_or_local:
resource = cl.get_value("run", "run name", "run005")
first = cl.get_value("first", "first frame to plot", 1)
last = cl.get_value("last",
"last frame to plot [0 to go to the end]",
max(1, first), 0)
else:
resource = cl.get_value("flist", "file list",
cline.Fname("files.lis", hcam.LIST))
first = 1
trim = cl.get_value("trim", "do you want to trim edges of windows?",
True)
if trim:
ncol = cl.get_value("ncol",
"number of columns to trim from windows", 0)
nrow = cl.get_value("nrow", "number of rows to trim from windows",
0)
else:
ncol, nrow = None, None
# define the panel grid. first get the labels and maximum dimensions
ccdinf = spooler.get_ccd_pars(source, resource)
nxdef = cl.get_default("nx", 3)
if len(ccdinf) > 1:
ccd = cl.get_value("ccd", "CCD(s) to plot [0 for all]", "0")
if ccd == "0":
ccds = list(ccdinf.keys())
else:
ccds = ccd.split()
check = set(ccdinf.keys())
if not set(ccds) <= check:
raise hcam.HipercamError(
"At least one invalid CCD label supplied")
if len(ccds) > 1:
nxdef = min(len(ccds), nxdef)
cl.set_default("nx", nxdef)
nx = cl.get_value("nx", "number of panels in X", 3, 1)
else:
nx = 1
else:
nx = 1
ccds = list(ccdinf.keys())
# bias frame (if any)
bias = cl.get_value(
"bias",
"bias frame ['none' to ignore]",
cline.Fname("bias", hcam.HCAM),
ignore="none",
)
if bias is not None:
# read the bias frame
bias = hcam.MCCD.read(bias)
fprompt = "flat frame ['none' to ignore]"
else:
fprompt = "flat frame ['none' is normal choice with no bias]"
# dark (if any)
dark = cl.get_value("dark",
"dark frame to subtract ['none' to ignore]",
cline.Fname("dark", hcam.HCAM),
ignore="none")
if dark is not None:
# read the dark frame
dark = hcam.MCCD.read(dark)
# flat (if any)
flat = cl.get_value("flat",
fprompt,
cline.Fname("flat", hcam.HCAM),
ignore="none")
if flat is not None:
# read the flat frame
flat = hcam.MCCD.read(flat)
# fringe file (if any)
fmap = cl.get_value(
"fmap",
"fringe map ['none' to ignore]",
cline.Fname("fmap", hcam.HCAM),
ignore="none",
)
if fmap is not None:
# read the fringe map
fmap = hcam.MCCD.read(fmap)
fpair = cl.get_value("fpair", "fringe pair file",
cline.Fname("fpair", hcam.FRNG))
fpair = fringe.MccdFringePair.read(fpair)
nhalf = cl.get_value("nhalf",
"half-size of fringe measurement regions", 2,
0)
rmin = cl.get_value("rmin", "minimum fringe pair ratio", -0.2)
rmax = cl.get_value("rmax", "maximum fringe pair ratio", 1.0)
# defect file (if any)
dfct = cl.get_value(
"defect",
"defect file ['none' to ignore]",
cline.Fname("defect", hcam.DFCT),
ignore="none",
)
if dfct is not None:
# read the defect frame
dfct = defect.MccdDefect.read(dfct)
# reduce log file (if any)
rlog = cl.get_value(
"log",
"reduce log file ['none' to ignore]",
cline.Fname("reduce", hcam.LOG),
ignore="none",
)
if rlog is not None:
# Read reduce file
hlg = hlog.Hlog.rascii(rlog)
targ = cl.get_value("targ", "target aperture", "1")
comp = cl.get_value("comp", "comparison aperture", "2")
fmin = cl.get_value("ymin", "minimum Y value for light curve plot",
0.)
fmax = cl.get_value("ymax", "maxmum Y value for light curve plot",
1.)
# need to check that the default has the right number of
# items, if not overr-ride it
ynorm = cl.get_default("ynorm")
if ynorm is not None:
if len(ynorm) > len(ccds):
cl.set_default("ynorm", ynorm[:len(ccds)])
elif len(ynorm) < len(ccds):
cl.set_default("ynorm",
ynorm + (len(ccds) - len(ynorm)) * [1.])
ynorm = cl.get_value(
"ynorm",
"normalisation factors for light curves (one per CCD)",
len(ccds) * [1.])
yoffset = cl.get_default("yoffset")
if yoffset is not None:
if len(yoffset) > len(ccds):
cl.set_default("yoffset", yoffset[:len(ccds)])
elif len(yoffset) < len(ccds):
cl.set_default("yoffset",
yoffset + (len(ccds) - len(yoffset)) * [0.])
yoffset = cl.get_value(
"yoffset", "vertical offsets for light curves (one per CCD)",
len(ccds) * [0.])
location = cl.get_value(
"location",
"position of light curve plot relative to images",
"s",
lvals=['s', 'e', 'S', 'E'])
if location.lower() == 's':
fraction = cl.get_value(
"fraction",
"fraction of figure in terms of height occupied by light curve",
0.5)
elif location.lower() == 'e':
fraction = cl.get_value(
"fraction",
"fraction of figure in terms of width occupied by light curve",
0.67)
lpad = cl.get_value(
"lpad",
"padding (left,bottom,right,top) around light curve plot",
(0.05, 0.05, 0.02, 0.02))
# trim down to the specified frames
lcs = []
T0, tmax = None, None
for cnam, yn, yo in zip(ccds, ynorm, yoffset):
nframes = hlg[cnam]['nframe']
keep = (nframes >= first) and (last == 0 or (nframes <= last))
hlg[cnam] = hlg[cnam][keep]
lc = (hlg.tseries(cnam, targ) /
hlg.tseries(cnam, comp)) / yn + yo
if T0 is None:
T0 = lc.t.min()
lc.ttrans(lambda t: 1440 * (t - T0))
if tmax is None:
tmax = lc.t.max()
else:
tmax = max(tmax, lc.t.max())
lcs.append((nframes[keep], lc))
# Some settings for the image plots
cmap = cl.get_value("cmap",
"colour map to use ['none' for mpl default]",
"Greys")
cmap = None if cmap == "none" else cmap
width = cl.get_value("width", "plot width [inches]", 10., 0.5)
height = cl.get_value("height", "plot height [inches]", 10., 0.5)
dstore = cl.get_value("dstore", "directory for images", "tmp")
if not os.path.isdir(dstore):
raise hcam.HipercamError(f"'{dstore}' is not a directory")
ndigit = cl.get_value("ndigit",
"number of digits for appended frame counter", 4,
1)
fext = cl.get_value("fext",
"file extension for images",
"png",
lvals=["png", "jpg"])
# define the display intensities
msub = cl.get_value("msub", "subtract median from each window?", True)
iset = cl.get_value(
"iset",
"set intensity a(utomatically),"
" d(irectly) or with p(ercentiles)?",
"a",
lvals=["a", "d", "p"],
)
iset = iset.lower()
plo, phi = 5, 95
ilos, ihis = len(ccds) * [0], len(ccds) * [1000]
if iset == "d":
# fiddle with the defaults
ilo = cl.get_default("ilo")
if ilo is not None:
if len(ilo) > len(ccds):
cl.set_default("ilo", ilo[:len(ccds)])
elif len(ilo) < len(ccds):
cl.set_default("ilo", ilo + (len(ccds) - len(ilo)) * [0.])
ihi = cl.get_default("ihi")
if ihi is not None:
if len(ihi) > len(ccds):
cl.set_default("ihi", ihi[:len(ccds)])
elif len(ihi) < len(ccds):
cl.set_default("ihi", ihi + (len(ccds) - len(ihi))[1000.])
ilos = cl.get_value("ilo", "lower intensity limit",
len(ccds) * [0.])
ihis = cl.get_value("ihi", "upper intensity limit",
len(ccds) * [1000.])
elif iset == "p":
plo = cl.get_value("plo", "lower intensity limit percentile", 5.0,
0.0, 100.0)
phi = cl.get_value("phi", "upper intensity limit percentile", 95.0,
0.0, 100.0)
# region to plot
for i, cnam in enumerate(ccds):
nxtot, nytot, nxpad, nypad = ccdinf[cnam]
if i == 0:
xmin, xmax = float(-nxpad), float(nxtot + nxpad + 1)
ymin, ymax = float(-nypad), float(nytot + nypad + 1)
else:
xmin = min(xmin, float(-nxpad))
xmax = max(xmax, float(nxtot + nxpad + 1))
ymin = min(ymin, float(-nypad))
ymax = max(ymax, float(nytot + nypad + 1))
xlo = cl.get_value("xlo",
"left-hand X value",
xmin,
xmin,
xmax,
enforce=False)
xhi = cl.get_value("xhi",
"right-hand X value",
xmax,
xmin,
xmax,
enforce=False)
ylo = cl.get_value("ylo",
"lower Y value",
ymin,
ymin,
ymax,
enforce=False)
yhi = cl.get_value("yhi",
"upper Y value",
ymax,
ymin,
ymax,
enforce=False)
dpi = cl.get_value("dpi", "dots per inch", 200)
if rlog is not None:
style = cl.get_value("style",
"light curve plot style",
"dots",
lvals=('dots', 'line', 'both'))
ms, lw = 0, 0
if style == 'dots' or style == 'both':
ms = cl.get_value("ms", "markersize", 2.)
if style == 'line' or style == 'both':
lw = cl.get_value("lw", "line width", 2.)
###############################################################################
# Phew. We finally have all the inputs and now can now display stuff.
# track which CCDs have been plotted at least once for the profile fits
nccd = len(ccds)
plotted = np.array(nccd * [False])
current_ccds = nccd * [None]
ny = nccd // nx if nccd % nx == 0 else nccd // nx + 1
first_fig = True
# Now go through data
with spooler.data_source(source, resource, first, full=False) as spool:
for nframe, mccd in enumerate(spool):
# Trim the frames: ULTRACAM windowed data has bad columns
# and rows on the sides of windows closest to the readout
# which can badly affect reduction. This option strips
# them.
if trim:
hcam.ccd.trim_ultracam(mccd, ncol, nrow)
# indicate progress
tstamp = Time(mccd.head["TIMSTAMP"], format="isot", precision=3)
print(f'{mccd.head.get("NFRAME",nframe+1)}, utc= {tstamp.iso}')
if nframe == 0:
# get the bias and flat into shape first time through
if bias is not None:
# crop the bias on the first frame only
bias = bias.crop(mccd)
bexpose = bias.head.get("EXPTIME", 0.0)
else:
bexpose = 0.
if dark is not None:
# crop the dark on the first frame only
dark = dark.crop(mccd)
if flat is not None:
# crop the flat on the first frame only
flat = flat.crop(mccd)
if fmap is not None:
# crop the fringe map and pair file
fmap = fmap.crop(mccd)
fpair = fpair.crop(mccd, nhalf)
# wind through the CCDs to display, accumulating stuff
# to send to the plot manager
message = ""
skipped = True
for nc, cnam in enumerate(ccds):
ccd = mccd[cnam]
if ccd.is_data():
# "is_data" indicates genuine data as opposed to junk
# that results from nskip > 0.
plotted[nc] = True
# subtract the bias
if bias is not None:
ccd -= bias[cnam]
if dark is not None:
dexpose = dark.head["EXPTIME"]
cexpose = ccd.head["EXPTIME"]
scale = (cexpose - bexpose) / dexpose
ccd -= scale * dark[cnam]
# divide out the flat
if flat is not None:
ccd /= flat[cnam]
# Remove fringes
if fmap is not None and cnam in fmap and cnam in fpair:
fscale = fpair[cnam].scale(ccd, fmap[cnam], nhalf,
rmin, rmax)
ccd -= fscale * fmap[cnam]
if msub:
# subtract median from each window
for wind in ccd.values():
wind -= wind.median()
# keep list of current CCDs
current_ccds[nc] = ccd
# at this point current_ccds contains the current set of CCDs to plot
if plotted.all():
# Finally have at least one proper exposure of all
# CCDs and can make plots every time with skipped
# frames staying unchanged Set up the plot. ny by nx
# rows x columns of images + 1 for optional light
# curve which is either South or East of images
prop_cycle = plt.rcParams['axes.prop_cycle']
colors = prop_cycle.by_key()['color'][:len(ccds)]
# create the figure just once to avoid memory
# problems, i.e. we re-use the same figure for every
# plot. plt.close() which I thought work instead
# didn't and the programme went a bit bonkers at the
# end releasing resources. Putting '0' didn't work either.
if first_fig:
fig = plt.figure(figsize=(width, height))
first_fig = False
if rlog is not None:
# plot light curve
# first the location
if location.lower() == 's':
# light curve "South" of the images
rect = [
lpad[0], fraction * lpad[1], 1 - lpad[0] - lpad[2],
fraction * (1 - lpad[1] - lpad[3])
]
gs = GridSpec(ny, nx, figure=fig, bottom=fraction)
elif location.lower() == 'e':
# light curve "East" of the images
rect = [
1 - fraction + fraction * lpad[0], lpad[1],
fraction * (1 - lpad[0] - lpad[2]),
1 - lpad[1] - lpad[3]
]
gs = GridSpec(ny, nx, right=1 - fraction)
# light curve axes
ax = fig.add_axes(rect)
ax.set_xlim(0, tmax)
ax.set_ylim(fmin, fmax)
ax.set_xlabel(f'Time [mins, since MJD = {T0:.4f}]')
ax.set_ylabel(f'Target / Comparison')
ax.tick_params(axis="x", direction="in")
ax.tick_params(axis="y", direction="in", rotation=90)
ax.tick_params(bottom=True,
top=True,
left=True,
right=True)
for cnam, (nframes, lc), col in zip(ccds, lcs, colors):
plot = (nframes >= first) & (nframes <= first + nframe)
lct = lc[plot]
if style == 'dots':
fmt = '.'
color = None
elif style == 'line':
fmt = '-'
color = None
col = None
elif style == 'both':
fmt = '.-'
color = '0.7'
lct.mplot(ax,
fmt=fmt,
color=color,
mfc=col,
mec=col,
ms=ms,
lw=lw,
label=f'CCD {cnam}')
if len(ccds) > 1:
ax.legend(loc='upper right')
else:
# images only
gs = GridSpec(ny, nx, figure=fig)
# plot images
for n, (cnam, ccd, ilo,
ihi) in enumerate(zip(ccds, current_ccds, ilos, ihis)):
ax = fig.add_subplot(gs[n // nx, n % nx])
mpl.pCcd(ax,
ccd,
iset,
plo,
phi,
ilo,
ihi,
f'CCD {cnam}',
xlo=xlo,
xhi=xhi,
ylo=ylo,
yhi=yhi,
cmap=cmap)
ax.set_xlim(xlo, xhi)
ax.set_ylim(ylo, yhi)
ax.tick_params(axis="x", direction="in")
ax.tick_params(axis="y", direction="in", rotation=90)
ax.tick_params(bottom=True,
top=True,
left=True,
right=True)
# save to disk, clear figure
oname = os.path.join(
dstore,
f'{os.path.basename(resource)}_{first+nframe:0{ndigit}d}.{fext}'
)
plt.savefig(oname, dpi=dpi)
plt.clf()
print(f' written figure to {oname}')
if last and nframe + first == last:
break