def psf_reduce(args=None):
"""``psf_reduce [source] rfile (run first twait tmax | flist) log lplot implot
(ccd nx msub xlo xhi ylo yhi iset (ilo ihi | plo phi))``
Performs PSF photometry on a sequence of multi-CCD images, plotting
lightcurves as images come in. It performs PSF photometry on specific
targets, defined in an aperture file using |psf_setaper| or |setaper|.
Aperture repositioning is identical to |reduce|, and the PSF used in photometry
is set by fitting well separated reference stars. For the best results, a
suitable strategy is to only reduce data from a small window near the target.
Set well separated bright stars as reference apertures and link fainter, or
blended objects to the references.
|psf_reduce| can source data from both the ULTRACAM and HiPERCAM servers, from
local 'raw' ULTRACAM and HiPERCAM files (i.e. .xml + .dat for ULTRACAM, 3D
FITS files for HiPERCAM) and from lists of HiPERCAM '.hcm' files. If you
have data from a different instrument you should convert into the
FITS-based hcm format.
|psf_reduce| is primarily configured from a file with extension ".red". This
contains a series of directives, e.g. to say how to re-position the apertures.
An initial reduce file is best generated with the script |genred| after you
have created an aperture file. This contains lots of help on what to do.
A |psf_reduce| run can be terminated at any point with ctrl-C without doing
any harm. You may often want to do this at the start in order to adjust
parameters of the reduce file.
Parameters:
source : string [hidden]
Data source, five options:
| 'hs': HiPERCAM server
| 'hl': local HiPERCAM FITS file
| 'us': ULTRACAM server
| 'ul': local ULTRACAM .xml/.dat files
| 'hf': list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab' or
converted from foreign data formats.
rfile : string
the "reduce" file, i.e. ASCII text file suitable for reading by
ConfigParser. Best seen by example as it has many parts. Generate
a starter file with |genred|.
run : string [if source ends 's' or 'l']
run number to access, e.g. 'run034'
first : int [if source ends 's' or 'l']
exposure number to start from. 1 = first frame; set = 0 to
always try to get the most recent frame (if it has changed)
last : int [if source ends 's' or 'l', hidden]
last frame to reduce. 0 to just continue until the end. This is
not prompted for by default and must be set explicitly. It
defaults to 0 if not set. Its purpose is to allow accurate
profiling tests.
twait : float [if source ends 's'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [if source ends 's'; hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
flist : string [if source ends 'f']
name of file list
log : string
log file for the results
tkeep : float
maximum number of minutes of data to store in internal buffers, 0
for the lot. When large numbers of frames are stored, performance
can be slowed (although I am not entirely clear why) in which case
it makes sense to lose the earlier points (without affecting the
saving to disk). This parameter also gives operation similar to that
of "max_xrange" parameter in the ULTRACAM pipeline whereby just
the last few minutes are shown.
lplot : bool
flag to indicate you want to plot the light curve. Saves time not
to especially in high-speed runs.
implot : bool
flag to indicate you want to plot images.
ccd : string [if implot]
CCD(s) to plot, '0' for all, '1 3' to plot '1' and '3' only, etc.
nx : int [if implot]
number of panels across to display.
msub : bool [if implot]
subtract the median from each window before scaling for the
image display or not. This happens after any bias subtraction.
xlo : float [if implot]
left-hand X-limit for plot
xhi : float [if implot]
right-hand X-limit for plot (can actually be < xlo)
ylo : float [if implot]
lower Y-limit for plot
yhi : float [if implot]
upper Y-limit for plot (can be < ylo)
iset : string [if implot]
determines how the intensities are determined. There are three
options: 'a' for automatic simply scales from the minimum to the
maximum value found on a per CCD basis. 'd' for direct just takes
two numbers from the user. 'p' for percentile dtermines levels
based upon percentiles determined from the entire CCD on a per CCD
basis.
ilo : float [if implot and iset='d']
lower intensity level
ihi : float [if implot and iset='d']
upper intensity level
plo : float [if implot and iset='p']
lower percentile level
phi : float [if implot and iset='p']
upper percentile level
"""
command, args = utils.script_args(args)
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("source", Cline.GLOBAL, Cline.HIDE)
cl.register("rfile", Cline.GLOBAL, Cline.PROMPT)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("last", Cline.LOCAL, Cline.HIDE)
cl.register("twait", Cline.LOCAL, Cline.HIDE)
cl.register("tmax", Cline.LOCAL, Cline.HIDE)
cl.register("flist", Cline.LOCAL, Cline.PROMPT)
cl.register("log", Cline.GLOBAL, Cline.PROMPT)
cl.register("tkeep", Cline.GLOBAL, Cline.PROMPT)
cl.register("lplot", Cline.LOCAL, Cline.PROMPT)
cl.register("implot", Cline.LOCAL, Cline.PROMPT)
cl.register("ccd", Cline.LOCAL, Cline.PROMPT)
cl.register("nx", Cline.LOCAL, Cline.PROMPT)
cl.register("msub", Cline.GLOBAL, Cline.PROMPT)
cl.register("iset", Cline.GLOBAL, Cline.PROMPT)
cl.register("ilo", Cline.GLOBAL, Cline.PROMPT)
cl.register("ihi", Cline.GLOBAL, Cline.PROMPT)
cl.register("plo", Cline.GLOBAL, Cline.PROMPT)
cl.register("phi", Cline.LOCAL, Cline.PROMPT)
cl.register("xlo", Cline.GLOBAL, Cline.PROMPT)
cl.register("xhi", Cline.GLOBAL, Cline.PROMPT)
cl.register("ylo", Cline.GLOBAL, Cline.PROMPT)
cl.register("yhi", Cline.GLOBAL, Cline.PROMPT)
# get inputs
source = cl.get_value(
"source",
"data source [hs, hl, us, ul, hf]",
"hl",
lvals=("hs", "hl", "us", "ul", "hf"),
)
# set some flags
server_or_local = source.endswith("s") or source.endswith("l")
# the reduce file
rfilen = cl.get_value("rfile", "reduce file",
cline.Fname("reduce.red", hcam.RED))
try:
rfile = Rfile.read(rfilen)
except hcam.HipercamError as err:
# abort on failure to read as there are many ways to get reduce
# files wrong
print(err, file=sys.stderr)
exit(1)
if server_or_local:
resource = cl.get_value("run", "run name", "run005")
first = cl.get_value("first", "first frame to reduce", 1, 0)
cl.set_default("last", 0)
last = cl.get_value("last", "last frame to reduce", 0, 0)
if last and last < first:
print("Cannot set last < first unless last == 0")
print("*** psf_reduce aborted")
exit(1)
twait = cl.get_value("twait",
"time to wait for a new frame [secs]", 1.0,
0.0)
tmx = cl.get_value("tmax",
"maximum time to wait for a new frame [secs]",
10.0, 0.0)
else:
resource = cl.get_value("flist", "file list",
cline.Fname("files.lis", hcam.LIST))
first = 1
last = 0
log = cl.get_value(
"log",
"name of log file to store results",
cline.Fname("reduce.log", hcam.LOG, cline.Fname.NEW),
)
tkeep = cl.get_value(
"tkeep",
"number of minute of data to"
" keep in internal buffers (0 for all)",
0.0,
0.0,
)
lplot = cl.get_value("lplot", "do you want to plot light curves?",
True)
implot = cl.get_value("implot", "do you want to plot images?", True)
if implot:
# define the panel grid. first get the labels and maximum
# dimensions
ccdinf = spooler.get_ccd_pars(source, resource)
nxdef = cl.get_default("nx", 3)
if len(ccdinf) > 1:
ccd = cl.get_value("ccd", "CCD(s) to plot [0 for all]", "0")
if ccd == "0":
ccds = list(ccdinf.keys())
else:
ccds = ccd.split()
if len(ccds) > 1:
nxdef = min(len(ccds), nxdef)
cl.set_default("nx", nxdef)
nx = cl.get_value("nx", "number of panels in X", 3, 1)
else:
nx = 1
else:
nx = 1
ccds = list(ccdinf.keys())
# define the display intensities
msub = cl.get_value("msub", "subtract median from each window?",
True)
iset = cl.get_value(
"iset",
"set intensity a(utomatically),"
" d(irectly) or with p(ercentiles)?",
"a",
lvals=["a", "d", "p"],
)
plo, phi = 5, 95
ilo, ihi = 0, 1000
if iset == "d":
ilo = cl.get_value("ilo", "lower intensity limit", 0.0)
ihi = cl.get_value("ihi", "upper intensity limit", 1000.0)
elif iset == "p":
plo = cl.get_value("plo", "lower intensity limit percentile",
5.0, 0.0, 100.0)
phi = cl.get_value("phi", "upper intensity limit percentile",
95.0, 0.0, 100.0)
# region to plot
for i, cnam in enumerate(ccds):
nxtot, nytot, nxpad, nypad = ccdinf[cnam]
if i == 0:
xmin, xmax = float(-nxpad), float(nxtot + nxpad + 1)
ymin, ymax = float(-nypad), float(nytot + nypad + 1)
else:
xmin = min(xmin, float(-nxpad))
xmax = max(xmax, float(nxtot + nxpad + 1))
ymin = min(ymin, float(-nypad))
ymax = max(ymax, float(nytot + nypad + 1))
xlo = cl.get_value("xlo",
"left-hand X value",
xmin,
xmin,
xmax,
enforce=False)
xhi = cl.get_value("xhi",
"right-hand X value",
xmax,
xmin,
xmax,
enforce=False)
ylo = cl.get_value("ylo",
"lower Y value",
ymin,
ymin,
ymax,
enforce=False)
yhi = cl.get_value("yhi",
"upper Y value",
ymax,
ymin,
ymax,
enforce=False)
else:
xlo, xhi, ylo, yhi = None, None, None, None
# save list of parameter values for writing to the reduction file
plist = cl.list()
################################################################
#
# all the inputs have now been obtained. Get on with doing stuff
if implot:
plot_lims = (xlo, xhi, ylo, yhi)
else:
plot_lims = None
imdev, lcdev, spanel, tpanel, xpanel, ypanel, lpanel = setup_plots(
rfile, ccds, nx, plot_lims, implot, lplot)
# a couple of initialisations
total_time = 0 # time waiting for new frame
# dictionary of dictionaries for looking up the window associated with a
# given aperture, i.e. mccdwins[cnam][apnam] give the name of the Window.
mccdwins = {}
if lplot:
lbuffer, xbuffer, ybuffer, tbuffer, sbuffer = setup_plot_buffers(rfile)
else:
lbuffer, xbuffer, ybuffer, tbuffer, sbuffer = None, None, None, None, None
############################################
#
# open the log file and write headers
#
with LogWriter(log, rfile, hipercam_version, plist) as logfile:
ncpu = rfile["general"]["ncpu"]
if ncpu > 1:
pool = multiprocessing.Pool(processes=ncpu)
else:
pool = None
# whether settings have been initialised
initialised = False
# containers for the processed and raw MCCD groups
# and their frame numbers
pccds, mccds, nframes = [], [], []
##############################################
#
# Finally, start winding through the frames
#
with spooler.data_source(source, resource, first, full=False) as spool:
# 'spool' is an iterable source of MCCDs
for nf, mccd in enumerate(spool):
if server_or_local:
# Handle the waiting game ...
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmx, total_time)
if give_up:
# Giving up, but need to handle any partially filled
# frame group
if len(mccds):
# finish processing remaining frames. This step
# will only occur if we have at least once passed
# to later stages during which read and gain will
# be set up
results = processor(pccds, mccds, nframes)
# write out results to the log file
alerts = logfile.write_results(results)
# print out any accumulated alert messages
if len(alerts):
print("\n".join(alerts))
update_plots(
results,
rfile,
implot,
lplot,
imdev,
lcdev,
pccd,
ccds,
msub,
nx,
iset,
plo,
phi,
ilo,
ihi,
xlo,
xhi,
ylo,
yhi,
lpanel,
xpanel,
ypanel,
tpanel,
spanel,
tkeep,
lbuffer,
xbuffer,
ybuffer,
tbuffer,
sbuffer,
)
print("psf_reduce stopped")
break
elif try_again:
continue
# indicate progress
if "NFRAME" in mccd.head:
nframe = mccd.head["NFRAME"]
else:
nframe = nf + 1
if last and nframe > last:
if len(mccds):
# finish processing remaining frames
results = processor(pccds, mccds, nframes)
# write out results to the log file
alerts = logfile.write_results(results)
# print out any accumulated alert messages
if len(alerts):
print("\n".join(alerts))
update_plots(
results,
rfile,
implot,
lplot,
imdev,
lcdev,
pccd,
ccds,
msub,
nx,
iset,
plo,
phi,
ilo,
ihi,
xlo,
xhi,
ylo,
yhi,
lpanel,
xpanel,
ypanel,
tpanel,
spanel,
tkeep,
lbuffer,
xbuffer,
ybuffer,
tbuffer,
sbuffer,
)
print("\nHave reduced up to the last frame set.")
print("psf_reduce finished")
break
print(
"Frame {:d}: {:s} [{:s}]".format(
nframe,
mccd.head["TIMSTAMP"],
"OK" if mccd.head.get("GOODTIME", True) else "NOK",
),
end="" if implot else "\n",
)
if not initialised:
# This is the first frame which allows us to make
# some checks and initialisations.
read, gain, ok = initial_checks(mccd, rfile)
# Define the CCD processor function object
processor = ProcessCCDs(rfile, read, gain, ccdproc, pool)
# set flag to show we are set
if not ok:
break
initialised = True
# De-bias the data. Retain a copy of the raw data as 'mccd'
# in order to judge saturation. Processed data called 'pccd'
if rfile.bias is not None:
# subtract bias
pccd = mccd - rfile.bias
else:
# no bias subtraction
pccd = mccd.copy()
if rfile.flat is not None:
# apply flat field to processed frame
pccd /= rfile.flat
# Acummulate frames into processing groups for faster
# parallelisation
pccds.append(pccd)
mccds.append(mccd)
nframes.append(nframe)
if len(pccds) == rfile["general"]["ngroup"]:
# parallel processing. This should usually be the first
# points at which it takes place
results = processor(pccds, mccds, nframes)
# write out results to the log file
alerts = logfile.write_results(results)
# print out any accumulated alert messages
if len(alerts):
print("\n".join(alerts))
update_plots(
results,
rfile,
implot,
lplot,
imdev,
lcdev,
pccds[-1],
ccds,
msub,
nx,
iset,
plo,
phi,
ilo,
ihi,
xlo,
xhi,
ylo,
yhi,
lpanel,
xpanel,
ypanel,
tpanel,
spanel,
tkeep,
lbuffer,
xbuffer,
ybuffer,
tbuffer,
sbuffer,
)
# Reset the frame buffers
pccds, mccds, nframes = [], [], []
def rtplot(args=None):
"""``rtplot [source device width height] (run first (trim [ncol nrow])
twait tmax | flist) [pause plotall] ccd (nx) bias flat defect
setup [hurl] msub iset (ilo ihi | plo phi) xlo xhi ylo yhi (profit
[fdevice fwidth fheight method beta fwhm fwhm_min shbox smooth
splot fhbox hmin read gain thresh])``
Plots a sequence of images as a movie in near 'real time', hence
'rt'. Designed to be used to look at images coming in while at the
telescope, 'rtplot' comes with many options, a large number of which are
hidden by default, and many of which are only prompted if other arguments
are set correctly. If you want to see them all, invoke as 'rtplot prompt'.
This is worth doing once to know rtplot's capabilities.
rtplot can source data from both the ULTRACAM and HiPERCAM servers, from
local 'raw' ULTRACAM and HiPERCAM files (i.e. .xml + .dat for ULTRACAM, 3D
FITS files for HiPERCAM) and from lists of HiPERCAM '.hcm' files.
rtplot optionally allows the selection of targets to be fitted with
gaussian or moffat profiles, and, if successful, will plot circles of 2x
the measured FWHM in green over the selected targets. This option only
works if a single CCD is being plotted.
Parameters:
source : string [hidden]
Data source, five options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : ULTRACAM server
| 'ul' : local ULTRACAM .xml/.dat files
| 'hf' : list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab' or
converted from foreign data formats.
device : string [hidden]
Plot device. PGPLOT is used so this should be a PGPLOT-style name,
e.g. '/xs', '1/xs' etc. At the moment only ones ending /xs are
supported.
width : float [hidden]
plot width (inches). Set = 0 to let the program choose.
height : float [hidden]
plot height (inches). Set = 0 to let the program choose. BOTH width
AND height must be non-zero to have any effect
run : string [if source ends 's' or 'l']
run number to access, e.g. 'run034'
flist : string [if source ends 'f']
name of file list
first : int [if source ends 's' or 'l']
exposure number to start from. 1 = first frame; set = 0 to always
try to get the most recent frame (if it has changed). For data
from the |hiper| server, a negative number tries to get a frame not
quite at the end. i.e. -10 will try to get 10 from the last
frame. This is mainly to sidestep a difficult bug with the
acquisition system.
trim : bool [if source starts with 'u']
True to trim columns and/or rows off the edges of windows nearest
the readout. This is particularly for ULTRACAM windowed data where
the first few rows and columns can contain bad data.
ncol : int [if trim, hidden]
Number of columns to remove (on left of left-hand window, and right
of right-hand windows)
nrow : int [if trim, hidden]
Number of rows to remove (bottom of windows)
twait : float [if source ends 's' or 'l'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [if source ends 's' or 'l'; hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
pause : float [hidden]
seconds to pause between frames (defaults to 0)
plotall : bool [hidden]
plot all frames regardless of status (i.e. including blank frames
when nskips are enabled (defaults to False). The profile fitting
will still be disabled for bad frames.
ccd : string
CCD(s) to plot, '0' for all, '1 3' to plot '1' and '3' only, etc.
nx : int [if more than 1 CCD]
number of panels across to display.
bias : string
Name of bias frame to subtract, 'none' to ignore.
flat : string
Name of flat field to divide by, 'none' to ignore. Should normally
only be used in conjunction with a bias, although it does allow you
to specify a flat even if you haven't specified a bias.
defect : string
Name of defect file, 'none' to ignore.
setup : bool
True/yes to access the current windows from hdriver. Useful
during observing when seeting up windows, but not normally
otherwise. Next argument (hidden) is the URL to get to
hdriver. Once setup, you should probably turn this off to
avoid overloading hdriver, especially if in drift mode as
it makes a request for the windows for every frame.
hurl : string [if setup; hidden]
URL needed to access window setting from hdriver. The internal
server in hdriver must be switched on using rtplot_server_on
in the hdriver config file.
msub : bool
subtract the median from each window before scaling for the
image display or not. This happens after any bias subtraction.
iset : string [single character]
determines how the intensities are determined. There are three
options: 'a' for automatic simply scales from the minimum to the
maximum value found on a per CCD basis. 'd' for direct just takes
two numbers from the user. 'p' for percentile dtermines levels
based upon percentiles determined from the entire CCD on a per CCD
basis.
ilo : float [if iset='d']
lower intensity level
ihi : float [if iset='d']
upper intensity level
plo : float [if iset='p']
lower percentile level
phi : float [if iset='p']
upper percentile level
xlo : float
left-hand X-limit for plot
xhi : float
right-hand X-limit for plot (can actually be < xlo)
ylo : float
lower Y-limit for plot
yhi : float
upper Y-limit for plot (can be < ylo)
profit : bool [if plotting a single CCD only]
carry out profile fits or not. If you say yes, then on the first
plot, you will have the option to pick objects with a cursor. The
program will then attempt to track these from frame to frame, and
fit their profile. You may need to adjust 'first' to see anything.
The parameters used for profile fits are hidden and you may want to
invoke the command with 'prompt' the first time you try profile
fitting.
fdevice : string [if profit; hidden]
plot device for profile fits, PGPLOT-style name.
e.g. '/xs', '2/xs' etc.
fwidth : float [if profit; hidden]
fit plot width (inches). Set = 0 to let the program choose.
fheight : float [if profit; hidden]
fit plot height (inches). Set = 0 to let the program choose.
BOTH fwidth AND fheight must be non-zero to have any effect
method : string [if profit; hidden]
this defines the profile fitting method, either a gaussian or a
moffat profile. The latter is usually best.
beta : float [if profit and method == 'm'; hidden]
default Moffat exponent
fwhm : float [if profit; hidden]
default FWHM, unbinned pixels.
fwhm_min : float [if profit; hidden]
minimum FWHM to allow, unbinned pixels.
shbox : float [if profit; hidden]
half width of box for searching for a star, unbinned pixels. The
brightest target in a region +/- shbox around an intial position
will be found. 'shbox' should be large enough to allow for likely
changes in position from frame to frame, but try to keep it as
small as you can to avoid jumping to different targets and to
reduce the chances of interference by cosmic rays.
smooth : float [if profit; hidden]
FWHM for gaussian smoothing, binned pixels. The initial position
for fitting is determined by finding the maximum flux in a smoothed
version of the image in a box of width +/- shbox around the starter
position. Typically should be comparable to the stellar width. Its
main purpose is to combat cosmi rays which tend only to occupy a
single pixel.
splot : bool [if profit; hidden]
Controls whether an outline of the search box and a target number
is plotted (in red) or not.
fhbox : float [if profit; hidden]
half width of box for profile fit, unbinned pixels. The fit box is
centred on the position located by the initial search. It should
normally be > ~2x the expected FWHM.
hmin : float [if profit; hidden]
height threshold to accept a fit. If the height is below this
value, the position will not be updated. This is to help in cloudy
conditions.
read : float [if profit; hidden]
readout noise, RMS ADU, for assigning uncertainties
gain : float [if profit; hidden]
gain, ADU/count, for assigning uncertainties
thresh : float [if profit; hidden]
sigma rejection threshold for fits
"""
command, args = utils.script_args(args)
# get the inputs
with Cline('HIPERCAM_ENV', '.hipercam', command, args) as cl:
# register parameters
cl.register('source', Cline.GLOBAL, Cline.HIDE)
cl.register('device', Cline.LOCAL, Cline.HIDE)
cl.register('width', Cline.LOCAL, Cline.HIDE)
cl.register('height', Cline.LOCAL, Cline.HIDE)
cl.register('run', Cline.GLOBAL, Cline.PROMPT)
cl.register('first', Cline.LOCAL, Cline.PROMPT)
cl.register('trim', Cline.GLOBAL, Cline.PROMPT)
cl.register('ncol', Cline.GLOBAL, Cline.HIDE)
cl.register('nrow', Cline.GLOBAL, Cline.HIDE)
cl.register('twait', Cline.LOCAL, Cline.HIDE)
cl.register('tmax', Cline.LOCAL, Cline.HIDE)
cl.register('flist', Cline.LOCAL, Cline.PROMPT)
cl.register('ccd', Cline.LOCAL, Cline.PROMPT)
cl.register('pause', Cline.LOCAL, Cline.HIDE)
cl.register('plotall', Cline.LOCAL, Cline.HIDE)
cl.register('nx', Cline.LOCAL, Cline.PROMPT)
cl.register('bias', Cline.GLOBAL, Cline.PROMPT)
cl.register('flat', Cline.GLOBAL, Cline.PROMPT)
cl.register('defect', Cline.GLOBAL, Cline.PROMPT)
cl.register('setup', Cline.GLOBAL, Cline.PROMPT)
cl.register('hurl', Cline.GLOBAL, Cline.HIDE)
cl.register('msub', Cline.GLOBAL, Cline.PROMPT)
cl.register('iset', Cline.GLOBAL, Cline.PROMPT)
cl.register('ilo', Cline.GLOBAL, Cline.PROMPT)
cl.register('ihi', Cline.GLOBAL, Cline.PROMPT)
cl.register('plo', Cline.GLOBAL, Cline.PROMPT)
cl.register('phi', Cline.LOCAL, Cline.PROMPT)
cl.register('xlo', Cline.GLOBAL, Cline.PROMPT)
cl.register('xhi', Cline.GLOBAL, Cline.PROMPT)
cl.register('ylo', Cline.GLOBAL, Cline.PROMPT)
cl.register('yhi', Cline.GLOBAL, Cline.PROMPT)
cl.register('profit', Cline.LOCAL, Cline.PROMPT)
cl.register('fdevice', Cline.LOCAL, Cline.HIDE)
cl.register('fwidth', Cline.LOCAL, Cline.HIDE)
cl.register('fheight', Cline.LOCAL, Cline.HIDE)
cl.register('method', Cline.LOCAL, Cline.HIDE)
cl.register('beta', Cline.LOCAL, Cline.HIDE)
cl.register('fwhm', Cline.LOCAL, Cline.HIDE)
cl.register('fwhm_min', Cline.LOCAL, Cline.HIDE)
cl.register('shbox', Cline.LOCAL, Cline.HIDE)
cl.register('smooth', Cline.LOCAL, Cline.HIDE)
cl.register('splot', Cline.LOCAL, Cline.HIDE)
cl.register('fhbox', Cline.LOCAL, Cline.HIDE)
cl.register('hmin', Cline.LOCAL, Cline.HIDE)
cl.register('read', Cline.LOCAL, Cline.HIDE)
cl.register('gain', Cline.LOCAL, Cline.HIDE)
cl.register('thresh', Cline.LOCAL, Cline.HIDE)
# get inputs
source = cl.get_value('source', 'data source [hs, hl, us, ul, hf]',
'hl', lvals=('hs','hl','us','ul','hf'))
# set some flags
server_or_local = source.endswith('s') or source.endswith('l')
# plot device stuff
device = cl.get_value('device', 'plot device', '1/xs')
width = cl.get_value('width', 'plot width (inches)', 0.)
height = cl.get_value('height', 'plot height (inches)', 0.)
if server_or_local:
resource = cl.get_value('run', 'run name', 'run005')
if source == 'hs':
first = cl.get_value('first', 'first frame to plot', 1)
else:
first = cl.get_value('first', 'first frame to plot', 1, 0)
if source.startswith('u'):
trim = cl.get_value(
'trim',
'do you want to trim edges of windows? (ULTRACAM only)',
True
)
if trim:
ncol = cl.get_value(
'ncol', 'number of columns to trim from windows', 0)
nrow = cl.get_value(
'nrow', 'number of rows to trim from windows', 0)
else:
trim = False
twait = cl.get_value(
'twait', 'time to wait for a new frame [secs]', 1., 0.)
tmax = cl.get_value(
'tmax', 'maximum time to wait for a new frame [secs]', 10., 0.)
else:
resource = cl.get_value(
'flist', 'file list', cline.Fname('files.lis',hcam.LIST)
)
first = 1
trim = False
# define the panel grid. first get the labels and maximum dimensions
ccdinf = spooler.get_ccd_pars(source, resource)
try:
nxdef = cl.get_default('nx')
except:
nxdef = 3
if len(ccdinf) > 1:
ccd = cl.get_value('ccd', 'CCD(s) to plot [0 for all]', '0')
if ccd == '0':
ccds = list(ccdinf.keys())
else:
ccds = ccd.split()
check = set(ccdinf.keys())
if not set(ccds) <= check:
raise hcam.HipercamError(
'At least one invalid CCD label supplied'
)
if len(ccds) > 1:
nxdef = min(len(ccds), nxdef)
cl.set_default('nx', nxdef)
nx = cl.get_value('nx', 'number of panels in X', 3, 1)
else:
nx = 1
else:
nx = 1
ccds = list(ccdinf.keys())
cl.set_default('pause', 0.)
pause = cl.get_value(
'pause', 'time delay to add between'
' frame plots [secs]', 0., 0.
)
cl.set_default('plotall', False)
plotall = cl.get_value('plotall', 'plot all frames,'
' regardless of status?', False)
# bias frame (if any)
bias = cl.get_value(
'bias', "bias frame ['none' to ignore]",
cline.Fname('bias', hcam.HCAM), ignore='none'
)
if bias is not None:
# read the bias frame
bias = hcam.MCCD.read(bias)
fprompt = "flat frame ['none' to ignore]"
else:
fprompt = "flat frame ['none' is normal choice with no bias]"
# flat (if any)
flat = cl.get_value(
'flat', fprompt,
cline.Fname('flat', hcam.HCAM), ignore='none'
)
if flat is not None:
# read the flat frame
flat = hcam.MCCD.read(flat)
# defect file (if any)
dfct = cl.get_value(
'defect', "defect file ['none' to ignore]",
cline.Fname('defect', hcam.DFCT), ignore='none'
)
if dfct is not None:
# read the defect frame
dfct = defect.MccdDefect.read(dfct)
# Get windows from hdriver
setup = cl.get_value(
'setup', 'display current hdriver window settings', False
)
if setup:
hurl = cl.get_value(
'hurl', 'URL for hdriver windows', 'http://192.168.1.2:5100'
)
# define the display intensities
msub = cl.get_value('msub', 'subtract median from each window?', True)
iset = cl.get_value(
'iset', 'set intensity a(utomatically),'
' d(irectly) or with p(ercentiles)?',
'a', lvals=['a','d','p'])
iset = iset.lower()
plo, phi = 5, 95
ilo, ihi = 0, 1000
if iset == 'd':
ilo = cl.get_value('ilo', 'lower intensity limit', 0.)
ihi = cl.get_value('ihi', 'upper intensity limit', 1000.)
elif iset == 'p':
plo = cl.get_value('plo', 'lower intensity limit percentile',
5., 0., 100.)
phi = cl.get_value('phi', 'upper intensity limit percentile',
95., 0., 100.)
# region to plot
for i, cnam in enumerate(ccds):
nxtot, nytot, nxpad, nypad = ccdinf[cnam]
if i == 0:
xmin, xmax = float(-nxpad), float(nxtot + nxpad + 1)
ymin, ymax = float(-nypad), float(nytot + nypad + 1)
else:
xmin = min(xmin, float(-nxpad))
xmax = max(xmax, float(nxtot + nxpad + 1))
ymin = min(ymin, float(-nypad))
ymax = max(ymax, float(nytot + nypad + 1))
xlo = cl.get_value('xlo', 'left-hand X value', xmin, xmin, xmax)
xhi = cl.get_value('xhi', 'right-hand X value', xmax, xmin, xmax)
ylo = cl.get_value('ylo', 'lower Y value', ymin, ymin, ymax)
yhi = cl.get_value('yhi', 'upper Y value', ymax, ymin, ymax)
# profile fitting if just one CCD chosen
if len(ccds) == 1:
# many parameters for profile fits, although most are not plotted
# by default
profit = cl.get_value('profit', 'do you want profile fits?', False)
if profit:
fdevice = cl.get_value('fdevice', 'plot device for fits',
'2/xs')
fwidth = cl.get_value('fwidth', 'fit plot width (inches)', 0.)
fheight = cl.get_value(
'fheight', 'fit plot height (inches)', 0.)
method = cl.get_value(
'method', 'fit method g(aussian) or m(offat)',
'm', lvals=['g','m'])
if method == 'm':
beta = cl.get_value(
'beta', 'initial exponent for Moffat fits', 5., 0.5)
else:
beta = 0.
fwhm_min = cl.get_value(
'fwhm_min', 'minimum FWHM to allow [unbinned pixels]',
1.5, 0.01
)
fwhm = cl.get_value(
'fwhm', 'initial FWHM [unbinned pixels] for profile fits',
6., fwhm_min
)
shbox = cl.get_value(
'shbox', 'half width of box for initial location'
' of target [unbinned pixels]', 11., 2.)
smooth = cl.get_value(
'smooth', 'FWHM for smoothing for initial object'
' detection [binned pixels]', 6.)
splot = cl.get_value(
'splot', 'plot outline of search box?', True)
fhbox = cl.get_value(
'fhbox', 'half width of box for profile fit'
' [unbinned pixels]', 21., 3.)
hmin = cl.get_value(
'hmin', 'minimum peak height to accept the fit', 50.)
read = cl.get_value('read', 'readout noise, RMS ADU', 3.)
gain = cl.get_value('gain', 'gain, ADU/e-', 1.)
thresh = cl.get_value('thresh', 'number of RMS to reject at', 4.)
else:
profit = False
################################################################
#
# all the inputs have now been obtained. Get on with doing stuff
# open image plot device
imdev = hcam.pgp.Device(device)
if width > 0 and height > 0:
pgpap(width,height/width)
# set up panels and axes
nccd = len(ccds)
ny = nccd // nx if nccd % nx == 0 else nccd // nx + 1
# slice up viewport
pgsubp(nx,ny)
# plot axes, labels, titles. Happens once only
for cnam in ccds:
pgsci(hcam.pgp.Params['axis.ci'])
pgsch(hcam.pgp.Params['axis.number.ch'])
pgenv(xlo, xhi, ylo, yhi, 1, 0)
pglab('X','Y','CCD {:s}'.format(cnam))
# initialisations. 'last_ok' is used to store the last OK frames of each
# CCD for retrieval when coping with skipped data.
total_time = 0 # time waiting for new frame
fpos = [] # list of target positions to fit
fframe = True # waiting for first valid frame with profit
# plot images
with spooler.data_source(source, resource, first, full=False) as spool:
# 'spool' is an iterable source of MCCDs
n = 0
for mccd in spool:
if server_or_local:
# Handle the waiting game ...
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmax, total_time
)
if give_up:
print('rtplot stopped')
break
elif try_again:
continue
# Trim the frames: ULTRACAM windowed data has bad columns
# and rows on the sides of windows closest to the readout
# which can badly affect reduction. This option strips
# them.
if trim:
hcam.ccd.trim_ultracam(mccd, ncol, nrow)
# indicate progress
tstamp = Time(mccd.head['TIMSTAMP'], format='isot', precision=3)
print(
'{:d}, utc = {:s}, '.format(
mccd.head['NFRAME'], tstamp.iso), end=''
)
# accumulate errors
emessages = []
if n == 0:
if bias is not None:
# crop the bias on the first frame only
bias = bias.crop(mccd)
if flat is not None:
# crop the flat on the first frame only
flat = flat.crop(mccd)
if setup:
# Get windows from hdriver. Fair bit of error checking
# needed. 'got_windows' indicates if anything useful
# found, 'hwindows' is a list of (llx,lly,nx,ny) tuples
# if somthing is found.
try:
r = requests.get(
'http://192.168.1.2:5100',timeout=0.2
)
if r.text.strip() == 'No valid data available':
emessages.append(
'** bad return from hdriver = {:s}'.format(r.text.strip())
)
got_windows = False
elif r.text.strip() == 'fullframe':
# to help Stu out a bit, effectively just ignore this one
got_windows = False
else:
# OK, got something
lines = r.text.split('\r\n')
xbinh,ybinh,nwinh = lines[0].split()
xbinh,ybinh,nwinh = int(xbinh),int(ybinh),int(nwinh)
hwindows = []
for line in lines[1:nwinh+1]:
llxh,llyh,nxh,nyh = line.split()
hwindows.append(
(int(llxh),int(llyh),int(nxh),int(nyh))
)
if nwinh != len(hwindows):
emessages.append(
('** expected {:d} windows from'
' hdriver but got {:d}').format(nwinh,len(hwindows))
)
got_windows = False
got_windows = True
except (requests.exceptions.ConnectionError, socket.timeout,
requests.exceptions.Timeout) as err:
emessages.append(' ** hdriver error: {!r}'.format(err))
got_windows = False
else:
got_windows = False
# display the CCDs chosen
message = ''
pgbbuf()
for nc, cnam in enumerate(ccds):
ccd = mccd[cnam]
if plotall or ccd.is_data():
# this should be data as opposed to a blank frame
# between data frames that occur with nskip > 0
# subtract the bias
if bias is not None:
ccd -= bias[cnam]
# divide out the flat
if flat is not None:
ccd /= flat[cnam]
if msub:
# subtract median from each window
for wind in ccd.values():
wind -= wind.median()
# set to the correct panel and then plot CCD
ix = (nc % nx) + 1
iy = nc // nx + 1
pgpanl(ix,iy)
vmin, vmax = hcam.pgp.pCcd(
ccd,iset,plo,phi,ilo,ihi,
xlo=xlo, xhi=xhi, ylo=ylo, yhi=yhi
)
if got_windows:
# plot the current hdriver windows
pgsci(hcam.CNAMS['yellow'])
pgsls(2)
for llxh,llyh,nxh,nyh in hwindows:
pgrect(llxh-0.5,llxh+nxh-0.5,llyh-0.5,llyh+nyh-0.5)
if dfct is not None and cnam in dfct:
# plot defects
hcam.pgp.pCcdDefect(dfct[cnam])
# accumulate string of image scalings
if nc:
message += ', ccd {:s}: {:.1f}, {:.1f}, exp: {:.4f}'.format(
cnam,vmin,vmax,mccd.head['EXPTIME']
)
else:
message += 'ccd {:s}: {:.1f}, {:.1f}, exp: {:.4f}'.format(
cnam,vmin,vmax,mccd.head['EXPTIME']
)
pgebuf()
# end of CCD display loop
print(message)
for emessage in emessages:
print(emessage)
if ccd.is_data() and profit and fframe:
fframe = False
# cursor selection of targets after first plot, if profit
# accumulate list of starter positions
print('Please select targets for profile fitting. You can select as many as you like.')
x, y, reply = (xlo+xhi)/2, (ylo+yhi)/2, ''
ntarg = 0
pgsci(2)
pgslw(2)
while reply != 'Q':
print("Place cursor on fit target. Any key to register, 'q' to quit")
x, y, reply = pgcurs(x, y)
if reply == 'q':
break
else:
# check that the position is inside a window
wnam = ccd.inside(x, y, 2)
if wnam is not None:
# store the position, Window label, target number,
# box size fwhm, beta
ntarg += 1
fpos.append(Fpar(x,y,wnam,ntarg,shbox,fwhm,beta))
# report information, overplot search box
print(
('Target {:d} selected at {:.1f},'
'{:.1f} in window {:s}').format(ntarg,x,y,wnam)
)
if splot:
fpos[-1].plot()
if len(fpos):
print(len(fpos),'targets selected')
# if some targets were selected, open the fit plot device
fdev = hcam.pgp.Device(fdevice)
if fwidth > 0 and fheight > 0:
pgpap(fwidth,fheight/fwidth)
if ccd.is_data():
# carry out fits. Nothing happens if fpos is empty
for fpar in fpos:
# switch to the image plot
imdev.select()
# plot search box
if splot:
fpar.plot()
try:
# extract search box from the CCD. 'fpar' is updated later
# if the fit is successful to reflect the new position
swind = fpar.swind(ccd)
# carry out initial search
x,y,peak = swind.search(smooth, fpar.x, fpar.y, hmin, False)
# now for a more refined fit. First extract fit Window
fwind = ccd[fpar.wnam].window(
x-fhbox, x+fhbox, y-fhbox, y+fhbox
)
# crude estimate of sky background
sky = np.percentile(fwind.data, 50)
# refine the Aperture position by fitting the profile
(sky, height, x, y, fwhm, beta), epars, \
(wfit, X, Y, sigma, chisq, nok,
nrej, npar, message) = hcam.fitting.combFit(
fwind, method, sky, peak-sky,
x, y, fpar.fwhm, fwhm_min, False,
fpar.beta, read, gain, thresh
)
print('Targ {:d}: {:s}'.format(fpar.ntarg,message))
if peak > hmin and ccd[fpar.wnam].distance(x,y) > 1:
# update some initial parameters for next time
if method == 'g':
fpar.x, fpar.y, fpar.fwhm = x, y, fwhm
elif method == 'm':
fpar.x, fpar.y, fpar.fwhm, \
fpar.beta = x, y, fwhm, beta
# plot values versus radial distance
ok = sigma > 0
R = np.sqrt((X-x)**2+(Y-y)**2)
fdev.select()
vmin = min(sky, sky+height, fwind.min())
vmax = max(sky, sky+height, fwind.max())
extent = vmax-vmin
pgeras()
pgvstd()
pgswin(0,R.max(),vmin-0.05*extent,vmax+0.05*extent)
pgsci(4)
pgbox('bcnst',0,0,'bcnst',0,0)
pgsci(2)
pglab('Radial distance [unbinned pixels]','Counts','')
pgsci(1)
pgpt(R[ok].flat, fwind.data[ok].flat, 1)
if nrej:
pgsci(2)
pgpt(R[~ok].flat, fwind.data[~ok].flat, 5)
# line fit
pgsci(3)
r = np.linspace(0,R.max(),400)
if method == 'g':
alpha = 4*np.log(2.)/fwhm**2
f = sky+height*np.exp(-alpha*r**2)
elif method == 'm':
alpha = 4*(2**(1/beta)-1)/fwhm**2
f = sky+height/(1+alpha*r**2)**beta
pgline(r,f)
# back to the image to plot a circle of radius FWHM
imdev.select()
pgsci(3)
pgcirc(x,y,fwhm)
else:
print(' *** below detection threshold; position & FWHM will not updated')
pgsci(2)
# plot location on image as a cross
pgpt1(x, y, 5)
except hcam.HipercamError as err:
print(' >> Targ {:d}: fit failed ***: {!s}'.format(
fpar.ntarg, err)
)
pgsci(2)
if pause > 0.:
# pause between frames
time.sleep(pause)
# update the frame number
n += 1
def ncal(args=None):
"""``ncal [source] (run first last [twait tmax] | flist) trim ([ncol
nrow]) (ccd) bias dark flat xybox read gain grain``
Calibrates noise characteristics of CCDs by plotting estimator
of RMS vs signal level from a series of frames. The estimate
is the mean of the absolute difference between each pixel
and the mean of its 8 near-neighbours. This is very local and
fairly robust. Assuming gaussian noise, the RMS is sqrt(4*Pi/9)
times this value, and this is what is plotted as the RMS by this
routine. `ncal` is best to applied to a series of frames with
a large dynamic range, ideally starting from bias-like frames
to well exposed sky flats. A long flat-field run going to low
levels, or a run into twilight at the end of the night could be
good places to start.
Parameters:
source : str [hidden]
Data source, five options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : ULTRACAM server
| 'ul' : local ULTRACAM .xml/.dat files
| 'hf' : list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab'
or converted from foreign data formats. The standard
start-off default for ``source`` can be set using the
environment variable HIPERCAM_DEFAULT_SOURCE. e.g. in bash
:code:`export HIPERCAM_DEFAULT_SOURCE="us"` would ensure it
always started with the ULTRACAM server by default. If
unspecified, it defaults to 'hl'.
run : str [if source ends 's' or 'l']
run number to access, e.g. 'run034'
flist : str [if source ends 'f']
name of file list
first : int [if source ends 's' or 'l']
exposure number to start from. 1 = first frame; set = 0 to always
try to get the most recent frame (if it has changed). For data
from the |hiper| server, a negative number tries to get a frame not
quite at the end. i.e. -10 will try to get 10 from the last
frame. This is mainly to sidestep a difficult bug with the
acquisition system.
last : int [if source ends 's' or 'l']
Last frame to access, 0 for the lot
twait : float [if source ends 's' or 'l'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [if source ends 's' or 'l'; hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
trim : bool [if source starts with 'u']
True to trim columns and/or rows off the edges of windows nearest
the readout which can sometimes contain bad data.
ncol : int [if trim, hidden]
Number of columns to remove (on left of left-hand window, and right
of right-hand windows)
nrow : int [if trim, hidden]
Number of rows to remove (bottom of windows)
ccd : str
The CCD to analyse.
bias : str
Name of bias frame to subtract, 'none' to ignore.
dark : str
Name of dark frame to subtract, 'none' to ignore
flat : str
Name of flat field to divide by, 'none' to ignore. Should normally
only be used in conjunction with a bias, although it does allow you
to specify a flat even if you haven't specified a bias.
xybox : int
the stats will be taken over boxes of xybox-squared pixels
to keep the number of points and scatter under control.
read : float
readout noise, RMS ADU, for overplotting a model
gain : float
gain, e-/count, for overploting a model
grain : float
fractional RMS variation due to flat-field variations,
if you didn't include a flat field.
"""
command, args = utils.script_args(args)
# get the inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("source", Cline.GLOBAL, Cline.HIDE)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("last", Cline.LOCAL, Cline.PROMPT)
cl.register("trim", Cline.GLOBAL, Cline.PROMPT)
cl.register("ncol", Cline.GLOBAL, Cline.HIDE)
cl.register("nrow", Cline.GLOBAL, Cline.HIDE)
cl.register("twait", Cline.LOCAL, Cline.HIDE)
cl.register("tmax", Cline.LOCAL, Cline.HIDE)
cl.register("flist", Cline.LOCAL, Cline.PROMPT)
cl.register("ccd", Cline.LOCAL, Cline.PROMPT)
cl.register("bias", Cline.GLOBAL, Cline.PROMPT)
cl.register("dark", Cline.GLOBAL, Cline.PROMPT)
cl.register("flat", Cline.GLOBAL, Cline.PROMPT)
cl.register("xybox", Cline.LOCAL, Cline.PROMPT)
cl.register("read", Cline.LOCAL, Cline.PROMPT)
cl.register("gain", Cline.LOCAL, Cline.PROMPT)
cl.register("grain", Cline.LOCAL, Cline.PROMPT)
# get inputs
default_source = os.environ.get('HIPERCAM_DEFAULT_SOURCE', 'hl')
source = cl.get_value(
"source",
"data source [hs, hl, us, ul, hf]",
default_source,
lvals=("hs", "hl", "us", "ul", "hf"),
)
# set some flags
server_or_local = source.endswith("s") or source.endswith("l")
if server_or_local:
resource = cl.get_value("run", "run name", "run005")
if source == "hs":
first = cl.get_value("first", "first frame to plot", 1)
else:
first = cl.get_value("first", "first frame to plot", 1, 0)
last = cl.get_value("last", "last frame to grab", 0)
if last < first and last != 0:
sys.stderr.write("last must be >= first or 0")
sys.exit(1)
twait = cl.get_value("twait",
"time to wait for a new frame [secs]", 1.0,
0.0)
tmax = cl.get_value("tmax",
"maximum time to wait for a new frame [secs]",
10.0, 0.0)
else:
resource = cl.get_value("flist", "file list",
cline.Fname("files.lis", hcam.LIST))
first = 1
trim = cl.get_value("trim", "do you want to trim edges of windows?",
True)
if trim:
ncol = cl.get_value("ncol",
"number of columns to trim from windows", 0)
nrow = cl.get_value("nrow", "number of rows to trim from windows",
0)
else:
ncol, nrow = None, None
# define the panel grid. first get the labels and maximum dimensions
ccdinf = spooler.get_ccd_pars(source, resource)
if len(ccdinf) > 1:
cnam = cl.get_value("ccd",
"CCD to analyse",
"1",
lvals=list(ccdinf.keys()))
else:
cnam = ccdinf.keys()[0]
# bias frame (if any)
bias = cl.get_value(
"bias",
"bias frame ['none' to ignore]",
cline.Fname("bias", hcam.HCAM),
ignore="none",
)
if bias is not None:
# read the bias frame
bias = hcam.MCCD.read(bias)
fprompt = "flat frame ['none' to ignore]"
else:
fprompt = "flat frame ['none' is normal choice with no bias]"
# dark (if any)
dark = cl.get_value("dark",
"dark frame to subtract ['none' to ignore]",
cline.Fname("dark", hcam.HCAM),
ignore="none")
if dark is not None:
# read the dark frame
dark = hcam.MCCD.read(dark)
# flat (if any)
flat = cl.get_value("flat",
fprompt,
cline.Fname("flat", hcam.HCAM),
ignore="none")
if flat is not None:
# read the flat frame
flat = hcam.MCCD.read(flat)
xybox = cl.get_value("xybox",
"box size for averaging results [binned pixels]",
11, 1)
read = cl.get_value("read", "readout noise, RMS ADU", 4.0, 0.0)
gain = cl.get_value("gain", "gain, ADU/e-", 1.0, 0.001)
grain = cl.get_value("grain", "flat field graininess", 0.01, 0.)
######################################
# Phew. We finally have all the inputs
# Now on with the analysis
total_time, nframe = 0, 0
with spooler.data_source(source, resource, first, full=False) as spool:
for mccd in spool:
if server_or_local:
# Handle the waiting game ... some awkward stuff
# involving updating on a cycle faster than twait to
# make the plots more responsive, if twait is long.
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmax, total_time)
if give_up:
print("ncal stopped")
break
elif try_again:
continue
# Trim the frames: ULTRACAM windowed data has bad columns
# and rows on the sides of windows closest to the readout
# which can badly affect reduction. This option strips
# them.
if trim:
hcam.ccd.trim_ultracam(mccd, ncol, nrow)
# indicate progress
tstamp = Time(mccd.head["TIMSTAMP"], format="isot", precision=3)
nfrm = mccd.head.get("NFRAME", nframe + 1)
print(
f'{nfrm}, utc= {tstamp.iso} ({"ok" if mccd.head.get("GOODTIME", True) else "nok"}), '
)
if nframe == 0:
# get the bias, dark, flat
# into shape first time through
if bias is not None:
# crop the bias on the first frame only
bias = bias.crop(mccd)
bexpose = bias.head.get("EXPTIME", 0.0)
else:
bexpose = 0.
if dark is not None:
# crop the dark on the first frame only
dark = dark.crop(mccd)
if flat is not None:
# crop the flat on the first frame only
flat = flat.crop(mccd)
# extract the CCD of interest
ccd = mccd[cnam]
if ccd.is_data():
# "is_data" indicates genuine data as opposed to junk
# that results from nskip > 0.
# subtract the bias
if bias is not None:
ccd -= bias[cnam]
# subtract the dark
if dark is not None:
dexpose = dark.head["EXPTIME"]
cexpose = ccd.head["EXPTIME"]
scale = (cexpose - bexpose) / dexpose
ccd -= scale * dark[cnam]
# divide out the flat
if flat is not None:
ccd /= flat[cnam]
# at this point we have the data in the right state to
# start processing.
for wind in ccd.values():
data = wind.data
ny, nx = data.shape
if nx - 2 >= xybox and ny - 2 >= xybox:
means, stds = procdata(wind.data, xybox)
plt.loglog(means, stds, ',b')
# update the frame number
nframe += 1
if server_or_local and last and nframe > last:
break
count = 10**np.linspace(0, 5, 200)
sigma = np.sqrt(read**2 + count / gain + (grain * count)**2)
plt.plot(count, sigma, 'r', lw=2)
plt.xlim(1, 100000)
plt.ylim(1, 1000)
plt.xlabel('Mean count level')
plt.ylabel('RMS [counts]')
plt.title(f'RMS vs level, CCD {cnam}')
plt.show()
def aligntool(args=None):
"""
Measures alignment of HiperCAM CCDs w.r.t a reference image of the blue CCD.
Arguments::
ref : (string)
name of an MCCD file to use as reference , as produced
by e.g. 'grab'
rccd : (string)
CCD to use as reference
thresh : (float)
threshold for object detection, in multiples of background RMS
source : (string) [hidden]
Data source, five options::
'hs' : HiPERCAM server
'hl' : local HiPERCAM FITS file
'us' : ULTRACAM server
'ul' : local ULTRACAM .xml/.dat files
'hf' : list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab' or
converted from foreign data formats.
device : (string) [hidden]
Plot device. PGPLOT is used so this should be a PGPLOT-style name,
e.g. '/xs', '1/xs' etc. At the moment only ones ending /xs are
supported.
width : (float) [hidden]
plot width (inches). Set = 0 to let the program choose.
height : (float) [hidden]
plot height (inches). Set = 0 to let the program choose. BOTH width
AND height must be non-zero to have any effect
run : (string) [if source == 's' or 'l']
run number to access, e.g. 'run034'
flist : (string) [if source == 'f']
name of file list
first : (int) [if source='s' or 'l']
exposure number to start from. 1 = first frame; set = 0 to
always try to get the most recent frame (if it has changed)
twait : (float) [if source == 's'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : (float) [if source == 's'; hidden]
maximum time to wait between attempts to find a new exposure, seconds.
pause : (float) [hidden]
seconds to pause between frames (defaults to 0)
ccd : (string)
CCD to measure aligment of.
bias : (string)
Name of bias frame to subtract, 'none' to ignore.
msub : (bool)
subtract the median from each window before scaling for the
image display or not. This happens after any bias subtraction.
iset : (string) [single character]
determines how the intensities are determined. There are three
options: 'a' for automatic simply scales from the minimum to the
maximum value found on a per CCD basis. 'd' for direct just takes
two numbers from the user. 'p' for percentile dtermines levels
based upon percentiles determined from the entire CCD on a per CCD
basis.
ilo : (float) [if iset='d']
lower intensity level
ihi : (float) [if iset='d']
upper intensity level
plo : (float) [if iset='p']
lower percentile level
phi : (float) [if iset='p']
upper percentile level
small_spots : (bool) [hidden]
use the small spots for alignment instead of the big spots
"""
command, args = utils.script_args(args)
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("ref", Cline.LOCAL, Cline.PROMPT)
cl.register("thresh", Cline.LOCAL, Cline.PROMPT)
cl.register("small_spots", Cline.LOCAL, Cline.HIDE)
cl.register("source", Cline.LOCAL, Cline.HIDE)
cl.register("device", Cline.LOCAL, Cline.HIDE)
cl.register("width", Cline.LOCAL, Cline.HIDE)
cl.register("height", Cline.LOCAL, Cline.HIDE)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("twait", Cline.LOCAL, Cline.HIDE)
cl.register("tmax", Cline.LOCAL, Cline.HIDE)
cl.register("flist", Cline.LOCAL, Cline.PROMPT)
cl.register("ccd", Cline.LOCAL, Cline.PROMPT)
cl.register("rccd", Cline.LOCAL, Cline.PROMPT)
cl.register("pause", Cline.LOCAL, Cline.HIDE)
cl.register("bias", Cline.GLOBAL, Cline.PROMPT)
cl.register("msub", Cline.GLOBAL, Cline.PROMPT)
cl.register("iset", Cline.GLOBAL, Cline.PROMPT)
cl.register("ilo", Cline.GLOBAL, Cline.PROMPT)
cl.register("ihi", Cline.GLOBAL, Cline.PROMPT)
cl.register("plo", Cline.GLOBAL, Cline.PROMPT)
cl.register("phi", Cline.LOCAL, Cline.PROMPT)
cl.register("xlo", Cline.GLOBAL, Cline.PROMPT)
cl.register("xhi", Cline.GLOBAL, Cline.PROMPT)
cl.register("ylo", Cline.GLOBAL, Cline.PROMPT)
cl.register("yhi", Cline.GLOBAL, Cline.PROMPT)
# get inputs
reference = cl.get_value("ref", "reference frame to align to",
cline.Fname("hcam", hcam.HCAM))
ref_mccd = hcam.MCCD.read(reference)
thresh = cl.get_value("thresh", "detection threshold (sigma)", 5.0)
small_spots = cl.get_value("small_spots",
"use small spots for alignment", False)
# get inputs
source = cl.get_value(
"source",
"data source [hs, hl, us, ul, hf]",
"hl",
lvals=("hs", "hl", "us", "ul", "hf"),
)
# set some flags
server_or_local = source.endswith("s") or source.endswith("l")
inst = source[0]
# plot device stuff
device = cl.get_value("device", "plot device", "1/xs")
width = cl.get_value("width", "plot width (inches)", 0.0)
height = cl.get_value("height", "plot height (inches)", 0.0)
if server_or_local:
resource = cl.get_value("run", "run name", "run005")
first = cl.get_value("first", "first frame to plot", 1, 1)
twait = cl.get_value("twait",
"time to wait for a new frame [secs]", 1.0,
0.0)
tmax = cl.get_value("tmax",
"maximum time to wait for a new frame [secs]",
10.0, 0.0)
else:
# set inst = 'h' as only lists of HiPERCAM files are supported
inst = "h"
resource = cl.get_value("flist", "file list",
cline.Fname("files.lis", hcam.LIST))
first = 1
flist = source.endswith("f")
server = source.endswith("s")
if inst == "u":
instrument = "ULTRA"
elif inst == "h":
instrument = "HIPER"
# get the labels and maximum dimensions and padding factors
ccdinf = spooler.get_ccd_pars(source, resource)
if len(ccdinf) > 1:
ccdnam = cl.get_value("ccd", "CCD to plot alignment of", "1")
rccdnam = cl.get_value("rccd", "CCD to use as reference", "5")
cl.set_default("pause", 0.0)
pause = cl.get_value("pause",
"time delay to add between frame plots [secs]",
0.0, 0.0)
# bias frame (if any)
bias = cl.get_value(
"bias",
"bias frame ['none' to ignore]",
cline.Fname("bias", hcam.HCAM),
ignore="none",
)
if bias is not None:
# read the bias frame
bias = hcam.MCCD.read(bias)
msub = cl.get_value("msub", "subtract median from each window?", True)
iset = cl.get_value(
"iset",
"set intensity a(utomatically),"
" d(irectly) or with p(ercentiles)?",
"a",
lvals=["a", "A", "d", "D", "p", "P"],
)
iset = iset.lower()
plo, phi = 5, 95
ilo, ihi = 0, 1000
if iset == "d":
ilo = cl.get_value("ilo", "lower intensity limit", 0.0)
ihi = cl.get_value("ihi", "upper intensity limit", 1000.0)
elif iset == "p":
plo = cl.get_value("plo", "lower intensity limit percentile", 5.0,
0.0, 100.0)
phi = cl.get_value("phi", "upper intensity limit percentile", 95.0,
0.0, 100.0)
# region to plot
nxtot, nytot, nxpad, nypad = ccdinf[ccdnam]
xmin, xmax = float(-nxpad), float(nxtot + nxpad + 1)
ymin, ymax = float(-nypad), float(nytot + nypad + 1)
xlo = cl.get_value("xlo", "left-hand X value", xmin, xmin, xmax)
xhi = cl.get_value("xhi", "right-hand X value", xmax, xmin, xmax)
ylo = cl.get_value("ylo", "lower Y value", ymin, ymin, ymax)
yhi = cl.get_value("yhi", "upper Y value", ymax, ymin, ymax)
# arguments defined, let's do something!
# open image plot device
imdev = hcam.pgp.Device(device)
if width > 0 and height > 0:
pgpap(width, height / width)
pgsubp(1, 2)
for i in range(2):
pgsci(hcam.pgp.Params["axis.ci"])
pgsch(hcam.pgp.Params["axis.number.ch"])
pgenv(xlo, xhi, ylo, yhi, 1, 0)
pglab("X", "Y", ("reference", "data")[i])
total_time = 0 # time waiting for new frame
with spooler.data_source(source, resource, first) as spool:
# 'spool' is an iterable source of MCCDs
for n, mccd in enumerate(spool):
if server_or_local:
# Handle the waiting game ...
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmax, total_time)
if give_up:
print("alignment tool stopped")
break
elif try_again:
continue
# indicate progress
print(
"Frame {:d}, time = {:s}; ".format(mccd.head["NFRAME"],
mccd.head["TIMSTAMP"]),
end="",
)
if n == 0 and bias is not None:
# crop the bias on the first frame only
bias = bias.crop(mccd)
ccd = mccd[ccdnam]
ref_ccd = ref_mccd[rccdnam]
if bias is not None:
ccd -= bias[ccdnam]
if msub:
for wind in ccd.values():
wind -= wind.median()
for wind in ref_ccd.values():
wind -= wind.median()
# plot images
message = ""
for i in range(2):
this_ccd = (ref_ccd, ccd)[i]
name = ("ref: ", "data: ")[i]
pgpanl(1, i + 1)
vmin, vmax = hcam.pgp.pCcd(this_ccd, iset, plo, phi, ilo, ihi)
message += " {:s}: {:.2f} to {:.2f}".format(name, vmin, vmax)
print(message)
# time for measurement of spots!
try:
xpos, ypos, fwhm, flux = measureSpots(mccd, ccdnam, thresh)
lx, ly, lf, lp, bx, by, bf, bp = separateBigSmallSpots(
xpos, ypos, fwhm, flux)
if not small_spots:
# use big spots
x, y, f, flux = bx, by, bf, bp
else:
x, y, f, flux = lx, ly, lf, lp
except Exception as err:
print("Failed to find spots in frame: ", end=" ")
print(str(err))
continue
# also measure spots in reference image
try:
xpos, ypos, fwhm, rflux = measureSpots(ref_mccd, rccdnam,
thresh)
lx, ly, lf, lp, bx, by, bf, bp = separateBigSmallSpots(
xpos, ypos, fwhm, rflux)
if not small_spots:
# use big spots
xref, yref, fref = bx, by, bf
else:
xref, yref, fref = lx, ly, lf
except Exception as err:
print("Failed to find spots in reference: ", end=" ")
print(str(err))
continue
plt.clf()
displayFWHM(x, y, xref, yref, f, flux, fix_fwhm_scale=False)
plt.draw()
plt.pause(0.05)
plt.ioff()
plt.show()
def filtid(args=None):
"""``filtid [source] (run first last [twait tmax] |
flist) trim ([ncol nrow]) ccdref maxlev plot``
Computes the ratio of the mean values in each CCD versus those of a reference CCD.
This is supposed to be applied to flat fields. The hope is that this might be
characteristic of the filter combination. Obviously requires > 1 CCD, i.e. not
ULTRASPEC.
Parameters:
source : str [hidden]
Data source, five options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : ULTRACAM server
| 'ul' : local ULTRACAM .xml/.dat files
| 'hf' : list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab'
or converted from foreign data formats. The standard
start-off default for ``source'' can be set using the
environment variable HIPERCAM_DEFAULT_SOURCE. e.g. in bash
:code:`export HIPERCAM_DEFAULT_SOURCE="us"` would ensure it
always started with the ULTRACAM server by default. If
unspecified, it defaults to 'hl'.
run : str [if source ends 's' or 'l']
run number to access, e.g. 'run034'
flist : str [if source ends 'f']
name of file list
first : int [if source ends 's' or 'l']
exposure number to start from. 1 = first frame; set = 0 to always
try to get the most recent frame (if it has changed). For data
from the |hiper| server, a negative number tries to get a frame not
quite at the end. i.e. -10 will try to get 10 from the last
frame. This is mainly to sidestep a difficult bug with the
acquisition system.
last : int [if source ends 's' or 'l']
last frame to reduce. 0 to just continue until the end.
twait : float [if source ends 's' or 'l'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [if source ends 's' or 'l'; hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
trim : bool [if source starts with 'u']
True to trim columns and/or rows off the edges of windows nearest
the readout which can sometimes contain bad data.
ncol : int [if trim, hidden]
Number of columns to remove (on left of left-hand window, and right
of right-hand windows)
nrow : int [if trim, hidden]
Number of rows to remove (bottom of windows)
ccdref : str
The reference CCD (usually choose the g-band one)
bias : str
bias frame to subtract (required)
lower : list of floats
Lower limits to the mean count level for a flat to be
included (after bias subtraction). Should be the same
number as the number of CCDs, and will be assumed to be in
the same order. Separate with spaces. Prevents low exposure
data from being included.
upper : list of floats
Upper limits to the mean count level for a flat to be
included. Should be the same number as the selected CCDs,
and will be assumed to be in the same order. Use to
eliminate saturated, peppered or non-linear
frames. Suggested hipercam values: 58000, 58000, 58000,
40000 and 40000 for CCDs 1, 2, 3, 4 and 5. Enter values
separated by spaces. ULTRACAM values 49000, 29000, 27000
for CCDs 1, 2 and 3.
plot: bool
Plot the fit or not.
.. Note::
This is currently adapted specifically for ULTRACAM data
"""
command, args = utils.script_args(args)
# get the inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("source", Cline.GLOBAL, Cline.HIDE)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("last", Cline.LOCAL, Cline.PROMPT)
cl.register("trim", Cline.GLOBAL, Cline.PROMPT)
cl.register("ncol", Cline.GLOBAL, Cline.HIDE)
cl.register("nrow", Cline.GLOBAL, Cline.HIDE)
cl.register("twait", Cline.LOCAL, Cline.HIDE)
cl.register("tmax", Cline.LOCAL, Cline.HIDE)
cl.register("flist", Cline.LOCAL, Cline.PROMPT)
cl.register("ccdref", Cline.LOCAL, Cline.PROMPT)
cl.register("bias", Cline.LOCAL, Cline.PROMPT)
cl.register("lower", Cline.LOCAL, Cline.PROMPT)
cl.register("upper", Cline.LOCAL, Cline.PROMPT)
cl.register("plot", Cline.LOCAL, Cline.PROMPT)
# get inputs
default_source = os.environ.get('HIPERCAM_DEFAULT_SOURCE', 'hl')
source = cl.get_value(
"source",
"data source [hs, hl, us, ul, hf]",
default_source,
lvals=("hs", "hl", "us", "ul", "hf"),
)
# set some flags
server_or_local = source.endswith("s") or source.endswith("l")
if server_or_local:
resource = cl.get_value("run", "run name", "run005")
first = cl.get_value("first", "first frame to consider", 1, 1)
last = cl.get_value("last", "last frame to consider", 0, 0)
if last and last < first + 2:
raise hcam.HipercamError(
"Must consider at least 2 exposures, and preferably more")
twait = cl.get_value("twait",
"time to wait for a new frame [secs]", 1.0,
0.0)
tmax = cl.get_value("tmax",
"maximum time to wait for a new frame [secs]",
10.0, 0.0)
else:
resource = cl.get_value("flist", "file list",
cline.Fname("files.lis", hcam.LIST))
first = 1
trim = cl.get_value("trim", "do you want to trim edges of windows?",
True)
if trim:
ncol = cl.get_value("ncol",
"number of columns to trim from windows", 0)
nrow = cl.get_value("nrow", "number of rows to trim from windows",
0)
# define the panel grid. first get the labels and maximum dimensions
ccdinf = spooler.get_ccd_pars(source, resource)
if len(ccdinf) == 1:
raise hcam.HipercamError(
"Only one CCD; this routine only works for > 1 CCD")
ccds = list(ccdinf.keys())
ccdref = cl.get_value("ccdref",
"the reference CCD for the ratios",
ccds[1],
lvals=ccds)
bias = cl.get_value("bias", "bias frame",
cline.Fname("bias", hcam.HCAM))
# read the bias frame
bias = hcam.MCCD.read(bias)
if len(bias) != len(ccds):
raise ValueError('bias has an incompatible number of CCDs')
# need to check that the default has the right number of items, if not,
# over-ride it
lowers = cl.get_default("lower")
if lowers is not None and len(lowers) != len(ccds):
cl.set_default("lower", len(ccds) * (5000, ))
lowers = cl.get_value(
"lower",
"lower limits on mean count level for included flats, 1 per CCD",
len(ccds) * (5000, ),
)
lowers = {k: v for k, v in zip(ccds, lowers)}
uppers = cl.get_default("upper")
if uppers is not None and len(uppers) != len(ccds):
cl.set_default("upper", len(ccds) * (50000, ))
uppers = cl.get_value(
"upper",
"lower limits on mean count level for included flats, 1 per CCD",
len(ccds) * (50000, ),
)
uppers = {k: v for k, v in zip(ccds, uppers)}
plot = cl.get_value("plot", "plot the results?", True)
################################################################
#
# all the inputs have now been obtained. Get on with doing stuff
total_time = 0.
xdata, ydata = {}, {}
# only want to look at CCDs other than ccdref
ccds.remove(ccdref)
# access images
with spooler.data_source(source, resource, first, full=False) as spool:
# 'spool' is an iterable source of MCCDs
nframe = 0
for mccd in spool:
if server_or_local:
# Handle the waiting game ...
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmax, total_time)
if give_up:
print("rtplot stopped")
break
elif try_again:
continue
# Trim the frames: ULTRACAM windowed data has bad columns
# and rows on the sides of windows closest to the readout
# which can badly affect reduction. This option strips
# them.
if trim:
hcam.ccd.trim_ultracam(mccd, ncol, nrow)
if nframe == 0:
# crop the bias on the first frame only
bias = bias.crop(mccd)
# bias subtraction
mccd -= bias
# indicate progress
tstamp = Time(mccd.head["TIMSTAMP"], format="isot", precision=3)
print(f"{mccd.head.get('NFRAME',nframe+1)}, utc= {tstamp.iso} " +
f"({'ok' if mccd.head.get('GOODTIME', True) else 'nok'})")
# add in data if it is in range
xd = mccd[ccdref].mean()
if xd > lowers[ccdref] and xd < uppers[ccdref]:
for cnam in ccds:
yd = mccd[cnam].mean()
if yd > lowers[cnam] and yd < uppers[cnam]:
if cnam in xdata:
xdata[cnam].append(xd)
ydata[cnam].append(yd)
else:
xdata[cnam] = [xd]
ydata[cnam] = [yd]
if last and nframe == last:
break
# update the frame number
nframe += 1
print()
invalid = True
for cnam in ccds:
if cnam in xdata:
xdata[cnam] = np.array(xdata[cnam])
ydata[cnam] = np.array(ydata[cnam])
print(
f'{cnam}-vs-{ccdref}: found {len(xdata[cnam])} valid points ranging from'
+
f' {xdata[cnam].min():.1f} to {xdata[cnam].max():.1f} (CCD {ccdref}), '
+
f' {ydata[cnam].min():.1f} to {ydata[cnam].max():.1f} (CCD {cnam})'
)
invalid = False
if invalid:
raise hcam.HipercamError(f"No valid points found at all")
print()
if plot:
fig, axes = plt.subplots(len(ccds), 1, sharex=True)
for cnam, ax in zip(ccds, axes):
if cnam in xdata:
xds = xdata[cnam]
ratios = ydata[cnam] / xds
mratio = ratios.mean()
ax.plot(xds, ratios, '.g')
ax.plot([xds.min(), xds.max()], [mratio, mratio], 'r--')
ax.set_ylabel(f'Ratio CCD {cnam} / CCD {ccdref}')
if cnam == ccds[-1]:
ax.set_xlabel(f'CCD = {ccdref} counts/pixel')
print(f'{cnam}-vs-{ccdref} = {mratio:.4f}')
plt.show()
else:
for cnam in ccds:
if cnam in xdata:
ratios = ydata[cnam] / xdata[cnam]
mratio = ratios.mean()
print(f'{cnam}-vs-{ccdref} = {mratio:.4f}')
def mstats(args=None):
"""``mstats [source] run [temp] (ndigit) first last [twait tmax] bias
[dtype]``
This downloads a sequence of images from a raw data file and writes
out stats (min, max, mean, median, rms) for each window to a file
Parameters:
source : string [hidden]
Data source, four options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : ULTRACAM server
| 'ul' : local ULTRACAM .xml/.dat files
run : string
run name to access
first : int
First frame to access
last : int
Last frame to access, 0 for the lot
twait : float [hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
bias : string
Name of bias frame to subtract, 'none' to ignore.
format : string
output format for numbers. e.g. the default '9.3f'
might give 12345.678 (9 characters, 3 digits after d.p.)
outfile : string
file for output (extension .stats)
"""
command, args = utils.script_args(args)
# get inputs
with Cline('HIPERCAM_ENV', '.hipercam', command, args) as cl:
# register parameters
cl.register('source', Cline.GLOBAL, Cline.HIDE)
cl.register('run', Cline.GLOBAL, Cline.PROMPT)
cl.register('first', Cline.LOCAL, Cline.PROMPT)
cl.register('last', Cline.LOCAL, Cline.PROMPT)
cl.register('twait', Cline.LOCAL, Cline.HIDE)
cl.register('tmax', Cline.LOCAL, Cline.HIDE)
cl.register('bias', Cline.GLOBAL, Cline.PROMPT)
cl.register('format', Cline.LOCAL, Cline.HIDE)
cl.register('outfile', Cline.LOCAL, Cline.PROMPT)
# get inputs
source = cl.get_value('source',
'data source [hs, hl, us, ul]',
'hl',
lvals=('hs', 'hl', 'us', 'ul'))
# OK, more inputs
resource = cl.get_value('run', 'run name', 'run005')
first = cl.get_value('first', 'first frame to grab', 1, 0)
last = cl.get_value('last', 'last frame to grab', 0)
if last < first and last != 0:
sys.stderr.write('last must be >= first or 0')
sys.exit(1)
twait = cl.get_value('twait', 'time to wait for a new frame [secs]',
1., 0.)
tmax = cl.get_value('tmax',
'maximum time to wait for a new frame [secs]', 10.,
0.)
bias = cl.get_value('bias',
"bias frame ['none' to ignore]",
cline.Fname('bias', hcam.HCAM),
ignore='none')
cl.set_default('format', '9.3f')
form = cl.get_value('format', 'output format for stats', '9.3f')
outfile = cl.get_value('outfile', 'output file for stats',
cline.Fname('stats', '.stats', cline.Fname.NEW))
# Now the actual work.
# strip off extensions
if resource.endswith(hcam.HRAW):
resource = resource[:resource.find(hcam.HRAW)]
# initialisations
total_time = 0 # time waiting for new frame
nframe = first
root = os.path.basename(resource)
bframe = None
with spooler.data_source(source, resource, first) as spool:
with open(outfile, 'w') as stats:
stats.write("""#
# This file was generated by mstats running on file {run}
#
# The columns are:
#
# nframe ccd window minimum maximum mean median rms
#
# where ccd and window are string labels, nframe is the frame
# number an an integer, while the rest are floats.
#
""".format(run=resource))
for mccd in spool:
# Handle the waiting game ...
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmax, total_time)
if give_up:
print('mstats stopped')
break
elif try_again:
continue
if bias is not None:
# read bias after first frame so we can
# chop the format
if bframe is None:
# read the bias frame
bframe = hcam.MCCD.read(bias)
# reformat
bframe = bframe.crop(mccd)
mccd -= bframe
for cnam, ccd in mccd.items():
for wnam, wind in ccd.items():
stats.write(
('{1:5d} {2:5s} {3:5s} {4:{0:s}} {5:{0:s}}'
' {6:{0:s}} {7:{0:s}} {8:{0:s}}\n').format(
form, nframe, cnam, wnam, wind.min(),
wind.max(), wind.mean(), wind.median(),
wind.std()))
# flush the output
stats.flush()
# progress info
print('Written stats of frame {:d} to {:s}'.format(
nframe, outfile))
# update the frame number
nframe += 1
if last and nframe > last:
break
def reduce(args=None):
"""``reduce [source] rfile (run first last (trim [ncol nrow]) twait
tmax | flist) log lplot implot (ccd nx msub xlo xhi ylo yhi iset
(ilo ihi | plo phi))``
Reduces a sequence of multi-CCD images, plotting lightcurves as images
come in. It can extract with either simple aperture photometry or Tim
Naylor's optimal photometry, on specific targets defined in an aperture
file using |setaper|.
reduce can source data from both the ULTRACAM and HiPERCAM servers, from
local 'raw' ULTRACAM and HiPERCAM files (i.e. .xml + .dat for ULTRACAM, 3D
FITS files for HiPERCAM) and from lists of HiPERCAM '.hcm' files. If you
have data from a different instrument you should convert into the
FITS-based hcm format.
reduce is primarily configured from a file with extension ".red". This
contains a series of directives, e.g. to say how to re-position and
re-size the apertures. An initial reduce file is best generated with
the script |genred| after you have created an aperture file. This contains
lots of help on what to do.
A reduce run can be terminated at any point with ctrl-C without doing
any harm. You may often want to do this at the start in order to adjust
parameters of the reduce file.
Parameters:
source : string [hidden]
Data source, five options:
| 'hs': HiPERCAM server
| 'hl': local HiPERCAM FITS file
| 'us': ULTRACAM server
| 'ul': local ULTRACAM .xml/.dat files
| 'hf': list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab' or
converted from foreign data formats.
rfile : string
the "reduce" file, i.e. ASCII text file suitable for reading by
ConfigParser. Best seen by example as it has many parts.
run : string [if source ends 's' or 'l']
run number to access, e.g. 'run034'
first : int [if source ends 's' or 'l']
first frame to reduce. 1 = first frame; set = 0 to always try to
get the most recent frame (if it has changed).
last : int [if source ends 's' or 'l', hidden]
last frame to reduce. 0 to just continue until the end. This is
not prompted for by default and must be set explicitly. It
defaults to 0 if not set. Its purpose is to allow accurate
profiling tests.
trim : bool [if source starts with 'u']
True to trim columns and/or rows off the edges of windows nearest
the readout. This is particularly for ULTRACAM windowed data where
the first few rows and columns can contain bad data.
ncol : int [if trim]
Number of columns to remove (on left of left-hand window, and right
of right-hand windows)
nrow : int [if trim]
Number of rows to remove (bottom of windows)
twait : float [if source ends 's'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [if source ends 's'; hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
flist : string [if source ends 'f']
name of file list
log : string
log file for the results
tkeep : float
maximum number of minutes of data to store in internal buffers, 0
for the lot. When large numbers of frames are stored, performance
can be slowed (although I am not entirely clear why) in which case
it makes sense to lose the earlier points (without affecting the
saving to disk). This parameter also gives operation similar to that
of "max_xrange" parameter in the ULTRACAM pipeline whereby just
the last few minutes are shown.
lplot : bool
flag to indicate you want to plot the light curve. Saves time not
to especially in high-speed runs.
implot : bool
flag to indicate you want to plot images.
ccd : string [if implot]
CCD(s) to plot, '0' for all, '1 3' to plot '1' and '3' only, etc.
nx : int [if implot]
number of panels across to display.
msub : bool [if implot]
subtract the median from each window before scaling for the
image display or not. This happens after any bias subtraction.
xlo : float [if implot]
left-hand X-limit for plot
xhi : float [if implot]
right-hand X-limit for plot (can actually be < xlo)
ylo : float [if implot]
lower Y-limit for plot
yhi : float [if implot]
upper Y-limit for plot (can be < ylo)
iset : string [if implot]
determines how the intensities are determined. There are three
options: 'a' for automatic simply scales from the minimum to the
maximum value found on a per CCD basis. 'd' for direct just takes
two numbers from the user. 'p' for percentile dtermines levels
based upon percentiles determined from the entire CCD on a per CCD
basis.
ilo : float [if implot and iset='d']
lower intensity level
ihi : float [if implot and iset='d']
upper intensity level
plo : float [if implot and iset='p']
lower percentile level
phi : float [if implot and iset='p']
upper percentile level
.. Warning::
The transmission plot generated with reduce is not reliable in the
case of optimal photometry since it is highly correlated with the
seeing. If you are worried about the transmission during observing,
you should always use normal aperture photometry.
"""
command, args = utils.script_args(args)
with Cline('HIPERCAM_ENV', '.hipercam', command, args) as cl:
# register parameters
cl.register('source', Cline.GLOBAL, Cline.HIDE)
cl.register('rfile', Cline.GLOBAL, Cline.PROMPT)
cl.register('run', Cline.GLOBAL, Cline.PROMPT)
cl.register('first', Cline.LOCAL, Cline.PROMPT)
cl.register('last', Cline.LOCAL, Cline.HIDE)
cl.register('trim', Cline.GLOBAL, Cline.PROMPT)
cl.register('ncol', Cline.GLOBAL, Cline.HIDE)
cl.register('nrow', Cline.GLOBAL, Cline.HIDE)
cl.register('twait', Cline.LOCAL, Cline.HIDE)
cl.register('tmax', Cline.LOCAL, Cline.HIDE)
cl.register('flist', Cline.LOCAL, Cline.PROMPT)
cl.register('log', Cline.GLOBAL, Cline.PROMPT)
cl.register('tkeep', Cline.GLOBAL, Cline.PROMPT)
cl.register('lplot', Cline.LOCAL, Cline.PROMPT)
cl.register('implot', Cline.LOCAL, Cline.PROMPT)
cl.register('ccd', Cline.LOCAL, Cline.PROMPT)
cl.register('nx', Cline.LOCAL, Cline.PROMPT)
cl.register('msub', Cline.GLOBAL, Cline.PROMPT)
cl.register('iset', Cline.GLOBAL, Cline.PROMPT)
cl.register('ilo', Cline.GLOBAL, Cline.PROMPT)
cl.register('ihi', Cline.GLOBAL, Cline.PROMPT)
cl.register('plo', Cline.GLOBAL, Cline.PROMPT)
cl.register('phi', Cline.LOCAL, Cline.PROMPT)
cl.register('xlo', Cline.GLOBAL, Cline.PROMPT)
cl.register('xhi', Cline.GLOBAL, Cline.PROMPT)
cl.register('ylo', Cline.GLOBAL, Cline.PROMPT)
cl.register('yhi', Cline.GLOBAL, Cline.PROMPT)
# get inputs
source = cl.get_value('source',
'data source [hs, hl, us, ul, hf]',
'hl',
lvals=('hs', 'hl', 'us', 'ul', 'hf'))
# set some flags
server_or_local = source.endswith('s') or source.endswith('l')
# the reduce file
rfilen = cl.get_value('rfile', 'reduce file',
cline.Fname('reduce.red', hcam.RED))
try:
rfile = Rfile.read(rfilen)
except hcam.HipercamError as err:
# abort on failure to read as there are many ways to get reduce
# files wrong
print(err, file=sys.stderr)
print('*** reduce aborted')
exit(1)
if server_or_local:
resource = cl.get_value('run', 'run name', 'run005')
first = cl.get_value('first', 'first frame to reduce', 1, 0)
cl.set_default('last', 0)
last = cl.get_value('last', 'last frame to reduce', 0, 0)
if last and last < first:
print('Cannot set last < first unless last == 0')
print('*** reduce aborted')
exit(1)
if source.startswith('u'):
trim = cl.get_value(
'trim',
'do you want to trim edges of windows? (ULTRACAM only)',
True)
if trim:
ncol = cl.get_value(
'ncol', 'number of columns to trim from windows', 0)
nrow = cl.get_value('nrow',
'number of rows to trim from windows',
0)
else:
trim = False
twait = cl.get_value('twait',
'time to wait for a new frame [secs]', 1., 0.)
tmx = cl.get_value('tmax',
'maximum time to wait for a new frame [secs]',
10., 0.)
else:
resource = cl.get_value('flist', 'file list',
cline.Fname('files.lis', hcam.LIST))
first = 1
last = 0
trim = False
log = cl.get_value(
'log', 'name of log file to store results',
cline.Fname('reduce.log', hcam.LOG, cline.Fname.NEW))
tkeep = cl.get_value(
'tkeep', 'number of minute of data to'
' keep in internal buffers (0 for all)', 0., 0.)
lplot = cl.get_value('lplot', 'do you want to plot light curves?',
True)
implot = cl.get_value('implot', 'do you want to plot images?', True)
if implot:
# define the panel grid. first get the labels and maximum
# dimensions
ccdinf = spooler.get_ccd_pars(source, resource)
try:
nxdef = cl.get_default('nx')
except KeyError:
nxdef = 3
if len(ccdinf) > 1:
ccd = cl.get_value('ccd', 'CCD(s) to plot [0 for all]', '0')
if ccd == '0':
ccds = list(ccdinf.keys())
else:
ccds = ccd.split()
if len(ccds) > 1:
nxdef = min(len(ccds), nxdef)
cl.set_default('nx', nxdef)
nx = cl.get_value('nx', 'number of panels in X', 3, 1)
else:
nx = 1
else:
nx = 1
ccds = list(ccdinf.keys())
# define the display intensities
msub = cl.get_value('msub', 'subtract median from each window?',
True)
iset = cl.get_value('iset', 'set intensity a(utomatically),'
' d(irectly) or with p(ercentiles)?',
'a',
lvals=['a', 'd', 'p'])
plo, phi = 5, 95
ilo, ihi = 0, 1000
if iset == 'd':
ilo = cl.get_value('ilo', 'lower intensity limit', 0.)
ihi = cl.get_value('ihi', 'upper intensity limit', 1000.)
elif iset == 'p':
plo = cl.get_value('plo', 'lower intensity limit percentile',
5., 0., 100.)
phi = cl.get_value('phi', 'upper intensity limit percentile',
95., 0., 100.)
# region to plot
for i, cnam in enumerate(ccds):
nxtot, nytot, nxpad, nypad = ccdinf[cnam]
if i == 0:
xmin, xmax = float(-nxpad), float(nxtot + nxpad + 1)
ymin, ymax = float(-nypad), float(nytot + nypad + 1)
else:
xmin = min(xmin, float(-nxpad))
xmax = max(xmax, float(nxtot + nxpad + 1))
ymin = min(ymin, float(-nypad))
ymax = max(ymax, float(nytot + nypad + 1))
xlo = cl.get_value('xlo', 'left-hand X value', xmin, xmin, xmax)
xhi = cl.get_value('xhi', 'right-hand X value', xmax, xmin, xmax)
ylo = cl.get_value('ylo', 'lower Y value', ymin, ymin, ymax)
yhi = cl.get_value('yhi', 'upper Y value', ymax, ymin, ymax)
else:
ccds, nx, msub, iset = None, None, None, None
ilo, ihi, plo, phi = None, None, None, None
xlo, xhi, ylo, yhi = None, None, None, None
# save list of parameter values for writing to the reduction file
plist = cl.list()
################################################################
#
# all the inputs have now been obtained. Get on with doing stuff
if implot:
plot_lims = (xlo, xhi, ylo, yhi)
else:
plot_lims = None
imdev, lcdev, spanel, tpanel, xpanel, ypanel, lpanel = setup_plots(
rfile, ccds, nx, plot_lims, implot, lplot)
# a couple of initialisations
total_time = 0 # time waiting for new frame
if lplot:
lbuffer, xbuffer, ybuffer, tbuffer, sbuffer = setup_plot_buffers(rfile)
else:
lbuffer, xbuffer, ybuffer, tbuffer, sbuffer = None, None, None, None, None
############################################
#
# open the log file and write headers
#
with LogWriter(log, rfile, hipercam_version, plist) as logfile:
ncpu = rfile['general']['ncpu']
if ncpu > 1:
pool = multiprocessing.Pool(processes=ncpu)
else:
pool = None
# whether a tzero has been set
tzset = False
# containers for the processed and raw MCCD groups
# and their frame numbers
pccds, mccds, nframes = [], [], []
##############################################
#
# Finally, start winding through the frames
#
with spooler.data_source(source, resource, first, full=False) as spool:
# 'spool' is an iterable source of MCCDs
for nf, mccd in enumerate(spool):
if server_or_local:
# Handle the waiting game ...
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmx, total_time)
if give_up:
# Giving up, but need to handle any partially filled
# frame group
if len(mccds):
# finish processing remaining frames. This step
# will only occur if we have at least once passed
# to later stages during which read and gain will
# be set up
results = processor(pccds, mccds, nframes)
# write out results to the log file
alerts = logfile.write_results(results)
# print out any accumulated alert messages
if len(alerts):
print('\n'.join(alerts))
update_plots(results, rfile, implot, lplot, imdev,
lcdev, pccd, ccds, msub, nx, iset,
plo, phi, ilo, ihi, xlo, xhi, ylo,
yhi, tzero, lpanel, xpanel, ypanel,
tpanel, spanel, tkeep, lbuffer,
xbuffer, ybuffer, tbuffer, sbuffer)
mccds = []
print('reduce finished')
break
elif try_again:
continue
# Trim the frames: ULTRACAM windowed data has bad
# columns and rows on the sides of windows closest to
# the readout which can badly affect reduction. This
# option strips them.
if trim:
hcam.ccd.trim_ultracam(mccd, ncol, nrow)
# indicate progress
if 'NFRAME' in mccd.head:
nframe = mccd.head['NFRAME']
else:
nframe = nf + 1
if source != 'hf' and last and nframe > last:
# finite last frame number
if len(mccds):
# finish processing remaining frames
results = processor(pccds, mccds, nframes)
# write out results to the log file
alerts = logfile.write_results(results)
# print out any accumulated alert messages
if len(alerts):
print('\n'.join(alerts))
update_plots(results, rfile, implot, lplot, imdev,
lcdev, pccd, ccds, msub, nx, iset, plo,
phi, ilo, ihi, xlo, xhi, ylo, yhi, tzero,
lpanel, xpanel, ypanel, tpanel, spanel,
tkeep, lbuffer, xbuffer, ybuffer, tbuffer,
sbuffer)
mccds = []
print('\nHave reduced up to the last frame set.')
print('reduce finished')
break
print('Frame {:d}: {:s} [{:s}]'.format(
nframe, mccd.head['TIMSTAMP'],
'OK' if mccd.head.get('GOODTIME', True) else 'NOK'),
end='' if implot else '\n')
if not tzset:
# This is the first frame which allows us to make
# some checks and initialisations.
tzero, read, gain, ok = initial_checks(mccd, rfile)
# Define the CCD processor function object
processor = ProcessCCDs(rfile, read, gain, ccdproc, pool)
# set flag to show we are set
if not ok:
break
tzset = True
# De-bias the data. Retain a copy of the raw data as 'mccd'
# in order to judge saturation. Processed data called 'pccd'
if rfile.bias is not None:
# subtract bias
pccd = mccd - rfile.bias
bexpose = rfile.bias.head.get('EXPTIME', 0.)
else:
# no bias subtraction
pccd = mccd.copy()
bexpose = 0.
if rfile.dark is not None:
# subtract dark, CCD by CCD
dexpose = rfile.dark.head['EXPTIME']
for cnam in pccd:
ccd = pccd[cnam]
cexpose = ccd.head['EXPTIME']
scale = (cexpose - bexpose) / dexpose
ccd -= scale * rfile.dark[cnam]
if rfile.flat is not None:
# apply flat field to processed frame
pccd /= rfile.flat
# Acummulate frames into processing groups for faster
# parallelisation
pccds.append(pccd)
mccds.append(mccd)
nframes.append(nframe)
if len(pccds) == rfile['general']['ngroup']:
# parallel processing. This should usually be the first
# points at which it takes place
results = processor(pccds, mccds, nframes)
# write out results to the log file
alerts = logfile.write_results(results)
# print out any accumulated alert messages
if len(alerts):
print('\n'.join(alerts))
update_plots(results, rfile, implot, lplot, imdev, lcdev,
pccds[-1], ccds, msub, nx, iset, plo, phi,
ilo, ihi, xlo, xhi, ylo, yhi, tzero, lpanel,
xpanel, ypanel, tpanel, spanel, tkeep,
lbuffer, xbuffer, ybuffer, tbuffer, sbuffer)
# Reset the frame buffers
pccds, mccds, nframes = [], [], []
if len(mccds):
# out of loop now. Finish processing any remaining
# frames.
results = processor(pccds, mccds, nframes)
# write out results to the log file
alerts = logfile.write_results(results)
# print out any accumulated alert messages
if len(alerts):
print('\n'.join(alerts))
update_plots(results, rfile, implot, lplot, imdev, lcdev, pccd,
ccds, msub, nx, iset, plo, phi, ilo, ihi, xlo, xhi,
ylo, yhi, tzero, lpanel, xpanel, ypanel, tpanel,
spanel, tkeep, lbuffer, xbuffer, ybuffer, tbuffer,
sbuffer)
print('reduce finished')
def reduce(args=None):
"""``reduce [source] rfile (run first last twait tmax | flist) trim
([ncol nrow]) log lplot implot (ccd nx msub xlo xhi ylo yhi iset
(ilo ihi | plo phi))``
Reduces a sequence of multi-CCD images, plotting lightcurves as images
come in. It can extract with either simple aperture photometry or Tim
Naylor's optimal photometry, on specific targets defined in an aperture
file using |setaper|.
reduce can source data from both the ULTRACAM and HiPERCAM servers, from
local 'raw' ULTRACAM and HiPERCAM files (i.e. .xml + .dat for ULTRACAM, 3D
FITS files for HiPERCAM) and from lists of HiPERCAM '.hcm' files. If you
have data from a different instrument you should convert into the
FITS-based hcm format.
reduce is primarily configured from a file with extension ".red". This
contains a series of directives, e.g. to say how to re-position and
re-size the apertures. An initial reduce file is best generated with
the script |genred| after you have created an aperture file. This contains
lots of help on what to do.
A reduce run can be terminated at any point with ctrl-C without doing
any harm. You may often want to do this at the start in order to adjust
parameters of the reduce file.
Parameters:
source : str [hidden]
Data source, five options:
| 'hs': HiPERCAM server
| 'hl': local HiPERCAM FITS file
| 'us': ULTRACAM server
| 'ul': local ULTRACAM .xml/.dat files
| 'hf': list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab' or
converted from foreign data formats.
run : str [if source ends 's' or 'l']
run number to access, e.g. 'run034' or a file list. If a run,
then reduce and log below will be set to have the same name by
default.
first : int [if source ends 's' or 'l']
first frame to reduce. 1 = first frame; set = 0 to always try to
get the most recent frame (if it has changed).
last : int [if source ends 's' or 'l', hidden]
last frame to reduce. 0 to just continue until the end. This is
not prompted for by default and must be set explicitly. It
defaults to 0 if not set. Its purpose is to allow accurate
profiling tests.
twait : float [if source ends 's'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [if source ends 's'; hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
flist : string [if source ends 'f']
name of file list
trim : bool
True to trim columns and/or rows off the edges of windows nearest
the readout. Particularly useful with ULTRACAM windowed data where
the first few rows and columns can contain bad data.
ncol : int [if trim, hidden]
Number of columns to remove (on left of left-hand window, and right
of right-hand windows)
nrow : int [if trim, hidden]
Number of rows to remove (bottom of windows)
rfile : str
the "reduce" file, i.e. ASCII text file suitable for reading by
ConfigParser. Best seen by example as it has many parts. If you
are reducing a run, this will be set to have the same root name by
default (but a different extension to avoid name clashes).
log : str
log file for the results. If you are reducing a run, this
will be set to have the same root name by default (but a
different extension to avoid name clashes)
tkeep : float
maximum number of minutes of data to store in internal buffers, 0
for the lot. When large numbers of frames are stored, performance
can be slowed (although I am not entirely clear why) in which case
it makes sense to lose the earlier points (without affecting the
saving to disk). This parameter also gives operation similar to that
of "max_xrange" parameter in the ULTRACAM pipeline whereby just
the last few minutes are shown.
lplot : bool
flag to indicate you want to plot the light curve. Saves time not
to especially in high-speed runs.
implot : bool
flag to indicate you want to plot images.
ccd : string [if implot]
CCD(s) to plot, '0' for all, '1 3' to plot '1' and '3' only, etc.
nx : int [if implot]
number of panels across to display.
msub : bool [if implot]
subtract the median from each window before scaling for the
image display or not. This happens after any bias subtraction.
xlo : float [if implot]
left-hand X-limit for plot
xhi : float [if implot]
right-hand X-limit for plot (can actually be < xlo)
ylo : float [if implot]
lower Y-limit for plot
yhi : float [if implot]
upper Y-limit for plot (can be < ylo)
iset : string [if implot]
determines how the intensities are determined. There are three
options: 'a' for automatic simply scales from the minimum to the
maximum value found on a per CCD basis. 'd' for direct just takes
two numbers from the user. 'p' for percentile dtermines levels
based upon percentiles determined from the entire CCD on a per CCD
basis.
ilo : float [if implot and iset='d']
lower intensity level
ihi : float [if implot and iset='d']
upper intensity level
plo : float [if implot and iset='p']
lower percentile level
phi : float [if implot and iset='p']
upper percentile level
.. Warning::
The transmission plot generated with reduce is not reliable in the
case of optimal photometry since it is highly correlated with the
seeing. If you are worried about the transmission during observing,
you should always use normal aperture photometry.
"""
command, args = utils.script_args(args)
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("source", Cline.GLOBAL, Cline.HIDE)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("last", Cline.LOCAL, Cline.HIDE)
cl.register("twait", Cline.LOCAL, Cline.HIDE)
cl.register("tmax", Cline.LOCAL, Cline.HIDE)
cl.register("flist", Cline.LOCAL, Cline.PROMPT)
cl.register("trim", Cline.GLOBAL, Cline.PROMPT)
cl.register("ncol", Cline.GLOBAL, Cline.HIDE)
cl.register("nrow", Cline.GLOBAL, Cline.HIDE)
cl.register("rfile", Cline.GLOBAL, Cline.PROMPT)
cl.register("log", Cline.GLOBAL, Cline.PROMPT)
cl.register("tkeep", Cline.GLOBAL, Cline.PROMPT)
cl.register("lplot", Cline.LOCAL, Cline.PROMPT)
cl.register("implot", Cline.LOCAL, Cline.PROMPT)
cl.register("ccd", Cline.LOCAL, Cline.PROMPT)
cl.register("nx", Cline.LOCAL, Cline.PROMPT)
cl.register("msub", Cline.GLOBAL, Cline.PROMPT)
cl.register("iset", Cline.GLOBAL, Cline.PROMPT)
cl.register("ilo", Cline.GLOBAL, Cline.PROMPT)
cl.register("ihi", Cline.GLOBAL, Cline.PROMPT)
cl.register("plo", Cline.GLOBAL, Cline.PROMPT)
cl.register("phi", Cline.LOCAL, Cline.PROMPT)
cl.register("xlo", Cline.GLOBAL, Cline.PROMPT)
cl.register("xhi", Cline.GLOBAL, Cline.PROMPT)
cl.register("ylo", Cline.GLOBAL, Cline.PROMPT)
cl.register("yhi", Cline.GLOBAL, Cline.PROMPT)
# get inputs
source = cl.get_value(
"source",
"data source [hs, hl, us, ul, hf]",
"hl",
lvals=("hs", "hl", "us", "ul", "hf"),
)
# set some flags
server_or_local = source.endswith("s") or source.endswith("l")
if server_or_local:
resource = cl.get_value("run", "run name", "run005")
first = cl.get_value("first", "first frame to reduce", 1, 0)
cl.set_default("last", 0)
last = cl.get_value("last", "last frame to reduce", 0, 0)
if last and last < first:
print("Cannot set last < first unless last == 0")
print("*** reduce aborted")
exit(1)
twait = cl.get_value("twait",
"time to wait for a new frame [secs]", 1.0,
0.0)
tmx = cl.get_value("tmax",
"maximum time to wait for a new frame [secs]",
10.0, 0.0)
# keep the reduce name and log in sync to save errors
root = os.path.basename(resource)
cl.set_default('rfile', cline.Fname(root, hcam.RED))
cl.set_default('log', cline.Fname(root, hcam.LOG, cline.Fname.NEW))
else:
resource = cl.get_value("flist", "file list",
cline.Fname("files.lis", hcam.LIST))
first = 1
last = 0
trim = cl.get_value("trim", "do you want to trim edges of windows?",
True)
if trim:
ncol = cl.get_value("ncol",
"number of columns to trim from windows", 0)
nrow = cl.get_value("nrow", "number of rows to trim from windows",
0)
# the reduce file
rfilen = cl.get_value("rfile", "reduce file",
cline.Fname("reduce.red", hcam.RED))
try:
rfile = Rfile.read(rfilen)
except hcam.HipercamError as err:
# abort on failure to read as there are many ways to get reduce
# files wrong
print(err, file=sys.stderr)
print("*** reduce aborted")
exit(1)
log = cl.get_value(
"log",
"name of log file to store results",
cline.Fname("reduce.log", hcam.LOG, cline.Fname.NEW),
)
tkeep = cl.get_value(
"tkeep",
"number of minute of data to"
" keep in internal buffers (0 for all)",
0.0,
0.0,
)
lplot = cl.get_value("lplot", "do you want to plot light curves?",
True)
implot = cl.get_value("implot", "do you want to plot images?", True)
if implot:
# define the panel grid. first get the labels and maximum
# dimensions
ccdinf = spooler.get_ccd_pars(source, resource)
try:
nxdef = cl.get_default("nx")
except KeyError:
nxdef = 3
if len(ccdinf) > 1:
ccd = cl.get_value("ccd", "CCD(s) to plot [0 for all]", "0")
if ccd == "0":
ccds = list(ccdinf.keys())
else:
ccds = ccd.split()
if len(ccds) > 1:
nxdef = min(len(ccds), nxdef)
cl.set_default("nx", nxdef)
nx = cl.get_value("nx", "number of panels in X", 3, 1)
else:
nx = 1
else:
nx = 1
ccds = list(ccdinf.keys())
# define the display intensities
msub = cl.get_value("msub", "subtract median from each window?",
True)
iset = cl.get_value(
"iset",
"set intensity a(utomatically),"
" d(irectly) or with p(ercentiles)?",
"a",
lvals=["a", "d", "p"],
)
plo, phi = 5, 95
ilo, ihi = 0, 1000
if iset == "d":
ilo = cl.get_value("ilo", "lower intensity limit", 0.0)
ihi = cl.get_value("ihi", "upper intensity limit", 1000.0)
elif iset == "p":
plo = cl.get_value("plo", "lower intensity limit percentile",
5.0, 0.0, 100.0)
phi = cl.get_value("phi", "upper intensity limit percentile",
95.0, 0.0, 100.0)
# region to plot
for i, cnam in enumerate(ccds):
nxtot, nytot, nxpad, nypad = ccdinf[cnam]
if i == 0:
xmin, xmax = float(-nxpad), float(nxtot + nxpad + 1)
ymin, ymax = float(-nypad), float(nytot + nypad + 1)
else:
xmin = min(xmin, float(-nxpad))
xmax = max(xmax, float(nxtot + nxpad + 1))
ymin = min(ymin, float(-nypad))
ymax = max(ymax, float(nytot + nypad + 1))
xlo = cl.get_value("xlo", "left-hand X value", xmin, xmin, xmax)
xhi = cl.get_value("xhi", "right-hand X value", xmax, xmin, xmax)
ylo = cl.get_value("ylo", "lower Y value", ymin, ymin, ymax)
yhi = cl.get_value("yhi", "upper Y value", ymax, ymin, ymax)
else:
ccds, nx, msub, iset = None, None, None, None
ilo, ihi, plo, phi = None, None, None, None
xlo, xhi, ylo, yhi = None, None, None, None
# save list of parameter values for writing to the reduction file
plist = cl.list()
################################################################
#
# all the inputs have now been obtained. Get on with doing stuff
if implot:
plot_lims = (xlo, xhi, ylo, yhi)
else:
plot_lims = None
imdev, lcdev, spanel, tpanel, xpanel, ypanel, lpanel = setup_plots(
rfile, ccds, nx, plot_lims, implot, lplot)
# a couple of initialisations
total_time = 0 # time waiting for new frame
if lplot:
lbuffer, xbuffer, ybuffer, tbuffer, sbuffer = setup_plot_buffers(rfile)
else:
lbuffer, xbuffer, ybuffer, tbuffer, sbuffer = None, None, None, None, None
############################################
#
# open the log file and write headers
#
with LogWriter(log, rfile, hipercam_version, plist) as logfile:
ncpu = rfile["general"]["ncpu"]
if ncpu > 1:
pool = multiprocessing.Pool(processes=ncpu)
else:
pool = None
# whether some parameters have been initialised
initialised = False
# containers for the processed and raw MCCD groups
# and their frame numbers
pccds, mccds, nframes = [], [], []
##############################################
#
# Finally, start winding through the frames
#
with spooler.data_source(source, resource, first, full=False) as spool:
# 'spool' is an iterable source of MCCDs
for nf, mccd in enumerate(spool):
if server_or_local:
# Handle the waiting game ...
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmx, total_time)
if give_up:
# Giving up, but need to handle any partially filled
# frame group
if len(mccds):
# finish processing remaining frames. This step
# will only occur if we have at least once passed
# to later stages during which read and gain will
# be set up
results = processor(pccds, mccds, nframes)
# write out results to the log file
alerts = logfile.write_results(results)
# print out any accumulated alert messages
if len(alerts):
print("\n".join(alerts))
update_plots(
results,
rfile,
implot,
lplot,
imdev,
lcdev,
pccd,
ccds,
msub,
nx,
iset,
plo,
phi,
ilo,
ihi,
xlo,
xhi,
ylo,
yhi,
lpanel,
xpanel,
ypanel,
tpanel,
spanel,
tkeep,
lbuffer,
xbuffer,
ybuffer,
tbuffer,
sbuffer,
)
mccds = []
print("reduce finished")
break
elif try_again:
continue
# Trim the frames: ULTRACAM windowed data has bad
# columns and rows on the sides of windows closest to
# the readout which can badly affect reduction. This
# option strips them.
if trim:
hcam.ccd.trim_ultracam(mccd, ncol, nrow)
# indicate progress
if "NFRAME" in mccd.head:
nframe = mccd.head["NFRAME"]
else:
nframe = nf + 1
if source != "hf" and last and nframe > last:
# finite last frame number
if len(mccds):
# finish processing remaining frames
results = processor(pccds, mccds, nframes)
# write out results to the log file
alerts = logfile.write_results(results)
# print out any accumulated alert messages
if len(alerts):
print("\n".join(alerts))
update_plots(
results,
rfile,
implot,
lplot,
imdev,
lcdev,
pccd,
ccds,
msub,
nx,
iset,
plo,
phi,
ilo,
ihi,
xlo,
xhi,
ylo,
yhi,
lpanel,
xpanel,
ypanel,
tpanel,
spanel,
tkeep,
lbuffer,
xbuffer,
ybuffer,
tbuffer,
sbuffer,
)
mccds = []
print("\nHave reduced up to the last frame set.")
print("reduce finished")
break
print(
"Frame {:d}: {:s} ({:s})".format(
nframe,
mccd.head["TIMSTAMP"],
"ok" if mccd.head.get("GOODTIME", True) else "nok",
),
end="" if implot else "\n",
)
if not initialised:
# This is the first frame which allows us to make
# some checks and initialisations.
read, gain, ok = initial_checks(mccd, rfile)
# Define the CCD processor function object
processor = ProcessCCDs(rfile, read, gain, ccdproc, pool)
# set flag to show we are set
if not ok:
break
initialised = True
# De-bias the data. Retain a copy of the raw data as 'mccd'
# in order to judge saturation. Processed data called 'pccd'
if rfile.bias is not None:
# subtract bias
pccd = mccd - rfile.bias
bexpose = rfile.bias.head.get("EXPTIME", 0.0)
else:
# no bias subtraction
pccd = mccd.copy()
bexpose = 0.0
if rfile.dark is not None:
# subtract dark, CCD by CCD
dexpose = rfile.dark.head["EXPTIME"]
for cnam in pccd:
ccd = pccd[cnam]
cexpose = ccd.head["EXPTIME"]
scale = (cexpose - bexpose) / dexpose
ccd -= scale * rfile.dark[cnam]
if rfile.flat is not None:
# apply flat field to processed frame
pccd /= rfile.flat
if rfile["focal_mask"]["demask"]:
# attempt to correct for poorly placed frame
# transfer mask causing a step illumination in the
# y-direction. Loop through all windows of all
# CCDs. Also include a stage where we average in
# the Y direction to try to eliminate high pixels.
dthresh = rfile["focal_mask"]["dthresh"]
for cnam, ccd in pccd.items():
for wnam, wind in ccd.items():
# form mean in Y direction, then try to
# mask out high pixels
ymean = np.mean(wind.data, 0)
xmask = ymean == ymean
while 1:
# rejection cycle, rejecting
# overly positive pixels
ave = ymean[xmask].mean()
rms = ymean[xmask].std()
diff = ymean - ave
diff[~xmask] = 0
imax = np.argmax(diff)
if diff[imax] > dthresh * rms:
xmask[imax] = False
else:
break
# form median in X direction
xmedian = np.median(wind.data[:, xmask], 1)
# subtract it's median to avoid removing
# general background
xmedian -= np.median(xmedian)
# now subtract from 2D image using
# broadcasting rules
wind.data -= xmedian.reshape((len(xmedian), 1))
# Acummulate frames into processing groups for faster
# parallelisation
pccds.append(pccd)
mccds.append(mccd)
nframes.append(nframe)
if len(pccds) == rfile["general"]["ngroup"]:
# parallel processing. This should usually be the first
# points at which it takes place
results = processor(pccds, mccds, nframes)
# write out results to the log file
alerts = logfile.write_results(results)
# print out any accumulated alert messages
if len(alerts):
print("\n".join(alerts))
update_plots(
results,
rfile,
implot,
lplot,
imdev,
lcdev,
pccds[-1],
ccds,
msub,
nx,
iset,
plo,
phi,
ilo,
ihi,
xlo,
xhi,
ylo,
yhi,
lpanel,
xpanel,
ypanel,
tpanel,
spanel,
tkeep,
lbuffer,
xbuffer,
ybuffer,
tbuffer,
sbuffer,
)
# Reset the frame buffers
pccds, mccds, nframes = [], [], []
if len(mccds):
# out of loop now. Finish processing any remaining
# frames.
results = processor(pccds, mccds, nframes)
# write out results to the log file
alerts = logfile.write_results(results)
# print out any accumulated alert messages
if len(alerts):
print("\n".join(alerts))
update_plots(
results,
rfile,
implot,
lplot,
imdev,
lcdev,
pccd,
ccds,
msub,
nx,
iset,
plo,
phi,
ilo,
ihi,
xlo,
xhi,
ylo,
yhi,
lpanel,
xpanel,
ypanel,
tpanel,
spanel,
tkeep,
lbuffer,
xbuffer,
ybuffer,
tbuffer,
sbuffer,
)
print("reduce finished")
def grab(args=None):
"""``grab [source] run [temp] (ndigit) first last (trim [ncol nrow])
[twait tmax] bias [dtype]``
This downloads a sequence of images from a raw data file and writes them
out to a series CCD / MCCD files.
Parameters:
source : string [hidden]
Data source, four options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : ULTRACAM server
| 'ul' : local ULTRACAM .xml/.dat files
run : string
run name to access
temp : bool [hidden, defaults to False]
True to indicate that the frames should be written to temporary
files with automatically-generated names in the expectation of
deleting them later. This also writes out a file listing the names.
The aim of this is to allow grab to be used as a feeder for other
routines in larger scripts. If temp == True, grab will return with
the name of the list of hcm files. Look at 'makebias' for an
example that uses this feature.
ndigit : int [if not temp]
Files created will be written as 'run005_0013.fits' etc. `ndigit`
is the number of digits used for the frame number (4 in this
case). Any pre-existing files of the same name will be
over-written.
first : int
First frame to access
last : int
Last frame to access, 0 for the lot
trim : bool [if source starts with 'u']
True to trim columns and/or rows off the edges of windows nearest
the readout. This is particularly for ULTRACAM windowed data where
the first few rows and columns can contain bad data.
ncol : int [if trim]
Number of columns to remove (on left of left-hand window, and right
of right-hand windows)
nrow : int [if trim]
Number of rows to remove (bottom of windows)
twait : float [hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
bias : string
Name of bias frame to subtract, 'none' to ignore.
dtype : string [hidden, defaults to 'f32']
Data type on output. Options:
| 'f32' : output as 32-bit floats (default)
| 'f64' : output as 64-bit floats.
| 'u16' : output as 16-bit unsigned integers. A warning will be
issued if loss of precision occurs; an exception will
be raised if the data are outside the range 0 to 65535.
"""
command, args = utils.script_args(args)
# get inputs
with Cline('HIPERCAM_ENV', '.hipercam', command, args) as cl:
# register parameters
cl.register('source', Cline.GLOBAL, Cline.HIDE)
cl.register('run', Cline.GLOBAL, Cline.PROMPT)
cl.register('temp', Cline.GLOBAL, Cline.HIDE)
cl.register('ndigit', Cline.LOCAL, Cline.PROMPT)
cl.register('first', Cline.LOCAL, Cline.PROMPT)
cl.register('last', Cline.LOCAL, Cline.PROMPT)
cl.register('trim', Cline.GLOBAL, Cline.PROMPT)
cl.register('ncol', Cline.GLOBAL, Cline.HIDE)
cl.register('nrow', Cline.GLOBAL, Cline.HIDE)
cl.register('twait', Cline.LOCAL, Cline.HIDE)
cl.register('tmax', Cline.LOCAL, Cline.HIDE)
cl.register('bias', Cline.GLOBAL, Cline.PROMPT)
cl.register('dtype', Cline.LOCAL, Cline.HIDE)
# get inputs
source = cl.get_value('source',
'data source [hs, hl, us, ul]',
'hl',
lvals=('hs', 'hl', 'us', 'ul'))
# OK, more inputs
resource = cl.get_value('run', 'run name', 'run005')
cl.set_default('temp', False)
temp = cl.get_value(
'temp', 'save to temporary automatically-generated file names?',
True)
if not temp:
ndigit = cl.get_value('ndigit',
'number of digits in frame identifier', 3, 0)
first = cl.get_value('first', 'first frame to grab', 1, 0)
last = cl.get_value('last', 'last frame to grab', 0)
if last < first and last != 0:
sys.stderr.write('last must be >= first or 0')
sys.exit(1)
if source.startswith('u'):
trim = cl.get_value(
'trim',
'do you want to trim edges of windows? (ULTRACAM only)', True)
if trim:
ncol = cl.get_value('ncol',
'number of columns to trim from windows',
0)
nrow = cl.get_value('nrow',
'number of rows to trim from windows', 0)
else:
trim = False
twait = cl.get_value('twait', 'time to wait for a new frame [secs]',
1., 0.)
tmax = cl.get_value('tmax',
'maximum time to wait for a new frame [secs]', 10.,
0.)
bias = cl.get_value('bias',
"bias frame ['none' to ignore]",
cline.Fname('bias', hcam.HCAM),
ignore='none')
cl.set_default('dtype', 'f32')
dtype = cl.get_value('dtype',
'data type [f32, f64, u16]',
'f32',
lvals=('f32', 'f64', 'u16'))
# Now the actual work.
# strip off extensions
if resource.endswith(hcam.HRAW):
resource = resource[:resource.find(hcam.HRAW)]
# initialisations
total_time = 0 # time waiting for new frame
nframe = first
root = os.path.basename(resource)
bframe = None
# Finally, we can go
if temp:
# create a directory on temp for the temporary file to avoid polluting
# it too much
fnames = []
tdir = os.path.join(tempfile.gettempdir(),
'hipercam-{:s}'.format(getpass.getuser()))
os.makedirs(tdir, exist_ok=True)
with spooler.data_source(source, resource, first) as spool:
try:
for mccd in spool:
# Handle the waiting game ...
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmax, total_time)
if give_up:
print('grab stopped')
break
elif try_again:
continue
# Trim the frames: ULTRACAM windowed data has bad
# columns and rows on the sides of windows closest to
# the readout which can badly affect reduction. This
# option strips them.
if trim:
hcam.ccd.trim_ultracam(mccd, ncol, nrow)
if bias is not None:
# read bias after first frame so we can
# chop the format
if bframe is None:
# read the bias frame
bframe = hcam.MCCD.read(bias)
# reformat
bframe = bframe.crop(mccd)
mccd -= bframe
if dtype == 'u16':
mccd.uint16()
elif dtype == 'f32':
mccd.float32()
elif dtype == 'f64':
mccd.float64()
# write to disk
if temp:
# generate name automatically
fd, fname = tempfile.mkstemp(suffix=hcam.HCAM, dir=tdir)
mccd.write(fname, True)
os.close(fd)
fnames.append(fname)
else:
fname = '{:s}_{:0{:d}}{:s}'.format(root, nframe, ndigit,
hcam.HCAM)
mccd.write(fname, True)
print('Written frame {:d} to {:s}'.format(nframe, fname))
# update the frame number
nframe += 1
if last and nframe > last:
break
except KeyboardInterrupt:
# trap ctrl-C so we can delete temporary files if temp
if temp:
for fname in fnames:
os.remove(fname)
print('\ntemporary files deleted')
print('grab aborted')
else:
print('\ngrab aborted')
sys.exit(1)
if temp:
if len(fnames) == 0:
raise hcam.HipercamError(
'no files were grabbed; please check input parameters, especially "first"'
)
# write the file names to a list
fd, fname = tempfile.mkstemp(suffix=hcam.LIST, dir=tdir)
with open(fname, 'w') as fout:
for fnam in fnames:
fout.write(fnam + '\n')
os.close(fd)
print('temporary file names written to {:s}'.format(fname))
# return the name of the file list
return fname
def ftargets(args=None):
"""``ftargets [source device width height] (run first [twait tmax] |
flist) trim ([ncol nrow]) (ccd (nx)) [pause] thresh fwhm minpix
output bias flat msub iset (ilo ihi | plo phi) xlo xhi ylo yhi``
This script carries out the following steps for each of a series
of images:
(1) detects the sources,
(2) identifies isolated targets suited to profile fits,
(3) fits 2D Moffat profiles to these,
(4) Saves results to disk.
The profile fits are carried out because `sep` does not return
anything that can be used reliably for a FWHM.
Several parameters depends on the object detection threshold
retuned by the source detection. This is referred to as
`threshold`. The source detection is carried out using `sep` which
runs according to the usual source extractor algorithm of Bertin.
The script plots the frames, with ellipses at 3*a,3*b indicated in
red, green boxes indicating the range of pixels identified by
`sep`, and blue boxes marking the targets selected for FWHM
fitting (the boxes indicate the fit region).
Parameters:
source : string [hidden]
Data source, five options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : ULTRACAM server
| 'ul' : local ULTRACAM .xml/.dat files
| 'hf' : list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab' or
converted from foreign data formats.
device : string [hidden]
Plot device. PGPLOT is used so this should be a PGPLOT-style name,
e.g. '/xs', '1/xs' etc. At the moment only ones ending /xs are
supported.
width : float [hidden]
plot width (inches). Set = 0 to let the program choose.
height : float [hidden]
plot height (inches). Set = 0 to let the program choose. BOTH width
AND height must be non-zero to have any effect
run : string [if source ends 's' or 'l']
run number to access, e.g. 'run034'
flist : string [if source ends 'f']
name of file list
first : int [if source ends 's' or 'l']
exposure number to start from. 1 = first frame; set = 0 to always
try to get the most recent frame (if it has changed). For data
from the |hiper| server, a negative number tries to get a frame not
quite at the end. i.e. -10 will try to get 10 from the last
frame. This is mainly to sidestep a difficult bug with the
acquisition system.
twait : float [if source ends 's' or 'l'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [if source ends 's' or 'l'; hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
trim : bool [if source starts with 'u']
True to trim columns and/or rows off the edges of windows nearest
the readout which can sometimes contain bad data.
ncol : int [if trim, hidden]
Number of columns to remove (on left of left-hand window, and right
of right-hand windows)
nrow : int [if trim, hidden]
Number of rows to remove (bottom of windows)
ccd : string
CCD(s) to plot, '0' for all, '1 3' to plot '1' and '3' only, etc.
nx : int [if more than 1 CCD]
number of panels across to display.
pause : float [hidden]
seconds to pause between frames (defaults to 0)
thresh : float
threshold (mutiple of RMS) to use for object detection. Typical
values 2.5 to 4. The higher it is, the fewer objects will be located,
but the fewer false detections will be made.
fwhm : float
FWHM to use for smoothing during object detection. Should be
comparable to the seeing.
minpix : int
Minimum number of pixels above threshold before convolution to count
as a detection. Useful in getting rid of cosmics and high dark count
pixels.
rmin : float
Closest distance of any other detected object for an attempt
to be made to fit the FWHM of an object [unbinned pixels].
pmin : float
Minimum peak height for an attempt to be made to fit the
FWHM of an object. This should be a multiple of the object
detection threshold (returned by `sep` for each object).
pmax : float
Maximum peak height for an attempt to be made to fit the
FWHM of an object. Use to exclude saturated targets
[counts]
emax : float
Maximum elongation (major/minor axis ratio = a/b), > 1. Use
to reduce very non-stellar profiles.
nmax : int
Maximum number of FWHMs to measure. Will take the brightest first,
judging by the flux.
bias : string
Name of bias frame to subtract, 'none' to ignore.
flat : string
Name of flat field to divide by, 'none' to ignore. Should normally
only be used in conjunction with a bias, although it does allow you
to specify a flat even if you haven't specified a bias.
output: string
Name of file for storage of results. Will be a fits file, with
results saved to the HDU 1 as a table.
iset : string [single character]
determines how the intensities are determined. There are three
options: 'a' for automatic simply scales from the minimum to the
maximum value found on a per CCD basis. 'd' for direct just takes
two numbers from the user. 'p' for percentile dtermines levels
based upon percentiles determined from the entire CCD on a per CCD
basis.
ilo : float [if iset='d']
lower intensity level
ihi : float [if iset='d']
upper intensity level
plo : float [if iset='p']
lower percentile level
phi : float [if iset='p']
upper percentile level
xlo : float
left-hand X-limit for plot
xhi : float
right-hand X-limit for plot (can actually be < xlo)
ylo : float
lower Y-limit for plot
yhi : float
upper Y-limit for plot (can be < ylo)
"""
command, args = utils.script_args(args)
# get the inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("source", Cline.GLOBAL, Cline.HIDE)
cl.register("device", Cline.LOCAL, Cline.HIDE)
cl.register("width", Cline.LOCAL, Cline.HIDE)
cl.register("height", Cline.LOCAL, Cline.HIDE)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("trim", Cline.GLOBAL, Cline.PROMPT)
cl.register("ncol", Cline.GLOBAL, Cline.HIDE)
cl.register("nrow", Cline.GLOBAL, Cline.HIDE)
cl.register("twait", Cline.LOCAL, Cline.HIDE)
cl.register("tmax", Cline.LOCAL, Cline.HIDE)
cl.register("flist", Cline.LOCAL, Cline.PROMPT)
cl.register("ccd", Cline.LOCAL, Cline.PROMPT)
cl.register("nx", Cline.LOCAL, Cline.PROMPT)
cl.register("pause", Cline.LOCAL, Cline.HIDE)
cl.register("thresh", Cline.LOCAL, Cline.PROMPT)
cl.register("fwhm", Cline.LOCAL, Cline.PROMPT)
cl.register("minpix", Cline.LOCAL, Cline.PROMPT)
cl.register("rmin", Cline.LOCAL, Cline.PROMPT)
cl.register("pmin", Cline.LOCAL, Cline.PROMPT)
cl.register("pmax", Cline.LOCAL, Cline.PROMPT)
cl.register("emax", Cline.LOCAL, Cline.PROMPT)
cl.register("nmax", Cline.LOCAL, Cline.PROMPT)
cl.register("gain", Cline.LOCAL, Cline.PROMPT)
cl.register("rej", Cline.LOCAL, Cline.PROMPT)
cl.register("bias", Cline.GLOBAL, Cline.PROMPT)
cl.register("flat", Cline.GLOBAL, Cline.PROMPT)
cl.register("output", Cline.LOCAL, Cline.PROMPT)
cl.register("iset", Cline.GLOBAL, Cline.PROMPT)
cl.register("ilo", Cline.GLOBAL, Cline.PROMPT)
cl.register("ihi", Cline.GLOBAL, Cline.PROMPT)
cl.register("plo", Cline.GLOBAL, Cline.PROMPT)
cl.register("phi", Cline.LOCAL, Cline.PROMPT)
cl.register("xlo", Cline.GLOBAL, Cline.PROMPT)
cl.register("xhi", Cline.GLOBAL, Cline.PROMPT)
cl.register("ylo", Cline.GLOBAL, Cline.PROMPT)
cl.register("yhi", Cline.GLOBAL, Cline.PROMPT)
# get inputs
source = cl.get_value(
"source",
"data source [hs, hl, us, ul, hf]",
"hl",
lvals=("hs", "hl", "us", "ul", "hf"),
)
# set some flags
server_or_local = source.endswith("s") or source.endswith("l")
# plot device stuff
device = cl.get_value("device", "plot device", "1/xs")
width = cl.get_value("width", "plot width (inches)", 0.0)
height = cl.get_value("height", "plot height (inches)", 0.0)
if server_or_local:
resource = cl.get_value("run", "run name", "run005")
if source == "hs":
first = cl.get_value("first", "first frame to plot", 1)
else:
first = cl.get_value("first", "first frame to plot", 1, 0)
twait = cl.get_value(
"twait", "time to wait for a new frame [secs]", 1.0, 0.0
)
tmax = cl.get_value(
"tmax", "maximum time to wait for a new frame [secs]", 10.0, 0.0
)
else:
resource = cl.get_value(
"flist", "file list", cline.Fname("files.lis", hcam.LIST)
)
first = 1
trim = cl.get_value("trim", "do you want to trim edges of windows?", True)
if trim:
ncol = cl.get_value("ncol", "number of columns to trim from windows", 0)
nrow = cl.get_value("nrow", "number of rows to trim from windows", 0)
# define the panel grid. first get the labels and maximum dimensions
ccdinf = spooler.get_ccd_pars(source, resource)
try:
nxdef = cl.get_default("nx")
except:
nxdef = 3
if len(ccdinf) > 1:
ccd = cl.get_value("ccd", "CCD(s) to plot [0 for all]", "0")
if ccd == "0":
ccds = list(ccdinf.keys())
else:
ccds = ccd.split()
check = set(ccdinf.keys())
if not set(ccds) <= check:
raise hcam.HipercamError("At least one invalid CCD label supplied")
if len(ccds) > 1:
nxdef = min(len(ccds), nxdef)
cl.set_default("nx", nxdef)
nx = cl.get_value("nx", "number of panels in X", 3, 1)
else:
nx = 1
else:
nx = 1
ccds = list(ccdinf.keys())
cl.set_default("pause", 0.0)
pause = cl.get_value(
"pause", "time delay to add between" " frame plots [secs]", 0.0, 0.0
)
thresh = cl.get_value("thresh", "source detection threshold [RMS]", 3.0)
fwhm = cl.get_value("fwhm", "FWHM for source detection [binned pixels]", 4.0)
minpix = cl.get_value("minpix", "minimum number of pixels above threshold", 3)
rmin = cl.get_value(
"rmin", "nearest neighbour for FWHM fits [unbinned pixels]", 20.0
)
pmin = cl.get_value(
"pmin", "minimum peak value for profile fits [multiple of threshold]", 5.0
)
pmax = cl.get_value(
"pmax", "maximum peak value for profile fits [counts]", 60000.0
)
emax = cl.get_value(
"emax", "maximum elongation (a/b) for profile fits", 1.2, 1.0
)
nmax = cl.get_value("nmax", "maximum number of profile fits", 10, 1)
gain = cl.get_value("gain", "CCD gain [electrons/ADU]", 1.0, 0.0)
rej = cl.get_value(
"rej", "rejection threshold for profile fits [RMS]", 6.0, 2.0
)
# bias frame (if any)
bias = cl.get_value(
"bias",
"bias frame ['none' to ignore]",
cline.Fname("bias", hcam.HCAM),
ignore="none",
)
if bias is not None:
# read the bias frame
bias = hcam.MCCD.read(bias)
fprompt = "flat frame ['none' to ignore]"
else:
fprompt = "flat frame ['none' is normal choice with no bias]"
# flat (if any)
flat = cl.get_value(
"flat", fprompt, cline.Fname("flat", hcam.HCAM), ignore="none"
)
if flat is not None:
# read the flat frame
flat = hcam.MCCD.read(flat)
output = cl.get_value(
"output",
"output file for results",
cline.Fname("sources", hcam.SEP, cline.Fname.NEW),
)
iset = cl.get_value(
"iset",
"set intensity a(utomatically)," " d(irectly) or with p(ercentiles)?",
"a",
lvals=["a", "d", "p"],
)
iset = iset.lower()
plo, phi = 5, 95
ilo, ihi = 0, 1000
if iset == "d":
ilo = cl.get_value("ilo", "lower intensity limit", 0.0)
ihi = cl.get_value("ihi", "upper intensity limit", 1000.0)
elif iset == "p":
plo = cl.get_value(
"plo", "lower intensity limit percentile", 5.0, 0.0, 100.0
)
phi = cl.get_value(
"phi", "upper intensity limit percentile", 95.0, 0.0, 100.0
)
# region to plot
for i, cnam in enumerate(ccds):
nxtot, nytot, nxpad, nypad = ccdinf[cnam]
if i == 0:
xmin, xmax = float(-nxpad), float(nxtot + nxpad + 1)
ymin, ymax = float(-nypad), float(nytot + nypad + 1)
else:
xmin = min(xmin, float(-nxpad))
xmax = max(xmax, float(nxtot + nxpad + 1))
ymin = min(ymin, float(-nypad))
ymax = max(ymax, float(nytot + nypad + 1))
xlo = cl.get_value("xlo", "left-hand X value", xmin, xmin, xmax)
xhi = cl.get_value("xhi", "right-hand X value", xmax, xmin, xmax)
ylo = cl.get_value("ylo", "lower Y value", ymin, ymin, ymax)
yhi = cl.get_value("yhi", "upper Y value", ymax, ymin, ymax)
################################################################
#
# all the inputs have now been obtained. Get on with doing stuff
# open image plot device
imdev = hcam.pgp.Device(device)
if width > 0 and height > 0:
pgpap(width, height / width)
# set up panels and axes
nccd = len(ccds)
ny = nccd // nx if nccd % nx == 0 else nccd // nx + 1
# slice up viewport
pgsubp(nx, ny)
# plot axes, labels, titles. Happens once only
for cnam in ccds:
pgsci(hcam.pgp.Params["axis.ci"])
pgsch(hcam.pgp.Params["axis.number.ch"])
pgenv(xlo, xhi, ylo, yhi, 1, 0)
pglab("X", "Y", "CCD {:s}".format(cnam))
# initialisations. 'last_ok' is used to store the last OK frames of each
# CCD for retrieval when coping with skipped data.
total_time = 0 # time waiting for new frame
nhdu = len(ccds) * [0]
thetas = np.linspace(0, 2 * np.pi, 100)
# values of various parameters
fwhm_min, beta, beta_min, beta_max, readout, max_nfev = (
2.0,
4.0,
1.5,
10.0,
5.5,
100,
)
# number of failed fits
nfail = 0
# open the output file for results
with fitsio.FITS(output, "rw", clobber=True) as fout:
# plot images
with spooler.data_source(source, resource, first, full=False) as spool:
# 'spool' is an iterable source of MCCDs
n = 0
for nf, mccd in enumerate(spool):
if server_or_local:
# Handle the waiting game ...
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmax, total_time
)
if give_up:
print("ftargets stopped")
break
elif try_again:
continue
# Trim the frames: ULTRACAM windowed data has bad columns
# and rows on the sides of windows closest to the readout
# which can badly affect reduction. This option strips
# them.
if trim:
hcam.ccd.trim_ultracam(mccd, ncol, nrow)
# indicate progress
tstamp = Time(mccd.head["TIMSTAMP"], format="isot", precision=3)
print(
"{:d}, utc= {:s} ({:s}), ".format(
mccd.head["NFRAME"],
tstamp.iso,
"ok" if mccd.head.get("GOODTIME", True) else "nok",
),
end="",
)
# accumulate errors
emessages = []
if n == 0:
if bias is not None:
# crop the bias on the first frame only
bias = bias.crop(mccd)
if flat is not None:
# crop the flat on the first frame only
flat = flat.crop(mccd)
# compute maximum length of window name strings
lsmax = 0
for ccd in mccd.values():
for wnam in ccd:
lsmax = max(lsmax, len(wnam))
# display the CCDs chosen
message = ""
pgbbuf()
for nc, cnam in enumerate(ccds):
ccd = mccd[cnam]
if ccd.is_data():
# this should be data as opposed to a blank frame
# between data frames that occur with nskip > 0
# subtract the bias
if bias is not None:
ccd -= bias[cnam]
# divide out the flat
if flat is not None:
ccd /= flat[cnam]
# Where the fancy stuff happens ...
# estimate sky background, look for stars
objs, dofwhms = [], []
nobj = 0
for wnam in ccd:
try:
# chop window, find objects
wind = ccd[wnam].window(xlo, xhi, ylo, yhi)
wind.data = wind.data.astype("float")
objects, bkg = findStars(wind, thresh, fwhm, True)
# subtract background from frame for display
# purposes.
bkg.subfrom(wind.data)
ccd[wnam] = wind
# remove targets with too few pixels
objects = objects[objects["tnpix"] >= minpix]
# run nearest neighbour search on all
# objects, but select only a subset
# with right count levels for FWHM
# measurement, and which are not too
# elongated
results = isolated(objects["x"], objects["y"], rmin)
peaks = objects["peak"]
ok = (
(peaks < pmax)
& (peaks > pmin * objects["thresh"])
& (objects["a"] < emax * objects["b"])
)
dfwhms = np.array(
[
i
for i in range(len(results))
if ok[i] and len(results[i]) == 1
], dtype=int
)
# pick the brightest. 'dfwhms' are the
# indices of the selected targets for
# FWHM measurement
if len(dfwhms):
fluxes = objects["flux"][dfwhms]
isort = np.argsort(fluxes)[::-1]
dfwhms = dfwhms[isort[:nmax]]
# buffer for storing the FWHMs, including NaNs
# for the ones thet are skipped
fwhms = np.zeros_like(peaks, dtype=np.float32)
betas = np.zeros_like(peaks, dtype=np.float32)
nfevs = np.zeros_like(peaks, dtype=np.int32)
for i, (x, y, peak, fwhm) in enumerate(
zip(
objects["x"],
objects["y"],
peaks,
objects["fwhm"],
)
):
# fit FWHMs of selected targets
if i in dfwhms:
try:
# extract fit Window
fwind = wind.window(
x - rmin, x + rmin, y - rmin, y + rmin
)
# fit profile
ofwhm = fwhm
obeta = beta
(
(sky, height, x, y, fwhm, beta),
epars,
(
wfit,
X,
Y,
sigma,
chisq,
nok,
nrej,
npar,
nfev,
),
) = hcam.fitting.fitMoffat(
fwind,
None,
peak,
x,
y,
fwhm,
2.0,
False,
beta,
beta_max,
False,
readout,
gain,
rej,
1,
max_nfev,
)
fwhms[i] = fwhm
betas[i] = beta
nfevs[i] = nfev
# keep value of beta for next round under control
beta = min(beta_max, max(beta_min, beta))
except hcam.HipercamError as err:
emessages.append(
" >> Targ {:d}: fit failed ***: {!s}".format(
i, err
)
)
fwhms[i] = np.nan
betas[i] = np.nan
nfevs[i] = 0
nfail += 1
else:
# skip this one
fwhms[i] = np.nan
betas[i] = np.nan
nfevs[i] = 0
# tack on frame number & window name
frames = (nf + first) * np.ones(
len(objects), dtype=np.int32
)
wnams = np.array(
len(objects) * [wnam], dtype="U{:d}".format(lsmax)
)
objects = append_fields(
objects,
("ffwhm", "beta", "nfev", "nframe", "wnam"),
(fwhms, betas, nfevs, frames, wnams),
)
# save the objects and the
objs.append(objects)
dofwhms.append(dfwhms + nobj)
print(dfwhms,nobj,dofwhms)
nobj += len(objects)
except hcam.HipercamError:
# window may have no overlap with xlo, xhi
# ylo, yhi
pass
if len(objs):
# Plot targets found
# concatenate results of all Windows
objs = np.concatenate(objs)
dofwhms = np.concatenate(dofwhms)
# set to the correct panel and then plot CCD
ix = (nc % nx) + 1
iy = nc // nx + 1
pgpanl(ix, iy)
vmin, vmax = hcam.pgp.pCcd(
ccd,
iset,
plo,
phi,
ilo,
ihi,
xlo=xlo,
xhi=xhi,
ylo=ylo,
yhi=yhi,
)
pgsci(core.CNAMS["red"])
As, Bs, Thetas, Xs, Ys = (
objs["a"],
objs["b"],
objs["theta"],
objs["x"],
objs["y"],
)
for a, b, theta0, x, y in zip(As, Bs, Thetas, Xs, Ys):
xs = x + 3 * a * np.cos(thetas + theta0)
ys = y + 3 * b * np.sin(thetas + theta0)
pgline(xs, ys)
pgsci(core.CNAMS["green"])
for xmin, xmax, ymin, ymax in zip(
objs["xmin"], objs["xmax"], objs["ymin"], objs["ymax"]
):
xs = [xmin, xmax, xmax, xmin, xmin]
ys = [ymin, ymin, ymax, ymax, ymin]
pgline(xs, ys)
pgsci(core.CNAMS["blue"])
if len(dofwhms):
print(dofwhms)
for x, y in zip(objs["x"][dofwhms], objs["y"][dofwhms]):
xs = [x - rmin, x + rmin, x + rmin, x - rmin, x - rmin]
ys = [y - rmin, y - rmin, y + rmin, y + rmin, y - rmin]
pgline(xs, ys)
# remove some less useful fields to save a
# bit more space prior to saving to disk
objs = remove_field_names(
objs,
(
"x2",
"y2",
"xy",
"cxx",
"cxy",
"cyy",
"hfd",
"cflux",
"cpeak",
"xcpeak",
"ycpeak",
"xmin",
"xmax",
"ymin",
"ymax",
),
)
# save to disk
if nhdu[nc]:
fout[nhdu[nc]].append(objs)
else:
fout.write(objs)
nhdu[nc] = nc + 1
# accumulate string of image scalings
if nc:
message += ", ccd {:s}: {:.1f}, {:.1f}, exp: {:.4f}".format(
cnam, vmin, vmax, mccd.head["EXPTIME"]
)
else:
message += "ccd {:s}: {:.1f}, {:.1f}, exp: {:.4f}".format(
cnam, vmin, vmax, mccd.head["EXPTIME"]
)
pgebuf()
# end of CCD display loop
print(message)
for emessage in emessages:
print(emessage)
if pause > 0.0:
# pause between frames
time.sleep(pause)
# update the frame number
n += 1
print("Number of failed fits =", nfail)
def ltimes(args=None):
"""``ltimes [source] run first last [twait tmax tdigit edigit]``
Lists timing information from a run in machine-readable space-separated
column style. Integer status flags are used, 1=OK, 0=not OK.
For HiPERCAM, the following items are listed: (1) the frame
number, (2) the raw GPS timestamp as an MJD, (3) the raw GPS
timestamp as a UTC string, (4) a status flag, 0 or 1, then for
each CCD: (i) the CCD number [1-5], (ii) the mid exposure time as
an MJD, (iii) the mid-exposure time as a UTC string, (iv) the
exposure length (seconds), (v) a status flag (0 or 1), determined
by the NSKIP parameters for each CCD.
For ULTRACAM, the following items are listed: (1) the frame
number, (2) the raw GPS timestamp as an MJD, (3) the raw GPS
timestamp as a UTC string, (4) the red & green mid exposure time
as an MJD, (5) the red & green mid-exposure time as a UTC string,
(6) a status flag to indicate whether the mid-exposure time is
considered to be good, (7) the exposure time in seconds, (8) the
blue mid-exposure time as an MJD, (9) the blue mid-exposure time
as a UTC string, (10) status flag of the blue data, which reflects
the "nblue" option where only some of the blue frames are valid
data.
For ULTRASPEC, the following items are listed: (1) the frame
number, (2) the raw GPS timestamp as an MJD, (3) the raw GPS
timestamp as a UTC string, (4) the mid-exposure time as an MJD,
(5) the mid-exposure time as a UTC string, (6) a status flag to
indicate whether the mid-exposure time is considered to be good,
(7) the exposure time in seconds.
Parameters:
source : string [hidden]
Data source, two options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : local ULTRA(CAM|SPEC) file
| 'ul' : ULTRA(CAM|SPEC) server
run : string
run number to access, e.g. 'run0034'
first : int
exposure number to start from. 1 = first frame; set = 0 to
always try to get the most recent frame (if it changes)
last : int
last exposure number, 0 for all.
twait : float [hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [hidden]
maximum time to wait between attempts to find a new exposure,
seconds. Set to 0 to avoid waiting and extra output messages.
tdigit : int [hidden]
number of digits of precision for the seconds after the decimal
point when reporting the times.
edigit : int [hidden]
number of digits of precision after the decimal point when
reporting the exposure times
.. Warning::
This routine does not yet work with ULTRACAM data.
"""
command, args = utils.script_args(args)
# get the inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("source", Cline.GLOBAL, Cline.HIDE)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("last", Cline.LOCAL, Cline.PROMPT)
cl.register("twait", Cline.LOCAL, Cline.HIDE)
cl.register("tmax", Cline.LOCAL, Cline.HIDE)
cl.register("tdigit", Cline.LOCAL, Cline.HIDE)
cl.register("edigit", Cline.LOCAL, Cline.HIDE)
# get inputs
source = cl.get_value(
"source",
"data source [hs, hl, us, ul]",
"hl",
lvals=("hs", "hl", "us", "ul"),
)
resource = cl.get_value("run", "run name", "run005")
if source == "hs":
first = cl.get_value("first", "first frame to list", 1)
else:
first = cl.get_value("first", "first frame to list", 1, 0)
last = cl.get_value("last", "last frame to list", 0, 0)
if last < first and last != 0:
sys.stderr.write("last must be >= first or 0")
sys.exit(1)
twait = cl.get_value("twait", "time to wait for a new frame [secs]",
1.0, 0.0)
tmax = cl.get_value("tmax",
"maximum time to wait for a new frame [secs]",
10.0, 0.0)
tdigit = cl.get_value("tdigit", "digits after decimal point for times",
6, 1, 9)
edigit = cl.get_value("edigit",
"digits after decimal point for exposure times",
3, 1, 9)
################################################################
#
# all the inputs have now been obtained. Get on with doing stuff
# open the run as an Rtime with exact type depending on the source
if source.startswith("h"):
rtime = hcam.hcam.Rtime(resource, first, source.endswith("s"))
else:
rtime = hcam.ucam.Rtime(resource, first, source.endswith("s"))
total_time = 0
nframe = first
for tdata in rtime:
# Handle the waiting game ...
give_up, try_again, total_time = spooler.hang_about(
tdata, twait, tmax, total_time)
if give_up:
if tmax > 0:
print("times stopped")
break
elif try_again:
continue
# indicate progress
if len(tdata) == 3:
# HiPERCAM
tstamp, tinfo, tflag = tdata
tstamp.precision = tdigit
print(
"{:d} {:.12f} {:s} {:d}".format(nframe, tstamp.mjd, tstamp.iso,
1 if tflag else 0),
end="",
)
message = ""
for nccd, (mjd, exptime, flag) in enumerate(tinfo):
ts = Time(mjd, format="mjd", precision=tdigit)
message += " {:d} {:.12f} {:s} {:.{:d}f} {:d}".format(
nccd + 1, mjd, ts.hms_custom, exptime, edigit,
1 if flag else 0)
print(message)
elif len(tdata) == 2:
# ULTRASPEC
time, tinfo = tdata
ts = Time(time.mjd, format="mjd", precision=tdigit)
gps = Time(tinfo["gps"], format="mjd", precision=tdigit)
print("{:d}, {:.12f} {:s} {:.12f} {:s} {:d} {:.5f}".format(
nframe,
tinfo["gps"],
gps.hms_custom,
time.mjd,
ts.hms_custom,
1 if time.good else 0,
time.expose,
))
elif len(tdata) == 4:
# ULTRACAM
time, tinfo, btime, bbad = tdata
ts = Time(time.mjd, format="mjd", precision=tdigit)
gps = Time(tinfo["gps"], format="mjd", precision=tdigit)
bts = Time(btime.mjd, format="mjd", precision=tdigit)
print(
"{:d} {:.12f} {:s} {:.12f} {:s} {:d} {:.5f} {:.12f} {:s} {:d}".
format(
nframe,
tinfo["gps"],
gps.hms_custom,
time.mjd,
ts.hms_custom,
1 if time.good else 0,
time.expose,
btime.mjd,
bts.hms_custom,
0 if bbad else 1,
))
if last > 0 and nframe == last:
break
# increment the frame
nframe += 1
def times(args=None):
"""``times [source] run first last [twait tmax tdigit edigit]``
Lists timing information from a run. For each frame the associated GPS
timestamp is listed (UTC, date then HMS), and then for each CCD the CCD
number [1-5], the mid exposure time (UTC, HMS only), the exposure length
(seconds) and a status flag are listed in parentheses (). The status flag
is set by the NSKIP parameters for each CCD. Only the valid frames will
have status = 'T'.
Parameters:
source : string [hidden]
Data source, two options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
run : string
run number to access, e.g. 'run0034'
first : int
exposure number to start from. 1 = first frame; set = 0 to
always try to get the most recent frame (if it changes)
last : int
last exposure number, 0 for all.
twait : float [hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [hidden]
maximum time to wait between attempts to find a new exposure,
seconds. Set to 0 to avoid waiting and extra output messages.
tdigit : int [hidden]
number of digits of precision for the seconds after the decimal
point when reporting the times.
edigit : int [hidden]
number of digits of precision after the decimal point when
reporting the exposure times
.. Warning::
This routine does not yet work with ULTRACAM data.
"""
command, args = utils.script_args(args)
# get the inputs
with Cline('HIPERCAM_ENV', '.hipercam', command, args) as cl:
# register parameters
cl.register('source', Cline.GLOBAL, Cline.HIDE)
cl.register('run', Cline.GLOBAL, Cline.PROMPT)
cl.register('first', Cline.LOCAL, Cline.PROMPT)
cl.register('last', Cline.LOCAL, Cline.PROMPT)
cl.register('twait', Cline.LOCAL, Cline.HIDE)
cl.register('tmax', Cline.LOCAL, Cline.HIDE)
cl.register('tdigit', Cline.LOCAL, Cline.HIDE)
cl.register('edigit', Cline.LOCAL, Cline.HIDE)
# get inputs
source = cl.get_value('source',
'data source [hs, hl]',
'hl',
lvals=('hs', 'hl'))
resource = cl.get_value('run', 'run name', 'run005')
first = cl.get_value('first', 'first frame to list', 1, 0)
last = cl.get_value('last', 'last frame to list', 0, 0)
if last < first and last != 0:
sys.stderr.write('last must be >= first or 0')
sys.exit(1)
twait = cl.get_value('twait', 'time to wait for a new frame [secs]',
1., 0.)
tmax = cl.get_value('tmax',
'maximum time to wait for a new frame [secs]', 10.,
0.)
tdigit = cl.get_value('tdigit', 'digits after decimal point for times',
6, 1, 9)
edigit = cl.get_value('edigit',
'digits after decimal point for exposure times',
3, 1, 9)
################################################################
#
# all the inputs have now been obtained. Get on with doing stuff
# open the run as an Rtime
rtime = hcam.hcam.Rtime(resource, first, source.endswith('s'))
total_time = 0
nframe = first
for tdata in rtime:
# Handle the waiting game ...
give_up, try_again, total_time = spooler.hang_about(
tdata, twait, tmax, total_time)
if give_up:
if tmax > 0:
print('times stopped')
break
elif try_again:
continue
# indicate progress
tstamp, tinfo, tflag = tdata
tstamp.precision = tdigit
print('Frame {:d}, GPS = {:s} [{:s}]'.format(nframe, tstamp.iso,
'OK' if tflag else 'NOK'),
end='')
message = ''
for nccd, (mjd, exptime, flag) in enumerate(tinfo):
ts = Time(mjd, format='mjd', precision=tdigit)
message += ', ({:d}, {:s}, {:.{:d}f}, {:s})'.format(
nccd + 1, ts.hms_custom, exptime, edigit, 'T' if flag else 'F')
print(message)
# increment the frame
nframe += 1
def grab(args=None):
"""``grab [source temp] (run first last (ndigit) [twait tmax] | flist
(prefix)) trim ([ncol nrow]) bias dark flat fmap (fpair [nhalf rmin
rmax verbose]) [dtype]``
This downloads a sequence of images from a raw data file and writes them
out to a series CCD / MCCD files.
Parameters:
source : str [hidden]
Data source, four options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : ULTRACAM server
| 'ul' : local ULTRACAM .xml/.dat files
| 'hf' : list of HiPERCAM hcm FITS-format files
The standard start-off default for ``source`` can be set
using the environment variable
HIPERCAM_DEFAULT_SOURCE. e.g. in bash :code:`export
HIPERCAM_DEFAULT_SOURCE="us"` would ensure it always
started with the ULTRACAM server by default. If
unspecified, it defaults to 'hl'. 'hf' is useful if you
want to apply pipeline calibrations (bias, flat, etc) to
files imported from a 'foreign' format. In this case the
output files will have the same name as the inputs but
with a prefix added.
temp : bool [hidden, defaults to False]
True to indicate that the frames should be written to
temporary files with automatically-generated names in the
expectation of deleting them later. This also writes out a
file listing the names. The aim of this is to allow grab
to be used as a feeder for other routines in larger
scripts. If temp == True, grab will return with the name
of the list of hcm files. Look at 'makebias' for an example
that uses this feature.
run : str [if source ends 's' or 'l']
run number to access, e.g. 'run034'
first : int [if source ends 's' or 'l']
First frame to access
last : int [if source ends 's' or 'l']
Last frame to access, 0 for the lot
ndigit : int [if source ends 's' or 'l' and not temp]
Files created will be written as 'run005_0013.fits'
etc. `ndigit` is the number of digits used for the frame
number (4 in this case). Any pre-existing files of the same
name will be over-written.
twait : float [hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
flist : str [if source ends 'f']
name of file list. Output files will have the same names
as the input files with a prefix added
prefix : str [if source ends 'f' and not temp]
prefix to add to create output file names
trim : bool
True to trim columns and/or rows off the edges of windows
nearest the readout. Particularly useful for ULTRACAM
windowed data where the first few rows and columns can
contain bad data.
ncol : int [if trim, hidden]
Number of columns to remove (on left of left-hand window,
and right of right-hand windows)
nrow : int [if trim, hidden]
Number of rows to remove (bottom of windows)
bias : str
Name of bias frame to subtract, 'none' to ignore.
dark : str
Name of dark frame, 'none' to ignore.
flat : str
Name of flat field to divide by, 'none' to ignore. Should normally
only be used in conjunction with a bias, although it does allow you
to specify a flat even if you haven't specified a bias.
fmap : str
Name of fringe map (see e.g. `makefringe`), 'none' to ignore.
fpair : str [if fmap is not 'none']
Name of fringe pair file (see e.g. `setfringe`). Required if
a fringe map has been specified.
nhalf : int [if fmap is not 'none', hidden]
When calculating the differences for fringe measurement,
a region extending +/-nhalf binned pixels will be used when
measuring the amplitudes. Basically helps the stats.
rmin : float [if fmap is not 'none', hidden]
Minimum individual ratio to accept prior to calculating the overall
median in order to reduce the effect of outliers. Although all ratios
should be positive, you might want to set this a little below zero
to allow for some statistical fluctuation.
rmax : float [if fmap is not 'none', hidden]
Maximum individual ratio to accept prior to calculating the overall
median in order to reduce the effect of outliers. Probably typically
< 1 if fringe map was created from longer exposure data.
verbose : bool
True to print lots of details of fringe ratios
dtype : string [hidden, defaults to 'f32']
Data type on output. Options:
| 'f32' : output as 32-bit floats (default)
| 'f64' : output as 64-bit floats.
| 'u16' : output as 16-bit unsigned integers. A warning will be
issued if loss of precision occurs; an exception will
be raised if the data are outside the range 0 to 65535.
.. Note::
|grab| is used by several other scripts such as |averun| so take great
care when changing anything to do with its input parameters.
If you use the "temp" option to write to temporary files, then those
files will be deleted if you interrup with CTRL-C. This is to prevent
the accumulation of such frames.
"""
command, args = utils.script_args(args)
# get inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("source", Cline.GLOBAL, Cline.HIDE)
cl.register("temp", Cline.GLOBAL, Cline.HIDE)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("last", Cline.LOCAL, Cline.PROMPT)
cl.register("ndigit", Cline.LOCAL, Cline.PROMPT)
cl.register("twait", Cline.LOCAL, Cline.HIDE)
cl.register("tmax", Cline.LOCAL, Cline.HIDE)
cl.register("flist", Cline.GLOBAL, Cline.PROMPT)
cl.register("prefix", Cline.GLOBAL, Cline.PROMPT)
cl.register("trim", Cline.GLOBAL, Cline.PROMPT)
cl.register("ncol", Cline.GLOBAL, Cline.HIDE)
cl.register("nrow", Cline.GLOBAL, Cline.HIDE)
cl.register("bias", Cline.LOCAL, Cline.PROMPT)
cl.register("flat", Cline.LOCAL, Cline.PROMPT)
cl.register("dark", Cline.LOCAL, Cline.PROMPT)
cl.register("fmap", Cline.LOCAL, Cline.PROMPT)
cl.register("fpair", Cline.LOCAL, Cline.PROMPT)
cl.register("nhalf", Cline.GLOBAL, Cline.HIDE)
cl.register("rmin", Cline.GLOBAL, Cline.HIDE)
cl.register("rmax", Cline.GLOBAL, Cline.HIDE)
cl.register("verbose", Cline.LOCAL, Cline.HIDE)
cl.register("dtype", Cline.LOCAL, Cline.HIDE)
# get inputs
default_source = os.environ.get('HIPERCAM_DEFAULT_SOURCE', 'hl')
source = cl.get_value(
"source",
"data source [hs, hl, us, ul, hf]",
default_source,
lvals=("hs", "hl", "us", "ul", "hf"),
)
cl.set_default("temp", False)
temp = cl.get_value(
"temp", "save to temporary automatically-generated file names?",
True)
# set some flags
server_or_local = source.endswith("s") or source.endswith("l")
if server_or_local:
resource = cl.get_value("run", "run name", "run005")
first = cl.get_value("first", "first frame to grab", 1, 0)
last = cl.get_value("last", "last frame to grab", 0)
if last < first and last != 0:
sys.stderr.write("last must be >= first or 0")
sys.exit(1)
if not temp:
ndigit = cl.get_value("ndigit",
"number of digits in frame identifier",
3, 0)
twait = cl.get_value("twait",
"time to wait for a new frame [secs]", 1.0,
0.0)
tmax = cl.get_value("tmax",
"maximum time to wait for a new frame [secs]",
10.0, 0.0)
else:
resource = cl.get_value("flist", "file list",
cline.Fname("files.lis", hcam.LIST))
if not temp:
prefix = cl.get_value("prefix",
"prefix to add to file names on output",
"pfx_")
first = 1
trim = cl.get_value("trim", "do you want to trim edges of windows?",
True)
if trim:
ncol = cl.get_value("ncol",
"number of columns to trim from windows", 0)
nrow = cl.get_value("nrow", "number of rows to trim from windows",
0)
bias = cl.get_value(
"bias",
"bias frame ['none' to ignore]",
cline.Fname("bias", hcam.HCAM),
ignore="none",
)
if bias is not None:
# read the bias frame
bias = hcam.MCCD.read(bias)
# dark
dark = cl.get_value("dark",
"dark frame ['none' to ignore]",
cline.Fname("dark", hcam.HCAM),
ignore="none")
if dark is not None:
# read the dark frame
dark = hcam.MCCD.read(dark)
# flat
flat = cl.get_value("flat",
"flat field frame ['none' to ignore]",
cline.Fname("flat", hcam.HCAM),
ignore="none")
if flat is not None:
# read the flat field frame
flat = hcam.MCCD.read(flat)
# fringe file (if any)
fmap = cl.get_value(
"fmap",
"fringe map ['none' to ignore]",
cline.Fname("fmap", hcam.HCAM),
ignore="none",
)
if fmap is not None:
# read the fringe frame and prompt other parameters
fmap = hcam.MCCD.read(fmap)
fpair = cl.get_value("fpair", "fringe pair file",
cline.Fname("fringe", hcam.FRNG))
fpair = fringe.MccdFringePair.read(fpair)
nhalf = cl.get_value("nhalf",
"half-size of fringe measurement regions", 2,
0)
rmin = cl.get_value("rmin", "minimum fringe pair ratio", -0.2)
rmax = cl.get_value("rmax", "maximum fringe pair ratio", 1.0)
verbose = cl.get_value("verbose", "verbose output", False)
cl.set_default("dtype", "f32")
dtype = cl.get_value("dtype",
"data type [f32, f64, u16]",
"f32",
lvals=("f32", "f64", "u16"))
# Now the actual work.
# strip off extensions
if resource.endswith(hcam.HRAW):
resource = resource[:resource.find(hcam.HRAW)]
# initialisations
total_time = 0 # time waiting for new frame
nframe = first
root = os.path.basename(resource)
bframe = None
# Finally, we can go
fnames = []
if temp:
tdir = utils.temp_dir()
with CleanUp(fnames, temp) as cleanup:
# The above line to handle ctrl-c and temporaries
with spooler.data_source(source, resource, first) as spool:
for mccd in spool:
if server_or_local:
# Handle the waiting game ...
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmax, total_time)
if give_up:
print("grab stopped")
break
elif try_again:
continue
# Trim the frames: ULTRACAM windowed data has bad
# columns and rows on the sides of windows closest to
# the readout which can badly affect reduction. This
# option strips them.
if trim:
hcam.ccd.trim_ultracam(mccd, ncol, nrow)
if nframe == first:
# First time through, need to manipulate calibration data
if bias is not None:
bias = bias.crop(mccd)
bexpose = bias.head.get("EXPTIME", 0.0)
else:
bexpose = 0.
if flat is not None:
flat = flat.crop(mccd)
if dark is not None:
dark = dark.crop(mccd)
if fmap is not None:
fmap = fmap.crop(mccd)
fpair = fpair.crop(mccd, nhalf)
# now any time through, apply calibrations
if bias is not None:
mccd -= bias
if dark is not None:
# subtract dark, CCD by CCD
dexpose = dark.head["EXPTIME"]
for cnam in mccd:
ccd = mccd[cnam]
if ccd.is_data():
cexpose = ccd.head["EXPTIME"]
scale = (cexpose - bexpose) / dexpose
ccd -= scale * dark[cnam]
if flat is not None:
mccd /= flat
if fmap is not None:
# apply CCD by CCD
for cnam in fmap:
if cnam in fpair:
ccd = mccd[cnam]
if ccd.is_data():
if verbose:
print(f' CCD {cnam}')
fscale = fpair[cnam].scale(ccd,
fmap[cnam],
nhalf,
rmin,
rmax,
verbose=verbose)
if verbose:
print(f' Median scale factor = {fscale}')
ccd -= fscale * fmap[cnam]
if dtype == "u16":
mccd.uint16()
elif dtype == "f32":
mccd.float32()
elif dtype == "f64":
mccd.float64()
# write to disk
if temp:
# generate name automatically
fd, fname = tempfile.mkstemp(suffix=hcam.HCAM, dir=tdir)
fnames.append(fname)
mccd.write(fname, True)
os.close(fd)
elif server_or_local:
fname = f"{root}_{nframe:0{ndigit}}{hcam.HCAM}"
mccd.write(fname, True)
else:
fname = utils.add_extension(
f"{prefix}{mccd.head['FILENAME']}")
mccd.write(fname, True)
print("Written frame {:d} to {:s}".format(nframe, fname))
# update the frame number
nframe += 1
if server_or_local and last and nframe > last:
break
if temp:
if len(fnames) == 0:
raise hcam.HipercamError(
'no files were grabbed; please check'
' input parameters, especially "first"')
# write the file names to a list
fd, fname = tempfile.mkstemp(suffix=hcam.LIST, dir=tdir)
cleanup.flist = fname
with open(fname, "w") as fout:
for fnam in fnames:
fout.write(fnam + "\n")
os.close(fd)
print("temporary file names written to {:s}".format(fname))
# return the name of the file list
return fname
print("grab finished")
