def hinfo(args=None):
"""``hinfo input``
Prints out an hcm file along with its name and the number of CCDs. You
will get information CCD-by-CCD first followed by the top level header.
You will also get at least partial printouts of the data arrays. If
you want still more detail, I recommend the FITS viewer tool 'fv', or
'ds9' for examining images (also :hplot:).
Parameters:
input : string
name of the MCCD file
"""
command, args = utils.script_args(args)
# get input section
with Cline('HIPERCAM_ENV', '.hipercam', command, args) as cl:
# register parameters
cl.register('input', Cline.LOCAL, Cline.PROMPT)
# get inputs
frame = cl.get_value('input', 'frame to plot',
cline.Fname('hcam', hcam.HCAM))
mccd = hcam.MCCD.read(frame)
print('Name of file = {:s}'.format(frame))
print('Number of CCDs = {:d}\n'.format(len(mccd)))
print(mccd)
def stats(args=None):
"""``stats input [format]``
Lists basic stats of a multi-CCD image, i.e. the minimum, maximum, mean,
median and standard deviation of each window of each CCD. The output
format can be altered to suit preference.
Parameters:
input : string
name of the MCCD file
format : string [hidden, defaults to 9.3f]
C-style format code as used in Python format statements for output of
the numerical values. e.g. '300.00' is '6.2f' (6 characters toal, 2
after the decimal point), '1.22e24' is '.2e' (as many characters as
needed, 2 after the decimal point)
"""
command, args = utils.script_args(args)
# get input section
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("input", Cline.LOCAL, Cline.PROMPT)
cl.register("format", Cline.LOCAL, Cline.HIDE)
# get inputs
frame = cl.get_value("input", "frame to lists stats of",
cline.Fname("hcam", hcam.HCAM))
mccd = hcam.MCCD.read(frame)
cl.set_default("format", "9.3f")
form = cl.get_value("format", "output format for numbers", "9.3f")
for cnam, ccd in mccd.items():
for wnam, wind in ccd.items():
print(
"CCD {0:s}, window {1:s}: min = {3:{2:s}}, max = {4:{2:s}}, mean = {5:{2:s}}, median = {6:{2:s}}, std = {7:{2:s}}"
.format(
cnam,
wnam,
form,
wind.min(),
wind.max(),
wind.mean(),
wind.median(),
wind.std(),
))
def averun(args=None):
"""``averun [source] (run first last twait tmax | flist) trim ([ncol
nrow]) bias dark flat [method sigma adjust clobber] output``
Averages images from a run using median combination, skipping the junk
frames that result from NSKIP / NBLUE options in HiPERCAM and ULTRACAM
data. `averun` is meant to be a simple tool to create median frames
suitable prior to aperture selection with `setaper`. See `combine` if you
want more fine-grained control over frame averaging. (`averun` uses a
combination of `grab` [if needed] and `combine`).
Parameters:
source : string [hidden]
Data source, five options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : ULTRACAM server
| 'ul' : local ULTRACAM .xml/.dat files
| 'hf' : list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab' or
converted from foreign data formats.
run : string [if source ends 's' or 'l']
run number to access, e.g. 'run034'
flist : string [if source ends 'f']
name of file list
first : int [if source ends 's' or 'l']
exposure number to start from. 1 = first frame ('0' is
not supported).
last : int [if source ends 's' or 'l']
last exposure number must be >= first.
twait : float [if source ends 's' or 'l'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [if source ends 's' or 'l'; hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
trim : bool
True to trim columns and/or rows off the edges of windows nearest
the readout. Useful for ULTRACAM particularly.
ncol : int [if trim, hidden]
Number of columns to remove (on left of left-hand window, and right
of right-hand windows)
nrow : int [if trim, hidden]
Number of rows to remove (bottom of windows)
bias : string
Name of bias frame to subtract, 'none' to ignore.
dark : string
Name of dark frame to subtract, 'none' to ignore.
flat : string
Name of flat field to divide by, 'none' to ignore.
method : string [hidden, defaults to 'm']
'm' for median, 'c' for clipped mean. See below for pros and cons.
sigma : float [hidden; if method == 'c']
With clipped mean combination, pixels that deviate by more than
sigma RMS from the mean are kicked out. This is carried out in an
iterative manner. sigma <= 0 implies no rejection, just a straight
average. sigma=3 is typical.
adjust : string [hidden; defaults to 'i']
adjustments to make: 'i' = ignore; 'n' = normalise the mean of all
frames to match the first; 'b' = add offsets so that the mean of
all frames is the same as the first. Option 'n' is useful for
twilight flats; 'b' for combining biases.
clobber : bool [hidden]
clobber any pre-existing output files
output : string
output file
"""
command, args = utils.script_args(args)
# get the inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("source", Cline.GLOBAL, Cline.HIDE)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("last", Cline.LOCAL, Cline.PROMPT)
cl.register("twait", Cline.LOCAL, Cline.HIDE)
cl.register("tmax", Cline.LOCAL, Cline.HIDE)
cl.register("trim", Cline.GLOBAL, Cline.PROMPT)
cl.register("ncol", Cline.GLOBAL, Cline.HIDE)
cl.register("nrow", Cline.GLOBAL, Cline.HIDE)
cl.register("flist", Cline.LOCAL, Cline.PROMPT)
cl.register("bias", Cline.LOCAL, Cline.PROMPT)
cl.register("dark", Cline.LOCAL, Cline.PROMPT)
cl.register("flat", Cline.LOCAL, Cline.PROMPT)
cl.register("method", Cline.LOCAL, Cline.HIDE)
cl.register("sigma", Cline.LOCAL, Cline.HIDE)
cl.register("adjust", Cline.LOCAL, Cline.HIDE)
cl.register("clobber", Cline.LOCAL, Cline.HIDE)
cl.register("output", Cline.LOCAL, Cline.PROMPT)
# get inputs
source = cl.get_value(
"source",
"data source [hs, hl, us, ul, hf]",
"hl",
lvals=("hs", "hl", "us", "ul", "hf"),
)
# set a flag
server_or_local = source.endswith("s") or source.endswith("l")
if server_or_local:
run = cl.get_value("run", "run name", "run005")
first = cl.get_value("first", "first frame to average", 1, 1)
last = cl.get_value("last", "last frame to average", first, first)
twait = cl.get_value("twait",
"time to wait for a new frame [secs]", 1.0,
0.0)
tmax = cl.get_value("tmax",
"maximum time to wait for a new frame [secs]",
10.0, 0.0)
else:
flist = cl.get_value("flist", "file list",
cline.Fname("files.lis", hcam.LIST))
first = 1
trim = cl.get_value("trim", "do you want to trim edges of windows?",
True)
if trim:
ncol = cl.get_value("ncol",
"number of columns to trim from windows", 0)
nrow = cl.get_value("nrow", "number of rows to trim from windows",
0)
# bias frame (if any)
bias = cl.get_value(
"bias",
"bias frame ['none' to ignore]",
cline.Fname("bias", hcam.HCAM),
ignore="none",
)
# dark frame (if any)
dark = cl.get_value(
"dark",
"dark frame ['none' to ignore]",
cline.Fname("dark", hcam.HCAM),
ignore="none",
)
# flat field frame (if any)
flat = cl.get_value(
"flat",
"flat field frame ['none' to ignore]",
cline.Fname("flat", hcam.HCAM),
ignore="none",
)
cl.set_default("method", "m")
method = cl.get_value("method",
"c(lipped mean), m(edian)",
"c",
lvals=("c", "m"))
if method == "c":
sigma = cl.get_value("sigma", "number of RMS deviations to clip",
3.0)
cl.set_default("adjust", "i")
adjust = cl.get_value("adjust",
"i(gnore), n(ormalise) b(ias offsets)",
"i",
lvals=("i", "n", "b"))
clobber = cl.get_value("clobber",
"clobber any pre-existing files on output",
False)
output = cl.get_value(
"output",
"output average",
cline.Fname("hcam", hcam.HCAM,
cline.Fname.NEW if clobber else cline.Fname.NOCLOBBER),
)
# inputs done with. Now do the work with 'grab' and 'combine'
if server_or_local:
print("\nCalling 'grab' ...")
if trim:
args = [
None,
"prompt",
source,
run,
"yes",
str(first),
str(last),
"yes",
str(ncol),
str(nrow),
str(twait),
str(tmax),
"none",
"f32",
]
else:
args = [
None,
"prompt",
source,
run,
"yes",
str(first),
str(last),
"no",
str(twait),
str(tmax),
"none",
"f32",
]
print("arg =", args)
flist = hcam.scripts.grab(args)
try:
print("\nCalling 'combine' ...")
if method == "m":
args = [
None,
"prompt",
flist,
"none" if bias is None else bias,
"none" if dark is None else dark,
"none" if flat is None else flat,
method,
adjust,
"usemean=yes",
"plot=no",
"yes" if clobber else "no",
output,
]
else:
args = [
None,
"prompt",
flist,
"none" if bias is None else bias,
"none" if dark is None else dark,
"none" if flat is None else flat,
method,
str(sigma),
adjust,
"usemean=yes",
"plot=no",
"yes" if clobber else "no",
output,
]
hcam.scripts.combine(args)
# remove temporary files
with open(flist) as fin:
for fname in fin:
fname = fname.strip()
os.remove(fname)
os.remove(flist)
print("\ntemporary files have been deleted")
print("averun finished")
except KeyboardInterrupt:
# this to ensure we delete the temporary files
with open(flist) as fin:
for fname in fin:
fname = fname.strip()
os.remove(fname)
os.remove(flist)
print("\ntemporary files have been deleted")
print("averun aborted")
def filtid(args=None):
"""``filtid [source] (run first last [twait tmax] |
flist) trim ([ncol nrow]) ccdref maxlev plot``
Computes the ratio of the mean values in each CCD versus those of a reference CCD.
This is supposed to be applied to flat fields. The hope is that this might be
characteristic of the filter combination. Obviously requires > 1 CCD, i.e. not
ULTRASPEC.
Parameters:
source : str [hidden]
Data source, five options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : ULTRACAM server
| 'ul' : local ULTRACAM .xml/.dat files
| 'hf' : list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab'
or converted from foreign data formats. The standard
start-off default for ``source'' can be set using the
environment variable HIPERCAM_DEFAULT_SOURCE. e.g. in bash
:code:`export HIPERCAM_DEFAULT_SOURCE="us"` would ensure it
always started with the ULTRACAM server by default. If
unspecified, it defaults to 'hl'.
run : str [if source ends 's' or 'l']
run number to access, e.g. 'run034'
flist : str [if source ends 'f']
name of file list
first : int [if source ends 's' or 'l']
exposure number to start from. 1 = first frame; set = 0 to always
try to get the most recent frame (if it has changed). For data
from the |hiper| server, a negative number tries to get a frame not
quite at the end. i.e. -10 will try to get 10 from the last
frame. This is mainly to sidestep a difficult bug with the
acquisition system.
last : int [if source ends 's' or 'l']
last frame to reduce. 0 to just continue until the end.
twait : float [if source ends 's' or 'l'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [if source ends 's' or 'l'; hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
trim : bool [if source starts with 'u']
True to trim columns and/or rows off the edges of windows nearest
the readout which can sometimes contain bad data.
ncol : int [if trim, hidden]
Number of columns to remove (on left of left-hand window, and right
of right-hand windows)
nrow : int [if trim, hidden]
Number of rows to remove (bottom of windows)
ccdref : str
The reference CCD (usually choose the g-band one)
bias : str
bias frame to subtract (required)
lower : list of floats
Lower limits to the mean count level for a flat to be
included (after bias subtraction). Should be the same
number as the number of CCDs, and will be assumed to be in
the same order. Separate with spaces. Prevents low exposure
data from being included.
upper : list of floats
Upper limits to the mean count level for a flat to be
included. Should be the same number as the selected CCDs,
and will be assumed to be in the same order. Use to
eliminate saturated, peppered or non-linear
frames. Suggested hipercam values: 58000, 58000, 58000,
40000 and 40000 for CCDs 1, 2, 3, 4 and 5. Enter values
separated by spaces. ULTRACAM values 49000, 29000, 27000
for CCDs 1, 2 and 3.
plot: bool
Plot the fit or not.
.. Note::
This is currently adapted specifically for ULTRACAM data
"""
command, args = utils.script_args(args)
# get the inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("source", Cline.GLOBAL, Cline.HIDE)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("last", Cline.LOCAL, Cline.PROMPT)
cl.register("trim", Cline.GLOBAL, Cline.PROMPT)
cl.register("ncol", Cline.GLOBAL, Cline.HIDE)
cl.register("nrow", Cline.GLOBAL, Cline.HIDE)
cl.register("twait", Cline.LOCAL, Cline.HIDE)
cl.register("tmax", Cline.LOCAL, Cline.HIDE)
cl.register("flist", Cline.LOCAL, Cline.PROMPT)
cl.register("ccdref", Cline.LOCAL, Cline.PROMPT)
cl.register("bias", Cline.LOCAL, Cline.PROMPT)
cl.register("lower", Cline.LOCAL, Cline.PROMPT)
cl.register("upper", Cline.LOCAL, Cline.PROMPT)
cl.register("plot", Cline.LOCAL, Cline.PROMPT)
# get inputs
default_source = os.environ.get('HIPERCAM_DEFAULT_SOURCE', 'hl')
source = cl.get_value(
"source",
"data source [hs, hl, us, ul, hf]",
default_source,
lvals=("hs", "hl", "us", "ul", "hf"),
)
# set some flags
server_or_local = source.endswith("s") or source.endswith("l")
if server_or_local:
resource = cl.get_value("run", "run name", "run005")
first = cl.get_value("first", "first frame to consider", 1, 1)
last = cl.get_value("last", "last frame to consider", 0, 0)
if last and last < first + 2:
raise hcam.HipercamError(
"Must consider at least 2 exposures, and preferably more")
twait = cl.get_value("twait",
"time to wait for a new frame [secs]", 1.0,
0.0)
tmax = cl.get_value("tmax",
"maximum time to wait for a new frame [secs]",
10.0, 0.0)
else:
resource = cl.get_value("flist", "file list",
cline.Fname("files.lis", hcam.LIST))
first = 1
trim = cl.get_value("trim", "do you want to trim edges of windows?",
True)
if trim:
ncol = cl.get_value("ncol",
"number of columns to trim from windows", 0)
nrow = cl.get_value("nrow", "number of rows to trim from windows",
0)
# define the panel grid. first get the labels and maximum dimensions
ccdinf = spooler.get_ccd_pars(source, resource)
if len(ccdinf) == 1:
raise hcam.HipercamError(
"Only one CCD; this routine only works for > 1 CCD")
ccds = list(ccdinf.keys())
ccdref = cl.get_value("ccdref",
"the reference CCD for the ratios",
ccds[1],
lvals=ccds)
bias = cl.get_value("bias", "bias frame",
cline.Fname("bias", hcam.HCAM))
# read the bias frame
bias = hcam.MCCD.read(bias)
if len(bias) != len(ccds):
raise ValueError('bias has an incompatible number of CCDs')
# need to check that the default has the right number of items, if not,
# over-ride it
lowers = cl.get_default("lower")
if lowers is not None and len(lowers) != len(ccds):
cl.set_default("lower", len(ccds) * (5000, ))
lowers = cl.get_value(
"lower",
"lower limits on mean count level for included flats, 1 per CCD",
len(ccds) * (5000, ),
)
lowers = {k: v for k, v in zip(ccds, lowers)}
uppers = cl.get_default("upper")
if uppers is not None and len(uppers) != len(ccds):
cl.set_default("upper", len(ccds) * (50000, ))
uppers = cl.get_value(
"upper",
"lower limits on mean count level for included flats, 1 per CCD",
len(ccds) * (50000, ),
)
uppers = {k: v for k, v in zip(ccds, uppers)}
plot = cl.get_value("plot", "plot the results?", True)
################################################################
#
# all the inputs have now been obtained. Get on with doing stuff
total_time = 0.
xdata, ydata = {}, {}
# only want to look at CCDs other than ccdref
ccds.remove(ccdref)
# access images
with spooler.data_source(source, resource, first, full=False) as spool:
# 'spool' is an iterable source of MCCDs
nframe = 0
for mccd in spool:
if server_or_local:
# Handle the waiting game ...
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmax, total_time)
if give_up:
print("rtplot stopped")
break
elif try_again:
continue
# Trim the frames: ULTRACAM windowed data has bad columns
# and rows on the sides of windows closest to the readout
# which can badly affect reduction. This option strips
# them.
if trim:
hcam.ccd.trim_ultracam(mccd, ncol, nrow)
if nframe == 0:
# crop the bias on the first frame only
bias = bias.crop(mccd)
# bias subtraction
mccd -= bias
# indicate progress
tstamp = Time(mccd.head["TIMSTAMP"], format="isot", precision=3)
print(f"{mccd.head.get('NFRAME',nframe+1)}, utc= {tstamp.iso} " +
f"({'ok' if mccd.head.get('GOODTIME', True) else 'nok'})")
# add in data if it is in range
xd = mccd[ccdref].mean()
if xd > lowers[ccdref] and xd < uppers[ccdref]:
for cnam in ccds:
yd = mccd[cnam].mean()
if yd > lowers[cnam] and yd < uppers[cnam]:
if cnam in xdata:
xdata[cnam].append(xd)
ydata[cnam].append(yd)
else:
xdata[cnam] = [xd]
ydata[cnam] = [yd]
if last and nframe == last:
break
# update the frame number
nframe += 1
print()
invalid = True
for cnam in ccds:
if cnam in xdata:
xdata[cnam] = np.array(xdata[cnam])
ydata[cnam] = np.array(ydata[cnam])
print(
f'{cnam}-vs-{ccdref}: found {len(xdata[cnam])} valid points ranging from'
+
f' {xdata[cnam].min():.1f} to {xdata[cnam].max():.1f} (CCD {ccdref}), '
+
f' {ydata[cnam].min():.1f} to {ydata[cnam].max():.1f} (CCD {cnam})'
)
invalid = False
if invalid:
raise hcam.HipercamError(f"No valid points found at all")
print()
if plot:
fig, axes = plt.subplots(len(ccds), 1, sharex=True)
for cnam, ax in zip(ccds, axes):
if cnam in xdata:
xds = xdata[cnam]
ratios = ydata[cnam] / xds
mratio = ratios.mean()
ax.plot(xds, ratios, '.g')
ax.plot([xds.min(), xds.max()], [mratio, mratio], 'r--')
ax.set_ylabel(f'Ratio CCD {cnam} / CCD {ccdref}')
if cnam == ccds[-1]:
ax.set_xlabel(f'CCD = {ccdref} counts/pixel')
print(f'{cnam}-vs-{ccdref} = {mratio:.4f}')
plt.show()
else:
for cnam in ccds:
if cnam in xdata:
ratios = ydata[cnam] / xdata[cnam]
mratio = ratios.mean()
print(f'{cnam}-vs-{ccdref} = {mratio:.4f}')
def setdefect(args=None):
"""``setdefect mccd defect ccd [linput width height] rtarg rsky1 rsky2 nx
msub iset (ilo ihi | plo phi) [profit method beta fwmin fwhm fwfix
shbox smooth splot fhbox read gain thresh]``
Interactive definition of CCD defects. This is a matplotlib-based routine
allowing you to define defects using the cursor.
Parameters:
mccd : string
name of an MCCD file, as produced by e.g. 'grab'
defect : string
the name of a defect file. If it exists it will be read so that
defects can be added to it. If it does not exist, it will be
created on exiting the routine. The defect files are in a fairly
readable / editiable text format
ccd : string
CCD(s) to plot, '0' for all. If not '0' then '1', '2' or even '3 4'
are possible inputs (without the quotes). '3 4' will plot CCD '3' and
CCD '4'. If you want to plot more than one CCD, then you will be
prompted for the number of panels in the X direction. This parameter
will not be prompted if there is only one CCD in the file.
width : float [hidden]
plot width (inches). Set = 0 to let the program choose.
height : float [hidden]
plot height (inches). Set = 0 to let the program choose. BOTH width
AND height must be non-zero to have any effect
nx : int
number of panels across to display, prompted if more than one CCD is
to be plotted.
msub : bool
True/False to subtract median from each window before scaling
invert : bool [if msub]
If msub is True, then you can invert the image values (-ve to +ve)
with this parameter. Can make it easier to spot bad values.
ffield : bool
If True, all defects will be assumed to be flat-field or poor
charge transfer defects as opposed to hot pixels. The latter are
best set from dark frames, and have a different impact than the
first two types in that they are worst for faint targets. Hot pixels
and flat-field defects are shown with the same colours for moderate
and severe, but different symbols (filled circles for flat-field
defects, stars for hot pixels). If you say no to add hot pixels,
the line defect option is not available.
hsbox : int
half-width in binned pixels of stats box as offset from central pixel
hsbox = 1 gives a 3x3 box; hsbox = 2 gives 5x5 etc. This is used by
the "show" option when setting defects.
iset : string [single character]
determines how the intensities are determined. There are three
options: 'a' for automatic simply scales from the minimum to the
maximum value found on a per CCD basis. 'd' for direct just takes two
numbers from the user. 'p' for percentile dtermines levels based upon
percentiles determined from the entire CCD on a per CCD bais.
ilo : float [if iset=='d']
lower intensity level
ihi : float [if iset=='d']
upper intensity level
plo : float [if iset=='p']
lower percentile level
phi : float [if iset=='p']
upper percentile level
There are a few conveniences to make setdefect easier:
1. The plot is initialised in pan mode whereby you can move around and
scale using the left and right mouse buttons.
2. All input is accomplished with the keyboard; the mouse buttons are
only for navigating the image.
3. The label input can be switched between sequential numerical,
single- and multi-character input ('linput').
Various standard keyboard shortcuts (e.g. 's' to save) are disabled as
they just confuse things and are of limited use in setdefect in any case.
Some aspects of the usage of matplotlib in setdefect are tricky. It is
possible that particular 'backends' will cause problems. I have tested
this with Qt4Agg, Qt5agg and GTK3Agg. One aspect is the cursor icon in pan
mode is a rather indistinct hand where one can't tell what is being
pointed at. I have therefore suppressed this, but only for the tested
backends. Others would need require further investigation.
NB At the end of this routine, it re-orders the defects so that the severe
ones follows the moderates. This helps emphasize the severe ones over the
moderates when running rtplot.
"""
command, args = utils.script_args(args)
# get input section
with Cline('HIPERCAM_ENV', '.hipercam', command, args) as cl:
# register parameters
cl.register('mccd', Cline.LOCAL, Cline.PROMPT)
cl.register('defect', Cline.LOCAL, Cline.PROMPT)
cl.register('ccd', Cline.LOCAL, Cline.PROMPT)
cl.register('width', Cline.LOCAL, Cline.HIDE)
cl.register('height', Cline.LOCAL, Cline.HIDE)
cl.register('nx', Cline.LOCAL, Cline.PROMPT)
cl.register('msub', Cline.GLOBAL, Cline.PROMPT)
cl.register('invert', Cline.GLOBAL, Cline.PROMPT)
cl.register('ffield', Cline.GLOBAL, Cline.PROMPT)
cl.register('hsbox', Cline.GLOBAL, Cline.HIDE)
cl.register('iset', Cline.GLOBAL, Cline.PROMPT)
cl.register('ilo', Cline.GLOBAL, Cline.PROMPT)
cl.register('ihi', Cline.GLOBAL, Cline.PROMPT)
cl.register('plo', Cline.GLOBAL, Cline.PROMPT)
cl.register('phi', Cline.GLOBAL, Cline.PROMPT)
# get inputs
mccd = cl.get_value('mccd', 'frame to plot',
cline.Fname('hcam', hcam.HCAM))
mccd = hcam.MCCD.read(mccd)
dfct = cl.get_value('defect', 'name of defect file',
cline.Fname('defect', hcam.DFCT, exist=False))
if os.path.exists(dfct):
# read in old defects
mccd_dfct = defect.MccdDefect.read(dfct)
print('Loaded existing file = {:s}'.format(dfct))
else:
# create empty container
mccd_dfct = defect.MccdDefect()
print('No file called {:s} exists; '
'will create from scratch'.format(dfct))
# define the panel grid
try:
nxdef = cl.get_default('nx')
except:
nxdef = 3
max_ccd = len(mccd)
if max_ccd > 1:
ccd = cl.get_value('ccd', 'CCD(s) to plot [0 for all]', '0')
if ccd == '0':
ccds = list(mccd.keys())
else:
ccds = ccd.split()
else:
ccds = list(mccd.keys())
width = cl.get_value('width', 'plot width (inches)', 0.)
height = cl.get_value('height', 'plot height (inches)', 0.)
# number of panels in X
if len(ccds) > 1:
nxdef = min(len(ccds), nxdef)
cl.set_default('nx', nxdef)
nx = cl.get_value('nx', 'number of panels in X', 3, 1)
else:
nx = 1
# define the display intensities
msub = cl.get_value('msub', 'subtract median from each window?', True)
if msub:
invert = cl.get_value('invert', 'invert image intensities?', True)
ffield = cl.get_value('ffield',
'flat field defects? [else hot pixels]', True)
hsbox = cl.get_value('hsbox',
'half-width of stats box (binned pixels)', 2, 1)
iset = cl.get_value('iset', 'set intensity a(utomatically),'
' d(irectly) or with p(ercentiles)?',
'a',
lvals=['a', 'A', 'd', 'D', 'p', 'P'])
iset = iset.lower()
plo, phi = 5, 95
ilo, ihi = 0, 1000
if iset == 'd':
ilo = cl.get_value('ilo', 'lower intensity limit', 0.)
ihi = cl.get_value('ihi', 'upper intensity limit', 1000.)
elif iset == 'p':
plo = cl.get_value('plo', 'lower intensity limit percentile', 5.,
0., 100.)
phi = cl.get_value('phi', 'upper intensity limit percentile', 95.,
0., 100.)
nxmax, nymax = 0, 0
for cnam in ccds:
nxmax = max(nxmax, mccd[cnam].nxtot)
nymax = max(nymax, mccd[cnam].nytot)
# might be worth trying to improve this at some point
xlo, xhi, ylo, yhi = 0, nxmax + 1, 0, nymax + 1
# Inputs obtained.
# re-configure keyboard shortcuts to avoid otherwise confusing behaviour
# quit_all does not seem to be universal, hence the try/except
try:
mpl.rcParams['keymap.back'] = ''
mpl.rcParams['keymap.forward'] = ''
mpl.rcParams['keymap.fullscreen'] = ''
mpl.rcParams['keymap.grid'] = ''
mpl.rcParams['keymap.home'] = ''
mpl.rcParams['keymap.pan'] = ''
mpl.rcParams['keymap.quit'] = ''
mpl.rcParams['keymap.save'] = ''
mpl.rcParams['keymap.pan'] = ''
mpl.rcParams['keymap.save'] = ''
mpl.rcParams['keymap.xscale'] = ''
mpl.rcParams['keymap.yscale'] = ''
mpl.rcParams['keymap.zoom'] = ''
except KeyError:
pass
# start plot
if width > 0 and height > 0:
fig = plt.figure(figsize=(width, height))
else:
fig = plt.figure()
# get the navigation toolbar. Go straight into pan mode
# where we want to stay.
toolbar = fig.canvas.manager.toolbar
toolbar.pan()
nccd = len(ccds)
ny = nccd // nx if nccd % nx == 0 else nccd // nx + 1
# we need to store some stuff
ax = None
cnams = {}
anams = {}
# this is a container for all the objects used to plot Defects to allow
# deletion. This is Group of Group objects supporting tuple storage. The
# idea is that pobjs[cnam][anam] returns the objects used to plot Defect
# anam of CCD cnam. It is initially empty,
pobjs = hcam.Group(hcam.Group)
for n, cnam in enumerate(ccds):
if ax is None:
axes = ax = fig.add_subplot(ny, nx, n + 1)
axes.set_aspect('equal', adjustable='box')
axes.set_xlim(xlo, xhi)
axes.set_ylim(ylo, yhi)
else:
axes = fig.add_subplot(ny, nx, n + 1, sharex=ax, sharey=ax)
axes.set_aspect('equal', adjustable='datalim')
if msub:
# subtract median from each window
for wind in mccd[cnam].values():
wind -= wind.median()
if invert:
mccd *= -1
hcam.mpl.pCcd(axes, mccd[cnam], iset, plo, phi, ilo, ihi,
'CCD {:s}'.format(cnam))
# keep track of the CCDs associated with each axes
cnams[axes] = cnam
# and axes associated with each CCD
anams[cnam] = axes
if cnam in mccd_dfct:
# plot any pre-existing Defects, keeping track of
# the plot objects
pobjs[cnam] = hcam.mpl.pCcdDefect(axes, mccd_dfct[cnam])
else:
# add in an empty CcdDefect for any CCD not already present
mccd_dfct[cnam] = defect.CcdDefect()
# and an empty container for any new plot objects
pobjs[cnam] = hcam.Group(tuple)
# create the Defect picker (see below for class def)
picker = PickDefect(mccd, cnams, anams, toolbar, fig, mccd_dfct, dfct,
ffield, hsbox, pobjs)
plt.tight_layout()
picker.action_prompt(False)
# squeeze space a bit
plt.subplots_adjust(wspace=0.1, hspace=0.1)
# finally show stuff ....
plt.show()
def averun(args=None):
"""``averun [source] (run first last twait tmax | flist) trim ([ncol
nrow]) bias dark flat fmap (fpair [nhalf rmin rmax]) [method sigma
adjust clobber] output``
Averages images from a run using median combination, skipping the junk
frames that result from NSKIP / NBLUE options in HiPERCAM and ULTRACAM
data. `averun` is meant to be a simple tool to create median frames
suitable prior to aperture selection with `setaper`. See `combine` if you
want more fine-grained control over frame averaging. (`averun` uses a
combination of `grab` [if needed] and `combine`).
Parameters:
source : str [hidden]
Data source, five options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : ULTRACAM server
| 'ul' : local ULTRACAM .xml/.dat files
| 'hf' : list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab'
or converted from foreign data formats. The standard
start-off default for ``source`` can be set using the
environment variable HIPERCAM_DEFAULT_SOURCE. e.g. in bash
:code:`export HIPERCAM_DEFAULT_SOURCE="us"` would ensure it
always started with the ULTRACAM server by default. If
unspecified, it defaults to 'hl'.
run : str [if source ends 's' or 'l']
run number to access, e.g. 'run034'
flist : str [if source ends 'f']
name of file list
first : int [if source ends 's' or 'l']
exposure number to start from. 1 = first frame ('0' is
not supported).
last : int [if source ends 's' or 'l']
last exposure number must be >= first, or 0 for the lot
twait : float [if source ends 's' or 'l'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [if source ends 's' or 'l'; hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
trim : bool
True to trim columns and/or rows off the edges of windows nearest
the readout. Useful for ULTRACAM particularly.
ncol : int [if trim, hidden]
Number of columns to remove (on left of left-hand window, and right
of right-hand windows)
nrow : int [if trim, hidden]
Number of rows to remove (bottom of windows)
bias : str
Name of bias frame to subtract, 'none' to ignore.
dark : str
Name of dark frame to subtract, 'none' to ignore.
flat : str
Name of flat field to divide by, 'none' to ignore.
fmap : str
Name of fringe map (see e.g. `makefringe`), 'none' to ignore.
fpair : str [if fmap is not 'none']
Name of fringe pair file (see e.g. `setfringe`). Required if
a fringe map has been specified.
nhalf : int [if fmap is not 'none', hidden]
When calculating the differences for fringe measurement,
a region extending +/-nhalf binned pixels will be used when
measuring the amplitudes. Basically helps the stats.
rmin : float [if fmap is not 'none', hidden]
Minimum individual ratio to accept prior to calculating the overall
median in order to reduce the effect of outliers. Although all ratios
should be positive, you might want to set this a little below zero
to allow for some statistical fluctuation.
rmax : float [if fmap is not 'none', hidden]
Maximum individual ratio to accept prior to calculating the overall
median in order to reduce the effect of outliers. Probably typically
< 1 if fringe map was created from longer exposure data.
method : str [hidden, defaults to 'm']
'm' for median, 'c' for clipped mean. See below for pros and cons.
sigma : float [hidden; if method == 'c']
With clipped mean combination, pixels that deviate by more than
sigma RMS from the mean are kicked out. This is carried out in an
iterative manner. sigma <= 0 implies no rejection, just a straight
average. sigma=3 is typical.
adjust : str [hidden; defaults to 'i']
adjustments to make: 'i' = ignore; 'n' = normalise the mean of all
frames to match the first; 'b' = add offsets so that the mean of
all frames is the same as the first. Option 'n' is useful for
twilight flats; 'b' for combining biases.
clobber : bool [hidden]
clobber any pre-existing output files
output : str
output file. Set by default to match the last part of "run"
(but it will have a different extension so they won't
clash)
"""
command, args = utils.script_args(args)
# get the inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("source", Cline.GLOBAL, Cline.HIDE)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("last", Cline.LOCAL, Cline.PROMPT)
cl.register("twait", Cline.LOCAL, Cline.HIDE)
cl.register("tmax", Cline.LOCAL, Cline.HIDE)
cl.register("trim", Cline.GLOBAL, Cline.PROMPT)
cl.register("ncol", Cline.GLOBAL, Cline.HIDE)
cl.register("nrow", Cline.GLOBAL, Cline.HIDE)
cl.register("flist", Cline.LOCAL, Cline.PROMPT)
cl.register("bias", Cline.LOCAL, Cline.PROMPT)
cl.register("dark", Cline.GLOBAL, Cline.PROMPT)
cl.register("flat", Cline.GLOBAL, Cline.PROMPT)
cl.register("fmap", Cline.GLOBAL, Cline.PROMPT)
cl.register("fpair", Cline.GLOBAL, Cline.PROMPT)
cl.register("nhalf", Cline.GLOBAL, Cline.HIDE)
cl.register("rmin", Cline.GLOBAL, Cline.HIDE)
cl.register("rmax", Cline.GLOBAL, Cline.HIDE)
cl.register("method", Cline.LOCAL, Cline.HIDE)
cl.register("sigma", Cline.LOCAL, Cline.HIDE)
cl.register("adjust", Cline.LOCAL, Cline.HIDE)
cl.register("clobber", Cline.LOCAL, Cline.HIDE)
cl.register("output", Cline.LOCAL, Cline.PROMPT)
# get inputs
default_source = os.environ.get('HIPERCAM_DEFAULT_SOURCE','hl')
source = cl.get_value(
"source",
"data source [hs, hl, us, ul, hf]",
default_source,
lvals=("hs", "hl", "us", "ul", "hf"),
)
# set a flag
server_or_local = source.endswith("s") or source.endswith("l")
if server_or_local:
resource = cl.get_value("run", "run name", "run005")
root = os.path.basename(resource)
cl.set_default('output', cline.Fname(root, hcam.HCAM))
first = cl.get_value("first", "first frame to average", 1, 1)
last = cl.get_value("last", "last frame to average", first, 0)
twait = cl.get_value(
"twait", "time to wait for a new frame [secs]", 1.0, 0.0
)
tmax = cl.get_value(
"tmax", "maximum time to wait for a new frame [secs]", 10.0, 0.0
)
else:
resource = cl.get_value(
"flist", "file list", cline.Fname("files.lis", hcam.LIST)
)
first = 1
trim = cl.get_value("trim", "do you want to trim edges of windows?", True)
if trim:
ncol = cl.get_value("ncol", "number of columns to trim from windows", 0)
nrow = cl.get_value("nrow", "number of rows to trim from windows", 0)
# bias frame (if any)
bias = cl.get_value(
"bias",
"bias frame ['none' to ignore]",
cline.Fname("bias", hcam.HCAM),
ignore="none",
)
# dark frame (if any)
dark = cl.get_value(
"dark",
"dark frame ['none' to ignore]",
cline.Fname("dark", hcam.HCAM),
ignore="none",
)
# flat field frame (if any)
flat = cl.get_value(
"flat",
"flat field frame ['none' to ignore]",
cline.Fname("flat", hcam.HCAM),
ignore="none",
)
# fringe file (if any)
fmap = cl.get_value(
"fmap",
"fringe map ['none' to ignore]",
cline.Fname("fmap", hcam.HCAM),
ignore="none",
)
if fmap is not None:
fpair = cl.get_value(
"fpair", "fringe pair file",
cline.Fname("fringe", hcam.FRNG)
)
nhalf = cl.get_value(
"nhalf", "half-size of fringe measurement regions",
2, 0
)
rmin = cl.get_value(
"rmin", "minimum fringe pair ratio", -0.2
)
rmax = cl.get_value(
"rmax", "maximum fringe pair ratio", 1.0
)
cl.set_default("method", "m")
method = cl.get_value(
"method", "c(lipped mean), m(edian)", "c", lvals=("c", "m")
)
if method == "c":
sigma = cl.get_value("sigma", "number of RMS deviations to clip", 3.0)
cl.set_default("adjust", "i")
adjust = cl.get_value(
"adjust", "i(gnore), n(ormalise) b(ias offsets)", "i", lvals=("i", "n", "b")
)
clobber = cl.get_value(
"clobber", "clobber any pre-existing files on output", False
)
output = cl.get_value(
"output",
"output average",
cline.Fname(
"hcam", hcam.HCAM, cline.Fname.NEW if clobber else cline.Fname.NOCLOBBER
),
)
# inputs done with. Now do the work with 'grab' and 'combine'
if server_or_local or bias is not None or dark is not None or \
flat is not None or fmap is not None:
print("\nCalling 'grab' ...")
# Build argument list
args = [None,"prompt",source,"yes",resource]
if server_or_local:
args += [str(first), str(last), str(twait), str(tmax)]
if trim:
args += ["yes",str(ncol),str(nrow)]
else:
args += ["no"]
args += [
"none" if bias is None else bias,
"none" if dark is None else dark,
"none" if flat is None else flat,
]
if fmap is None:
args += ["none", "f32"]
else:
args += [fmap,fpair,str(nhalf),str(rmin),str(rmax),"false","f32"]
resource = hcam.scripts.grab(args)
# at this point 'resource' is a list of files, no matter the input
# method.
with CleanUp(
resource, server_or_local or bias is not None or dark is not None
or flat is not None or fmap is not None
) as cleanup:
print("\nCalling 'combine' ...")
args = [
None, "prompt", resource,
"none", "none", "none",
method
]
if method != "m":
args += [str(sigma)]
args += [
adjust, "usemean=yes",
"plot=no", "yes" if clobber else "no",
output,
]
hcam.scripts.combine(args)
print("averun finished")
def rtplot(args=None):
"""``rtplot [source device width height] (run first (trim [ncol nrow])
twait tmax | flist) [pause plotall] ccd (nx) bias flat defect
setup [hurl] msub iset (ilo ihi | plo phi) xlo xhi ylo yhi (profit
[fdevice fwidth fheight method beta fwhm fwhm_min shbox smooth
splot fhbox hmin read gain thresh])``
Plots a sequence of images as a movie in near 'real time', hence
'rt'. Designed to be used to look at images coming in while at the
telescope, 'rtplot' comes with many options, a large number of which are
hidden by default, and many of which are only prompted if other arguments
are set correctly. If you want to see them all, invoke as 'rtplot prompt'.
This is worth doing once to know rtplot's capabilities.
rtplot can source data from both the ULTRACAM and HiPERCAM servers, from
local 'raw' ULTRACAM and HiPERCAM files (i.e. .xml + .dat for ULTRACAM, 3D
FITS files for HiPERCAM) and from lists of HiPERCAM '.hcm' files.
rtplot optionally allows the selection of targets to be fitted with
gaussian or moffat profiles, and, if successful, will plot circles of 2x
the measured FWHM in green over the selected targets. This option only
works if a single CCD is being plotted.
Parameters:
source : string [hidden]
Data source, five options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : ULTRACAM server
| 'ul' : local ULTRACAM .xml/.dat files
| 'hf' : list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab' or
converted from foreign data formats.
device : string [hidden]
Plot device. PGPLOT is used so this should be a PGPLOT-style name,
e.g. '/xs', '1/xs' etc. At the moment only ones ending /xs are
supported.
width : float [hidden]
plot width (inches). Set = 0 to let the program choose.
height : float [hidden]
plot height (inches). Set = 0 to let the program choose. BOTH width
AND height must be non-zero to have any effect
run : string [if source ends 's' or 'l']
run number to access, e.g. 'run034'
flist : string [if source ends 'f']
name of file list
first : int [if source ends 's' or 'l']
exposure number to start from. 1 = first frame; set = 0 to always
try to get the most recent frame (if it has changed). For data
from the |hiper| server, a negative number tries to get a frame not
quite at the end. i.e. -10 will try to get 10 from the last
frame. This is mainly to sidestep a difficult bug with the
acquisition system.
trim : bool [if source starts with 'u']
True to trim columns and/or rows off the edges of windows nearest
the readout. This is particularly for ULTRACAM windowed data where
the first few rows and columns can contain bad data.
ncol : int [if trim, hidden]
Number of columns to remove (on left of left-hand window, and right
of right-hand windows)
nrow : int [if trim, hidden]
Number of rows to remove (bottom of windows)
twait : float [if source ends 's' or 'l'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [if source ends 's' or 'l'; hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
pause : float [hidden]
seconds to pause between frames (defaults to 0)
plotall : bool [hidden]
plot all frames regardless of status (i.e. including blank frames
when nskips are enabled (defaults to False). The profile fitting
will still be disabled for bad frames.
ccd : string
CCD(s) to plot, '0' for all, '1 3' to plot '1' and '3' only, etc.
nx : int [if more than 1 CCD]
number of panels across to display.
bias : string
Name of bias frame to subtract, 'none' to ignore.
flat : string
Name of flat field to divide by, 'none' to ignore. Should normally
only be used in conjunction with a bias, although it does allow you
to specify a flat even if you haven't specified a bias.
defect : string
Name of defect file, 'none' to ignore.
setup : bool
True/yes to access the current windows from hdriver. Useful
during observing when seeting up windows, but not normally
otherwise. Next argument (hidden) is the URL to get to
hdriver. Once setup, you should probably turn this off to
avoid overloading hdriver, especially if in drift mode as
it makes a request for the windows for every frame.
hurl : string [if setup; hidden]
URL needed to access window setting from hdriver. The internal
server in hdriver must be switched on using rtplot_server_on
in the hdriver config file.
msub : bool
subtract the median from each window before scaling for the
image display or not. This happens after any bias subtraction.
iset : string [single character]
determines how the intensities are determined. There are three
options: 'a' for automatic simply scales from the minimum to the
maximum value found on a per CCD basis. 'd' for direct just takes
two numbers from the user. 'p' for percentile dtermines levels
based upon percentiles determined from the entire CCD on a per CCD
basis.
ilo : float [if iset='d']
lower intensity level
ihi : float [if iset='d']
upper intensity level
plo : float [if iset='p']
lower percentile level
phi : float [if iset='p']
upper percentile level
xlo : float
left-hand X-limit for plot
xhi : float
right-hand X-limit for plot (can actually be < xlo)
ylo : float
lower Y-limit for plot
yhi : float
upper Y-limit for plot (can be < ylo)
profit : bool [if plotting a single CCD only]
carry out profile fits or not. If you say yes, then on the first
plot, you will have the option to pick objects with a cursor. The
program will then attempt to track these from frame to frame, and
fit their profile. You may need to adjust 'first' to see anything.
The parameters used for profile fits are hidden and you may want to
invoke the command with 'prompt' the first time you try profile
fitting.
fdevice : string [if profit; hidden]
plot device for profile fits, PGPLOT-style name.
e.g. '/xs', '2/xs' etc.
fwidth : float [if profit; hidden]
fit plot width (inches). Set = 0 to let the program choose.
fheight : float [if profit; hidden]
fit plot height (inches). Set = 0 to let the program choose.
BOTH fwidth AND fheight must be non-zero to have any effect
method : string [if profit; hidden]
this defines the profile fitting method, either a gaussian or a
moffat profile. The latter is usually best.
beta : float [if profit and method == 'm'; hidden]
default Moffat exponent
fwhm : float [if profit; hidden]
default FWHM, unbinned pixels.
fwhm_min : float [if profit; hidden]
minimum FWHM to allow, unbinned pixels.
shbox : float [if profit; hidden]
half width of box for searching for a star, unbinned pixels. The
brightest target in a region +/- shbox around an intial position
will be found. 'shbox' should be large enough to allow for likely
changes in position from frame to frame, but try to keep it as
small as you can to avoid jumping to different targets and to
reduce the chances of interference by cosmic rays.
smooth : float [if profit; hidden]
FWHM for gaussian smoothing, binned pixels. The initial position
for fitting is determined by finding the maximum flux in a smoothed
version of the image in a box of width +/- shbox around the starter
position. Typically should be comparable to the stellar width. Its
main purpose is to combat cosmi rays which tend only to occupy a
single pixel.
splot : bool [if profit; hidden]
Controls whether an outline of the search box and a target number
is plotted (in red) or not.
fhbox : float [if profit; hidden]
half width of box for profile fit, unbinned pixels. The fit box is
centred on the position located by the initial search. It should
normally be > ~2x the expected FWHM.
hmin : float [if profit; hidden]
height threshold to accept a fit. If the height is below this
value, the position will not be updated. This is to help in cloudy
conditions.
read : float [if profit; hidden]
readout noise, RMS ADU, for assigning uncertainties
gain : float [if profit; hidden]
gain, ADU/count, for assigning uncertainties
thresh : float [if profit; hidden]
sigma rejection threshold for fits
"""
command, args = utils.script_args(args)
# get the inputs
with Cline('HIPERCAM_ENV', '.hipercam', command, args) as cl:
# register parameters
cl.register('source', Cline.GLOBAL, Cline.HIDE)
cl.register('device', Cline.LOCAL, Cline.HIDE)
cl.register('width', Cline.LOCAL, Cline.HIDE)
cl.register('height', Cline.LOCAL, Cline.HIDE)
cl.register('run', Cline.GLOBAL, Cline.PROMPT)
cl.register('first', Cline.LOCAL, Cline.PROMPT)
cl.register('trim', Cline.GLOBAL, Cline.PROMPT)
cl.register('ncol', Cline.GLOBAL, Cline.HIDE)
cl.register('nrow', Cline.GLOBAL, Cline.HIDE)
cl.register('twait', Cline.LOCAL, Cline.HIDE)
cl.register('tmax', Cline.LOCAL, Cline.HIDE)
cl.register('flist', Cline.LOCAL, Cline.PROMPT)
cl.register('ccd', Cline.LOCAL, Cline.PROMPT)
cl.register('pause', Cline.LOCAL, Cline.HIDE)
cl.register('plotall', Cline.LOCAL, Cline.HIDE)
cl.register('nx', Cline.LOCAL, Cline.PROMPT)
cl.register('bias', Cline.GLOBAL, Cline.PROMPT)
cl.register('flat', Cline.GLOBAL, Cline.PROMPT)
cl.register('defect', Cline.GLOBAL, Cline.PROMPT)
cl.register('setup', Cline.GLOBAL, Cline.PROMPT)
cl.register('hurl', Cline.GLOBAL, Cline.HIDE)
cl.register('msub', Cline.GLOBAL, Cline.PROMPT)
cl.register('iset', Cline.GLOBAL, Cline.PROMPT)
cl.register('ilo', Cline.GLOBAL, Cline.PROMPT)
cl.register('ihi', Cline.GLOBAL, Cline.PROMPT)
cl.register('plo', Cline.GLOBAL, Cline.PROMPT)
cl.register('phi', Cline.LOCAL, Cline.PROMPT)
cl.register('xlo', Cline.GLOBAL, Cline.PROMPT)
cl.register('xhi', Cline.GLOBAL, Cline.PROMPT)
cl.register('ylo', Cline.GLOBAL, Cline.PROMPT)
cl.register('yhi', Cline.GLOBAL, Cline.PROMPT)
cl.register('profit', Cline.LOCAL, Cline.PROMPT)
cl.register('fdevice', Cline.LOCAL, Cline.HIDE)
cl.register('fwidth', Cline.LOCAL, Cline.HIDE)
cl.register('fheight', Cline.LOCAL, Cline.HIDE)
cl.register('method', Cline.LOCAL, Cline.HIDE)
cl.register('beta', Cline.LOCAL, Cline.HIDE)
cl.register('fwhm', Cline.LOCAL, Cline.HIDE)
cl.register('fwhm_min', Cline.LOCAL, Cline.HIDE)
cl.register('shbox', Cline.LOCAL, Cline.HIDE)
cl.register('smooth', Cline.LOCAL, Cline.HIDE)
cl.register('splot', Cline.LOCAL, Cline.HIDE)
cl.register('fhbox', Cline.LOCAL, Cline.HIDE)
cl.register('hmin', Cline.LOCAL, Cline.HIDE)
cl.register('read', Cline.LOCAL, Cline.HIDE)
cl.register('gain', Cline.LOCAL, Cline.HIDE)
cl.register('thresh', Cline.LOCAL, Cline.HIDE)
# get inputs
source = cl.get_value('source', 'data source [hs, hl, us, ul, hf]',
'hl', lvals=('hs','hl','us','ul','hf'))
# set some flags
server_or_local = source.endswith('s') or source.endswith('l')
# plot device stuff
device = cl.get_value('device', 'plot device', '1/xs')
width = cl.get_value('width', 'plot width (inches)', 0.)
height = cl.get_value('height', 'plot height (inches)', 0.)
if server_or_local:
resource = cl.get_value('run', 'run name', 'run005')
if source == 'hs':
first = cl.get_value('first', 'first frame to plot', 1)
else:
first = cl.get_value('first', 'first frame to plot', 1, 0)
if source.startswith('u'):
trim = cl.get_value(
'trim',
'do you want to trim edges of windows? (ULTRACAM only)',
True
)
if trim:
ncol = cl.get_value(
'ncol', 'number of columns to trim from windows', 0)
nrow = cl.get_value(
'nrow', 'number of rows to trim from windows', 0)
else:
trim = False
twait = cl.get_value(
'twait', 'time to wait for a new frame [secs]', 1., 0.)
tmax = cl.get_value(
'tmax', 'maximum time to wait for a new frame [secs]', 10., 0.)
else:
resource = cl.get_value(
'flist', 'file list', cline.Fname('files.lis',hcam.LIST)
)
first = 1
trim = False
# define the panel grid. first get the labels and maximum dimensions
ccdinf = spooler.get_ccd_pars(source, resource)
try:
nxdef = cl.get_default('nx')
except:
nxdef = 3
if len(ccdinf) > 1:
ccd = cl.get_value('ccd', 'CCD(s) to plot [0 for all]', '0')
if ccd == '0':
ccds = list(ccdinf.keys())
else:
ccds = ccd.split()
check = set(ccdinf.keys())
if not set(ccds) <= check:
raise hcam.HipercamError(
'At least one invalid CCD label supplied'
)
if len(ccds) > 1:
nxdef = min(len(ccds), nxdef)
cl.set_default('nx', nxdef)
nx = cl.get_value('nx', 'number of panels in X', 3, 1)
else:
nx = 1
else:
nx = 1
ccds = list(ccdinf.keys())
cl.set_default('pause', 0.)
pause = cl.get_value(
'pause', 'time delay to add between'
' frame plots [secs]', 0., 0.
)
cl.set_default('plotall', False)
plotall = cl.get_value('plotall', 'plot all frames,'
' regardless of status?', False)
# bias frame (if any)
bias = cl.get_value(
'bias', "bias frame ['none' to ignore]",
cline.Fname('bias', hcam.HCAM), ignore='none'
)
if bias is not None:
# read the bias frame
bias = hcam.MCCD.read(bias)
fprompt = "flat frame ['none' to ignore]"
else:
fprompt = "flat frame ['none' is normal choice with no bias]"
# flat (if any)
flat = cl.get_value(
'flat', fprompt,
cline.Fname('flat', hcam.HCAM), ignore='none'
)
if flat is not None:
# read the flat frame
flat = hcam.MCCD.read(flat)
# defect file (if any)
dfct = cl.get_value(
'defect', "defect file ['none' to ignore]",
cline.Fname('defect', hcam.DFCT), ignore='none'
)
if dfct is not None:
# read the defect frame
dfct = defect.MccdDefect.read(dfct)
# Get windows from hdriver
setup = cl.get_value(
'setup', 'display current hdriver window settings', False
)
if setup:
hurl = cl.get_value(
'hurl', 'URL for hdriver windows', 'http://192.168.1.2:5100'
)
# define the display intensities
msub = cl.get_value('msub', 'subtract median from each window?', True)
iset = cl.get_value(
'iset', 'set intensity a(utomatically),'
' d(irectly) or with p(ercentiles)?',
'a', lvals=['a','d','p'])
iset = iset.lower()
plo, phi = 5, 95
ilo, ihi = 0, 1000
if iset == 'd':
ilo = cl.get_value('ilo', 'lower intensity limit', 0.)
ihi = cl.get_value('ihi', 'upper intensity limit', 1000.)
elif iset == 'p':
plo = cl.get_value('plo', 'lower intensity limit percentile',
5., 0., 100.)
phi = cl.get_value('phi', 'upper intensity limit percentile',
95., 0., 100.)
# region to plot
for i, cnam in enumerate(ccds):
nxtot, nytot, nxpad, nypad = ccdinf[cnam]
if i == 0:
xmin, xmax = float(-nxpad), float(nxtot + nxpad + 1)
ymin, ymax = float(-nypad), float(nytot + nypad + 1)
else:
xmin = min(xmin, float(-nxpad))
xmax = max(xmax, float(nxtot + nxpad + 1))
ymin = min(ymin, float(-nypad))
ymax = max(ymax, float(nytot + nypad + 1))
xlo = cl.get_value('xlo', 'left-hand X value', xmin, xmin, xmax)
xhi = cl.get_value('xhi', 'right-hand X value', xmax, xmin, xmax)
ylo = cl.get_value('ylo', 'lower Y value', ymin, ymin, ymax)
yhi = cl.get_value('yhi', 'upper Y value', ymax, ymin, ymax)
# profile fitting if just one CCD chosen
if len(ccds) == 1:
# many parameters for profile fits, although most are not plotted
# by default
profit = cl.get_value('profit', 'do you want profile fits?', False)
if profit:
fdevice = cl.get_value('fdevice', 'plot device for fits',
'2/xs')
fwidth = cl.get_value('fwidth', 'fit plot width (inches)', 0.)
fheight = cl.get_value(
'fheight', 'fit plot height (inches)', 0.)
method = cl.get_value(
'method', 'fit method g(aussian) or m(offat)',
'm', lvals=['g','m'])
if method == 'm':
beta = cl.get_value(
'beta', 'initial exponent for Moffat fits', 5., 0.5)
else:
beta = 0.
fwhm_min = cl.get_value(
'fwhm_min', 'minimum FWHM to allow [unbinned pixels]',
1.5, 0.01
)
fwhm = cl.get_value(
'fwhm', 'initial FWHM [unbinned pixels] for profile fits',
6., fwhm_min
)
shbox = cl.get_value(
'shbox', 'half width of box for initial location'
' of target [unbinned pixels]', 11., 2.)
smooth = cl.get_value(
'smooth', 'FWHM for smoothing for initial object'
' detection [binned pixels]', 6.)
splot = cl.get_value(
'splot', 'plot outline of search box?', True)
fhbox = cl.get_value(
'fhbox', 'half width of box for profile fit'
' [unbinned pixels]', 21., 3.)
hmin = cl.get_value(
'hmin', 'minimum peak height to accept the fit', 50.)
read = cl.get_value('read', 'readout noise, RMS ADU', 3.)
gain = cl.get_value('gain', 'gain, ADU/e-', 1.)
thresh = cl.get_value('thresh', 'number of RMS to reject at', 4.)
else:
profit = False
################################################################
#
# all the inputs have now been obtained. Get on with doing stuff
# open image plot device
imdev = hcam.pgp.Device(device)
if width > 0 and height > 0:
pgpap(width,height/width)
# set up panels and axes
nccd = len(ccds)
ny = nccd // nx if nccd % nx == 0 else nccd // nx + 1
# slice up viewport
pgsubp(nx,ny)
# plot axes, labels, titles. Happens once only
for cnam in ccds:
pgsci(hcam.pgp.Params['axis.ci'])
pgsch(hcam.pgp.Params['axis.number.ch'])
pgenv(xlo, xhi, ylo, yhi, 1, 0)
pglab('X','Y','CCD {:s}'.format(cnam))
# initialisations. 'last_ok' is used to store the last OK frames of each
# CCD for retrieval when coping with skipped data.
total_time = 0 # time waiting for new frame
fpos = [] # list of target positions to fit
fframe = True # waiting for first valid frame with profit
# plot images
with spooler.data_source(source, resource, first, full=False) as spool:
# 'spool' is an iterable source of MCCDs
n = 0
for mccd in spool:
if server_or_local:
# Handle the waiting game ...
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmax, total_time
)
if give_up:
print('rtplot stopped')
break
elif try_again:
continue
# Trim the frames: ULTRACAM windowed data has bad columns
# and rows on the sides of windows closest to the readout
# which can badly affect reduction. This option strips
# them.
if trim:
hcam.ccd.trim_ultracam(mccd, ncol, nrow)
# indicate progress
tstamp = Time(mccd.head['TIMSTAMP'], format='isot', precision=3)
print(
'{:d}, utc = {:s}, '.format(
mccd.head['NFRAME'], tstamp.iso), end=''
)
# accumulate errors
emessages = []
if n == 0:
if bias is not None:
# crop the bias on the first frame only
bias = bias.crop(mccd)
if flat is not None:
# crop the flat on the first frame only
flat = flat.crop(mccd)
if setup:
# Get windows from hdriver. Fair bit of error checking
# needed. 'got_windows' indicates if anything useful
# found, 'hwindows' is a list of (llx,lly,nx,ny) tuples
# if somthing is found.
try:
r = requests.get(
'http://192.168.1.2:5100',timeout=0.2
)
if r.text.strip() == 'No valid data available':
emessages.append(
'** bad return from hdriver = {:s}'.format(r.text.strip())
)
got_windows = False
elif r.text.strip() == 'fullframe':
# to help Stu out a bit, effectively just ignore this one
got_windows = False
else:
# OK, got something
lines = r.text.split('\r\n')
xbinh,ybinh,nwinh = lines[0].split()
xbinh,ybinh,nwinh = int(xbinh),int(ybinh),int(nwinh)
hwindows = []
for line in lines[1:nwinh+1]:
llxh,llyh,nxh,nyh = line.split()
hwindows.append(
(int(llxh),int(llyh),int(nxh),int(nyh))
)
if nwinh != len(hwindows):
emessages.append(
('** expected {:d} windows from'
' hdriver but got {:d}').format(nwinh,len(hwindows))
)
got_windows = False
got_windows = True
except (requests.exceptions.ConnectionError, socket.timeout,
requests.exceptions.Timeout) as err:
emessages.append(' ** hdriver error: {!r}'.format(err))
got_windows = False
else:
got_windows = False
# display the CCDs chosen
message = ''
pgbbuf()
for nc, cnam in enumerate(ccds):
ccd = mccd[cnam]
if plotall or ccd.is_data():
# this should be data as opposed to a blank frame
# between data frames that occur with nskip > 0
# subtract the bias
if bias is not None:
ccd -= bias[cnam]
# divide out the flat
if flat is not None:
ccd /= flat[cnam]
if msub:
# subtract median from each window
for wind in ccd.values():
wind -= wind.median()
# set to the correct panel and then plot CCD
ix = (nc % nx) + 1
iy = nc // nx + 1
pgpanl(ix,iy)
vmin, vmax = hcam.pgp.pCcd(
ccd,iset,plo,phi,ilo,ihi,
xlo=xlo, xhi=xhi, ylo=ylo, yhi=yhi
)
if got_windows:
# plot the current hdriver windows
pgsci(hcam.CNAMS['yellow'])
pgsls(2)
for llxh,llyh,nxh,nyh in hwindows:
pgrect(llxh-0.5,llxh+nxh-0.5,llyh-0.5,llyh+nyh-0.5)
if dfct is not None and cnam in dfct:
# plot defects
hcam.pgp.pCcdDefect(dfct[cnam])
# accumulate string of image scalings
if nc:
message += ', ccd {:s}: {:.1f}, {:.1f}, exp: {:.4f}'.format(
cnam,vmin,vmax,mccd.head['EXPTIME']
)
else:
message += 'ccd {:s}: {:.1f}, {:.1f}, exp: {:.4f}'.format(
cnam,vmin,vmax,mccd.head['EXPTIME']
)
pgebuf()
# end of CCD display loop
print(message)
for emessage in emessages:
print(emessage)
if ccd.is_data() and profit and fframe:
fframe = False
# cursor selection of targets after first plot, if profit
# accumulate list of starter positions
print('Please select targets for profile fitting. You can select as many as you like.')
x, y, reply = (xlo+xhi)/2, (ylo+yhi)/2, ''
ntarg = 0
pgsci(2)
pgslw(2)
while reply != 'Q':
print("Place cursor on fit target. Any key to register, 'q' to quit")
x, y, reply = pgcurs(x, y)
if reply == 'q':
break
else:
# check that the position is inside a window
wnam = ccd.inside(x, y, 2)
if wnam is not None:
# store the position, Window label, target number,
# box size fwhm, beta
ntarg += 1
fpos.append(Fpar(x,y,wnam,ntarg,shbox,fwhm,beta))
# report information, overplot search box
print(
('Target {:d} selected at {:.1f},'
'{:.1f} in window {:s}').format(ntarg,x,y,wnam)
)
if splot:
fpos[-1].plot()
if len(fpos):
print(len(fpos),'targets selected')
# if some targets were selected, open the fit plot device
fdev = hcam.pgp.Device(fdevice)
if fwidth > 0 and fheight > 0:
pgpap(fwidth,fheight/fwidth)
if ccd.is_data():
# carry out fits. Nothing happens if fpos is empty
for fpar in fpos:
# switch to the image plot
imdev.select()
# plot search box
if splot:
fpar.plot()
try:
# extract search box from the CCD. 'fpar' is updated later
# if the fit is successful to reflect the new position
swind = fpar.swind(ccd)
# carry out initial search
x,y,peak = swind.search(smooth, fpar.x, fpar.y, hmin, False)
# now for a more refined fit. First extract fit Window
fwind = ccd[fpar.wnam].window(
x-fhbox, x+fhbox, y-fhbox, y+fhbox
)
# crude estimate of sky background
sky = np.percentile(fwind.data, 50)
# refine the Aperture position by fitting the profile
(sky, height, x, y, fwhm, beta), epars, \
(wfit, X, Y, sigma, chisq, nok,
nrej, npar, message) = hcam.fitting.combFit(
fwind, method, sky, peak-sky,
x, y, fpar.fwhm, fwhm_min, False,
fpar.beta, read, gain, thresh
)
print('Targ {:d}: {:s}'.format(fpar.ntarg,message))
if peak > hmin and ccd[fpar.wnam].distance(x,y) > 1:
# update some initial parameters for next time
if method == 'g':
fpar.x, fpar.y, fpar.fwhm = x, y, fwhm
elif method == 'm':
fpar.x, fpar.y, fpar.fwhm, \
fpar.beta = x, y, fwhm, beta
# plot values versus radial distance
ok = sigma > 0
R = np.sqrt((X-x)**2+(Y-y)**2)
fdev.select()
vmin = min(sky, sky+height, fwind.min())
vmax = max(sky, sky+height, fwind.max())
extent = vmax-vmin
pgeras()
pgvstd()
pgswin(0,R.max(),vmin-0.05*extent,vmax+0.05*extent)
pgsci(4)
pgbox('bcnst',0,0,'bcnst',0,0)
pgsci(2)
pglab('Radial distance [unbinned pixels]','Counts','')
pgsci(1)
pgpt(R[ok].flat, fwind.data[ok].flat, 1)
if nrej:
pgsci(2)
pgpt(R[~ok].flat, fwind.data[~ok].flat, 5)
# line fit
pgsci(3)
r = np.linspace(0,R.max(),400)
if method == 'g':
alpha = 4*np.log(2.)/fwhm**2
f = sky+height*np.exp(-alpha*r**2)
elif method == 'm':
alpha = 4*(2**(1/beta)-1)/fwhm**2
f = sky+height/(1+alpha*r**2)**beta
pgline(r,f)
# back to the image to plot a circle of radius FWHM
imdev.select()
pgsci(3)
pgcirc(x,y,fwhm)
else:
print(' *** below detection threshold; position & FWHM will not updated')
pgsci(2)
# plot location on image as a cross
pgpt1(x, y, 5)
except hcam.HipercamError as err:
print(' >> Targ {:d}: fit failed ***: {!s}'.format(
fpar.ntarg, err)
)
pgsci(2)
if pause > 0.:
# pause between frames
time.sleep(pause)
# update the frame number
n += 1
def psf_reduce(args=None):
"""``psf_reduce [source] rfile (run first twait tmax | flist) log lplot implot
(ccd nx msub xlo xhi ylo yhi iset (ilo ihi | plo phi))``
Performs PSF photometry on a sequence of multi-CCD images, plotting
lightcurves as images come in. It performs PSF photometry on specific
targets, defined in an aperture file using |psf_setaper| or |setaper|.
Aperture repositioning is identical to |reduce|, and the PSF used in photometry
is set by fitting well separated reference stars. For the best results, a
suitable strategy is to only reduce data from a small window near the target.
Set well separated bright stars as reference apertures and link fainter, or
blended objects to the references.
|psf_reduce| can source data from both the ULTRACAM and HiPERCAM servers, from
local 'raw' ULTRACAM and HiPERCAM files (i.e. .xml + .dat for ULTRACAM, 3D
FITS files for HiPERCAM) and from lists of HiPERCAM '.hcm' files. If you
have data from a different instrument you should convert into the
FITS-based hcm format.
|psf_reduce| is primarily configured from a file with extension ".red". This
contains a series of directives, e.g. to say how to re-position the apertures.
An initial reduce file is best generated with the script |genred| after you
have created an aperture file. This contains lots of help on what to do.
A |psf_reduce| run can be terminated at any point with ctrl-C without doing
any harm. You may often want to do this at the start in order to adjust
parameters of the reduce file.
Parameters:
source : string [hidden]
Data source, five options:
| 'hs': HiPERCAM server
| 'hl': local HiPERCAM FITS file
| 'us': ULTRACAM server
| 'ul': local ULTRACAM .xml/.dat files
| 'hf': list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab' or
converted from foreign data formats.
rfile : string
the "reduce" file, i.e. ASCII text file suitable for reading by
ConfigParser. Best seen by example as it has many parts. Generate
a starter file with |genred|.
run : string [if source ends 's' or 'l']
run number to access, e.g. 'run034'
first : int [if source ends 's' or 'l']
exposure number to start from. 1 = first frame; set = 0 to
always try to get the most recent frame (if it has changed)
last : int [if source ends 's' or 'l', hidden]
last frame to reduce. 0 to just continue until the end. This is
not prompted for by default and must be set explicitly. It
defaults to 0 if not set. Its purpose is to allow accurate
profiling tests.
twait : float [if source ends 's'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [if source ends 's'; hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
flist : string [if source ends 'f']
name of file list
log : string
log file for the results
tkeep : float
maximum number of minutes of data to store in internal buffers, 0
for the lot. When large numbers of frames are stored, performance
can be slowed (although I am not entirely clear why) in which case
it makes sense to lose the earlier points (without affecting the
saving to disk). This parameter also gives operation similar to that
of "max_xrange" parameter in the ULTRACAM pipeline whereby just
the last few minutes are shown.
lplot : bool
flag to indicate you want to plot the light curve. Saves time not
to especially in high-speed runs.
implot : bool
flag to indicate you want to plot images.
ccd : string [if implot]
CCD(s) to plot, '0' for all, '1 3' to plot '1' and '3' only, etc.
nx : int [if implot]
number of panels across to display.
msub : bool [if implot]
subtract the median from each window before scaling for the
image display or not. This happens after any bias subtraction.
xlo : float [if implot]
left-hand X-limit for plot
xhi : float [if implot]
right-hand X-limit for plot (can actually be < xlo)
ylo : float [if implot]
lower Y-limit for plot
yhi : float [if implot]
upper Y-limit for plot (can be < ylo)
iset : string [if implot]
determines how the intensities are determined. There are three
options: 'a' for automatic simply scales from the minimum to the
maximum value found on a per CCD basis. 'd' for direct just takes
two numbers from the user. 'p' for percentile dtermines levels
based upon percentiles determined from the entire CCD on a per CCD
basis.
ilo : float [if implot and iset='d']
lower intensity level
ihi : float [if implot and iset='d']
upper intensity level
plo : float [if implot and iset='p']
lower percentile level
phi : float [if implot and iset='p']
upper percentile level
"""
command, args = utils.script_args(args)
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("source", Cline.GLOBAL, Cline.HIDE)
cl.register("rfile", Cline.GLOBAL, Cline.PROMPT)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("last", Cline.LOCAL, Cline.HIDE)
cl.register("twait", Cline.LOCAL, Cline.HIDE)
cl.register("tmax", Cline.LOCAL, Cline.HIDE)
cl.register("flist", Cline.LOCAL, Cline.PROMPT)
cl.register("log", Cline.GLOBAL, Cline.PROMPT)
cl.register("tkeep", Cline.GLOBAL, Cline.PROMPT)
cl.register("lplot", Cline.LOCAL, Cline.PROMPT)
cl.register("implot", Cline.LOCAL, Cline.PROMPT)
cl.register("ccd", Cline.LOCAL, Cline.PROMPT)
cl.register("nx", Cline.LOCAL, Cline.PROMPT)
cl.register("msub", Cline.GLOBAL, Cline.PROMPT)
cl.register("iset", Cline.GLOBAL, Cline.PROMPT)
cl.register("ilo", Cline.GLOBAL, Cline.PROMPT)
cl.register("ihi", Cline.GLOBAL, Cline.PROMPT)
cl.register("plo", Cline.GLOBAL, Cline.PROMPT)
cl.register("phi", Cline.LOCAL, Cline.PROMPT)
cl.register("xlo", Cline.GLOBAL, Cline.PROMPT)
cl.register("xhi", Cline.GLOBAL, Cline.PROMPT)
cl.register("ylo", Cline.GLOBAL, Cline.PROMPT)
cl.register("yhi", Cline.GLOBAL, Cline.PROMPT)
# get inputs
source = cl.get_value(
"source",
"data source [hs, hl, us, ul, hf]",
"hl",
lvals=("hs", "hl", "us", "ul", "hf"),
)
# set some flags
server_or_local = source.endswith("s") or source.endswith("l")
# the reduce file
rfilen = cl.get_value("rfile", "reduce file",
cline.Fname("reduce.red", hcam.RED))
try:
rfile = Rfile.read(rfilen)
except hcam.HipercamError as err:
# abort on failure to read as there are many ways to get reduce
# files wrong
print(err, file=sys.stderr)
exit(1)
if server_or_local:
resource = cl.get_value("run", "run name", "run005")
first = cl.get_value("first", "first frame to reduce", 1, 0)
cl.set_default("last", 0)
last = cl.get_value("last", "last frame to reduce", 0, 0)
if last and last < first:
print("Cannot set last < first unless last == 0")
print("*** psf_reduce aborted")
exit(1)
twait = cl.get_value("twait",
"time to wait for a new frame [secs]", 1.0,
0.0)
tmx = cl.get_value("tmax",
"maximum time to wait for a new frame [secs]",
10.0, 0.0)
else:
resource = cl.get_value("flist", "file list",
cline.Fname("files.lis", hcam.LIST))
first = 1
last = 0
log = cl.get_value(
"log",
"name of log file to store results",
cline.Fname("reduce.log", hcam.LOG, cline.Fname.NEW),
)
tkeep = cl.get_value(
"tkeep",
"number of minute of data to"
" keep in internal buffers (0 for all)",
0.0,
0.0,
)
lplot = cl.get_value("lplot", "do you want to plot light curves?",
True)
implot = cl.get_value("implot", "do you want to plot images?", True)
if implot:
# define the panel grid. first get the labels and maximum
# dimensions
ccdinf = spooler.get_ccd_pars(source, resource)
nxdef = cl.get_default("nx", 3)
if len(ccdinf) > 1:
ccd = cl.get_value("ccd", "CCD(s) to plot [0 for all]", "0")
if ccd == "0":
ccds = list(ccdinf.keys())
else:
ccds = ccd.split()
if len(ccds) > 1:
nxdef = min(len(ccds), nxdef)
cl.set_default("nx", nxdef)
nx = cl.get_value("nx", "number of panels in X", 3, 1)
else:
nx = 1
else:
nx = 1
ccds = list(ccdinf.keys())
# define the display intensities
msub = cl.get_value("msub", "subtract median from each window?",
True)
iset = cl.get_value(
"iset",
"set intensity a(utomatically),"
" d(irectly) or with p(ercentiles)?",
"a",
lvals=["a", "d", "p"],
)
plo, phi = 5, 95
ilo, ihi = 0, 1000
if iset == "d":
ilo = cl.get_value("ilo", "lower intensity limit", 0.0)
ihi = cl.get_value("ihi", "upper intensity limit", 1000.0)
elif iset == "p":
plo = cl.get_value("plo", "lower intensity limit percentile",
5.0, 0.0, 100.0)
phi = cl.get_value("phi", "upper intensity limit percentile",
95.0, 0.0, 100.0)
# region to plot
for i, cnam in enumerate(ccds):
nxtot, nytot, nxpad, nypad = ccdinf[cnam]
if i == 0:
xmin, xmax = float(-nxpad), float(nxtot + nxpad + 1)
ymin, ymax = float(-nypad), float(nytot + nypad + 1)
else:
xmin = min(xmin, float(-nxpad))
xmax = max(xmax, float(nxtot + nxpad + 1))
ymin = min(ymin, float(-nypad))
ymax = max(ymax, float(nytot + nypad + 1))
xlo = cl.get_value("xlo",
"left-hand X value",
xmin,
xmin,
xmax,
enforce=False)
xhi = cl.get_value("xhi",
"right-hand X value",
xmax,
xmin,
xmax,
enforce=False)
ylo = cl.get_value("ylo",
"lower Y value",
ymin,
ymin,
ymax,
enforce=False)
yhi = cl.get_value("yhi",
"upper Y value",
ymax,
ymin,
ymax,
enforce=False)
else:
xlo, xhi, ylo, yhi = None, None, None, None
# save list of parameter values for writing to the reduction file
plist = cl.list()
################################################################
#
# all the inputs have now been obtained. Get on with doing stuff
if implot:
plot_lims = (xlo, xhi, ylo, yhi)
else:
plot_lims = None
imdev, lcdev, spanel, tpanel, xpanel, ypanel, lpanel = setup_plots(
rfile, ccds, nx, plot_lims, implot, lplot)
# a couple of initialisations
total_time = 0 # time waiting for new frame
# dictionary of dictionaries for looking up the window associated with a
# given aperture, i.e. mccdwins[cnam][apnam] give the name of the Window.
mccdwins = {}
if lplot:
lbuffer, xbuffer, ybuffer, tbuffer, sbuffer = setup_plot_buffers(rfile)
else:
lbuffer, xbuffer, ybuffer, tbuffer, sbuffer = None, None, None, None, None
############################################
#
# open the log file and write headers
#
with LogWriter(log, rfile, hipercam_version, plist) as logfile:
ncpu = rfile["general"]["ncpu"]
if ncpu > 1:
pool = multiprocessing.Pool(processes=ncpu)
else:
pool = None
# whether settings have been initialised
initialised = False
# containers for the processed and raw MCCD groups
# and their frame numbers
pccds, mccds, nframes = [], [], []
##############################################
#
# Finally, start winding through the frames
#
with spooler.data_source(source, resource, first, full=False) as spool:
# 'spool' is an iterable source of MCCDs
for nf, mccd in enumerate(spool):
if server_or_local:
# Handle the waiting game ...
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmx, total_time)
if give_up:
# Giving up, but need to handle any partially filled
# frame group
if len(mccds):
# finish processing remaining frames. This step
# will only occur if we have at least once passed
# to later stages during which read and gain will
# be set up
results = processor(pccds, mccds, nframes)
# write out results to the log file
alerts = logfile.write_results(results)
# print out any accumulated alert messages
if len(alerts):
print("\n".join(alerts))
update_plots(
results,
rfile,
implot,
lplot,
imdev,
lcdev,
pccd,
ccds,
msub,
nx,
iset,
plo,
phi,
ilo,
ihi,
xlo,
xhi,
ylo,
yhi,
lpanel,
xpanel,
ypanel,
tpanel,
spanel,
tkeep,
lbuffer,
xbuffer,
ybuffer,
tbuffer,
sbuffer,
)
print("psf_reduce stopped")
break
elif try_again:
continue
# indicate progress
if "NFRAME" in mccd.head:
nframe = mccd.head["NFRAME"]
else:
nframe = nf + 1
if last and nframe > last:
if len(mccds):
# finish processing remaining frames
results = processor(pccds, mccds, nframes)
# write out results to the log file
alerts = logfile.write_results(results)
# print out any accumulated alert messages
if len(alerts):
print("\n".join(alerts))
update_plots(
results,
rfile,
implot,
lplot,
imdev,
lcdev,
pccd,
ccds,
msub,
nx,
iset,
plo,
phi,
ilo,
ihi,
xlo,
xhi,
ylo,
yhi,
lpanel,
xpanel,
ypanel,
tpanel,
spanel,
tkeep,
lbuffer,
xbuffer,
ybuffer,
tbuffer,
sbuffer,
)
print("\nHave reduced up to the last frame set.")
print("psf_reduce finished")
break
print(
"Frame {:d}: {:s} [{:s}]".format(
nframe,
mccd.head["TIMSTAMP"],
"OK" if mccd.head.get("GOODTIME", True) else "NOK",
),
end="" if implot else "\n",
)
if not initialised:
# This is the first frame which allows us to make
# some checks and initialisations.
read, gain, ok = initial_checks(mccd, rfile)
# Define the CCD processor function object
processor = ProcessCCDs(rfile, read, gain, ccdproc, pool)
# set flag to show we are set
if not ok:
break
initialised = True
# De-bias the data. Retain a copy of the raw data as 'mccd'
# in order to judge saturation. Processed data called 'pccd'
if rfile.bias is not None:
# subtract bias
pccd = mccd - rfile.bias
else:
# no bias subtraction
pccd = mccd.copy()
if rfile.flat is not None:
# apply flat field to processed frame
pccd /= rfile.flat
# Acummulate frames into processing groups for faster
# parallelisation
pccds.append(pccd)
mccds.append(mccd)
nframes.append(nframe)
if len(pccds) == rfile["general"]["ngroup"]:
# parallel processing. This should usually be the first
# points at which it takes place
results = processor(pccds, mccds, nframes)
# write out results to the log file
alerts = logfile.write_results(results)
# print out any accumulated alert messages
if len(alerts):
print("\n".join(alerts))
update_plots(
results,
rfile,
implot,
lplot,
imdev,
lcdev,
pccds[-1],
ccds,
msub,
nx,
iset,
plo,
phi,
ilo,
ihi,
xlo,
xhi,
ylo,
yhi,
lpanel,
xpanel,
ypanel,
tpanel,
spanel,
tkeep,
lbuffer,
xbuffer,
ybuffer,
tbuffer,
sbuffer,
)
# Reset the frame buffers
pccds, mccds, nframes = [], [], []
def genred(args=None):
"""``genred apfile rfile bias dark flat fmap fpair seeing (sworst
binfac) template (inst (nccd (ccd) nonlin sat scale readout
gain))``
Generates a reduce file as needed by |reduce| or |psf_reduce|. You
give it the name of an aperture file, calibration frames and a few
other parameters and it will write out a reduce file which you can
then refine by hand. The aim is to try to get a reduce file that
is self-consistent and works to get observing started as soon as
possible. The parameters are not necessarily the best.
It is assumed that the target is called '1', the main comparison '2'.
To avoid endless prompts, genred does not try to prompt for all
parameters even via hidden ones (a change from its previous
behaviour). However, if you find yourself always making the samed
edits to the file produced by genred, you may start wanting a way
to alter some of the otherwise fixed parameters. You can do so by
supplying a template file, which can just be from a previous run
of genred where such parameters can be altered by editing with a
standard text editor before running genred.
genred recognises some standard instrument telescope combinations
which it uses to set parameters such as the pixel scale and
readout noise, but if you choose 'other', you will be prompted for
these details. The main parameters it does prompt for are
calibration files, and it allows you to define a "seeing" which
then controls the setting of various profile fit parameters. This
can be ignored by setting = 0, in which case any template
parameters will be passed unchanged.
Parameters:
apfile : str
the input aperture file created using |setaper| (default
extension .ape). This will be read for the targets. The
main target will be assumed to have been called '1', the
main comparison '2'. If there is a '3' it will be plotted
relative to '2'; all others will be ignored for plotting
purposes. Target '2' will be used to define the position
and transmission plots for one CCD only [user
definable]. Target '1' will be used for the seeing plot
unless it is linked when target '2' will be used instead.
rfile : str
the output reduce file created by this script. The main
target will be assumed to have been called '1', the main
comparison '2'. If there is a '3' it will be plotted
relative to '2'; all others will be ignored for plotting
purposes.
bias : str
Name of bias frame; 'none' to ignore.
dark : str
Name of dark frame; 'none' to ignore.
flat : str
Name of flat field frame; 'none' to ignore.
fmap : str
Name of fringe frame; 'none' to ignore.
fpair : str [if fmap not 'none']
File defining pairs to measure fringe amplitudes.
seeing : float
estimate of seeing which will be used to define several of
the profile fitting parameters. Enter 0 to ignore and use
defaults from the template (or genred if no template)
instead.
sworst : float [if seeing > 0]
worst seeing expected during run. Expands the search and
fitting boxes which can cause difficulties if they are
too small. It also sets the maximum target aperture radius
which will set to ~ 1.8*sworst, converted to pixels, and the
sky annulus radii. It has to be at least 2*seeing. Always
remember that you can override the automated settings using
a pre-edited template and seeing = 0.
binfac : int [if seeing > 0]
binning factor. e.g. 4 if using 4x4. Needed to optimise some
profile parameters.
template : str
Reduce file to use as a template for any parameters not set
by genred. This allows one to tweak settings by running
genred without a template, then modifying the resultant
file and using it as a template. 'none' to ignore, and then
these values will be set by genred by default. Some will be
modified anyway (e.g. the calibration file names), and if
seeing > 0, then several of the profile fitting paramaters
will be adapted to the value given along with the
instrument parameters. genred will try to keep as many of
the 'extraction', 'position', 'transmission', 'light' etc
lines associated with particular CCDs in this case without
modification, and will pass readout, saturation levels etc
unchanged from the template.
inst : str [if template == 'none']
the instrument. 'hipercam-gtc', 'ultracam-ntt', 'ultraspec-tnt',
'ultracam-wht', 'ultracam-vlt', or 'other'. Sets pixel scale and
standard readnoise params.
nccd : int [if inst == 'other']
number of CCDs.
ccd : int [if nccd > 1]
the CCD to use for position plots
nonlin : list(float) [if inst == 'other']
values for warning of non linearity
sat : list(float) [if inst == 'other']
values for warning of saturation
scale : float [if inst == 'other']
image scale in arcsec/pixel
readout : float | string [if inst == 'other']
readout noise, RMS ADU. Can either be a single value or an
hcm file.
gain : float | string [if inst == 'other']
gain, electrons per ADU. Can either be a single value or an
hcm file.
"""
command, args = utils.script_args(args)
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("apfile", Cline.LOCAL, Cline.PROMPT)
cl.register("rfile", Cline.GLOBAL, Cline.PROMPT)
cl.register("comment", Cline.LOCAL, Cline.PROMPT)
cl.register("bias", Cline.LOCAL, Cline.PROMPT)
cl.register("dark", Cline.LOCAL, Cline.PROMPT)
cl.register("flat", Cline.LOCAL, Cline.PROMPT)
cl.register("fmap", Cline.LOCAL, Cline.PROMPT)
cl.register("fpair", Cline.LOCAL, Cline.PROMPT)
cl.register("seeing", Cline.LOCAL, Cline.PROMPT)
cl.register("sworst", Cline.LOCAL, Cline.PROMPT)
cl.register("binfac", Cline.LOCAL, Cline.PROMPT)
cl.register("inst", Cline.LOCAL, Cline.PROMPT)
cl.register("nccd", Cline.LOCAL, Cline.PROMPT)
cl.register("ccd", Cline.LOCAL, Cline.PROMPT)
cl.register("nonlin", Cline.LOCAL, Cline.PROMPT)
cl.register("sat", Cline.LOCAL, Cline.PROMPT)
cl.register("scale", Cline.LOCAL, Cline.PROMPT)
cl.register("readout", Cline.LOCAL, Cline.PROMPT)
cl.register("gain", Cline.LOCAL, Cline.PROMPT)
cl.register("template", Cline.LOCAL, Cline.PROMPT)
# get inputs
# the aperture file
apfile = cl.get_value(
"apfile", "aperture input file", cline.Fname("aper.ape", hcam.APER)
)
# Read the aperture file
aper = hcam.MccdAper.read(apfile)
# the reduce file
root = os.path.splitext(os.path.basename(apfile))[0]
cl.set_default("rfile", cline.Fname(root, hcam.RED, cline.Fname.NEW))
rfile = cl.get_value(
"rfile", "reduce output file",
cline.Fname("reduce.red", hcam.RED, cline.Fname.NEW),
)
# calibrations
# bias frame
bias = cl.get_value(
"bias",
"bias frame ['none' to ignore]",
cline.Fname("bias", hcam.HCAM),
ignore="none",
)
# dark frame
dark = cl.get_value(
"dark",
"dark field frame ['none' to ignore]",
cline.Fname("dark", hcam.HCAM),
ignore="none",
)
# flat field frame
flat = cl.get_value(
"flat",
"flat field frame ['none' to ignore]",
cline.Fname("flat", hcam.HCAM),
ignore="none",
)
# fringe frame
fmap = cl.get_value(
"fmap",
"fringe frame ['none' to ignore]",
cline.Fname("fringe", hcam.HCAM),
ignore="none",
)
if fmap is not None:
fpair = cl.get_value(
"fpair", "fringe pair file",
cline.Fname("fpair", hcam.FRNG),
)
else:
fpair = None
seeing = cl.get_value(
"seeing",
"approximate seeing [arcsec, 0 to ignore]",
1.0, 0.
)
if seeing > 0.:
sworst = cl.get_value(
"sworst",
"worst expected seeing [arcsec]",
max(3., 2.5*seeing), 2.*seeing
)
binfac = cl.get_value(
"binfac", "binning factor",
1, 1
)
# template
template = cl.get_value(
"template",
"template reduce file ['none' to ignore]",
cline.Fname("template", hcam.RED),
ignore="none",
)
if template is None:
instrument = cl.get_value(
"inst",
"the instrument-telescope",
"hipercam-gtc",
lvals=[
"hipercam-gtc", "ultracam-ntt",
"ultracam-wht", "ultracam-vlt",
"ultraspec-tnt", "other"
],
)
if instrument.startswith("hipercam"):
warn_levels = """
warn = 1 50000 64000
warn = 2 50000 64000
warn = 3 50000 64000
warn = 4 50000 64000
warn = 5 50000 64000
"""
maxcpu = 5
skipbadt = False
scale = 0.081
readout = '4.2'
gain = '1.1'
nccd = 5
ccd = '2'
elif instrument.startswith("ultracam"):
warn_levels = """
warn = 1 28000 64000
warn = 2 28000 64000
warn = 3 50000 64000
"""
maxcpu = 3
skipbadt = True
readout = '2.8'
gain = '1.1'
nccd = 3
ccd = '2'
if instrument.endswith("ntt"):
scale = 0.35
elif instrument.endswith("wht"):
scale = 0.30
elif instrument.endswith("vlt"):
scale = 0.15
elif instrument.startswith("ultraspec"):
warn_levels = """
warn = 1 60000 64000
"""
maxcpu = 1
skipbadt = True
scale = 0.45
readout = '5.0'
gain = '0.8'
nccd = 1
ccd = '1'
elif instrument == "other":
# unrecognised instrument need extra stuff
nccd = cl.get_value("nccd", "how many CCDs?", 3, 1)
if nccd > 1:
ccd = cl.get_value("ccd", "which CCD for the position plot?", 1, 1, nccd)
ccd = str(ccd)
else:
ccd = '1'
# non-linearity warning levels
nonlin = cl.get_default("nonlin")
if nonlin is not None and len(nonlin) != nccd:
cl.set_default("nonlin", nccd * (50000.,))
nonlins = cl.get_value(
"nonlin",
"levels to indicate non-linear behaviour, 1 per CCD",
nccd*(50000.,)
)
# saturation warning levels
sat = cl.get_default("sat")
if sat is not None and len(sat) != nccd:
cl.set_default("sat", nccd * (62000.,))
sats = cl.get_value(
"sat",
"levels to indicate saturation, 1 per CCD",
nccd*(62000.,)
)
warn_levels = ""
for n, (nonlin,sat) in enumerate(zip(nonlins,sats)):
warn_levels += \
f"warn = {n+1} {nonlin} {sat}\n"
maxcpu = nccd
scale = cl.get_value(
"scale", "image scale [arcsec/unbinned pixel]",
0.3, 0.001
)
readout = cl.get_value(
"readout", "readout noise, RMS ADU (float or file name)", "4.5"
)
gain = cl.get_value(
"gain", "gain, electrons/ADU, (float or file name)", "1.1"
)
else:
raise hcam.HipercamError(
"Programming error: instrument unrecognised"
)
if ccd not in aper:
# make sure 'ccd' is in aper, even if our
# favoured one isn't
for cnam in aper:
ccd = cnam
break
################################################################
#
# all the inputs have now been obtained. Get on with doing stuff
if template is not None:
# read the template to define many unprompted values
# shortcut names define.
tvals = reduction.Rfile.read(template, False)
gsec = tvals['general']
instrument = gsec.get('instrument')
scale = gsec.get('scale')
warn_levels = ""
for value in gsec['warn']:
warn_levels += f'warn = {value}\n'
asec = tvals['apertures']
psfsec = tvals['psf_photom']
sksec = tvals['sky']
csec = tvals['calibration']
lcsec = tvals['lcplot']
lsec = tvals['light']
psec = tvals.get('position',None)
tsec = tvals.get('transmission',None)
ssec = tvals.get('seeing',None)
fsec = tvals.get('focal_mask',None)
else:
# define default values for unprompted params.
if seeing == 0.:
seeing = 1.5
sworst = 4.0
binfac = 1
# same name as read into template
tvals = {}
# general section
gsec = tvals['general'] = {}
gsec['instrument'] = instrument
gsec['scale'] = scale
gsec['ldevice'] = '1/xs'
gsec['lwidth'] = 0.
gsec['lheight'] = 0.
gsec['idevice'] = '2/xs'
gsec['iwidth'] = 0.
gsec['iheight'] = 0.
gsec['toffset'] = 0
gsec['skipbadt'] = 'yes'
gsec['ncpu'] = 1
gsec['ngroup'] = 1
# aperture section
asec = tvals['apertures'] = {}
asec['location'] = 'variable'
asec['search_smooth_fft'] = 'no'
asec['fit_method'] = 'moffat'
asec['fit_beta'] = 5.0
asec['fit_beta_max'] = 20.0
asec['fit_ndiv'] = 0
asec['fit_fwhm_fixed'] = 'no'
asec['fit_thresh'] = 8.
asec['fit_height_min_ref'] = 40.
asec['fit_height_min_nrf'] = 10.
asec['fit_alpha'] = 0.1
# sky
sksec = tvals['sky'] = {}
sksec['error'] = 'variance'
sksec['method'] = 'clipped'
sksec['thresh'] = 3.0
# calibration
csec = tvals['calibration'] = {}
csec['crop'] = 'yes'
csec['nhalf'] = 3
csec['rmin'] = -1.
csec['rmax'] = 2.
csec['readout'] = readout
csec['gain'] = gain
# PSF photom
psfsec = tvals['psf_photom'] = {}
psfsec['gfac'] = 3.0
psfsec['fit_half_width'] = 15.0
psfsec['positions'] = 'fixed'
# light curve
lcsec = tvals['lcplot'] = {}
lcsec['xrange'] = 0
lcsec['extend_x'] = 10.0
# light
lsec = tvals['light'] = {}
lsec['linear'] = 'no'
lsec['y_fixed'] = 'no'
lsec['y1'] = 0
lsec['y2'] = 0
lsec['extend_y'] = 0.1
# position
psec = tvals['position'] = {}
psec['height'] = 0.5
psec['x_fixed'] = 'no'
psec['x_min'] = -5.0
psec['x_max'] = +5.0
psec['y_fixed'] = 'no'
psec['y_min'] = -5.0
psec['y_max'] = +5.0
psec['extend_y'] = 0.2
# transmission
tsec = tvals['transmission'] = {}
tsec['height'] = 0.5
tsec['ymax'] = 110.
# seeing
ssec = tvals['seeing'] = {}
ssec['height'] = 0.5
ssec['ymax'] = 2.999
ssec['y_fixed'] = 'no'
ssec['extend_y'] = 0.2
fsec = tvals['focal_mask'] = {}
fsec['demask'] = 'no'
fsec['dthresh'] = 3.0
# apply seeing/instrument-related fixes...
# *always* gets run in the no template case
if seeing > 0.:
asec['search_half_width'] = max(5., sworst/scale)
asec['search_smooth_fwhm'] = max(2., seeing/scale/binfac)
asec['fwhm_min'] = 1.5*binfac
asec['fwhm'] = max(1.5*binfac, seeing/scale)
asec['fwhm_max'] = 3.*sworst/scale
asec['fit_max_shift'] = max(1., sworst/scale/4.)
asec['fit_half_width'] = max(5., 2*sworst/scale)
asec['fit_diff'] = max(1., sworst/scale/6.)
rfac = 1.8
ramin = 3*binfac
ramax = max(ramin,rfac*sworst/scale)
sinner = ramax
souter = 1.5*sinner
# standard colours for CCDs
if instrument.startswith("hipercam"):
CCD_COLS = {
"1": "blue",
"2": "green",
"3": "orange",
"4": "red",
"5": "darkred",
}
elif instrument.startswith("ultracam"):
CCD_COLS = {
"1": "red",
"2": "green",
"3": "blue",
}
elif instrument.startswith("ultraspec"):
CCD_COLS = {
"1": "green",
}
elif instrument == "other":
# 'other'
CCD_COLS = {
"1": "green",
"2": "blue",
"3": "red",
"4": "orange",
"5": "purple",
}
else:
raise hcam.HipercamError(
"Programming error: instrument unrecognised"
)
# Extraction lines
extraction = ""
if template is None:
# generate one for each CCD
for cnam in aper:
extraction += (
f"{cnam} = variable normal"
f" {rfac:.2f} {ramin:.1f} {ramax:.1f}"
f" 2.5 {sinner:.1f} {sinner:.1f}"
f" 3.0 {souter:.1f} {souter:.1f}\n"
)
else:
# pass through unchanged where possible if
# a template being used but check for consistency
esec = tvals['extraction']
for cnam, lst in esec.items():
if cnam in aper:
# unpack items
ltype,etype,rfac,ramin,ramax,sifac,simin,simax,sofac,somin,somax = lst
extraction += (
f"{cnam} = variable normal"
f" {rfac:.2f} {ramin:.1f} {ramax:.1f}"
f" {sifac:.2f} {simin:.1f} {simax:.1f}"
f" {sofac:.2f} {somin:.1f} {somax:.1f}\n"
)
else:
warnings.warn(
f'CCD {cnam} has an extraction line in {template} but no apertures in {apfile} and will be skipped'
)
# warn if there are apertures for a CCD but no extraction line
for cnam in aper:
if cnam not in esec:
warnings.warn(
f'CCD {cnam} has apertures defined in {apfile} but no extraction line in {template}'
)
# Generate the light curve plot lines
light_plot = ""
no_light = True
if template is None:
# Here we assume target in aperture 1, main comparison in aperture 2
for cnam in aper:
ccdaper = aper[cnam]
if "1" in ccdaper and "2" in ccdaper:
# favour 2 as the comparison
light_plot += (
f'plot = {cnam} 1 2 0 1 {CCD_COLS.get(cnam,"black")} ! '
" # ccd, targ, comp, off, fac, dcol, ecol\n"
)
no_light = False
elif "1" in ccdaper and "3" in ccdaper:
# but try 3 if need be
light_plot += (
f'plot = {cnam} 1 3 0 1 {CCD_COLS.get(cnam,"black")} ! '
" # ccd, targ, comp, off, fac, dcol, ecol\n"
)
no_light = False
elif "1":
# just plot 1 as last resort
light_plot += (
f'plot = {cnam} 1 ! 0 1 {CCD_COLS.get(cnam,"black")} ! '
" # ccd, targ, comp, off, fac, dcol, ecol\n"
)
no_light = False
else:
# template case: pass as many of the lines as we can but
# run checks of apertures
plots = tvals['light']['plot']
for pl in plots:
cnam, targ, comp = pl['ccd'], pl['targ'], pl['comp']
off, fac, dcol, ecol = pl['off'], pl['fac'], ct(pl['dcol']), ct(pl['ecol'])
if cnam in aper and targ in aper[cnam] and (comp == '!' or comp in aper[cnam]):
light_plot += (
f"plot = {cnam} {targ} {comp} {off} {fac} {dcol} {ecol}"
" # ccd, targ, comp, off, fac, dcol, ecol\n"
)
no_light = False
else:
warnings.warn(
"Light curve plot line:\n"
f"plot = {cnam} {targ} {comp} {off} {fac} {dcol} {ecol}\n"
f"from {template} is incompatible with {apfile} and has been skipped"
)
if no_light:
raise hcam.HipercamError(
"Found no targets for light curve plots in any CCD; cannot make light "
"curve plot. Needs at least aperture '1' defined and preferably '2' as well,"
"for at least one CCD"
)
# Generate the position plot lines. Favour the comparisons first, then the target.
position_plot = ""
no_position = True
if template is None:
ccdaper = aper[ccd]
for targ in ('2','3','1'):
if targ in ccdaper:
position_plot += (
f'plot = {ccd} {targ} '
f'{CCD_COLS.get(cnam,"black")} ! '
'# ccd, targ, dcol, ecol\n'
)
no_position = False
break
if no_position:
warnings.warn(
f"Targets 2, 1, or 3 not found in CCD = {ccd} in {apfile}"
)
elif psec is not None:
plots = psec['plot']
for pl in plots:
cnam, targ, dcol, ecol = pl['ccd'], pl['targ'], ct(pl['dcol']), ct(pl['ecol'])
if cnam in aper and targ in aper[cnam]:
position_plot += (
f'plot = {cnam} {targ} {dcol} {ecol} # ccd, targ, dcol, ecol\n'
)
no_position = False
else:
warnings.warn(
f"Position target {targ} / CCD {cnam} not found in {apfile}; will skip"
)
if no_position:
warnings.warn(f"No position plot will be made")
# Generate the transmission plot lines, again comparisons first,
# target only if desperate.
transmission_plot = ""
no_transmission = True
if template is None:
for cnam in aper:
ccdaper = aper[cnam]
for targ in ('2','3','1'):
if targ in ccdaper:
transmission_plot += (
f'plot = {cnam} {targ} '
f'{CCD_COLS.get(cnam,"black")} ! '
f"# ccd, targ, dcol, ecol\n"
)
no_transmission = False
break
if no_transmission:
warnings.warn(
f"Targets 2, 3, or 1 not found in any CCD within {apfile}; no transmission plot"
)
elif tsec is not None:
plots = tsec['plot']
for pl in plots:
cnam, targ, dcol, ecol = pl['ccd'], pl['targ'], ct(pl['dcol']), ct(pl['ecol'])
if cnam in aper and targ in aper[cnam]:
transmission_plot += (
f'plot = {cnam} {targ} {dcol} {ecol} # ccd, targ, dcol, ecol\n'
)
no_transmission = False
else:
warnings.warn(
f"Transmission target {targ} / CCD {cnam} not found in {apfile}; will skip"
)
if no_transmission:
warnings.warn(f"No transmission plot will be made")
# Generate the seeing plot lines. Prioritise the target in this case.
seeing_plot = ""
no_seeing = True
if template is None:
for cnam in aper:
ccdaper = aper[cnam]
for targ in ('1','2','3'):
if targ in ccdaper and not ccdaper[targ].linked:
seeing_plot += (
f"plot = {cnam} {targ}"
f' {CCD_COLS.get(cnam,"black")} ! '
"# ccd, targ, dcol, ecol\n"
)
no_seeing = False
break
if no_seeing:
raise hcam.HipercamError(
"Targets 1, 2 and 3 not found in any CCD"
" (or they are linked); cannot make seeing plot"
)
elif ssec is not None:
plots = tsec['plot']
for pl in plots:
cnam, targ, dcol, ecol = pl['ccd'], pl['targ'], ct(pl['dcol']), ct(pl['ecol'])
if cnam in aper and targ in aper[cnam] and not aper[cnam][targ].linked:
seeing_plot += (
f'plot = {cnam} {targ} {dcol} {ecol} # ccd, targ, dcol, ecol\n'
)
no_seeing = False
else:
warnings.warn(
f"Seeing target {targ} / CCD {cnam} not found or linked in {apfile}; will skip"
)
if no_seeing:
warnings.warn(f"No seeing plot will be made")
# monitor targets (all of them by default)
if template is None:
targs = set()
for cnam in aper:
ccdaper = aper[cnam]
for targ in ccdaper:
targs.add(targ)
monitor = ""
for targ in sorted(targs):
monitor += (
f"{targ} = NO_EXTRACTION TARGET_SATURATED TARGET_AT_EDGE"
" TARGET_NONLINEAR NO_SKY NO_FWHM NO_DATA SKY_AT_EDGE\n"
)
else:
monitor = ""
msec = tvals['monitor']
rlook = dict([(k,v) for v,k in hcam.FLAGS])
for targ, masks in msec.items():
smasks = ' '.join([rlook[mask] for mask in masks])
monitor += f'{targ} = {smasks}\n'
# time stamp
tstamp = strftime("%d %b %Y %H:%M:%S (UTC)", gmtime())
# Finally, write out the reduce file ...
with open(rfile, "w") as fout:
# write out file
fout.write(
f"""#
# This is a HiPERCAM "reduce file" which defines the operation of the
# reduce script. It was written by the HiPERCAM pipeline command
# 'genred'. It consists of a series of sections each of which
# contains a number of parameters. The file is self-documenting on the
# meaning of these parameters. The idea is that these are to large
# extent unchanging and it would be annoying to be prompted every time
# for them, but it also acts as a record of how reduction was carried
# out and is fed into the log file produce by 'reduce'.
# File written on {tstamp}
#
# HiPERCAM pipeline version: {hcam.REDUCE_FILE_VERSION}
#
# Start with some general items that tend not to change
# much. 'version' is the version of the reduce file format which
# changes more slowly than the software does. It must match the same
# parameter in 'reduce' for reduction to proceed. This is
# automatically the case at the time of creation, but old versions of
# the reduce file may become incompatible with later versions of
# reduce. Either they will require updating to be used, or the
# software version can be rolled back to give a compatible version of
# reduce using 'git'. The script 'rupdate', which attempts automatic
# update, may be worth trying if you need to update. It attempts to
# make the minimum changes needed to an old reduce file to run with
# later version dates.
[general]
version = {hcam.REDUCE_FILE_VERSION} # must be compatible with the version in reduce
instrument = {gsec['instrument']} # instrument
scale = {gsec['scale']} # pixel scale, arcsec/pixel
ldevice = {gsec['ldevice']} # PGPLOT plot device for light curve plots
lwidth = {gsec['lwidth']} # light curve plot width, inches, 0 to let program choose
lheight = {gsec['lheight']} # light curve plot height, inches
idevice = {gsec['idevice']} # PGPLOT plot device for image plots [if implot True]
iwidth = {gsec['iwidth']} # image curve plot width, inches, 0 to let program choose
iheight = {gsec['iheight']} # image curve plot height, inches
toffset = {gsec['toffset']} # integer offset to subtract from the MJD
# skip points with bad times in plots. HiPERCAM has a problem in not
# correctly indicating bad times so one does not usually want to
# skip "bad time" points, whereas one should for ULTRACAM and ULTRASPEC.
skipbadt = {gsec['skipbadt']}
# series of count levels at which warnings will be triggered for (a)
# non linearity and (b) saturation. Each line starts 'warn =', and is
# then followed by the CCD label, the non-linearity level and the
# saturation level
{warn_levels}
# The aperture reposition and extraction stages can be run in separate
# CPUs in parallel for each CCD potentially offering speed
# advantages. 'ncpu' is the number of CPUs to use for this. The
# maximum useful and best number to use is the number of CCDs in the
# instrument, e.g. 5 for HiPERCAM. You probably also want to leave at
# least one CPU to do other stuff, but if you have more than 2 CPUs,
# this parameter may help speed things. If you use this option (ncpu >
# 1), then there is also an advantage in terms of reducing
# parallelisation overheads in reading frames a few at a time before
# processing. This is controlled using 'ngroup'. i.e. with ngroup=10,
# 10 full frames are read before being processed. This parameter is
# ignored if ncpu==1
# I have had mixed results with this, which could depend a lot on the
# installation. On my home desktop is works better with ncpu=1 because
# it already seems to parallelise and it only gets worse if I split things
# up. But no harm trying.
ncpu = {gsec['ncpu']}
ngroup = {gsec['ngroup']}
# The next section '[apertures]' defines how the apertures are
# re-positioned from frame to frame. Apertures are re-positioned
# through a combination of a search near a start location followed by
# a 2D profile fit. Several parameters below are associated with this
# process and setting these right can be the key to a successful
# reduction. If there are reference apertures, they are located first
# to give a mean shift. This is used to avoid the initial search for
# any non-reference apertures which has the advantage of reducing the
# chance of problems. The search is carried out by first extracting a
# sub-window centred on the last good position of a target. This is
# then smoothed by a gaussian (width 'search_smooth_fwhm'), and the
# closest peak to the last valid position higher than
# 'fit_height_min_ref' above background (median of the square box) is
# taken as the initial position for later profile fits. The smoothing
# serves to make the process more robust against cosmic rays. The
# width of the search box ('search_half_width') depends on how good
# the telescope guiding is. It should be large enough to cope with the
# largest likely shift in position between any two consecutive
# frames. Well-chosen reference targets, which should be isolated and
# bright, can help this process a great deal. The threshold is applied
# to the *smoothed* image. This means that it can be significantly
# lower than simply the raw peak height. e.g. a target might have a
# typical peak height around 100, in seeing of 4 pixels FWHM. If you
# smooth by 10 pixels, the peak height will drop to
# 100*4**2/(4**2+10**2) = 14 counts. It will be much more stable as a
# result, but you should then probably choose a threshold of 7 when
# you might have thought 50 was appropriate. The smoothing itself can
# be carried out by direct convolution or by an FFT-based method. The
# end-result is the same either way but for large values of
# 'search_smooth_fwhm', i.e. >> 1, FFTs may offer an advantage
# speed-wise. But the only way to tell is by explicity running with
# 'search_smooth_fft' switched from 'no' to 'yes'.
# The boxes for the fits ('fit_half_width') need to be large enough to
# include the target and a bit of sky to ensure that the FWHM is
# accurately measured, remembering that seeing can flare of course. If
# your target was defocussed, a gaussian or Moffat function will be a
# poor fit and you may be better keeping the FWHM fixed at a large
# value comparable to the widths of your defoccused images (and use
# the gaussian option in such cases). If the apertures are chosen to
# be fixed, there will be no search or fit carried out in which case
# you must choose 'fixed' as well when it comes the extraction since
# otherwise it needs a FWHM. 'fixed' is a last resort and you will
# very likely need to use large aperture radii in the extraction
# section.
# An obscure parameter is 'fit_ndiv'. If this is made > 0, the fit
# routine attempts to allow for pixellation by evaluating the profile
# at multiple points within each pixel of the fit. First it will
# evaluate the profile for every unbinned pixel within a binned pixel
# if the pixels are binned; second, it will evaluate the profile over
# an ndiv by ndiv square grid within each unbinned pixel. Thus ndiv=1
# means one evaluation at the centre of each unbinned pixel,
# etc. Obviously this will slow things, but it could help if your
# images are under-sampled. I would always start with fit_ndiv=0, and
# only raise it if the measured FWHM seem to be close to or below two
# binned pixels.
# If you use reference targets (you should if possible), the initial
# positions for the non-reference targets should be good. You can then
# guard further against problems using the parameter 'fit_max_shift'
# to reject positions for the non-reference targets that shift too far
# from the initial guess. 'fit_alpha' is another parameter that
# applies only in this case. If reference apertures are being used,
# the expected locations of non-reference apertures can be predicted
# with some confidence. In this case when the non-reference aperture's
# position is measured, its position will be adjusted by 'fit_alpha'
# times the measured change in its position. Its value is bounded by 0
# < fit_alpha <= 1. "1" just means use the full measured change from
# the current frame to update the position. Anything < 1 builds in a
# bit of past history. The hope is that this could make the aperture
# positioning, especially for faint targets, more robust to cosmic
# rays and other issues. Of course it will correlate the positions
# from frame to frame. fit_alpha = 0.1 for instance will lead to a
# correlation length ~ 10 frames.
# If you use > 1 reference targets, then the parameter 'fit_diff'
# comes into play. Multiple reference targets should move together
# and give very consistent shifts. If they don't, then a problem may
# have occurred, e.g. one or more have been affected by a meteor trail
# for instance. The maximum acceptable differential shift is defined
# by 'fit_diff'. If exceeded, then the entire extraction will be
# aborted and positions held fixed.
# To get and idea of the right values of some of these parameters, in
# particular the 'search_half_width', the height thresholds,
# 'fit_max_shift' and 'fit_diff', the easiest approach is probably to
# run a reduction with loose values and see how it goes.
[apertures]
aperfile = {apfile} # file of software apertures for each CCD
location = {asec['location']} # aperture locations: 'fixed' or 'variable'
search_half_width = {asec['search_half_width']:.1f} # for initial search for objects around previous position, unbinned pixels
search_smooth_fwhm = {asec['search_smooth_fwhm']:.1f} # smoothing FWHM, binned pixels
search_smooth_fft = {asec['search_smooth_fft']} # use FFTs for smoothing, 'yes' or 'no'.
fit_method = {asec['fit_method']} # gaussian or moffat
fit_beta = {asec['fit_beta']:.1f} # Moffat exponent
fit_beta_max = {asec['fit_beta_max']:.1f} # max Moffat expt
fit_fwhm = {asec['fwhm']:.1f} # FWHM, unbinned pixels
fit_fwhm_min = {asec['fwhm_min']:.1f} # Minimum FWHM, unbinned pixels [MUST be < fit_fwhm!]
fit_fwhm_max = {asec['fwhm_max']:.1f} # Maximum FWHM, unbinned pixels
fit_ndiv = {asec['fit_ndiv']} # sub-pixellation factor
fit_fwhm_fixed = {asec['fit_fwhm_fixed']} # Might want to set = 'yes' for defocussed images
fit_half_width = {asec['fit_half_width']:.1f} # for fit, unbinned pixels
fit_thresh = {asec['fit_thresh']:.2f} # RMS rejection threshold for fits
fit_height_min_ref = {asec['fit_height_min_ref']:.1f} # minimum height to accept a fit, reference aperture
fit_height_min_nrf = {asec['fit_height_min_nrf']:.1f} # minimum height to accept a fit, non-reference aperture
fit_max_shift = {asec['fit_max_shift']:.1f} # max. non-ref. shift, unbinned pixels.
fit_alpha = {asec['fit_alpha']:.2f} # Fraction of non-reference aperture shift to apply
fit_diff = {asec['fit_diff']:.2f} # Maximum differential shift of multiple reference apertures, unbinned
# The next lines define how the apertures will be re-sized and how the
# flux will be extracted from the aperture. There is one line per CCD
# with format:
# <CCD label> = <resize> <extract method> [scale min max] [scale min max]
# [scale min max]
# where: <CCD label> is the CCD label; <resize> is either 'variable'
# or 'fixed' and sets how the aperture size will be determined -- if
# variable it will be scaled relative to the FWHM, so profile fitting
# will be attempted; <extract method> is either 'normal' or 'optimal'
# to say how the flux will be extracted -- 'normal' means a straight
# sum of sky subtracted flux over the aperture, 'optimal' use Tim
# Naylor's profile weighting, and requires profile fits to
# work. Finally there follow a series of numbers in three triplets,
# each of which is a scale factor relative to the FWHM for the
# aperture radius if the 'variable' option was chosen, then a minimum
# and a maximum aperture radius in unbinned pixels. The three triples
# correspond to the target aperture radius, the inner sky radius and
# finally the outer sky radius. The mininum and maximum also apply if
# you choose 'fixed' apertures and can be used to override whatever
# value comes from the aperture file. A common approach is set them
# equal to each other to give a fixed value, especially for the sky
# where one does not necessarily want the radii to vary. For PSF
# photometry, all these settings have no effect, but this section can
# still be used to determine which CCDs have fluxes extracted.
[extraction]
{extraction}
# The next lines are specific to the PSF photometry option. 'gfac' is
# used to label the sources according to groups, such that stars
# closer than 'gfac' times the FWHM are labelled in the same
# group. Each group has a PSF model fit independently. The reason
# behind the construction of groups is to reduce the dimensionality of
# the fitting procedure. Usually you want closely seperated stars to
# be fit simultaneously, but too large a value will mean fitting a
# model with many free parameters, which can fail to converge. The
# size of the box over which data is collected for fitting is set by
# 'fit_half_width'. Finally, 'positions' determines whether the star's
# positions should be considered variable in the PSF fitting. If this
# is set to fixed, the positions are held at the locations found in
# the aperture repositioning step, otherwise the positions are refined
# during PSF fitting. This step can fail for PSF photometry of faint
# sources.
[psf_photom]
gfac = {psfsec['gfac']} # multiple of the FWHM to use in grouping objects
fit_half_width = {psfsec['fit_half_width']} # size of window used to collect the data to do the fitting
positions = {psfsec['positions']} # 'fixed' or 'variable'
# Next lines determine how the sky background level is
# calculated. Note you can only set error = variance if method =
# 'clipped'. 'median' should usually be avoided as it can cause
# noticable steps in light curves. It's here as a comparator.
[sky]
method = {sksec['method']} # 'clipped' | 'median'
error = {sksec['error']} # 'variance' | 'photon': first uses actual variance of sky
thresh = {sksec['thresh']:.1f} # threshold in terms of RMS for 'clipped'
# Calibration frames and constants
# If you specify "!" for the readout, an attempt to estimate it from
# +/- 1 sigma percentiles will be made. This could help if you have no
# bias (and hence variance calculation from the count level will be
# wrong)
# The fringe parameters are the most complex. To de-fringe you need
# first a fringe map, e.g. as returned by makefringe. Then you need a
# file of FringePairs as returned by 'setfringe'. These are pairs of
# points placed at peaks and troughs of fringes. They will be used to
# measure a series of differences in each frame of interest and the
# fringe map from which a series of ratios is formed. The differences
# are measured from a group of pixels extending +/-nhlaf around the
# central pixel of each point of a FringePair. Thus nhalf=2 would give
# a 5x5 array (unbinned pixels). Finally to reduce the effect of bad
# values the individual ratios can be pruned if they lie outside a
# range rmin to rmax prior to their overall median being calculated.
# rmin should be small, but probably slightly less than zero to allow
# for a bit of statistical fluctuation, depending on your setup. One
# would normally expect rmax < 1 if the fringe map came from frames with
# a long exposure compared to the data.
[calibration]
crop = {csec['crop']} # Crop calibrations to match the data
bias = {'' if bias is None else bias} # Bias frame, blank to ignore
flat = {'' if flat is None else flat} # Flat field frame, blank to ignore
dark = {'' if dark is None else dark} # Dark frame, blank to ignore
fmap = {'' if fmap is None else fmap} # Fringe map frame, blank to ignore
fpair = {'' if fpair is None else fpair} # File of fringe pairs, ignored if fmap blank
nhalf = {csec['nhalf']} # Half-size of region used for fringe ratios, binned pix, ignored if fmap blank=""
rmin = {csec['rmin']} # Mininum acceptable individual fmap ratio, ignored if fmap blank
rmax = {csec['rmax']} # Maximum acceptable individual fmap ratio, ignored if fmap blank
readout = {csec['readout']} # RMS ADU. Float or string name of a file or "!" to estimate on the fly
gain = {csec['gain']} # Gain, electrons/ADU. Float or string name of a file
# The light curve plot which consists of light curves, X & Y
# poistions, the transmission and seeing. All but the light curves can
# be switched off by commenting them out (in full). First a couple of
# general parameters.
[lcplot]
xrange = {lcsec['xrange']} # maximum range in X to plot (minutes), <= 0 for everything
extend_x = {lcsec['extend_x']} # amount by which to extend xrange, minutes.
# The light curve panel (must be present). Mostly obvious, then a
# series of lines, each starting 'plot' which specify one light curve
# to be plotted giving CCD, target, comparison ('!' if you don't want
# a comparison), an additive offset, a multiplicative scaling factor
# and then a colour for the data and a colour for the error bar There
# will always be a light curve plot, whereas later elements are
# optional, therefore the light curve panel is defined to have unit
# height and all others are scaled relative to this.
[light]
linear = {lsec['linear']} # linear vertical scale (else magnitudes): 'yes' or 'no'
y_fixed = {lsec['y_fixed']} # keep a fixed vertical range or not: 'yes' or 'no'
y1 = {lsec['y1']} # initial lower y value
y2 = {lsec['y2']} # initial upper y value. y1=y2 for auto scaling
extend_y = {lsec['extend_y']} # fraction of plot height to extend when rescaling
# line or lines defining the targets to plot
{light_plot}
"""
)
# position plot (optional)
if no_position:
fout.write(
f"""# The X,Y position panel. Uncomment to show
#[position]
#height = 0.5 # height relative to light curve plot
#x_fixed = no # keep X-position vertical range fixed
#x_min = -5 # lower limit for X-position
#x_max = +5 # upper limit for X-position
#y_fixed = no # keep Y-position vertical range fixed
#y_min = -5 # lower limit for Y-position
#y_max = +5 # upper limit for Y-position
#extend_y = 0.2 # Vertical extension fraction if limits exceeded
## line or lines defining the targets to plot
# plot = 1 1 darkred ! # ccd, targ, dcol, ecol
"""
)
else:
fout.write(
f"""# The X,Y position panel. Comment *every* line
# if you don't want it.
[position]
height = {psec['height']} # height relative to light curve plot
x_fixed = {psec['x_fixed']} # keep X-position vertical range fixed
x_min = {psec['x_min']} # lower limit for X-position
x_max = {psec['x_max']} # upper limit for X-position
y_fixed = {psec['y_fixed']} # keep Y-position vertical range fixed
y_min = {psec['y_min']} # lower limit for Y-position
y_max = {psec['y_max']} # upper limit for Y-position
extend_y = {psec['extend_y']} # Vertical extension fraction if limits exceeded
# line or lines defining the targets to plot
{position_plot}
"""
)
# Transmission plot, optional
if no_transmission:
fout.write(
f"""# The transmission panel. Uncomment if you want it
# Simply plots the flux in whatever apertures are chosen,
# scaling them by their maximum (hence one can sometimes
# find that what you thought was 100% transmission was
# actually only 50% revealed as the cloud clears).
#[transmission]
#height = 0.5 # height relative to the light curve plot
#ymax = 110 # Maximum transmission to plot (>= 100 to slow replotting)
## line or lines defining the targets to plot
# plot = 1 2 green ! # ccd, targ, dcol, ecol
"""
)
else:
fout.write(
f"""# The transmission panel. If you comment it out,
# comment *every* line.
# Simply plots the flux in whatever apertures are chosen,
# scaling them by their maximum (hence one can sometimes
# find that what you thought was 100% transmission was
# actually only 50% revealed as the cloud clears).
[transmission]
height = {tsec['height']} # height relative to the light curve plot
ymax = {tsec['ymax']} # Maximum transmission to plot (>= 100 to slow replotting)
# line or lines defining the targets to plot
{transmission_plot}
"""
)
if no_seeing:
fout.write(
f"""# The seeing panel. Uncomment if you want it.
# You can have multiple plot lines. Don't choose linked
# targets as their FWHMs are not measured.
#[seeing]
#height = 0.5 # height relative to the light curve plot
#ymax = 2.999 # Initial maximum seeing
#y_fixed = no # fix the seeing scale (or not)
#extend_y = 0.2 # Y extension fraction if out of range and not fixed
## line or lines defining the targets to plot
# plot = 1 2 green ! # ccd, targ, dcol, ecol
"""
)
else:
fout.write(
f"""# The seeing panel. Can be commented out if you don't want one but make
# sure to comment it out completely, section name and all
# parameters. You can have multiple plot lines. Don't choose linked
# targets as their FWHMs are not measured.
[seeing]
height = {ssec['height']} # height relative to the light curve plot
ymax = {ssec['ymax']} # Initial maximum seeing
y_fixed = {ssec['y_fixed']} # fix the seeing scale (or not)
extend_y = {ssec['extend_y']} # Y extension fraction if out of range and not fixed
# line or lines defining the targets to plot
{seeing_plot}
"""
)
# Finish up
fout.write(
f"""
# This option attempts to correct for a badly-positioned focal plane mask
# in drift mode which combined with a high background can lead to steps in
# illumination in the Y direction. This tries to subtract the median in the
# X-direction of each window. 'dthresh' is a threshold used to reject X
# pixels prior to taking the median. The purpose is to prevent the stars
# from distorting the median. Take care with this option which is experimental.
[focal_mask]
demask = {fsec['demask']}
dthresh = {fsec['dthresh']}
# Monitor section. This section allows you to monitor particular
# targets for problems. If they occur, then messages will be printed
# to the terminal during reduce. The messages are determined by the
# bitmask flag set during the extraction of each
# target. Possibilities:
# NO_FWHM : no FWHM measured
# NO_SKY : no sky pixels at all
# SKY_AT_EDGE : sky aperture off edge of window
# TARGET_AT_EDGE : target aperture off edge of window
# TARGET_SATURATED : at least one pixel in target above saturation level
# TARGET_NONLINEAR : at least one pixel in target above nonlinear level
# NO_EXTRACTION : no extraction possible
# NO_DATA : no valid pixels in aperture
# For a target you want to monitor, type its label, '=', then the
# bitmask patterns you want to be flagged up if they are set. This is
# designed mainly for observing, as there is less you can do once the
# data have been taken, but it still may prove useful.
[monitor]
{monitor}
"""
)
print("Reduce file written to {:s}".format(rfile))
def makemccd(args=None):
"""Script to generate multi-CCD test data given a set of parameters
defined in a config file (parsed using configparser). This allows
things such as a bias frame, flat field variations offsets, scale
factor and rotations between CCDs, and temporal variations.
Arguments::
config : string
file defining the parameters.
parallel : bool
True / yes etc to run in parallel. Be warned: it does not
always make things faster, which I assume is the result of
overheads when parallelising. It will simply use whatever
CPUs are available.
Depending upon the setting in the config file, this could generate a large
number of different files and so the first time you run it, you may want
to do so in a clean directory.
Config file format: see the documentation of configparser for the general
format of the config files expected by this routine. Essentially there are
a series of sections, e.g.:
[ccd 1]
nxtot = 2048
nytot = 1048
.
.
.
which define all the parameters needed. There are many others to simulate
a bias offset, a flat field; see the example file ??? for a fully-documented
version.
"""
import configparser
global _gframe, _gfield
command, args = utils.script_args(args)
# get inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# Register parameters
cl.register("config", Cline.LOCAL, Cline.PROMPT)
cl.register("parallel", Cline.LOCAL, Cline.PROMPT)
# Prompt for them
config = cl.get_value("config", "configuration file",
cline.Fname("config"))
parallel = cl.get_value("parallel", "add targets in parallel?", False)
# Read the config file
conf = configparser.ConfigParser()
conf.read(config)
# Determine whether files get overwritten or not
overwrite = (conf.getboolean("general", "overwrite")
if "overwrite" in conf["general"] else False)
dtype = conf["general"]["dtype"] if "dtype" in conf["general"] else None
# Top-level header
thead = fits.Header()
thead.add_history("Created by makedata")
# Store the CCD labels and parameters and their dimensions. Determine
# maximum dimensions for later use when adding targets
ccd_pars = Odict()
maxnx = 0
maxny = 0
for key in conf:
if key.startswith("ccd"):
# translate parameters
nxtot = int(conf[key]["nxtot"])
nytot = int(conf[key]["nytot"])
xcen = float(conf[key]["xcen"])
ycen = float(conf[key]["ycen"])
angle = float(conf[key]["angle"])
scale = float(conf[key]["scale"])
xoff = float(conf[key]["xoff"])
yoff = float(conf[key]["yoff"])
fscale = float(conf[key]["fscale"])
toff = float(conf[key]["toff"])
field = (hcam.Field.rjson(conf[key]["field"])
if "field" in conf[key] else None)
ndiv = int(conf[key]["ndiv"]) if field is not None else None
back = float(conf[key]["back"])
# determine maximum total dimension
maxnx = max(maxnx, nxtot)
maxny = max(maxny, nytot)
# store parameters
ccd_pars[key[3:].strip()] = {
"nxtot": nxtot,
"nytot": nytot,
"xcen": xcen,
"ycen": ycen,
"angle": angle,
"scale": scale,
"xoff": xoff,
"yoff": yoff,
"fscale": fscale,
"toff": toff,
"field": field,
"ndiv": ndiv,
"back": back,
}
if not len(ccd_pars):
raise ValueError("hipercam.makedata: no CCDs found in " + config)
# get the timing data
utc_start = Time(conf["timing"]["utc_start"])
exposure = float(conf["timing"]["exposure"])
deadtime = float(conf["timing"]["deadtime"])
# Generate the CCDs, store the read / gain values
ccds = hcam.Group(hcam.CCD)
rgs = {}
for cnam, pars in ccd_pars.items():
# Generate header with timing data
head = fits.Header()
td = TimeDelta(pars["toff"], format="sec")
utc = utc_start + td
head["UTC"] = (utc.isot, "UTC at mid exposure")
head["MJD"] = (utc.mjd, "MJD at mid exposure")
head["EXPOSE"] = (exposure, "Exposure time, seconds")
head["TIMEOK"] = (True, "Time status flag")
# Generate the Windows
winds = hcam.Group(hcam.Window)
rgs[cnam] = {}
for key in conf:
if key.startswith("window"):
iccd, wnam = key[6:].split()
if iccd == cnam:
llx = int(conf[key]["llx"])
lly = int(conf[key]["lly"])
nx = int(conf[key]["nx"])
ny = int(conf[key]["ny"])
xbin = int(conf[key]["xbin"])
ybin = int(conf[key]["ybin"])
if len(winds):
wind = hcam.Window(
hcam.Winhead(llx, lly, nx, ny, xbin, ybin))
else:
# store the header in the first Window
wind = hcam.Window(
hcam.Winhead(llx, lly, nx, ny, xbin, ybin, head))
# Store the Window
winds[wnam] = wind
# Store read / gain value
rgs[cnam][wnam] = (
float(conf[key]["read"]),
float(conf[key]["gain"]),
)
# Accumulate CCDs
ccds[cnam] = hcam.CCD(winds, pars["nxtot"], pars["nytot"])
# Make the template MCCD
mccd = hcam.MCCD(ccds, thead)
# Make a flat field
flat = mccd.copy()
if "flat" in conf:
rms = float(conf["flat"]["rms"])
for ccd in flat.values():
# Generate dust
nspeck = int(conf["flat"]["nspeck"])
if nspeck:
radius = float(conf["flat"]["radius"])
depth = float(conf["flat"]["depth"])
specks = []
for n in range(nspeck):
x = np.random.uniform(0.5, ccd.nxtot + 0.5)
y = np.random.uniform(0.5, ccd.nytot + 0.5)
specks.append(Dust(x, y, radius, depth))
# Set the flat field values
for wind in ccd.values():
wind.data = np.random.normal(1.0, rms, (wind.ny, wind.nx))
if nspeck:
wind.add_fxy(specks)
flat.head["DATATYPE"] = ("Flat field", "Artificially generated")
fname = utils.add_extension(conf["flat"]["flat"], hcam.HCAM)
flat.write(fname, overwrite)
print("Saved flat field to ", fname)
else:
# Set the flat to unity
flat.set_const(1.0)
print("No flat field generated")
# Make a bias frame
bias = mccd.copy()
if "bias" in conf:
mean = float(conf["bias"]["mean"])
rms = float(conf["bias"]["rms"])
for ccd in bias.values():
for wind in ccd.values():
wind.data = np.random.normal(mean, rms, (wind.ny, wind.nx))
bias.head["DATATYPE"] = ("Bias frame", "Artificially generated")
fname = utils.add_extension(conf["bias"]["bias"], hcam.HCAM)
bias.write(fname, overwrite)
print("Saved bias frame to ", fname)
else:
# Set the bias to zero
bias.set_const(0.0)
print("No bias frame generated")
# Everything is set to go, so now generate data files
nfiles = int(conf["files"]["nfiles"])
if nfiles == 0:
out = mccd * flat + bias
fname = utils.add_extension(conf["files"]["root"], hcam.HCAM)
out.write(fname, overwrite)
print("Written data to", fname)
else:
# file naming info
root = conf["files"]["root"]
ndigit = int(conf["files"]["ndigit"])
# movement
xdrift = float(conf["movement"]["xdrift"])
ydrift = float(conf["movement"]["ydrift"])
nreset = int(conf["movement"]["nreset"])
jitter = float(conf["movement"]["jitter"])
print("Now generating data")
tdelta = TimeDelta(exposure + deadtime, format="sec")
for nfile in range(nfiles):
# copy over template (into a global variable for multiprocessing speed)
_gframe = mccd.copy()
# get x,y offset
xoff = np.random.normal(xdrift * (nfile % nreset), jitter)
yoff = np.random.normal(ydrift * (nfile % nreset), jitter)
# create target fields for each CCD, add background
_gfield = {}
for cnam in _gframe.keys():
p = ccd_pars[cnam]
_gframe[cnam] += p["back"]
if p["field"] is not None:
# get field modification settings
transform = Transform(
p["nxtot"],
p["nytot"],
p["xcen"],
p["ycen"],
p["angle"],
p["scale"],
p["xoff"] + xoff,
p["yoff"] + yoff,
)
fscale = p["fscale"]
_gfield[cnam] = p["field"].modify(transform, fscale)
# add the targets in (slow step)
if parallel:
# run in parallel on whatever cores are available
args = [(cnam, ccd_pars[cnam]["ndiv"]) for cnam in _gfield]
with Pool() as pool:
ccds = pool.map(worker, args)
for cnam in _gfield:
_gframe[cnam] = ccds.pop(0)
else:
# single core
for cnam in _gfield:
ccd = _gframe[cnam]
ndiv = ccd_pars[cnam]["ndiv"]
field = _gfield[cnam]
for wind in ccd.values():
field.add(wind, ndiv)
# Apply flat
_gframe *= flat
# Add noise
for cnam, ccd in _gframe.items():
for wnam, wind in ccd.items():
readout, gain = rgs[cnam][wnam]
wind.add_noise(readout, gain)
# Apply bias
_gframe += bias
# data type on output
if dtype == "float32":
_gframe.float32()
elif dtype == "uint16":
_gframe.uint16()
# Save
fname = "{0:s}{1:0{2:d}d}{3:s}".format(root, nfile + 1, ndigit,
hcam.HCAM)
_gframe.write(fname, overwrite)
print("Written file {0:d} to {1:s}".format(nfile + 1, fname))
# update times in template
for ccd in mccd.values():
head = ccd.head
utc = Time(head["UTC"]) + tdelta
head["UTC"] = (utc.isot, "UTC at mid exposure")
head["MJD"] = (utc.mjd, "MJD at mid exposure")
def makefringe(args=None):
"""``makefringe [source] (run first last [twait tmax] | flist) bias
dark flat fpair ([nhalf]) ccd fwhm [clobber] output``
Averages a set of images to make a frame for defringing (referred
to elsewhere as a "fringe map").
At long wavelengths, CCDs suffer what is known as "fringing",
which in terms of structure looks something like the coloured
patterns you see when there is a thin layer of oil on water. Both
patterns are caused by interference. In the case of CCDs this is
caused by sky emission lines and is variable in strength, and
additive in nature, but seems mostly fairly fixed in position.
De-fringing in |hiper| is implemented according to the method
suggested by Snodgrass & Carry (2013Msngr.152...14S). The idea is
to create a set of pairs of points marking the peak and troughs of
fringes. The differences in intensity between these point pairs in
the data to be corrected is compared with the differences for the
same pairs in a reference "fringe map" by taking their
ratios. Many pairs are used so that the median can be taken to
eliminate ratios affected by celestial targets or cosmic rays. The
median ratio is used to scale the reference data and thereby
subtract the fringes. ``makefringe`` is used to make the reference
fringe map. It works as follows: given a single run or a list of
files, it reads them all in, optionally debiases, dark-subtracts
and flat-fields them, calculates a median count level of each one
which is subtracted from each CCD individually. The pixel-by-pixel
median of all frames is then calculated. It is also possible (and
a good idea) to apply the fringe pair ratio measurements to scale
each frame before combining them, which allows frames of similar
pattern but varying amplitude to be combined. Finally, the output
can be smoothed because fringes are usually a medium- to
large-scale pattern.
Parameters:
source : string [hidden]
Data source, five options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : ULTRACAM server
| 'ul' : local ULTRACAM .xml/.dat files
| 'hf' : list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab' or
converted from foreign data formats.
run : string [if source ends 's' or 'l']
run number to access, e.g. 'run034'
first : int [if source ends 's' or 'l']
exposure number to start from. 1 = first frame ('0' is
not supported).
last : int [if source ends 's' or 'l']
last exposure number must be >= first or 0 for the whole lot.
twait : float [if source ends 's' or 'l'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [if source ends 's' or 'l'; hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
flist : string [if source ends 'f']
name of file list. Assumed that bias, flat-fielding etc have been applied to
these already.
bias : str
Name of bias frame to subtract, 'none' to ignore.
dark : str
Name of dark frame to subtract, 'none' to ignore.
flat : str
Name of flat field, 'none' to ignore.
fpair : str
FringePair file (see setfringe), or 'none' to ignore. If specified
if will be used to scale the frames prior to combining.
nhalf : int [if fpair is not 'none']
When calculating the differences for fringe measurement,
a region extending +/-nhalf binned pixels will be used when
measuring the amplitudes. Basically helps the stats.
ccd : str
CCD(s) to process, '0' for all, '1 3' for '1' and '3' only, etc.
fwhm : float
FWHM (unbinned pixels) of gaussian smoothing to apply to the output.
0 to ignore. Should be << smallest scale between fringes. Probably
no more than 4.
clobber : bool [hidden]
clobber any pre-existing output files
output : str
output file. Set by default to match the last part of "run"
(but it will have a different extension so they won't
clash). The default is
.. Note::
This routine writes the files returned by 'grab' to
automatically generated files, typically in .hipercam/tmp, to
avoid polluting the working directory. These are removed at
the end, but may not be if you ctrl-C. You should check
.hipercam/tmp for redundant files every so often
"""
command, args = utils.script_args(args)
# get the inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("source", Cline.GLOBAL, Cline.HIDE)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("last", Cline.LOCAL, Cline.PROMPT)
cl.register("twait", Cline.LOCAL, Cline.HIDE)
cl.register("tmax", Cline.LOCAL, Cline.HIDE)
cl.register("flist", Cline.LOCAL, Cline.PROMPT)
cl.register("bias", Cline.LOCAL, Cline.PROMPT)
cl.register("dark", Cline.LOCAL, Cline.PROMPT)
cl.register("flat", Cline.LOCAL, Cline.PROMPT)
cl.register("fpair", Cline.LOCAL, Cline.PROMPT)
cl.register("nhalf", Cline.LOCAL, Cline.PROMPT)
cl.register("ccd", Cline.LOCAL, Cline.PROMPT)
cl.register("fwhm", Cline.LOCAL, Cline.PROMPT)
cl.register("clobber", Cline.LOCAL, Cline.HIDE)
cl.register("output", Cline.LOCAL, Cline.PROMPT)
# get inputs
source = cl.get_value(
"source",
"data source [hs, hl, us, ul, hf]",
"hl",
lvals=("hs", "hl", "us", "ul", "hf"),
)
# set a flag
server_or_local = source.endswith("s") or source.endswith("l")
if server_or_local:
resource = cl.get_value("run", "run name", "run005")
root = os.path.basename(resource)
cl.set_default('output', cline.Fname(root, hcam.HCAM))
first = cl.get_value("first", "first frame to average", 1, 1)
last = cl.get_value("last", "last frame to average (0 for all)",
first, 0)
twait = cl.get_value("twait",
"time to wait for a new frame [secs]", 1.0,
0.0)
tmax = cl.get_value("tmax",
"maximum time to wait for a new frame [secs]",
10.0, 0.0)
else:
resource = cl.get_value("flist", "file list",
cline.Fname("files.lis", hcam.LIST))
first = 1
# bias frame (if any)
bias = cl.get_value(
"bias",
"bias frame ['none' to ignore]",
cline.Fname("bias", hcam.HCAM),
ignore="none",
)
# dark frame (if any)
dark = cl.get_value(
"dark",
"dark frame ['none' to ignore]",
cline.Fname("dark", hcam.HCAM),
ignore="none",
)
# flat field (if any)
flat = cl.get_value(
"flat",
"flat field frame ['none' to ignore]",
cline.Fname("flat", hcam.HCAM),
ignore="none",
)
fpair = cl.get_value("fpair",
"fringe pair file",
cline.Fname("fringe", hcam.FRNG),
ignore="none")
if fpair is not None:
nhalf = cl.get_value("nhalf",
"half-size of fringe measurement region", 2,
0)
ccdinf = spooler.get_ccd_pars(source, resource)
if len(ccdinf) > 1:
ccd = cl.get_value("ccd", "CCD(s) to process [0 for all]", "0")
if ccd == "0":
ccds = list(ccdinf.keys())
else:
ccds = ccd.split()
else:
ccds = list(ccdinf.keys())
fwhm = cl.get_value(
"fwhm", "FWHM of gaussian smoothing to apply"
" to final result [unbinned pixels, 0 to ignore]", 4., 0.)
clobber = cl.get_value("clobber",
"clobber any pre-existing files on output",
False)
output = cl.get_value(
"output",
"median output fringe frame",
cline.Fname("hcam", hcam.HCAM,
cline.Fname.NEW if clobber else cline.Fname.NOCLOBBER),
)
# inputs done with.
if server_or_local or bias is not None or dark is not None or flat is not None:
print("\nCalling 'grab' ...")
args = [None, "prompt", source, "yes", resource]
if server_or_local:
args += [str(first), str(last), str(twait), str(tmax)]
args += [
"no",
"none" if bias is None else bias,
"none" if dark is None else dark,
"none" if flat is None else flat,
"none",
"f32",
]
resource = hcam.scripts.grab(args)
# at this point 'resource' is a list of files, no matter the input
# method.
fnames = []
with CleanUp(resource, fnames, server_or_local or bias is not None
or dark is not None) as cleanup:
# Read all the files to determine mean levels (after bias
# subtraction) save the bias-subtracted, flat-fielded,
# mean-level subtracted and normalised results to temporary
# files
print("Reading all files in to determine their mean levels")
bframe, fframe, dframe, fpairobj = None, None, None, None
medians = {}
for cnam in ccds:
medians[cnam] = {}
# We might have a load of temporaries from grab, but we are about to
# make some more to save the bias-subtracted normalised versions.
tdir = utils.temp_dir()
mtype = []
fref = {}
with spooler.HcamListSpool(resource) as spool:
for mccd in spool:
if fpair is not None:
# prepare fringepairs
if fpairobj is None:
fpairobj = fringe.MccdFringePair.read(fpair)
fpairobj = fpairobj.crop(mccd, nhalf)
# here we determine the median levels, subtract them
# then (optionally) normalise by the ratio of fringe
# amplitudes
# generate the name to save to automatically
fd, fname = tempfile.mkstemp(suffix=hcam.HCAM, dir=tdir)
for cnam in ccds:
# its unlikely that fringe frames would be taken
# with skips, but you never know. Eliminate them
# from consideration now.
ccd = mccd[cnam]
if ccd.is_data():
cmedian = mccd[cnam].median()
medians[cnam][fname] = cmedian
mccd[cnam] -= cmedian
if cnam not in fref:
# save reference
fref[cnam] = mccd[cnam]
elif fpair is not None:
fscale = fpairobj[cnam].scale(
mccd[cnam], fref[cnam], nhalf)
mccd[cnam] *= fscale
print(f'{cnam}, fscale = {fscale:.3f}')
# write to disk, save the name, close the filehandle
fnames.append(fname)
mccd.write(fname)
os.close(fd)
# a bit of progress info
print(f"Saved {', '.join(mtype)} frame to {fname}")
# now we go through CCD by CCD, using the first as a template
# for the window names in which we will also store the results.
template = hcam.MCCD.read(fnames[0])
# Now process each file CCD by CCD to reduce the memory
# footprint
for cnam in ccds:
# Read files into memory, insisting that they
# all have the same set of CCDs
print(f"\nLoading all CCDs labelled '{cnam}'")
accds = []
with spooler.HcamListSpool(fnames, cnam) as spool:
mean = None
for ccd in spool:
if ccd.is_data():
accds.append(ccd)
if len(accds) == 0:
raise hcam.HipercamError(
f"Found no valid examples of CCD {cnam}"
f" in {fnames}")
else:
print(f"Loaded {len(accds)} CCDs")
# Median combine
for wnam, wind in template[cnam].items():
# build list of all data arrays
arrs = [ccd[wnam].data for ccd in accds]
# convert to 3D numpy array
arr3d = np.stack(arrs)
wind.data = np.median(arr3d, axis=0)
# Add history
template[cnam].head.add_history(
f"Median combine of {len(accds)} images")
print("Computed and stored their pixel-by-pixel median")
# Remove any CCDs not included to avoid impression of having done
# something to them
dcnams = []
for cnam in template.keys():
if cnam not in ccds:
dcnams.append(cnam)
for cnam in dcnams:
del template[cnam]
if fwhm > 0:
# apply gaussian smoothing
print(f'Smoothing with FWHM = {fwhm}')
sigma = fwhm / np.sqrt(8 * np.log(2))
kern = Gaussian2DKernel(sigma)
for ccd in template.values():
for wind in ccd.values():
wind.data = convolve_fft(wind.data, kern, "fill",
wind.median())
# write out
template.write(output, clobber)
print(f"\nFinal result written to {output}")
print('makefringe finished')
def makeflat(args=None):
"""``makeflat [source] (run first last [twait tmax] | flist) ngroup bias
dark ccd [clobber] output``
Averages a set of images to make a flat field.
Typically flat-fields for HiPERCAM and ULTRA(CAM|SPEC) are taken with a
strongly time-variable twilight sky as the Sun sets or rises. A typical
flat field run may start out bright, or even saturated, but by the end be
only a few thousand counts above bias. Moreover, there are very often
stars visible in the images, so we usually take them while offsetting the
telescope in a spiral pattern. The challenge is to combine these images
while rejecting the stars and saturated frames and giving due weight to
the better exposed images. This moreover has to be done for each CCD which
vary significantly in sensitivity.
'makeflat' does this as follows: given an input list of files (or
optionally a single run), it reads them all in, debiases them
(optionally), and calculates the mean count level in each CCD,
normalises by the mean and writes out the results to temporary
files. For each CCD it then sorts the files by their (original)
mean level, and for those that lie between defined limits it takes
the median of the mean-mormalised frames in groups of defined
size. Thus, say one had 75 OK images, then these would be divided
into 10 groups, the first 9 having 7 frames, the last having
16. The median average of each of these would be taken. In each
case the mean levels would be adjusted to be the same before
taking the average to overcome the problem of taking a median of a
time-variable sky. The assumption is that while the level may
vary, the pattern of the image does not. It is up to the user to
check that this is correct. Each of the medians is adjusted to
have a mean equal to the sum of the means of the input
frames. Finally the normal average of all of these median frames
is taken and the mean level of the final output normalised to
1. The first step, taking the median in groups is designed to
remove the stars assuming that the telescope was spiralled. The
size of the groups ('ngroup' below is a crucial parameter in
whether this works). A good strategy is to run makeflat for a
succession of ever larger 'ngroup' and then to divide the results
into each other to see if stars are visible.
The final step, the average of the medians with adjusted mean
levels, is to ensure that the flats are combined in a way that
reflects the level of signal that they have, i.e. to avoid giving
equal weights to the median of a series of flats with 20,000 counts
per pixel and another series with 1,000 counts per pixel. This
somewhat complex procedure is implemented through a series of
temporary files which are written and read as the script runs, but
deleted at its end. This allows very large numbers to be combined
as long as there is enough memory to load 'ngroup' CCDs
simultaneously, which should usually be fine.
Parameters:
source : str [hidden]
Data source, five options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : ULTRACAM server
| 'ul' : local ULTRACAM .xml/.dat files
| 'hf' : list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab' or
converted from foreign data formats.
run : str [if source ends 's' or 'l']
run number to access, e.g. 'run034'
flist : str [if source ends 'f']
name of file list
first : int [if source ends 's' or 'l']
exposure number to start from. 1 = first frame ('0' is
not supported).
last : int [if source ends 's' or 'l']
last exposure number must be >= first or 0 for the whole lot.
twait : float [if source ends 's' or 'l'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [if source ends 's' or 'l'; hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
ngroup : int
the number of frames. Probably should be at least 5, preferably
more. Experiment to see its effect.
bias : string
Name of bias frame to subtract, 'none' to ignore.
dark : str
Name of dark frame to subtract, 'none' to ignore. Note that
it is assumed all CCDs have the same exposure time when making
a dark correction.
ccd : str
CCD(s) to process, '0' for all, '1 3' for '1' and '3' only, etc.
Would almost always expect this to be set = '0'.
lower : list of floats
Lower limits to the mean count level for a flat to be included. The
count level is determined after bias subtraction. Should be the
same number as the selected CCDs, and will be assumed to be in the
same order. Use this to elminate frames that are of so low a level
that the accuracy of the bias subtraction could be a worry.
Suggested hipercam values: 3000 for each CCD. Enter values separated
by spaces.
upper : list of floats
Upper limits to the mean count level for a flat to be included. The
count level is determined *after* bias subtraction. Should be the
same number as the selected CCDs, and will be assumed to be in the
same order. Use this to eliminate saturated, peppered or non-linear
frames. Suggested hipercam values: 58000, 58000, 58000, 40000 and
40000 for CCDs 1, 2, 3, 4 and 5. Enter values separated by spaces.
ULTRACAM values 49000, 29000, 27000 for CCDs 1, 2 and 3.
clobber : bool [hidden]
clobber any pre-existing output files
output : str
output file
"""
command, args = utils.script_args(args)
# get the inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("source", Cline.GLOBAL, Cline.HIDE)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("last", Cline.LOCAL, Cline.PROMPT)
cl.register("twait", Cline.LOCAL, Cline.HIDE)
cl.register("tmax", Cline.LOCAL, Cline.HIDE)
cl.register("flist", Cline.LOCAL, Cline.PROMPT)
cl.register("ngroup", Cline.LOCAL, Cline.PROMPT)
cl.register("bias", Cline.LOCAL, Cline.PROMPT)
cl.register("dark", Cline.LOCAL, Cline.PROMPT)
cl.register("ccd", Cline.LOCAL, Cline.PROMPT)
cl.register("lower", Cline.LOCAL, Cline.PROMPT)
cl.register("upper", Cline.LOCAL, Cline.PROMPT)
cl.register("clobber", Cline.LOCAL, Cline.HIDE)
cl.register("output", Cline.LOCAL, Cline.PROMPT)
# get inputs
source = cl.get_value(
"source",
"data source [hs, hl, us, ul, hf]",
"hl",
lvals=("hs", "hl", "us", "ul", "hf"),
)
# set a flag
server_or_local = source.endswith("s") or source.endswith("l")
if server_or_local:
resource = cl.get_value("run", "run name", "run005")
first = cl.get_value("first", "first frame to average", 1, 1)
last = cl.get_value("last", "last frame to average (0 for all)",
first, 0)
twait = cl.get_value("twait",
"time to wait for a new frame [secs]", 1.0,
0.0)
tmax = cl.get_value("tmax",
"maximum time to wait for a new frame [secs]",
10.0, 0.0)
else:
resource = cl.get_value("flist", "file list",
cline.Fname("files.lis", hcam.LIST))
first = 1
ngroup = cl.get_value("ngroup",
"number of frames per median average group", 3,
1)
# bias frame (if any)
bias = cl.get_value(
"bias",
"bias frame ['none' to ignore]",
cline.Fname("bias", hcam.HCAM),
ignore="none",
)
# dark frame (if any)
dark = cl.get_value(
"dark",
"dark frame ['none' to ignore]",
cline.Fname("dark", hcam.HCAM),
ignore="none",
)
ccdinf = spooler.get_ccd_pars(source, resource)
if len(ccdinf) > 1:
ccd = cl.get_value("ccd", "CCD(s) to process [0 for all]", "0")
if ccd == "0":
ccds = list(ccdinf.keys())
else:
ccds = ccd.split()
else:
ccds = list(ccdinf.keys())
# need to check that the default has the right number of items, if not
# overr-ride it
lowers = cl.get_default("lower")
if len(lowers) != len(ccds):
cl.set_default("lower", len(ccds) * (5000, ))
lowers = cl.get_value(
"lower",
"lower limits on mean count level for included flats, 1 per CCD",
len(ccds) * (5000, ),
)
uppers = cl.get_default("upper")
if len(uppers) != len(ccds):
cl.set_default("upper", len(ccds) * (50000, ))
uppers = cl.get_value(
"upper",
"lower limits on mean count level for included flats, 1 per CCD",
len(ccds) * (50000, ),
)
clobber = cl.get_value("clobber",
"clobber any pre-existing files on output",
False)
output = cl.get_value(
"output",
"output average",
cline.Fname("hcam", hcam.HCAM,
cline.Fname.NEW if clobber else cline.Fname.NOCLOBBER),
)
# inputs done with.
try:
# big try / except section here to trap ctrl-C to allow the temporary
# files to be deleted. First make a directory for the temporary files
if server_or_local:
print("\nCalling 'grab' ...")
args = [
None,
"prompt",
source,
resource,
"yes",
str(first),
str(last),
"no",
str(twait),
str(tmax),
"none",
"f32",
]
resource = hcam.scripts.grab(args)
# at this point 'resource' is a list of files, no matter the input
# method.
# Read all the files to determine mean levels (after bias subtraction)
# save the bias-subtracted, mean-level normalised results to temporary
# files
print("Reading all files in to determine their mean levels")
bframe, dframe = None, None
means = {}
for cnam in ccds:
means[cnam] = {}
# We might have a load of temporaries from grab, but we are about to
# make some more to save the bias-subtracted normalised versions.
tdir = os.path.join(tempfile.gettempdir(),
"hipercam-{:s}".format(getpass.getuser()))
os.makedirs(tdir, exist_ok=True)
fnames = []
with spooler.HcamListSpool(resource) as spool:
for mccd in spool:
if bias is not None:
# bias subtraction
if bframe is None:
bframe = hcam.MCCD.read(bias)
bframe = bframe.crop(mccd)
mccd -= bframe
bexpose = bframe.head.get("EXPTIME", 0.0)
else:
bexpose = 0.0
if dark is not None:
# dark subtraction
if dframe is None:
dframe = hcam.MCCD.read(dark)
dframe = dframe.crop(mccd)
# Factor to scale the dark frame by before
# subtracting from flat. Assumes that all
# frames have same exposure time.
scale = (mccd.head["EXPTIME"] -
bexpose) / dframe.head["EXPTIME"]
# make dark correction
mccd -= scale * dframe
# here we determine the mean levels, store them
# then normalise the CCDs by them and save the files
# to disk
# generate the name to save to automatically
fd, fname = tempfile.mkstemp(suffix=hcam.HCAM, dir=tdir)
for cnam in ccds:
# its unlikely that flats would be taken with skips, but
# you never know. Eliminate them from consideration now.
ccd = mccd[cnam]
if ccd.is_data():
cmean = mccd[cnam].mean()
means[cnam][fname] = cmean
mccd[cnam] /= cmean
# write the disk, save the name, close the filehandle
mccd.write(fname)
fnames.append(fname)
os.close(fd)
# a bit of progress info
if bias is not None:
print("Saved debiassed, normalised"
" flat to {:s}".format(fname))
else:
print("Saved normalised flat to {:s}".format(fname))
# now we go through CCD by CCD, using the first as a template
# for the window names in which we will also store the results.
template = hcam.MCCD.read(fnames[0])
for cnam, lower, upper in zip(ccds, lowers, uppers):
tccd = template[cnam]
# get the keys (filenames) and corresponding mean values
mkeys = np.array(list(means[cnam].keys()))
mvals = np.array(list(means[cnam].values()))
# chop down to acceptable ones
ok = (mvals > lower) & (mvals < upper)
mkeys = mkeys[ok]
mvals = mvals[ok]
# some more progress info
print("Found {:d} frames for CCD {:s}".format(len(mkeys), cnam))
if len(mkeys) == 0:
print((".. cannot average 0 frames;"
" will skip CCD {:s}").format(cnam))
continue
elif len(mkeys) < ngroup:
print(("WARNING: fewer than ngroup = {:d} frames"
" found. Output for CCD {:s} could be poor").format(
ngroup, cnam))
nchunk = len(mkeys) // ngroup
if nchunk == 0:
nchunk = 1
# sort by mean value
isort = mvals.argsort()
mvals = mvals[isort]
mkeys = mkeys[isort]
# wsum used to sum all the eight factors to allow overall
# normalisation at the end of the loop
wsum = 0.0
for n in range(nchunk):
# loop through in chunks of ngroup at a time with a
# potentially larger group to sweep up the end ones.
n1 = ngroup * n
n2 = n1 + ngroup
if n == nchunk:
n2 = len(mkeys)
# load the CCDs of this group
ccdgroup = []
with spooler.HcamListSpool(list(mkeys[n1:n2]), cnam) as spool:
for ccd in spool:
ccdgroup.append(ccd)
# take median of the group to get rid of jumping
# stars. 'weight' used to weight the results when summing the
# results together. this stage is like the 'n' option of
# 'combine' except we have already cut out any junk frames and
# we have normalised the remainder
weight = mvals[n1:n2].sum()
wsum += weight
for wnam, wind in tccd.items():
# go through each window, building a list of all data
# arrays
arrs = [ccd[wnam].data for ccd in ccdgroup]
arr3d = np.stack(arrs)
# at this point, arr3d is a 3D array, with the first
# dimension (axis=0) running over the images. We take the
# median over this axis. The first time through we put
# this straight into the output Window. afterwards we add
# it in (with the appropriate weight)
if n == 0:
wind.data = weight * np.median(arr3d, axis=0)
else:
wind.data += weight * np.median(arr3d, axis=0)
# Normalise the final result to a mean = 1.
tccd /= wsum
# Add some history
tccd.head.add_history(
("result of makeflat on {:d}"
" frames, ngroup = {:d}").format(len(mkeys), ngroup))
# Remove any CCDs not included to avoid impression of having done
# something to them
for cnam in template:
if cnam not in ccds:
del template[cnam]
# write out
template.write(output, clobber)
print("\nFinal result written to {:s}".format(output))
except KeyboardInterrupt:
print("\nmakeflat aborted")
if server_or_local:
# grab has created a load of temporaries, including the file list
# 'resource'
with open(resource) as fin:
for fname in fin:
os.remove(fname.strip())
os.remove(resource)
# and another load were created during the later running of the script
for fname in fnames:
os.remove(fname)
print("temporary files have been removed.")
def hfilter(args=None):
"""``hfilter input ccd nx filter [fwhm] output``
Filters a multi-CCD image. e.g. smooths it. Can be useful
in some analysis steps, e.g. for picking out defects, division
by a smoother version of an image can be useful.
Parameters:
input : string
name of MCCD input file
ccd : string
CCD(s) to filter, '0' for all. If not '0' then '1', '2' or even '3 4'
are possible inputs (without the quotes).
filter : string [single character]
type of filter. 'g' = gaussian. Uses an FFT-based approach which
regards the boundaries as periodic, so you will get significant
edge-effects if the values on opposite edges of a window are
significantly different from each other.
clobber : bool [hidden]
clobber any pre-existing output files
output : string
name of MCCD output file
"""
command, args = utils.script_args(args)
# get input section
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("input", Cline.LOCAL, Cline.PROMPT)
cl.register("ccd", Cline.LOCAL, Cline.PROMPT)
cl.register("filter", Cline.LOCAL, Cline.PROMPT)
cl.register("fwhm", Cline.LOCAL, Cline.PROMPT)
cl.register("clobber", Cline.LOCAL, Cline.HIDE)
cl.register("output", Cline.LOCAL, Cline.PROMPT)
# get inputs
frame = cl.get_value("input", "frame to filter",
cline.Fname("hcam", hcam.HCAM))
mccd = hcam.MCCD.read(frame)
max_ccd = len(mccd)
if max_ccd > 1:
ccd = cl.get_value("ccd", "CCD(s) to filter [0 for all]", "0")
if ccd == "0":
ccds = list(mccd.keys())
else:
ccds = ccd.split()
else:
ccds = list(mccd.keys())
# define the display intensities
filt = cl.get_value("filter",
"filter to apply: g(aussian)",
"g",
lvals=[
"g",
])
if filt == "g":
fwhm = cl.get_value("fwhm",
"FWHM for gaussian filter (binned pixels)",
4.0, 0.01)
clobber = cl.get_value("clobber",
"clobber any pre-existing files on output",
False)
output = cl.get_value(
"output",
"output file",
cline.Fname(
"filtered",
hcam.HCAM,
cline.Fname.NEW if clobber else cline.Fname.NOCLOBBER,
),
)
# all inputs obtained, filter
if filt == "g":
# create filter kernal
kern = Gaussian2DKernel(fwhm / np.sqrt(8 * np.log(2)))
for cnam in ccds:
ccd = mccd[cnam]
for wnam, wind in ccd.items():
if filt == "g":
wind.data = convolve_fft(wind.data, kern, "wrap")
print("Filtered CCD {:s}".format(cnam))
# write out
mccd.write(output, clobber)
print("\nFiltered result written to {:s}".format(output))
def psfaper(args=None):
"""``psfaper mccd aper ccd [linput width height] nx method thresh
niters gfac msub iset (ilo ihi | plo phi) [beta fwmin fwhm
shbox smooth splot fhbox read gain rejthresh]``
Definition of apertures for PSF photometry. This occurs in three
steps - first the user draws a box around the region of interest,
then the user selects a reference star which is used to measure the
PSF. Finally, all the stars in the region of interest are located
following an iterative application of FINDing stars, FITting the image
using a PSF model, SUBTRACTing the model.
An aperture file is automatically created, which can be edited
with |setaper| if necessary.
Parameters:
mccd : string
name of an MCCD file, as produced by e.g. 'grab'
aper : string
the name of an aperture file. If it exists it will be read so that
apertures can be added to it. If it does not exist, it will be
created on exiting the routine. The aperture files are is a fairly
readable / editiable text format
ccd : string
CCD(s) to plot, '0' for all. If not '0' then '1', '2' or even '3 4'
are possible inputs (without the quotes). '3 4' will plot CCD '3' and
CCD '4'. If you want to plot more than one CCD, then you will be
prompted for the number of panels in the X direction. This parameter
will not be prompted if there is only one CCD in the file.
linput : string [hidden]
sets the way in which the apertures are labelled. 'n' = numerical
input, with the program just incrementing by 1 for each successive
aperture; 's' = single character (without requiring the user to hit
hit <CR>); 'm' = multi-character, ending with <CR>.
Allowed characters are 0-9, a-z, A-Z, no spaces or punctuation, but a
single '0' on its own is not permitted.
width : float [hidden]
plot width (inches). Set = 0 to let the program choose.
height : float [hidden]
plot height (inches). Set = 0 to let the program choose. BOTH width
AND height must be non-zero to have any effect
nx : int
number of panels across to display, prompted if more than one CCD is
to be plotted.
method : string
this defines the profile function. Either a gaussian or a moffat
profile, 'g' or 'm'. The latter should usually be best.
thresh : float
this sets the threshold for detection of stars in the image,
in multiples of the background RMS
niters : int
when detecting stars, this sets the number of iterations of the
FIND-FIT-SUBTRACT loop.
gfac : float
in PSF fitting, stars are split into groups, and each group is fit
seperately. This is to avoid fitting models with large numbers of
free parameters. This number, multiplied by the FWHM, gives the
maximum seperation for stars to be fitted within the same group.
msub : bool
True/False to subtract median from each window before scaling
iset : string [single character]
determines how the intensities are determined. There are three
options: 'a' for automatic simply scales from the minimum to the
maximum value found on a per CCD basis. 'd' for direct just takes two
numbers from the user. 'p' for percentile dtermines levels based upon
percentiles determined from the entire CCD on a per CCD bais.
ilo : float [if iset=='d']
lower intensity level
ihi : float [if iset=='d']
upper intensity level
plo : float [if iset=='p']
lower percentile level
phi : float [if iset=='p']
upper percentile level
beta : float [if method == 'm'; hidden]
default Moffat exponent
fwmin : float [hidden]
minimum FWHM to allow, unbinned pixels.
fwhm : float [hidden]
default FWHM, unbinned pixels.
shbox : float [hidden]
half width of box for searching for a star, unbinned pixels. The
brightest target in a region +/- shbox around an intial position will
be found. 'shbox' should be large enough to allow for likely changes
in position from frame to frame, but try to keep it as small as you
can to avoid jumping to different targets and to reduce the chances
of interference by cosmic rays.
smooth : float [hidden]
FWHM for gaussian smoothing, binned pixels. The initial position for
fitting is determined by finding the maximum flux in a smoothed
version of the image in a box of width +/- shbox around the starter
position. Typically should be comparable to the stellar width. Its
main purpose is to combat cosmic rays which tend only to occupy a
single pixel.
fhbox : float [hidden]
half width of box for profile fit, unbinned pixels. The fit box is
centred on the position located by the initial search. It should
normally be > ~2x the expected FWHM.
read : float [hidden]
readout noise, RMS ADU, for assigning uncertainties
gain : float [hidden]
gain, ADU/count, for assigning uncertainties
rejthresh : float [hidden]
thresh rejection threshold
Various standard keyboard shortcuts (e.g. 's' to save) are disabled as
they just confuse things and are of limited use in setaper in any case.
Some aspects of the usage of matplotlib in psfaper are tricky. It is
possible that particular 'backends' will cause problems. I have tested
this with Qt4Agg, Qt5agg and GTK3Agg. One aspect is the cursor icon in pan
mode is a rather indistinct hand where one can't tell what is being
pointed at. I have therefore suppressed this, but only for the tested
backends. Others would need require further investigation.
"""
command, args = utils.script_args(args)
# get input section
with Cline('HIPERCAM_ENV', '.hipercam', command, args) as cl:
# register parameters
cl.register('mccd', Cline.LOCAL, Cline.PROMPT)
cl.register('aper', Cline.LOCAL, Cline.PROMPT)
cl.register('ccd', Cline.LOCAL, Cline.PROMPT)
cl.register('linput', Cline.LOCAL, Cline.HIDE)
cl.register('width', Cline.LOCAL, Cline.HIDE)
cl.register('height', Cline.LOCAL, Cline.HIDE)
cl.register('nx', Cline.LOCAL, Cline.PROMPT)
cl.register('method', Cline.LOCAL, Cline.PROMPT)
cl.register('thresh', Cline.LOCAL, Cline.PROMPT)
cl.register('niters', Cline.LOCAL, Cline.PROMPT)
cl.register('gfac', Cline.LOCAL, Cline.PROMPT)
cl.register('msub', Cline.GLOBAL, Cline.PROMPT)
cl.register('iset', Cline.GLOBAL, Cline.PROMPT)
cl.register('ilo', Cline.GLOBAL, Cline.PROMPT)
cl.register('ihi', Cline.GLOBAL, Cline.PROMPT)
cl.register('plo', Cline.GLOBAL, Cline.PROMPT)
cl.register('phi', Cline.GLOBAL, Cline.PROMPT)
cl.register('beta', Cline.LOCAL, Cline.HIDE)
cl.register('fwhm', Cline.LOCAL, Cline.HIDE)
cl.register('fwmin', Cline.LOCAL, Cline.HIDE)
cl.register('shbox', Cline.LOCAL, Cline.HIDE)
cl.register('smooth', Cline.LOCAL, Cline.HIDE)
cl.register('fhbox', Cline.LOCAL, Cline.HIDE)
cl.register('read', Cline.LOCAL, Cline.HIDE)
cl.register('gain', Cline.LOCAL, Cline.HIDE)
cl.register('rejthresh', Cline.LOCAL, Cline.HIDE)
# get inputs
mccd = cl.get_value('mccd', 'frame to plot',
cline.Fname('hcam', hcam.HCAM))
mccd = hcam.MCCD.read(mccd)
aper = cl.get_value('aper', 'name of aperture file',
cline.Fname('hcam', hcam.APER, exist=False))
if os.path.exists(aper):
# read in old apertures
mccdaper = hcam.MccdAper.read(aper)
print('Loaded existing file = {:s}'.format(aper))
else:
# create empty container
mccdaper = hcam.MccdAper()
print('No file called {:s} exists; '
'will create from scratch'.format(aper))
# define the panel grid
try:
nxdef = cl.get_default('nx')
except KeyError:
nxdef = 3
max_ccd = len(mccd)
if max_ccd > 1:
ccd = cl.get_value('ccd', 'CCD(s) to plot [0 for all]', '0')
if ccd == '0':
ccds = list(mccd.keys())
else:
ccds = ccd.split()
else:
ccds = list(mccd.keys())
width = cl.get_value('width', 'plot width (inches)', 0.)
height = cl.get_value('height', 'plot height (inches)', 0.)
# number of panels in X
if len(ccds) > 1:
nxdef = min(len(ccds), nxdef)
cl.set_default('nx', nxdef)
nx = cl.get_value('nx', 'number of panels in X', 3, 1)
else:
nx = 1
# PSF fitting stuff
method = cl.get_value('method',
'fit method g(aussian) or m(offat)',
'm',
lvals=['g', 'm'])
if method == 'm':
beta = cl.get_value('beta', 'initial exponent for Moffat fits', 5.,
0.5)
else:
beta = 0
fwhm_min = cl.get_value('fwmin',
'minimum FWHM to allow [unbinned pixels]', 1.5,
0.01)
fwhm = cl.get_value('fwhm',
'initial FWHM [unbinned pixels] for profile fits',
6., fwhm_min)
gfac = cl.get_value(
'gfac', 'multiple of FWHM used to group stars for fitting', 2.0,
0.1)
thresh = cl.get_value(
'thresh', 'threshold for object detection (multiple of sky RMS)',
3.0, 0.1)
niters = cl.get_value('niters',
'number of iterations of FIND-FIT-SUBTRACT', 2,
1)
# define the display intensities
msub = cl.get_value('msub', 'subtract median from each window?', True)
iset = cl.get_value('iset', 'set intensity a(utomatically),'
' d(irectly) or with p(ercentiles)?',
'a',
lvals=['a', 'A', 'd', 'D', 'p', 'P'])
iset = iset.lower()
plo, phi = 5, 95
ilo, ihi = 0, 1000
if iset == 'd':
ilo = cl.get_value('ilo', 'lower intensity limit', 0.)
ihi = cl.get_value('ihi', 'upper intensity limit', 1000.)
elif iset == 'p':
plo = cl.get_value('plo', 'lower intensity limit percentile', 5.,
0., 100.)
phi = cl.get_value('phi', 'upper intensity limit percentile', 95.,
0., 100.)
nxmax, nymax = 0, 0
for cnam in ccds:
nxmax = max(nxmax, mccd[cnam].nxtot)
nymax = max(nymax, mccd[cnam].nytot)
# might be worth trying to improve this at some point
xlo, xhi, ylo, yhi = 0, nxmax + 1, 0, nymax + 1
shbox = cl.get_value(
'shbox', 'half width of box for initial'
' location of target [unbinned pixels]', 11., 2.)
smooth = cl.get_value(
'smooth', 'FWHM for smoothing for initial object'
' detection [binned pixels]', 6.)
fhbox = cl.get_value(
'fhbox', 'half width of box for profile fit'
' [unbinned pixels]', 21., 3.)
read = cl.get_value('read', 'readout noise, RMS ADU', 3.)
gain = cl.get_value('gain', 'gain, ADU/e-', 1.)
rejthresh = cl.get_value('rejthresh',
'RMS rejection threshold for sky fitting', 4.)
# Inputs obtained.
# re-configure keyboard shortcuts to avoid otherwise confusing behaviour
# quit_all does not seem to be universal, hence the try/except
try:
mpl.rcParams['keymap.back'] = ''
mpl.rcParams['keymap.forward'] = ''
mpl.rcParams['keymap.fullscreen'] = ''
mpl.rcParams['keymap.grid'] = ''
mpl.rcParams['keymap.home'] = ''
mpl.rcParams['keymap.pan'] = ''
mpl.rcParams['keymap.quit'] = ''
mpl.rcParams['keymap.save'] = ''
mpl.rcParams['keymap.pan'] = ''
mpl.rcParams['keymap.save'] = ''
mpl.rcParams['keymap.xscale'] = ''
mpl.rcParams['keymap.yscale'] = ''
mpl.rcParams['keymap.zoom'] = ''
except KeyError:
pass
# start plot
if width > 0 and height > 0:
fig = plt.figure(figsize=(width, height))
else:
fig = plt.figure()
# get the navigation toolbar.
toolbar = fig.canvas.manager.toolbar
nccd = len(ccds)
ny = nccd // nx if nccd % nx == 0 else nccd // nx + 1
# we need to store some stuff
ax = None
cnams = {}
anams = {}
# this is a container for all the objects used to plot apertures to allow
# deletion. This is Group of Group objects supporting tuple storage. The
# idea is that pobjs[cnam][anam] returns the objects used to plot aperture
# anam of CCD cnam. It is initially empty,
pobjs = hcam.Group(hcam.Group)
for n, cnam in enumerate(ccds):
if ax is None:
axes = ax = fig.add_subplot(ny, nx, n + 1)
axes.set_aspect('equal', adjustable='box')
axes.set_xlim(xlo, xhi)
axes.set_ylim(ylo, yhi)
else:
axes = fig.add_subplot(ny, nx, n + 1, sharex=ax, sharey=ax)
axes.set_aspect('equal')
if msub:
# subtract median from each window
pccd = deepcopy(mccd)
for wind in pccd[cnam].values():
wind -= wind.median()
else:
pccd = mccd
hcam.mpl.pCcd(axes, pccd[cnam], iset, plo, phi, ilo, ihi,
'CCD {:s}'.format(cnam))
# keep track of the CCDs associated with each axes
cnams[axes] = cnam
# and axes associated with each CCD
anams[cnam] = axes
if cnam in mccdaper:
# plot any pre-existing apertures, keeping track of
# the plot objects
pobjs[cnam] = hcam.mpl.pCcdAper(axes, mccdaper[cnam])
else:
# add in an empty CcdApers for any CCD not already present
mccdaper[cnam] = hcam.CcdAper()
# and an empty container for any new plot objects
pobjs[cnam] = hcam.Group(tuple)
print("""
Now use the mouse and the pan/zoom tools to zoom in to the region of interest (ROI) for
PSF photometry. All existing apertures inside this region will be retained, but apertures
outside this region will be deleted.
Once you are happy with your selection, use the key commands listed below to select one or
more bright, isolated, stars to use as references to determine the PSF. These stars will also be
set as reference objects in the final aperture file.
Key commands:
a(dd) : add an aperture
d(elete) : delete an aperture
u(nlink) : unlink axes for all CCDs so that you can tweak ROI for CCDs independently
q(uit) : quit interactive step and find all stars in ROI.
Hitting 'd' will delete the aperture nearest to the cursor, as long as it is
close enough.
""")
try:
plt.tight_layout()
except:
pass
# create a class for picking reference star and zooming in
picker = PickRef(mccd, cnams, anams, toolbar, fig, mccdaper, method, beta,
fwhm, gfac, niters, thresh, fwhm_min, shbox, smooth,
fhbox, read, gain, rejthresh, pobjs, aper)
# squeeze space a bit
plt.subplots_adjust(wspace=0.1, hspace=0.1)
# finally show stuff ....
plt.show()
def hlogger(args=None):
"""``hlogger server (dirnam)``
Generates html logs for hipercam runs.
hlogger expects to work on directories containing the runs for each
night. It should be run from the top-level of the hipercam/logs directory.
It extracts information from each run file which it writes to a file in
JSON format and also as an html file. The JSON file acts as a short-cut
for any re-run.
Parameters:
server : bool
True if the log is to be compiled from the server data. This option
might be useful during observing. If server is True, the JSON and
html files will be written in the current working directory.
dirnam : string [if not server]
Directory name to work on. This is run through 'glob' which gives
UNIX-style name matching, so e.g. '2017*' will match 2017-10-17,
2017-10-18 etc. The JSON and html files will be written into the
directory or directories. In this case this expects the data to be
available as a link called 'data' within the night directory. This
is the standard configuration of my 'logs' directory as created by
'digest'. This script should be run from the log directory. To run
it on the command line, to prevent the shell from globbing, you
must escape or quote the glob as in "hlogger no '*'" or "hlogger no
\*".
"""
command, args = utils.script_args(args)
# get the inputs
with Cline('HIPERCAM_ENV', '.hipercam', command, args) as cl:
# register parameters
cl.register('server', Cline.GLOBAL, Cline.PROMPT)
# get inputs
server = cl.get_value('server', 'access data via the server?', False)
if server:
raise NotImplementedError('server option yet to be enabled')
else:
if os.path.basename(os.getcwd()) != 'logs':
print("** hlogger must be run in a directory called 'logs'")
print('hlogger aborted',file=sys.stderr)
return
# next are regular expressions to match run directories, nights, and
# run files
rre = re.compile('^\d\d\d\d-\d\d$')
nre = re.compile('^\d\d\d\d-\d\d-\d\d$')
fre = re.compile('^run\d\d\d\d\.fits$')
# Get list of run directories
rnames = [rname for rname in os.listdir('.') if \
rre.match(rname) and \
os.path.isdir(rname) and \
os.path.isfile(os.path.join(rname, 'telescope'))
]
rnames.sort()
if len(rnames) == 0:
print("there were no run directories of the form "
"YYYY-MM with a file called 'telescope' in them", file=sys.stderr)
print('hlogger aborted',file=sys.stderr)
return
with open('index.html','w') as ihtml:
# start the top level index html file
ihtml.write(INTRODUCTION_HEADER)
for rname in rnames:
print('\nProcessing run {:s}'.format(rname))
# write in run date, start table of nights
rn = os.path.basename(rname)
year, month = rn.split('-')
with open(os.path.join(rname,'telescope')) as tel:
telescope = tel.read().strip()
ihtml.write('<h2>{:s} {:s}, {:s}</h2>\n'.format(MONTHS[month],year,telescope))
ihtml.write('\n<p>\n<table>\n')
# get night directories
nnames = [os.path.join(rname, ndir) for ndir in os.listdir(rname) if \
nre.match(ndir) and \
os.path.isdir(os.path.join(rname, ndir)) and \
os.path.isdir(os.path.join(rname, ndir, 'data'))
]
nnames.sort()
if len(nnames) == 0:
print(
"found no night directories of the form YYYY-MM-DD with"
" 'data' sub-directories in {:s}".format(rname),
file=sys.stderr
)
print('hlogger aborted',file=sys.stderr)
return
for nn, nname in enumerate(nnames):
print(' night {:s}'.format(nname))
links = '\n<p><a href="../">Run index</a>'
if nn > 0:
bnight = os.path.basename(nnames[nn-1])
links += ', <a href="../{0:s}/{0:s}.html">Previous night</a>'.format(bnight)
if nn < len(nnames)-1:
anight = os.path.basename(nnames[nn+1])
links += ', <a href="../{0:s}/{0:s}.html">Next night</a>\n</p>\n'.format(anight)
links += '\n</p>\n'
# Write an entry for each night linking to the log for that night.
night = os.path.basename(nname)
fname = '{0:s}/{0:s}.html'.format(night)
ihtml.write('<tr><td><a href="{:s}">{:s}</a><td></tr>\n'.format(fname, night))
# Create the html file for the night
date = '{:s}, {:s}'.format(night, telescope)
with open(fname,'w') as nhtml:
# write header of night file
nhtml.write(NIGHT_HEADER1)
nhtml.write(NIGHT_HEADER2.format(date))
nhtml.write(links)
nhtml.write(TABLE_TOP)
# read and store the hand written log
handlog = os.path.join(night,'{:s}.dat'.format(night))
with open(handlog) as fin:
hlog = {}
for line in fin:
if line.startswith('run'):
arr = line.split()
hlog[arr[0]] = ' '.join(arr[1:])
# load all the run names
ddir = os.path.join(night,'data')
runs = [run[:-5] for run in os.listdir(ddir) if \
fre.match(run)]
runs.sort()
# now wind through the runs getting basic info and
# writing a row of info to the html file for the night
# in question.
for nrun, run in enumerate(runs):
if nrun % 20 == 0:
# write table header
nhtml.write(TABLE_HEADER)
# open the run file as an Rtime
rname = os.path.join(night,'data',run)
try:
rtime = hcam.hcam.Rtime(rname)
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_tb(exc_traceback, limit=1, file=sys.stdout)
traceback.print_exc(file=sys.stdout)
print('Problem on run = ',rname)
# dummy info line just to allow us to proceed
nhtml.write('<tr>\n')
# run number
nhtml.write('<td class="lalert">{:s}</td>'.format(run[3:]))
nhtml.write('</tr>\n')
continue
hd = rtime.header
# start the row
nhtml.write('<tr>\n')
# run number
nhtml.write('<td class="left">{:s}</td>'.format(run[3:]))
# object name
nhtml.write(
'<td class="left">{:s}</td>'.format(
hd['OBJECT'])
)
# RA, Dec
ra, dec = correct_ra_dec(hd['RA'], hd['Dec'])
nhtml.write(
'<td class="left">{:s}</td><td class="left">{:s}</td>'.format(
ra, dec)
)
# timing info
ntotal = rtime.ntotal()
texps, toffs, nskips, tdead = rtime.tinfo()
# total = total time on target
# duty = worst duty cycle, percent
# tsamp = shortest sample time
ttotal = 0.
duty = 100
tsamp = 99000.
for texp, nskip in zip(texps, nskips):
ttotal = max(ttotal, (texp+tdead)*(ntotal // nskip))
duty = min(duty, 100.*texp/(texp+tdead))
tsamp = min(tsamp, texp+tdead)
# First & last timestamp
try:
tstamp, tinfo, tflag1 = rtime(1)
tstart = tstamp.isot
tstamp, tinfo, tflag2 = rtime(ntotal)
tend = tstamp.isot
nhtml.write(
'<td class="cen">{:s}</td><td class="cen">{:s}</td><td class="cen">{:s}</td><td class="cen">{:s}</td>'.format(
tstart[:tstart.find('T')],
tstart[tstart.find('T')+1:tstart.rfind('.')],
tend[tend.find('T')+1:tend.rfind('.')],
'OK' if tflag1 and tflag2 else 'NOK'
)
)
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_tb(exc_traceback, limit=1, file=sys.stdout)
traceback.print_exc(file=sys.stdout)
print('Run =',rname)
nhtml.write('<td class="cen">----</td><td class="cen">----</td><td class="cen">----</td><td>NOK</td>')
# sample time
nhtml.write('<td class="right">{:.3f}</td>'.format(tsamp))
# duty cycle
nhtml.write('<td class="right">{:.1f}</td>'.format(duty))
# number of frames
nhtml.write('<td class="right">{:d}</td>'.format(ntotal))
# total exposure time
nhtml.write('<td class="right">{:d}</td>'.format(
int(round(ttotal)))
)
# filters used
nhtml.write('<td class="cen">{:s}</td>'.format(
hd.get('filters','----')
))
# run type
nhtml.write(
'<td class="left">{:s}</td>'.format(
hd.get('IMAGETYP', '---'))
)
# readout mode
nhtml.write('<td class="cen">{:s}</td>'.format(
TRANSLATE_MODE[rtime.mode])
)
# cycle nums
nhtml.write('<td class="cen">{:s}</td>'.format(
','.join([str(nskip) for nskip in nskips]))
)
# window formats
nhtml.write(
'<td class="cen">{:s}</td>'.format(rtime.wforms[0])
)
nhtml.write(
'<td class="cen">{:s}</td>'.format(
rtime.wforms[1] if len(rtime.wforms) > 1 else '')
)
# binning
nhtml.write(
'<td class="cen">{:d}x{:d}</td>'.format(
rtime.xbin,rtime.ybin)
)
# clear
nhtml.write(
'<td class="cen">{:s}</td>'.format(
'On' if rtime.clear else 'Off')
)
# dummy output in use
nhtml.write(
'<td class="cen">{:s}</td>'.format(
'On' if rtime.dummy else 'Off')
)
# LED on
led = hd.get('ESO DET EXPLED','--')
if led == 0:
led = 'Off'
elif led == 1:
led = 'On'
nhtml.write(
'<td class="cen">{:s}</td>'.format(led)
)
# overscan/prescan
nhtml.write(
'<td class="cen">{:s},{:s}</td>'.format(
'On' if rtime.oscan else 'Off',
'On' if rtime.pscan else 'Off')
)
# CCD speed
speed = hd.get('ESO DET SPEED','--')
if speed == 0:
speed = 'Slow'
elif speed == 1:
speed = 'Fast'
nhtml.write(
'<td class="cen">{:s}</td>'.format(speed)
)
# Fast clocks
fclock = hd.get('ESO DET FASTCLK','--')
if fclock == 0:
fclock = 'No'
elif fclock == 1:
fclock = 'Yes'
nhtml.write(
'<td class="cen">{!s}</td>'.format(fclock)
)
# Tbytes problem
nhtml.write(
'<td class="cen">{:s}</td>'.format(
'OK' if rtime.ntbytes == 36 else 'NOK'
)
)
# Focal plane slide
nhtml.write(
'<td class="cen">{!s}</td>'.format(
hd.get('FPslide','----')
)
)
# instr PA [GTC]
nhtml.write(
'<td class="cen">{!s}</td>'.format(
hd.get('INSTRPA','UNDEF')
))
# CCD temps
nhtml.write(
'<td class="cen">{:.1f},{:.1f},{:.1f},{:.1f},{:.1f}</td>'.format(
hd.get('CCD1TEMP',0.0),hd.get('CCD2TEMP',0.0),hd.get('CCD3TEMP',0.0),hd.get('CCD4TEMP',0.0),hd.get('CCD5TEMP',0.0)
))
# PI
nhtml.write(
'<td class="left">{:s}</td>'.format(
hd.get('PI', '---'))
)
# Program ID
nhtml.write(
'<td class="left">{:s}</td>'.format(
hd.get('PROGRM', '---'))
)
# run number again
nhtml.write(
'<td class="left">{:s}</td>'.format(run[3:])
)
# comments
pcomm = hd.get('RUNCOM', '').strip()
if pcomm == 'None' or pcomm == 'UNDEF':
pcomm = ''
if pcomm != '':
if not pcomm.endswith('.'):
pcomm += ' '
else:
pcomm += '. '
nhtml.write(
'<td class="left">{:s}{:s}</td>'.format(pcomm,hlog[run])
)
# end the row
nhtml.write('\n</tr>\n')
# finish off the night file
nhtml.write('</table>\n{:s}'.format(links))
nhtml.write(NIGHT_FOOTER)
# finish off the run table
ihtml.write('\n</table>\n\n')
# finish the main index
ihtml.write(INTRODUCTION_FOOTER)
# write out the css file
with open('hiper.css','w') as fout:
fout.write(CSS)
def hpackage(args=None):
"""``hpackage runs dname tar``
``hpackage`` looks for standard reduce data products and bundles
them up into a single directory and optionally creates a tar
file. The idea is to copy all the files needed to be able to
re-run the reduction with the pipeline, while also adding a few
helpful extras where possible. Given 'run123' for instance, it
looks for:
run123.hcm -- typically the result from a run of |averun|
run123.ape -- file of photometric apertures
run123.red -- reduce file as made by |genred|
run123.log -- result from |reduce|
It also looks for calibration files inside the reduce file and
copies them. It requires them to be within the same directory and
will fail if they are not.
It produces several extra files which are:
run123.fits -- FITS version of the log file
run123_ccd1.fits -- joined-up ds9-able version of run123.hcm
(and ccd2 etc) [but only if the windows are
in sync.
run123_ccd1.reg -- ds9-region file representing the apertures
from run123.ape [see above re synced windows]
README -- a file of explanation.
The files are all copied to a temporary directory.
Arguments:
run : str
Series of run names of the ones to copy, separated by spaces.
dname : str
Name for the directory to store all files forming the root of
any output tar file created from it.
tar : bool
Make a tar.gz file of the directory at the end; the directory and
the files in it will be deleted. Otherwise, no tar file is made and
the directory is left untouched. The directory will however be deleted
if the program is aborted early.
"""
command, args = utils.script_args(args)
FEXTS = (hcam.HCAM, hcam.APER, hcam.LOG, hcam.RED)
# get the inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("runs", Cline.LOCAL, Cline.PROMPT)
cl.register("dname", Cline.LOCAL, Cline.PROMPT)
cl.register("tar", Cline.LOCAL, Cline.PROMPT)
runs = cl.get_value(
"runs", "run names [space separated]",
'run005'
)
runs = runs.split()
for run in runs:
if os.path.dirname(run) != '':
raise hcam.HipercamError(
'hpackage only runs on files in the working directory'
)
for fext in FEXTS:
if not os.path.exists(run + fext):
raise hcam.HipercamError(
f'could not find {run+fext}'
)
dname = cl.get_value(
"dname", "name of directory for storage of files (will be used to any tar file as well)",
'hdata'
)
tar = cl.get_value(
"tar", "make a tar file (and delete temporary directory at end)?", True
)
# Make name of temporary directory
tdir = utils.temp_dir()
tmpdir = os.path.join(tdir, dname)
with CleanUp(tmpdir, tar) as cleanup:
# create directory
os.makedirs(tmpdir, exist_ok=True)
print(f'Will write files to {tmpdir}')
# Get on
for run in runs:
# strip extension
root = os.path.splitext(run)[0]
# need to read the file to determine
# the number of CCDs
print(
run,root,utils.add_extension(run,hcam.HCAM)
)
mccd = hcam.MCCD.read(
utils.add_extension(run,hcam.HCAM)
)
try:
# convert the hcm and ape files using joinup
args = [
None,'prompt','list','hf',run,'no'
]
if len(mccd) > 1:
args += ['0']
args += [
root + hcam.APER,
'none','none','none','none','no',
'float32',str(100),str(100),
'no','rice',tmpdir
]
hcam.scripts.joinup(args)
except:
print('failed to joinup hcm / aper files')
# convert log to fits as well
args = [
None,'prompt',run,'h',tmpdir
]
hcam.scripts.hlog2fits(args)
# copy standard files over
for fext in FEXTS:
source = utils.add_extension(root,fext)
target = os.path.join(tmpdir,source)
shutil.copyfile(source, target)
print(f'copied {source} to {target}')
# now the calibrations
rfile = hcam.reduction.Rfile.read(run + hcam.RED)
csec = rfile['calibration']
if rfile.bias is not None:
source = utils.add_extension(
csec['bias'], hcam.HCAM
)
if os.path.dirname(source) != '':
raise HipercamError(
f'bias = {source} is not in the present working directory'
)
target = os.path.join(tmpdir,source)
shutil.copyfile(source, target)
print(f'copied {source} to {target}')
if rfile.dark is not None:
source = utils.add_extension(
csec['dark'], hcam.HCAM
)
if os.path.dirname(source) != '':
raise HipercamError(
f'dark = {source} is not in the present working directory'
)
target = os.path.join(tmpdir,source)
shutil.copyfile(source, target)
print(f'copied {source} to {target}')
if rfile.flat is not None:
source = utils.add_extension(
csec['flat'], hcam.HCAM
)
if os.path.dirname(source) != '':
raise HipercamError(
f'flat = {source} is not in the present working directory'
)
target = os.path.join(tmpdir,source)
shutil.copyfile(source, target)
print(f'copied {source} to {target}')
if rfile.fmap is not None:
source = utils.add_extension(
csec['fmap'], hcam.HCAM
)
if os.path.dirname(source) != '':
raise HipercamError(
f'fringe map = {source} is not in the present working directory'
)
target = os.path.join(tmpdir,source)
shutil.copyfile(source, target)
print(f'copied {source} to {target}')
if rfile.fpair is not None:
source = utils.add_extension(
csec['fpair'], hcam.FRNG
)
if os.path.dirname(source) != '':
raise HipercamError(
f'fringe peak/trough pair file = {source}'
' is not in the present working directory'
)
target = os.path.join(tmpdir,source)
shutil.copyfile(source, target)
print(f'copied {source} to {target}')
readme = os.path.join(tmpdir,'README')
with open(readme,'w') as fp:
fp.write(README)
# tar up the results
args = ['tar','cvfz',f'{dname}.tar.gz','-C',tdir,dname]
subprocess.run(args)
def combine(args=None):
"""``combine list bias dark flat method (sigma) adjust (usemean) [plot clobber]
output``
Combines a series of images defined by a list using median or clipped
mean combination. Only combines those CCDs for which is_data() is true
(i.e. it skips blank frames caused by NSKIP / NBLUE options)
Parameters:
list : string
list of hcm files with images to combine. The formats of the
images should all match
bias : string
Name of bias frame to subtract, 'none' to ignore.
dark : string
Name of dark frame to subtract, 'none' to ignore.
flat : string
Name of flat field frame to subtract, 'none' to ignore.
method : string
'm' for median, 'c' for clipped mean. See below for pros and cons.
sigma : float [if method == 'c']
With clipped mean combination, pixels that deviate by more than
sigma RMS from the mean are kicked out. This is carried out in an
iterative manner. sigma <= 0 implies no rejection, just a straight
average. sigma=3 is typical.
adjust : string
adjustments to make: 'i' = ignore; 'n' = normalise the mean or
median of all frames to match the first; 'b' = add offsets so that
the mean or median of all frames is the same as the first. Option
'n' is useful for twilight flats and fringe frames; 'b' is good
for combining biases.
usemean : bool [if adjust == 'n' or 'b']
True to use the mean rather than the median for normalisation or
biass offsetting.
plot : bool [hidden, if adjust == 'n' or 'b'; defaults to False]
make a plot of the mean versus frame number. This can provide a
quick check that the frames are not too different.
clobber : bool [hidden]
clobber any pre-existing output files
output : string
output file
Clipped mean can work well for large numbers of frames but gets worse
for small numbers as the RMS can be heavily influenced by a single bad
value. The median can be better in such cases, but has the downside of
digitisation noise. For instance, the average of 100 bias frames could
have a noise level significantly below 1 count, depending upon the
readout noise, and the +/- 0.5 count uncertainty of median combination
may be worse than this.
.. Note::
This routine reads all inputs into memory, so can be a bit of a
hog. However, it does so one CCD at a time to alleviate this. It will
fail if it cannot find a valid frame for any CCD.
"""
command, args = utils.script_args(args)
# get the inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("list", Cline.GLOBAL, Cline.PROMPT)
cl.register("bias", Cline.LOCAL, Cline.PROMPT)
cl.register("dark", Cline.LOCAL, Cline.PROMPT)
cl.register("flat", Cline.LOCAL, Cline.PROMPT)
cl.register("method", Cline.LOCAL, Cline.PROMPT)
cl.register("sigma", Cline.LOCAL, Cline.PROMPT)
cl.register("adjust", Cline.LOCAL, Cline.PROMPT)
cl.register("usemean", Cline.LOCAL, Cline.PROMPT)
cl.register("plot", Cline.LOCAL, Cline.HIDE)
cl.register("clobber", Cline.LOCAL, Cline.HIDE)
cl.register("output", Cline.LOCAL, Cline.PROMPT)
# get inputs
flist = cl.get_value("list", "list of files to combine",
cline.Fname("files", hcam.LIST))
# bias frame (if any)
bias = cl.get_value(
"bias",
"bias frame ['none' to ignore]",
cline.Fname("bias", hcam.HCAM),
ignore="none",
)
if bias is not None:
# read the bias frame
bias = hcam.MCCD.read(bias)
# dark frame (if any)
dark = cl.get_value(
"dark",
"dark frame ['none' to ignore]",
cline.Fname("dark", hcam.HCAM),
ignore="none",
)
if dark is not None:
# read the dark frame
dark = hcam.MCCD.read(dark)
# flat frame (if any)
flat = cl.get_value(
"flat",
"flat frame ['none' to ignore]",
cline.Fname("flat", hcam.HCAM),
ignore="none",
)
if flat is not None:
# read the flat frame
flat = hcam.MCCD.read(flat)
method = cl.get_value("method",
"c(lipped mean), m(edian)",
"c",
lvals=("c", "m"))
if method == "c":
sigma = cl.get_value("sigma", "number of RMS deviations to clip",
3.0)
adjust = cl.get_value(
"adjust",
"i(gnore), n(ormalise), b(ias offsets)",
"i",
lvals=("i", "n", "b"),
)
if adjust == "n" or adjust == "b":
usemean = cl.get_value(
"usemean",
"use the mean for normalisation / offsetting [else median]",
True,
)
plot = cl.get_value(
"plot",
"plot mean levels versus frame number?"
if usemean else "plot median levels versus frame number?",
False,
)
else:
plot = False
usemean = False
clobber = cl.get_value("clobber",
"clobber any pre-existing files on output",
False)
outfile = cl.get_value(
"output",
"output file",
cline.Fname("hcam", hcam.HCAM,
cline.Fname.NEW if clobber else cline.Fname.NOCLOBBER),
)
# inputs done with
# Read the first file of the list to act as a template
# for the CCD names etc.
with open(flist) as fin:
for line in fin:
if not line.startswith("#") and not line.isspace():
template_name = line.strip()
break
else:
raise hcam.HipercamError("List = {:s} is empty".format(flist))
template = hcam.MCCD.read(utils.add_extension(template_name, hcam.HCAM))
if bias is not None:
# crop the bias
bias = bias.crop(template)
if dark is not None:
# crop the dark
dark = dark.crop(template)
if flat is not None:
# crop the flat
flat = flat.crop(template)
# Now process each file CCD by CCD to reduce the memory
# footprint
for cnam in template:
# Read files into memory, insisting that they
# all have the same set of CCDs
print("\nLoading all CCDs labelled '{:s}' from {:s}".format(
cnam, flist))
ccds, means = [], []
nrej, ntot = 0, 0
with spooler.HcamListSpool(flist, cnam) as spool:
if bias is not None:
# extract relevant CCD from the bias
bccd = bias[cnam]
bexpose = bias.head.get("EXPTIME", 0.0)
else:
bexpose = 0.0
if dark is not None:
# extract relevant CCD from the dark
dccd = dark[cnam]
dexpose = dark.head["EXPTIME"]
if flat is not None:
# extract relevant CCD from the flat
fccd = flat[cnam]
mean = None
for ccd in spool:
if ccd.is_data():
if bias is not None:
# subtract bias
ccd -= bccd
if dark is not None:
# subtract dark
scale = (ccd.head["EXPTIME"] - bexpose) / dexpose
ccd -= scale * dccd
if flat is not None:
# apply flat
ccd /= fccd
# keep the result
ccds.append(ccd)
if (adjust == "b" or adjust == "n") and mean is None:
# store the first mean [median]
if usemean:
mean = ccd.mean()
else:
mean = ccd.median()
if len(ccds) == 0:
raise hcam.HipercamError("Found no valid examples of CCD {:s}"
" in list = {:s}".format(cnam, flist))
else:
print("Loaded {:d} CCDs".format(len(ccds)))
if adjust == "b" or adjust == "n":
# adjust the means
print("Computing and adjusting their mean levels")
# now carry out the adjustments
for ccd in ccds:
if usemean:
cmean = ccd.mean()
else:
cmean = ccd.median()
means.append(cmean)
if adjust == "b":
ccd += mean - cmean
elif adjust == "n":
ccd *= mean / cmean
if plot:
plt.plot(means)
plt.plot(means, ".k")
plt.text(len(means) + 1,
means[-1],
cnam,
va="center",
ha="left")
# Finally, combine
if method == "m":
print("Combining them (median) and storing the result")
elif method == "c":
print("Combining them (clipped mean) and storing the result")
else:
raise NotImplementedError(
"method = {:s} not implemented".format(method))
for wnam, wind in template[cnam].items():
# build list of all data arrays
arrs = [ccd[wnam].data for ccd in ccds]
arr3d = np.stack(arrs)
# at this point, arr3d is a 3D array, with the first dimension
# (axis=0) running over the images. We want to average / median
# over this axis.
if method == "m":
# median
wind.data = np.median(arr3d, axis=0)
elif method == "c":
# Cython routine avgstd requires np.float32 input
arr3d = arr3d.astype(np.float32)
if sigma > 0.0:
avg, std, num = support.avgstd(arr3d, sigma)
nrej += len(ccds) * num.size - num.sum()
ntot += len(ccds) * num.size
else:
avg = np.mean(arr3d, axis=0)
wind.data = avg
# Add history
if method == "m":
template[cnam].head.add_history(
"Median combine of {:d} images".format(len(ccds)))
elif method == "c":
print("Rejected {:d} pixels = {:.3f}% of the total".format(
nrej, 100 * nrej / ntot))
template[cnam].head.add_history(
"Clipped mean combine of {:d} images, sigma = {:.1f}".format(
len(ccds), sigma))
# write out
template.write(outfile, clobber)
print("\nFinal result written to {:s}".format(outfile))
# finally
if plot:
plt.xlabel("Frame number")
plt.ylabel("Mean counts")
plt.show()
def redanal(args=None):
"""``redanal aper log``
This provides some basic stats on a log file such as the fraction of duff
points for each aperture of each CCD, how much targets have moved and the
like. The aim is to help with setting parameters in the reduce file in
difficult cases and to diagnose problems. It will probably be added to with
time.
Parameters:
aper : string
the aperture file used for the reduction.
log : string
the log file.
"""
command, args = utils.script_args(args)
# get input section
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("aper", Cline.LOCAL, Cline.PROMPT)
cl.register("log", Cline.LOCAL, Cline.PROMPT)
# get inputs
aper_file = cl.get_value(
"aper", "aperture file used for reduction", cline.Fname("run001", hcam.APER)
)
aper = hcam.MccdAper.read(aper_file)
log_file = cl.get_value(
"log",
"ASCII reduction log file to analyse",
cline.Fname("run001", hcam.LOG),
)
log = hcam.hlog.Hlog.rascii(log_file)
for cnam in sorted(log):
print()
mjds = log[cnam]["MJD"]
mjdoks = log[cnam]["MJDok"]
diffs = mjds[1:] - mjds[:-1]
tgap_max = diffs.max()
tgap_mean = diffs.mean()
print(
"CCD {:s} mean / maximum time gap = {:.2f} / {:.2f} seconds".format(
cnam, 86400.0 * tgap_mean, 86400.0 * tgap_max
)
)
apnams = log.apnames[cnam]
for apnam in apnams:
if not aper[cnam][apnam].linked:
x = log[cnam]["x_{:s}".format(apnam)]
xe = log[cnam]["xe_{:s}".format(apnam)]
y = log[cnam]["y_{:s}".format(apnam)]
ye = log[cnam]["ye_{:s}".format(apnam)]
ok = (xe > 0) & (ye > 0)
xdiffs = x[ok][1:] - x[ok][:-1]
ydiffs = x[ok][1:] - x[ok][:-1]
jitt = np.sqrt(xdiffs ** 2 + ydiffs ** 2)
print(
"CCD {:s}, ap {:s} has {:d}/{:d}/{:d} OK/NOK/total points. Min/mean/median/max jitter = {:.2f}/{:.2f}/{:.2f}/{:.2f} pixels {:s}".format(
cnam,
apnam,
len(x[ok]),
len(x) - len(x[ok]),
len(x),
jitt.min(),
jitt.mean(),
np.median(jitt),
jitt.max(),
"[reference]" if aper[cnam][apnam].ref else "[non-reference]",
)
)
else:
print(
"CCD {:s}, aperture {:s} is linked to aperture {:s}".format(
cnam, apnam, aper[cnam][apnam].link
)
)
def makefield(args=None):
"""Script to generate an artificial star field which is saved to a disk
file, a first step in generating fake data. The resulting files are needed
by makedata if you want to add in artificial star fields. If the name
supplied corresponds to an existing file, an attempt will be made to read
it in first and then add to it. In this way a complex field can be
generated. The targets are distributed at random, with random peak heights
based on constant luminosity objects distributed throughout 3D space, and
random angles uniform over the input range. All targets otherwise have the
same shape thus multiple calls are needed to generate a field of objects
of multiple shapes. Ellipsoidal "Moffat" functions [1/(1+r^2)^beta] are
used.
Arguments::
fname : (string)
file to add to. Will be created if it does not exist. [input, optional / output]
ntarg : (int)
The number of targets to add to the field.
x1 : (float)
The left-hand limit of the field [unbinned pixels]
x2 : (float)
The right-hand limit of the field [unbinned pixels]
y1 : (float)
The lowest limit of the field [unbinned pixels]
y2 : (float)
The highest limit of the field [unbinned pixels]
h1 : (float)
Minimum peak height [counts per unbinned pixel]
h2 : (float)
Maximum peak height [counts per unbinned pixel]
angle1 : (float)
Lower limit on axis 1, anti-clockwise from X-axis [degrees]
angle2 : (float)
Upper limit on axis 1, anti-clockwise from X-axis [degrees]
fwhm1 : (float)
FWHM along axis 1 [unbinned pixels]
fwhm2 : (float)
FWHM along axis 2 [unbinned pixels]
beta : (float)
Moffat function exponent
"""
command, args = utils.script_args(args)
# create Cline object
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("fname", Cline.LOCAL, Cline.PROMPT)
cl.register("ntarg", Cline.LOCAL, Cline.PROMPT)
cl.register("x1", Cline.LOCAL, Cline.PROMPT)
cl.register("x2", Cline.LOCAL, Cline.PROMPT)
cl.register("y1", Cline.LOCAL, Cline.PROMPT)
cl.register("y2", Cline.LOCAL, Cline.PROMPT)
cl.register("h1", Cline.LOCAL, Cline.PROMPT)
cl.register("h2", Cline.LOCAL, Cline.PROMPT)
cl.register("angle1", Cline.LOCAL, Cline.PROMPT)
cl.register("angle2", Cline.LOCAL, Cline.PROMPT)
cl.register("fwhm1", Cline.LOCAL, Cline.PROMPT)
cl.register("fwhm2", Cline.LOCAL, Cline.PROMPT)
cl.register("beta", Cline.LOCAL, Cline.PROMPT)
cl.register("fmin", Cline.LOCAL, Cline.PROMPT)
# get inputs
fname = cl.get_value(
"fname",
"file to save field to",
cline.Fname("field", hcam.FIELD, cline.Fname.NEW),
)
if os.path.exists(fname):
# Initialise the field from a file
field = hcam.Field.rjson(fname)
print(">> Loaded a field of", len(field), "objects from", fname)
else:
# Create an empty field
field = hcam.Field()
print(">> Created an empty field.")
ntarg = cl.get_value("ntarg", "number of targets", 100, 1)
x1 = cl.get_value("x1", "left-hand limit of field", -10.0)
x2 = cl.get_value("x2", "right-hand limit of field", 2000.0, x1)
y1 = cl.get_value("y1", "lower limit of field", -10.0)
y2 = cl.get_value("y2", "upper limit of field", 1000.0, y1)
h1 = cl.get_value("h1", "lower peak height limit", 0.1, 1.0e-6)
h2 = cl.get_value("h2", "upper peak height limit", 1000.0, h1)
angle1 = cl.get_value("angle1", "lower limit of axis 1 angle", 0.0,
-360.0, 360.0)
angle2 = cl.get_value("angle2", "angle of major axis", 0.0, angle1,
360.0)
fwhm1 = cl.get_value("fwhm1", "FWHM along axis 1", 4.0, 1.0e-6)
fwhm2 = cl.get_value("fwhm2", "FWHM along axis 2", 4.0, 1.0e-6)
beta = cl.get_value("beta", "Moffat exponent", 4.0, 1.0)
fmin = cl.get_value("fmin", "minimum flux level (counts/pix)", 0.001,
0.0)
# add targets
field.add_random(ntarg, x1, x2, y1, y2, h1, h2, angle1, angle2, fwhm1,
fwhm2, beta, fmin)
# save result
field.wjson(fname)
print(">> Saved a field of", len(field), "objects to", fname)
def mstats(args=None):
"""``mstats [source] run [temp] (ndigit) first last [twait tmax] bias
[dtype]``
This downloads a sequence of images from a raw data file and writes
out stats (min, max, mean, median, rms) for each window to a file
Parameters:
source : string [hidden]
Data source, four options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : ULTRACAM server
| 'ul' : local ULTRACAM .xml/.dat files
run : string
run name to access
first : int
First frame to access
last : int
Last frame to access, 0 for the lot
twait : float [hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
bias : string
Name of bias frame to subtract, 'none' to ignore.
format : string
output format for numbers. e.g. the default '9.3f'
might give 12345.678 (9 characters, 3 digits after d.p.)
outfile : string
file for output (extension .stats)
"""
command, args = utils.script_args(args)
# get inputs
with Cline('HIPERCAM_ENV', '.hipercam', command, args) as cl:
# register parameters
cl.register('source', Cline.GLOBAL, Cline.HIDE)
cl.register('run', Cline.GLOBAL, Cline.PROMPT)
cl.register('first', Cline.LOCAL, Cline.PROMPT)
cl.register('last', Cline.LOCAL, Cline.PROMPT)
cl.register('twait', Cline.LOCAL, Cline.HIDE)
cl.register('tmax', Cline.LOCAL, Cline.HIDE)
cl.register('bias', Cline.GLOBAL, Cline.PROMPT)
cl.register('format', Cline.LOCAL, Cline.HIDE)
cl.register('outfile', Cline.LOCAL, Cline.PROMPT)
# get inputs
source = cl.get_value('source',
'data source [hs, hl, us, ul]',
'hl',
lvals=('hs', 'hl', 'us', 'ul'))
# OK, more inputs
resource = cl.get_value('run', 'run name', 'run005')
first = cl.get_value('first', 'first frame to grab', 1, 0)
last = cl.get_value('last', 'last frame to grab', 0)
if last < first and last != 0:
sys.stderr.write('last must be >= first or 0')
sys.exit(1)
twait = cl.get_value('twait', 'time to wait for a new frame [secs]',
1., 0.)
tmax = cl.get_value('tmax',
'maximum time to wait for a new frame [secs]', 10.,
0.)
bias = cl.get_value('bias',
"bias frame ['none' to ignore]",
cline.Fname('bias', hcam.HCAM),
ignore='none')
cl.set_default('format', '9.3f')
form = cl.get_value('format', 'output format for stats', '9.3f')
outfile = cl.get_value('outfile', 'output file for stats',
cline.Fname('stats', '.stats', cline.Fname.NEW))
# Now the actual work.
# strip off extensions
if resource.endswith(hcam.HRAW):
resource = resource[:resource.find(hcam.HRAW)]
# initialisations
total_time = 0 # time waiting for new frame
nframe = first
root = os.path.basename(resource)
bframe = None
with spooler.data_source(source, resource, first) as spool:
with open(outfile, 'w') as stats:
stats.write("""#
# This file was generated by mstats running on file {run}
#
# The columns are:
#
# nframe ccd window minimum maximum mean median rms
#
# where ccd and window are string labels, nframe is the frame
# number an an integer, while the rest are floats.
#
""".format(run=resource))
for mccd in spool:
# Handle the waiting game ...
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmax, total_time)
if give_up:
print('mstats stopped')
break
elif try_again:
continue
if bias is not None:
# read bias after first frame so we can
# chop the format
if bframe is None:
# read the bias frame
bframe = hcam.MCCD.read(bias)
# reformat
bframe = bframe.crop(mccd)
mccd -= bframe
for cnam, ccd in mccd.items():
for wnam, wind in ccd.items():
stats.write(
('{1:5d} {2:5s} {3:5s} {4:{0:s}} {5:{0:s}}'
' {6:{0:s}} {7:{0:s}} {8:{0:s}}\n').format(
form, nframe, cnam, wnam, wind.min(),
wind.max(), wind.mean(), wind.median(),
wind.std()))
# flush the output
stats.flush()
# progress info
print('Written stats of frame {:d} to {:s}'.format(
nframe, outfile))
# update the frame number
nframe += 1
if last and nframe > last:
break
def aligntool(args=None):
"""
Measures alignment of HiperCAM CCDs w.r.t a reference image of the blue CCD.
Arguments::
ref : (string)
name of an MCCD file to use as reference , as produced
by e.g. 'grab'
rccd : (string)
CCD to use as reference
thresh : (float)
threshold for object detection, in multiples of background RMS
source : (string) [hidden]
Data source, five options::
'hs' : HiPERCAM server
'hl' : local HiPERCAM FITS file
'us' : ULTRACAM server
'ul' : local ULTRACAM .xml/.dat files
'hf' : list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab' or
converted from foreign data formats.
device : (string) [hidden]
Plot device. PGPLOT is used so this should be a PGPLOT-style name,
e.g. '/xs', '1/xs' etc. At the moment only ones ending /xs are
supported.
width : (float) [hidden]
plot width (inches). Set = 0 to let the program choose.
height : (float) [hidden]
plot height (inches). Set = 0 to let the program choose. BOTH width
AND height must be non-zero to have any effect
run : (string) [if source == 's' or 'l']
run number to access, e.g. 'run034'
flist : (string) [if source == 'f']
name of file list
first : (int) [if source='s' or 'l']
exposure number to start from. 1 = first frame; set = 0 to
always try to get the most recent frame (if it has changed)
twait : (float) [if source == 's'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : (float) [if source == 's'; hidden]
maximum time to wait between attempts to find a new exposure, seconds.
pause : (float) [hidden]
seconds to pause between frames (defaults to 0)
ccd : (string)
CCD to measure aligment of.
bias : (string)
Name of bias frame to subtract, 'none' to ignore.
msub : (bool)
subtract the median from each window before scaling for the
image display or not. This happens after any bias subtraction.
iset : (string) [single character]
determines how the intensities are determined. There are three
options: 'a' for automatic simply scales from the minimum to the
maximum value found on a per CCD basis. 'd' for direct just takes
two numbers from the user. 'p' for percentile dtermines levels
based upon percentiles determined from the entire CCD on a per CCD
basis.
ilo : (float) [if iset='d']
lower intensity level
ihi : (float) [if iset='d']
upper intensity level
plo : (float) [if iset='p']
lower percentile level
phi : (float) [if iset='p']
upper percentile level
small_spots : (bool) [hidden]
use the small spots for alignment instead of the big spots
"""
command, args = utils.script_args(args)
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("ref", Cline.LOCAL, Cline.PROMPT)
cl.register("thresh", Cline.LOCAL, Cline.PROMPT)
cl.register("small_spots", Cline.LOCAL, Cline.HIDE)
cl.register("source", Cline.LOCAL, Cline.HIDE)
cl.register("device", Cline.LOCAL, Cline.HIDE)
cl.register("width", Cline.LOCAL, Cline.HIDE)
cl.register("height", Cline.LOCAL, Cline.HIDE)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("twait", Cline.LOCAL, Cline.HIDE)
cl.register("tmax", Cline.LOCAL, Cline.HIDE)
cl.register("flist", Cline.LOCAL, Cline.PROMPT)
cl.register("ccd", Cline.LOCAL, Cline.PROMPT)
cl.register("rccd", Cline.LOCAL, Cline.PROMPT)
cl.register("pause", Cline.LOCAL, Cline.HIDE)
cl.register("bias", Cline.GLOBAL, Cline.PROMPT)
cl.register("msub", Cline.GLOBAL, Cline.PROMPT)
cl.register("iset", Cline.GLOBAL, Cline.PROMPT)
cl.register("ilo", Cline.GLOBAL, Cline.PROMPT)
cl.register("ihi", Cline.GLOBAL, Cline.PROMPT)
cl.register("plo", Cline.GLOBAL, Cline.PROMPT)
cl.register("phi", Cline.LOCAL, Cline.PROMPT)
cl.register("xlo", Cline.GLOBAL, Cline.PROMPT)
cl.register("xhi", Cline.GLOBAL, Cline.PROMPT)
cl.register("ylo", Cline.GLOBAL, Cline.PROMPT)
cl.register("yhi", Cline.GLOBAL, Cline.PROMPT)
# get inputs
reference = cl.get_value("ref", "reference frame to align to",
cline.Fname("hcam", hcam.HCAM))
ref_mccd = hcam.MCCD.read(reference)
thresh = cl.get_value("thresh", "detection threshold (sigma)", 5.0)
small_spots = cl.get_value("small_spots",
"use small spots for alignment", False)
# get inputs
source = cl.get_value(
"source",
"data source [hs, hl, us, ul, hf]",
"hl",
lvals=("hs", "hl", "us", "ul", "hf"),
)
# set some flags
server_or_local = source.endswith("s") or source.endswith("l")
inst = source[0]
# plot device stuff
device = cl.get_value("device", "plot device", "1/xs")
width = cl.get_value("width", "plot width (inches)", 0.0)
height = cl.get_value("height", "plot height (inches)", 0.0)
if server_or_local:
resource = cl.get_value("run", "run name", "run005")
first = cl.get_value("first", "first frame to plot", 1, 1)
twait = cl.get_value("twait",
"time to wait for a new frame [secs]", 1.0,
0.0)
tmax = cl.get_value("tmax",
"maximum time to wait for a new frame [secs]",
10.0, 0.0)
else:
# set inst = 'h' as only lists of HiPERCAM files are supported
inst = "h"
resource = cl.get_value("flist", "file list",
cline.Fname("files.lis", hcam.LIST))
first = 1
flist = source.endswith("f")
server = source.endswith("s")
if inst == "u":
instrument = "ULTRA"
elif inst == "h":
instrument = "HIPER"
# get the labels and maximum dimensions and padding factors
ccdinf = spooler.get_ccd_pars(source, resource)
if len(ccdinf) > 1:
ccdnam = cl.get_value("ccd", "CCD to plot alignment of", "1")
rccdnam = cl.get_value("rccd", "CCD to use as reference", "5")
cl.set_default("pause", 0.0)
pause = cl.get_value("pause",
"time delay to add between frame plots [secs]",
0.0, 0.0)
# bias frame (if any)
bias = cl.get_value(
"bias",
"bias frame ['none' to ignore]",
cline.Fname("bias", hcam.HCAM),
ignore="none",
)
if bias is not None:
# read the bias frame
bias = hcam.MCCD.read(bias)
msub = cl.get_value("msub", "subtract median from each window?", True)
iset = cl.get_value(
"iset",
"set intensity a(utomatically),"
" d(irectly) or with p(ercentiles)?",
"a",
lvals=["a", "A", "d", "D", "p", "P"],
)
iset = iset.lower()
plo, phi = 5, 95
ilo, ihi = 0, 1000
if iset == "d":
ilo = cl.get_value("ilo", "lower intensity limit", 0.0)
ihi = cl.get_value("ihi", "upper intensity limit", 1000.0)
elif iset == "p":
plo = cl.get_value("plo", "lower intensity limit percentile", 5.0,
0.0, 100.0)
phi = cl.get_value("phi", "upper intensity limit percentile", 95.0,
0.0, 100.0)
# region to plot
nxtot, nytot, nxpad, nypad = ccdinf[ccdnam]
xmin, xmax = float(-nxpad), float(nxtot + nxpad + 1)
ymin, ymax = float(-nypad), float(nytot + nypad + 1)
xlo = cl.get_value("xlo", "left-hand X value", xmin, xmin, xmax)
xhi = cl.get_value("xhi", "right-hand X value", xmax, xmin, xmax)
ylo = cl.get_value("ylo", "lower Y value", ymin, ymin, ymax)
yhi = cl.get_value("yhi", "upper Y value", ymax, ymin, ymax)
# arguments defined, let's do something!
# open image plot device
imdev = hcam.pgp.Device(device)
if width > 0 and height > 0:
pgpap(width, height / width)
pgsubp(1, 2)
for i in range(2):
pgsci(hcam.pgp.Params["axis.ci"])
pgsch(hcam.pgp.Params["axis.number.ch"])
pgenv(xlo, xhi, ylo, yhi, 1, 0)
pglab("X", "Y", ("reference", "data")[i])
total_time = 0 # time waiting for new frame
with spooler.data_source(source, resource, first) as spool:
# 'spool' is an iterable source of MCCDs
for n, mccd in enumerate(spool):
if server_or_local:
# Handle the waiting game ...
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmax, total_time)
if give_up:
print("alignment tool stopped")
break
elif try_again:
continue
# indicate progress
print(
"Frame {:d}, time = {:s}; ".format(mccd.head["NFRAME"],
mccd.head["TIMSTAMP"]),
end="",
)
if n == 0 and bias is not None:
# crop the bias on the first frame only
bias = bias.crop(mccd)
ccd = mccd[ccdnam]
ref_ccd = ref_mccd[rccdnam]
if bias is not None:
ccd -= bias[ccdnam]
if msub:
for wind in ccd.values():
wind -= wind.median()
for wind in ref_ccd.values():
wind -= wind.median()
# plot images
message = ""
for i in range(2):
this_ccd = (ref_ccd, ccd)[i]
name = ("ref: ", "data: ")[i]
pgpanl(1, i + 1)
vmin, vmax = hcam.pgp.pCcd(this_ccd, iset, plo, phi, ilo, ihi)
message += " {:s}: {:.2f} to {:.2f}".format(name, vmin, vmax)
print(message)
# time for measurement of spots!
try:
xpos, ypos, fwhm, flux = measureSpots(mccd, ccdnam, thresh)
lx, ly, lf, lp, bx, by, bf, bp = separateBigSmallSpots(
xpos, ypos, fwhm, flux)
if not small_spots:
# use big spots
x, y, f, flux = bx, by, bf, bp
else:
x, y, f, flux = lx, ly, lf, lp
except Exception as err:
print("Failed to find spots in frame: ", end=" ")
print(str(err))
continue
# also measure spots in reference image
try:
xpos, ypos, fwhm, rflux = measureSpots(ref_mccd, rccdnam,
thresh)
lx, ly, lf, lp, bx, by, bf, bp = separateBigSmallSpots(
xpos, ypos, fwhm, rflux)
if not small_spots:
# use big spots
xref, yref, fref = bx, by, bf
else:
xref, yref, fref = lx, ly, lf
except Exception as err:
print("Failed to find spots in reference: ", end=" ")
print(str(err))
continue
plt.clf()
displayFWHM(x, y, xref, yref, f, flux, fix_fwhm_scale=False)
plt.draw()
plt.pause(0.05)
plt.ioff()
plt.show()
def hplot(args=None):
"""``hplot input [device] ccd nx msub ([cmap]) hsbox iset (ilo ihi | plo
phi) xlo xhi ylo yhi [width height]``
Plots a multi-CCD image. Can use PGPLOT or matplotlib. The matplotlib
version is slightly clunky in its choice of the viewing area but has some
features that could be useful, in particular, the interactive plot
(device='/mpl') allows one to pan and zoom and to compare the same part of
multiple CCDs easily.
Parameters:
input : string
name of MCCD file
device : string [hidden]
Plot device name. Uses characters after a final trailing '/' to
identify the type in PGPLOT style. Thus:
| /xs : PGPLOT xserver interactive plot
| 1/xs : PGPLOT xserver interactive plot called '1'
| plot.ps/cps : PGPLOT colour postscript called 'plot.ps'
| plot.ps/vps : PGPLOT B&W portrait oriented plot
| /mpl : matplotlib interactive plot
| plot.pdf/mpl : matplotlib PDF plot
ccd : string
CCD(s) to plot, '0' for all. If not '0' then '1', '2' or even '3 4'
are possible inputs (without the quotes). '3 4' will plot CCD '3' and
CCD '4'. If you want to plot more than one CCD, then you will be
prompted for the number of panels in the X direction. This parameter
will not be prompted if there is only one CCD in the file.
nx : int
number of panels across to display, prompted if more than one CCD is
to be plotted.
msub : bool
True/False to subtract median from each window before scaling
cmap : str [if matplotlib; hidden]
The colour map to use. "Greys" is the usual, but there are
many others. Typing an incorrect one will give a list. "none"
for matplotlib default.
hsbox : int [if device = '/mpl'; hidden]
half-width in binned pixels of stats box as offset from central pixel
hsbox = 1 gives a 3x3 box; hsbox = 2 gives 5x5 etc.
iset : string [single character]
determines how the intensities are determined. There are three
options: 'a' for automatic simply scales from the minimum to the
maximum value found on a per CCD basis. 'd' for direct just takes two
numbers from the user. 'p' for percentile dtermines levels based upon
percentiles determined from the entire CCD on a per CCD bais.
ilo : float [if iset=='d']
lower intensity level
ihi : float [if iset=='d']
upper intensity level
plo : float [if iset=='p']
lower percentile level
phi : float [if iset=='p']
upper percentile level
xlo : float
left X-limit, for PGPLOT plots. Applies to matplotlib plots
to restrict region used to compute percentile limits. This is
useful in case where bias strips otherwise distort the plot
limits (e.g. ultraspec full frame images)
xhi : float
right X-limit. See comments for xlo as well.
ylo : float
bottom Y-limit. See comments for xlo as well.
yhi : float
top Y-limit. See comments for xlo as well.
width : float [hidden]
plot width (inches). Set = 0 to let the program choose.
height : float [hidden]
plot height (inches). Set = 0 to let the program choose. BOTH width
AND height must be non-zero to have any effect
"""
global fig, mccd, caxes, hsbox
command, args = utils.script_args(args)
# get input section
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("input", Cline.LOCAL, Cline.PROMPT)
cl.register("device", Cline.LOCAL, Cline.HIDE)
cl.register("ccd", Cline.LOCAL, Cline.PROMPT)
cl.register("nx", Cline.LOCAL, Cline.PROMPT)
cl.register("msub", Cline.GLOBAL, Cline.PROMPT)
cl.register("cmap", Cline.LOCAL, Cline.HIDE)
cl.register("hsbox", Cline.GLOBAL, Cline.HIDE)
cl.register("iset", Cline.GLOBAL, Cline.PROMPT)
cl.register("ilo", Cline.GLOBAL, Cline.PROMPT)
cl.register("ihi", Cline.GLOBAL, Cline.PROMPT)
cl.register("plo", Cline.GLOBAL, Cline.PROMPT)
cl.register("phi", Cline.GLOBAL, Cline.PROMPT)
cl.register("xlo", Cline.GLOBAL, Cline.PROMPT)
cl.register("xhi", Cline.GLOBAL, Cline.PROMPT)
cl.register("ylo", Cline.GLOBAL, Cline.PROMPT)
cl.register("yhi", Cline.GLOBAL, Cline.PROMPT)
cl.register("width", Cline.LOCAL, Cline.HIDE)
cl.register("height", Cline.LOCAL, Cline.HIDE)
# get inputs
frame = cl.get_value("input", "frame to plot",
cline.Fname("hcam", hcam.HCAM))
mccd = hcam.MCCD.read(frame)
device = cl.get_value("device", "plot device name", "/mpl")
# set type of plot (PGPLOT or matplotlib) and the name of the file
# if any in the case of matplotlib
fslash = device.rfind("/")
if fslash > -1:
if device[fslash + 1:] == "mpl":
ptype = "MPL"
hard = device[:fslash].strip()
else:
ptype = "PGP"
else:
raise ValueError(
"Could not identify plot type from device = {:s}".format(
device))
# define the panel grid
nxdef = cl.get_default("nx", 3)
max_ccd = len(mccd)
if max_ccd > 1:
ccd = cl.get_value("ccd", "CCD(s) to plot [0 for all]", "0")
if ccd == "0":
ccds = list(mccd.keys())
else:
ccds = ccd.split()
if len(ccds) > 1:
nxdef = min(len(ccds), nxdef)
cl.set_default("nx", nxdef)
nx = cl.get_value("nx", "number of panels in X", 3, 1)
else:
nx = 1
else:
ccds = list(mccd.keys())
nx = 1
# define the display intensities
msub = cl.get_value("msub", "subtract median from each window?", True)
if ptype == "MPL":
cmap = cl.get_value("cmap",
"colour map to use ['none' for mpl default]",
"Greys")
cmap = None if cmap == "none" else cmap
if ptype == "MPL" and hard == "":
hsbox = cl.get_value("hsbox",
"half-width of stats box (binned pixels)", 2,
1)
iset = cl.get_value(
"iset",
"set intensity a(utomatically), d(irectly) or with p(ercentiles)?",
"a",
lvals=["a", "A", "d", "D", "p", "P"],
)
iset = iset.lower()
plo, phi = 5, 95
ilo, ihi = 0, 1000
if iset == "d":
ilo = cl.get_value("ilo", "lower intensity limit", 0.0)
ihi = cl.get_value("ihi", "upper intensity limit", 1000.0)
elif iset == "p":
plo = cl.get_value("plo", "lower intensity limit percentile", 5.0,
0.0, 100.0)
phi = cl.get_value("phi", "upper intensity limit percentile", 95.0,
0.0, 100.0)
# region to plot
for i, cnam in enumerate(ccds):
ccd = mccd[cnam]
nxtot, nytot, nxpad, nypad = ccd.nxtot, ccd.nytot, ccd.nxpad, ccd.nypad
if i == 0:
xmin, xmax = float(-nxpad), float(nxtot + nxpad + 1)
ymin, ymax = float(-nypad), float(nytot + nypad + 1)
else:
xmin = min(xmin, float(-nxpad))
xmax = max(xmax, float(nxtot + nxpad + 1))
ymin = min(ymin, float(-nypad))
ymax = max(ymax, float(nytot + nypad + 1))
xlo = cl.get_value("xlo",
"left-hand X value",
xmin,
xmin,
xmax,
enforce=False)
xhi = cl.get_value("xhi",
"right-hand X value",
xmax,
xmin,
xmax,
enforce=False)
ylo = cl.get_value("ylo",
"lower Y value",
ymin,
ymin,
ymax,
enforce=False)
yhi = cl.get_value("yhi",
"upper Y value",
ymax,
ymin,
ymax,
enforce=False)
width = cl.get_value("width", "plot width (inches)", 0.0)
height = cl.get_value("height", "plot height (inches)", 0.0)
# all inputs obtained, plot
if ptype == "MPL":
if width > 0 and height > 0:
fig = plt.figure(figsize=(width, height))
else:
fig = plt.figure()
mpl.rcParams["xtick.labelsize"] = hcam.mpl.Params["axis.number.fs"]
mpl.rcParams["ytick.labelsize"] = hcam.mpl.Params["axis.number.fs"]
nccd = len(ccds)
ny = nccd // nx if nccd % nx == 0 else nccd // nx + 1
ax = None
caxes = {}
for n, cnam in enumerate(ccds):
if ax is None:
axes = ax = fig.add_subplot(ny, nx, n + 1)
axes.set_aspect("equal", adjustable="box")
else:
axes = fig.add_subplot(ny, nx, n + 1, sharex=ax, sharey=ax)
axes.set_aspect("equal")
# store the CCD associated with these axes for the cursor callback
caxes[axes] = cnam
axes.set_xlim(xlo, xhi)
axes.set_ylim(ylo, yhi)
if msub:
# subtract median from each window
for wind in mccd[cnam].values():
wind -= wind.median()
vmin, vmax, _ = hcam.mpl.pCcd(axes,
mccd[cnam],
iset,
plo,
phi,
ilo,
ihi,
"CCD {:s}".format(cnam),
xlo=xlo,
xhi=xhi,
ylo=ylo,
yhi=yhi,
cmap=cmap)
print("CCD =", cnam, "plot range =", vmin, "to", vmax)
try:
plt.tight_layout()
except:
pass
if hard == "":
# add in the callback
fig.canvas.mpl_connect("button_press_event", buttonPressEvent)
print("\nClick points in windows for stats in a {:d}x{:d} box".
format(2 * hsbox + 1, 2 * hsbox + 1))
plt.subplots_adjust(wspace=0.1, hspace=0.1)
plt.show()
else:
plt.savefig(hard, bbox_inches="tight", pad_inches=0)
elif ptype == "PGP":
# open the plot
dev = hcam.pgp.Device(device)
if width > 0 and height > 0:
pgpap(width, height / width)
nccd = len(ccds)
ny = nccd // nx if nccd % nx == 0 else nccd // nx + 1
# set up panels and axes
pgsubp(nx, ny)
for cnam in ccds:
pgsci(hcam.pgp.Params["axis.ci"])
pgsch(hcam.pgp.Params["axis.number.ch"])
pgenv(xlo, xhi, ylo, yhi, 1, 0)
vmin, vmax = hcam.pgp.pCcd(mccd[cnam], iset, plo, phi, ilo, ihi,
"CCD {:s}".format(cnam))
print("CCD =", cnam, "plot range =", vmin, "to", vmax)
def flagcloud(args=None):
"""``flagcloud hlog aper1 aper2 ccd delta output``
Interactive flagging of cloud-affected or otherwise bad points in a
|hiper| log file. You either mark a range of times as cloudy,
or individual points as junk. If you mark a time range, then *all*
apertures of *all* CCDs will be flagged with the bitmask value
CLOUDS. Individual points will be flagged as JUNK. Note that nothing
is done to the data apart from changing the bitmask flags, so it is
then up to you to test for these later on. It is also possible to
flag individual points as CLOUDS but these will not propagate across
CCDs so it is probably not advisable to do this.
Junk points are marked red, cloudy points orange. OK aperture 1
points are plotted green, aperture 2 blue, their ratio black. What
is meant by 'junk' as opposed to 'cloud' is really down to the
user. I tend to reserve junk for one-off points affected by bad
cosmic rays and satellites, but in general it is probably sensible
to think of junk as points you never want to see again versus
clouds meaning data that you might want to mask or use down the
line or perhaps just grey out in plots. Some genuine "cloudy" data
will be so bad that it will be better flagged as junk however. You can
flag the same point as both "cloud" and "junk", but "junk" is the
stronger condition.
Bitmasks propagate when data are combined so a point flagged junk in
aperture 2 but not aperture 1 will be flagged junk in the ratio of 1
divided by 2. You can also recover points in this routine; doing so will
clear both their junk and/or cloud status. At the moment this is only
possible on a point-by-point basis.
Interaction is via the cursor and hitting specific keys. Common options are
lower case; less common ones upper case. Hitting the X on the plot will abort
without saving the results. 'q' to quite saves the results. 'h' will give some
help on the options.
Parameters:
hlog : str
ASCII log file, as produced by |reduce|.
aper1 : str
the name of first aperture to look at
aper2 : str
the name of second aperture to look at. The ratio aper1 / aper2
will be plotted along with the two separately, scaled by their
maximum, all in the same panel.
ccd : str
CCD(s) to plot, '0' for all. If not '0' then '1', '2', or even '1 2 3'
are possible inputs (without the quotes). Note the space separation when multiple
CCDs are specified. If you want to plot more than one CCD, then you will get multiple
panels in the Y direction, but their X-axes are kept in lock step when panning or zooming.
delta : float
separation to use to space the plots in a given panel, each of which is normalised
to 1. A value of 1 is recommended because then the second aperture should end with
a typical level of 0, and any dips below this show the extent of the cloud. e.g. -0.9
would suggest an approximate 90% loss of flux due to cloud, ignoring extinction.
output : str
name of modified version of the Hlog for output. Can overwrite the original if you dare.
"""
command, args = utils.script_args(args)
# get input section
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("hlog", Cline.LOCAL, Cline.PROMPT)
cl.register("aper1", Cline.LOCAL, Cline.PROMPT)
cl.register("aper2", Cline.LOCAL, Cline.PROMPT)
cl.register("ccd", Cline.LOCAL, Cline.PROMPT)
cl.register("delta", Cline.LOCAL, Cline.PROMPT)
cl.register("output", Cline.LOCAL, Cline.PROMPT)
# get inputs
hlog = cl.get_value("hlog", "hipercam ASCII log file",
cline.Fname("hcam", hcam.LOG))
hlog = hcam.hlog.Hlog.read(hlog)
cnams = sorted(hlog.keys())
apnames = set()
for cnam in cnams:
apnames |= set(hlog.apnames[cnam])
apnames = sorted(apnames)
aper1 = cl.get_value("aper1",
"first aperture",
apnames[0],
lvals=apnames)
if len(apnames) > 1:
aper2 = cl.get_value("aper2",
"second aperture",
apnames[-1],
lvals=apnames)
else:
aper2 = None
max_ccd = len(hlog)
if max_ccd > 1:
ccd = cl.get_value("ccd", "CCD(s) to plot [0 for all]", "0")
if ccd == "0":
ccds = cnams
else:
ccds = ccd.split()
if set(ccds) <= set(cnams):
print(
f' selected CCDs ({ccds}) not amongst those in the reduce log ({cnams})'
)
return
else:
ccds = cnams
delta = cl.get_value("delta", "vertical separation between plots", 1,
0.)
output = cl.get_value(
"output",
"name for output log file",
cline.Fname("run", hcam.LOG, cline.Fname.NEW),
)
# Inputs obtained.
print()
# re-configure keyboard shortcuts to avoid otherwise confusing behaviour
# quit_all does not seem to be universal, hence the try/except
try:
mpl.rcParams["keymap.back"] = ""
mpl.rcParams["keymap.forward"] = ""
mpl.rcParams["keymap.fullscreen"] = ""
mpl.rcParams["keymap.grid"] = ""
mpl.rcParams["keymap.home"] = ""
mpl.rcParams["keymap.pan"] = ""
mpl.rcParams["keymap.quit"] = ""
mpl.rcParams["keymap.save"] = ""
mpl.rcParams["keymap.pan"] = ""
mpl.rcParams["keymap.save"] = ""
mpl.rcParams["keymap.xscale"] = ""
mpl.rcParams["keymap.yscale"] = ""
mpl.rcParams["keymap.zoom"] = ""
except KeyError:
pass
cnams, anams, plots = {}, {}, {}
ax = None
fig, axs = plt.subplots(len(ccds), 1, sharex=True)
if len(ccds) == 1:
axs = [axs]
# get the navigation toolbar. Go straight into pan mode where we
# want to stay.
toolbar = fig.canvas.manager.toolbar
if backend != "TkAgg":
toolbar.pan()
ny = len(ccds)
T0 = None
for cnam, ax in zip(ccds, axs):
# prep data
a1 = hlog.tseries(cnam, aper1)
a1.normalise()
if T0 is None:
T0 = a1.t[0]
if aper2:
a2 = hlog.tseries(cnam, aper2)
a2.normalise()
rat = a1 / a2
a2.t -= T0
rat.t -= T0
rat += delta
a2 -= delta
rat.mplot(ax, COL_RAT, bitmask=hcam.JUNK | hcam.CLOUDS)
rat.mplot(ax, COL_CLOUD, bitmask=hcam.CLOUDS, flagged=True)
rat.mplot(ax, COL_JUNK, bitmask=hcam.JUNK, flagged=True)
a2.mplot(ax, COL_AP2, bitmask=hcam.JUNK | hcam.CLOUDS)
a2.mplot(ax, COL_CLOUD, bitmask=hcam.CLOUDS, flagged=True)
a2.mplot(ax, COL_JUNK, bitmask=hcam.JUNK, flagged=True)
a1.t -= T0
a1.mplot(ax, COL_AP1, bitmask=hcam.JUNK | hcam.CLOUDS)
a1.mplot(ax, COL_CLOUD, bitmask=hcam.CLOUDS, flagged=True)
a1.mplot(ax, COL_JUNK, bitmask=hcam.JUNK, flagged=True)
# store the plots needed to identify which point has been selected
# along with names of apertures needed to flag points
plots[cnam] = {
"aper1": aper1,
"a1": a1,
"aper2": aper2,
"a2": a2 if aper2 is not None else None,
"rat": rat if aper2 is not None else None
}
# keep track of the CCD associated with each axes
cnams[ax] = cnam
# and the axes associated with each CCD
anams[cnam] = ax
ax.set_ylabel("CCD {:s}".format(cnam))
ax.set_xlabel(f'Time [MJD - {T0}]')
# create the picker
picker = PickPoints(fig, hlog, cnams, anams, plots, T0, output)
try:
plt.tight_layout()
except:
pass
PickPoints.action_prompt(False)
# squeeze space a bit
plt.subplots_adjust(hspace=0.1)
# finally show stuff ....
plt.xlabel("Time [MJD - {:.7f}]".format(T0))
plt.show()
def flagcloud(args=None):
"""``flagcloud hlog aper1 aper2 ccd output``
Interactive flagging of cloud-affected or bad points in a |hipercam| log file.
Parameters:
hlog : string
ASCII log file.
aper1 : string
the name of first aperture to look at
aper2 : string
the name of second aperture to look at. The ratio aper1 / aper2
will be plotted along with the two separately, scaled by their
maximum, all in the same panel.
ccd : string
CCD(s) to plot, '0' for all. If not '0' then '1', '2' or even '3 4'
are possible inputs (without the quotes). '3 4' will plot CCD '3' and
CCD '4'. If you want to plot more than one CCD, then you will be
prompted for the number of panels in the X direction.
output : string
name of modified version of the Hlog for output.
"""
command, args = utils.script_args(args)
# get input section
with Cline('HIPERCAM_ENV', '.hipercam', command, args) as cl:
# register parameters
cl.register('hlog', Cline.LOCAL, Cline.PROMPT)
cl.register('aper1', Cline.LOCAL, Cline.PROMPT)
cl.register('aper2', Cline.LOCAL, Cline.PROMPT)
cl.register('ccd', Cline.LOCAL, Cline.PROMPT)
cl.register('output', Cline.LOCAL, Cline.PROMPT)
# get inputs
hlog = cl.get_value(
'hlog', 'hipercam ASCII log file',
cline.Fname('hcam', hcam.LOG)
)
hlog = hcam.hlog.Hlog.read(hlog)
aper1 = cl.get_value('aper1', 'first aperture', '2')
aper2 = cl.get_value('aper2', 'second aperture', '3')
max_ccd = len(hlog)
if max_ccd > 1:
ccd = cl.get_value('ccd', 'CCD(s) to plot [0 for all]', '0')
if ccd == '0':
ccds = list(hlog.keys())
else:
ccds = ccd.split()
else:
ccds = list(hlog.keys())
output = cl.get_value(
'output', 'name for output log file',
cline.Fname('run', hcam.LOG, cline.Fname.NEW)
)
# Inputs obtained.
# re-configure keyboard shortcuts to avoid otherwise confusing behaviour
# quit_all does not seem to be universal, hence the try/except
try:
mpl.rcParams['keymap.back'] = ''
mpl.rcParams['keymap.forward'] = ''
mpl.rcParams['keymap.fullscreen'] = ''
mpl.rcParams['keymap.grid'] = ''
mpl.rcParams['keymap.home'] = ''
mpl.rcParams['keymap.pan'] = ''
mpl.rcParams['keymap.quit'] = ''
mpl.rcParams['keymap.save'] = ''
mpl.rcParams['keymap.pan'] = ''
mpl.rcParams['keymap.save'] = ''
mpl.rcParams['keymap.xscale'] = ''
mpl.rcParams['keymap.yscale'] = ''
mpl.rcParams['keymap.zoom'] = ''
except KeyError:
pass
cnams, anams = {}, {}
plots = {}
ax = None
fig = plt.figure()
ny = len(ccds)
T0 = None
for n, cnam in enumerate(ccds):
if ax is None:
axes = ax = fig.add_subplot(ny, 1, n+1)
else:
axes = fig.add_subplot(ny, 1, n+1, sharex=ax)
# prep data
a1 = hlog.tseries(cnam,aper1)
a2 = hlog.tseries(cnam,aper1)
rat = (a1/a2).normalise()
a1 /= np.percentile(a1.y,99)
a2 /= np.percentile(a2.y,99)
if T0 is None:
T0 = a1.t[0]
a1.t -= T0
a2.t -= T0
rat.t -= T0
# three vector plots
(rat+0.1).mplot(plt,'k')
a1.mplot(plt,'g')
(a2-0.1).mplot(plt,'b')
# store the plots needed to identify which point has been selected
plots[cnam] = {'a1' : a1, 'a2' : a2-0.1, 'rat' : rat+0.1}
# keep track of the CCD associated with each axes
cnams[axes] = cnam
# and the axes associated with each CCD
anams[cnam] = axes
plt.ylabel('CCD {:s}'.format(cnam))
# create the picker
picker = PickPoints(fig, hlog, cnams, anams, plots, output)
try:
plt.tight_layout()
except:
pass
PickPoints.action_prompt(False)
# squeeze space a bit
plt.subplots_adjust(hspace=0.1)
# finally show stuff ....
plt.xlabel('Time [MJD - {:.7f}]'.format(T0))
plt.show()
def hlog2fits(args=None):
"""``hlog2fits log [origin dir]``
Converts a |hiper| ASCII log into a FITS file. As well as a modest
reduction in file size (~40%, the ASCII logs are written relatively
efficiently), the resulting file is faster to read than the ASCII log
so this may be useful for very large log files [test of 78,000 frame file:
12.9 seconds to read the ASCII file, 1.9 to read the FITS version]. The FITS
log is also much easier to understand than the ASCII files, but they don't
have all the header information, so are not a replacement. At the
moment no significant header information is transferred beyond the CCD
names. Each CCD appears as a single binary table, starting at the second
HDU (or HDU 1 if you number them 0,1,2 ..). This can be read using
:meth:`hipercam.hlog.Hlog.from_fits`.
Parameters:
log : str
name of the log file (should end .log). The output FITS file
will have the same root name but end .fits. The routine will abort
if there is a pre-existing file of the same name.
origin : str [hidden]
'h' or 'u' depending upon whether the log file was created with
the hipercam or old ultracam pipeline. Defaults to 'h'.
dir : str [hidden]
directory for the output; defaults to the present working directory
NB Because of the danger of over-writing raw data (also ends
.fits), this routine will not over-write pre-existing files. You
should delete clashing files if you really want to proceed.
"""
command, args = utils.script_args(args)
# get input section
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("log", Cline.LOCAL, Cline.PROMPT)
cl.register("origin", Cline.LOCAL, Cline.HIDE)
cl.register("dir", Cline.LOCAL, Cline.HIDE)
# get inputs
log = cl.get_value(
"log",
'name of log file from "reduce" to convert to FITS',
cline.Fname("red", hcam.LOG),
)
cl.set_default('origin', 'h')
origin = cl.get_value("origin",
"h(ipercam) or u(ltracam) pipeline?",
"h",
lvals=["h", "u"])
cl.set_default('dir', '.')
dir = cl.get_value(
"dir",
"directory for output",
".",
)
root = os.path.splitext(os.path.basename(log))[0]
oname = os.path.join(dir, root + ".fits")
if os.path.exists(oname):
raise hcam.HipercamError(f"A file called {oname} already exists and"
" will not be over-written; aborting")
# Read in the ASCII log
if origin == "h":
hlg = hcam.hlog.Hlog.rascii(log)
elif origin == "u":
hlg = hcam.hlog.Hlog.fulog(log)
print(f"Loaded ASCII log = {log}")
# Generate HDU list
# First the primary HDU (no data)
phdr = fits.Header()
phdr["LOGFILE"] = (os.path.basename(log), "Original log file")
phdu = fits.PrimaryHDU(header=phdr)
hdul = [
phdu,
]
# Now a BinTable for each CCD
for cnam in sorted(hlg):
hdr = fits.Header()
hdr["CCDNAME"] = (cnam, "CCD name")
hdul.append(fits.BinTableHDU(hlg[cnam], header=hdr,
name=f"CCD {cnam}"))
hdul = fits.HDUList(hdul)
# finally write to disk
print(f"Writing to disk in file = {oname}")
hdul.writeto(oname)
print(f"Converted {log} to {oname}")
def rupdate(args=None):
"""``rupdate rfile``
As changes are made to 'reduce', old reduce files can become
obsolete. This script tries to bring old reduce files up to date
by adding in the new options but in a way that should give the old
behaviour. The file is modified in place.
There are some cases where a full update might not be possible and
a further manual edit of the reduce file might be required.
Parameters:
rfile : string
the output reduce file created using |setaper|. This will be read
for the targets. The main target will be assumed to have been
called '1', the main comparison '2'. If there is a '3' it will be
plotted relative to '2'; all others will be ignored for plotting
purposes.
"""
command, args = utils.script_args(args)
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("rfile", Cline.GLOBAL, Cline.PROMPT)
# get inputs
# the reduce file
rfile = cl.get_value(
"rfile",
"reduce output file",
cline.Fname("reduce.red", hcam.RED, cline.Fname.OLD),
)
lines = []
fversion = False
nversion = 0
with open(rfile) as fin:
for line in fin:
if line.startswith("version ="):
# extract the version number
version = line[9:].strip()
if version.find(" ") > -1:
version = version[:version.find(" ")]
if version.find("#") > -1:
version = version[:version.find("#")]
if version == hcam.REDUCE_FILE_VERSION:
print(f"reduce file = {rfile} is up to date")
exit(0)
elif version == "20181107":
# update version number
line = ("version = {:s} # must be"
" compatible with the"
" version in reduce\n").format(
hcam.REDUCE_FILE_VERSION)
# Insert modified version and extra lines which go into the 'general'
# section along with the version number.
lines.append(line)
lines.append(
"\n# Next line was automatically added by rupdate\n")
lines.append("skipbadt = no\n\n")
# record version in case we need other actions later
nversion = 1
elif version == "20200207":
# update version number
line = ("version = {:s} # must be"
" compatible with the"
" version in reduce\n").format(
hcam.REDUCE_FILE_VERSION)
lines.append(line)
# record version in case we need other actions later
nversion = 2
elif version == "20200223":
# update version number
line = ("version = {:s} # must be"
" compatible with the"
" version in reduce\n").format(
hcam.REDUCE_FILE_VERSION)
lines.append(line)
# record version in case we need other actions later
nversion = 3
elif version == "20200318":
# update version number
lines.append(
f"version = {hcam.REDUCE_FILE_VERSION} # must be"
" compatible with the version in reduce\n")
# record version in case we need other actions later
nversion = 4
elif version == "20210523":
# update version number
lines.append(
f"version = {hcam.REDUCE_FILE_VERSION} # must be"
" compatible with the version in reduce\n")
# record version in case we need other actions later
nversion = 5
else:
print("Version = {:s} not recognised".format(version))
print("Aborting update; nothing changed.")
exit(1)
if nversion <= 5:
lines.append(
"\n# Next lines were automatically added by rupdate\n")
lines.append(
"instrument = UNKNOWN # instrument-telescope\n")
lines.append(
"scale = UNKNOWN # scale, arcsec/unbinned pixel\n")
elif line.startswith("fit_fwhm_min ="):
lines.append(line)
if nversion <= 3:
lines.append(
"\n# Next line was automatically added by rupdate\n")
lines.append(
"fit_fwhm_max = 1000 # Maximum FWHM, unbinned pixels\n"
)
elif line.startswith("dark ="):
lines.append(line)
if nversion <= 4:
lines.append(
"\n# Next lines were automatically added by rupdate\n")
lines.append("fmap = # Fringe map, blank to ignore\n")
lines.append(
"fpair = # FringePair file, ignored if fringe blank\n")
lines.append(
"nhalf = 3 # Half-width, ignored if fringe blank\n")
lines.append(
"rmin = -2 # minimum ratio, ignored if fringe blank\n")
lines.append(
"rmax = 1 # maximum ratio ignored if fringe blank\n\n")
elif line.startswith("scale =") and nversion <= 5:
arr = line.split()
try:
scale = float(arr[2])
except:
scale = 'UNKNOWN'
else:
# Default action is just to store save the line
lines.append(line)
# Write out modified file
with open(rfile, "w") as fout:
for line in lines:
if line.startswith('scale =') and scale != 'UNKNOWN':
fout.write(
f'scale = {scale:.3f} # scale, arcsec per unbinned pixel\n'
)
else:
fout.write(line)
# This could be the point at which extra lines are tacked
# on to the end of the file.
# if nversion == XX etc
if nversion == 1 or nversion == 2:
fout.write("""#
# Next lines were added automatically by 'rupdate' to bring this reduce file
# up to date. The changes are designed to produce the same behaviour as the
# old version of the reduce file as far as is possible.
[focal_mask]
demask = no
dthresh = 4
""")
print(f"Updated reduce file = {rfile}")
def arith(args=None):
"""
Carries out operations of the form output = input1 [op] input2 where [op]
is '+', '-', '*' or '/' referred to by 'add', 'sub', 'mul' and
'div'.
Parameters:
input1 : string
first input hcm file
input2 : string
second input hcm file
ccd : string [hidden, defaults to 'all']
the CCD or CCDs to apply the operation to. 'all' for the whole lot
which it returns to by default. Can be several e.g. '2 4' or just
one '3'
win : string [hidden, defaults to 'all']
the CCD or CCDs to apply the operation to. 'all' for the whole lot
which it returns to by default. Can be several e.g. 'E2 G1' or just
one 'H1'. If you specify windows in this manner, it is assumed that
all the CCDs chosen in the previous input have the named windows;
'all' just applies the operation to all windows regardless.
crop : bool [hidden, defaults to False]
set True to try to crop input2 to have the same format as input1 (it
must enclose it and have compatible binning for this to work).
output : string
output hcm file name. Can be same as either input1 or input2
in which case the input file will be over-written.
"""
command, args = utils.script_args(args)
# get inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("input1", Cline.LOCAL, Cline.PROMPT)
cl.register("input2", Cline.LOCAL, Cline.PROMPT)
cl.register("ccd", Cline.LOCAL, Cline.HIDE)
cl.register("win", Cline.LOCAL, Cline.HIDE)
cl.register("crop", Cline.LOCAL, Cline.HIDE)
cl.register("output", Cline.LOCAL, Cline.PROMPT)
prompts = {
"add": "add",
"sub": "subtract",
"mul": "multiply by",
"div": "divide by",
}
infile1 = cl.get_value("input1", "first input file",
cline.Fname("hcam", hcam.HCAM))
mccd1 = hcam.MCCD.read(infile1)
infile2 = cl.get_value("input2", "second input file",
cline.Fname("hcam", hcam.HCAM))
mccd2 = hcam.MCCD.read(infile2)
if len(mccd1) > 1:
cl.set_default("ccd", "all")
ccd = cl.get_value("ccd", "CCD(s) to process", "all")
if ccd == "all":
ccds = list(mccd1.keys())
else:
ccds = ccd.split()
else:
ccd = "all"
ccds = list(mccd1.keys())
tccd = mccd1[ccds[0]]
if len(tccd) > 1:
cl.set_default("win", "all")
win = cl.get_value("win", "window(s) to process", "all")
if win == "all":
wins = "all"
else:
wins = win.split()
else:
win = "all"
wins = "all"
cl.set_default("crop", False)
crop = cl.get_value("crop",
"try to crop input2 to the same format as input1",
False)
if crop:
mccd2 = mccd2.crop(mccd1)
print("cropped {:s} to match {:s} before operation".format(
infile2, infile1))
outfile = cl.get_value("output", "output file",
cline.Fname("hcam", hcam.HCAM, cline.Fname.NEW))
# carry out operation
if command == "add":
# addition
for cnam in ccds:
ccd1 = mccd1[cnam]
ccd2 = mccd2[cnam]
if wins == "all":
ccd1 += ccd2
else:
for wnam in wins:
ccd1[wnam] += ccd2[wnam]
elif command == "sub":
# subtraction
for cnam in ccds:
ccd1 = mccd1[cnam]
ccd2 = mccd2[cnam]
if wins == "all":
ccd1 -= ccd2
else:
for wnam in wins:
ccd1[wnam] -= ccd2[wnam]
elif command == "mul":
# multiplication
for cnam in ccds:
ccd1 = mccd1[cnam]
ccd2 = mccd2[cnam]
if wins == "all":
ccd1 *= ccd2
else:
for wnam in wins:
ccd1[wnam] *= ccd2[wnam]
elif command == "div":
# multiplication
for cnam in ccds:
ccd1 = mccd1[cnam]
ccd2 = mccd2[cnam]
if wins == "all":
ccd1 /= ccd2
else:
for wnam in wins:
ccd1[wnam] /= ccd2[wnam]
# Add a history line
mccd1.head.add_history("{:s} {:s} {:s} {:s} {:s} {:s}".format(
command,
utils.sub_extension(infile1, hcam.HCAM),
utils.sub_extension(infile2, hcam.HCAM),
ccd,
win,
utils.sub_extension(outfile, hcam.HCAM),
))
# save result
mccd1.write(outfile, True)
def pfolder(args=None):
"""``pfolder log [device width height] ccd aper``
Folds data from reduce logs on user specified ephemeris (in MJD)
and plots the results. Rather specific routine to help with timing
tests where the LED switches on on the second and off on the half second.
Parameters:
log : string
name of reduce log file (text file with loads of columns)
device : string [hidden, defaults to 'term']
'term' for interactive plot, file name such as 'plot.pdf'
for a hardcopy.
width : float [hidden]
plot width (inches). Set = 0 to let the program choose.
height : float [hidden]
plot height (inches). Set = 0 to let the program choose. BOTH
width AND height must be non-zero to have any effect
ccd : string
the CCD to consider, e.g. '1'
aper : string
the aperture to consider
t0 : float
zero point of the ephemeris in MJD (i.e. no light travel correction)
period : float
period to fold on [seconds]
"""
command, args = utils.script_args(args)
# get input section
with Cline('HIPERCAM_ENV', '.hipercam', command, args) as cl:
# register parameters
cl.register('log', Cline.LOCAL, Cline.PROMPT)
cl.register('device', Cline.LOCAL, Cline.HIDE)
cl.register('width', Cline.LOCAL, Cline.HIDE)
cl.register('height', Cline.LOCAL, Cline.HIDE)
cl.register('ccd', Cline.LOCAL, Cline.PROMPT)
cl.register('aper', Cline.LOCAL, Cline.PROMPT)
cl.register('t0', Cline.LOCAL, Cline.PROMPT)
cl.register('period', Cline.LOCAL, Cline.PROMPT)
# get inputs
log = cl.get_value(
'log', 'reduce log file to plot',
cline.Fname('hcam', hcam.LOG)
)
device = cl.get_value('device', 'plot device name', 'term')
width = cl.get_value('width', 'plot width (inches)', 0.)
height = cl.get_value('height', 'plot height (inches)', 0.)
ccd = cl.get_value('ccd', 'first CCD to plot', '1')
aper = cl.get_value('aper', 'first aperture', '1')
t0 = cl.get_value('t0', 'zero point of ephemeris [MJD]', 55000.)
period = cl.get_value('period', 'period of ephemeris [seconds]',
1., 1e-6)
# load the reduce log
hlog = hcam.hlog.Hlog.fromLog(log)
if width > 0 and height > 0:
fig = plt.figure(figsize=(width,height))
else:
fig = plt.figure()
# load counts data, fold, plot two cycles
data = hlog.tseries(ccd, aper, 'counts')
data.t = np.mod(86400.*(data.t-t0)/period,1)
xlabel = 'Phase [cycles]'
ylabel = 'Counts'
data.mplot(plt, mask=hcam.ALL_OK)
data.t += 1
data.mplot(plt, mask=hcam.ALL_OK)
plt.title('{:s}, CCD {:s}, Aperture {:s}'.format(log,ccd,aper))
if device == 'term':
plt.show()
else:
plt.savefig(device)
def plog(args=None):
"""``plog log [device width height] ccd1 aper1 param1 ccd2 (aper2 param2
scheme) [title]``
Provides quick-look plots of HiPERCAM |reduce| logs.
Parameters:
log : string
name of |reduce| ASCII log file (text file with loads of columns)
device : string [hidden, defaults to 'term']
'term' for interactive plot, file name such as 'plot.pdf'
for a hardcopy.
width : float [hidden]
plot width (inches). Set = 0 to let the program choose.
height : float [hidden]
plot height (inches). Set = 0 to let the program choose. BOTH
width AND height must be non-zero to have any effect
ccd1 : str
first CCD to consider, e.g. '1'
aper1 : str
first aperture to consider
param1 : str
first parameter to consider. Choices are 'x' = X position,
'y' = Y position, 'f' = FWHM, 'b' = Moffat beta, 's' = sky.
ccd2 : str
second CCD to consider; '!' to ignore. Can be (and typically
would be) the same as ccd1.
aper2 : str [if ccd2 != '!']
second aperture to consider
param2 : str [if ccd2 != '!']
second parameter. See param1 for choices
scheme : str [if ccd2 != '!']
how to plot if both apertures are chosen. Choices:
| 'd' = difference, i.e. plot 1-2
| 'b' = both plotted on same panel
| 'r' = ratio, i.e. 1 / 2, good for relative photom
| 's' = scatter plot, 2 vs 1.
title : str [hidden]
plot title. Defaults to the run number if not specified
.. Note::
Points with negative errors are ignored. Be careful with linked
apertures where all x, y, FWHM, beta automatically have negative errors
since they are not fitted.
"""
command, args = utils.script_args(args)
# get input section
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("log", Cline.LOCAL, Cline.PROMPT)
cl.register("device", Cline.LOCAL, Cline.HIDE)
cl.register("width", Cline.LOCAL, Cline.HIDE)
cl.register("height", Cline.LOCAL, Cline.HIDE)
cl.register("ccd1", Cline.LOCAL, Cline.PROMPT)
cl.register("aper1", Cline.LOCAL, Cline.PROMPT)
cl.register("param1", Cline.LOCAL, Cline.PROMPT)
cl.register("ccd2", Cline.LOCAL, Cline.PROMPT)
cl.register("aper2", Cline.LOCAL, Cline.PROMPT)
cl.register("param2", Cline.LOCAL, Cline.PROMPT)
cl.register("scheme", Cline.LOCAL, Cline.PROMPT)
cl.register("title", Cline.LOCAL, Cline.HIDE)
# get inputs
log = cl.get_value("log", "reduce log file to plot",
cline.Fname("hcam", hcam.LOG))
device = cl.get_value("device", "plot device name", "term")
width = cl.get_value("width", "plot width (inches)", 0.0)
height = cl.get_value("height", "plot height (inches)", 0.0)
ccd1 = cl.get_value("ccd1", "first CCD to plot", "1")
aper1 = cl.get_value("aper1", "first aperture", "1")
param1 = cl.get_value(
"param1",
"first parameter [x, y, f(whm), b(eta), c(ounts), s(ky)]",
"c",
lvals=("x", "y", "f", "b", "c", "s"),
)
lab1 = "[CCD={:s},ap={:s},p={:s}]".format(ccd1, aper1, param1)
ccd2 = cl.get_value("ccd2", "second CCD to plot ['!' to ignore]", ccd1)
if ccd2 != "!":
aper2 = cl.get_value("aper2", "second aperture", "1")
param2 = cl.get_value(
"param2",
"second parameter [x, y, f(whm), b(eta), c(ounts), s(ky)]",
"c",
lvals=("x", "y", "f", "b", "c", "s"),
)
lab2 = "[CCD={:s},ap={:s},p={:s}]".format(ccd2, aper2, param2)
# map to names used in reduce log files
MAP = {
"x": "x",
"y": "y",
"f": "fwhm",
"b": "beta",
"c": "counts",
"s": "sky"
}
pname1 = MAP[param1]
if ccd2 != "!":
pname2 = MAP[param2]
scheme = cl.get_value(
"scheme",
"b(oth), d(ifference), r(atio), s(catter)",
"b",
lvals=("b", "d", "r", "s"),
)
cl.set_default("title", log)
title = cl.get_value("title", "plot title", "Plot Title")
# load the reduce log
hlog = hcam.hlog.Hlog.read(log)
if width > 0 and height > 0:
fig = plt.figure(figsize=(width, height))
else:
fig = plt.figure()
dat1 = hlog.tseries(ccd1, aper1, pname1)
if ccd2 != "!":
dat2 = hlog.tseries(ccd2, aper2, pname2)
if scheme == "b":
# plots both together
dat1.mplot(plt, "b", bitmask=hcam.BAD_TIME)
dat2.mplot(plt, "r", bitmask=hcam.BAD_TIME)
xlabel = "Time [MJD]"
ylabel = "{:s} & {:s}".format(lab1, lab2)
elif scheme == "r":
# ratio
ratio = dat1 / dat2
ratio.mplot(plt, "b", bitmask=hcam.BAD_TIME)
xlabel = "Time [MJD]"
ylabel = "{:s} / {:s}".format(lab1, lab2)
elif scheme == "d":
# difference
diff = dat1 - dat2
diff.mplot(plt, "b", bitmask=hcam.BAD_TIME)
xlabel = "Time [MJD]"
ylabel = "{:s} - {:s}".format(lab1, lab2)
elif scheme == "s":
mask1 = dat1.get_mask(bitmask=hcam.BAD_TIME, invert=True)
mask2 = dat1.get_mask(bitmask=hcam.BAD_TIME, invert=True)
ok = ~mask1 & ~mask2
plt.errorbar(dat1.y[ok],
dat2.y[ok],
dat2.ye[ok],
dat1.ye[ok],
".",
capsize=0)
xlabel = lab1
ylabel = lab2
else:
# just one
dat1.mplot(plt, bitmask=hcam.BAD_TIME)
xlabel = "Time [MJD]"
ylabel = lab1
plt.xlabel(xlabel)
plt.ylabel(ylabel)
plt.title(title)
if device == "term":
plt.show()
else:
plt.savefig(device)
def ncal(args=None):
"""``ncal [source] (run first last [twait tmax] | flist) trim ([ncol
nrow]) (ccd) bias dark flat xybox read gain grain``
Calibrates noise characteristics of CCDs by plotting estimator
of RMS vs signal level from a series of frames. The estimate
is the mean of the absolute difference between each pixel
and the mean of its 8 near-neighbours. This is very local and
fairly robust. Assuming gaussian noise, the RMS is sqrt(4*Pi/9)
times this value, and this is what is plotted as the RMS by this
routine. `ncal` is best to applied to a series of frames with
a large dynamic range, ideally starting from bias-like frames
to well exposed sky flats. A long flat-field run going to low
levels, or a run into twilight at the end of the night could be
good places to start.
Parameters:
source : str [hidden]
Data source, five options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : ULTRACAM server
| 'ul' : local ULTRACAM .xml/.dat files
| 'hf' : list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab'
or converted from foreign data formats. The standard
start-off default for ``source`` can be set using the
environment variable HIPERCAM_DEFAULT_SOURCE. e.g. in bash
:code:`export HIPERCAM_DEFAULT_SOURCE="us"` would ensure it
always started with the ULTRACAM server by default. If
unspecified, it defaults to 'hl'.
run : str [if source ends 's' or 'l']
run number to access, e.g. 'run034'
flist : str [if source ends 'f']
name of file list
first : int [if source ends 's' or 'l']
exposure number to start from. 1 = first frame; set = 0 to always
try to get the most recent frame (if it has changed). For data
from the |hiper| server, a negative number tries to get a frame not
quite at the end. i.e. -10 will try to get 10 from the last
frame. This is mainly to sidestep a difficult bug with the
acquisition system.
last : int [if source ends 's' or 'l']
Last frame to access, 0 for the lot
twait : float [if source ends 's' or 'l'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [if source ends 's' or 'l'; hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
trim : bool [if source starts with 'u']
True to trim columns and/or rows off the edges of windows nearest
the readout which can sometimes contain bad data.
ncol : int [if trim, hidden]
Number of columns to remove (on left of left-hand window, and right
of right-hand windows)
nrow : int [if trim, hidden]
Number of rows to remove (bottom of windows)
ccd : str
The CCD to analyse.
bias : str
Name of bias frame to subtract, 'none' to ignore.
dark : str
Name of dark frame to subtract, 'none' to ignore
flat : str
Name of flat field to divide by, 'none' to ignore. Should normally
only be used in conjunction with a bias, although it does allow you
to specify a flat even if you haven't specified a bias.
xybox : int
the stats will be taken over boxes of xybox-squared pixels
to keep the number of points and scatter under control.
read : float
readout noise, RMS ADU, for overplotting a model
gain : float
gain, e-/count, for overploting a model
grain : float
fractional RMS variation due to flat-field variations,
if you didn't include a flat field.
"""
command, args = utils.script_args(args)
# get the inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("source", Cline.GLOBAL, Cline.HIDE)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("last", Cline.LOCAL, Cline.PROMPT)
cl.register("trim", Cline.GLOBAL, Cline.PROMPT)
cl.register("ncol", Cline.GLOBAL, Cline.HIDE)
cl.register("nrow", Cline.GLOBAL, Cline.HIDE)
cl.register("twait", Cline.LOCAL, Cline.HIDE)
cl.register("tmax", Cline.LOCAL, Cline.HIDE)
cl.register("flist", Cline.LOCAL, Cline.PROMPT)
cl.register("ccd", Cline.LOCAL, Cline.PROMPT)
cl.register("bias", Cline.GLOBAL, Cline.PROMPT)
cl.register("dark", Cline.GLOBAL, Cline.PROMPT)
cl.register("flat", Cline.GLOBAL, Cline.PROMPT)
cl.register("xybox", Cline.LOCAL, Cline.PROMPT)
cl.register("read", Cline.LOCAL, Cline.PROMPT)
cl.register("gain", Cline.LOCAL, Cline.PROMPT)
cl.register("grain", Cline.LOCAL, Cline.PROMPT)
# get inputs
default_source = os.environ.get('HIPERCAM_DEFAULT_SOURCE', 'hl')
source = cl.get_value(
"source",
"data source [hs, hl, us, ul, hf]",
default_source,
lvals=("hs", "hl", "us", "ul", "hf"),
)
# set some flags
server_or_local = source.endswith("s") or source.endswith("l")
if server_or_local:
resource = cl.get_value("run", "run name", "run005")
if source == "hs":
first = cl.get_value("first", "first frame to plot", 1)
else:
first = cl.get_value("first", "first frame to plot", 1, 0)
last = cl.get_value("last", "last frame to grab", 0)
if last < first and last != 0:
sys.stderr.write("last must be >= first or 0")
sys.exit(1)
twait = cl.get_value("twait",
"time to wait for a new frame [secs]", 1.0,
0.0)
tmax = cl.get_value("tmax",
"maximum time to wait for a new frame [secs]",
10.0, 0.0)
else:
resource = cl.get_value("flist", "file list",
cline.Fname("files.lis", hcam.LIST))
first = 1
trim = cl.get_value("trim", "do you want to trim edges of windows?",
True)
if trim:
ncol = cl.get_value("ncol",
"number of columns to trim from windows", 0)
nrow = cl.get_value("nrow", "number of rows to trim from windows",
0)
else:
ncol, nrow = None, None
# define the panel grid. first get the labels and maximum dimensions
ccdinf = spooler.get_ccd_pars(source, resource)
if len(ccdinf) > 1:
cnam = cl.get_value("ccd",
"CCD to analyse",
"1",
lvals=list(ccdinf.keys()))
else:
cnam = ccdinf.keys()[0]
# bias frame (if any)
bias = cl.get_value(
"bias",
"bias frame ['none' to ignore]",
cline.Fname("bias", hcam.HCAM),
ignore="none",
)
if bias is not None:
# read the bias frame
bias = hcam.MCCD.read(bias)
fprompt = "flat frame ['none' to ignore]"
else:
fprompt = "flat frame ['none' is normal choice with no bias]"
# dark (if any)
dark = cl.get_value("dark",
"dark frame to subtract ['none' to ignore]",
cline.Fname("dark", hcam.HCAM),
ignore="none")
if dark is not None:
# read the dark frame
dark = hcam.MCCD.read(dark)
# flat (if any)
flat = cl.get_value("flat",
fprompt,
cline.Fname("flat", hcam.HCAM),
ignore="none")
if flat is not None:
# read the flat frame
flat = hcam.MCCD.read(flat)
xybox = cl.get_value("xybox",
"box size for averaging results [binned pixels]",
11, 1)
read = cl.get_value("read", "readout noise, RMS ADU", 4.0, 0.0)
gain = cl.get_value("gain", "gain, ADU/e-", 1.0, 0.001)
grain = cl.get_value("grain", "flat field graininess", 0.01, 0.)
######################################
# Phew. We finally have all the inputs
# Now on with the analysis
total_time, nframe = 0, 0
with spooler.data_source(source, resource, first, full=False) as spool:
for mccd in spool:
if server_or_local:
# Handle the waiting game ... some awkward stuff
# involving updating on a cycle faster than twait to
# make the plots more responsive, if twait is long.
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmax, total_time)
if give_up:
print("ncal stopped")
break
elif try_again:
continue
# Trim the frames: ULTRACAM windowed data has bad columns
# and rows on the sides of windows closest to the readout
# which can badly affect reduction. This option strips
# them.
if trim:
hcam.ccd.trim_ultracam(mccd, ncol, nrow)
# indicate progress
tstamp = Time(mccd.head["TIMSTAMP"], format="isot", precision=3)
nfrm = mccd.head.get("NFRAME", nframe + 1)
print(
f'{nfrm}, utc= {tstamp.iso} ({"ok" if mccd.head.get("GOODTIME", True) else "nok"}), '
)
if nframe == 0:
# get the bias, dark, flat
# into shape first time through
if bias is not None:
# crop the bias on the first frame only
bias = bias.crop(mccd)
bexpose = bias.head.get("EXPTIME", 0.0)
else:
bexpose = 0.
if dark is not None:
# crop the dark on the first frame only
dark = dark.crop(mccd)
if flat is not None:
# crop the flat on the first frame only
flat = flat.crop(mccd)
# extract the CCD of interest
ccd = mccd[cnam]
if ccd.is_data():
# "is_data" indicates genuine data as opposed to junk
# that results from nskip > 0.
# subtract the bias
if bias is not None:
ccd -= bias[cnam]
# subtract the dark
if dark is not None:
dexpose = dark.head["EXPTIME"]
cexpose = ccd.head["EXPTIME"]
scale = (cexpose - bexpose) / dexpose
ccd -= scale * dark[cnam]
# divide out the flat
if flat is not None:
ccd /= flat[cnam]
# at this point we have the data in the right state to
# start processing.
for wind in ccd.values():
data = wind.data
ny, nx = data.shape
if nx - 2 >= xybox and ny - 2 >= xybox:
means, stds = procdata(wind.data, xybox)
plt.loglog(means, stds, ',b')
# update the frame number
nframe += 1
if server_or_local and last and nframe > last:
break
count = 10**np.linspace(0, 5, 200)
sigma = np.sqrt(read**2 + count / gain + (grain * count)**2)
plt.plot(count, sigma, 'r', lw=2)
plt.xlim(1, 100000)
plt.ylim(1, 1000)
plt.xlabel('Mean count level')
plt.ylabel('RMS [counts]')
plt.title(f'RMS vs level, CCD {cnam}')
plt.show()