def hpackage(args=None):
"""``hpackage runs dname tar``
``hpackage`` looks for standard reduce data products and bundles
them up into a single directory and optionally creates a tar
file. The idea is to copy all the files needed to be able to
re-run the reduction with the pipeline, while also adding a few
helpful extras where possible. Given 'run123' for instance, it
looks for:
run123.hcm -- typically the result from a run of |averun|
run123.ape -- file of photometric apertures
run123.red -- reduce file as made by |genred|
run123.log -- result from |reduce|
It also looks for calibration files inside the reduce file and
copies them. It requires them to be within the same directory and
will fail if they are not.
It produces several extra files which are:
run123.fits -- FITS version of the log file
run123_ccd1.fits -- joined-up ds9-able version of run123.hcm
(and ccd2 etc) [but only if the windows are
in sync.
run123_ccd1.reg -- ds9-region file representing the apertures
from run123.ape [see above re synced windows]
README -- a file of explanation.
The files are all copied to a temporary directory.
Arguments:
run : str
Series of run names of the ones to copy, separated by spaces.
dname : str
Name for the directory to store all files forming the root of
any output tar file created from it.
tar : bool
Make a tar.gz file of the directory at the end; the directory and
the files in it will be deleted. Otherwise, no tar file is made and
the directory is left untouched. The directory will however be deleted
if the program is aborted early.
"""
command, args = utils.script_args(args)
FEXTS = (hcam.HCAM, hcam.APER, hcam.LOG, hcam.RED)
# get the inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("runs", Cline.LOCAL, Cline.PROMPT)
cl.register("dname", Cline.LOCAL, Cline.PROMPT)
cl.register("tar", Cline.LOCAL, Cline.PROMPT)
runs = cl.get_value(
"runs", "run names [space separated]",
'run005'
)
runs = runs.split()
for run in runs:
if os.path.dirname(run) != '':
raise hcam.HipercamError(
'hpackage only runs on files in the working directory'
)
for fext in FEXTS:
if not os.path.exists(run + fext):
raise hcam.HipercamError(
f'could not find {run+fext}'
)
dname = cl.get_value(
"dname", "name of directory for storage of files (will be used to any tar file as well)",
'hdata'
)
tar = cl.get_value(
"tar", "make a tar file (and delete temporary directory at end)?", True
)
# Make name of temporary directory
tdir = utils.temp_dir()
tmpdir = os.path.join(tdir, dname)
with CleanUp(tmpdir, tar) as cleanup:
# create directory
os.makedirs(tmpdir, exist_ok=True)
print(f'Will write files to {tmpdir}')
# Get on
for run in runs:
# strip extension
root = os.path.splitext(run)[0]
# need to read the file to determine
# the number of CCDs
print(
run,root,utils.add_extension(run,hcam.HCAM)
)
mccd = hcam.MCCD.read(
utils.add_extension(run,hcam.HCAM)
)
try:
# convert the hcm and ape files using joinup
args = [
None,'prompt','list','hf',run,'no'
]
if len(mccd) > 1:
args += ['0']
args += [
root + hcam.APER,
'none','none','none','none','no',
'float32',str(100),str(100),
'no','rice',tmpdir
]
hcam.scripts.joinup(args)
except:
print('failed to joinup hcm / aper files')
# convert log to fits as well
args = [
None,'prompt',run,'h',tmpdir
]
hcam.scripts.hlog2fits(args)
# copy standard files over
for fext in FEXTS:
source = utils.add_extension(root,fext)
target = os.path.join(tmpdir,source)
shutil.copyfile(source, target)
print(f'copied {source} to {target}')
# now the calibrations
rfile = hcam.reduction.Rfile.read(run + hcam.RED)
csec = rfile['calibration']
if rfile.bias is not None:
source = utils.add_extension(
csec['bias'], hcam.HCAM
)
if os.path.dirname(source) != '':
raise HipercamError(
f'bias = {source} is not in the present working directory'
)
target = os.path.join(tmpdir,source)
shutil.copyfile(source, target)
print(f'copied {source} to {target}')
if rfile.dark is not None:
source = utils.add_extension(
csec['dark'], hcam.HCAM
)
if os.path.dirname(source) != '':
raise HipercamError(
f'dark = {source} is not in the present working directory'
)
target = os.path.join(tmpdir,source)
shutil.copyfile(source, target)
print(f'copied {source} to {target}')
if rfile.flat is not None:
source = utils.add_extension(
csec['flat'], hcam.HCAM
)
if os.path.dirname(source) != '':
raise HipercamError(
f'flat = {source} is not in the present working directory'
)
target = os.path.join(tmpdir,source)
shutil.copyfile(source, target)
print(f'copied {source} to {target}')
if rfile.fmap is not None:
source = utils.add_extension(
csec['fmap'], hcam.HCAM
)
if os.path.dirname(source) != '':
raise HipercamError(
f'fringe map = {source} is not in the present working directory'
)
target = os.path.join(tmpdir,source)
shutil.copyfile(source, target)
print(f'copied {source} to {target}')
if rfile.fpair is not None:
source = utils.add_extension(
csec['fpair'], hcam.FRNG
)
if os.path.dirname(source) != '':
raise HipercamError(
f'fringe peak/trough pair file = {source}'
' is not in the present working directory'
)
target = os.path.join(tmpdir,source)
shutil.copyfile(source, target)
print(f'copied {source} to {target}')
readme = os.path.join(tmpdir,'README')
with open(readme,'w') as fp:
fp.write(README)
# tar up the results
args = ['tar','cvfz',f'{dname}.tar.gz','-C',tdir,dname]
subprocess.run(args)
def daophot(cnam, ccd, xlo, xhi, ylo, yhi, niters, method, fwhm, beta, gfac,
thresh, rejthresh):
"""
Perform iterative PSF photometry and star finding on region of CCD
"""
print(xlo, ylo, xhi, yhi)
# first check we are within a single window
wnam1 = ccd.inside(xlo, ylo, 2)
wnam2 = ccd.inside(xhi, yhi, 2)
if wnam1 != wnam2:
raise hcam.HipercamError(
'PSF photometry cannot currently be run across seperate windows')
wnam = wnam1
print(wnam)
# background stats from whole windpw
# estimate background RMS
wind = ccd[wnam]
rms_func = MADStdBackgroundRMS(sigma_clip=SigmaClip(sigma=rejthresh))
bkg_rms = rms_func(wind.data)
bkg_func = MMMBackground(sigma_clip=SigmaClip(sigma=rejthresh))
bkg = bkg_func(wind.data)
print(' Background estimate = {}, BKG RMS = {}'.format(bkg, bkg_rms))
# crop window to ROI
wind = ccd[wnam].window(xlo, xhi, ylo, yhi)
# correct FWHM for binning
fwhm /= wind.xbin
if method == 'm':
psf_model = MoffatPSF(fwhm, beta)
print(' FWHM = {:.1f}, BETA={:.1f}'.format(fwhm, beta))
else:
psf_model = IntegratedGaussianPRF(sigma=fwhm * gaussian_fwhm_to_sigma)
print(' FWHM = {:.1f}'.format(fwhm))
# region to extract around positions for fits
fitshape = int(5 * fwhm)
# ensure odd
if fitshape % 2 == 0:
fitshape += 1
photometry_task = DAOPhotPSFPhotometry(gfac * fwhm,
thresh * bkg_rms,
fwhm,
psf_model,
fitshape,
niters=niters,
sigma=rejthresh)
with warnings.catch_warnings():
warnings.simplefilter('ignore')
results = photometry_task(wind.data - bkg)
# filter out junk fits
tiny = 1e-30
bad_errs = (results['flux_unc'] < tiny) | (results['x_0_unc'] < tiny) | (
results['y_0_unc'] < tiny)
results = results[~bad_errs]
results.write('table_{}.fits'.format(cnam))
print(' found {} stars'.format(len(results)))
xlocs, ylocs = results['x_fit'], results['y_fit']
# convert to device coordinates
xlocs = wind.x(xlocs)
ylocs = wind.y(ylocs)
return xlocs, ylocs
def filtid(args=None):
"""``filtid [source] (run first last [twait tmax] |
flist) trim ([ncol nrow]) ccdref maxlev plot``
Computes the ratio of the mean values in each CCD versus those of a reference CCD.
This is supposed to be applied to flat fields. The hope is that this might be
characteristic of the filter combination. Obviously requires > 1 CCD, i.e. not
ULTRASPEC.
Parameters:
source : str [hidden]
Data source, five options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : ULTRACAM server
| 'ul' : local ULTRACAM .xml/.dat files
| 'hf' : list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab'
or converted from foreign data formats. The standard
start-off default for ``source'' can be set using the
environment variable HIPERCAM_DEFAULT_SOURCE. e.g. in bash
:code:`export HIPERCAM_DEFAULT_SOURCE="us"` would ensure it
always started with the ULTRACAM server by default. If
unspecified, it defaults to 'hl'.
run : str [if source ends 's' or 'l']
run number to access, e.g. 'run034'
flist : str [if source ends 'f']
name of file list
first : int [if source ends 's' or 'l']
exposure number to start from. 1 = first frame; set = 0 to always
try to get the most recent frame (if it has changed). For data
from the |hiper| server, a negative number tries to get a frame not
quite at the end. i.e. -10 will try to get 10 from the last
frame. This is mainly to sidestep a difficult bug with the
acquisition system.
last : int [if source ends 's' or 'l']
last frame to reduce. 0 to just continue until the end.
twait : float [if source ends 's' or 'l'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [if source ends 's' or 'l'; hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
trim : bool [if source starts with 'u']
True to trim columns and/or rows off the edges of windows nearest
the readout which can sometimes contain bad data.
ncol : int [if trim, hidden]
Number of columns to remove (on left of left-hand window, and right
of right-hand windows)
nrow : int [if trim, hidden]
Number of rows to remove (bottom of windows)
ccdref : str
The reference CCD (usually choose the g-band one)
bias : str
bias frame to subtract (required)
lower : list of floats
Lower limits to the mean count level for a flat to be
included (after bias subtraction). Should be the same
number as the number of CCDs, and will be assumed to be in
the same order. Separate with spaces. Prevents low exposure
data from being included.
upper : list of floats
Upper limits to the mean count level for a flat to be
included. Should be the same number as the selected CCDs,
and will be assumed to be in the same order. Use to
eliminate saturated, peppered or non-linear
frames. Suggested hipercam values: 58000, 58000, 58000,
40000 and 40000 for CCDs 1, 2, 3, 4 and 5. Enter values
separated by spaces. ULTRACAM values 49000, 29000, 27000
for CCDs 1, 2 and 3.
plot: bool
Plot the fit or not.
.. Note::
This is currently adapted specifically for ULTRACAM data
"""
command, args = utils.script_args(args)
# get the inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("source", Cline.GLOBAL, Cline.HIDE)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("last", Cline.LOCAL, Cline.PROMPT)
cl.register("trim", Cline.GLOBAL, Cline.PROMPT)
cl.register("ncol", Cline.GLOBAL, Cline.HIDE)
cl.register("nrow", Cline.GLOBAL, Cline.HIDE)
cl.register("twait", Cline.LOCAL, Cline.HIDE)
cl.register("tmax", Cline.LOCAL, Cline.HIDE)
cl.register("flist", Cline.LOCAL, Cline.PROMPT)
cl.register("ccdref", Cline.LOCAL, Cline.PROMPT)
cl.register("bias", Cline.LOCAL, Cline.PROMPT)
cl.register("lower", Cline.LOCAL, Cline.PROMPT)
cl.register("upper", Cline.LOCAL, Cline.PROMPT)
cl.register("plot", Cline.LOCAL, Cline.PROMPT)
# get inputs
default_source = os.environ.get('HIPERCAM_DEFAULT_SOURCE', 'hl')
source = cl.get_value(
"source",
"data source [hs, hl, us, ul, hf]",
default_source,
lvals=("hs", "hl", "us", "ul", "hf"),
)
# set some flags
server_or_local = source.endswith("s") or source.endswith("l")
if server_or_local:
resource = cl.get_value("run", "run name", "run005")
first = cl.get_value("first", "first frame to consider", 1, 1)
last = cl.get_value("last", "last frame to consider", 0, 0)
if last and last < first + 2:
raise hcam.HipercamError(
"Must consider at least 2 exposures, and preferably more")
twait = cl.get_value("twait",
"time to wait for a new frame [secs]", 1.0,
0.0)
tmax = cl.get_value("tmax",
"maximum time to wait for a new frame [secs]",
10.0, 0.0)
else:
resource = cl.get_value("flist", "file list",
cline.Fname("files.lis", hcam.LIST))
first = 1
trim = cl.get_value("trim", "do you want to trim edges of windows?",
True)
if trim:
ncol = cl.get_value("ncol",
"number of columns to trim from windows", 0)
nrow = cl.get_value("nrow", "number of rows to trim from windows",
0)
# define the panel grid. first get the labels and maximum dimensions
ccdinf = spooler.get_ccd_pars(source, resource)
if len(ccdinf) == 1:
raise hcam.HipercamError(
"Only one CCD; this routine only works for > 1 CCD")
ccds = list(ccdinf.keys())
ccdref = cl.get_value("ccdref",
"the reference CCD for the ratios",
ccds[1],
lvals=ccds)
bias = cl.get_value("bias", "bias frame",
cline.Fname("bias", hcam.HCAM))
# read the bias frame
bias = hcam.MCCD.read(bias)
if len(bias) != len(ccds):
raise ValueError('bias has an incompatible number of CCDs')
# need to check that the default has the right number of items, if not,
# over-ride it
lowers = cl.get_default("lower")
if lowers is not None and len(lowers) != len(ccds):
cl.set_default("lower", len(ccds) * (5000, ))
lowers = cl.get_value(
"lower",
"lower limits on mean count level for included flats, 1 per CCD",
len(ccds) * (5000, ),
)
lowers = {k: v for k, v in zip(ccds, lowers)}
uppers = cl.get_default("upper")
if uppers is not None and len(uppers) != len(ccds):
cl.set_default("upper", len(ccds) * (50000, ))
uppers = cl.get_value(
"upper",
"lower limits on mean count level for included flats, 1 per CCD",
len(ccds) * (50000, ),
)
uppers = {k: v for k, v in zip(ccds, uppers)}
plot = cl.get_value("plot", "plot the results?", True)
################################################################
#
# all the inputs have now been obtained. Get on with doing stuff
total_time = 0.
xdata, ydata = {}, {}
# only want to look at CCDs other than ccdref
ccds.remove(ccdref)
# access images
with spooler.data_source(source, resource, first, full=False) as spool:
# 'spool' is an iterable source of MCCDs
nframe = 0
for mccd in spool:
if server_or_local:
# Handle the waiting game ...
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmax, total_time)
if give_up:
print("rtplot stopped")
break
elif try_again:
continue
# Trim the frames: ULTRACAM windowed data has bad columns
# and rows on the sides of windows closest to the readout
# which can badly affect reduction. This option strips
# them.
if trim:
hcam.ccd.trim_ultracam(mccd, ncol, nrow)
if nframe == 0:
# crop the bias on the first frame only
bias = bias.crop(mccd)
# bias subtraction
mccd -= bias
# indicate progress
tstamp = Time(mccd.head["TIMSTAMP"], format="isot", precision=3)
print(f"{mccd.head.get('NFRAME',nframe+1)}, utc= {tstamp.iso} " +
f"({'ok' if mccd.head.get('GOODTIME', True) else 'nok'})")
# add in data if it is in range
xd = mccd[ccdref].mean()
if xd > lowers[ccdref] and xd < uppers[ccdref]:
for cnam in ccds:
yd = mccd[cnam].mean()
if yd > lowers[cnam] and yd < uppers[cnam]:
if cnam in xdata:
xdata[cnam].append(xd)
ydata[cnam].append(yd)
else:
xdata[cnam] = [xd]
ydata[cnam] = [yd]
if last and nframe == last:
break
# update the frame number
nframe += 1
print()
invalid = True
for cnam in ccds:
if cnam in xdata:
xdata[cnam] = np.array(xdata[cnam])
ydata[cnam] = np.array(ydata[cnam])
print(
f'{cnam}-vs-{ccdref}: found {len(xdata[cnam])} valid points ranging from'
+
f' {xdata[cnam].min():.1f} to {xdata[cnam].max():.1f} (CCD {ccdref}), '
+
f' {ydata[cnam].min():.1f} to {ydata[cnam].max():.1f} (CCD {cnam})'
)
invalid = False
if invalid:
raise hcam.HipercamError(f"No valid points found at all")
print()
if plot:
fig, axes = plt.subplots(len(ccds), 1, sharex=True)
for cnam, ax in zip(ccds, axes):
if cnam in xdata:
xds = xdata[cnam]
ratios = ydata[cnam] / xds
mratio = ratios.mean()
ax.plot(xds, ratios, '.g')
ax.plot([xds.min(), xds.max()], [mratio, mratio], 'r--')
ax.set_ylabel(f'Ratio CCD {cnam} / CCD {ccdref}')
if cnam == ccds[-1]:
ax.set_xlabel(f'CCD = {ccdref} counts/pixel')
print(f'{cnam}-vs-{ccdref} = {mratio:.4f}')
plt.show()
else:
for cnam in ccds:
if cnam in xdata:
ratios = ydata[cnam] / xdata[cnam]
mratio = ratios.mean()
print(f'{cnam}-vs-{ccdref} = {mratio:.4f}')
def combine(args=None):
"""``combine list bias dark flat method (sigma) adjust (usemean) [plot clobber]
output``
Combines a series of images defined by a list using median or clipped
mean combination. Only combines those CCDs for which is_data() is true
(i.e. it skips blank frames caused by NSKIP / NBLUE options)
Parameters:
list : string
list of hcm files with images to combine. The formats of the
images should all match
bias : string
Name of bias frame to subtract, 'none' to ignore.
dark : string
Name of dark frame to subtract, 'none' to ignore.
flat : string
Name of flat field frame to subtract, 'none' to ignore.
method : string
'm' for median, 'c' for clipped mean. See below for pros and cons.
sigma : float [if method == 'c']
With clipped mean combination, pixels that deviate by more than
sigma RMS from the mean are kicked out. This is carried out in an
iterative manner. sigma <= 0 implies no rejection, just a straight
average. sigma=3 is typical.
adjust : string
adjustments to make: 'i' = ignore; 'n' = normalise the mean or
median of all frames to match the first; 'b' = add offsets so that
the mean or median of all frames is the same as the first. Option
'n' is useful for twilight flats and fringe frames; 'b' is good
for combining biases.
usemean : bool [if adjust == 'n' or 'b']
True to use the mean rather than the median for normalisation or
biass offsetting.
plot : bool [hidden, if adjust == 'n' or 'b'; defaults to False]
make a plot of the mean versus frame number. This can provide a
quick check that the frames are not too different.
clobber : bool [hidden]
clobber any pre-existing output files
output : string
output file
Clipped mean can work well for large numbers of frames but gets worse
for small numbers as the RMS can be heavily influenced by a single bad
value. The median can be better in such cases, but has the downside of
digitisation noise. For instance, the average of 100 bias frames could
have a noise level significantly below 1 count, depending upon the
readout noise, and the +/- 0.5 count uncertainty of median combination
may be worse than this.
.. Note::
This routine reads all inputs into memory, so can be a bit of a
hog. However, it does so one CCD at a time to alleviate this. It will
fail if it cannot find a valid frame for any CCD.
"""
command, args = utils.script_args(args)
# get the inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("list", Cline.GLOBAL, Cline.PROMPT)
cl.register("bias", Cline.LOCAL, Cline.PROMPT)
cl.register("dark", Cline.LOCAL, Cline.PROMPT)
cl.register("flat", Cline.LOCAL, Cline.PROMPT)
cl.register("method", Cline.LOCAL, Cline.PROMPT)
cl.register("sigma", Cline.LOCAL, Cline.PROMPT)
cl.register("adjust", Cline.LOCAL, Cline.PROMPT)
cl.register("usemean", Cline.LOCAL, Cline.PROMPT)
cl.register("plot", Cline.LOCAL, Cline.HIDE)
cl.register("clobber", Cline.LOCAL, Cline.HIDE)
cl.register("output", Cline.LOCAL, Cline.PROMPT)
# get inputs
flist = cl.get_value("list", "list of files to combine",
cline.Fname("files", hcam.LIST))
# bias frame (if any)
bias = cl.get_value(
"bias",
"bias frame ['none' to ignore]",
cline.Fname("bias", hcam.HCAM),
ignore="none",
)
if bias is not None:
# read the bias frame
bias = hcam.MCCD.read(bias)
# dark frame (if any)
dark = cl.get_value(
"dark",
"dark frame ['none' to ignore]",
cline.Fname("dark", hcam.HCAM),
ignore="none",
)
if dark is not None:
# read the dark frame
dark = hcam.MCCD.read(dark)
# flat frame (if any)
flat = cl.get_value(
"flat",
"flat frame ['none' to ignore]",
cline.Fname("flat", hcam.HCAM),
ignore="none",
)
if flat is not None:
# read the flat frame
flat = hcam.MCCD.read(flat)
method = cl.get_value("method",
"c(lipped mean), m(edian)",
"c",
lvals=("c", "m"))
if method == "c":
sigma = cl.get_value("sigma", "number of RMS deviations to clip",
3.0)
adjust = cl.get_value(
"adjust",
"i(gnore), n(ormalise), b(ias offsets)",
"i",
lvals=("i", "n", "b"),
)
if adjust == "n" or adjust == "b":
usemean = cl.get_value(
"usemean",
"use the mean for normalisation / offsetting [else median]",
True,
)
plot = cl.get_value(
"plot",
"plot mean levels versus frame number?"
if usemean else "plot median levels versus frame number?",
False,
)
else:
plot = False
usemean = False
clobber = cl.get_value("clobber",
"clobber any pre-existing files on output",
False)
outfile = cl.get_value(
"output",
"output file",
cline.Fname("hcam", hcam.HCAM,
cline.Fname.NEW if clobber else cline.Fname.NOCLOBBER),
)
# inputs done with
# Read the first file of the list to act as a template
# for the CCD names etc.
with open(flist) as fin:
for line in fin:
if not line.startswith("#") and not line.isspace():
template_name = line.strip()
break
else:
raise hcam.HipercamError("List = {:s} is empty".format(flist))
template = hcam.MCCD.read(utils.add_extension(template_name, hcam.HCAM))
if bias is not None:
# crop the bias
bias = bias.crop(template)
if dark is not None:
# crop the dark
dark = dark.crop(template)
if flat is not None:
# crop the flat
flat = flat.crop(template)
# Now process each file CCD by CCD to reduce the memory
# footprint
for cnam in template:
# Read files into memory, insisting that they
# all have the same set of CCDs
print("\nLoading all CCDs labelled '{:s}' from {:s}".format(
cnam, flist))
ccds, means = [], []
nrej, ntot = 0, 0
with spooler.HcamListSpool(flist, cnam) as spool:
if bias is not None:
# extract relevant CCD from the bias
bccd = bias[cnam]
bexpose = bias.head.get("EXPTIME", 0.0)
else:
bexpose = 0.0
if dark is not None:
# extract relevant CCD from the dark
dccd = dark[cnam]
dexpose = dark.head["EXPTIME"]
if flat is not None:
# extract relevant CCD from the flat
fccd = flat[cnam]
mean = None
for ccd in spool:
if ccd.is_data():
if bias is not None:
# subtract bias
ccd -= bccd
if dark is not None:
# subtract dark
scale = (ccd.head["EXPTIME"] - bexpose) / dexpose
ccd -= scale * dccd
if flat is not None:
# apply flat
ccd /= fccd
# keep the result
ccds.append(ccd)
if (adjust == "b" or adjust == "n") and mean is None:
# store the first mean [median]
if usemean:
mean = ccd.mean()
else:
mean = ccd.median()
if len(ccds) == 0:
raise hcam.HipercamError("Found no valid examples of CCD {:s}"
" in list = {:s}".format(cnam, flist))
else:
print("Loaded {:d} CCDs".format(len(ccds)))
if adjust == "b" or adjust == "n":
# adjust the means
print("Computing and adjusting their mean levels")
# now carry out the adjustments
for ccd in ccds:
if usemean:
cmean = ccd.mean()
else:
cmean = ccd.median()
means.append(cmean)
if adjust == "b":
ccd += mean - cmean
elif adjust == "n":
ccd *= mean / cmean
if plot:
plt.plot(means)
plt.plot(means, ".k")
plt.text(len(means) + 1,
means[-1],
cnam,
va="center",
ha="left")
# Finally, combine
if method == "m":
print("Combining them (median) and storing the result")
elif method == "c":
print("Combining them (clipped mean) and storing the result")
else:
raise NotImplementedError(
"method = {:s} not implemented".format(method))
for wnam, wind in template[cnam].items():
# build list of all data arrays
arrs = [ccd[wnam].data for ccd in ccds]
arr3d = np.stack(arrs)
# at this point, arr3d is a 3D array, with the first dimension
# (axis=0) running over the images. We want to average / median
# over this axis.
if method == "m":
# median
wind.data = np.median(arr3d, axis=0)
elif method == "c":
# Cython routine avgstd requires np.float32 input
arr3d = arr3d.astype(np.float32)
if sigma > 0.0:
avg, std, num = support.avgstd(arr3d, sigma)
nrej += len(ccds) * num.size - num.sum()
ntot += len(ccds) * num.size
else:
avg = np.mean(arr3d, axis=0)
wind.data = avg
# Add history
if method == "m":
template[cnam].head.add_history(
"Median combine of {:d} images".format(len(ccds)))
elif method == "c":
print("Rejected {:d} pixels = {:.3f}% of the total".format(
nrej, 100 * nrej / ntot))
template[cnam].head.add_history(
"Clipped mean combine of {:d} images, sigma = {:.1f}".format(
len(ccds), sigma))
# write out
template.write(outfile, clobber)
print("\nFinal result written to {:s}".format(outfile))
# finally
if plot:
plt.xlabel("Frame number")
plt.ylabel("Mean counts")
plt.show()
def hlog2fits(args=None):
"""``hlog2fits log [origin dir]``
Converts a |hiper| ASCII log into a FITS file. As well as a modest
reduction in file size (~40%, the ASCII logs are written relatively
efficiently), the resulting file is faster to read than the ASCII log
so this may be useful for very large log files [test of 78,000 frame file:
12.9 seconds to read the ASCII file, 1.9 to read the FITS version]. The FITS
log is also much easier to understand than the ASCII files, but they don't
have all the header information, so are not a replacement. At the
moment no significant header information is transferred beyond the CCD
names. Each CCD appears as a single binary table, starting at the second
HDU (or HDU 1 if you number them 0,1,2 ..). This can be read using
:meth:`hipercam.hlog.Hlog.from_fits`.
Parameters:
log : str
name of the log file (should end .log). The output FITS file
will have the same root name but end .fits. The routine will abort
if there is a pre-existing file of the same name.
origin : str [hidden]
'h' or 'u' depending upon whether the log file was created with
the hipercam or old ultracam pipeline. Defaults to 'h'.
dir : str [hidden]
directory for the output; defaults to the present working directory
NB Because of the danger of over-writing raw data (also ends
.fits), this routine will not over-write pre-existing files. You
should delete clashing files if you really want to proceed.
"""
command, args = utils.script_args(args)
# get input section
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("log", Cline.LOCAL, Cline.PROMPT)
cl.register("origin", Cline.LOCAL, Cline.HIDE)
cl.register("dir", Cline.LOCAL, Cline.HIDE)
# get inputs
log = cl.get_value(
"log",
'name of log file from "reduce" to convert to FITS',
cline.Fname("red", hcam.LOG),
)
cl.set_default('origin', 'h')
origin = cl.get_value("origin",
"h(ipercam) or u(ltracam) pipeline?",
"h",
lvals=["h", "u"])
cl.set_default('dir', '.')
dir = cl.get_value(
"dir",
"directory for output",
".",
)
root = os.path.splitext(os.path.basename(log))[0]
oname = os.path.join(dir, root + ".fits")
if os.path.exists(oname):
raise hcam.HipercamError(f"A file called {oname} already exists and"
" will not be over-written; aborting")
# Read in the ASCII log
if origin == "h":
hlg = hcam.hlog.Hlog.rascii(log)
elif origin == "u":
hlg = hcam.hlog.Hlog.fulog(log)
print(f"Loaded ASCII log = {log}")
# Generate HDU list
# First the primary HDU (no data)
phdr = fits.Header()
phdr["LOGFILE"] = (os.path.basename(log), "Original log file")
phdu = fits.PrimaryHDU(header=phdr)
hdul = [
phdu,
]
# Now a BinTable for each CCD
for cnam in sorted(hlg):
hdr = fits.Header()
hdr["CCDNAME"] = (cnam, "CCD name")
hdul.append(fits.BinTableHDU(hlg[cnam], header=hdr,
name=f"CCD {cnam}"))
hdul = fits.HDUList(hdul)
# finally write to disk
print(f"Writing to disk in file = {oname}")
hdul.writeto(oname)
print(f"Converted {log} to {oname}")
def makemovie(args=None):
"""``makemovie [source] (run first last | flist) trim ([ncol nrow])
(ccd (nx)) bias flat defect log (targ comp ymin ymax yscales
yoffset location fraction lpad) cmap width height dstore ndigit fext msub iset
(ilo ihi | plo phi) xlo xhi ylo yhi [dpi (style (ms lw))]``
``makemovie`` is for generating stills to combine into a movie for
presentations. It can optionally also read a log file from the run
to display an evolving light curve. There are lots of fiddly
parameters mostly to do with the plot positioning, so try it out
on a small number of frames first before going mad.
Parameters:
source : str [hidden]
Data source, five options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : ULTRACAM server
| 'ul' : local ULTRACAM .xml/.dat files
| 'hf' : list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab'
or converted from foreign data formats. The standard
start-off default for ``source`` can be set using the
environment variable HIPERCAM_DEFAULT_SOURCE. e.g. in bash
:code:`export HIPERCAM_DEFAULT_SOURCE="us"` would ensure it
always started with the ULTRACAM server by default. If
unspecified, it defaults to 'hl'.
run : str [if source ends 's' or 'l']
run number to access, e.g. 'run034'
flist : str [if source ends 'f']
name of file list
first : int [if source ends 's' or 'l']
exposure number to start from. 1 = first frame.
last : int [if source ends 's' or 'l']
last exposure
trim : bool [if source starts with 'u']
True to trim columns and/or rows off the edges of windows nearest
the readout which can sometimes contain bad data.
ncol : int [if trim, hidden]
Number of columns to remove (on left of left-hand window, and right
of right-hand windows)
nrow : int [if trim, hidden]
Number of rows to remove (bottom of windows)
ccd : str
CCD(s) to plot, '0' for all, '1 3' to plot '1' and '3' only, etc. If
you plot more than one, then a legend is added to any light curve panel
to distinguish the light curves.
nx : int [if more than 1 CCD]
number of panels across to display for the image display.
bias : str
Name of bias frame to subtract, 'none' to ignore.
dark : str
Name of dark frame to subtract, 'none' to ignore.
flat : str
Name of flat field to divide by, 'none' to ignore. Should normally
only be used in conjunction with a bias, although it does allow you
to specify a flat even if you haven't specified a bias.
defect : str
Name of defect file, 'none' to ignore.
fmap : str
Name of fringe map (see e.g. `makefringe`), 'none' to ignore.
fpair : str [if fringe is not 'none']
Name of fringe pair file (see e.g. `setfringe`). Required if
a fringe map has been specified.
nhalf : int [if fringe is not 'none', hidden]
When calculating the differences for fringe measurement,
a region extending +/-nhalf binned pixels will be used when
measuring the amplitudes. Basically helps the stats.
rmin : float [if fringe is not 'none', hidden]
Minimum individual ratio to accept prior to calculating the overall
median in order to reduce the effect of outliers. Although all ratios
should be positive, you might want to set this a little below zero
to allow for some statistical fluctuation.
rmax : float [if fringe is not 'none', hidden]
Maximum individual ratio to accept prior to calculating the overall
median in order to reduce the effect of outliers. Probably typically
< 1 if fringe map was created from longer exposure data.
log : str
Name of reduce log file for light curve plot, 'none' to ignore
targ : str [if log defined]
Target aperture
comp : str [if log defined]
Comparison aperture
ymin : float [if log defined]
Minimum Y value for light curve plot
ymax : float [if log defined]
Maximum Y value for light curve plot
ynorm : list(float) [if log defined]
Normalisation factors, one per CCD for light curve plot
yoffset : list(float) [if log defined]
Offsets, one per CCD for light curve plot
location : str [if log defined]
Offsets, one per CCD for light curve plot
fraction : float [if log defined]
Fraction of figure to occupy, by height if location is South,
by width if it is East
lpad : tuple(float) [if log defined]
padding on left, bottom, right and top of light curve plot as fraction
of allocated width and height
cmap : str
The matplotlib colour map to use. "Greys" gives the usual greyscale.
"none" will give whatever the current default is. Many other choices:
"viridis", "jet", "hot", "Oranges", etc. Enter an invalid one and
the program will fail but return a huge list of possibles in the
process.
width : float
plot width in inches.
height : float
plot height in inches.
dstore : str
root directory for plot files. Will get names like dstore/run003_001.png.
ndigit : int
number of digits in frame counter, i.e. the '001' of the previous section.
fext : str
file extension 'png', 'jpeg' for images generated
msub : bool
subtract the median from each window before scaling for the
image display or not. This happens after any bias subtraction.
iset : str [single character]
determines how the intensities are determined. There are three
options: 'a' for automatic simply scales from the minimum to the
maximum value found on a per CCD basis. 'd' for direct just takes
two numbers from the user. 'p' for percentile dtermines levels
based upon percentiles determined from the entire CCD on a per CCD
basis.
ilo : list(float) [if iset='d']
lower intensity level, one per image
ihi : list(float) [if iset='d']
upper intensity level, one per image
plo : float [if iset='p']
lower percentile level
phi : float [if iset='p']
upper percentile level
xlo : float
left-hand X-limit for plot, initially at least since it is
possible to re-size. For iset='p' these limits also set the
region of the frame over which the percentil will be
calculated. You will usually want yhi-ylo ~ xhi-xlo in
magnitude because the aspect ratio is preserved.
xhi : float
right-hand X-limit for plot (can be < xlo to invert the display)
ylo : float
lower Y-limit for plot
yhi : float
upper Y-limit for plot (can be < ylo)
dpi : int [hidden]
dots per inch of output. Default 72. Allows control over font size versus image size,
in combination with width and height.
style : str [hidden, if log defined]
style for light curves 'dots', 'line', 'both'. The line will be grey for the 'both' option.
ms : float [hidden, if log defined and style==dots or both]
markersize. Controls dot size which is useful when fiddling with dpi
lw : float [hidden, if log defined and style==line or both]
line width
"""
command, args = utils.script_args(args)
# get the inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("source", Cline.GLOBAL, Cline.HIDE)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("last", Cline.LOCAL, Cline.PROMPT)
cl.register("trim", Cline.GLOBAL, Cline.PROMPT)
cl.register("ncol", Cline.GLOBAL, Cline.HIDE)
cl.register("nrow", Cline.GLOBAL, Cline.HIDE)
cl.register("flist", Cline.LOCAL, Cline.PROMPT)
cl.register("ccd", Cline.LOCAL, Cline.PROMPT)
cl.register("nx", Cline.LOCAL, Cline.PROMPT)
cl.register("bias", Cline.GLOBAL, Cline.PROMPT)
cl.register("dark", Cline.GLOBAL, Cline.PROMPT)
cl.register("flat", Cline.GLOBAL, Cline.PROMPT)
cl.register("fmap", Cline.GLOBAL, Cline.PROMPT)
cl.register("fpair", Cline.GLOBAL, Cline.PROMPT)
cl.register("nhalf", Cline.GLOBAL, Cline.HIDE)
cl.register("rmin", Cline.GLOBAL, Cline.HIDE)
cl.register("rmax", Cline.GLOBAL, Cline.HIDE)
cl.register("defect", Cline.GLOBAL, Cline.PROMPT)
cl.register("log", Cline.LOCAL, Cline.PROMPT)
cl.register("targ", Cline.LOCAL, Cline.PROMPT)
cl.register("comp", Cline.LOCAL, Cline.PROMPT)
cl.register("ymin", Cline.LOCAL, Cline.PROMPT)
cl.register("ymax", Cline.LOCAL, Cline.PROMPT)
cl.register("ynorm", Cline.LOCAL, Cline.PROMPT)
cl.register("yoffset", Cline.LOCAL, Cline.PROMPT)
cl.register("location", Cline.LOCAL, Cline.PROMPT)
cl.register("fraction", Cline.LOCAL, Cline.PROMPT)
cl.register("lpad", Cline.LOCAL, Cline.PROMPT)
cl.register("cmap", Cline.LOCAL, Cline.PROMPT)
cl.register("width", Cline.LOCAL, Cline.PROMPT)
cl.register("height", Cline.LOCAL, Cline.PROMPT)
cl.register("dstore", Cline.LOCAL, Cline.PROMPT)
cl.register("ndigit", Cline.LOCAL, Cline.PROMPT)
cl.register("fext", Cline.LOCAL, Cline.PROMPT)
cl.register("msub", Cline.GLOBAL, Cline.PROMPT)
cl.register("iset", Cline.GLOBAL, Cline.PROMPT)
cl.register("ilo", Cline.LOCAL, Cline.PROMPT)
cl.register("ihi", Cline.LOCAL, Cline.PROMPT)
cl.register("plo", Cline.GLOBAL, Cline.PROMPT)
cl.register("phi", Cline.LOCAL, Cline.PROMPT)
cl.register("xlo", Cline.GLOBAL, Cline.PROMPT)
cl.register("xhi", Cline.GLOBAL, Cline.PROMPT)
cl.register("ylo", Cline.GLOBAL, Cline.PROMPT)
cl.register("yhi", Cline.GLOBAL, Cline.PROMPT)
cl.register("dpi", Cline.LOCAL, Cline.HIDE)
cl.register("style", Cline.LOCAL, Cline.HIDE)
cl.register("ms", Cline.LOCAL, Cline.HIDE)
cl.register("lw", Cline.LOCAL, Cline.HIDE)
# get inputs
default_source = os.environ.get('HIPERCAM_DEFAULT_SOURCE', 'hl')
source = cl.get_value(
"source",
"data source [hs, hl, us, ul, hf]",
default_source,
lvals=("hs", "hl", "us", "ul", "hf"),
)
# set some flags
server_or_local = source.endswith("s") or source.endswith("l")
if server_or_local:
resource = cl.get_value("run", "run name", "run005")
first = cl.get_value("first", "first frame to plot", 1)
last = cl.get_value("last",
"last frame to plot [0 to go to the end]",
max(1, first), 0)
else:
resource = cl.get_value("flist", "file list",
cline.Fname("files.lis", hcam.LIST))
first = 1
trim = cl.get_value("trim", "do you want to trim edges of windows?",
True)
if trim:
ncol = cl.get_value("ncol",
"number of columns to trim from windows", 0)
nrow = cl.get_value("nrow", "number of rows to trim from windows",
0)
else:
ncol, nrow = None, None
# define the panel grid. first get the labels and maximum dimensions
ccdinf = spooler.get_ccd_pars(source, resource)
nxdef = cl.get_default("nx", 3)
if len(ccdinf) > 1:
ccd = cl.get_value("ccd", "CCD(s) to plot [0 for all]", "0")
if ccd == "0":
ccds = list(ccdinf.keys())
else:
ccds = ccd.split()
check = set(ccdinf.keys())
if not set(ccds) <= check:
raise hcam.HipercamError(
"At least one invalid CCD label supplied")
if len(ccds) > 1:
nxdef = min(len(ccds), nxdef)
cl.set_default("nx", nxdef)
nx = cl.get_value("nx", "number of panels in X", 3, 1)
else:
nx = 1
else:
nx = 1
ccds = list(ccdinf.keys())
# bias frame (if any)
bias = cl.get_value(
"bias",
"bias frame ['none' to ignore]",
cline.Fname("bias", hcam.HCAM),
ignore="none",
)
if bias is not None:
# read the bias frame
bias = hcam.MCCD.read(bias)
fprompt = "flat frame ['none' to ignore]"
else:
fprompt = "flat frame ['none' is normal choice with no bias]"
# dark (if any)
dark = cl.get_value("dark",
"dark frame to subtract ['none' to ignore]",
cline.Fname("dark", hcam.HCAM),
ignore="none")
if dark is not None:
# read the dark frame
dark = hcam.MCCD.read(dark)
# flat (if any)
flat = cl.get_value("flat",
fprompt,
cline.Fname("flat", hcam.HCAM),
ignore="none")
if flat is not None:
# read the flat frame
flat = hcam.MCCD.read(flat)
# fringe file (if any)
fmap = cl.get_value(
"fmap",
"fringe map ['none' to ignore]",
cline.Fname("fmap", hcam.HCAM),
ignore="none",
)
if fmap is not None:
# read the fringe map
fmap = hcam.MCCD.read(fmap)
fpair = cl.get_value("fpair", "fringe pair file",
cline.Fname("fpair", hcam.FRNG))
fpair = fringe.MccdFringePair.read(fpair)
nhalf = cl.get_value("nhalf",
"half-size of fringe measurement regions", 2,
0)
rmin = cl.get_value("rmin", "minimum fringe pair ratio", -0.2)
rmax = cl.get_value("rmax", "maximum fringe pair ratio", 1.0)
# defect file (if any)
dfct = cl.get_value(
"defect",
"defect file ['none' to ignore]",
cline.Fname("defect", hcam.DFCT),
ignore="none",
)
if dfct is not None:
# read the defect frame
dfct = defect.MccdDefect.read(dfct)
# reduce log file (if any)
rlog = cl.get_value(
"log",
"reduce log file ['none' to ignore]",
cline.Fname("reduce", hcam.LOG),
ignore="none",
)
if rlog is not None:
# Read reduce file
hlg = hlog.Hlog.rascii(rlog)
targ = cl.get_value("targ", "target aperture", "1")
comp = cl.get_value("comp", "comparison aperture", "2")
fmin = cl.get_value("ymin", "minimum Y value for light curve plot",
0.)
fmax = cl.get_value("ymax", "maxmum Y value for light curve plot",
1.)
# need to check that the default has the right number of
# items, if not overr-ride it
ynorm = cl.get_default("ynorm")
if ynorm is not None:
if len(ynorm) > len(ccds):
cl.set_default("ynorm", ynorm[:len(ccds)])
elif len(ynorm) < len(ccds):
cl.set_default("ynorm",
ynorm + (len(ccds) - len(ynorm)) * [1.])
ynorm = cl.get_value(
"ynorm",
"normalisation factors for light curves (one per CCD)",
len(ccds) * [1.])
yoffset = cl.get_default("yoffset")
if yoffset is not None:
if len(yoffset) > len(ccds):
cl.set_default("yoffset", yoffset[:len(ccds)])
elif len(yoffset) < len(ccds):
cl.set_default("yoffset",
yoffset + (len(ccds) - len(yoffset)) * [0.])
yoffset = cl.get_value(
"yoffset", "vertical offsets for light curves (one per CCD)",
len(ccds) * [0.])
location = cl.get_value(
"location",
"position of light curve plot relative to images",
"s",
lvals=['s', 'e', 'S', 'E'])
if location.lower() == 's':
fraction = cl.get_value(
"fraction",
"fraction of figure in terms of height occupied by light curve",
0.5)
elif location.lower() == 'e':
fraction = cl.get_value(
"fraction",
"fraction of figure in terms of width occupied by light curve",
0.67)
lpad = cl.get_value(
"lpad",
"padding (left,bottom,right,top) around light curve plot",
(0.05, 0.05, 0.02, 0.02))
# trim down to the specified frames
lcs = []
T0, tmax = None, None
for cnam, yn, yo in zip(ccds, ynorm, yoffset):
nframes = hlg[cnam]['nframe']
keep = (nframes >= first) and (last == 0 or (nframes <= last))
hlg[cnam] = hlg[cnam][keep]
lc = (hlg.tseries(cnam, targ) /
hlg.tseries(cnam, comp)) / yn + yo
if T0 is None:
T0 = lc.t.min()
lc.ttrans(lambda t: 1440 * (t - T0))
if tmax is None:
tmax = lc.t.max()
else:
tmax = max(tmax, lc.t.max())
lcs.append((nframes[keep], lc))
# Some settings for the image plots
cmap = cl.get_value("cmap",
"colour map to use ['none' for mpl default]",
"Greys")
cmap = None if cmap == "none" else cmap
width = cl.get_value("width", "plot width [inches]", 10., 0.5)
height = cl.get_value("height", "plot height [inches]", 10., 0.5)
dstore = cl.get_value("dstore", "directory for images", "tmp")
if not os.path.isdir(dstore):
raise hcam.HipercamError(f"'{dstore}' is not a directory")
ndigit = cl.get_value("ndigit",
"number of digits for appended frame counter", 4,
1)
fext = cl.get_value("fext",
"file extension for images",
"png",
lvals=["png", "jpg"])
# define the display intensities
msub = cl.get_value("msub", "subtract median from each window?", True)
iset = cl.get_value(
"iset",
"set intensity a(utomatically),"
" d(irectly) or with p(ercentiles)?",
"a",
lvals=["a", "d", "p"],
)
iset = iset.lower()
plo, phi = 5, 95
ilos, ihis = len(ccds) * [0], len(ccds) * [1000]
if iset == "d":
# fiddle with the defaults
ilo = cl.get_default("ilo")
if ilo is not None:
if len(ilo) > len(ccds):
cl.set_default("ilo", ilo[:len(ccds)])
elif len(ilo) < len(ccds):
cl.set_default("ilo", ilo + (len(ccds) - len(ilo)) * [0.])
ihi = cl.get_default("ihi")
if ihi is not None:
if len(ihi) > len(ccds):
cl.set_default("ihi", ihi[:len(ccds)])
elif len(ihi) < len(ccds):
cl.set_default("ihi", ihi + (len(ccds) - len(ihi))[1000.])
ilos = cl.get_value("ilo", "lower intensity limit",
len(ccds) * [0.])
ihis = cl.get_value("ihi", "upper intensity limit",
len(ccds) * [1000.])
elif iset == "p":
plo = cl.get_value("plo", "lower intensity limit percentile", 5.0,
0.0, 100.0)
phi = cl.get_value("phi", "upper intensity limit percentile", 95.0,
0.0, 100.0)
# region to plot
for i, cnam in enumerate(ccds):
nxtot, nytot, nxpad, nypad = ccdinf[cnam]
if i == 0:
xmin, xmax = float(-nxpad), float(nxtot + nxpad + 1)
ymin, ymax = float(-nypad), float(nytot + nypad + 1)
else:
xmin = min(xmin, float(-nxpad))
xmax = max(xmax, float(nxtot + nxpad + 1))
ymin = min(ymin, float(-nypad))
ymax = max(ymax, float(nytot + nypad + 1))
xlo = cl.get_value("xlo",
"left-hand X value",
xmin,
xmin,
xmax,
enforce=False)
xhi = cl.get_value("xhi",
"right-hand X value",
xmax,
xmin,
xmax,
enforce=False)
ylo = cl.get_value("ylo",
"lower Y value",
ymin,
ymin,
ymax,
enforce=False)
yhi = cl.get_value("yhi",
"upper Y value",
ymax,
ymin,
ymax,
enforce=False)
dpi = cl.get_value("dpi", "dots per inch", 200)
if rlog is not None:
style = cl.get_value("style",
"light curve plot style",
"dots",
lvals=('dots', 'line', 'both'))
ms, lw = 0, 0
if style == 'dots' or style == 'both':
ms = cl.get_value("ms", "markersize", 2.)
if style == 'line' or style == 'both':
lw = cl.get_value("lw", "line width", 2.)
###############################################################################
# Phew. We finally have all the inputs and now can now display stuff.
# track which CCDs have been plotted at least once for the profile fits
nccd = len(ccds)
plotted = np.array(nccd * [False])
current_ccds = nccd * [None]
ny = nccd // nx if nccd % nx == 0 else nccd // nx + 1
first_fig = True
# Now go through data
with spooler.data_source(source, resource, first, full=False) as spool:
for nframe, mccd in enumerate(spool):
# Trim the frames: ULTRACAM windowed data has bad columns
# and rows on the sides of windows closest to the readout
# which can badly affect reduction. This option strips
# them.
if trim:
hcam.ccd.trim_ultracam(mccd, ncol, nrow)
# indicate progress
tstamp = Time(mccd.head["TIMSTAMP"], format="isot", precision=3)
print(f'{mccd.head.get("NFRAME",nframe+1)}, utc= {tstamp.iso}')
if nframe == 0:
# get the bias and flat into shape first time through
if bias is not None:
# crop the bias on the first frame only
bias = bias.crop(mccd)
bexpose = bias.head.get("EXPTIME", 0.0)
else:
bexpose = 0.
if dark is not None:
# crop the dark on the first frame only
dark = dark.crop(mccd)
if flat is not None:
# crop the flat on the first frame only
flat = flat.crop(mccd)
if fmap is not None:
# crop the fringe map and pair file
fmap = fmap.crop(mccd)
fpair = fpair.crop(mccd, nhalf)
# wind through the CCDs to display, accumulating stuff
# to send to the plot manager
message = ""
skipped = True
for nc, cnam in enumerate(ccds):
ccd = mccd[cnam]
if ccd.is_data():
# "is_data" indicates genuine data as opposed to junk
# that results from nskip > 0.
plotted[nc] = True
# subtract the bias
if bias is not None:
ccd -= bias[cnam]
if dark is not None:
dexpose = dark.head["EXPTIME"]
cexpose = ccd.head["EXPTIME"]
scale = (cexpose - bexpose) / dexpose
ccd -= scale * dark[cnam]
# divide out the flat
if flat is not None:
ccd /= flat[cnam]
# Remove fringes
if fmap is not None and cnam in fmap and cnam in fpair:
fscale = fpair[cnam].scale(ccd, fmap[cnam], nhalf,
rmin, rmax)
ccd -= fscale * fmap[cnam]
if msub:
# subtract median from each window
for wind in ccd.values():
wind -= wind.median()
# keep list of current CCDs
current_ccds[nc] = ccd
# at this point current_ccds contains the current set of CCDs to plot
if plotted.all():
# Finally have at least one proper exposure of all
# CCDs and can make plots every time with skipped
# frames staying unchanged Set up the plot. ny by nx
# rows x columns of images + 1 for optional light
# curve which is either South or East of images
prop_cycle = plt.rcParams['axes.prop_cycle']
colors = prop_cycle.by_key()['color'][:len(ccds)]
# create the figure just once to avoid memory
# problems, i.e. we re-use the same figure for every
# plot. plt.close() which I thought work instead
# didn't and the programme went a bit bonkers at the
# end releasing resources. Putting '0' didn't work either.
if first_fig:
fig = plt.figure(figsize=(width, height))
first_fig = False
if rlog is not None:
# plot light curve
# first the location
if location.lower() == 's':
# light curve "South" of the images
rect = [
lpad[0], fraction * lpad[1], 1 - lpad[0] - lpad[2],
fraction * (1 - lpad[1] - lpad[3])
]
gs = GridSpec(ny, nx, figure=fig, bottom=fraction)
elif location.lower() == 'e':
# light curve "East" of the images
rect = [
1 - fraction + fraction * lpad[0], lpad[1],
fraction * (1 - lpad[0] - lpad[2]),
1 - lpad[1] - lpad[3]
]
gs = GridSpec(ny, nx, right=1 - fraction)
# light curve axes
ax = fig.add_axes(rect)
ax.set_xlim(0, tmax)
ax.set_ylim(fmin, fmax)
ax.set_xlabel(f'Time [mins, since MJD = {T0:.4f}]')
ax.set_ylabel(f'Target / Comparison')
ax.tick_params(axis="x", direction="in")
ax.tick_params(axis="y", direction="in", rotation=90)
ax.tick_params(bottom=True,
top=True,
left=True,
right=True)
for cnam, (nframes, lc), col in zip(ccds, lcs, colors):
plot = (nframes >= first) & (nframes <= first + nframe)
lct = lc[plot]
if style == 'dots':
fmt = '.'
color = None
elif style == 'line':
fmt = '-'
color = None
col = None
elif style == 'both':
fmt = '.-'
color = '0.7'
lct.mplot(ax,
fmt=fmt,
color=color,
mfc=col,
mec=col,
ms=ms,
lw=lw,
label=f'CCD {cnam}')
if len(ccds) > 1:
ax.legend(loc='upper right')
else:
# images only
gs = GridSpec(ny, nx, figure=fig)
# plot images
for n, (cnam, ccd, ilo,
ihi) in enumerate(zip(ccds, current_ccds, ilos, ihis)):
ax = fig.add_subplot(gs[n // nx, n % nx])
mpl.pCcd(ax,
ccd,
iset,
plo,
phi,
ilo,
ihi,
f'CCD {cnam}',
xlo=xlo,
xhi=xhi,
ylo=ylo,
yhi=yhi,
cmap=cmap)
ax.set_xlim(xlo, xhi)
ax.set_ylim(ylo, yhi)
ax.tick_params(axis="x", direction="in")
ax.tick_params(axis="y", direction="in", rotation=90)
ax.tick_params(bottom=True,
top=True,
left=True,
right=True)
# save to disk, clear figure
oname = os.path.join(
dstore,
f'{os.path.basename(resource)}_{first+nframe:0{ndigit}d}.{fext}'
)
plt.savefig(oname, dpi=dpi)
plt.clf()
print(f' written figure to {oname}')
if last and nframe + first == last:
break
def grab(args=None):
"""``grab [source] run [temp] (ndigit) first last (trim [ncol nrow])
[twait tmax] bias [dtype]``
This downloads a sequence of images from a raw data file and writes them
out to a series CCD / MCCD files.
Parameters:
source : string [hidden]
Data source, four options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : ULTRACAM server
| 'ul' : local ULTRACAM .xml/.dat files
run : string
run name to access
temp : bool [hidden, defaults to False]
True to indicate that the frames should be written to temporary
files with automatically-generated names in the expectation of
deleting them later. This also writes out a file listing the names.
The aim of this is to allow grab to be used as a feeder for other
routines in larger scripts. If temp == True, grab will return with
the name of the list of hcm files. Look at 'makebias' for an
example that uses this feature.
ndigit : int [if not temp]
Files created will be written as 'run005_0013.fits' etc. `ndigit`
is the number of digits used for the frame number (4 in this
case). Any pre-existing files of the same name will be
over-written.
first : int
First frame to access
last : int
Last frame to access, 0 for the lot
trim : bool [if source starts with 'u']
True to trim columns and/or rows off the edges of windows nearest
the readout. This is particularly for ULTRACAM windowed data where
the first few rows and columns can contain bad data.
ncol : int [if trim]
Number of columns to remove (on left of left-hand window, and right
of right-hand windows)
nrow : int [if trim]
Number of rows to remove (bottom of windows)
twait : float [hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
bias : string
Name of bias frame to subtract, 'none' to ignore.
dtype : string [hidden, defaults to 'f32']
Data type on output. Options:
| 'f32' : output as 32-bit floats (default)
| 'f64' : output as 64-bit floats.
| 'u16' : output as 16-bit unsigned integers. A warning will be
issued if loss of precision occurs; an exception will
be raised if the data are outside the range 0 to 65535.
"""
command, args = utils.script_args(args)
# get inputs
with Cline('HIPERCAM_ENV', '.hipercam', command, args) as cl:
# register parameters
cl.register('source', Cline.GLOBAL, Cline.HIDE)
cl.register('run', Cline.GLOBAL, Cline.PROMPT)
cl.register('temp', Cline.GLOBAL, Cline.HIDE)
cl.register('ndigit', Cline.LOCAL, Cline.PROMPT)
cl.register('first', Cline.LOCAL, Cline.PROMPT)
cl.register('last', Cline.LOCAL, Cline.PROMPT)
cl.register('trim', Cline.GLOBAL, Cline.PROMPT)
cl.register('ncol', Cline.GLOBAL, Cline.HIDE)
cl.register('nrow', Cline.GLOBAL, Cline.HIDE)
cl.register('twait', Cline.LOCAL, Cline.HIDE)
cl.register('tmax', Cline.LOCAL, Cline.HIDE)
cl.register('bias', Cline.GLOBAL, Cline.PROMPT)
cl.register('dtype', Cline.LOCAL, Cline.HIDE)
# get inputs
source = cl.get_value('source',
'data source [hs, hl, us, ul]',
'hl',
lvals=('hs', 'hl', 'us', 'ul'))
# OK, more inputs
resource = cl.get_value('run', 'run name', 'run005')
cl.set_default('temp', False)
temp = cl.get_value(
'temp', 'save to temporary automatically-generated file names?',
True)
if not temp:
ndigit = cl.get_value('ndigit',
'number of digits in frame identifier', 3, 0)
first = cl.get_value('first', 'first frame to grab', 1, 0)
last = cl.get_value('last', 'last frame to grab', 0)
if last < first and last != 0:
sys.stderr.write('last must be >= first or 0')
sys.exit(1)
if source.startswith('u'):
trim = cl.get_value(
'trim',
'do you want to trim edges of windows? (ULTRACAM only)', True)
if trim:
ncol = cl.get_value('ncol',
'number of columns to trim from windows',
0)
nrow = cl.get_value('nrow',
'number of rows to trim from windows', 0)
else:
trim = False
twait = cl.get_value('twait', 'time to wait for a new frame [secs]',
1., 0.)
tmax = cl.get_value('tmax',
'maximum time to wait for a new frame [secs]', 10.,
0.)
bias = cl.get_value('bias',
"bias frame ['none' to ignore]",
cline.Fname('bias', hcam.HCAM),
ignore='none')
cl.set_default('dtype', 'f32')
dtype = cl.get_value('dtype',
'data type [f32, f64, u16]',
'f32',
lvals=('f32', 'f64', 'u16'))
# Now the actual work.
# strip off extensions
if resource.endswith(hcam.HRAW):
resource = resource[:resource.find(hcam.HRAW)]
# initialisations
total_time = 0 # time waiting for new frame
nframe = first
root = os.path.basename(resource)
bframe = None
# Finally, we can go
if temp:
# create a directory on temp for the temporary file to avoid polluting
# it too much
fnames = []
tdir = os.path.join(tempfile.gettempdir(),
'hipercam-{:s}'.format(getpass.getuser()))
os.makedirs(tdir, exist_ok=True)
with spooler.data_source(source, resource, first) as spool:
try:
for mccd in spool:
# Handle the waiting game ...
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmax, total_time)
if give_up:
print('grab stopped')
break
elif try_again:
continue
# Trim the frames: ULTRACAM windowed data has bad
# columns and rows on the sides of windows closest to
# the readout which can badly affect reduction. This
# option strips them.
if trim:
hcam.ccd.trim_ultracam(mccd, ncol, nrow)
if bias is not None:
# read bias after first frame so we can
# chop the format
if bframe is None:
# read the bias frame
bframe = hcam.MCCD.read(bias)
# reformat
bframe = bframe.crop(mccd)
mccd -= bframe
if dtype == 'u16':
mccd.uint16()
elif dtype == 'f32':
mccd.float32()
elif dtype == 'f64':
mccd.float64()
# write to disk
if temp:
# generate name automatically
fd, fname = tempfile.mkstemp(suffix=hcam.HCAM, dir=tdir)
mccd.write(fname, True)
os.close(fd)
fnames.append(fname)
else:
fname = '{:s}_{:0{:d}}{:s}'.format(root, nframe, ndigit,
hcam.HCAM)
mccd.write(fname, True)
print('Written frame {:d} to {:s}'.format(nframe, fname))
# update the frame number
nframe += 1
if last and nframe > last:
break
except KeyboardInterrupt:
# trap ctrl-C so we can delete temporary files if temp
if temp:
for fname in fnames:
os.remove(fname)
print('\ntemporary files deleted')
print('grab aborted')
else:
print('\ngrab aborted')
sys.exit(1)
if temp:
if len(fnames) == 0:
raise hcam.HipercamError(
'no files were grabbed; please check input parameters, especially "first"'
)
# write the file names to a list
fd, fname = tempfile.mkstemp(suffix=hcam.LIST, dir=tdir)
with open(fname, 'w') as fout:
for fnam in fnames:
fout.write(fnam + '\n')
os.close(fd)
print('temporary file names written to {:s}'.format(fname))
# return the name of the file list
return fname
def extractFlux(cnam, ccd, rccd, read, gain, ccdwin, rfile, store):
"""This extracts the flux of all apertures of a given CCD.
The steps are (1) creation of PSF model, (2) PSF fitting, (3)
flux extraction. The apertures are assumed to be correctly positioned.
It returns the results as a dictionary keyed on the aperture label. Each
entry returns a list:
[x, ex, y, ey, fwhm, efwhm, beta, ebeta, counts, countse, sky, esky,
nsky, nrej, flag]
flag = bitmask. See hipercam.core to see all the options which are
referred to by name in the code e.g. ALL_OK. The various flags can
signal that there no sky pixels (NO_SKY), the sky aperture was off
the edge of the window (SKY_AT_EDGE), etc.
This code::
>> bset = flag & TARGET_SATURATED
determines whether the data saturation flag is set for example.
Arguments::
cnam : string
CCD identifier label
ccd : CCD
the debiassed, flat-fielded CCD.
rccd : CCD
corresponding raw CCD, used to work out whether data are
saturated in target aperture.
read : CCD
readnoise divided by the flat-field
gain : CCD
gain multiplied by the flat field
ccdwin : dictionary of strings
the Window label corresponding to each Aperture
rfile : Rfile
reduce file configuration parameters
store : dict of dicts
see moveApers for what this contains.
"""
# initialise flag
flag = hcam.ALL_OK
ccdaper = rfile.aper[cnam]
results = {}
# get profile params from aperture store
mfwhm = store["mfwhm"]
mbeta = store["mbeta"]
method = "m" if mbeta > 0.0 else "g"
if mfwhm <= 0:
# die hard, die soon as there's nothing we can do.
print((" *** WARNING: CCD {:s}: no measured FWHM to create PSF model"
"; no extraction possible").format(cnam))
# set flag to indicate no FWHM
flag = hcam.NO_FWHM
for apnam, aper in ccdaper.items():
info = store[apnam]
results[apnam] = {
"x": aper.x,
"xe": info["xe"],
"y": aper.y,
"ye": info["ye"],
"fwhm": info["fwhm"],
"fwhme": info["fwhme"],
"beta": info["beta"],
"betae": info["betae"],
"counts": 0.0,
"countse": -1,
"sky": 0.0,
"skye": 0.0,
"nsky": 0,
"nrej": 0,
"flag": flag,
}
return results
# all apertures have to be in the same window, or we can't easily make a
# postage stamp of the data
wnames = set(ccdwin.values())
if len(wnames) != 1:
print((" *** WARNING: CCD {:s}: not all apertures"
" lie within the same window; no extraction possible"
).format(cnam))
# set flag to indicate no extraction
flag = hcam.NO_EXTRACTION
# return empty results
for apnam, aper in ccdaper.items():
info = store[apnam]
results[apnam] = {
"x": aper.x,
"xe": info["xe"],
"y": aper.y,
"ye": info["ye"],
"fwhm": info["fwhm"],
"fwhme": info["fwhme"],
"beta": info["beta"],
"betae": info["betae"],
"counts": 0.0,
"countse": -1,
"sky": 0.0,
"skye": 0.0,
"nsky": 0,
"nrej": 0,
"flag": flag,
}
return results
wnam = wnames.pop()
# PSF params are in binned pixels, so find binning
bin_fac = ccd[wnam].xbin
# create PSF model
if method == "m":
psf_model = MoffatPSF(beta=mbeta, fwhm=mfwhm / bin_fac)
else:
psf_model = IntegratedGaussianPRF(sigma=mfwhm *
gaussian_fwhm_to_sigma / bin_fac)
# force photometry only at aperture positions
# this means PSF shape and positions are fixed, we are only fitting flux
if rfile["psf_photom"]["positions"] == "fixed":
psf_model.x_0.fixed = True
psf_model.y_0.fixed = True
# create instances for PSF photometry
gfac = float(rfile["psf_photom"]["gfac"])
sclip = float(rfile["sky"]["thresh"])
daogroup = DAOGroup(gfac * mfwhm / bin_fac)
mmm_bkg = MMMBackground(sigma_clip=SigmaClip(sclip))
fitter = LevMarLSQFitter()
fitshape_box_size = int(2 * int(rfile["psf_photom"]["fit_half_width"]) + 1)
fitshape = (fitshape_box_size, fitshape_box_size)
photometry_task = BasicPSFPhotometry(
group_maker=daogroup,
bkg_estimator=mmm_bkg,
psf_model=psf_model,
fitter=fitter,
fitshape=fitshape,
)
# initialise flag
flag = hcam.ALL_OK
# extract Windows relevant for these apertures
wdata = ccd[wnam]
wraw = rccd[wnam]
# extract sub-windows that include all of the apertures, plus a little
# extra around the edges.
x1 = min([ap.x - ap.rsky2 - wdata.xbin for ap in ccdaper.values()])
x2 = max([ap.x + ap.rsky2 + wdata.xbin for ap in ccdaper.values()])
y1 = min([ap.y - ap.rsky2 - wdata.ybin for ap in ccdaper.values()])
y2 = max([ap.y + ap.rsky2 + wdata.ybin for ap in ccdaper.values()])
# extract sub-Windows
swdata = wdata.window(x1, x2, y1, y2)
swraw = wraw.window(x1, x2, y1, y2)
# compute pixel positions of apertures in windows
xpos, ypos = zip(*((swdata.x_pixel(ap.x), swdata.y_pixel(ap.y))
for ap in ccdaper.values()))
positions = Table(names=["x_0", "y_0"], data=(xpos, ypos))
# do the PSF photometry
photom_results = photometry_task(swdata.data, init_guesses=positions)
slevel = mmm_bkg(swdata.data)
# unpack the results and check apertures
for apnam, aper in ccdaper.items():
try:
# reset flag
flag = hcam.ALL_OK
result_row = photom_results[photom_results["id"] == int(apnam)]
if len(result_row) == 0:
flag |= hcam.NO_DATA
raise hcam.HipercamError(
"no source in PSF photometry for this aperture")
elif len(result_row) > 1:
flag |= hcam.NO_EXTRACTION
raise hcam.HipercamError(
"ambiguous lookup for this aperture in PSF photometry")
else:
result_row = result_row[0]
# compute X, Y arrays over the sub-window relative to the centre
# of the aperture and the distance squared from the centre (Rsq)
# to save a little effort.
x = swdata.x(np.arange(swdata.nx)) - aper.x
y = swdata.y(np.arange(swdata.ny)) - aper.y
X, Y = np.meshgrid(x, y)
Rsq = X**2 + Y**2
# size of a pixel which is used to taper pixels as they approach
# the edge of the aperture to reduce pixellation noise
size = np.sqrt(wdata.xbin * wdata.ybin)
# target selection, accounting for extra apertures and allowing
# pixels to contribute if their centres are as far as size/2 beyond
# the edge of the circle (but with a tapered weight)
dok = Rsq < (aper.rtarg + size / 2.0)**2
if not dok.any():
# check there are some valid pixels
flag |= hcam.NO_DATA
raise hcam.HipercamError("no valid pixels in aperture")
# check for saturation and nonlinearity
if cnam in rfile.warn:
if swraw.data[dok].max() >= rfile.warn[cnam]["saturation"]:
flag |= hcam.TARGET_SATURATED
if swraw.data[dok].max() >= rfile.warn[cnam]["nonlinear"]:
flag |= hcam.TARGET_NONLINEAR
else:
warnings.warn(
"CCD {:s} has no nonlinearity or saturation levels set")
counts = result_row["flux_fit"]
countse = result_row["flux_unc"]
info = store[apnam]
results[apnam] = {
"x": aper.x,
"xe": info["xe"],
"y": aper.y,
"ye": info["ye"],
"fwhm": info["fwhm"],
"fwhme": info["fwhme"],
"beta": info["beta"],
"betae": info["betae"],
"counts": counts,
"countse": countse,
"sky": slevel,
"skye": 0,
"nsky": 0,
"nrej": 0,
"flag": flag,
}
except hcam.HipercamError as err:
info = store[apnam]
flag |= hcam.NO_EXTRACTION
results[apnam] = {
"x": aper.x,
"xe": info["xe"],
"y": aper.y,
"ye": info["ye"],
"fwhm": info["fwhm"],
"fwhme": info["fwhme"],
"beta": info["beta"],
"betae": info["betae"],
"counts": 0.0,
"countse": -1,
"sky": 0.0,
"skye": 0.0,
"nsky": 0,
"nrej": 0,
"flag": flag,
}
# finally, we are done
return results
def hlog2fits(args=None):
"""``hlog2fits log``
Converts a |hiper| ASCII log into a FITS file. As well as a modest
reduction in file size (~40%, the ASCII logs are written relatively
efficiently), the resulting file is much faster to read than the ASCII log
so this may be useful for very large log files [test of 78,000 frame file:
12.9 seconds to read the ASCII file, 1.9 to read the FITS version]. At the
moment no significant header information is transferred beyond the CCD
names. Each CCD appears as a single binary table, starting at the second
HDU (or HDU 1 if you number them 0,1,2 ..). This can be read using
:meth:`hipercam.hlog.Hlog.from_fits`.
Parameters:
log : string
name of the log file (should end .log). The output FITS file
will have the same root name but end .fits. The routine will abort
if there is a pre-existing file of the same name.
NB Because of the danger of over-writing raw data (also ends .fits), this
routine will not over-write pre-existing files. You should delete clashing
files if you really want to proceed.
"""
command, args = utils.script_args(args)
# get input section
with Cline('HIPERCAM_ENV', '.hipercam', command, args) as cl:
# register parameters
cl.register('log', Cline.LOCAL, Cline.PROMPT)
cl.register('origin', Cline.LOCAL, Cline.PROMPT)
# get inputs
log = cl.get_value(
'log', 'name of log file from "reduce" to convert to FITS',
cline.Fname('red', hcam.LOG))
origin = cl.get_value('origin',
'h(ipercam) or u(ltracam)',
'h',
lvals=['h', 'u'])
oname = os.path.basename(log)
oname = oname[:oname.rfind('.')] + '.fits'
if os.path.exists(oname):
raise hcam.HipercamError(
('A file called {:s} already exists and'
' will not be over-written; aborting').format(oname))
# Read in the ASCII log
if origin == 'h':
hlg = hcam.hlog.Hlog.read(log)
elif origin == 'u':
hlg = hcam.hlog.Hlog.fulog(log)
print('Loaded ASCII log = {:s}'.format(log))
# Generate HDU list
# First the primary HDU (no data)
phdr = fits.Header()
phdr['LOGFILE'] = (os.path.basename(log), 'Original log file')
phdu = fits.PrimaryHDU(header=phdr)
hdul = [
phdu,
]
# Now a BinTable for each CCD
for cnam in sorted(hlg):
hdr = fits.Header()
hdr['CCDNAME'] = (cnam, 'CCD name')
hdul.append(
fits.BinTableHDU(hlg[cnam],
header=hdr,
name='CCD {:s}'.format(cnam)))
hdul = fits.HDUList(hdul)
# finally write to disk
print('Writing to disk in file = {:s}'.format(oname))
hdul.writeto(oname)
print('Converted {:s} to {:s}'.format(log, oname))
def splice(args=None):
"""``splice input1 input2 ccd [win] output``
Merges two multi-CCD images so that parts of input1 are replaced
by equivalent parts of input2. This can be useful e.g. to create
a combined flat field out of different frames.
Parameters:
input1 : str
first input frame
input2 : str
second input frame
ccd : str
the CCD(s) to transfer, '0' for all of them. '1 3', i.e.
separate with spaces for more than one CCD.
win : str [hidden]
the window(s) to transfer, '0' for all of them (default). The
same windows must exist in the selected CCDs of both inputs
output : str
output file.
"""
command, args = utils.script_args(args)
# get inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("input1", Cline.GLOBAL, Cline.PROMPT)
cl.register("input2", Cline.GLOBAL, Cline.PROMPT)
cl.register("ccd", Cline.LOCAL, Cline.PROMPT)
cl.register("win", Cline.LOCAL, Cline.HIDE)
cl.register("output", Cline.LOCAL, Cline.PROMPT)
# get inputs
infile1 = cl.get_value(
"input1", "first input frame", cline.Fname("frame1", hcam.HCAM)
)
mccd1 = hcam.MCCD.read(infile1)
infile2 = cl.get_value(
"input2", "second input frame", cline.Fname("frame2", hcam.HCAM)
)
mccd2 = hcam.MCCD.read(infile2)
if len(mccd1) > 1:
ccd = cl.get_value("ccd", "CCD(s) to transfer ('0' for all)", "1")
if ccd == "0":
ccds = list(mccd1.keys())
else:
ccds = ccd.split()
else:
ccds = list(mccd1.keys())
# Find window names in common across all selected CCDs
# (intersection of sets)
wins = None
for cnam in ccds:
if wins is None:
wins = set(mccd1[cnam].keys())
else:
wins &= set(mccd1[cnam].keys())
if len(wins) == 0:
raise hcam.HipercamError(
"There were no windows in common across the selected CCDs"
)
elif len(wins) > 1:
cl.set_default("win", "0")
win = cl.get_value("win", "windows to transfer ('0' for all)", "0")
if win != "0":
wins = wins.split()
outfile = cl.get_value(
"output", "output file", cline.Fname("output", hcam.HCAM, cline.Fname.NEW)
)
# Now the (trivial) actual work.
for cnam in ccds:
for wnam in wins:
mccd1[cnam][wnam] = mccd2[cnam][wnam]
mccd1.write(outfile, True)
def tanalysis(args=None):
"""``tanalysis [source] run [mintim dcmax] details (plot [check (output)])``
.. Warning::
This script should only be run if you know what you are doing.
Analyses timing data in |hiper|, ULTRACAM or ULTRASPEC data. You
must first have made copy of the timing bytes (see ``tbytes'' and
``atbytes''). It looks for a timing bytes file in a sub-directory
of the directory the script is run from called "tbytes". If the
file does not exist, the script will exit.
On occasion there are problems with ULTRACAM, ULTRASPEC and, very
rarely, |hiper| timestamps. The main purpose of this script is to
provide an analysis of the timing to spot these is as clear a way
as possible. Some of the problems are::
1) ULTRASPEC null timestamps: ULTRASPEC runs occasionally feature
weird null timestamps. They don't replace the proper ones but
push them down the line so they turn up later.
2) Extra OK-looking timestamps: on three occasions, genuine but
spurious timestamps have been added to the FIFO buffer. Again
these are extra, and push the real ones down the line.
Parameters:
source : string [hidden]
Data source, two options:
| 'hl' : local HiPERCAM FITS file
| 'ul' : ULTRA(CAM|SPEC) server
run : string
run number to access, e.g. 'run0034'. This will also be
used to generate the name for the timing bytes file
(extension '.tbts'). If a file of this name already exists,
the script will attempt to read and compare the bytes of
the two files and report any changes. The timing bytes
file will be written to the present working directory, not
necessarily the location of the data file.
mintim : int [hidden]
Minimum number of times to attempt to do anything
with. This must be at least 4 so that there are 3+ time
differences to try to get a median time, but in practice it
is probably advisable to use a larger number still.
dcmax : float [hidden]
Maximum differential in terms of cycle number from exact integers
to use to indicate "failed" times. Pre 2010-02 needs to be looser
than post 2010-02. Number of order 0.002 is fine post 2010-02,
whereas 0.02 is more like it pre 2010-02.
details : bool
Print out detailed information on problem times
plot : bool
True to make diagnostic plots if problem runs found
check : bool [if plot, hidden]
True simply to check a run for timing problems, not try to
fix them. Defaults to True, and is also hidden by
default. If you set check=False, the code also attempts to
fix the times and generate a new data file. At the moment
it is very specific code set to try to correct for spurious
extra timestamps as happened for the following runs:
2011-10-31 run022, 2013-12-31 run031 and 2015-06-22
run024. It works by calculating its best estimate of cycle
numbers and rejecting any too far off being integers. The
corrected times are analysed for duplications and gaps. You
do not want any duplications (and ideally no gaps either).
You will be asked one final time if you want to proceed.
output : str [if not check]
Name of the modified data file. For safety this will not overwrite
any existing file. It is up to the user to move / copy / rename files
appropriately. Take care to check the results before deleting any
original data. Keeping a copy is probably advisable.
.. Note::
The final frame on many runs can be terminated early and thus ends with
an out of sequence timestamp (it comes earlier than expected). The script
regards such a run as being "OK" if there are not other issues.
"""
command, args = utils.script_args(args)
# get the inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("source", Cline.LOCAL, Cline.HIDE)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("mintim", Cline.LOCAL, Cline.HIDE)
cl.register("dcmax", Cline.LOCAL, Cline.HIDE)
cl.register("plot", Cline.LOCAL, Cline.PROMPT)
cl.register("details", Cline.LOCAL, Cline.PROMPT)
cl.register("check", Cline.LOCAL, Cline.HIDE)
cl.register("output", Cline.LOCAL, Cline.PROMPT)
# get inputs
source = cl.get_value("source",
"data source [hl, ul]",
"hl",
lvals=("hl", "ul"))
run = cl.get_value("run", "run name", "run005")
if run.endswith(".fits"):
run = run[:-5]
mintim = cl.get_value("mintim", "minimum number of times needed", 6, 4)
dcmax = cl.get_value("dcmax",
"maximum cycle number offset from an integer",
0.003, 0)
details = cl.get_value("details",
"print detailed diagnostics of problems", True)
plot = cl.get_value("plot", "make diagnostic plots", True)
if plot:
cl.set_default("check", True)
check = cl.get_value("check",
"check for, but do not fix, any problems",
True)
if not check:
output = cl.get_value(
"output", "output file for data with modified times",
cline.Fname("modified", ftype=cline.Fname.NOCLOBBER))
else:
check = True
# create name of timing file
tfile = os.path.join("tbytes", os.path.basename(run) + hcam.TBTS)
if not os.path.isfile(tfile):
raise hcam.HipercamError(
"Could not find timing file = {}".format(tfile))
# create name of run file and the copy which will only get made
# later if problems are picked up
if source == "hl":
rfile = run + ".fits"
else:
rfile = run + ".dat"
# first load all old time stamps, and compute MJDs
if source == "ul":
# Load up the time stamps from the timing data file. Need also
# the header of the original run to know how many bytes to
# read and how to interpret them.
rhead = hcam.ucam.Rhead(run)
with open(tfile, "rb") as fin:
atbytes, mjds, tflags, gflags = [], [], [], []
nframe = 0
while 1:
tbytes = fin.read(rhead.ntbytes)
if len(tbytes) != rhead.ntbytes:
break
nframe += 1
atbytes.append(tbytes)
# interpret times
mjd, tflag, gflag, uformat = u_tbytes_to_mjd(
tbytes, rhead, nframe)
mjds.append(mjd)
tflags.append(tflag)
gflags.append(gflag)
else:
raise NotImplementedError(
"source = {} not yet implemented".format(source))
if len(mjds) < mintim:
# Must have specified minimum of times to work with. This is
# checked more stringently later, but this avoids silly crashes
# when there are no times.
print(
run,
"has too few frames to work with ({} vs minimum = {})".format(
len(mjds), mintim),
)
return
# Independent of source, at this stage 'atbytes' is a list of all
# timestamp bytes, while 'mjds' is a list of all equivalent MJDs,
# and 'tflags' are bools which are True for OK times, False for
# null timestamps.
mjds = np.array(mjds)
tflags = np.array(tflags)
gflags = np.array(gflags)
inds = np.arange(len(mjds))
nulls = ~tflags
nulls_present = nulls.any()
btimes = ~gflags
bad_times_present = btimes.any()
# Remove null timestamps
mjds_ok = mjds[tflags & gflags]
inds_ok = inds[tflags & gflags]
# Must have specified minimum of times to work with.
if len(mjds_ok) < mintim:
print(
run,
"has too few non-null times to work with ({} vs minimum = {})".
format(len(mjds_ok), mintim),
)
return
# Median time difference of GPS timestamps with guard against identical
# times that can dominate leading to mdiff = 0
diffs = mjds_ok[1:] - mjds_ok[:-1]
pos = diffs > 1.0e-9
if len(diffs[pos]) == 0:
print(run, "has zero positive time differences to work with")
return
mdiff = np.median(diffs[pos])
# Maximum deviation from median separation to allow (days)
# 1.e-9 ~ 1e-4 seconds, ~30x shorter than shortest ULTRACAM
# cycle time. This could allow some bad stamps through but
# they will be spotted later by looking at cycle numbers.
MDIFF = 1.0e-9
# Identify OK timestamps. Could miss some at this point, but
# that's OK) kick off by marking all timestamps as False
ok = mjds_ok == mjds_ok + 1
for n in range(len(mjds_ok) - 1):
if abs(mjds_ok[n + 1] - mjds_ok[n] - mdiff) < MDIFF:
# if two timestamps differ by the right amount, mark both
# as ok
ok[n] = ok[n + 1] = True
gmjds = mjds_ok[ok]
ginds = inds_ok[ok]
if len(gmjds) < 2:
print("{}: found fewer than 2 good timestamps".format(run))
return
# Work out integer cycle numbers. First OK timestamp is given
# cycle number = 0 automatically by the method used. The cycle
# numbers can go wrong if the median is not precise enough leading
# to jumps in the cycle number on runs with large numbers of short
# exposures, so we build up to the full fit in stages, doubling the
# number fitted each time
NMAX = 100
cycles = (gmjds[:NMAX] - gmjds[0]) / mdiff
moffset = (cycles - np.round(cycles)).mean()
icycles = np.round(cycles - moffset).astype(int)
while NMAX < len(gmjds):
# fit linear trend of first NMAX times where hopefully NMAX is small
# enough for there not to be a big error but large enough to allow extrapolation.
slope, intercept, r, p, err = stats.linregress(icycles[:NMAX],
gmjds[:NMAX])
NMAX *= 1.5
NMAX = int(NMAX)
icycles = np.round((gmjds[:NMAX] - intercept) / slope).astype(int)
# hope the cycle numbers should be good across the board for all the "gmjds"
# at this point. Carry out final fits to try to refine the intercept and
# slope because it is possible to be upset by jumps in time of less
# than a cycle (i.e 2002-09-20, run014)
c*k = icycles == icycles
cinds = np.arange(len(c*k))
nrej = 0
while 1:
# fit to ok ones
slope, intercept, r, p, err = stats.linregress(icycles[c*k],
gmjds[c*k])
cycles = (gmjds - intercept) / slope
# cdiffs, acdiffs -- cycle differences and their absolute
# values
cdiffs = cycles - icycles
acdiffs = np.abs(cdiffs)
dmax = np.argmax(acdiffs[c*k])
if acdiffs[c*k][dmax] > dcmax:
# at least one is out of spec. reject it
c*k[cinds[c*k][dmax]] = False
nrej += 1
else:
break
print("Rejected {} times when fitting nominally good times".format(nrej))
# now try to compute cycle numbers for *all* non-null timestamps
acycles = (mjds_ok - intercept) / slope
iacycles = np.round(acycles).astype(int)
if iacycles[-1] == iacycles[-2]:
# because the last frame can be terminated early, it is
# not uncommon for it to end with same cycle number as the
# penultimate one. Correct this here
iacycles[-1] += 1
cadiffs = acycles - iacycles
monotonic = (iacycles[1:] - iacycles[:-1] > 0).all()
# check for early termination on run causing last frame to appear
# early
terminated_early = cadiffs[-1] < -dcmax
# check that all is OK
if (not nulls_present and monotonic
and ((terminated_early and (np.abs(cadiffs[:-1]) < dcmax).all()) or
(not terminated_early and (np.abs(cadiffs) < dcmax).all()))):
if terminated_early:
mdev = np.abs(cadiffs[:-1]).max()
print(
"{} times are OK; {} frames; max. dev. all bar last = {:.2g} cyc, {:.2g} msec; last = {:.2g} cyc"
.format(run, len(iacycles), mdev, 86400 * 1000 * slope * mdev,
cadiffs[-1]))
else:
mdev = np.abs(cadiffs).max()
print(
"{} times are OK; {} frames; max. dev. all = {:.2g} cyc, {:.2g} msec"
.format(run, len(iacycles), mdev, 86400 * 1000 * slope * mdev))
# to prevent crash during plot
fails = cadiffs != cadiffs
run_ok = True
else:
# search for duplicate cycle numbers
u, c = np.unique(iacycles, return_counts=True)
dupes = u[c > 1]
dupes_present = len(dupes) > 0
ntot = len(mjds)
ndupe = len(dupes)
nnull = len(mjds[nulls])
nbad = len(mjds[btimes])
fails = np.abs(cadiffs) > dcmax
if terminated_early:
nfail = len(cadiffs[fails]) - 1
mdev = np.abs(cadiffs[:-1]).max()
else:
nfail = len(cadiffs[fails])
mdev = np.abs(cadiffs).max()
if ndupe == 0 and nbad == 0 and nnull == 0 and inds_ok[fails][
-1] < nfail:
# save some runs with trivial level issues from being flagged
if nfail < len(cadiffs):
mdev = np.abs(cadiffs[nfail:]).max()
comment = " [All fails are at start]"
run_ok = True
else:
comment = ""
run_ok = False
# Summarise problems
print(
"{} timestamps corrupt. TOT,DUP,NULL,BAD,FAIL,FOK = {},{},{},{},{},{}; max dev = {:.2g} cyc, {:.2g} sec.{}"
.format(
run,
ntot,
ndupe,
nnull,
nbad,
nfail,
inds_ok[ok][0] + 1,
mdev,
86400 * slope * mdev,
comment,
))
if details:
# Some details
fcadiffs = iacycles[1:] - iacycles[:-1]
back = fcadiffs <= 0
oinds = inds_ok[:-1][back]
ninds = np.arange(len(fcadiffs))[back]
cycs = iacycles[:-1][back]
ncycs = iacycles[1:][back]
tims = mjds_ok[:-1][back]
if len(oinds):
print("\nInverted order times:")
for oind, nind, cyc, ncyc, mjd in zip(oinds, ninds, cycs,
ncycs, tims):
print(
" Old index = {}, new index = {}, cycle = {}, next cycle = {}, time = {}"
.format(oind, nind, cyc, ncyc, mjd))
if nfail:
print("\nFailed times:")
inds = inds_ok[fails]
cycs = iacycles[fails]
cydiffs = cadiffs[fails]
tims = mjds_ok[fails]
for ind, cyc, cdiff, mjd in zip(inds, cycs, cydiffs, tims):
print(
" Index = {}, cycle = {}, cycle diff. = {:.2g}, time = {}"
.format(ind, cyc, cdiff, mjd))
if plot:
# diagnostic plot: cycle differences vs cycle numbers
def c2t(x):
return 86400 * slope * x
def t2c(x):
return x / slope / 86400
fig = plt.figure()
ax = fig.add_subplot()
ax.plot(iacycles[~fails], cadiffs[~fails], ".b")
ax.plot(iacycles[fails], cadiffs[fails], ".r")
ax.set_xlabel("Cycle number")
ax.set_ylabel('Cycle - nint(Cycle) [cycles]')
secxax = ax.secondary_xaxis("top", functions=(c2t, t2c))
secxax.set_xlabel("Time [MJD - {}] (seconds)".format(intercept))
secyax = ax.secondary_yaxis("right",
functions=(lambda x: 1000 * c2t(x),
lambda x: t2c(x) / 1000))
secyax.set_xlabel("$\Delta t$ (msec)".format(intercept))
plt.show()
if not plot or run_ok or check:
# go no further if run is OK, no plot has been made, or we are in check mode
return
# very specific code at this point for the post-2010 problem runs: remove duplicate
# time stamps.
if ginds[0] != 0 or nfail == 0 or (source == 'ul' and uformat != 2):
# this case needs checking
raise hcam.HipercamError(
'Cannot yet handle cases where the first timestamp is no good, nfail == 0, or format==2 ultracam data'
)
# these frames have been ID-ed as junk
skip_inds = inds_ok[fails]
inds_keep = inds == inds
inds_keep[skip_inds] = False
# inds_keep : False for any Frame we want to forget
new_mjds, new_atbytes = [], []
nframe = 0
# manipulate the frame numbers to avoid warnings
for keep, mjd, tbytes in zip(inds_keep, mjds, atbytes):
if keep:
nframe += 1
new_mjds.append(mjd)
new_atbytes.append(tbytes[:4] + struct.pack("<I", nframe) +
tbytes[8:])
nskip = len(mjds) - len(new_mjds)
# modify the last few times to make up for the skips,
# extrapolating the times using the fitted time step per cycle
# from before. We don't simply add "slope" to the last MJD one by
# one because of subtle round-off issues.
mjds_nskip = mjd + slope * np.linspace(1, nskip, nskip)
for tbytes, mjd in zip(atbytes[-nskip:], mjds_nskip):
unix_sec = ucam.DSEC * (mjd - ucam.UNIX)
nsec = int(np.floor(unix_sec))
nnsec = int(round(1.e7 * (unix_sec - nsec)))
new_mjds.append(ucam.UNIX + float(nsec + nnsec / 1.0e7) / ucam.DSEC)
nframe += 1
tbytes = tbytes[:4] + struct.pack("<I",
nframe) + tbytes[8:12] + struct.pack(
"<II", nsec, nnsec) + tbytes[20:]
new_atbytes.append(tbytes)
# transfer byte 0 over as this contains a flag indicating the status of the blue data for the NBLUE option
# (god, this is painful)
for n in range(len(atbytes)):
new_atbytes[n] = atbytes[n][0:1] + new_atbytes[n][1:]
new_mjds_ok = np.array(new_mjds)
new_mjds_ok = new_mjds_ok[tflags & gflags]
acycles = (new_mjds_ok - intercept) / slope
iacycles = np.round(acycles).astype(int)
diffs = acycles - iacycles
# search for duplicates
u, c = np.unique(iacycles, return_counts=True)
ndupes = len(u[c > 1])
# search for gaps
csteps = iacycles[1:] - iacycles[:-1]
ngaps = len(iacycles[1:][csteps > 1])
print(
'\nFound {} spurious time stamps; new times have {} duplications and {} gaps > 1'
.format(nskip, ndupes, ngaps))
print('Ideally, the last two numbers should be 0 and 0.')
plt.plot(iacycles[:-nskip], diffs[:-nskip], '.g')
plt.plot(iacycles[-nskip:], diffs[-nskip:], '.b')
plt.xlabel('Cycle number')
plt.ylabel('Cycle - int(Cycle) [cycles]')
plt.show()
tdiffs = 86400 * (mjds_ok - new_mjds_ok)
plt.plot(iacycles[:-nskip], tdiffs[:-nskip], '.g')
plt.plot(iacycles[-nskip:], tdiffs[-nskip:], '.b')
plt.xlabel('Cycle number')
plt.ylabel('Old time - new [seconds]')
plt.show()
reply = input('Generate data with corrected time stamps? [no] ')
if reply != 'yes':
print('time correction aborted')
return
# OK, finally, let's go for it!
if source == "ul":
with open(output, "wb") as fout:
with open(rfile, "rb") as fin:
nframe = 0
while 1:
# Read a whole frame
buff = fin.read(rhead.framesize)
if len(buff) != rhead.framesize:
break
# write to output, writing modified timing bytes
# in place of old ones
fout.write(new_atbytes[nframe])
fout.write(buff[rhead.ntbytes:])
nframe += 1
print('Processed frame {} / {}'.format(
nframe, len(atbytes)))
else:
raise NotImplementedError(
"source = {} not yet implemented".format(source))
def ftargets(args=None):
"""``ftargets [source device width height] (run first [twait tmax] |
flist) trim ([ncol nrow]) (ccd (nx)) [pause] thresh fwhm minpix
output bias flat msub iset (ilo ihi | plo phi) xlo xhi ylo yhi``
This script carries out the following steps for each of a series
of images:
(1) detects the sources,
(2) identifies isolated targets suited to profile fits,
(3) fits 2D Moffat profiles to these,
(4) Saves results to disk.
The profile fits are carried out because `sep` does not return
anything that can be used reliably for a FWHM.
Several parameters depends on the object detection threshold
retuned by the source detection. This is referred to as
`threshold`. The source detection is carried out using `sep` which
runs according to the usual source extractor algorithm of Bertin.
The script plots the frames, with ellipses at 3*a,3*b indicated in
red, green boxes indicating the range of pixels identified by
`sep`, and blue boxes marking the targets selected for FWHM
fitting (the boxes indicate the fit region).
Parameters:
source : string [hidden]
Data source, five options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : ULTRACAM server
| 'ul' : local ULTRACAM .xml/.dat files
| 'hf' : list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab' or
converted from foreign data formats.
device : string [hidden]
Plot device. PGPLOT is used so this should be a PGPLOT-style name,
e.g. '/xs', '1/xs' etc. At the moment only ones ending /xs are
supported.
width : float [hidden]
plot width (inches). Set = 0 to let the program choose.
height : float [hidden]
plot height (inches). Set = 0 to let the program choose. BOTH width
AND height must be non-zero to have any effect
run : string [if source ends 's' or 'l']
run number to access, e.g. 'run034'
flist : string [if source ends 'f']
name of file list
first : int [if source ends 's' or 'l']
exposure number to start from. 1 = first frame; set = 0 to always
try to get the most recent frame (if it has changed). For data
from the |hiper| server, a negative number tries to get a frame not
quite at the end. i.e. -10 will try to get 10 from the last
frame. This is mainly to sidestep a difficult bug with the
acquisition system.
twait : float [if source ends 's' or 'l'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [if source ends 's' or 'l'; hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
trim : bool [if source starts with 'u']
True to trim columns and/or rows off the edges of windows nearest
the readout which can sometimes contain bad data.
ncol : int [if trim, hidden]
Number of columns to remove (on left of left-hand window, and right
of right-hand windows)
nrow : int [if trim, hidden]
Number of rows to remove (bottom of windows)
ccd : string
CCD(s) to plot, '0' for all, '1 3' to plot '1' and '3' only, etc.
nx : int [if more than 1 CCD]
number of panels across to display.
pause : float [hidden]
seconds to pause between frames (defaults to 0)
thresh : float
threshold (mutiple of RMS) to use for object detection. Typical
values 2.5 to 4. The higher it is, the fewer objects will be located,
but the fewer false detections will be made.
fwhm : float
FWHM to use for smoothing during object detection. Should be
comparable to the seeing.
minpix : int
Minimum number of pixels above threshold before convolution to count
as a detection. Useful in getting rid of cosmics and high dark count
pixels.
rmin : float
Closest distance of any other detected object for an attempt
to be made to fit the FWHM of an object [unbinned pixels].
pmin : float
Minimum peak height for an attempt to be made to fit the
FWHM of an object. This should be a multiple of the object
detection threshold (returned by `sep` for each object).
pmax : float
Maximum peak height for an attempt to be made to fit the
FWHM of an object. Use to exclude saturated targets
[counts]
emax : float
Maximum elongation (major/minor axis ratio = a/b), > 1. Use
to reduce very non-stellar profiles.
nmax : int
Maximum number of FWHMs to measure. Will take the brightest first,
judging by the flux.
bias : string
Name of bias frame to subtract, 'none' to ignore.
flat : string
Name of flat field to divide by, 'none' to ignore. Should normally
only be used in conjunction with a bias, although it does allow you
to specify a flat even if you haven't specified a bias.
output: string
Name of file for storage of results. Will be a fits file, with
results saved to the HDU 1 as a table.
iset : string [single character]
determines how the intensities are determined. There are three
options: 'a' for automatic simply scales from the minimum to the
maximum value found on a per CCD basis. 'd' for direct just takes
two numbers from the user. 'p' for percentile dtermines levels
based upon percentiles determined from the entire CCD on a per CCD
basis.
ilo : float [if iset='d']
lower intensity level
ihi : float [if iset='d']
upper intensity level
plo : float [if iset='p']
lower percentile level
phi : float [if iset='p']
upper percentile level
xlo : float
left-hand X-limit for plot
xhi : float
right-hand X-limit for plot (can actually be < xlo)
ylo : float
lower Y-limit for plot
yhi : float
upper Y-limit for plot (can be < ylo)
"""
command, args = utils.script_args(args)
# get the inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("source", Cline.GLOBAL, Cline.HIDE)
cl.register("device", Cline.LOCAL, Cline.HIDE)
cl.register("width", Cline.LOCAL, Cline.HIDE)
cl.register("height", Cline.LOCAL, Cline.HIDE)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("trim", Cline.GLOBAL, Cline.PROMPT)
cl.register("ncol", Cline.GLOBAL, Cline.HIDE)
cl.register("nrow", Cline.GLOBAL, Cline.HIDE)
cl.register("twait", Cline.LOCAL, Cline.HIDE)
cl.register("tmax", Cline.LOCAL, Cline.HIDE)
cl.register("flist", Cline.LOCAL, Cline.PROMPT)
cl.register("ccd", Cline.LOCAL, Cline.PROMPT)
cl.register("nx", Cline.LOCAL, Cline.PROMPT)
cl.register("pause", Cline.LOCAL, Cline.HIDE)
cl.register("thresh", Cline.LOCAL, Cline.PROMPT)
cl.register("fwhm", Cline.LOCAL, Cline.PROMPT)
cl.register("minpix", Cline.LOCAL, Cline.PROMPT)
cl.register("rmin", Cline.LOCAL, Cline.PROMPT)
cl.register("pmin", Cline.LOCAL, Cline.PROMPT)
cl.register("pmax", Cline.LOCAL, Cline.PROMPT)
cl.register("emax", Cline.LOCAL, Cline.PROMPT)
cl.register("nmax", Cline.LOCAL, Cline.PROMPT)
cl.register("gain", Cline.LOCAL, Cline.PROMPT)
cl.register("rej", Cline.LOCAL, Cline.PROMPT)
cl.register("bias", Cline.GLOBAL, Cline.PROMPT)
cl.register("flat", Cline.GLOBAL, Cline.PROMPT)
cl.register("output", Cline.LOCAL, Cline.PROMPT)
cl.register("iset", Cline.GLOBAL, Cline.PROMPT)
cl.register("ilo", Cline.GLOBAL, Cline.PROMPT)
cl.register("ihi", Cline.GLOBAL, Cline.PROMPT)
cl.register("plo", Cline.GLOBAL, Cline.PROMPT)
cl.register("phi", Cline.LOCAL, Cline.PROMPT)
cl.register("xlo", Cline.GLOBAL, Cline.PROMPT)
cl.register("xhi", Cline.GLOBAL, Cline.PROMPT)
cl.register("ylo", Cline.GLOBAL, Cline.PROMPT)
cl.register("yhi", Cline.GLOBAL, Cline.PROMPT)
# get inputs
source = cl.get_value(
"source",
"data source [hs, hl, us, ul, hf]",
"hl",
lvals=("hs", "hl", "us", "ul", "hf"),
)
# set some flags
server_or_local = source.endswith("s") or source.endswith("l")
# plot device stuff
device = cl.get_value("device", "plot device", "1/xs")
width = cl.get_value("width", "plot width (inches)", 0.0)
height = cl.get_value("height", "plot height (inches)", 0.0)
if server_or_local:
resource = cl.get_value("run", "run name", "run005")
if source == "hs":
first = cl.get_value("first", "first frame to plot", 1)
else:
first = cl.get_value("first", "first frame to plot", 1, 0)
twait = cl.get_value(
"twait", "time to wait for a new frame [secs]", 1.0, 0.0
)
tmax = cl.get_value(
"tmax", "maximum time to wait for a new frame [secs]", 10.0, 0.0
)
else:
resource = cl.get_value(
"flist", "file list", cline.Fname("files.lis", hcam.LIST)
)
first = 1
trim = cl.get_value("trim", "do you want to trim edges of windows?", True)
if trim:
ncol = cl.get_value("ncol", "number of columns to trim from windows", 0)
nrow = cl.get_value("nrow", "number of rows to trim from windows", 0)
# define the panel grid. first get the labels and maximum dimensions
ccdinf = spooler.get_ccd_pars(source, resource)
try:
nxdef = cl.get_default("nx")
except:
nxdef = 3
if len(ccdinf) > 1:
ccd = cl.get_value("ccd", "CCD(s) to plot [0 for all]", "0")
if ccd == "0":
ccds = list(ccdinf.keys())
else:
ccds = ccd.split()
check = set(ccdinf.keys())
if not set(ccds) <= check:
raise hcam.HipercamError("At least one invalid CCD label supplied")
if len(ccds) > 1:
nxdef = min(len(ccds), nxdef)
cl.set_default("nx", nxdef)
nx = cl.get_value("nx", "number of panels in X", 3, 1)
else:
nx = 1
else:
nx = 1
ccds = list(ccdinf.keys())
cl.set_default("pause", 0.0)
pause = cl.get_value(
"pause", "time delay to add between" " frame plots [secs]", 0.0, 0.0
)
thresh = cl.get_value("thresh", "source detection threshold [RMS]", 3.0)
fwhm = cl.get_value("fwhm", "FWHM for source detection [binned pixels]", 4.0)
minpix = cl.get_value("minpix", "minimum number of pixels above threshold", 3)
rmin = cl.get_value(
"rmin", "nearest neighbour for FWHM fits [unbinned pixels]", 20.0
)
pmin = cl.get_value(
"pmin", "minimum peak value for profile fits [multiple of threshold]", 5.0
)
pmax = cl.get_value(
"pmax", "maximum peak value for profile fits [counts]", 60000.0
)
emax = cl.get_value(
"emax", "maximum elongation (a/b) for profile fits", 1.2, 1.0
)
nmax = cl.get_value("nmax", "maximum number of profile fits", 10, 1)
gain = cl.get_value("gain", "CCD gain [electrons/ADU]", 1.0, 0.0)
rej = cl.get_value(
"rej", "rejection threshold for profile fits [RMS]", 6.0, 2.0
)
# bias frame (if any)
bias = cl.get_value(
"bias",
"bias frame ['none' to ignore]",
cline.Fname("bias", hcam.HCAM),
ignore="none",
)
if bias is not None:
# read the bias frame
bias = hcam.MCCD.read(bias)
fprompt = "flat frame ['none' to ignore]"
else:
fprompt = "flat frame ['none' is normal choice with no bias]"
# flat (if any)
flat = cl.get_value(
"flat", fprompt, cline.Fname("flat", hcam.HCAM), ignore="none"
)
if flat is not None:
# read the flat frame
flat = hcam.MCCD.read(flat)
output = cl.get_value(
"output",
"output file for results",
cline.Fname("sources", hcam.SEP, cline.Fname.NEW),
)
iset = cl.get_value(
"iset",
"set intensity a(utomatically)," " d(irectly) or with p(ercentiles)?",
"a",
lvals=["a", "d", "p"],
)
iset = iset.lower()
plo, phi = 5, 95
ilo, ihi = 0, 1000
if iset == "d":
ilo = cl.get_value("ilo", "lower intensity limit", 0.0)
ihi = cl.get_value("ihi", "upper intensity limit", 1000.0)
elif iset == "p":
plo = cl.get_value(
"plo", "lower intensity limit percentile", 5.0, 0.0, 100.0
)
phi = cl.get_value(
"phi", "upper intensity limit percentile", 95.0, 0.0, 100.0
)
# region to plot
for i, cnam in enumerate(ccds):
nxtot, nytot, nxpad, nypad = ccdinf[cnam]
if i == 0:
xmin, xmax = float(-nxpad), float(nxtot + nxpad + 1)
ymin, ymax = float(-nypad), float(nytot + nypad + 1)
else:
xmin = min(xmin, float(-nxpad))
xmax = max(xmax, float(nxtot + nxpad + 1))
ymin = min(ymin, float(-nypad))
ymax = max(ymax, float(nytot + nypad + 1))
xlo = cl.get_value("xlo", "left-hand X value", xmin, xmin, xmax)
xhi = cl.get_value("xhi", "right-hand X value", xmax, xmin, xmax)
ylo = cl.get_value("ylo", "lower Y value", ymin, ymin, ymax)
yhi = cl.get_value("yhi", "upper Y value", ymax, ymin, ymax)
################################################################
#
# all the inputs have now been obtained. Get on with doing stuff
# open image plot device
imdev = hcam.pgp.Device(device)
if width > 0 and height > 0:
pgpap(width, height / width)
# set up panels and axes
nccd = len(ccds)
ny = nccd // nx if nccd % nx == 0 else nccd // nx + 1
# slice up viewport
pgsubp(nx, ny)
# plot axes, labels, titles. Happens once only
for cnam in ccds:
pgsci(hcam.pgp.Params["axis.ci"])
pgsch(hcam.pgp.Params["axis.number.ch"])
pgenv(xlo, xhi, ylo, yhi, 1, 0)
pglab("X", "Y", "CCD {:s}".format(cnam))
# initialisations. 'last_ok' is used to store the last OK frames of each
# CCD for retrieval when coping with skipped data.
total_time = 0 # time waiting for new frame
nhdu = len(ccds) * [0]
thetas = np.linspace(0, 2 * np.pi, 100)
# values of various parameters
fwhm_min, beta, beta_min, beta_max, readout, max_nfev = (
2.0,
4.0,
1.5,
10.0,
5.5,
100,
)
# number of failed fits
nfail = 0
# open the output file for results
with fitsio.FITS(output, "rw", clobber=True) as fout:
# plot images
with spooler.data_source(source, resource, first, full=False) as spool:
# 'spool' is an iterable source of MCCDs
n = 0
for nf, mccd in enumerate(spool):
if server_or_local:
# Handle the waiting game ...
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmax, total_time
)
if give_up:
print("ftargets stopped")
break
elif try_again:
continue
# Trim the frames: ULTRACAM windowed data has bad columns
# and rows on the sides of windows closest to the readout
# which can badly affect reduction. This option strips
# them.
if trim:
hcam.ccd.trim_ultracam(mccd, ncol, nrow)
# indicate progress
tstamp = Time(mccd.head["TIMSTAMP"], format="isot", precision=3)
print(
"{:d}, utc= {:s} ({:s}), ".format(
mccd.head["NFRAME"],
tstamp.iso,
"ok" if mccd.head.get("GOODTIME", True) else "nok",
),
end="",
)
# accumulate errors
emessages = []
if n == 0:
if bias is not None:
# crop the bias on the first frame only
bias = bias.crop(mccd)
if flat is not None:
# crop the flat on the first frame only
flat = flat.crop(mccd)
# compute maximum length of window name strings
lsmax = 0
for ccd in mccd.values():
for wnam in ccd:
lsmax = max(lsmax, len(wnam))
# display the CCDs chosen
message = ""
pgbbuf()
for nc, cnam in enumerate(ccds):
ccd = mccd[cnam]
if ccd.is_data():
# this should be data as opposed to a blank frame
# between data frames that occur with nskip > 0
# subtract the bias
if bias is not None:
ccd -= bias[cnam]
# divide out the flat
if flat is not None:
ccd /= flat[cnam]
# Where the fancy stuff happens ...
# estimate sky background, look for stars
objs, dofwhms = [], []
nobj = 0
for wnam in ccd:
try:
# chop window, find objects
wind = ccd[wnam].window(xlo, xhi, ylo, yhi)
wind.data = wind.data.astype("float")
objects, bkg = findStars(wind, thresh, fwhm, True)
# subtract background from frame for display
# purposes.
bkg.subfrom(wind.data)
ccd[wnam] = wind
# remove targets with too few pixels
objects = objects[objects["tnpix"] >= minpix]
# run nearest neighbour search on all
# objects, but select only a subset
# with right count levels for FWHM
# measurement, and which are not too
# elongated
results = isolated(objects["x"], objects["y"], rmin)
peaks = objects["peak"]
ok = (
(peaks < pmax)
& (peaks > pmin * objects["thresh"])
& (objects["a"] < emax * objects["b"])
)
dfwhms = np.array(
[
i
for i in range(len(results))
if ok[i] and len(results[i]) == 1
], dtype=int
)
# pick the brightest. 'dfwhms' are the
# indices of the selected targets for
# FWHM measurement
if len(dfwhms):
fluxes = objects["flux"][dfwhms]
isort = np.argsort(fluxes)[::-1]
dfwhms = dfwhms[isort[:nmax]]
# buffer for storing the FWHMs, including NaNs
# for the ones thet are skipped
fwhms = np.zeros_like(peaks, dtype=np.float32)
betas = np.zeros_like(peaks, dtype=np.float32)
nfevs = np.zeros_like(peaks, dtype=np.int32)
for i, (x, y, peak, fwhm) in enumerate(
zip(
objects["x"],
objects["y"],
peaks,
objects["fwhm"],
)
):
# fit FWHMs of selected targets
if i in dfwhms:
try:
# extract fit Window
fwind = wind.window(
x - rmin, x + rmin, y - rmin, y + rmin
)
# fit profile
ofwhm = fwhm
obeta = beta
(
(sky, height, x, y, fwhm, beta),
epars,
(
wfit,
X,
Y,
sigma,
chisq,
nok,
nrej,
npar,
nfev,
),
) = hcam.fitting.fitMoffat(
fwind,
None,
peak,
x,
y,
fwhm,
2.0,
False,
beta,
beta_max,
False,
readout,
gain,
rej,
1,
max_nfev,
)
fwhms[i] = fwhm
betas[i] = beta
nfevs[i] = nfev
# keep value of beta for next round under control
beta = min(beta_max, max(beta_min, beta))
except hcam.HipercamError as err:
emessages.append(
" >> Targ {:d}: fit failed ***: {!s}".format(
i, err
)
)
fwhms[i] = np.nan
betas[i] = np.nan
nfevs[i] = 0
nfail += 1
else:
# skip this one
fwhms[i] = np.nan
betas[i] = np.nan
nfevs[i] = 0
# tack on frame number & window name
frames = (nf + first) * np.ones(
len(objects), dtype=np.int32
)
wnams = np.array(
len(objects) * [wnam], dtype="U{:d}".format(lsmax)
)
objects = append_fields(
objects,
("ffwhm", "beta", "nfev", "nframe", "wnam"),
(fwhms, betas, nfevs, frames, wnams),
)
# save the objects and the
objs.append(objects)
dofwhms.append(dfwhms + nobj)
print(dfwhms,nobj,dofwhms)
nobj += len(objects)
except hcam.HipercamError:
# window may have no overlap with xlo, xhi
# ylo, yhi
pass
if len(objs):
# Plot targets found
# concatenate results of all Windows
objs = np.concatenate(objs)
dofwhms = np.concatenate(dofwhms)
# set to the correct panel and then plot CCD
ix = (nc % nx) + 1
iy = nc // nx + 1
pgpanl(ix, iy)
vmin, vmax = hcam.pgp.pCcd(
ccd,
iset,
plo,
phi,
ilo,
ihi,
xlo=xlo,
xhi=xhi,
ylo=ylo,
yhi=yhi,
)
pgsci(core.CNAMS["red"])
As, Bs, Thetas, Xs, Ys = (
objs["a"],
objs["b"],
objs["theta"],
objs["x"],
objs["y"],
)
for a, b, theta0, x, y in zip(As, Bs, Thetas, Xs, Ys):
xs = x + 3 * a * np.cos(thetas + theta0)
ys = y + 3 * b * np.sin(thetas + theta0)
pgline(xs, ys)
pgsci(core.CNAMS["green"])
for xmin, xmax, ymin, ymax in zip(
objs["xmin"], objs["xmax"], objs["ymin"], objs["ymax"]
):
xs = [xmin, xmax, xmax, xmin, xmin]
ys = [ymin, ymin, ymax, ymax, ymin]
pgline(xs, ys)
pgsci(core.CNAMS["blue"])
if len(dofwhms):
print(dofwhms)
for x, y in zip(objs["x"][dofwhms], objs["y"][dofwhms]):
xs = [x - rmin, x + rmin, x + rmin, x - rmin, x - rmin]
ys = [y - rmin, y - rmin, y + rmin, y + rmin, y - rmin]
pgline(xs, ys)
# remove some less useful fields to save a
# bit more space prior to saving to disk
objs = remove_field_names(
objs,
(
"x2",
"y2",
"xy",
"cxx",
"cxy",
"cyy",
"hfd",
"cflux",
"cpeak",
"xcpeak",
"ycpeak",
"xmin",
"xmax",
"ymin",
"ymax",
),
)
# save to disk
if nhdu[nc]:
fout[nhdu[nc]].append(objs)
else:
fout.write(objs)
nhdu[nc] = nc + 1
# accumulate string of image scalings
if nc:
message += ", ccd {:s}: {:.1f}, {:.1f}, exp: {:.4f}".format(
cnam, vmin, vmax, mccd.head["EXPTIME"]
)
else:
message += "ccd {:s}: {:.1f}, {:.1f}, exp: {:.4f}".format(
cnam, vmin, vmax, mccd.head["EXPTIME"]
)
pgebuf()
# end of CCD display loop
print(message)
for emessage in emessages:
print(emessage)
if pause > 0.0:
# pause between frames
time.sleep(pause)
# update the frame number
n += 1
print("Number of failed fits =", nfail)
def setdefect(args=None):
"""``setdefect mccd defect ccd [width height] nx msub [cmap] ffield
(auto (hlo hhi severity)) hsbox iset (ilo ihi | plo phi)``
Interactive definition of CCD defects. This is a matplotlib-based routine
allowing you to define defects using the cursor. Various routines allow
you to overplot defects, above all |rtplot| / |nrtplot|. Point-like and
line-like defects of two different colours (orange for moderate, red for
severe can be marked). Hot pixels can be marked by their count rate as
measured in dark frames.
You should apply this in stages using bias frames and flat fields
to set the point and line defects, and a darkframe from |makedark|
to set hot pixels. The latter can be set semi-automatically by
flagging pixels between pre-set rate values.
Parameters:
mccd : str
name of an MCCD file, as produced by e.g. 'grab'
defect : str
the name of a defect file. If it exists it will be read so that
defects can be added to it. If it does not exist, it will be
created on exiting the routine. The defect files are in a fairly
readable / editiable text format
ccd : str
CCD(s) to plot, '0' for all. If not '0' then '1', '2' or even '3 4'
are possible inputs (without the quotes). '3 4' will plot CCD '3' and
CCD '4'. If you want to plot more than one CCD, then you will be
prompted for the number of panels in the X direction. This parameter
will not be prompted if there is only one CCD in the file.
width : float [hidden]
plot width (inches). Set = 0 to let the program choose.
height : float [hidden]
plot height (inches). Set = 0 to let the program choose. BOTH width
AND height must be non-zero to have any effect
nx : int
number of panels across to display, prompted if more than one CCD is
to be plotted.
msub : bool
True/False to subtract median from each window before scaling
cmap : str [hidden]
The colour map to use. "Greys" is the usual; "Greys_r" reverses it.
There are many others; typing an incorrect one will give a list. "none"
for matplotlib default.
ffield : bool
If True, all defects will be assumed to be flat-field or poor
charge transfer defects as opposed to hot pixels. The latter
should be set from dark frames, and have a different impact
than the first two types in that they are worst for faint
targets. Hot pixels and flat-field defects are shown with the
same colours for moderate and severe, but different symbols
(filled circles for flat-field defects, stars for hot
pixels). If you say "no" in order to add hot pixels, the line
defect option is not available.
auto : bool [if ffield==False]
Hot pixels can be set automatically if wanted. If so then a
few extra parameters are needed. This only makes sense if you
are feeding setdefect with a dark frame produced by
|makedark| so that the intensity levels mean something.
hlo : float [if auto]
lower limit to flag as a hot pixel as a count rate per second
hhi : float [if auto]
upper limit to flag as a hot pixel as a count rate per second
severity : str [if auto]
'm' for moderate, 'n' for nasty. Controls colour used to plot
the hot pixel; it will be labelled with a rate as well.
hsbox : int
half-width in binned pixels of stats box as offset from central pixel
hsbox = 1 gives a 3x3 box; hsbox = 2 gives 5x5 etc. This is used by
the "show" option when setting defects.
iset : str [single character]
determines how the intensities are determined. There are three
options: 'a' for automatic simply scales from the minimum to the
maximum value found on a per CCD basis. 'd' for direct just takes two
numbers from the user. 'p' for percentile dtermines levels based upon
percentiles determined from the entire CCD on a per CCD bais.
ilo : float [if iset=='d']
lower intensity level
ihi : float [if iset=='d']
upper intensity level
plo : float [if iset=='p']
lower percentile level
phi : float [if iset=='p']
upper percentile level
There are a few conveniences to make setdefect easier:
1. The plot is initialised in pan mode whereby you can move around and
scale using the left and right mouse buttons.
2. All input is accomplished with the keyboard; the mouse buttons are
only for navigating the image.
3. The label input can be switched between sequential numerical,
single- and multi-character input ('linput').
Various standard keyboard shortcuts (e.g. 's' to save) are disabled as
they just confuse things and are of limited use in setdefect in any case.
Some aspects of the usage of matplotlib in setdefect are tricky. It is
possible that particular 'backends' will cause problems. I have tested
this with Qt4Agg, Qt5agg and GTK3Agg. One aspect is the cursor icon in pan
mode is a rather indistinct hand where one can't tell what is being
pointed at. I have therefore suppressed this, but only for the tested
backends. Others would need require further investigation.
NB At the end of this routine, it re-orders the defects so that the severe
ones follows the moderates. This helps emphasize the severe ones over the
moderates when running rtplot.
"""
command, args = utils.script_args(args)
# get input section
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("mccd", Cline.LOCAL, Cline.PROMPT)
cl.register("defect", Cline.LOCAL, Cline.PROMPT)
cl.register("ccd", Cline.LOCAL, Cline.PROMPT)
cl.register("width", Cline.LOCAL, Cline.HIDE)
cl.register("height", Cline.LOCAL, Cline.HIDE)
cl.register("nx", Cline.LOCAL, Cline.PROMPT)
cl.register("msub", Cline.GLOBAL, Cline.PROMPT)
cl.register("cmap", Cline.LOCAL, Cline.HIDE)
cl.register("ffield", Cline.LOCAL, Cline.PROMPT)
cl.register("auto", Cline.LOCAL, Cline.PROMPT)
cl.register("hlo", Cline.LOCAL, Cline.PROMPT)
cl.register("hhi", Cline.LOCAL, Cline.PROMPT)
cl.register("severity", Cline.LOCAL, Cline.PROMPT)
cl.register("hsbox", Cline.GLOBAL, Cline.HIDE)
cl.register("iset", Cline.GLOBAL, Cline.PROMPT)
cl.register("ilo", Cline.GLOBAL, Cline.PROMPT)
cl.register("ihi", Cline.GLOBAL, Cline.PROMPT)
cl.register("plo", Cline.GLOBAL, Cline.PROMPT)
cl.register("phi", Cline.GLOBAL, Cline.PROMPT)
# get inputs
mccd = cl.get_value("mccd", "frame to plot",
cline.Fname("hcam", hcam.HCAM))
mccd = hcam.MCCD.read(mccd)
dfct = cl.get_value(
"defect",
"name of defect file",
cline.Fname("defect", hcam.DFCT, exist=False),
)
if os.path.exists(dfct):
# read in old defects
mccd_dfct = defect.MccdDefect.read(dfct)
print("Loaded existing file = {:s}".format(dfct))
else:
# create empty container
mccd_dfct = defect.MccdDefect()
print("No file called {:s} exists; "
"will create from scratch".format(dfct))
max_ccd = len(mccd)
if max_ccd > 1:
ccd = cl.get_value("ccd", "CCD(s) to plot [0 for all]", "0")
if ccd == "0":
ccds = list(mccd.keys())
else:
ccds = ccd.split()
else:
ccds = list(mccd.keys())
width = cl.get_value("width", "plot width (inches)", 0.0)
height = cl.get_value("height", "plot height (inches)", 0.0)
nxdef = cl.get_default("nx", 3)
# number of panels in X
if len(ccds) > 1:
nxdef = min(len(ccds), nxdef)
cl.set_default("nx", nxdef)
nx = cl.get_value("nx", "number of panels in X", 3, 1)
else:
nx = 1
# define the display intensities
msub = cl.get_value("msub", "subtract median from each window?", True)
cmap = cl.get_value("cmap",
"colour map to use ['none' for mpl default]",
"Greys")
cmap = None if cmap == "none" else cmap
ffield = cl.get_value("ffield",
"flat field defects? [else hot pixels]", True)
if not ffield:
auto = cl.get_value("auto", "automatic assignation of hot pixels?",
False)
hlo = cl.get_value("hlo", "lower rate threshold", 10., 0.)
hhi = cl.get_value("hhi", "upper rate threshold", max(hlo, 100.),
hlo)
severity = cl.get_value("severity",
"severity of hot pixels",
'm',
lvals=('m', 'n'))
if severity == "m":
severity = defect.Severity.MODERATE
elif severity == "n":
severity = defect.Severity.SEVERE
else:
raise hcam.HipercamError(
f"Severity = {severity} not recognised; programming error")
else:
auto = False
hsbox = cl.get_value("hsbox",
"half-width of stats box (binned pixels)", 2, 1)
iset = cl.get_value(
"iset",
"set intensity a(utomatically),"
" d(irectly) or with p(ercentiles)?",
"a",
lvals=["a", "A", "d", "D", "p", "P"],
)
iset = iset.lower()
plo, phi = 5, 95
ilo, ihi = 0, 1000
if iset == "d":
ilo = cl.get_value("ilo", "lower intensity limit", 0.0)
ihi = cl.get_value("ihi", "upper intensity limit", 1000.0)
elif iset == "p":
plo = cl.get_value("plo", "lower intensity limit percentile", 5.0,
0.0, 100.0)
phi = cl.get_value("phi", "upper intensity limit percentile", 95.0,
0.0, 100.0)
nxmax, nymax = 0, 0
for cnam in ccds:
nxmax = max(nxmax, mccd[cnam].nxtot)
nymax = max(nymax, mccd[cnam].nytot)
# might be worth trying to improve this at some point
xlo, xhi, ylo, yhi = 0, nxmax + 1, 0, nymax + 1
# Inputs obtained.
# re-configure keyboard shortcuts to avoid otherwise confusing behaviour
# quit_all does not seem to be universal, hence the try/except
try:
mpl.rcParams["keymap.back"] = ""
mpl.rcParams["keymap.forward"] = ""
mpl.rcParams["keymap.fullscreen"] = ""
mpl.rcParams["keymap.grid"] = ""
mpl.rcParams["keymap.home"] = ""
mpl.rcParams["keymap.pan"] = ""
mpl.rcParams["keymap.quit"] = ""
mpl.rcParams["keymap.save"] = ""
mpl.rcParams["keymap.pan"] = ""
mpl.rcParams["keymap.save"] = ""
mpl.rcParams["keymap.xscale"] = ""
mpl.rcParams["keymap.yscale"] = ""
mpl.rcParams["keymap.zoom"] = ""
except KeyError:
pass
# start plot
nccd = len(ccds)
ny = nccd // nx if nccd % nx == 0 else nccd // nx + 1
if width > 0 and height > 0:
fig = plt.figure(figsize=(width, height))
else:
fig = plt.figure()
# get the navigation toolbar. Go straight into pan mode
# where we want to stay.
toolbar = fig.canvas.manager.toolbar
toolbar.pan()
# we need to store some stuff
ax = None
cnams = {}
anams = {}
# this is a container for all the objects used to plot Defects to
# allow deletion.
pobjs = {}
for n, cnam in enumerate(ccds):
ccd = mccd[cnam]
if ax is None:
axes = ax = fig.add_subplot(ny, nx, n + 1)
axes.set_aspect("equal", adjustable="box")
axes.set_xlim(xlo, xhi)
axes.set_ylim(ylo, yhi)
else:
axes = fig.add_subplot(ny, nx, n + 1, sharex=ax, sharey=ax)
axes.set_aspect("equal", adjustable="box")
if msub:
# subtract median from each window
for wind in ccd.values():
wind -= wind.median()
hcam.mpl.pCcd(axes,
ccd,
iset,
plo,
phi,
ilo,
ihi,
f"CCD {cnam}",
cmap=cmap)
# keep track of the CCDs associated with each axes
cnams[axes] = cnam
# and axes associated with each CCD
anams[cnam] = axes
if auto:
# add hot pixels if auto option enabled
if cnam in mccd_dfct:
cdfct = mccd_dfct[cnam]
else:
cdfct = mccd_dfct[cnam] = defect.CcdDefect()
# Calculate the largest number, label the new Defect
# with one more
high = 0
for ndfct in cdfct:
try:
high = max(high, int(ndfct))
except ValueError:
pass
high += 1
dexpose = mccd.head["EXPTIME"]
clo = hlo * dexpose
chi = hhi * dexpose
for wnam, wind in ccd.items():
flag = (wind.data > clo) & (wind.data < chi)
xs = wind.x(np.arange(wind.nx))
ys = wind.y(np.arange(wind.ny))
Xs, Ys = np.meshgrid(xs, ys)
for x, y, c in zip(Xs[flag], Ys[flag], wind.data[flag]):
cdfct[str(high)] = defect.Hot(severity, x, y, c / dexpose)
high += 1
if cnam in mccd_dfct:
# plot any pre-existing Defects, keeping track of
# the plot objects
pobjs[cnam] = hcam.mpl.pCcdDefect(axes, mccd_dfct[cnam])
else:
# add in an empty CcdDefect for any CCD not already present
mccd_dfct[cnam] = defect.CcdDefect()
# and an empty container for any new plot objects
pobjs[cnam] = {}
# create the Defect picker (see below for class def)
picker = PickDefect(mccd, cnams, anams, toolbar, fig, mccd_dfct, dfct,
ffield, hsbox, pobjs)
picker.action_prompt(False)
# squeeze space a bit
plt.subplots_adjust(wspace=0.1, hspace=0.1)
# finally show stuff ....
plt.show()
def target_lookup(target,
ra_tel=None,
dec_tel=None,
dist=5.5,
url='http://simbad.u-strasbg.fr'):
"""
Tries to determine coordinates of a target in
one of three ways:
1) First it attempts to find coordinates in simbad by running
a query ID with target (unless target is None or ''). If the
target name starts with "STD_" (GTC flux standard), that is
stripped off first.
2) If (1) fails, try to find coordinates within the name in the
form JHHMMSS.S[+/-]DDMMSS [can be more precise, but not less]
4) If (1) and (2) fail or target == '' or None, then attempt a coordinate
search in simbad within a limit of dist arcmin (default equals half
diagonal of ULTRASPEC. It needs a unique match in this case. The
search position comes from ra_tel and dec_tel
If successful, it comes back with (name, ra, dec), the matched name and position.
ra, dec are in decimal form (hours, degrees). If it fails, it returns ('UNDEF','UNDEF','UNDEF').
This is for use by the logging scripts.
"""
err_msgs = []
if url.endswith('/'):
url += 'simbad/sim-script'
else:
url += '/simbad/sim-script'
RESPC = re.compile('\s+')
if target is not None and target != '':
if target.startswith('STD_'):
target = target[4:]
# simbad query script
query = f"""set limit 2
format object form1 "Target: %IDLIST(1) | %COO(A D;ICRS)"
query id {target}"""
payload = {'submit': 'submit script', 'script': query}
rep = requests.post(url, data=payload)
data = False
found = False
fail = False
nres = 0
for line in rep.text.split('\n'):
if line.startswith('::data::'):
data = True
if line.startswith('::error::'):
err_msgs.append(
f' Error occurred querying simbad for {target}')
fail = True
break
if data:
if line.startswith('Target:'):
nres += 1
if nres > 1:
err_msgs.append(
f'More than one target returned when querying simbad for {target}'
)
fail = True
break
name, coords = line[7:].split(' | ')
found = True
if found and not fail:
# OK we have found one, but we are still not done --
# some SIMBAD lookups are no good
name = name.strip()
name = re.sub(RESPC, ' ', name)
try:
ra, dec, syst = str2radec(coords)
return (name, ra, dec)
except:
err_msgs.append(
f'Matched "{target}" with "{name}", but failed to translate position = {coords}'
)
# simbad ID lookup failed, so now try to read position from
# coordinates
REPOS = re.compile(
r'J(\d\d)(\d\d)(\d\d\.\d(?:\d*)?)([+-])(\d\d)(\d\d)(\d\d(?:\.\d*)?)$'
)
m = REPOS.search(target)
if m:
rah, ram, ras, decsgn, decd, decm, decs = m.group(
1, 2, 3, 4, 5, 6, 7)
rah, ram, ras, decd, decm, decs = int(rah), int(ram), float(
ras), int(decd), int(decm), float(decs)
if rah > 23 or ram > 59 or ras >= 60. or decd > 89 or decm > 59 or decs >= 60.:
err_msgs.append(
f'{target} matched the positional regular expression but the numbers were out of range'
)
else:
name = re.sub(RESPC, ' ', target)
ra = rah + ram / 60 + ras / 3600
dec = decd + decm / 60 + decs / 3600
dec = dec if decsgn == '+' else -dec
return (name, ra, dec)
err_msgs.append(f'Could not extract position from {target}')
if ra_tel is not None and dec_tel is not None and dist is not None:
# Second
query = f"""set limit 2
format object form1 "Target: %IDLIST(1) | %COO(A D;ICRS)"
query coo {ra_tel} {dec_tel} radius={dist}m"""
payload = {'submit': 'submit script', 'script': query}
rep = requests.post(url, data=payload)
data = False
found = False
fail = False
nres = 0
for line in rep.text.split('\n'):
if line.startswith('::data::'):
data = True
if line.startswith('::error::'):
err_msgs.append(
f'Error occured during simbad coordinate query for {target}'
)
fail = True
break
if data:
if line.startswith('Target:'):
nres += 1
if nres > 1:
err_msgs.append(
f'More than one target returned when querying simbad for telescope position = {ra_tel} {dec_tel} ({target})'
)
fail = True
break
name, coords = line[7:].split(' | ')
found = True
if found and not fail:
# OK we have found one, but we are still not done --
# some SIMBAD lookups are no good
name = name.strip()
name = re.sub(RESPC, ' ', name)
try:
ra, dec, syst = str2radec(coords)
ra_str = dec2sexg(ra, False, 2)
dec_str = dec2sexg(dec, True, 1)
print(
f' Matched telescope position = {coords} {dec} with "{name}", position = {ra_str} {dec_str}'
)
return (name, ra, dec)
except:
err_msgs.append(
f'Matched telescope position = {ra} {dec} ({target}) with "{name}", but failed to translate position = {coords}'
)
raise hcam.HipercamError('\n'.join(err_msgs))
def extractFlux(cnam, ccd, rccd, read, gain, ccdwin, rfile, store):
"""This extracts the flux of all apertures of a given CCD.
The steps are (1) aperture resizing, (2) sky background estimation, (3)
flux extraction. The apertures are assumed to be correctly positioned.
It returns the results as a dictionary keyed on the aperture label. Each
entry returns a list:
[x, ex, y, ey, fwhm, efwhm, beta, ebeta, counts, countse, sky, esky,
nsky, nrej, flag]
flag = bitmask. See hipercam.core to see all the options which are
referred to by name in the code e.g. ALL_OK. The various flags can
signal that there no sky pixels (NO_SKY), the sky aperture was off
the edge of the window (SKY_AT_EDGE), etc.
This code::
>> bset = flag & TARGET_SATURATED
determines whether the data saturation flag is set for example.
Arguments::
cnam : string
CCD identifier label
ccd : CCD
the debiassed, flat-fielded CCD.
rccd : CCD
corresponding raw CCD, used to work out whether data are
saturated in target aperture.
read : CCD
readnoise divided by the flat-field
gain : CCD
gain multiplied by the flat field
ccdwin : dictionary of strings
the Window label corresponding to each Aperture
rfile : Rfile
reduce file configuration parameters
store : dict of dicts
see moveApers for what this contains.
"""
# initialise flag
flag = hcam.ALL_OK
ccdaper = rfile.aper[cnam]
# get the control parameters
(
resize,
extype,
r1fac,
r1min,
r1max,
r2fac,
r2min,
r2max,
r3fac,
r3min,
r3max,
) = rfile["extraction"][cnam]
results = {}
mfwhm = store["mfwhm"]
if resize == "variable" or extype == "optimal":
if mfwhm <= 0:
# return early here as there is nothing we can do.
print((" *** WARNING: CCD {:s}: no measured FWHM to re-size"
" apertures or carry out optimal extraction; no"
" extraction possible").format(cnam))
# set flag to indicate no FWHM
flag = hcam.NO_FWHM
for apnam, aper in ccdaper.items():
info = store[apnam]
results[apnam] = {
"x": aper.x,
"xe": info["xe"],
"y": aper.y,
"ye": info["ye"],
"fwhm": info["fwhm"],
"fwhme": info["fwhme"],
"beta": info["beta"],
"betae": info["betae"],
"counts": NaN,
"countse": NaN,
"sky": NaN,
"skye": NaN,
"nsky": 0,
"nrej": 0,
"flag": flag,
"cmax": 0,
}
return results
else:
# Re-size the apertures
for aper in ccdaper.values():
aper.rtarg = max(r1min, min(r1max, r1fac * mfwhm))
aper.rsky1 = max(r2min, min(r2max, r2fac * mfwhm))
aper.rsky2 = max(r3min, min(r3max, r3fac * mfwhm))
elif resize == "fixed":
# just apply the max and min limits
for aper in ccdaper.values():
aper.rtarg = max(r1min, min(r1max, aper.rtarg))
aper.rsky1 = max(r2min, min(r2max, aper.rsky1))
aper.rsky2 = max(r3min, min(r3max, aper.rsky2))
else:
raise hcam.HipercamError(
"CCD {:s}: 'variable' and 'fixed' are the only"
" aperture resizing options".format(cnam))
# apertures have been positioned in moveApers and now re-sized. Finally
# we can extract something.
for apnam, aper in ccdaper.items():
# initialise flag
flag = hcam.ALL_OK
# extract Windows relevant for this aperture
wnam = ccdwin[apnam]
wdata = ccd[wnam]
wread = read[wnam]
wgain = gain[wnam]
wraw = rccd[wnam]
# extract sub-windows that include all of the pixels that could
# conceivably affect the aperture. We have to check that 'extra'
# apertures do not go beyond rsky2 which would normally be expected to
# be the default outer radius
rmax = aper.rsky2
for xoff, yoff in aper.extra:
rmax = max(rmax, np.sqrt(xoff**2 + yoff**2) + aper.rtarg)
# this is the region of interest
x1, x2, y1, y2 = (
aper.x - aper.rsky2 - wdata.xbin,
aper.x + aper.rsky2 + wdata.xbin,
aper.y - aper.rsky2 - wdata.ybin,
aper.y + aper.rsky2 + wdata.ybin,
)
try:
# extract sub-Windows
swdata = wdata.window(x1, x2, y1, y2)
swread = wread.window(x1, x2, y1, y2)
swgain = wgain.window(x1, x2, y1, y2)
swraw = wraw.window(x1, x2, y1, y2)
# some checks for possible problems. bitmask flags will be set if
# they are encountered.
xlo, xhi, ylo, yhi = swdata.extent()
if (xlo > aper.x - aper.rsky2 or xhi < aper.x + aper.rsky2
or ylo > aper.y - aper.rsky2 or yhi < aper.y + aper.rsky2):
# the sky aperture overlaps the edge of the window
flag |= hcam.SKY_AT_EDGE
if (xlo > aper.x - aper.rtarg or xhi < aper.x + aper.rtarg
or ylo > aper.y - aper.rtarg or yhi < aper.y + aper.rtarg):
# the target aperture overlaps the edge of the window
flag |= hcam.TARGET_AT_EDGE
for xoff, yoff in aper.extra:
rout = np.sqrt(xoff**2 + yoff**2) + aper.rtarg
if (xlo > aper.x - rout or xhi < aper.x + rout
or ylo > aper.y - rout or yhi < aper.y + rout):
# an extra target aperture overlaps the edge of the window
flag |= hcam.TARGET_AT_EDGE
# compute X, Y arrays over the sub-window relative to the centre
# of the aperture and the distance squared from the centre (Rsq)
# to save a little effort.
x = swdata.x(np.arange(swdata.nx)) - aper.x
y = swdata.y(np.arange(swdata.ny)) - aper.y
X, Y = np.meshgrid(x, y)
Rsq = X**2 + Y**2
# squared aperture radii for comparison
R1sq, R2sq, R3sq = aper.rtarg**2, aper.rsky1**2, aper.rsky2**2
# sky selection, accounting for masks and extra (which we assume
# acts like a sky mask as well)
sok = (Rsq > R2sq) & (Rsq < R3sq)
for xoff, yoff, radius in aper.mask:
sok &= (X - xoff)**2 + (Y - yoff)**2 > radius**2
for xoff, yoff in aper.extra:
sok &= (X - xoff)**2 + (Y - yoff)**2 > R1sq
# sky data
dsky = swdata.data[sok]
if len(dsky):
# we have some sky!
if rfile["sky"]["method"] == "clipped":
# clipped mean. Take average, compute RMS,
# reject pixels > thresh*rms from the mean.
# repeat until no new pixels are rejected.
thresh = rfile["sky"]["thresh"]
ok = np.ones_like(dsky, dtype=bool)
nrej = 1
while nrej:
slevel = dsky[ok].mean()
srms = dsky[ok].std()
nold = len(dsky[ok])
ok = ok & (np.abs(dsky - slevel) < thresh * srms)
nrej = nold - len(dsky[ok])
if len(dsky[ok]) == 0:
# no sky. will still return the flux in
# the aperture but set flag and the sky
# uncertainty to -1
flag |= hcam.NO_SKY
slevel = 0
serror = -1
nrej = 0
break
nsky = len(dsky[ok])
if nsky:
# serror -- error in the sky estimate.
serror = srms / np.sqrt(nsky)
else:
# 'median' goes with 'photon'
slevel = dsky.median()
nsky = len(dsky)
nrej = 0
# read*gain/flat and flat over sky region
dread = swread.data[sok]
dgain = swgain.data[sok]
serror = np.sqrt(
(dread**2 + np.max(0, dsky) / dgain).sum() / nsky**2)
else:
# no sky. will still return the flux in the aperture but set
# flag and the sky uncertainty to -1
flag |= hcam.NO_SKY
slevel = NaN
serror = NaN
nsky = 0
nrej = 0
# size of a pixel which is used to taper pixels as they approach
# the edge of the aperture to reduce pixellation noise
size = np.sqrt(wdata.xbin * wdata.ybin)
# target selection, accounting for extra apertures and allowing
# pixels to contribute if their centres are as far as size/2 beyond
# the edge of the circle (but with a tapered weight)
dok = Rsq < (aper.rtarg + size / 2.0)**2
if not dok.any():
# check there are some valid pixels
flag |= hcam.NO_DATA
raise hcam.HipercamError("no valid pixels in aperture")
# check for saturation and nonlinearity
cmax = int(swraw.data[dok].max())
if cnam in rfile.warn:
if cmax >= rfile.warn[cnam]["saturation"]:
flag |= hcam.TARGET_SATURATED
if cmax >= rfile.warn[cnam]["nonlinear"]:
flag |= hcam.TARGET_NONLINEAR
else:
warnings.warn(
"CCD {:s} has no nonlinearity or saturation levels set".
format(cnam))
# Pixellation amelioration:
#
# The weight of a pixel is set to 1 at the most and then linearly
# declines as it approaches the edge of the aperture. The scale over
# which it declines is set by 'size', the geometric mean of the
# binning factors. A pixel with its centre exactly on the edge
# gets a weight of 0.5.
wgt = np.minimum(
1,
np.maximum(0, (aper.rtarg + size / 2.0 - np.sqrt(Rsq)) / size))
for xoff, yoff in aper.extra:
rsq = (X - xoff)**2 + (Y - yoff)**2
dok |= rsq < (aper.rtarg + size / 2.0)**2
wg = np.minimum(
1,
np.maximum(0, (aper.rtarg + size / 2.0 - np.sqrt(rsq)) /
size))
wgt = np.maximum(wgt, wg)
# the values needed to extract the flux.
dtarg = swdata.data[dok]
dread = swread.data[dok]
dgain = swgain.data[dok]
wtarg = wgt[dok]
# 'override' to indicate we want to override the readout noise.
if nsky and rfile["sky"]["error"] == "variance":
# from sky variance
rd = srms
override = True
else:
rd = dread
override = False
# count above sky
diff = dtarg - slevel
if extype == "normal" or extype == "optimal":
if extype == "optimal":
# optimal extraction. Need the profile
warnings.warn("Transmission plot is not reliable"
" with optimal extraction")
mbeta = store["mbeta"]
if mbeta > 0.0:
prof = fitting.moffat(
X[dok],
Y[dok],
0.0,
1.0,
0.0,
0.0,
mfwhm,
mbeta,
wdata.xbin,
wdata.ybin,
rfile["apertures"]["fit_ndiv"],
)
else:
prof = fitting.gaussian(
X[dok],
Y[dok],
0.0,
1.0,
0.0,
0.0,
mfwhm,
wdata.xbin,
wdata.ybin,
rfile["apertures"]["fit_ndiv"],
)
# multiply weights by the profile
wtarg *= prof
# now extract
counts = (wtarg * diff).sum()
if override:
# in this case, the "readout noise" includes the component
# due to the sky background so we use the sky-subtracted
# counts above sky for the object contribution.
var = (wtarg**2 *
(rd**2 + np.maximum(0, diff) / dgain)).sum()
else:
# in this case we are using the true readout noise and we
# just use the data (which should be debiassed) without
# removal of the sky.
var = (wtarg**2 *
(rd**2 + np.maximum(0, dtarg) / dgain)).sum()
if serror > 0:
# add in factor due to uncertainty in sky estimate
var += (wtarg.sum() * serror)**2
countse = np.sqrt(var)
else:
raise hcam.HipercamError(
"extraction type = {:s} not recognised".format(extype))
info = store[apnam]
results[apnam] = {
"x": aper.x,
"xe": info["xe"],
"y": aper.y,
"ye": info["ye"],
"fwhm": info["fwhm"],
"fwhme": info["fwhme"],
"beta": info["beta"],
"betae": info["betae"],
"counts": counts,
"countse": countse,
"sky": slevel,
"skye": serror,
"nsky": nsky,
"nrej": nrej,
"flag": flag,
"cmax": cmax,
}
except hcam.HipercamError as err:
info = store[apnam]
flag |= hcam.NO_EXTRACTION
results[apnam] = {
"x": aper.x,
"xe": info["xe"],
"y": aper.y,
"ye": info["ye"],
"fwhm": info["fwhm"],
"fwhme": info["fwhme"],
"beta": info["beta"],
"betae": info["betae"],
"counts": NaN,
"countse": NaN,
"sky": NaN,
"skye": NaN,
"nsky": 0,
"nrej": 0,
"flag": flag,
"cmax": 0,
}
# finally, we are done
return results
def makefringe(args=None):
"""``makefringe [source] (run first last [twait tmax] | flist) bias
dark flat fpair ([nhalf]) ccd fwhm [clobber] output``
Averages a set of images to make a frame for defringing (referred
to elsewhere as a "fringe map").
At long wavelengths, CCDs suffer what is known as "fringing",
which in terms of structure looks something like the coloured
patterns you see when there is a thin layer of oil on water. Both
patterns are caused by interference. In the case of CCDs this is
caused by sky emission lines and is variable in strength, and
additive in nature, but seems mostly fairly fixed in position.
De-fringing in |hiper| is implemented according to the method
suggested by Snodgrass & Carry (2013Msngr.152...14S). The idea is
to create a set of pairs of points marking the peak and troughs of
fringes. The differences in intensity between these point pairs in
the data to be corrected is compared with the differences for the
same pairs in a reference "fringe map" by taking their
ratios. Many pairs are used so that the median can be taken to
eliminate ratios affected by celestial targets or cosmic rays. The
median ratio is used to scale the reference data and thereby
subtract the fringes. ``makefringe`` is used to make the reference
fringe map. It works as follows: given a single run or a list of
files, it reads them all in, optionally debiases, dark-subtracts
and flat-fields them, calculates a median count level of each one
which is subtracted from each CCD individually. The pixel-by-pixel
median of all frames is then calculated. It is also possible (and
a good idea) to apply the fringe pair ratio measurements to scale
each frame before combining them, which allows frames of similar
pattern but varying amplitude to be combined. Finally, the output
can be smoothed because fringes are usually a medium- to
large-scale pattern.
Parameters:
source : string [hidden]
Data source, five options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : ULTRACAM server
| 'ul' : local ULTRACAM .xml/.dat files
| 'hf' : list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab' or
converted from foreign data formats.
run : string [if source ends 's' or 'l']
run number to access, e.g. 'run034'
first : int [if source ends 's' or 'l']
exposure number to start from. 1 = first frame ('0' is
not supported).
last : int [if source ends 's' or 'l']
last exposure number must be >= first or 0 for the whole lot.
twait : float [if source ends 's' or 'l'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [if source ends 's' or 'l'; hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
flist : string [if source ends 'f']
name of file list. Assumed that bias, flat-fielding etc have been applied to
these already.
bias : str
Name of bias frame to subtract, 'none' to ignore.
dark : str
Name of dark frame to subtract, 'none' to ignore.
flat : str
Name of flat field, 'none' to ignore.
fpair : str
FringePair file (see setfringe), or 'none' to ignore. If specified
if will be used to scale the frames prior to combining.
nhalf : int [if fpair is not 'none']
When calculating the differences for fringe measurement,
a region extending +/-nhalf binned pixels will be used when
measuring the amplitudes. Basically helps the stats.
ccd : str
CCD(s) to process, '0' for all, '1 3' for '1' and '3' only, etc.
fwhm : float
FWHM (unbinned pixels) of gaussian smoothing to apply to the output.
0 to ignore. Should be << smallest scale between fringes. Probably
no more than 4.
clobber : bool [hidden]
clobber any pre-existing output files
output : str
output file. Set by default to match the last part of "run"
(but it will have a different extension so they won't
clash). The default is
.. Note::
This routine writes the files returned by 'grab' to
automatically generated files, typically in .hipercam/tmp, to
avoid polluting the working directory. These are removed at
the end, but may not be if you ctrl-C. You should check
.hipercam/tmp for redundant files every so often
"""
command, args = utils.script_args(args)
# get the inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("source", Cline.GLOBAL, Cline.HIDE)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("last", Cline.LOCAL, Cline.PROMPT)
cl.register("twait", Cline.LOCAL, Cline.HIDE)
cl.register("tmax", Cline.LOCAL, Cline.HIDE)
cl.register("flist", Cline.LOCAL, Cline.PROMPT)
cl.register("bias", Cline.LOCAL, Cline.PROMPT)
cl.register("dark", Cline.LOCAL, Cline.PROMPT)
cl.register("flat", Cline.LOCAL, Cline.PROMPT)
cl.register("fpair", Cline.LOCAL, Cline.PROMPT)
cl.register("nhalf", Cline.LOCAL, Cline.PROMPT)
cl.register("ccd", Cline.LOCAL, Cline.PROMPT)
cl.register("fwhm", Cline.LOCAL, Cline.PROMPT)
cl.register("clobber", Cline.LOCAL, Cline.HIDE)
cl.register("output", Cline.LOCAL, Cline.PROMPT)
# get inputs
source = cl.get_value(
"source",
"data source [hs, hl, us, ul, hf]",
"hl",
lvals=("hs", "hl", "us", "ul", "hf"),
)
# set a flag
server_or_local = source.endswith("s") or source.endswith("l")
if server_or_local:
resource = cl.get_value("run", "run name", "run005")
root = os.path.basename(resource)
cl.set_default('output', cline.Fname(root, hcam.HCAM))
first = cl.get_value("first", "first frame to average", 1, 1)
last = cl.get_value("last", "last frame to average (0 for all)",
first, 0)
twait = cl.get_value("twait",
"time to wait for a new frame [secs]", 1.0,
0.0)
tmax = cl.get_value("tmax",
"maximum time to wait for a new frame [secs]",
10.0, 0.0)
else:
resource = cl.get_value("flist", "file list",
cline.Fname("files.lis", hcam.LIST))
first = 1
# bias frame (if any)
bias = cl.get_value(
"bias",
"bias frame ['none' to ignore]",
cline.Fname("bias", hcam.HCAM),
ignore="none",
)
# dark frame (if any)
dark = cl.get_value(
"dark",
"dark frame ['none' to ignore]",
cline.Fname("dark", hcam.HCAM),
ignore="none",
)
# flat field (if any)
flat = cl.get_value(
"flat",
"flat field frame ['none' to ignore]",
cline.Fname("flat", hcam.HCAM),
ignore="none",
)
fpair = cl.get_value("fpair",
"fringe pair file",
cline.Fname("fringe", hcam.FRNG),
ignore="none")
if fpair is not None:
nhalf = cl.get_value("nhalf",
"half-size of fringe measurement region", 2,
0)
ccdinf = spooler.get_ccd_pars(source, resource)
if len(ccdinf) > 1:
ccd = cl.get_value("ccd", "CCD(s) to process [0 for all]", "0")
if ccd == "0":
ccds = list(ccdinf.keys())
else:
ccds = ccd.split()
else:
ccds = list(ccdinf.keys())
fwhm = cl.get_value(
"fwhm", "FWHM of gaussian smoothing to apply"
" to final result [unbinned pixels, 0 to ignore]", 4., 0.)
clobber = cl.get_value("clobber",
"clobber any pre-existing files on output",
False)
output = cl.get_value(
"output",
"median output fringe frame",
cline.Fname("hcam", hcam.HCAM,
cline.Fname.NEW if clobber else cline.Fname.NOCLOBBER),
)
# inputs done with.
if server_or_local or bias is not None or dark is not None or flat is not None:
print("\nCalling 'grab' ...")
args = [None, "prompt", source, "yes", resource]
if server_or_local:
args += [str(first), str(last), str(twait), str(tmax)]
args += [
"no",
"none" if bias is None else bias,
"none" if dark is None else dark,
"none" if flat is None else flat,
"none",
"f32",
]
resource = hcam.scripts.grab(args)
# at this point 'resource' is a list of files, no matter the input
# method.
fnames = []
with CleanUp(resource, fnames, server_or_local or bias is not None
or dark is not None) as cleanup:
# Read all the files to determine mean levels (after bias
# subtraction) save the bias-subtracted, flat-fielded,
# mean-level subtracted and normalised results to temporary
# files
print("Reading all files in to determine their mean levels")
bframe, fframe, dframe, fpairobj = None, None, None, None
medians = {}
for cnam in ccds:
medians[cnam] = {}
# We might have a load of temporaries from grab, but we are about to
# make some more to save the bias-subtracted normalised versions.
tdir = utils.temp_dir()
mtype = []
fref = {}
with spooler.HcamListSpool(resource) as spool:
for mccd in spool:
if fpair is not None:
# prepare fringepairs
if fpairobj is None:
fpairobj = fringe.MccdFringePair.read(fpair)
fpairobj = fpairobj.crop(mccd, nhalf)
# here we determine the median levels, subtract them
# then (optionally) normalise by the ratio of fringe
# amplitudes
# generate the name to save to automatically
fd, fname = tempfile.mkstemp(suffix=hcam.HCAM, dir=tdir)
for cnam in ccds:
# its unlikely that fringe frames would be taken
# with skips, but you never know. Eliminate them
# from consideration now.
ccd = mccd[cnam]
if ccd.is_data():
cmedian = mccd[cnam].median()
medians[cnam][fname] = cmedian
mccd[cnam] -= cmedian
if cnam not in fref:
# save reference
fref[cnam] = mccd[cnam]
elif fpair is not None:
fscale = fpairobj[cnam].scale(
mccd[cnam], fref[cnam], nhalf)
mccd[cnam] *= fscale
print(f'{cnam}, fscale = {fscale:.3f}')
# write to disk, save the name, close the filehandle
fnames.append(fname)
mccd.write(fname)
os.close(fd)
# a bit of progress info
print(f"Saved {', '.join(mtype)} frame to {fname}")
# now we go through CCD by CCD, using the first as a template
# for the window names in which we will also store the results.
template = hcam.MCCD.read(fnames[0])
# Now process each file CCD by CCD to reduce the memory
# footprint
for cnam in ccds:
# Read files into memory, insisting that they
# all have the same set of CCDs
print(f"\nLoading all CCDs labelled '{cnam}'")
accds = []
with spooler.HcamListSpool(fnames, cnam) as spool:
mean = None
for ccd in spool:
if ccd.is_data():
accds.append(ccd)
if len(accds) == 0:
raise hcam.HipercamError(
f"Found no valid examples of CCD {cnam}"
f" in {fnames}")
else:
print(f"Loaded {len(accds)} CCDs")
# Median combine
for wnam, wind in template[cnam].items():
# build list of all data arrays
arrs = [ccd[wnam].data for ccd in accds]
# convert to 3D numpy array
arr3d = np.stack(arrs)
wind.data = np.median(arr3d, axis=0)
# Add history
template[cnam].head.add_history(
f"Median combine of {len(accds)} images")
print("Computed and stored their pixel-by-pixel median")
# Remove any CCDs not included to avoid impression of having done
# something to them
dcnams = []
for cnam in template.keys():
if cnam not in ccds:
dcnams.append(cnam)
for cnam in dcnams:
del template[cnam]
if fwhm > 0:
# apply gaussian smoothing
print(f'Smoothing with FWHM = {fwhm}')
sigma = fwhm / np.sqrt(8 * np.log(2))
kern = Gaussian2DKernel(sigma)
for ccd in template.values():
for wind in ccd.values():
wind.data = convolve_fft(wind.data, kern, "fill",
wind.median())
# write out
template.write(output, clobber)
print(f"\nFinal result written to {output}")
print('makefringe finished')
def genred(args=None):
"""``genred apfile rfile comment bias flat dark linear inst [ncpu extendx
ccd location smoothfwhm method beta betamax fwhm fwhmmin searchwidth thresh
hminref hminnrf rfac rmin rmax sinner souter scale psfgfac psfwidth psfpostweak]``
Generates a reduce file as needed by |reduce| or |psf_reduce|. You give it
the name of an aperture file and a few other parameters and it will write
out a reduce file which you can then refine by hand. A few simplifying
assumptions are made, e.g. that the target is called '1'; see below for more.
This script effectively defines the format of reduce files. The script attempts
above all to generate a self-consistent reduce file. e.g. if there are no
apertures in CCD 5, it does not attempt to plot any corresponding light
curves.
To avoid excessive prompting, |genred| has many hidden parameters. The
very first time you use it on a run, specify ``prompt`` on the command line
to see all of these. They are chosen to be the parameters most likely to
vary with telescope or conditions; many others are left at default values
and require editing to change. If you find yourself repeatedly editing a
parameter, let me know and I will add it to this routine.
Parameters:
apfile : string
the input aperture file created using |setaper| (default extension
.ape). This will be read for the targets. The main target will be
assumed to have been called '1', the main comparison '2'. If there
is a '3' it will be plotted relative to '2'; all others will be
ignored for plotting purposes. Target '2' will be used to define
the position and transmission plots for one CCD only [user
definable]. Target '1' will be used for the seeing plot unless it
is linked when target '2' will be used instead.
rfile : string
the output reduce file created using |setaper|. This will be read
for the targets. The main target will be assumed to have been
called '1', the main comparison '2'. If there is a '3' it will be
plotted relative to '2'; all others will be ignored for plotting
purposes.
comment : string
comment to add near the top of the reduce file. Obvious things to say
are the target name and the name of the observer for instance.
bias : string
Name of bias frame; 'none' to ignore.
flat : string
Name of flat field frame; 'none' to ignore.
dark : string
Name of dark frame; 'none' to ignore.
linear : string
light curve plot linear (else magnitudes)
inst : string
the instrument (needed to set nonlinearity and saturation levels for
warning purposes. Possible options listed.
ncpu : int [hidden]
some increase in speed can be obtained by running the
reduction in parallel. This parameter is the number of CPUs
to use. The parellisation is over the CCDs, and there is no
point having ncpu greater than the number of CCDs, but it should
ideally equal the number of CCDs. There is no point using this
for single CCD reduction.
ngroup : int [hidden, if ncpu > 1]
to reduce parallelisation overheads, this parameter means that ngroup
frames are read before being split up for the parallisation step is applied.
extendx : float [hidden]
how many minutes to extend light curve plot by at a time
ccd : string [hidden]
label of the (single) CCD used for the position plot
location : string [hidden]
whether to reposition apertures or leave them fixed.
toffset : int [hidden]
integer offset to subtract from the MJD times in order to
reduce round-off. i.e. rather 55678.123456789, if you
specified toffset=55600, you would reduce the round-off
error by a factor ~1000. With typical MJDs, round off is
around the 0.5 microsecond level. If you want to improve on
that, set an appropriate offset value. It is set by default
to 0, i.e. it will be 0 unless you explicitly set it
otherwise. The value used is recorded in the log
file. toffset must be >= 0.
smoothfwhm : float [hidden]
FWHM to use for smoothing during initial search [binned pixels]
fft : bool [hidden]
whether or not to use FFTs when carrying out the convolution
operation used in the initial search. No effect on results,
but could be faster for large values of smoothfwhm.
method : string
profile fitting method. 'g' for gaussian, 'm' for moffat
beta : float [hidden]
default Moffat exponent to use to start fitting
betamax : float [hidden]
maximum Moffat exponent to pass on to subsequent fits. Prevents
it wandering to silly values which can happen.
fwhm : float [hidden]
the default FWHM to use when fitting [unbinned pixels].
fwhmmin : float [hidden]
the default FWHM to use when fitting [unbinned pixels].
searchwidth : int [hidden]
half width in (binned) pixels for the target searches
fitwidth : int [hidden]
half width in (binned) pixels for the profile fits
maxshift : float [hidden]
maximum shift of non-reference targets relative to the initial
positions derived from the reference targets. The reference targets
should give good initial positions, thus this can be set to quite a
small value to improve robustness against duff positions, caused
e.g. by cosmic rays or field crowding in periods of bad seeing. Use
linked apertures for more difficult cases still.
thresh : float [hidden]
RMS rejection threshold for profile fits.
hminref : float [hidden]
minimum peak height for a fit to a reference aperture to be
accepted. This applies to the peak height in the *smoothed* image
used during the initial search as well as the peak height after
profile fitting. It is the check on the smoothed image that is more
consistent since the seeing-limited peak height can be highly
variable, and the more stringent since smoothing reduces the peak
height by seeing**2/(seeing**2+smoothfwhm**2) where 'seeing' is the
FWHM of the seeing in binned pixels. If smoothfwhm is chosen to be
larger than seeing is likely to be, this makes the peak heights
more consistent so that the thresholding is better behaved. But, it
means you should use a value lower than you might guess from the
images. A peak of height `h` in a smoothed image will contain
about (seeing**2+smoothfwhm**2)*h counts in total (but fewer if
binned).
hminnrf : float [hidden]
minimum peak height for a fit to a non-reference aperture to be
accepted. This applies above all to the smoothed peak height as for
hminref.
alpha : float [hidden]
amount by which non-reference apertures are corrected relative to
their expected positions when reference apertures are enabled. The
idea is that the positions of non-reference relative to reference
apertures should vary little, so rather than simply re-positioning
independently every frame, one might want to build in a bit of past
history. This can be done by setting alpha small. If alpha = 1,
then that simply returns to fully independent positioning for each
frame.
diff : float [hidden]
maximum difference in the shifts of reference apertures, when more
than one has been defined on a CCD. If exceeded, no extraction is
performed. This is to guard against perturbations of the reference
apertures by meteors, cosmic rays and the like. [unbinned pixels]
rfac : float [hidden]
target aperture radius relative to the FWHM for 'variable' aperture
photometry. Usual values 1.5 to 2.5.
rmin : float [hidden]
minimum target aperture radius [unbinned pixels]
rmax : float [hidden]
maximum target aperture radius [unbinned pixels]
sinner : float [hidden]
inner sky aperture radius [unbinned pixels]
souter : float [hidden]
outer sky aperture radius [unbinned pixels]
readout : float | string [hidden]
readout noise, RMS ADU. Can either be a single value or an hcm file.
gain : float [hidden]
gain, electrons per ADU. Can either be a single value or an hcm file.
scale : float [hidden]
image scale in arcsec/pixel
psfgfac : float [hidden]
multiple of FWHM used to group objects together for PSF fitting
psfwidth : int [hidden]
half-width of box used to extract data around objects for PSF fitting
psfpostweak : string [hidden]
During PSF fitting, either hold positions at aperture location ('fixed'),
or fit as part of PSF model ('variable')
"""
# print(my_version)
command, args = utils.script_args(args)
with Cline('HIPERCAM_ENV', '.hipercam', command, args) as cl:
# register parameters
cl.register('apfile', Cline.LOCAL, Cline.PROMPT)
cl.register('rfile', Cline.GLOBAL, Cline.PROMPT)
cl.register('comment', Cline.LOCAL, Cline.PROMPT)
cl.register('bias', Cline.LOCAL, Cline.PROMPT)
cl.register('flat', Cline.LOCAL, Cline.PROMPT)
cl.register('dark', Cline.LOCAL, Cline.PROMPT)
cl.register('linear', Cline.LOCAL, Cline.PROMPT)
cl.register('inst', Cline.LOCAL, Cline.HIDE)
cl.register('ncpu', Cline.LOCAL, Cline.HIDE)
cl.register('ngroup', Cline.LOCAL, Cline.HIDE)
cl.register('extendx', Cline.LOCAL, Cline.HIDE)
cl.register('ccd', Cline.LOCAL, Cline.HIDE)
cl.register('location', Cline.LOCAL, Cline.HIDE)
cl.register('toffset', Cline.LOCAL, Cline.HIDE)
cl.register('smoothfwhm', Cline.LOCAL, Cline.HIDE)
cl.register('fft', Cline.LOCAL, Cline.HIDE)
cl.register('beta', Cline.LOCAL, Cline.HIDE)
cl.register('betamax', Cline.LOCAL, Cline.HIDE)
cl.register('fwhm', Cline.LOCAL, Cline.HIDE)
cl.register('method', Cline.LOCAL, Cline.HIDE)
cl.register('fwhmmin', Cline.LOCAL, Cline.HIDE)
cl.register('searchwidth', Cline.LOCAL, Cline.HIDE)
cl.register('fitwidth', Cline.LOCAL, Cline.HIDE)
cl.register('maxshift', Cline.LOCAL, Cline.HIDE)
cl.register('thresh', Cline.LOCAL, Cline.HIDE)
cl.register('hminref', Cline.LOCAL, Cline.HIDE)
cl.register('hminnrf', Cline.LOCAL, Cline.HIDE)
cl.register('alpha', Cline.LOCAL, Cline.HIDE)
cl.register('diff', Cline.LOCAL, Cline.HIDE)
cl.register('rfac', Cline.LOCAL, Cline.HIDE)
cl.register('rmin', Cline.LOCAL, Cline.HIDE)
cl.register('rmax', Cline.LOCAL, Cline.HIDE)
cl.register('sinner', Cline.LOCAL, Cline.HIDE)
cl.register('souter', Cline.LOCAL, Cline.HIDE)
cl.register('readout', Cline.LOCAL, Cline.HIDE)
cl.register('gain', Cline.LOCAL, Cline.HIDE)
cl.register('scale', Cline.LOCAL, Cline.HIDE)
cl.register('psfgfac', Cline.LOCAL, Cline.HIDE)
cl.register('psfwidth', Cline.LOCAL, Cline.HIDE)
cl.register('psfpostweak', Cline.LOCAL, Cline.HIDE)
# get inputs
# the aperture file
apfile = cl.get_value('apfile', 'aperture input file',
cline.Fname('aper.ape', hcam.APER))
# Read the aperture file
aper = hcam.MccdAper.read(apfile)
# the reduce file
rfile = cl.get_value(
'rfile', 'reduce output file',
cline.Fname('reduce.red', hcam.RED, cline.Fname.NEW))
# user comment string
comment = cl.get_value(
'comment', 'user comment to add [<cr>'
' for newline to get multilines]', '')
if comment == '':
comment = '# There was no user comment\n'
else:
comment_lines = comment.split('<cr>')
comment = '# User comment:\n#\n# ' + '\n# '.join(comment_lines)
# ones you might quite often want to change
# bias frame
bias = cl.get_value('bias',
"bias frame ['none' to ignore]",
cline.Fname('bias', hcam.HCAM),
ignore='none')
bias = '' if bias is None else bias
# flat field frame
flat = cl.get_value('flat',
"flat field frame ['none' to ignore]",
cline.Fname('flat', hcam.HCAM),
ignore='none')
flat = '' if flat is None else flat
# dark frame
dark = cl.get_value('dark',
"dark field frame ['none' to ignore]",
cline.Fname('dark', hcam.HCAM),
ignore='none')
dark = '' if dark is None else dark
inst = cl.get_value(
'inst',
'instrument (hipercam, ultracam, ultraspec)',
'hipercam',
lvals=['hipercam', 'ultracam', 'ultraspec', 'ignore'])
if inst == 'hipercam':
warn_levels = """# Warning levels for instrument = HiPERCAM
warn = 1 50000 64000
warn = 2 50000 64000
warn = 3 50000 64000
warn = 4 50000 64000
warn = 5 50000 64000
"""
maxcpu = 5
elif inst == 'ultracam':
warn_levels = """# Warning levels for instrument = ULTRACAM
warn = 1 28000 64000
warn = 2 28000 64000
warn = 3 50000 64000
"""
maxcpu = 3
elif inst == 'ultraspec':
warn_levels = """# Warning levels for instrument = ULTRASPEC
warn = 1 60000 64000
"""
maxcpu = 1
else:
warn_levels = """# No warning levels have been set!!"""
maxcpu = 20
if maxcpu > 1:
ncpu = cl.get_value('ncpu',
'number of CPUs to use (<= number of CCDs)', 1,
1, maxcpu)
else:
ncpu = 1
if ncpu > 1:
ngroup = cl.get_value(
'ngroup',
'number of frames per group to reduce parallelisation overheads',
1, 1)
else:
ngroup = 1
linear = cl.get_value('linear', 'linear light curve plot?', False)
linear = 'yes' if linear else 'no'
# hidden parameters
extendx = cl.get_value('extendx',
'how much to extend light curve plot [mins]',
10., 0.01)
ccd = cl.get_value('ccd',
'label for the CCD used for the position plot', '2')
if ccd not in aper:
raise hcam.HipercamError(
'CCD {:s} not found in aperture file {:s}'.format(ccd, apfile))
# hidden parameters
location = cl.get_value('location',
'aperture location, f(ixed) or v(ariable)',
'v',
lvals=['f', 'v'])
location = 'variable' if location == 'v' else 'fixed'
comm_seeing = '#' if location == 'fixed' else ''
comm_position = '#' if location == 'fixed' else ''
cl.set_default('toffset', 0)
toffset = cl.get_value(
'toffset',
'offset to subtract from the MJD times (to reduce round-off) [days]',
0, 0)
smooth_fwhm = cl.get_value('smoothfwhm',
'search smoothing FWHM [binned pixels]', 6.,
3.)
smooth_fft = cl.get_value('fft', 'use FFT when smoothing', False)
profile_type = cl.get_value(
'method',
'profile fit method, g(aussian) or m(offat)',
'g',
lvals=['g', 'm'])
profile_type = 'gaussian' if profile_type == 'g' else 'moffat'
beta = cl.get_value('beta', 'starting value of beta', 4., 3.)
beta_max = cl.get_value(
'betamax', 'maximum value of beta to start consecutive fits', 20.,
beta)
fwhm = cl.get_value('fwhm', 'starting FWHM, unbinned pixels', 5., 1.5)
fwhm_min = cl.get_value('fwhmmin', 'minimum FWHM, unbinned pixels',
1.5, 0.)
search_half_width = cl.get_value(
'searchwidth', 'half width for initial searches, unbinned pixels',
11, 3)
fit_half_width = cl.get_value(
'fitwidth', 'half width for profile fits, unbinned pixels', 21, 5)
fit_max_shift = cl.get_value(
'maxshift', 'maximum non-reference shift, unbinned pixels', 15.,
0.)
thresh = cl.get_value('thresh',
'RMS rejection threshold for fits (sigma)', 5.,
2.)
height_min_ref = cl.get_value(
'hminref',
'minimum peak height for a fit to reference aperture [counts]',
10., 0.)
height_min_nrf = cl.get_value(
'hminnrf',
'minimum peak height for a fit to non-reference aperture [counts]',
5., 0.)
fit_alpha = cl.get_value(
'alpha',
'non-reference aperture fractional shift parameter (range: (0,1])',
1., 1.e-5, 1.)
fit_diff = cl.get_value(
'diff', 'maximum differential reference aperture shift', 2., 1.e-5)
rfac = cl.get_value('rfac', 'target aperture scale factor', 1.8, 1.0)
rmin = cl.get_value(
'rmin', 'minimum target aperture radius [unbinned pixels]', 6., 1.)
rmax = cl.get_value(
'rmax', 'maximum target aperture radius [unbinned pixels]', 30.,
rmin)
sinner = cl.get_value('sinner',
'inner sky aperture radius [unbinned pixels]',
30., rmax)
souter = cl.get_value('souter',
'outer sky aperture radius [unbinned pixels]',
50., sinner + 1)
readout = cl.get_value('readout',
'readout noise, RMS ADU (float or file name)',
'4.5')
gain = cl.get_value('gain',
'gain, electrons/ADU, (float or file name)', '1.1')
scale = cl.get_value('scale', 'image scale [arcsec/unbinned pixel]',
0.3, 0.001)
psfgfac = cl.get_value(
'psfgfac',
'multiple of FWHM used to group objects for PSF fitting', 3, 0.1)
psfwidth = cl.get_value('psfwidth',
'half width for PSF fits, unbinned pixels', 15,
5)
psfpostweak = cl.get_value(
'psfpostweak',
'locations during PSF fitting stage, f(ixed) or v(ariable)',
'f',
lvals=['f', 'v'])
psfpostweak = 'variable' if psfpostweak == 'v' else 'fixed'
################################################################
#
# all the inputs have now been obtained. Get on with doing stuff
# Generate the extraction lines. Note that the aperture location
# parameter maps into the same names as the aperture re-size
# parameter
extraction = ''
for cnam in aper:
extraction += ('{:s} = {:s} normal'
' {:.2f} {:.1f} {:.1f}'
' 2.5 {:.1f} {:.1f}'
' 3.0 {:.1f} {:.1f}\n').format(cnam, location, rfac,
rmin, rmax, sinner,
sinner, souter, souter)
# standard colours for CCDs
if inst == 'hipercam':
CCD_COLS = {
'1': 'purple',
'2': 'green',
'3': 'orange',
'4': 'red',
'5': 'darkred'
}
elif inst == 'ultracam':
CCD_COLS = {
'1': 'red',
'2': 'green',
'3': 'blue',
}
elif inst == 'ultraspec':
CCD_COLS = {
'1': 'green',
}
# Generate the light curve plot lines
light_plot = ''
no_light = True
for cnam in aper:
ccdaper = aper[cnam]
if '1' in ccdaper and '2' in ccdaper:
light_plot += (
'plot = {:s} 1 2 0 1 {:10s} ! '
' # ccd, targ, comp, off, fac, dcol, ecol\n').format(
cnam, CCD_COLS[cnam])
no_light = False
elif '1' in ccdaper and '2' not in ccdaper:
light_plot += (
'plot = {:s} 1 ! 0 1 {:10s} ! '
' # ccd, targ, comp, off, fac, dcol, ecol\n').format(
cnam, CCD_COLS[cnam])
no_light = False
if '2' in ccdaper and '3' in ccdaper:
light_plot += (
'plot = {:s} 3 2 0 1 {:10s} ! '
' # ccd, targ, domp, off, fac, dcol, ecol\n').format(
cnam, CCD_COLS[cnam])
no_light = False
if no_light:
raise hcam.HipercamError(
'Found no targets for light curve plots in any CCD; cannot make light curve plot'
)
# Generate the position plot lines
position_plot = ''
ccdaper = aper[ccd]
no_position = True
if '2' in ccdaper:
position_plot += ('{:s}plot = {:s} 2 {:10s} ! '
' # ccd, targ, dcol, ecol\n').format(
comm_position, ccd, CCD_COLS[ccd])
no_position = False
elif '3' in ccdaper:
position_plot += ('{:s}plot = {:s} 3 {:10s} ! '
' # ccd, targ, dcol, ecol\n').format(
comm_position, ccd, CCD_COLS[ccd])
no_position = False
elif '1' in ccdaper:
position_plot += ('{:s}plot = {:s} 1 {:10s} ! '
' # ccd, targ, dcol, ecol\n').format(
comm_position, ccd, CCD_COLS[ccd])
no_position = False
if no_position:
raise hcam.HipercamError(
'Targets 1, 2 and 3 not found in '
'CCD = {:s}; cannot make position plot'.format(ccd))
# Generate the transmission plot lines
transmission_plot = ''
no_transmission = True
for cnam in aper:
ccdaper = aper[cnam]
if '2' in ccdaper:
transmission_plot += ('plot = {:s} 2 {:10s} ! '
' # ccd, targ, dcol, ecol\n').format(
cnam, CCD_COLS[cnam])
no_transmission = False
elif '3' in ccdaper:
transmission_plot += ('plot = {:s} 3 {:10s} ! '
' # ccd, targ, dcol, ecol\n').format(
cnam, CCD_COLS[cnam])
no_transmission = False
elif '1' in ccdaper:
transmission_plot += ('plot = {:s} 1 {:10s} ! '
' # ccd, targ, dcol, ecol\n').format(
cnam, CCD_COLS[cnam])
no_transmission = False
if no_transmission:
raise hcam.HipercamError('Targets 1, 2 and 3 not found in any CCDs;'
' cannot make transmission plot')
# Generate the seeing plot lines
seeing_plot = ''
no_seeing = True
for cnam in aper:
ccdaper = aper[cnam]
if '1' in ccdaper and not ccdaper['1'].linked:
seeing_plot += ('{:s}plot = {:s} 1 {:10s} ! '
' # ccd, targ, dcol, ecol\n').format(
comm_seeing, cnam, CCD_COLS[cnam])
no_seeing = False
elif '2' in ccdaper and not ccdaper['2'].linked:
seeing_plot += ('{:s}plot = {:s} 2 {:10s} ! '
' # ccd, targ, dcol, ecol\n').format(
comm_seeing, cnam, CCD_COLS[cnam])
no_seeing = False
elif '3' in ccdaper and not ccdaper['3'].linked:
seeing_plot += ('{:s}plot = {:s} 3 {:10s} ! '
' # ccd, targ, dcol, ecol\n').format(
comm_seeing, cnam, CCD_COLS[cnam])
no_seeing = False
if no_seeing:
raise hcam.HipercamError(
'Targets 1, 2 and 3 not found in any CCD'
' (or they are linked); cannot make seeing plot')
# monitor targets (whole lot by default)
targs = set()
for cnam in aper:
ccdaper = aper[cnam]
for targ in ccdaper:
targs.add(targ)
monitor = ''
for targ in sorted(targs):
monitor += ('{:s} = NO_EXTRACTION TARGET_SATURATED TARGET_AT_EDGE'
' TARGET_NONLINEAR NO_SKY NO_FWHM NO_DATA SKY_AT_EDGE\n'
).format(targ)
# time stamp
tstamp = strftime("%d %b %Y %H:%M:%S (UTC)", gmtime())
# finally write out the reduce file.
with open(rfile, 'w') as fout:
# write out file
fout.write(
TEMPLATE.format(version=hcam.REDUCE_FILE_VERSION,
apfile=apfile,
fwhm=fwhm,
fwhm_min=fwhm_min,
extraction=extraction,
bias=bias,
flat=flat,
dark=dark,
smooth_fwhm=smooth_fwhm,
linear=linear,
light_plot=light_plot,
position_plot=position_plot,
transmission_plot=transmission_plot,
seeing_plot=seeing_plot,
monitor=monitor,
comment=comment,
tstamp=tstamp,
hipercam_version=hipercam_version,
location=location,
comm_seeing=comm_seeing,
extendx=extendx,
comm_position=comm_position,
scale=scale,
warn_levels=warn_levels,
ncpu=ncpu,
ngroup=ngroup,
search_half_width=search_half_width,
fit_half_width=fit_half_width,
profile_type=profile_type,
height_min_ref=height_min_ref,
height_min_nrf=height_min_nrf,
beta=beta,
beta_max=beta_max,
thresh=thresh,
readout=readout,
gain=gain,
fit_max_shift=fit_max_shift,
fit_alpha=fit_alpha,
fit_diff=fit_diff,
psfgfac=psfgfac,
psfpostweak=psfpostweak,
psfwidth=psfwidth,
toffset=toffset,
smooth_fft='yes' if smooth_fft else 'no'))
print('Reduce file written to {:s}'.format(rfile))
def genred(args=None):
"""``genred apfile rfile bias dark flat fmap fpair seeing (sworst
binfac) template (inst (nccd (ccd) nonlin sat scale readout
gain))``
Generates a reduce file as needed by |reduce| or |psf_reduce|. You
give it the name of an aperture file, calibration frames and a few
other parameters and it will write out a reduce file which you can
then refine by hand. The aim is to try to get a reduce file that
is self-consistent and works to get observing started as soon as
possible. The parameters are not necessarily the best.
It is assumed that the target is called '1', the main comparison '2'.
To avoid endless prompts, genred does not try to prompt for all
parameters even via hidden ones (a change from its previous
behaviour). However, if you find yourself always making the samed
edits to the file produced by genred, you may start wanting a way
to alter some of the otherwise fixed parameters. You can do so by
supplying a template file, which can just be from a previous run
of genred where such parameters can be altered by editing with a
standard text editor before running genred.
genred recognises some standard instrument telescope combinations
which it uses to set parameters such as the pixel scale and
readout noise, but if you choose 'other', you will be prompted for
these details. The main parameters it does prompt for are
calibration files, and it allows you to define a "seeing" which
then controls the setting of various profile fit parameters. This
can be ignored by setting = 0, in which case any template
parameters will be passed unchanged.
Parameters:
apfile : str
the input aperture file created using |setaper| (default
extension .ape). This will be read for the targets. The
main target will be assumed to have been called '1', the
main comparison '2'. If there is a '3' it will be plotted
relative to '2'; all others will be ignored for plotting
purposes. Target '2' will be used to define the position
and transmission plots for one CCD only [user
definable]. Target '1' will be used for the seeing plot
unless it is linked when target '2' will be used instead.
rfile : str
the output reduce file created by this script. The main
target will be assumed to have been called '1', the main
comparison '2'. If there is a '3' it will be plotted
relative to '2'; all others will be ignored for plotting
purposes.
bias : str
Name of bias frame; 'none' to ignore.
dark : str
Name of dark frame; 'none' to ignore.
flat : str
Name of flat field frame; 'none' to ignore.
fmap : str
Name of fringe frame; 'none' to ignore.
fpair : str [if fmap not 'none']
File defining pairs to measure fringe amplitudes.
seeing : float
estimate of seeing which will be used to define several of
the profile fitting parameters. Enter 0 to ignore and use
defaults from the template (or genred if no template)
instead.
sworst : float [if seeing > 0]
worst seeing expected during run. Expands the search and
fitting boxes which can cause difficulties if they are
too small. It also sets the maximum target aperture radius
which will set to ~ 1.8*sworst, converted to pixels, and the
sky annulus radii. It has to be at least 2*seeing. Always
remember that you can override the automated settings using
a pre-edited template and seeing = 0.
binfac : int [if seeing > 0]
binning factor. e.g. 4 if using 4x4. Needed to optimise some
profile parameters.
template : str
Reduce file to use as a template for any parameters not set
by genred. This allows one to tweak settings by running
genred without a template, then modifying the resultant
file and using it as a template. 'none' to ignore, and then
these values will be set by genred by default. Some will be
modified anyway (e.g. the calibration file names), and if
seeing > 0, then several of the profile fitting paramaters
will be adapted to the value given along with the
instrument parameters. genred will try to keep as many of
the 'extraction', 'position', 'transmission', 'light' etc
lines associated with particular CCDs in this case without
modification, and will pass readout, saturation levels etc
unchanged from the template.
inst : str [if template == 'none']
the instrument. 'hipercam-gtc', 'ultracam-ntt', 'ultraspec-tnt',
'ultracam-wht', 'ultracam-vlt', or 'other'. Sets pixel scale and
standard readnoise params.
nccd : int [if inst == 'other']
number of CCDs.
ccd : int [if nccd > 1]
the CCD to use for position plots
nonlin : list(float) [if inst == 'other']
values for warning of non linearity
sat : list(float) [if inst == 'other']
values for warning of saturation
scale : float [if inst == 'other']
image scale in arcsec/pixel
readout : float | string [if inst == 'other']
readout noise, RMS ADU. Can either be a single value or an
hcm file.
gain : float | string [if inst == 'other']
gain, electrons per ADU. Can either be a single value or an
hcm file.
"""
command, args = utils.script_args(args)
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("apfile", Cline.LOCAL, Cline.PROMPT)
cl.register("rfile", Cline.GLOBAL, Cline.PROMPT)
cl.register("comment", Cline.LOCAL, Cline.PROMPT)
cl.register("bias", Cline.LOCAL, Cline.PROMPT)
cl.register("dark", Cline.LOCAL, Cline.PROMPT)
cl.register("flat", Cline.LOCAL, Cline.PROMPT)
cl.register("fmap", Cline.LOCAL, Cline.PROMPT)
cl.register("fpair", Cline.LOCAL, Cline.PROMPT)
cl.register("seeing", Cline.LOCAL, Cline.PROMPT)
cl.register("sworst", Cline.LOCAL, Cline.PROMPT)
cl.register("binfac", Cline.LOCAL, Cline.PROMPT)
cl.register("inst", Cline.LOCAL, Cline.PROMPT)
cl.register("nccd", Cline.LOCAL, Cline.PROMPT)
cl.register("ccd", Cline.LOCAL, Cline.PROMPT)
cl.register("nonlin", Cline.LOCAL, Cline.PROMPT)
cl.register("sat", Cline.LOCAL, Cline.PROMPT)
cl.register("scale", Cline.LOCAL, Cline.PROMPT)
cl.register("readout", Cline.LOCAL, Cline.PROMPT)
cl.register("gain", Cline.LOCAL, Cline.PROMPT)
cl.register("template", Cline.LOCAL, Cline.PROMPT)
# get inputs
# the aperture file
apfile = cl.get_value(
"apfile", "aperture input file", cline.Fname("aper.ape", hcam.APER)
)
# Read the aperture file
aper = hcam.MccdAper.read(apfile)
# the reduce file
root = os.path.splitext(os.path.basename(apfile))[0]
cl.set_default("rfile", cline.Fname(root, hcam.RED, cline.Fname.NEW))
rfile = cl.get_value(
"rfile", "reduce output file",
cline.Fname("reduce.red", hcam.RED, cline.Fname.NEW),
)
# calibrations
# bias frame
bias = cl.get_value(
"bias",
"bias frame ['none' to ignore]",
cline.Fname("bias", hcam.HCAM),
ignore="none",
)
# dark frame
dark = cl.get_value(
"dark",
"dark field frame ['none' to ignore]",
cline.Fname("dark", hcam.HCAM),
ignore="none",
)
# flat field frame
flat = cl.get_value(
"flat",
"flat field frame ['none' to ignore]",
cline.Fname("flat", hcam.HCAM),
ignore="none",
)
# fringe frame
fmap = cl.get_value(
"fmap",
"fringe frame ['none' to ignore]",
cline.Fname("fringe", hcam.HCAM),
ignore="none",
)
if fmap is not None:
fpair = cl.get_value(
"fpair", "fringe pair file",
cline.Fname("fpair", hcam.FRNG),
)
else:
fpair = None
seeing = cl.get_value(
"seeing",
"approximate seeing [arcsec, 0 to ignore]",
1.0, 0.
)
if seeing > 0.:
sworst = cl.get_value(
"sworst",
"worst expected seeing [arcsec]",
max(3., 2.5*seeing), 2.*seeing
)
binfac = cl.get_value(
"binfac", "binning factor",
1, 1
)
# template
template = cl.get_value(
"template",
"template reduce file ['none' to ignore]",
cline.Fname("template", hcam.RED),
ignore="none",
)
if template is None:
instrument = cl.get_value(
"inst",
"the instrument-telescope",
"hipercam-gtc",
lvals=[
"hipercam-gtc", "ultracam-ntt",
"ultracam-wht", "ultracam-vlt",
"ultraspec-tnt", "other"
],
)
if instrument.startswith("hipercam"):
warn_levels = """
warn = 1 50000 64000
warn = 2 50000 64000
warn = 3 50000 64000
warn = 4 50000 64000
warn = 5 50000 64000
"""
maxcpu = 5
skipbadt = False
scale = 0.081
readout = '4.2'
gain = '1.1'
nccd = 5
ccd = '2'
elif instrument.startswith("ultracam"):
warn_levels = """
warn = 1 28000 64000
warn = 2 28000 64000
warn = 3 50000 64000
"""
maxcpu = 3
skipbadt = True
readout = '2.8'
gain = '1.1'
nccd = 3
ccd = '2'
if instrument.endswith("ntt"):
scale = 0.35
elif instrument.endswith("wht"):
scale = 0.30
elif instrument.endswith("vlt"):
scale = 0.15
elif instrument.startswith("ultraspec"):
warn_levels = """
warn = 1 60000 64000
"""
maxcpu = 1
skipbadt = True
scale = 0.45
readout = '5.0'
gain = '0.8'
nccd = 1
ccd = '1'
elif instrument == "other":
# unrecognised instrument need extra stuff
nccd = cl.get_value("nccd", "how many CCDs?", 3, 1)
if nccd > 1:
ccd = cl.get_value("ccd", "which CCD for the position plot?", 1, 1, nccd)
ccd = str(ccd)
else:
ccd = '1'
# non-linearity warning levels
nonlin = cl.get_default("nonlin")
if nonlin is not None and len(nonlin) != nccd:
cl.set_default("nonlin", nccd * (50000.,))
nonlins = cl.get_value(
"nonlin",
"levels to indicate non-linear behaviour, 1 per CCD",
nccd*(50000.,)
)
# saturation warning levels
sat = cl.get_default("sat")
if sat is not None and len(sat) != nccd:
cl.set_default("sat", nccd * (62000.,))
sats = cl.get_value(
"sat",
"levels to indicate saturation, 1 per CCD",
nccd*(62000.,)
)
warn_levels = ""
for n, (nonlin,sat) in enumerate(zip(nonlins,sats)):
warn_levels += \
f"warn = {n+1} {nonlin} {sat}\n"
maxcpu = nccd
scale = cl.get_value(
"scale", "image scale [arcsec/unbinned pixel]",
0.3, 0.001
)
readout = cl.get_value(
"readout", "readout noise, RMS ADU (float or file name)", "4.5"
)
gain = cl.get_value(
"gain", "gain, electrons/ADU, (float or file name)", "1.1"
)
else:
raise hcam.HipercamError(
"Programming error: instrument unrecognised"
)
if ccd not in aper:
# make sure 'ccd' is in aper, even if our
# favoured one isn't
for cnam in aper:
ccd = cnam
break
################################################################
#
# all the inputs have now been obtained. Get on with doing stuff
if template is not None:
# read the template to define many unprompted values
# shortcut names define.
tvals = reduction.Rfile.read(template, False)
gsec = tvals['general']
instrument = gsec.get('instrument')
scale = gsec.get('scale')
warn_levels = ""
for value in gsec['warn']:
warn_levels += f'warn = {value}\n'
asec = tvals['apertures']
psfsec = tvals['psf_photom']
sksec = tvals['sky']
csec = tvals['calibration']
lcsec = tvals['lcplot']
lsec = tvals['light']
psec = tvals.get('position',None)
tsec = tvals.get('transmission',None)
ssec = tvals.get('seeing',None)
fsec = tvals.get('focal_mask',None)
else:
# define default values for unprompted params.
if seeing == 0.:
seeing = 1.5
sworst = 4.0
binfac = 1
# same name as read into template
tvals = {}
# general section
gsec = tvals['general'] = {}
gsec['instrument'] = instrument
gsec['scale'] = scale
gsec['ldevice'] = '1/xs'
gsec['lwidth'] = 0.
gsec['lheight'] = 0.
gsec['idevice'] = '2/xs'
gsec['iwidth'] = 0.
gsec['iheight'] = 0.
gsec['toffset'] = 0
gsec['skipbadt'] = 'yes'
gsec['ncpu'] = 1
gsec['ngroup'] = 1
# aperture section
asec = tvals['apertures'] = {}
asec['location'] = 'variable'
asec['search_smooth_fft'] = 'no'
asec['fit_method'] = 'moffat'
asec['fit_beta'] = 5.0
asec['fit_beta_max'] = 20.0
asec['fit_ndiv'] = 0
asec['fit_fwhm_fixed'] = 'no'
asec['fit_thresh'] = 8.
asec['fit_height_min_ref'] = 40.
asec['fit_height_min_nrf'] = 10.
asec['fit_alpha'] = 0.1
# sky
sksec = tvals['sky'] = {}
sksec['error'] = 'variance'
sksec['method'] = 'clipped'
sksec['thresh'] = 3.0
# calibration
csec = tvals['calibration'] = {}
csec['crop'] = 'yes'
csec['nhalf'] = 3
csec['rmin'] = -1.
csec['rmax'] = 2.
csec['readout'] = readout
csec['gain'] = gain
# PSF photom
psfsec = tvals['psf_photom'] = {}
psfsec['gfac'] = 3.0
psfsec['fit_half_width'] = 15.0
psfsec['positions'] = 'fixed'
# light curve
lcsec = tvals['lcplot'] = {}
lcsec['xrange'] = 0
lcsec['extend_x'] = 10.0
# light
lsec = tvals['light'] = {}
lsec['linear'] = 'no'
lsec['y_fixed'] = 'no'
lsec['y1'] = 0
lsec['y2'] = 0
lsec['extend_y'] = 0.1
# position
psec = tvals['position'] = {}
psec['height'] = 0.5
psec['x_fixed'] = 'no'
psec['x_min'] = -5.0
psec['x_max'] = +5.0
psec['y_fixed'] = 'no'
psec['y_min'] = -5.0
psec['y_max'] = +5.0
psec['extend_y'] = 0.2
# transmission
tsec = tvals['transmission'] = {}
tsec['height'] = 0.5
tsec['ymax'] = 110.
# seeing
ssec = tvals['seeing'] = {}
ssec['height'] = 0.5
ssec['ymax'] = 2.999
ssec['y_fixed'] = 'no'
ssec['extend_y'] = 0.2
fsec = tvals['focal_mask'] = {}
fsec['demask'] = 'no'
fsec['dthresh'] = 3.0
# apply seeing/instrument-related fixes...
# *always* gets run in the no template case
if seeing > 0.:
asec['search_half_width'] = max(5., sworst/scale)
asec['search_smooth_fwhm'] = max(2., seeing/scale/binfac)
asec['fwhm_min'] = 1.5*binfac
asec['fwhm'] = max(1.5*binfac, seeing/scale)
asec['fwhm_max'] = 3.*sworst/scale
asec['fit_max_shift'] = max(1., sworst/scale/4.)
asec['fit_half_width'] = max(5., 2*sworst/scale)
asec['fit_diff'] = max(1., sworst/scale/6.)
rfac = 1.8
ramin = 3*binfac
ramax = max(ramin,rfac*sworst/scale)
sinner = ramax
souter = 1.5*sinner
# standard colours for CCDs
if instrument.startswith("hipercam"):
CCD_COLS = {
"1": "blue",
"2": "green",
"3": "orange",
"4": "red",
"5": "darkred",
}
elif instrument.startswith("ultracam"):
CCD_COLS = {
"1": "red",
"2": "green",
"3": "blue",
}
elif instrument.startswith("ultraspec"):
CCD_COLS = {
"1": "green",
}
elif instrument == "other":
# 'other'
CCD_COLS = {
"1": "green",
"2": "blue",
"3": "red",
"4": "orange",
"5": "purple",
}
else:
raise hcam.HipercamError(
"Programming error: instrument unrecognised"
)
# Extraction lines
extraction = ""
if template is None:
# generate one for each CCD
for cnam in aper:
extraction += (
f"{cnam} = variable normal"
f" {rfac:.2f} {ramin:.1f} {ramax:.1f}"
f" 2.5 {sinner:.1f} {sinner:.1f}"
f" 3.0 {souter:.1f} {souter:.1f}\n"
)
else:
# pass through unchanged where possible if
# a template being used but check for consistency
esec = tvals['extraction']
for cnam, lst in esec.items():
if cnam in aper:
# unpack items
ltype,etype,rfac,ramin,ramax,sifac,simin,simax,sofac,somin,somax = lst
extraction += (
f"{cnam} = variable normal"
f" {rfac:.2f} {ramin:.1f} {ramax:.1f}"
f" {sifac:.2f} {simin:.1f} {simax:.1f}"
f" {sofac:.2f} {somin:.1f} {somax:.1f}\n"
)
else:
warnings.warn(
f'CCD {cnam} has an extraction line in {template} but no apertures in {apfile} and will be skipped'
)
# warn if there are apertures for a CCD but no extraction line
for cnam in aper:
if cnam not in esec:
warnings.warn(
f'CCD {cnam} has apertures defined in {apfile} but no extraction line in {template}'
)
# Generate the light curve plot lines
light_plot = ""
no_light = True
if template is None:
# Here we assume target in aperture 1, main comparison in aperture 2
for cnam in aper:
ccdaper = aper[cnam]
if "1" in ccdaper and "2" in ccdaper:
# favour 2 as the comparison
light_plot += (
f'plot = {cnam} 1 2 0 1 {CCD_COLS.get(cnam,"black")} ! '
" # ccd, targ, comp, off, fac, dcol, ecol\n"
)
no_light = False
elif "1" in ccdaper and "3" in ccdaper:
# but try 3 if need be
light_plot += (
f'plot = {cnam} 1 3 0 1 {CCD_COLS.get(cnam,"black")} ! '
" # ccd, targ, comp, off, fac, dcol, ecol\n"
)
no_light = False
elif "1":
# just plot 1 as last resort
light_plot += (
f'plot = {cnam} 1 ! 0 1 {CCD_COLS.get(cnam,"black")} ! '
" # ccd, targ, comp, off, fac, dcol, ecol\n"
)
no_light = False
else:
# template case: pass as many of the lines as we can but
# run checks of apertures
plots = tvals['light']['plot']
for pl in plots:
cnam, targ, comp = pl['ccd'], pl['targ'], pl['comp']
off, fac, dcol, ecol = pl['off'], pl['fac'], ct(pl['dcol']), ct(pl['ecol'])
if cnam in aper and targ in aper[cnam] and (comp == '!' or comp in aper[cnam]):
light_plot += (
f"plot = {cnam} {targ} {comp} {off} {fac} {dcol} {ecol}"
" # ccd, targ, comp, off, fac, dcol, ecol\n"
)
no_light = False
else:
warnings.warn(
"Light curve plot line:\n"
f"plot = {cnam} {targ} {comp} {off} {fac} {dcol} {ecol}\n"
f"from {template} is incompatible with {apfile} and has been skipped"
)
if no_light:
raise hcam.HipercamError(
"Found no targets for light curve plots in any CCD; cannot make light "
"curve plot. Needs at least aperture '1' defined and preferably '2' as well,"
"for at least one CCD"
)
# Generate the position plot lines. Favour the comparisons first, then the target.
position_plot = ""
no_position = True
if template is None:
ccdaper = aper[ccd]
for targ in ('2','3','1'):
if targ in ccdaper:
position_plot += (
f'plot = {ccd} {targ} '
f'{CCD_COLS.get(cnam,"black")} ! '
'# ccd, targ, dcol, ecol\n'
)
no_position = False
break
if no_position:
warnings.warn(
f"Targets 2, 1, or 3 not found in CCD = {ccd} in {apfile}"
)
elif psec is not None:
plots = psec['plot']
for pl in plots:
cnam, targ, dcol, ecol = pl['ccd'], pl['targ'], ct(pl['dcol']), ct(pl['ecol'])
if cnam in aper and targ in aper[cnam]:
position_plot += (
f'plot = {cnam} {targ} {dcol} {ecol} # ccd, targ, dcol, ecol\n'
)
no_position = False
else:
warnings.warn(
f"Position target {targ} / CCD {cnam} not found in {apfile}; will skip"
)
if no_position:
warnings.warn(f"No position plot will be made")
# Generate the transmission plot lines, again comparisons first,
# target only if desperate.
transmission_plot = ""
no_transmission = True
if template is None:
for cnam in aper:
ccdaper = aper[cnam]
for targ in ('2','3','1'):
if targ in ccdaper:
transmission_plot += (
f'plot = {cnam} {targ} '
f'{CCD_COLS.get(cnam,"black")} ! '
f"# ccd, targ, dcol, ecol\n"
)
no_transmission = False
break
if no_transmission:
warnings.warn(
f"Targets 2, 3, or 1 not found in any CCD within {apfile}; no transmission plot"
)
elif tsec is not None:
plots = tsec['plot']
for pl in plots:
cnam, targ, dcol, ecol = pl['ccd'], pl['targ'], ct(pl['dcol']), ct(pl['ecol'])
if cnam in aper and targ in aper[cnam]:
transmission_plot += (
f'plot = {cnam} {targ} {dcol} {ecol} # ccd, targ, dcol, ecol\n'
)
no_transmission = False
else:
warnings.warn(
f"Transmission target {targ} / CCD {cnam} not found in {apfile}; will skip"
)
if no_transmission:
warnings.warn(f"No transmission plot will be made")
# Generate the seeing plot lines. Prioritise the target in this case.
seeing_plot = ""
no_seeing = True
if template is None:
for cnam in aper:
ccdaper = aper[cnam]
for targ in ('1','2','3'):
if targ in ccdaper and not ccdaper[targ].linked:
seeing_plot += (
f"plot = {cnam} {targ}"
f' {CCD_COLS.get(cnam,"black")} ! '
"# ccd, targ, dcol, ecol\n"
)
no_seeing = False
break
if no_seeing:
raise hcam.HipercamError(
"Targets 1, 2 and 3 not found in any CCD"
" (or they are linked); cannot make seeing plot"
)
elif ssec is not None:
plots = tsec['plot']
for pl in plots:
cnam, targ, dcol, ecol = pl['ccd'], pl['targ'], ct(pl['dcol']), ct(pl['ecol'])
if cnam in aper and targ in aper[cnam] and not aper[cnam][targ].linked:
seeing_plot += (
f'plot = {cnam} {targ} {dcol} {ecol} # ccd, targ, dcol, ecol\n'
)
no_seeing = False
else:
warnings.warn(
f"Seeing target {targ} / CCD {cnam} not found or linked in {apfile}; will skip"
)
if no_seeing:
warnings.warn(f"No seeing plot will be made")
# monitor targets (all of them by default)
if template is None:
targs = set()
for cnam in aper:
ccdaper = aper[cnam]
for targ in ccdaper:
targs.add(targ)
monitor = ""
for targ in sorted(targs):
monitor += (
f"{targ} = NO_EXTRACTION TARGET_SATURATED TARGET_AT_EDGE"
" TARGET_NONLINEAR NO_SKY NO_FWHM NO_DATA SKY_AT_EDGE\n"
)
else:
monitor = ""
msec = tvals['monitor']
rlook = dict([(k,v) for v,k in hcam.FLAGS])
for targ, masks in msec.items():
smasks = ' '.join([rlook[mask] for mask in masks])
monitor += f'{targ} = {smasks}\n'
# time stamp
tstamp = strftime("%d %b %Y %H:%M:%S (UTC)", gmtime())
# Finally, write out the reduce file ...
with open(rfile, "w") as fout:
# write out file
fout.write(
f"""#
# This is a HiPERCAM "reduce file" which defines the operation of the
# reduce script. It was written by the HiPERCAM pipeline command
# 'genred'. It consists of a series of sections each of which
# contains a number of parameters. The file is self-documenting on the
# meaning of these parameters. The idea is that these are to large
# extent unchanging and it would be annoying to be prompted every time
# for them, but it also acts as a record of how reduction was carried
# out and is fed into the log file produce by 'reduce'.
# File written on {tstamp}
#
# HiPERCAM pipeline version: {hcam.REDUCE_FILE_VERSION}
#
# Start with some general items that tend not to change
# much. 'version' is the version of the reduce file format which
# changes more slowly than the software does. It must match the same
# parameter in 'reduce' for reduction to proceed. This is
# automatically the case at the time of creation, but old versions of
# the reduce file may become incompatible with later versions of
# reduce. Either they will require updating to be used, or the
# software version can be rolled back to give a compatible version of
# reduce using 'git'. The script 'rupdate', which attempts automatic
# update, may be worth trying if you need to update. It attempts to
# make the minimum changes needed to an old reduce file to run with
# later version dates.
[general]
version = {hcam.REDUCE_FILE_VERSION} # must be compatible with the version in reduce
instrument = {gsec['instrument']} # instrument
scale = {gsec['scale']} # pixel scale, arcsec/pixel
ldevice = {gsec['ldevice']} # PGPLOT plot device for light curve plots
lwidth = {gsec['lwidth']} # light curve plot width, inches, 0 to let program choose
lheight = {gsec['lheight']} # light curve plot height, inches
idevice = {gsec['idevice']} # PGPLOT plot device for image plots [if implot True]
iwidth = {gsec['iwidth']} # image curve plot width, inches, 0 to let program choose
iheight = {gsec['iheight']} # image curve plot height, inches
toffset = {gsec['toffset']} # integer offset to subtract from the MJD
# skip points with bad times in plots. HiPERCAM has a problem in not
# correctly indicating bad times so one does not usually want to
# skip "bad time" points, whereas one should for ULTRACAM and ULTRASPEC.
skipbadt = {gsec['skipbadt']}
# series of count levels at which warnings will be triggered for (a)
# non linearity and (b) saturation. Each line starts 'warn =', and is
# then followed by the CCD label, the non-linearity level and the
# saturation level
{warn_levels}
# The aperture reposition and extraction stages can be run in separate
# CPUs in parallel for each CCD potentially offering speed
# advantages. 'ncpu' is the number of CPUs to use for this. The
# maximum useful and best number to use is the number of CCDs in the
# instrument, e.g. 5 for HiPERCAM. You probably also want to leave at
# least one CPU to do other stuff, but if you have more than 2 CPUs,
# this parameter may help speed things. If you use this option (ncpu >
# 1), then there is also an advantage in terms of reducing
# parallelisation overheads in reading frames a few at a time before
# processing. This is controlled using 'ngroup'. i.e. with ngroup=10,
# 10 full frames are read before being processed. This parameter is
# ignored if ncpu==1
# I have had mixed results with this, which could depend a lot on the
# installation. On my home desktop is works better with ncpu=1 because
# it already seems to parallelise and it only gets worse if I split things
# up. But no harm trying.
ncpu = {gsec['ncpu']}
ngroup = {gsec['ngroup']}
# The next section '[apertures]' defines how the apertures are
# re-positioned from frame to frame. Apertures are re-positioned
# through a combination of a search near a start location followed by
# a 2D profile fit. Several parameters below are associated with this
# process and setting these right can be the key to a successful
# reduction. If there are reference apertures, they are located first
# to give a mean shift. This is used to avoid the initial search for
# any non-reference apertures which has the advantage of reducing the
# chance of problems. The search is carried out by first extracting a
# sub-window centred on the last good position of a target. This is
# then smoothed by a gaussian (width 'search_smooth_fwhm'), and the
# closest peak to the last valid position higher than
# 'fit_height_min_ref' above background (median of the square box) is
# taken as the initial position for later profile fits. The smoothing
# serves to make the process more robust against cosmic rays. The
# width of the search box ('search_half_width') depends on how good
# the telescope guiding is. It should be large enough to cope with the
# largest likely shift in position between any two consecutive
# frames. Well-chosen reference targets, which should be isolated and
# bright, can help this process a great deal. The threshold is applied
# to the *smoothed* image. This means that it can be significantly
# lower than simply the raw peak height. e.g. a target might have a
# typical peak height around 100, in seeing of 4 pixels FWHM. If you
# smooth by 10 pixels, the peak height will drop to
# 100*4**2/(4**2+10**2) = 14 counts. It will be much more stable as a
# result, but you should then probably choose a threshold of 7 when
# you might have thought 50 was appropriate. The smoothing itself can
# be carried out by direct convolution or by an FFT-based method. The
# end-result is the same either way but for large values of
# 'search_smooth_fwhm', i.e. >> 1, FFTs may offer an advantage
# speed-wise. But the only way to tell is by explicity running with
# 'search_smooth_fft' switched from 'no' to 'yes'.
# The boxes for the fits ('fit_half_width') need to be large enough to
# include the target and a bit of sky to ensure that the FWHM is
# accurately measured, remembering that seeing can flare of course. If
# your target was defocussed, a gaussian or Moffat function will be a
# poor fit and you may be better keeping the FWHM fixed at a large
# value comparable to the widths of your defoccused images (and use
# the gaussian option in such cases). If the apertures are chosen to
# be fixed, there will be no search or fit carried out in which case
# you must choose 'fixed' as well when it comes the extraction since
# otherwise it needs a FWHM. 'fixed' is a last resort and you will
# very likely need to use large aperture radii in the extraction
# section.
# An obscure parameter is 'fit_ndiv'. If this is made > 0, the fit
# routine attempts to allow for pixellation by evaluating the profile
# at multiple points within each pixel of the fit. First it will
# evaluate the profile for every unbinned pixel within a binned pixel
# if the pixels are binned; second, it will evaluate the profile over
# an ndiv by ndiv square grid within each unbinned pixel. Thus ndiv=1
# means one evaluation at the centre of each unbinned pixel,
# etc. Obviously this will slow things, but it could help if your
# images are under-sampled. I would always start with fit_ndiv=0, and
# only raise it if the measured FWHM seem to be close to or below two
# binned pixels.
# If you use reference targets (you should if possible), the initial
# positions for the non-reference targets should be good. You can then
# guard further against problems using the parameter 'fit_max_shift'
# to reject positions for the non-reference targets that shift too far
# from the initial guess. 'fit_alpha' is another parameter that
# applies only in this case. If reference apertures are being used,
# the expected locations of non-reference apertures can be predicted
# with some confidence. In this case when the non-reference aperture's
# position is measured, its position will be adjusted by 'fit_alpha'
# times the measured change in its position. Its value is bounded by 0
# < fit_alpha <= 1. "1" just means use the full measured change from
# the current frame to update the position. Anything < 1 builds in a
# bit of past history. The hope is that this could make the aperture
# positioning, especially for faint targets, more robust to cosmic
# rays and other issues. Of course it will correlate the positions
# from frame to frame. fit_alpha = 0.1 for instance will lead to a
# correlation length ~ 10 frames.
# If you use > 1 reference targets, then the parameter 'fit_diff'
# comes into play. Multiple reference targets should move together
# and give very consistent shifts. If they don't, then a problem may
# have occurred, e.g. one or more have been affected by a meteor trail
# for instance. The maximum acceptable differential shift is defined
# by 'fit_diff'. If exceeded, then the entire extraction will be
# aborted and positions held fixed.
# To get and idea of the right values of some of these parameters, in
# particular the 'search_half_width', the height thresholds,
# 'fit_max_shift' and 'fit_diff', the easiest approach is probably to
# run a reduction with loose values and see how it goes.
[apertures]
aperfile = {apfile} # file of software apertures for each CCD
location = {asec['location']} # aperture locations: 'fixed' or 'variable'
search_half_width = {asec['search_half_width']:.1f} # for initial search for objects around previous position, unbinned pixels
search_smooth_fwhm = {asec['search_smooth_fwhm']:.1f} # smoothing FWHM, binned pixels
search_smooth_fft = {asec['search_smooth_fft']} # use FFTs for smoothing, 'yes' or 'no'.
fit_method = {asec['fit_method']} # gaussian or moffat
fit_beta = {asec['fit_beta']:.1f} # Moffat exponent
fit_beta_max = {asec['fit_beta_max']:.1f} # max Moffat expt
fit_fwhm = {asec['fwhm']:.1f} # FWHM, unbinned pixels
fit_fwhm_min = {asec['fwhm_min']:.1f} # Minimum FWHM, unbinned pixels [MUST be < fit_fwhm!]
fit_fwhm_max = {asec['fwhm_max']:.1f} # Maximum FWHM, unbinned pixels
fit_ndiv = {asec['fit_ndiv']} # sub-pixellation factor
fit_fwhm_fixed = {asec['fit_fwhm_fixed']} # Might want to set = 'yes' for defocussed images
fit_half_width = {asec['fit_half_width']:.1f} # for fit, unbinned pixels
fit_thresh = {asec['fit_thresh']:.2f} # RMS rejection threshold for fits
fit_height_min_ref = {asec['fit_height_min_ref']:.1f} # minimum height to accept a fit, reference aperture
fit_height_min_nrf = {asec['fit_height_min_nrf']:.1f} # minimum height to accept a fit, non-reference aperture
fit_max_shift = {asec['fit_max_shift']:.1f} # max. non-ref. shift, unbinned pixels.
fit_alpha = {asec['fit_alpha']:.2f} # Fraction of non-reference aperture shift to apply
fit_diff = {asec['fit_diff']:.2f} # Maximum differential shift of multiple reference apertures, unbinned
# The next lines define how the apertures will be re-sized and how the
# flux will be extracted from the aperture. There is one line per CCD
# with format:
# <CCD label> = <resize> <extract method> [scale min max] [scale min max]
# [scale min max]
# where: <CCD label> is the CCD label; <resize> is either 'variable'
# or 'fixed' and sets how the aperture size will be determined -- if
# variable it will be scaled relative to the FWHM, so profile fitting
# will be attempted; <extract method> is either 'normal' or 'optimal'
# to say how the flux will be extracted -- 'normal' means a straight
# sum of sky subtracted flux over the aperture, 'optimal' use Tim
# Naylor's profile weighting, and requires profile fits to
# work. Finally there follow a series of numbers in three triplets,
# each of which is a scale factor relative to the FWHM for the
# aperture radius if the 'variable' option was chosen, then a minimum
# and a maximum aperture radius in unbinned pixels. The three triples
# correspond to the target aperture radius, the inner sky radius and
# finally the outer sky radius. The mininum and maximum also apply if
# you choose 'fixed' apertures and can be used to override whatever
# value comes from the aperture file. A common approach is set them
# equal to each other to give a fixed value, especially for the sky
# where one does not necessarily want the radii to vary. For PSF
# photometry, all these settings have no effect, but this section can
# still be used to determine which CCDs have fluxes extracted.
[extraction]
{extraction}
# The next lines are specific to the PSF photometry option. 'gfac' is
# used to label the sources according to groups, such that stars
# closer than 'gfac' times the FWHM are labelled in the same
# group. Each group has a PSF model fit independently. The reason
# behind the construction of groups is to reduce the dimensionality of
# the fitting procedure. Usually you want closely seperated stars to
# be fit simultaneously, but too large a value will mean fitting a
# model with many free parameters, which can fail to converge. The
# size of the box over which data is collected for fitting is set by
# 'fit_half_width'. Finally, 'positions' determines whether the star's
# positions should be considered variable in the PSF fitting. If this
# is set to fixed, the positions are held at the locations found in
# the aperture repositioning step, otherwise the positions are refined
# during PSF fitting. This step can fail for PSF photometry of faint
# sources.
[psf_photom]
gfac = {psfsec['gfac']} # multiple of the FWHM to use in grouping objects
fit_half_width = {psfsec['fit_half_width']} # size of window used to collect the data to do the fitting
positions = {psfsec['positions']} # 'fixed' or 'variable'
# Next lines determine how the sky background level is
# calculated. Note you can only set error = variance if method =
# 'clipped'. 'median' should usually be avoided as it can cause
# noticable steps in light curves. It's here as a comparator.
[sky]
method = {sksec['method']} # 'clipped' | 'median'
error = {sksec['error']} # 'variance' | 'photon': first uses actual variance of sky
thresh = {sksec['thresh']:.1f} # threshold in terms of RMS for 'clipped'
# Calibration frames and constants
# If you specify "!" for the readout, an attempt to estimate it from
# +/- 1 sigma percentiles will be made. This could help if you have no
# bias (and hence variance calculation from the count level will be
# wrong)
# The fringe parameters are the most complex. To de-fringe you need
# first a fringe map, e.g. as returned by makefringe. Then you need a
# file of FringePairs as returned by 'setfringe'. These are pairs of
# points placed at peaks and troughs of fringes. They will be used to
# measure a series of differences in each frame of interest and the
# fringe map from which a series of ratios is formed. The differences
# are measured from a group of pixels extending +/-nhlaf around the
# central pixel of each point of a FringePair. Thus nhalf=2 would give
# a 5x5 array (unbinned pixels). Finally to reduce the effect of bad
# values the individual ratios can be pruned if they lie outside a
# range rmin to rmax prior to their overall median being calculated.
# rmin should be small, but probably slightly less than zero to allow
# for a bit of statistical fluctuation, depending on your setup. One
# would normally expect rmax < 1 if the fringe map came from frames with
# a long exposure compared to the data.
[calibration]
crop = {csec['crop']} # Crop calibrations to match the data
bias = {'' if bias is None else bias} # Bias frame, blank to ignore
flat = {'' if flat is None else flat} # Flat field frame, blank to ignore
dark = {'' if dark is None else dark} # Dark frame, blank to ignore
fmap = {'' if fmap is None else fmap} # Fringe map frame, blank to ignore
fpair = {'' if fpair is None else fpair} # File of fringe pairs, ignored if fmap blank
nhalf = {csec['nhalf']} # Half-size of region used for fringe ratios, binned pix, ignored if fmap blank=""
rmin = {csec['rmin']} # Mininum acceptable individual fmap ratio, ignored if fmap blank
rmax = {csec['rmax']} # Maximum acceptable individual fmap ratio, ignored if fmap blank
readout = {csec['readout']} # RMS ADU. Float or string name of a file or "!" to estimate on the fly
gain = {csec['gain']} # Gain, electrons/ADU. Float or string name of a file
# The light curve plot which consists of light curves, X & Y
# poistions, the transmission and seeing. All but the light curves can
# be switched off by commenting them out (in full). First a couple of
# general parameters.
[lcplot]
xrange = {lcsec['xrange']} # maximum range in X to plot (minutes), <= 0 for everything
extend_x = {lcsec['extend_x']} # amount by which to extend xrange, minutes.
# The light curve panel (must be present). Mostly obvious, then a
# series of lines, each starting 'plot' which specify one light curve
# to be plotted giving CCD, target, comparison ('!' if you don't want
# a comparison), an additive offset, a multiplicative scaling factor
# and then a colour for the data and a colour for the error bar There
# will always be a light curve plot, whereas later elements are
# optional, therefore the light curve panel is defined to have unit
# height and all others are scaled relative to this.
[light]
linear = {lsec['linear']} # linear vertical scale (else magnitudes): 'yes' or 'no'
y_fixed = {lsec['y_fixed']} # keep a fixed vertical range or not: 'yes' or 'no'
y1 = {lsec['y1']} # initial lower y value
y2 = {lsec['y2']} # initial upper y value. y1=y2 for auto scaling
extend_y = {lsec['extend_y']} # fraction of plot height to extend when rescaling
# line or lines defining the targets to plot
{light_plot}
"""
)
# position plot (optional)
if no_position:
fout.write(
f"""# The X,Y position panel. Uncomment to show
#[position]
#height = 0.5 # height relative to light curve plot
#x_fixed = no # keep X-position vertical range fixed
#x_min = -5 # lower limit for X-position
#x_max = +5 # upper limit for X-position
#y_fixed = no # keep Y-position vertical range fixed
#y_min = -5 # lower limit for Y-position
#y_max = +5 # upper limit for Y-position
#extend_y = 0.2 # Vertical extension fraction if limits exceeded
## line or lines defining the targets to plot
# plot = 1 1 darkred ! # ccd, targ, dcol, ecol
"""
)
else:
fout.write(
f"""# The X,Y position panel. Comment *every* line
# if you don't want it.
[position]
height = {psec['height']} # height relative to light curve plot
x_fixed = {psec['x_fixed']} # keep X-position vertical range fixed
x_min = {psec['x_min']} # lower limit for X-position
x_max = {psec['x_max']} # upper limit for X-position
y_fixed = {psec['y_fixed']} # keep Y-position vertical range fixed
y_min = {psec['y_min']} # lower limit for Y-position
y_max = {psec['y_max']} # upper limit for Y-position
extend_y = {psec['extend_y']} # Vertical extension fraction if limits exceeded
# line or lines defining the targets to plot
{position_plot}
"""
)
# Transmission plot, optional
if no_transmission:
fout.write(
f"""# The transmission panel. Uncomment if you want it
# Simply plots the flux in whatever apertures are chosen,
# scaling them by their maximum (hence one can sometimes
# find that what you thought was 100% transmission was
# actually only 50% revealed as the cloud clears).
#[transmission]
#height = 0.5 # height relative to the light curve plot
#ymax = 110 # Maximum transmission to plot (>= 100 to slow replotting)
## line or lines defining the targets to plot
# plot = 1 2 green ! # ccd, targ, dcol, ecol
"""
)
else:
fout.write(
f"""# The transmission panel. If you comment it out,
# comment *every* line.
# Simply plots the flux in whatever apertures are chosen,
# scaling them by their maximum (hence one can sometimes
# find that what you thought was 100% transmission was
# actually only 50% revealed as the cloud clears).
[transmission]
height = {tsec['height']} # height relative to the light curve plot
ymax = {tsec['ymax']} # Maximum transmission to plot (>= 100 to slow replotting)
# line or lines defining the targets to plot
{transmission_plot}
"""
)
if no_seeing:
fout.write(
f"""# The seeing panel. Uncomment if you want it.
# You can have multiple plot lines. Don't choose linked
# targets as their FWHMs are not measured.
#[seeing]
#height = 0.5 # height relative to the light curve plot
#ymax = 2.999 # Initial maximum seeing
#y_fixed = no # fix the seeing scale (or not)
#extend_y = 0.2 # Y extension fraction if out of range and not fixed
## line or lines defining the targets to plot
# plot = 1 2 green ! # ccd, targ, dcol, ecol
"""
)
else:
fout.write(
f"""# The seeing panel. Can be commented out if you don't want one but make
# sure to comment it out completely, section name and all
# parameters. You can have multiple plot lines. Don't choose linked
# targets as their FWHMs are not measured.
[seeing]
height = {ssec['height']} # height relative to the light curve plot
ymax = {ssec['ymax']} # Initial maximum seeing
y_fixed = {ssec['y_fixed']} # fix the seeing scale (or not)
extend_y = {ssec['extend_y']} # Y extension fraction if out of range and not fixed
# line or lines defining the targets to plot
{seeing_plot}
"""
)
# Finish up
fout.write(
f"""
# This option attempts to correct for a badly-positioned focal plane mask
# in drift mode which combined with a high background can lead to steps in
# illumination in the Y direction. This tries to subtract the median in the
# X-direction of each window. 'dthresh' is a threshold used to reject X
# pixels prior to taking the median. The purpose is to prevent the stars
# from distorting the median. Take care with this option which is experimental.
[focal_mask]
demask = {fsec['demask']}
dthresh = {fsec['dthresh']}
# Monitor section. This section allows you to monitor particular
# targets for problems. If they occur, then messages will be printed
# to the terminal during reduce. The messages are determined by the
# bitmask flag set during the extraction of each
# target. Possibilities:
# NO_FWHM : no FWHM measured
# NO_SKY : no sky pixels at all
# SKY_AT_EDGE : sky aperture off edge of window
# TARGET_AT_EDGE : target aperture off edge of window
# TARGET_SATURATED : at least one pixel in target above saturation level
# TARGET_NONLINEAR : at least one pixel in target above nonlinear level
# NO_EXTRACTION : no extraction possible
# NO_DATA : no valid pixels in aperture
# For a target you want to monitor, type its label, '=', then the
# bitmask patterns you want to be flagged up if they are set. This is
# designed mainly for observing, as there is less you can do once the
# data have been taken, but it still may prove useful.
[monitor]
{monitor}
"""
)
print("Reduce file written to {:s}".format(rfile))
def tbytes(args=None):
"""``tbytes [source] ([old]) run``
Reads all timing bytes from a |hiper|, ULTRACAM or ULTRASPEC run
and dumps them to a disk file. Designed as a safety fallback for
correcting timing issues where one wants to manipulate the raw
data.
Parameters:
source : string [hidden]
Data source, two options:
| 'hl' : local HiPERCAM FITS file
| 'ul' : ULTRA(CAM|SPEC) server
old : bool [if source=='ul', hidden, defaults to False]
Will attempt first to read timing bytes from an old version
of the data file, i.e. one with extension '.dat.old' rather
than plain '.dat' (applies to ULTRA(CAM|SPEC) data only. Such
files are generated by the null timestamp correction script
routinely applied to ULTRASPEC data for example. 'old' will
switch to False by default if not explicitly set.
run : str
run number to access, e.g. 'run0034'. This will also be
used to generate the name for the timing bytes file
(extension '.tbts'). If a file of this name already exists,
the script will attempt to read and compare the bytes of
the two files and report any changes. The timing bytes
file will be written to the present working directory,
unless it contains a sub-directory named "tbytes", in which
case the timing data will be read/written from/to there.
"""
command, args = utils.script_args(args)
# get the inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("source", Cline.LOCAL, Cline.HIDE)
cl.register("old", Cline.LOCAL, Cline.HIDE)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
# get inputs
source = cl.get_value(
"source", "data source [hl, ul]", "hl", lvals=("hl", "ul")
)
if source == "ul":
cl.set_default("old", False)
old = cl.get_value("old", "try to access .dat.old files first?", False)
run = cl.get_value("run", "run name", "run005")
if run.endswith(".fits"):
run = run[:-5]
# create name of timing bytes file
if os.path.isdir('tbytes'):
ofile = os.path.join('tbytes', os.path.basename(run) + hcam.TBTS)
else:
ofile = os.path.basename(run) + hcam.TBTS
if source == "hl":
nframe = 0
if os.path.exists(ofile):
# interpret timing bytes. In this case the file of timing
# bytes already exists. We read it and the run and compare
# the times, reporting any differences.
ndiffer = 0
with spooler.HcamTbytesSpool(run) as rtbytes:
with open(ofile, "rb") as fin:
for tbytes in rtbytes:
otbytes = fin.read(rtbytes.ntbytes)
if len(otbytes) != rtbytes.ntbytes:
raise hcam.HipercamError(
"Failed to read", rtbytes.ntbytes, "bytes from", ofile
)
nframe += 1
if tbytes != otbytes:
# now need to interpret times
nmjd = h_tbytes_to_mjd(tbytes, nframe)
omjd = h_tbytes_to_mjd(otbytes, nframe)
print(
"Frame {:d}, new vs old GPS timestamp (MJD) = {:.12f} vs {:.12f}".format(
nframe, nmjd, omjd
)
)
ndiffer += 1
print(
"{:s} vs {:s}: there were {:d} time stamp differences in {:d} frames".format(
run, ofile, ndiffer, nframe
)
)
else:
# save timing bytes to disk
with spooler.HcamTbytesSpool(run) as rtbytes:
with open(ofile, "wb") as fout:
for tbytes in rtbytes:
fout.write(tbytes)
nframe += 1
print("Found", nframe, "frames in", run, "\nWrote timing data to", ofile)
elif source == "ul":
nframe = 0
if os.path.exists(ofile):
# interpret timing bytes. In this case the file of timing
# bytes already exists. We read it and the run and compare
# the times, reporting any differences.
ndiffer = 0
with spooler.UcamTbytesSpool(run, old=old) as rtbytes:
with open(ofile, "rb") as fin:
for tbytes in rtbytes:
otbytes = fin.read(rtbytes.ntbytes)
if len(otbytes) != rtbytes.ntbytes:
raise hcam.HipercamError(
"Failed to read", rtbytes.ntbytes, "bytes from", ofile
)
nframe += 1
if tbytes != otbytes:
# now need to interpret times
nmjd = u_tbytes_to_mjd(tbytes, rtbytes, nframe)
omjd = u_tbytes_to_mjd(otbytes, rtbytes, nframe)
print(
"Frame {:d}, new vs old GPS timestamp (MJD) = {:.12f} vs {:.12f}".format(
nframe, nmjd, omjd
)
)
ndiffer += 1
print(
"{:s} vs {:s}: there were {:d} time stamp differences in {:d} frames".format(
run, ofile, ndiffer, nframe
)
)
else:
# no timing bytes files exists; save to disk
with spooler.UcamTbytesSpool(run, old=old) as rtbytes:
with open(ofile, "wb") as fout:
for tbytes in rtbytes:
fout.write(tbytes)
nframe += 1
print("Found", nframe, "frames in", run, "\nWrote timing data to", ofile)
def rtplot(args=None):
"""``rtplot [source device width height] (run first (trim [ncol nrow])
twait tmax | flist) [pause plotall] ccd (nx) bias flat defect
setup [hurl] msub iset (ilo ihi | plo phi) xlo xhi ylo yhi (profit
[fdevice fwidth fheight method beta fwhm fwhm_min shbox smooth
splot fhbox hmin read gain thresh])``
Plots a sequence of images as a movie in near 'real time', hence
'rt'. Designed to be used to look at images coming in while at the
telescope, 'rtplot' comes with many options, a large number of which are
hidden by default, and many of which are only prompted if other arguments
are set correctly. If you want to see them all, invoke as 'rtplot prompt'.
This is worth doing once to know rtplot's capabilities.
rtplot can source data from both the ULTRACAM and HiPERCAM servers, from
local 'raw' ULTRACAM and HiPERCAM files (i.e. .xml + .dat for ULTRACAM, 3D
FITS files for HiPERCAM) and from lists of HiPERCAM '.hcm' files.
rtplot optionally allows the selection of targets to be fitted with
gaussian or moffat profiles, and, if successful, will plot circles of 2x
the measured FWHM in green over the selected targets. This option only
works if a single CCD is being plotted.
Parameters:
source : string [hidden]
Data source, five options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : ULTRACAM server
| 'ul' : local ULTRACAM .xml/.dat files
| 'hf' : list of HiPERCAM hcm FITS-format files
'hf' is used to look at sets of frames generated by 'grab' or
converted from foreign data formats.
device : string [hidden]
Plot device. PGPLOT is used so this should be a PGPLOT-style name,
e.g. '/xs', '1/xs' etc. At the moment only ones ending /xs are
supported.
width : float [hidden]
plot width (inches). Set = 0 to let the program choose.
height : float [hidden]
plot height (inches). Set = 0 to let the program choose. BOTH width
AND height must be non-zero to have any effect
run : string [if source ends 's' or 'l']
run number to access, e.g. 'run034'
flist : string [if source ends 'f']
name of file list
first : int [if source ends 's' or 'l']
exposure number to start from. 1 = first frame; set = 0 to always
try to get the most recent frame (if it has changed). For data
from the |hiper| server, a negative number tries to get a frame not
quite at the end. i.e. -10 will try to get 10 from the last
frame. This is mainly to sidestep a difficult bug with the
acquisition system.
trim : bool [if source starts with 'u']
True to trim columns and/or rows off the edges of windows nearest
the readout. This is particularly for ULTRACAM windowed data where
the first few rows and columns can contain bad data.
ncol : int [if trim, hidden]
Number of columns to remove (on left of left-hand window, and right
of right-hand windows)
nrow : int [if trim, hidden]
Number of rows to remove (bottom of windows)
twait : float [if source ends 's' or 'l'; hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [if source ends 's' or 'l'; hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
pause : float [hidden]
seconds to pause between frames (defaults to 0)
plotall : bool [hidden]
plot all frames regardless of status (i.e. including blank frames
when nskips are enabled (defaults to False). The profile fitting
will still be disabled for bad frames.
ccd : string
CCD(s) to plot, '0' for all, '1 3' to plot '1' and '3' only, etc.
nx : int [if more than 1 CCD]
number of panels across to display.
bias : string
Name of bias frame to subtract, 'none' to ignore.
flat : string
Name of flat field to divide by, 'none' to ignore. Should normally
only be used in conjunction with a bias, although it does allow you
to specify a flat even if you haven't specified a bias.
defect : string
Name of defect file, 'none' to ignore.
setup : bool
True/yes to access the current windows from hdriver. Useful
during observing when seeting up windows, but not normally
otherwise. Next argument (hidden) is the URL to get to
hdriver. Once setup, you should probably turn this off to
avoid overloading hdriver, especially if in drift mode as
it makes a request for the windows for every frame.
hurl : string [if setup; hidden]
URL needed to access window setting from hdriver. The internal
server in hdriver must be switched on using rtplot_server_on
in the hdriver config file.
msub : bool
subtract the median from each window before scaling for the
image display or not. This happens after any bias subtraction.
iset : string [single character]
determines how the intensities are determined. There are three
options: 'a' for automatic simply scales from the minimum to the
maximum value found on a per CCD basis. 'd' for direct just takes
two numbers from the user. 'p' for percentile dtermines levels
based upon percentiles determined from the entire CCD on a per CCD
basis.
ilo : float [if iset='d']
lower intensity level
ihi : float [if iset='d']
upper intensity level
plo : float [if iset='p']
lower percentile level
phi : float [if iset='p']
upper percentile level
xlo : float
left-hand X-limit for plot
xhi : float
right-hand X-limit for plot (can actually be < xlo)
ylo : float
lower Y-limit for plot
yhi : float
upper Y-limit for plot (can be < ylo)
profit : bool [if plotting a single CCD only]
carry out profile fits or not. If you say yes, then on the first
plot, you will have the option to pick objects with a cursor. The
program will then attempt to track these from frame to frame, and
fit their profile. You may need to adjust 'first' to see anything.
The parameters used for profile fits are hidden and you may want to
invoke the command with 'prompt' the first time you try profile
fitting.
fdevice : string [if profit; hidden]
plot device for profile fits, PGPLOT-style name.
e.g. '/xs', '2/xs' etc.
fwidth : float [if profit; hidden]
fit plot width (inches). Set = 0 to let the program choose.
fheight : float [if profit; hidden]
fit plot height (inches). Set = 0 to let the program choose.
BOTH fwidth AND fheight must be non-zero to have any effect
method : string [if profit; hidden]
this defines the profile fitting method, either a gaussian or a
moffat profile. The latter is usually best.
beta : float [if profit and method == 'm'; hidden]
default Moffat exponent
fwhm : float [if profit; hidden]
default FWHM, unbinned pixels.
fwhm_min : float [if profit; hidden]
minimum FWHM to allow, unbinned pixels.
shbox : float [if profit; hidden]
half width of box for searching for a star, unbinned pixels. The
brightest target in a region +/- shbox around an intial position
will be found. 'shbox' should be large enough to allow for likely
changes in position from frame to frame, but try to keep it as
small as you can to avoid jumping to different targets and to
reduce the chances of interference by cosmic rays.
smooth : float [if profit; hidden]
FWHM for gaussian smoothing, binned pixels. The initial position
for fitting is determined by finding the maximum flux in a smoothed
version of the image in a box of width +/- shbox around the starter
position. Typically should be comparable to the stellar width. Its
main purpose is to combat cosmi rays which tend only to occupy a
single pixel.
splot : bool [if profit; hidden]
Controls whether an outline of the search box and a target number
is plotted (in red) or not.
fhbox : float [if profit; hidden]
half width of box for profile fit, unbinned pixels. The fit box is
centred on the position located by the initial search. It should
normally be > ~2x the expected FWHM.
hmin : float [if profit; hidden]
height threshold to accept a fit. If the height is below this
value, the position will not be updated. This is to help in cloudy
conditions.
read : float [if profit; hidden]
readout noise, RMS ADU, for assigning uncertainties
gain : float [if profit; hidden]
gain, ADU/count, for assigning uncertainties
thresh : float [if profit; hidden]
sigma rejection threshold for fits
"""
command, args = utils.script_args(args)
# get the inputs
with Cline('HIPERCAM_ENV', '.hipercam', command, args) as cl:
# register parameters
cl.register('source', Cline.GLOBAL, Cline.HIDE)
cl.register('device', Cline.LOCAL, Cline.HIDE)
cl.register('width', Cline.LOCAL, Cline.HIDE)
cl.register('height', Cline.LOCAL, Cline.HIDE)
cl.register('run', Cline.GLOBAL, Cline.PROMPT)
cl.register('first', Cline.LOCAL, Cline.PROMPT)
cl.register('trim', Cline.GLOBAL, Cline.PROMPT)
cl.register('ncol', Cline.GLOBAL, Cline.HIDE)
cl.register('nrow', Cline.GLOBAL, Cline.HIDE)
cl.register('twait', Cline.LOCAL, Cline.HIDE)
cl.register('tmax', Cline.LOCAL, Cline.HIDE)
cl.register('flist', Cline.LOCAL, Cline.PROMPT)
cl.register('ccd', Cline.LOCAL, Cline.PROMPT)
cl.register('pause', Cline.LOCAL, Cline.HIDE)
cl.register('plotall', Cline.LOCAL, Cline.HIDE)
cl.register('nx', Cline.LOCAL, Cline.PROMPT)
cl.register('bias', Cline.GLOBAL, Cline.PROMPT)
cl.register('flat', Cline.GLOBAL, Cline.PROMPT)
cl.register('defect', Cline.GLOBAL, Cline.PROMPT)
cl.register('setup', Cline.GLOBAL, Cline.PROMPT)
cl.register('hurl', Cline.GLOBAL, Cline.HIDE)
cl.register('msub', Cline.GLOBAL, Cline.PROMPT)
cl.register('iset', Cline.GLOBAL, Cline.PROMPT)
cl.register('ilo', Cline.GLOBAL, Cline.PROMPT)
cl.register('ihi', Cline.GLOBAL, Cline.PROMPT)
cl.register('plo', Cline.GLOBAL, Cline.PROMPT)
cl.register('phi', Cline.LOCAL, Cline.PROMPT)
cl.register('xlo', Cline.GLOBAL, Cline.PROMPT)
cl.register('xhi', Cline.GLOBAL, Cline.PROMPT)
cl.register('ylo', Cline.GLOBAL, Cline.PROMPT)
cl.register('yhi', Cline.GLOBAL, Cline.PROMPT)
cl.register('profit', Cline.LOCAL, Cline.PROMPT)
cl.register('fdevice', Cline.LOCAL, Cline.HIDE)
cl.register('fwidth', Cline.LOCAL, Cline.HIDE)
cl.register('fheight', Cline.LOCAL, Cline.HIDE)
cl.register('method', Cline.LOCAL, Cline.HIDE)
cl.register('beta', Cline.LOCAL, Cline.HIDE)
cl.register('fwhm', Cline.LOCAL, Cline.HIDE)
cl.register('fwhm_min', Cline.LOCAL, Cline.HIDE)
cl.register('shbox', Cline.LOCAL, Cline.HIDE)
cl.register('smooth', Cline.LOCAL, Cline.HIDE)
cl.register('splot', Cline.LOCAL, Cline.HIDE)
cl.register('fhbox', Cline.LOCAL, Cline.HIDE)
cl.register('hmin', Cline.LOCAL, Cline.HIDE)
cl.register('read', Cline.LOCAL, Cline.HIDE)
cl.register('gain', Cline.LOCAL, Cline.HIDE)
cl.register('thresh', Cline.LOCAL, Cline.HIDE)
# get inputs
source = cl.get_value('source', 'data source [hs, hl, us, ul, hf]',
'hl', lvals=('hs','hl','us','ul','hf'))
# set some flags
server_or_local = source.endswith('s') or source.endswith('l')
# plot device stuff
device = cl.get_value('device', 'plot device', '1/xs')
width = cl.get_value('width', 'plot width (inches)', 0.)
height = cl.get_value('height', 'plot height (inches)', 0.)
if server_or_local:
resource = cl.get_value('run', 'run name', 'run005')
if source == 'hs':
first = cl.get_value('first', 'first frame to plot', 1)
else:
first = cl.get_value('first', 'first frame to plot', 1, 0)
if source.startswith('u'):
trim = cl.get_value(
'trim',
'do you want to trim edges of windows? (ULTRACAM only)',
True
)
if trim:
ncol = cl.get_value(
'ncol', 'number of columns to trim from windows', 0)
nrow = cl.get_value(
'nrow', 'number of rows to trim from windows', 0)
else:
trim = False
twait = cl.get_value(
'twait', 'time to wait for a new frame [secs]', 1., 0.)
tmax = cl.get_value(
'tmax', 'maximum time to wait for a new frame [secs]', 10., 0.)
else:
resource = cl.get_value(
'flist', 'file list', cline.Fname('files.lis',hcam.LIST)
)
first = 1
trim = False
# define the panel grid. first get the labels and maximum dimensions
ccdinf = spooler.get_ccd_pars(source, resource)
try:
nxdef = cl.get_default('nx')
except:
nxdef = 3
if len(ccdinf) > 1:
ccd = cl.get_value('ccd', 'CCD(s) to plot [0 for all]', '0')
if ccd == '0':
ccds = list(ccdinf.keys())
else:
ccds = ccd.split()
check = set(ccdinf.keys())
if not set(ccds) <= check:
raise hcam.HipercamError(
'At least one invalid CCD label supplied'
)
if len(ccds) > 1:
nxdef = min(len(ccds), nxdef)
cl.set_default('nx', nxdef)
nx = cl.get_value('nx', 'number of panels in X', 3, 1)
else:
nx = 1
else:
nx = 1
ccds = list(ccdinf.keys())
cl.set_default('pause', 0.)
pause = cl.get_value(
'pause', 'time delay to add between'
' frame plots [secs]', 0., 0.
)
cl.set_default('plotall', False)
plotall = cl.get_value('plotall', 'plot all frames,'
' regardless of status?', False)
# bias frame (if any)
bias = cl.get_value(
'bias', "bias frame ['none' to ignore]",
cline.Fname('bias', hcam.HCAM), ignore='none'
)
if bias is not None:
# read the bias frame
bias = hcam.MCCD.read(bias)
fprompt = "flat frame ['none' to ignore]"
else:
fprompt = "flat frame ['none' is normal choice with no bias]"
# flat (if any)
flat = cl.get_value(
'flat', fprompt,
cline.Fname('flat', hcam.HCAM), ignore='none'
)
if flat is not None:
# read the flat frame
flat = hcam.MCCD.read(flat)
# defect file (if any)
dfct = cl.get_value(
'defect', "defect file ['none' to ignore]",
cline.Fname('defect', hcam.DFCT), ignore='none'
)
if dfct is not None:
# read the defect frame
dfct = defect.MccdDefect.read(dfct)
# Get windows from hdriver
setup = cl.get_value(
'setup', 'display current hdriver window settings', False
)
if setup:
hurl = cl.get_value(
'hurl', 'URL for hdriver windows', 'http://192.168.1.2:5100'
)
# define the display intensities
msub = cl.get_value('msub', 'subtract median from each window?', True)
iset = cl.get_value(
'iset', 'set intensity a(utomatically),'
' d(irectly) or with p(ercentiles)?',
'a', lvals=['a','d','p'])
iset = iset.lower()
plo, phi = 5, 95
ilo, ihi = 0, 1000
if iset == 'd':
ilo = cl.get_value('ilo', 'lower intensity limit', 0.)
ihi = cl.get_value('ihi', 'upper intensity limit', 1000.)
elif iset == 'p':
plo = cl.get_value('plo', 'lower intensity limit percentile',
5., 0., 100.)
phi = cl.get_value('phi', 'upper intensity limit percentile',
95., 0., 100.)
# region to plot
for i, cnam in enumerate(ccds):
nxtot, nytot, nxpad, nypad = ccdinf[cnam]
if i == 0:
xmin, xmax = float(-nxpad), float(nxtot + nxpad + 1)
ymin, ymax = float(-nypad), float(nytot + nypad + 1)
else:
xmin = min(xmin, float(-nxpad))
xmax = max(xmax, float(nxtot + nxpad + 1))
ymin = min(ymin, float(-nypad))
ymax = max(ymax, float(nytot + nypad + 1))
xlo = cl.get_value('xlo', 'left-hand X value', xmin, xmin, xmax)
xhi = cl.get_value('xhi', 'right-hand X value', xmax, xmin, xmax)
ylo = cl.get_value('ylo', 'lower Y value', ymin, ymin, ymax)
yhi = cl.get_value('yhi', 'upper Y value', ymax, ymin, ymax)
# profile fitting if just one CCD chosen
if len(ccds) == 1:
# many parameters for profile fits, although most are not plotted
# by default
profit = cl.get_value('profit', 'do you want profile fits?', False)
if profit:
fdevice = cl.get_value('fdevice', 'plot device for fits',
'2/xs')
fwidth = cl.get_value('fwidth', 'fit plot width (inches)', 0.)
fheight = cl.get_value(
'fheight', 'fit plot height (inches)', 0.)
method = cl.get_value(
'method', 'fit method g(aussian) or m(offat)',
'm', lvals=['g','m'])
if method == 'm':
beta = cl.get_value(
'beta', 'initial exponent for Moffat fits', 5., 0.5)
else:
beta = 0.
fwhm_min = cl.get_value(
'fwhm_min', 'minimum FWHM to allow [unbinned pixels]',
1.5, 0.01
)
fwhm = cl.get_value(
'fwhm', 'initial FWHM [unbinned pixels] for profile fits',
6., fwhm_min
)
shbox = cl.get_value(
'shbox', 'half width of box for initial location'
' of target [unbinned pixels]', 11., 2.)
smooth = cl.get_value(
'smooth', 'FWHM for smoothing for initial object'
' detection [binned pixels]', 6.)
splot = cl.get_value(
'splot', 'plot outline of search box?', True)
fhbox = cl.get_value(
'fhbox', 'half width of box for profile fit'
' [unbinned pixels]', 21., 3.)
hmin = cl.get_value(
'hmin', 'minimum peak height to accept the fit', 50.)
read = cl.get_value('read', 'readout noise, RMS ADU', 3.)
gain = cl.get_value('gain', 'gain, ADU/e-', 1.)
thresh = cl.get_value('thresh', 'number of RMS to reject at', 4.)
else:
profit = False
################################################################
#
# all the inputs have now been obtained. Get on with doing stuff
# open image plot device
imdev = hcam.pgp.Device(device)
if width > 0 and height > 0:
pgpap(width,height/width)
# set up panels and axes
nccd = len(ccds)
ny = nccd // nx if nccd % nx == 0 else nccd // nx + 1
# slice up viewport
pgsubp(nx,ny)
# plot axes, labels, titles. Happens once only
for cnam in ccds:
pgsci(hcam.pgp.Params['axis.ci'])
pgsch(hcam.pgp.Params['axis.number.ch'])
pgenv(xlo, xhi, ylo, yhi, 1, 0)
pglab('X','Y','CCD {:s}'.format(cnam))
# initialisations. 'last_ok' is used to store the last OK frames of each
# CCD for retrieval when coping with skipped data.
total_time = 0 # time waiting for new frame
fpos = [] # list of target positions to fit
fframe = True # waiting for first valid frame with profit
# plot images
with spooler.data_source(source, resource, first, full=False) as spool:
# 'spool' is an iterable source of MCCDs
n = 0
for mccd in spool:
if server_or_local:
# Handle the waiting game ...
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmax, total_time
)
if give_up:
print('rtplot stopped')
break
elif try_again:
continue
# Trim the frames: ULTRACAM windowed data has bad columns
# and rows on the sides of windows closest to the readout
# which can badly affect reduction. This option strips
# them.
if trim:
hcam.ccd.trim_ultracam(mccd, ncol, nrow)
# indicate progress
tstamp = Time(mccd.head['TIMSTAMP'], format='isot', precision=3)
print(
'{:d}, utc = {:s}, '.format(
mccd.head['NFRAME'], tstamp.iso), end=''
)
# accumulate errors
emessages = []
if n == 0:
if bias is not None:
# crop the bias on the first frame only
bias = bias.crop(mccd)
if flat is not None:
# crop the flat on the first frame only
flat = flat.crop(mccd)
if setup:
# Get windows from hdriver. Fair bit of error checking
# needed. 'got_windows' indicates if anything useful
# found, 'hwindows' is a list of (llx,lly,nx,ny) tuples
# if somthing is found.
try:
r = requests.get(
'http://192.168.1.2:5100',timeout=0.2
)
if r.text.strip() == 'No valid data available':
emessages.append(
'** bad return from hdriver = {:s}'.format(r.text.strip())
)
got_windows = False
elif r.text.strip() == 'fullframe':
# to help Stu out a bit, effectively just ignore this one
got_windows = False
else:
# OK, got something
lines = r.text.split('\r\n')
xbinh,ybinh,nwinh = lines[0].split()
xbinh,ybinh,nwinh = int(xbinh),int(ybinh),int(nwinh)
hwindows = []
for line in lines[1:nwinh+1]:
llxh,llyh,nxh,nyh = line.split()
hwindows.append(
(int(llxh),int(llyh),int(nxh),int(nyh))
)
if nwinh != len(hwindows):
emessages.append(
('** expected {:d} windows from'
' hdriver but got {:d}').format(nwinh,len(hwindows))
)
got_windows = False
got_windows = True
except (requests.exceptions.ConnectionError, socket.timeout,
requests.exceptions.Timeout) as err:
emessages.append(' ** hdriver error: {!r}'.format(err))
got_windows = False
else:
got_windows = False
# display the CCDs chosen
message = ''
pgbbuf()
for nc, cnam in enumerate(ccds):
ccd = mccd[cnam]
if plotall or ccd.is_data():
# this should be data as opposed to a blank frame
# between data frames that occur with nskip > 0
# subtract the bias
if bias is not None:
ccd -= bias[cnam]
# divide out the flat
if flat is not None:
ccd /= flat[cnam]
if msub:
# subtract median from each window
for wind in ccd.values():
wind -= wind.median()
# set to the correct panel and then plot CCD
ix = (nc % nx) + 1
iy = nc // nx + 1
pgpanl(ix,iy)
vmin, vmax = hcam.pgp.pCcd(
ccd,iset,plo,phi,ilo,ihi,
xlo=xlo, xhi=xhi, ylo=ylo, yhi=yhi
)
if got_windows:
# plot the current hdriver windows
pgsci(hcam.CNAMS['yellow'])
pgsls(2)
for llxh,llyh,nxh,nyh in hwindows:
pgrect(llxh-0.5,llxh+nxh-0.5,llyh-0.5,llyh+nyh-0.5)
if dfct is not None and cnam in dfct:
# plot defects
hcam.pgp.pCcdDefect(dfct[cnam])
# accumulate string of image scalings
if nc:
message += ', ccd {:s}: {:.1f}, {:.1f}, exp: {:.4f}'.format(
cnam,vmin,vmax,mccd.head['EXPTIME']
)
else:
message += 'ccd {:s}: {:.1f}, {:.1f}, exp: {:.4f}'.format(
cnam,vmin,vmax,mccd.head['EXPTIME']
)
pgebuf()
# end of CCD display loop
print(message)
for emessage in emessages:
print(emessage)
if ccd.is_data() and profit and fframe:
fframe = False
# cursor selection of targets after first plot, if profit
# accumulate list of starter positions
print('Please select targets for profile fitting. You can select as many as you like.')
x, y, reply = (xlo+xhi)/2, (ylo+yhi)/2, ''
ntarg = 0
pgsci(2)
pgslw(2)
while reply != 'Q':
print("Place cursor on fit target. Any key to register, 'q' to quit")
x, y, reply = pgcurs(x, y)
if reply == 'q':
break
else:
# check that the position is inside a window
wnam = ccd.inside(x, y, 2)
if wnam is not None:
# store the position, Window label, target number,
# box size fwhm, beta
ntarg += 1
fpos.append(Fpar(x,y,wnam,ntarg,shbox,fwhm,beta))
# report information, overplot search box
print(
('Target {:d} selected at {:.1f},'
'{:.1f} in window {:s}').format(ntarg,x,y,wnam)
)
if splot:
fpos[-1].plot()
if len(fpos):
print(len(fpos),'targets selected')
# if some targets were selected, open the fit plot device
fdev = hcam.pgp.Device(fdevice)
if fwidth > 0 and fheight > 0:
pgpap(fwidth,fheight/fwidth)
if ccd.is_data():
# carry out fits. Nothing happens if fpos is empty
for fpar in fpos:
# switch to the image plot
imdev.select()
# plot search box
if splot:
fpar.plot()
try:
# extract search box from the CCD. 'fpar' is updated later
# if the fit is successful to reflect the new position
swind = fpar.swind(ccd)
# carry out initial search
x,y,peak = swind.search(smooth, fpar.x, fpar.y, hmin, False)
# now for a more refined fit. First extract fit Window
fwind = ccd[fpar.wnam].window(
x-fhbox, x+fhbox, y-fhbox, y+fhbox
)
# crude estimate of sky background
sky = np.percentile(fwind.data, 50)
# refine the Aperture position by fitting the profile
(sky, height, x, y, fwhm, beta), epars, \
(wfit, X, Y, sigma, chisq, nok,
nrej, npar, message) = hcam.fitting.combFit(
fwind, method, sky, peak-sky,
x, y, fpar.fwhm, fwhm_min, False,
fpar.beta, read, gain, thresh
)
print('Targ {:d}: {:s}'.format(fpar.ntarg,message))
if peak > hmin and ccd[fpar.wnam].distance(x,y) > 1:
# update some initial parameters for next time
if method == 'g':
fpar.x, fpar.y, fpar.fwhm = x, y, fwhm
elif method == 'm':
fpar.x, fpar.y, fpar.fwhm, \
fpar.beta = x, y, fwhm, beta
# plot values versus radial distance
ok = sigma > 0
R = np.sqrt((X-x)**2+(Y-y)**2)
fdev.select()
vmin = min(sky, sky+height, fwind.min())
vmax = max(sky, sky+height, fwind.max())
extent = vmax-vmin
pgeras()
pgvstd()
pgswin(0,R.max(),vmin-0.05*extent,vmax+0.05*extent)
pgsci(4)
pgbox('bcnst',0,0,'bcnst',0,0)
pgsci(2)
pglab('Radial distance [unbinned pixels]','Counts','')
pgsci(1)
pgpt(R[ok].flat, fwind.data[ok].flat, 1)
if nrej:
pgsci(2)
pgpt(R[~ok].flat, fwind.data[~ok].flat, 5)
# line fit
pgsci(3)
r = np.linspace(0,R.max(),400)
if method == 'g':
alpha = 4*np.log(2.)/fwhm**2
f = sky+height*np.exp(-alpha*r**2)
elif method == 'm':
alpha = 4*(2**(1/beta)-1)/fwhm**2
f = sky+height/(1+alpha*r**2)**beta
pgline(r,f)
# back to the image to plot a circle of radius FWHM
imdev.select()
pgsci(3)
pgcirc(x,y,fwhm)
else:
print(' *** below detection threshold; position & FWHM will not updated')
pgsci(2)
# plot location on image as a cross
pgpt1(x, y, 5)
except hcam.HipercamError as err:
print(' >> Targ {:d}: fit failed ***: {!s}'.format(
fpar.ntarg, err)
)
pgsci(2)
if pause > 0.:
# pause between frames
time.sleep(pause)
# update the frame number
n += 1
def grab(args=None):
"""``grab [source temp] (run first last (ndigit) [twait tmax] | flist
(prefix)) trim ([ncol nrow]) bias dark flat fmap (fpair [nhalf rmin
rmax verbose]) [dtype]``
This downloads a sequence of images from a raw data file and writes them
out to a series CCD / MCCD files.
Parameters:
source : str [hidden]
Data source, four options:
| 'hs' : HiPERCAM server
| 'hl' : local HiPERCAM FITS file
| 'us' : ULTRACAM server
| 'ul' : local ULTRACAM .xml/.dat files
| 'hf' : list of HiPERCAM hcm FITS-format files
The standard start-off default for ``source`` can be set
using the environment variable
HIPERCAM_DEFAULT_SOURCE. e.g. in bash :code:`export
HIPERCAM_DEFAULT_SOURCE="us"` would ensure it always
started with the ULTRACAM server by default. If
unspecified, it defaults to 'hl'. 'hf' is useful if you
want to apply pipeline calibrations (bias, flat, etc) to
files imported from a 'foreign' format. In this case the
output files will have the same name as the inputs but
with a prefix added.
temp : bool [hidden, defaults to False]
True to indicate that the frames should be written to
temporary files with automatically-generated names in the
expectation of deleting them later. This also writes out a
file listing the names. The aim of this is to allow grab
to be used as a feeder for other routines in larger
scripts. If temp == True, grab will return with the name
of the list of hcm files. Look at 'makebias' for an example
that uses this feature.
run : str [if source ends 's' or 'l']
run number to access, e.g. 'run034'
first : int [if source ends 's' or 'l']
First frame to access
last : int [if source ends 's' or 'l']
Last frame to access, 0 for the lot
ndigit : int [if source ends 's' or 'l' and not temp]
Files created will be written as 'run005_0013.fits'
etc. `ndigit` is the number of digits used for the frame
number (4 in this case). Any pre-existing files of the same
name will be over-written.
twait : float [hidden]
time to wait between attempts to find a new exposure, seconds.
tmax : float [hidden]
maximum time to wait between attempts to find a new exposure,
seconds.
flist : str [if source ends 'f']
name of file list. Output files will have the same names
as the input files with a prefix added
prefix : str [if source ends 'f' and not temp]
prefix to add to create output file names
trim : bool
True to trim columns and/or rows off the edges of windows
nearest the readout. Particularly useful for ULTRACAM
windowed data where the first few rows and columns can
contain bad data.
ncol : int [if trim, hidden]
Number of columns to remove (on left of left-hand window,
and right of right-hand windows)
nrow : int [if trim, hidden]
Number of rows to remove (bottom of windows)
bias : str
Name of bias frame to subtract, 'none' to ignore.
dark : str
Name of dark frame, 'none' to ignore.
flat : str
Name of flat field to divide by, 'none' to ignore. Should normally
only be used in conjunction with a bias, although it does allow you
to specify a flat even if you haven't specified a bias.
fmap : str
Name of fringe map (see e.g. `makefringe`), 'none' to ignore.
fpair : str [if fmap is not 'none']
Name of fringe pair file (see e.g. `setfringe`). Required if
a fringe map has been specified.
nhalf : int [if fmap is not 'none', hidden]
When calculating the differences for fringe measurement,
a region extending +/-nhalf binned pixels will be used when
measuring the amplitudes. Basically helps the stats.
rmin : float [if fmap is not 'none', hidden]
Minimum individual ratio to accept prior to calculating the overall
median in order to reduce the effect of outliers. Although all ratios
should be positive, you might want to set this a little below zero
to allow for some statistical fluctuation.
rmax : float [if fmap is not 'none', hidden]
Maximum individual ratio to accept prior to calculating the overall
median in order to reduce the effect of outliers. Probably typically
< 1 if fringe map was created from longer exposure data.
verbose : bool
True to print lots of details of fringe ratios
dtype : string [hidden, defaults to 'f32']
Data type on output. Options:
| 'f32' : output as 32-bit floats (default)
| 'f64' : output as 64-bit floats.
| 'u16' : output as 16-bit unsigned integers. A warning will be
issued if loss of precision occurs; an exception will
be raised if the data are outside the range 0 to 65535.
.. Note::
|grab| is used by several other scripts such as |averun| so take great
care when changing anything to do with its input parameters.
If you use the "temp" option to write to temporary files, then those
files will be deleted if you interrup with CTRL-C. This is to prevent
the accumulation of such frames.
"""
command, args = utils.script_args(args)
# get inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# register parameters
cl.register("source", Cline.GLOBAL, Cline.HIDE)
cl.register("temp", Cline.GLOBAL, Cline.HIDE)
cl.register("run", Cline.GLOBAL, Cline.PROMPT)
cl.register("first", Cline.LOCAL, Cline.PROMPT)
cl.register("last", Cline.LOCAL, Cline.PROMPT)
cl.register("ndigit", Cline.LOCAL, Cline.PROMPT)
cl.register("twait", Cline.LOCAL, Cline.HIDE)
cl.register("tmax", Cline.LOCAL, Cline.HIDE)
cl.register("flist", Cline.GLOBAL, Cline.PROMPT)
cl.register("prefix", Cline.GLOBAL, Cline.PROMPT)
cl.register("trim", Cline.GLOBAL, Cline.PROMPT)
cl.register("ncol", Cline.GLOBAL, Cline.HIDE)
cl.register("nrow", Cline.GLOBAL, Cline.HIDE)
cl.register("bias", Cline.LOCAL, Cline.PROMPT)
cl.register("flat", Cline.LOCAL, Cline.PROMPT)
cl.register("dark", Cline.LOCAL, Cline.PROMPT)
cl.register("fmap", Cline.LOCAL, Cline.PROMPT)
cl.register("fpair", Cline.LOCAL, Cline.PROMPT)
cl.register("nhalf", Cline.GLOBAL, Cline.HIDE)
cl.register("rmin", Cline.GLOBAL, Cline.HIDE)
cl.register("rmax", Cline.GLOBAL, Cline.HIDE)
cl.register("verbose", Cline.LOCAL, Cline.HIDE)
cl.register("dtype", Cline.LOCAL, Cline.HIDE)
# get inputs
default_source = os.environ.get('HIPERCAM_DEFAULT_SOURCE', 'hl')
source = cl.get_value(
"source",
"data source [hs, hl, us, ul, hf]",
default_source,
lvals=("hs", "hl", "us", "ul", "hf"),
)
cl.set_default("temp", False)
temp = cl.get_value(
"temp", "save to temporary automatically-generated file names?",
True)
# set some flags
server_or_local = source.endswith("s") or source.endswith("l")
if server_or_local:
resource = cl.get_value("run", "run name", "run005")
first = cl.get_value("first", "first frame to grab", 1, 0)
last = cl.get_value("last", "last frame to grab", 0)
if last < first and last != 0:
sys.stderr.write("last must be >= first or 0")
sys.exit(1)
if not temp:
ndigit = cl.get_value("ndigit",
"number of digits in frame identifier",
3, 0)
twait = cl.get_value("twait",
"time to wait for a new frame [secs]", 1.0,
0.0)
tmax = cl.get_value("tmax",
"maximum time to wait for a new frame [secs]",
10.0, 0.0)
else:
resource = cl.get_value("flist", "file list",
cline.Fname("files.lis", hcam.LIST))
if not temp:
prefix = cl.get_value("prefix",
"prefix to add to file names on output",
"pfx_")
first = 1
trim = cl.get_value("trim", "do you want to trim edges of windows?",
True)
if trim:
ncol = cl.get_value("ncol",
"number of columns to trim from windows", 0)
nrow = cl.get_value("nrow", "number of rows to trim from windows",
0)
bias = cl.get_value(
"bias",
"bias frame ['none' to ignore]",
cline.Fname("bias", hcam.HCAM),
ignore="none",
)
if bias is not None:
# read the bias frame
bias = hcam.MCCD.read(bias)
# dark
dark = cl.get_value("dark",
"dark frame ['none' to ignore]",
cline.Fname("dark", hcam.HCAM),
ignore="none")
if dark is not None:
# read the dark frame
dark = hcam.MCCD.read(dark)
# flat
flat = cl.get_value("flat",
"flat field frame ['none' to ignore]",
cline.Fname("flat", hcam.HCAM),
ignore="none")
if flat is not None:
# read the flat field frame
flat = hcam.MCCD.read(flat)
# fringe file (if any)
fmap = cl.get_value(
"fmap",
"fringe map ['none' to ignore]",
cline.Fname("fmap", hcam.HCAM),
ignore="none",
)
if fmap is not None:
# read the fringe frame and prompt other parameters
fmap = hcam.MCCD.read(fmap)
fpair = cl.get_value("fpair", "fringe pair file",
cline.Fname("fringe", hcam.FRNG))
fpair = fringe.MccdFringePair.read(fpair)
nhalf = cl.get_value("nhalf",
"half-size of fringe measurement regions", 2,
0)
rmin = cl.get_value("rmin", "minimum fringe pair ratio", -0.2)
rmax = cl.get_value("rmax", "maximum fringe pair ratio", 1.0)
verbose = cl.get_value("verbose", "verbose output", False)
cl.set_default("dtype", "f32")
dtype = cl.get_value("dtype",
"data type [f32, f64, u16]",
"f32",
lvals=("f32", "f64", "u16"))
# Now the actual work.
# strip off extensions
if resource.endswith(hcam.HRAW):
resource = resource[:resource.find(hcam.HRAW)]
# initialisations
total_time = 0 # time waiting for new frame
nframe = first
root = os.path.basename(resource)
bframe = None
# Finally, we can go
fnames = []
if temp:
tdir = utils.temp_dir()
with CleanUp(fnames, temp) as cleanup:
# The above line to handle ctrl-c and temporaries
with spooler.data_source(source, resource, first) as spool:
for mccd in spool:
if server_or_local:
# Handle the waiting game ...
give_up, try_again, total_time = spooler.hang_about(
mccd, twait, tmax, total_time)
if give_up:
print("grab stopped")
break
elif try_again:
continue
# Trim the frames: ULTRACAM windowed data has bad
# columns and rows on the sides of windows closest to
# the readout which can badly affect reduction. This
# option strips them.
if trim:
hcam.ccd.trim_ultracam(mccd, ncol, nrow)
if nframe == first:
# First time through, need to manipulate calibration data
if bias is not None:
bias = bias.crop(mccd)
bexpose = bias.head.get("EXPTIME", 0.0)
else:
bexpose = 0.
if flat is not None:
flat = flat.crop(mccd)
if dark is not None:
dark = dark.crop(mccd)
if fmap is not None:
fmap = fmap.crop(mccd)
fpair = fpair.crop(mccd, nhalf)
# now any time through, apply calibrations
if bias is not None:
mccd -= bias
if dark is not None:
# subtract dark, CCD by CCD
dexpose = dark.head["EXPTIME"]
for cnam in mccd:
ccd = mccd[cnam]
if ccd.is_data():
cexpose = ccd.head["EXPTIME"]
scale = (cexpose - bexpose) / dexpose
ccd -= scale * dark[cnam]
if flat is not None:
mccd /= flat
if fmap is not None:
# apply CCD by CCD
for cnam in fmap:
if cnam in fpair:
ccd = mccd[cnam]
if ccd.is_data():
if verbose:
print(f' CCD {cnam}')
fscale = fpair[cnam].scale(ccd,
fmap[cnam],
nhalf,
rmin,
rmax,
verbose=verbose)
if verbose:
print(f' Median scale factor = {fscale}')
ccd -= fscale * fmap[cnam]
if dtype == "u16":
mccd.uint16()
elif dtype == "f32":
mccd.float32()
elif dtype == "f64":
mccd.float64()
# write to disk
if temp:
# generate name automatically
fd, fname = tempfile.mkstemp(suffix=hcam.HCAM, dir=tdir)
fnames.append(fname)
mccd.write(fname, True)
os.close(fd)
elif server_or_local:
fname = f"{root}_{nframe:0{ndigit}}{hcam.HCAM}"
mccd.write(fname, True)
else:
fname = utils.add_extension(
f"{prefix}{mccd.head['FILENAME']}")
mccd.write(fname, True)
print("Written frame {:d} to {:s}".format(nframe, fname))
# update the frame number
nframe += 1
if server_or_local and last and nframe > last:
break
if temp:
if len(fnames) == 0:
raise hcam.HipercamError(
'no files were grabbed; please check'
' input parameters, especially "first"')
# write the file names to a list
fd, fname = tempfile.mkstemp(suffix=hcam.LIST, dir=tdir)
cleanup.flist = fname
with open(fname, "w") as fout:
for fnam in fnames:
fout.write(fnam + "\n")
os.close(fd)
print("temporary file names written to {:s}".format(fname))
# return the name of the file list
return fname
print("grab finished")
