def hmeta(args=None):
description = \
"""hmeta
This command is to be run in the "raw_data" directory containing
night-by-night directories of data for |hipercam|, ULTRACAM or
ULTRASPEC. It computes data on every runs it can find listing the
mininum, maximum, mean and median and the following percentiles:
1.0,5.0,15.865,84.135,95.0,99.0. It does this for all frames in a
run up to a maximum of 100-200 frames. The data are listed along
with the frame number separately for each CCD since this only looks
at valid data, i.e. it takes into account nblue for ULTRACAM and nskips
for |hiper|. This means there may be different numbers of frames listed
for each CCD. It writes a file runXXX[X].csv for each run containing data
to a sub-directory called meta.
The program only considers genuine frames, i.e. it copes with the
nblue and nskip options of ULTRACAM and |hipercam|. It also knows
about the different amplifier outputs of the instruments and
starts by subtracting the medians of each amplifier output to
avoid being disturbed by variable mean bias offsets. These offsets
are recorded as well.
hmeta takes a good while to run, so be nice when running it.
"""
parser = argparse.ArgumentParser(description=description)
parser.add_argument(
"-n",
dest="night",
help="name of specific night [for testing purposes]",
)
args = parser.parse_args()
cwd = os.getcwd()
if os.path.basename(cwd) != "raw_data":
print("** hmeta must be run in a directory called 'raw_data'")
print("hmeta aborted", file=sys.stderr)
return
if cwd.find("ultracam") > -1:
instrument = "ULTRACAM"
itype = 'U'
source = 'ul'
cnams = ('1', '2', '3')
elif cwd.find("ultraspec") > -1:
instrument = "ULTRASPEC"
itype = 'U'
source = 'ul'
cnams = ('1', )
elif cwd.find("hipercam") > -1:
instrument = "HiPERCAM"
itype = 'H'
source = 'hl'
cnams = ('1', '2', '3', '4', '5')
else:
print("** hmeta: cannot find either ultracam, "
"ultraspec or hipercam in path")
print("hmeta aborted", file=sys.stderr)
return
warnings.filterwarnings('ignore')
linstrument = instrument.lower()
# Now the actual work. Next are regular expressions to match run
# directories, nights, and run files
nre = re.compile("^\d\d\d\d-\d\d-\d\d$")
ure = re.compile("^run\d\d\d\.xml$")
hre = re.compile("^run\d\d\d\d\.fits$")
# Get list of night directories
nnames = [
nname for nname in os.listdir(".")
if nre.match(nname) and os.path.isdir(nname) and (
args.night is None or nname == args.night)
]
nnames.sort()
if len(nnames) == 0:
print("no night directories found", file=sys.stderr)
print("hmeta aborted", file=sys.stderr)
return
# This defines approx how many we will analyse; could be up to
# double this in practice
NLIM = 100
for nname in nnames:
print(f"Night {nname}")
# load all the run names
if itype == 'U':
runs = [
run[:-4] for run in os.listdir(nname) if ure.match(run)
and os.path.exists(os.path.join(nname, run[:-4] + '.dat'))
]
else:
runs = [run[:-5] for run in os.listdir(nname) if hre.match(run)]
runs.sort()
if len(runs) == 0:
print(f' No runs with data found in {nname}; skipping')
continue
# create directory for any meta info such as the times
meta = os.path.join(nname, 'meta')
os.makedirs(meta, exist_ok=True)
# Accumulate results in an array
for run in runs:
dfile = os.path.join(nname, run)
# check for existence of data files
for cnam in cnams:
oname = os.path.join(meta, f'{run}_{cnam}.csv')
if not os.path.exists(oname):
break
else:
if len(cnams) > 1:
print(
f' {dfile}: stats files already exist and will not be re-created'
)
else:
print(
f' {dfile}: stats file already exists and will not be re-created'
)
continue
try:
if itype == 'U':
rdat = hcam.ucam.Rdata(dfile)
else:
rdat = hcam.hcam.Rdata(dfile, 1, False, False)
except hcam.ucam.PowerOnOffError:
print(f' {dfile} -- power on/off; skipping')
continue
except:
# some other failure
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_tb(exc_traceback, limit=1, file=sys.stderr)
traceback.print_exc(file=sys.stderr)
print(f' {dfile} -- problem occurred; skipping')
continue
# For speed, analyse a maximum of NLIM to 2*NLIM-1
# images of each CCD from any given run. Have to take
# into account the skips / nblue parameters.
ntotal = rdat.ntotal()
if ntotal == 0:
print(f' {dfile} -- zero frames; skipping')
continue
ncframes = {}
if linstrument == 'ultraspec':
# just the one CCD here. nstep defines how often we sample.
# note that ntotal has to be 2*NLIM before nstep > 1.
nstep = ntotal // min(ntotal, NLIM)
ncframes['1'] = list(range(1, ntotal + 1, nstep))
nframes = ncframes['1']
elif linstrument == 'ultracam':
# CCD 1, 2 read out each time, but 3 can be skipped
nstep = ntotal // min(ntotal, NLIM)
ncframes['1'] = list(range(1, ntotal + 1, nstep))
ncframes['2'] = ncframes['1']
nb = rdat.nblue
nbstep = nb * max(1, ntotal // min(ntotal, NLIM * nb))
ncframes['3'] = list(range(nb, ntotal + 1, nbstep))
nframes = sorted(set(ncframes['1'] + ncframes['3']))
else:
# All CCDs potentially skippable
nskips = rdat.nskips
nsteps = []
for cnam, nskip in zip(cnams, nskips):
nstep = (nskip + 1) * max(
1, ntotal // min(ntotal, NLIM * (nskip + 1)))
ncframes[cnam] = list(range(nskip + 1, ntotal + 1, nstep))
# sorted list of frames to access
nframes = sorted(
set(ncframes['1'] + ncframes['2'] + ncframes['3'] +
ncframes['4'] + ncframes['5']))
# index counters
ns = dict(zip(cnams, len(cnams) * [0]))
# define arrays for holding the stats
medians, means, mins, p1s, p5s, p16s, p84s, p95s, p99s, maxs = \
{}, {}, {}, {}, {}, {}, {}, {}, {}, {}
for cnam, ncframe in ncframes.items():
if linstrument == 'ultraspec':
medians[cnam] = np.empty_like(ncframe, dtype=np.float)
elif linstrument == 'ultracam':
medians[cnam] = {
'L': np.empty_like(ncframe, dtype=np.float),
'R': np.empty_like(ncframe, dtype=np.float)
}
elif linstrument == 'hipercam':
medians[cnam] = {
'E': np.empty_like(ncframe, dtype=np.float),
'F': np.empty_like(ncframe, dtype=np.float),
'G': np.empty_like(ncframe, dtype=np.float),
'H': np.empty_like(ncframe, dtype=np.float)
}
means[cnam] = np.empty_like(ncframe, dtype=np.float)
mins[cnam] = np.empty_like(ncframe, dtype=np.float)
p1s[cnam] = np.empty_like(ncframe, dtype=np.float)
p5s[cnam] = np.empty_like(ncframe, dtype=np.float)
p16s[cnam] = np.empty_like(ncframe, dtype=np.float)
p84s[cnam] = np.empty_like(ncframe, dtype=np.float)
p95s[cnam] = np.empty_like(ncframe, dtype=np.float)
p99s[cnam] = np.empty_like(ncframe, dtype=np.float)
maxs[cnam] = np.empty_like(ncframe, dtype=np.float)
# now access the data and calculate stats. we have to
# remember that ultracam and hipercam have different
# readout amps so we subtract median values calculated for
# each amp separately, which is a little painful.
for n, nf in enumerate(nframes):
try:
mccd = rdat(nf)
for cnam, ncframe in ncframes.items():
if nf in ncframe:
ccd = mccd[cnam]
nc = ns[cnam]
if linstrument == 'ultraspec':
medval = ccd.median()
ccd -= medval
medians[cnam][nc] = medval
elif linstrument == 'ultracam':
wl = hcam.Group(hcam.Window)
wr = hcam.Group(hcam.Window)
for nw, wnam in enumerate(ccd):
if nw % 2 == 0:
wl[wnam] = ccd[wnam]
else:
wr[wnam] = ccd[wnam]
ccdl = hcam.CCD(wl, ccd.nxtot, ccd.nytot)
medl = ccdl.median()
ccdl -= medl
ccdr = hcam.CCD(wr, ccd.nxtot, ccd.nytot)
medr = ccdr.median()
ccdr -= medr
medians[cnam]['L'][nc] = medl
medians[cnam]['R'][nc] = medr
elif linstrument == 'hipercam':
we = hcam.Group(hcam.Window)
wf = hcam.Group(hcam.Window)
wg = hcam.Group(hcam.Window)
wh = hcam.Group(hcam.Window)
for nw, wnam in enumerate(ccd):
if wnam.startswith('E'):
we[wnam] = ccd[wnam]
elif wnam.startswith('F'):
wf[wnam] = ccd[wnam]
elif wnam.startswith('G'):
wg[wnam] = ccd[wnam]
elif wnam.startswith('H'):
wh[wnam] = ccd[wnam]
ccde = hcam.CCD(we, ccd.nxtot, ccd.nytot)
mede = ccde.median()
ccde -= mede
ccdf = hcam.CCD(wf, ccd.nxtot, ccd.nytot)
medf = ccdf.median()
ccdf -= medf
ccdg = hcam.CCD(wg, ccd.nxtot, ccd.nytot)
medg = ccdg.median()
ccdg -= medg
ccdh = hcam.CCD(wh, ccd.nxtot, ccd.nytot)
medh = ccdh.median()
ccdh -= medh
medians[cnam]['E'][nc] = mede
medians[cnam]['F'][nc] = medf
medians[cnam]['G'][nc] = medg
medians[cnam]['H'][nc] = medh
# At this stage the median value should
# have been subtracted from the CCD on a
# per output basis. Remaining stats
# calculated from these median subtracted
# images.
means[cnam][nc] = ccd.mean()
mins[cnam][nc] = ccd.min()
p1, p5, p16, p84, p95, p99 = ccd.percentile(
[1.0, 5.0, 15.865, 84.135, 95.0, 99.0])
p1s[cnam][nc] = p1
p5s[cnam][nc] = p5
p16s[cnam][nc] = p16
p84s[cnam][nc] = p84
p95s[cnam][nc] = p95
p99s[cnam][nc] = p99
maxs[cnam][nc] = ccd.max()
ns[cnam] += 1
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_tb(exc_traceback, limit=1, file=sys.stderr)
traceback.print_exc(file=sys.stderr)
continue
# All numbers extracted from the run. Now want to create
# pandas dataframe for easy output
# Create pandas dataframe for easy output. First names and
# datatypes
cnames, dtypes = get_cnames_dtypes(linstrument)
for cnam in cnams:
# one file per CCD because of potentially different
# column lengths
oname = os.path.join(meta, f'{run}_{cnam}.csv')
# Form output array
barr = [
ncframes[cnam],
]
if linstrument == 'ultraspec':
barr.append(medians[cnam])
elif linstrument == 'ultracam':
barr += [medians[cnam]['L'], medians[cnam]['R']]
elif linstrument == 'hipercam':
barr += [
medians[cnam]['E'], medians[cnam]['F'],
medians[cnam]['G'], medians[cnam]['H']
]
barr += [
means[cnam], mins[cnam], p1s[cnam], p5s[cnam], p16s[cnam],
p84s[cnam], p95s[cnam], p99s[cnam], maxs[cnam]
]
# Make pandas dataframe
table = pd.DataFrame(data=np.column_stack(barr),
columns=cnames)
table = table.astype(dtypes)
table.to_csv(oname, index=False)
print(f' {dfile}, written stats to {oname}')
def makemccd(args=None):
"""Script to generate multi-CCD test data given a set of parameters
defined in a config file (parsed using configparser). This allows
things such as a bias frame, flat field variations offsets, scale
factor and rotations between CCDs, and temporal variations.
Arguments::
config : string
file defining the parameters.
parallel : bool
True / yes etc to run in parallel. Be warned: it does not
always make things faster, which I assume is the result of
overheads when parallelising. It will simply use whatever
CPUs are available.
Depending upon the setting in the config file, this could generate a large
number of different files and so the first time you run it, you may want
to do so in a clean directory.
Config file format: see the documentation of configparser for the general
format of the config files expected by this routine. Essentially there are
a series of sections, e.g.:
[ccd 1]
nxtot = 2048
nytot = 1048
.
.
.
which define all the parameters needed. There are many others to simulate
a bias offset, a flat field; see the example file ??? for a fully-documented
version.
"""
import configparser
global _gframe, _gfield
command, args = utils.script_args(args)
# get inputs
with Cline("HIPERCAM_ENV", ".hipercam", command, args) as cl:
# Register parameters
cl.register("config", Cline.LOCAL, Cline.PROMPT)
cl.register("parallel", Cline.LOCAL, Cline.PROMPT)
# Prompt for them
config = cl.get_value("config", "configuration file",
cline.Fname("config"))
parallel = cl.get_value("parallel", "add targets in parallel?", False)
# Read the config file
conf = configparser.ConfigParser()
conf.read(config)
# Determine whether files get overwritten or not
overwrite = (conf.getboolean("general", "overwrite")
if "overwrite" in conf["general"] else False)
dtype = conf["general"]["dtype"] if "dtype" in conf["general"] else None
# Top-level header
thead = fits.Header()
thead.add_history("Created by makedata")
# Store the CCD labels and parameters and their dimensions. Determine
# maximum dimensions for later use when adding targets
ccd_pars = Odict()
maxnx = 0
maxny = 0
for key in conf:
if key.startswith("ccd"):
# translate parameters
nxtot = int(conf[key]["nxtot"])
nytot = int(conf[key]["nytot"])
xcen = float(conf[key]["xcen"])
ycen = float(conf[key]["ycen"])
angle = float(conf[key]["angle"])
scale = float(conf[key]["scale"])
xoff = float(conf[key]["xoff"])
yoff = float(conf[key]["yoff"])
fscale = float(conf[key]["fscale"])
toff = float(conf[key]["toff"])
field = (hcam.Field.rjson(conf[key]["field"])
if "field" in conf[key] else None)
ndiv = int(conf[key]["ndiv"]) if field is not None else None
back = float(conf[key]["back"])
# determine maximum total dimension
maxnx = max(maxnx, nxtot)
maxny = max(maxny, nytot)
# store parameters
ccd_pars[key[3:].strip()] = {
"nxtot": nxtot,
"nytot": nytot,
"xcen": xcen,
"ycen": ycen,
"angle": angle,
"scale": scale,
"xoff": xoff,
"yoff": yoff,
"fscale": fscale,
"toff": toff,
"field": field,
"ndiv": ndiv,
"back": back,
}
if not len(ccd_pars):
raise ValueError("hipercam.makedata: no CCDs found in " + config)
# get the timing data
utc_start = Time(conf["timing"]["utc_start"])
exposure = float(conf["timing"]["exposure"])
deadtime = float(conf["timing"]["deadtime"])
# Generate the CCDs, store the read / gain values
ccds = hcam.Group(hcam.CCD)
rgs = {}
for cnam, pars in ccd_pars.items():
# Generate header with timing data
head = fits.Header()
td = TimeDelta(pars["toff"], format="sec")
utc = utc_start + td
head["UTC"] = (utc.isot, "UTC at mid exposure")
head["MJD"] = (utc.mjd, "MJD at mid exposure")
head["EXPOSE"] = (exposure, "Exposure time, seconds")
head["TIMEOK"] = (True, "Time status flag")
# Generate the Windows
winds = hcam.Group(hcam.Window)
rgs[cnam] = {}
for key in conf:
if key.startswith("window"):
iccd, wnam = key[6:].split()
if iccd == cnam:
llx = int(conf[key]["llx"])
lly = int(conf[key]["lly"])
nx = int(conf[key]["nx"])
ny = int(conf[key]["ny"])
xbin = int(conf[key]["xbin"])
ybin = int(conf[key]["ybin"])
if len(winds):
wind = hcam.Window(
hcam.Winhead(llx, lly, nx, ny, xbin, ybin))
else:
# store the header in the first Window
wind = hcam.Window(
hcam.Winhead(llx, lly, nx, ny, xbin, ybin, head))
# Store the Window
winds[wnam] = wind
# Store read / gain value
rgs[cnam][wnam] = (
float(conf[key]["read"]),
float(conf[key]["gain"]),
)
# Accumulate CCDs
ccds[cnam] = hcam.CCD(winds, pars["nxtot"], pars["nytot"])
# Make the template MCCD
mccd = hcam.MCCD(ccds, thead)
# Make a flat field
flat = mccd.copy()
if "flat" in conf:
rms = float(conf["flat"]["rms"])
for ccd in flat.values():
# Generate dust
nspeck = int(conf["flat"]["nspeck"])
if nspeck:
radius = float(conf["flat"]["radius"])
depth = float(conf["flat"]["depth"])
specks = []
for n in range(nspeck):
x = np.random.uniform(0.5, ccd.nxtot + 0.5)
y = np.random.uniform(0.5, ccd.nytot + 0.5)
specks.append(Dust(x, y, radius, depth))
# Set the flat field values
for wind in ccd.values():
wind.data = np.random.normal(1.0, rms, (wind.ny, wind.nx))
if nspeck:
wind.add_fxy(specks)
flat.head["DATATYPE"] = ("Flat field", "Artificially generated")
fname = utils.add_extension(conf["flat"]["flat"], hcam.HCAM)
flat.write(fname, overwrite)
print("Saved flat field to ", fname)
else:
# Set the flat to unity
flat.set_const(1.0)
print("No flat field generated")
# Make a bias frame
bias = mccd.copy()
if "bias" in conf:
mean = float(conf["bias"]["mean"])
rms = float(conf["bias"]["rms"])
for ccd in bias.values():
for wind in ccd.values():
wind.data = np.random.normal(mean, rms, (wind.ny, wind.nx))
bias.head["DATATYPE"] = ("Bias frame", "Artificially generated")
fname = utils.add_extension(conf["bias"]["bias"], hcam.HCAM)
bias.write(fname, overwrite)
print("Saved bias frame to ", fname)
else:
# Set the bias to zero
bias.set_const(0.0)
print("No bias frame generated")
# Everything is set to go, so now generate data files
nfiles = int(conf["files"]["nfiles"])
if nfiles == 0:
out = mccd * flat + bias
fname = utils.add_extension(conf["files"]["root"], hcam.HCAM)
out.write(fname, overwrite)
print("Written data to", fname)
else:
# file naming info
root = conf["files"]["root"]
ndigit = int(conf["files"]["ndigit"])
# movement
xdrift = float(conf["movement"]["xdrift"])
ydrift = float(conf["movement"]["ydrift"])
nreset = int(conf["movement"]["nreset"])
jitter = float(conf["movement"]["jitter"])
print("Now generating data")
tdelta = TimeDelta(exposure + deadtime, format="sec")
for nfile in range(nfiles):
# copy over template (into a global variable for multiprocessing speed)
_gframe = mccd.copy()
# get x,y offset
xoff = np.random.normal(xdrift * (nfile % nreset), jitter)
yoff = np.random.normal(ydrift * (nfile % nreset), jitter)
# create target fields for each CCD, add background
_gfield = {}
for cnam in _gframe.keys():
p = ccd_pars[cnam]
_gframe[cnam] += p["back"]
if p["field"] is not None:
# get field modification settings
transform = Transform(
p["nxtot"],
p["nytot"],
p["xcen"],
p["ycen"],
p["angle"],
p["scale"],
p["xoff"] + xoff,
p["yoff"] + yoff,
)
fscale = p["fscale"]
_gfield[cnam] = p["field"].modify(transform, fscale)
# add the targets in (slow step)
if parallel:
# run in parallel on whatever cores are available
args = [(cnam, ccd_pars[cnam]["ndiv"]) for cnam in _gfield]
with Pool() as pool:
ccds = pool.map(worker, args)
for cnam in _gfield:
_gframe[cnam] = ccds.pop(0)
else:
# single core
for cnam in _gfield:
ccd = _gframe[cnam]
ndiv = ccd_pars[cnam]["ndiv"]
field = _gfield[cnam]
for wind in ccd.values():
field.add(wind, ndiv)
# Apply flat
_gframe *= flat
# Add noise
for cnam, ccd in _gframe.items():
for wnam, wind in ccd.items():
readout, gain = rgs[cnam][wnam]
wind.add_noise(readout, gain)
# Apply bias
_gframe += bias
# data type on output
if dtype == "float32":
_gframe.float32()
elif dtype == "uint16":
_gframe.uint16()
# Save
fname = "{0:s}{1:0{2:d}d}{3:s}".format(root, nfile + 1, ndigit,
hcam.HCAM)
_gframe.write(fname, overwrite)
print("Written file {0:d} to {1:s}".format(nfile + 1, fname))
# update times in template
for ccd in mccd.values():
head = ccd.head
utc = Time(head["UTC"]) + tdelta
head["UTC"] = (utc.isot, "UTC at mid exposure")
head["MJD"] = (utc.mjd, "MJD at mid exposure")
def setdefect(args=None):
"""``setdefect mccd defect ccd [linput width height] rtarg rsky1 rsky2 nx
msub iset (ilo ihi | plo phi) [profit method beta fwmin fwhm fwfix
shbox smooth splot fhbox read gain thresh]``
Interactive definition of CCD defects. This is a matplotlib-based routine
allowing you to define defects using the cursor.
Parameters:
mccd : string
name of an MCCD file, as produced by e.g. 'grab'
defect : string
the name of a defect file. If it exists it will be read so that
defects can be added to it. If it does not exist, it will be
created on exiting the routine. The defect files are in a fairly
readable / editiable text format
ccd : string
CCD(s) to plot, '0' for all. If not '0' then '1', '2' or even '3 4'
are possible inputs (without the quotes). '3 4' will plot CCD '3' and
CCD '4'. If you want to plot more than one CCD, then you will be
prompted for the number of panels in the X direction. This parameter
will not be prompted if there is only one CCD in the file.
width : float [hidden]
plot width (inches). Set = 0 to let the program choose.
height : float [hidden]
plot height (inches). Set = 0 to let the program choose. BOTH width
AND height must be non-zero to have any effect
nx : int
number of panels across to display, prompted if more than one CCD is
to be plotted.
msub : bool
True/False to subtract median from each window before scaling
invert : bool [if msub]
If msub is True, then you can invert the image values (-ve to +ve)
with this parameter. Can make it easier to spot bad values.
ffield : bool
If True, all defects will be assumed to be flat-field or poor
charge transfer defects as opposed to hot pixels. The latter are
best set from dark frames, and have a different impact than the
first two types in that they are worst for faint targets. Hot pixels
and flat-field defects are shown with the same colours for moderate
and severe, but different symbols (filled circles for flat-field
defects, stars for hot pixels). If you say no to add hot pixels,
the line defect option is not available.
hsbox : int
half-width in binned pixels of stats box as offset from central pixel
hsbox = 1 gives a 3x3 box; hsbox = 2 gives 5x5 etc. This is used by
the "show" option when setting defects.
iset : string [single character]
determines how the intensities are determined. There are three
options: 'a' for automatic simply scales from the minimum to the
maximum value found on a per CCD basis. 'd' for direct just takes two
numbers from the user. 'p' for percentile dtermines levels based upon
percentiles determined from the entire CCD on a per CCD bais.
ilo : float [if iset=='d']
lower intensity level
ihi : float [if iset=='d']
upper intensity level
plo : float [if iset=='p']
lower percentile level
phi : float [if iset=='p']
upper percentile level
There are a few conveniences to make setdefect easier:
1. The plot is initialised in pan mode whereby you can move around and
scale using the left and right mouse buttons.
2. All input is accomplished with the keyboard; the mouse buttons are
only for navigating the image.
3. The label input can be switched between sequential numerical,
single- and multi-character input ('linput').
Various standard keyboard shortcuts (e.g. 's' to save) are disabled as
they just confuse things and are of limited use in setdefect in any case.
Some aspects of the usage of matplotlib in setdefect are tricky. It is
possible that particular 'backends' will cause problems. I have tested
this with Qt4Agg, Qt5agg and GTK3Agg. One aspect is the cursor icon in pan
mode is a rather indistinct hand where one can't tell what is being
pointed at. I have therefore suppressed this, but only for the tested
backends. Others would need require further investigation.
NB At the end of this routine, it re-orders the defects so that the severe
ones follows the moderates. This helps emphasize the severe ones over the
moderates when running rtplot.
"""
command, args = utils.script_args(args)
# get input section
with Cline('HIPERCAM_ENV', '.hipercam', command, args) as cl:
# register parameters
cl.register('mccd', Cline.LOCAL, Cline.PROMPT)
cl.register('defect', Cline.LOCAL, Cline.PROMPT)
cl.register('ccd', Cline.LOCAL, Cline.PROMPT)
cl.register('width', Cline.LOCAL, Cline.HIDE)
cl.register('height', Cline.LOCAL, Cline.HIDE)
cl.register('nx', Cline.LOCAL, Cline.PROMPT)
cl.register('msub', Cline.GLOBAL, Cline.PROMPT)
cl.register('invert', Cline.GLOBAL, Cline.PROMPT)
cl.register('ffield', Cline.GLOBAL, Cline.PROMPT)
cl.register('hsbox', Cline.GLOBAL, Cline.HIDE)
cl.register('iset', Cline.GLOBAL, Cline.PROMPT)
cl.register('ilo', Cline.GLOBAL, Cline.PROMPT)
cl.register('ihi', Cline.GLOBAL, Cline.PROMPT)
cl.register('plo', Cline.GLOBAL, Cline.PROMPT)
cl.register('phi', Cline.GLOBAL, Cline.PROMPT)
# get inputs
mccd = cl.get_value('mccd', 'frame to plot',
cline.Fname('hcam', hcam.HCAM))
mccd = hcam.MCCD.read(mccd)
dfct = cl.get_value('defect', 'name of defect file',
cline.Fname('defect', hcam.DFCT, exist=False))
if os.path.exists(dfct):
# read in old defects
mccd_dfct = defect.MccdDefect.read(dfct)
print('Loaded existing file = {:s}'.format(dfct))
else:
# create empty container
mccd_dfct = defect.MccdDefect()
print('No file called {:s} exists; '
'will create from scratch'.format(dfct))
# define the panel grid
try:
nxdef = cl.get_default('nx')
except:
nxdef = 3
max_ccd = len(mccd)
if max_ccd > 1:
ccd = cl.get_value('ccd', 'CCD(s) to plot [0 for all]', '0')
if ccd == '0':
ccds = list(mccd.keys())
else:
ccds = ccd.split()
else:
ccds = list(mccd.keys())
width = cl.get_value('width', 'plot width (inches)', 0.)
height = cl.get_value('height', 'plot height (inches)', 0.)
# number of panels in X
if len(ccds) > 1:
nxdef = min(len(ccds), nxdef)
cl.set_default('nx', nxdef)
nx = cl.get_value('nx', 'number of panels in X', 3, 1)
else:
nx = 1
# define the display intensities
msub = cl.get_value('msub', 'subtract median from each window?', True)
if msub:
invert = cl.get_value('invert', 'invert image intensities?', True)
ffield = cl.get_value('ffield',
'flat field defects? [else hot pixels]', True)
hsbox = cl.get_value('hsbox',
'half-width of stats box (binned pixels)', 2, 1)
iset = cl.get_value('iset', 'set intensity a(utomatically),'
' d(irectly) or with p(ercentiles)?',
'a',
lvals=['a', 'A', 'd', 'D', 'p', 'P'])
iset = iset.lower()
plo, phi = 5, 95
ilo, ihi = 0, 1000
if iset == 'd':
ilo = cl.get_value('ilo', 'lower intensity limit', 0.)
ihi = cl.get_value('ihi', 'upper intensity limit', 1000.)
elif iset == 'p':
plo = cl.get_value('plo', 'lower intensity limit percentile', 5.,
0., 100.)
phi = cl.get_value('phi', 'upper intensity limit percentile', 95.,
0., 100.)
nxmax, nymax = 0, 0
for cnam in ccds:
nxmax = max(nxmax, mccd[cnam].nxtot)
nymax = max(nymax, mccd[cnam].nytot)
# might be worth trying to improve this at some point
xlo, xhi, ylo, yhi = 0, nxmax + 1, 0, nymax + 1
# Inputs obtained.
# re-configure keyboard shortcuts to avoid otherwise confusing behaviour
# quit_all does not seem to be universal, hence the try/except
try:
mpl.rcParams['keymap.back'] = ''
mpl.rcParams['keymap.forward'] = ''
mpl.rcParams['keymap.fullscreen'] = ''
mpl.rcParams['keymap.grid'] = ''
mpl.rcParams['keymap.home'] = ''
mpl.rcParams['keymap.pan'] = ''
mpl.rcParams['keymap.quit'] = ''
mpl.rcParams['keymap.save'] = ''
mpl.rcParams['keymap.pan'] = ''
mpl.rcParams['keymap.save'] = ''
mpl.rcParams['keymap.xscale'] = ''
mpl.rcParams['keymap.yscale'] = ''
mpl.rcParams['keymap.zoom'] = ''
except KeyError:
pass
# start plot
if width > 0 and height > 0:
fig = plt.figure(figsize=(width, height))
else:
fig = plt.figure()
# get the navigation toolbar. Go straight into pan mode
# where we want to stay.
toolbar = fig.canvas.manager.toolbar
toolbar.pan()
nccd = len(ccds)
ny = nccd // nx if nccd % nx == 0 else nccd // nx + 1
# we need to store some stuff
ax = None
cnams = {}
anams = {}
# this is a container for all the objects used to plot Defects to allow
# deletion. This is Group of Group objects supporting tuple storage. The
# idea is that pobjs[cnam][anam] returns the objects used to plot Defect
# anam of CCD cnam. It is initially empty,
pobjs = hcam.Group(hcam.Group)
for n, cnam in enumerate(ccds):
if ax is None:
axes = ax = fig.add_subplot(ny, nx, n + 1)
axes.set_aspect('equal', adjustable='box')
axes.set_xlim(xlo, xhi)
axes.set_ylim(ylo, yhi)
else:
axes = fig.add_subplot(ny, nx, n + 1, sharex=ax, sharey=ax)
axes.set_aspect('equal', adjustable='datalim')
if msub:
# subtract median from each window
for wind in mccd[cnam].values():
wind -= wind.median()
if invert:
mccd *= -1
hcam.mpl.pCcd(axes, mccd[cnam], iset, plo, phi, ilo, ihi,
'CCD {:s}'.format(cnam))
# keep track of the CCDs associated with each axes
cnams[axes] = cnam
# and axes associated with each CCD
anams[cnam] = axes
if cnam in mccd_dfct:
# plot any pre-existing Defects, keeping track of
# the plot objects
pobjs[cnam] = hcam.mpl.pCcdDefect(axes, mccd_dfct[cnam])
else:
# add in an empty CcdDefect for any CCD not already present
mccd_dfct[cnam] = defect.CcdDefect()
# and an empty container for any new plot objects
pobjs[cnam] = hcam.Group(tuple)
# create the Defect picker (see below for class def)
picker = PickDefect(mccd, cnams, anams, toolbar, fig, mccd_dfct, dfct,
ffield, hsbox, pobjs)
plt.tight_layout()
picker.action_prompt(False)
# squeeze space a bit
plt.subplots_adjust(wspace=0.1, hspace=0.1)
# finally show stuff ....
plt.show()
def psfaper(args=None):
"""``psfaper mccd aper ccd [linput width height] nx method thresh
niters gfac msub iset (ilo ihi | plo phi) [beta fwmin fwhm
shbox smooth splot fhbox read gain rejthresh]``
Definition of apertures for PSF photometry. This occurs in three
steps - first the user draws a box around the region of interest,
then the user selects a reference star which is used to measure the
PSF. Finally, all the stars in the region of interest are located
following an iterative application of FINDing stars, FITting the image
using a PSF model, SUBTRACTing the model.
An aperture file is automatically created, which can be edited
with |setaper| if necessary.
Parameters:
mccd : string
name of an MCCD file, as produced by e.g. 'grab'
aper : string
the name of an aperture file. If it exists it will be read so that
apertures can be added to it. If it does not exist, it will be
created on exiting the routine. The aperture files are is a fairly
readable / editiable text format
ccd : string
CCD(s) to plot, '0' for all. If not '0' then '1', '2' or even '3 4'
are possible inputs (without the quotes). '3 4' will plot CCD '3' and
CCD '4'. If you want to plot more than one CCD, then you will be
prompted for the number of panels in the X direction. This parameter
will not be prompted if there is only one CCD in the file.
linput : string [hidden]
sets the way in which the apertures are labelled. 'n' = numerical
input, with the program just incrementing by 1 for each successive
aperture; 's' = single character (without requiring the user to hit
hit <CR>); 'm' = multi-character, ending with <CR>.
Allowed characters are 0-9, a-z, A-Z, no spaces or punctuation, but a
single '0' on its own is not permitted.
width : float [hidden]
plot width (inches). Set = 0 to let the program choose.
height : float [hidden]
plot height (inches). Set = 0 to let the program choose. BOTH width
AND height must be non-zero to have any effect
nx : int
number of panels across to display, prompted if more than one CCD is
to be plotted.
method : string
this defines the profile function. Either a gaussian or a moffat
profile, 'g' or 'm'. The latter should usually be best.
thresh : float
this sets the threshold for detection of stars in the image,
in multiples of the background RMS
niters : int
when detecting stars, this sets the number of iterations of the
FIND-FIT-SUBTRACT loop.
gfac : float
in PSF fitting, stars are split into groups, and each group is fit
seperately. This is to avoid fitting models with large numbers of
free parameters. This number, multiplied by the FWHM, gives the
maximum seperation for stars to be fitted within the same group.
msub : bool
True/False to subtract median from each window before scaling
iset : string [single character]
determines how the intensities are determined. There are three
options: 'a' for automatic simply scales from the minimum to the
maximum value found on a per CCD basis. 'd' for direct just takes two
numbers from the user. 'p' for percentile dtermines levels based upon
percentiles determined from the entire CCD on a per CCD bais.
ilo : float [if iset=='d']
lower intensity level
ihi : float [if iset=='d']
upper intensity level
plo : float [if iset=='p']
lower percentile level
phi : float [if iset=='p']
upper percentile level
beta : float [if method == 'm'; hidden]
default Moffat exponent
fwmin : float [hidden]
minimum FWHM to allow, unbinned pixels.
fwhm : float [hidden]
default FWHM, unbinned pixels.
shbox : float [hidden]
half width of box for searching for a star, unbinned pixels. The
brightest target in a region +/- shbox around an intial position will
be found. 'shbox' should be large enough to allow for likely changes
in position from frame to frame, but try to keep it as small as you
can to avoid jumping to different targets and to reduce the chances
of interference by cosmic rays.
smooth : float [hidden]
FWHM for gaussian smoothing, binned pixels. The initial position for
fitting is determined by finding the maximum flux in a smoothed
version of the image in a box of width +/- shbox around the starter
position. Typically should be comparable to the stellar width. Its
main purpose is to combat cosmic rays which tend only to occupy a
single pixel.
fhbox : float [hidden]
half width of box for profile fit, unbinned pixels. The fit box is
centred on the position located by the initial search. It should
normally be > ~2x the expected FWHM.
read : float [hidden]
readout noise, RMS ADU, for assigning uncertainties
gain : float [hidden]
gain, ADU/count, for assigning uncertainties
rejthresh : float [hidden]
thresh rejection threshold
Various standard keyboard shortcuts (e.g. 's' to save) are disabled as
they just confuse things and are of limited use in setaper in any case.
Some aspects of the usage of matplotlib in psfaper are tricky. It is
possible that particular 'backends' will cause problems. I have tested
this with Qt4Agg, Qt5agg and GTK3Agg. One aspect is the cursor icon in pan
mode is a rather indistinct hand where one can't tell what is being
pointed at. I have therefore suppressed this, but only for the tested
backends. Others would need require further investigation.
"""
command, args = utils.script_args(args)
# get input section
with Cline('HIPERCAM_ENV', '.hipercam', command, args) as cl:
# register parameters
cl.register('mccd', Cline.LOCAL, Cline.PROMPT)
cl.register('aper', Cline.LOCAL, Cline.PROMPT)
cl.register('ccd', Cline.LOCAL, Cline.PROMPT)
cl.register('linput', Cline.LOCAL, Cline.HIDE)
cl.register('width', Cline.LOCAL, Cline.HIDE)
cl.register('height', Cline.LOCAL, Cline.HIDE)
cl.register('nx', Cline.LOCAL, Cline.PROMPT)
cl.register('method', Cline.LOCAL, Cline.PROMPT)
cl.register('thresh', Cline.LOCAL, Cline.PROMPT)
cl.register('niters', Cline.LOCAL, Cline.PROMPT)
cl.register('gfac', Cline.LOCAL, Cline.PROMPT)
cl.register('msub', Cline.GLOBAL, Cline.PROMPT)
cl.register('iset', Cline.GLOBAL, Cline.PROMPT)
cl.register('ilo', Cline.GLOBAL, Cline.PROMPT)
cl.register('ihi', Cline.GLOBAL, Cline.PROMPT)
cl.register('plo', Cline.GLOBAL, Cline.PROMPT)
cl.register('phi', Cline.GLOBAL, Cline.PROMPT)
cl.register('beta', Cline.LOCAL, Cline.HIDE)
cl.register('fwhm', Cline.LOCAL, Cline.HIDE)
cl.register('fwmin', Cline.LOCAL, Cline.HIDE)
cl.register('shbox', Cline.LOCAL, Cline.HIDE)
cl.register('smooth', Cline.LOCAL, Cline.HIDE)
cl.register('fhbox', Cline.LOCAL, Cline.HIDE)
cl.register('read', Cline.LOCAL, Cline.HIDE)
cl.register('gain', Cline.LOCAL, Cline.HIDE)
cl.register('rejthresh', Cline.LOCAL, Cline.HIDE)
# get inputs
mccd = cl.get_value('mccd', 'frame to plot',
cline.Fname('hcam', hcam.HCAM))
mccd = hcam.MCCD.read(mccd)
aper = cl.get_value('aper', 'name of aperture file',
cline.Fname('hcam', hcam.APER, exist=False))
if os.path.exists(aper):
# read in old apertures
mccdaper = hcam.MccdAper.read(aper)
print('Loaded existing file = {:s}'.format(aper))
else:
# create empty container
mccdaper = hcam.MccdAper()
print('No file called {:s} exists; '
'will create from scratch'.format(aper))
# define the panel grid
try:
nxdef = cl.get_default('nx')
except KeyError:
nxdef = 3
max_ccd = len(mccd)
if max_ccd > 1:
ccd = cl.get_value('ccd', 'CCD(s) to plot [0 for all]', '0')
if ccd == '0':
ccds = list(mccd.keys())
else:
ccds = ccd.split()
else:
ccds = list(mccd.keys())
width = cl.get_value('width', 'plot width (inches)', 0.)
height = cl.get_value('height', 'plot height (inches)', 0.)
# number of panels in X
if len(ccds) > 1:
nxdef = min(len(ccds), nxdef)
cl.set_default('nx', nxdef)
nx = cl.get_value('nx', 'number of panels in X', 3, 1)
else:
nx = 1
# PSF fitting stuff
method = cl.get_value('method',
'fit method g(aussian) or m(offat)',
'm',
lvals=['g', 'm'])
if method == 'm':
beta = cl.get_value('beta', 'initial exponent for Moffat fits', 5.,
0.5)
else:
beta = 0
fwhm_min = cl.get_value('fwmin',
'minimum FWHM to allow [unbinned pixels]', 1.5,
0.01)
fwhm = cl.get_value('fwhm',
'initial FWHM [unbinned pixels] for profile fits',
6., fwhm_min)
gfac = cl.get_value(
'gfac', 'multiple of FWHM used to group stars for fitting', 2.0,
0.1)
thresh = cl.get_value(
'thresh', 'threshold for object detection (multiple of sky RMS)',
3.0, 0.1)
niters = cl.get_value('niters',
'number of iterations of FIND-FIT-SUBTRACT', 2,
1)
# define the display intensities
msub = cl.get_value('msub', 'subtract median from each window?', True)
iset = cl.get_value('iset', 'set intensity a(utomatically),'
' d(irectly) or with p(ercentiles)?',
'a',
lvals=['a', 'A', 'd', 'D', 'p', 'P'])
iset = iset.lower()
plo, phi = 5, 95
ilo, ihi = 0, 1000
if iset == 'd':
ilo = cl.get_value('ilo', 'lower intensity limit', 0.)
ihi = cl.get_value('ihi', 'upper intensity limit', 1000.)
elif iset == 'p':
plo = cl.get_value('plo', 'lower intensity limit percentile', 5.,
0., 100.)
phi = cl.get_value('phi', 'upper intensity limit percentile', 95.,
0., 100.)
nxmax, nymax = 0, 0
for cnam in ccds:
nxmax = max(nxmax, mccd[cnam].nxtot)
nymax = max(nymax, mccd[cnam].nytot)
# might be worth trying to improve this at some point
xlo, xhi, ylo, yhi = 0, nxmax + 1, 0, nymax + 1
shbox = cl.get_value(
'shbox', 'half width of box for initial'
' location of target [unbinned pixels]', 11., 2.)
smooth = cl.get_value(
'smooth', 'FWHM for smoothing for initial object'
' detection [binned pixels]', 6.)
fhbox = cl.get_value(
'fhbox', 'half width of box for profile fit'
' [unbinned pixels]', 21., 3.)
read = cl.get_value('read', 'readout noise, RMS ADU', 3.)
gain = cl.get_value('gain', 'gain, ADU/e-', 1.)
rejthresh = cl.get_value('rejthresh',
'RMS rejection threshold for sky fitting', 4.)
# Inputs obtained.
# re-configure keyboard shortcuts to avoid otherwise confusing behaviour
# quit_all does not seem to be universal, hence the try/except
try:
mpl.rcParams['keymap.back'] = ''
mpl.rcParams['keymap.forward'] = ''
mpl.rcParams['keymap.fullscreen'] = ''
mpl.rcParams['keymap.grid'] = ''
mpl.rcParams['keymap.home'] = ''
mpl.rcParams['keymap.pan'] = ''
mpl.rcParams['keymap.quit'] = ''
mpl.rcParams['keymap.save'] = ''
mpl.rcParams['keymap.pan'] = ''
mpl.rcParams['keymap.save'] = ''
mpl.rcParams['keymap.xscale'] = ''
mpl.rcParams['keymap.yscale'] = ''
mpl.rcParams['keymap.zoom'] = ''
except KeyError:
pass
# start plot
if width > 0 and height > 0:
fig = plt.figure(figsize=(width, height))
else:
fig = plt.figure()
# get the navigation toolbar.
toolbar = fig.canvas.manager.toolbar
nccd = len(ccds)
ny = nccd // nx if nccd % nx == 0 else nccd // nx + 1
# we need to store some stuff
ax = None
cnams = {}
anams = {}
# this is a container for all the objects used to plot apertures to allow
# deletion. This is Group of Group objects supporting tuple storage. The
# idea is that pobjs[cnam][anam] returns the objects used to plot aperture
# anam of CCD cnam. It is initially empty,
pobjs = hcam.Group(hcam.Group)
for n, cnam in enumerate(ccds):
if ax is None:
axes = ax = fig.add_subplot(ny, nx, n + 1)
axes.set_aspect('equal', adjustable='box')
axes.set_xlim(xlo, xhi)
axes.set_ylim(ylo, yhi)
else:
axes = fig.add_subplot(ny, nx, n + 1, sharex=ax, sharey=ax)
axes.set_aspect('equal')
if msub:
# subtract median from each window
pccd = deepcopy(mccd)
for wind in pccd[cnam].values():
wind -= wind.median()
else:
pccd = mccd
hcam.mpl.pCcd(axes, pccd[cnam], iset, plo, phi, ilo, ihi,
'CCD {:s}'.format(cnam))
# keep track of the CCDs associated with each axes
cnams[axes] = cnam
# and axes associated with each CCD
anams[cnam] = axes
if cnam in mccdaper:
# plot any pre-existing apertures, keeping track of
# the plot objects
pobjs[cnam] = hcam.mpl.pCcdAper(axes, mccdaper[cnam])
else:
# add in an empty CcdApers for any CCD not already present
mccdaper[cnam] = hcam.CcdAper()
# and an empty container for any new plot objects
pobjs[cnam] = hcam.Group(tuple)
print("""
Now use the mouse and the pan/zoom tools to zoom in to the region of interest (ROI) for
PSF photometry. All existing apertures inside this region will be retained, but apertures
outside this region will be deleted.
Once you are happy with your selection, use the key commands listed below to select one or
more bright, isolated, stars to use as references to determine the PSF. These stars will also be
set as reference objects in the final aperture file.
Key commands:
a(dd) : add an aperture
d(elete) : delete an aperture
u(nlink) : unlink axes for all CCDs so that you can tweak ROI for CCDs independently
q(uit) : quit interactive step and find all stars in ROI.
Hitting 'd' will delete the aperture nearest to the cursor, as long as it is
close enough.
""")
try:
plt.tight_layout()
except:
pass
# create a class for picking reference star and zooming in
picker = PickRef(mccd, cnams, anams, toolbar, fig, mccdaper, method, beta,
fwhm, gfac, niters, thresh, fwhm_min, shbox, smooth,
fhbox, read, gain, rejthresh, pobjs, aper)
# squeeze space a bit
plt.subplots_adjust(wspace=0.1, hspace=0.1)
# finally show stuff ....
plt.show()
def hmeta(args=None):
description = \
"""hmeta
This command is to be run in the "raw_data" directory containing
night-by-night directories of data for |hipercam|, ULTRACAM or
ULTRASPEC. It attempts to generate meta data on any run data it
can find and write this to a file 'statistics.csv' in a
sub-directory called meta. These data can be picked up by logging
scripts. The sort of data it produces are means, medians, etc of
the frames (or some of the frames -- up to a maximum of 100-200)
of each run. The program only considers genuine frames, i.e. it
copes with the nblue and nskip options of ULTRACAM and |hipercam|.
It also know about the different amplifier outputs of the
instruments and starts by subtracting the medians of each
amplifier output to avoif being disturbed by variable mean bias
offsets.
hmeta takes a good while to run, so be nice when running it.
"""
parser = argparse.ArgumentParser(description=description)
parser.add_argument(
"-f",
dest="full",
action="store_true",
help="carry out full re-computation of stats for all valid nights",
)
args = parser.parse_args()
cwd = os.getcwd()
if os.path.basename(cwd) != "raw_data":
print("** hmeta must be run in a directory called 'raw_data'")
print("hmeta aborted", file=sys.stderr)
return
if cwd.find("ultracam") > -1:
instrument = "ULTRACAM"
itype = 'U'
source = 'ul'
cnams = ('1', '2', '3')
elif cwd.find("ultraspec") > -1:
instrument = "ULTRASPEC"
itype = 'U'
source = 'ul'
cnams = ('1', )
elif cwd.find("hipercam") > -1:
instrument = "HiPERCAM"
itype = 'H'
source = 'hl'
cnams = ('1', '2', '3', '4', '5')
else:
print(
"** hmeta: cannot find either ultracam, ultraspec or hipercam in path"
)
print("hmeta aborted", file=sys.stderr)
return
warnings.filterwarnings('ignore')
linstrument = instrument.lower()
# Now the actual work. Next are regular expressions to match run
# directories, nights, and run files
nre = re.compile("^\d\d\d\d-\d\d-\d\d$")
ure = re.compile("^run\d\d\d\.xml$")
hre = re.compile("^run\d\d\d\d\.fits$")
# Get list of night directories
nnames = [
nname for nname in os.listdir(".")
if nre.match(nname) and os.path.isdir(nname)
]
nnames.sort()
if len(nnames) == 0:
print("no night directories found", file=sys.stderr)
print("hmeta aborted", file=sys.stderr)
return
for nname in nnames:
print(f"Night {nname}")
# load all the run names
if itype == 'U':
runs = [
run[:-4] for run in os.listdir(nname) if ure.match(run)
and os.path.exists(os.path.join(nname, run[:-4] + '.dat'))
]
else:
runs = [run[:-5] for run in os.listdir(nname) if hre.match(run)]
runs.sort()
if len(runs) == 0:
print(f' No runs with data found in {nname}; skipping')
continue
# create directory for any meta info such as the times
meta = os.path.join(nname, 'meta')
os.makedirs(meta, exist_ok=True)
# name of stats file
stats = os.path.join(meta, 'statistics.csv')
if not args.full and os.path.exists(stats):
# if file already present and we are re-doing things
# in full, don't attempt to re-compute
continue
# Accumulate results in an array
barr = []
for run in runs:
dfile = os.path.join(nname, run)
try:
if itype == 'U':
rdat = hcam.ucam.Rdata(dfile)
else:
rdat = hcam.hcam.Rdata(dfile, 1, False, False)
print(f" {dfile}")
except hcam.ucam.PowerOnOffError:
print(f' {dfile} -- power on/off; skipping')
continue
except:
# some other failure
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_tb(exc_traceback, limit=1, file=sys.stderr)
traceback.print_exc(file=sys.stderr)
print(f' {dfile} -- problem occurred; skipping')
continue
# For speed, analyse a maximum of 100-200
# images of each CCD from any given run. Have to take
# into account the skips / nblue parameters.
ntotal = rdat.ntotal()
if ntotal == 0:
print(f' {dfile} -- zero frames; skipping')
continue
ncframes = {}
if instrument == 'ULTRASPEC':
# just the one CCD here
nstep = ntotal // min(ntotal, 100)
ncframes['1'] = list(range(1, ntotal + 1, nstep))
nframes = ncframes['1']
ns = {'1': 0}
elif instrument == 'ULTRACAM':
# CCD 1, 2 read out each time, but 3 can be skipped
nstep = ntotal // min(ntotal, 100)
ncframes['1'] = list(range(1, ntotal + 1, nstep))
ncframes['2'] = ncframes['1']
nb = rdat.nblue
nbstep = nb * max(1, ntotal // min(ntotal, 100 * nb))
ncframes['3'] = list(range(nb, ntotal + 1, nbstep))
nframes = sorted(set(ncframes['1'] + ncframes['3']))
ns = {'1': 0, '2': 0, '3': 0}
else:
raise NotImplementedError('HiPERCAM case not done yet')
# define arrays for holding the stats
medians, means, p1s, p16s, rmsps, p84s, p99s = {}, {}, {}, {}, {}, {}, {}
for cnam, ncframe in ncframes.items():
if instrument == 'ULTRASPEC':
medians[cnam] = np.empty_like(ncframe, dtype=np.float)
elif instrument == 'ULTRACAM':
medians[cnam] = {
'L': np.empty_like(ncframe, dtype=np.float),
'R': np.empty_like(ncframe, dtype=np.float)
}
else:
raise NotImplementedError('HiPERCAM case not done yet')
means[cnam] = np.empty_like(ncframe, dtype=np.float)
p1s[cnam] = np.empty_like(ncframe, dtype=np.float)
p16s[cnam] = np.empty_like(ncframe, dtype=np.float)
p84s[cnam] = np.empty_like(ncframe, dtype=np.float)
rmsps[cnam] = np.empty_like(ncframe, dtype=np.float)
p99s[cnam] = np.empty_like(ncframe, dtype=np.float)
# now access the data and calculate stats. we have to
# remember that ultracam and hipercam have different
# readout amps so we subtract median values calculated for
# each amp separately, which is a little painful.
for n, nf in enumerate(nframes):
try:
mccd = rdat(nf)
for cnam, ncframe in ncframes.items():
if nf in ncframe:
ccd = mccd[cnam]
nc = ns[cnam]
if instrument == 'ULTRASPEC':
medval = ccd.median()
ccd -= medval
medians[cnam][nc] = medval
elif instrument == 'ULTRACAM':
wl = hcam.Group(hcam.Window)
wr = hcam.Group(hcam.Window)
for nw, wnam in enumerate(ccd):
if nw % 2 == 0:
wl[wnam] = ccd[wnam]
else:
wr[wnam] = ccd[wnam]
ccdl = hcam.CCD(wl, ccd.nxtot, ccd.nytot)
medl = ccdl.median()
ccdl -= medl
ccdr = hcam.CCD(wr, ccd.nxtot, ccd.nytot)
medr = ccdr.median()
ccdr -= medr
medians[cnam]['L'][nc] = medl
medians[cnam]['R'][nc] = medr
else:
raise NotImplementedError(
'HiPERCAM case not done yet')
# At this stage the median value should have been subtracted
# from the CCD on a per output basis. Remaining stats calculated
# from these median subtracted images.
means[cnam][nc] = ccd.mean()
p1, p16, p84, p99 = ccd.percentile(
[1.0, 15.865, 84.135, 99.0])
p1s[cnam][nc] = p1
p16s[cnam][nc] = p16
rmsps[cnam][nc] = (p84 - p16) / 2
p84s[cnam][nc] = p84
p99s[cnam][nc] = p99
ns[cnam] += 1
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_tb(exc_traceback, limit=1, file=sys.stderr)
traceback.print_exc(file=sys.stderr)
continue
# All extracted from the run; take and store medians
# of the extracted stats
brow = [
run,
]
for cnam in cnams:
if len(means[cnam]) == 0:
if instrument == 'ULTRASPEC':
brow += [0] + 21 * [None]
elif instrument == 'ULTRACAM':
brow += [0] + 24 * [None]
else:
raise NotImplementedError('HiPERCAM case not done yet')
else:
if instrument == 'ULTRASPEC':
min_med = np.min(medians[cnam])
med_med = np.median(medians[cnam])
max_med = np.max(medians[cnam])
brow += [len(means[cnam]), min_med, med_med, max_med]
elif instrument == 'ULTRACAM':
min_medl = np.min(medians[cnam]['L'])
med_medl = np.median(medians[cnam]['L'])
max_medl = np.max(medians[cnam]['L'])
min_medr = np.min(medians[cnam]['R'])
med_medr = np.median(medians[cnam]['R'])
max_medr = np.max(medians[cnam]['R'])
brow += [
len(means[cnam]), min_medl, med_medl, max_medl,
min_medr, med_medr, max_medr
]
else:
raise NotImplementedError('HiPERCAM case not done yet')
brow += [
np.min(means[cnam]),
np.median(means[cnam]),
np.max(means[cnam])
]
brow += [
np.min(p1s[cnam]),
np.median(p1s[cnam]),
np.max(p1s[cnam])
]
brow += [
np.min(p16s[cnam]),
np.median(p16s[cnam]),
np.max(p16s[cnam])
]
brow += [
np.min(rmsps[cnam]),
np.median(rmsps[cnam]),
np.max(rmsps[cnam])
]
brow += [
np.min(p84s[cnam]),
np.median(p84s[cnam]),
np.max(p84s[cnam])
]
brow += [
np.min(p99s[cnam]),
np.median(p99s[cnam]),
np.max(p99s[cnam])
]
barr.append(brow)
# Create pandas dataframe for easy output
if instrument == 'ULTRASPEC':
colnames = ULTRASPEC_META_COLNAMES
elif instrument == 'ULTRACAM':
colnames = ULTRACAM_META_COLNAMES
else:
raise NotImplementedError('HiPERCAM case not done yet')
cnames, dtypes = [], {}
for cname, dtype, defn in colnames:
cnames.append(cname)
dtypes[cname] = dtype
table = pd.DataFrame(data=barr, columns=cnames)
table = table.astype(dtypes)
table.to_csv(stats, index=False)
print(f'Written statistics to {stats}\n')
def makedata(args=None):
"""Script to generate multi-CCD test data given a set of parameters defined in
a config file (parsed using configparser). This allows things such as a
bias frame, flat field variations offsets, scale factor and rotations
between CCDs, and temporal variations.
Arguments::
config : (string)
file defining the parameters.
parallel : (bool)
True / yes etc to run in parallel. Be warned: it does not always make things faster,
which I assume is the result of overheads when parallelising. It will simply use whatever
CPUs are available.
Depending upon the setting in the config file, this could generate a large
number of different files and so the first time you run it, you may want
to do so in a clean directory.
Config file format: see the documentation of configparser for the general
format of the config files expected by this routine. Essentially there are
a series of sections, e.g.:
[ccd 1]
nxtot = 2048
nytot = 1048
.
.
.
which define all the parameters needed. There are many others to simulate
a bias offset, a flat field; see the example file ??? for a fully-documented
version.
"""
import configparser
global _gframe, _gfield
command, args = utils.script_args(args)
# get inputs
with Cline('HIPERCAM_ENV', '.hipercam', command, args) as cl:
# Register parameters
cl.register('config', Cline.LOCAL, Cline.PROMPT)
cl.register('parallel', Cline.LOCAL, Cline.PROMPT)
config = cl.get_value('config', 'configuration file',
cline.Fname('config'))
parallel = cl.get_value('parallel', 'add targets in parallel?', False)
# Read the config file
conf = configparser.ConfigParser()
conf.read(config)
# Determine whether files get overwritten or not
overwrite = conf.getboolean('general', 'overwrite') \
if 'overwrite' in conf['general'] else False
dtype = conf['general']['dtype'] \
if 'dtype' in conf['general'] else None
# Top-level header
thead = fits.Header()
thead.add_history('Created by makedata')
# Store the CCD labels and parameters and their dimensions. Determine
# maximum dimensions for later use when adding targets
ccd_pars = Odict()
maxnx = 0
maxny = 0
for key in conf:
if key.startswith('ccd'):
# translate parameters
nxtot = int(conf[key]['nxtot'])
nytot = int(conf[key]['nytot'])
xcen = float(conf[key]['xcen'])
ycen = float(conf[key]['ycen'])
angle = float(conf[key]['angle'])
scale = float(conf[key]['scale'])
xoff = float(conf[key]['xoff'])
yoff = float(conf[key]['yoff'])
fscale = float(conf[key]['fscale'])
toff = float(conf[key]['toff'])
field = hcam.Field.rjson(conf[key]['field']) \
if 'field' in conf[key] else None
ndiv = int(conf[key]['ndiv']) \
if field is not None else None
back = float(conf[key]['back'])
# determine maximum total dimension
maxnx = max(maxnx, nxtot)
maxny = max(maxny, nytot)
# store parameters
ccd_pars[key[3:].strip()] = {
'nxtot': nxtot,
'nytot': nytot,
'xcen': xcen,
'ycen': ycen,
'angle': angle,
'scale': scale,
'xoff': xoff,
'yoff': yoff,
'fscale': fscale,
'toff': toff,
'field': field,
'ndiv': ndiv,
'back': back,
}
if not len(ccd_pars):
raise ValueError('hipercam.makedata: no CCDs found in ' + config)
# get the timing data
utc_start = Time(conf['timing']['utc_start'])
exposure = float(conf['timing']['exposure'])
deadtime = float(conf['timing']['deadtime'])
# Generate the CCDs, store the read / gain values
ccds = hcam.Group(hcam.CCD)
rgs = {}
for cnam, pars in ccd_pars.items():
# Generate header with timing data
head = fits.Header()
td = TimeDelta(pars['toff'], format='sec')
utc = utc_start + td
head['UTC'] = (utc.isot, 'UTC at mid exposure')
head['MJD'] = (utc.mjd, 'MJD at mid exposure')
head['EXPOSE'] = (exposure, 'Exposure time, seconds')
head['TIMEOK'] = (True, 'Time status flag')
# Generate the Windows
winds = hcam.Group(hcam.Window)
rgs[cnam] = {}
for key in conf:
if key.startswith('window'):
iccd, wnam = key[6:].split()
if iccd == cnam:
llx = int(conf[key]['llx'])
lly = int(conf[key]['lly'])
nx = int(conf[key]['nx'])
ny = int(conf[key]['ny'])
xbin = int(conf[key]['xbin'])
ybin = int(conf[key]['ybin'])
if len(winds):
wind = hcam.Window(
hcam.Winhead(llx, lly, nx, ny, xbin, ybin))
else:
# store the header in the first Window
wind = hcam.Window(
hcam.Winhead(llx, lly, nx, ny, xbin, ybin, head))
# Store the Window
winds[wnam] = wind
# Store read / gain value
rgs[cnam][wnam] = (float(conf[key]['read']),
float(conf[key]['gain']))
# Accumulate CCDs
ccds[cnam] = hcam.CCD(winds, pars['nxtot'], pars['nytot'])
# Make the template MCCD
mccd = hcam.MCCD(ccds, thead)
# Make a flat field
flat = mccd.copy()
if 'flat' in conf:
rms = float(conf['flat']['rms'])
for ccd in flat.values():
# Generate dust
nspeck = int(conf['flat']['nspeck'])
if nspeck:
radius = float(conf['flat']['radius'])
depth = float(conf['flat']['depth'])
specks = []
for n in range(nspeck):
x = np.random.uniform(0.5, ccd.nxtot + 0.5)
y = np.random.uniform(0.5, ccd.nytot + 0.5)
specks.append(Dust(x, y, radius, depth))
# Set the flat field values
for wind in ccd.values():
wind.data = np.random.normal(1., rms, (wind.ny, wind.nx))
if nspeck:
wind.add_fxy(specks)
flat.head['DATATYPE'] = ('Flat field', 'Artificially generated')
fname = utils.add_extension(conf['flat']['flat'], hcam.HCAM)
flat.write(fname, overwrite)
print('Saved flat field to ', fname)
else:
# Set the flat to unity
flat.set_const(1.0)
print('No flat field generated')
# Make a bias frame
bias = mccd.copy()
if 'bias' in conf:
mean = float(conf['bias']['mean'])
rms = float(conf['bias']['rms'])
for ccd in bias.values():
for wind in ccd.values():
wind.data = np.random.normal(mean, rms, (wind.ny, wind.nx))
bias.head['DATATYPE'] = ('Bias frame', 'Artificially generated')
fname = utils.add_extension(conf['bias']['bias'], hcam.HCAM)
bias.write(fname, overwrite)
print('Saved bias frame to ', fname)
else:
# Set the bias to zero
bias.set_const(0.)
print('No bias frame generated')
# Everything is set to go, so now generate data files
nfiles = int(conf['files']['nfiles'])
if nfiles == 0:
out = mccd * flat + bias
fname = utils.add_extension(conf['files']['root'], hcam.HCAM)
out.write(fname, overwrite)
print('Written data to', fname)
else:
# file naming info
root = conf['files']['root']
ndigit = int(conf['files']['ndigit'])
# movement
xdrift = float(conf['movement']['xdrift'])
ydrift = float(conf['movement']['ydrift'])
nreset = int(conf['movement']['nreset'])
jitter = float(conf['movement']['jitter'])
print('Now generating data')
tdelta = TimeDelta(exposure + deadtime, format='sec')
for nfile in range(nfiles):
# copy over template (into a global variable for multiprocessing speed)
_gframe = mccd.copy()
# get x,y offset
xoff = np.random.normal(xdrift * (nfile % nreset), jitter)
yoff = np.random.normal(ydrift * (nfile % nreset), jitter)
# create target fields for each CCD, add background
_gfield = {}
for cnam in _gframe.keys():
p = ccd_pars[cnam]
_gframe[cnam] += p['back']
if p['field'] is not None:
# get field modification settings
transform = Transform(p['nxtot'], p['nytot'], p['xcen'],
p['ycen'], p['angle'], p['scale'],
p['xoff'] + xoff, p['yoff'] + yoff)
fscale = p['fscale']
_gfield[cnam] = p['field'].modify(transform, fscale)
# add the targets in (slow step)
if parallel:
# run in parallel on whatever cores are available
args = [(cnam, ccd_pars[cnam]['ndiv']) for cnam in _gfield]
with Pool() as pool:
ccds = pool.map(worker, args)
for cnam in _gfield:
_gframe[cnam] = ccds.pop(0)
else:
# single core
for cnam in _gfield:
ccd = _gframe[cnam]
ndiv = ccd_pars[cnam]['ndiv']
field = _gfield[cnam]
for wind in ccd.values():
field.add(wind, ndiv)
# Apply flat
_gframe *= flat
# Add noise
for cnam, ccd in _gframe.items():
for wnam, wind in ccd.items():
readout, gain = rgs[cnam][wnam]
wind.add_noise(readout, gain)
# Apply bias
_gframe += bias
# data type on output
if dtype == 'float32':
_gframe.float32()
elif dtype == 'uint16':
_gframe.uint16()
# Save
fname = '{0:s}{1:0{2:d}d}{3:s}'.format(root, nfile + 1, ndigit,
hcam.HCAM)
_gframe.write(fname, overwrite)
print('Written file {0:d} to {1:s}'.format(nfile + 1, fname))
# update times in template
for ccd in mccd.values():
head = ccd.head
utc = Time(head['UTC']) + tdelta
head['UTC'] = (utc.isot, 'UTC at mid exposure')
head['MJD'] = (utc.mjd, 'MJD at mid exposure')