def __call__(cls, image, null_mask, resaturate):
"""Create or update the mask plane of an image
:Parameters:
- `image`: the DESImage to operate upon. Mask plane is created if absent
- `null_mask`: Integer or list of BADPIX bit names that, when set in mask image,
will put weight=0 for that pixel.
- `resaturate`: if True, set data for every pixel with BADPIX_SATURATE set
to a value above the SATURATE keyword
"""
if image.mask is None:
raise NullWeightsError('Mask is missing in image')
if null_mask != 0:
logger.info('Nulling weight image from mask bits')
if image.weight is None and image.variance is None:
raise NullWeightsError('Weight is missing in image')
weight = image.get_weight()
kill = np.array(image.mask & null_mask, dtype=bool)
weight[kill] = 0.
image['HISTORY'] = time.asctime(time.localtime()) + \
' Null weights with mask 0x{:04X}'.format(null_mask)
logger.debug('Finished nulling weight image')
if resaturate:
logger.info('Re-saturating pixels from mask bits')
sat = np.array(image.mask & maskbits.BADPIX_SATURATE, dtype=bool)
try:
saturation_level = image['SATURATE']
except (ValueError, KeyError):
# If there is no SATURATE, try taking max of amps
maxsat = 0.
try:
for amp in decaminfo.amps:
maxsat = max(maxsat, image['SATURAT' + amp])
except:
logger.error('SATURATx header keywords not found')
raise NullWeightsError(
'SATURATx header keywords not found')
saturation_level = maxsat
logger.warning(
'Taking SATURATE as max of single-amp SATURATx values')
image.data[sat] = 1.01 * saturation_level
image['HISTORY'] = time.asctime(time.localtime()) + \
' Set saturated pixels to {:.0f}'.format(saturation_level)
logger.debug('Finished nulling weight image')
ret_code = 0
return ret_code
def run(cls, config):
"""Customized execution for streak connection. No single input or output images.
:Parameters:
- `config`: the configuration from which to get other parameters
"""
streak_name_in = config.get(cls.step_name, 'streak_name_in')
streak_name_out = config.get(cls.step_name, 'streak_name_out')
image_name_in = config.get(cls.step_name, 'image_name_in')
image_name_out = config.get(cls.step_name, 'image_name_out')
add_width = config.getfloat(cls.step_name, 'add_width')
max_extrapolate = config.getfloat(cls.step_name, 'max_extrapolate')
if config.has_option(cls.step_name, 'plotfile'):
plotfile = config.get(cls.step_name, 'plotfile')
else:
plotfile = None
try:
streak_list = filelist_to_list(
config.get(cls.step_name, 'streak_file'))
except:
logger.error('Failure reading streak file names from {:s}'.format(
streak_list))
return 1
try:
image_list = filelist_to_list(
config.get(cls.step_name, 'image_file'))
except:
logger.error('Failure reading image file names from {:s}'.format(
image_list))
return 1
ret_code = cls.__call__(streak_list=streak_list,
image_list=image_list,
streak_name_in=streak_name_in,
streak_name_out=streak_name_out,
image_name_in=image_name_in,
image_name_out=image_name_out,
add_width=add_width,
max_extrapolate=max_extrapolate,
plotfile=plotfile)
return ret_code
def __call__(cls,
streak_list,
image_list,
streak_name_in,
streak_name_out,
image_name_in,
image_name_out,
add_width,
max_extrapolate,
plotfile=None):
"""
Read input list of streak detections and predict where a streak
crossed a CCD but was missed. Then create new copies of images,
altering masks to set STREAK bit in new streaks.
:Parameters:
- `streak_list`: list of input streak file names
- `image_list`: list of names of image files to be updated
- `streak_name_in`: string to replace in input streak filenames
- `streak_name_out`: replacement string for output streak filenames
- `image_name_in`: string to replace in input image filenames
- `image_name_out`: replacement string for output image filenames
- `add_width`: number of pixels to grow (or shrink) streak width
- `max_extrapolate`: farthest to start a new streak from endpoint of an existing one (degrees)
- `plotfile`: if given, a diagram of streaks is drawn into this file
"""
logger.info('Reading {:d} streak files'.format(len(streak_list)))
# Read in all the streak RA/Dec, into a dictionary keyed by CCDNUM,
# which should be in the primary header. Also save a dictionary of
# the file names for these
streak_corners = {}
streak_names = {}
for streakfile in streak_list:
logger.info(f"Reading streak file {streakfile}")
with fitsio.FITS(streakfile, 'r') as fits:
ccdnum = fits[0].read_header()['CCDNUM']
streak_names[ccdnum] = streakfile
tab = fits[1].read()
if len(tab) > 0:
streak_corners[ccdnum] = fits[1].read()['CORNERS_WCS']
logger.info('Reading WCS from {:d} CCDs'.format(len(image_list)))
# Read in the WCS for each CCD for which we have an image,
# also put into dicts keyed by CCDNUM
# Will get these directly from FITS instead of using DESImage in order
# to save reading all of the data.
wcs = {}
crval1 = []
crval2 = []
for imgfile in image_list:
try:
hdr = fitsio.read_header(imgfile, 0)
ccd = hdr['CCDNUM']
crval1.append(hdr['CRVAL1'])
crval2.append(hdr['CRVAL2'])
# Due to a bug in fitsio 1.0.0rc1+0, we need to clean up the
# header before feeding it to wcsutil and remove the 'None' and other problematic items
for k in hdr:
# Try to access the item, if failed we have to remove it
if not k:
hdr.delete(k)
continue
try:
_ = hdr[k]
except:
logger.info(
"Removing keyword: {:s} from header".format(k))
hdr.delete(k)
wcs[ccd] = wcsutil.WCS(hdr)
except Exception as e:
print(e) ###
logger.error('Failure reading WCS from {:s}'.format(imgfile))
return 1
# Determine a center for local gnomonic projection
ra0 = np.median(crval1)
dec0 = np.median(crval2)
# Calculate upper and lower bounds of each CCD in the local
# gnomonic system.
ccd_x1 = np.zeros(63, dtype=float)
ccd_x2 = np.zeros(63, dtype=float)
ccd_y1 = np.zeros(63, dtype=float)
ccd_y2 = np.zeros(63, dtype=float)
ccd_xmin = 1.
ccd_xmax = 2048.
ccd_ymin = 1.
ccd_ymax = 4096.
ccd_corners_xpix = np.array([ccd_xmin, ccd_xmin, ccd_xmax, ccd_xmax])
ccd_corners_ypix = np.array([ccd_ymin, ccd_ymax, ccd_ymax, ccd_ymin])
for ccd, w in wcs.items():
ra, dec = w.image2sky(ccd_corners_xpix, ccd_corners_ypix)
x_corners, y_corners = gnomonic(ra, dec, ra0, dec0)
ccd_x1[ccd] = np.min(x_corners)
ccd_y1[ccd] = np.min(y_corners)
ccd_x2[ccd] = np.max(x_corners)
ccd_y2[ccd] = np.max(y_corners)
# Now collect information on all of the streak segments that we have
ccdnum = []
ra_corner = []
dec_corner = []
for ccd, streaks in streak_corners.items():
if ccd not in wcs:
# Skip segments on CCDs that have no WCS
logger.warning(
'No WCS found for streaks on CCD {:d}'.format(ccd))
continue
n1, _, _ = streaks.shape
for i in range(n1):
ccdnum.append(ccd)
ra_corner.append(streaks[i, :, 0])
dec_corner.append(streaks[i, :, 1])
# Put streak corners into gnomonic system for this exposure
x1, y1 = gnomonic(np.array([r[0] for r in ra_corner], dtype=float),
np.array([d[0] for d in dec_corner], dtype=float),
ra0, dec0)
x2, y2 = gnomonic(np.array([r[1] for r in ra_corner], dtype=float),
np.array([d[1] for d in dec_corner], dtype=float),
ra0, dec0)
x3, y3 = gnomonic(np.array([r[2] for r in ra_corner], dtype=float),
np.array([d[2] for d in dec_corner], dtype=float),
ra0, dec0)
x4, y4 = gnomonic(np.array([r[3] for r in ra_corner], dtype=float),
np.array([d[3] for d in dec_corner], dtype=float),
ra0, dec0)
ccdnum = np.array(ccdnum, dtype=int)
# Describe each segmet by two endpoints at the midpoints of short sides
# Will need to decide which is the short side
d12 = np.hypot(x2 - x1, y2 - y1)
d23 = np.hypot(x3 - x2, y3 - y2)
xleft = np.where(d12 < d23, 0.5 * (x1 + x2), 0.5 * (x2 + x3))
yleft = np.where(d12 < d23, 0.5 * (y1 + y2), 0.5 * (y2 + y3))
xright = np.where(d12 < d23, 0.5 * (x3 + x4), 0.5 * (x4 + x1))
yright = np.where(d12 < d23, 0.5 * (y3 + y4), 0.5 * (y4 + y1))
dx = xright - xleft
dy = yright - yleft
# Calculate a width as 2x the
# largest perp distance from a vertex to this line
w1 = np.abs(dx * (y1 - yleft) - dy * (x1 - xleft)) / np.hypot(dx, dy)
w2 = np.abs(dx * (y2 - yleft) - dy * (x2 - xleft)) / np.hypot(dx, dy)
w3 = np.abs(dx * (y3 - yleft) - dy * (x3 - xleft)) / np.hypot(dx, dy)
w4 = np.abs(dx * (y4 - yleft) - dy * (x4 - xleft)) / np.hypot(dx, dy)
wmax = np.maximum(w1, w2)
wmax = np.maximum(wmax, w3)
wmax = np.maximum(wmax, w4)
wmax = 2 * wmax
# Rearrange so that xleft <= xright
swapit = xright < xleft
tmp = np.where(swapit, xleft, xright)
xleft = np.where(swapit, xright, xleft)
xright = np.array(tmp)
tmp = np.where(swapit, yleft, yright)
yleft = np.where(swapit, yright, yleft)
yright = np.array(tmp)
# Get the crossing points of the lines into CCDs
xc1, xc2, yc1, yc2 = boxCross(xleft, yleft, dx, dy, ccd_x1[ccdnum],
ccd_x2[ccdnum], ccd_y1[ccdnum],
ccd_y2[ccdnum])
# Get rid of segments that appear to miss their host CCDs
miss = xc2 < xc1
# Take 1st crossing point instead of left point if it has higher x, or vertical
# with higher y, i.e. truncate the track segment at the edge of the CCD.
replace = np.where(dx == 0, yc1 > yleft, xc1 > xleft)
xc1 = np.where(replace, xc1, xleft)
yc1 = np.where(replace, yc1, yleft)
# Likewise truncate segment at right-hand crossing
replace = np.where(dx == 0, yc2 < yright, xc2 < xright)
xc2 = np.where(replace, xc2, xright)
yc2 = np.where(replace, yc2, yright)
# Backfill the non-intersections again - note that above
# maneuvers will leave xc2<xc1 for streaks that miss their CCDs,
# unless vertical ???
xc1[miss] = 0.
xc2[miss] = -1.
# Get a final verdict on hit or miss
miss = np.where(dx == 0, yc2 < yc1, xc2 < xc1)
# Save information on all valid streaks
xc1 = xc1[~miss]
xc2 = xc2[~miss]
yc1 = yc1[~miss]
yc2 = yc2[~miss]
wmax = wmax[~miss]
ccdnum = ccdnum[~miss]
# Express segments as slopes and midpoints
dx = xc2 - xc1
dy = yc2 - yc1
mx = dx / np.hypot(dx, dy)
my = dy / np.hypot(dx, dy)
# Mark segments that are probably spurious edge detections
EDGE_SLOPE = 2. # Degrees from horizontal for edge streaks
EDGE_DISTANCE = 0.005 # Max degrees from streak center to CCD edge for spurious streaks
horizontal = np.abs(my) < np.sin(EDGE_SLOPE * np.pi / 180.)
ymid = 0.5 * (yc1 + yc2)
nearedge = np.logical_or(ccd_y2[ccdnum] - ymid < EDGE_DISTANCE,
ymid - ccd_y1[ccdnum] < EDGE_DISTANCE)
nearedge = np.logical_and(nearedge, horizontal)
# Check short edges too
vertical = np.abs(mx) < np.sin(EDGE_SLOPE * np.pi / 180.)
xmid = 0.5 * (xc1 + xc2)
tmp = np.logical_or(ccd_x2[ccdnum] - xmid < EDGE_DISTANCE,
xmid - ccd_x1[ccdnum] < EDGE_DISTANCE)
nearedge = np.logical_or(nearedge, np.logical_and(tmp, vertical))
# Decide which segments are "friends" of each other.
# To be a friend, the center of each must be close
# to the extension of the line of the other.
# Accumulate a list of tracks, each track is a list of
# individual streaks that are friends of friends
tracks = []
for i in range(len(xc1)):
if nearedge[i]:
continue # Do not use edge tracks
itstrack = [i] # start new track with just this
for t in tracks:
# Search other tracks for friends
for j in t:
if friends(xc1, xc2, yc1, yc2, mx, my, ccdnum, i, j):
itstrack += t # Merge track
tracks.remove(t) # Get rid of old one
break # No need to check others
tracks.append(itstrack)
# Now iterate through tracks, seeing if they have missing segments
# Create arrays to hold information on new tracks
new_ccdnum = []
new_xc1 = []
new_xc2 = []
new_yc1 = []
new_yc2 = []
new_ra1 = []
new_ra2 = []
new_dec1 = []
new_dec2 = []
new_width = []
new_extrapolated = []
new_nearest = []
for t in tracks:
if len(t) < 2:
continue # Do not extrapolate singlet tracks
ids = np.array(
t) # Make an array of indices of segments in this track
# Fit a quadratic path to the streak endpoints
xx = np.concatenate((xc1[ids], xc2[ids]))
yy = np.concatenate((yc1[ids], yc2[ids]))
# If the track slope is mostly along x, then we'll have the independent
# variable xx be x and dependent yy will be y. But if track
# is more vertical, then we'll look at functions x(y) instead.
xOrder = np.median(np.abs(mx[ids])) > np.median(np.abs(my[ids]))
if not xOrder:
xx, yy = yy, xx
# Record limits of detected tracks' independent variable
xxmin = np.min(xx)
xxmax = np.max(xx)
# Fit a quadratic to the points, or
# linear if only one streak
# Allow up to nclip points to clip
RESID_TOLERANCE = 6. / 3600. # Clip >6" deviants
nclip = 2
for i in range(nclip + 1):
if len(xx) > 2:
A = np.vstack((np.ones_like(xx), xx, xx * xx))
else:
A = np.vstack((np.ones_like(xx), xx))
coeffs = np.linalg.lstsq(A.T, yy, rcond=None)[0]
resid = yy - np.dot(A.T, coeffs)
j = np.argmax(np.abs(resid))
if i == nclip or np.abs(resid[j]) < RESID_TOLERANCE:
break
xx = np.delete(xx, j)
yy = np.delete(yy, j)
# Calculate the y(x1),y(x2) where tracks
# cross the left/right of every CCD, then
# find the ones that will cross CCD's y.
# These are CCD bounds, with xx being the quadratic's argument
if xOrder:
xx1 = ccd_x1
xx2 = ccd_x2
yy1 = ccd_y1
yy2 = ccd_y2
else:
xx1 = ccd_y1
xx2 = ccd_y2
yy1 = ccd_x1
yy2 = ccd_x2
if len(coeffs) == 2:
A2 = np.vstack((np.ones_like(xx2), xx2)).T
A1 = np.vstack((np.ones_like(xx1), xx1)).T
else:
A2 = np.vstack((np.ones_like(xx2), xx2, xx2 * xx2)).T
A1 = np.vstack((np.ones_like(xx1), xx1, xx1 * xx1)).T
# yyc[12] are the dependent coordinate at crossings of xx[12] bounds
yyc1 = np.dot(A1, coeffs)
yyc2 = np.dot(A2, coeffs)
# Now we ask whether the y value of streak at either edge crossing
# is in the y range of a CCD
missed = np.logical_or(
np.maximum(yyc1, yyc2) < yy1,
np.minimum(yyc1, yyc2) > yy2)
# Also skip any CCD where we already have a streak
for iccd in ccdnum[ids]:
missed[iccd] = True
missed[0] = True # There is no CCD0
missed[61] = True # Never use this one either, it's always dead
# Now find intersection of new streaks with edges of their CCDs
# Define a function for the streak path that we'll use for solving
def poly(x, coeffs, ysolve):
y = coeffs[0] + x * coeffs[1]
if len(coeffs) > 2:
y += coeffs[2] * x * x
return y - ysolve
EDGE_TOLERANCE = 0.2 / 3600. # Find x/y of edge to this accuracy (0.2 arcsec)
for iccd in np.where(~missed)[0]:
# This is a loop over every CCD that the track crosses but has no detected segment
# Determine an (xx,yy) pair for its entry and exit from the CCD
new_yy1 = yyc1[iccd]
new_yy2 = yyc2[iccd]
new_xx1 = xx1[iccd]
new_xx2 = xx2[iccd]
# left side:
if new_yy1 < yy1[iccd]:
new_xx1 = newton(poly,
new_xx1,
args=(coeffs, yy1[iccd]),
tol=EDGE_TOLERANCE)
elif new_yy1 > yy2[iccd]:
new_xx1 = newton(poly,
new_xx1,
args=(coeffs, yy2[iccd]),
tol=EDGE_TOLERANCE)
new_yy1 = poly(new_xx1, coeffs, 0.)
# right side
if new_yy2 < yy1[iccd]:
new_xx2 = newton(poly,
new_xx2,
args=(coeffs, yy1[iccd]),
tol=EDGE_TOLERANCE)
elif new_yy2 > yy2[iccd]:
new_xx2 = newton(poly,
new_xx2,
args=(coeffs, yy2[iccd]),
tol=EDGE_TOLERANCE)
new_yy2 = poly(new_xx2, coeffs, 0.)
# Does the solution lie outside the input streaks?
extrapolated = new_xx1 < xxmin or new_xx2 > xxmax
width = np.median(wmax[ids])
# Calculate distance to nearest unclipped streak member
nearest = min(np.min(np.hypot(xx - new_xx1, yy - new_yy1)),
np.min(np.hypot(xx - new_xx2, yy - new_yy2)))
if not xOrder:
# swap xx,yy back if we had y as the independent variable
new_xx1, new_yy1 = new_yy1, new_xx1
new_xx2, new_yy2 = new_yy2, new_xx2
# Project the coordinates back to RA, Dec
ra1, dec1 = gnomonicInverse(new_xx1, new_yy1, ra0, dec0)
ra2, dec2 = gnomonicInverse(new_xx2, new_yy2, ra0, dec0)
# Append this streak to list of new ones
new_ccdnum.append(iccd)
new_xc1.append(new_xx1)
new_xc2.append(new_xx2)
new_yc1.append(new_yy1)
new_yc2.append(new_yy2)
new_ra1.append(ra1)
new_ra2.append(ra2)
new_dec1.append(dec1)
new_dec2.append(dec2)
new_width.append(width)
new_extrapolated.append(extrapolated)
new_nearest.append(nearest)
# Make all lists into arrays
new_ccdnum = np.array(new_ccdnum, dtype=int)
new_xc1 = np.array(new_xc1, dtype=float)
new_xc2 = np.array(new_xc2, dtype=float)
new_yc1 = np.array(new_yc1, dtype=float)
new_yc2 = np.array(new_yc2, dtype=float)
new_ra1 = np.array(new_ra1, dtype=float)
new_ra2 = np.array(new_ra2, dtype=float)
new_dec1 = np.array(new_dec1, dtype=float)
new_dec2 = np.array(new_dec2, dtype=float)
new_width = np.array(new_width, dtype=float)
new_extrapolated = np.array(new_extrapolated, dtype=bool)
new_nearest = np.array(new_nearest, dtype=float)
# Decide which new segments will be masked
maskit = np.logical_or(~new_extrapolated,
new_nearest <= max_extrapolate)
logger.info('Identified {:d} missing streak segments for masking'.format(\
np.count_nonzero(maskit)))
# Make the diagnostic plot if desired
if plotfile is not None:
pl.figure(figsize=(6, 6))
pl.xlim(-1.1, 1.1)
pl.ylim(-1.1, 1.1)
pl.gca().set_aspect('equal')
# Draw CCD outlines and numbers
for ccd, w in wcs.items():
ra, dec = w.image2sky(ccd_corners_xpix, ccd_corners_ypix)
x_corners, y_corners = gnomonic(ra, dec, ra0, dec0)
x = x_corners.tolist()
y = y_corners.tolist()
x.append(x[0])
y.append(y[0])
pl.plot(x, y, 'k-', label=None)
x = np.mean(x_corners)
y = np.mean(y_corners)
pl.text(x,
y,
str(ccd),
horizontalalignment='center',
verticalalignment='center',
fontsize=14)
# Draw input streaks marked as edge
labelled = False
for i in np.where(nearedge)[0]:
x = (xc1[i], xc2[i])
y = (yc1[i], yc2[i])
if not labelled:
pl.plot(x, y, 'm-', lw=2, label='edge')
labelled = True
else:
pl.plot(x, y, 'm-', lw=2, label=None)
# Draw linked tracks
s = set()
for t in tracks:
if len(t) > 1:
s = s.union(set(t))
labelled = False
for i in s:
x = (xc1[i], xc2[i])
y = (yc1[i], yc2[i])
if not labelled:
pl.plot(x, y, 'b-', lw=2, label='connected')
labelled = True
else:
pl.plot(x, y, 'b-', lw=2, label=None)
# Draw singleton tracks as those that are neither edge nor connected
s = s.union(set(np.where(nearedge)[0]))
single = set(range(len(xc1)))
single = single.difference(s)
labelled = False
for i in single:
x = (xc1[i], xc2[i])
y = (yc1[i], yc2[i])
if not labelled:
pl.plot(x, y, 'c-', lw=2, label='unconnected')
labelled = True
else:
pl.plot(x, y, 'c-', lw=2, label=None)
# Draw missed tracks that will be masked
labelled = False
for i in np.where(maskit)[0]:
x = (new_xc1[i], new_xc2[i])
y = (new_yc1[i], new_yc2[i])
if not labelled:
pl.plot(x, y, 'r-', lw=2, label='new masked')
labelled = True
else:
pl.plot(x, y, 'r-', lw=2, label=None)
# Draw missed tracks that will not be masked
labelled = False
for i in np.where(~maskit)[0]:
x = (new_xc1[i], new_xc2[i])
y = (new_yc1[i], new_yc2[i])
if not labelled:
pl.plot(x, y, 'r:', lw=2, label='new skipped')
labelled = True
else:
pl.plot(x, y, 'r:', lw=2, label=None)
# legend
pl.legend(framealpha=0.3, fontsize='small')
pl.savefig(plotfile)
# Now accumulate pixel coordinates of corners of all new streaks to mask
added_streak_ccds = []
added_streak_corners = []
for id, ccd in enumerate(new_ccdnum):
ccd = new_ccdnum[id]
if not maskit[id]:
continue # Only proceed with the ones to be masked
# Get a pixel scale from the WCS, in arcsec/pix
xmid = np.mean(ccd_corners_xpix)
ymid = np.mean(ccd_corners_ypix)
ra, dec = wcs[ccd].image2sky(xmid, ymid)
ra2, dec2 = wcs[ccd].image2sky(xmid + 1, ymid)
pixscale = np.hypot(
np.cos(dec * np.pi / 180.) * (ra - ra2), dec - dec2)
# width of streak, in pixels
w = new_width[id] / pixscale + add_width
if w <= 0.:
continue # Don't mask streaks of zero width
# Make RA/Dec of track endpoints
x = np.array([new_xc1[id], new_xc2[id]])
y = np.array([new_yc1[id], new_yc2[id]])
ra, dec = gnomonicInverse(x, y, ra0, dec0)
# Convert to pixel coordinates
x, y = wcs[ccd].sky2image(ra, dec)
line = Line(x[0], y[0], x[1], y[1])
# Create bounding rectangle of track
corners_pix = boxTrack(line, w, ccd_xmin, ccd_xmax, ccd_ymin,
ccd_ymax)
added_streak_ccds.append(ccd)
added_streak_corners.append(np.array(corners_pix))
added_streak_ccds = np.array(added_streak_ccds)
# Make new copies of streak files, adding new ones
logger.debug('Rewriting streak files')
for ccd, streakfile_in in streak_names.items():
nmatch = len(re.findall(streak_name_in, streakfile_in))
if nmatch != 1:
logger.error('Could not update streak file named <' +
streakfile_in + '>')
return 1
streakfile_out = re.sub(streak_name_in, streak_name_out,
streakfile_in)
# Use file system to make fresh copy of table's FITS file
shutil.copy2(streakfile_in, streakfile_out)
# Find new streaks for this ccd
add_ids = np.where(added_streak_ccds == ccd)[0]
if len(add_ids) > 0:
# Open the table and add new streaks' info
try:
fits = fitsio.FITS(streakfile_out, 'rw')
addit = np.recarray(len(add_ids),
dtype=[('LABEL', '>i4'),
('CORNERS', '>f4', (4, 2)),
('CORNERS_WCS', '>f8', (4, 2))])
if fits[1]['LABEL'][:]:
first_label = np.max(fits[1]['LABEL'][:]) + 1
else:
first_label = 1
addit.LABEL = np.arange(first_label,
first_label + len(addit))
for i, id in enumerate(add_ids):
corners_pix = added_streak_corners[id]
addit.CORNERS[i] = corners_pix
ra, dec = wcs[ccd].image2sky(corners_pix[:, 0],
corners_pix[:, 1])
addit.CORNERS_WCS[i] = np.vstack((ra, dec)).T
fits[1].append(addit)
fits.close()
except Exception as e:
print(e)
logger.error('Failure updating streak file <{:s}>'.format(
streakfile_out))
return 1
logger.debug('Remasking images')
for imgfile_in in image_list:
# Make the name needed for output
nmatch = len(re.findall(image_name_in, imgfile_in))
if nmatch != 1:
logger.error(
'Could not create output name for image file named <' +
imgfile_in + '>')
return 1
imgfile_out = re.sub(image_name_in, image_name_out, imgfile_in)
logger.info(f"Loading image: {imgfile_in}")
sci = DESImage.load(imgfile_in)
ccd = sci.header['CCDNUM']
# Find added streaks for this ccd
add_ids = np.where(added_streak_ccds == ccd)[0]
if len(add_ids) > 0:
shape = sci.mask.shape
yy, xx = np.indices(shape)
points = np.vstack((xx.flatten(), yy.flatten())).T
inside = None
for id in add_ids:
# From Alex's immask routine: mark interior pixels
# for each added streak
v = added_streak_corners[id]
vertices = [(v[0, 0], v[0, 1]), (v[1, 0], v[1, 1]),
(v[2, 0], v[2, 1]), (v[3, 0], v[3, 1]),
(v[0, 0], v[0, 1])]
path = matplotlib.path.Path(vertices)
if inside is None:
inside = path.contains_points(points)
else:
inside = np.logical_or(inside,
path.contains_points(points))
# Make the list of masked pixels
if inside is None:
ymask, xmask = np.array(0, dtype=int), np.array(0,
dtype=int)
else:
ymask, xmask = np.nonzero(inside.reshape(shape))
sci.mask[ymask, xmask] |= parse_badpix_mask('STREAK')
# Write something into the image header
sci['DESCNCTS'] = time.asctime(time.localtime()) + \
' Mask {:d} new streaks'.format(len(add_ids))
# sci['HISTORY'] = time.asctime(time.localtime()) + \
# ' Mask {:d} new streaks'.format(len(add_ids))
logger.info(f"Saving to: {imgfile_out}")
sci.save(imgfile_out)
logger.info('Finished connecting streaks')
ret_code = 0
return ret_code
def streakMask(self, streak_file, addWidth=0., addLength=100., maxExtrapolate=0):
'''
Produce a list of pixels in the image that should be masked for
streaks in the input table. streaktab is the output table of new
streaks to add image is a FITS HDU, with header and image data
addWidth is additional number of pixels to add to half-width
addLength is length added to each end of streak (pixels)
Returns:
ypix, xpix: 1d arrays with indices of affected pixels
nStreaks: number of new streaks masked
'''
# Read the streaks table first
try:
tab = fitsio.FITS(streak_file)
streaktab = tab[1].read()
except:
logger.error('Could not read streak file {:s}'.format(streak_file))
sys.exit(1)
image_header = self.sci.header
image_data = self.sci.data
# Pixscale in degrees
pixscale = astrometry.get_pixelscale(image_header, units='arcsec') / 3600.
shape = image_data.shape
# # Due to a bug in fitsio 1.0.0rc1+0, we need to clean up the
# # header before feeding it to wcsutil and remove the 'None' and other problematic items
# for k in image_header:
# # Try to access the item, if failed we hace to remove it
# try:
# item = image_header[k]
# except:
# logger.info("Removing keyword: {:s} from header".format(k))
# image_header.delete(k)
w = wcsutil.WCS(image_header)
# WE NEED TO UPDATE THIS WHEN THE TABLE IS PER EXPNUM
use = np.logical_and(streaktab['expnum'] == image_header['EXPNUM'],
streaktab['ccdnum'] == image_header['CCDNUM'])
logger.info('{:d} streaks found to mask'.format(np.count_nonzero(use)))
nStreaks = 0
inside = None
for row in streaktab[use]:
if maxExtrapolate > 0:
if row['extrapolated'] and row['nearest'] > maxExtrapolate:
logger.info('Skipping extrapolated streak')
continue
width = row['width']
ra = np.array((row['ra1'], row['ra2']))
dec = np.array((row['dec1'], row['dec2']))
x, y = w.sky2image(ra, dec)
x1, x2, y1, y2 = x[0], x[1], y[0], y[1]
# Slope of the line, cos/sin form
mx = (x2 - x1) / np.hypot(x2 - x1, y2 -y1)
my = (y2 - y1) / np.hypot(x2 - x1, y2 -y1)
#displacement for width of streak:
wx = width / pixscale + addWidth
wy = wx * mx
wx = wx * -my
# grow length
x1 -= addLength * mx
x2 += addLength * mx
y1 -= addLength * my
y2 += addLength * my
# From Alex's immask routine: mark interior pixels
vertices = [(x1 + wx, y1 + wy), (x2 + wx, y2 + wy), (x2 - wx, y2 - wy), (x1 - wx, y1 - wy)]
vertices.append(vertices[0]) # Close the path
if inside is None:
# Set up coordinate arrays
yy, xx = np.indices(shape)
points = np.vstack((xx.flatten(), yy.flatten())).T
path = matplotlib.path.Path(vertices)
inside = path.contains_points(points)
else:
# use logical_and for additional streaks
path = matplotlib.path.Path(vertices)
inside = np.logical_or(inside, path.contains_points(points))
nStreaks = nStreaks + 1
logger.info('Masked {:d} new streaks'.format(nStreaks))
# Make the list of masked pixels
if inside is None:
ymask, xmask = np.array(0, dtype=int), np.array(0, dtype=int)
else:
ymask, xmask = np.nonzero(inside.reshape(shape))
logger.info('Setting bits in MSK image for STREAK: {:d}'.format(parse_badpix_mask('STREAK')))
self.sci.mask[ymask, xmask] |= parse_badpix_mask('STREAK')
def __call__(cls,
image,
min_cols=DEFAULT_MINCOLS,
max_cols=DEFAULT_MAXCOLS,
interp_mask=DEFAULT_INTERP_MASK,
invalid_mask=DEFAULT_INVALID_MASK):
"""
Interpolate over selected pixels by inserting average of pixels to left and right
of any bunch of adjacent selected pixels. If the interpolation region touches an
edge, or the adjacent pixel has flags marking it as invalid, than the value at
other border is used for interpolation. No interpolation is done if both
boundary pixels are invalid.
:Parameters:
- `image`: DESImage to fix.
- `min_cols`: Minimum width of region to be interpolated.
- `max_cols`: Maximum width of region to be interpolated.
- `interp_mask`: Mask bits that will trigger interpolation
- `invalid_mask`: Mask bits invalidating a pixel as interpolation source.
"""
logger.info('Interpolating along rows')
if image.mask is None:
logger.error('Input image does not have mask')
return 1
interpolate = np.array(image.mask & interp_mask, dtype=bool)
# Make arrays noting where a run of bad pixels starts or ends
# Then make arrays has_?? which says whether left side is valid
# and an array with the value just to the left/right of the run.
work = np.array(interpolate)
work[:, 1:] = np.logical_and(interpolate[:, 1:], ~interpolate[:, :-1])
ystart, xstart = np.where(work)
work = np.array(interpolate)
work[:, :-1] = np.logical_and(interpolate[:, :-1], ~interpolate[:, 1:])
yend, xend = np.where(work)
xend += 1 # Make the value one-past-end
# If we've done this correctly, every run has a start and an end.
if not np.all(ystart == yend):
logger.error("Logic problem, ystart and yend not equal.")
return 1
# Narrow our list to runs of the desired length range
use = xend - xstart >= min_cols
if max_cols is not None:
use = np.logical_and(xend - xstart <= max_cols, use)
xstart = xstart[use]
xend = xend[use]
ystart = ystart[use]
# Now determine which runs have valid data at left/right
xleft = np.maximum(0, xstart - 1)
has_left = ~np.array(image.mask[ystart, xleft] & invalid_mask,
dtype=bool)
has_left = np.logical_and(xstart >= 1, has_left)
left_value = image.data[ystart, xleft]
xright = np.minimum(work.shape[1] - 1, xend)
has_right = ~np.array(image.mask[ystart, xright] & invalid_mask,
dtype=bool)
has_right = np.logical_and(xend < work.shape[1], has_right)
right_value = image.data[ystart, xright]
# Assign right-side value to runs having just right data
for run in np.where(np.logical_and(~has_left, has_right))[0]:
image.data[ystart[run], xstart[run]:xend[run]] = right_value[run]
image.mask[ystart[run],
xstart[run]:xend[run]] |= maskbits.BADPIX_INTERP
# Assign left-side value to runs having just left data
for run in np.where(np.logical_and(has_left, ~has_right))[0]:
image.data[ystart[run], xstart[run]:xend[run]] = left_value[run]
image.mask[ystart[run],
xstart[run]:xend[run]] |= maskbits.BADPIX_INTERP
# Assign mean of left and right to runs having both sides
for run in np.where(np.logical_and(has_left, has_right))[0]:
image.data[ystart[run], xstart[run]:xend[run]] = \
0.5*(left_value[run] + right_value[run])
image.mask[ystart[run],
xstart[run]:xend[run]] |= maskbits.BADPIX_INTERP
# Add to image history
image['HISTORY'] = time.asctime(time.localtime()) + \
' row_interp over mask 0x{:04X}'.format(interp_mask)
logger.debug('Finished interpolating')
ret_code = 0
return ret_code
def __call__(cls, image, fit_filename, pc_filename, weight, dome,
skymodel_filename):
"""
Subtract sky from image using previous principal-components fit. Optionally
build weight image from fitted sky or all counts, in which case the dome flat
is needed and proper gain values are expected in the image header.
:Parameters:
- `image`: DESImage that has been flattened with dome already and fit
- `fit_filename`: filename with the coefficients from minisky fitting. Sky
subtraction is skipped if this is None.
- `pc_filename`: filename for the stored full-res sky principal components
- `weight`: 'none' to skip weights, 'sky' to calculate weight at sky level,
'all' to use all counts
- `dome`: DESImage for the dome flat, needed if weight is not 'none'.
- `skymodel_filename`: optional output filename for 'sky'
"""
if weight == 'sky' and fit_filename is None:
raise SkyError(
'Cannot make sky-only weight map without doing sky subtraction'
)
if fit_filename is not None:
logger.info('Subtracting sky')
mini = skyinfo.MiniDecam.load(fit_filename)
templates = skyinfo.SkyPC.load(pc_filename)
if templates.detpos != image['DETPOS']:
# Quit if we don't have the right CCD to subtract
logger.error(
'Image DETPOS {:s} does not match sky template {:s}'.
format(templates.detpos, image['DETPOS']))
return 1
try:
image['BAND']
except:
image['BAND'] = decaminfo.get_band(image['FILTER'])
try:
items_must_match(image, mini.header, 'BAND', 'EXPNUM')
items_must_match(image, templates.header, 'BAND')
# ??? Could check that template and image use same dome flat
except:
return 1
sky = templates.sky(mini.coeffs)
image.data -= sky
image.write_key('SKYSBFIL',
path.basename(pc_filename),
comment='Sky subtraction template file')
for i, c in enumerate(mini.coeffs):
image.write_key('SKYPC{:>02d}'.format(i),
c,
comment='Sky template coefficient')
logger.info('Finished sky subtraction')
#
# Optionally write the sky model that was subtracted from the image.
#
if skymodel_filename is not None:
# Create HDU for output skymodel, add some header info, save output to file
logger.info('Optional output of skymodel requested')
skymodel_image = DESDataImage(sky)
skymodel_image.write_key(
'SKYSBFIL',
path.basename(pc_filename),
comment='Sky subtraction template file')
for i, c in enumerate(mini.coeffs):
skymodel_image.write_key(
'SKYPC{:>02d}'.format(i),
c,
comment='Sky template coefficient')
skymodel_image.write_key('BAND', image['BAND'], comment='Band')
skymodel_image.write_key('EXPNUM',
image['EXPNUM'],
comment='Exposure Number')
skymodel_image.write_key('CCDNUM',
image['CCDNUM'],
comment='CCD Number')
skymodel_image.write_key('NITE',
image['NITE'],
comment='Night')
# skymodel_image.copy_header_info(image, cls.propagate, require=False)
## ?? catch exception from write error below?
skymodel_image.save(skymodel_filename)
else:
sky = None
if weight == 'none':
do_weight = False
sky_weight = False
elif weight == 'sky':
do_weight = True
sky_weight = True
elif weight == 'all':
do_weight = True
sky_weight = False
else:
raise SkyError('Invalid weight value: ' + weight)
if do_weight:
if dome is None:
raise SkyError(
'sky_subtract needs dome flat when making weights')
if sky_weight:
logger.info('Constructing weight image from sky image')
data = sky
else:
logger.info('Constructing weight image from all counts')
if sky is None:
# If we did not subtract a sky, the image data gives total counts
data = image.data
else:
# Add sky back in to get total counts
data = image.data + sky
if image.weight is not None or image.variance is not None:
image.weight = None
image.variance = None
logger.warning('Overwriting existing weight image')
"""
We assume in constructing the weight (=inverse variance) image that
the input image here has been divided by the dome flat already, and that
its GAIN[AB] keywords are correct for a pixel that has been divided
by the FLATMED[AB] of the flat image. So the number of *electrons* that
were read in a pixel whose current value=sky is
e = sky * (dome/FLATMED) * GAIN
The variance has three parts: read noise, and sky Poisson noise, and
multiplicative errors from noise in the flat field.
The read noise variance, in electrons, is
Var = RDNOISE^2
...and the shot noise from sky was, in electrons,
Var = sky * (dome/FLATMED) * GAIN
This means the total variance in the image, in its present form, is
Var = (RDNOISE * FLATMED / dome / GAIN)^2 + (FLATMED/GAIN)*sky/dome
We can also add the uncertainty propagated from shot noise in the dome flat,
if the dome image has a weight or variance. In which case we would add
Var += var(dome) * sky^2 / dome^2
(remembering that sky has already been divided by the dome).
If sky_weight = False, we can substitute the image data for sky in the above
calculations.
"""
# Transform the sky image into a variance image
var = np.array(data, dtype=weight_dtype)
for amp in decaminfo.amps:
sec = section2slice(image['DATASEC' + amp])
invgain = (image['FLATMED' + amp] /
image['GAIN' + amp]) / dome.data[sec]
var[sec] += image['RDNOISE' + amp]**2 * invgain
var[sec] *= invgain
# Add noise from the dome flat shot noise, if present
if dome.weight is not None:
var += data * data / (dome.weight * dome.data * dome.data)
elif dome.variance is not None:
var += data * data * dome.variance / (dome.data * dome.data)
image.variance = var
# Now there are statistics desired for the output image header.
# First, the median variance at sky level on the two amps, SKYVAR[AB]
meds = []
for amp in decaminfo.amps:
sec = section2slice(image['DATASEC' + amp])
v = np.median(var[sec][::4, ::4])
image.write_key(
'SKYVAR' + amp,
v,
comment='Median noise variance at sky level, amp ' + amp)
meds.append(v)
# SKYSIGMA is overall average noise level
image.write_key('SKYSIGMA',
np.sqrt(np.mean(meds)),
comment='RMS noise at sky level')
# SKYBRITE is a measure of sky brightness. Use the sky image if we've got it, else
# use the data
if sky is None:
skybrite = np.median(data[::4, ::4])
else:
skybrite = np.median(sky[::2, ::2])
image.write_key('SKYBRITE',
skybrite,
comment='Median sky brightness')
logger.debug('Finished weight construction')
# Run null_mask or resaturate if requested in the command-line
if cls.do_step('null_mask') or cls.do_step('resaturate'):
logger.info("Running null_weights")
# We need to fix the step_name if we want to call 'step_run'
null_weights.__class__.step_name = config_section
#null_weights.__class__.step_name = cls.config_section
null_weights.step_run(image, cls.config)
ret_code = 0
return ret_code
def __call__(cls,
in_filename,
out_filename,
ccdnum,
input_template=None,
input_list=None,
good_filename=None,
reject_rms=None,
mem_use=8.,
bitmask=skyinfo.DEFAULT_SKYMASK):
"""
Create full-resolution sky templates based on previous PCA.
Does this pixel by pixel, via robust fitting of the data in the input
full-res images to the PCA coefficients. The full-res input images' filenames
are determined from the EXPNUM _either_ by python formatting of the string
given in -input_template _or_ by looking at the list of expnum, filename pairs
in the file specified by -input-list.
Output FITS image has an extension NGOOD giving number of
images used in fit at each pixel.
:Parameters:
- `in_filename`: the file holding the PCA outputs on compressed sky
- `out_filename`: filename for the output template
- `ccdnum`: which CCD to produce templates for
- `input_template`: string that can be formatted with the expnum to yield
filename of the DESImage holding the full-res data.
- `input_list`: name of a file containing expnum, filename pairs, one pair per
line, separated by whitespace.
- `good_filename`: Name of a FITS file in which to save number of images
contributing to each pixel's fit. No output if None.
- `reject_rms`: Exclude exposures with fractional RMS residual sky above this.
If this is None, just uses the exposures that PCA used.
- `mem_use:` Number of GB to target for memory usage (Default = 8)
- `bitmask:` Applied to MASK extension of images for initial bad-pixel
exclusion.
"""
logger.info('Starting sky template construction')
# Need exactly one of these two arguments:
if not input_template is None ^ input_list is None:
logger.error(
'Need exactly one of input_template and input_list to be given'
)
return 1
# Acquire PCA information, including the table of info on input exposures
pc = skyinfo.MiniskyPC.load(in_filename)
pctab = skyinfo.MiniskyPC.get_exposures(in_filename)
# Build a MiniDECam that has our choice of CCDs that we can use for indexing.
mini = pc.get_pc(0)
# Quit if we are requesting template for a CCD that was not compressed
detpos = decaminfo.detpos_dict[ccdnum]
try:
mini.index_of(detpos, 1, 1)
except skyinfo.SkyError:
logger.error('Template requested for CCDNUM not included in PCA')
return 1
# Select exposures we'll use
if reject_rms is None:
# If no RMS threshold is specified, use the same exposures
# that were kept during PCA of compressed skies
use = np.array(pctab['USE'])
else:
# Choose our own threshold
use = pctab['RMS'] < reject_rms
# Get filenames for the full-res images from list:
if input_list is not None:
filenames = {}
flist = np.loadtxt(input_list, dtype=str)
for expnum, filename in flist:
filenames[int(expnum)] = filename
del flist
# Now warn if we are missing expnums and remove from usable exposure list
for i, val in enumerate(use):
if val and int(pctab['EXPNUM'][i]) not in filenames.keys():
use[i] = False
logger.warning('No input filename given for expnum ' +
str(expnum))
nimg = np.count_nonzero(use)
expnums = []
vv = []
for i, val in enumerate(use):
if val:
vv.append(pctab['COEFFS'][i])
expnums.append(pctab['EXPNUM'][i])
V = np.vstack(vv)
del vv
# We'll re-normalize each exposure, and its coefficients, by V[0]
norms = np.array(V[:, 0])
V = V.T / norms # V is now of shape (npc,nimg)
# The linear solutions will require this:
ainv = np.linalg.inv(np.dot(V, V.T))
nexp = V.shape[1]
npc = pc.U.shape[1]
ySize = decaminfo.shape[0]
xSize = decaminfo.shape[1]
# Create the output array
out = np.zeros((npc, ySize, xSize), dtype=np.float32)
# And an array to hold the number of exposures used at each pixel:
if good_filename is not None:
ngood = np.zeros((ySize, xSize), dtype=np.int16)
# Decide how many rows of blocks we'll read from files at a time
bytes_per_row = 4 * xSize * pc.blocksize * nimg
xBlocks = xSize / pc.blocksize
yBlocks = min(int(np.floor(0.8 * mem_use * (2**30) / bytes_per_row)),
ySize / pc.blocksize)
if yBlocks < 1:
logger.warning(
'Proceeding even though mem_use is not enough to store 1 row of blocks'
)
yBlocks = 1
d = {'ccd': ccdnum} # A dictionary used to assign names to files
hdr = {} # A dictionary of information to go into output image header
# A mask of zero is equivalent to no masking:
if bitmask == 0:
bitmask = None
nonConvergentBlocks = 0 # Keep count of blocks where clipping does not converge.
# Collect input data in chunks of yBlocks rows of blocks, then process one block at a time.
for yStart in range(0, ySize, yBlocks * pc.blocksize):
# Acquire the pixel data into a 3d array
yStop = min(ySize, yStart + yBlocks * pc.blocksize)
logger.info('Working on rows {:d} -- {:d}'.format(yStart, yStop))
data = np.zeros((nimg, yStop - yStart, xSize), dtype=np.float32)
# Mask image:
mask = np.zeros((nimg, yStop - yStart, xSize), dtype=bool)
for i, expnum in enumerate(expnums):
d['expnum'] = expnum
if input_template is None:
# Get the filename from the input list
filename = filenames[expnum]
else:
# Get the filename from formatting the template
filename = input_template.format(**d)
logger.debug('Getting pixels from ' + filename)
with fitsio.FITS(filename) as fits:
data[i, :, :] = fits['SCI'][yStart:yStop, :xSize]
if bitmask is None:
mask[i, :, :] = True
else:
m = np.array(fits['MSK'][yStart:yStop, :xSize],
dtype=np.int16)
mask[i, :, :] = (m & bitmask) == 0
del m
if yStart == 0:
# First time through the images we will be collecting/checking
# header information from the contributing images
hdrin = fits['SCI'].read_header()
usehdr = {}
if 'BAND' in hdrin.keys():
usehdr['BAND'] = hdrin['BAND']
elif 'FILTER' in hdrin.keys():
usehdr['BAND'] = decaminfo.get_band(
hdrin['FILTER'])
else:
logger.error('No BAND or FILTER in ' + filename)
return 1
if 'NITE' in hdrin.keys():
usehdr['NITE'] = hdrin['NITE']
elif 'DATE-OBS' in hdrin.keys():
usehdr['NITE'] = decaminfo.get_nite(
hdrin['DATE-OBS'])
else:
logger.error('No NITE or DATE-OBS in ' + filename)
return 1
if 'FLATFIL' in hdrin.keys():
usehdr['FLATFIL'] = hdrin['FLATFIL']
else:
logger.error('No FLATFIL in ' + filename)
return 1
if 'CCDNUM' in hdrin.keys():
usehdr['CCDNUM'] = hdrin['CCDNUM']
else:
logger.error('No CCDNUM in ' + filename)
return 1
if hdr:
# First exposure will establish values for the output
hdr['BAND'] = usehdr['BAND']
hdr['MINNITE'] = usehdr['NITE']
hdr['MAXNITE'] = usehdr['NITE']
hdr['CCDNUM'] = usehdr['CCDNUM']
if hdr['CCDNUM'] != ccdnum:
logger.error(
'Wrong ccdnum {:d} in {:s}'.format(
ccdnum, filename))
hdr['FLATFIL'] = usehdr['FLATFIL']
else:
# Check that this exposure matches the others
try:
items_must_match(hdr, usehdr, 'BAND', 'CCDNUM',
'FLATFIL')
except:
return 1
hdr['MINNITE'] = min(hdr['MINNITE'],
usehdr['NITE'])
hdr['MAXNITE'] = max(hdr['MAXNITE'],
usehdr['NITE'])
data /= norms[:, np.newaxis,
np.newaxis] # Apply norms to be near unity
# Now cycle through all blocks
for jb in range((yStop - yStart) / pc.blocksize):
for ib in range(xSize / pc.blocksize):
logger.debug('Fitting for block ({:d},{:d})'.format(
jb + yStart / pc.blocksize, ib))
if ccdnum == decaminfo.ccdnums['S7'] and \
pc.halfS7 and \
ib >= xSize / pc.blocksize / 2:
# If we are looking at the bad amp of S7, we'll just
# store the median of the normalized images in PC0.
# The other PC's stay at zero.
out[0,
yStart + jb * pc.blocksize: yStart + (jb + 1) * pc.blocksize,
ib * pc.blocksize: (ib + 1) * pc.blocksize] = \
np.median(data[:,
jb * pc.blocksize: (jb + 1) * pc.blocksize,
ib * pc.blocksize: (ib + 1) * pc.blocksize],
axis=0)
continue
# Use PCA of this block as starting guess at solution
index = mini.index_of(detpos, yStart / pc.blocksize + jb,
ib)
guess = np.array(pc.U[index, :])
# Extract the data for this block into (nexp,npix) array
block = np.array(
data[:, jb * pc.blocksize:(jb + 1) * pc.blocksize,
ib * pc.blocksize:(ib + 1) * pc.blocksize])
block.resize(nexp, pc.blocksize * pc.blocksize)
bmask = np.array(
mask[:, jb * pc.blocksize:(jb + 1) * pc.blocksize,
ib * pc.blocksize:(ib + 1) * pc.blocksize])
bmask.resize(nexp, pc.blocksize * pc.blocksize)
# We'll scale the guess in each pixel by the typical ratio
# of this pixel's data to the PCA model for the block, and
# also estimate noise as dispersion about this guess
model = np.dot(guess, V)
ratio = block / model[:, np.newaxis]
scale, var, n = clippedMean(ratio, 4, axis=0)
clip = 3. * np.sqrt(var.data) * scale.data
# First guess at solution is the outer product of superblock PCA
# with the scaling per pixel
soln = guess[:, np.newaxis] * scale.data
del scale, var, ratio, n
# Linear solution with clipping iteration
MAX_ITERATIONS = 20
TOLERANCE = 0.0001
for i in range(MAX_ITERATIONS):
model = np.dot(V.T, soln)
# Residuals from model are used to clip
resid = block - model
# Find clipped points and masked ones
good = np.logical_and(resid < clip, resid > -clip)
good = np.logical_and(good, bmask)
# Set residual to zero at bad pixels
resid[~good] = 0.
# Get shift in linear solution from residuals:
dsoln = np.dot(ainv, np.dot(V, resid))
soln += dsoln
# Calculate largest change in model as convergence criterion
shift = np.max(np.abs(np.dot(V.T, dsoln)))
logger.debug('Iteration {:d}, model shift {:f}'.format(
i, shift))
if shift < TOLERANCE:
break
if i == MAX_ITERATIONS - 1:
nonConvergentBlocks = nonConvergentBlocks + 1
# Save results into big matrices
soln.resize(npc, pc.blocksize, pc.blocksize)
out[:, yStart + jb * pc.blocksize:yStart +
(jb + 1) * pc.blocksize,
ib * pc.blocksize:(ib + 1) * pc.blocksize] = soln
if good_filename is not None:
# Gin up a masked array because it allows counting along an axis
nblock = np.ma.count_masked(\
np.ma.masked_array(np.zeros_like(good), good), axis=0)
nblock.resize(pc.blocksize, pc.blocksize)
ngood[yStart + jb * pc.blocksize:yStart +
(jb + 1) * pc.blocksize, ib *
pc.blocksize:(ib + 1) * pc.blocksize] = nblock
del nblock
del resid, model, good, dsoln, block
del data
if nonConvergentBlocks > 0:
logger.warning(
'Clipping did not converge for {:d} blocks out of {:d}'.format(
nonConvergentBlocks, xBlocks * (ySize / pc.blocksize)))
# Add a history line about creation here
hdr['HISTORY'] = time.asctime(time.localtime()) + \
' Build sky template from PCA file {:s}'.format(path.basename(in_filename))
# Save the template into the outfile
spc = skyinfo.SkyPC(out, detpos, header=hdr)
spc.save(out_filename)
del out
# Save the number of good sky pixels in another extension
if good_filename is not None:
gimg = DESDataImage(ngood,
header={
'DETPOS': detpos,
'CCDNUM': ccdnum
})
logger.debug('Writing ngood to ' + good_filename)
gimg.save(good_filename)
del gimg, ngood
logger.debug('Finished sky template')
ret_code = 0
return ret_code
def __call__(cls, image, fname_lincor):
"""Apply a linearity correction
:Parameters:
- `image`: the DESImage to determine and apply an ovescan correction
- `fname_lincor`: the linearity correction FITS table (contains look-up tables)
Applies the correction "in place"
"""
#
# Discover the HDU in the linearity correction FITS table that contains data for a specific CCD
#
fits_inventory = DESFITSInventory(fname_lincor)
lincor_hdu = fits_inventory.ccd_hdus(image['CCDNUM'])
if len(lincor_hdu) != 1:
if not lincor_hdu:
logger.error(
'Unable to locate HDU in %s containing linearity correction for CCDNUM %d. Aborting!',
fname_lincor, image['CCDNUM'])
else:
logger.error(
'Found multiple HDUs in %s containing linearity correction for CCDNUM %d. Aborting!',
fname_lincor, image['CCDNUM'])
raise Exception()
logger.info('Reading Linearity Correction from %s', fname_lincor)
cat_fits = fitsio.FITS(fname_lincor, 'r')
cat_hdu = lincor_hdu[0]
cols_retrieve = ["ADU", "ADU_LINEAR_A", "ADU_LINEAR_B"]
CAT = cat_fits[cat_hdu].read(columns=cols_retrieve)
#
# If columns do not get put into CAT in a predefined order then these utilities
# may be needed. RAG has them and can implement... left this way for now since it
# currently duplicates imcorrect exactly
#
# CATcol=cat_fits[cat_hdu].get_colnames()
# cdict=MkCatDict(CATcol,cols_retrieve)
#
# Define the correction being made.
#
nonlinear = []
linearA = []
linearB = []
for row in CAT:
nonlinear.append(row[0])
linearA.append(row[1])
linearB.append(row[2])
nonlinear = np.array(nonlinear)
linearA = np.array(linearA)
linearB = np.array(linearB)
interpA = interpolate.interp1d(nonlinear,
linearA,
kind='linear',
copy=True)
interpB = interpolate.interp1d(nonlinear,
linearB,
kind='linear',
copy=True)
logger.info('Applying Linearity Correction')
#
# Slice over the datasecs for each amplifier.
# Apply the correction
#
seca = section2slice(image['DATASECA'])
secb = section2slice(image['DATASECB'])
# Only fix pixels that are in the range of the nonlinearity table
in_range = np.logical_and(image.data[seca] >= np.min(nonlinear),
image.data[seca] <= np.max(nonlinear))
image.data[seca][in_range] = interpA(image.data[seca][in_range])
in_range = np.logical_and(image.data[secb] >= np.min(nonlinear),
image.data[secb] <= np.max(nonlinear))
image.data[secb][in_range] = interpB(image.data[secb][in_range])
image.write_key('LINCFIL',
path.basename(fname_lincor),
comment='Nonlinearity correction file')
ret_code = 0
return ret_code
def __call__(cls,
imageIn,
imageOut,
min_cols=DEFAULT_MINCOLS,
max_cols=DEFAULT_MAXCOLS,
weight_threshold=DEFAULT_WEIGHT_THRESHOLD,
weight_value=DEFAULT_WEIGHT_VALUE):
"""
Add a mask plane to imageIn and set bits wherever the weight value is <= given
threshold. Then interpolate the data plane along columns to replace masked pixels.
Set weight to weight_value at the masked pixels too. Masked pixels along edges are
not interpolated, and are left with input weight.
:Parameters:
- `imageIn`: filename of input image
- `imageOut`: filename of output image
- `min_cols`: Minimum width of region to be interpolated.
- `max_cols`: Maximum width of region to be interpolated.
- `weight_threshold`: Upper bound for weight values to mark as bad
- `weight_value`: New weight value for bad pixels, enter <0 to use neighbor weights
"""
logger.info('Preparing coadd {:s} for SExtractor'.format(imageIn))
# Read weight plane and science plane
sci, scihdr = fitsio.read(imageIn, ext=0, header=True)
wt, wthdr = fitsio.read(imageIn, ext=1, header=True)
# Make mask plane
mask = wt <= float(weight_threshold)
# Identify column runs to interpolate, start by marking beginnings of runs
work = np.array(mask)
work[1:, :] = np.logical_and(mask[1:, :], ~mask[:-1, :])
xstart, ystart = np.where(work.T)
# Now ends of runs
work = np.array(mask)
work[:-1, :] = np.logical_and(mask[:-1, :], ~mask[1:, :])
xend, yend = np.where(work.T)
yend = yend + 1 # Make the value one-past-end
# If we've done this correctly, every run has a start and an end, on same col
if not np.all(xstart == xend):
logger.error("Logic problem, xstart and xend not equal.")
print(xstart, xend) ###
return 1
# Narrow our list to runs of the desired length range and
# not touching the edges
use = yend - ystart >= min_cols
if max_cols is not None:
use = np.logical_and(yend - ystart <= max_cols, use)
use = np.logical_and(ystart > 0, use)
use = np.logical_and(yend < mask.shape[0], use)
ystart = ystart[use]
yend = yend[use]
xstart = xstart[use]
# Assign mean of top and bottom to runs, and fill in weight plane
for run in range(len(xstart)):
sci[ystart[run]:yend[run], xstart[run]] = \
0.5 * (sci[ystart[run] - 1, xstart[run]] +
sci[yend[run], xstart[run]])
if weight_value < 0:
fill_weight = 0.5 * (wt[ystart[run] - 1, xstart[run]] \
+ wt[yend[run], xstart[run]])
else:
fill_weight = weight_value
wt[ystart[run]:yend[run], xstart[run]] = fill_weight
# Add to image history
scihdr['HISTORY'] = time.asctime(time.localtime()) + \
' coadd_prepare with weight threshold {:f}'.format(weight_threshold)
# Write out all three planes
mask = np.array(mask, dtype=np.int16) * cls.MASK_VALUE
logger.debug('Writing output images')
with fitsio.FITS(imageOut, mode=fitsio.READWRITE, clobber=True) as ff:
ff.write(sci, extname='SCI', header=scihdr, clobber=True)
ff.write(mask, extname='MSK')
ff.write(wt, extname='WGT', header=wthdr)
logger.debug('Finished coadd_prepare')
ret_code = 0
return ret_code
def __call__(cls, in_filenames, out_filename, npc, reject_rms):
"""
Perform robust PCA of a collection of MiniDecam images.
:Parameters:
- `in_filenames`: list of filenames of exposure mini-sky image
- `out_filename`: filename for the output PCA
- `npc`: Number of principal components to retain
- `reject_rms`: Exclude exposures with fractional RMS above this
"""
logger.info('Collecting images for PCA')
if npc > skyinfo.MAX_PC:
raise skyinfo.SkyError(
"Requested number of sky pc's {:d} is above MAX_PC".format(
npc))
mm = [] # will hold the data vectors for each exposure
expnums = [] # Collect the exposure numbers of exposures being used
data_length = None
blocksize = None
hdr = {}
for f in in_filenames:
mini = skyinfo.MiniDecam.load(f)
v = mini.vector()
if data_length is None:
data_length = len(v)
elif len(v) != data_length:
logger.error('Mismatched sky data vector length in file ' + f)
raise skyinfo.SkyError(
'Mismatched sky data vector length in file ' + f)
# Also check for agreement of the mini-sky setup
if blocksize is None:
blocksize = mini.blocksize
mask_value = mini.mask_value
invalid = mini.invalid
halfS7 = mini.halfS7
hdr['BAND'] = mini.header['BAND']
hdr['MINNITE'] = mini.header['NITE']
hdr['MAXNITE'] = mini.header['NITE']
else:
if mini.blocksize != blocksize \
or mini.invalid != invalid \
or mini.halfS7 != halfS7:
logger.error('Mismatched minisky configuration in file ' +
f)
raise skyinfo.SkyError(
'Mismatched minisky configuration in file ' + f)
try:
# Die if there is a filter mismatch among exposures
items_must_match(hdr, mini.header, 'BAND')
except:
return 1
hdr['MINNITE'] = min(hdr['MINNITE'], mini.header['NITE'])
hdr['MAXNITE'] = max(hdr['MAXNITE'], mini.header['NITE'])
mm.append(np.array(v))
expnums.append(int(mini.header['EXPNUM']))
m = np.vstack(mm).transpose()
del mm
logger.info("Start first PCA cycle")
U, S, v = process(m, npc)
pc = skyinfo.MiniskyPC(U,
blocksize=blocksize,
mask_value=mask_value,
invalid=invalid,
header=hdr,
halfS7=halfS7)
# Refit each exposure to this PCA,
nexp = len(in_filenames)
V = np.zeros((nexp, U.shape[1]), dtype=float)
rms = np.zeros(nexp, dtype=float)
frac = np.zeros(nexp, dtype=float)
for i in range(nexp):
mini.fill_from(m[:, i])
pc.fit(mini, clip_sigma=3.)
rms[i] = mini.rms
use = rms <= reject_rms
logger.info('Retained {:d} out of {:d} exposures'.format(
np.count_nonzero(use), nexp))
# New PCA excluding outliers
logger.info("Start second PCA cycle")
U, S, v = process(m[:, use], npc)
pc.U = U
pc.save(out_filename)
# Recollect statistics and save
logger.info("Collecting statistics")
for i in range(nexp):
mini.fill_from(m[:, i])
pc.fit(mini, clip_sigma=3.)
V[i, :] = mini.coeffs
rms[i] = mini.rms
frac[i] = mini.frac
# Write V and a results table
skyinfo.MiniskyPC.save_exposures(out_filename, expnums, V, rms, frac,
use, S)
logger.debug('Finished PCA')
ret_code = 0
return ret_code
def __call__(cls, in_filename, out_filename, ccdnum, img_template,
reject_rms, mem_use):
"""
Create full-resolution sky templates based on previous PCA.
Does this pixel by pixel, via robust fitting of the data in the input
full-res images to the PCA coefficients.
:Parameters:
- `in_filename`: the file holding the PCA outputs on compressed sky
- `out_filename`: filename for the output template
- `ccdnum`: which CCD to produce templates for
- `img_template`: string that can be formatted with the expnum to yield
filename of the DESImage holding the full-res data.
- `reject_rms`: Exclude exposures with fractional RMS residual sky above this.
If this is None, just uses the exposures that PCA used.
- `mem_use:` Number of GB to target for memory usage
"""
logger.info('Starting sky template construction')
# Acquire PCA information, including the table of info on input exposures
pc = skyinfo.MiniskyPC.load(in_filename)
# ??? Should make this table part of the MiniskyPC class:
pctab = fitsio.read(in_filename, ext='EXPOSURES')
# Build a MiniDECam that has our choice of CCDs that we can use for indexing.
mini = pc.get_pc(0)
# Quit if we are requesting template for a CCD that was not compressed
detpos = decaminfo.detpos_dict[ccdnum]
try:
mini.index_of(detpos, 1, 1)
except skyinfo.SkyError:
logger.error('Template requested for CCDNUM not included in PCA')
return 1
# Select exposures we'll use
if reject_rms is None:
# If no RMS threshold is specified, use the same exposures
# that were kept during PCA of compressed skies
use = np.array(pctab['USE'])
else:
# Choose our own threshold
use = pctab['RMS'] < reject_rms
nimg = np.count_nonzero(use)
expnums = []
vv = []
for i, val in enumerate(use):
if val:
vv.append(pctab['COEFFS'][i])
expnums.append(pctab['EXPNUM'][i])
V = np.vstack(vv)
del vv
# We'll re-normalize each exposure, and its coefficients, by V[0]
norms = np.array(V[:, 0])
V = V.T / norms # V is now of shape (npc,nimg)
npc = pc.U.shape[1]
ySize = decaminfo.shape[0]
xSize = decaminfo.shape[1]
# Create the output array
out = np.zeros((npc, ySize, xSize), dtype=np.float32)
# Only fill half of it for the bad amp:
if ccdnum == decaminfo.ccdnums['S7'] and pc.halfS7:
xSize /= 2
# Decide how many rows of blocks we'll read from files at a time
bytes_per_row = 4 * xSize * pc.blocksize * nimg
#xBlocks = xSize / pc.blocksize
yBlocks = min(int(np.floor(mem_use * (2**30) / bytes_per_row)),
ySize / pc.blocksize)
if yBlocks < 1:
logger.warning(
'Proceeding even though mem_use is not enough to store 1 row of blocks'
)
yBlocks = 1
d = {'ccd': ccdnum}
# Collect input data in chunks of yBlocks rows of blocks, then process one block at a time.
for yStart in range(0, ySize, yBlocks * pc.blocksize):
# Acquire the pixel data into a 3d array
yStop = min(ySize, yStart + yBlocks * pc.blocksize)
logger.info('Working on rows {:d} -- {:d}'.format(yStart, yStop))
data = np.zeros((nimg, yStop - yStart, xSize), dtype=np.float32)
for i, expnum in enumerate(expnums):
d['expnum'] = expnum
filename = img_template.format(**d)
logger.debug('Getting pixels from ' + filename)
with fitsio.FITS(filename) as fits:
data[i, :, :] = fits['SCI'][yStart:yStop, :xSize]
data /= norms[:, np.newaxis,
np.newaxis] # Apply norms to be near zero
# Now cycle through all blocks
for jb in range((yStop - yStart) / pc.blocksize):
for ib in range(xSize / pc.blocksize):
logger.debug('Fitting for block ({:d},{:d})'.format(
jb + yStart / pc.blocksize, ib))
# Use PCA of this block as starting guess at solution
index = mini.index_of(detpos, yStart / pc.blocksize + jb,
ib)
guess = np.array(pc.U[index, :])
# We'll scale the guess in each pixel by the typical ratio
# of this pixel's data to the PCA model for the block:
model = np.dot(guess, V)
ratio = data[:, jb * pc.blocksize:(jb + 1) * pc.blocksize,
ib * pc.blocksize:(ib + 1) *
pc.blocksize] / model[:, np.newaxis,
np.newaxis]
scale, var, n = clippedMean(ratio, 4, axis=0)
logger.debug('Var, scale, ratio shapes: ' + str(var.shape) + \
' ' + str(scale.shape) + ' ' + str(ratio.shape))
del ratio, n
# Solve each pixel in the block:
for jp in range(pc.blocksize):
for ip in range(pc.blocksize):
cost = skyinfo.ClippedCost(3 *
np.sqrt(var[jp, ip]))
# Execute and save the fit
out[:, yStart + jb * pc.blocksize + jp, ib * pc.blocksize + ip] = \
skyinfo.linearFit(data[:, jb * pc.blocksize + jp, ib * pc.blocksize + ip],
V,
guess * scale[jp, ip],
cost)
del data
# Save the template into the outfile
spc = skyinfo.SkyPC(out, detpos)
spc.save(out_filename)
logger.debug('Finished sky template')
ret_code = 0
return ret_code