def process(self, cache, dataId):
"""
Add fakes to individual CCDs.
Return None if we are skipping the CCD
"""
cache.result = None
ignoreCcdList = self.config.ignoreCcdList
if (dataId["ccd"] in ignoreCcdList) or (dataId['ccd'] > 103):
self.log.warn("Ignoring %s: CCD in ignoreCcdList" % (dataId, ))
return None
"""
Try to deal with missing CCDs gracefully
Song Huang
"""
try:
dataRef = hscButler.getDataRef(cache.butler, dataId)
ccdId = dataRef.get("ccdExposureId")
with self.logOperation("processing %s (ccdId=%d)" %
(dataId, ccdId)):
self.log.info("Loading... %s - %s" % (dataId, ccdId))
exposure = dataRef.get('calexp', immediate=True)
self.log.info("Running... %s - %s" % (dataId, ccdId))
self.fakes.run(exposure, None)
self.log.info("Finishing... %s - %s" % (dataId, ccdId))
""" Remove unused mask plane CR, and UNMASKEDNAN """
self.log.info("Removing unused mask plane")
try:
exposure.getMaskedImage().getMask(
).removeAndClearMaskPlane('UNMASKEDNAN', True)
except Exception:
self.log.info("Can not remove the UNMASKEDNAN plane")
try:
exposure.getMaskedImage().getMask(
).removeAndClearMaskPlane('FAKE', True)
except Exception:
self.log.info("Can not remove the FAKE plane")
dataRef.put(exposure, "calexp")
return 0
except Exception, errMsg:
with open(self.missingLog, "a") as mlog:
try:
mlog.write("%s , %d\n" % (dataId, ccdId))
fcntl.flock(mlog, fcntl.LOCK_UN)
except IOError:
pass
self.log.warn(str(errMsg))
self.log.warn("Something is wrong for CCD %s (ccdId=%d)" %
(dataId, ccdId))
return None
def process(self, cache, dataId):
"""
Add fakes to individual CCDs.
Return None if we are skipping the CCD
"""
cache.result = None
ignoreCcdList = self.config.ignoreCcdList
if (dataId["ccd"] in ignoreCcdList) or (dataId['ccd'] > 103):
self.log.warn("Ignoring %s: CCD in ignoreCcdList" % (dataId,))
return None
"""
Try to deal with missing CCDs gracefully
Song Huang
"""
try:
dataRef = hscButler.getDataRef(cache.butler, dataId)
ccdId = dataRef.get("ccdExposureId")
with self.logOperation("processing %s (ccdId=%d)" % (dataId,
ccdId)):
self.log.info("Loading... %s - %s" % (dataId, ccdId))
exposure = dataRef.get('calexp', immediate=True)
self.log.info("Running... %s - %s" % (dataId, ccdId))
self.fakes.run(exposure, None)
self.log.info("Finishing... %s - %s" % (dataId, ccdId))
""" Remove unused mask plane CR, and UNMASKEDNAN """
self.log.info("Removing unused mask plane")
maskPlane = exposure.getMaskedImage().getMask()
try:
maskPlane.removeAndClearMaskPlane('CR', True)
except Exception:
self.log.info("Can not remove the CR plane")
try:
maskPlane.removeAndClearMaskPlane('UNMASKEDNAN', True)
except Exception:
self.log.info("Can not remove the UNMASKEDNAN plane")
dataRef.put(exposure, "calexp")
return 0
except Exception, errMsg:
with open(self.missingLog, "a") as mlog:
try:
mlog.write("%s , %d\n" % (dataId, ccdId))
fcntl.flock(mlog, fcntl.LOCK_UN)
except IOError:
pass
self.log.warn(str(errMsg))
self.log.warn("Something is wrong for CCD %s (ccdId=%d)" % (dataId,
ccdId))
return None
def process(self, cache, dataId):
"""
add fakes to individual CCDs
return None if we are skipping the CCD
"""
cache.result = None
if (dataId["ccd"] in self.config.ignoreCcdList) or (dataId['ccd'] > 103):
self.log.warn("Ignoring %s: CCD in ignoreCcdList" % (dataId,))
return None
dataRef = hscButler.getDataRef(cache.butler, dataId)
ccdId = dataRef.get("ccdExposureId")
with self.logOperation("processing %s (ccdId=%d)" %(dataId, ccdId)):
exposure = dataRef.get('calexp', immediate=True)
self.fakes.run(exposure,None)
dataRef.put(exposure,"calexp")
return 0
def process(self, cache, dataId):
"""
add fakes to individual CCDs
return None if we are skipping the CCD
"""
cache.result = None
if (dataId["ccd"]
in self.config.ignoreCcdList) or (dataId['ccd'] > 103):
self.log.warn("Ignoring %s: CCD in ignoreCcdList" % (dataId, ))
return None
dataRef = hscButler.getDataRef(cache.butler, dataId)
ccdId = dataRef.get("ccdExposureId")
with self.logOperation("processing %s (ccdId=%d)" % (dataId, ccdId)):
exposure = dataRef.get('calexp', immediate=True)
self.fakes.run(exposure, None)
dataRef.put(exposure, "calexp")
return 0
def process(self, cache, dataId):
"""
Add fakes to individual Patch.
Return None if we are skipping the Patch
"""
cache.result = None
ignorePatchList = self.config.ignorePatchList
#if dataId["patch"] in ignorePatchList:
# self.log.warn("Ignoring %s Patch" % (dataId,))
# return None
"""
Try to deal with missing CCDs gracefully
Song Huang
"""
#try:
self.log.info("Reading...%s" % (dataId,))
dataRef = hscButler.getDataRef(cache.butler, dataId,
self.config.coaddName + 'Coadd_calexp')
with self.logOperation("processing %s" % (dataId,)):
self.log.info("Loading... %s" % (dataId,))
exposure = dataRef.get(self.config.coaddName + 'Coadd_calexp',
immediate=True)
self.log.info("Running... %s" % (dataId,))
self.fakes.run(exposure, None)
self.log.info("Finishing... %s" % (dataId,))
""" Remove unused mask plane CR, and UNMASKEDNAN """
self.log.info("Removing unused mask plane")
maskPlane = exposure.getMaskedImage().getMask()
try:
maskPlane.removeAndClearMaskPlane('CROSSTALK', True)
except Exception:
self.log.info("Can not remove the CROSSTALK plane")
try:
maskPlane.removeAndClearMaskPlane('UNMASKEDNAN', True)
except Exception:
self.log.info("Can not remove the UNMASKEDNAN plane")
dataRef.put(exposure, "deepCoadd_calexp")
return 0
def process(self, cache, dataId):
"""
Add fakes to individual Patch.
Return None if we are skipping the Patch
"""
cache.result = None
ignorePatchList = self.config.ignorePatchList
#if dataId["patch"] in ignorePatchList:
# self.log.warn("Ignoring %s Patch" % (dataId,))
# return None
"""
Try to deal with missing CCDs gracefully
Song Huang
"""
#try:
self.log.info("Reading...%s" % (dataId, ))
dataRef = hscButler.getDataRef(cache.butler, dataId,
self.config.coaddName + 'Coadd_calexp')
with self.logOperation("processing %s" % (dataId, )):
self.log.info("Loading... %s" % (dataId, ))
exposure = dataRef.get(self.config.coaddName + 'Coadd_calexp',
immediate=True)
self.log.info("Running... %s" % (dataId, ))
self.fakes.run(exposure, None)
self.log.info("Finishing... %s" % (dataId, ))
""" Remove unused mask plane CR, and UNMASKEDNAN """
self.log.info("Removing unused mask plane")
maskPlane = exposure.getMaskedImage().getMask()
try:
maskPlane.removeAndClearMaskPlane('CROSSTALK', True)
except Exception:
self.log.info("Can not remove the CROSSTALK plane")
try:
maskPlane.removeAndClearMaskPlane('UNMASKEDNAN', True)
except Exception:
self.log.info("Can not remove the UNMASKEDNAN plane")
dataRef.put(exposure, "deepCoadd_calexp")
return 0
def process(self, cache, dataId):
"""Process this dataId."""
dataRef = hscButler.getDataRef(cache.butler, dataId, datasetType="src")
ccdId = dataRef.get("ccdExposureId")
with self.logOperation("Started satellite %s (ccdId=%d) on %s" % (dataId, ccdId, basePool.NODE)):
try:
# *** HERE's the important call ***
result = self.satellite.run(dataRef)
except Exception, e:
self.log.warn("Satellite failed %s: %s\n" % (dataId, e))
import traceback
traceback.print_exc()
return None
if result is not None:
# Cache the results (in particular, the image)
cache.result = result
self.log.info("Finished satellite %s (ccdId=%d) on %s" % (dataId, ccdId, basePool.NODE))
return pipeBase.Struct(ccdId=ccdId)
def process(self, cache, dataId):
"""Process this dataId."""
dataRef = hscButler.getDataRef(cache.butler, dataId, datasetType="src")
ccdId = dataRef.get("ccdExposureId")
with self.logOperation("Started satellite %s (ccdId=%d) on %s" %
(dataId, ccdId, basePool.NODE)):
try:
# *** HERE's the important call ***
result = self.satellite.run(dataRef)
except Exception, e:
self.log.warn("Satellite failed %s: %s\n" % (dataId, e))
import traceback
traceback.print_exc()
return None
if result is not None:
# Cache the results (in particular, the image)
cache.result = result
self.log.info("Finished satellite %s (ccdId=%d) on %s" %
(dataId, ccdId, basePool.NODE))
return pipeBase.Struct(ccdId=ccdId)
def main(root,
threads,
output,
input=None,
kind=None,
visit=None,
sensor=None,
candidateSet=None):
if not candidateSet:
candidateSet = candi.knownCandidates
butler = dafPersist.Butler(root)
rMax, thetaMax = 100.0, 0.15
allMessages = ""
###################################################################
# Create an eventList for all kinds of candiate Trail
eventLists = {}
nMasked = {}
for k in candi.Candidate.validKinds:
eventLists[k] = EventList()
nMasked[k] = 0
eventLists['all'] = EventList()
nMasked['all'] = 0
candidateLookup = collections.defaultdict(list)
for candidate in candidateSet:
did = {'visit': candidate.visit, 'ccd': candidate.ccd}
candidateLookup[hashDataId(did)].append(candidate)
###################################################################
# Add jobs to the map
if input is None:
mp = mapr.MapFunc(threads, process)
alreadyProcessing = set()
for candidate in candidateSet:
rightKind = kind is None or (candidate.kind in kind)
rightVisit = visit is None or (candidate.visit in visit)
rightSensor = sensor is None or (candidate.ccd in sensor)
if rightKind and rightVisit and rightSensor and \
(candidate.visit,candidate.ccd) not in alreadyProcessing:
dataId = {'visit': candidate.visit, 'ccd': candidate.ccd}
dataRef = hscButler.getDataRef(butler, dataId)
mp.add(dataRef)
alreadyProcessing.add((candidate.visit, candidate.ccd))
results = mp.run()
else:
with open(input, 'r') as fp:
results = pickle.load(fp)
if output is not None:
with open(output, 'w') as fp:
pickle.dump(results, fp)
if kind is not None:
resultsTmp = []
for result in results:
dataHash, foundTrails, runtime = result
candidates = candidateLookup[dataHash]
rightKind = False
for candidate in candidates:
if candidate.kind in kind:
rightKind = True
if rightKind:
resultsTmp.append(result)
results = resultsTmp
falsePos, falseNeg = [], []
####################################################################
# Tally the results and see how we did
runtimes = []
widths = collections.defaultdict(list)
for result in results:
dataHash, foundTrails, runtime = result
runtimes.append(runtime)
# if there's no candidate for this data
candidates = candidateLookup[dataHash]
if len(candidates) == 0:
resultMsg = ""
# if we found something it's a false positive
if len(foundTrails) > 0:
for iTrail, fTrail in enumerate(foundTrails):
resultMsg += "\n %s: %s (%s)" % (clr.color(
"FALSE-POS", "red"), fTrail, "no-candidate")
eventLists['empty'].append(Event(False, True))
eventLists['all'].append(Event(False, True))
falsePos.append((dataHash, fTrail))
# otherwise, it's a true negative
else:
resultMsg += "\n %s: %s (%s)" % (clr.color(
"TRUE-NEG", "green"), "No Trail", "no-candidate")
eventLists['empty'].append(Event(False, False))
eventLists['all'].append(Event(False, False))
allMessages += resultMsg
for candidate in candidateLookup[dataHash]:
nTrail = len(foundTrails)
eList = EventList()
t = candidate.trail
resultMsg = ""
#########################################
# True positives - does result match candidate
#########################################
claimed = [False] * nTrail
for iTrail, fTrail in enumerate(foundTrails):
widths[candidate.kind].append(fTrail.width)
nMasked[candidate.kind] += fTrail.nMaskedPixels
nMasked['all'] += fTrail.nMaskedPixels
if t and (candidate.kind in candi.Candidate.positiveKinds) and \
(fTrail.isNear(t, rMax, thetaMax)):
resultMsg += "\n %s: %s (%s)" % (clr.color(
"TRUE-POS", "green"), fTrail, candidate.kind)
claimed[iTrail] = True
eList.append(Event(True, True))
##########################################
# False positives
##########################################
nUnclaimed = len(claimed) - sum(claimed)
if nUnclaimed > 0:
nIgnored = 0
resultTmp = ""
for iClaim, claim in enumerate(claimed):
if not claim:
isIgnored = t and (candidate.kind in candi.Candidate.ignoredKinds) and \
(foundTrails[iClaim].isNear(t, rMax, thetaMax))
isNegative = t and (candidate.kind in candi.Candidate.negativeKinds) and \
(foundTrails[iClaim].isNear(t, rMax, thetaMax))
isOtherCandidate = False
for cand in candidateLookup[dataHash]:
if cand == candidate:
continue
if cand.trail and foundTrails[iClaim].isNear(
cand.trail, rMax, thetaMax):
isOtherCandidate = True
if isNegative:
tag = "Known-bad"
elif isIgnored:
tag = "Ignored"
nIgnored += 1
resultMsg += "\n %s: %s (%s)" % (clr.color(
"IGNORED-POS",
"yellow"), fTrail, candidate.kind)
continue
elif isOtherCandidate:
nIgnored += 1
continue
else:
tag = "Unclaimed"
resultTmp += " --> %s: %s\n" % (
tag, clr.color(str(foundTrails[iClaim]), 'yellow'))
eList.append(Event(False, True))
falsePos.append((dataHash, foundTrails[iClaim]))
if nUnclaimed > nIgnored:
resultMsg += "\n %s: %d Unclaimed trails (total=%d) (%s).\n" % \
(clr.color("FALSE-POS", "red"), nUnclaimed, nTrail, candidate.kind)
resultMsg += resultTmp
#########################################
# True negatives and False ones
#########################################
if not resultMsg:
# result does not match ... and it shouldn't
if t is None or (candidate.kind in \
(candi.Candidate.negativeKinds | candi.Candidate.ignoredKinds)):
resultMsg += "\n %s: %s (%s)" % (
clr.color("TRUE-NEG", "green"),
"No trails present, and none found.", candidate.kind)
eList.append(Event(False, False))
# result does not match ... but it should have
else:
resultMsg += "\n %s: %s (%s)" % (clr.color(
"FALSE-NEG", "red"), t, candidate.kind)
eList.append(Event(True, False))
falseNeg.append((dataHash, t, candidate.kind))
did = "(%d, %d)" % (candidate.visit, candidate.ccd)
if t:
msg = "%s [r=%.1f theta=%.3f wid=%.1f]." % (clr.color(
did, "cyan"), t.r, t.theta, t.width)
else:
msg = "%s [empty candidate]" % (clr.color(did, "cyan"))
msg += resultMsg
allMessages += msg + "\n"
print "\n" + msg + "\n"
eventLists[candidate.kind] += eList
if candidate.kind not in candi.Candidate.ignoredKinds:
eventLists['all'] += eList
print clr.color("=== Summary ===", "magenta")
print allMessages
for kind, eventList in eventLists.items():
print clr.color("=== %s ===" % (kind), "magenta")
if len(eventList) == 0:
print "No events."
continue
nPos = eventList.positiveDetections or 1
print "TP=%d, TN=%d, FP=%d, FN=%d" % (
eventList.truePositives, eventList.trueNegatives,
eventList.falsePositives, eventList.falseNegatives)
print "Recall,precision,f1: %4.2f %4.2f %4.2f" % (
eventList.recall, eventList.precision, eventList.f1)
print "Masked pixels: %10d" % (nMasked[kind]), \
" (all: n = %d, %.2f%%)" % (len(eventList),100.0*nMasked[kind]/(len(eventList)*2048*4096)), \
" (det: n = %d, %.2f%%)" % (eventList.positiveDetections,100.0*nMasked[kind]/(nPos*2048*4096))
rt = np.array(runtimes)
print "Runtimes: mean=%.2f med=%.2f std=%.2f min=%.2f max=%.2f\n" % \
(rt.mean(), np.median(rt), rt.std(), rt.min(), rt.max())
for k, v in widths.items():
print "Widths: %16s mean=%6.2f med=%6.2f std=%6.2f min=%6.2f max=%6.2f" % \
(k, np.mean(v), np.median(v), np.std(v), np.min(v), np.max(v))
with open("falsePositives.txt", 'w') as fp:
for d, f in falsePos:
fp.write("%s %s\n" % (str(d), str(f)))
with open("falseNegatives.txt", 'w') as fp:
for d, f, k in falseNeg:
fp.write("%s %s %s\n" % (str(d), k, str(f)))
def main(rerun,
visit1,
visit2,
op,
ccdno,
datatype='calexp',
scale=None,
root=None,
invert=False,
cmap='gray',
vmax=None,
annotate=None,
bins=16,
rerun2=None,
showCbar=False,
vsig=None,
showAnnotate=False,
percent=None,
hilite=None):
vsigDefault = 5.0
visit1 = int(visit1)
if visit2:
visit2 = int(visit2)
ccdno = set([int(x) for x in hscUtil.idSplit(ccdno)])
butler1 = hscUtil.getButler(rerun, root=root)
if rerun2:
butler2 = hscUtil.getButler(rerun2, root=root)
else:
butler2 = butler1
if datatype not in calibTypes:
dataIds1 = butler1.queryMetadata(datatype,
"ccd",
format=["visit", "ccd"],
dataId={'visit': visit1})
dataIds1 = [{'visit': x[0], 'ccd': x[1]} for x in dataIds1]
else:
dataIds1 = [{'visit': visit1, 'ccd': x} for x in range(104)]
dataRef1 = None
dataRef2 = None
if visit2:
dataIds2 = butler2.queryMetadata(datatype,
"ccd",
format=["visit", "ccd"],
dataId={'visit': visit2})
dataIds2 = [{'visit': x[0], 'ccd': x[1]} for x in dataIds2]
# flip the color map
if invert:
cmap = re.sub("_r$", "", cmap) if re.match('_r$', cmap) else '_r'
# sleezy, but if vmax isn't set and we're linear, just use ccd in the middle to norm
if vmax is None and scale != 'histeq':
vmax = 'c049'
# handle the gray scale normalization
vmin = 0.0
mdRef1 = None
mdRef2 = None
if vmax:
if scale == 'histeq':
raise ValueError("Cannot specify vmax with histeq scaling.")
# if it identifies a CCD to use
if re.match("^c", vmax):
vmaxCcd = int(re.sub("c", "", vmax))
try:
dataRef1 = hscButler.getDataRef(butler1, {
'visit': visit1,
'ccd': vmaxCcd
})
imgRef1 = dataRef1.get(
datatype).getMaskedImage().getImage().getArray()
if datatype not in calibTypes:
mdRef1 = dataRef1.get(datatype + '_md', immediate=True)
if visit2:
dataRef2 = hscButler.getDataRef(butler2, {
'visit': visit2,
'ccd': vmaxCcd
})
imgRef2 = dataRef2.get(
datatype).getMaskedImage().getImage().getArray()
mdRef2 = dataRef2.get(datatype + '_md', immediate=True)
img_op = hscUtil.rebin(compute(imgRef1, imgRef2, op), bins)
med = numpy.median(img_op)
std = numpy.std(img_op)
if not vsig:
vsig = vsigDefault
delta = vsig * std
if percent:
delta = percent * med
vmin = med - abs(delta)
vmax = med + abs(delta)
else:
if showAnnotate:
exp1 = dataRef1.get(datatype)
aval = float(exp1.getMetadata().get(annotate))
med = aval
delta = vsig
if not vsig:
delta = 0.5 * aval
if percent:
delta = percent * aval
vmin = med - abs(delta)
vmax = med + abs(delta)
else:
img_op = imgRef1
med = numpy.median(img_op)
sig = numpy.sqrt(med)
delta = vsigDefault * sig
if vsig:
delta = vsig * sig
if percent:
delta = percent * med
vmin = med - abs(delta)
vmax = med + abs(delta)
if not vsig and not percent:
vmin = 0.5 * med
vmax = med + 5.0 * sig
except Exception, e:
raise RuntimeError("Could not get stats on vmax CCD" +
str(vmax) + " Exiting." + str(e))
elif re.search(":", vmax):
vmaxCcd = None
vmin, vmax = [float(x) for x in vmax.split(":")]
med = 0.5 * (vmax + vmin)
else:
vmaxCcd = None
vmax = float(vmax)
med = 0.5 * (vmax + vmin)
im_ax = None
i_show = 0
for dataId1 in dataIds1:
if dataId1['ccd'] not in ccdno:
continue
print dataId1,
if visit2:
dataId2 = {'visit': visit2, 'ccd': dataId1['ccd']}
print dataId2
else:
print ""
try:
dataRef1 = hscButler.getDataRef(butler1, dataId1)
exp1 = dataRef1.get(datatype)
img1 = None
if not showAnnotate:
img1 = exp1.getMaskedImage().getImage().getArray()
img1 = img1
if visit2:
dataRef2 = hscButler.getDataRef(butler2, dataId2)
exp2 = dataRef2.get(datatype)
img2 = exp2.getMaskedImage().getImage().getArray()
except Exception, e:
#raise
print "getDataRef() failed for ", visit1, visit2, ccdno, str(e)
continue
ax = fpa_fig.getAxes(dataId1['ccd'])
def main(visit, rerun, out, ccd="0..103", root=None,
point=None, minchips=1, showLabels=False, boresightOnly=False, edge=0.9):
pcen, rad, pcenVisit = None, None, None
if point:
pcenArgs = map(float, point.split(":"))
if len(pcenArgs) == 2:
pcenVisit, rad = pcenArgs
elif len(pcenArgs) == 3:
cx, cy, rad = pcenArgs
pcen = afwCoord.Fk5Coord(afwGeom.Point2D(cx, cy))
else:
raise ValueError("Unable to parse center 'point', must be ra:dec:rad or visit:rad")
print visit, rerun
butler = hscUtil.getButler(rerun, root=root)
visitWidths = []
visitHeights = []
dataIds = visitCcdToDataId(visit, ccd)
corners = {}
boresights = {}
for dataId in dataIds:
visit, ccd = dataId['visit'], dataId['ccd']
if boresightOnly and visit in boresights:
continue
print "Getting ", visit, ccd
try:
dataRef = hscButler.getDataRef(butler, dataId)
except Exception, e:
print "getDataRef() failed for ", visit, ccd, str(e)
continue
calfile = dataRef.get("calexp_filename", immediate=True)[0]
noCal = False
if not os.path.exists(calfile):
print calfile+" missing."
noCal = True
else:
# get the calexp
calmd = dataRef.get("calexp_md", immediate=True)
wcs = afwImage.makeWcs(calmd)
w, h = calmd.get("NAXIS1"), calmd.get("NAXIS2")
boresights[dataId['visit']] = [calmd.get("RA2000"), calmd.get("DEC2000")]
# get the corners in RA,Dec
ll = wcs.pixelToSky(afwGeom.Point2D(1,1)).toFk5()
lr = wcs.pixelToSky(afwGeom.Point2D(w,1)).toFk5()
ul = wcs.pixelToSky(afwGeom.Point2D(1,h)).toFk5()
ur = wcs.pixelToSky(afwGeom.Point2D(w,h)).toFk5()
if noCal:
rawfile = dataRef.get("raw_filename", immediate=True)[0]
if not os.path.exists(rawfile):
print rawfile+" missing too. continuing."
continue
else:
rawmd = dataRef.get("raw_md", immediate=True)
raS, decS = rawmd.get("RA2000"), rawmd.get("DEC2000")
boresights[dataId['visit']] = [raS, decS]
#ra, dec = c.getRa().asDegrees(), c.getDec().asDegrees()
crv1, crv2 = map(float, [rawmd.get("CRVAL1"), rawmd.get("CRVAL2")])
crp1, crp2 = map(float, [rawmd.get("CRPIX1"), rawmd.get("CRPIX2")])
useDelt = True
try:
cdelt1, cdelt2 = map(float, [rawmd.get("CDELT1"), rawmd.get("CDELT1")])
except:
useDelt= False
if not useDelt:
cd11, cd12, cd21, cd22 = map(float, [rawmd.get("CD1_1"),
rawmd.get("CD1_2"),
rawmd.get("CD2_1"),
rawmd.get("CD2_2")])
cdelt1 = 3600*(cd11 + cd12)/206265
cdelt2 = 3600*(cd21 + cd22)/206265
#cdelt1, cdelt2 = 1.0, 1.0
nx, ny = map(int, [rawmd.get("NAXIS1"), rawmd.get("NAXIS2")])
def trans(x, y):
ra = crv1 + (crp1-x)*cdelt1
dec = crv2 + (crp2-y)*cdelt2
return ra, dec
ll = afwCoord.Fk5Coord(afwGeom.Point2D(*trans(1, 1)))
lr = afwCoord.Fk5Coord(afwGeom.Point2D(*trans(nx, 1)))
ul = afwCoord.Fk5Coord(afwGeom.Point2D(*trans(1, ny)))
ur = afwCoord.Fk5Coord(afwGeom.Point2D(*trans(nx, ny)))
if pcenVisit and not pcen:
if int(pcenVisit) == int(visit):
cx, cy = boresights[pcenVisit]
pcen = afwCoord.Fk5Coord(cx, cy)
if visit not in corners:
corners[visit] = []
corners[visit].append([ll, lr, ur, ul, ccd])
def main(visit,
rerun,
out,
ccd="0..103",
root=None,
point=None,
minchips=1,
showLabels=False,
boresightOnly=False,
edge=0.9):
pcen, rad, pcenVisit = None, None, None
if point:
pcenArgs = map(float, point.split(":"))
if len(pcenArgs) == 2:
pcenVisit, rad = pcenArgs
elif len(pcenArgs) == 3:
cx, cy, rad = pcenArgs
pcen = afwCoord.Fk5Coord(afwGeom.Point2D(cx, cy))
else:
raise ValueError(
"Unable to parse center 'point', must be ra:dec:rad or visit:rad"
)
print visit, rerun
butler = hscUtil.getButler(rerun, root=root)
visitWidths = []
visitHeights = []
dataIds = visitCcdToDataId(visit, ccd)
corners = {}
boresights = {}
for dataId in dataIds:
visit, ccd = dataId['visit'], dataId['ccd']
if boresightOnly and visit in boresights:
continue
print "Getting ", visit, ccd
try:
dataRef = hscButler.getDataRef(butler, dataId)
except Exception, e:
print "getDataRef() failed for ", visit, ccd, str(e)
continue
calfile = dataRef.get("calexp_filename", immediate=True)[0]
noCal = False
if not os.path.exists(calfile):
print calfile + " missing."
noCal = True
else:
# get the calexp
calmd = dataRef.get("calexp_md", immediate=True)
wcs = afwImage.makeWcs(calmd)
w, h = calmd.get("NAXIS1"), calmd.get("NAXIS2")
boresights[dataId['visit']] = [
calmd.get("RA2000"), calmd.get("DEC2000")
]
# get the corners in RA,Dec
ll = wcs.pixelToSky(afwGeom.Point2D(1, 1)).toFk5()
lr = wcs.pixelToSky(afwGeom.Point2D(w, 1)).toFk5()
ul = wcs.pixelToSky(afwGeom.Point2D(1, h)).toFk5()
ur = wcs.pixelToSky(afwGeom.Point2D(w, h)).toFk5()
if noCal:
rawfile = dataRef.get("raw_filename", immediate=True)[0]
if not os.path.exists(rawfile):
print rawfile + " missing too. continuing."
continue
else:
rawmd = dataRef.get("raw_md", immediate=True)
raS, decS = rawmd.get("RA2000"), rawmd.get("DEC2000")
boresights[dataId['visit']] = [raS, decS]
#ra, dec = c.getRa().asDegrees(), c.getDec().asDegrees()
crv1, crv2 = map(float,
[rawmd.get("CRVAL1"),
rawmd.get("CRVAL2")])
crp1, crp2 = map(float,
[rawmd.get("CRPIX1"),
rawmd.get("CRPIX2")])
useDelt = True
try:
cdelt1, cdelt2 = map(
float, [rawmd.get("CDELT1"),
rawmd.get("CDELT1")])
except:
useDelt = False
if not useDelt:
cd11, cd12, cd21, cd22 = map(float, [
rawmd.get("CD1_1"),
rawmd.get("CD1_2"),
rawmd.get("CD2_1"),
rawmd.get("CD2_2")
])
cdelt1 = 3600 * (cd11 + cd12) / 206265
cdelt2 = 3600 * (cd21 + cd22) / 206265
#cdelt1, cdelt2 = 1.0, 1.0
nx, ny = map(int, [rawmd.get("NAXIS1"), rawmd.get("NAXIS2")])
def trans(x, y):
ra = crv1 + (crp1 - x) * cdelt1
dec = crv2 + (crp2 - y) * cdelt2
return ra, dec
ll = afwCoord.Fk5Coord(afwGeom.Point2D(*trans(1, 1)))
lr = afwCoord.Fk5Coord(afwGeom.Point2D(*trans(nx, 1)))
ul = afwCoord.Fk5Coord(afwGeom.Point2D(*trans(1, ny)))
ur = afwCoord.Fk5Coord(afwGeom.Point2D(*trans(nx, ny)))
if pcenVisit and not pcen:
if int(pcenVisit) == int(visit):
cx, cy = boresights[pcenVisit]
pcen = afwCoord.Fk5Coord(cx, cy)
if visit not in corners:
corners[visit] = []
corners[visit].append([ll, lr, ur, ul, ccd])
def main(rerun, visit1, visit2, op, ccdno,
datatype='calexp', scale=None, root=None, invert=False, cmap='gray',
vmax=None, annotate=None, bins=16, rerun2=None, showCbar=False, vsig=None,
showAnnotate=False, percent=None, hilite=None):
vsigDefault = 5.0
visit1 = int(visit1)
if visit2:
visit2 = int(visit2)
ccdno = set([int(x) for x in hscUtil.idSplit(ccdno)])
butler1 = hscUtil.getButler(rerun, root=root)
if rerun2:
butler2 = hscUtil.getButler(rerun2, root=root)
else:
butler2 = butler1
if datatype not in calibTypes:
dataIds1 = butler1.queryMetadata(datatype, "ccd", format=["visit", "ccd"], dataId={'visit':visit1})
dataIds1 = [{'visit':x[0], 'ccd':x[1]} for x in dataIds1]
else:
dataIds1 = [{'visit': visit1, 'ccd':x} for x in range(104)]
dataRef1 = None
dataRef2 = None
if visit2:
dataIds2 = butler2.queryMetadata(datatype, "ccd", format=["visit", "ccd"], dataId={'visit':visit2})
dataIds2 = [{'visit':x[0], 'ccd':x[1]} for x in dataIds2]
# flip the color map
if invert:
cmap = re.sub("_r$", "", cmap) if re.match('_r$', cmap) else '_r'
# sleezy, but if vmax isn't set and we're linear, just use ccd in the middle to norm
if vmax is None and scale != 'histeq':
vmax = 'c049'
# handle the gray scale normalization
vmin = 0.0
mdRef1 = None
mdRef2 = None
if vmax:
if scale == 'histeq':
raise ValueError("Cannot specify vmax with histeq scaling.")
# if it identifies a CCD to use
if re.match("^c", vmax):
vmaxCcd = int(re.sub("c", "", vmax))
try:
dataRef1 = hscButler.getDataRef(butler1, {'visit':visit1, 'ccd':vmaxCcd})
imgRef1 = dataRef1.get(datatype).getMaskedImage().getImage().getArray()
if datatype not in calibTypes:
mdRef1 = dataRef1.get(datatype+'_md', immediate=True)
if visit2:
dataRef2 = hscButler.getDataRef(butler2, {'visit':visit2, 'ccd':vmaxCcd})
imgRef2 = dataRef2.get(datatype).getMaskedImage().getImage().getArray()
mdRef2 = dataRef2.get(datatype+'_md', immediate=True)
img_op = hscUtil.rebin(compute(imgRef1, imgRef2, op), bins)
med = numpy.median(img_op)
std = numpy.std(img_op)
if not vsig:
vsig = vsigDefault
delta = vsig*std
if percent:
delta = percent*med
vmin = med - abs(delta)
vmax = med + abs(delta)
else:
if showAnnotate:
exp1 = dataRef1.get(datatype)
aval = float(exp1.getMetadata().get(annotate))
med = aval
delta = vsig
if not vsig:
delta = 0.5*aval
if percent:
delta = percent*aval
vmin = med - abs(delta)
vmax = med + abs(delta)
else:
img_op = imgRef1
med = numpy.median(img_op)
sig = numpy.sqrt(med)
delta = vsigDefault*sig
if vsig:
delta = vsig*sig
if percent:
delta = percent*med
vmin = med - abs(delta)
vmax = med + abs(delta)
if not vsig and not percent:
vmin = 0.5*med
vmax = med + 5.0*sig
except Exception, e:
raise RuntimeError("Could not get stats on vmax CCD" + str(vmax)+ " Exiting." + str(e))
elif re.search(":", vmax):
vmaxCcd = None
vmin, vmax = [float(x) for x in vmax.split(":")]
med = 0.5*(vmax + vmin)
else:
vmaxCcd = None
vmax = float(vmax)
med = 0.5*(vmax + vmin)
im_ax = None
i_show = 0
for dataId1 in dataIds1:
if dataId1['ccd'] not in ccdno:
continue
print dataId1,
if visit2:
dataId2 = {'visit':visit2, 'ccd':dataId1['ccd']}
print dataId2
else:
print ""
try:
dataRef1 = hscButler.getDataRef(butler1, dataId1)
exp1 = dataRef1.get(datatype)
img1 = None
if not showAnnotate:
img1 = exp1.getMaskedImage().getImage().getArray()
img1 = img1
if visit2:
dataRef2 = hscButler.getDataRef(butler2, dataId2)
exp2 = dataRef2.get(datatype)
img2 = exp2.getMaskedImage().getImage().getArray()
except Exception, e:
#raise
print "getDataRef() failed for ", visit1, visit2, ccdno, str(e)
continue
ax = fpa_fig.getAxes(dataId1['ccd'])
def main(root, threads, output, input=None, kind=None, visit=None, sensor=None, candidateSet=None):
if not candidateSet:
candidateSet = candi.knownCandidates
butler = dafPersist.Butler(root)
rMax, thetaMax = 100.0, 0.15
allMessages = ""
###################################################################
# Create an eventList for all kinds of candiate Trail
eventLists = {}
nMasked = {}
for k in candi.Candidate.validKinds:
eventLists[k] = EventList()
nMasked[k] = 0
eventLists['all'] = EventList()
nMasked['all'] = 0
candidateLookup = collections.defaultdict(list)
for candidate in candidateSet:
did = {'visit': candidate.visit, 'ccd':candidate.ccd }
candidateLookup[hashDataId(did)].append( candidate )
###################################################################
# Add jobs to the map
if input is None:
mp = mapr.MapFunc(threads, process)
alreadyProcessing = set()
for candidate in candidateSet:
rightKind = kind is None or (candidate.kind in kind)
rightVisit = visit is None or (candidate.visit in visit)
rightSensor = sensor is None or (candidate.ccd in sensor)
if rightKind and rightVisit and rightSensor and \
(candidate.visit,candidate.ccd) not in alreadyProcessing:
dataId = {'visit': candidate.visit, 'ccd': candidate.ccd}
dataRef = hscButler.getDataRef(butler, dataId)
mp.add(dataRef)
alreadyProcessing.add( (candidate.visit, candidate.ccd) )
results = mp.run()
else:
with open(input, 'r') as fp:
results = pickle.load(fp)
if output is not None:
with open(output, 'w') as fp:
pickle.dump(results, fp)
if kind is not None:
resultsTmp = []
for result in results:
dataHash, foundTrails, runtime = result
candidates = candidateLookup[dataHash]
rightKind = False
for candidate in candidates:
if candidate.kind in kind:
rightKind = True
if rightKind:
resultsTmp.append(result)
results = resultsTmp
falsePos, falseNeg = [], []
####################################################################
# Tally the results and see how we did
runtimes = []
widths = collections.defaultdict(list)
for result in results:
dataHash, foundTrails, runtime = result
runtimes.append(runtime)
# if there's no candidate for this data
candidates = candidateLookup[dataHash]
if len(candidates) == 0:
resultMsg = ""
# if we found something it's a false positive
if len(foundTrails) > 0:
for iTrail, fTrail in enumerate(foundTrails):
resultMsg += "\n %s: %s (%s)" % (clr.color("FALSE-POS", "red"), fTrail, "no-candidate")
eventLists['empty'].append(Event(False, True))
eventLists['all'].append(Event(False, True))
falsePos.append((dataHash, fTrail))
# otherwise, it's a true negative
else:
resultMsg += "\n %s: %s (%s)" % (clr.color("TRUE-NEG", "green"), "No Trail", "no-candidate")
eventLists['empty'].append(Event(False, False))
eventLists['all'].append(Event(False, False))
allMessages += resultMsg
for candidate in candidateLookup[dataHash]:
nTrail = len(foundTrails)
eList = EventList()
t = candidate.trail
resultMsg = ""
#########################################
# True positives - does result match candidate
#########################################
claimed = [False]*nTrail
for iTrail,fTrail in enumerate(foundTrails):
widths[candidate.kind].append(fTrail.width)
nMasked[candidate.kind] += fTrail.nMaskedPixels
nMasked['all'] += fTrail.nMaskedPixels
if t and (candidate.kind in candi.Candidate.positiveKinds) and \
(fTrail.isNear(t, rMax, thetaMax)):
resultMsg += "\n %s: %s (%s)" % (clr.color("TRUE-POS", "green"), fTrail, candidate.kind)
claimed[iTrail] = True
eList.append(Event(True, True))
##########################################
# False positives
##########################################
nUnclaimed = len(claimed) - sum(claimed)
if nUnclaimed > 0:
nIgnored = 0
resultTmp = ""
for iClaim,claim in enumerate(claimed):
if not claim:
isIgnored = t and (candidate.kind in candi.Candidate.ignoredKinds) and \
(foundTrails[iClaim].isNear(t, rMax, thetaMax))
isNegative = t and (candidate.kind in candi.Candidate.negativeKinds) and \
(foundTrails[iClaim].isNear(t, rMax, thetaMax))
isOtherCandidate = False
for cand in candidateLookup[dataHash]:
if cand == candidate:
continue
if cand.trail and foundTrails[iClaim].isNear(cand.trail, rMax,thetaMax):
isOtherCandidate=True
if isNegative:
tag = "Known-bad"
elif isIgnored:
tag = "Ignored"
nIgnored += 1
resultMsg += "\n %s: %s (%s)" % (clr.color("IGNORED-POS", "yellow"),
fTrail, candidate.kind)
continue
elif isOtherCandidate:
nIgnored += 1
continue
else:
tag = "Unclaimed"
resultTmp += " --> %s: %s\n" % (tag, clr.color(str(foundTrails[iClaim]), 'yellow'))
eList.append(Event(False, True))
falsePos.append((dataHash, foundTrails[iClaim]))
if nUnclaimed > nIgnored:
resultMsg += "\n %s: %d Unclaimed trails (total=%d) (%s).\n" % \
(clr.color("FALSE-POS", "red"), nUnclaimed, nTrail, candidate.kind)
resultMsg += resultTmp
#########################################
# True negatives and False ones
#########################################
if not resultMsg:
# result does not match ... and it shouldn't
if t is None or (candidate.kind in \
(candi.Candidate.negativeKinds | candi.Candidate.ignoredKinds)):
resultMsg += "\n %s: %s (%s)" % (clr.color("TRUE-NEG", "green"),
"No trails present, and none found.", candidate.kind)
eList.append(Event(False, False))
# result does not match ... but it should have
else:
resultMsg += "\n %s: %s (%s)" % (clr.color("FALSE-NEG", "red"), t, candidate.kind)
eList.append(Event(True, False))
falseNeg.append((dataHash, t, candidate.kind))
did = "(%d, %d)" % (candidate.visit, candidate.ccd)
if t:
msg = "%s [r=%.1f theta=%.3f wid=%.1f]." % (clr.color(did, "cyan"), t.r, t.theta, t.width)
else:
msg = "%s [empty candidate]" % (clr.color(did, "cyan"))
msg += resultMsg
allMessages += msg+"\n"
print "\n"+msg+"\n"
eventLists[candidate.kind] += eList
if candidate.kind not in candi.Candidate.ignoredKinds:
eventLists['all'] += eList
print clr.color("=== Summary ===", "magenta")
print allMessages
for kind, eventList in eventLists.items():
print clr.color("=== %s ===" % (kind), "magenta")
if len(eventList) == 0:
print "No events."
continue
nPos = eventList.positiveDetections or 1
print "TP=%d, TN=%d, FP=%d, FN=%d" % (eventList.truePositives, eventList.trueNegatives,
eventList.falsePositives, eventList.falseNegatives)
print "Recall,precision,f1: %4.2f %4.2f %4.2f"%(eventList.recall, eventList.precision, eventList.f1)
print "Masked pixels: %10d" % (nMasked[kind]), \
" (all: n = %d, %.2f%%)" % (len(eventList),100.0*nMasked[kind]/(len(eventList)*2048*4096)), \
" (det: n = %d, %.2f%%)" % (eventList.positiveDetections,100.0*nMasked[kind]/(nPos*2048*4096))
rt = np.array(runtimes)
print "Runtimes: mean=%.2f med=%.2f std=%.2f min=%.2f max=%.2f\n" % \
(rt.mean(), np.median(rt), rt.std(), rt.min(), rt.max())
for k,v in widths.items():
print "Widths: %16s mean=%6.2f med=%6.2f std=%6.2f min=%6.2f max=%6.2f" % \
(k, np.mean(v), np.median(v), np.std(v), np.min(v), np.max(v))
with open("falsePositives.txt", 'w') as fp:
for d, f in falsePos:
fp.write("%s %s\n" % (str(d), str(f)))
with open("falseNegatives.txt", 'w') as fp:
for d, f, k in falseNeg:
fp.write("%s %s %s\n" % (str(d), k, str(f)))
