def process(self, cache, dataId): """ Add fakes to individual CCDs. Return None if we are skipping the CCD """ cache.result = None ignoreCcdList = self.config.ignoreCcdList if (dataId["ccd"] in ignoreCcdList) or (dataId['ccd'] > 103): self.log.warn("Ignoring %s: CCD in ignoreCcdList" % (dataId, )) return None """ Try to deal with missing CCDs gracefully Song Huang """ try: dataRef = hscButler.getDataRef(cache.butler, dataId) ccdId = dataRef.get("ccdExposureId") with self.logOperation("processing %s (ccdId=%d)" % (dataId, ccdId)): self.log.info("Loading... %s - %s" % (dataId, ccdId)) exposure = dataRef.get('calexp', immediate=True) self.log.info("Running... %s - %s" % (dataId, ccdId)) self.fakes.run(exposure, None) self.log.info("Finishing... %s - %s" % (dataId, ccdId)) """ Remove unused mask plane CR, and UNMASKEDNAN """ self.log.info("Removing unused mask plane") try: exposure.getMaskedImage().getMask( ).removeAndClearMaskPlane('UNMASKEDNAN', True) except Exception: self.log.info("Can not remove the UNMASKEDNAN plane") try: exposure.getMaskedImage().getMask( ).removeAndClearMaskPlane('FAKE', True) except Exception: self.log.info("Can not remove the FAKE plane") dataRef.put(exposure, "calexp") return 0 except Exception, errMsg: with open(self.missingLog, "a") as mlog: try: mlog.write("%s , %d\n" % (dataId, ccdId)) fcntl.flock(mlog, fcntl.LOCK_UN) except IOError: pass self.log.warn(str(errMsg)) self.log.warn("Something is wrong for CCD %s (ccdId=%d)" % (dataId, ccdId)) return None
def process(self, cache, dataId): """ Add fakes to individual CCDs. Return None if we are skipping the CCD """ cache.result = None ignoreCcdList = self.config.ignoreCcdList if (dataId["ccd"] in ignoreCcdList) or (dataId['ccd'] > 103): self.log.warn("Ignoring %s: CCD in ignoreCcdList" % (dataId,)) return None """ Try to deal with missing CCDs gracefully Song Huang """ try: dataRef = hscButler.getDataRef(cache.butler, dataId) ccdId = dataRef.get("ccdExposureId") with self.logOperation("processing %s (ccdId=%d)" % (dataId, ccdId)): self.log.info("Loading... %s - %s" % (dataId, ccdId)) exposure = dataRef.get('calexp', immediate=True) self.log.info("Running... %s - %s" % (dataId, ccdId)) self.fakes.run(exposure, None) self.log.info("Finishing... %s - %s" % (dataId, ccdId)) """ Remove unused mask plane CR, and UNMASKEDNAN """ self.log.info("Removing unused mask plane") maskPlane = exposure.getMaskedImage().getMask() try: maskPlane.removeAndClearMaskPlane('CR', True) except Exception: self.log.info("Can not remove the CR plane") try: maskPlane.removeAndClearMaskPlane('UNMASKEDNAN', True) except Exception: self.log.info("Can not remove the UNMASKEDNAN plane") dataRef.put(exposure, "calexp") return 0 except Exception, errMsg: with open(self.missingLog, "a") as mlog: try: mlog.write("%s , %d\n" % (dataId, ccdId)) fcntl.flock(mlog, fcntl.LOCK_UN) except IOError: pass self.log.warn(str(errMsg)) self.log.warn("Something is wrong for CCD %s (ccdId=%d)" % (dataId, ccdId)) return None
def process(self, cache, dataId): """ add fakes to individual CCDs return None if we are skipping the CCD """ cache.result = None if (dataId["ccd"] in self.config.ignoreCcdList) or (dataId['ccd'] > 103): self.log.warn("Ignoring %s: CCD in ignoreCcdList" % (dataId,)) return None dataRef = hscButler.getDataRef(cache.butler, dataId) ccdId = dataRef.get("ccdExposureId") with self.logOperation("processing %s (ccdId=%d)" %(dataId, ccdId)): exposure = dataRef.get('calexp', immediate=True) self.fakes.run(exposure,None) dataRef.put(exposure,"calexp") return 0
def process(self, cache, dataId): """ add fakes to individual CCDs return None if we are skipping the CCD """ cache.result = None if (dataId["ccd"] in self.config.ignoreCcdList) or (dataId['ccd'] > 103): self.log.warn("Ignoring %s: CCD in ignoreCcdList" % (dataId, )) return None dataRef = hscButler.getDataRef(cache.butler, dataId) ccdId = dataRef.get("ccdExposureId") with self.logOperation("processing %s (ccdId=%d)" % (dataId, ccdId)): exposure = dataRef.get('calexp', immediate=True) self.fakes.run(exposure, None) dataRef.put(exposure, "calexp") return 0
def process(self, cache, dataId): """ Add fakes to individual Patch. Return None if we are skipping the Patch """ cache.result = None ignorePatchList = self.config.ignorePatchList #if dataId["patch"] in ignorePatchList: # self.log.warn("Ignoring %s Patch" % (dataId,)) # return None """ Try to deal with missing CCDs gracefully Song Huang """ #try: self.log.info("Reading...%s" % (dataId,)) dataRef = hscButler.getDataRef(cache.butler, dataId, self.config.coaddName + 'Coadd_calexp') with self.logOperation("processing %s" % (dataId,)): self.log.info("Loading... %s" % (dataId,)) exposure = dataRef.get(self.config.coaddName + 'Coadd_calexp', immediate=True) self.log.info("Running... %s" % (dataId,)) self.fakes.run(exposure, None) self.log.info("Finishing... %s" % (dataId,)) """ Remove unused mask plane CR, and UNMASKEDNAN """ self.log.info("Removing unused mask plane") maskPlane = exposure.getMaskedImage().getMask() try: maskPlane.removeAndClearMaskPlane('CROSSTALK', True) except Exception: self.log.info("Can not remove the CROSSTALK plane") try: maskPlane.removeAndClearMaskPlane('UNMASKEDNAN', True) except Exception: self.log.info("Can not remove the UNMASKEDNAN plane") dataRef.put(exposure, "deepCoadd_calexp") return 0
def process(self, cache, dataId): """ Add fakes to individual Patch. Return None if we are skipping the Patch """ cache.result = None ignorePatchList = self.config.ignorePatchList #if dataId["patch"] in ignorePatchList: # self.log.warn("Ignoring %s Patch" % (dataId,)) # return None """ Try to deal with missing CCDs gracefully Song Huang """ #try: self.log.info("Reading...%s" % (dataId, )) dataRef = hscButler.getDataRef(cache.butler, dataId, self.config.coaddName + 'Coadd_calexp') with self.logOperation("processing %s" % (dataId, )): self.log.info("Loading... %s" % (dataId, )) exposure = dataRef.get(self.config.coaddName + 'Coadd_calexp', immediate=True) self.log.info("Running... %s" % (dataId, )) self.fakes.run(exposure, None) self.log.info("Finishing... %s" % (dataId, )) """ Remove unused mask plane CR, and UNMASKEDNAN """ self.log.info("Removing unused mask plane") maskPlane = exposure.getMaskedImage().getMask() try: maskPlane.removeAndClearMaskPlane('CROSSTALK', True) except Exception: self.log.info("Can not remove the CROSSTALK plane") try: maskPlane.removeAndClearMaskPlane('UNMASKEDNAN', True) except Exception: self.log.info("Can not remove the UNMASKEDNAN plane") dataRef.put(exposure, "deepCoadd_calexp") return 0
def process(self, cache, dataId): """Process this dataId.""" dataRef = hscButler.getDataRef(cache.butler, dataId, datasetType="src") ccdId = dataRef.get("ccdExposureId") with self.logOperation("Started satellite %s (ccdId=%d) on %s" % (dataId, ccdId, basePool.NODE)): try: # *** HERE's the important call *** result = self.satellite.run(dataRef) except Exception, e: self.log.warn("Satellite failed %s: %s\n" % (dataId, e)) import traceback traceback.print_exc() return None if result is not None: # Cache the results (in particular, the image) cache.result = result self.log.info("Finished satellite %s (ccdId=%d) on %s" % (dataId, ccdId, basePool.NODE)) return pipeBase.Struct(ccdId=ccdId)
def main(root, threads, output, input=None, kind=None, visit=None, sensor=None, candidateSet=None): if not candidateSet: candidateSet = candi.knownCandidates butler = dafPersist.Butler(root) rMax, thetaMax = 100.0, 0.15 allMessages = "" ################################################################### # Create an eventList for all kinds of candiate Trail eventLists = {} nMasked = {} for k in candi.Candidate.validKinds: eventLists[k] = EventList() nMasked[k] = 0 eventLists['all'] = EventList() nMasked['all'] = 0 candidateLookup = collections.defaultdict(list) for candidate in candidateSet: did = {'visit': candidate.visit, 'ccd': candidate.ccd} candidateLookup[hashDataId(did)].append(candidate) ################################################################### # Add jobs to the map if input is None: mp = mapr.MapFunc(threads, process) alreadyProcessing = set() for candidate in candidateSet: rightKind = kind is None or (candidate.kind in kind) rightVisit = visit is None or (candidate.visit in visit) rightSensor = sensor is None or (candidate.ccd in sensor) if rightKind and rightVisit and rightSensor and \ (candidate.visit,candidate.ccd) not in alreadyProcessing: dataId = {'visit': candidate.visit, 'ccd': candidate.ccd} dataRef = hscButler.getDataRef(butler, dataId) mp.add(dataRef) alreadyProcessing.add((candidate.visit, candidate.ccd)) results = mp.run() else: with open(input, 'r') as fp: results = pickle.load(fp) if output is not None: with open(output, 'w') as fp: pickle.dump(results, fp) if kind is not None: resultsTmp = [] for result in results: dataHash, foundTrails, runtime = result candidates = candidateLookup[dataHash] rightKind = False for candidate in candidates: if candidate.kind in kind: rightKind = True if rightKind: resultsTmp.append(result) results = resultsTmp falsePos, falseNeg = [], [] #################################################################### # Tally the results and see how we did runtimes = [] widths = collections.defaultdict(list) for result in results: dataHash, foundTrails, runtime = result runtimes.append(runtime) # if there's no candidate for this data candidates = candidateLookup[dataHash] if len(candidates) == 0: resultMsg = "" # if we found something it's a false positive if len(foundTrails) > 0: for iTrail, fTrail in enumerate(foundTrails): resultMsg += "\n %s: %s (%s)" % (clr.color( "FALSE-POS", "red"), fTrail, "no-candidate") eventLists['empty'].append(Event(False, True)) eventLists['all'].append(Event(False, True)) falsePos.append((dataHash, fTrail)) # otherwise, it's a true negative else: resultMsg += "\n %s: %s (%s)" % (clr.color( "TRUE-NEG", "green"), "No Trail", "no-candidate") eventLists['empty'].append(Event(False, False)) eventLists['all'].append(Event(False, False)) allMessages += resultMsg for candidate in candidateLookup[dataHash]: nTrail = len(foundTrails) eList = EventList() t = candidate.trail resultMsg = "" ######################################### # True positives - does result match candidate ######################################### claimed = [False] * nTrail for iTrail, fTrail in enumerate(foundTrails): widths[candidate.kind].append(fTrail.width) nMasked[candidate.kind] += fTrail.nMaskedPixels nMasked['all'] += fTrail.nMaskedPixels if t and (candidate.kind in candi.Candidate.positiveKinds) and \ (fTrail.isNear(t, rMax, thetaMax)): resultMsg += "\n %s: %s (%s)" % (clr.color( "TRUE-POS", "green"), fTrail, candidate.kind) claimed[iTrail] = True eList.append(Event(True, True)) ########################################## # False positives ########################################## nUnclaimed = len(claimed) - sum(claimed) if nUnclaimed > 0: nIgnored = 0 resultTmp = "" for iClaim, claim in enumerate(claimed): if not claim: isIgnored = t and (candidate.kind in candi.Candidate.ignoredKinds) and \ (foundTrails[iClaim].isNear(t, rMax, thetaMax)) isNegative = t and (candidate.kind in candi.Candidate.negativeKinds) and \ (foundTrails[iClaim].isNear(t, rMax, thetaMax)) isOtherCandidate = False for cand in candidateLookup[dataHash]: if cand == candidate: continue if cand.trail and foundTrails[iClaim].isNear( cand.trail, rMax, thetaMax): isOtherCandidate = True if isNegative: tag = "Known-bad" elif isIgnored: tag = "Ignored" nIgnored += 1 resultMsg += "\n %s: %s (%s)" % (clr.color( "IGNORED-POS", "yellow"), fTrail, candidate.kind) continue elif isOtherCandidate: nIgnored += 1 continue else: tag = "Unclaimed" resultTmp += " --> %s: %s\n" % ( tag, clr.color(str(foundTrails[iClaim]), 'yellow')) eList.append(Event(False, True)) falsePos.append((dataHash, foundTrails[iClaim])) if nUnclaimed > nIgnored: resultMsg += "\n %s: %d Unclaimed trails (total=%d) (%s).\n" % \ (clr.color("FALSE-POS", "red"), nUnclaimed, nTrail, candidate.kind) resultMsg += resultTmp ######################################### # True negatives and False ones ######################################### if not resultMsg: # result does not match ... and it shouldn't if t is None or (candidate.kind in \ (candi.Candidate.negativeKinds | candi.Candidate.ignoredKinds)): resultMsg += "\n %s: %s (%s)" % ( clr.color("TRUE-NEG", "green"), "No trails present, and none found.", candidate.kind) eList.append(Event(False, False)) # result does not match ... but it should have else: resultMsg += "\n %s: %s (%s)" % (clr.color( "FALSE-NEG", "red"), t, candidate.kind) eList.append(Event(True, False)) falseNeg.append((dataHash, t, candidate.kind)) did = "(%d, %d)" % (candidate.visit, candidate.ccd) if t: msg = "%s [r=%.1f theta=%.3f wid=%.1f]." % (clr.color( did, "cyan"), t.r, t.theta, t.width) else: msg = "%s [empty candidate]" % (clr.color(did, "cyan")) msg += resultMsg allMessages += msg + "\n" print "\n" + msg + "\n" eventLists[candidate.kind] += eList if candidate.kind not in candi.Candidate.ignoredKinds: eventLists['all'] += eList print clr.color("=== Summary ===", "magenta") print allMessages for kind, eventList in eventLists.items(): print clr.color("=== %s ===" % (kind), "magenta") if len(eventList) == 0: print "No events." continue nPos = eventList.positiveDetections or 1 print "TP=%d, TN=%d, FP=%d, FN=%d" % ( eventList.truePositives, eventList.trueNegatives, eventList.falsePositives, eventList.falseNegatives) print "Recall,precision,f1: %4.2f %4.2f %4.2f" % ( eventList.recall, eventList.precision, eventList.f1) print "Masked pixels: %10d" % (nMasked[kind]), \ " (all: n = %d, %.2f%%)" % (len(eventList),100.0*nMasked[kind]/(len(eventList)*2048*4096)), \ " (det: n = %d, %.2f%%)" % (eventList.positiveDetections,100.0*nMasked[kind]/(nPos*2048*4096)) rt = np.array(runtimes) print "Runtimes: mean=%.2f med=%.2f std=%.2f min=%.2f max=%.2f\n" % \ (rt.mean(), np.median(rt), rt.std(), rt.min(), rt.max()) for k, v in widths.items(): print "Widths: %16s mean=%6.2f med=%6.2f std=%6.2f min=%6.2f max=%6.2f" % \ (k, np.mean(v), np.median(v), np.std(v), np.min(v), np.max(v)) with open("falsePositives.txt", 'w') as fp: for d, f in falsePos: fp.write("%s %s\n" % (str(d), str(f))) with open("falseNegatives.txt", 'w') as fp: for d, f, k in falseNeg: fp.write("%s %s %s\n" % (str(d), k, str(f)))
def main(rerun, visit1, visit2, op, ccdno, datatype='calexp', scale=None, root=None, invert=False, cmap='gray', vmax=None, annotate=None, bins=16, rerun2=None, showCbar=False, vsig=None, showAnnotate=False, percent=None, hilite=None): vsigDefault = 5.0 visit1 = int(visit1) if visit2: visit2 = int(visit2) ccdno = set([int(x) for x in hscUtil.idSplit(ccdno)]) butler1 = hscUtil.getButler(rerun, root=root) if rerun2: butler2 = hscUtil.getButler(rerun2, root=root) else: butler2 = butler1 if datatype not in calibTypes: dataIds1 = butler1.queryMetadata(datatype, "ccd", format=["visit", "ccd"], dataId={'visit': visit1}) dataIds1 = [{'visit': x[0], 'ccd': x[1]} for x in dataIds1] else: dataIds1 = [{'visit': visit1, 'ccd': x} for x in range(104)] dataRef1 = None dataRef2 = None if visit2: dataIds2 = butler2.queryMetadata(datatype, "ccd", format=["visit", "ccd"], dataId={'visit': visit2}) dataIds2 = [{'visit': x[0], 'ccd': x[1]} for x in dataIds2] # flip the color map if invert: cmap = re.sub("_r$", "", cmap) if re.match('_r$', cmap) else '_r' # sleezy, but if vmax isn't set and we're linear, just use ccd in the middle to norm if vmax is None and scale != 'histeq': vmax = 'c049' # handle the gray scale normalization vmin = 0.0 mdRef1 = None mdRef2 = None if vmax: if scale == 'histeq': raise ValueError("Cannot specify vmax with histeq scaling.") # if it identifies a CCD to use if re.match("^c", vmax): vmaxCcd = int(re.sub("c", "", vmax)) try: dataRef1 = hscButler.getDataRef(butler1, { 'visit': visit1, 'ccd': vmaxCcd }) imgRef1 = dataRef1.get( datatype).getMaskedImage().getImage().getArray() if datatype not in calibTypes: mdRef1 = dataRef1.get(datatype + '_md', immediate=True) if visit2: dataRef2 = hscButler.getDataRef(butler2, { 'visit': visit2, 'ccd': vmaxCcd }) imgRef2 = dataRef2.get( datatype).getMaskedImage().getImage().getArray() mdRef2 = dataRef2.get(datatype + '_md', immediate=True) img_op = hscUtil.rebin(compute(imgRef1, imgRef2, op), bins) med = numpy.median(img_op) std = numpy.std(img_op) if not vsig: vsig = vsigDefault delta = vsig * std if percent: delta = percent * med vmin = med - abs(delta) vmax = med + abs(delta) else: if showAnnotate: exp1 = dataRef1.get(datatype) aval = float(exp1.getMetadata().get(annotate)) med = aval delta = vsig if not vsig: delta = 0.5 * aval if percent: delta = percent * aval vmin = med - abs(delta) vmax = med + abs(delta) else: img_op = imgRef1 med = numpy.median(img_op) sig = numpy.sqrt(med) delta = vsigDefault * sig if vsig: delta = vsig * sig if percent: delta = percent * med vmin = med - abs(delta) vmax = med + abs(delta) if not vsig and not percent: vmin = 0.5 * med vmax = med + 5.0 * sig except Exception, e: raise RuntimeError("Could not get stats on vmax CCD" + str(vmax) + " Exiting." + str(e)) elif re.search(":", vmax): vmaxCcd = None vmin, vmax = [float(x) for x in vmax.split(":")] med = 0.5 * (vmax + vmin) else: vmaxCcd = None vmax = float(vmax) med = 0.5 * (vmax + vmin)
im_ax = None i_show = 0 for dataId1 in dataIds1: if dataId1['ccd'] not in ccdno: continue print dataId1, if visit2: dataId2 = {'visit': visit2, 'ccd': dataId1['ccd']} print dataId2 else: print "" try: dataRef1 = hscButler.getDataRef(butler1, dataId1) exp1 = dataRef1.get(datatype) img1 = None if not showAnnotate: img1 = exp1.getMaskedImage().getImage().getArray() img1 = img1 if visit2: dataRef2 = hscButler.getDataRef(butler2, dataId2) exp2 = dataRef2.get(datatype) img2 = exp2.getMaskedImage().getImage().getArray() except Exception, e: #raise print "getDataRef() failed for ", visit1, visit2, ccdno, str(e) continue ax = fpa_fig.getAxes(dataId1['ccd'])
def main(visit, rerun, out, ccd="0..103", root=None, point=None, minchips=1, showLabels=False, boresightOnly=False, edge=0.9): pcen, rad, pcenVisit = None, None, None if point: pcenArgs = map(float, point.split(":")) if len(pcenArgs) == 2: pcenVisit, rad = pcenArgs elif len(pcenArgs) == 3: cx, cy, rad = pcenArgs pcen = afwCoord.Fk5Coord(afwGeom.Point2D(cx, cy)) else: raise ValueError("Unable to parse center 'point', must be ra:dec:rad or visit:rad") print visit, rerun butler = hscUtil.getButler(rerun, root=root) visitWidths = [] visitHeights = [] dataIds = visitCcdToDataId(visit, ccd) corners = {} boresights = {} for dataId in dataIds: visit, ccd = dataId['visit'], dataId['ccd'] if boresightOnly and visit in boresights: continue print "Getting ", visit, ccd try: dataRef = hscButler.getDataRef(butler, dataId) except Exception, e: print "getDataRef() failed for ", visit, ccd, str(e) continue calfile = dataRef.get("calexp_filename", immediate=True)[0] noCal = False if not os.path.exists(calfile): print calfile+" missing." noCal = True else: # get the calexp calmd = dataRef.get("calexp_md", immediate=True) wcs = afwImage.makeWcs(calmd) w, h = calmd.get("NAXIS1"), calmd.get("NAXIS2") boresights[dataId['visit']] = [calmd.get("RA2000"), calmd.get("DEC2000")] # get the corners in RA,Dec ll = wcs.pixelToSky(afwGeom.Point2D(1,1)).toFk5() lr = wcs.pixelToSky(afwGeom.Point2D(w,1)).toFk5() ul = wcs.pixelToSky(afwGeom.Point2D(1,h)).toFk5() ur = wcs.pixelToSky(afwGeom.Point2D(w,h)).toFk5() if noCal: rawfile = dataRef.get("raw_filename", immediate=True)[0] if not os.path.exists(rawfile): print rawfile+" missing too. continuing." continue else: rawmd = dataRef.get("raw_md", immediate=True) raS, decS = rawmd.get("RA2000"), rawmd.get("DEC2000") boresights[dataId['visit']] = [raS, decS] #ra, dec = c.getRa().asDegrees(), c.getDec().asDegrees() crv1, crv2 = map(float, [rawmd.get("CRVAL1"), rawmd.get("CRVAL2")]) crp1, crp2 = map(float, [rawmd.get("CRPIX1"), rawmd.get("CRPIX2")]) useDelt = True try: cdelt1, cdelt2 = map(float, [rawmd.get("CDELT1"), rawmd.get("CDELT1")]) except: useDelt= False if not useDelt: cd11, cd12, cd21, cd22 = map(float, [rawmd.get("CD1_1"), rawmd.get("CD1_2"), rawmd.get("CD2_1"), rawmd.get("CD2_2")]) cdelt1 = 3600*(cd11 + cd12)/206265 cdelt2 = 3600*(cd21 + cd22)/206265 #cdelt1, cdelt2 = 1.0, 1.0 nx, ny = map(int, [rawmd.get("NAXIS1"), rawmd.get("NAXIS2")]) def trans(x, y): ra = crv1 + (crp1-x)*cdelt1 dec = crv2 + (crp2-y)*cdelt2 return ra, dec ll = afwCoord.Fk5Coord(afwGeom.Point2D(*trans(1, 1))) lr = afwCoord.Fk5Coord(afwGeom.Point2D(*trans(nx, 1))) ul = afwCoord.Fk5Coord(afwGeom.Point2D(*trans(1, ny))) ur = afwCoord.Fk5Coord(afwGeom.Point2D(*trans(nx, ny))) if pcenVisit and not pcen: if int(pcenVisit) == int(visit): cx, cy = boresights[pcenVisit] pcen = afwCoord.Fk5Coord(cx, cy) if visit not in corners: corners[visit] = [] corners[visit].append([ll, lr, ur, ul, ccd])
def main(visit, rerun, out, ccd="0..103", root=None, point=None, minchips=1, showLabels=False, boresightOnly=False, edge=0.9): pcen, rad, pcenVisit = None, None, None if point: pcenArgs = map(float, point.split(":")) if len(pcenArgs) == 2: pcenVisit, rad = pcenArgs elif len(pcenArgs) == 3: cx, cy, rad = pcenArgs pcen = afwCoord.Fk5Coord(afwGeom.Point2D(cx, cy)) else: raise ValueError( "Unable to parse center 'point', must be ra:dec:rad or visit:rad" ) print visit, rerun butler = hscUtil.getButler(rerun, root=root) visitWidths = [] visitHeights = [] dataIds = visitCcdToDataId(visit, ccd) corners = {} boresights = {} for dataId in dataIds: visit, ccd = dataId['visit'], dataId['ccd'] if boresightOnly and visit in boresights: continue print "Getting ", visit, ccd try: dataRef = hscButler.getDataRef(butler, dataId) except Exception, e: print "getDataRef() failed for ", visit, ccd, str(e) continue calfile = dataRef.get("calexp_filename", immediate=True)[0] noCal = False if not os.path.exists(calfile): print calfile + " missing." noCal = True else: # get the calexp calmd = dataRef.get("calexp_md", immediate=True) wcs = afwImage.makeWcs(calmd) w, h = calmd.get("NAXIS1"), calmd.get("NAXIS2") boresights[dataId['visit']] = [ calmd.get("RA2000"), calmd.get("DEC2000") ] # get the corners in RA,Dec ll = wcs.pixelToSky(afwGeom.Point2D(1, 1)).toFk5() lr = wcs.pixelToSky(afwGeom.Point2D(w, 1)).toFk5() ul = wcs.pixelToSky(afwGeom.Point2D(1, h)).toFk5() ur = wcs.pixelToSky(afwGeom.Point2D(w, h)).toFk5() if noCal: rawfile = dataRef.get("raw_filename", immediate=True)[0] if not os.path.exists(rawfile): print rawfile + " missing too. continuing." continue else: rawmd = dataRef.get("raw_md", immediate=True) raS, decS = rawmd.get("RA2000"), rawmd.get("DEC2000") boresights[dataId['visit']] = [raS, decS] #ra, dec = c.getRa().asDegrees(), c.getDec().asDegrees() crv1, crv2 = map(float, [rawmd.get("CRVAL1"), rawmd.get("CRVAL2")]) crp1, crp2 = map(float, [rawmd.get("CRPIX1"), rawmd.get("CRPIX2")]) useDelt = True try: cdelt1, cdelt2 = map( float, [rawmd.get("CDELT1"), rawmd.get("CDELT1")]) except: useDelt = False if not useDelt: cd11, cd12, cd21, cd22 = map(float, [ rawmd.get("CD1_1"), rawmd.get("CD1_2"), rawmd.get("CD2_1"), rawmd.get("CD2_2") ]) cdelt1 = 3600 * (cd11 + cd12) / 206265 cdelt2 = 3600 * (cd21 + cd22) / 206265 #cdelt1, cdelt2 = 1.0, 1.0 nx, ny = map(int, [rawmd.get("NAXIS1"), rawmd.get("NAXIS2")]) def trans(x, y): ra = crv1 + (crp1 - x) * cdelt1 dec = crv2 + (crp2 - y) * cdelt2 return ra, dec ll = afwCoord.Fk5Coord(afwGeom.Point2D(*trans(1, 1))) lr = afwCoord.Fk5Coord(afwGeom.Point2D(*trans(nx, 1))) ul = afwCoord.Fk5Coord(afwGeom.Point2D(*trans(1, ny))) ur = afwCoord.Fk5Coord(afwGeom.Point2D(*trans(nx, ny))) if pcenVisit and not pcen: if int(pcenVisit) == int(visit): cx, cy = boresights[pcenVisit] pcen = afwCoord.Fk5Coord(cx, cy) if visit not in corners: corners[visit] = [] corners[visit].append([ll, lr, ur, ul, ccd])
def main(rerun, visit1, visit2, op, ccdno, datatype='calexp', scale=None, root=None, invert=False, cmap='gray', vmax=None, annotate=None, bins=16, rerun2=None, showCbar=False, vsig=None, showAnnotate=False, percent=None, hilite=None): vsigDefault = 5.0 visit1 = int(visit1) if visit2: visit2 = int(visit2) ccdno = set([int(x) for x in hscUtil.idSplit(ccdno)]) butler1 = hscUtil.getButler(rerun, root=root) if rerun2: butler2 = hscUtil.getButler(rerun2, root=root) else: butler2 = butler1 if datatype not in calibTypes: dataIds1 = butler1.queryMetadata(datatype, "ccd", format=["visit", "ccd"], dataId={'visit':visit1}) dataIds1 = [{'visit':x[0], 'ccd':x[1]} for x in dataIds1] else: dataIds1 = [{'visit': visit1, 'ccd':x} for x in range(104)] dataRef1 = None dataRef2 = None if visit2: dataIds2 = butler2.queryMetadata(datatype, "ccd", format=["visit", "ccd"], dataId={'visit':visit2}) dataIds2 = [{'visit':x[0], 'ccd':x[1]} for x in dataIds2] # flip the color map if invert: cmap = re.sub("_r$", "", cmap) if re.match('_r$', cmap) else '_r' # sleezy, but if vmax isn't set and we're linear, just use ccd in the middle to norm if vmax is None and scale != 'histeq': vmax = 'c049' # handle the gray scale normalization vmin = 0.0 mdRef1 = None mdRef2 = None if vmax: if scale == 'histeq': raise ValueError("Cannot specify vmax with histeq scaling.") # if it identifies a CCD to use if re.match("^c", vmax): vmaxCcd = int(re.sub("c", "", vmax)) try: dataRef1 = hscButler.getDataRef(butler1, {'visit':visit1, 'ccd':vmaxCcd}) imgRef1 = dataRef1.get(datatype).getMaskedImage().getImage().getArray() if datatype not in calibTypes: mdRef1 = dataRef1.get(datatype+'_md', immediate=True) if visit2: dataRef2 = hscButler.getDataRef(butler2, {'visit':visit2, 'ccd':vmaxCcd}) imgRef2 = dataRef2.get(datatype).getMaskedImage().getImage().getArray() mdRef2 = dataRef2.get(datatype+'_md', immediate=True) img_op = hscUtil.rebin(compute(imgRef1, imgRef2, op), bins) med = numpy.median(img_op) std = numpy.std(img_op) if not vsig: vsig = vsigDefault delta = vsig*std if percent: delta = percent*med vmin = med - abs(delta) vmax = med + abs(delta) else: if showAnnotate: exp1 = dataRef1.get(datatype) aval = float(exp1.getMetadata().get(annotate)) med = aval delta = vsig if not vsig: delta = 0.5*aval if percent: delta = percent*aval vmin = med - abs(delta) vmax = med + abs(delta) else: img_op = imgRef1 med = numpy.median(img_op) sig = numpy.sqrt(med) delta = vsigDefault*sig if vsig: delta = vsig*sig if percent: delta = percent*med vmin = med - abs(delta) vmax = med + abs(delta) if not vsig and not percent: vmin = 0.5*med vmax = med + 5.0*sig except Exception, e: raise RuntimeError("Could not get stats on vmax CCD" + str(vmax)+ " Exiting." + str(e)) elif re.search(":", vmax): vmaxCcd = None vmin, vmax = [float(x) for x in vmax.split(":")] med = 0.5*(vmax + vmin) else: vmaxCcd = None vmax = float(vmax) med = 0.5*(vmax + vmin)
im_ax = None i_show = 0 for dataId1 in dataIds1: if dataId1['ccd'] not in ccdno: continue print dataId1, if visit2: dataId2 = {'visit':visit2, 'ccd':dataId1['ccd']} print dataId2 else: print "" try: dataRef1 = hscButler.getDataRef(butler1, dataId1) exp1 = dataRef1.get(datatype) img1 = None if not showAnnotate: img1 = exp1.getMaskedImage().getImage().getArray() img1 = img1 if visit2: dataRef2 = hscButler.getDataRef(butler2, dataId2) exp2 = dataRef2.get(datatype) img2 = exp2.getMaskedImage().getImage().getArray() except Exception, e: #raise print "getDataRef() failed for ", visit1, visit2, ccdno, str(e) continue ax = fpa_fig.getAxes(dataId1['ccd'])
def main(root, threads, output, input=None, kind=None, visit=None, sensor=None, candidateSet=None): if not candidateSet: candidateSet = candi.knownCandidates butler = dafPersist.Butler(root) rMax, thetaMax = 100.0, 0.15 allMessages = "" ################################################################### # Create an eventList for all kinds of candiate Trail eventLists = {} nMasked = {} for k in candi.Candidate.validKinds: eventLists[k] = EventList() nMasked[k] = 0 eventLists['all'] = EventList() nMasked['all'] = 0 candidateLookup = collections.defaultdict(list) for candidate in candidateSet: did = {'visit': candidate.visit, 'ccd':candidate.ccd } candidateLookup[hashDataId(did)].append( candidate ) ################################################################### # Add jobs to the map if input is None: mp = mapr.MapFunc(threads, process) alreadyProcessing = set() for candidate in candidateSet: rightKind = kind is None or (candidate.kind in kind) rightVisit = visit is None or (candidate.visit in visit) rightSensor = sensor is None or (candidate.ccd in sensor) if rightKind and rightVisit and rightSensor and \ (candidate.visit,candidate.ccd) not in alreadyProcessing: dataId = {'visit': candidate.visit, 'ccd': candidate.ccd} dataRef = hscButler.getDataRef(butler, dataId) mp.add(dataRef) alreadyProcessing.add( (candidate.visit, candidate.ccd) ) results = mp.run() else: with open(input, 'r') as fp: results = pickle.load(fp) if output is not None: with open(output, 'w') as fp: pickle.dump(results, fp) if kind is not None: resultsTmp = [] for result in results: dataHash, foundTrails, runtime = result candidates = candidateLookup[dataHash] rightKind = False for candidate in candidates: if candidate.kind in kind: rightKind = True if rightKind: resultsTmp.append(result) results = resultsTmp falsePos, falseNeg = [], [] #################################################################### # Tally the results and see how we did runtimes = [] widths = collections.defaultdict(list) for result in results: dataHash, foundTrails, runtime = result runtimes.append(runtime) # if there's no candidate for this data candidates = candidateLookup[dataHash] if len(candidates) == 0: resultMsg = "" # if we found something it's a false positive if len(foundTrails) > 0: for iTrail, fTrail in enumerate(foundTrails): resultMsg += "\n %s: %s (%s)" % (clr.color("FALSE-POS", "red"), fTrail, "no-candidate") eventLists['empty'].append(Event(False, True)) eventLists['all'].append(Event(False, True)) falsePos.append((dataHash, fTrail)) # otherwise, it's a true negative else: resultMsg += "\n %s: %s (%s)" % (clr.color("TRUE-NEG", "green"), "No Trail", "no-candidate") eventLists['empty'].append(Event(False, False)) eventLists['all'].append(Event(False, False)) allMessages += resultMsg for candidate in candidateLookup[dataHash]: nTrail = len(foundTrails) eList = EventList() t = candidate.trail resultMsg = "" ######################################### # True positives - does result match candidate ######################################### claimed = [False]*nTrail for iTrail,fTrail in enumerate(foundTrails): widths[candidate.kind].append(fTrail.width) nMasked[candidate.kind] += fTrail.nMaskedPixels nMasked['all'] += fTrail.nMaskedPixels if t and (candidate.kind in candi.Candidate.positiveKinds) and \ (fTrail.isNear(t, rMax, thetaMax)): resultMsg += "\n %s: %s (%s)" % (clr.color("TRUE-POS", "green"), fTrail, candidate.kind) claimed[iTrail] = True eList.append(Event(True, True)) ########################################## # False positives ########################################## nUnclaimed = len(claimed) - sum(claimed) if nUnclaimed > 0: nIgnored = 0 resultTmp = "" for iClaim,claim in enumerate(claimed): if not claim: isIgnored = t and (candidate.kind in candi.Candidate.ignoredKinds) and \ (foundTrails[iClaim].isNear(t, rMax, thetaMax)) isNegative = t and (candidate.kind in candi.Candidate.negativeKinds) and \ (foundTrails[iClaim].isNear(t, rMax, thetaMax)) isOtherCandidate = False for cand in candidateLookup[dataHash]: if cand == candidate: continue if cand.trail and foundTrails[iClaim].isNear(cand.trail, rMax,thetaMax): isOtherCandidate=True if isNegative: tag = "Known-bad" elif isIgnored: tag = "Ignored" nIgnored += 1 resultMsg += "\n %s: %s (%s)" % (clr.color("IGNORED-POS", "yellow"), fTrail, candidate.kind) continue elif isOtherCandidate: nIgnored += 1 continue else: tag = "Unclaimed" resultTmp += " --> %s: %s\n" % (tag, clr.color(str(foundTrails[iClaim]), 'yellow')) eList.append(Event(False, True)) falsePos.append((dataHash, foundTrails[iClaim])) if nUnclaimed > nIgnored: resultMsg += "\n %s: %d Unclaimed trails (total=%d) (%s).\n" % \ (clr.color("FALSE-POS", "red"), nUnclaimed, nTrail, candidate.kind) resultMsg += resultTmp ######################################### # True negatives and False ones ######################################### if not resultMsg: # result does not match ... and it shouldn't if t is None or (candidate.kind in \ (candi.Candidate.negativeKinds | candi.Candidate.ignoredKinds)): resultMsg += "\n %s: %s (%s)" % (clr.color("TRUE-NEG", "green"), "No trails present, and none found.", candidate.kind) eList.append(Event(False, False)) # result does not match ... but it should have else: resultMsg += "\n %s: %s (%s)" % (clr.color("FALSE-NEG", "red"), t, candidate.kind) eList.append(Event(True, False)) falseNeg.append((dataHash, t, candidate.kind)) did = "(%d, %d)" % (candidate.visit, candidate.ccd) if t: msg = "%s [r=%.1f theta=%.3f wid=%.1f]." % (clr.color(did, "cyan"), t.r, t.theta, t.width) else: msg = "%s [empty candidate]" % (clr.color(did, "cyan")) msg += resultMsg allMessages += msg+"\n" print "\n"+msg+"\n" eventLists[candidate.kind] += eList if candidate.kind not in candi.Candidate.ignoredKinds: eventLists['all'] += eList print clr.color("=== Summary ===", "magenta") print allMessages for kind, eventList in eventLists.items(): print clr.color("=== %s ===" % (kind), "magenta") if len(eventList) == 0: print "No events." continue nPos = eventList.positiveDetections or 1 print "TP=%d, TN=%d, FP=%d, FN=%d" % (eventList.truePositives, eventList.trueNegatives, eventList.falsePositives, eventList.falseNegatives) print "Recall,precision,f1: %4.2f %4.2f %4.2f"%(eventList.recall, eventList.precision, eventList.f1) print "Masked pixels: %10d" % (nMasked[kind]), \ " (all: n = %d, %.2f%%)" % (len(eventList),100.0*nMasked[kind]/(len(eventList)*2048*4096)), \ " (det: n = %d, %.2f%%)" % (eventList.positiveDetections,100.0*nMasked[kind]/(nPos*2048*4096)) rt = np.array(runtimes) print "Runtimes: mean=%.2f med=%.2f std=%.2f min=%.2f max=%.2f\n" % \ (rt.mean(), np.median(rt), rt.std(), rt.min(), rt.max()) for k,v in widths.items(): print "Widths: %16s mean=%6.2f med=%6.2f std=%6.2f min=%6.2f max=%6.2f" % \ (k, np.mean(v), np.median(v), np.std(v), np.min(v), np.max(v)) with open("falsePositives.txt", 'w') as fp: for d, f in falsePos: fp.write("%s %s\n" % (str(d), str(f))) with open("falseNegatives.txt", 'w') as fp: for d, f, k in falseNeg: fp.write("%s %s %s\n" % (str(d), k, str(f)))