def mask_gap(in_path: Path, out_path: Path): """Attempt to mask out pixels beyond the central DNs of the median based on gaps in the histogram. This approach worked well based on 'ideal' reverse-clocked data or 'dark' images, but in 'real' HiRISE images of Mars, the reality is that there are 'gaps' everywhere along the DN range, and this approach ends up being too 'dumb'. """ hist = isis.Histogram(in_path) median = math.trunc(float(hist["Median"])) # std = math.trunc(float(hist['Std Deviation'])) high = find_gap( hist, median, math.trunc(float(hist["Maximum"])), findfirst=True ) low = find_gap( hist, median, math.trunc(float(hist["Minimum"])), findfirst=True ) highdist = high - median lowdist = median - low maskmax = find_gap(hist, median, (median + (2 * highdist))) maskmin = find_gap(hist, median, (median - (2 * lowdist))) isis.mask(in_path, to=out_path, minimum=maskmin, maximum=maskmax) return
def test_getHistVal(self): if tuple(isisversion.version_info()[:3]) < (4, 3, 0): col = "DN" else: col = "MaxExclusive" histogram = isis.Histogram(f"""Total Pixels: 2048000 Null Pixels: 0 Lis Pixels: 0 {col},Pixels,CumulativePixels,Percent,CumulativePercent 3889,1,1,4.88281e-05,4.88281e-05 3924,1,2,4.88281e-05,9.76563e-05 3960,2,4,9.76563e-05,0.000195313 3995,1,5,4.88281e-05,0.000244141 4030,4,9,0.000195313,0.000439453 6841,17215,2020810,0.840576,98.6724 6887,9753,2030563,0.476221,99.1486 7258,460,2045676,0.0224609,99.8865 7304,364,2046040,0.0177734,99.9043 7350,279,2046319,0.013623,99.9179 7759,108,2047734,0.00527344,99.987 7826,104,2047838,0.00507813,99.9921 7893,78,2047916,0.00380859,99.9959 7961,37,2047953,0.00180664,99.9977 8028,21,2047974,0.00102539,99.9987 8095,12,2047986,0.000585937,99.9993 8163,12,2047998,0.000585937,99.9999 8230,2,2048000,9.76563e-05,100""") conf = dict( NoiseFilter_HighEnd_Percent=99.999, NoiseFilter_Hard_Tolmax=1.5, NoiseFilter_Hard_HighEnd_Percent=99.9, ) self.assertEqual((0, 8095), hc.getHistVal(histogram, conf)) bad_conf = dict(conf) bad_conf["NoiseFilter_HighEnd_Percent"] = 150 self.assertRaises(ValueError, hc.getHistVal, histogram, bad_conf) histogram.dictionary["Lis Pixels"] = int(histogram["Total Pixels"]) / 2 self.assertEqual((50, 7304), hc.getHistVal(histogram, conf))
def get_dncnt(cub: os.PathLike, hmin=0.01, hmax=99.99, keep=False) -> int: """Extract DN count from the histogram of a cub file""" # I'm not sure about this method. # The code below is the exact logic that the original Perl program has, # but this is just counting the number of histogram bins # that are within the boundaries, not the number of DN. # And the # of bins is automatically computed by isis.hist, # so could be different for each cube. # logger.info(get_dncnt.__doc__) histfile = Path(cub).with_suffix(".hist") if not histfile.is_file(): isis.hist(cub, to=histfile) h = isis.Histogram(histfile) count: int = 0 for row in h: if hmin <= float(row.CumulativePercent) <= hmax: count += 1 if not keep: histfile.unlink() return count
def setUp(self): self.h = isis.Histogram('''Cube: foo.cub Band: 1 Average: 6490.68 Std Deviation: 166.739 Variance: 27801.9 Median: 6489 Mode: 6489 Skew: 0.0302975 Minimum: 3889 Maximum: 8230 Total Pixels: 2048000 Valid Pixels: 2048000 Null Pixels: 0 Lis Pixels: 0 Lrs Pixels: 0 His Pixels: 0 Hrs Pixels: 0 DN,Pixels,CumulativePixels,Percent,CumulativePercent 3889,1,1,4.88281e-05,4.88281e-05 3924,1,2,4.88281e-05,9.76563e-05 3960,2,4,9.76563e-05,0.000195313 3995,1,5,4.88281e-05,0.000244141 4030,4,9,0.000195313,0.000439453 4065,6,15,0.000292969,0.000732422 4101,12,27,0.000585937,0.00131836 4136,13,40,0.000634766,0.00195312 4171,11,51,0.000537109,0.00249023 4207,11,62,0.000537109,0.00302734 4242,10,72,0.000488281,0.00351562 4277,14,86,0.000683594,0.00419922 4312,4,90,0.000195313,0.00439453 4348,17,107,0.000830078,0.00522461 4383,23,130,0.00112305,0.00634766 4418,9,139,0.000439453,0.00678711 4454,10,149,0.000488281,0.00727539 4489,17,166,0.000830078,0.00810547 4525,17,183,0.000830078,0.00893555 4561,21,204,0.00102539,0.00996094 4597,24,228,0.00117187,0.0111328 4633,19,247,0.000927734,0.0120605 4669,26,273,0.00126953,0.0133301 4705,25,298,0.0012207,0.0145508 4741,16,314,0.00078125,0.015332 4778,20,334,0.000976562,0.0163086 4814,16,350,0.00078125,0.0170898 4850,28,378,0.00136719,0.018457 4886,40,418,0.00195312,0.0204102 4922,24,442,0.00117187,0.021582 4958,45,487,0.00219727,0.0237793 4995,53,540,0.00258789,0.0263672 5031,54,594,0.00263672,0.0290039 5067,64,658,0.003125,0.0321289 5103,57,715,0.0027832,0.0349121 5139,76,791,0.00371094,0.038623 5175,69,860,0.00336914,0.0419922 5212,74,934,0.00361328,0.0456055 5248,97,1031,0.00473633,0.0503418 5284,90,1121,0.00439453,0.0547363 5320,101,1222,0.00493164,0.059668 5356,101,1323,0.00493164,0.0645996 5393,139,1462,0.00678711,0.0713867 5429,134,1596,0.00654297,0.0779297 5465,144,1740,0.00703125,0.0849609 5501,162,1902,0.00791016,0.0928711 5537,167,2069,0.0081543,0.101025 5573,196,2265,0.00957031,0.110596 5610,225,2490,0.0109863,0.121582 5646,271,2761,0.0132324,0.134814 5682,352,3113,0.0171875,0.152002 5718,420,3533,0.0205078,0.17251 5754,524,4057,0.0255859,0.198096 5790,658,4715,0.0321289,0.230225 5827,745,5460,0.036377,0.266602 5863,881,6341,0.0430176,0.309619 5899,1124,7465,0.0548828,0.364502 5935,1329,8794,0.0648926,0.429395 5971,1767,10561,0.0862793,0.515674 6007,2522,13083,0.123145,0.638818 6044,3499,16582,0.17085,0.809668 6081,5372,21954,0.262305,1.07197 6118,8701,30655,0.424854,1.49683 6155,14642,45297,0.714941,2.21177 6192,24158,69455,1.17959,3.39136 6229,39133,108588,1.91079,5.30215 6266,58840,167428,2.87305,8.1752 6303,84919,252347,4.14644,12.3216 6340,115400,367747,5.63477,17.9564 6378,153026,520773,7.47197,25.4284 6415,179963,700736,8.78726,34.2156 6452,204294,905030,9.97529,44.1909 6489,215791,1120821,10.5367,54.7276 6526,208155,1328976,10.1638,64.8914 6563,185376,1514352,9.05156,73.943 6600,151221,1665573,7.38384,81.3268 6637,117121,1782694,5.7188,87.0456 6674,85864,1868558,4.19258,91.2382 6711,59945,1928503,2.927,94.1652 6751,45028,1973531,2.19863,96.3638 6795,30064,2003595,1.46797,97.8318 6841,17215,2020810,0.840576,98.6724 6887,9753,2030563,0.476221,99.1486 6934,5370,2035933,0.262207,99.4108 6980,3296,2039229,0.160937,99.5717 7026,2113,2041342,0.103174,99.6749 7073,1411,2042753,0.0688965,99.7438 7119,1067,2043820,0.0520996,99.7959 7165,818,2044638,0.0399414,99.8358 7211,578,2045216,0.0282227,99.8641 7258,460,2045676,0.0224609,99.8865 7304,364,2046040,0.0177734,99.9043 7350,279,2046319,0.013623,99.9179 7397,276,2046595,0.0134766,99.9314 7443,225,2046820,0.0109863,99.9424 7494,228,2047048,0.0111328,99.9535 7557,240,2047288,0.0117188,99.9652 7624,196,2047484,0.00957031,99.9748 7691,142,2047626,0.00693359,99.9817 7759,108,2047734,0.00527344,99.987 7826,104,2047838,0.00507813,99.9921 7893,78,2047916,0.00380859,99.9959 7961,37,2047953,0.00180664,99.9977 8028,21,2047974,0.00102539,99.9987 8095,12,2047986,0.000585937,99.9993 8163,12,2047998,0.000585937,99.9999 8230,2,2048000,9.76563e-05,100''')
def test_len(self): h = isis.Histogram(self.histfile) self.assertEqual(107, len(h))
def test_contains(self): h = isis.Histogram(self.histfile) self.assertTrue('Std Deviation' in h)
def test_listlike(self): h = isis.Histogram(self.histfile) self.assertEqual(5, len(h[0]))
def test_dictlike(self): h = isis.Histogram(self.histfile) self.assertEqual(self.cube.name, h['Cube'])
def test_init_str(self): h = isis.Histogram(isis.hist_k(self.cube)) self.assertIsInstance(h, isis.Histogram)
def test_init_cube(self): h = isis.Histogram(self.cube) self.assertIsInstance(h, isis.Histogram)
def test_init_histfile(self): h = isis.Histogram(self.histfile) self.assertIsInstance(h, isis.Histogram)
def unflip(in_p: Path, out_p: Path, keep=False): """Attempt to indentify DNs whose bits have been flipped in the ISIS cube indicated by *in_p*, and unflip them. Only operates on image-area. """ to_del = isis.PathSet() # This full suite of deltas works well for the reverse-clock area # and even 'dark' images, but not 'real' images. # deltas = (8192, 4096, 2048, 1024, 512, 256, 128, 64) # Restricting the number of deltas might help, but this seems # arbitrary. deltas = (8192, 4096, 2048, 1024) count = 0 suffix = ".bf{}-{}{}.cub" this_p = to_del.add(in_p.with_suffix(suffix.format(count, 0, 0))) this_p.symlink_to(in_p) median = math.trunc(float(isis.stats_k(in_p)["Median"])) for (sign, pm, extrema) in ((+1, "m", "Maximum"), (-1, "p", "Minimum")): # logging.info(pm) for delt in deltas: d = sign * delt far = median + d near = median + (d / 2) hist = isis.Histogram(this_p) logger.info( f"bitflip position {pm}{delt}, near: {near} " f"far: {far}, extrema: {hist[extrema]}" ) if (sign > 0 and far < float(hist[extrema])) or ( sign < 0 and far > float(hist[extrema]) ): count += 1 s = suffix.format(count, pm, delt) next_p = to_del.add(this_p.with_suffix("").with_suffix(s)) try: thresh = get_unflip_thresh(hist, far, near, d) except ValueError as err: logger.info(err) count -= 1 break subtract_over_thresh(this_p, next_p, thresh, d, keep=keep) this_p = next_p else: logger.info( "The far value was beyond the extrema. " "Didn't bother." ) shutil.move(this_p, out_p) if not keep: to_del.remove(this_p) to_del.unlink() return