def __init__(self, pathlike, cubenormstep=False): self.path = Path(pathlike) self.nextpath = self.path super().__init__(hirise.get_CCDID_fromfile(self.path)) self.ns = int(isis.getkey_k(self.path, "Dimensions", "Samples")) self.nl = int(isis.getkey_k(self.path, "Dimensions", "Lines")) self.bin = int(isis.getkey_k(self.path, "Instrument", "Summing")) self.cubenormstep = cubenormstep self.cubenorm_stddev = None self.sl_cubenorm = None self.nl_cubenorm = None self.ss_balance_left = None self.ns_balance_left = None self.ss_balance_right = None self.ns_balance_right = None self.sl_balance = None self.nl_balance = None self.smag = None self.lmag = None self.ls_path = None self.rs_path = None self.lm_path = None self.rm_path = None self.rstats = None self.lstats = None self.correction = None self.hical_status = None self.snr_list = list()
def get_binning_from(self, color_code, pids=None, db_paths=None): dbs = list() if pids is None: pids = self.pids parent = self.path.parent if db_paths is None: db_paths = list() for pid in pids: db_paths.extend(list(parent.glob(f"{pid}*.json"))) for pid in pids: if color_code in pid: if len(db_paths) > 0: for p in db_paths: with open(p, "r") as f: dbs.append(json.load(f)) else: for p in parent.glob(f"{pid}*.cub"): temp_d = dict() temp_d["PRODUCT_ID"] = isis.getkey_k( p, "Archive", "ProductId") temp_d["BINNING"] = int( isis.getkey_k(p, "Instrument", "Summing")) dbs.append(temp_d) return dbs
def __init__(self, pathlike): self.path = Path(pathlike) super().__init__(hirise.get_CCDID_fromfile(self.path)) self.bin = int(isis.getkey_k(self.path, "Instrument", "Summing")) self.tdi = int(isis.getkey_k(self.path, "Instrument", "TDI")) self.lines = int(isis.getkey_k(self.path, "Dimensions", "Lines")) self.samps = int(isis.getkey_k(self.path, "Dimensions", "Samples"))
def EDR_Stats( img: os.PathLike, out_path: os.PathLike, gainsinfo: dict, histmin=0.01, histmax=99.99, keep=False, ) -> dict: logger.info(f"EDR_Stats start: {img}") try: logger.info("The LUT for this file is: " + str(check_lut(img))) except KeyError as err: logger.error("The LUT header area is either corrupted or has a gap.") raise err # Convert to .cub isis.hi2isis(img, to=out_path) histat_complete = isis.histat( out_path, useoffsets=True, leftimage=0, rightimage=1, leftcalbuffer=3, rightcalbuffer=1, leftcaldark=3, rightcaldark=1, leftbuffer=3, rightbuffer=1, leftdark=3, rightdark=1, ) histats = parse_histat(histat_complete.stdout) # Get some info from the new cube: histats["PRODUCT_ID"] = isis.getkey_k(out_path, "Archive", "ProductId") histats["IMAGE_LINES"] = int( isis.getkey_k(out_path, "Dimensions", "Lines") ) histats["LINE_SAMPLES"] = int( isis.getkey_k(out_path, "Dimensions", "Samples") ) histats["BINNING"] = int(isis.getkey_k(out_path, "Instrument", "Summing")) histats["STD_DN_LEVELS"] = get_dncnt(out_path, histmin, histmax, keep=keep) histats["IMAGE_SIGNAL_TO_NOISE_RATIO"] = calc_snr( out_path, gainsinfo, histats ) histats["GAP_PIXELS_PERCENT"] = ( histats["GAP_PIXELS"] / (int(histats["IMAGE_LINES"]) * int(histats["LINE_SAMPLES"])) ) * 100.0 tdi_bin_check(out_path, histats) lut_check(out_path, histats) logger.info(f"EDR_Stats done: {out_path}") return histats
def test_Hidestripe(self): to_del = isis.PathSet() calcube = to_del.add(Path("test_Hidestripe-out.hical.cub")) isis.hical(self.cube, to=calcube) to_del.add(Path(str(self.pid)).with_suffix(".hical.log")) outcube = to_del.add(Path("test_Hidestripe-out.cub")) samps = int(isis.getkey_k(calcube, "Dimensions", "Samples")) self.assertRaises( KeyError, hc.Hidestripe, self.cube, outcube, self.binning, minimum=0.0, maximum=1.5, hidcorr="ADD", line_samples=samps, keep=False, ) self.assertAlmostEqual( 0.000101402295171637, hc.Hidestripe( calcube, outcube, self.binning, minimum=0.0, maximum=1.5, hidcorr="ADD", line_samples=samps, keep=False, ), ) to_del.unlink()
def setUp(self): self.cube = imgs[0].with_suffix(".TestHiCal.cub") self.pid = hirise.get_ChannelID_fromfile(self.cube) self.db = edr.EDR_Stats(imgs[0], self.cube, gains) self.binning = int(isis.getkey_k(self.cube, "Instrument", "Summing")) self.conf = conf # self.conf['HiGainFx'] = pvl.load(str(hgf_conf))['HiGainFx'] self.conf["NoiseFilter"] = nf_conf["NoiseFilter"]
def sort_input_cubes(a_cub: os.PathLike, b_cub: os.PathLike) -> tuple: """Figures out which one is Channel 0 and which is Channel 1.""" channel_of = dict() channel_of[a_cub] = isis.getkey_k(a_cub, "Instrument", "ChannelNumber") channel_of[b_cub] = isis.getkey_k(b_cub, "Instrument", "ChannelNumber") if channel_of[a_cub] == channel_of[b_cub]: raise RuntimeError(f"{a_cub} and {b_cub} have the same channel: " "{channel_of[a_cub]}.") for (cub, chan) in channel_of.items(): if int(chan) != 0 and int(chan) != 1: raise RuntimeError( f"{cub} has a channel other than 0 or 1: {chan}") sorted_cubchan = sorted(channel_of.items(), key=lambda x: x[1]) return (sorted_cubchan[0][0], sorted_cubchan[1][0])
def _get_fromfile(path: os.PathLike, IDclass, name, archivekey): p = Path(path) try: return IDclass(isis.getkey_k(p, "Archive", archivekey)) except (subprocess.CalledProcessError, ValueError): # The CalledProcessError is if there is some problem with running # getkey, the ValueError is if a CCDID can't be extracted from # the labels. This allows this function to be called on any kind of # file, as it will just try and read the CCDID from the filename # or could also reverse recurse up to directory names. return _reverse_recurse(p, IDclass, name)
def tdi_bin_check(cube: os.PathLike, histats: dict): """This function only logs warnings and returns nothing.""" # TDI and binning check if float(histats["IMAGE_MEAN"]) >= 8000: logger.warning( "Channel mean greater than 8000 (TDI or binning too high)." ) elif float(histats["IMAGE_MEAN"]) < 2500: tdi = isis.getkey_k(cube, "Instrument", "Tdi") if tdi == "32" or tdi == "64": logger.warning("TDI too low.") return
def SpecialProcessingFlags(cube: HiccdStitchCube): """Set the special processing flags in the ISIS label.""" status = "NOMINAL" if cube.hical_status == "BadCal": status = "BADCAL" if cube.cubenormstep: status = "CUBENORM" try: isis.getkey_k(cube.nextpath, "Instrument", "Special_Processing_Flag") option = "MODKEY" except subprocess.CalledProcessError: option = "ADDKEY" isis.editlab( cube.nextpath, option=option, grpname="Instrument", keyword="Special_Processing_Flag", value=status, )
def __init__(self, pathlike, dbs=None): self.path = Path(pathlike) match = re.search(r"COLOR(\d)", str(pathlike)) if match: self.ccdnumber = match.group(1) else: raise ValueError(f"Could not extract a COLOR from {pathlike}") super().__init__(hirise.get_ObsID_fromfile(self.path)) self.lines = int(isis.getkey_k(self.path, "Dimensions", "Lines")) self.samps = int(isis.getkey_k(self.path, "Dimensions", "Samples")) self.bands = int(isis.getkey_k(self.path, "Dimensions", "Bands")) if self.bands != 3: raise Exception("{} must have 3 bands.".format(self.path)) # centers = isis.getkey_k(self.path, 'BandBin', 'Center') # match_c = re.search(r'900,\s+700,\s+500', centers) if not match: raise Exception( f"{self.path} must have BandBin/Center of 900, 700, 500.") self.band = {"IR": 1, "RED": 2, "BG": 3} self.pids = isis.getkey_k(self.path, "Mosaic", "SourceProductId").split(", ") self.ir_bin = self.get_binning("IR", dbs) self.red_bin = self.get_binning("RED", dbs) # self.bg_bin = get_binning('BG') self.mask_path = {"IR": None, "BG": None} self.crop_path = {"IR": None, "BG": None} self.nrm_path = {"IR": None, "BG": None} self.crop_sline = None self.crop_lines = None self.final_path = None
def tdi_bin_check(cube: os.PathLike, histats: dict, cid=None): """This function only logs warnings and returns nothing.""" if cid is None: try: cid = histats['PRODUCT_ID'] except KeyError: cid = hirise.get_ChannelID_fromfile(cube) # TDI and binning check if float(histats["IMAGE_MEAN"]) >= 8000: logger.warning( f"{cid}: " f"Channel mean greater than 8000 (TDI or binning too high)." ) elif float(histats["IMAGE_MEAN"]) < 2500: tdi = isis.getkey_k(cube, "Instrument", "Tdi") if tdi == "32" or tdi == "64": logger.warning(f"{cid}: TDI too low.") return
def fix_labels(cubes: list, path: os.PathLike, matched_cube: str, prodid: str) -> None: isis.editlab( path, option="modkey", grpname="Archive", keyword="ProductId", value=prodid, ) isis.editlab( path, option="modkey", grpname="Instrument", keyword="MatchedCube", value=str(matched_cube), ) # Fix ck kernel in InstrumentPointing in RED label # This doesn't seem to be needed, maybe it was HiROC-specific. # Add SourceProductIds to Archive group in label logger.info("Original Perl just assumes that both channels are included " "in the balance cube.") source_ids = list() for c in cubes: source_ids.append( isis.getkey_k(c.path, "Instrument", "StitchedProductIds")) isis.editlab( path, option="ADDKEY", grpname="Archive", keyword="SourceProductId", value="({})".format(", ".join(source_ids)), ) return
def test_getkey_k(self): truth = 'HIRISE' key = isis.getkey_k(self.cub, 'Instrument', 'InstrumentId') self.assertEqual(truth, key)
def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "-o", "--output", required=False, default=".mdr.iof.cub" ) parser.add_argument("-e", "--edr", required=True) parser.add_argument( "-c", "--conf", required=False, type=argparse.FileType('r'), default=pkg_resources.resource_stream( __name__, 'data/hical.pipelines.conf' ), ) parser.add_argument("mdr", metavar="MDR_file") parser.add_argument( "-l", "--log", required=False, default="WARNING", help="The log level to show for this program, can " "be a named log level or a numerical level.", ) parser.add_argument( "-k", "--keep", required=False, default=False, action="store_true", help="Normally, the program will clean up any " "intermediary files, but if this option is given, it " "won't.", ) args = parser.parse_args() util.set_logger(args.verbose, args.logfile, args.log) edr_path = Path(args.edr) mdr_path = Path(args.mdr) to_del = isis.PathSet() h2i_path = to_del.add(edr_path.with_suffix(".hi2isis.cub")) out_path = util.path_w_suffix(args.output, edr_path) # The first thing Alan's program did was to crop the image down to only the # 'imaging' parts. We're not doing that so the resultant file has a # geometry similar to what comes out of ISIS hical. # Convert the EDR to a cube file isis.hi2isis(edr_path, to=h2i_path) # Convert Alan's MDR to a cube file mdr_cub_path = to_del.add(mdr_path.with_suffix(".alan.cub")) logger.info(f"Running gdal_translate {mdr_path} -of ISIS3 {mdr_cub_path}") gdal.Translate(str(mdr_cub_path), str(mdr_path), format="ISIS3") h2i_s = int(isis.getkey_k(h2i_path, "Dimensions", "Samples")) h2i_l = int(isis.getkey_k(h2i_path, "Dimensions", "Lines")) mdr_s = int(isis.getkey_k(mdr_cub_path, "Dimensions", "Samples")) mdr_l = int(isis.getkey_k(mdr_cub_path, "Dimensions", "Lines")) if h2i_s != mdr_s: label = pvl.load(str(h2i_path)) hirise_cal_info = get_one( label, "Table", "HiRISE Calibration Ancillary" ) buffer_pixels = get_one(hirise_cal_info, "Field", "BufferPixels")[ "Size" ] dark_pixels = get_one(hirise_cal_info, "Field", "DarkPixels")["Size"] rev_mask_tdi_lines = hirise_cal_info["Records"] if h2i_s + buffer_pixels + dark_pixels == mdr_s: logger.info( f"The file {mdr_cub_path} has " f"{buffer_pixels + dark_pixels} more sample pixels " f"than {h2i_path}, assuming those are dark and " "buffer pixels and will crop accordingly." ) if h2i_l + rev_mask_tdi_lines != mdr_l: logger.critical( 'Even assuming this is a "full" channel ' "image, this has the wrong number of lines. " f"{mdr_cub_path} should have " f"{h2i_l + rev_mask_tdi_lines}, but " f"has {mdr_l} lines. Exiting" ) sys.exit() else: crop_path = to_del.add(mdr_cub_path.with_suffix(".crop.cub")) # We want to start with the next pixel (+1) after the cal # pixels. isis.crop( mdr_cub_path, to=crop_path, sample=buffer_pixels + 1, nsamples=h2i_s, line=rev_mask_tdi_lines + 1, ) mdr_cub_path = crop_path mdr_l = int(isis.getkey_k(mdr_cub_path, "Dimensions", "Lines")) else: logger.critical( f"The number of samples in {h2i_path} ({h2i_s}) " f"and {mdr_cub_path} ({mdr_s}) are different. " "Exiting." ) sys.exit() if h2i_l != mdr_l: logger.critical( f"The number of lines in {h2i_path} ({h2i_l}) " f"and {mdr_cub_path} ({mdr_l}) are different. " "Exiting." ) sys.exit() # Convert the EDR to the right bit type for post-HiCal Pipeline: h2i_16b_p = to_del.add(h2i_path.with_suffix(".16bit.cub")) isis.bit2bit( h2i_path, to=h2i_16b_p, bit="16bit", clip="minmax", minval=0, maxval=1.5, ) shutil.copyfile(h2i_16b_p, out_path) # If it is a channel 1 file, Alan mirrored it so that he could process # the two channels in an identical way (which we also took advantage # of above if the buffer and dark pixels were included), so we need to # mirror it back. cid = hirise.get_ChannelID_fromfile(h2i_16b_p) if cid.channel == "1": mirror_path = to_del.add(mdr_cub_path.with_suffix(".mirror.cub")) isis.mirror(mdr_cub_path, to=mirror_path) mdr_cub_path = mirror_path # Is the MDR in DN or I/F? maximum_pxl = float( pvl.loads(isis.stats(mdr_cub_path).stdout)["Results"]["Maximum"] ) if maximum_pxl < 1.5: logger.info("MDR is already in I/F units.") mdr_16b_p = to_del.add(mdr_cub_path.with_suffix(".16bit.cub")) isis.bit2bit( mdr_cub_path, to=mdr_16b_p, bit="16bit", clip="minmax", minval=0, maxval=1.5, ) isis.handmos(mdr_16b_p, mosaic=out_path) else: logger.info("MDR is in DN units and will be converted to I/F.") fpa_t = statistics.mean( [ float( isis.getkey_k( h2i_16b_p, "Instrument", "FpaPositiveYTemperature" ) ), float( isis.getkey_k( h2i_16b_p, "Instrument", "FpaNegativeYTemperature" ) ), ] ) print(f"fpa_t {fpa_t}") conf = pvl.load(args.conf) tdg = t_dep_gain(get_one(conf["Hical"], "Profile", cid.ccdname), fpa_t) suncorr = solar_correction() sclk = isis.getkey_k( h2i_16b_p, "Instrument", "SpacecraftClockStartCount" ) target = isis.getkey_k(h2i_16b_p, "Instrument", "TargetName") suncorr = solar_correction(sunDistanceAU(sclk, target)) sed = float( isis.getkey_k(h2i_16b_p, "Instrument", "LineExposureDuration") ) zbin = get_one(conf["Hical"], "Profile", "GainUnitConversion")[ "GainUnitConversionBinFactor" ] # The 'ziof' name is from the ISIS HiCal/GainUnitConversion.h, it is a # divisor in the calibration equation. print(f"zbin {zbin}") print(f"tdg {tdg}") print(f"sed {sed}") print(f"suncorr {suncorr}") ziof = zbin * tdg * sed * 1e-6 * suncorr eqn = f"\(F1 / {ziof})" # noqa W605 mdriof_p = to_del.add(mdr_cub_path.with_suffix(".iof.cub")) to_s = "{}+SignedWord+{}:{}".format(mdriof_p, 0, 1.5) isis.fx(f1=mdr_cub_path, to=to_s, equ=eqn) isis.handmos(mdriof_p, mosaic=out_path) if not args.keep: to_del.unlink()
def lut_check(cube: os.PathLike, histats: dict, cid=None): # LUT check lut = int(isis.getkey_k(cube, "Instrument", "LookupTableNumber")) orbit_number = int(isis.getkey_k(cube, "Archive", "OrbitNumber")) threshhold = dict() if lut != -9998: if orbit_number > 65881: # After orbit 65881, RED1 RED2 RED3 moved to to DN 900-1000 offset threshhold["RED0"] = ( (6814, 22), (5341, 23), (3869, 24), (3133, 25), (2397, 26), (1200, 27), ) threshhold["RED1"] = ( (6619, 1), (5116, 2), (3614, 3), (2863, 4), (2112, 5), (900, 6), ) threshhold["RED2"] = threshhold["RED1"] threshhold["RED3"] = threshhold["RED1"] threshhold["RED4"] = ( (6684, 8), (5191, 9), (3699, 10), (2953, 11), (2207, 12), (1000, 13) ) threshhold["RED5"] = ( (6749, 15), (5266, 16), (3784, 17), (3043, 18), (2302, 19), (1100, 20), ) threshhold["RED6"] = threshhold["RED4"] threshhold["RED7"] = threshhold["RED4"] threshhold["RED8"] = threshhold["RED0"] threshhold["RED9"] = threshhold["RED4"] threshhold["IR10"] = threshhold["RED1"] threshhold["IR11"] = threshhold["RED4"] threshhold["BG12"] = threshhold["RED1"] threshhold["BG13"] = threshhold["RED1"] elif orbit_number > 13057: # After orbit 13057, IR10 moved to DN 900-1000 offset threshhold["RED0"] = ( (6814, 22), (5341, 23), (3869, 24), (3133, 25), (2397, 26), (1200, 27), ) threshhold["RED1"] = ( (6684, 8), (5191, 9), (3699, 10), (2953, 11), (2207, 12), (1000, 13), ) threshhold["RED2"] = threshhold["RED1"] threshhold["RED3"] = threshhold["RED1"] threshhold["RED4"] = threshhold["RED1"] threshhold["RED5"] = ( (6749, 15), (5266, 16), (3784, 17), (3043, 18), (2302, 19), (1100, 20), ) threshhold["RED6"] = threshhold["RED1"] threshhold["RED7"] = threshhold["RED1"] threshhold["RED8"] = threshhold["RED0"] threshhold["RED9"] = threshhold["RED1"] threshhold["IR10"] = ( (6619, 1), (5116, 2), (3614, 3), (2863, 4), (2112, 5), (900, 6), ) threshhold["IR11"] = threshhold["RED1"] threshhold["BG12"] = threshhold["IR10"] threshhold["BG13"] = threshhold["IR10"] elif orbit_number > 11710: # After orbit 11710, RED6 moved to DN 1000-1100 offset threshhold["RED0"] = ( (6814, 22), (5341, 23), (3869, 24), (3133, 25), (2397, 26), (1200, 27), ) threshhold["RED1"] = ( (6684, 8), (5191, 9), (3699, 10), (2953, 11), (2207, 12), (1000, 13), ) threshhold["RED2"] = threshhold["RED1"] threshhold["RED3"] = threshhold["RED1"] threshhold["RED4"] = threshhold["RED1"] threshhold["RED5"] = ( (6749, 15), (5266, 16), (3784, 17), (3043, 18), (2302, 19), (1100, 20), ) threshhold["RED6"] = threshhold["RED1"] threshhold["RED7"] = threshhold["RED1"] threshhold["RED8"] = threshhold["RED0"] threshhold["RED9"] = threshhold["RED1"] threshhold["IR10"] = threshhold["RED1"] threshhold["IR11"] = threshhold["RED1"] threshhold["BG12"] = ( (6619, 1), (5116, 2), (3614, 3), (2863, 4), (2112, 5), (900, 6), ) threshhold["BG13"] = threshhold["BG12"] elif orbit_number > 2660: # New LUT table after orbit 2660 threshhold["RED0"] = ( (6814, 22), (5341, 23), (3869, 24), (3133, 25), (2397, 26), (1200, 27), ) threshhold["RED1"] = ( (6684, 8), (5191, 9), (3699, 10), (2953, 11), (2207, 12), (1000, 13), ) threshhold["RED2"] = threshhold["RED1"] threshhold["RED3"] = threshhold["RED1"] threshhold["RED4"] = threshhold["RED1"] threshhold["RED5"] = ( (6749, 15), (5266, 16), (3784, 17), (3043, 18), (2302, 19), (1100, 20), ) threshhold["RED6"] = ( (6619, 1), (5116, 2), (3614, 3), (2863, 4), (2112, 5), (900, 6), ) threshhold["RED7"] = threshhold["RED1"] threshhold["RED8"] = threshhold["RED0"] threshhold["RED9"] = threshhold["RED1"] threshhold["IR10"] = threshhold["RED1"] threshhold["IR11"] = threshhold["RED1"] threshhold["BG12"] = threshhold["RED6"] threshhold["BG13"] = threshhold["RED6"] else: # Original LUTs prior to 2661 threshhold["RED0"] = ( (14057, 21), (11676, 22), (9295, 23), (7152, 24), (5248, 25), (3343, 26), (1200, 27), ) threshhold["RED1"] = ( (13857 + 1, 14), (11476 + 1, 15), (9095 + 1, 16), (6952 + 1, 17), (5048 + 1, 18), (3143 + 1, 19), (1000, 20), ) threshhold["RED2"] = threshhold["RED1"] threshhold["RED3"] = threshhold["RED1"] threshhold["RED4"] = threshhold["RED1"] threshhold["RED5"] = threshhold["RED1"] threshhold["RED6"] = ( (13657 + 1, 7), (11276 + 1, 8), (8895 + 1, 9), (6752 + 1, 10), (4848 + 1, 11), (2943 + 1, 12), (800, 13), ) threshhold["RED7"] = threshhold["RED1"] threshhold["RED8"] = threshhold["RED0"] threshhold["RED9"] = threshhold["RED1"] threshhold["IR10"] = threshhold["RED1"] threshhold["IR11"] = threshhold["RED1"] threshhold["BG12"] = threshhold["RED6"] threshhold["BG13"] = threshhold["RED6"] ccd = hirise.ChannelID( isis.getkey_k(cube, "Archive", "ProductId") ).get_ccd() if cid is None: cid = ccd for (th, ex) in threshhold[ccd]: if float(histats["IMAGE_MEAN"]) >= th: lut_diff = lut - ex if lut_diff >= 1 or lut_diff <= -1: if lut_diff > 0: direction = "to the right" else: direction = "to the left" logger.warning( f"{cid}: LUT is {lut_diff} column(s) ({direction}) " f"from ideal settings - image overcompressed." ) break else: logger.warning( f"{cid}: DN value, {histats['IMAGE_MEAN']}, lower than lowest " f"DN value with defined LUT for this channel." ) return
def test_calc_snr(self): histats = edr.parse_histat(isis.histat(self.hicube).stdout) histats["BINNING"] = isis.getkey_k(self.hicube, "Instrument", "Summing") s = edr.calc_snr(self.hicube, pvl.load(gains), histats) self.assertAlmostEqual(s, 291.80442197)
def setUp(self): self.cube = imgs[0].with_suffix(".TestHiCal_TestNeedISISCube.cub") isis.hi2isis(imgs[0], to=self.cube) self.pid = hirise.get_ChannelID_fromfile(self.cube) self.binning = int(isis.getkey_k(self.cube, "Instrument", "Summing"))
def HiFurrow_Fix(in_cube: os.PathLike, out_cube: os.PathLike, max_mean: float, keep=False): """Perform a normalization of the furrow region of bin 2 or 4 HiRISE images. The input to this script is a HiRISE stitch product containing both channels of a CCD. """ in_cub = Path(in_cube) binning = int(isis.getkey_k(in_cub, "Instrument", "Summing")) lines = int(isis.getkey_k(in_cub, "Dimensions", "Lines")) samps = int(isis.getkey_k(in_cub, "Dimensions", "Samples")) if binning != 2 and binning != 4: raise ValueError("HiFurrow_Fix only supports correction for " "bin 2 or 4 data.") if binning == 2 and samps != 1024: raise ValueError(f"HiFurrowFix: improper number of samples: {samps}, " "for a stitch product with bin 2 (should be 1024).") # This string will get placed in the filename for all of our # temporary files. It will (hopefully) prevent collisions with # existing files and also allow for easy clean-up if keep=True temp_token = datetime.now().strftime("HFF-%y%m%d%H%M%S") to_del = isis.PathSet() # For bin2 and bin4 imaging, specify width of furrow based on # image average DN range range_low = {2: (512, 513), 4: (256, 257)} # 2 pixel furrow width range_mid = {2: (511, 514), 4: (255, 258)} # 4 pixel furrow width range_hgh = {2: (511, 514), 4: (255, 258)} # 4 pixel furrow width range_max = {2: (510, 515), 4: (254, 259)} # 6 pixel furrow width # Original code had low/mid/hgh for bin2 and bin4, but they # were hard-coded to be identical. dn_range_low = 9000 dn_range_mid = 10000 dn_range_hgh = 12000 if max_mean > dn_range_hgh: dn_range = range_max[binning] elif max_mean > dn_range_mid: dn_range = range_hgh[binning] elif max_mean > dn_range_low: dn_range = range_mid[binning] else: dn_range = range_low[binning] lpf_samp = int((dn_range[1] - dn_range[0] + 1) / 2) * 4 + 1 lpf_line = int(lpf_samp / 2) * 20 + 1 # Create a mask file # DN=1 for non-furrow area # DN=0 for furrow area eqn = rf"\(1*(sample<{dn_range[0]})+ 1*(sample>{dn_range[1]}) + 0)" fx_cub = to_del.add(in_cub.with_suffix(f".{temp_token}.fx.cub")) isis.fx(to=fx_cub, mode="OUTPUTONLY", lines=lines, samples=samps, equation=eqn) # Create a file where the furrow area is set to null mask1_cub = to_del.add(in_cub.with_suffix(f".{temp_token}.mask1.cub")) isis.mask( in_cub, mask=fx_cub, to=mask1_cub, min_=1, max_=1, preserve="INSIDE", spixels="NULL", ) # Lowpass filter to fill in the null pixel area lpf_cub = to_del.add(in_cub.with_suffix(f".{temp_token}.lpf.cub")) isis.lowpass( mask1_cub, to=lpf_cub, sample=lpf_samp, line=lpf_line, null=True, hrs=False, his=False, lis=False, ) # Create a file where non-furrow columns are set to null mask2_cub = to_del.add(in_cub.with_suffix(f".{temp_token}.mask2.cub")) isis.mask( in_cub, mask=fx_cub, to=mask2_cub, min_=0, max_=0, preserve="INSIDE", spixels="NULL", ) # Highpass filter the furrow region hpf_cub = to_del.add(in_cub.with_suffix(f".{temp_token}.hpf.cub")) isis.highpass(mask2_cub, to=hpf_cub, sample=1, line=lpf_line) # Add lowpass and highpass together to achieve desired result alg_cub = to_del.add(in_cub.with_suffix(f".{temp_token}.alg.cub")) isis.algebra(from_=lpf_cub, from2=hpf_cub, to=alg_cub, operator="ADD", A=1.0, B=1.0) # copy the input file to the output file then mosaic the # furrow area as needed. logger.info(f"Copy {in_cub} to {out_cube}.") shutil.copyfile(in_cub, out_cube) isis.handmos( alg_cub, mosaic=out_cube, outsample=1, outline=1, outband=1, insample=1, inline=1, inband=1, create="NO", ) if not keep: to_del.unlink() return
def HiStitch( cube0: os.PathLike, cube1: os.PathLike, db0: dict, db1: dict, out_cube: os.PathLike, conf: dict, keep=False, ) -> tuple: logger.info(f"HiStitch start: {cube0} {cube1}") # GetConfigurationParameters() conf_check(conf) # GetProductFiles() if not cube1: cubes = (Path(cube0), ) else: cubes = sort_input_cubes(Path(cube0), Path(cube1)) chids = get_chids(cubes) out_path = set_outpath(out_cube, hirise.CCDID(chids[0]), cubes[0].parent) # PrepareDBStatements() # select from HiCat.EDR_Products db_paths = list() db_paths.append(db0) if len(cubes) == 2: db_paths.append(db1) dbs = sort_databases([db0, db1], chids) ccd_number = int(chids[0].ccdnumber) # Allows for indexing in lists ordered by bin value. b = 1, 2, 4, 8, 16 # This string will get placed in the filename for all of our # temporary files. It will (hopefully) prevent collisions with # existing files and also allow for easy clean-up if keep=True temp_token = datetime.now().strftime("HiStitch-%y%m%d%H%M%S") # ProcessingStep() - mostly sets up stuff max_mean = max(map(lambda x: float(x["IMAGE_MEAN"]), dbs)) flags = set_flags(conf["HiStitch"], dbs, ccd_number, b.index(int(dbs[0]["BINNING"])), max_mean) # HiStitchStep() - runs HiStitch, and originally inserted to db, now we # just return at the bottom of this function. logger.info( HiStitchStep( cubes, out_path, dbs[0]["BINNING"], ccd_number, conf["HiStitch"], flags.balance, flags.equalize, ).args) if len(cubes) == 2 and flags.balance: # run getkey from ?HiStitch_output? truthchannel = isis.getkey_k(out_path, "HiStitch", "TruthChannel") balanceratio = isis.getkey_k(out_path, "HiStitch", "BalanceRatio") else: truthchannel = None balanceratio = None if flags.furrow: furrow_file = out_path.with_suffix(f".{temp_token}.temp.cub") HiFurrow_Fix(out_path, furrow_file, max_mean, keep=keep) furrow_file.rename(out_path) # insert ObsID, pid0.ccdname+pid0.ccdnumber, truthchannel, balanceratio # into HiCat.CCD_Processing_Statistics db = dict() db["OBSERVATION_ID"] = str(chids[0].get_obsid()) db["CCD"] = chids[0].get_ccd() db["CONTROL_CHANNEL"] = truthchannel db["CHANNEL_MATCHING_CORRECTION"] = balanceratio logger.info(f"HiStitch done: {out_path}") return db, out_path
def make_flats(cubes, common_cube, conf, temp_token, keep=False): # If the flat files already exist, don't remake them. # "$OBS_ID"."_".$ccd."-".$common.".flat.tab" jitter_cubes = list() n_row = int( common_cube.lines / conf["AutoRegistration"]["ControlNet"]["Control_Lines"] ) for c in cubes: if c == common_cube: continue jitter_cubes.append( hjr.JitterCube( c.next_path, matchccd=common_cube.get_ccd(), config=conf ) ) successful_flats = list() if not all(x.flattab_path.exists() for x in jitter_cubes): confauto = conf["AutoRegistration"] for c in jitter_cubes: params = { "ROWS": n_row, "TOLERANCE": confauto["Algorithm"]["Tolerance"], } min_fraction_good = confauto["AnaylyzeFlat"]["Minimum_Good"] if c.ccdname == "RED": redcolor = "Red" else: redcolor = "Color" params["GROUP"] = "ResolveJitter" params["COLS"] = confauto["ControlNet"]["Control_Cols_" + redcolor] params["PATTERN_SAMPLES"] = confauto["PatternChip" + redcolor][ "Samples" ] params["PATTERN_LINES"] = confauto["PatternChip" + redcolor][ "Lines" ] params["SEARCH_SAMPLES"] = confauto["SearchChip" + redcolor][ "Samples" ] params["SEARCH_LINES"] = confauto["SearchChip" + redcolor]["Lines"] if c.ccdname != "RED": channels = isis.getkey_k( c.path, "Instrument", "StitchedProductIds" ) if len(channels) < 2: logger.info( f"Increasing columns because {c.path} is " "missing a channel." ) params["COLS"] += 1 min_fraction_good *= 0.5 logger.info( "The minimum allowable Fraction Good " f"Matches = {min_fraction_good}" ) step = 0 while step <= confauto["Algorithm"]["Steps"]: logger.info(f"Step {step} begin") hjr.run_HiJitReg( common_cube.next_path, c, params, temp_token, keep=keep ) ret = hjr.Analyze_Flat( c, 0, (min_fraction_good * 2), hijitreg=False ) if ret == 1: successful_flats.append(c.flattab_path) break else: step += 1 c.regdef_path.unlink() c.flattab_path.unlink() c.cnet_path.unlink() params["TOLERANCE"] -= ( confauto["Algorithm"]["Increment"] * step ) else: raise RuntimeError( f"Flat file for {c} is not within tolerances." ) else: successful_flats = list(x.flattab_path for x in jitter_cubes) return successful_flats
def jitter_iter( red: hicolor.HiColorCube, color: hicolor.HiColorCube, conf: dict, keep=False, ) -> bool: """Iterates through hijitreg for the color cube.""" temp_token = datetime.now().strftime("HiJitReg-%y%m%d%H%M%S") bin_ratio = color.bin / red.bin jit_param = dict() conf_ar = conf["AutoRegistration"] jit_param["GROUP"] = "HiJitReg" jit_param["COLS"] = conf_ar["ControlNet"]["Control_Cols"] jit_param["ROWS"] = conf_ar["ControlNet"]["Control_Lines"] jit_param["TOLERANCE"] = conf_ar["Algorithm"]["Tolerance"] jit_param["PATTERN_SAMPLES"] = conf_ar["PatternChip"]["Samples"] jit_param["PATTERN_LINES"] = conf_ar["PatternChip"]["Lines"] jit_param["SEARCH_SAMPLES"] = conf_ar["SearchChip"]["Samples"] jit_param["SEARCH_LINES"] = conf_ar["SearchChip"]["Lines"] jit_param["SEARCHLONGER_SAMPLES"] = conf_ar["SearchLongerChip"]["Samples"] jit_param["SEARCHLONGER_LINES"] = conf_ar["SearchLongerChip"]["Lines"] if bin_ratio > 3: jit_param["TOLERANCE"] -= conf_ar["Algorithm"]["INCREMENT"] channels = isis.getkey_k(color.path, "Instrument", "StitchedProductIds") coverage = 1.0 if len(channels) < 2: coverage /= 2 jit_param["COLS"] += jit_param["COLS"] / 2 # A two-step process with completely different outcomes at each step, # so we can't really make a loop. step = 1 logger.info(f"Attempting hijitreg iteration #{step} for {color}") color_jitter = JitterCube(color, conf) run_HiJitReg(red.path, color_jitter, jit_param, temp_token, keep=keep) ret = Analyze_Flat(color_jitter, step, coverage) if ret == -1: # edgy or suspect points only if jit_param["SEARCH_LINES"] == jit_param["SEARCHLONGER_LINES"]: return True else: # use larger search box for all subsequent iterations # (other CCDs too) jit_param["SEARCH_SAMPLES"] = jit_param["SEARCHLONGER_SAMPLES"] jit_param["SEARCH_LINES"] = jit_param["SEARCHLONGER_LINES"] elif ret == 0: # not enough points found # increase grid density jit_param["ROWS"] = jit_param["ROWS"] * 2 if len(channels) >= 2: jit_param["COLS"] += 2 coverage /= 2 else: return True step += 1 logger.info(f"Attempting hijitreg iteration #{step} for {color}") # second pass run_HiJitReg(red.path, color_jitter, jit_param, temp_token, keep=keep) # analyze output again ret = Analyze_Flat(color_jitter, step, coverage) if ret == 0: logger.info(f"Jitter registration failed for {color}") return False elif ret < 0: logger.info("!!! Validation Required !!!") return True else: return True
def test_getkey_k(self): truth = 'HIRISE' gk = Mock(stdout=f'{truth}\n') with patch('kalasiris.k_funcs.isis.getkey', return_value=gk): key = isis.getkey_k('dummy.cub', 'Instrument', 'InstrumentId') self.assertEqual(truth, key)