def per_color(cube, temp_token, color_code, keep=False): tt = f"{temp_token}_{color_code}" numerator = "{}+{}".format(cube.path, cube.band[color_code]) denominator = "{}+{}".format(cube.path, cube.band["RED"]) # Generate the IR/RED and BG/RED ratios for each of the COLOR products rat_p = cube.path.with_suffix(f".{tt}.ratio.cub") isis.ratio(num=numerator, den=denominator, to=rat_p) # mask out invalid pixels # Possible future update: Make mosaic of ratio files then run cubnorm # correction on these for the unfiltered products, to avoid any null # pixels created during the ratio process. mask_p = cube.path.with_suffix(f".{tt}.mask.cub") isis.mask( rat_p, mask=rat_p, to=mask_p, minimum=0.0, maximum=4.0, preserve="INSIDE", ) # Generate the crop files crop_p = cube.path.with_suffix(f".{tt}.ratcrop.cub") isis.crop(mask_p, to=crop_p, line=cube.crop_sline, nlines=cube.crop_lines) if not keep: rat_p.unlink() return mask_p, crop_p
def CubeNormStep(cube, hconf: dict, keep=False) -> HiccdStitchCube: to_del = isis.PathSet() # crop removing the top and bottom portion of the image crop_p = to_del.add(cube.nextpath.with_suffix(".crop.cub")) isis.crop( cube.nextpath, to=crop_p, line=cube.sl_cubenorm, nlines=cube.nl_cubenorm, ) # run cubenorm to get statistics of the cropped area stats_p = to_del.add(cube.nextpath.with_suffix(".cubenorm.tab")) isis.cubenorm( crop_p, stats=stats_p, format_="TABLE", direction="COLUMN", normalizer="AVERAGE", MODE=hconf["HiccdStitch_Cubenorm_Method"], PRESERVE=True, ) stats_filtered_p = to_del.add(cube.nextpath.with_suffix(".cubenorm2.tab")) # Original Perl: write cubenorm_stdev to the DB (again?) but we'll just cube.cubenorm_stddev = AnalyzeStats(stats_p, stats_filtered_p) # Original Perl: if HiccdStitch: cubenorm_stdev is written to the DB here # Original Perl: if HiccdStitchC: write cubenorm_stdev to a PVL file? # run cubenorm again, this time make the correction to the file next_path = cube.nextpath.with_suffix(".cubenorm.cub") to_s = "{}+SignedWord+{}:{}".format( next_path, hconf["HiccdStitch_Normalization_Minimum"], hconf["HiccdStitch_Normalization_Maximum"], ) isis.cubenorm( cube.nextpath, to=to_s, fromstats=stats_filtered_p, statsource="TABLE", direction="COLUMN", normalizer="AVERAGE", MODE=hconf["HiccdStitch_Cubenorm_Method"], PRESERVE=True, ) cube.nextpath = next_path if not keep: to_del.unlink() return cube
def crop_and_scale(cubes: list) -> list: to_del = isis.PathSet() for i, c in enumerate(cubes): # First step in balancing process is to crop out the left and # right overlap areas of each CCD lc_path = to_del.add(c.nextpath.with_suffix(".left.crop.cub")) rc_path = to_del.add(c.nextpath.with_suffix(".right.crop.cub")) for t, s, n in zip( [lc_path, rc_path], [c.ss_balance_left, c.ss_balance_right], [c.ns_balance_left, c.ns_balance_right], ): isis.crop( c.nextpath, to=t, sample=s, nsamples=n, line=c.sl_balance, nlines=c.nl_balance, ) # Second step is to scale all of the croped files to have the # same lines and samples, needed for mask step if c.smag == 1 and c.lmag == 1: cubes[i].set_ls_path(lc_path) cubes[i].set_rs_path(rc_path) else: cubes[i].set_ls_path( to_del.add(c.nextpath.with_suffix(".left.scale.cub"))) cubes[i].set_rs_path( to_del.add(c.nextpath.with_suffix(".right.scale.cub"))) for lc, ls in zip([lc_path, rc_path], [cubes[i].ls_path, cubes[i].rs_path]): isis.enlarge(lc, to=ls, sscale=c.smag, lscale=c.lmag, interp="CUBIC") return (cubes, to_del)
def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "-o", "--output", required=False, default=".mdr.iof.cub" ) parser.add_argument("-e", "--edr", required=True) parser.add_argument( "-c", "--conf", required=False, type=argparse.FileType('r'), default=pkg_resources.resource_stream( __name__, 'data/hical.pipelines.conf' ), ) parser.add_argument("mdr", metavar="MDR_file") parser.add_argument( "-l", "--log", required=False, default="WARNING", help="The log level to show for this program, can " "be a named log level or a numerical level.", ) parser.add_argument( "-k", "--keep", required=False, default=False, action="store_true", help="Normally, the program will clean up any " "intermediary files, but if this option is given, it " "won't.", ) args = parser.parse_args() util.set_logger(args.verbose, args.logfile, args.log) edr_path = Path(args.edr) mdr_path = Path(args.mdr) to_del = isis.PathSet() h2i_path = to_del.add(edr_path.with_suffix(".hi2isis.cub")) out_path = util.path_w_suffix(args.output, edr_path) # The first thing Alan's program did was to crop the image down to only the # 'imaging' parts. We're not doing that so the resultant file has a # geometry similar to what comes out of ISIS hical. # Convert the EDR to a cube file isis.hi2isis(edr_path, to=h2i_path) # Convert Alan's MDR to a cube file mdr_cub_path = to_del.add(mdr_path.with_suffix(".alan.cub")) logger.info(f"Running gdal_translate {mdr_path} -of ISIS3 {mdr_cub_path}") gdal.Translate(str(mdr_cub_path), str(mdr_path), format="ISIS3") h2i_s = int(isis.getkey_k(h2i_path, "Dimensions", "Samples")) h2i_l = int(isis.getkey_k(h2i_path, "Dimensions", "Lines")) mdr_s = int(isis.getkey_k(mdr_cub_path, "Dimensions", "Samples")) mdr_l = int(isis.getkey_k(mdr_cub_path, "Dimensions", "Lines")) if h2i_s != mdr_s: label = pvl.load(str(h2i_path)) hirise_cal_info = get_one( label, "Table", "HiRISE Calibration Ancillary" ) buffer_pixels = get_one(hirise_cal_info, "Field", "BufferPixels")[ "Size" ] dark_pixels = get_one(hirise_cal_info, "Field", "DarkPixels")["Size"] rev_mask_tdi_lines = hirise_cal_info["Records"] if h2i_s + buffer_pixels + dark_pixels == mdr_s: logger.info( f"The file {mdr_cub_path} has " f"{buffer_pixels + dark_pixels} more sample pixels " f"than {h2i_path}, assuming those are dark and " "buffer pixels and will crop accordingly." ) if h2i_l + rev_mask_tdi_lines != mdr_l: logger.critical( 'Even assuming this is a "full" channel ' "image, this has the wrong number of lines. " f"{mdr_cub_path} should have " f"{h2i_l + rev_mask_tdi_lines}, but " f"has {mdr_l} lines. Exiting" ) sys.exit() else: crop_path = to_del.add(mdr_cub_path.with_suffix(".crop.cub")) # We want to start with the next pixel (+1) after the cal # pixels. isis.crop( mdr_cub_path, to=crop_path, sample=buffer_pixels + 1, nsamples=h2i_s, line=rev_mask_tdi_lines + 1, ) mdr_cub_path = crop_path mdr_l = int(isis.getkey_k(mdr_cub_path, "Dimensions", "Lines")) else: logger.critical( f"The number of samples in {h2i_path} ({h2i_s}) " f"and {mdr_cub_path} ({mdr_s}) are different. " "Exiting." ) sys.exit() if h2i_l != mdr_l: logger.critical( f"The number of lines in {h2i_path} ({h2i_l}) " f"and {mdr_cub_path} ({mdr_l}) are different. " "Exiting." ) sys.exit() # Convert the EDR to the right bit type for post-HiCal Pipeline: h2i_16b_p = to_del.add(h2i_path.with_suffix(".16bit.cub")) isis.bit2bit( h2i_path, to=h2i_16b_p, bit="16bit", clip="minmax", minval=0, maxval=1.5, ) shutil.copyfile(h2i_16b_p, out_path) # If it is a channel 1 file, Alan mirrored it so that he could process # the two channels in an identical way (which we also took advantage # of above if the buffer and dark pixels were included), so we need to # mirror it back. cid = hirise.get_ChannelID_fromfile(h2i_16b_p) if cid.channel == "1": mirror_path = to_del.add(mdr_cub_path.with_suffix(".mirror.cub")) isis.mirror(mdr_cub_path, to=mirror_path) mdr_cub_path = mirror_path # Is the MDR in DN or I/F? maximum_pxl = float( pvl.loads(isis.stats(mdr_cub_path).stdout)["Results"]["Maximum"] ) if maximum_pxl < 1.5: logger.info("MDR is already in I/F units.") mdr_16b_p = to_del.add(mdr_cub_path.with_suffix(".16bit.cub")) isis.bit2bit( mdr_cub_path, to=mdr_16b_p, bit="16bit", clip="minmax", minval=0, maxval=1.5, ) isis.handmos(mdr_16b_p, mosaic=out_path) else: logger.info("MDR is in DN units and will be converted to I/F.") fpa_t = statistics.mean( [ float( isis.getkey_k( h2i_16b_p, "Instrument", "FpaPositiveYTemperature" ) ), float( isis.getkey_k( h2i_16b_p, "Instrument", "FpaNegativeYTemperature" ) ), ] ) print(f"fpa_t {fpa_t}") conf = pvl.load(args.conf) tdg = t_dep_gain(get_one(conf["Hical"], "Profile", cid.ccdname), fpa_t) suncorr = solar_correction() sclk = isis.getkey_k( h2i_16b_p, "Instrument", "SpacecraftClockStartCount" ) target = isis.getkey_k(h2i_16b_p, "Instrument", "TargetName") suncorr = solar_correction(sunDistanceAU(sclk, target)) sed = float( isis.getkey_k(h2i_16b_p, "Instrument", "LineExposureDuration") ) zbin = get_one(conf["Hical"], "Profile", "GainUnitConversion")[ "GainUnitConversionBinFactor" ] # The 'ziof' name is from the ISIS HiCal/GainUnitConversion.h, it is a # divisor in the calibration equation. print(f"zbin {zbin}") print(f"tdg {tdg}") print(f"sed {sed}") print(f"suncorr {suncorr}") ziof = zbin * tdg * sed * 1e-6 * suncorr eqn = f"\(F1 / {ziof})" # noqa W605 mdriof_p = to_del.add(mdr_cub_path.with_suffix(".iof.cub")) to_s = "{}+SignedWord+{}:{}".format(mdriof_p, 0, 1.5) isis.fx(f1=mdr_cub_path, to=to_s, equ=eqn) isis.handmos(mdriof_p, mosaic=out_path) if not args.keep: to_del.unlink()
def per_band(cubes, out_p, temp_token, color_code, furrow_flag, unfiltered, keep=False) -> float: to_del = isis.PathSet() # Generate the handmos of both the ratio and the croped ratio files if len(cubes) == 2: maxlines = max(c.lines for c in cubes) maxsamps = sum(c.samps for c in cubes) # Generate the mosaic of the ratio image mos_p = to_del.add( out_p.with_suffix(f".{temp_token}.{color_code}.mos.cub")) make_LR_mosaic( cubes[0].mask_path[color_code], cubes[1].mask_path[color_code], cubes[0].samps, mos_p, maxlines, maxsamps, ) # Generate the mosaic of the crop ratio image maxlines = max(c.crop_lines for c in cubes) crpmos_p = to_del.add( out_p.with_suffix(f".{temp_token}.{color_code}.moscrop.cub")) make_LR_mosaic( cubes[0].crop_path[color_code], cubes[1].crop_path[color_code], cubes[0].samps, crpmos_p, maxlines, maxsamps, ) else: mos_p = cubes[0].mask_path[color_code] crpmos_p = cubes[0].crop_path[color_code] mosnrm_p = to_del.add( out_p.with_suffix(f".{temp_token}.{color_code}.mosnorm.cub")) ratio_stddev = cubenorm_stats(crpmos_p, mos_p, mosnrm_p, keep=keep) # from the handmos cubes, pull out the individual CCDs with crop if len(cubes) == 2: samp = 1 for i, c in enumerate(cubes): cubes[i].nrm_path[color_code] = c.path.with_suffix( f".{temp_token}.{color_code}.norm.cub") isis.crop( mosnrm_p, to=cubes[i].nrm_path[color_code], sample=samp, nsamples=c.samps, ) samp += c.samps else: # If there was only one file then all we need to do is rename files cubes[0].nrm_path[color_code] = mosnrm_p if unfiltered: # Create the unfiltered color cubes, if needed for c in cubes: make_unfiltered( c.path, c.nrm_path[color_code], temp_token, color_code, c.band[color_code], keep=keep, ) # low pass filter the files for c in cubes: # If $lpfz=1 then we're only interpolating null pixels caused by # noise or furrow removal lowpass_args = dict( minopt="PERCENT", minimum=25, low=0.00, high=2.0, null=True, hrs=True, lis=True, lrs=True, ) # As the comment below states, I *think* that the OP had a copy # and paste error here, because even though it kept track of both # the IR and BG boxcar dimensions, they were always identical. logger.warning("The original Perl calculates the boxcar size for " "both the IR and BG based only on the ratio of the " "IR to the RED.") if c.get_boxcar_size(c.ir_bin) == 1: lowpass_args["lines"] = 3 lowpass_args["samples"] = 3 lowpass_args["filter"] = "OUTSIDE" else: lowpass_args["lines"] = 1 lowpass_args["samples"] = 1 lpf_path = to_del.add( c.path.with_suffix(f".{temp_token}.{color_code}.lpf.cub")) isis.lowpass(c.nrm_path[color_code], to=lpf_path, **lowpass_args) # Perform lpfz filters to interpolate null pixels due to furrows or # bad pixels if furrow_flag: lpfz_path = c.path.with_suffix( f".{temp_token}.{color_code}.lpfz.cub") lpfz_triplefilter(lpf_path, lpfz_path, temp_token, keep=keep) else: lpfz_path = lpf_path # run ISIS algebra program to created (normalized-IR/RED)*RED files alg_path = to_del.add( c.path.with_suffix(f".{temp_token}.{color_code}.algebra.cub")) isis.algebra(lpfz_path, from2=f"{c.path}+2", to=alg_path, operator="MULTIPLY") # Update the output file with the normalized IR and BG bands isis.handmos( alg_path, mosaic=c.final_path, outsample=1, outline=1, outband=c.band[color_code], matchbandbin=False, ) if not keep: to_del.unlink() return ratio_stddev