def add_offsets(cubes: list, base_ccdnumber: int, temp_token: str, keep=False) -> tuple: flats = isis.PathSet() for i, c in enumerate(cubes[:-1]): pair = "{}-{}".format(cubes[i + 1].get_ccd(), cubes[i].get_ccd()) flat_p = flats.add( c.path.with_suffix(f".{temp_token}.{pair}.flat.tab")) (avg_line_offset, avg_samp_offset) = get_offsets(cubes[i].next_path, cubes[i + 1].next_path, flat_p) j = i if int(c.ccdnumber) >= base_ccdnumber: j = i + 1 cubes[j].line_offset = avg_line_offset cubes[j].samp_offset = avg_samp_offset if not keep: flats.unlink() return (cubes, None) else: return (cubes, flats)
def test_Hidestripe(self): to_del = isis.PathSet() calcube = to_del.add(Path("test_Hidestripe-out.hical.cub")) isis.hical(self.cube, to=calcube) to_del.add(Path(str(self.pid)).with_suffix(".hical.log")) outcube = to_del.add(Path("test_Hidestripe-out.cub")) samps = int(isis.getkey_k(calcube, "Dimensions", "Samples")) self.assertRaises( KeyError, hc.Hidestripe, self.cube, outcube, self.binning, minimum=0.0, maximum=1.5, hidcorr="ADD", line_samples=samps, keep=False, ) self.assertAlmostEqual( 0.000101402295171637, hc.Hidestripe( calcube, outcube, self.binning, minimum=0.0, maximum=1.5, hidcorr="ADD", line_samples=samps, keep=False, ), ) to_del.unlink()
def test_unlink(self): ps = isis.PathSet(self.paths) for p in ps: p.touch() ps.unlink() for p in ps: self.assertFalse(p.exists())
def CubeNormStep(cube, hconf: dict, keep=False) -> HiccdStitchCube: to_del = isis.PathSet() # crop removing the top and bottom portion of the image crop_p = to_del.add(cube.nextpath.with_suffix(".crop.cub")) isis.crop( cube.nextpath, to=crop_p, line=cube.sl_cubenorm, nlines=cube.nl_cubenorm, ) # run cubenorm to get statistics of the cropped area stats_p = to_del.add(cube.nextpath.with_suffix(".cubenorm.tab")) isis.cubenorm( crop_p, stats=stats_p, format_="TABLE", direction="COLUMN", normalizer="AVERAGE", MODE=hconf["HiccdStitch_Cubenorm_Method"], PRESERVE=True, ) stats_filtered_p = to_del.add(cube.nextpath.with_suffix(".cubenorm2.tab")) # Original Perl: write cubenorm_stdev to the DB (again?) but we'll just cube.cubenorm_stddev = AnalyzeStats(stats_p, stats_filtered_p) # Original Perl: if HiccdStitch: cubenorm_stdev is written to the DB here # Original Perl: if HiccdStitchC: write cubenorm_stdev to a PVL file? # run cubenorm again, this time make the correction to the file next_path = cube.nextpath.with_suffix(".cubenorm.cub") to_s = "{}+SignedWord+{}:{}".format( next_path, hconf["HiccdStitch_Normalization_Minimum"], hconf["HiccdStitch_Normalization_Maximum"], ) isis.cubenorm( cube.nextpath, to=to_s, fromstats=stats_filtered_p, statsource="TABLE", direction="COLUMN", normalizer="AVERAGE", MODE=hconf["HiccdStitch_Cubenorm_Method"], PRESERVE=True, ) cube.nextpath = next_path if not keep: to_del.unlink() return cube
def lpfz_triplefilter(from_path: os.PathLike, to_path: os.PathLike, keep=False) -> None: to_del = isis.PathSet() from_p = Path(from_path) z1 = to_del.add(from_p.with_suffix(".z1.cub")) lpfz_filtering(from_p, z1, 11, 5) z2 = to_del.add(from_p.with_suffix(".z2.cub")) lpfz_filtering(z1, z2, 21, 9) lpfz_filtering(z2, to_path, 41, 11) if not keep: to_del.unlink() return
def crop_and_scale(cubes: list) -> list: to_del = isis.PathSet() for i, c in enumerate(cubes): # First step in balancing process is to crop out the left and # right overlap areas of each CCD lc_path = to_del.add(c.nextpath.with_suffix(".left.crop.cub")) rc_path = to_del.add(c.nextpath.with_suffix(".right.crop.cub")) for t, s, n in zip( [lc_path, rc_path], [c.ss_balance_left, c.ss_balance_right], [c.ns_balance_left, c.ns_balance_right], ): isis.crop( c.nextpath, to=t, sample=s, nsamples=n, line=c.sl_balance, nlines=c.nl_balance, ) # Second step is to scale all of the croped files to have the # same lines and samples, needed for mask step if c.smag == 1 and c.lmag == 1: cubes[i].set_ls_path(lc_path) cubes[i].set_rs_path(rc_path) else: cubes[i].set_ls_path( to_del.add(c.nextpath.with_suffix(".left.scale.cub"))) cubes[i].set_rs_path( to_del.add(c.nextpath.with_suffix(".right.scale.cub"))) for lc, ls in zip([lc_path, rc_path], [cubes[i].ls_path, cubes[i].rs_path]): isis.enlarge(lc, to=ls, sscale=c.smag, lscale=c.lmag, interp="CUBIC") return (cubes, to_del)
def HiColorNorm(cubes: list, output, conf: dict, make_unfiltered=True, db_list=None, keep=False): logger.info(f"HiColorNorm start: {cubes}") # GetConfigurationParameters() conf_check(conf) cubes = list(map(ColorCube, cubes, repeat(db_list))) cubes.sort() outcub_path = set_outpath(output, cubes) temp_token = datetime.now().strftime("HiColorNorm-%y%m%d%H%M%S") out_p = Path(outcub_path) furrow_flag = FurrowCheck(cubes) to_del = isis.PathSet() for i, _ in enumerate(cubes): cubes[i].set_crop_lines(conf) for i, c in enumerate(cubes): # Protect the processing from erroneous/spurious pixels mask_list = list() for b in (1, 2, 3): tmp_p = to_del.add( c.path.with_suffix(f".{temp_token}.temp{b}.cub")) isis.mask( f"{c.path}+{b}", mask=f"{c.path}+{b}", to=tmp_p, minimum=0.0, maximum=2.0, preserve="INSIDE", ) mask_list.append(tmp_p) c.final_path = c.path.with_suffix(f".HiColorNorm.cub") isis.cubeit_k(mask_list, to=c.final_path) (cubes[i].mask_path["IR"], cubes[i].crop_path["IR"]) = per_color(c, temp_token, "IR", keep=keep) (cubes[i].mask_path["BG"], cubes[i].crop_path["BG"]) = per_color(c, temp_token, "BG", keep=keep) ir_ratio_stddev = per_band(cubes, out_p, temp_token, "IR", furrow_flag, make_unfiltered, keep=keep) bg_ratio_stddev = per_band(cubes, out_p, temp_token, "BG", furrow_flag, make_unfiltered, keep=keep) if conf["HiColorNorm"]["HiColorNorm_Make_Stitch"]: # listpath = to_del.add(c.path.with_suffix(f'.{temp_token}.list.txt')) # listpath.write_text( # '\n'.join(str(c.final_path) for c in cubes) + '\n') with isis.fromlist.temp([str(c.final_path) for c in cubes]) as f: isis.hiccdstitch(fromlist=f, to=out_p) for c in cubes: to_del.add(c.final_path) if not keep: to_del.unlink() for c in cubes: for cc in ("IR", "BG"): c.mask_path[cc].unlink() c.crop_path[cc].unlink() c.nrm_path[cc].unlink() logger.info(f"HiColorNorm done.") return ir_ratio_stddev, bg_ratio_stddev
def HiJACK( cube_paths: list, conf_dir: os.PathLike, outdir=Path("./HiJACK"), base_ccd_number=5, plot=True, keep=False, ): cubes = list(map(hnp.Cube, cube_paths)) cubes.sort() logger.info(f"HiJACK start: {', '.join(map(str, cubes))}") base_cube = check(cubes, base_ccd_number) temp_token = datetime.now().strftime("HiJACK-%y%m%d%H%M%S") outdir_p = Path(outdir) outdir_p.mkdir(exist_ok=True) to_del = isis.PathSet() # Pre-HiJACK for c in cubes: c.next_path = to_del.add( (outdir_p / c.path.name).with_suffix( f".{temp_token}.prehijack.cub" ) ) flat_path = outdir_p / "{}.{}.{}".format( cubes[0].get_obsid(), temp_token, "RED5-RED4_prehijack.flat.tab" ) match_red(cubes, base_cube, flat_path) # jitter_path = outdir_p / (str(cubes[0].get_obsid()) + "_jitter_cpp.txt") jitter_path = outdir_p / (str(cubes[0].get_obsid()) + "_jitter_py.txt") if not jitter_path.exists(): resolvejitter_conf = Path(conf_dir) / "ResolveJitter.conf" ResolveJitter( cubes, pvl.load(str(resolvejitter_conf)), jitter_path, temp_token, keep=keep, ) # SmearStats just updates the db, not needed. # HiJACK proper: hijack_conf = pvl.load(str(Path(conf_dir) / "HiJACK.conf")) hnp.conf_check(hijack_conf["HiJACK"]) polar = False if hijack_conf["HiJACK"]["Shape"] == "USER": polar = hnp.is_polar(cubes, hijack_conf["Pole_Tolerance"], temp_token) for c in cubes: hnp.copy_and_spice( c.next_path, to_del.add(c.path.with_suffix(f".{temp_token}.spiced.cub")), hijack_conf["HiJACK"], polar, ) # Run hijitter inlist = list() outlist = list() for c in cubes: c.next_path = ( (outdir_p / c.path.name) .with_suffix("") .with_suffix(".dejittered.cub") ) inlist.append(c.path) outlist.append(c.next_path) jitterck_p = outdir_p / (str(cubes[0].get_obsid()) + ".jittery.bc") hijitregdef_p = ( Path(os.environ["ISIS3DATA"]) / "mro/calibration/hijitreg.p1745.s3070.def" ) # inlist_p = to_del.add(isis.fromlist.make(inlist, # (outdir_p / # (str(cubes[0].get_obsid()) + # '_hijitter.inlst')))) # outlist_p = to_del.add(isis.fromlist.make(outlist, # inlist_p.with_suffix('.outlst'))) with isis.fromlist.temp(inlist) as inlist_f: with isis.fromlist.temp(outlist) as outlist_f: isis.hijitter( fromlist=inlist_f, jitter=jitter_path, regdef=hijitregdef_p, tolist=outlist_f, jitterck=jitterck_p, ) # Mosaic dejittered cubs # Remember hijitter makes all the individual cubes the size of the # entire image. With the image data in the appropriate space for the ccd. out_root = outdir_p / str(cubes[0].get_obsid()) # All REDs red_p = out_root.with_name(out_root.name + "_RED.NOPROJ.cub") red_5 = list( filter(lambda x: x.ccdname == "RED" and x.ccdnumber == "5", cubes) )[0] # shutil.copyfile(base_cube.next_path, red_p) shutil.copyfile(red_5.next_path, red_p) logger.info( "Original Perl hard codes this file copy from RED5, even if " "another cube is selected as the base_ccd." ) (red_cubes, red_flat_files) = hnp.add_offsets( list(filter(lambda x: x.ccdname == "RED", cubes)), 5, temp_token, keep=True, ) hnp.handmos_side(red_cubes, red_5, red_p, left=True) hnp.handmos_side(red_cubes, red_5, red_p, left=False) hnp.fix_labels( red_cubes, red_p, red_5, "{}_RED".format(str(red_cubes[0].get_obsid())) ) # Center RED for color center_red_p = out_root.with_name(out_root.name + "_RED4-5.NOPROJ.cub") center_red_cubes = list( filter(lambda x: x.ccdnumber == "4" or x.ccdnumber == "5", red_cubes) ) shutil.copyfile(center_red_cubes[1].next_path, center_red_p) mosaic_dejittered( center_red_cubes, center_red_p, "{}_RED4-5".format(str(center_red_cubes[0].get_obsid())), ) ir_p = out_root.with_name(out_root.name + "_IR.NOPROJ.cub") (ir_cubes, _) = hnp.add_offsets( list(filter(lambda x: x.ccdname == "IR", cubes)), 5, temp_token, keep=keep, ) shutil.copyfile(ir_cubes[1].next_path, ir_p) mosaic_dejittered( ir_cubes, ir_p, "{}_IR".format(str(ir_cubes[0].get_obsid())) ) # BG next bg_p = out_root.with_name(out_root.name + "_BG.NOPROJ.cub") (bg_cubes, _) = hnp.add_offsets( list(filter(lambda x: x.ccdname == "BG", cubes)), 5, temp_token, keep=keep, ) shutil.copyfile(bg_cubes[1].next_path, bg_p) mosaic_dejittered( bg_cubes, bg_p, "{}_BG".format(str(bg_cubes[0].get_obsid())) ) # Create color product irb_p = out_root.with_name(out_root.name + "_IRB.NOPROJ.cub") with isis.fromlist.temp([ir_p, center_red_p, bg_p]) as f: isis.cubeit(fromlist=f, to=irb_p, proplab=center_red_p) if plot: # Make plot of before and after flat.tab results dejit_flat = list( filter(lambda x: x.match("*RED5-RED4*"), red_flat_files) )[0] plot_flats(flat_path, dejit_flat) if not keep: flat_path.unlink() red_flat_files.unlink() to_del.unlink() logger.info(f"HiJACK done") return
def HiNoProj(cubes: list, conf: dict, output="_RED.NOPROJ.cub", base=5, keep=False): logger.info(f"HiNoProj start: {', '.join(map(str, cubes))}") cubes = list(map(Cube, cubes)) cubes.sort() if not all(c.ccdname == "RED" for c in cubes): raise ValueError("Not all of the input files are RED CCD files.") sequences = list() for k, g in itertools.groupby( (int(c.ccdnumber) for c in cubes), lambda x, c=itertools.count(): next(c) - x, ): sequences.append(list(g)) if len(sequences) != 1: raise ValueError("The given cubes are not a single run of sequential " "HiRISE CCDs, instead there are " f"{len(sequences)} groups with these " f"CCD numbers: {sequences}.") if not isinstance(base, int): base = hirise.get_CCDID_fromfile(base) base_ccd = list(filter(lambda x: x.ccdnumber == str(base), cubes)) if len(base_ccd) != 1: raise ValueError(f"The base ccd, number {base}, " "is not one of the given cubes.") base_cube = base_ccd[0] conf_check(conf["HiNoProj"]) conf = conf["HiNoProj"] out_p = hcn.set_outpath(output, cubes) temp_token = datetime.now().strftime("HiNoProj-%y%m%d%H%M%S") to_del = isis.PathSet() polar = False if conf["Shape"] == "USER": polar = is_polar(cubes, conf["Pole_Tolerance"], temp_token) for c in cubes: temp_p = to_del.add(c.path.with_suffix(f".{temp_token}.spiced.cub")) copy_and_spice(c.path, temp_p, conf, polar) isis.spicefit(temp_p) c.next_path = to_del.add( c.path.with_suffix(f".{temp_token}.noproj.cub")) c.path = temp_p for c in cubes: isis.noproj(c.path, match=base_cube.path, to=c.next_path, source="frommatch") # Run hijitreg on adjacent noproj'ed ccds to get average line/sample offset (cubes, _) = add_offsets(cubes, int(base_cube.ccdnumber), temp_token, keep=keep) # Mosaic noproj'ed ccds using average line/sample offset shutil.copyfile(base_cube.next_path, out_p) logger.info("Original Perl hard codes this file copy from RED5, even if " "another cube is selected as the base_ccd.") handmos_side(cubes, base_cube, out_p, left=True) handmos_side(cubes, base_cube, out_p, left=False) isis.editlab( out_p, option="addkey", grpname="Instrument", keyword="ImageJitterCorrected", value=0, ) fix_labels( cubes, out_p, base_cube, "{}_{}".format(str(cubes[0].get_obsid()), cubes[0].ccdname), ) if not keep: to_del.unlink() logger.info(f"HiNoProj done: {out_p}") return
def test_init(self): ps = isis.PathSet() self.assertIsInstance(ps, isis.PathSet)
def BalanceStep(cubes, conf, keep=False) -> list: to_del = isis.PathSet() # Sort the cubes so that they are in CCD order cubes.sort() cubes, to_delete = crop_and_scale(cubes) to_del.update(to_delete) # Original Perl: Generate CCD number array for each CCD file, not needed # now, since we can just query the object. # Original Perl: Used $0 (the program name) instead of the index 0 # here, but I've arranged things with the set_ls_path() and set_rs_path() # to just make these a full copy so there's no # need to mess with a conditional assignment and also streamlines # the following logic, since they're identical except when you # explicitly change them. # The third step is to mask the left and right overlap areas. We # want to zap pixels where there is not common coverage. for i, c in enumerate(cubes): if i + 1 < len(cubes) and int(cubes[i].ccdnumber) + 1 == int( cubes[i + 1].ccdnumber): cubes[i].rm_path = to_del.add( c.nextpath.with_suffix(".right.mask.cub")) cubes[i + 1].lm_path = to_del.add( cubes[i + 1].nextpath.with_suffix(".left.mask.cub")) for f, m, t in zip( [cubes[i].rs_path, cubes[i + 1].ls_path], [cubes[i + 1].ls_path, cubes[i].rs_path], [cubes[i].rm_path, cubes[i + 1].lm_path], ): isis.mask( f, mask=m, to=t, preserve="INSIDE", min_=conf["HiccdStitch_Normalization_Minimum"], max_=conf["HiccdStitch_Normalization_Maximum"], ) # The fourth step is to get image statistics for left and right # overlap areas of each CCD image. cubes = get_stats(cubes) # Look for a break in joining CCDs, defined by a break in the CCD number, # or the right or left statistics are undefined, due to an all null channel # # In the original Perl there was a loop to determine if there was a break, # but then nothing was done with that information? Oh, it was used # differently: the code past that point develops a series of sequences # from $first to $last. If there are no breaks, then it only runs a # single sequence. If there are breaks, it runs the sequences it finds. # # Here's the pythonic version: cubes.sort() for (offset, group) in get_group_i(cubes): logger.info("Correction before redistribution.") for ccd in group: i = ccd + offset cubes[i].correction = get_correction( cubes[i], cubes[i - 1], conf["HiccdStitch_Balance_Correction"], i, ) logger.info( f"CCDID: {cubes[i]}, correction: {cubes[i].correction}") normalization = get_normalization(cubes, group, offset, conf["HiccdStitch_Control_CCD"]) logger.info("Correction after redistribution.") for ccd in group: i = ccd + offset cubes[i].correction /= normalization logger.info( f"CCDID: {cubes[i]}, correction: {cubes[i].correction}, " f"left: {cubes[i].lstats}, right: {cubes[i].rstats}") # In the original Perl, they wrote out to the DB here, but we'll # do it later. There was also a distinction that if it was # HiccdStitchC that the data was written out to a PVL file. Not # sure why. # Create the balance cubes for ccd in group: i = ccd + offset balance_path = cubes[i].nextpath.with_suffix(".balance.cub") make_balance(cubes[i], conf, balance_path) cubes[i].nextpath = balance_path if not keep: to_del.unlink() return cubes
def HiccdStitch(cubes: list, out_path: os.PathLike, conf: dict, sline=None, eline=None, keep=False) -> tuple: logger.info(f"HiccdStitch start: {', '.join(map(str, cubes))}") # Perl: GetConfigurationParameters() # conf = pvl.load(str(conf_path)) conf_check(conf) out_p = set_outpath(out_path, cubes) # Perl: GetImageDims() cubes = GetImageDims(cubes, conf, sline, eline) # This string will get placed in the filename for all of our # temporary files. It will (hopefully) prevent collisions with # existing files and also allow for easy clean-up if keep=True # temp_token = datetime.now().strftime('HiccdStitch-%y%m%d%H%M%S') to_delete = isis.PathSet() # Perl: MakeList() # This makes a file, but doesn't really do anything with it, because # a different file is actually made later to give to hiccdstitch for i, c in enumerate(cubes): if c.cubenormstep: cubes[i] = CubeNormStep(c, conf["HiccdStitch"], keep=keep) if conf["HiccdStitch"]["HiccdStitch_Balance"]: cubes = BalanceStep(cubes, conf["HiccdStitch"], keep=keep) for c in cubes: SpecialProcessingFlags(c) cubes.sort() logger.info("The Original Perl looked for a custom file for " "hiccdstitch's shiftdef parameter, but the default ISIS " "file seems better, so this isn't implemented.") with isis.fromlist.temp([str(c.nextpath) for c in cubes]) as f: isis.hiccdstitch( fromlist=f, to=out_p, interp=conf["HiccdStitch"]["HiccdStitch_Interpolation"], ) SNR_Check(cubes, conf["HiccdStitch"]["HiccdStitch_SNR_Threshold"]) logger.info("HiccdStitch done.") if not keep: to_delete.unlink() # Afterwards inserts into CCD_Processing_Statistics table. db = {"OBSERVATION_ID": str(cubes[0].get_obsid())} for c in cubes: ccd_db = { "CCDID": str(c), "RADIOMETRIC_MATCHING_CORRECTION": c.correction, "LEFT_OVERLAP_AVERAGE": c.lstats, "RIGHT_OVERLAP_AVERAGE": c.rstats, "CUBENORM_COLUMN_CORRECTION_STANDARD_DEVIATION": c.cubenorm_stddev, } db[c.get_ccd()] = ccd_db logger.info(f"HiccdStitch done: {out_p}") return db, out_p
def test_add_Error(self): ps = isis.PathSet() p0 = self.paths[0] ps.add(p0) self.assertRaises(TypeError, ps.add, 'not a Path') self.assertRaises(ValueError, ps.add, p0)
def test_add(self): ps = isis.PathSet() path1 = self.paths[0] added_path = ps.add(path1) self.assertEqual(path1, added_path)
def test_init_iterable(self): ps = isis.PathSet(self.paths) self.assertEqual(3, len(ps))
def per_band(cubes, out_p, temp_token, color_code, furrow_flag, unfiltered, keep=False) -> float: to_del = isis.PathSet() # Generate the handmos of both the ratio and the croped ratio files if len(cubes) == 2: maxlines = max(c.lines for c in cubes) maxsamps = sum(c.samps for c in cubes) # Generate the mosaic of the ratio image mos_p = to_del.add( out_p.with_suffix(f".{temp_token}.{color_code}.mos.cub")) make_LR_mosaic( cubes[0].mask_path[color_code], cubes[1].mask_path[color_code], cubes[0].samps, mos_p, maxlines, maxsamps, ) # Generate the mosaic of the crop ratio image maxlines = max(c.crop_lines for c in cubes) crpmos_p = to_del.add( out_p.with_suffix(f".{temp_token}.{color_code}.moscrop.cub")) make_LR_mosaic( cubes[0].crop_path[color_code], cubes[1].crop_path[color_code], cubes[0].samps, crpmos_p, maxlines, maxsamps, ) else: mos_p = cubes[0].mask_path[color_code] crpmos_p = cubes[0].crop_path[color_code] mosnrm_p = to_del.add( out_p.with_suffix(f".{temp_token}.{color_code}.mosnorm.cub")) ratio_stddev = cubenorm_stats(crpmos_p, mos_p, mosnrm_p, keep=keep) # from the handmos cubes, pull out the individual CCDs with crop if len(cubes) == 2: samp = 1 for i, c in enumerate(cubes): cubes[i].nrm_path[color_code] = c.path.with_suffix( f".{temp_token}.{color_code}.norm.cub") isis.crop( mosnrm_p, to=cubes[i].nrm_path[color_code], sample=samp, nsamples=c.samps, ) samp += c.samps else: # If there was only one file then all we need to do is rename files cubes[0].nrm_path[color_code] = mosnrm_p if unfiltered: # Create the unfiltered color cubes, if needed for c in cubes: make_unfiltered( c.path, c.nrm_path[color_code], temp_token, color_code, c.band[color_code], keep=keep, ) # low pass filter the files for c in cubes: # If $lpfz=1 then we're only interpolating null pixels caused by # noise or furrow removal lowpass_args = dict( minopt="PERCENT", minimum=25, low=0.00, high=2.0, null=True, hrs=True, lis=True, lrs=True, ) # As the comment below states, I *think* that the OP had a copy # and paste error here, because even though it kept track of both # the IR and BG boxcar dimensions, they were always identical. logger.warning("The original Perl calculates the boxcar size for " "both the IR and BG based only on the ratio of the " "IR to the RED.") if c.get_boxcar_size(c.ir_bin) == 1: lowpass_args["lines"] = 3 lowpass_args["samples"] = 3 lowpass_args["filter"] = "OUTSIDE" else: lowpass_args["lines"] = 1 lowpass_args["samples"] = 1 lpf_path = to_del.add( c.path.with_suffix(f".{temp_token}.{color_code}.lpf.cub")) isis.lowpass(c.nrm_path[color_code], to=lpf_path, **lowpass_args) # Perform lpfz filters to interpolate null pixels due to furrows or # bad pixels if furrow_flag: lpfz_path = c.path.with_suffix( f".{temp_token}.{color_code}.lpfz.cub") lpfz_triplefilter(lpf_path, lpfz_path, temp_token, keep=keep) else: lpfz_path = lpf_path # run ISIS algebra program to created (normalized-IR/RED)*RED files alg_path = to_del.add( c.path.with_suffix(f".{temp_token}.{color_code}.algebra.cub")) isis.algebra(lpfz_path, from2=f"{c.path}+2", to=alg_path, operator="MULTIPLY") # Update the output file with the normalized IR and BG bands isis.handmos( alg_path, mosaic=c.final_path, outsample=1, outline=1, outband=c.band[color_code], matchbandbin=False, ) if not keep: to_del.unlink() return ratio_stddev
def HiFurrow_Fix(in_cube: os.PathLike, out_cube: os.PathLike, max_mean: float, keep=False): """Perform a normalization of the furrow region of bin 2 or 4 HiRISE images. The input to this script is a HiRISE stitch product containing both channels of a CCD. """ in_cub = Path(in_cube) binning = int(isis.getkey_k(in_cub, "Instrument", "Summing")) lines = int(isis.getkey_k(in_cub, "Dimensions", "Lines")) samps = int(isis.getkey_k(in_cub, "Dimensions", "Samples")) if binning != 2 and binning != 4: raise ValueError("HiFurrow_Fix only supports correction for " "bin 2 or 4 data.") if binning == 2 and samps != 1024: raise ValueError(f"HiFurrowFix: improper number of samples: {samps}, " "for a stitch product with bin 2 (should be 1024).") # This string will get placed in the filename for all of our # temporary files. It will (hopefully) prevent collisions with # existing files and also allow for easy clean-up if keep=True temp_token = datetime.now().strftime("HFF-%y%m%d%H%M%S") to_del = isis.PathSet() # For bin2 and bin4 imaging, specify width of furrow based on # image average DN range range_low = {2: (512, 513), 4: (256, 257)} # 2 pixel furrow width range_mid = {2: (511, 514), 4: (255, 258)} # 4 pixel furrow width range_hgh = {2: (511, 514), 4: (255, 258)} # 4 pixel furrow width range_max = {2: (510, 515), 4: (254, 259)} # 6 pixel furrow width # Original code had low/mid/hgh for bin2 and bin4, but they # were hard-coded to be identical. dn_range_low = 9000 dn_range_mid = 10000 dn_range_hgh = 12000 if max_mean > dn_range_hgh: dn_range = range_max[binning] elif max_mean > dn_range_mid: dn_range = range_hgh[binning] elif max_mean > dn_range_low: dn_range = range_mid[binning] else: dn_range = range_low[binning] lpf_samp = int((dn_range[1] - dn_range[0] + 1) / 2) * 4 + 1 lpf_line = int(lpf_samp / 2) * 20 + 1 # Create a mask file # DN=1 for non-furrow area # DN=0 for furrow area eqn = rf"\(1*(sample<{dn_range[0]})+ 1*(sample>{dn_range[1]}) + 0)" fx_cub = to_del.add(in_cub.with_suffix(f".{temp_token}.fx.cub")) isis.fx(to=fx_cub, mode="OUTPUTONLY", lines=lines, samples=samps, equation=eqn) # Create a file where the furrow area is set to null mask1_cub = to_del.add(in_cub.with_suffix(f".{temp_token}.mask1.cub")) isis.mask( in_cub, mask=fx_cub, to=mask1_cub, min_=1, max_=1, preserve="INSIDE", spixels="NULL", ) # Lowpass filter to fill in the null pixel area lpf_cub = to_del.add(in_cub.with_suffix(f".{temp_token}.lpf.cub")) isis.lowpass( mask1_cub, to=lpf_cub, sample=lpf_samp, line=lpf_line, null=True, hrs=False, his=False, lis=False, ) # Create a file where non-furrow columns are set to null mask2_cub = to_del.add(in_cub.with_suffix(f".{temp_token}.mask2.cub")) isis.mask( in_cub, mask=fx_cub, to=mask2_cub, min_=0, max_=0, preserve="INSIDE", spixels="NULL", ) # Highpass filter the furrow region hpf_cub = to_del.add(in_cub.with_suffix(f".{temp_token}.hpf.cub")) isis.highpass(mask2_cub, to=hpf_cub, sample=1, line=lpf_line) # Add lowpass and highpass together to achieve desired result alg_cub = to_del.add(in_cub.with_suffix(f".{temp_token}.alg.cub")) isis.algebra(from_=lpf_cub, from2=hpf_cub, to=alg_cub, operator="ADD", A=1.0, B=1.0) # copy the input file to the output file then mosaic the # furrow area as needed. logger.info(f"Copy {in_cub} to {out_cube}.") shutil.copyfile(in_cub, out_cube) isis.handmos( alg_cub, mosaic=out_cube, outsample=1, outline=1, outband=1, insample=1, inline=1, inband=1, create="NO", ) if not keep: to_del.unlink() return
def HiBeautify(cube_paths: list, conf: dict, out_irb="_IRB.cub", out_rgb="_RGB.cub", keep=False): logger.info(f"HiBeautify start: {', '.join(map(str, cube_paths))}") # GetConfigurationParameters() cubes = list(map(hcn.ColorCube, cube_paths)) cubes.sort() irb_out_p = hcn.set_outpath(out_irb, cubes) rgb_out_p = hcn.set_outpath(out_rgb, cubes) temp_token = datetime.now().strftime("HiBeautify-%y%m%d%H%M%S") to_del = isis.PathSet() # Create an IRB mosaic from the HiColorNorm halves. # If a half is missing, we create a mosaic with the proper width and place # the half in it at the proper location. # Logic ofr image_midpoint and total_width come from original HiColorInit # irbmerged_p = to_del.add(out_p.with_suffix(f'.{temp_token}_IRB.cub')) image_midpoint = int((2000 / cubes[0].red_bin) + 1) outsample = image_midpoint if cubes[0].ccdnumber == "4": outsample = 1 total_width = int(2 * cubes[0].samps - (48 / cubes[0].red_bin)) isis.handmos( cubes[0].path, mosaic=irb_out_p, outline=1, outsample=outsample, outband=1, create="Y", nlines=cubes[0].lines, nsamp=total_width, nbands=3, ) if len(cubes) == 1: logger.info("Warning, missing one half!") else: logger.info("Using both halves") isis.handmos( cubes[1].path, mosaic=irb_out_p, outline=1, outsample=image_midpoint, outband=1, ) # Nothing is actually done to the pixels here regarding the FrostStats, so # I'm just going to skip them here. # # # Determine if Frost/ICE may be present using FrostStats module. # frost = None # if args.frost: # frost = True # logging.info('Frost override: disabling auto-detection and using ' # 'the frost/ice color stretch') # if args.nofrost: # frost = False # logging.info('Frost override: disable auto-detection and not using ' # 'the frost/ice color stretch') # if frost is None: # pass # # get frost stats # Subtract the unaltered RED band from the high pass filtered BG for # synthetic blue. logger.info("Creating synthetic B, subtracting RED from BG") rgbsynthb_p = to_del.add(irb_out_p.with_suffix(f".{temp_token}_B.cub")) isis.algebra( f"{irb_out_p}+3", from2=f"{irb_out_p}+2", op="subtract", to=rgbsynthb_p, A=conf["Beautify"]["Synthetic_A_Coefficient"], B=conf["Beautify"]["Synthetic_B_Coefficient"], ) # Adjust the BandBin group isis.editlab(rgbsynthb_p, grpname="BandBin", keyword="Name", value="Synthetic Blue") isis.editlab(rgbsynthb_p, grpname="BandBin", keyword="Center", value="0") isis.editlab(rgbsynthb_p, grpname="BandBin", keyword="Width", value="0") # HiBeautify gathers and writes a bunch of statistics to PVL that is # important to the HiRISE GDS, but not relevant to just producing pixels # so I'm omitting it. # # # Determine the min and max DN values of each band (RED, IR, BG, B) we're # # working with. # (upper, lower) = conf['Beautify']['Stretch_Exclude_Lines'] # if upper == 0 and lower == 0: # synthbcrp_p = rgbsynthb_p # irbmrgcrp_p = irbmerged_p # else: # synthbcrp_p = to_del.add( # out_p.with_suffix(f'.{temp_token}_Bx.cub')) # irbmrgcrp_p = to_del.add( # out_p.with_suffix(f'.{temp_token}_IRBx.cub')) # # for (f, t) in ((rgbsynthb_p, synthbcrp_p), # (irbmerged_p, irbmrgcrp_p)): # logging.info(isis.crop(f, to=t, propspice=False, # line=(1 + upper), # nlines=( # cubes[0].lines - lower + upper)).args) # # stats = dict() # stats['B'] = Get_MinMax(synthbcrp_p, # conf['Beautify']['Stretch_Reduction_Factor'], # temp_token, keep=keep) # # for band in cubes[0].band.keys(): # stats[band] = Get_MinMax('{}+{}'.format(str(irbmrgcrp_p), # cubes[0].band[band]), # conf['Beautify']['Stretch_Reduction_Factor'], # temp_token, keep=keep) # Create an RGB cube using the RED from the IRB mosaic, # the BG from the IRB mosaic and the synthetic B that we just made. isis.cubeit_k([f"{irb_out_p}+2", f"{irb_out_p}+3", rgbsynthb_p], to=rgb_out_p) if not keep: to_del.unlink() return
def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "-o", "--output", required=False, default=".mdr.iof.cub" ) parser.add_argument("-e", "--edr", required=True) parser.add_argument( "-c", "--conf", required=False, type=argparse.FileType('r'), default=pkg_resources.resource_stream( __name__, 'data/hical.pipelines.conf' ), ) parser.add_argument("mdr", metavar="MDR_file") parser.add_argument( "-l", "--log", required=False, default="WARNING", help="The log level to show for this program, can " "be a named log level or a numerical level.", ) parser.add_argument( "-k", "--keep", required=False, default=False, action="store_true", help="Normally, the program will clean up any " "intermediary files, but if this option is given, it " "won't.", ) args = parser.parse_args() util.set_logger(args.verbose, args.logfile, args.log) edr_path = Path(args.edr) mdr_path = Path(args.mdr) to_del = isis.PathSet() h2i_path = to_del.add(edr_path.with_suffix(".hi2isis.cub")) out_path = util.path_w_suffix(args.output, edr_path) # The first thing Alan's program did was to crop the image down to only the # 'imaging' parts. We're not doing that so the resultant file has a # geometry similar to what comes out of ISIS hical. # Convert the EDR to a cube file isis.hi2isis(edr_path, to=h2i_path) # Convert Alan's MDR to a cube file mdr_cub_path = to_del.add(mdr_path.with_suffix(".alan.cub")) logger.info(f"Running gdal_translate {mdr_path} -of ISIS3 {mdr_cub_path}") gdal.Translate(str(mdr_cub_path), str(mdr_path), format="ISIS3") h2i_s = int(isis.getkey_k(h2i_path, "Dimensions", "Samples")) h2i_l = int(isis.getkey_k(h2i_path, "Dimensions", "Lines")) mdr_s = int(isis.getkey_k(mdr_cub_path, "Dimensions", "Samples")) mdr_l = int(isis.getkey_k(mdr_cub_path, "Dimensions", "Lines")) if h2i_s != mdr_s: label = pvl.load(str(h2i_path)) hirise_cal_info = get_one( label, "Table", "HiRISE Calibration Ancillary" ) buffer_pixels = get_one(hirise_cal_info, "Field", "BufferPixels")[ "Size" ] dark_pixels = get_one(hirise_cal_info, "Field", "DarkPixels")["Size"] rev_mask_tdi_lines = hirise_cal_info["Records"] if h2i_s + buffer_pixels + dark_pixels == mdr_s: logger.info( f"The file {mdr_cub_path} has " f"{buffer_pixels + dark_pixels} more sample pixels " f"than {h2i_path}, assuming those are dark and " "buffer pixels and will crop accordingly." ) if h2i_l + rev_mask_tdi_lines != mdr_l: logger.critical( 'Even assuming this is a "full" channel ' "image, this has the wrong number of lines. " f"{mdr_cub_path} should have " f"{h2i_l + rev_mask_tdi_lines}, but " f"has {mdr_l} lines. Exiting" ) sys.exit() else: crop_path = to_del.add(mdr_cub_path.with_suffix(".crop.cub")) # We want to start with the next pixel (+1) after the cal # pixels. isis.crop( mdr_cub_path, to=crop_path, sample=buffer_pixels + 1, nsamples=h2i_s, line=rev_mask_tdi_lines + 1, ) mdr_cub_path = crop_path mdr_l = int(isis.getkey_k(mdr_cub_path, "Dimensions", "Lines")) else: logger.critical( f"The number of samples in {h2i_path} ({h2i_s}) " f"and {mdr_cub_path} ({mdr_s}) are different. " "Exiting." ) sys.exit() if h2i_l != mdr_l: logger.critical( f"The number of lines in {h2i_path} ({h2i_l}) " f"and {mdr_cub_path} ({mdr_l}) are different. " "Exiting." ) sys.exit() # Convert the EDR to the right bit type for post-HiCal Pipeline: h2i_16b_p = to_del.add(h2i_path.with_suffix(".16bit.cub")) isis.bit2bit( h2i_path, to=h2i_16b_p, bit="16bit", clip="minmax", minval=0, maxval=1.5, ) shutil.copyfile(h2i_16b_p, out_path) # If it is a channel 1 file, Alan mirrored it so that he could process # the two channels in an identical way (which we also took advantage # of above if the buffer and dark pixels were included), so we need to # mirror it back. cid = hirise.get_ChannelID_fromfile(h2i_16b_p) if cid.channel == "1": mirror_path = to_del.add(mdr_cub_path.with_suffix(".mirror.cub")) isis.mirror(mdr_cub_path, to=mirror_path) mdr_cub_path = mirror_path # Is the MDR in DN or I/F? maximum_pxl = float( pvl.loads(isis.stats(mdr_cub_path).stdout)["Results"]["Maximum"] ) if maximum_pxl < 1.5: logger.info("MDR is already in I/F units.") mdr_16b_p = to_del.add(mdr_cub_path.with_suffix(".16bit.cub")) isis.bit2bit( mdr_cub_path, to=mdr_16b_p, bit="16bit", clip="minmax", minval=0, maxval=1.5, ) isis.handmos(mdr_16b_p, mosaic=out_path) else: logger.info("MDR is in DN units and will be converted to I/F.") fpa_t = statistics.mean( [ float( isis.getkey_k( h2i_16b_p, "Instrument", "FpaPositiveYTemperature" ) ), float( isis.getkey_k( h2i_16b_p, "Instrument", "FpaNegativeYTemperature" ) ), ] ) print(f"fpa_t {fpa_t}") conf = pvl.load(args.conf) tdg = t_dep_gain(get_one(conf["Hical"], "Profile", cid.ccdname), fpa_t) suncorr = solar_correction() sclk = isis.getkey_k( h2i_16b_p, "Instrument", "SpacecraftClockStartCount" ) target = isis.getkey_k(h2i_16b_p, "Instrument", "TargetName") suncorr = solar_correction(sunDistanceAU(sclk, target)) sed = float( isis.getkey_k(h2i_16b_p, "Instrument", "LineExposureDuration") ) zbin = get_one(conf["Hical"], "Profile", "GainUnitConversion")[ "GainUnitConversionBinFactor" ] # The 'ziof' name is from the ISIS HiCal/GainUnitConversion.h, it is a # divisor in the calibration equation. print(f"zbin {zbin}") print(f"tdg {tdg}") print(f"sed {sed}") print(f"suncorr {suncorr}") ziof = zbin * tdg * sed * 1e-6 * suncorr eqn = f"\(F1 / {ziof})" # noqa W605 mdriof_p = to_del.add(mdr_cub_path.with_suffix(".iof.cub")) to_s = "{}+SignedWord+{}:{}".format(mdriof_p, 0, 1.5) isis.fx(f1=mdr_cub_path, to=to_s, equ=eqn) isis.handmos(mdriof_p, mosaic=out_path) if not args.keep: to_del.unlink()
def unflip(in_p: Path, out_p: Path, keep=False): """Attempt to indentify DNs whose bits have been flipped in the ISIS cube indicated by *in_p*, and unflip them. Only operates on image-area. """ to_del = isis.PathSet() # This full suite of deltas works well for the reverse-clock area # and even 'dark' images, but not 'real' images. # deltas = (8192, 4096, 2048, 1024, 512, 256, 128, 64) # Restricting the number of deltas might help, but this seems # arbitrary. deltas = (8192, 4096, 2048, 1024) count = 0 suffix = ".bf{}-{}{}.cub" this_p = to_del.add(in_p.with_suffix(suffix.format(count, 0, 0))) this_p.symlink_to(in_p) median = math.trunc(float(isis.stats_k(in_p)["Median"])) for (sign, pm, extrema) in ((+1, "m", "Maximum"), (-1, "p", "Minimum")): # logging.info(pm) for delt in deltas: d = sign * delt far = median + d near = median + (d / 2) hist = isis.Histogram(this_p) logger.info( f"bitflip position {pm}{delt}, near: {near} " f"far: {far}, extrema: {hist[extrema]}" ) if (sign > 0 and far < float(hist[extrema])) or ( sign < 0 and far > float(hist[extrema]) ): count += 1 s = suffix.format(count, pm, delt) next_p = to_del.add(this_p.with_suffix("").with_suffix(s)) try: thresh = get_unflip_thresh(hist, far, near, d) except ValueError as err: logger.info(err) count -= 1 break subtract_over_thresh(this_p, next_p, thresh, d, keep=keep) this_p = next_p else: logger.info( "The far value was beyond the extrema. " "Didn't bother." ) shutil.move(this_p, out_p) if not keep: to_del.remove(this_p) to_del.unlink() return