def add_offsets(cubes: list,
base_ccdnumber: int,
temp_token: str,
keep=False) -> tuple:
flats = isis.PathSet()
for i, c in enumerate(cubes[:-1]):
pair = "{}-{}".format(cubes[i + 1].get_ccd(), cubes[i].get_ccd())
flat_p = flats.add(
c.path.with_suffix(f".{temp_token}.{pair}.flat.tab"))
(avg_line_offset,
avg_samp_offset) = get_offsets(cubes[i].next_path,
cubes[i + 1].next_path, flat_p)
j = i
if int(c.ccdnumber) >= base_ccdnumber:
j = i + 1
cubes[j].line_offset = avg_line_offset
cubes[j].samp_offset = avg_samp_offset
if not keep:
flats.unlink()
return (cubes, None)
else:
return (cubes, flats)
def test_Hidestripe(self):
to_del = isis.PathSet()
calcube = to_del.add(Path("test_Hidestripe-out.hical.cub"))
isis.hical(self.cube, to=calcube)
to_del.add(Path(str(self.pid)).with_suffix(".hical.log"))
outcube = to_del.add(Path("test_Hidestripe-out.cub"))
samps = int(isis.getkey_k(calcube, "Dimensions", "Samples"))
self.assertRaises(
KeyError,
hc.Hidestripe,
self.cube,
outcube,
self.binning,
minimum=0.0,
maximum=1.5,
hidcorr="ADD",
line_samples=samps,
keep=False,
)
self.assertAlmostEqual(
0.000101402295171637,
hc.Hidestripe(
calcube,
outcube,
self.binning,
minimum=0.0,
maximum=1.5,
hidcorr="ADD",
line_samples=samps,
keep=False,
),
)
to_del.unlink()
def test_unlink(self):
ps = isis.PathSet(self.paths)
for p in ps:
p.touch()
ps.unlink()
for p in ps:
self.assertFalse(p.exists())
def CubeNormStep(cube, hconf: dict, keep=False) -> HiccdStitchCube:
to_del = isis.PathSet()
# crop removing the top and bottom portion of the image
crop_p = to_del.add(cube.nextpath.with_suffix(".crop.cub"))
isis.crop(
cube.nextpath,
to=crop_p,
line=cube.sl_cubenorm,
nlines=cube.nl_cubenorm,
)
# run cubenorm to get statistics of the cropped area
stats_p = to_del.add(cube.nextpath.with_suffix(".cubenorm.tab"))
isis.cubenorm(
crop_p,
stats=stats_p,
format_="TABLE",
direction="COLUMN",
normalizer="AVERAGE",
MODE=hconf["HiccdStitch_Cubenorm_Method"],
PRESERVE=True,
)
stats_filtered_p = to_del.add(cube.nextpath.with_suffix(".cubenorm2.tab"))
# Original Perl: write cubenorm_stdev to the DB (again?) but we'll just
cube.cubenorm_stddev = AnalyzeStats(stats_p, stats_filtered_p)
# Original Perl: if HiccdStitch: cubenorm_stdev is written to the DB here
# Original Perl: if HiccdStitchC: write cubenorm_stdev to a PVL file?
# run cubenorm again, this time make the correction to the file
next_path = cube.nextpath.with_suffix(".cubenorm.cub")
to_s = "{}+SignedWord+{}:{}".format(
next_path,
hconf["HiccdStitch_Normalization_Minimum"],
hconf["HiccdStitch_Normalization_Maximum"],
)
isis.cubenorm(
cube.nextpath,
to=to_s,
fromstats=stats_filtered_p,
statsource="TABLE",
direction="COLUMN",
normalizer="AVERAGE",
MODE=hconf["HiccdStitch_Cubenorm_Method"],
PRESERVE=True,
)
cube.nextpath = next_path
if not keep:
to_del.unlink()
return cube
def lpfz_triplefilter(from_path: os.PathLike,
to_path: os.PathLike,
keep=False) -> None:
to_del = isis.PathSet()
from_p = Path(from_path)
z1 = to_del.add(from_p.with_suffix(".z1.cub"))
lpfz_filtering(from_p, z1, 11, 5)
z2 = to_del.add(from_p.with_suffix(".z2.cub"))
lpfz_filtering(z1, z2, 21, 9)
lpfz_filtering(z2, to_path, 41, 11)
if not keep:
to_del.unlink()
return
def crop_and_scale(cubes: list) -> list:
to_del = isis.PathSet()
for i, c in enumerate(cubes):
# First step in balancing process is to crop out the left and
# right overlap areas of each CCD
lc_path = to_del.add(c.nextpath.with_suffix(".left.crop.cub"))
rc_path = to_del.add(c.nextpath.with_suffix(".right.crop.cub"))
for t, s, n in zip(
[lc_path, rc_path],
[c.ss_balance_left, c.ss_balance_right],
[c.ns_balance_left, c.ns_balance_right],
):
isis.crop(
c.nextpath,
to=t,
sample=s,
nsamples=n,
line=c.sl_balance,
nlines=c.nl_balance,
)
# Second step is to scale all of the croped files to have the
# same lines and samples, needed for mask step
if c.smag == 1 and c.lmag == 1:
cubes[i].set_ls_path(lc_path)
cubes[i].set_rs_path(rc_path)
else:
cubes[i].set_ls_path(
to_del.add(c.nextpath.with_suffix(".left.scale.cub")))
cubes[i].set_rs_path(
to_del.add(c.nextpath.with_suffix(".right.scale.cub")))
for lc, ls in zip([lc_path, rc_path],
[cubes[i].ls_path, cubes[i].rs_path]):
isis.enlarge(lc,
to=ls,
sscale=c.smag,
lscale=c.lmag,
interp="CUBIC")
return (cubes, to_del)
def HiColorNorm(cubes: list,
output,
conf: dict,
make_unfiltered=True,
db_list=None,
keep=False):
logger.info(f"HiColorNorm start: {cubes}")
# GetConfigurationParameters()
conf_check(conf)
cubes = list(map(ColorCube, cubes, repeat(db_list)))
cubes.sort()
outcub_path = set_outpath(output, cubes)
temp_token = datetime.now().strftime("HiColorNorm-%y%m%d%H%M%S")
out_p = Path(outcub_path)
furrow_flag = FurrowCheck(cubes)
to_del = isis.PathSet()
for i, _ in enumerate(cubes):
cubes[i].set_crop_lines(conf)
for i, c in enumerate(cubes):
# Protect the processing from erroneous/spurious pixels
mask_list = list()
for b in (1, 2, 3):
tmp_p = to_del.add(
c.path.with_suffix(f".{temp_token}.temp{b}.cub"))
isis.mask(
f"{c.path}+{b}",
mask=f"{c.path}+{b}",
to=tmp_p,
minimum=0.0,
maximum=2.0,
preserve="INSIDE",
)
mask_list.append(tmp_p)
c.final_path = c.path.with_suffix(f".HiColorNorm.cub")
isis.cubeit_k(mask_list, to=c.final_path)
(cubes[i].mask_path["IR"],
cubes[i].crop_path["IR"]) = per_color(c, temp_token, "IR", keep=keep)
(cubes[i].mask_path["BG"],
cubes[i].crop_path["BG"]) = per_color(c, temp_token, "BG", keep=keep)
ir_ratio_stddev = per_band(cubes,
out_p,
temp_token,
"IR",
furrow_flag,
make_unfiltered,
keep=keep)
bg_ratio_stddev = per_band(cubes,
out_p,
temp_token,
"BG",
furrow_flag,
make_unfiltered,
keep=keep)
if conf["HiColorNorm"]["HiColorNorm_Make_Stitch"]:
# listpath = to_del.add(c.path.with_suffix(f'.{temp_token}.list.txt'))
# listpath.write_text(
# '\n'.join(str(c.final_path) for c in cubes) + '\n')
with isis.fromlist.temp([str(c.final_path) for c in cubes]) as f:
isis.hiccdstitch(fromlist=f, to=out_p)
for c in cubes:
to_del.add(c.final_path)
if not keep:
to_del.unlink()
for c in cubes:
for cc in ("IR", "BG"):
c.mask_path[cc].unlink()
c.crop_path[cc].unlink()
c.nrm_path[cc].unlink()
logger.info(f"HiColorNorm done.")
return ir_ratio_stddev, bg_ratio_stddev
def HiJACK(
cube_paths: list,
conf_dir: os.PathLike,
outdir=Path("./HiJACK"),
base_ccd_number=5,
plot=True,
keep=False,
):
cubes = list(map(hnp.Cube, cube_paths))
cubes.sort()
logger.info(f"HiJACK start: {', '.join(map(str, cubes))}")
base_cube = check(cubes, base_ccd_number)
temp_token = datetime.now().strftime("HiJACK-%y%m%d%H%M%S")
outdir_p = Path(outdir)
outdir_p.mkdir(exist_ok=True)
to_del = isis.PathSet()
# Pre-HiJACK
for c in cubes:
c.next_path = to_del.add(
(outdir_p / c.path.name).with_suffix(
f".{temp_token}.prehijack.cub"
)
)
flat_path = outdir_p / "{}.{}.{}".format(
cubes[0].get_obsid(), temp_token, "RED5-RED4_prehijack.flat.tab"
)
match_red(cubes, base_cube, flat_path)
# jitter_path = outdir_p / (str(cubes[0].get_obsid()) + "_jitter_cpp.txt")
jitter_path = outdir_p / (str(cubes[0].get_obsid()) + "_jitter_py.txt")
if not jitter_path.exists():
resolvejitter_conf = Path(conf_dir) / "ResolveJitter.conf"
ResolveJitter(
cubes,
pvl.load(str(resolvejitter_conf)),
jitter_path,
temp_token,
keep=keep,
)
# SmearStats just updates the db, not needed.
# HiJACK proper:
hijack_conf = pvl.load(str(Path(conf_dir) / "HiJACK.conf"))
hnp.conf_check(hijack_conf["HiJACK"])
polar = False
if hijack_conf["HiJACK"]["Shape"] == "USER":
polar = hnp.is_polar(cubes, hijack_conf["Pole_Tolerance"], temp_token)
for c in cubes:
hnp.copy_and_spice(
c.next_path,
to_del.add(c.path.with_suffix(f".{temp_token}.spiced.cub")),
hijack_conf["HiJACK"],
polar,
)
# Run hijitter
inlist = list()
outlist = list()
for c in cubes:
c.next_path = (
(outdir_p / c.path.name)
.with_suffix("")
.with_suffix(".dejittered.cub")
)
inlist.append(c.path)
outlist.append(c.next_path)
jitterck_p = outdir_p / (str(cubes[0].get_obsid()) + ".jittery.bc")
hijitregdef_p = (
Path(os.environ["ISIS3DATA"])
/ "mro/calibration/hijitreg.p1745.s3070.def"
)
# inlist_p = to_del.add(isis.fromlist.make(inlist,
# (outdir_p /
# (str(cubes[0].get_obsid()) +
# '_hijitter.inlst'))))
# outlist_p = to_del.add(isis.fromlist.make(outlist,
# inlist_p.with_suffix('.outlst')))
with isis.fromlist.temp(inlist) as inlist_f:
with isis.fromlist.temp(outlist) as outlist_f:
isis.hijitter(
fromlist=inlist_f,
jitter=jitter_path,
regdef=hijitregdef_p,
tolist=outlist_f,
jitterck=jitterck_p,
)
# Mosaic dejittered cubs
# Remember hijitter makes all the individual cubes the size of the
# entire image. With the image data in the appropriate space for the ccd.
out_root = outdir_p / str(cubes[0].get_obsid())
# All REDs
red_p = out_root.with_name(out_root.name + "_RED.NOPROJ.cub")
red_5 = list(
filter(lambda x: x.ccdname == "RED" and x.ccdnumber == "5", cubes)
)[0]
# shutil.copyfile(base_cube.next_path, red_p)
shutil.copyfile(red_5.next_path, red_p)
logger.info(
"Original Perl hard codes this file copy from RED5, even if "
"another cube is selected as the base_ccd."
)
(red_cubes, red_flat_files) = hnp.add_offsets(
list(filter(lambda x: x.ccdname == "RED", cubes)),
5,
temp_token,
keep=True,
)
hnp.handmos_side(red_cubes, red_5, red_p, left=True)
hnp.handmos_side(red_cubes, red_5, red_p, left=False)
hnp.fix_labels(
red_cubes, red_p, red_5, "{}_RED".format(str(red_cubes[0].get_obsid()))
)
# Center RED for color
center_red_p = out_root.with_name(out_root.name + "_RED4-5.NOPROJ.cub")
center_red_cubes = list(
filter(lambda x: x.ccdnumber == "4" or x.ccdnumber == "5", red_cubes)
)
shutil.copyfile(center_red_cubes[1].next_path, center_red_p)
mosaic_dejittered(
center_red_cubes,
center_red_p,
"{}_RED4-5".format(str(center_red_cubes[0].get_obsid())),
)
ir_p = out_root.with_name(out_root.name + "_IR.NOPROJ.cub")
(ir_cubes, _) = hnp.add_offsets(
list(filter(lambda x: x.ccdname == "IR", cubes)),
5,
temp_token,
keep=keep,
)
shutil.copyfile(ir_cubes[1].next_path, ir_p)
mosaic_dejittered(
ir_cubes, ir_p, "{}_IR".format(str(ir_cubes[0].get_obsid()))
)
# BG next
bg_p = out_root.with_name(out_root.name + "_BG.NOPROJ.cub")
(bg_cubes, _) = hnp.add_offsets(
list(filter(lambda x: x.ccdname == "BG", cubes)),
5,
temp_token,
keep=keep,
)
shutil.copyfile(bg_cubes[1].next_path, bg_p)
mosaic_dejittered(
bg_cubes, bg_p, "{}_BG".format(str(bg_cubes[0].get_obsid()))
)
# Create color product
irb_p = out_root.with_name(out_root.name + "_IRB.NOPROJ.cub")
with isis.fromlist.temp([ir_p, center_red_p, bg_p]) as f:
isis.cubeit(fromlist=f, to=irb_p, proplab=center_red_p)
if plot:
# Make plot of before and after flat.tab results
dejit_flat = list(
filter(lambda x: x.match("*RED5-RED4*"), red_flat_files)
)[0]
plot_flats(flat_path, dejit_flat)
if not keep:
flat_path.unlink()
red_flat_files.unlink()
to_del.unlink()
logger.info(f"HiJACK done")
return
def HiNoProj(cubes: list,
conf: dict,
output="_RED.NOPROJ.cub",
base=5,
keep=False):
logger.info(f"HiNoProj start: {', '.join(map(str, cubes))}")
cubes = list(map(Cube, cubes))
cubes.sort()
if not all(c.ccdname == "RED" for c in cubes):
raise ValueError("Not all of the input files are RED CCD files.")
sequences = list()
for k, g in itertools.groupby(
(int(c.ccdnumber) for c in cubes),
lambda x, c=itertools.count(): next(c) - x,
):
sequences.append(list(g))
if len(sequences) != 1:
raise ValueError("The given cubes are not a single run of sequential "
"HiRISE CCDs, instead there are "
f"{len(sequences)} groups with these "
f"CCD numbers: {sequences}.")
if not isinstance(base, int):
base = hirise.get_CCDID_fromfile(base)
base_ccd = list(filter(lambda x: x.ccdnumber == str(base), cubes))
if len(base_ccd) != 1:
raise ValueError(f"The base ccd, number {base}, "
"is not one of the given cubes.")
base_cube = base_ccd[0]
conf_check(conf["HiNoProj"])
conf = conf["HiNoProj"]
out_p = hcn.set_outpath(output, cubes)
temp_token = datetime.now().strftime("HiNoProj-%y%m%d%H%M%S")
to_del = isis.PathSet()
polar = False
if conf["Shape"] == "USER":
polar = is_polar(cubes, conf["Pole_Tolerance"], temp_token)
for c in cubes:
temp_p = to_del.add(c.path.with_suffix(f".{temp_token}.spiced.cub"))
copy_and_spice(c.path, temp_p, conf, polar)
isis.spicefit(temp_p)
c.next_path = to_del.add(
c.path.with_suffix(f".{temp_token}.noproj.cub"))
c.path = temp_p
for c in cubes:
isis.noproj(c.path,
match=base_cube.path,
to=c.next_path,
source="frommatch")
# Run hijitreg on adjacent noproj'ed ccds to get average line/sample offset
(cubes, _) = add_offsets(cubes,
int(base_cube.ccdnumber),
temp_token,
keep=keep)
# Mosaic noproj'ed ccds using average line/sample offset
shutil.copyfile(base_cube.next_path, out_p)
logger.info("Original Perl hard codes this file copy from RED5, even if "
"another cube is selected as the base_ccd.")
handmos_side(cubes, base_cube, out_p, left=True)
handmos_side(cubes, base_cube, out_p, left=False)
isis.editlab(
out_p,
option="addkey",
grpname="Instrument",
keyword="ImageJitterCorrected",
value=0,
)
fix_labels(
cubes,
out_p,
base_cube,
"{}_{}".format(str(cubes[0].get_obsid()), cubes[0].ccdname),
)
if not keep:
to_del.unlink()
logger.info(f"HiNoProj done: {out_p}")
return
def test_init(self):
ps = isis.PathSet()
self.assertIsInstance(ps, isis.PathSet)
def BalanceStep(cubes, conf, keep=False) -> list:
to_del = isis.PathSet()
# Sort the cubes so that they are in CCD order
cubes.sort()
cubes, to_delete = crop_and_scale(cubes)
to_del.update(to_delete)
# Original Perl: Generate CCD number array for each CCD file, not needed
# now, since we can just query the object.
# Original Perl: Used $0 (the program name) instead of the index 0
# here, but I've arranged things with the set_ls_path() and set_rs_path()
# to just make these a full copy so there's no
# need to mess with a conditional assignment and also streamlines
# the following logic, since they're identical except when you
# explicitly change them.
# The third step is to mask the left and right overlap areas. We
# want to zap pixels where there is not common coverage.
for i, c in enumerate(cubes):
if i + 1 < len(cubes) and int(cubes[i].ccdnumber) + 1 == int(
cubes[i + 1].ccdnumber):
cubes[i].rm_path = to_del.add(
c.nextpath.with_suffix(".right.mask.cub"))
cubes[i + 1].lm_path = to_del.add(
cubes[i + 1].nextpath.with_suffix(".left.mask.cub"))
for f, m, t in zip(
[cubes[i].rs_path, cubes[i + 1].ls_path],
[cubes[i + 1].ls_path, cubes[i].rs_path],
[cubes[i].rm_path, cubes[i + 1].lm_path],
):
isis.mask(
f,
mask=m,
to=t,
preserve="INSIDE",
min_=conf["HiccdStitch_Normalization_Minimum"],
max_=conf["HiccdStitch_Normalization_Maximum"],
)
# The fourth step is to get image statistics for left and right
# overlap areas of each CCD image.
cubes = get_stats(cubes)
# Look for a break in joining CCDs, defined by a break in the CCD number,
# or the right or left statistics are undefined, due to an all null channel
#
# In the original Perl there was a loop to determine if there was a break,
# but then nothing was done with that information? Oh, it was used
# differently: the code past that point develops a series of sequences
# from $first to $last. If there are no breaks, then it only runs a
# single sequence. If there are breaks, it runs the sequences it finds.
#
# Here's the pythonic version:
cubes.sort()
for (offset, group) in get_group_i(cubes):
logger.info("Correction before redistribution.")
for ccd in group:
i = ccd + offset
cubes[i].correction = get_correction(
cubes[i],
cubes[i - 1],
conf["HiccdStitch_Balance_Correction"],
i,
)
logger.info(
f"CCDID: {cubes[i]}, correction: {cubes[i].correction}")
normalization = get_normalization(cubes, group, offset,
conf["HiccdStitch_Control_CCD"])
logger.info("Correction after redistribution.")
for ccd in group:
i = ccd + offset
cubes[i].correction /= normalization
logger.info(
f"CCDID: {cubes[i]}, correction: {cubes[i].correction}, "
f"left: {cubes[i].lstats}, right: {cubes[i].rstats}")
# In the original Perl, they wrote out to the DB here, but we'll
# do it later. There was also a distinction that if it was
# HiccdStitchC that the data was written out to a PVL file. Not
# sure why.
# Create the balance cubes
for ccd in group:
i = ccd + offset
balance_path = cubes[i].nextpath.with_suffix(".balance.cub")
make_balance(cubes[i], conf, balance_path)
cubes[i].nextpath = balance_path
if not keep:
to_del.unlink()
return cubes
def HiccdStitch(cubes: list,
out_path: os.PathLike,
conf: dict,
sline=None,
eline=None,
keep=False) -> tuple:
logger.info(f"HiccdStitch start: {', '.join(map(str, cubes))}")
# Perl: GetConfigurationParameters()
# conf = pvl.load(str(conf_path))
conf_check(conf)
out_p = set_outpath(out_path, cubes)
# Perl: GetImageDims()
cubes = GetImageDims(cubes, conf, sline, eline)
# This string will get placed in the filename for all of our
# temporary files. It will (hopefully) prevent collisions with
# existing files and also allow for easy clean-up if keep=True
# temp_token = datetime.now().strftime('HiccdStitch-%y%m%d%H%M%S')
to_delete = isis.PathSet()
# Perl: MakeList()
# This makes a file, but doesn't really do anything with it, because
# a different file is actually made later to give to hiccdstitch
for i, c in enumerate(cubes):
if c.cubenormstep:
cubes[i] = CubeNormStep(c, conf["HiccdStitch"], keep=keep)
if conf["HiccdStitch"]["HiccdStitch_Balance"]:
cubes = BalanceStep(cubes, conf["HiccdStitch"], keep=keep)
for c in cubes:
SpecialProcessingFlags(c)
cubes.sort()
logger.info("The Original Perl looked for a custom file for "
"hiccdstitch's shiftdef parameter, but the default ISIS "
"file seems better, so this isn't implemented.")
with isis.fromlist.temp([str(c.nextpath) for c in cubes]) as f:
isis.hiccdstitch(
fromlist=f,
to=out_p,
interp=conf["HiccdStitch"]["HiccdStitch_Interpolation"],
)
SNR_Check(cubes, conf["HiccdStitch"]["HiccdStitch_SNR_Threshold"])
logger.info("HiccdStitch done.")
if not keep:
to_delete.unlink()
# Afterwards inserts into CCD_Processing_Statistics table.
db = {"OBSERVATION_ID": str(cubes[0].get_obsid())}
for c in cubes:
ccd_db = {
"CCDID": str(c),
"RADIOMETRIC_MATCHING_CORRECTION": c.correction,
"LEFT_OVERLAP_AVERAGE": c.lstats,
"RIGHT_OVERLAP_AVERAGE": c.rstats,
"CUBENORM_COLUMN_CORRECTION_STANDARD_DEVIATION": c.cubenorm_stddev,
}
db[c.get_ccd()] = ccd_db
logger.info(f"HiccdStitch done: {out_p}")
return db, out_p
def test_add_Error(self):
ps = isis.PathSet()
p0 = self.paths[0]
ps.add(p0)
self.assertRaises(TypeError, ps.add, 'not a Path')
self.assertRaises(ValueError, ps.add, p0)
def test_add(self):
ps = isis.PathSet()
path1 = self.paths[0]
added_path = ps.add(path1)
self.assertEqual(path1, added_path)
def test_init_iterable(self):
ps = isis.PathSet(self.paths)
self.assertEqual(3, len(ps))
def per_band(cubes,
out_p,
temp_token,
color_code,
furrow_flag,
unfiltered,
keep=False) -> float:
to_del = isis.PathSet()
# Generate the handmos of both the ratio and the croped ratio files
if len(cubes) == 2:
maxlines = max(c.lines for c in cubes)
maxsamps = sum(c.samps for c in cubes)
# Generate the mosaic of the ratio image
mos_p = to_del.add(
out_p.with_suffix(f".{temp_token}.{color_code}.mos.cub"))
make_LR_mosaic(
cubes[0].mask_path[color_code],
cubes[1].mask_path[color_code],
cubes[0].samps,
mos_p,
maxlines,
maxsamps,
)
# Generate the mosaic of the crop ratio image
maxlines = max(c.crop_lines for c in cubes)
crpmos_p = to_del.add(
out_p.with_suffix(f".{temp_token}.{color_code}.moscrop.cub"))
make_LR_mosaic(
cubes[0].crop_path[color_code],
cubes[1].crop_path[color_code],
cubes[0].samps,
crpmos_p,
maxlines,
maxsamps,
)
else:
mos_p = cubes[0].mask_path[color_code]
crpmos_p = cubes[0].crop_path[color_code]
mosnrm_p = to_del.add(
out_p.with_suffix(f".{temp_token}.{color_code}.mosnorm.cub"))
ratio_stddev = cubenorm_stats(crpmos_p, mos_p, mosnrm_p, keep=keep)
# from the handmos cubes, pull out the individual CCDs with crop
if len(cubes) == 2:
samp = 1
for i, c in enumerate(cubes):
cubes[i].nrm_path[color_code] = c.path.with_suffix(
f".{temp_token}.{color_code}.norm.cub")
isis.crop(
mosnrm_p,
to=cubes[i].nrm_path[color_code],
sample=samp,
nsamples=c.samps,
)
samp += c.samps
else:
# If there was only one file then all we need to do is rename files
cubes[0].nrm_path[color_code] = mosnrm_p
if unfiltered:
# Create the unfiltered color cubes, if needed
for c in cubes:
make_unfiltered(
c.path,
c.nrm_path[color_code],
temp_token,
color_code,
c.band[color_code],
keep=keep,
)
# low pass filter the files
for c in cubes:
# If $lpfz=1 then we're only interpolating null pixels caused by
# noise or furrow removal
lowpass_args = dict(
minopt="PERCENT",
minimum=25,
low=0.00,
high=2.0,
null=True,
hrs=True,
lis=True,
lrs=True,
)
# As the comment below states, I *think* that the OP had a copy
# and paste error here, because even though it kept track of both
# the IR and BG boxcar dimensions, they were always identical.
logger.warning("The original Perl calculates the boxcar size for "
"both the IR and BG based only on the ratio of the "
"IR to the RED.")
if c.get_boxcar_size(c.ir_bin) == 1:
lowpass_args["lines"] = 3
lowpass_args["samples"] = 3
lowpass_args["filter"] = "OUTSIDE"
else:
lowpass_args["lines"] = 1
lowpass_args["samples"] = 1
lpf_path = to_del.add(
c.path.with_suffix(f".{temp_token}.{color_code}.lpf.cub"))
isis.lowpass(c.nrm_path[color_code], to=lpf_path, **lowpass_args)
# Perform lpfz filters to interpolate null pixels due to furrows or
# bad pixels
if furrow_flag:
lpfz_path = c.path.with_suffix(
f".{temp_token}.{color_code}.lpfz.cub")
lpfz_triplefilter(lpf_path, lpfz_path, temp_token, keep=keep)
else:
lpfz_path = lpf_path
# run ISIS algebra program to created (normalized-IR/RED)*RED files
alg_path = to_del.add(
c.path.with_suffix(f".{temp_token}.{color_code}.algebra.cub"))
isis.algebra(lpfz_path,
from2=f"{c.path}+2",
to=alg_path,
operator="MULTIPLY")
# Update the output file with the normalized IR and BG bands
isis.handmos(
alg_path,
mosaic=c.final_path,
outsample=1,
outline=1,
outband=c.band[color_code],
matchbandbin=False,
)
if not keep:
to_del.unlink()
return ratio_stddev
def HiFurrow_Fix(in_cube: os.PathLike,
out_cube: os.PathLike,
max_mean: float,
keep=False):
"""Perform a normalization of the furrow region of bin 2 or 4
HiRISE images. The input to this script is a HiRISE stitch
product containing both channels of a CCD.
"""
in_cub = Path(in_cube)
binning = int(isis.getkey_k(in_cub, "Instrument", "Summing"))
lines = int(isis.getkey_k(in_cub, "Dimensions", "Lines"))
samps = int(isis.getkey_k(in_cub, "Dimensions", "Samples"))
if binning != 2 and binning != 4:
raise ValueError("HiFurrow_Fix only supports correction for "
"bin 2 or 4 data.")
if binning == 2 and samps != 1024:
raise ValueError(f"HiFurrowFix: improper number of samples: {samps}, "
"for a stitch product with bin 2 (should be 1024).")
# This string will get placed in the filename for all of our
# temporary files. It will (hopefully) prevent collisions with
# existing files and also allow for easy clean-up if keep=True
temp_token = datetime.now().strftime("HFF-%y%m%d%H%M%S")
to_del = isis.PathSet()
# For bin2 and bin4 imaging, specify width of furrow based on
# image average DN range
range_low = {2: (512, 513), 4: (256, 257)} # 2 pixel furrow width
range_mid = {2: (511, 514), 4: (255, 258)} # 4 pixel furrow width
range_hgh = {2: (511, 514), 4: (255, 258)} # 4 pixel furrow width
range_max = {2: (510, 515), 4: (254, 259)} # 6 pixel furrow width
# Original code had low/mid/hgh for bin2 and bin4, but they
# were hard-coded to be identical.
dn_range_low = 9000
dn_range_mid = 10000
dn_range_hgh = 12000
if max_mean > dn_range_hgh:
dn_range = range_max[binning]
elif max_mean > dn_range_mid:
dn_range = range_hgh[binning]
elif max_mean > dn_range_low:
dn_range = range_mid[binning]
else:
dn_range = range_low[binning]
lpf_samp = int((dn_range[1] - dn_range[0] + 1) / 2) * 4 + 1
lpf_line = int(lpf_samp / 2) * 20 + 1
# Create a mask file
# DN=1 for non-furrow area
# DN=0 for furrow area
eqn = rf"\(1*(sample<{dn_range[0]})+ 1*(sample>{dn_range[1]}) + 0)"
fx_cub = to_del.add(in_cub.with_suffix(f".{temp_token}.fx.cub"))
isis.fx(to=fx_cub,
mode="OUTPUTONLY",
lines=lines,
samples=samps,
equation=eqn)
# Create a file where the furrow area is set to null
mask1_cub = to_del.add(in_cub.with_suffix(f".{temp_token}.mask1.cub"))
isis.mask(
in_cub,
mask=fx_cub,
to=mask1_cub,
min_=1,
max_=1,
preserve="INSIDE",
spixels="NULL",
)
# Lowpass filter to fill in the null pixel area
lpf_cub = to_del.add(in_cub.with_suffix(f".{temp_token}.lpf.cub"))
isis.lowpass(
mask1_cub,
to=lpf_cub,
sample=lpf_samp,
line=lpf_line,
null=True,
hrs=False,
his=False,
lis=False,
)
# Create a file where non-furrow columns are set to null
mask2_cub = to_del.add(in_cub.with_suffix(f".{temp_token}.mask2.cub"))
isis.mask(
in_cub,
mask=fx_cub,
to=mask2_cub,
min_=0,
max_=0,
preserve="INSIDE",
spixels="NULL",
)
# Highpass filter the furrow region
hpf_cub = to_del.add(in_cub.with_suffix(f".{temp_token}.hpf.cub"))
isis.highpass(mask2_cub, to=hpf_cub, sample=1, line=lpf_line)
# Add lowpass and highpass together to achieve desired result
alg_cub = to_del.add(in_cub.with_suffix(f".{temp_token}.alg.cub"))
isis.algebra(from_=lpf_cub,
from2=hpf_cub,
to=alg_cub,
operator="ADD",
A=1.0,
B=1.0)
# copy the input file to the output file then mosaic the
# furrow area as needed.
logger.info(f"Copy {in_cub} to {out_cube}.")
shutil.copyfile(in_cub, out_cube)
isis.handmos(
alg_cub,
mosaic=out_cube,
outsample=1,
outline=1,
outband=1,
insample=1,
inline=1,
inband=1,
create="NO",
)
if not keep:
to_del.unlink()
return
def HiBeautify(cube_paths: list,
conf: dict,
out_irb="_IRB.cub",
out_rgb="_RGB.cub",
keep=False):
logger.info(f"HiBeautify start: {', '.join(map(str, cube_paths))}")
# GetConfigurationParameters()
cubes = list(map(hcn.ColorCube, cube_paths))
cubes.sort()
irb_out_p = hcn.set_outpath(out_irb, cubes)
rgb_out_p = hcn.set_outpath(out_rgb, cubes)
temp_token = datetime.now().strftime("HiBeautify-%y%m%d%H%M%S")
to_del = isis.PathSet()
# Create an IRB mosaic from the HiColorNorm halves.
# If a half is missing, we create a mosaic with the proper width and place
# the half in it at the proper location.
# Logic ofr image_midpoint and total_width come from original HiColorInit
# irbmerged_p = to_del.add(out_p.with_suffix(f'.{temp_token}_IRB.cub'))
image_midpoint = int((2000 / cubes[0].red_bin) + 1)
outsample = image_midpoint
if cubes[0].ccdnumber == "4":
outsample = 1
total_width = int(2 * cubes[0].samps - (48 / cubes[0].red_bin))
isis.handmos(
cubes[0].path,
mosaic=irb_out_p,
outline=1,
outsample=outsample,
outband=1,
create="Y",
nlines=cubes[0].lines,
nsamp=total_width,
nbands=3,
)
if len(cubes) == 1:
logger.info("Warning, missing one half!")
else:
logger.info("Using both halves")
isis.handmos(
cubes[1].path,
mosaic=irb_out_p,
outline=1,
outsample=image_midpoint,
outband=1,
)
# Nothing is actually done to the pixels here regarding the FrostStats, so
# I'm just going to skip them here.
#
# # Determine if Frost/ICE may be present using FrostStats module.
# frost = None
# if args.frost:
# frost = True
# logging.info('Frost override: disabling auto-detection and using '
# 'the frost/ice color stretch')
# if args.nofrost:
# frost = False
# logging.info('Frost override: disable auto-detection and not using '
# 'the frost/ice color stretch')
# if frost is None:
# pass
# # get frost stats
# Subtract the unaltered RED band from the high pass filtered BG for
# synthetic blue.
logger.info("Creating synthetic B, subtracting RED from BG")
rgbsynthb_p = to_del.add(irb_out_p.with_suffix(f".{temp_token}_B.cub"))
isis.algebra(
f"{irb_out_p}+3",
from2=f"{irb_out_p}+2",
op="subtract",
to=rgbsynthb_p,
A=conf["Beautify"]["Synthetic_A_Coefficient"],
B=conf["Beautify"]["Synthetic_B_Coefficient"],
)
# Adjust the BandBin group
isis.editlab(rgbsynthb_p,
grpname="BandBin",
keyword="Name",
value="Synthetic Blue")
isis.editlab(rgbsynthb_p, grpname="BandBin", keyword="Center", value="0")
isis.editlab(rgbsynthb_p, grpname="BandBin", keyword="Width", value="0")
# HiBeautify gathers and writes a bunch of statistics to PVL that is
# important to the HiRISE GDS, but not relevant to just producing pixels
# so I'm omitting it.
#
# # Determine the min and max DN values of each band (RED, IR, BG, B) we're
# # working with.
# (upper, lower) = conf['Beautify']['Stretch_Exclude_Lines']
# if upper == 0 and lower == 0:
# synthbcrp_p = rgbsynthb_p
# irbmrgcrp_p = irbmerged_p
# else:
# synthbcrp_p = to_del.add(
# out_p.with_suffix(f'.{temp_token}_Bx.cub'))
# irbmrgcrp_p = to_del.add(
# out_p.with_suffix(f'.{temp_token}_IRBx.cub'))
#
# for (f, t) in ((rgbsynthb_p, synthbcrp_p),
# (irbmerged_p, irbmrgcrp_p)):
# logging.info(isis.crop(f, to=t, propspice=False,
# line=(1 + upper),
# nlines=(
# cubes[0].lines - lower + upper)).args)
#
# stats = dict()
# stats['B'] = Get_MinMax(synthbcrp_p,
# conf['Beautify']['Stretch_Reduction_Factor'],
# temp_token, keep=keep)
#
# for band in cubes[0].band.keys():
# stats[band] = Get_MinMax('{}+{}'.format(str(irbmrgcrp_p),
# cubes[0].band[band]),
# conf['Beautify']['Stretch_Reduction_Factor'],
# temp_token, keep=keep)
# Create an RGB cube using the RED from the IRB mosaic,
# the BG from the IRB mosaic and the synthetic B that we just made.
isis.cubeit_k([f"{irb_out_p}+2", f"{irb_out_p}+3", rgbsynthb_p],
to=rgb_out_p)
if not keep:
to_del.unlink()
return
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"-o", "--output", required=False, default=".mdr.iof.cub"
)
parser.add_argument("-e", "--edr", required=True)
parser.add_argument(
"-c",
"--conf",
required=False,
type=argparse.FileType('r'),
default=pkg_resources.resource_stream(
__name__,
'data/hical.pipelines.conf'
),
)
parser.add_argument("mdr", metavar="MDR_file")
parser.add_argument(
"-l",
"--log",
required=False,
default="WARNING",
help="The log level to show for this program, can "
"be a named log level or a numerical level.",
)
parser.add_argument(
"-k",
"--keep",
required=False,
default=False,
action="store_true",
help="Normally, the program will clean up any "
"intermediary files, but if this option is given, it "
"won't.",
)
args = parser.parse_args()
util.set_logger(args.verbose, args.logfile, args.log)
edr_path = Path(args.edr)
mdr_path = Path(args.mdr)
to_del = isis.PathSet()
h2i_path = to_del.add(edr_path.with_suffix(".hi2isis.cub"))
out_path = util.path_w_suffix(args.output, edr_path)
# The first thing Alan's program did was to crop the image down to only the
# 'imaging' parts. We're not doing that so the resultant file has a
# geometry similar to what comes out of ISIS hical.
# Convert the EDR to a cube file
isis.hi2isis(edr_path, to=h2i_path)
# Convert Alan's MDR to a cube file
mdr_cub_path = to_del.add(mdr_path.with_suffix(".alan.cub"))
logger.info(f"Running gdal_translate {mdr_path} -of ISIS3 {mdr_cub_path}")
gdal.Translate(str(mdr_cub_path), str(mdr_path), format="ISIS3")
h2i_s = int(isis.getkey_k(h2i_path, "Dimensions", "Samples"))
h2i_l = int(isis.getkey_k(h2i_path, "Dimensions", "Lines"))
mdr_s = int(isis.getkey_k(mdr_cub_path, "Dimensions", "Samples"))
mdr_l = int(isis.getkey_k(mdr_cub_path, "Dimensions", "Lines"))
if h2i_s != mdr_s:
label = pvl.load(str(h2i_path))
hirise_cal_info = get_one(
label, "Table", "HiRISE Calibration Ancillary"
)
buffer_pixels = get_one(hirise_cal_info, "Field", "BufferPixels")[
"Size"
]
dark_pixels = get_one(hirise_cal_info, "Field", "DarkPixels")["Size"]
rev_mask_tdi_lines = hirise_cal_info["Records"]
if h2i_s + buffer_pixels + dark_pixels == mdr_s:
logger.info(
f"The file {mdr_cub_path} has "
f"{buffer_pixels + dark_pixels} more sample pixels "
f"than {h2i_path}, assuming those are dark and "
"buffer pixels and will crop accordingly."
)
if h2i_l + rev_mask_tdi_lines != mdr_l:
logger.critical(
'Even assuming this is a "full" channel '
"image, this has the wrong number of lines. "
f"{mdr_cub_path} should have "
f"{h2i_l + rev_mask_tdi_lines}, but "
f"has {mdr_l} lines. Exiting"
)
sys.exit()
else:
crop_path = to_del.add(mdr_cub_path.with_suffix(".crop.cub"))
# We want to start with the next pixel (+1) after the cal
# pixels.
isis.crop(
mdr_cub_path,
to=crop_path,
sample=buffer_pixels + 1,
nsamples=h2i_s,
line=rev_mask_tdi_lines + 1,
)
mdr_cub_path = crop_path
mdr_l = int(isis.getkey_k(mdr_cub_path, "Dimensions", "Lines"))
else:
logger.critical(
f"The number of samples in {h2i_path} ({h2i_s}) "
f"and {mdr_cub_path} ({mdr_s}) are different. "
"Exiting."
)
sys.exit()
if h2i_l != mdr_l:
logger.critical(
f"The number of lines in {h2i_path} ({h2i_l}) "
f"and {mdr_cub_path} ({mdr_l}) are different. "
"Exiting."
)
sys.exit()
# Convert the EDR to the right bit type for post-HiCal Pipeline:
h2i_16b_p = to_del.add(h2i_path.with_suffix(".16bit.cub"))
isis.bit2bit(
h2i_path,
to=h2i_16b_p,
bit="16bit",
clip="minmax",
minval=0,
maxval=1.5,
)
shutil.copyfile(h2i_16b_p, out_path)
# If it is a channel 1 file, Alan mirrored it so that he could process
# the two channels in an identical way (which we also took advantage
# of above if the buffer and dark pixels were included), so we need to
# mirror it back.
cid = hirise.get_ChannelID_fromfile(h2i_16b_p)
if cid.channel == "1":
mirror_path = to_del.add(mdr_cub_path.with_suffix(".mirror.cub"))
isis.mirror(mdr_cub_path, to=mirror_path)
mdr_cub_path = mirror_path
# Is the MDR in DN or I/F?
maximum_pxl = float(
pvl.loads(isis.stats(mdr_cub_path).stdout)["Results"]["Maximum"]
)
if maximum_pxl < 1.5:
logger.info("MDR is already in I/F units.")
mdr_16b_p = to_del.add(mdr_cub_path.with_suffix(".16bit.cub"))
isis.bit2bit(
mdr_cub_path,
to=mdr_16b_p,
bit="16bit",
clip="minmax",
minval=0,
maxval=1.5,
)
isis.handmos(mdr_16b_p, mosaic=out_path)
else:
logger.info("MDR is in DN units and will be converted to I/F.")
fpa_t = statistics.mean(
[
float(
isis.getkey_k(
h2i_16b_p, "Instrument", "FpaPositiveYTemperature"
)
),
float(
isis.getkey_k(
h2i_16b_p, "Instrument", "FpaNegativeYTemperature"
)
),
]
)
print(f"fpa_t {fpa_t}")
conf = pvl.load(args.conf)
tdg = t_dep_gain(get_one(conf["Hical"], "Profile", cid.ccdname), fpa_t)
suncorr = solar_correction()
sclk = isis.getkey_k(
h2i_16b_p, "Instrument", "SpacecraftClockStartCount"
)
target = isis.getkey_k(h2i_16b_p, "Instrument", "TargetName")
suncorr = solar_correction(sunDistanceAU(sclk, target))
sed = float(
isis.getkey_k(h2i_16b_p, "Instrument", "LineExposureDuration")
)
zbin = get_one(conf["Hical"], "Profile", "GainUnitConversion")[
"GainUnitConversionBinFactor"
]
# The 'ziof' name is from the ISIS HiCal/GainUnitConversion.h, it is a
# divisor in the calibration equation.
print(f"zbin {zbin}")
print(f"tdg {tdg}")
print(f"sed {sed}")
print(f"suncorr {suncorr}")
ziof = zbin * tdg * sed * 1e-6 * suncorr
eqn = f"\(F1 / {ziof})" # noqa W605
mdriof_p = to_del.add(mdr_cub_path.with_suffix(".iof.cub"))
to_s = "{}+SignedWord+{}:{}".format(mdriof_p, 0, 1.5)
isis.fx(f1=mdr_cub_path, to=to_s, equ=eqn)
isis.handmos(mdriof_p, mosaic=out_path)
if not args.keep:
to_del.unlink()
def unflip(in_p: Path, out_p: Path, keep=False):
"""Attempt to indentify DNs whose bits have been flipped in the
ISIS cube indicated by *in_p*, and unflip them.
Only operates on image-area.
"""
to_del = isis.PathSet()
# This full suite of deltas works well for the reverse-clock area
# and even 'dark' images, but not 'real' images.
# deltas = (8192, 4096, 2048, 1024, 512, 256, 128, 64)
# Restricting the number of deltas might help, but this seems
# arbitrary.
deltas = (8192, 4096, 2048, 1024)
count = 0
suffix = ".bf{}-{}{}.cub"
this_p = to_del.add(in_p.with_suffix(suffix.format(count, 0, 0)))
this_p.symlink_to(in_p)
median = math.trunc(float(isis.stats_k(in_p)["Median"]))
for (sign, pm, extrema) in ((+1, "m", "Maximum"), (-1, "p", "Minimum")):
# logging.info(pm)
for delt in deltas:
d = sign * delt
far = median + d
near = median + (d / 2)
hist = isis.Histogram(this_p)
logger.info(
f"bitflip position {pm}{delt}, near: {near} "
f"far: {far}, extrema: {hist[extrema]}"
)
if (sign > 0 and far < float(hist[extrema])) or (
sign < 0 and far > float(hist[extrema])
):
count += 1
s = suffix.format(count, pm, delt)
next_p = to_del.add(this_p.with_suffix("").with_suffix(s))
try:
thresh = get_unflip_thresh(hist, far, near, d)
except ValueError as err:
logger.info(err)
count -= 1
break
subtract_over_thresh(this_p, next_p, thresh, d, keep=keep)
this_p = next_p
else:
logger.info(
"The far value was beyond the extrema. " "Didn't bother."
)
shutil.move(this_p, out_p)
if not keep:
to_del.remove(this_p)
to_del.unlink()
return
