def test_lpfz_triplefilter(self, m_low):
hcn.lpfz_triplefilter("from.cub", "to.cub", keep=True)
self.assertListEqual(
m_low.call_args_list,
[
call(
Path("from.cub"),
HIS=True,
HRS=True,
LIS=True,
LRS=True,
filter="OUTSIDE",
high=2.0,
lines=11,
low=0.0,
minimum=25,
minopt="PERCENTAGE",
null=True,
samples=5,
to=Path("from.z1.cub"),
),
call(
Path("from.z1.cub"),
HIS=True,
HRS=True,
LIS=True,
LRS=True,
filter="OUTSIDE",
high=2.0,
lines=21,
low=0.0,
minimum=25,
minopt="PERCENTAGE",
null=True,
samples=9,
to=Path("from.z2.cub"),
),
call(
Path("from.z2.cub"),
HIS=True,
HRS=True,
LIS=True,
LRS=True,
filter="OUTSIDE",
high=2.0,
lines=41,
low=0.0,
minimum=25,
minopt="PERCENTAGE",
null=True,
samples=11,
to="to.cub",
),
],
)
def test_FurrowCheck(self, m_open):
self.assertFalse(hcn.FurrowCheck(self.cubes, None))
with patch("hiproc.HiColorNorm.json.load",
return_value={"zapped": True}):
self.assertTrue(
hcn.FurrowCheck(self.cubes,
["dummy_db_path1", "dummy_db_path2"]))
with patch("hiproc.HiColorNorm.json.load",
return_value={"zapped": False}):
self.assertFalse(
hcn.FurrowCheck(self.cubes,
["dummy_db_path1", "dummy_db_path2"]))
def test_HiColorNorm(self, m_getkey, m_mask, m_Pathwrite, m_cubeit, m_pc,
m_pb, m_hcs, m_gb):
conf = {
"HiColorNorm": {
"HiColorNorm_Crop_Top": 1000,
"HiColorNorm_Crop_Bot": 2000,
"HiColorNorm_Make_Stitch": True,
"HiColorNorm_NoiseFilter_IR10": True,
"HiColorNorm_Normalization_Minimum": 0,
"HiColorNorm_Normalization_Maximum": 16000,
}
}
self.assertEqual(
(3, 3),
hcn.HiColorNorm(
[
Path("dummy/PSP_010502_2090_COLOR4"),
Path("dummy/PSP_010502_2090_COLOR5")
],
"out_path.cub",
conf,
make_unfiltered=True,
keep=True,
),
)
self.assertEqual(6, m_mask.call_count)
self.assertEqual(2, m_cubeit.call_count)
self.assertEqual(4, m_pc.call_count)
self.assertEqual(2, m_pb.call_count)
def test_make_unfiltered(self, m_copy, m_alg, m_hand):
self.assertEqual(
Path("in_UNFILTERED_COLOR4.cub"),
hcn.make_unfiltered("in_COLOR4.cub",
"nrm.cub",
"ttoken",
"CC",
2,
keep=True),
)
self.assertEqual(
m_alg.call_args_list,
[
call(
"nrm.cub",
from2="in_COLOR4.cub+2",
operator="MULTIPLY",
to=Path("in_COLOR4.ttoken_CC.algebra.cub"),
)
],
)
self.assertEqual(
m_hand.call_args_list,
[
call(
Path("in_COLOR4.ttoken_CC.algebra.cub"),
matchbandbin=False,
mosaic=Path("in_UNFILTERED_COLOR4.cub"),
outband=2,
outline=1,
outsample=1,
)
],
)
def test_cubenorm_stats(self, m_cubenorm, m_open, m_reader):
self.assertAlmostEqual(
0.70710678,
hcn.cubenorm_stats("crpmos.cub",
"mos.cub",
"mosnrm.cub",
keep=True),
)
expected = [
call(
"crpmos.cub",
direction="column",
format="table",
norm="average",
stats=Path("crpmos.cubenorm.txt"),
),
call(
"mos.cub",
direction="column",
format="table",
fromstats=Path("crpmos.cubenorm.txt"),
norm="average",
preserve=True,
statsource="table",
to="mosnrm.cub",
),
]
self.assertListEqual(m_cubenorm.call_args_list, expected)
def test_set_outpath(self):
self.assertEqual(Path("unchanged"),
hcn.set_outpath("unchanged", self.cubes))
self.assertEqual(
Path("dummy/PSP_010502_2090_COLOR.cub"),
hcn.set_outpath("_COLOR.cub", self.cubes),
)
t_getkey = getkey
with patch("hiproc.HiColorNorm.ColorCube.get_binning", return_value=2):
with patch("hiproc.HiColorNorm.isis.getkey_k",
side_effect=t_getkey):
with patch(
"hiproc.hirise.get_ObsID_fromfile",
return_value=hirise.ObservationID("ESP_060680_3180"),
):
oddball = hcn.ColorCube("dummy/ESP_060680_3180_COLOR4")
newlist = self.cubes + [oddball]
self.assertRaises(ValueError, hcn.set_outpath, "unchanged",
newlist)
def test_lpfz_filtering(self, m_low):
hcn.lpfz_filtering("from.cub", "to.cub", 3, 5)
self.assertListEqual(
m_low.call_args_list,
[
call(
"from.cub",
HIS=True,
HRS=True,
LIS=True,
LRS=True,
filter="OUTSIDE",
high=2.0,
lines=3,
low=0.0,
minimum=25,
minopt="PERCENTAGE",
null=True,
samples=5,
to="to.cub",
)
],
)
def test_make_LR_mosaic(self, m_handmos):
hcn.make_LR_mosaic("left.cub", "right.cub", 10, "mosaic.cub", 100, 20)
expected = [
call(
"left.cub",
create="YES",
mosaic="mosaic.cub",
nbands=1,
nlines=100,
nsamp=20,
outband=1,
outline=1,
outsamp=1,
),
call(
"right.cub",
mosaic="mosaic.cub",
outband=1,
outline=1,
outsamp=11,
),
]
self.assertListEqual(m_handmos.call_args_list, expected)
def test_per_band(
self,
m_LR,
m_cn,
m_crop,
m_unfilt,
m_low,
m_3filt,
m_algebra,
m_handmos,
):
conf = {
"HiColorNorm": {
"HiColorNorm_Crop_Top": 1000,
"HiColorNorm_Crop_Bot": 2000,
}
}
for c in self.cubes:
c.set_crop_lines(conf)
self.assertEqual(
3,
hcn.per_band(
self.cubes,
Path("out_path.cub"),
"ttoken",
"IR",
True,
True,
keep=True,
),
)
num_cubes = len(self.cubes)
m_cn.assert_called_once()
self.assertEqual(num_cubes, m_LR.call_count)
self.assertEqual(num_cubes, m_crop.call_count)
self.assertEqual(num_cubes, m_unfilt.call_count)
self.assertEqual(num_cubes, m_low.call_count)
self.assertEqual(num_cubes, m_3filt.call_count)
self.assertEqual(num_cubes, m_algebra.call_count)
self.assertEqual(num_cubes, m_handmos.call_count)
def color(obsid, conf_dir: Path, parent: Path, db_list: list, keep=False):
colors = get_cubes(f"{obsid}_COLOR*.cub", parent)
# HiColorNorm - only for color
# takes *COLOR[4|5].cub
# creates *UNFILTERED_COLOR[4|5].cub and *COLOR[4|5].HiColorNorm.cub
HiColorNorm.HiColorNorm(colors,
"_COLOR.cub",
pvl.load(conf_dir / "HiColorNorm.conf"),
db_list=db_list,
keep=keep)
# HiBeautify - only for color
# takes tmp/*.HiColorNorm.cub
# creates *IRB.cub and *RGB.cub
for x in colors:
x.with_suffix(".HiColorNorm.cub")
HiBeautify.start(
[x.with_suffix(".HiColorNorm.cub") for x in colors],
pvl.load(conf_dir / "HiBeautify.conf"),
)
return
def test_per_color(self, m_rat, m_mask, m_crop):
mask_p = Path("dummy/PSP_010502_2090_COLOR4.ttoken_IR.mask.cub")
ratc_p = Path("dummy/PSP_010502_2090_COLOR4.ttoken_IR.ratcrop.cub")
rati_p = Path("dummy/PSP_010502_2090_COLOR4.ttoken_IR.ratio.cub")
self.assertTupleEqual(
(mask_p, ratc_p),
hcn.per_color(self.cubes[0], "ttoken", "IR", keep=True),
)
self.assertEqual(
m_rat.call_args_list,
[
call(
den="dummy/PSP_010502_2090_COLOR4+2",
num="dummy/PSP_010502_2090_COLOR4+1",
to=rati_p,
)
],
)
self.assertEqual(
m_mask.call_args_list,
[
call(
rati_p,
mask=rati_p,
maximum=4.0,
minimum=0.0,
preserve="INSIDE",
to=mask_p,
)
],
)
self.assertEqual(
m_crop.call_args_list,
[call(mask_p, line=None, nlines=None, to=ratc_p)],
)
def HiNoProj(cubes: list,
conf: dict,
output="_RED.NOPROJ.cub",
base=5,
keep=False):
logger.info(f"HiNoProj start: {', '.join(map(str, cubes))}")
cubes = list(map(Cube, cubes))
cubes.sort()
if not all(c.ccdname == "RED" for c in cubes):
raise ValueError("Not all of the input files are RED CCD files.")
sequences = list()
for k, g in itertools.groupby(
(int(c.ccdnumber) for c in cubes),
lambda x, c=itertools.count(): next(c) - x,
):
sequences.append(list(g))
if len(sequences) != 1:
raise ValueError("The given cubes are not a single run of sequential "
"HiRISE CCDs, instead there are "
f"{len(sequences)} groups with these "
f"CCD numbers: {sequences}.")
if not isinstance(base, int):
base = hirise.get_CCDID_fromfile(base)
base_ccd = list(filter(lambda x: x.ccdnumber == str(base), cubes))
if len(base_ccd) != 1:
raise ValueError(f"The base ccd, number {base}, "
"is not one of the given cubes.")
base_cube = base_ccd[0]
conf_check(conf["HiNoProj"])
conf = conf["HiNoProj"]
out_p = hcn.set_outpath(output, cubes)
temp_token = datetime.now().strftime("HiNoProj-%y%m%d%H%M%S")
to_del = isis.PathSet()
polar = False
if conf["Shape"] == "USER":
polar = is_polar(cubes, conf["Pole_Tolerance"], temp_token)
for c in cubes:
temp_p = to_del.add(c.path.with_suffix(f".{temp_token}.spiced.cub"))
copy_and_spice(c.path, temp_p, conf, polar)
isis.spicefit(temp_p)
c.next_path = to_del.add(
c.path.with_suffix(f".{temp_token}.noproj.cub"))
c.path = temp_p
for c in cubes:
isis.noproj(c.path,
match=base_cube.path,
to=c.next_path,
source="frommatch")
# Run hijitreg on adjacent noproj'ed ccds to get average line/sample offset
(cubes, _) = add_offsets(cubes,
int(base_cube.ccdnumber),
temp_token,
keep=keep)
# Mosaic noproj'ed ccds using average line/sample offset
shutil.copyfile(base_cube.next_path, out_p)
logger.info("Original Perl hard codes this file copy from RED5, even if "
"another cube is selected as the base_ccd.")
handmos_side(cubes, base_cube, out_p, left=True)
handmos_side(cubes, base_cube, out_p, left=False)
isis.editlab(
out_p,
option="addkey",
grpname="Instrument",
keyword="ImageJitterCorrected",
value=0,
)
fix_labels(
cubes,
out_p,
base_cube,
"{}_{}".format(str(cubes[0].get_obsid()), cubes[0].ccdname),
)
if not keep:
to_del.unlink()
logger.info(f"HiNoProj done: {out_p}")
return
def HiBeautify(cube_paths: list,
conf: dict,
out_irb="_IRB.cub",
out_rgb="_RGB.cub",
keep=False):
logger.info(f"HiBeautify start: {', '.join(map(str, cube_paths))}")
# GetConfigurationParameters()
cubes = list(map(hcn.ColorCube, cube_paths))
cubes.sort()
irb_out_p = hcn.set_outpath(out_irb, cubes)
rgb_out_p = hcn.set_outpath(out_rgb, cubes)
temp_token = datetime.now().strftime("HiBeautify-%y%m%d%H%M%S")
to_del = isis.PathSet()
# Create an IRB mosaic from the HiColorNorm halves.
# If a half is missing, we create a mosaic with the proper width and place
# the half in it at the proper location.
# Logic ofr image_midpoint and total_width come from original HiColorInit
# irbmerged_p = to_del.add(out_p.with_suffix(f'.{temp_token}_IRB.cub'))
image_midpoint = int((2000 / cubes[0].red_bin) + 1)
outsample = image_midpoint
if cubes[0].ccdnumber == "4":
outsample = 1
total_width = int(2 * cubes[0].samps - (48 / cubes[0].red_bin))
isis.handmos(
cubes[0].path,
mosaic=irb_out_p,
outline=1,
outsample=outsample,
outband=1,
create="Y",
nlines=cubes[0].lines,
nsamp=total_width,
nbands=3,
)
if len(cubes) == 1:
logger.info("Warning, missing one half!")
else:
logger.info("Using both halves")
isis.handmos(
cubes[1].path,
mosaic=irb_out_p,
outline=1,
outsample=image_midpoint,
outband=1,
)
# Nothing is actually done to the pixels here regarding the FrostStats, so
# I'm just going to skip them here.
#
# # Determine if Frost/ICE may be present using FrostStats module.
# frost = None
# if args.frost:
# frost = True
# logging.info('Frost override: disabling auto-detection and using '
# 'the frost/ice color stretch')
# if args.nofrost:
# frost = False
# logging.info('Frost override: disable auto-detection and not using '
# 'the frost/ice color stretch')
# if frost is None:
# pass
# # get frost stats
# Subtract the unaltered RED band from the high pass filtered BG for
# synthetic blue.
logger.info("Creating synthetic B, subtracting RED from BG")
rgbsynthb_p = to_del.add(irb_out_p.with_suffix(f".{temp_token}_B.cub"))
isis.algebra(
f"{irb_out_p}+3",
from2=f"{irb_out_p}+2",
op="subtract",
to=rgbsynthb_p,
A=conf["Beautify"]["Synthetic_A_Coefficient"],
B=conf["Beautify"]["Synthetic_B_Coefficient"],
)
# Adjust the BandBin group
isis.editlab(rgbsynthb_p,
grpname="BandBin",
keyword="Name",
value="Synthetic Blue")
isis.editlab(rgbsynthb_p, grpname="BandBin", keyword="Center", value="0")
isis.editlab(rgbsynthb_p, grpname="BandBin", keyword="Width", value="0")
# HiBeautify gathers and writes a bunch of statistics to PVL that is
# important to the HiRISE GDS, but not relevant to just producing pixels
# so I'm omitting it.
#
# # Determine the min and max DN values of each band (RED, IR, BG, B) we're
# # working with.
# (upper, lower) = conf['Beautify']['Stretch_Exclude_Lines']
# if upper == 0 and lower == 0:
# synthbcrp_p = rgbsynthb_p
# irbmrgcrp_p = irbmerged_p
# else:
# synthbcrp_p = to_del.add(
# out_p.with_suffix(f'.{temp_token}_Bx.cub'))
# irbmrgcrp_p = to_del.add(
# out_p.with_suffix(f'.{temp_token}_IRBx.cub'))
#
# for (f, t) in ((rgbsynthb_p, synthbcrp_p),
# (irbmerged_p, irbmrgcrp_p)):
# logging.info(isis.crop(f, to=t, propspice=False,
# line=(1 + upper),
# nlines=(
# cubes[0].lines - lower + upper)).args)
#
# stats = dict()
# stats['B'] = Get_MinMax(synthbcrp_p,
# conf['Beautify']['Stretch_Reduction_Factor'],
# temp_token, keep=keep)
#
# for band in cubes[0].band.keys():
# stats[band] = Get_MinMax('{}+{}'.format(str(irbmrgcrp_p),
# cubes[0].band[band]),
# conf['Beautify']['Stretch_Reduction_Factor'],
# temp_token, keep=keep)
# Create an RGB cube using the RED from the IRB mosaic,
# the BG from the IRB mosaic and the synthetic B that we just made.
isis.cubeit_k([f"{irb_out_p}+2", f"{irb_out_p}+3", rgbsynthb_p],
to=rgb_out_p)
if not keep:
to_del.unlink()
return
def test_conf_check(self):
conf_path = Path("data") / "HiColorNorm.conf"
c = pvl.load(str(conf_path))
self.assertIsNone(hcn.conf_check(c))
def setUp(self, m_getkey, m_get_binning):
c4 = hcn.ColorCube("dummy/PSP_010502_2090_COLOR4")
c5 = hcn.ColorCube("dummy/PSP_010502_2090_COLOR5")
self.cubes = [c4, c5]
def setUp(self, m_getkey, m_get_binning):
self.c = hcn.ColorCube("dummy/PSP_010502_2090_COLOR5")