def test_Analyze_Flat(self):
with patch("hiproc.HiJitReg.open", mock_open(read_data=flat_tab_text)):
with patch(
"hiproc.HiJitReg.pvl.load",
side_effect=[reg_def, cnet_pvl, cnet_pvl],
):
self.assertEqual(hjr.Analyze_Flat(self.j, 1, 1), 0)
with patch(
"hiproc.HiJitReg.pvl.load",
side_effect=[reg_def, cnet_pvl, cnet_pvl],
):
self.j["CanSlither"] = True
self.assertEqual(hjr.Analyze_Flat(self.j, 1, 1), 1)
def test_jitter_iter(self, m_getkey, m_Path, m_hijitreg, m_cnetbin2pvl):
with patch("hiproc.HiJitReg.open", mock_open(read_data=flat_tab_text)):
with patch(
"hiproc.HiJitReg.pvl.load",
side_effect=[reg_def, cnet_pvl, reg_def, cnet_pvl],
):
self.assertFalse(
hjr.jitter_iter(self.r, self.c, self.conf, keep=True))
with patch(
"hiproc.HiJitReg.pvl.load",
side_effect=[reg_def, cnet_pvl, reg_def, cnet_pvl],
):
with patch("hiproc.HiJitReg.Analyze_Flat", return_value=1):
self.assertTrue(
hjr.jitter_iter(self.r, self.c, self.conf, keep=True))
def setUp(self, mock_getkey):
self.r = hci.HiColorCube("dummy/PSP_010502_2090_RED4_0")
self.c = hci.HiColorCube("dummy/PSP_010502_2090_IR10_0")
self.j = hjr.JitterCube(self.c)
self.conf = {
"AutoRegistration": {
"ControlNet": {
"Control_Cols": 4,
"Control_Lines": 200
},
"Algorithm": {
"Tolerance": 0.5,
"Increment": 0.1
},
"PatternChip": {
"Samples": 100,
"Lines": 60
},
"SearchChip": {
"Samples": 120,
"Lines": 100
},
"SearchLongerChip": {
"Samples": 130,
"Lines": 140
},
},
"Smoothing": {
"Exclude_Limit": 2,
"Badness_Limit": 1,
"Boxcar_Length": 10,
},
}
def test_parseFlatTab(self, mock_getkey):
c = hjr.JitterCube("dummy/PSP_010502_2090_IR10_0")
with patch("hiproc.HiJitReg.pvl.load", return_value=reg_def):
c.parseRegDefs()
with patch("hiproc.HiJitReg.open", mock_open(read_data=flat_tab_text)):
c.parseFlatTab()
self.assertEqual(c["MatchedLineCount"], 5)
self.assertEqual(c["AvgSampleOffset"], 0.4470)
def test_parseCNetPVL(self, mock_getkey):
c = hjr.JitterCube("dummy/PSP_010502_2090_IR10_0")
with patch("hiproc.HiJitReg.pvl.load", return_value=reg_def):
c.parseRegDefs()
with patch("hiproc.HiJitReg.open", mock_open(read_data=flat_tab_text)):
c.parseFlatTab()
with patch("hiproc.HiJitReg.pvl.load", return_value=cnet_pvl):
c.parseCNetPVL()
self.assertFalse(c["CanSlither"])
self.assertEqual(c["MatchedLineCount"], 1)
def test_filterCNetPVL(self, mock_getkey):
c = hjr.JitterCube("dummy/PSP_010502_2090_IR10_0")
with patch("hiproc.HiJitReg.pvl.load", return_value=reg_def):
c.parseRegDefs()
with patch("hiproc.HiJitReg.open", mock_open(read_data=flat_tab_text)):
c.parseFlatTab()
c.IgnoredPoints.add("Row 2 Column 0")
with patch("hiproc.HiJitReg.pvl.load", return_value=cnet_pvl):
with patch("hiproc.HiJitReg.open", mock_open()) as m:
c.filterCNetPVL()
self.assertEqual(
str(m().write.call_args[0][0]).count("Ignore"), 2)
def test_HiJitReg(self, m_jit_it, m_getkey):
c04 = Path("dummy/PSP_010502_2090_RED4")
c05 = Path("dummy/PSP_010502_2090_RED5")
c10 = Path("dummy/PSP_010502_2090_IR10")
c11 = Path("dummy/PSP_010502_2090_IR11")
c12 = Path("dummy/PSP_010502_2090_BG12")
c13 = Path("dummy/PSP_010502_2090_BG13")
conf = dict(dummy="yes")
# self.assertRaises(
# RuntimeError, hjr.HiJitReg, [c04, c05, c10, c11, c12, c13], conf
# )
self.assertListEqual([hci.HiColorCube(c12)],
hjr.HiJitReg([c04, c12, c13], conf))
self.assertListEqual(
[
hci.HiColorCube(c10),
hci.HiColorCube(c12),
hci.HiColorCube(c11),
hci.HiColorCube(c13)
],
hjr.HiJitReg([c04, c05, c10, c11, c12, c13], conf),
)
def edr2stitch(
images,
conf_dir,
bitflipwidth=0,
lis_tolerance=1.0,
max_workers=None,
keep=False,
):
chids = list()
edr_conf = pvl.load(conf_dir / "EDR_Stats_gains_config.pvl")
hical_conf = HiCal.conf_setup(
pvl.load(conf_dir / "HiCal.conf"),
pvl.load(conf_dir / "NoiseFilter.conf"),
)
with concurrent.futures.ProcessPoolExecutor(
max_workers=max_workers
) as executor:
future_dbs = dict()
for i in images:
out_edr = util.path_w_suffix(".EDR_Stats.cub", i)
out_hical = util.path_w_suffix(".HiCal.cub", out_edr)
f = executor.submit(
edr_cal,
i,
out_edr,
out_hical,
edr_conf,
hical_conf,
conf_dir / "HiCal.conf",
bitflipwidth=bitflipwidth,
lis_tolerance=lis_tolerance,
keep=False,
)
future_dbs[f] = out_hical
for future in concurrent.futures.as_completed(future_dbs):
out_hical = future_dbs[future]
chids.append(ChannelCube(out_hical, future.result()))
# HiStitch
# get Channel pairs
cids = list()
stitch_conf = pvl.load(conf_dir / "HiStitch.conf")
with concurrent.futures.ProcessPoolExecutor(
max_workers=max_workers
) as executor:
future_tuples = list()
for chid1, chid2 in get_CCDpairs(chids):
f = executor.submit(
# (db, o_path) = HiStitch.HiStitch(
HiStitch.HiStitch,
chid1.nextpath,
chid2.nextpath,
chid1.db,
chid2.db,
".HiStitch.cub",
stitch_conf,
keep=keep,
)
future_tuples.append(f)
for future in concurrent.futures.as_completed(future_tuples):
(db, o_path) = future.result()
cid = HiccdStitch.HiccdStitchCube(o_path)
cid.gather_from_db(db)
cids.append(cid)
# HiccdStitch, makes balance cubes
# need to separate by color:
color_groups = get_color_groups(cids)
for color_group in color_groups.values():
db, out_stitch = HiccdStitch.HiccdStitch(
color_group,
".HiccdStitch.cub",
pvl.load(conf_dir / "HiccdStitch.conf"),
sline=None,
eline=None,
keep=keep,
)
# HiColorInit
# takes *balance.cub
# creates *[IR|BG]*.balance.precolor.cub
# Can then run JitPlot on these *.balance.precolor.cub
HiColorInit.HiColorInit(
[c.nextpath for c in cids], ".precolor.cub", keep=keep
)
# HiJitReg
# takes tmp/*balance.cub tmp/*balance.precolor.cub
# creates *regdef.pvl and *flat.tab files
for_jitreg = list()
for color, balcubes in color_groups.items():
if color == "RED":
for c in balcubes:
for_jitreg.append(c.nextpath)
else:
for c in balcubes:
for_jitreg.append(c.nextpath.with_suffix(".precolor.cub"))
HiJitReg.HiJitReg(
for_jitreg, pvl.load(conf_dir / "HiJitReg.conf"), keep=keep
)
# HiSlither
# takes same as HiJitReg (and assumes its products are available.
# creates *slither.txt, *slither.cub, and *COLOR[4|5].cub
# Can then run SlitherStats on the *slither.txt
HiSlither.HiSlither(for_jitreg)
return chids
def main():
args = arg_parser().parse_args()
util.set_logger(args.verbose, args.logfile, args.log)
conf = pvl.load(args.conf)
with util.main_exceptions(args.verbose):
cubes = list(map(hicolor.HiColorCube, args.cubes))
(red4, red5, ir10, ir11, bg12, bg13) = hicolor.separate_ccds(cubes)
plt.ioff()
fig, axes = plt.subplots(1, 4, sharex=True, sharey=True)
axes[0].set_ylabel("Line")
fig.text(0.5,
0.04,
"Error Magnitude (pixels)",
ha="center",
va="center")
for c, ax in zip([ir10, ir11, bg12, bg13], axes):
if c is None:
continue
logger.info(f"Working on {c}")
j = hjr.JitterCube(c, conf)
j.reset()
(accepted, rejected, smoothed) = filter_and_smooth(j)
ax.set_title(j.get_ccd())
size = 5
if len(accepted) > 0:
acceptA = np.array(accepted)
ax.scatter(
acceptA[:, 1],
acceptA[:, 0],
s=size,
label="Accepted",
facecolors="none",
edgecolors="black",
)
if len(rejected) > 0:
rejectedA = np.array(rejected)
ax.scatter(
rejectedA[:, 1],
rejectedA[:, 0],
c="red",
s=size,
label="Rejected",
)
if len(smoothed) > 0:
smoothedA = np.array(smoothed)
ax.scatter(
smoothedA[:, 1],
smoothedA[:, 0],
c="blue",
s=size,
label="Smoothed",
)
fig.suptitle(str(j.get_obsid()))
bottom, top = plt.ylim()
plt.ylim(top, bottom)
plt.xlim(0, 10)
plt.legend(loc="upper right", bbox_to_anchor=(1, -0.05), ncol=3)
plt.show()
return
def test_run_HiJitReg(self, m_Path, m_hijitreg, m_cnetbin2pvl):
hjr.run_HiJitReg(self.r.path, self.j, jitter_params, "foo", keep=True)
m_hijitreg.assert_called_once()
m_cnetbin2pvl.assert_called_once()
def test_write_regdef(self, m_Path):
hjr.write_regdef("dummy/out.pvl", jitter_params)
self.assertEqual(len(m_Path.mock_calls), 2)
def test_dict(self, mock_getkey):
c = hjr.JitterCube("dummy/PSP_010502_2090_IR10_0")
self.assertEqual(c["bin"], c.bin)
self.assertEqual(c["ExcludeLimit"], 2)
def test_parseCNetPVL_file(self):
c = hjr.JitterCube(
"tmp/PSP_010502_2090_IR10.HiStitch.balance.precolor.cub")
c.parseRegDefs()
c.parseFlatTab()
c.parseCNetPVL("tmp/test.pvl")
def test_parseRegDefs(self, mock_getkey):
c = hjr.JitterCube("dummy/PSP_010502_2090_IR10_0")
with patch("hiproc.HiJitReg.pvl.load", return_value=reg_def):
c.parseRegDefs()
self.assertEqual(c["PatternSamples"], 100)
def edr2stitch(images, conf_dir, bitflipwidth=0, lis_tolerance=1, keep=False):
chids = list()
for i in images:
out_edr = util.path_w_suffix(".EDR_Stats.cub", i)
# EDR_Stats
db = EDR_Stats.EDR_Stats(i,
out_edr,
pvl.load(conf_dir /
"EDR_Stats_gains_config.pvl"),
keep=keep)
# HiCal
out_hical = util.path_w_suffix(".HiCal.cub", out_edr)
db = HiCal.HiCal(
out_edr,
out_hical,
db,
HiCal.conf_setup(pvl.load(conf_dir / "HiCal.conf"),
pvl.load(conf_dir / "NoiseFilter.conf")),
conf_dir / "HiCal.conf",
None,
None,
bitflipwidth,
lis_tolerance,
keep=keep,
)
chids.append(ChannelCube(out_hical, db))
# HiStitch
# get Channel pairs
cids = list()
for chid1, chid2 in get_CCDpairs(chids):
(db, o_path) = HiStitch.HiStitch(
chid1.nextpath,
chid2.nextpath,
chid1.db,
chid2.db,
".HiStitch.cub",
pvl.load(conf_dir / "HiStitch.conf"),
keep=keep,
)
cid = HiccdStitch.HiccdStitchCube(o_path)
cid.gather_from_db(db)
cids.append(cid)
# HiccdStitch, makes balance cubes
# need to separate by color:
color_groups = get_color_groups(cids)
for color_group in color_groups.values():
db, out_stitch = HiccdStitch.HiccdStitch(
color_group,
".HiccdStitch.cub",
pvl.load(conf_dir / "HiccdStitch.conf"),
sline=None,
eline=None,
keep=keep,
)
# HiColorInit
# takes *balance.cub
# creates *[IR|BG]*.balance.precolor.cub
# Can then run JitPlot on these *.balance.precolor.cub
HiColorInit.HiColorInit([c.nextpath for c in cids],
".precolor.cub",
keep=keep)
# HiJitReg
# takes tmp/*balance.cub tmp/*balance.precolor.cub
# creates *regdef.pvl and *flat.tab files
for_jitreg = list()
for color, balcubes in color_groups.items():
if color == "RED":
for c in balcubes:
for_jitreg.append(c.nextpath)
else:
for c in balcubes:
for_jitreg.append(c.nextpath.with_suffix(".precolor.cub"))
HiJitReg.HiJitReg(for_jitreg,
pvl.load(conf_dir / "HiJitReg.conf"),
keep=keep)
# HiSlither
# takes same as HiJitReg (and assumes its products are available.
# creates *slither.txt, *slither.cub, and *COLOR[4|5].cub
# Can then run SliterStats on the *slither.txt
HiSlither.HiSlither(for_jitreg)
return chids
def make_flats(cubes, common_cube, conf, temp_token, keep=False):
# If the flat files already exist, don't remake them.
# "$OBS_ID"."_".$ccd."-".$common.".flat.tab"
jitter_cubes = list()
n_row = int(
common_cube.lines
/ conf["AutoRegistration"]["ControlNet"]["Control_Lines"]
)
for c in cubes:
if c == common_cube:
continue
jitter_cubes.append(
hjr.JitterCube(
c.next_path, matchccd=common_cube.get_ccd(), config=conf
)
)
successful_flats = list()
if not all(x.flattab_path.exists() for x in jitter_cubes):
confauto = conf["AutoRegistration"]
for c in jitter_cubes:
params = {
"ROWS": n_row,
"TOLERANCE": confauto["Algorithm"]["Tolerance"],
}
min_fraction_good = confauto["AnaylyzeFlat"]["Minimum_Good"]
if c.ccdname == "RED":
redcolor = "Red"
else:
redcolor = "Color"
params["GROUP"] = "ResolveJitter"
params["COLS"] = confauto["ControlNet"]["Control_Cols_" + redcolor]
params["PATTERN_SAMPLES"] = confauto["PatternChip" + redcolor][
"Samples"
]
params["PATTERN_LINES"] = confauto["PatternChip" + redcolor][
"Lines"
]
params["SEARCH_SAMPLES"] = confauto["SearchChip" + redcolor][
"Samples"
]
params["SEARCH_LINES"] = confauto["SearchChip" + redcolor]["Lines"]
if c.ccdname != "RED":
channels = isis.getkey_k(
c.path, "Instrument", "StitchedProductIds"
)
if len(channels) < 2:
logger.info(
f"Increasing columns because {c.path} is "
"missing a channel."
)
params["COLS"] += 1
min_fraction_good *= 0.5
logger.info(
"The minimum allowable Fraction Good "
f"Matches = {min_fraction_good}"
)
step = 0
while step <= confauto["Algorithm"]["Steps"]:
logger.info(f"Step {step} begin")
hjr.run_HiJitReg(
common_cube.next_path, c, params, temp_token, keep=keep
)
ret = hjr.Analyze_Flat(
c, 0, (min_fraction_good * 2), hijitreg=False
)
if ret == 1:
successful_flats.append(c.flattab_path)
break
else:
step += 1
c.regdef_path.unlink()
c.flattab_path.unlink()
c.cnet_path.unlink()
params["TOLERANCE"] -= (
confauto["Algorithm"]["Increment"] * step
)
else:
raise RuntimeError(
f"Flat file for {c} is not within tolerances."
)
else:
successful_flats = list(x.flattab_path for x in jitter_cubes)
return successful_flats
def test_init(self, mock_getkey):
c = hci.HiColorCube("dummy/PSP_010502_2090_IR10_0")
j = hjr.JitterCube(c)
self.assertEqual(j.cnet_path,
Path("dummy/PSP_010502_2090_RED4-IR10.control.pvl"))
