def test_Analyze_Flat(self): with patch("hiproc.HiJitReg.open", mock_open(read_data=flat_tab_text)): with patch( "hiproc.HiJitReg.pvl.load", side_effect=[reg_def, cnet_pvl, cnet_pvl], ): self.assertEqual(hjr.Analyze_Flat(self.j, 1, 1), 0) with patch( "hiproc.HiJitReg.pvl.load", side_effect=[reg_def, cnet_pvl, cnet_pvl], ): self.j["CanSlither"] = True self.assertEqual(hjr.Analyze_Flat(self.j, 1, 1), 1)
def test_jitter_iter(self, m_getkey, m_Path, m_hijitreg, m_cnetbin2pvl): with patch("hiproc.HiJitReg.open", mock_open(read_data=flat_tab_text)): with patch( "hiproc.HiJitReg.pvl.load", side_effect=[reg_def, cnet_pvl, reg_def, cnet_pvl], ): self.assertFalse( hjr.jitter_iter(self.r, self.c, self.conf, keep=True)) with patch( "hiproc.HiJitReg.pvl.load", side_effect=[reg_def, cnet_pvl, reg_def, cnet_pvl], ): with patch("hiproc.HiJitReg.Analyze_Flat", return_value=1): self.assertTrue( hjr.jitter_iter(self.r, self.c, self.conf, keep=True))
def setUp(self, mock_getkey): self.r = hci.HiColorCube("dummy/PSP_010502_2090_RED4_0") self.c = hci.HiColorCube("dummy/PSP_010502_2090_IR10_0") self.j = hjr.JitterCube(self.c) self.conf = { "AutoRegistration": { "ControlNet": { "Control_Cols": 4, "Control_Lines": 200 }, "Algorithm": { "Tolerance": 0.5, "Increment": 0.1 }, "PatternChip": { "Samples": 100, "Lines": 60 }, "SearchChip": { "Samples": 120, "Lines": 100 }, "SearchLongerChip": { "Samples": 130, "Lines": 140 }, }, "Smoothing": { "Exclude_Limit": 2, "Badness_Limit": 1, "Boxcar_Length": 10, }, }
def test_parseFlatTab(self, mock_getkey): c = hjr.JitterCube("dummy/PSP_010502_2090_IR10_0") with patch("hiproc.HiJitReg.pvl.load", return_value=reg_def): c.parseRegDefs() with patch("hiproc.HiJitReg.open", mock_open(read_data=flat_tab_text)): c.parseFlatTab() self.assertEqual(c["MatchedLineCount"], 5) self.assertEqual(c["AvgSampleOffset"], 0.4470)
def test_parseCNetPVL(self, mock_getkey): c = hjr.JitterCube("dummy/PSP_010502_2090_IR10_0") with patch("hiproc.HiJitReg.pvl.load", return_value=reg_def): c.parseRegDefs() with patch("hiproc.HiJitReg.open", mock_open(read_data=flat_tab_text)): c.parseFlatTab() with patch("hiproc.HiJitReg.pvl.load", return_value=cnet_pvl): c.parseCNetPVL() self.assertFalse(c["CanSlither"]) self.assertEqual(c["MatchedLineCount"], 1)
def test_filterCNetPVL(self, mock_getkey): c = hjr.JitterCube("dummy/PSP_010502_2090_IR10_0") with patch("hiproc.HiJitReg.pvl.load", return_value=reg_def): c.parseRegDefs() with patch("hiproc.HiJitReg.open", mock_open(read_data=flat_tab_text)): c.parseFlatTab() c.IgnoredPoints.add("Row 2 Column 0") with patch("hiproc.HiJitReg.pvl.load", return_value=cnet_pvl): with patch("hiproc.HiJitReg.open", mock_open()) as m: c.filterCNetPVL() self.assertEqual( str(m().write.call_args[0][0]).count("Ignore"), 2)
def test_HiJitReg(self, m_jit_it, m_getkey): c04 = Path("dummy/PSP_010502_2090_RED4") c05 = Path("dummy/PSP_010502_2090_RED5") c10 = Path("dummy/PSP_010502_2090_IR10") c11 = Path("dummy/PSP_010502_2090_IR11") c12 = Path("dummy/PSP_010502_2090_BG12") c13 = Path("dummy/PSP_010502_2090_BG13") conf = dict(dummy="yes") # self.assertRaises( # RuntimeError, hjr.HiJitReg, [c04, c05, c10, c11, c12, c13], conf # ) self.assertListEqual([hci.HiColorCube(c12)], hjr.HiJitReg([c04, c12, c13], conf)) self.assertListEqual( [ hci.HiColorCube(c10), hci.HiColorCube(c12), hci.HiColorCube(c11), hci.HiColorCube(c13) ], hjr.HiJitReg([c04, c05, c10, c11, c12, c13], conf), )
def edr2stitch( images, conf_dir, bitflipwidth=0, lis_tolerance=1.0, max_workers=None, keep=False, ): chids = list() edr_conf = pvl.load(conf_dir / "EDR_Stats_gains_config.pvl") hical_conf = HiCal.conf_setup( pvl.load(conf_dir / "HiCal.conf"), pvl.load(conf_dir / "NoiseFilter.conf"), ) with concurrent.futures.ProcessPoolExecutor( max_workers=max_workers ) as executor: future_dbs = dict() for i in images: out_edr = util.path_w_suffix(".EDR_Stats.cub", i) out_hical = util.path_w_suffix(".HiCal.cub", out_edr) f = executor.submit( edr_cal, i, out_edr, out_hical, edr_conf, hical_conf, conf_dir / "HiCal.conf", bitflipwidth=bitflipwidth, lis_tolerance=lis_tolerance, keep=False, ) future_dbs[f] = out_hical for future in concurrent.futures.as_completed(future_dbs): out_hical = future_dbs[future] chids.append(ChannelCube(out_hical, future.result())) # HiStitch # get Channel pairs cids = list() stitch_conf = pvl.load(conf_dir / "HiStitch.conf") with concurrent.futures.ProcessPoolExecutor( max_workers=max_workers ) as executor: future_tuples = list() for chid1, chid2 in get_CCDpairs(chids): f = executor.submit( # (db, o_path) = HiStitch.HiStitch( HiStitch.HiStitch, chid1.nextpath, chid2.nextpath, chid1.db, chid2.db, ".HiStitch.cub", stitch_conf, keep=keep, ) future_tuples.append(f) for future in concurrent.futures.as_completed(future_tuples): (db, o_path) = future.result() cid = HiccdStitch.HiccdStitchCube(o_path) cid.gather_from_db(db) cids.append(cid) # HiccdStitch, makes balance cubes # need to separate by color: color_groups = get_color_groups(cids) for color_group in color_groups.values(): db, out_stitch = HiccdStitch.HiccdStitch( color_group, ".HiccdStitch.cub", pvl.load(conf_dir / "HiccdStitch.conf"), sline=None, eline=None, keep=keep, ) # HiColorInit # takes *balance.cub # creates *[IR|BG]*.balance.precolor.cub # Can then run JitPlot on these *.balance.precolor.cub HiColorInit.HiColorInit( [c.nextpath for c in cids], ".precolor.cub", keep=keep ) # HiJitReg # takes tmp/*balance.cub tmp/*balance.precolor.cub # creates *regdef.pvl and *flat.tab files for_jitreg = list() for color, balcubes in color_groups.items(): if color == "RED": for c in balcubes: for_jitreg.append(c.nextpath) else: for c in balcubes: for_jitreg.append(c.nextpath.with_suffix(".precolor.cub")) HiJitReg.HiJitReg( for_jitreg, pvl.load(conf_dir / "HiJitReg.conf"), keep=keep ) # HiSlither # takes same as HiJitReg (and assumes its products are available. # creates *slither.txt, *slither.cub, and *COLOR[4|5].cub # Can then run SlitherStats on the *slither.txt HiSlither.HiSlither(for_jitreg) return chids
def main(): args = arg_parser().parse_args() util.set_logger(args.verbose, args.logfile, args.log) conf = pvl.load(args.conf) with util.main_exceptions(args.verbose): cubes = list(map(hicolor.HiColorCube, args.cubes)) (red4, red5, ir10, ir11, bg12, bg13) = hicolor.separate_ccds(cubes) plt.ioff() fig, axes = plt.subplots(1, 4, sharex=True, sharey=True) axes[0].set_ylabel("Line") fig.text(0.5, 0.04, "Error Magnitude (pixels)", ha="center", va="center") for c, ax in zip([ir10, ir11, bg12, bg13], axes): if c is None: continue logger.info(f"Working on {c}") j = hjr.JitterCube(c, conf) j.reset() (accepted, rejected, smoothed) = filter_and_smooth(j) ax.set_title(j.get_ccd()) size = 5 if len(accepted) > 0: acceptA = np.array(accepted) ax.scatter( acceptA[:, 1], acceptA[:, 0], s=size, label="Accepted", facecolors="none", edgecolors="black", ) if len(rejected) > 0: rejectedA = np.array(rejected) ax.scatter( rejectedA[:, 1], rejectedA[:, 0], c="red", s=size, label="Rejected", ) if len(smoothed) > 0: smoothedA = np.array(smoothed) ax.scatter( smoothedA[:, 1], smoothedA[:, 0], c="blue", s=size, label="Smoothed", ) fig.suptitle(str(j.get_obsid())) bottom, top = plt.ylim() plt.ylim(top, bottom) plt.xlim(0, 10) plt.legend(loc="upper right", bbox_to_anchor=(1, -0.05), ncol=3) plt.show() return
def test_run_HiJitReg(self, m_Path, m_hijitreg, m_cnetbin2pvl): hjr.run_HiJitReg(self.r.path, self.j, jitter_params, "foo", keep=True) m_hijitreg.assert_called_once() m_cnetbin2pvl.assert_called_once()
def test_write_regdef(self, m_Path): hjr.write_regdef("dummy/out.pvl", jitter_params) self.assertEqual(len(m_Path.mock_calls), 2)
def test_dict(self, mock_getkey): c = hjr.JitterCube("dummy/PSP_010502_2090_IR10_0") self.assertEqual(c["bin"], c.bin) self.assertEqual(c["ExcludeLimit"], 2)
def test_parseCNetPVL_file(self): c = hjr.JitterCube( "tmp/PSP_010502_2090_IR10.HiStitch.balance.precolor.cub") c.parseRegDefs() c.parseFlatTab() c.parseCNetPVL("tmp/test.pvl")
def test_parseRegDefs(self, mock_getkey): c = hjr.JitterCube("dummy/PSP_010502_2090_IR10_0") with patch("hiproc.HiJitReg.pvl.load", return_value=reg_def): c.parseRegDefs() self.assertEqual(c["PatternSamples"], 100)
def edr2stitch(images, conf_dir, bitflipwidth=0, lis_tolerance=1, keep=False): chids = list() for i in images: out_edr = util.path_w_suffix(".EDR_Stats.cub", i) # EDR_Stats db = EDR_Stats.EDR_Stats(i, out_edr, pvl.load(conf_dir / "EDR_Stats_gains_config.pvl"), keep=keep) # HiCal out_hical = util.path_w_suffix(".HiCal.cub", out_edr) db = HiCal.HiCal( out_edr, out_hical, db, HiCal.conf_setup(pvl.load(conf_dir / "HiCal.conf"), pvl.load(conf_dir / "NoiseFilter.conf")), conf_dir / "HiCal.conf", None, None, bitflipwidth, lis_tolerance, keep=keep, ) chids.append(ChannelCube(out_hical, db)) # HiStitch # get Channel pairs cids = list() for chid1, chid2 in get_CCDpairs(chids): (db, o_path) = HiStitch.HiStitch( chid1.nextpath, chid2.nextpath, chid1.db, chid2.db, ".HiStitch.cub", pvl.load(conf_dir / "HiStitch.conf"), keep=keep, ) cid = HiccdStitch.HiccdStitchCube(o_path) cid.gather_from_db(db) cids.append(cid) # HiccdStitch, makes balance cubes # need to separate by color: color_groups = get_color_groups(cids) for color_group in color_groups.values(): db, out_stitch = HiccdStitch.HiccdStitch( color_group, ".HiccdStitch.cub", pvl.load(conf_dir / "HiccdStitch.conf"), sline=None, eline=None, keep=keep, ) # HiColorInit # takes *balance.cub # creates *[IR|BG]*.balance.precolor.cub # Can then run JitPlot on these *.balance.precolor.cub HiColorInit.HiColorInit([c.nextpath for c in cids], ".precolor.cub", keep=keep) # HiJitReg # takes tmp/*balance.cub tmp/*balance.precolor.cub # creates *regdef.pvl and *flat.tab files for_jitreg = list() for color, balcubes in color_groups.items(): if color == "RED": for c in balcubes: for_jitreg.append(c.nextpath) else: for c in balcubes: for_jitreg.append(c.nextpath.with_suffix(".precolor.cub")) HiJitReg.HiJitReg(for_jitreg, pvl.load(conf_dir / "HiJitReg.conf"), keep=keep) # HiSlither # takes same as HiJitReg (and assumes its products are available. # creates *slither.txt, *slither.cub, and *COLOR[4|5].cub # Can then run SliterStats on the *slither.txt HiSlither.HiSlither(for_jitreg) return chids
def make_flats(cubes, common_cube, conf, temp_token, keep=False): # If the flat files already exist, don't remake them. # "$OBS_ID"."_".$ccd."-".$common.".flat.tab" jitter_cubes = list() n_row = int( common_cube.lines / conf["AutoRegistration"]["ControlNet"]["Control_Lines"] ) for c in cubes: if c == common_cube: continue jitter_cubes.append( hjr.JitterCube( c.next_path, matchccd=common_cube.get_ccd(), config=conf ) ) successful_flats = list() if not all(x.flattab_path.exists() for x in jitter_cubes): confauto = conf["AutoRegistration"] for c in jitter_cubes: params = { "ROWS": n_row, "TOLERANCE": confauto["Algorithm"]["Tolerance"], } min_fraction_good = confauto["AnaylyzeFlat"]["Minimum_Good"] if c.ccdname == "RED": redcolor = "Red" else: redcolor = "Color" params["GROUP"] = "ResolveJitter" params["COLS"] = confauto["ControlNet"]["Control_Cols_" + redcolor] params["PATTERN_SAMPLES"] = confauto["PatternChip" + redcolor][ "Samples" ] params["PATTERN_LINES"] = confauto["PatternChip" + redcolor][ "Lines" ] params["SEARCH_SAMPLES"] = confauto["SearchChip" + redcolor][ "Samples" ] params["SEARCH_LINES"] = confauto["SearchChip" + redcolor]["Lines"] if c.ccdname != "RED": channels = isis.getkey_k( c.path, "Instrument", "StitchedProductIds" ) if len(channels) < 2: logger.info( f"Increasing columns because {c.path} is " "missing a channel." ) params["COLS"] += 1 min_fraction_good *= 0.5 logger.info( "The minimum allowable Fraction Good " f"Matches = {min_fraction_good}" ) step = 0 while step <= confauto["Algorithm"]["Steps"]: logger.info(f"Step {step} begin") hjr.run_HiJitReg( common_cube.next_path, c, params, temp_token, keep=keep ) ret = hjr.Analyze_Flat( c, 0, (min_fraction_good * 2), hijitreg=False ) if ret == 1: successful_flats.append(c.flattab_path) break else: step += 1 c.regdef_path.unlink() c.flattab_path.unlink() c.cnet_path.unlink() params["TOLERANCE"] -= ( confauto["Algorithm"]["Increment"] * step ) else: raise RuntimeError( f"Flat file for {c} is not within tolerances." ) else: successful_flats = list(x.flattab_path for x in jitter_cubes) return successful_flats
def test_init(self, mock_getkey): c = hci.HiColorCube("dummy/PSP_010502_2090_IR10_0") j = hjr.JitterCube(c) self.assertEqual(j.cnet_path, Path("dummy/PSP_010502_2090_RED4-IR10.control.pvl"))