def get_chids(cubes: list) -> tuple: cid_a = hirise.get_ChannelID_fromfile(cubes[0]) if len(cubes) == 2: cid_b = hirise.get_ChannelID_fromfile(cubes[1]) if hirise.CCDID(cid_a) == hirise.CCDID(cid_b): return (cid_a, cid_b) else: raise ValueError("These do not appear to be channels " f"from the same CCD:\n{cubes[0]}: " f"{cid_a} and\n{cubes[1]}: {cid_b}") else: return (cid_a, )
def calc_snr( cub: os.PathLike, gainsinfo: dict, histats: dict, cid=None ) -> float: """Calculate the signal to noise ratio.""" if cid is None: cid = hirise.get_ChannelID_fromfile(cub) # logger.info(f"{cid}: " + calc_snr.__doc__) ccdchan = f"{cid.get_ccd()}_{cid.channel}" # gainspvl = pvl.load(str(gainsfile)) gain = float(gainsinfo["Gains"][ccdchan]["Bin" + str(histats["BINNING"])]) img_mean = float(histats["IMAGE_MEAN"]) lis_pixels = float(histats["LOW_SATURATED_PIXELS"]) buf_mean = float(histats["IMAGE_BUFFER_MEAN"]) snr = -9999 r = 90 # Note from original file about r: # 150 e *Changed value to 90 e- 1/31/2012 to bring closer # to HIPHOP value for read noise. SM if 0 == lis_pixels and img_mean > 0.0 and buf_mean > 0.0: s = (img_mean - buf_mean) * gain snr = s / math.sqrt(s + r * r) logger.info(f"{cid}: Calculation of Signal/Noise Ratio:") logger.info(f"{cid}: \tIMAGE_MEAN: {img_mean}") logger.info(f"{cid}: \tIMAGE_BUFFER_MEAN: {buf_mean}") logger.info(f"{cid}: \tR (electrons/DN): {r}") logger.info(f"{cid}: \tGain: {gain}") logger.info(f"{cid}: Signal/Noise ratio: {snr}") return snr
def setUp(self): self.cube = imgs[0].with_suffix(".TestHiCal.cub") self.pid = hirise.get_ChannelID_fromfile(self.cube) self.db = edr.EDR_Stats(imgs[0], self.cube, gains) self.binning = int(isis.getkey_k(self.cube, "Instrument", "Summing")) self.conf = conf # self.conf['HiGainFx'] = pvl.load(str(hgf_conf))['HiGainFx'] self.conf["NoiseFilter"] = nf_conf["NoiseFilter"]
def pid_path_w_suffix(in_path: str, template_path: os.PathLike) -> Path: """A little extra twist to look for the db file.""" p = path_w_suffix(in_path, template_path) if p.exists(): return p elif in_path.startswith("."): pid = hirise.get_ChannelID_fromfile(template_path) t_path = Path(template_path) if t_path.is_dir(): d = t_path else: d = t_path.parent pid_path = d / Path(str(pid)).with_suffix(in_path) if pid_path.exists(): return pid_path else: raise FileNotFoundError(f"Could not find {pid_path}") else: raise FileNotFoundError(f"Could not find {p}")
def tdi_bin_check(cube: os.PathLike, histats: dict, cid=None): """This function only logs warnings and returns nothing.""" if cid is None: try: cid = histats['PRODUCT_ID'] except KeyError: cid = hirise.get_ChannelID_fromfile(cube) # TDI and binning check if float(histats["IMAGE_MEAN"]) >= 8000: logger.warning( f"{cid}: " f"Channel mean greater than 8000 (TDI or binning too high)." ) elif float(histats["IMAGE_MEAN"]) < 2500: tdi = isis.getkey_k(cube, "Instrument", "Tdi") if tdi == "32" or tdi == "64": logger.warning(f"{cid}: TDI too low.") return
def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "-o", "--output", required=False, default=".mdr.iof.cub" ) parser.add_argument("-e", "--edr", required=True) parser.add_argument( "-c", "--conf", required=False, type=argparse.FileType('r'), default=pkg_resources.resource_stream( __name__, 'data/hical.pipelines.conf' ), ) parser.add_argument("mdr", metavar="MDR_file") parser.add_argument( "-l", "--log", required=False, default="WARNING", help="The log level to show for this program, can " "be a named log level or a numerical level.", ) parser.add_argument( "-k", "--keep", required=False, default=False, action="store_true", help="Normally, the program will clean up any " "intermediary files, but if this option is given, it " "won't.", ) args = parser.parse_args() util.set_logger(args.verbose, args.logfile, args.log) edr_path = Path(args.edr) mdr_path = Path(args.mdr) to_del = isis.PathSet() h2i_path = to_del.add(edr_path.with_suffix(".hi2isis.cub")) out_path = util.path_w_suffix(args.output, edr_path) # The first thing Alan's program did was to crop the image down to only the # 'imaging' parts. We're not doing that so the resultant file has a # geometry similar to what comes out of ISIS hical. # Convert the EDR to a cube file isis.hi2isis(edr_path, to=h2i_path) # Convert Alan's MDR to a cube file mdr_cub_path = to_del.add(mdr_path.with_suffix(".alan.cub")) logger.info(f"Running gdal_translate {mdr_path} -of ISIS3 {mdr_cub_path}") gdal.Translate(str(mdr_cub_path), str(mdr_path), format="ISIS3") h2i_s = int(isis.getkey_k(h2i_path, "Dimensions", "Samples")) h2i_l = int(isis.getkey_k(h2i_path, "Dimensions", "Lines")) mdr_s = int(isis.getkey_k(mdr_cub_path, "Dimensions", "Samples")) mdr_l = int(isis.getkey_k(mdr_cub_path, "Dimensions", "Lines")) if h2i_s != mdr_s: label = pvl.load(str(h2i_path)) hirise_cal_info = get_one( label, "Table", "HiRISE Calibration Ancillary" ) buffer_pixels = get_one(hirise_cal_info, "Field", "BufferPixels")[ "Size" ] dark_pixels = get_one(hirise_cal_info, "Field", "DarkPixels")["Size"] rev_mask_tdi_lines = hirise_cal_info["Records"] if h2i_s + buffer_pixels + dark_pixels == mdr_s: logger.info( f"The file {mdr_cub_path} has " f"{buffer_pixels + dark_pixels} more sample pixels " f"than {h2i_path}, assuming those are dark and " "buffer pixels and will crop accordingly." ) if h2i_l + rev_mask_tdi_lines != mdr_l: logger.critical( 'Even assuming this is a "full" channel ' "image, this has the wrong number of lines. " f"{mdr_cub_path} should have " f"{h2i_l + rev_mask_tdi_lines}, but " f"has {mdr_l} lines. Exiting" ) sys.exit() else: crop_path = to_del.add(mdr_cub_path.with_suffix(".crop.cub")) # We want to start with the next pixel (+1) after the cal # pixels. isis.crop( mdr_cub_path, to=crop_path, sample=buffer_pixels + 1, nsamples=h2i_s, line=rev_mask_tdi_lines + 1, ) mdr_cub_path = crop_path mdr_l = int(isis.getkey_k(mdr_cub_path, "Dimensions", "Lines")) else: logger.critical( f"The number of samples in {h2i_path} ({h2i_s}) " f"and {mdr_cub_path} ({mdr_s}) are different. " "Exiting." ) sys.exit() if h2i_l != mdr_l: logger.critical( f"The number of lines in {h2i_path} ({h2i_l}) " f"and {mdr_cub_path} ({mdr_l}) are different. " "Exiting." ) sys.exit() # Convert the EDR to the right bit type for post-HiCal Pipeline: h2i_16b_p = to_del.add(h2i_path.with_suffix(".16bit.cub")) isis.bit2bit( h2i_path, to=h2i_16b_p, bit="16bit", clip="minmax", minval=0, maxval=1.5, ) shutil.copyfile(h2i_16b_p, out_path) # If it is a channel 1 file, Alan mirrored it so that he could process # the two channels in an identical way (which we also took advantage # of above if the buffer and dark pixels were included), so we need to # mirror it back. cid = hirise.get_ChannelID_fromfile(h2i_16b_p) if cid.channel == "1": mirror_path = to_del.add(mdr_cub_path.with_suffix(".mirror.cub")) isis.mirror(mdr_cub_path, to=mirror_path) mdr_cub_path = mirror_path # Is the MDR in DN or I/F? maximum_pxl = float( pvl.loads(isis.stats(mdr_cub_path).stdout)["Results"]["Maximum"] ) if maximum_pxl < 1.5: logger.info("MDR is already in I/F units.") mdr_16b_p = to_del.add(mdr_cub_path.with_suffix(".16bit.cub")) isis.bit2bit( mdr_cub_path, to=mdr_16b_p, bit="16bit", clip="minmax", minval=0, maxval=1.5, ) isis.handmos(mdr_16b_p, mosaic=out_path) else: logger.info("MDR is in DN units and will be converted to I/F.") fpa_t = statistics.mean( [ float( isis.getkey_k( h2i_16b_p, "Instrument", "FpaPositiveYTemperature" ) ), float( isis.getkey_k( h2i_16b_p, "Instrument", "FpaNegativeYTemperature" ) ), ] ) print(f"fpa_t {fpa_t}") conf = pvl.load(args.conf) tdg = t_dep_gain(get_one(conf["Hical"], "Profile", cid.ccdname), fpa_t) suncorr = solar_correction() sclk = isis.getkey_k( h2i_16b_p, "Instrument", "SpacecraftClockStartCount" ) target = isis.getkey_k(h2i_16b_p, "Instrument", "TargetName") suncorr = solar_correction(sunDistanceAU(sclk, target)) sed = float( isis.getkey_k(h2i_16b_p, "Instrument", "LineExposureDuration") ) zbin = get_one(conf["Hical"], "Profile", "GainUnitConversion")[ "GainUnitConversionBinFactor" ] # The 'ziof' name is from the ISIS HiCal/GainUnitConversion.h, it is a # divisor in the calibration equation. print(f"zbin {zbin}") print(f"tdg {tdg}") print(f"sed {sed}") print(f"suncorr {suncorr}") ziof = zbin * tdg * sed * 1e-6 * suncorr eqn = f"\(F1 / {ziof})" # noqa W605 mdriof_p = to_del.add(mdr_cub_path.with_suffix(".iof.cub")) to_s = "{}+SignedWord+{}:{}".format(mdriof_p, 0, 1.5) isis.fx(f1=mdr_cub_path, to=to_s, equ=eqn) isis.handmos(mdriof_p, mosaic=out_path) if not args.keep: to_del.unlink()
def EDR_Stats( img: os.PathLike, out_path: os.PathLike, gainsinfo: dict, histmin=0.01, histmax=99.99, keep=False, ) -> dict: cid = hirise.get_ChannelID_fromfile(img) logger.info(f"{cid}: EDR_Stats start: {img}") try: logger.info( f"{cid}: The LUT for this file is: " + str(check_lut(img)) ) except KeyError as err: logger.error( f"{cid}: The LUT header area is either corrupted or has a gap." ) raise err # Convert to .cub isis.hi2isis(img, to=out_path) histat_complete = isis.histat( out_path, useoffsets=True, leftimage=0, rightimage=1, leftcalbuffer=3, rightcalbuffer=1, leftcaldark=3, rightcaldark=1, leftbuffer=3, rightbuffer=1, leftdark=3, rightdark=1, ) histats = parse_histat(histat_complete.stdout) # Get some info from the new cube: histats["PRODUCT_ID"] = isis.getkey_k(out_path, "Archive", "ProductId") histats["IMAGE_LINES"] = int( isis.getkey_k(out_path, "Dimensions", "Lines") ) histats["LINE_SAMPLES"] = int( isis.getkey_k(out_path, "Dimensions", "Samples") ) histats["BINNING"] = int(isis.getkey_k(out_path, "Instrument", "Summing")) histats["STD_DN_LEVELS"] = get_dncnt(out_path, histmin, histmax, keep=keep) histats["IMAGE_SIGNAL_TO_NOISE_RATIO"] = calc_snr( out_path, gainsinfo, histats, cid=cid ) histats["GAP_PIXELS_PERCENT"] = ( histats["GAP_PIXELS"] / (int(histats["IMAGE_LINES"]) * int(histats["LINE_SAMPLES"])) ) * 100.0 tdi_bin_check(out_path, histats, cid=cid) lut_check(out_path, histats) logger.info(f"{cid}: EDR_Stats done: {out_path}") return histats
def setUp(self): self.cube = imgs[0].with_suffix(".TestHiCal_TestNeedISISCube.cub") isis.hi2isis(imgs[0], to=self.cube) self.pid = hirise.get_ChannelID_fromfile(self.cube) self.binning = int(isis.getkey_k(self.cube, "Instrument", "Summing"))
def __init__(self, pathlike, db=None): self.path = Path(pathlike) self.nextpath = self.path super().__init__(hirise.get_ChannelID_fromfile(self.path)) self.db = db
def fix(in_path: os.PathLike, out_path: os.PathLike, tolerance=0.4) -> bool: """The ISIS cube file at *out_path* will be the result of running lis-fix processing of the file at *in-path*, if allowed by *tolerance*. If the *tolerance* setting would result in the application of lis-fix, then no file will be created at *out_path* and this function will return False. Otherwise there will be a file there, and this function will return True. If the fraction of LIS pixels in the Reverse-Clock areas is greater than *tolerance*, then all LIS pixels in the rev-clock will be converted to a real DN value based on modeling the slope of the ramp area. If the fraction of LIS pixels in the Buffer Pixel area is greater than *tolerance*, this algorithm will attempt to fix those LIS pixels, as well. If the file at *in_path* isn't an ISIS cube, a ValueError will be raised. This function anticipates a HiRISE cube that has the following table objects in its label: HiRISE Calibration Ancillary, HiRISE Calibration Image, HiRISE Ancillary. If they are not present a KeyError will be raised. The HiRISE Calibration Ancillary table contains the BufferPixels and DarkPixels from either side of the HiRISE Calibration Image. The HiRISE Calibration Image table contains the Reverse-Clock, Mask, and Ramp Image areas. The HiRISE Ancillary table contains the BufferPixels and DarkPixels from either side of the main Image Area. """ in_p = Path(in_path) out_p = Path(out_path) fixed = False label = pvl.load(in_p) if "IsisCube" not in label: raise ValueError(f"The file at {in_p} is not an ISIS Cube.") binning = label["IsisCube"]["Instrument"]["Summing"] specialpix = getattr(isis.specialpixels, label["IsisCube"]["Core"]["Pixels"]["Type"]) # Get Rev-Clock data t_name = "HiRISE Calibration Image" hci_dict = isis.cube.get_table(in_path, t_name) cal_vals = np.array(hci_dict["Calibration"]) cal_image = np.ma.masked_outside(cal_vals, specialpix.Min, specialpix.Max) mask_lines = int(20 / binning) ramp_start = 20 + mask_lines # Get the average slope of the dark ramp hca_dict = isis.cube.get_table(in_path, "HiRISE Calibration Ancillary") caldark_vals = np.array(hca_dict["DarkPixels"]) caldark = np.ma.masked_outside(caldark_vals, specialpix.Min, specialpix.Max) # print(caldark.shape) dark_slopes = np.ma.apply_along_axis(get_ramp_slope, 0, caldark[ramp_start:, 1:]) # print(dark_slopes.shape) # print(dark_slopes) # CORRECT FOR MASKED LINES # ASSUME SLOPE IS DEFINED BY DARK COLUMNS dark_slope_mean = np.mean(dark_slopes) logger.info(f"Dark Ramp mean slope: {dark_slope_mean}") # FOR # J = 12, SZ(1) - 1 # DO # zero.line(j) = zero.line(j) - SL_D * ( # ystart(j) + 1 + 20. / info.bin * 103. / 89) zero_correction = dark_slope_mean * (20 / binning * 103 / 89) revclk_lisfrac = np.ma.count_masked( cal_image[:20, :]) / cal_image[:20, :].size logger.info( f"Fraction of LIS pixels in Reverse-Clock area: {revclk_lisfrac:.2}") if revclk_lisfrac > tolerance: fixed = True else: logger.info( f"Less than tolerance ({tolerance}), will not fix Reverse-Clock.") # Get Buffer Data calbuf_vals = np.array(hca_dict["BufferPixels"]) calbuf = np.ma.masked_outside(calbuf_vals, specialpix.Min, specialpix.Max) ha_dict = isis.cube.get_table(in_path, "HiRISE Ancillary") dark_vals = np.array(ha_dict["DarkPixels"]) dark = np.ma.masked_outside(dark_vals, specialpix.Min, specialpix.Max) # print(dark.shape) buffer_vals = np.array(ha_dict["BufferPixels"]) buffer = np.ma.masked_outside(buffer_vals, specialpix.Min, specialpix.Max) # print(buffer) # print(buffer.dtype) # print(buffer.shape) buffer_lisfrac = np.ma.count_masked(buffer) / buffer.size logger.info(f"Fraction of LIS pixels in Buffer area: {buffer_lisfrac:.2}") if buffer_lisfrac > tolerance: fixed = True else: logger.info(f"Less than tolerance ({tolerance}), will not fix Buffer.") ramp_lisfrac = np.ma.count_masked( cal_image[ramp_start:]) / cal_image[ramp_start:].size if ramp_lisfrac > tolerance: fixed = False logger.warning( f"The fraction of LIS pixels in the ramp area ({ramp_lisfrac}) " f"is greater than the tolerance ({tolerance}), which prevents " f"lisfix processing.") if fixed: shutil.copy(in_path, out_path) else: return False # Fix rev-clock if revclk_lisfrac > tolerance: rev_model = ramp_rev_clock( cal_image[ramp_start:], label["IsisCube"]["Instrument"]["ChannelNumber"], zero_correction).astype(int) # # Mask out all of the rev-clock for testing, so we get wholesale # # replacement # orig_cal_image = np.ma.copy(cal_image) # o_mean = np.ma.mean(orig_cal_image[:20, :], axis=0) # y_err_upper = np.ma.max(orig_cal_image[:20, :], axis=0) - o_mean # y_err_lower = o_mean - np.ma.min(orig_cal_image[:20, :], axis=0) # cal_image[:20, :] = np.ma.masked # # Some plotting for debugging: # import matplotlib.pyplot as plt # plt.plot(o_mean, 'k', label='Mean of Rev-Clock columns') # # plt.errorbar( # # np.linspace(0, o_mean.size, num=o_mean.size, endpoint=False), # # o_mean, # # yerr=[y_err_lower, y_err_upper], # # fmt='k', # # label='Mean of Rev-Clock columns' # # ) # for i in range(19): # plt.plot(orig_cal_image[i, :], 'k.') # plt.plot(orig_cal_image[19, :], 'k.', label='Original Rev-Clock') # plt.plot(rev_model, 'r', label="Fixed") # plt.legend(loc='best') # plt.show() # sys.exit() # When we "fix" pixels, we must ensure that we are only fixing the LIS # pixels. The cal_image has all special pixels masked out, so # we need to create a new structure that only masks the LIS pixels. cal_lismasked = np.ma.masked_equal(cal_vals, specialpix.Lis) logger.info("Fixing Reverse-Clock LIS pixels.") fixed_cal = np.ma.apply_along_axis( fix_rev_clock, 0, np.ma.concatenate( (cal_lismasked, rev_model.reshape((1, rev_model.size))))) logger.info("Writing out Reverse-Clock pixels.") hci_dict["Calibration"] = apply_special_pixels( fixed_cal, specialpix).data.tolist() isis.cube.overwrite_table(out_p, t_name, hci_dict) # Fix the buffer area: if buffer_lisfrac > tolerance: # first_im_line = 19+(20+label["IsisCube"]["Instrument"]["Tdi"])/binning sp_frac = np.ma.count_masked(calbuf[:20, :]) / calbuf[:20, :].size if sp_frac <= tolerance: refr = int(np.ma.median(calbuf[:20, :])) logger.info( f"Median of first 20 lines of Calibration Buffer Pixels: {refr}" ) else: logger.info(f"More than {tolerance:.1%} of the Calibration Buffer " f"Pixels have special pixel values: {sp_frac:.2%}.") isp = pvl.loads(isis.catoriglab( in_path).stdout)["INSTRUMENT_SETTING_PARAMETERS"] adc = label["IsisCube"]["Instrument"]["ADCTimingSetting"] if adc == -9999: adc = isp["MRO:ADC_TIMING_SETTINGS"] cid = get_ChannelID_fromfile(in_path) refr = int( revclock_model( cid.ccdname + cid.ccdnumber + "_" + cid.channel, binning, label["IsisCube"]["Instrument"] ["FpaPositiveYTemperature"].value, adc)) logger.info(f"Using a model-based substitute ({refr}).") # import hiproc.img as img # lut = img.LUT_Table(isp["MRO:LOOKUP_CONVERSION_TABLE"]) # if refr <= lut.table[1]: # logger.error(f"Using a model-based substitute ({refr}).") # else: # logger.error( # f"Model ({refr}) was greater than LUT floor " # f"({lut.table[1]}). Setting to LUT floor." # ) # refr = lut.table[1] if np.ma.count_masked(dark[:20]) / dark[:20].size <= tolerance: refd = int(np.ma.median(dark[:20])) logger.info(f"Median of first 20 lines of Dark Pixels: {refd}") else: refd = int(np.ma.median(fixed_cal[:20, :])) logger.info( f"More than {tolerance:.1%} of the first 20 lines of " f"Dark Pixels have a real " f"value: {np.ma.count_masked(dark[:20]) / dark[:20].size} " f"Using the rev-clock median ({refd}).") model_buffer = dark[:, 2:14] - refd + refr # print(model_buffer) # print(model_buffer.dtype) # print(model_buffer.shape) # # Mask out all of the buffer for testing, so we get wholesale # # replacement # buffer.mask = True logger.info("Fixing Buffer pixels.") buffer_lismasked = np.ma.masked_equal(buffer_vals, specialpix.Lis) fixed_buf = np.ma.apply_along_axis( fix_buffer, 1, np.ma.concatenate((buffer_lismasked, model_buffer), 1), buffer_lismasked.shape[1]) # print(fixed_buf) # print(fixed_buf.dtype) # print(fixed_buf.shape) ha_dict["BufferPixels"] = apply_special_pixels( fixed_buf, specialpix).data.tolist() logger.info("Writing out Buffer pixels.") isis.cube.overwrite_table(out_p, "HiRISE Ancillary", ha_dict) return fixed