def get_chids(cubes: list) -> tuple:
cid_a = hirise.get_ChannelID_fromfile(cubes[0])
if len(cubes) == 2:
cid_b = hirise.get_ChannelID_fromfile(cubes[1])
if hirise.CCDID(cid_a) == hirise.CCDID(cid_b):
return (cid_a, cid_b)
else:
raise ValueError("These do not appear to be channels "
f"from the same CCD:\n{cubes[0]}: "
f"{cid_a} and\n{cubes[1]}: {cid_b}")
else:
return (cid_a, )
def calc_snr(
cub: os.PathLike, gainsinfo: dict, histats: dict, cid=None
) -> float:
"""Calculate the signal to noise ratio."""
if cid is None:
cid = hirise.get_ChannelID_fromfile(cub)
# logger.info(f"{cid}: " + calc_snr.__doc__)
ccdchan = f"{cid.get_ccd()}_{cid.channel}"
# gainspvl = pvl.load(str(gainsfile))
gain = float(gainsinfo["Gains"][ccdchan]["Bin" + str(histats["BINNING"])])
img_mean = float(histats["IMAGE_MEAN"])
lis_pixels = float(histats["LOW_SATURATED_PIXELS"])
buf_mean = float(histats["IMAGE_BUFFER_MEAN"])
snr = -9999
r = 90
# Note from original file about r:
# 150 e *Changed value to 90 e- 1/31/2012 to bring closer
# to HIPHOP value for read noise. SM
if 0 == lis_pixels and img_mean > 0.0 and buf_mean > 0.0:
s = (img_mean - buf_mean) * gain
snr = s / math.sqrt(s + r * r)
logger.info(f"{cid}: Calculation of Signal/Noise Ratio:")
logger.info(f"{cid}: \tIMAGE_MEAN: {img_mean}")
logger.info(f"{cid}: \tIMAGE_BUFFER_MEAN: {buf_mean}")
logger.info(f"{cid}: \tR (electrons/DN): {r}")
logger.info(f"{cid}: \tGain: {gain}")
logger.info(f"{cid}: Signal/Noise ratio: {snr}")
return snr
def setUp(self):
self.cube = imgs[0].with_suffix(".TestHiCal.cub")
self.pid = hirise.get_ChannelID_fromfile(self.cube)
self.db = edr.EDR_Stats(imgs[0], self.cube, gains)
self.binning = int(isis.getkey_k(self.cube, "Instrument", "Summing"))
self.conf = conf
# self.conf['HiGainFx'] = pvl.load(str(hgf_conf))['HiGainFx']
self.conf["NoiseFilter"] = nf_conf["NoiseFilter"]
def pid_path_w_suffix(in_path: str, template_path: os.PathLike) -> Path:
"""A little extra twist to look for the db file."""
p = path_w_suffix(in_path, template_path)
if p.exists():
return p
elif in_path.startswith("."):
pid = hirise.get_ChannelID_fromfile(template_path)
t_path = Path(template_path)
if t_path.is_dir():
d = t_path
else:
d = t_path.parent
pid_path = d / Path(str(pid)).with_suffix(in_path)
if pid_path.exists():
return pid_path
else:
raise FileNotFoundError(f"Could not find {pid_path}")
else:
raise FileNotFoundError(f"Could not find {p}")
def tdi_bin_check(cube: os.PathLike, histats: dict, cid=None):
"""This function only logs warnings and returns nothing."""
if cid is None:
try:
cid = histats['PRODUCT_ID']
except KeyError:
cid = hirise.get_ChannelID_fromfile(cube)
# TDI and binning check
if float(histats["IMAGE_MEAN"]) >= 8000:
logger.warning(
f"{cid}: "
f"Channel mean greater than 8000 (TDI or binning too high)."
)
elif float(histats["IMAGE_MEAN"]) < 2500:
tdi = isis.getkey_k(cube, "Instrument", "Tdi")
if tdi == "32" or tdi == "64":
logger.warning(f"{cid}: TDI too low.")
return
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"-o", "--output", required=False, default=".mdr.iof.cub"
)
parser.add_argument("-e", "--edr", required=True)
parser.add_argument(
"-c",
"--conf",
required=False,
type=argparse.FileType('r'),
default=pkg_resources.resource_stream(
__name__,
'data/hical.pipelines.conf'
),
)
parser.add_argument("mdr", metavar="MDR_file")
parser.add_argument(
"-l",
"--log",
required=False,
default="WARNING",
help="The log level to show for this program, can "
"be a named log level or a numerical level.",
)
parser.add_argument(
"-k",
"--keep",
required=False,
default=False,
action="store_true",
help="Normally, the program will clean up any "
"intermediary files, but if this option is given, it "
"won't.",
)
args = parser.parse_args()
util.set_logger(args.verbose, args.logfile, args.log)
edr_path = Path(args.edr)
mdr_path = Path(args.mdr)
to_del = isis.PathSet()
h2i_path = to_del.add(edr_path.with_suffix(".hi2isis.cub"))
out_path = util.path_w_suffix(args.output, edr_path)
# The first thing Alan's program did was to crop the image down to only the
# 'imaging' parts. We're not doing that so the resultant file has a
# geometry similar to what comes out of ISIS hical.
# Convert the EDR to a cube file
isis.hi2isis(edr_path, to=h2i_path)
# Convert Alan's MDR to a cube file
mdr_cub_path = to_del.add(mdr_path.with_suffix(".alan.cub"))
logger.info(f"Running gdal_translate {mdr_path} -of ISIS3 {mdr_cub_path}")
gdal.Translate(str(mdr_cub_path), str(mdr_path), format="ISIS3")
h2i_s = int(isis.getkey_k(h2i_path, "Dimensions", "Samples"))
h2i_l = int(isis.getkey_k(h2i_path, "Dimensions", "Lines"))
mdr_s = int(isis.getkey_k(mdr_cub_path, "Dimensions", "Samples"))
mdr_l = int(isis.getkey_k(mdr_cub_path, "Dimensions", "Lines"))
if h2i_s != mdr_s:
label = pvl.load(str(h2i_path))
hirise_cal_info = get_one(
label, "Table", "HiRISE Calibration Ancillary"
)
buffer_pixels = get_one(hirise_cal_info, "Field", "BufferPixels")[
"Size"
]
dark_pixels = get_one(hirise_cal_info, "Field", "DarkPixels")["Size"]
rev_mask_tdi_lines = hirise_cal_info["Records"]
if h2i_s + buffer_pixels + dark_pixels == mdr_s:
logger.info(
f"The file {mdr_cub_path} has "
f"{buffer_pixels + dark_pixels} more sample pixels "
f"than {h2i_path}, assuming those are dark and "
"buffer pixels and will crop accordingly."
)
if h2i_l + rev_mask_tdi_lines != mdr_l:
logger.critical(
'Even assuming this is a "full" channel '
"image, this has the wrong number of lines. "
f"{mdr_cub_path} should have "
f"{h2i_l + rev_mask_tdi_lines}, but "
f"has {mdr_l} lines. Exiting"
)
sys.exit()
else:
crop_path = to_del.add(mdr_cub_path.with_suffix(".crop.cub"))
# We want to start with the next pixel (+1) after the cal
# pixels.
isis.crop(
mdr_cub_path,
to=crop_path,
sample=buffer_pixels + 1,
nsamples=h2i_s,
line=rev_mask_tdi_lines + 1,
)
mdr_cub_path = crop_path
mdr_l = int(isis.getkey_k(mdr_cub_path, "Dimensions", "Lines"))
else:
logger.critical(
f"The number of samples in {h2i_path} ({h2i_s}) "
f"and {mdr_cub_path} ({mdr_s}) are different. "
"Exiting."
)
sys.exit()
if h2i_l != mdr_l:
logger.critical(
f"The number of lines in {h2i_path} ({h2i_l}) "
f"and {mdr_cub_path} ({mdr_l}) are different. "
"Exiting."
)
sys.exit()
# Convert the EDR to the right bit type for post-HiCal Pipeline:
h2i_16b_p = to_del.add(h2i_path.with_suffix(".16bit.cub"))
isis.bit2bit(
h2i_path,
to=h2i_16b_p,
bit="16bit",
clip="minmax",
minval=0,
maxval=1.5,
)
shutil.copyfile(h2i_16b_p, out_path)
# If it is a channel 1 file, Alan mirrored it so that he could process
# the two channels in an identical way (which we also took advantage
# of above if the buffer and dark pixels were included), so we need to
# mirror it back.
cid = hirise.get_ChannelID_fromfile(h2i_16b_p)
if cid.channel == "1":
mirror_path = to_del.add(mdr_cub_path.with_suffix(".mirror.cub"))
isis.mirror(mdr_cub_path, to=mirror_path)
mdr_cub_path = mirror_path
# Is the MDR in DN or I/F?
maximum_pxl = float(
pvl.loads(isis.stats(mdr_cub_path).stdout)["Results"]["Maximum"]
)
if maximum_pxl < 1.5:
logger.info("MDR is already in I/F units.")
mdr_16b_p = to_del.add(mdr_cub_path.with_suffix(".16bit.cub"))
isis.bit2bit(
mdr_cub_path,
to=mdr_16b_p,
bit="16bit",
clip="minmax",
minval=0,
maxval=1.5,
)
isis.handmos(mdr_16b_p, mosaic=out_path)
else:
logger.info("MDR is in DN units and will be converted to I/F.")
fpa_t = statistics.mean(
[
float(
isis.getkey_k(
h2i_16b_p, "Instrument", "FpaPositiveYTemperature"
)
),
float(
isis.getkey_k(
h2i_16b_p, "Instrument", "FpaNegativeYTemperature"
)
),
]
)
print(f"fpa_t {fpa_t}")
conf = pvl.load(args.conf)
tdg = t_dep_gain(get_one(conf["Hical"], "Profile", cid.ccdname), fpa_t)
suncorr = solar_correction()
sclk = isis.getkey_k(
h2i_16b_p, "Instrument", "SpacecraftClockStartCount"
)
target = isis.getkey_k(h2i_16b_p, "Instrument", "TargetName")
suncorr = solar_correction(sunDistanceAU(sclk, target))
sed = float(
isis.getkey_k(h2i_16b_p, "Instrument", "LineExposureDuration")
)
zbin = get_one(conf["Hical"], "Profile", "GainUnitConversion")[
"GainUnitConversionBinFactor"
]
# The 'ziof' name is from the ISIS HiCal/GainUnitConversion.h, it is a
# divisor in the calibration equation.
print(f"zbin {zbin}")
print(f"tdg {tdg}")
print(f"sed {sed}")
print(f"suncorr {suncorr}")
ziof = zbin * tdg * sed * 1e-6 * suncorr
eqn = f"\(F1 / {ziof})" # noqa W605
mdriof_p = to_del.add(mdr_cub_path.with_suffix(".iof.cub"))
to_s = "{}+SignedWord+{}:{}".format(mdriof_p, 0, 1.5)
isis.fx(f1=mdr_cub_path, to=to_s, equ=eqn)
isis.handmos(mdriof_p, mosaic=out_path)
if not args.keep:
to_del.unlink()
def EDR_Stats(
img: os.PathLike,
out_path: os.PathLike,
gainsinfo: dict,
histmin=0.01,
histmax=99.99,
keep=False,
) -> dict:
cid = hirise.get_ChannelID_fromfile(img)
logger.info(f"{cid}: EDR_Stats start: {img}")
try:
logger.info(
f"{cid}: The LUT for this file is: " + str(check_lut(img))
)
except KeyError as err:
logger.error(
f"{cid}: The LUT header area is either corrupted or has a gap."
)
raise err
# Convert to .cub
isis.hi2isis(img, to=out_path)
histat_complete = isis.histat(
out_path,
useoffsets=True,
leftimage=0,
rightimage=1,
leftcalbuffer=3,
rightcalbuffer=1,
leftcaldark=3,
rightcaldark=1,
leftbuffer=3,
rightbuffer=1,
leftdark=3,
rightdark=1,
)
histats = parse_histat(histat_complete.stdout)
# Get some info from the new cube:
histats["PRODUCT_ID"] = isis.getkey_k(out_path, "Archive", "ProductId")
histats["IMAGE_LINES"] = int(
isis.getkey_k(out_path, "Dimensions", "Lines")
)
histats["LINE_SAMPLES"] = int(
isis.getkey_k(out_path, "Dimensions", "Samples")
)
histats["BINNING"] = int(isis.getkey_k(out_path, "Instrument", "Summing"))
histats["STD_DN_LEVELS"] = get_dncnt(out_path, histmin, histmax, keep=keep)
histats["IMAGE_SIGNAL_TO_NOISE_RATIO"] = calc_snr(
out_path, gainsinfo, histats, cid=cid
)
histats["GAP_PIXELS_PERCENT"] = (
histats["GAP_PIXELS"]
/ (int(histats["IMAGE_LINES"]) * int(histats["LINE_SAMPLES"]))
) * 100.0
tdi_bin_check(out_path, histats, cid=cid)
lut_check(out_path, histats)
logger.info(f"{cid}: EDR_Stats done: {out_path}")
return histats
def setUp(self):
self.cube = imgs[0].with_suffix(".TestHiCal_TestNeedISISCube.cub")
isis.hi2isis(imgs[0], to=self.cube)
self.pid = hirise.get_ChannelID_fromfile(self.cube)
self.binning = int(isis.getkey_k(self.cube, "Instrument", "Summing"))
def __init__(self, pathlike, db=None):
self.path = Path(pathlike)
self.nextpath = self.path
super().__init__(hirise.get_ChannelID_fromfile(self.path))
self.db = db
def fix(in_path: os.PathLike, out_path: os.PathLike, tolerance=0.4) -> bool:
"""The ISIS cube file at *out_path* will be the result of running lis-fix
processing of the file at *in-path*, if allowed by *tolerance*. If the
*tolerance* setting would result in the application of lis-fix, then
no file will be created at *out_path* and this function will return
False. Otherwise there will be a file there, and this function
will return True.
If the fraction of LIS pixels in the Reverse-Clock areas is
greater than *tolerance*, then all LIS pixels in the rev-clock
will be converted to a real DN value based on modeling the slope
of the ramp area.
If the fraction of LIS pixels in the Buffer Pixel area is greater
than *tolerance*, this algorithm will attempt to fix those LIS pixels,
as well.
If the file at *in_path* isn't an ISIS cube, a ValueError will be raised.
This function anticipates a HiRISE cube that has the following
table objects in its label: HiRISE Calibration Ancillary,
HiRISE Calibration Image, HiRISE Ancillary. If they are not present
a KeyError will be raised.
The HiRISE Calibration Ancillary table contains the BufferPixels
and DarkPixels from either side of the HiRISE Calibration Image.
The HiRISE Calibration Image table contains the Reverse-Clock,
Mask, and Ramp Image areas. The HiRISE Ancillary table contains
the BufferPixels and DarkPixels from either side of the main
Image Area.
"""
in_p = Path(in_path)
out_p = Path(out_path)
fixed = False
label = pvl.load(in_p)
if "IsisCube" not in label:
raise ValueError(f"The file at {in_p} is not an ISIS Cube.")
binning = label["IsisCube"]["Instrument"]["Summing"]
specialpix = getattr(isis.specialpixels,
label["IsisCube"]["Core"]["Pixels"]["Type"])
# Get Rev-Clock data
t_name = "HiRISE Calibration Image"
hci_dict = isis.cube.get_table(in_path, t_name)
cal_vals = np.array(hci_dict["Calibration"])
cal_image = np.ma.masked_outside(cal_vals, specialpix.Min, specialpix.Max)
mask_lines = int(20 / binning)
ramp_start = 20 + mask_lines
# Get the average slope of the dark ramp
hca_dict = isis.cube.get_table(in_path, "HiRISE Calibration Ancillary")
caldark_vals = np.array(hca_dict["DarkPixels"])
caldark = np.ma.masked_outside(caldark_vals, specialpix.Min,
specialpix.Max)
# print(caldark.shape)
dark_slopes = np.ma.apply_along_axis(get_ramp_slope, 0,
caldark[ramp_start:, 1:])
# print(dark_slopes.shape)
# print(dark_slopes)
# CORRECT FOR MASKED LINES
# ASSUME SLOPE IS DEFINED BY DARK COLUMNS
dark_slope_mean = np.mean(dark_slopes)
logger.info(f"Dark Ramp mean slope: {dark_slope_mean}")
# FOR # J = 12, SZ(1) - 1 # DO
# zero.line(j) = zero.line(j) - SL_D * (
# ystart(j) + 1 + 20. / info.bin * 103. / 89)
zero_correction = dark_slope_mean * (20 / binning * 103 / 89)
revclk_lisfrac = np.ma.count_masked(
cal_image[:20, :]) / cal_image[:20, :].size
logger.info(
f"Fraction of LIS pixels in Reverse-Clock area: {revclk_lisfrac:.2}")
if revclk_lisfrac > tolerance:
fixed = True
else:
logger.info(
f"Less than tolerance ({tolerance}), will not fix Reverse-Clock.")
# Get Buffer Data
calbuf_vals = np.array(hca_dict["BufferPixels"])
calbuf = np.ma.masked_outside(calbuf_vals, specialpix.Min, specialpix.Max)
ha_dict = isis.cube.get_table(in_path, "HiRISE Ancillary")
dark_vals = np.array(ha_dict["DarkPixels"])
dark = np.ma.masked_outside(dark_vals, specialpix.Min, specialpix.Max)
# print(dark.shape)
buffer_vals = np.array(ha_dict["BufferPixels"])
buffer = np.ma.masked_outside(buffer_vals, specialpix.Min, specialpix.Max)
# print(buffer)
# print(buffer.dtype)
# print(buffer.shape)
buffer_lisfrac = np.ma.count_masked(buffer) / buffer.size
logger.info(f"Fraction of LIS pixels in Buffer area: {buffer_lisfrac:.2}")
if buffer_lisfrac > tolerance:
fixed = True
else:
logger.info(f"Less than tolerance ({tolerance}), will not fix Buffer.")
ramp_lisfrac = np.ma.count_masked(
cal_image[ramp_start:]) / cal_image[ramp_start:].size
if ramp_lisfrac > tolerance:
fixed = False
logger.warning(
f"The fraction of LIS pixels in the ramp area ({ramp_lisfrac}) "
f"is greater than the tolerance ({tolerance}), which prevents "
f"lisfix processing.")
if fixed:
shutil.copy(in_path, out_path)
else:
return False
# Fix rev-clock
if revclk_lisfrac > tolerance:
rev_model = ramp_rev_clock(
cal_image[ramp_start:],
label["IsisCube"]["Instrument"]["ChannelNumber"],
zero_correction).astype(int)
# # Mask out all of the rev-clock for testing, so we get wholesale
# # replacement
# orig_cal_image = np.ma.copy(cal_image)
# o_mean = np.ma.mean(orig_cal_image[:20, :], axis=0)
# y_err_upper = np.ma.max(orig_cal_image[:20, :], axis=0) - o_mean
# y_err_lower = o_mean - np.ma.min(orig_cal_image[:20, :], axis=0)
# cal_image[:20, :] = np.ma.masked
# # Some plotting for debugging:
# import matplotlib.pyplot as plt
# plt.plot(o_mean, 'k', label='Mean of Rev-Clock columns')
# # plt.errorbar(
# # np.linspace(0, o_mean.size, num=o_mean.size, endpoint=False),
# # o_mean,
# # yerr=[y_err_lower, y_err_upper],
# # fmt='k',
# # label='Mean of Rev-Clock columns'
# # )
# for i in range(19):
# plt.plot(orig_cal_image[i, :], 'k.')
# plt.plot(orig_cal_image[19, :], 'k.', label='Original Rev-Clock')
# plt.plot(rev_model, 'r', label="Fixed")
# plt.legend(loc='best')
# plt.show()
# sys.exit()
# When we "fix" pixels, we must ensure that we are only fixing the LIS
# pixels. The cal_image has all special pixels masked out, so
# we need to create a new structure that only masks the LIS pixels.
cal_lismasked = np.ma.masked_equal(cal_vals, specialpix.Lis)
logger.info("Fixing Reverse-Clock LIS pixels.")
fixed_cal = np.ma.apply_along_axis(
fix_rev_clock, 0,
np.ma.concatenate(
(cal_lismasked, rev_model.reshape((1, rev_model.size)))))
logger.info("Writing out Reverse-Clock pixels.")
hci_dict["Calibration"] = apply_special_pixels(
fixed_cal, specialpix).data.tolist()
isis.cube.overwrite_table(out_p, t_name, hci_dict)
# Fix the buffer area:
if buffer_lisfrac > tolerance:
# first_im_line = 19+(20+label["IsisCube"]["Instrument"]["Tdi"])/binning
sp_frac = np.ma.count_masked(calbuf[:20, :]) / calbuf[:20, :].size
if sp_frac <= tolerance:
refr = int(np.ma.median(calbuf[:20, :]))
logger.info(
f"Median of first 20 lines of Calibration Buffer Pixels: {refr}"
)
else:
logger.info(f"More than {tolerance:.1%} of the Calibration Buffer "
f"Pixels have special pixel values: {sp_frac:.2%}.")
isp = pvl.loads(isis.catoriglab(
in_path).stdout)["INSTRUMENT_SETTING_PARAMETERS"]
adc = label["IsisCube"]["Instrument"]["ADCTimingSetting"]
if adc == -9999:
adc = isp["MRO:ADC_TIMING_SETTINGS"]
cid = get_ChannelID_fromfile(in_path)
refr = int(
revclock_model(
cid.ccdname + cid.ccdnumber + "_" + cid.channel, binning,
label["IsisCube"]["Instrument"]
["FpaPositiveYTemperature"].value, adc))
logger.info(f"Using a model-based substitute ({refr}).")
# import hiproc.img as img
# lut = img.LUT_Table(isp["MRO:LOOKUP_CONVERSION_TABLE"])
# if refr <= lut.table[1]:
# logger.error(f"Using a model-based substitute ({refr}).")
# else:
# logger.error(
# f"Model ({refr}) was greater than LUT floor "
# f"({lut.table[1]}). Setting to LUT floor."
# )
# refr = lut.table[1]
if np.ma.count_masked(dark[:20]) / dark[:20].size <= tolerance:
refd = int(np.ma.median(dark[:20]))
logger.info(f"Median of first 20 lines of Dark Pixels: {refd}")
else:
refd = int(np.ma.median(fixed_cal[:20, :]))
logger.info(
f"More than {tolerance:.1%} of the first 20 lines of "
f"Dark Pixels have a real "
f"value: {np.ma.count_masked(dark[:20]) / dark[:20].size} "
f"Using the rev-clock median ({refd}).")
model_buffer = dark[:, 2:14] - refd + refr
# print(model_buffer)
# print(model_buffer.dtype)
# print(model_buffer.shape)
# # Mask out all of the buffer for testing, so we get wholesale
# # replacement
# buffer.mask = True
logger.info("Fixing Buffer pixels.")
buffer_lismasked = np.ma.masked_equal(buffer_vals, specialpix.Lis)
fixed_buf = np.ma.apply_along_axis(
fix_buffer, 1,
np.ma.concatenate((buffer_lismasked, model_buffer), 1),
buffer_lismasked.shape[1])
# print(fixed_buf)
# print(fixed_buf.dtype)
# print(fixed_buf.shape)
ha_dict["BufferPixels"] = apply_special_pixels(
fixed_buf, specialpix).data.tolist()
logger.info("Writing out Buffer pixels.")
isis.cube.overwrite_table(out_p, "HiRISE Ancillary", ha_dict)
return fixed