def main():
args = arg_parser().parse_args()
util.set_logger(args.verbose, args.logfile, args.log)
if len(args.img) > 1 and not args.output.startswith("."):
logger.critical(
"With more than one input IMG file, the --output must start with "
f"a period, and it does not: {args.output}"
)
sys.exit()
gainsinfo = pvl.load(args.gains)
for i in args.img:
out_p = util.path_w_suffix(args.output, i)
with util.main_exceptions(args.verbose):
histats = EDR_Stats(
i, out_p, gainsinfo, args.histmin, args.histmax, keep=args.keep
)
# DB stuff
# for k, v in histats.items():
# print(f'{k}: {v}')
db_path = util.path_w_suffix(args.db, i)
with open(db_path, "w") as f:
json.dump(histats, f, indent=0, sort_keys=True)
logger.info(f"Wrote {db_path}")
return
def main():
try:
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
parents=[util.parent_parser()],
)
parser.add_argument(
"-o", "--output", required=False, default=".bitunflip.cub"
)
parser.add_argument("cube", help="ISIS Cube file.")
args = parser.parse_args()
util.set_logger(args.verbose, args.logfile, args.log)
out_p = util.path_w_suffix(args.output, args.cube)
unflip(Path(args.cube), out_p, keep=args.keep)
except subprocess.CalledProcessError as err:
print("Had an ISIS error:", file=sys.stderr)
print(" ".join(err.cmd), file=sys.stderr)
print(err.stdout, file=sys.stderr)
print(err.stderr, file=sys.stderr)
def main():
# The original Perl needed the specific output of cubenorm from a
# particular step in the HiCal pipeline before here. However, rather
# than keep track of those files, open them, and read them again, we
# can (and did) perform the relevant check in HiCal.py, and then save
# that in the db that we can check now.
args = arg_parser().parse_args()
util.set_logger(args.verbose, args.logfile, args.log)
with util.main_exceptions(args.verbose):
(db, outcub_path) = HiStitch(
args.cube0,
args.cube1,
get_db(util.pid_path_w_suffix(args.db, args.cube0)),
get_db(util.pid_path_w_suffix(args.db2, args.cube1)),
args.output,
pvl.load(args.conf),
keep=args.keep,
)
db_path = set_outpath(args.dbout, hirise.get_CCDID_fromfile(outcub_path),
outcub_path.parent)
with open(db_path, "w") as f:
json.dump(db, f, indent=0, sort_keys=True)
logger.info(f"Wrote {db_path}")
return
def main():
args = arg_parser().parse_args()
util.set_logger(args.verbose, args.logfile, args.log)
with util.main_exceptions(args.verbose):
HiSlither(args.cubes, keep=args.keep)
return
def main():
args = arg_parser().parse_args()
util.set_logger(args.verbose, args.logfile, args.log)
for s in args.slithertxts:
slitherstats = Polyfit(s, args.plot)
print(f"Stats for {s}")
print("Max Diff: {:.2f}\nAve Diff: {:.2f}\nStd Dev: {:.2f}".format(
*slitherstats))
def main():
args = arg_parser().parse_args()
util.set_logger(args.verbose, args.logfile, args.log)
if len(args.img) > 1:
if not args.output.startswith("."):
logger.critical(
"With more than one input IMG file, the --output must start "
f"with a period, and it does not: {args.output}"
)
sys.exit()
if not args.db.startswith("."):
logger.critical(
"With more than one input IMG file, the --db must start with "
f"a period, and it does not: {args.db}"
)
sys.exit()
gainsinfo = pvl.load(args.gains)
with util.main_exceptions(args.verbose):
if len(args.img) == 1:
# No need to fire up the multiprocessing for just one image.
db_path = write_json(
EDR_Stats(
args.img[0],
util.path_w_suffix(args.output, args.img[0]),
gainsinfo,
args.histmin,
args.histmax,
keep=args.keep
),
args.db,
args.img[0],
)
logger.info(f"Wrote {db_path}")
else:
with concurrent.futures.ProcessPoolExecutor(
max_workers=args.max_workers
) as executor:
for img, histats, in zip(args.img, executor.map(
EDR_Stats,
args.img,
map(util.path_w_suffix, repeat(args.output), args.img),
repeat(gainsinfo),
repeat(args.histmin),
repeat(args.histmax),
repeat(args.keep),
)):
db_path = write_json(histats, args.db, img)
logger.info(f"Wrote {db_path}")
return
def main():
args = arg_parser().parse_args()
util.set_logger(args.verbose, args.logfile, args.log)
with util.main_exceptions(args.verbose):
HiBeautify(args.cubes,
pvl.load(args.conf),
args.output_irb,
args.output_rgb,
keep=args.keep)
return
def main():
args = arg_parser().parse_args()
util.set_logger(args.verbose, args.logfile, args.log)
with util.main_exceptions(args.verbose):
(ir_ratio, bg_ratio) = HiColorNorm(
args.cubes,
args.output,
pvl.load(args.conf),
make_unfiltered=args.Make_Unfiltered,
keep=args.keep,
)
def main():
args = arg_parser().parse_args()
util.set_logger(args.verbose, args.logfile, args.log)
with util.main_exceptions(args.verbose):
successful_ccds = HiJitReg(args.cubes,
pvl.load(args.conf),
keep=args.keep)
print("Successful CCDs are:")
for c in successful_ccds:
print("\t{}".format(str(c)))
return
def main():
args = arg_parser().parse_args()
util.set_logger(args.verbose, args.logfile, args.log)
with util.main_exceptions(args.verbose):
HiNoProj(
args.cubes,
pvl.load(args.conf),
args.output,
args.base_ccd_number,
keep=args.keep,
)
return
def main():
args = arg_parser().parse_args()
util.set_logger(args.verbose, args.logfile, args.log)
if not args.output_suffix.startswith("."):
logger.critical("--output_suffix must start with a period, and it "
f"does not: {args.output_suffix}")
sys.exit()
with util.main_exceptions(args.verbose):
HiColorInit(args.cubes, args.output_suffix, keep=args.keep)
return
def main():
args = arg_parser().parse_args()
util.set_logger(args.verbose, args.logfile, args.log)
with util.main_exceptions(args.verbose):
HiJACK(
args.cubes,
args.conf_dir,
args.out_dir,
args.base_ccd_number,
plot=args.plot,
keep=args.keep,
)
return
def main():
args = arg_parser().parse_args()
util.set_logger(args.verbose, args.logfile, args.log)
oid = hirise.get_ObsID_fromfile(args.img[0])
parent = Path(args.img[0]).parent
if args.img == 1:
imgs = parent.glob(f"{oid}*.img") + parent.glob(f"{oid}*.IMG")
else:
imgs = args.img
db_list = None
with util.main_exceptions(args.verbose):
try:
get_cubes(f"{oid}*balance.cub", parent)
except FileNotFoundError:
chancubes = edr2stitch(
imgs,
args.conf_dir,
max_workers=args.max_workers,
keep=args.keep
)
db_list = [x.db for x in chancubes]
if args.color:
color(oid, args.conf_dir, parent, db_list, keep=args.keep)
if args.precision or args.jack:
jack = False
if args.jack:
jack = True
else:
slithers = list(parent.glob("*.slither.txt"))
# HiPrecisionInit to determine if you need to HiNoProj or
# HiJACK takes *slither.txt
(HiJACK_flags, _, _) = HiPrecisionInit.check(
slithers, pvl.load(args.conf_dir / "HiPrecisionInit.conf")
)
if any(HiJACK_flags):
jack = True
precision(oid, args.conf_dir, parent, hijack=jack, keep=args.keep)
return
def main():
args = arg_parser().parse_args()
util.set_logger(args.verbose, args.logfile, args.log)
out_p = util.path_w_suffix(args.output, args.file)
with util.main_exceptions(args.verbose):
fixed = fix(
args.file,
out_p,
)
if not fixed:
sys.exit(f"{args.file} did not need lisfix cleaning.")
return
def main():
args = arg_parser().parse_args()
# Ignore args.log to always print info when run from the command line.
util.set_logger("info", args.logfile, args.log)
with util.main_exceptions(args.verbose):
hijack, averages, thresh = check(args.slither_text,
pvl.load(args.conf))
print(f"Mean_Jitter_Magnitude_Threshold: {thresh}")
print("Average\tProcess \tFile Name")
for a, j, s in zip(averages, hijack, args.slither_text):
print("{:.2}\t{}\t{}".format(a, "HiJACK " if j else "HiNoProj", s))
if any(hijack):
print("Probably should run HiJACK.")
else:
print("Can safely run NoProj.")
return
def main():
# The Original Perl took a .pvl file as input which mostly just had the
# filenames of the ccd files to stitch together. We'll just take those
# on the command line and into args.cubes.
args = arg_parser().parse_args()
util.set_logger(args.verbose, args.logfile, args.log)
# outcub_path = set_outcube(args.output, pid0)
if args.cubenorm is not None:
if len(args.cubenorm) != len(args.cubes):
logger.critical(f"The number of cubes ({len(args.cubes)}) and "
"the number of cubenorm flags given "
f"({len(args.cubenorm)}) did not match. Exiting.")
sys.exit()
else:
args.cubenorm = [False] * len(args.cubes)
# Perl: GetConfigurationParameters()
conf = pvl.load(args.conf)
conf_check(conf)
# We may not need to read anything from the DB, only write to it?
# Nope, we need to read in each of the CHANNEL!(!) HiCat files to get
# their hical_status value. Hmm. Rather than devise a
# command line strategy for manually loading them, I think we'll just
# use the args.cubes filenames to find them.
# Perl: GetPVLParameters()
# Gets some options and items from the pvl file that the.
# Need to recreate?
# Looks like there's a True/False for each input about Cubenorming or not
# Then a HiccdStitch/Start_line, HiccdStitch/End_Line,
# and a HiccdStitch/Reduce_Factor
# If those aren't present then then default to 0, 0, -9999. Probably
# should all be 'None's.
# Upon inspection of the HiStitch_Next_Pipe, there is no logic to set these
# values, they are simply always FALSE for each CCD, 0, 0, and -9999.
# So maybe this was put in for manual use, but not a 'normal' part of the
# pipeline?
# Adding as command line arguments
if len(args.cubes) == 1:
# Assume this is a filepath to a file of cube names
cubes = list(
map(
HiccdStitchCube,
Path(args.cubes[0]).read_text().splitlines(),
args.cubenorm,
))
else:
cubes = list(map(HiccdStitchCube, args.cubes, args.cubenorm))
for c in cubes:
c.gather_from_db()
with util.main_exceptions(args.verbose):
(db, outpath) = HiccdStitch(
cubes,
args.output,
conf,
args.sline,
args.eline,
keep=args.keep,
)
db_path = util.path_w_suffix(args.db, outpath)
with open(db_path, "w") as f:
json.dump(db, f, indent=0, sort_keys=True)
return
def main():
args = arg_parser().parse_args()
util.set_logger(args.verbose, args.logfile, args.log)
conf = pvl.load(args.conf)
with util.main_exceptions(args.verbose):
cubes = list(map(hicolor.HiColorCube, args.cubes))
(red4, red5, ir10, ir11, bg12, bg13) = hicolor.separate_ccds(cubes)
plt.ioff()
fig, axes = plt.subplots(1, 4, sharex=True, sharey=True)
axes[0].set_ylabel("Line")
fig.text(0.5,
0.04,
"Error Magnitude (pixels)",
ha="center",
va="center")
for c, ax in zip([ir10, ir11, bg12, bg13], axes):
if c is None:
continue
logger.info(f"Working on {c}")
j = hjr.JitterCube(c, conf)
j.reset()
(accepted, rejected, smoothed) = filter_and_smooth(j)
ax.set_title(j.get_ccd())
size = 5
if len(accepted) > 0:
acceptA = np.array(accepted)
ax.scatter(
acceptA[:, 1],
acceptA[:, 0],
s=size,
label="Accepted",
facecolors="none",
edgecolors="black",
)
if len(rejected) > 0:
rejectedA = np.array(rejected)
ax.scatter(
rejectedA[:, 1],
rejectedA[:, 0],
c="red",
s=size,
label="Rejected",
)
if len(smoothed) > 0:
smoothedA = np.array(smoothed)
ax.scatter(
smoothedA[:, 1],
smoothedA[:, 0],
c="blue",
s=size,
label="Smoothed",
)
fig.suptitle(str(j.get_obsid()))
bottom, top = plt.ylim()
plt.ylim(top, bottom)
plt.xlim(0, 10)
plt.legend(loc="upper right", bbox_to_anchor=(1, -0.05), ncol=3)
plt.show()
return
def main():
parser = arg_parser()
args = parser.parse_args()
util.set_logger(args.verbose, args.logfile, args.log)
if len(args.files) == 3:
# With just three arguments, these are the expected flat files.
fp1, fp2, fp3 = map(Path, args.files)
which1 = args.which1
which2 = args.which2
which3 = args.which3
elif len(args.files) == 9:
# This is the old-style positional calling
oldparser = argparse.ArgumentParser()
oldparser.add_argument("outdir", type=Path)
oldparser.add_argument("outprefix", type=str)
oldparser.add_argument("lineinterval", type=float)
oldparser.add_argument("file_path1", type=Path)
oldparser.add_argument("which1", type=int, choices=[-1, 1])
oldparser.add_argument("file_path2", type=Path)
oldparser.add_argument("which2", type=int, choices=[-1, 1])
oldparser.add_argument("file_path3", type=Path)
oldparser.add_argument("which3", type=int, choices=[-1, 1])
args = oldparser.parse_args(args.files, namespace=args)
fp1 = args.file_path1
fp2 = args.file_path2
fp3 = args.file_path3
which1 = True if args.which1 != 1 else False
which2 = True if args.which2 != 1 else False
which3 = True if args.which2 != 1 else False
else:
parser.error("Only takes 3 or 9 positional arguments.")
if args.lineinterval is None:
args.lineinterval = pvl.load(args.conf)["AutoRegistration"][
"ControlNet"
]["Control_Lines"]
elif args.lineinterval <= 0:
raise ValueError("--lineinterval must be positive.")
if args.outdir is None:
outdir = fp1.parent
else:
outdir = args.outdir
fp1 = set_file_path(outdir, fp1)
fp2 = set_file_path(outdir, fp2)
fp3 = set_file_path(outdir, fp3)
with util.main_exceptions(args.verbose):
start(
fp1,
which1,
fp2,
which2,
fp3,
which3,
line_interval=args.lineinterval,
outdir=outdir,
outprefix=args.outprefix,
plotshow=args.plot,
plotsave=args.saveplot,
writecsv=args.csv,
)
return
def test_logging(self):
util.set_logger(loglvl="WARNING")
logger = logging.getLogger()
self.assertEqual(30, logger.getEffectiveLevel())
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"-o", "--output", required=False, default=".mdr.iof.cub"
)
parser.add_argument("-e", "--edr", required=True)
parser.add_argument(
"-c",
"--conf",
required=False,
type=argparse.FileType('r'),
default=pkg_resources.resource_stream(
__name__,
'data/hical.pipelines.conf'
),
)
parser.add_argument("mdr", metavar="MDR_file")
parser.add_argument(
"-l",
"--log",
required=False,
default="WARNING",
help="The log level to show for this program, can "
"be a named log level or a numerical level.",
)
parser.add_argument(
"-k",
"--keep",
required=False,
default=False,
action="store_true",
help="Normally, the program will clean up any "
"intermediary files, but if this option is given, it "
"won't.",
)
args = parser.parse_args()
util.set_logger(args.verbose, args.logfile, args.log)
edr_path = Path(args.edr)
mdr_path = Path(args.mdr)
to_del = isis.PathSet()
h2i_path = to_del.add(edr_path.with_suffix(".hi2isis.cub"))
out_path = util.path_w_suffix(args.output, edr_path)
# The first thing Alan's program did was to crop the image down to only the
# 'imaging' parts. We're not doing that so the resultant file has a
# geometry similar to what comes out of ISIS hical.
# Convert the EDR to a cube file
isis.hi2isis(edr_path, to=h2i_path)
# Convert Alan's MDR to a cube file
mdr_cub_path = to_del.add(mdr_path.with_suffix(".alan.cub"))
logger.info(f"Running gdal_translate {mdr_path} -of ISIS3 {mdr_cub_path}")
gdal.Translate(str(mdr_cub_path), str(mdr_path), format="ISIS3")
h2i_s = int(isis.getkey_k(h2i_path, "Dimensions", "Samples"))
h2i_l = int(isis.getkey_k(h2i_path, "Dimensions", "Lines"))
mdr_s = int(isis.getkey_k(mdr_cub_path, "Dimensions", "Samples"))
mdr_l = int(isis.getkey_k(mdr_cub_path, "Dimensions", "Lines"))
if h2i_s != mdr_s:
label = pvl.load(str(h2i_path))
hirise_cal_info = get_one(
label, "Table", "HiRISE Calibration Ancillary"
)
buffer_pixels = get_one(hirise_cal_info, "Field", "BufferPixels")[
"Size"
]
dark_pixels = get_one(hirise_cal_info, "Field", "DarkPixels")["Size"]
rev_mask_tdi_lines = hirise_cal_info["Records"]
if h2i_s + buffer_pixels + dark_pixels == mdr_s:
logger.info(
f"The file {mdr_cub_path} has "
f"{buffer_pixels + dark_pixels} more sample pixels "
f"than {h2i_path}, assuming those are dark and "
"buffer pixels and will crop accordingly."
)
if h2i_l + rev_mask_tdi_lines != mdr_l:
logger.critical(
'Even assuming this is a "full" channel '
"image, this has the wrong number of lines. "
f"{mdr_cub_path} should have "
f"{h2i_l + rev_mask_tdi_lines}, but "
f"has {mdr_l} lines. Exiting"
)
sys.exit()
else:
crop_path = to_del.add(mdr_cub_path.with_suffix(".crop.cub"))
# We want to start with the next pixel (+1) after the cal
# pixels.
isis.crop(
mdr_cub_path,
to=crop_path,
sample=buffer_pixels + 1,
nsamples=h2i_s,
line=rev_mask_tdi_lines + 1,
)
mdr_cub_path = crop_path
mdr_l = int(isis.getkey_k(mdr_cub_path, "Dimensions", "Lines"))
else:
logger.critical(
f"The number of samples in {h2i_path} ({h2i_s}) "
f"and {mdr_cub_path} ({mdr_s}) are different. "
"Exiting."
)
sys.exit()
if h2i_l != mdr_l:
logger.critical(
f"The number of lines in {h2i_path} ({h2i_l}) "
f"and {mdr_cub_path} ({mdr_l}) are different. "
"Exiting."
)
sys.exit()
# Convert the EDR to the right bit type for post-HiCal Pipeline:
h2i_16b_p = to_del.add(h2i_path.with_suffix(".16bit.cub"))
isis.bit2bit(
h2i_path,
to=h2i_16b_p,
bit="16bit",
clip="minmax",
minval=0,
maxval=1.5,
)
shutil.copyfile(h2i_16b_p, out_path)
# If it is a channel 1 file, Alan mirrored it so that he could process
# the two channels in an identical way (which we also took advantage
# of above if the buffer and dark pixels were included), so we need to
# mirror it back.
cid = hirise.get_ChannelID_fromfile(h2i_16b_p)
if cid.channel == "1":
mirror_path = to_del.add(mdr_cub_path.with_suffix(".mirror.cub"))
isis.mirror(mdr_cub_path, to=mirror_path)
mdr_cub_path = mirror_path
# Is the MDR in DN or I/F?
maximum_pxl = float(
pvl.loads(isis.stats(mdr_cub_path).stdout)["Results"]["Maximum"]
)
if maximum_pxl < 1.5:
logger.info("MDR is already in I/F units.")
mdr_16b_p = to_del.add(mdr_cub_path.with_suffix(".16bit.cub"))
isis.bit2bit(
mdr_cub_path,
to=mdr_16b_p,
bit="16bit",
clip="minmax",
minval=0,
maxval=1.5,
)
isis.handmos(mdr_16b_p, mosaic=out_path)
else:
logger.info("MDR is in DN units and will be converted to I/F.")
fpa_t = statistics.mean(
[
float(
isis.getkey_k(
h2i_16b_p, "Instrument", "FpaPositiveYTemperature"
)
),
float(
isis.getkey_k(
h2i_16b_p, "Instrument", "FpaNegativeYTemperature"
)
),
]
)
print(f"fpa_t {fpa_t}")
conf = pvl.load(args.conf)
tdg = t_dep_gain(get_one(conf["Hical"], "Profile", cid.ccdname), fpa_t)
suncorr = solar_correction()
sclk = isis.getkey_k(
h2i_16b_p, "Instrument", "SpacecraftClockStartCount"
)
target = isis.getkey_k(h2i_16b_p, "Instrument", "TargetName")
suncorr = solar_correction(sunDistanceAU(sclk, target))
sed = float(
isis.getkey_k(h2i_16b_p, "Instrument", "LineExposureDuration")
)
zbin = get_one(conf["Hical"], "Profile", "GainUnitConversion")[
"GainUnitConversionBinFactor"
]
# The 'ziof' name is from the ISIS HiCal/GainUnitConversion.h, it is a
# divisor in the calibration equation.
print(f"zbin {zbin}")
print(f"tdg {tdg}")
print(f"sed {sed}")
print(f"suncorr {suncorr}")
ziof = zbin * tdg * sed * 1e-6 * suncorr
eqn = f"\(F1 / {ziof})" # noqa W605
mdriof_p = to_del.add(mdr_cub_path.with_suffix(".iof.cub"))
to_s = "{}+SignedWord+{}:{}".format(mdriof_p, 0, 1.5)
isis.fx(f1=mdr_cub_path, to=to_s, equ=eqn)
isis.handmos(mdriof_p, mosaic=out_path)
if not args.keep:
to_del.unlink()
def main():
# Set the numpy type for elements in the main data area of the .dat file.
float_type = np.float32
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
parents=[util.parent_parser()],
)
parser.add_argument("-o",
"--output",
required=False,
default=".bitflip.dat")
parser.add_argument(
"-w",
"--width",
required=False,
default=5,
help="The number of medstd widths for bit-flip "
"cleaning.",
)
parser.add_argument(
"-r",
"--replacement",
required=False,
default=float_type(0),
type=float_type,
help="By default, the program will replace "
"identified pixels with an appropriate NULL data "
"value, but if provided this value will be used "
"instead.",
)
parser.add_argument(
"-p",
"--plot",
required=False,
action="store_true",
help="Displays plot for each area.",
)
parser.add_argument(
"-n",
"--dryrun",
required=False,
action="store_true",
help="Does not produce a cleaned output file.",
)
parser.add_argument("file", help="A .dat file to clean.")
args = parser.parse_args()
util.set_logger(args.verbose, args.logfile, args.log)
out_p = util.path_w_suffix(args.output, args.file)
clean(
args.file,
out_p,
args.replacement,
width=args.width,
plot=args.plot,
dryrun=args.dryrun,
)
sys.exit(0)
