def main(): args = arg_parser().parse_args() util.set_logger(args.verbose, args.logfile, args.log) if len(args.img) > 1 and not args.output.startswith("."): logger.critical( "With more than one input IMG file, the --output must start with " f"a period, and it does not: {args.output}" ) sys.exit() gainsinfo = pvl.load(args.gains) for i in args.img: out_p = util.path_w_suffix(args.output, i) with util.main_exceptions(args.verbose): histats = EDR_Stats( i, out_p, gainsinfo, args.histmin, args.histmax, keep=args.keep ) # DB stuff # for k, v in histats.items(): # print(f'{k}: {v}') db_path = util.path_w_suffix(args.db, i) with open(db_path, "w") as f: json.dump(histats, f, indent=0, sort_keys=True) logger.info(f"Wrote {db_path}") return
def main(): try: parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, parents=[util.parent_parser()], ) parser.add_argument( "-o", "--output", required=False, default=".bitunflip.cub" ) parser.add_argument("cube", help="ISIS Cube file.") args = parser.parse_args() util.set_logger(args.verbose, args.logfile, args.log) out_p = util.path_w_suffix(args.output, args.cube) unflip(Path(args.cube), out_p, keep=args.keep) except subprocess.CalledProcessError as err: print("Had an ISIS error:", file=sys.stderr) print(" ".join(err.cmd), file=sys.stderr) print(err.stdout, file=sys.stderr) print(err.stderr, file=sys.stderr)
def main(): # The original Perl needed the specific output of cubenorm from a # particular step in the HiCal pipeline before here. However, rather # than keep track of those files, open them, and read them again, we # can (and did) perform the relevant check in HiCal.py, and then save # that in the db that we can check now. args = arg_parser().parse_args() util.set_logger(args.verbose, args.logfile, args.log) with util.main_exceptions(args.verbose): (db, outcub_path) = HiStitch( args.cube0, args.cube1, get_db(util.pid_path_w_suffix(args.db, args.cube0)), get_db(util.pid_path_w_suffix(args.db2, args.cube1)), args.output, pvl.load(args.conf), keep=args.keep, ) db_path = set_outpath(args.dbout, hirise.get_CCDID_fromfile(outcub_path), outcub_path.parent) with open(db_path, "w") as f: json.dump(db, f, indent=0, sort_keys=True) logger.info(f"Wrote {db_path}") return
def main(): args = arg_parser().parse_args() util.set_logger(args.verbose, args.logfile, args.log) with util.main_exceptions(args.verbose): HiSlither(args.cubes, keep=args.keep) return
def main(): args = arg_parser().parse_args() util.set_logger(args.verbose, args.logfile, args.log) for s in args.slithertxts: slitherstats = Polyfit(s, args.plot) print(f"Stats for {s}") print("Max Diff: {:.2f}\nAve Diff: {:.2f}\nStd Dev: {:.2f}".format( *slitherstats))
def main(): args = arg_parser().parse_args() util.set_logger(args.verbose, args.logfile, args.log) if len(args.img) > 1: if not args.output.startswith("."): logger.critical( "With more than one input IMG file, the --output must start " f"with a period, and it does not: {args.output}" ) sys.exit() if not args.db.startswith("."): logger.critical( "With more than one input IMG file, the --db must start with " f"a period, and it does not: {args.db}" ) sys.exit() gainsinfo = pvl.load(args.gains) with util.main_exceptions(args.verbose): if len(args.img) == 1: # No need to fire up the multiprocessing for just one image. db_path = write_json( EDR_Stats( args.img[0], util.path_w_suffix(args.output, args.img[0]), gainsinfo, args.histmin, args.histmax, keep=args.keep ), args.db, args.img[0], ) logger.info(f"Wrote {db_path}") else: with concurrent.futures.ProcessPoolExecutor( max_workers=args.max_workers ) as executor: for img, histats, in zip(args.img, executor.map( EDR_Stats, args.img, map(util.path_w_suffix, repeat(args.output), args.img), repeat(gainsinfo), repeat(args.histmin), repeat(args.histmax), repeat(args.keep), )): db_path = write_json(histats, args.db, img) logger.info(f"Wrote {db_path}") return
def main(): args = arg_parser().parse_args() util.set_logger(args.verbose, args.logfile, args.log) with util.main_exceptions(args.verbose): HiBeautify(args.cubes, pvl.load(args.conf), args.output_irb, args.output_rgb, keep=args.keep) return
def main(): args = arg_parser().parse_args() util.set_logger(args.verbose, args.logfile, args.log) with util.main_exceptions(args.verbose): (ir_ratio, bg_ratio) = HiColorNorm( args.cubes, args.output, pvl.load(args.conf), make_unfiltered=args.Make_Unfiltered, keep=args.keep, )
def main(): args = arg_parser().parse_args() util.set_logger(args.verbose, args.logfile, args.log) with util.main_exceptions(args.verbose): successful_ccds = HiJitReg(args.cubes, pvl.load(args.conf), keep=args.keep) print("Successful CCDs are:") for c in successful_ccds: print("\t{}".format(str(c))) return
def main(): args = arg_parser().parse_args() util.set_logger(args.verbose, args.logfile, args.log) with util.main_exceptions(args.verbose): HiNoProj( args.cubes, pvl.load(args.conf), args.output, args.base_ccd_number, keep=args.keep, ) return
def main(): args = arg_parser().parse_args() util.set_logger(args.verbose, args.logfile, args.log) if not args.output_suffix.startswith("."): logger.critical("--output_suffix must start with a period, and it " f"does not: {args.output_suffix}") sys.exit() with util.main_exceptions(args.verbose): HiColorInit(args.cubes, args.output_suffix, keep=args.keep) return
def main(): args = arg_parser().parse_args() util.set_logger(args.verbose, args.logfile, args.log) with util.main_exceptions(args.verbose): HiJACK( args.cubes, args.conf_dir, args.out_dir, args.base_ccd_number, plot=args.plot, keep=args.keep, ) return
def main(): args = arg_parser().parse_args() util.set_logger(args.verbose, args.logfile, args.log) oid = hirise.get_ObsID_fromfile(args.img[0]) parent = Path(args.img[0]).parent if args.img == 1: imgs = parent.glob(f"{oid}*.img") + parent.glob(f"{oid}*.IMG") else: imgs = args.img db_list = None with util.main_exceptions(args.verbose): try: get_cubes(f"{oid}*balance.cub", parent) except FileNotFoundError: chancubes = edr2stitch( imgs, args.conf_dir, max_workers=args.max_workers, keep=args.keep ) db_list = [x.db for x in chancubes] if args.color: color(oid, args.conf_dir, parent, db_list, keep=args.keep) if args.precision or args.jack: jack = False if args.jack: jack = True else: slithers = list(parent.glob("*.slither.txt")) # HiPrecisionInit to determine if you need to HiNoProj or # HiJACK takes *slither.txt (HiJACK_flags, _, _) = HiPrecisionInit.check( slithers, pvl.load(args.conf_dir / "HiPrecisionInit.conf") ) if any(HiJACK_flags): jack = True precision(oid, args.conf_dir, parent, hijack=jack, keep=args.keep) return
def main(): args = arg_parser().parse_args() util.set_logger(args.verbose, args.logfile, args.log) out_p = util.path_w_suffix(args.output, args.file) with util.main_exceptions(args.verbose): fixed = fix( args.file, out_p, ) if not fixed: sys.exit(f"{args.file} did not need lisfix cleaning.") return
def main(): args = arg_parser().parse_args() # Ignore args.log to always print info when run from the command line. util.set_logger("info", args.logfile, args.log) with util.main_exceptions(args.verbose): hijack, averages, thresh = check(args.slither_text, pvl.load(args.conf)) print(f"Mean_Jitter_Magnitude_Threshold: {thresh}") print("Average\tProcess \tFile Name") for a, j, s in zip(averages, hijack, args.slither_text): print("{:.2}\t{}\t{}".format(a, "HiJACK " if j else "HiNoProj", s)) if any(hijack): print("Probably should run HiJACK.") else: print("Can safely run NoProj.") return
def main(): # The Original Perl took a .pvl file as input which mostly just had the # filenames of the ccd files to stitch together. We'll just take those # on the command line and into args.cubes. args = arg_parser().parse_args() util.set_logger(args.verbose, args.logfile, args.log) # outcub_path = set_outcube(args.output, pid0) if args.cubenorm is not None: if len(args.cubenorm) != len(args.cubes): logger.critical(f"The number of cubes ({len(args.cubes)}) and " "the number of cubenorm flags given " f"({len(args.cubenorm)}) did not match. Exiting.") sys.exit() else: args.cubenorm = [False] * len(args.cubes) # Perl: GetConfigurationParameters() conf = pvl.load(args.conf) conf_check(conf) # We may not need to read anything from the DB, only write to it? # Nope, we need to read in each of the CHANNEL!(!) HiCat files to get # their hical_status value. Hmm. Rather than devise a # command line strategy for manually loading them, I think we'll just # use the args.cubes filenames to find them. # Perl: GetPVLParameters() # Gets some options and items from the pvl file that the. # Need to recreate? # Looks like there's a True/False for each input about Cubenorming or not # Then a HiccdStitch/Start_line, HiccdStitch/End_Line, # and a HiccdStitch/Reduce_Factor # If those aren't present then then default to 0, 0, -9999. Probably # should all be 'None's. # Upon inspection of the HiStitch_Next_Pipe, there is no logic to set these # values, they are simply always FALSE for each CCD, 0, 0, and -9999. # So maybe this was put in for manual use, but not a 'normal' part of the # pipeline? # Adding as command line arguments if len(args.cubes) == 1: # Assume this is a filepath to a file of cube names cubes = list( map( HiccdStitchCube, Path(args.cubes[0]).read_text().splitlines(), args.cubenorm, )) else: cubes = list(map(HiccdStitchCube, args.cubes, args.cubenorm)) for c in cubes: c.gather_from_db() with util.main_exceptions(args.verbose): (db, outpath) = HiccdStitch( cubes, args.output, conf, args.sline, args.eline, keep=args.keep, ) db_path = util.path_w_suffix(args.db, outpath) with open(db_path, "w") as f: json.dump(db, f, indent=0, sort_keys=True) return
def main(): args = arg_parser().parse_args() util.set_logger(args.verbose, args.logfile, args.log) conf = pvl.load(args.conf) with util.main_exceptions(args.verbose): cubes = list(map(hicolor.HiColorCube, args.cubes)) (red4, red5, ir10, ir11, bg12, bg13) = hicolor.separate_ccds(cubes) plt.ioff() fig, axes = plt.subplots(1, 4, sharex=True, sharey=True) axes[0].set_ylabel("Line") fig.text(0.5, 0.04, "Error Magnitude (pixels)", ha="center", va="center") for c, ax in zip([ir10, ir11, bg12, bg13], axes): if c is None: continue logger.info(f"Working on {c}") j = hjr.JitterCube(c, conf) j.reset() (accepted, rejected, smoothed) = filter_and_smooth(j) ax.set_title(j.get_ccd()) size = 5 if len(accepted) > 0: acceptA = np.array(accepted) ax.scatter( acceptA[:, 1], acceptA[:, 0], s=size, label="Accepted", facecolors="none", edgecolors="black", ) if len(rejected) > 0: rejectedA = np.array(rejected) ax.scatter( rejectedA[:, 1], rejectedA[:, 0], c="red", s=size, label="Rejected", ) if len(smoothed) > 0: smoothedA = np.array(smoothed) ax.scatter( smoothedA[:, 1], smoothedA[:, 0], c="blue", s=size, label="Smoothed", ) fig.suptitle(str(j.get_obsid())) bottom, top = plt.ylim() plt.ylim(top, bottom) plt.xlim(0, 10) plt.legend(loc="upper right", bbox_to_anchor=(1, -0.05), ncol=3) plt.show() return
def main(): parser = arg_parser() args = parser.parse_args() util.set_logger(args.verbose, args.logfile, args.log) if len(args.files) == 3: # With just three arguments, these are the expected flat files. fp1, fp2, fp3 = map(Path, args.files) which1 = args.which1 which2 = args.which2 which3 = args.which3 elif len(args.files) == 9: # This is the old-style positional calling oldparser = argparse.ArgumentParser() oldparser.add_argument("outdir", type=Path) oldparser.add_argument("outprefix", type=str) oldparser.add_argument("lineinterval", type=float) oldparser.add_argument("file_path1", type=Path) oldparser.add_argument("which1", type=int, choices=[-1, 1]) oldparser.add_argument("file_path2", type=Path) oldparser.add_argument("which2", type=int, choices=[-1, 1]) oldparser.add_argument("file_path3", type=Path) oldparser.add_argument("which3", type=int, choices=[-1, 1]) args = oldparser.parse_args(args.files, namespace=args) fp1 = args.file_path1 fp2 = args.file_path2 fp3 = args.file_path3 which1 = True if args.which1 != 1 else False which2 = True if args.which2 != 1 else False which3 = True if args.which2 != 1 else False else: parser.error("Only takes 3 or 9 positional arguments.") if args.lineinterval is None: args.lineinterval = pvl.load(args.conf)["AutoRegistration"][ "ControlNet" ]["Control_Lines"] elif args.lineinterval <= 0: raise ValueError("--lineinterval must be positive.") if args.outdir is None: outdir = fp1.parent else: outdir = args.outdir fp1 = set_file_path(outdir, fp1) fp2 = set_file_path(outdir, fp2) fp3 = set_file_path(outdir, fp3) with util.main_exceptions(args.verbose): start( fp1, which1, fp2, which2, fp3, which3, line_interval=args.lineinterval, outdir=outdir, outprefix=args.outprefix, plotshow=args.plot, plotsave=args.saveplot, writecsv=args.csv, ) return
def test_logging(self): util.set_logger(loglvl="WARNING") logger = logging.getLogger() self.assertEqual(30, logger.getEffectiveLevel())
def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "-o", "--output", required=False, default=".mdr.iof.cub" ) parser.add_argument("-e", "--edr", required=True) parser.add_argument( "-c", "--conf", required=False, type=argparse.FileType('r'), default=pkg_resources.resource_stream( __name__, 'data/hical.pipelines.conf' ), ) parser.add_argument("mdr", metavar="MDR_file") parser.add_argument( "-l", "--log", required=False, default="WARNING", help="The log level to show for this program, can " "be a named log level or a numerical level.", ) parser.add_argument( "-k", "--keep", required=False, default=False, action="store_true", help="Normally, the program will clean up any " "intermediary files, but if this option is given, it " "won't.", ) args = parser.parse_args() util.set_logger(args.verbose, args.logfile, args.log) edr_path = Path(args.edr) mdr_path = Path(args.mdr) to_del = isis.PathSet() h2i_path = to_del.add(edr_path.with_suffix(".hi2isis.cub")) out_path = util.path_w_suffix(args.output, edr_path) # The first thing Alan's program did was to crop the image down to only the # 'imaging' parts. We're not doing that so the resultant file has a # geometry similar to what comes out of ISIS hical. # Convert the EDR to a cube file isis.hi2isis(edr_path, to=h2i_path) # Convert Alan's MDR to a cube file mdr_cub_path = to_del.add(mdr_path.with_suffix(".alan.cub")) logger.info(f"Running gdal_translate {mdr_path} -of ISIS3 {mdr_cub_path}") gdal.Translate(str(mdr_cub_path), str(mdr_path), format="ISIS3") h2i_s = int(isis.getkey_k(h2i_path, "Dimensions", "Samples")) h2i_l = int(isis.getkey_k(h2i_path, "Dimensions", "Lines")) mdr_s = int(isis.getkey_k(mdr_cub_path, "Dimensions", "Samples")) mdr_l = int(isis.getkey_k(mdr_cub_path, "Dimensions", "Lines")) if h2i_s != mdr_s: label = pvl.load(str(h2i_path)) hirise_cal_info = get_one( label, "Table", "HiRISE Calibration Ancillary" ) buffer_pixels = get_one(hirise_cal_info, "Field", "BufferPixels")[ "Size" ] dark_pixels = get_one(hirise_cal_info, "Field", "DarkPixels")["Size"] rev_mask_tdi_lines = hirise_cal_info["Records"] if h2i_s + buffer_pixels + dark_pixels == mdr_s: logger.info( f"The file {mdr_cub_path} has " f"{buffer_pixels + dark_pixels} more sample pixels " f"than {h2i_path}, assuming those are dark and " "buffer pixels and will crop accordingly." ) if h2i_l + rev_mask_tdi_lines != mdr_l: logger.critical( 'Even assuming this is a "full" channel ' "image, this has the wrong number of lines. " f"{mdr_cub_path} should have " f"{h2i_l + rev_mask_tdi_lines}, but " f"has {mdr_l} lines. Exiting" ) sys.exit() else: crop_path = to_del.add(mdr_cub_path.with_suffix(".crop.cub")) # We want to start with the next pixel (+1) after the cal # pixels. isis.crop( mdr_cub_path, to=crop_path, sample=buffer_pixels + 1, nsamples=h2i_s, line=rev_mask_tdi_lines + 1, ) mdr_cub_path = crop_path mdr_l = int(isis.getkey_k(mdr_cub_path, "Dimensions", "Lines")) else: logger.critical( f"The number of samples in {h2i_path} ({h2i_s}) " f"and {mdr_cub_path} ({mdr_s}) are different. " "Exiting." ) sys.exit() if h2i_l != mdr_l: logger.critical( f"The number of lines in {h2i_path} ({h2i_l}) " f"and {mdr_cub_path} ({mdr_l}) are different. " "Exiting." ) sys.exit() # Convert the EDR to the right bit type for post-HiCal Pipeline: h2i_16b_p = to_del.add(h2i_path.with_suffix(".16bit.cub")) isis.bit2bit( h2i_path, to=h2i_16b_p, bit="16bit", clip="minmax", minval=0, maxval=1.5, ) shutil.copyfile(h2i_16b_p, out_path) # If it is a channel 1 file, Alan mirrored it so that he could process # the two channels in an identical way (which we also took advantage # of above if the buffer and dark pixels were included), so we need to # mirror it back. cid = hirise.get_ChannelID_fromfile(h2i_16b_p) if cid.channel == "1": mirror_path = to_del.add(mdr_cub_path.with_suffix(".mirror.cub")) isis.mirror(mdr_cub_path, to=mirror_path) mdr_cub_path = mirror_path # Is the MDR in DN or I/F? maximum_pxl = float( pvl.loads(isis.stats(mdr_cub_path).stdout)["Results"]["Maximum"] ) if maximum_pxl < 1.5: logger.info("MDR is already in I/F units.") mdr_16b_p = to_del.add(mdr_cub_path.with_suffix(".16bit.cub")) isis.bit2bit( mdr_cub_path, to=mdr_16b_p, bit="16bit", clip="minmax", minval=0, maxval=1.5, ) isis.handmos(mdr_16b_p, mosaic=out_path) else: logger.info("MDR is in DN units and will be converted to I/F.") fpa_t = statistics.mean( [ float( isis.getkey_k( h2i_16b_p, "Instrument", "FpaPositiveYTemperature" ) ), float( isis.getkey_k( h2i_16b_p, "Instrument", "FpaNegativeYTemperature" ) ), ] ) print(f"fpa_t {fpa_t}") conf = pvl.load(args.conf) tdg = t_dep_gain(get_one(conf["Hical"], "Profile", cid.ccdname), fpa_t) suncorr = solar_correction() sclk = isis.getkey_k( h2i_16b_p, "Instrument", "SpacecraftClockStartCount" ) target = isis.getkey_k(h2i_16b_p, "Instrument", "TargetName") suncorr = solar_correction(sunDistanceAU(sclk, target)) sed = float( isis.getkey_k(h2i_16b_p, "Instrument", "LineExposureDuration") ) zbin = get_one(conf["Hical"], "Profile", "GainUnitConversion")[ "GainUnitConversionBinFactor" ] # The 'ziof' name is from the ISIS HiCal/GainUnitConversion.h, it is a # divisor in the calibration equation. print(f"zbin {zbin}") print(f"tdg {tdg}") print(f"sed {sed}") print(f"suncorr {suncorr}") ziof = zbin * tdg * sed * 1e-6 * suncorr eqn = f"\(F1 / {ziof})" # noqa W605 mdriof_p = to_del.add(mdr_cub_path.with_suffix(".iof.cub")) to_s = "{}+SignedWord+{}:{}".format(mdriof_p, 0, 1.5) isis.fx(f1=mdr_cub_path, to=to_s, equ=eqn) isis.handmos(mdriof_p, mosaic=out_path) if not args.keep: to_del.unlink()
def main(): # Set the numpy type for elements in the main data area of the .dat file. float_type = np.float32 parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, parents=[util.parent_parser()], ) parser.add_argument("-o", "--output", required=False, default=".bitflip.dat") parser.add_argument( "-w", "--width", required=False, default=5, help="The number of medstd widths for bit-flip " "cleaning.", ) parser.add_argument( "-r", "--replacement", required=False, default=float_type(0), type=float_type, help="By default, the program will replace " "identified pixels with an appropriate NULL data " "value, but if provided this value will be used " "instead.", ) parser.add_argument( "-p", "--plot", required=False, action="store_true", help="Displays plot for each area.", ) parser.add_argument( "-n", "--dryrun", required=False, action="store_true", help="Does not produce a cleaned output file.", ) parser.add_argument("file", help="A .dat file to clean.") args = parser.parse_args() util.set_logger(args.verbose, args.logfile, args.log) out_p = util.path_w_suffix(args.output, args.file) clean( args.file, out_p, args.replacement, width=args.width, plot=args.plot, dryrun=args.dryrun, ) sys.exit(0)