def test_get_all(self): after, before = datetime.min, datetime.max exp_res = self.info ls = utils.get_images(self.wd, after, before) self.assertEqual(len(ls), len(exp_res)) self.assertEqual(ls, exp_res) ls = utils.get_images(self.wd) self.assertEqual(len(ls), len(exp_res)) self.assertEqual(ls, exp_res)
def main(args): nc_paths = get_paths(args.sim_dir) c = cdo.Cdo() for date_str, nc_list in nc_paths.items(): print("%s IN:" % date_str) start_list, stop_list = [], [] for nc in nc_list: start, stop = get_dt_range(c, nc) print(" %s (%s to %s)" % (basename(nc), start, stop)) start_list.append(start) stop_list.append(stop) start, stop = min(start_list), max(stop_list) out_subd = join(args.out_dir, date_str) print("%s OUT:" % date_str) print(" %s (%s to %s)" % (date_str, start, stop)) sys.stdout.flush() try: os.makedirs(out_subd) except FileExistsError: pass pairs = utils.get_images(args.radar_dir, after=start, before=stop) for dt, src in pairs: out_name = "%s.png" % strftime(dt, utils.FMT) dst = join(out_subd, out_name) with open(src, "rb") as fi, open(dst, "wb") as fo: fo.write(fi.read()) print()
def main(args): try: os.makedirs(args.out_dir) except FileExistsError: pass ga = utils.GeoAdapter(args.footprint) dt_path_pairs = utils.get_images(args.img_dir) groups = None if args.resolution: group_gen = utils.group_images(dt_path_pairs, args.resolution) groups = [(dt, list(g)) for dt, g in group_gen] nt, t0 = len(groups), groups[0][0] rr_stream = utils.avg_rainfall(groups) report_int = 1 else: nt, t0 = len(dt_path_pairs), dt_path_pairs[0][0] rr_stream = get_rr_stream(dt_path_pairs) report_int = 100 if args.format == "nc": ds_path = os.path.join(args.out_dir, "%s.nc" % strftime(t0, utils.FMT)) writer = cfio.NCWriter(ds_path, ga, nt, t0, t_chunks=args.t_chunks) elif args.format == "tif": writer = tiffio.GTiffWriter(args.out_dir, ga) print(" 0/%d" % nt) for i, (dt, rr) in enumerate(rr_stream): if ((i + 1) % report_int == 0): print(" %d/%d" % (i + 1, nt)) writer.write(i, dt, rr) writer.close()
def main(args): host, port, out_dir = hdfs.path.split(args.out_dir) fs = hdfs.hdfs(host, port) fs.create_directory(out_dir) join = os.path.join for dt, path in get_images(args.in_dir): out_path = join(out_dir, f"{dt.strftime(OUT_FMT)}.png") if not args.overwrite and fs.exists(out_path): continue with io.open(path, "rb") as fi: with fs.open_file(out_path, "wb") as fo: fo.write(fi.read())
def main(args): gtiff_map = utils.scan_gtiffs(args.gtiff_dir) dt_path_pairs = utils.get_images(args.png_dir) if args.resolution: groups = utils.group_images(dt_path_pairs, args.resolution) dt_rr_stream = utils.avg_rainfall(groups) else: dt_rr_stream = ((dt, utils.estimate_rainfall(utils.get_image_data(_))) for (dt, _) in dt_path_pairs) ga = utils.GeoAdapter(args.footprint) for dt, rr in dt_rr_stream: assert dt in gtiff_map check(ga, rr, dt, gtiff_map[dt])
def main(args): print("scanning %s" % args.in_dir) dt_path_pairs = utils.get_images(args.in_dir) fmt = utils.FMT for event in events.split(dt_path_pairs, min_len=args.min_len): start_str = strftime(event[0][0], fmt) out_subdir = join(args.out_dir, start_str) try: os.makedirs(out_subdir) except FileExistsError: pass print(" event from: %s (%d time points)" % (start_str, len(event))) for dt, p in event: out_p = join(out_subdir, strftime(dt, fmt)) with open(p, "rb") as fi, open(out_p, "wb") as fo: fo.write(fi.read())
def main(args): dt_path_pairs = utils.get_images(args.png_img_dir) ga = utils.GeoAdapter(args.footprint) gtiff_map = utils.scan_gtiffs(args.gtiff_img_dir) assert {_[0] for _ in dt_path_pairs}.issubset(gtiff_map) wd = tempfile.mkdtemp(prefix="tdm_") in_fn = os.path.join(wd, "orig.tif") warped_fn = os.path.join(wd, "warped.tif") t_srs = "EPSG:4326" n_pairs = len(dt_path_pairs) for i, (dt, path) in enumerate(dt_path_pairs): rm_f(in_fn, warped_fn) print("checking %s (%d/%d)" % (gtiff_map[dt], i + 1, n_pairs)) signal = utils.get_image_data(path) rain = utils.estimate_rainfall(signal) metadata = {"TIFFTAG_DATETIME": strftime(dt, TIFF_DT_FMT)} ga.save_as_gtiff(in_fn, rain, metadata) subprocess.check_call(["gdalwarp", "-t_srs", t_srs, in_fn, warped_fn]) compare_gtiff(gtiff_map[dt], warped_fn) shutil.rmtree(wd)
def main(args): try: os.makedirs(args.out_dir) except FileExistsError: pass pairs = utils.get_images(args.in_dir) if len(pairs) == 0: raise RuntimeError(f"no radar image found in ${args.in_dir}") dts, paths = zip(*pairs) print(f"{args.in_dir}: {len(pairs)} images, from {dts[0]} to {dts[-1]}") print(f" 0/{len(paths)}") for i, p in enumerate(paths): if ((i + 1) % 10 == 0): print(f" {i + 1}/{len(paths)}") out_p = os.path.join(args.out_dir, os.path.basename(p)) if args.verbose: print(f" writing {out_p}") data = imread(p) r_data = rotate(data, args.angle, reshape=False, cval=CVAL) imwrite(out_p, r_data, optimize=args.optimize)
def test_get(self): exp_res = self.info[2:-1] ls = utils.get_images(self.wd, self.AFTER, self.BEFORE) self.assertEqual(len(ls), len(exp_res)) self.assertEqual(ls, exp_res)
def check(nc_path, img_dir, footprint, resolution=None): dts, paths = zip(*utils.get_images(img_dir)) ds = Dataset(nc_path, "r") check_geo(ds, footprint) check_time(ds, dts, resolution=resolution) check_rainfall_rate(ds, dts, img_dir, resolution=resolution)