def test_get_all(self):
after, before = datetime.min, datetime.max
exp_res = self.info
ls = utils.get_images(self.wd, after, before)
self.assertEqual(len(ls), len(exp_res))
self.assertEqual(ls, exp_res)
ls = utils.get_images(self.wd)
self.assertEqual(len(ls), len(exp_res))
self.assertEqual(ls, exp_res)
def main(args):
nc_paths = get_paths(args.sim_dir)
c = cdo.Cdo()
for date_str, nc_list in nc_paths.items():
print("%s IN:" % date_str)
start_list, stop_list = [], []
for nc in nc_list:
start, stop = get_dt_range(c, nc)
print(" %s (%s to %s)" % (basename(nc), start, stop))
start_list.append(start)
stop_list.append(stop)
start, stop = min(start_list), max(stop_list)
out_subd = join(args.out_dir, date_str)
print("%s OUT:" % date_str)
print(" %s (%s to %s)" % (date_str, start, stop))
sys.stdout.flush()
try:
os.makedirs(out_subd)
except FileExistsError:
pass
pairs = utils.get_images(args.radar_dir, after=start, before=stop)
for dt, src in pairs:
out_name = "%s.png" % strftime(dt, utils.FMT)
dst = join(out_subd, out_name)
with open(src, "rb") as fi, open(dst, "wb") as fo:
fo.write(fi.read())
print()
def main(args):
try:
os.makedirs(args.out_dir)
except FileExistsError:
pass
ga = utils.GeoAdapter(args.footprint)
dt_path_pairs = utils.get_images(args.img_dir)
groups = None
if args.resolution:
group_gen = utils.group_images(dt_path_pairs, args.resolution)
groups = [(dt, list(g)) for dt, g in group_gen]
nt, t0 = len(groups), groups[0][0]
rr_stream = utils.avg_rainfall(groups)
report_int = 1
else:
nt, t0 = len(dt_path_pairs), dt_path_pairs[0][0]
rr_stream = get_rr_stream(dt_path_pairs)
report_int = 100
if args.format == "nc":
ds_path = os.path.join(args.out_dir, "%s.nc" % strftime(t0, utils.FMT))
writer = cfio.NCWriter(ds_path, ga, nt, t0, t_chunks=args.t_chunks)
elif args.format == "tif":
writer = tiffio.GTiffWriter(args.out_dir, ga)
print(" 0/%d" % nt)
for i, (dt, rr) in enumerate(rr_stream):
if ((i + 1) % report_int == 0):
print(" %d/%d" % (i + 1, nt))
writer.write(i, dt, rr)
writer.close()
def main(args):
host, port, out_dir = hdfs.path.split(args.out_dir)
fs = hdfs.hdfs(host, port)
fs.create_directory(out_dir)
join = os.path.join
for dt, path in get_images(args.in_dir):
out_path = join(out_dir, f"{dt.strftime(OUT_FMT)}.png")
if not args.overwrite and fs.exists(out_path):
continue
with io.open(path, "rb") as fi:
with fs.open_file(out_path, "wb") as fo:
fo.write(fi.read())
def main(args):
gtiff_map = utils.scan_gtiffs(args.gtiff_dir)
dt_path_pairs = utils.get_images(args.png_dir)
if args.resolution:
groups = utils.group_images(dt_path_pairs, args.resolution)
dt_rr_stream = utils.avg_rainfall(groups)
else:
dt_rr_stream = ((dt, utils.estimate_rainfall(utils.get_image_data(_)))
for (dt, _) in dt_path_pairs)
ga = utils.GeoAdapter(args.footprint)
for dt, rr in dt_rr_stream:
assert dt in gtiff_map
check(ga, rr, dt, gtiff_map[dt])
def main(args):
print("scanning %s" % args.in_dir)
dt_path_pairs = utils.get_images(args.in_dir)
fmt = utils.FMT
for event in events.split(dt_path_pairs, min_len=args.min_len):
start_str = strftime(event[0][0], fmt)
out_subdir = join(args.out_dir, start_str)
try:
os.makedirs(out_subdir)
except FileExistsError:
pass
print(" event from: %s (%d time points)" % (start_str, len(event)))
for dt, p in event:
out_p = join(out_subdir, strftime(dt, fmt))
with open(p, "rb") as fi, open(out_p, "wb") as fo:
fo.write(fi.read())
def main(args):
dt_path_pairs = utils.get_images(args.png_img_dir)
ga = utils.GeoAdapter(args.footprint)
gtiff_map = utils.scan_gtiffs(args.gtiff_img_dir)
assert {_[0] for _ in dt_path_pairs}.issubset(gtiff_map)
wd = tempfile.mkdtemp(prefix="tdm_")
in_fn = os.path.join(wd, "orig.tif")
warped_fn = os.path.join(wd, "warped.tif")
t_srs = "EPSG:4326"
n_pairs = len(dt_path_pairs)
for i, (dt, path) in enumerate(dt_path_pairs):
rm_f(in_fn, warped_fn)
print("checking %s (%d/%d)" % (gtiff_map[dt], i + 1, n_pairs))
signal = utils.get_image_data(path)
rain = utils.estimate_rainfall(signal)
metadata = {"TIFFTAG_DATETIME": strftime(dt, TIFF_DT_FMT)}
ga.save_as_gtiff(in_fn, rain, metadata)
subprocess.check_call(["gdalwarp", "-t_srs", t_srs, in_fn, warped_fn])
compare_gtiff(gtiff_map[dt], warped_fn)
shutil.rmtree(wd)
def main(args):
try:
os.makedirs(args.out_dir)
except FileExistsError:
pass
pairs = utils.get_images(args.in_dir)
if len(pairs) == 0:
raise RuntimeError(f"no radar image found in ${args.in_dir}")
dts, paths = zip(*pairs)
print(f"{args.in_dir}: {len(pairs)} images, from {dts[0]} to {dts[-1]}")
print(f" 0/{len(paths)}")
for i, p in enumerate(paths):
if ((i + 1) % 10 == 0):
print(f" {i + 1}/{len(paths)}")
out_p = os.path.join(args.out_dir, os.path.basename(p))
if args.verbose:
print(f" writing {out_p}")
data = imread(p)
r_data = rotate(data, args.angle, reshape=False, cval=CVAL)
imwrite(out_p, r_data, optimize=args.optimize)
def test_get(self):
exp_res = self.info[2:-1]
ls = utils.get_images(self.wd, self.AFTER, self.BEFORE)
self.assertEqual(len(ls), len(exp_res))
self.assertEqual(ls, exp_res)
def check(nc_path, img_dir, footprint, resolution=None):
dts, paths = zip(*utils.get_images(img_dir))
ds = Dataset(nc_path, "r")
check_geo(ds, footprint)
check_time(ds, dts, resolution=resolution)
check_rainfall_rate(ds, dts, img_dir, resolution=resolution)