def map(self, context):
# TODO: look for a way to avoid the local write
path, signal = context.key, context.value
rr = utils.estimate_rainfall(signal)
dt_string = os.path.splitext(hdfs.path.basename(path))[0]
out_name = "%s.tif" % dt_string
dt = datetime.strptime(dt_string, IN_FMT)
metadata = {tiffio.DT_TAG: dt.strftime(tiffio.DT_FMT)}
self.ga.save_as_gtiff(out_name, rr, metadata=metadata)
with io.open(out_name, "rb") as f:
value = f.read()
context.emit(out_name, value)
def main(args):
gtiff_map = utils.scan_gtiffs(args.gtiff_dir)
dt_path_pairs = utils.get_images(args.png_dir)
if args.resolution:
groups = utils.group_images(dt_path_pairs, args.resolution)
dt_rr_stream = utils.avg_rainfall(groups)
else:
dt_rr_stream = ((dt, utils.estimate_rainfall(utils.get_image_data(_)))
for (dt, _) in dt_path_pairs)
ga = utils.GeoAdapter(args.footprint)
for dt, rr in dt_rr_stream:
assert dt in gtiff_map
check(ga, rr, dt, gtiff_map[dt])
def main(args):
dt_path_pairs = utils.get_images(args.png_img_dir)
ga = utils.GeoAdapter(args.footprint)
gtiff_map = utils.scan_gtiffs(args.gtiff_img_dir)
assert {_[0] for _ in dt_path_pairs}.issubset(gtiff_map)
wd = tempfile.mkdtemp(prefix="tdm_")
in_fn = os.path.join(wd, "orig.tif")
warped_fn = os.path.join(wd, "warped.tif")
t_srs = "EPSG:4326"
n_pairs = len(dt_path_pairs)
for i, (dt, path) in enumerate(dt_path_pairs):
rm_f(in_fn, warped_fn)
print("checking %s (%d/%d)" % (gtiff_map[dt], i + 1, n_pairs))
signal = utils.get_image_data(path)
rain = utils.estimate_rainfall(signal)
metadata = {"TIFFTAG_DATETIME": strftime(dt, TIFF_DT_FMT)}
ga.save_as_gtiff(in_fn, rain, metadata)
subprocess.check_call(["gdalwarp", "-t_srs", t_srs, in_fn, warped_fn])
compare_gtiff(gtiff_map[dt], warped_fn)
shutil.rmtree(wd)
def check_rainfall_rate(dataset, dts, img_dir, resolution=None):
rr = dataset.variables["rainfall_rate"]
dt_rr_pairs = []
for i, dt in enumerate(dts):
name = "%s.png" % strftime(dt, "%Y-%m-%d_%H:%M:%S")
signal = utils.get_image_data(os.path.join(img_dir, name))
rainfall = utils.estimate_rainfall(signal)
if resolution:
dt_rr_pairs.append((dt, rainfall))
else:
assert np.ma.allclose(rr[i], rainfall, atol=1e-4)
if not resolution:
return
delta = datetime.timedelta(seconds=resolution)
chunks = [[ma for dt, ma in g]
for _, g in utils.group_images(dt_rr_pairs, delta)]
assert len(chunks) == len(rr)
for i, c in enumerate(chunks):
avg_rr = np.ma.mean(c, axis=0)
assert np.ma.allclose(rr[i], avg_rr, atol=1e-4)
def test_gtiff(self):
signal = utils.get_image_data(self.raw_fn)
rain = utils.estimate_rainfall(signal)
ga = utils.GeoAdapter(self.template)
out_fn = os.path.join(self.wd, "sample.tif")
ga.save_as_gtiff(out_fn, rain)
dataset = gdal.Open(out_fn)
self.assertEqual(dataset.RasterCount, 1)
band = dataset.GetRasterBand(1)
m_band = band.GetMaskBand()
self.assertEqual(band.GetMaskFlags(), gdal.GMF_NODATA)
ma = np.ma.masked_array(band.ReadAsArray(),
mask=(m_band.ReadAsArray() == 0),
fill_value=band.GetNoDataValue())
self.assertTrue(np.array_equal(ma.mask, rain.mask))
self.assertTrue(np.ma.allclose(ma, rain))
self.assertEqual(dataset.GetGeoTransform(),
(ga.oX, ga.pxlW, 0, ga.oY, 0, ga.pxlH))
sr = gdal.osr.SpatialReference(wkt=dataset.GetProjectionRef())
self.assertTrue(sr.IsSame(ga.sr))
# check band_to_ma
ma2 = utils.band_to_ma(band)
self.assertTrue(np.array_equal(ma2.mask, ma.mask))
self.assertTrue(np.ma.allclose(ma2, ma))
def get_rr_stream(dt_path_pairs):
for dt, path in dt_path_pairs:
signal = utils.get_image_data(path)
yield dt, utils.estimate_rainfall(signal)