def test_quality_field_name(self):
obj = _acrr.new()
self.assertEqual("se.smhi.composite.distance.radar",
obj.quality_field_name)
obj.quality_field_name = "se.smhi.composite.distance.radar2"
self.assertEqual("se.smhi.composite.distance.radar2",
obj.quality_field_name)
def test_sum_different_quantity(self):
p1 = _cartesianparam.new()
p1.quantity = "DBZH"
p1.setData(reshape(array((NO_DATA, 111, 0, 111), uint8), (2, 2)))
p1.nodata, p1.undetect, p1.gain, p1.offset = NO_DATA, UNDETECT, 0.5, -32.5
d1 = _ravefield.new()
d1.addAttribute("what/gain", DEFAULT_DISTANCE_GAIN)
d1.setData(reshape(array((0, 100, 75, 50), uint8), (2, 2)))
d1.addAttribute("how/task", "se.smhi.composite.distance.radar")
p1.addQualityField(d1)
p2 = _cartesianparam.new()
p2.quantity = "TH"
p2.setData(reshape(array((NO_DATA, 111, 0, 111), uint8), (2, 2)))
p2.nodata, p2.undetect, p2.gain, p2.offset = NO_DATA, UNDETECT, 0.5, -32.5
d2 = _ravefield.new()
d2.addAttribute("what/gain", DEFAULT_DISTANCE_GAIN)
d2.setData(reshape(array((0, 100, 75, 50), uint8), (2, 2)))
d2.addAttribute("how/task", "se.smhi.composite.distance.radar")
p2.addQualityField(d2)
obj = _acrr.new()
obj.sum(p1, 200.0, 1.6)
try:
obj.sum(p2, 200.0, 1.6)
self.fail("Expected IOError")
except IOError:
pass
self.assertEqual(True, obj.isInitialized())
self.assertEqual("DBZH", obj.getQuantity())
def test_accumulate_distfield_nodata(self):
distance_field1 = (10, 20, 30, ACRR_NO_DATA)
distance_field2 = (10, 20, 30, ACRR_NO_DATA)
p1 = _cartesianparam.new()
p1.quantity = "DBZH"
p1.setData(reshape(array((111, 111, UNDETECT, NO_DATA), uint8),
(2, 2)))
p1.nodata, p1.undetect, p1.gain, p1.offset = NO_DATA, UNDETECT, 0.5, -32.5
d1 = _ravefield.new()
d1.addAttribute("what/gain", DEFAULT_DISTANCE_GAIN)
d1.setData(reshape(array(distance_field1, uint8), (2, 2)))
d1.addAttribute("how/task", "se.smhi.composite.distance.radar")
p1.addQualityField(d1)
p2 = _cartesianparam.new()
p2.quantity = "DBZH"
p2.setData(reshape(array((111, 111, 111, NO_DATA), uint8), (2, 2)))
p2.nodata, p2.undetect, p2.gain, p2.offset = NO_DATA, UNDETECT, 0.5, -32.5
d2 = _ravefield.new()
d2.addAttribute("what/gain", DEFAULT_DISTANCE_GAIN)
d2.setData(reshape(array(distance_field2, uint8), (2, 2)))
d2.addAttribute("how/task", "se.smhi.composite.distance.radar")
p2.addQualityField(d2)
obj = _acrr.new()
obj.nodata = ACRR_NO_DATA
obj.undetect = UNDETECT
obj.sum(p1, 200.0, 1.6)
obj.sum(p2, 200.0, 1.6)
result = obj.accumulate(0.0, 2, 1.0)
self.assertAlmostEqual(1.0, result.getAttribute("what/prodpar"), 4)
self.assertEqual("ACRR", result.quantity)
self.assertAlmostEqual(ACRR_NO_DATA, result.nodata, 4)
self.assertAlmostEqual(UNDETECT, result.undetect, 4)
refAcrr = [1.0, 1.0, 0.5, ACRR_NO_DATA]
refDist = self.calculateAvgDistanceField(
[distance_field1, distance_field2], DEFAULT_DISTANCE_GAIN)
Acrr = result.getData().flatten()
qfield = result.getQualityFieldByHowTask(
"se.smhi.composite.distance.radar")
self.assertAlmostEqual(0.0, qfield.getAttribute("what/offset"), 4)
self.assertAlmostEqual(1000.0, qfield.getAttribute("what/gain"), 4)
Dist = qfield.getData().flatten()
for i in range(len(refAcrr)):
self.assertAlmostEqual(Acrr[i], refAcrr[i], 2)
self.assertAlmostEqual(Dist[i], refDist[i], 2)
def test_initialize_without_qualityfield(self):
p1 = _cartesianparam.new()
p1.quantity = "DBZH"
p1.setData(reshape(array((NO_DATA, 111, 0, 111), uint8), (2, 2)))
p1.nodata, p1.undetect, p1.gain, p1.offset = NO_DATA, UNDETECT, 0.5, -32.5
obj = _acrr.new()
try:
obj.sum(p1, 200.0, 1.6)
self.fail("Expected IOError")
except IOError:
pass
self.assertEqual(False, obj.isInitialized())
self.assertEqual(None, obj.getQuantity())
def test_after_initialization(self):
p1 = _cartesianparam.new()
p1.quantity = "DBZH"
p1.setData(reshape(array((NO_DATA, 111, 0, 111), uint8), (2, 2)))
p1.nodata, p1.undetect, p1.gain, p1.offset = NO_DATA, UNDETECT, 0.5, -32.5
d1 = _ravefield.new()
d1.addAttribute("what/gain", DEFAULT_DISTANCE_GAIN)
d1.setData(reshape(array((0, 100, 75, 50), uint8), (2, 2)))
d1.addAttribute("how/task", "se.smhi.composite.distance.radar")
p1.addQualityField(d1)
obj = _acrr.new()
obj.sum(p1, 200.0, 1.6)
self.assertEqual(True, obj.isInitialized())
self.assertEqual("DBZH", obj.getQuantity())
def test_accumulate_tofewfiles(self):
p1 = _cartesianparam.new()
p1.quantity = "DBZH"
p1.setData(reshape(array((NO_DATA, 0, 0, 0), uint8), (2, 2)))
p1.nodata, p1.undetect, p1.gain, p1.offset = NO_DATA, UNDETECT, 0.5, -32.5
d1 = _ravefield.new()
d1.addAttribute("what/gain", DEFAULT_DISTANCE_GAIN)
d1.setData(reshape(array((0, 100, 75, 50), uint8), (2, 2)))
d1.addAttribute("how/task", "se.smhi.composite.distance.radar")
p1.addQualityField(d1)
p2 = _cartesianparam.new()
p2.quantity = "DBZH"
p2.setData(reshape(array((0, NO_DATA, 0, 0), uint8), (2, 2)))
p2.nodata, p2.undetect, p2.gain, p2.offset = NO_DATA, UNDETECT, 0.5, -32.5
d2 = _ravefield.new()
d2.addAttribute("what/gain", DEFAULT_DISTANCE_GAIN)
d2.setData(reshape(array((10, 20, 25, 50), uint8), (2, 2)))
d2.addAttribute("how/task", "se.smhi.composite.distance.radar")
p2.addQualityField(d2)
obj = _acrr.new()
obj.nodata = ACRR_NO_DATA
obj.undetect = UNDETECT
obj.sum(p1, 200.0, 1.6)
obj.sum(p2, 200.0, 1.6)
result = obj.accumulate(0.0, 3, 1.0)
self.assertAlmostEqual(1.0, result.getAttribute("what/prodpar"), 4)
self.assertEqual("ACRR", result.quantity)
self.assertAlmostEqual(ACRR_NO_DATA, result.nodata, 4)
self.assertAlmostEqual(UNDETECT, result.undetect, 4)
refAcrr = [ACRR_NO_DATA, ACRR_NO_DATA, ACRR_NO_DATA, ACRR_NO_DATA]
Acrr = result.getData().flatten()
for i in range(len(refAcrr)):
self.assertAlmostEqual(Acrr[i], refAcrr[i], 2)
def generate(files, arguments):
args = arglist2dict(arguments)
zr_a = 200.0
zr_b = 1.6
quantity = "DBZH"
accept = 0.0
distancefield = "se.smhi.composite.distance.radar"
interval = 12
N = 13
adjustmentfile = None
#Accept is the required limit for how many nodata-pixels that are allowed in order for the
#data to be accumulated
#If we expect to have 10 observations and an accept limit of 20, then it can be 0, 1 or 2 observations
#with nodata for that position.
etime = args["time"]
edate = args["date"]
acrrproduct = None
if "zra" in args.keys():
zr_a = float(args["zra"])
if "zrb" in args.keys():
zr_b = float(args["zrb"])
if "quantity" in args.keys():
quantity = args["quantity"]
if "accept" in args.keys():
accept = float(args["accept"]) / 100.0
if "distancefield" in args.keys():
distancefield = args["distancefield"]
if "interval" in args.keys():
interval = int(args["interval"])
if "N" in args.keys():
N = int(args["N"])
if "adjustmentfile" in args.keys():
adjustmentfile = args["adjustmentfile"]
if distancefield == "eu.baltrad.composite.quality.distance.radar":
distancefield = "se.smhi.composite.distance.radar"
acrr = _acrr.new()
acrr.nodata = -1.0
acrr.undetect = 0.0
acrr.quality_field_name = distancefield
logger.info("rave_pgf_gra_plugin: Processing files")
for fname in files:
obj = None
if ravebdb != None:
obj = ravebdb.get_rave_object(fname)
else:
rio = _raveio.open(fname)
obj = rio.object
if _cartesianvolume.isCartesianVolume(obj):
obj = obj.getImage(0)
if not _cartesian.isCartesian(obj):
raise AttributeError("Must call plugin with cartesian products")
if acrrproduct == None:
acrrproduct = _cartesian.new()
acrrproduct.xscale = obj.xscale
acrrproduct.yscale = obj.yscale
acrrproduct.areaextent = obj.areaextent
acrrproduct.projection = obj.projection
acrrproduct.product = obj.product
acrrproduct.source = obj.source
acrrproduct.time = etime
acrrproduct.date = edate
if obj.xscale != acrrproduct.xscale or obj.yscale != acrrproduct.yscale or \
obj.projection.definition != acrrproduct.projection.definition:
raise AttributeError(
"Scale or projdef inconsistancy for used area")
par = obj.getParameter(quantity)
if par == None:
logger.warn("Could not find parameter (%s) for %s %s" %
(quantity, obj.date, obj.time))
else:
if par.getQualityFieldByHowTask(distancefield) != None:
acrr.sum(par, zr_a, zr_b)
logger.info("rave_pgf_gra_plugin: Running acrr accumulation")
# accept, N, hours
acrrparam = acrr.accumulate(accept, N, interval)
acrrproduct.addParameter(acrrparam)
db = rave_dom_db.create_db_from_conf()
try:
logger.info("rave_pgf_gra_plugin: Calculating gra coefficients")
calculate_gra_coefficient(distancefield, interval, adjustmentfile,
etime, edate, acrrproduct, db)
logger.info("rave_pgf_gra_plugin: Coefficients calculated")
except OperationalError as e:
if "server closed the connection unexpectedly" in e.message:
logger.warn(
"Got indication that connection reseted at server side, retrying gra coefficient generation"
)
calculate_gra_coefficient(distancefield, interval, adjustmentfile,
etime, edate, acrrproduct, db)
return None
def generate(files, arguments):
args = arglist2dict(arguments)
zr_a = 200.0
zr_b = 1.6
quantity = "DBZH"
accept = 0.0
distancefield = "se.smhi.composite.distance.radar"
hours = 1
N = 5
applygra = False
#Accept is the required limit for how many nodata-pixels that are allowed in order for the
#data to be accumulated
#If we expect to have 10 observations and an accept limit of 20, then it can be 0, 1 or 2 observations
#with nodata for that position.
etime = args["time"]
edate = args["date"]
img = None
if "zra" in args.keys():
zr_a = float(args["zra"])
if "zrb" in args.keys():
zr_b = float(args["zrb"])
if "quantity" in args.keys():
quantity = args["quantity"]
if "accept" in args.keys():
accept = float(args["accept"]) / 100.0
if "distancefield" in args.keys():
distancefield = args["distancefield"]
if "hours" in args.keys():
hours = int(args["hours"])
if "N" in args.keys():
N = int(args["N"])
if "applygra" in args:
applygra = True
if distancefield == "eu.baltrad.composite.quality.distance.radar":
distancefield = "se.smhi.composite.distance.radar"
pdatetime = datetime.datetime.strptime(
edate + etime, "%Y%m%d%H%M%S") - datetime.timedelta(minutes=60 * hours)
sdate = pdatetime.strftime("%Y%m%d")
stime = pdatetime.strftime("%H%M00")
acrr = _acrr.new()
acrr.nodata = -1.0
acrr.undetect = 0.0
acrr.quality_field_name = distancefield
nodes = None
for fname in files:
obj = None
if ravebdb != None:
obj = ravebdb.get_rave_object(fname)
else:
rio = _raveio.open(fname)
obj = rio.object
if _cartesianvolume.isCartesianVolume(obj):
obj = obj.getImage(0)
if not _cartesian.isCartesian(obj):
raise AttributeError("Must call plugin with cartesian products")
if img == None:
img = _cartesian.new()
img.xscale = obj.xscale
img.yscale = obj.yscale
img.areaextent = obj.areaextent
img.projection = obj.projection
img.product = _rave.Rave_ProductType_RR
img.source = obj.source
img.time = etime
img.date = edate
img.startdate = sdate
img.starttime = stime
img.enddate = edate
img.endtime = etime
if obj.hasAttribute("how/nodes") and nodes == None:
nodes = obj.getAttribute("how/nodes")
if obj.xscale != img.xscale or obj.yscale != img.yscale or \
obj.projection.definition != img.projection.definition:
raise AttributeError(
"Scale or projdef inconsistancy for used area")
par = obj.getParameter(quantity)
if par == None:
logger.warn("Could not find parameter (%s) for %s %s" %
(quantity, obj.date, obj.time))
else:
if par.getQualityFieldByHowTask(distancefield) != None:
acrr.sum(par, zr_a, zr_b)
# accept, N, hours
result = acrr.accumulate(accept, N, hours)
fileno, outfile = rave_tempfile.mktemp(suffix='.h5', close="True")
if nodes != None:
img.addAttribute("how/nodes", nodes)
img.addParameter(result)
#logger.info("Apply gra: %s"%`applygra`)
if applygra:
db = rave_dom_db.create_db_from_conf()
dt = datetime.datetime(int(edate[:4]), int(edate[4:6]), int(edate[6:]),
int(etime[:2]), int(etime[2:4]), 0)
dt = dt - datetime.timedelta(
seconds=3600 * 12) # 12 hours back in time for now..
gra = _gra.new()
gra.A = DEFAULTA
gra.B = DEFAULTB
gra.C = DEFAULTC
gra.zrA = zr_a
gra.zrb = zr_b
grac = db.get_gra_coefficient(dt)
if grac != None and not math.isnan(grac.a) and not math.isnan(
grac.b) and not math.isnan(grac.c):
logger.debug("Using gra coefficients from database, quantity: %s" %
quantity)
gra.A = grac.a
gra.B = grac.b
gra.C = grac.c
else:
logger.info(
"Could not find coefficients for given time, trying to get aged or climatologic coefficients"
)
nowdt = datetime.datetime(int(edate[:4]), int(edate[4:6]),
int(edate[6:]), int(etime[:2]),
int(etime[2:4]), 0)
agedt = nowdt - datetime.timedelta(
seconds=3600 * 48) # 2 days back
sig, pts, loss, r, rsig, corr, gra.A, gra.B, gra.C, mean, dev = get_backup_gra_coefficient(
db, agedt, nowdt)
dfield = result.getQualityFieldByHowTask(distancefield)
gra_field = gra.apply(dfield, result)
gra_field.quantity = result.quantity + "_CORR"
img.addParameter(gra_field)
ios = _raveio.new()
ios.object = img
ios.filename = outfile
ios.version = RAVE_IO_DEFAULT_VERSION
ios.save()
return outfile
def test_before_initalization(self):
obj = _acrr.new()
self.assertEqual(False, obj.isInitialized())
self.assertEqual(None, obj.getQuantity())
def test_undetect(self):
obj = _acrr.new()
self.assertAlmostEqual(UNDETECT, obj.undetect, 4)
obj.undetect = 11.0
self.assertAlmostEqual(11.0, obj.undetect, 4)
def test_nodata(self):
obj = _acrr.new()
self.assertAlmostEqual(ACRR_NO_DATA, obj.nodata, 4)
obj.nodata = 10.0
self.assertAlmostEqual(10.0, obj.nodata, 4)
def test_attribute_visibility(self):
attrs = ['nodata', 'undetect', 'quality_field_name']
acrr = _acrr.new()
alist = dir(acrr)
for a in attrs:
self.assertEqual(True, a in alist)
def test_new(self):
obj = _acrr.new()
self.assertNotEqual(-1, str(type(obj)).find("AcrrCore"))
def test_accumulate(self):
distance_field1 = (0, 100, 75, 50)
distance_field2 = (0, 0, 25, 50)
p1 = _cartesianparam.new()
p1.quantity = "DBZH"
p1.setData(reshape(array((NO_DATA, 111, UNDETECT, 111), uint8),
(2, 2)))
p1.nodata, p1.undetect, p1.gain, p1.offset = NO_DATA, UNDETECT, 0.5, -32.5
d1 = _ravefield.new()
d1.addAttribute("what/gain", DEFAULT_DISTANCE_GAIN)
d1.setData(reshape(array(distance_field1, uint8), (2, 2)))
d1.addAttribute("how/task", "se.smhi.composite.distance.radar")
p1.addQualityField(d1)
p2 = _cartesianparam.new()
p2.quantity = "DBZH"
p2.setData(reshape(array((NO_DATA, 111, 111, UNDETECT), uint8),
(2, 2)))
p2.nodata, p2.undetect, p2.gain, p2.offset = NO_DATA, UNDETECT, 0.5, -32.5
d2 = _ravefield.new()
d2.addAttribute("what/gain", DEFAULT_DISTANCE_GAIN)
d2.setData(reshape(array(distance_field2, uint8), (2, 2)))
d2.addAttribute("how/task", "se.smhi.composite.distance.radar")
d3 = _ravefield.new()
d3.addAttribute("what/gain", DEFAULT_DISTANCE_GAIN)
d3.setData(reshape(array((1, 1, 1, 1), uint8), (2, 2)))
d3.addAttribute("how/task", "se.smhi.composite.distance.notused")
p2.addQualityField(
d3
) # Add d3 before d2 so that we know that it performs lookup by name
p2.addQualityField(d2)
obj = _acrr.new()
obj.nodata = ACRR_NO_DATA
obj.undetect = UNDETECT
obj.sum(p1, 200.0, 1.6)
obj.sum(p2, 200.0, 1.6)
result = obj.accumulate(0.0, 2, 1.0)
self.assertAlmostEqual(1.0, result.getAttribute("what/prodpar"), 4)
self.assertEqual("ACRR", result.quantity)
self.assertAlmostEqual(ACRR_NO_DATA, result.nodata, 4)
self.assertAlmostEqual(UNDETECT, result.undetect, 4)
refAcrr = [ACRR_NO_DATA, 1.0, 0.5, 0.5]
refDist = self.calculateAvgDistanceField(
[distance_field1, distance_field2], DEFAULT_DISTANCE_GAIN)
refDist[
0] = ACRR_NO_DATA # shall be no_data since there is 'no_data' in the acrr-fields at this position
Acrr = result.getData().flatten()
qfield = result.getQualityFieldByHowTask(
"se.smhi.composite.distance.radar")
self.assertAlmostEqual(0.0, qfield.getAttribute("what/offset"), 4)
self.assertAlmostEqual(1000.0, qfield.getAttribute("what/gain"), 4)
Dist = qfield.getData().flatten()
for i in range(len(refAcrr)):
self.assertAlmostEqual(Acrr[i], refAcrr[i], 2)
self.assertAlmostEqual(Dist[i], refDist[i], 2)