def generate(files, arguments):
args = arglist2dict(arguments)
if not _rave.isCFConventionSupported():
logger.info("CF Conventions is not supported, ignoring export")
return None
if len(files) != 1:
raise AttributeError("Must provide one file to export")
if not "filename" in args.keys():
raise AttributeError("Must specify name of file to export")
obj = None
if ravebdb != None:
obj = ravebdb.get_rave_object(files[0])
else:
rio = _raveio.open(files[0])
obj = rio.object
if not _cartesianvolume.isCartesianVolume(
obj) and not _cartesian.isCartesian(obj):
raise AttributeError("Must call plugin with cartesian products")
filename = args["filename"]
rio = _raveio.new()
rio.object = obj
rio.file_format = _raveio.RaveIO_FileFormat_CF
rio.save(filename)
return None
def generate(self, dd, dt, area=None):
pyarea = my_area_registry.getarea(area)
if self.preprocess_qc and self.mp_process_qc and self.number_of_quality_control_processes > 1:
self.file_objects, self.nodes, self.how_tasks, all_files_malfunc = self._fetch_file_objects_mp(
)
else:
self.file_objects, self.nodes, self.how_tasks, all_files_malfunc = self._fetch_file_objects(
)
if all_files_malfunc:
self.logger.info(
"Content of all provided files were marked as 'malfunc'. Since option 'ignore_malfunc' is set, no composite is generated!"
)
return None
args = self._create_arguments(dd, dt, pyarea)
results = []
ntiles = len(args)
ncpucores = multiprocessing.cpu_count()
nrprocesses = ntiles
if not RAVE_TILE_COMPOSITING_PROCESSES is None:
if nrprocesses > RAVE_TILE_COMPOSITING_PROCESSES:
nrprocesses = RAVE_TILE_COMPOSITING_PROCESSES
if nrprocesses > ncpucores:
nrprocesses = ncpucores
if nrprocesses == ncpucores and ncpucores > 1:
nrprocesses = nrprocesses - 1 # We always want to leave at least one core for something else
pool = multiprocessing.Pool(nrprocesses)
r = pool.map_async(comp_generate, args, callback=results.append)
r.wait()
pool.terminate()
pool.join()
self.logger.info("Finished processing tiles, combining tiles")
objects = []
try:
for v in results[0]:
tile_file = v[1]
if tile_file == None:
self.logger.warn(
"No partial composite for tile area %s was created. This tile will therefore not be included in complete composite.",
v[0])
else:
o = _raveio.open(tile_file).object
if _cartesianvolume.isCartesianVolume(o):
o = o.getImage(0)
o.objectType = _rave.Rave_ObjectType_COMP
objects.append(o)
t = _transform.new()
self.logger.debug(
"Combining %d tiles into one composite for area %s.",
len(objects), area)
result = t.combine_tiles(pyarea, objects)
# Fix so that we get a valid place for /what/source and /how/nodes
result.source = "%s,CMT:%s" % (CENTER_ID, area)
result.addAttribute('how/nodes', self.nodes)
if self.how_tasks != "":
result.addAttribute('how/task', self.how_tasks)
self.logger.info("Tiles combined")
return result
finally:
if self._do_remove_temporary_files:
for fname in self.compositing.filenames:
try:
os.unlink(fname)
except:
logger.info("Failed to remove temporary file: %s" %
fname)
if results != None:
for v in results[0]:
if v != None and v[1] != None and os.path.exists(v[1]):
try:
os.unlink(v[1])
except Exception:
logger.exception("Failed to unlink %s" % v[1])
return None
def test_isCartesianVolumeFalse(self):
obj = _cartesian.new()
self.assertEqual(False, _cartesianvolume.isCartesianVolume(obj))
def generate(files, arguments):
args = arglist2dict(arguments)
zr_a = 200.0
zr_b = 1.6
quantity = "DBZH"
accept = 0.0
distancefield = "se.smhi.composite.distance.radar"
interval = 12
N = 13
adjustmentfile = None
#Accept is the required limit for how many nodata-pixels that are allowed in order for the
#data to be accumulated
#If we expect to have 10 observations and an accept limit of 20, then it can be 0, 1 or 2 observations
#with nodata for that position.
etime = args["time"]
edate = args["date"]
acrrproduct = None
if "zra" in args.keys():
zr_a = float(args["zra"])
if "zrb" in args.keys():
zr_b = float(args["zrb"])
if "quantity" in args.keys():
quantity = args["quantity"]
if "accept" in args.keys():
accept = float(args["accept"]) / 100.0
if "distancefield" in args.keys():
distancefield = args["distancefield"]
if "interval" in args.keys():
interval = int(args["interval"])
if "N" in args.keys():
N = int(args["N"])
if "adjustmentfile" in args.keys():
adjustmentfile = args["adjustmentfile"]
if distancefield == "eu.baltrad.composite.quality.distance.radar":
distancefield = "se.smhi.composite.distance.radar"
acrr = _acrr.new()
acrr.nodata = -1.0
acrr.undetect = 0.0
acrr.quality_field_name = distancefield
logger.info("rave_pgf_gra_plugin: Processing files")
for fname in files:
obj = None
if ravebdb != None:
obj = ravebdb.get_rave_object(fname)
else:
rio = _raveio.open(fname)
obj = rio.object
if _cartesianvolume.isCartesianVolume(obj):
obj = obj.getImage(0)
if not _cartesian.isCartesian(obj):
raise AttributeError("Must call plugin with cartesian products")
if acrrproduct == None:
acrrproduct = _cartesian.new()
acrrproduct.xscale = obj.xscale
acrrproduct.yscale = obj.yscale
acrrproduct.areaextent = obj.areaextent
acrrproduct.projection = obj.projection
acrrproduct.product = obj.product
acrrproduct.source = obj.source
acrrproduct.time = etime
acrrproduct.date = edate
if obj.xscale != acrrproduct.xscale or obj.yscale != acrrproduct.yscale or \
obj.projection.definition != acrrproduct.projection.definition:
raise AttributeError(
"Scale or projdef inconsistancy for used area")
par = obj.getParameter(quantity)
if par == None:
logger.warn("Could not find parameter (%s) for %s %s" %
(quantity, obj.date, obj.time))
else:
if par.getQualityFieldByHowTask(distancefield) != None:
acrr.sum(par, zr_a, zr_b)
logger.info("rave_pgf_gra_plugin: Running acrr accumulation")
# accept, N, hours
acrrparam = acrr.accumulate(accept, N, interval)
acrrproduct.addParameter(acrrparam)
db = rave_dom_db.create_db_from_conf()
try:
logger.info("rave_pgf_gra_plugin: Calculating gra coefficients")
calculate_gra_coefficient(distancefield, interval, adjustmentfile,
etime, edate, acrrproduct, db)
logger.info("rave_pgf_gra_plugin: Coefficients calculated")
except OperationalError as e:
if "server closed the connection unexpectedly" in e.message:
logger.warn(
"Got indication that connection reseted at server side, retrying gra coefficient generation"
)
calculate_gra_coefficient(distancefield, interval, adjustmentfile,
etime, edate, acrrproduct, db)
return None
def generate(files, arguments):
args = arglist2dict(arguments)
zr_a = 200.0
zr_b = 1.6
quantity = "DBZH"
accept = 0.0
distancefield = "se.smhi.composite.distance.radar"
hours = 1
N = 5
applygra = False
#Accept is the required limit for how many nodata-pixels that are allowed in order for the
#data to be accumulated
#If we expect to have 10 observations and an accept limit of 20, then it can be 0, 1 or 2 observations
#with nodata for that position.
etime = args["time"]
edate = args["date"]
img = None
if "zra" in args.keys():
zr_a = float(args["zra"])
if "zrb" in args.keys():
zr_b = float(args["zrb"])
if "quantity" in args.keys():
quantity = args["quantity"]
if "accept" in args.keys():
accept = float(args["accept"]) / 100.0
if "distancefield" in args.keys():
distancefield = args["distancefield"]
if "hours" in args.keys():
hours = int(args["hours"])
if "N" in args.keys():
N = int(args["N"])
if "applygra" in args:
applygra = True
if distancefield == "eu.baltrad.composite.quality.distance.radar":
distancefield = "se.smhi.composite.distance.radar"
pdatetime = datetime.datetime.strptime(
edate + etime, "%Y%m%d%H%M%S") - datetime.timedelta(minutes=60 * hours)
sdate = pdatetime.strftime("%Y%m%d")
stime = pdatetime.strftime("%H%M00")
acrr = _acrr.new()
acrr.nodata = -1.0
acrr.undetect = 0.0
acrr.quality_field_name = distancefield
nodes = None
for fname in files:
obj = None
if ravebdb != None:
obj = ravebdb.get_rave_object(fname)
else:
rio = _raveio.open(fname)
obj = rio.object
if _cartesianvolume.isCartesianVolume(obj):
obj = obj.getImage(0)
if not _cartesian.isCartesian(obj):
raise AttributeError("Must call plugin with cartesian products")
if img == None:
img = _cartesian.new()
img.xscale = obj.xscale
img.yscale = obj.yscale
img.areaextent = obj.areaextent
img.projection = obj.projection
img.product = _rave.Rave_ProductType_RR
img.source = obj.source
img.time = etime
img.date = edate
img.startdate = sdate
img.starttime = stime
img.enddate = edate
img.endtime = etime
if obj.hasAttribute("how/nodes") and nodes == None:
nodes = obj.getAttribute("how/nodes")
if obj.xscale != img.xscale or obj.yscale != img.yscale or \
obj.projection.definition != img.projection.definition:
raise AttributeError(
"Scale or projdef inconsistancy for used area")
par = obj.getParameter(quantity)
if par == None:
logger.warn("Could not find parameter (%s) for %s %s" %
(quantity, obj.date, obj.time))
else:
if par.getQualityFieldByHowTask(distancefield) != None:
acrr.sum(par, zr_a, zr_b)
# accept, N, hours
result = acrr.accumulate(accept, N, hours)
fileno, outfile = rave_tempfile.mktemp(suffix='.h5', close="True")
if nodes != None:
img.addAttribute("how/nodes", nodes)
img.addParameter(result)
#logger.info("Apply gra: %s"%`applygra`)
if applygra:
db = rave_dom_db.create_db_from_conf()
dt = datetime.datetime(int(edate[:4]), int(edate[4:6]), int(edate[6:]),
int(etime[:2]), int(etime[2:4]), 0)
dt = dt - datetime.timedelta(
seconds=3600 * 12) # 12 hours back in time for now..
gra = _gra.new()
gra.A = DEFAULTA
gra.B = DEFAULTB
gra.C = DEFAULTC
gra.zrA = zr_a
gra.zrb = zr_b
grac = db.get_gra_coefficient(dt)
if grac != None and not math.isnan(grac.a) and not math.isnan(
grac.b) and not math.isnan(grac.c):
logger.debug("Using gra coefficients from database, quantity: %s" %
quantity)
gra.A = grac.a
gra.B = grac.b
gra.C = grac.c
else:
logger.info(
"Could not find coefficients for given time, trying to get aged or climatologic coefficients"
)
nowdt = datetime.datetime(int(edate[:4]), int(edate[4:6]),
int(edate[6:]), int(etime[:2]),
int(etime[2:4]), 0)
agedt = nowdt - datetime.timedelta(
seconds=3600 * 48) # 2 days back
sig, pts, loss, r, rsig, corr, gra.A, gra.B, gra.C, mean, dev = get_backup_gra_coefficient(
db, agedt, nowdt)
dfield = result.getQualityFieldByHowTask(distancefield)
gra_field = gra.apply(dfield, result)
gra_field.quantity = result.quantity + "_CORR"
img.addParameter(gra_field)
ios = _raveio.new()
ios.object = img
ios.filename = outfile
ios.version = RAVE_IO_DEFAULT_VERSION
ios.save()
return outfile