def generate(inobj,
reprocess_quality_flag=True,
quality_control_mode=QUALITY_CONTROL_MODE_ANALYZE_AND_APPLY):
if _polarscan.isPolarScan(inobj) == False and _polarvolume.isPolarVolume(
inobj) == False:
raise IOError("Input file must be either polar scan or volume.")
if reprocess_quality_flag == False:
if _polarscan.isPolarScan(inobj) and inobj.findQualityFieldByHowTask(
"fi.fmi.ropo.detector.classification"):
return inobj
elif _polarvolume.isPolarVolume(inobj):
allprocessed = True
for i in range(inobj.getNumberOfScans()):
scan = inobj.getScan(i)
if not scan.findQualityFieldByHowTask(
"fi.fmi.ropo.detector.classification"):
allprocessed = False
break
if allprocessed:
return inobj
options = get_options(
inobj
) # Gets options/arguments for this radar. Fixes /what/source if required.
if _polarvolume.isPolarVolume(inobj):
ret = process_pvol(inobj, options, quality_control_mode)
elif _polarscan.isPolarScan(inobj):
month = int(inobj.date[4:6]) - 1
options.threshold = THRESHOLDS[options.threshold][month]
ret = process_scan(inobj, options, quality_control_mode)
copy_topwhat(inobj, ret)
return ret
def create_filename(self, pobj):
#_polarscan.isPolarScan(obj) and not _polarvolume.isPolarVolume(obj):
if _polarvolume.isPolarVolume(pobj):
ptype = "pvol"
elif _polarscan.isPolarScan(pobj):
ptype = "scan"
else:
try:
ptype = pobj.getAttribute("what/object").tolower()
except:
ptype = "unknowntype"
src = odim_source.NODfromSource(pobj)
dstr = "19700101"
tstr = "000000"
try:
dstr = pobj.date
tstr = pobj.time
except:
pass
t = tempfile.mkstemp(prefix="%s_%s_%s_%s_" % (ptype, src, dstr, tstr),
suffix=".h5",
dir=self.dumppath)
os.close(t[0])
return t[1]
def adjustFromRB5(obj):
if _polarvolume.isPolarVolume(obj):
for e in range(obj.getNumberOfScans()):
scan = obj.getScan(e)
adjustFromRB5(scan) # recursive call
elif _polarscan.isPolarScan(obj):
# First: convert RHOHV to unsigned 16-bit integer
rhohv = obj.getParameter("RHOHV")
data = rhohv.getData()
data = data.astype(np.uint16)
rhohv.setData(data)
rhohv.nodata = 256.0
# Second: check for sector blanking.
txPower = obj.getAttribute('how/TXpower')
threshold = 0.9 * np.max(txPower)
# Find out how many rays have tx powers beneath the threshold
blanked = np.less(txPower, threshold)
nrays2blank = np.sum(blanked)
# Only continue if we know we need to, blanking all moments/quantities
if nrays2blank:
for pname in obj.getParameterNames():
param = obj.getParameter(pname)
data = param.getData()
for ray in range(obj.nrays):
data[ray] = np.where(np.equal(blanked[ray], True),
param.nodata, data[ray])
param.setData(data)
def hacFilter(obj, quant="DBZH"):
if _polarvolume.isPolarVolume(obj):
filterPvol(obj, quant)
elif _polarscan.isPolarScan(obj):
filterScan(obj, quant)
else:
raise TypeError, "HAC filter received neither SCAN nor PVOL as input"
def checkDBZH(obj, ow=True):
if _polarvolume.isPolarVolume(obj):
for i in range(obj.getNumberOfScans()):
scan = obj.getScan(i)
copyDBZH(scan, ow)
elif _polarscan.isPolarScan(obj):
copyDBZH(obj, ow)
def drQC(pobject,
profile_fstr=None,
param_name="DBZH",
zdr_offset=0.0,
kernely=2,
kernelx=2,
param_thresh=35.0,
dr_thresh=-12.0,
Tw=True,
keepDR=True):
if profile_fstr: profile = ec_temperatureProfile.readProfile(profile_fstr)
else: profile = None
if _polarvolume.isPolarVolume(pobject):
nscans = pobject.getNumberOfScans(pobject)
for n in range(nscans):
scan = pobject.getScan(n)
drQCscan(scan, profile, param_name, zdr_offset, kernely, kernelx,
param_thresh, dr_thresh, Tw, keepDR)
elif _polarscan.isPolarScan(pobject):
drQCscan(pobject, profile, param_name, zdr_offset, kernely, kernelx,
param_thresh, dr_thresh, Tw, keepDR)
else:
raise IOError("Input object is neither polar volume nor scan")
def zdiff(obj, thresh=40.0):
if _polarscan.isPolarScan(obj):
zdiffScan(obj, thresh)
elif _polarvolume.isPolarVolume(obj):
zdiffPvol(obj, thresh)
else:
raise TypeError, "Input is expected to be a polar volume or scan"
def process(self, obj, reprocess_quality_flag=True, quality_control_mode=QUALITY_CONTROL_MODE_ANALYZE_AND_APPLY, arguments=None):
if obj != None:
try:
if _polarscan.isPolarScan(obj):
if reprocess_quality_flag == False and obj.findQualityFieldByHowTask("se.smhi.detector.beamblockage") != None:
return obj
bb = self._create_bb()
result = bb.getBlockage(obj, self._dblimit)
if quality_control_mode != QUALITY_CONTROL_MODE_ANALYZE:
_beamblockage.restore(obj, result, "DBZH", self._bblimit)
obj.addOrReplaceQualityField(result)
elif _polarvolume.isPolarVolume(obj):
for i in range(obj.getNumberOfScans()):
scan = obj.getScan(i)
if reprocess_quality_flag == False and scan.findQualityFieldByHowTask("se.smhi.detector.beamblockage") != None:
continue
bb = self._create_bb()
result = bb.getBlockage(scan, self._dblimit)
if quality_control_mode != QUALITY_CONTROL_MODE_ANALYZE:
_beamblockage.restore(scan, result, "DBZH", self._bblimit)
scan.addOrReplaceQualityField(result)
except:
logger.exception("Failed to generate beam blockage field")
return obj
def readRB5(filenamelist):
objects = readParameterFiles(filenamelist)
rio = _raveio.new()
if _polarscan.isPolarScan(objects[0]):
rio.object = compileScanParameters(objects)
elif _polarvolume.isPolarVolume(objects[0]):
rio.object = compileVolumeFromVolumes(objects)
return rio
def add_dealiased_param(obj):
import _polarvolume
if obj != None and _polarvolume.isPolarVolume(obj):
for i in range(obj.getNumberOfScans()):
scan = obj.getScan(i)
add_dealiased_param_for_scan(scan)
elif obj != None and _polarscan.isPolarScan(obj):
add_dealiased_param_for_scan(obj)
def fetch_objects(self):
nodes = ""
objects = {}
tasks = []
for fname in self.filenames:
obj = None
try:
if self.ravebdb != None:
obj = self.ravebdb.get_rave_object(fname)
else:
obj = _raveio.open(fname).object
except IOError:
self.logger.exception("Failed to open %s" % fname)
is_scan = _polarscan.isPolarScan(obj)
if is_scan:
is_pvol = False
else:
is_pvol = _polarvolume.isPolarVolume(obj)
if not is_scan and not is_pvol:
self.logger.info(
"Input file %s is neither polar scan or volume, ignoring."
% fname)
continue
if self.ignore_malfunc:
obj = rave_util.remove_malfunc(obj)
if obj is None:
continue
node = odim_source.NODfromSource(obj)
if len(nodes):
nodes += ",'%s'" % node
else:
nodes += "'%s'" % node
objects[fname] = obj
if is_scan:
self.logger.debug(
"Scan used in composite generation - UUID: %s, Node: %s, Nominal date and time: %sT%s",
fname, node, obj.date, obj.time)
self.add_how_task_from_scan(obj, tasks)
elif is_pvol:
self.logger.debug(
"PVOL used in composite generation - UUID: %s, Node: %s, Nominal date and time: %sT%s",
fname, node, obj.date, obj.time)
for i in range(obj.getNumberOfScans()):
scan = obj.getScan(i)
self.add_how_task_from_scan(scan, tasks)
how_tasks = ",".join(tasks)
return objects, nodes, how_tasks
def process(self, obj, reprocess_quality_flag=True, quality_control_mode=QUALITY_CONTROL_MODE_ANALYZE_AND_APPLY, arguments=None):
objinfo = self.get_object_information(obj)
if _polarscan.isPolarScan(obj):
self.processScan(objinfo, obj)
elif _polarvolume.isPolarVolume(obj):
nscans = obj.getNumberOfScans(obj)
for i in range(nscans):
self.processScan(objinfo, obj.getScan(i))
return obj, self.getQualityFields()
def process(self,
obj,
reprocess_quality_flag=True,
quality_control_mode=QUALITY_CONTROL_MODE_ANALYZE_AND_APPLY,
arguments=None):
try:
if _polarscan.isPolarScan(obj) or _polarvolume.isPolarVolume(obj):
odc_hac.zdiff(obj)
except:
logger.exception("Failure during zdiff processing")
return obj, self.getQualityFields()
def hacIncrement(obj, quant="DBZH"):
if _polarvolume.isPolarVolume(obj):
incrementPvol(obj, quant)
elif _polarscan.isPolarScan(obj):
incrementScan(obj, quant)
elif type(obj) == types.StringType:
if os.path.isfile(obj) and os.path.getsize(obj):
obj = _raveio.open(obj).object
hacIncrement(obj)
else:
raise TypeError, "HAC incrementor received a string without a matching file, or file is empty"
else:
raise TypeError, "HAC incrementor received neither SCAN nor PVOL as input object"
def remove_malfunc(obj):
result = obj
if _polarvolume.isPolarVolume(obj):
result = remove_malfunc_from_volume(obj)
if result != None and result.getNumberOfScans() == 0:
logger.debug(
"All scans of the volume were detected as malfunc. Complete volume therefore considered as malfunc."
)
result = None
elif _polarscan.isPolarScan(obj):
if get_malfunc_from_obj(obj):
result = None
return result
def is_polar_malfunc(obj):
result = False
if _polarvolume.isPolarVolume(obj):
result = get_malfunc_from_obj(obj)
if not result:
for i in range(obj.getNumberOfScans()):
result = get_malfunc_from_obj(obj.getScan(i))
if result:
break
elif _polarscan.isPolarScan(obj):
result = get_malfunc_from_obj(obj)
else:
raise Exception("Neither polar volume or polar scan")
return result
def zdiffScan(scan, thresh=40.0):
if _polarscan.isPolarScan(scan):
if not scan.hasParameter("DBZH") or not scan.hasParameter("TH"):
return
qind = _ravefield.new()
qind.setData(zeros((scan.nrays, scan.nbins), uint8))
qind.addAttribute("how/task", "eu.opera.odc.zdiff")
qind.addAttribute("how/task_args", thresh)
qind.addAttribute("what/gain", 1 / 255.0)
qind.addAttribute("what/offset", 0.0)
scan.addQualityField(qind)
ret = _odc_hac.zdiff(scan, thresh)
else:
raise TypeError, "Input is expected to be a polar scan. Got something else."
def should_perform_qc_process(reprocess, obj, how_task):
if reprocess:
return True
if _polarscan.isPolarScan(obj) and obj.findQualityFieldByHowTask(how_task):
return False
if _polarvolume.isPolarVolume(obj):
for i in range(obj.getNumberOfScans()):
scan = obj.getScan(i)
if not scan.findQualityFieldByHowTask(how_task):
return True
return False
return True
def MakeAreaFromPolarObjects(objects,
proj_id='llwgs84',
xscale=2000.0,
yscale=2000.0):
import _rave, _raveio
areas = []
for o in objects:
if _polarvolume.isPolarVolume(o):
scan = o.getScanWithMaxDistance()
elif _polarscan.isPolarScan(o):
scan = o
else:
raise IOError("Input object is not a polar scan or volume")
areas.append(MakeSingleAreaFromSCAN(scan, proj_id, xscale, yscale))
minx = 10e100
maxx = -10e100
miny = 10e100
maxy = -10e100
for a in areas:
if a.extent[0] < minx: minx = a.extent[0]
if a.extent[1] < miny: miny = a.extent[1]
if a.extent[2] > maxx: maxx = a.extent[2]
if a.extent[3] > maxy: maxy = a.extent[3]
# Expand to nearest pixel - buffering by one pixel was done in MakeSingleAreaFromSCAN
dx = (maxx - minx) / xscale
dx = (1.0 - (dx - int(dx))) * xscale
if dx < xscale:
minx -= dx
dy = (maxy - miny) / yscale
dy = (1.0 - (dy - int(dy))) * yscale
if dy < yscale:
miny -= dy
xsize = int(round((maxx - minx) / xscale, 0))
ysize = int(round((maxy - miny) / yscale, 0))
A = AREA()
A.xsize, A.ysize, A.xscale, A.yscale = xsize, ysize, xscale, yscale
A.extent = minx, miny, maxx, maxy
A.pcs = proj_id
return A
def _add_files_to_argument_list(self, args, tiled_areas):
self.logger.info("Distributing polar objects among %d tiles" %
len(args))
# Loop through tile areas
for i in range(len(tiled_areas)):
p = tiled_areas[i].projection
llx, lly, urx, ury = tiled_areas[i].extent
# Loop through radars
for k in self.file_objects.keys():
v = self.file_objects[k]
if not _polarscan.isPolarScan(
v) and not _polarvolume.isPolarVolume(v):
continue
if _polarvolume.isPolarVolume(v):
v = v.getScanWithMaxDistance()
scan = v
if self.compositing.quantity not in scan.getParameterNames():
self.logger.info("Quantity %s not in data from %s" %
(self.compositing.quantity, scan.source))
continue
bi = scan.nbins - 1
# Loop around the scan
for ai in range(scan.nrays):
lon, lat = scan.getLonLatFromIndex(bi, ai)
x, y = p.fwd((lon, lat))
# If this position is inside the tile, then add the radar's file string to the list and then bail
if x >= llx and x <= urx and y >= lly and y <= ury:
if not k in args[i][0].filenames:
args[i][0].filenames.append(k)
break # No need to continue
for idx in range(len(args)):
self.logger.info(
"Tile %s contains %d files and dimensions %i x %i" %
(args[idx][0].area_definition.id, len(args[idx][0].filenames),
args[idx][0].area_definition.xsize,
args[idx][0].area_definition.ysize))
self.logger.info("Finished splitting polar object")
def _add_object_to(self, srco, tgto):
if _polarvolume.isPolarVolume(srco) and _polarscan.isPolarScan(tgto):
raise TypeError, "Can not merge a volume into a scan"
# Is srco is a polarvolume, we use a recursive back with the individual scans
if _polarvolume.isPolarVolume(srco):
for i in range(srco.getNumberOfScans()):
s = srco.getScan(i)
self._add_object_to(s, tgto)
return
# From here on we know that srco always is a scan
tgtscan = tgto
if _polarvolume.isPolarVolume(tgto):
cscan = tgto.getScanClosestToElevation(srco.elangle, 0)
if cscan.elangle != srco.elangle:
tgto.addScan(srco)
else:
self._merge_parameters(srco, cscan)
else:
self._merge_parameters(srco, tgtscan)
def drQC(pobject,
param_name="DBZH",
zdr_offset=0.0,
kernely=2,
kernelx=2,
param_thresh=35.0,
dr_thresh=-12.0,
keepDR=True):
if _polarvolume.isPolarVolume(pobject):
nscans = pobject.getNumberOfScans(pobject)
for n in range(nscans):
scan = pobject.getScan(n)
drQCscan(scan, param_name, zdr_offset, kernely, kernelx,
param_thresh, dr_thresh, keepDR)
elif _polarscan.isPolarScan(pobject):
drQCscan(pobject, param_name, zdr_offset, kernely, kernelx,
param_thresh, dr_thresh, keepDR)
else:
raise IOError, "Input object is neither polar volume nor scan"
def _add_radar_index_value_to_argument_list(self, args):
ctr = 1
for k in self.file_objects.keys():
v = self.file_objects[k]
if not _polarscan.isPolarScan(
v) and not _polarvolume.isPolarVolume(v):
continue
sourceid = v.source
try:
osource = odim_source.ODIM_Source(v.source)
if osource.wmo:
sourceid = "WMO:%s" % osource.wmo
elif osource.rad:
sourceid = "RAD:%s" % osource.rad
elif osource.nod:
sourceid = "NOD:%s" % osource.nod
except:
pass
for arg in args:
arg[0].radar_index_mapping[sourceid] = ctr
ctr = ctr + 1
def _ensure_date_and_time_on_args(self, args):
dtstr = None
ddstr = None
for k in self.file_objects.keys():
v = self.file_objects[k]
if not _polarscan.isPolarScan(
v) and not _polarvolume.isPolarVolume(v):
continue
dtstr = v.time
ddstr = v.date
break
if dtstr is None or ddstr is None:
self.logger.info("Could not determine any date and time string")
return False
for arg in args:
if len(arg[0].filenames) == 0 and (arg[1] is None
or arg[2] is None):
arg[1] = ddstr
arg[2] = dtstr
return True
def ncar_PID(rio,
profile_fstr,
median_filter_len=0,
pid_thresholds=None,
zdr_offset=0.0,
derive_dr=0,
zdr_scale=1.0,
keepExtras=False):
profile = readProfile(profile_fstr, scale_height=1000.0)
pobject = rio.object
if _polarvolume.isPolarVolume(pobject):
nscans = pobject.getNumberOfScans(pobject)
for n in range(nscans):
scan = pobject.getScan(n)
pidScan(scan, profile, median_filter_len, pid_thresholds,
zdr_offset, derive_dr, zdr_scale, keepExtras)
elif _polarscan.isPolarScan(pobject):
pidScan(pobject, profile, median_filter_len, pid_thresholds,
zdr_offset, derive_dr, zdr_scale, keepExtras)
else:
raise IOError("Input object is neither polar volume nor scan")
def fetch_objects(self):
nodes = ""
objects = {}
tasks = []
malfunc_files = 0
preload_quantity = None
if self.use_lazy_loading and self.use_lazy_loading_preloads:
preload_quantity = self.quantity
for fname in self.filenames:
obj = None
try:
if self.ravebdb != None:
obj = self.ravebdb.get_rave_object(fname,
self.use_lazy_loading,
preload_quantity)
else:
obj = _raveio.open(fname, self.use_lazy_loading,
preload_quantity).object #self.quantity
except IOError:
self.logger.exception("Failed to open %s", fname)
is_scan = _polarscan.isPolarScan(obj)
if is_scan:
is_pvol = False
else:
is_pvol = _polarvolume.isPolarVolume(obj)
if not is_scan and not is_pvol:
self.logger.warn(
"Input file %s is neither polar scan or volume, ignoring.",
fname)
continue
# Force azimuthal nav information usage if requested
obj.use_azimuthal_nav_information = self.use_azimuthal_nav_information
if self.ignore_malfunc:
obj = rave_util.remove_malfunc(obj)
if obj is None:
self.logger.info(
"Input file %s detected as 'malfunc', ignoring.",
fname)
malfunc_files += 1
continue
node = odim_source.NODfromSource(obj)
if len(nodes):
nodes += ",'%s'" % node
else:
nodes += "'%s'" % node
objects[fname] = obj
if is_scan:
self.logger.debug(
"Scan used in composite generation - UUID: %s, Node: %s, Nominal date and time: %sT%s",
fname, node, obj.date, obj.time)
self.add_how_task_from_scan(obj, tasks)
elif is_pvol:
self.logger.debug(
"PVOL used in composite generation - UUID: %s, Node: %s, Nominal date and time: %sT%s",
fname, node, obj.date, obj.time)
for i in range(obj.getNumberOfScans()):
scan = obj.getScan(i)
self.add_how_task_from_scan(scan, tasks)
how_tasks = ",".join(tasks)
all_files_malfunc = (len(self.filenames) > 0
and malfunc_files == len(self.filenames))
return objects, nodes, how_tasks, all_files_malfunc
