def generate(inobj,
reprocess_quality_flag=True,
quality_control_mode=QUALITY_CONTROL_MODE_ANALYZE_AND_APPLY):
if _polarscan.isPolarScan(inobj) == False and _polarvolume.isPolarVolume(
inobj) == False:
raise IOError("Input file must be either polar scan or volume.")
if reprocess_quality_flag == False:
if _polarscan.isPolarScan(inobj) and inobj.findQualityFieldByHowTask(
"fi.fmi.ropo.detector.classification"):
return inobj
elif _polarvolume.isPolarVolume(inobj):
allprocessed = True
for i in range(inobj.getNumberOfScans()):
scan = inobj.getScan(i)
if not scan.findQualityFieldByHowTask(
"fi.fmi.ropo.detector.classification"):
allprocessed = False
break
if allprocessed:
return inobj
options = get_options(
inobj
) # Gets options/arguments for this radar. Fixes /what/source if required.
if _polarvolume.isPolarVolume(inobj):
ret = process_pvol(inobj, options, quality_control_mode)
elif _polarscan.isPolarScan(inobj):
month = int(inobj.date[4:6]) - 1
options.threshold = THRESHOLDS[options.threshold][month]
ret = process_scan(inobj, options, quality_control_mode)
copy_topwhat(inobj, ret)
return ret
def create_filename(self, pobj):
#_polarscan.isPolarScan(obj) and not _polarvolume.isPolarVolume(obj):
if _polarvolume.isPolarVolume(pobj):
ptype = "pvol"
elif _polarscan.isPolarScan(pobj):
ptype = "scan"
else:
try:
ptype = pobj.getAttribute("what/object").tolower()
except:
ptype = "unknowntype"
src = odim_source.NODfromSource(pobj)
dstr = "19700101"
tstr = "000000"
try:
dstr = pobj.date
tstr = pobj.time
except:
pass
t = tempfile.mkstemp(prefix="%s_%s_%s_%s_" % (ptype, src, dstr, tstr),
suffix=".h5",
dir=self.dumppath)
os.close(t[0])
return t[1]
def worker_collect_radar_metadata(input=None):
directory = input
D_metadata = {}
for fname in os.listdir(directory):
try:
filePath = os.path.join(directory, fname)
rio = _rave.open(filePath) # use ODIM h5 functionality
obj = rio.object
if not _polarvolume.isPolarVolume(obj): # rarely happened
sys.stderr.write('Warning: not polar volume: %s\n' % filePath)
continue
source = obj.source
sourceTuple = source.split(',')
id_radar = sourceTuple[0].split(':')[1]
# NOTE ussox threw error for 2019 data
plc = sourceTuple[1].split(':')[1]
D_metadata[id_radar] = {
'lon': obj.longitude,
'lat': obj.latitude,
'hgt': obj.height,
'plc': plc,
}
except:
sys.stderr.write('Warning, failed for %s\n' % fname)
# Level2_KMQT_20180510_0204.ar2v.qc.usmqt.h5
# Level2_KRGX_20180520_1501.ar2v.qc.usrgx.h5
return D_metadata
def process(self,
obj,
reprocess_quality_flag=True,
quality_control_mode=QUALITY_CONTROL_MODE_ANALYZE_AND_APPLY,
arguments=None):
if obj != None and _polarvolume.isPolarVolume(obj):
import _detectionrange
ascending = obj.isAscendingScans()
drgenerator = _detectionrange.new()
maxscan = obj.getScanWithMaxDistance()
if reprocess_quality_flag or not maxscan.findQualityFieldByHowTask(
"se.smhi.detector.poo"):
# We want to have same resolution as maxdistance scan since we are going to add the poo-field to it
# The second argument is dbz threshold, modify it accordingly
topfield = drgenerator.top(obj, maxscan.rscale,
-40.0) # Topfield is a scan
filterfield = drgenerator.filter(
topfield) # filterfield is a scan
poofield = drgenerator.analyze(
filterfield, 60, 0.1,
0.35) # poofield is a quality field, add it to maxscan
maxscan.addOrReplaceQualityField(poofield)
if ascending:
obj.sortByElevations(1)
return obj, self.getQualityFields()
def process(self, obj, reprocess_quality_flag=True, arguments=None):
#_rave.setDebugLevel(_rave.Debug_RAVE_DEBUG)
if not _polarvolume.isPolarVolume(obj):
raise Exception, "Input object is not a polar volume. Bailing ..."
# Using a dictionary lets us match tasks with payloads
dpvol = {"dopvol1a" : None,
"dopvol1b" : None,
"dopvol1c" : None,
"dopvol2" : None}
for i in range(len(arguments)):
dobj = _raveio.open(arguments[i]).object
task = "".join(dobj.getAttribute('how/task').lower().split("_"))
dpvol[task] = dobj
dopvol = ec_dopvolqc.mergeDopvol(dopvol1a = dpvol["dopvol1a"],
dopvol1b = dpvol["dopvol1b"],
dopvol1c = dpvol["dopvol1c"],
dopvol2 = dpvol["dopvol2"])
# Assume we always want spatial=1 filtering
ec_dopvolqc.dopvolFilter(obj, dopvol)
return obj
def drQC(pobject,
profile_fstr=None,
param_name="DBZH",
zdr_offset=0.0,
kernely=2,
kernelx=2,
param_thresh=35.0,
dr_thresh=-12.0,
Tw=True,
keepDR=True):
if profile_fstr: profile = ec_temperatureProfile.readProfile(profile_fstr)
else: profile = None
if _polarvolume.isPolarVolume(pobject):
nscans = pobject.getNumberOfScans(pobject)
for n in range(nscans):
scan = pobject.getScan(n)
drQCscan(scan, profile, param_name, zdr_offset, kernely, kernelx,
param_thresh, dr_thresh, Tw, keepDR)
elif _polarscan.isPolarScan(pobject):
drQCscan(pobject, profile, param_name, zdr_offset, kernely, kernelx,
param_thresh, dr_thresh, Tw, keepDR)
else:
raise IOError("Input object is neither polar volume nor scan")
def process(self, obj, reprocess_quality_flag=True, quality_control_mode=QUALITY_CONTROL_MODE_ANALYZE_AND_APPLY, arguments=None):
if obj != None:
try:
if _polarscan.isPolarScan(obj):
if reprocess_quality_flag == False and obj.findQualityFieldByHowTask("se.smhi.detector.beamblockage") != None:
return obj
bb = self._create_bb()
result = bb.getBlockage(obj, self._dblimit)
if quality_control_mode != QUALITY_CONTROL_MODE_ANALYZE:
_beamblockage.restore(obj, result, "DBZH", self._bblimit)
obj.addOrReplaceQualityField(result)
elif _polarvolume.isPolarVolume(obj):
for i in range(obj.getNumberOfScans()):
scan = obj.getScan(i)
if reprocess_quality_flag == False and scan.findQualityFieldByHowTask("se.smhi.detector.beamblockage") != None:
continue
bb = self._create_bb()
result = bb.getBlockage(scan, self._dblimit)
if quality_control_mode != QUALITY_CONTROL_MODE_ANALYZE:
_beamblockage.restore(scan, result, "DBZH", self._bblimit)
scan.addOrReplaceQualityField(result)
except:
logger.exception("Failed to generate beam blockage field")
return obj
def hacFilter(obj, quant="DBZH"):
if _polarvolume.isPolarVolume(obj):
filterPvol(obj, quant)
elif _polarscan.isPolarScan(obj):
filterScan(obj, quant)
else:
raise TypeError, "HAC filter received neither SCAN nor PVOL as input"
def zdiffPvol(pvol, thresh=40.0):
if _polarvolume.isPolarVolume(pvol):
for i in range(pvol.getNumberOfScans()):
scan = pvol.getScan(i)
zdiffScan(scan, thresh)
else:
raise TypeError, "Input is expected to be a polar volume. Got something else."
def zdiff(obj, thresh=40.0):
if _polarscan.isPolarScan(obj):
zdiffScan(obj, thresh)
elif _polarvolume.isPolarVolume(obj):
zdiffPvol(obj, thresh)
else:
raise TypeError, "Input is expected to be a polar volume or scan"
def checkDBZH(obj, ow=True):
if _polarvolume.isPolarVolume(obj):
for i in range(obj.getNumberOfScans()):
scan = obj.getScan(i)
copyDBZH(scan, ow)
elif _polarscan.isPolarScan(obj):
copyDBZH(obj, ow)
def adjustFromRB5(obj):
if _polarvolume.isPolarVolume(obj):
for e in range(obj.getNumberOfScans()):
scan = obj.getScan(e)
adjustFromRB5(scan) # recursive call
elif _polarscan.isPolarScan(obj):
# First: convert RHOHV to unsigned 16-bit integer
rhohv = obj.getParameter("RHOHV")
data = rhohv.getData()
data = data.astype(np.uint16)
rhohv.setData(data)
rhohv.nodata = 256.0
# Second: check for sector blanking.
txPower = obj.getAttribute('how/TXpower')
threshold = 0.9 * np.max(txPower)
# Find out how many rays have tx powers beneath the threshold
blanked = np.less(txPower, threshold)
nrays2blank = np.sum(blanked)
# Only continue if we know we need to, blanking all moments/quantities
if nrays2blank:
for pname in obj.getParameterNames():
param = obj.getParameter(pname)
data = param.getData()
for ray in range(obj.nrays):
data[ray] = np.where(np.equal(blanked[ray], True),
param.nodata, data[ray])
param.setData(data)
def readRB5(filenamelist):
objects = readParameterFiles(filenamelist)
rio = _raveio.new()
if _polarscan.isPolarScan(objects[0]):
rio.object = compileScanParameters(objects)
elif _polarvolume.isPolarVolume(objects[0]):
rio.object = compileVolumeFromVolumes(objects)
return rio
def add_dealiased_param(obj):
import _polarvolume
if obj != None and _polarvolume.isPolarVolume(obj):
for i in range(obj.getNumberOfScans()):
scan = obj.getScan(i)
add_dealiased_param_for_scan(scan)
elif obj != None and _polarscan.isPolarScan(obj):
add_dealiased_param_for_scan(obj)
def _add_files_to_argument_list(self, args, tiled_areas):
self.logger.info("Distributing polar objects among %d tiles" %
len(args))
# Loop through tile areas
for i in range(len(tiled_areas)):
p = tiled_areas[i].projection
llx, lly, urx, ury = tiled_areas[i].extent
# Loop through radars
for k in self.file_objects.keys():
v = self.file_objects[k]
if not _polarscan.isPolarScan(
v) and not _polarvolume.isPolarVolume(v):
continue
if _polarvolume.isPolarVolume(v):
v = v.getScanWithMaxDistance()
scan = v
if self.compositing.quantity not in scan.getParameterNames():
self.logger.info("Quantity %s not in data from %s" %
(self.compositing.quantity, scan.source))
continue
bi = scan.nbins - 1
# Loop around the scan
for ai in range(scan.nrays):
lon, lat = scan.getLonLatFromIndex(bi, ai)
x, y = p.fwd((lon, lat))
# If this position is inside the tile, then add the radar's file string to the list and then bail
if x >= llx and x <= urx and y >= lly and y <= ury:
if not k in args[i][0].filenames:
args[i][0].filenames.append(k)
break # No need to continue
for idx in range(len(args)):
self.logger.info(
"Tile %s contains %d files and dimensions %i x %i" %
(args[idx][0].area_definition.id, len(args[idx][0].filenames),
args[idx][0].area_definition.xsize,
args[idx][0].area_definition.ysize))
self.logger.info("Finished splitting polar object")
def fetch_objects(self):
nodes = ""
objects = {}
tasks = []
for fname in self.filenames:
obj = None
try:
if self.ravebdb != None:
obj = self.ravebdb.get_rave_object(fname)
else:
obj = _raveio.open(fname).object
except IOError:
self.logger.exception("Failed to open %s" % fname)
is_scan = _polarscan.isPolarScan(obj)
if is_scan:
is_pvol = False
else:
is_pvol = _polarvolume.isPolarVolume(obj)
if not is_scan and not is_pvol:
self.logger.info(
"Input file %s is neither polar scan or volume, ignoring."
% fname)
continue
if self.ignore_malfunc:
obj = rave_util.remove_malfunc(obj)
if obj is None:
continue
node = odim_source.NODfromSource(obj)
if len(nodes):
nodes += ",'%s'" % node
else:
nodes += "'%s'" % node
objects[fname] = obj
if is_scan:
self.logger.debug(
"Scan used in composite generation - UUID: %s, Node: %s, Nominal date and time: %sT%s",
fname, node, obj.date, obj.time)
self.add_how_task_from_scan(obj, tasks)
elif is_pvol:
self.logger.debug(
"PVOL used in composite generation - UUID: %s, Node: %s, Nominal date and time: %sT%s",
fname, node, obj.date, obj.time)
for i in range(obj.getNumberOfScans()):
scan = obj.getScan(i)
self.add_how_task_from_scan(scan, tasks)
how_tasks = ",".join(tasks)
return objects, nodes, how_tasks
def process(self, obj, reprocess_quality_flag=True, quality_control_mode=QUALITY_CONTROL_MODE_ANALYZE_AND_APPLY, arguments=None):
objinfo = self.get_object_information(obj)
if _polarscan.isPolarScan(obj):
self.processScan(objinfo, obj)
elif _polarvolume.isPolarVolume(obj):
nscans = obj.getNumberOfScans(obj)
for i in range(nscans):
self.processScan(objinfo, obj.getScan(i))
return obj, self.getQualityFields()
def _add_object_to(self, srco, tgto):
if _polarvolume.isPolarVolume(srco) and _polarscan.isPolarScan(tgto):
raise TypeError, "Can not merge a volume into a scan"
# Is srco is a polarvolume, we use a recursive back with the individual scans
if _polarvolume.isPolarVolume(srco):
for i in range(srco.getNumberOfScans()):
s = srco.getScan(i)
self._add_object_to(s, tgto)
return
# From here on we know that srco always is a scan
tgtscan = tgto
if _polarvolume.isPolarVolume(tgto):
cscan = tgto.getScanClosestToElevation(srco.elangle, 0)
if cscan.elangle != srco.elangle:
tgto.addScan(srco)
else:
self._merge_parameters(srco, cscan)
else:
self._merge_parameters(srco, tgtscan)
def process(self,
obj,
reprocess_quality_flag=True,
quality_control_mode=QUALITY_CONTROL_MODE_ANALYZE_AND_APPLY,
arguments=None):
try:
if _polarscan.isPolarScan(obj) or _polarvolume.isPolarVolume(obj):
odc_hac.zdiff(obj)
except:
logger.exception("Failure during zdiff processing")
return obj, self.getQualityFields()
def _verify_elangles(self, pos):
gotvolume = False
elangle = None
for po in pos:
if _polarvolume.isPolarVolume(po):
gotvolume = True
if not gotvolume:
for po in pos:
if elangle is None:
elangle = po.elangle
if po.elangle != elangle:
raise TypeError, "When merging scans elevation angles between files must be same."
def hacIncrement(obj, quant="DBZH"):
if _polarvolume.isPolarVolume(obj):
incrementPvol(obj, quant)
elif _polarscan.isPolarScan(obj):
incrementScan(obj, quant)
elif type(obj) == types.StringType:
if os.path.isfile(obj) and os.path.getsize(obj):
obj = _raveio.open(obj).object
hacIncrement(obj)
else:
raise TypeError, "HAC incrementor received a string without a matching file, or file is empty"
else:
raise TypeError, "HAC incrementor received neither SCAN nor PVOL as input object"
def remove_malfunc(obj):
result = obj
if _polarvolume.isPolarVolume(obj):
result = remove_malfunc_from_volume(obj)
if result != None and result.getNumberOfScans() == 0:
logger.debug(
"All scans of the volume were detected as malfunc. Complete volume therefore considered as malfunc."
)
result = None
elif _polarscan.isPolarScan(obj):
if get_malfunc_from_obj(obj):
result = None
return result
def is_polar_malfunc(obj):
result = False
if _polarvolume.isPolarVolume(obj):
result = get_malfunc_from_obj(obj)
if not result:
for i in range(obj.getNumberOfScans()):
result = get_malfunc_from_obj(obj.getScan(i))
if result:
break
elif _polarscan.isPolarScan(obj):
result = get_malfunc_from_obj(obj)
else:
raise Exception("Neither polar volume or polar scan")
return result
def should_perform_qc_process(reprocess, obj, how_task):
if reprocess:
return True
if _polarscan.isPolarScan(obj) and obj.findQualityFieldByHowTask(how_task):
return False
if _polarvolume.isPolarVolume(obj):
for i in range(obj.getNumberOfScans()):
scan = obj.getScan(i)
if not scan.findQualityFieldByHowTask(how_task):
return True
return False
return True
def process_file(out_dir, in_name):
print "in process_file"
print in_name
print h5py.File(in_name, 'r')
vol = _raveio.open(in_name).object
if _polarvolume.isPolarVolume(vol):
if getNod(vol) in SOURCES:
fname = get_filename(vol)
v2b = _pyvol2bird.new(vol)
# calculate a vertical profile of birds
vpr = v2b.vol2bird(vol)
write_file_to_outdir(out_dir, fname, vpr)
else:
print_log("Not a polar volume. Ignoring")
def MakeAreaFromPolarObjects(objects,
proj_id='llwgs84',
xscale=2000.0,
yscale=2000.0):
import _rave, _raveio
areas = []
for o in objects:
if _polarvolume.isPolarVolume(o):
scan = o.getScanWithMaxDistance()
elif _polarscan.isPolarScan(o):
scan = o
else:
raise IOError("Input object is not a polar scan or volume")
areas.append(MakeSingleAreaFromSCAN(scan, proj_id, xscale, yscale))
minx = 10e100
maxx = -10e100
miny = 10e100
maxy = -10e100
for a in areas:
if a.extent[0] < minx: minx = a.extent[0]
if a.extent[1] < miny: miny = a.extent[1]
if a.extent[2] > maxx: maxx = a.extent[2]
if a.extent[3] > maxy: maxy = a.extent[3]
# Expand to nearest pixel - buffering by one pixel was done in MakeSingleAreaFromSCAN
dx = (maxx - minx) / xscale
dx = (1.0 - (dx - int(dx))) * xscale
if dx < xscale:
minx -= dx
dy = (maxy - miny) / yscale
dy = (1.0 - (dy - int(dy))) * yscale
if dy < yscale:
miny -= dy
xsize = int(round((maxx - minx) / xscale, 0))
ysize = int(round((maxy - miny) / yscale, 0))
A = AREA()
A.xsize, A.ysize, A.xscale, A.yscale = xsize, ysize, xscale, yscale
A.extent = minx, miny, maxx, maxy
A.pcs = proj_id
return A
def _merge_files(self, pos):
result = None
# First we want to check if one of these items is a volume since it makes more sence to add
# scans to a volume than having to check if a later object is volume and then remerge
for i in range(len(pos)):
if _polarvolume.isPolarVolume(pos[i]):
result = pos[i]
del pos[i]
break
if result is None:
result = pos[0]
del pos[0]
for po in pos:
self._add_object_to(po, result)
return result
def remove_malfunc_from_volume(obj):
result = obj
if _polarvolume.isPolarVolume(obj):
if get_malfunc_from_obj(obj):
logger.debug(
"Malfunc volume found. Source: %s, Nominal date and time: %sT%s",
obj.source, obj.date, obj.time)
return None
for i in range(obj.getNumberOfScans() - 1, -1, -1):
scan = obj.getScan(i)
if get_malfunc_from_obj(scan):
logger.debug(
"Malfunc scan with elangle %f found. Removing from volume. Source: %s, Nominal date and time: %sT%s"
% ((scan.elangle * 180.0 / math.pi), obj.source, obj.date,
obj.time))
obj.removeScan(i)
return result
def drQC(pobject,
param_name="DBZH",
zdr_offset=0.0,
kernely=2,
kernelx=2,
param_thresh=35.0,
dr_thresh=-12.0,
keepDR=True):
if _polarvolume.isPolarVolume(pobject):
nscans = pobject.getNumberOfScans(pobject)
for n in range(nscans):
scan = pobject.getScan(n)
drQCscan(scan, param_name, zdr_offset, kernely, kernelx,
param_thresh, dr_thresh, keepDR)
elif _polarscan.isPolarScan(pobject):
drQCscan(pobject, param_name, zdr_offset, kernely, kernelx,
param_thresh, dr_thresh, keepDR)
else:
raise IOError, "Input object is neither polar volume nor scan"
def _add_radar_index_value_to_argument_list(self, args):
ctr = 1
for k in self.file_objects.keys():
v = self.file_objects[k]
if not _polarscan.isPolarScan(
v) and not _polarvolume.isPolarVolume(v):
continue
sourceid = v.source
try:
osource = odim_source.ODIM_Source(v.source)
if osource.wmo:
sourceid = "WMO:%s" % osource.wmo
elif osource.rad:
sourceid = "RAD:%s" % osource.rad
elif osource.nod:
sourceid = "NOD:%s" % osource.nod
except:
pass
for arg in args:
arg[0].radar_index_mapping[sourceid] = ctr
ctr = ctr + 1
