def generate(inobj, reprocess_quality_flag=True, quality_control_mode=QUALITY_CONTROL_MODE_ANALYZE_AND_APPLY): if _polarscan.isPolarScan(inobj) == False and _polarvolume.isPolarVolume( inobj) == False: raise IOError("Input file must be either polar scan or volume.") if reprocess_quality_flag == False: if _polarscan.isPolarScan(inobj) and inobj.findQualityFieldByHowTask( "fi.fmi.ropo.detector.classification"): return inobj elif _polarvolume.isPolarVolume(inobj): allprocessed = True for i in range(inobj.getNumberOfScans()): scan = inobj.getScan(i) if not scan.findQualityFieldByHowTask( "fi.fmi.ropo.detector.classification"): allprocessed = False break if allprocessed: return inobj options = get_options( inobj ) # Gets options/arguments for this radar. Fixes /what/source if required. if _polarvolume.isPolarVolume(inobj): ret = process_pvol(inobj, options, quality_control_mode) elif _polarscan.isPolarScan(inobj): month = int(inobj.date[4:6]) - 1 options.threshold = THRESHOLDS[options.threshold][month] ret = process_scan(inobj, options, quality_control_mode) copy_topwhat(inobj, ret) return ret
def create_filename(self, pobj): #_polarscan.isPolarScan(obj) and not _polarvolume.isPolarVolume(obj): if _polarvolume.isPolarVolume(pobj): ptype = "pvol" elif _polarscan.isPolarScan(pobj): ptype = "scan" else: try: ptype = pobj.getAttribute("what/object").tolower() except: ptype = "unknowntype" src = odim_source.NODfromSource(pobj) dstr = "19700101" tstr = "000000" try: dstr = pobj.date tstr = pobj.time except: pass t = tempfile.mkstemp(prefix="%s_%s_%s_%s_" % (ptype, src, dstr, tstr), suffix=".h5", dir=self.dumppath) os.close(t[0]) return t[1]
def adjustFromRB5(obj): if _polarvolume.isPolarVolume(obj): for e in range(obj.getNumberOfScans()): scan = obj.getScan(e) adjustFromRB5(scan) # recursive call elif _polarscan.isPolarScan(obj): # First: convert RHOHV to unsigned 16-bit integer rhohv = obj.getParameter("RHOHV") data = rhohv.getData() data = data.astype(np.uint16) rhohv.setData(data) rhohv.nodata = 256.0 # Second: check for sector blanking. txPower = obj.getAttribute('how/TXpower') threshold = 0.9 * np.max(txPower) # Find out how many rays have tx powers beneath the threshold blanked = np.less(txPower, threshold) nrays2blank = np.sum(blanked) # Only continue if we know we need to, blanking all moments/quantities if nrays2blank: for pname in obj.getParameterNames(): param = obj.getParameter(pname) data = param.getData() for ray in range(obj.nrays): data[ray] = np.where(np.equal(blanked[ray], True), param.nodata, data[ray]) param.setData(data)
def hacFilter(obj, quant="DBZH"): if _polarvolume.isPolarVolume(obj): filterPvol(obj, quant) elif _polarscan.isPolarScan(obj): filterScan(obj, quant) else: raise TypeError, "HAC filter received neither SCAN nor PVOL as input"
def checkDBZH(obj, ow=True): if _polarvolume.isPolarVolume(obj): for i in range(obj.getNumberOfScans()): scan = obj.getScan(i) copyDBZH(scan, ow) elif _polarscan.isPolarScan(obj): copyDBZH(obj, ow)
def drQC(pobject, profile_fstr=None, param_name="DBZH", zdr_offset=0.0, kernely=2, kernelx=2, param_thresh=35.0, dr_thresh=-12.0, Tw=True, keepDR=True): if profile_fstr: profile = ec_temperatureProfile.readProfile(profile_fstr) else: profile = None if _polarvolume.isPolarVolume(pobject): nscans = pobject.getNumberOfScans(pobject) for n in range(nscans): scan = pobject.getScan(n) drQCscan(scan, profile, param_name, zdr_offset, kernely, kernelx, param_thresh, dr_thresh, Tw, keepDR) elif _polarscan.isPolarScan(pobject): drQCscan(pobject, profile, param_name, zdr_offset, kernely, kernelx, param_thresh, dr_thresh, Tw, keepDR) else: raise IOError("Input object is neither polar volume nor scan")
def zdiff(obj, thresh=40.0): if _polarscan.isPolarScan(obj): zdiffScan(obj, thresh) elif _polarvolume.isPolarVolume(obj): zdiffPvol(obj, thresh) else: raise TypeError, "Input is expected to be a polar volume or scan"
def process(self, obj, reprocess_quality_flag=True, quality_control_mode=QUALITY_CONTROL_MODE_ANALYZE_AND_APPLY, arguments=None): if obj != None: try: if _polarscan.isPolarScan(obj): if reprocess_quality_flag == False and obj.findQualityFieldByHowTask("se.smhi.detector.beamblockage") != None: return obj bb = self._create_bb() result = bb.getBlockage(obj, self._dblimit) if quality_control_mode != QUALITY_CONTROL_MODE_ANALYZE: _beamblockage.restore(obj, result, "DBZH", self._bblimit) obj.addOrReplaceQualityField(result) elif _polarvolume.isPolarVolume(obj): for i in range(obj.getNumberOfScans()): scan = obj.getScan(i) if reprocess_quality_flag == False and scan.findQualityFieldByHowTask("se.smhi.detector.beamblockage") != None: continue bb = self._create_bb() result = bb.getBlockage(scan, self._dblimit) if quality_control_mode != QUALITY_CONTROL_MODE_ANALYZE: _beamblockage.restore(scan, result, "DBZH", self._bblimit) scan.addOrReplaceQualityField(result) except: logger.exception("Failed to generate beam blockage field") return obj
def readRB5(filenamelist): objects = readParameterFiles(filenamelist) rio = _raveio.new() if _polarscan.isPolarScan(objects[0]): rio.object = compileScanParameters(objects) elif _polarvolume.isPolarVolume(objects[0]): rio.object = compileVolumeFromVolumes(objects) return rio
def add_dealiased_param(obj): import _polarvolume if obj != None and _polarvolume.isPolarVolume(obj): for i in range(obj.getNumberOfScans()): scan = obj.getScan(i) add_dealiased_param_for_scan(scan) elif obj != None and _polarscan.isPolarScan(obj): add_dealiased_param_for_scan(obj)
def fetch_objects(self): nodes = "" objects = {} tasks = [] for fname in self.filenames: obj = None try: if self.ravebdb != None: obj = self.ravebdb.get_rave_object(fname) else: obj = _raveio.open(fname).object except IOError: self.logger.exception("Failed to open %s" % fname) is_scan = _polarscan.isPolarScan(obj) if is_scan: is_pvol = False else: is_pvol = _polarvolume.isPolarVolume(obj) if not is_scan and not is_pvol: self.logger.info( "Input file %s is neither polar scan or volume, ignoring." % fname) continue if self.ignore_malfunc: obj = rave_util.remove_malfunc(obj) if obj is None: continue node = odim_source.NODfromSource(obj) if len(nodes): nodes += ",'%s'" % node else: nodes += "'%s'" % node objects[fname] = obj if is_scan: self.logger.debug( "Scan used in composite generation - UUID: %s, Node: %s, Nominal date and time: %sT%s", fname, node, obj.date, obj.time) self.add_how_task_from_scan(obj, tasks) elif is_pvol: self.logger.debug( "PVOL used in composite generation - UUID: %s, Node: %s, Nominal date and time: %sT%s", fname, node, obj.date, obj.time) for i in range(obj.getNumberOfScans()): scan = obj.getScan(i) self.add_how_task_from_scan(scan, tasks) how_tasks = ",".join(tasks) return objects, nodes, how_tasks
def process(self, obj, reprocess_quality_flag=True, quality_control_mode=QUALITY_CONTROL_MODE_ANALYZE_AND_APPLY, arguments=None): objinfo = self.get_object_information(obj) if _polarscan.isPolarScan(obj): self.processScan(objinfo, obj) elif _polarvolume.isPolarVolume(obj): nscans = obj.getNumberOfScans(obj) for i in range(nscans): self.processScan(objinfo, obj.getScan(i)) return obj, self.getQualityFields()
def process(self, obj, reprocess_quality_flag=True, quality_control_mode=QUALITY_CONTROL_MODE_ANALYZE_AND_APPLY, arguments=None): try: if _polarscan.isPolarScan(obj) or _polarvolume.isPolarVolume(obj): odc_hac.zdiff(obj) except: logger.exception("Failure during zdiff processing") return obj, self.getQualityFields()
def hacIncrement(obj, quant="DBZH"): if _polarvolume.isPolarVolume(obj): incrementPvol(obj, quant) elif _polarscan.isPolarScan(obj): incrementScan(obj, quant) elif type(obj) == types.StringType: if os.path.isfile(obj) and os.path.getsize(obj): obj = _raveio.open(obj).object hacIncrement(obj) else: raise TypeError, "HAC incrementor received a string without a matching file, or file is empty" else: raise TypeError, "HAC incrementor received neither SCAN nor PVOL as input object"
def remove_malfunc(obj): result = obj if _polarvolume.isPolarVolume(obj): result = remove_malfunc_from_volume(obj) if result != None and result.getNumberOfScans() == 0: logger.debug( "All scans of the volume were detected as malfunc. Complete volume therefore considered as malfunc." ) result = None elif _polarscan.isPolarScan(obj): if get_malfunc_from_obj(obj): result = None return result
def is_polar_malfunc(obj): result = False if _polarvolume.isPolarVolume(obj): result = get_malfunc_from_obj(obj) if not result: for i in range(obj.getNumberOfScans()): result = get_malfunc_from_obj(obj.getScan(i)) if result: break elif _polarscan.isPolarScan(obj): result = get_malfunc_from_obj(obj) else: raise Exception("Neither polar volume or polar scan") return result
def zdiffScan(scan, thresh=40.0): if _polarscan.isPolarScan(scan): if not scan.hasParameter("DBZH") or not scan.hasParameter("TH"): return qind = _ravefield.new() qind.setData(zeros((scan.nrays, scan.nbins), uint8)) qind.addAttribute("how/task", "eu.opera.odc.zdiff") qind.addAttribute("how/task_args", thresh) qind.addAttribute("what/gain", 1 / 255.0) qind.addAttribute("what/offset", 0.0) scan.addQualityField(qind) ret = _odc_hac.zdiff(scan, thresh) else: raise TypeError, "Input is expected to be a polar scan. Got something else."
def should_perform_qc_process(reprocess, obj, how_task): if reprocess: return True if _polarscan.isPolarScan(obj) and obj.findQualityFieldByHowTask(how_task): return False if _polarvolume.isPolarVolume(obj): for i in range(obj.getNumberOfScans()): scan = obj.getScan(i) if not scan.findQualityFieldByHowTask(how_task): return True return False return True
def MakeAreaFromPolarObjects(objects, proj_id='llwgs84', xscale=2000.0, yscale=2000.0): import _rave, _raveio areas = [] for o in objects: if _polarvolume.isPolarVolume(o): scan = o.getScanWithMaxDistance() elif _polarscan.isPolarScan(o): scan = o else: raise IOError("Input object is not a polar scan or volume") areas.append(MakeSingleAreaFromSCAN(scan, proj_id, xscale, yscale)) minx = 10e100 maxx = -10e100 miny = 10e100 maxy = -10e100 for a in areas: if a.extent[0] < minx: minx = a.extent[0] if a.extent[1] < miny: miny = a.extent[1] if a.extent[2] > maxx: maxx = a.extent[2] if a.extent[3] > maxy: maxy = a.extent[3] # Expand to nearest pixel - buffering by one pixel was done in MakeSingleAreaFromSCAN dx = (maxx - minx) / xscale dx = (1.0 - (dx - int(dx))) * xscale if dx < xscale: minx -= dx dy = (maxy - miny) / yscale dy = (1.0 - (dy - int(dy))) * yscale if dy < yscale: miny -= dy xsize = int(round((maxx - minx) / xscale, 0)) ysize = int(round((maxy - miny) / yscale, 0)) A = AREA() A.xsize, A.ysize, A.xscale, A.yscale = xsize, ysize, xscale, yscale A.extent = minx, miny, maxx, maxy A.pcs = proj_id return A
def _add_files_to_argument_list(self, args, tiled_areas): self.logger.info("Distributing polar objects among %d tiles" % len(args)) # Loop through tile areas for i in range(len(tiled_areas)): p = tiled_areas[i].projection llx, lly, urx, ury = tiled_areas[i].extent # Loop through radars for k in self.file_objects.keys(): v = self.file_objects[k] if not _polarscan.isPolarScan( v) and not _polarvolume.isPolarVolume(v): continue if _polarvolume.isPolarVolume(v): v = v.getScanWithMaxDistance() scan = v if self.compositing.quantity not in scan.getParameterNames(): self.logger.info("Quantity %s not in data from %s" % (self.compositing.quantity, scan.source)) continue bi = scan.nbins - 1 # Loop around the scan for ai in range(scan.nrays): lon, lat = scan.getLonLatFromIndex(bi, ai) x, y = p.fwd((lon, lat)) # If this position is inside the tile, then add the radar's file string to the list and then bail if x >= llx and x <= urx and y >= lly and y <= ury: if not k in args[i][0].filenames: args[i][0].filenames.append(k) break # No need to continue for idx in range(len(args)): self.logger.info( "Tile %s contains %d files and dimensions %i x %i" % (args[idx][0].area_definition.id, len(args[idx][0].filenames), args[idx][0].area_definition.xsize, args[idx][0].area_definition.ysize)) self.logger.info("Finished splitting polar object")
def _add_object_to(self, srco, tgto): if _polarvolume.isPolarVolume(srco) and _polarscan.isPolarScan(tgto): raise TypeError, "Can not merge a volume into a scan" # Is srco is a polarvolume, we use a recursive back with the individual scans if _polarvolume.isPolarVolume(srco): for i in range(srco.getNumberOfScans()): s = srco.getScan(i) self._add_object_to(s, tgto) return # From here on we know that srco always is a scan tgtscan = tgto if _polarvolume.isPolarVolume(tgto): cscan = tgto.getScanClosestToElevation(srco.elangle, 0) if cscan.elangle != srco.elangle: tgto.addScan(srco) else: self._merge_parameters(srco, cscan) else: self._merge_parameters(srco, tgtscan)
def drQC(pobject, param_name="DBZH", zdr_offset=0.0, kernely=2, kernelx=2, param_thresh=35.0, dr_thresh=-12.0, keepDR=True): if _polarvolume.isPolarVolume(pobject): nscans = pobject.getNumberOfScans(pobject) for n in range(nscans): scan = pobject.getScan(n) drQCscan(scan, param_name, zdr_offset, kernely, kernelx, param_thresh, dr_thresh, keepDR) elif _polarscan.isPolarScan(pobject): drQCscan(pobject, param_name, zdr_offset, kernely, kernelx, param_thresh, dr_thresh, keepDR) else: raise IOError, "Input object is neither polar volume nor scan"
def _add_radar_index_value_to_argument_list(self, args): ctr = 1 for k in self.file_objects.keys(): v = self.file_objects[k] if not _polarscan.isPolarScan( v) and not _polarvolume.isPolarVolume(v): continue sourceid = v.source try: osource = odim_source.ODIM_Source(v.source) if osource.wmo: sourceid = "WMO:%s" % osource.wmo elif osource.rad: sourceid = "RAD:%s" % osource.rad elif osource.nod: sourceid = "NOD:%s" % osource.nod except: pass for arg in args: arg[0].radar_index_mapping[sourceid] = ctr ctr = ctr + 1
def _ensure_date_and_time_on_args(self, args): dtstr = None ddstr = None for k in self.file_objects.keys(): v = self.file_objects[k] if not _polarscan.isPolarScan( v) and not _polarvolume.isPolarVolume(v): continue dtstr = v.time ddstr = v.date break if dtstr is None or ddstr is None: self.logger.info("Could not determine any date and time string") return False for arg in args: if len(arg[0].filenames) == 0 and (arg[1] is None or arg[2] is None): arg[1] = ddstr arg[2] = dtstr return True
def ncar_PID(rio, profile_fstr, median_filter_len=0, pid_thresholds=None, zdr_offset=0.0, derive_dr=0, zdr_scale=1.0, keepExtras=False): profile = readProfile(profile_fstr, scale_height=1000.0) pobject = rio.object if _polarvolume.isPolarVolume(pobject): nscans = pobject.getNumberOfScans(pobject) for n in range(nscans): scan = pobject.getScan(n) pidScan(scan, profile, median_filter_len, pid_thresholds, zdr_offset, derive_dr, zdr_scale, keepExtras) elif _polarscan.isPolarScan(pobject): pidScan(pobject, profile, median_filter_len, pid_thresholds, zdr_offset, derive_dr, zdr_scale, keepExtras) else: raise IOError("Input object is neither polar volume nor scan")
def fetch_objects(self): nodes = "" objects = {} tasks = [] malfunc_files = 0 preload_quantity = None if self.use_lazy_loading and self.use_lazy_loading_preloads: preload_quantity = self.quantity for fname in self.filenames: obj = None try: if self.ravebdb != None: obj = self.ravebdb.get_rave_object(fname, self.use_lazy_loading, preload_quantity) else: obj = _raveio.open(fname, self.use_lazy_loading, preload_quantity).object #self.quantity except IOError: self.logger.exception("Failed to open %s", fname) is_scan = _polarscan.isPolarScan(obj) if is_scan: is_pvol = False else: is_pvol = _polarvolume.isPolarVolume(obj) if not is_scan and not is_pvol: self.logger.warn( "Input file %s is neither polar scan or volume, ignoring.", fname) continue # Force azimuthal nav information usage if requested obj.use_azimuthal_nav_information = self.use_azimuthal_nav_information if self.ignore_malfunc: obj = rave_util.remove_malfunc(obj) if obj is None: self.logger.info( "Input file %s detected as 'malfunc', ignoring.", fname) malfunc_files += 1 continue node = odim_source.NODfromSource(obj) if len(nodes): nodes += ",'%s'" % node else: nodes += "'%s'" % node objects[fname] = obj if is_scan: self.logger.debug( "Scan used in composite generation - UUID: %s, Node: %s, Nominal date and time: %sT%s", fname, node, obj.date, obj.time) self.add_how_task_from_scan(obj, tasks) elif is_pvol: self.logger.debug( "PVOL used in composite generation - UUID: %s, Node: %s, Nominal date and time: %sT%s", fname, node, obj.date, obj.time) for i in range(obj.getNumberOfScans()): scan = obj.getScan(i) self.add_how_task_from_scan(scan, tasks) how_tasks = ",".join(tasks) all_files_malfunc = (len(self.filenames) > 0 and malfunc_files == len(self.filenames)) return objects, nodes, how_tasks, all_files_malfunc