def generate(inobj, reprocess_quality_flag=True, quality_control_mode=QUALITY_CONTROL_MODE_ANALYZE_AND_APPLY): if _polarscan.isPolarScan(inobj) == False and _polarvolume.isPolarVolume( inobj) == False: raise IOError("Input file must be either polar scan or volume.") if reprocess_quality_flag == False: if _polarscan.isPolarScan(inobj) and inobj.findQualityFieldByHowTask( "fi.fmi.ropo.detector.classification"): return inobj elif _polarvolume.isPolarVolume(inobj): allprocessed = True for i in range(inobj.getNumberOfScans()): scan = inobj.getScan(i) if not scan.findQualityFieldByHowTask( "fi.fmi.ropo.detector.classification"): allprocessed = False break if allprocessed: return inobj options = get_options( inobj ) # Gets options/arguments for this radar. Fixes /what/source if required. if _polarvolume.isPolarVolume(inobj): ret = process_pvol(inobj, options, quality_control_mode) elif _polarscan.isPolarScan(inobj): month = int(inobj.date[4:6]) - 1 options.threshold = THRESHOLDS[options.threshold][month] ret = process_scan(inobj, options, quality_control_mode) copy_topwhat(inobj, ret) return ret
def create_filename(self, pobj): #_polarscan.isPolarScan(obj) and not _polarvolume.isPolarVolume(obj): if _polarvolume.isPolarVolume(pobj): ptype = "pvol" elif _polarscan.isPolarScan(pobj): ptype = "scan" else: try: ptype = pobj.getAttribute("what/object").tolower() except: ptype = "unknowntype" src = odim_source.NODfromSource(pobj) dstr = "19700101" tstr = "000000" try: dstr = pobj.date tstr = pobj.time except: pass t = tempfile.mkstemp(prefix="%s_%s_%s_%s_" % (ptype, src, dstr, tstr), suffix=".h5", dir=self.dumppath) os.close(t[0]) return t[1]
def worker_collect_radar_metadata(input=None): directory = input D_metadata = {} for fname in os.listdir(directory): try: filePath = os.path.join(directory, fname) rio = _rave.open(filePath) # use ODIM h5 functionality obj = rio.object if not _polarvolume.isPolarVolume(obj): # rarely happened sys.stderr.write('Warning: not polar volume: %s\n' % filePath) continue source = obj.source sourceTuple = source.split(',') id_radar = sourceTuple[0].split(':')[1] # NOTE ussox threw error for 2019 data plc = sourceTuple[1].split(':')[1] D_metadata[id_radar] = { 'lon': obj.longitude, 'lat': obj.latitude, 'hgt': obj.height, 'plc': plc, } except: sys.stderr.write('Warning, failed for %s\n' % fname) # Level2_KMQT_20180510_0204.ar2v.qc.usmqt.h5 # Level2_KRGX_20180520_1501.ar2v.qc.usrgx.h5 return D_metadata
def process(self, obj, reprocess_quality_flag=True, quality_control_mode=QUALITY_CONTROL_MODE_ANALYZE_AND_APPLY, arguments=None): if obj != None and _polarvolume.isPolarVolume(obj): import _detectionrange ascending = obj.isAscendingScans() drgenerator = _detectionrange.new() maxscan = obj.getScanWithMaxDistance() if reprocess_quality_flag or not maxscan.findQualityFieldByHowTask( "se.smhi.detector.poo"): # We want to have same resolution as maxdistance scan since we are going to add the poo-field to it # The second argument is dbz threshold, modify it accordingly topfield = drgenerator.top(obj, maxscan.rscale, -40.0) # Topfield is a scan filterfield = drgenerator.filter( topfield) # filterfield is a scan poofield = drgenerator.analyze( filterfield, 60, 0.1, 0.35) # poofield is a quality field, add it to maxscan maxscan.addOrReplaceQualityField(poofield) if ascending: obj.sortByElevations(1) return obj, self.getQualityFields()
def process(self, obj, reprocess_quality_flag=True, arguments=None): #_rave.setDebugLevel(_rave.Debug_RAVE_DEBUG) if not _polarvolume.isPolarVolume(obj): raise Exception, "Input object is not a polar volume. Bailing ..." # Using a dictionary lets us match tasks with payloads dpvol = {"dopvol1a" : None, "dopvol1b" : None, "dopvol1c" : None, "dopvol2" : None} for i in range(len(arguments)): dobj = _raveio.open(arguments[i]).object task = "".join(dobj.getAttribute('how/task').lower().split("_")) dpvol[task] = dobj dopvol = ec_dopvolqc.mergeDopvol(dopvol1a = dpvol["dopvol1a"], dopvol1b = dpvol["dopvol1b"], dopvol1c = dpvol["dopvol1c"], dopvol2 = dpvol["dopvol2"]) # Assume we always want spatial=1 filtering ec_dopvolqc.dopvolFilter(obj, dopvol) return obj
def drQC(pobject, profile_fstr=None, param_name="DBZH", zdr_offset=0.0, kernely=2, kernelx=2, param_thresh=35.0, dr_thresh=-12.0, Tw=True, keepDR=True): if profile_fstr: profile = ec_temperatureProfile.readProfile(profile_fstr) else: profile = None if _polarvolume.isPolarVolume(pobject): nscans = pobject.getNumberOfScans(pobject) for n in range(nscans): scan = pobject.getScan(n) drQCscan(scan, profile, param_name, zdr_offset, kernely, kernelx, param_thresh, dr_thresh, Tw, keepDR) elif _polarscan.isPolarScan(pobject): drQCscan(pobject, profile, param_name, zdr_offset, kernely, kernelx, param_thresh, dr_thresh, Tw, keepDR) else: raise IOError("Input object is neither polar volume nor scan")
def process(self, obj, reprocess_quality_flag=True, quality_control_mode=QUALITY_CONTROL_MODE_ANALYZE_AND_APPLY, arguments=None): if obj != None: try: if _polarscan.isPolarScan(obj): if reprocess_quality_flag == False and obj.findQualityFieldByHowTask("se.smhi.detector.beamblockage") != None: return obj bb = self._create_bb() result = bb.getBlockage(obj, self._dblimit) if quality_control_mode != QUALITY_CONTROL_MODE_ANALYZE: _beamblockage.restore(obj, result, "DBZH", self._bblimit) obj.addOrReplaceQualityField(result) elif _polarvolume.isPolarVolume(obj): for i in range(obj.getNumberOfScans()): scan = obj.getScan(i) if reprocess_quality_flag == False and scan.findQualityFieldByHowTask("se.smhi.detector.beamblockage") != None: continue bb = self._create_bb() result = bb.getBlockage(scan, self._dblimit) if quality_control_mode != QUALITY_CONTROL_MODE_ANALYZE: _beamblockage.restore(scan, result, "DBZH", self._bblimit) scan.addOrReplaceQualityField(result) except: logger.exception("Failed to generate beam blockage field") return obj
def hacFilter(obj, quant="DBZH"): if _polarvolume.isPolarVolume(obj): filterPvol(obj, quant) elif _polarscan.isPolarScan(obj): filterScan(obj, quant) else: raise TypeError, "HAC filter received neither SCAN nor PVOL as input"
def zdiffPvol(pvol, thresh=40.0): if _polarvolume.isPolarVolume(pvol): for i in range(pvol.getNumberOfScans()): scan = pvol.getScan(i) zdiffScan(scan, thresh) else: raise TypeError, "Input is expected to be a polar volume. Got something else."
def zdiff(obj, thresh=40.0): if _polarscan.isPolarScan(obj): zdiffScan(obj, thresh) elif _polarvolume.isPolarVolume(obj): zdiffPvol(obj, thresh) else: raise TypeError, "Input is expected to be a polar volume or scan"
def checkDBZH(obj, ow=True): if _polarvolume.isPolarVolume(obj): for i in range(obj.getNumberOfScans()): scan = obj.getScan(i) copyDBZH(scan, ow) elif _polarscan.isPolarScan(obj): copyDBZH(obj, ow)
def adjustFromRB5(obj): if _polarvolume.isPolarVolume(obj): for e in range(obj.getNumberOfScans()): scan = obj.getScan(e) adjustFromRB5(scan) # recursive call elif _polarscan.isPolarScan(obj): # First: convert RHOHV to unsigned 16-bit integer rhohv = obj.getParameter("RHOHV") data = rhohv.getData() data = data.astype(np.uint16) rhohv.setData(data) rhohv.nodata = 256.0 # Second: check for sector blanking. txPower = obj.getAttribute('how/TXpower') threshold = 0.9 * np.max(txPower) # Find out how many rays have tx powers beneath the threshold blanked = np.less(txPower, threshold) nrays2blank = np.sum(blanked) # Only continue if we know we need to, blanking all moments/quantities if nrays2blank: for pname in obj.getParameterNames(): param = obj.getParameter(pname) data = param.getData() for ray in range(obj.nrays): data[ray] = np.where(np.equal(blanked[ray], True), param.nodata, data[ray]) param.setData(data)
def readRB5(filenamelist): objects = readParameterFiles(filenamelist) rio = _raveio.new() if _polarscan.isPolarScan(objects[0]): rio.object = compileScanParameters(objects) elif _polarvolume.isPolarVolume(objects[0]): rio.object = compileVolumeFromVolumes(objects) return rio
def add_dealiased_param(obj): import _polarvolume if obj != None and _polarvolume.isPolarVolume(obj): for i in range(obj.getNumberOfScans()): scan = obj.getScan(i) add_dealiased_param_for_scan(scan) elif obj != None and _polarscan.isPolarScan(obj): add_dealiased_param_for_scan(obj)
def _add_files_to_argument_list(self, args, tiled_areas): self.logger.info("Distributing polar objects among %d tiles" % len(args)) # Loop through tile areas for i in range(len(tiled_areas)): p = tiled_areas[i].projection llx, lly, urx, ury = tiled_areas[i].extent # Loop through radars for k in self.file_objects.keys(): v = self.file_objects[k] if not _polarscan.isPolarScan( v) and not _polarvolume.isPolarVolume(v): continue if _polarvolume.isPolarVolume(v): v = v.getScanWithMaxDistance() scan = v if self.compositing.quantity not in scan.getParameterNames(): self.logger.info("Quantity %s not in data from %s" % (self.compositing.quantity, scan.source)) continue bi = scan.nbins - 1 # Loop around the scan for ai in range(scan.nrays): lon, lat = scan.getLonLatFromIndex(bi, ai) x, y = p.fwd((lon, lat)) # If this position is inside the tile, then add the radar's file string to the list and then bail if x >= llx and x <= urx and y >= lly and y <= ury: if not k in args[i][0].filenames: args[i][0].filenames.append(k) break # No need to continue for idx in range(len(args)): self.logger.info( "Tile %s contains %d files and dimensions %i x %i" % (args[idx][0].area_definition.id, len(args[idx][0].filenames), args[idx][0].area_definition.xsize, args[idx][0].area_definition.ysize)) self.logger.info("Finished splitting polar object")
def fetch_objects(self): nodes = "" objects = {} tasks = [] for fname in self.filenames: obj = None try: if self.ravebdb != None: obj = self.ravebdb.get_rave_object(fname) else: obj = _raveio.open(fname).object except IOError: self.logger.exception("Failed to open %s" % fname) is_scan = _polarscan.isPolarScan(obj) if is_scan: is_pvol = False else: is_pvol = _polarvolume.isPolarVolume(obj) if not is_scan and not is_pvol: self.logger.info( "Input file %s is neither polar scan or volume, ignoring." % fname) continue if self.ignore_malfunc: obj = rave_util.remove_malfunc(obj) if obj is None: continue node = odim_source.NODfromSource(obj) if len(nodes): nodes += ",'%s'" % node else: nodes += "'%s'" % node objects[fname] = obj if is_scan: self.logger.debug( "Scan used in composite generation - UUID: %s, Node: %s, Nominal date and time: %sT%s", fname, node, obj.date, obj.time) self.add_how_task_from_scan(obj, tasks) elif is_pvol: self.logger.debug( "PVOL used in composite generation - UUID: %s, Node: %s, Nominal date and time: %sT%s", fname, node, obj.date, obj.time) for i in range(obj.getNumberOfScans()): scan = obj.getScan(i) self.add_how_task_from_scan(scan, tasks) how_tasks = ",".join(tasks) return objects, nodes, how_tasks
def process(self, obj, reprocess_quality_flag=True, quality_control_mode=QUALITY_CONTROL_MODE_ANALYZE_AND_APPLY, arguments=None): objinfo = self.get_object_information(obj) if _polarscan.isPolarScan(obj): self.processScan(objinfo, obj) elif _polarvolume.isPolarVolume(obj): nscans = obj.getNumberOfScans(obj) for i in range(nscans): self.processScan(objinfo, obj.getScan(i)) return obj, self.getQualityFields()
def _add_object_to(self, srco, tgto): if _polarvolume.isPolarVolume(srco) and _polarscan.isPolarScan(tgto): raise TypeError, "Can not merge a volume into a scan" # Is srco is a polarvolume, we use a recursive back with the individual scans if _polarvolume.isPolarVolume(srco): for i in range(srco.getNumberOfScans()): s = srco.getScan(i) self._add_object_to(s, tgto) return # From here on we know that srco always is a scan tgtscan = tgto if _polarvolume.isPolarVolume(tgto): cscan = tgto.getScanClosestToElevation(srco.elangle, 0) if cscan.elangle != srco.elangle: tgto.addScan(srco) else: self._merge_parameters(srco, cscan) else: self._merge_parameters(srco, tgtscan)
def process(self, obj, reprocess_quality_flag=True, quality_control_mode=QUALITY_CONTROL_MODE_ANALYZE_AND_APPLY, arguments=None): try: if _polarscan.isPolarScan(obj) or _polarvolume.isPolarVolume(obj): odc_hac.zdiff(obj) except: logger.exception("Failure during zdiff processing") return obj, self.getQualityFields()
def _verify_elangles(self, pos): gotvolume = False elangle = None for po in pos: if _polarvolume.isPolarVolume(po): gotvolume = True if not gotvolume: for po in pos: if elangle is None: elangle = po.elangle if po.elangle != elangle: raise TypeError, "When merging scans elevation angles between files must be same."
def hacIncrement(obj, quant="DBZH"): if _polarvolume.isPolarVolume(obj): incrementPvol(obj, quant) elif _polarscan.isPolarScan(obj): incrementScan(obj, quant) elif type(obj) == types.StringType: if os.path.isfile(obj) and os.path.getsize(obj): obj = _raveio.open(obj).object hacIncrement(obj) else: raise TypeError, "HAC incrementor received a string without a matching file, or file is empty" else: raise TypeError, "HAC incrementor received neither SCAN nor PVOL as input object"
def remove_malfunc(obj): result = obj if _polarvolume.isPolarVolume(obj): result = remove_malfunc_from_volume(obj) if result != None and result.getNumberOfScans() == 0: logger.debug( "All scans of the volume were detected as malfunc. Complete volume therefore considered as malfunc." ) result = None elif _polarscan.isPolarScan(obj): if get_malfunc_from_obj(obj): result = None return result
def is_polar_malfunc(obj): result = False if _polarvolume.isPolarVolume(obj): result = get_malfunc_from_obj(obj) if not result: for i in range(obj.getNumberOfScans()): result = get_malfunc_from_obj(obj.getScan(i)) if result: break elif _polarscan.isPolarScan(obj): result = get_malfunc_from_obj(obj) else: raise Exception("Neither polar volume or polar scan") return result
def should_perform_qc_process(reprocess, obj, how_task): if reprocess: return True if _polarscan.isPolarScan(obj) and obj.findQualityFieldByHowTask(how_task): return False if _polarvolume.isPolarVolume(obj): for i in range(obj.getNumberOfScans()): scan = obj.getScan(i) if not scan.findQualityFieldByHowTask(how_task): return True return False return True
def process_file(out_dir, in_name): print "in process_file" print in_name print h5py.File(in_name, 'r') vol = _raveio.open(in_name).object if _polarvolume.isPolarVolume(vol): if getNod(vol) in SOURCES: fname = get_filename(vol) v2b = _pyvol2bird.new(vol) # calculate a vertical profile of birds vpr = v2b.vol2bird(vol) write_file_to_outdir(out_dir, fname, vpr) else: print_log("Not a polar volume. Ignoring")
def MakeAreaFromPolarObjects(objects, proj_id='llwgs84', xscale=2000.0, yscale=2000.0): import _rave, _raveio areas = [] for o in objects: if _polarvolume.isPolarVolume(o): scan = o.getScanWithMaxDistance() elif _polarscan.isPolarScan(o): scan = o else: raise IOError("Input object is not a polar scan or volume") areas.append(MakeSingleAreaFromSCAN(scan, proj_id, xscale, yscale)) minx = 10e100 maxx = -10e100 miny = 10e100 maxy = -10e100 for a in areas: if a.extent[0] < minx: minx = a.extent[0] if a.extent[1] < miny: miny = a.extent[1] if a.extent[2] > maxx: maxx = a.extent[2] if a.extent[3] > maxy: maxy = a.extent[3] # Expand to nearest pixel - buffering by one pixel was done in MakeSingleAreaFromSCAN dx = (maxx - minx) / xscale dx = (1.0 - (dx - int(dx))) * xscale if dx < xscale: minx -= dx dy = (maxy - miny) / yscale dy = (1.0 - (dy - int(dy))) * yscale if dy < yscale: miny -= dy xsize = int(round((maxx - minx) / xscale, 0)) ysize = int(round((maxy - miny) / yscale, 0)) A = AREA() A.xsize, A.ysize, A.xscale, A.yscale = xsize, ysize, xscale, yscale A.extent = minx, miny, maxx, maxy A.pcs = proj_id return A
def _merge_files(self, pos): result = None # First we want to check if one of these items is a volume since it makes more sence to add # scans to a volume than having to check if a later object is volume and then remerge for i in range(len(pos)): if _polarvolume.isPolarVolume(pos[i]): result = pos[i] del pos[i] break if result is None: result = pos[0] del pos[0] for po in pos: self._add_object_to(po, result) return result
def remove_malfunc_from_volume(obj): result = obj if _polarvolume.isPolarVolume(obj): if get_malfunc_from_obj(obj): logger.debug( "Malfunc volume found. Source: %s, Nominal date and time: %sT%s", obj.source, obj.date, obj.time) return None for i in range(obj.getNumberOfScans() - 1, -1, -1): scan = obj.getScan(i) if get_malfunc_from_obj(scan): logger.debug( "Malfunc scan with elangle %f found. Removing from volume. Source: %s, Nominal date and time: %sT%s" % ((scan.elangle * 180.0 / math.pi), obj.source, obj.date, obj.time)) obj.removeScan(i) return result
def drQC(pobject, param_name="DBZH", zdr_offset=0.0, kernely=2, kernelx=2, param_thresh=35.0, dr_thresh=-12.0, keepDR=True): if _polarvolume.isPolarVolume(pobject): nscans = pobject.getNumberOfScans(pobject) for n in range(nscans): scan = pobject.getScan(n) drQCscan(scan, param_name, zdr_offset, kernely, kernelx, param_thresh, dr_thresh, keepDR) elif _polarscan.isPolarScan(pobject): drQCscan(pobject, param_name, zdr_offset, kernely, kernelx, param_thresh, dr_thresh, keepDR) else: raise IOError, "Input object is neither polar volume nor scan"
def _add_radar_index_value_to_argument_list(self, args): ctr = 1 for k in self.file_objects.keys(): v = self.file_objects[k] if not _polarscan.isPolarScan( v) and not _polarvolume.isPolarVolume(v): continue sourceid = v.source try: osource = odim_source.ODIM_Source(v.source) if osource.wmo: sourceid = "WMO:%s" % osource.wmo elif osource.rad: sourceid = "RAD:%s" % osource.rad elif osource.nod: sourceid = "NOD:%s" % osource.nod except: pass for arg in args: arg[0].radar_index_mapping[sourceid] = ctr ctr = ctr + 1