def generate(files, arguments): args = arglist2dict(arguments) if not _rave.isCFConventionSupported(): logger.info("CF Conventions is not supported, ignoring export") return None if len(files) != 1: raise AttributeError("Must provide one file to export") if not "filename" in args.keys(): raise AttributeError("Must specify name of file to export") obj = None if ravebdb != None: obj = ravebdb.get_rave_object(files[0]) else: rio = _raveio.open(files[0]) obj = rio.object if not _cartesianvolume.isCartesianVolume( obj) and not _cartesian.isCartesian(obj): raise AttributeError("Must call plugin with cartesian products") filename = args["filename"] rio = _raveio.new() rio.object = obj rio.file_format = _raveio.RaveIO_FileFormat_CF rio.save(filename) return None
def generate(self, dd, dt, area=None): pyarea = my_area_registry.getarea(area) if self.preprocess_qc and self.mp_process_qc and self.number_of_quality_control_processes > 1: self.file_objects, self.nodes, self.how_tasks, all_files_malfunc = self._fetch_file_objects_mp( ) else: self.file_objects, self.nodes, self.how_tasks, all_files_malfunc = self._fetch_file_objects( ) if all_files_malfunc: self.logger.info( "Content of all provided files were marked as 'malfunc'. Since option 'ignore_malfunc' is set, no composite is generated!" ) return None args = self._create_arguments(dd, dt, pyarea) results = [] ntiles = len(args) ncpucores = multiprocessing.cpu_count() nrprocesses = ntiles if not RAVE_TILE_COMPOSITING_PROCESSES is None: if nrprocesses > RAVE_TILE_COMPOSITING_PROCESSES: nrprocesses = RAVE_TILE_COMPOSITING_PROCESSES if nrprocesses > ncpucores: nrprocesses = ncpucores if nrprocesses == ncpucores and ncpucores > 1: nrprocesses = nrprocesses - 1 # We always want to leave at least one core for something else pool = multiprocessing.Pool(nrprocesses) r = pool.map_async(comp_generate, args, callback=results.append) r.wait() pool.terminate() pool.join() self.logger.info("Finished processing tiles, combining tiles") objects = [] try: for v in results[0]: tile_file = v[1] if tile_file == None: self.logger.warn( "No partial composite for tile area %s was created. This tile will therefore not be included in complete composite.", v[0]) else: o = _raveio.open(tile_file).object if _cartesianvolume.isCartesianVolume(o): o = o.getImage(0) o.objectType = _rave.Rave_ObjectType_COMP objects.append(o) t = _transform.new() self.logger.debug( "Combining %d tiles into one composite for area %s.", len(objects), area) result = t.combine_tiles(pyarea, objects) # Fix so that we get a valid place for /what/source and /how/nodes result.source = "%s,CMT:%s" % (CENTER_ID, area) result.addAttribute('how/nodes', self.nodes) if self.how_tasks != "": result.addAttribute('how/task', self.how_tasks) self.logger.info("Tiles combined") return result finally: if self._do_remove_temporary_files: for fname in self.compositing.filenames: try: os.unlink(fname) except: logger.info("Failed to remove temporary file: %s" % fname) if results != None: for v in results[0]: if v != None and v[1] != None and os.path.exists(v[1]): try: os.unlink(v[1]) except Exception: logger.exception("Failed to unlink %s" % v[1]) return None
def test_isCartesianVolumeFalse(self): obj = _cartesian.new() self.assertEqual(False, _cartesianvolume.isCartesianVolume(obj))
def generate(files, arguments): args = arglist2dict(arguments) zr_a = 200.0 zr_b = 1.6 quantity = "DBZH" accept = 0.0 distancefield = "se.smhi.composite.distance.radar" interval = 12 N = 13 adjustmentfile = None #Accept is the required limit for how many nodata-pixels that are allowed in order for the #data to be accumulated #If we expect to have 10 observations and an accept limit of 20, then it can be 0, 1 or 2 observations #with nodata for that position. etime = args["time"] edate = args["date"] acrrproduct = None if "zra" in args.keys(): zr_a = float(args["zra"]) if "zrb" in args.keys(): zr_b = float(args["zrb"]) if "quantity" in args.keys(): quantity = args["quantity"] if "accept" in args.keys(): accept = float(args["accept"]) / 100.0 if "distancefield" in args.keys(): distancefield = args["distancefield"] if "interval" in args.keys(): interval = int(args["interval"]) if "N" in args.keys(): N = int(args["N"]) if "adjustmentfile" in args.keys(): adjustmentfile = args["adjustmentfile"] if distancefield == "eu.baltrad.composite.quality.distance.radar": distancefield = "se.smhi.composite.distance.radar" acrr = _acrr.new() acrr.nodata = -1.0 acrr.undetect = 0.0 acrr.quality_field_name = distancefield logger.info("rave_pgf_gra_plugin: Processing files") for fname in files: obj = None if ravebdb != None: obj = ravebdb.get_rave_object(fname) else: rio = _raveio.open(fname) obj = rio.object if _cartesianvolume.isCartesianVolume(obj): obj = obj.getImage(0) if not _cartesian.isCartesian(obj): raise AttributeError("Must call plugin with cartesian products") if acrrproduct == None: acrrproduct = _cartesian.new() acrrproduct.xscale = obj.xscale acrrproduct.yscale = obj.yscale acrrproduct.areaextent = obj.areaextent acrrproduct.projection = obj.projection acrrproduct.product = obj.product acrrproduct.source = obj.source acrrproduct.time = etime acrrproduct.date = edate if obj.xscale != acrrproduct.xscale or obj.yscale != acrrproduct.yscale or \ obj.projection.definition != acrrproduct.projection.definition: raise AttributeError( "Scale or projdef inconsistancy for used area") par = obj.getParameter(quantity) if par == None: logger.warn("Could not find parameter (%s) for %s %s" % (quantity, obj.date, obj.time)) else: if par.getQualityFieldByHowTask(distancefield) != None: acrr.sum(par, zr_a, zr_b) logger.info("rave_pgf_gra_plugin: Running acrr accumulation") # accept, N, hours acrrparam = acrr.accumulate(accept, N, interval) acrrproduct.addParameter(acrrparam) db = rave_dom_db.create_db_from_conf() try: logger.info("rave_pgf_gra_plugin: Calculating gra coefficients") calculate_gra_coefficient(distancefield, interval, adjustmentfile, etime, edate, acrrproduct, db) logger.info("rave_pgf_gra_plugin: Coefficients calculated") except OperationalError as e: if "server closed the connection unexpectedly" in e.message: logger.warn( "Got indication that connection reseted at server side, retrying gra coefficient generation" ) calculate_gra_coefficient(distancefield, interval, adjustmentfile, etime, edate, acrrproduct, db) return None
def generate(files, arguments): args = arglist2dict(arguments) zr_a = 200.0 zr_b = 1.6 quantity = "DBZH" accept = 0.0 distancefield = "se.smhi.composite.distance.radar" hours = 1 N = 5 applygra = False #Accept is the required limit for how many nodata-pixels that are allowed in order for the #data to be accumulated #If we expect to have 10 observations and an accept limit of 20, then it can be 0, 1 or 2 observations #with nodata for that position. etime = args["time"] edate = args["date"] img = None if "zra" in args.keys(): zr_a = float(args["zra"]) if "zrb" in args.keys(): zr_b = float(args["zrb"]) if "quantity" in args.keys(): quantity = args["quantity"] if "accept" in args.keys(): accept = float(args["accept"]) / 100.0 if "distancefield" in args.keys(): distancefield = args["distancefield"] if "hours" in args.keys(): hours = int(args["hours"]) if "N" in args.keys(): N = int(args["N"]) if "applygra" in args: applygra = True if distancefield == "eu.baltrad.composite.quality.distance.radar": distancefield = "se.smhi.composite.distance.radar" pdatetime = datetime.datetime.strptime( edate + etime, "%Y%m%d%H%M%S") - datetime.timedelta(minutes=60 * hours) sdate = pdatetime.strftime("%Y%m%d") stime = pdatetime.strftime("%H%M00") acrr = _acrr.new() acrr.nodata = -1.0 acrr.undetect = 0.0 acrr.quality_field_name = distancefield nodes = None for fname in files: obj = None if ravebdb != None: obj = ravebdb.get_rave_object(fname) else: rio = _raveio.open(fname) obj = rio.object if _cartesianvolume.isCartesianVolume(obj): obj = obj.getImage(0) if not _cartesian.isCartesian(obj): raise AttributeError("Must call plugin with cartesian products") if img == None: img = _cartesian.new() img.xscale = obj.xscale img.yscale = obj.yscale img.areaextent = obj.areaextent img.projection = obj.projection img.product = _rave.Rave_ProductType_RR img.source = obj.source img.time = etime img.date = edate img.startdate = sdate img.starttime = stime img.enddate = edate img.endtime = etime if obj.hasAttribute("how/nodes") and nodes == None: nodes = obj.getAttribute("how/nodes") if obj.xscale != img.xscale or obj.yscale != img.yscale or \ obj.projection.definition != img.projection.definition: raise AttributeError( "Scale or projdef inconsistancy for used area") par = obj.getParameter(quantity) if par == None: logger.warn("Could not find parameter (%s) for %s %s" % (quantity, obj.date, obj.time)) else: if par.getQualityFieldByHowTask(distancefield) != None: acrr.sum(par, zr_a, zr_b) # accept, N, hours result = acrr.accumulate(accept, N, hours) fileno, outfile = rave_tempfile.mktemp(suffix='.h5', close="True") if nodes != None: img.addAttribute("how/nodes", nodes) img.addParameter(result) #logger.info("Apply gra: %s"%`applygra`) if applygra: db = rave_dom_db.create_db_from_conf() dt = datetime.datetime(int(edate[:4]), int(edate[4:6]), int(edate[6:]), int(etime[:2]), int(etime[2:4]), 0) dt = dt - datetime.timedelta( seconds=3600 * 12) # 12 hours back in time for now.. gra = _gra.new() gra.A = DEFAULTA gra.B = DEFAULTB gra.C = DEFAULTC gra.zrA = zr_a gra.zrb = zr_b grac = db.get_gra_coefficient(dt) if grac != None and not math.isnan(grac.a) and not math.isnan( grac.b) and not math.isnan(grac.c): logger.debug("Using gra coefficients from database, quantity: %s" % quantity) gra.A = grac.a gra.B = grac.b gra.C = grac.c else: logger.info( "Could not find coefficients for given time, trying to get aged or climatologic coefficients" ) nowdt = datetime.datetime(int(edate[:4]), int(edate[4:6]), int(edate[6:]), int(etime[:2]), int(etime[2:4]), 0) agedt = nowdt - datetime.timedelta( seconds=3600 * 48) # 2 days back sig, pts, loss, r, rsig, corr, gra.A, gra.B, gra.C, mean, dev = get_backup_gra_coefficient( db, agedt, nowdt) dfield = result.getQualityFieldByHowTask(distancefield) gra_field = gra.apply(dfield, result) gra_field.quantity = result.quantity + "_CORR" img.addParameter(gra_field) ios = _raveio.new() ios.object = img ios.filename = outfile ios.version = RAVE_IO_DEFAULT_VERSION ios.save() return outfile