def generate(files, arguments): args = arglist2dict(arguments) for fname in files: logger.info("Input file: %s" % fname) obj = None if ravebdb != None: obj = ravebdb.get_rave_object(fname) else: rio = _raveio.open(fname) obj = rio.object v2b = _pyvol2bird.new(obj) # calculate a vertical profile of birds vpr = v2b.vol2bird(obj, 0.0) fileno, outfile = rave_tempfile.mktemp(suffix='.h5', close="True") logger.info("Output file: %s" % outfile) ios = _raveio.new() ios.object = vpr ios.filename = outfile ios.save() return outfile
def test_ctoscan(self): obj = _transform.new() cartesian = _raveio.open(self.FIXTURE_CARTESIAN_PPI).object volume = _raveio.open(self.FIXTURE_VOLUME).object radardef = _radardef.new() radardef.id = volume.source radardef.description = "ctop test" radardef.longitude = volume.longitude radardef.latitude = volume.latitude radardef.height = volume.height elangles = [] nangles = volume.getNumberOfScans() for i in range(nangles): scan = volume.getScan(i) elangles.append(scan.elangle) radardef.elangles = elangles scan = volume.getScan(0) radardef.nrays = scan.nrays radardef.nbins = scan.nbins radardef.scale = scan.rscale radardef.beamwidth = scan.beamwidth elangle = scan.elangle result = obj.ctoscan(cartesian, radardef, elangle, "DBZH") rio = _raveio.new() rio.object = result rio.save("ctop_polarscan.h5")
def makeODIM(): now = time.time() ifstrs = [] Afiles = glob.glob(os.path.join(DROOT, A) + '/*dBZ.azi') Bfiles = glob.glob(os.path.join(DROOT, B) + '/*dBZ.azi') Cfiles = glob.glob(os.path.join(DROOT, C) + '/*dBZ.azi') allfiles = Afiles + Bfiles + Cfiles # Only bother with recent files for f in allfiles: if (now - os.path.getmtime(f)) < DELAY: ifstrs.append(f) print f pvol = ec_rb5.makePvolFromFiles(ifstrs) pvol = odc_polarQC.QC(pvol) rio = _raveio.new() rio.object = pvol DATE = pvol.date TIME = pvol.time ofstr = "%s/caxka_pvol_%sT%sZ.h5" % (OPATH, pvol.date, pvol.time) rio.save(ofstr) return ofstr, pvol.date, pvol.time
def combineRB5(ifiles, out_fullfile=None, return_rio=False): big_obj = None nMEMBERs = len(ifiles) for iMEMBER in range(nMEMBERs): inp_fullfile = ifiles[iMEMBER] validate(inp_fullfile) mb = parse_tarball_member_name(inp_fullfile, ignoredir=True) if mb['rb5_ftype'] == "rawdata": rio = singleRB5( inp_fullfile, return_rio=True) ## w/ isRainbow5() check and handles .gz this_obj = rio.object if rio.objectType == _rave.Rave_ObjectType_PVOL: big_obj = compile_big_pvol(big_obj, this_obj, mb, iMEMBER) else: big_obj = compile_big_scan(big_obj, this_obj, mb) container = _raveio.new() container.object = big_obj if out_fullfile: container.save(out_fullfile) if return_rio: return container
def generate(files, arguments): args = arglist2dict(arguments) logger.debug("rave_pgf_apply_qc_plugin called with arguments: %s", args) # should only be one file fname = files[0] volume = generate_new_volume_with_qc(fname, args) if volume == None: logger.info("No volume with QC applied could be generated!") return None _, outfile = rave_tempfile.mktemp(suffix='.h5', close="True") ios = _raveio.new() ios.object = volume ios.filename = outfile ios.version = RAVE_IO_DEFAULT_VERSION ios.save() logger.info("Generated new volume with QC applied.") return outfile
def combineRB5(ifiles, out_fullfile=None, return_rio=False): big_obj = None nMEMBERs = len(ifiles) for iMEMBER in range(nMEMBERs): inp_fullfile = ifiles[iMEMBER] validate(inp_fullfile) mb = parse_tarball_member_name(inp_fullfile, ignoredir=True) if mb['rb5_ftype'] == "rawdata": isrb5 = _rb52odim.isRainbow5(inp_fullfile) if not isrb5: raise IOError, "%s is not a proper RB5 raw file" % inp_fullfile else: rio = _rb52odim.readRB5(inp_fullfile) ## by FILENAME this_obj = rio.object if rio.objectType == _rave.Rave_ObjectType_PVOL: big_obj = compile_big_pvol(big_obj, this_obj, mb, iMEMBER) else: big_obj = compile_big_scan(big_obj, this_obj, mb) container = _raveio.new() container.object = big_obj if out_fullfile: container.save(out_fullfile) if return_rio: return container
def generate(files, arguments): args = arglist2dict(arguments) quality_control_mode = QUALITY_CONTROL_MODE_ANALYZE_AND_APPLY volume = generateVolume(files, args) fileno, outfile = rave_tempfile.mktemp(suffix='.h5', close="True") if "anomaly-qc" in args.keys(): detectors = args["anomaly-qc"].split(",") else: detectors = [] if "qc-mode" in args.keys(): quality_control_mode = args["qc-mode"] volume = perform_quality_control(volume, detectors, quality_control_mode.lower()) ios = _raveio.new() ios.object = volume ios.filename = outfile ios.version = RAVE_IO_DEFAULT_VERSION ios.save() return outfile
def testPPI(self): volume = _raveio.open(self.VOLUMENAME).object transformer = _rave.transform() transformer.method = _rave.NEAREST a = area.area("ang_240") param = _rave.cartesianparam() param.nodata = 255.0 param.undetect = 0.0 param.quantity = "DBZH" param.setData(numpy.zeros((a.ysize, a.xsize), numpy.uint8)) cartesian = _rave.cartesian() cartesian.xscale = a.xscale cartesian.yscale = a.yscale cartesian.areaextent = a.extent cartesian.date = "20100101" cartesian.time = "090000" cartesian.source = volume.source cartesian.product = _rave.Rave_ProductType_CAPPI cartesian.objectType = _rave.Rave_ObjectType_IMAGE cartesian.areaextent = a.extent cartesian.projection = _rave.projection(a.Id, a.name, pcs.pcs(a.pcs).tostring()) cartesian.addParameter(param) cartesian.defaultParameter = "DBZH" scan = volume.getScan(0) transformer.ppi(scan, cartesian) rio = _raveio.new() rio.object = cartesian rio.save("cartesian_ppi.h5")
def generate(files, arguments): args = arglist2dict(arguments) if not _rave.isCFConventionSupported(): logger.info("CF Conventions is not supported, ignoring export") return None if len(files) != 1: raise AttributeError("Must provide one file to export") if not "filename" in args.keys(): raise AttributeError("Must specify name of file to export") obj = None if ravebdb != None: obj = ravebdb.get_rave_object(files[0]) else: rio = _raveio.open(files[0]) obj = rio.object if not _cartesianvolume.isCartesianVolume( obj) and not _cartesian.isCartesian(obj): raise AttributeError("Must call plugin with cartesian products") filename = args["filename"] rio = _raveio.new() rio.object = obj rio.file_format = _raveio.RaveIO_FileFormat_CF rio.save(filename) return None
def generate(infiles, DATE, TIME): comp = compositing() comp.filenames = infiles.split(",") comp.detectors = odc_polarQC.algorithm_ids comp.quantity = "DBZH" comp.set_product_from_string("PCAPPI") #comp.range = options.range comp.gain = 0.5 comp.offset = -32.0 comp.prodpar = 1000.0 comp.set_method_from_string("NEAREST_RADAR") comp.qitotal_field = None comp.pcsid = "gmaps" comp.xscale = 250.0 comp.yscale = 250.0 comp.zr_A = 200.0 comp.zr_b = 1.6 comp.applygapfilling = True comp.reprocess_quality_fields = True result = comp.generate(DATE, TIME, None) result.objectType = _rave.Rave_ObjectType_IMAGE rio = _raveio.new() rio.object = result ifstr = os.path.split(infiles)[1] ofstr = re.sub('pvol', 'pcappi', ifstr) ofstr = os.path.join(PPATH, ofstr) rio.save(ofstr) return ofstr
def readRB5(filenamelist): objects = readParameterFiles(filenamelist) rio = _raveio.new() if _polarscan.isPolarScan(objects[0]): rio.object = compileScanParameters(objects) elif _polarvolume.isPolarVolume(objects[0]): rio.object = compileVolumeFromVolumes(objects) return rio
def test_fillGap(self): obj = _transform.new() io = _raveio.new() cartesian = _raveio.open(self.FIXTURE_CARTESIAN_PCAPPI).object io.object = obj.fillGap(cartesian) io.filename = self.TRANSFORM_FILLGAP_FILENAME io.save()
def write_file_to_outdir(out_dir, outname, dst): fname = "%s/%s" % (out_dir, outname) #print fname rio = _raveio.new() rio.object = dst rio.save(fname) rio.close() print_log("Wrote %s" % fname)
def testGenerateVolumeAndSave(self): args = {"source":"sella","date":"20101023","time":"180000"} volume = rave_pgf_volume_plugin.generateVolume(self.FIXTURES, args) ios = _raveio.new() ios.object = volume ios.filename = self.TEMPORARY_FILE ios.save() ios = None
def test_top(self): dr = _detectionrange.new() o = _raveio.open(self.FIXTURE_VOLUME) result = dr.top(o.object, 2000, -40.0) os = _raveio.new() os.filename = self.TEMPORARY_FILE os.object = result os.save()
def testGeneratorSequence_generateClassification(self): a = _raveio.open(self.PVOL_RIX_TESTFILE).object.getScan(0) b = _ropogenerator.new(_fmiimage.fromRave(a, "DBZH")) c = b.speck(-20, 5).emitter(3, 6).classify().classification output = _raveio.new() output.object = c.toPolarScan("DBZH") output.filename = self.TEMPORARY_FILE output.save()
def test_third_generate_with_several_howattributes(self): pvol = _raveio.open(self.FIXTURE4).object wrwp = load_wrwp_defaults_to_obj() wrwp.emin = 4.0 fields = None exceptionTest = False try: fields = QUANTITIES vp = wrwp.generate(pvol, fields) uwnd = vp.getUWND() vwnd = vp.getVWND() hght = vp.getHGHT() nv = vp.getNV() self.assertEqual(1, uwnd.xsize) self.assertEqual(60, uwnd.ysize) self.assertEqual("UWND", uwnd.getAttribute("what/quantity")) self.assertEqual(1, vwnd.xsize) self.assertEqual(60, vwnd.ysize) self.assertEqual("VWND", vwnd.getAttribute("what/quantity")) self.assertEqual(1, hght.xsize) self.assertEqual(60, hght.ysize) self.assertEqual("HGHT", hght.getAttribute("what/quantity")) self.assertEqual(1, nv.xsize) self.assertEqual(60, nv.ysize) self.assertEqual("n", nv.getAttribute("what/quantity")) self.assertEqual(60, vp.getLevels()) self.assertEqual(200, vp.interval) self.assertEqual(0, vp.minheight) self.assertEqual(12000, vp.maxheight) self.assertEqual(pvol.source, vp.source) self.assertEqual(pvol.date, vp.date) self.assertEqual(pvol.time, vp.time) self.assertEqual(5.0, vp.getAttribute("how/minrange")) self.assertEqual(25.0, vp.getAttribute("how/maxrange")) self.assertEqual("4.0,8.0,14.0,24.0,40.0", vp.getAttribute("how/angles")) self.assertEqual("osd_zdr,osd_zdr_fastshort,osd_ldr_fastshort,osd_zdr_longshort", vp.getAttribute("how/task")) robj = _raveio.new() robj.object = vp robj.save("slask3.h5") except: exceptionTest = True self.assertEqual(False, exceptionTest)
def generate(flist, trim=True): try: # # Skip NEXRAD, K for CONUS, P for Alaska # if os.path.split(flist[0])[1][0] in ('K', 'P'): # return flist[0], "Skipped" files = [] for fstr in flist: if mimetypes.guess_type(fstr)[1] == 'gzip': files.append(gunzip(fstr)) else: files.append(fstr) path, fstr = os.path.split(files[0]) # Is Canadian? Starts with digits, whereas NEXRAD LII files start with letters if fstr[0] in string.digits: # Is IRIS? if _iris2odim.isIRIS(files[0]): rio = makeCanadianPETScan(files) #rio = makeCanadianPvol(files) # Else is McGill else: rio = _raveio.new() rio.object = ec_mcgill.file2pvol( files[0]) # Should only be one file # Else is Level II else: rio = ec_nexrad.readLevelII(files[0]) if trim: rio = ecWxR_trimRange.generate( "bobbe", RIO=rio) # dummy file string because it doesn't exist yet # Determine output file string ofstr = ec_filesys.MakePolarFileName(rio, root=OUTPATH, makepath=True, Round=True) rio.save(ofstr) # Clean up tmp directory for fstr in files: if os.path.split(fstr)[0] == TMPPATH: os.remove(fstr) except Exception, e: # Clean up tmp directory for fstr in files: if os.path.split(fstr)[0] == TMPPATH: os.remove(fstr) return flist, traceback.format_exc()
def X_test_generate_2(self): pvol = _raveio.open(self.FIXTURE).object wrwp = load_wrwp_defaults_to_obj() wrwp.hmax = 2000 wrwp.dz = 200 fields = None vp = wrwp.generate(pvol, fields) robj = _raveio.new() robj.object = vp robj.save("slasktest.h5")
def multi_generate(fstrs, DATE, TIME, sequential=False, trim=0, return_rio=True): import multiprocessing if trim > 0: import ecWxR_trimRange ca, us = [], [] # Split up Canada and US for fstr in fstrs: path, filename = os.path.split(fstr) if filename[:2] == 'us': us.append(fstr) else: ca.append(fstr) trimmed = ecWxR_trimRange.multi_generate(us) fstrs = ca + trimmed args = [] tile_areas, files_per_tile = addAreas(), {} for k in tile_areas.keys(): files_per_tile[k] = [] # Which files belong to which tiles? for fstr in fstrs: whichTile(fstr, AREA, tile_areas, files_per_tile) RESIDENT_LUT.write() # Update the file on disk for the next time the composite is generated # Access each tile's list of input files for k in files_per_tile.keys(): args.append([files_per_tile[k], k, DATE, TIME]) #print k, files_per_tile[k] #sys.exit() #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! results = [] if not sequential: pool = multiprocessing.Pool(len(tile_areas.keys())) r = pool.map_async(generate, args, callback=results.append) r.wait() # Wait on the results pool.terminate() pool.join() else: for a in args: #print a[1] results.append(generate(a)) # for i in range(len(results[0])): # print results[0][i] ocomp = mergeTiles(results[0]) rio = _raveio.new() rio.object = ocomp if return_rio: return rio rio.filename = ec_filesys.MakeCompositeFileName(rio, root=COMPROOT, prefix='qcomp_v7_', makepath=True) rio.save() return rio.filename
def concatenate_topo(): ios1 = _raveio.open("./W020N90.h5") src1 = ios1.object data1 = src1.getParameter("TOPO").getData() ios2 = _raveio.open("./E020N90.h5") src2 = ios2.object data2 = src2.getParameter("TOPO").getData() #data2 = numpy.zeros((6000,4800), numpy.int16) a1 = array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], numpy.int16) data = concatenate((a1, a1), 1) print(concatenate((a1, a1), 1)) print(data) a = _area.new() a.id = "BALTRAD" a.projection = src1.projection a.extent = (src1.areaextent[0], src1.areaextent[1], src2.areaextent[2], src2.areaextent[3]) a.xsize = src1.xsize + src2.xsize a.ysize = src1.ysize a.xscale = (a.extent[2] - a.extent[0]) / (a.xsize - 1) a.yscale = (a.extent[3] - a.extent[1]) / (a.ysize - 1) print(a.id) print(a.projection.definition) print(a.extent) print(a.xsize, a.ysize) print(a.xscale, a.yscale) src = _cartesian.new() src.init(a, _rave.RaveDataType_SHORT) src.time = src1.time src.date = src1.date src.objectType = src1.objectType src.product = src1.product src.source = src1.source param1 = src1.getParameter("TOPO") param = _cartesianparam.new() param.quantity = param1.quantity param.gain = param1.gain param.offset = param1.offset param.nodata = param1.nodata param.setData(data) ios = _raveio.new() ios.object = src ios.filename = "./BALTRAD_topo.h5" ios.save()
def test_analyze_and_write(self): dr = _detectionrange.new() o = _raveio.open(self.FIXTURE_VOLUME) topfield = dr.top(o.object, 2000, -40.0) filterfield = dr.filter(topfield) result = dr.analyze(filterfield, 60, 0.1, 0.5) param = _polarscanparam.fromField(result) scan = _polarscan.new() scan.addParameter(param) os = _raveio.new() os.filename = self.TEMPORARY_FILE os.object = scan os.save()
def test_nearest_first(self): a = self.create_area() obj = _cartesiancomposite.new() obj.method = _cartesiancomposite.SelectionMethod_FIRST for f in self.FIXTURES: obj.add(_raveio.open(f).object) obj.nodata = 255.0 obj.undetect = 0.0 result = obj.nearest(a) result.time = "120000" result.date = "20090501" result.source = "eua_gmaps" rio = _raveio.new() rio.object = result rio.save("cart_composite_first.h5")
def test_create_intersect(self): obj = _bitmapgenerator.new() cartesian = _raveio.open( self.FIXTURE_COMPOSITE_WITH_RADARINDEX).object.getImage(0) result = obj.create_intersect(cartesian.getParameter("DBZH"), "se.smhi.composite.index.radar") dbzh = cartesian.getParameter("DBZH") bitmap = result.getData() d = numpy.where(numpy.equal(bitmap, 1.0), 255.0, dbzh.getData()).astype(numpy.uint8) np = _cartesianparam.new() np.setData(d) np.quantity = "BRDR" cartesian.addParameter(np) cartesian.getParameter("DBZH").addQualityField(result) rio = _raveio.new() rio.object = cartesian rio.save(self.GENERATOR_RADARINDEX_FILENAME)
def testGeneratorSequence_fullExample(self): a = _raveio.open(self.PVOL_RIX_TESTFILE).object.getScan(0) b = _ropogenerator.new(_fmiimage.fromRave(a, "DBZH")) b.speck(-20, 5).speckNormOld(-20, 5, 16).emitter(-20, 4).emitter2(-10, 4, 2) classification = b.classify().classification.toRaveField() restored = b.restore(50).toRaveField() a.addQualityField(classification) a.addQualityField(restored) output = _raveio.new() output.object = a output.filename = self.TEMPORARY_FILE self.exportFile(output)
def test_create_surrounding(self): obj = _bitmapgenerator.new() cartesian = _raveio.open( self.FIXTURE_COMPOSITE_WITH_RADARINDEX).object.getImage(0) result = obj.create_surrounding(cartesian.getParameter("DBZH")) cartesian.getParameter("DBZH").addQualityField(result) dbzh = cartesian.getParameter("DBZH").getData() bitmap = result.getData() dbzh = numpy.where(numpy.equal(dbzh, 255.0), 0.0, dbzh).astype(numpy.uint8) d = numpy.where(numpy.equal(bitmap, 1.0), 255.0, dbzh).astype(numpy.uint8) np = _cartesianparam.new() np.setData(d) np.quantity = "BRDR" cartesian.addParameter(np) rio = _raveio.new() rio.object = cartesian rio.save(self.GENERATOR_SURROUNDING_FILENAME)
def test_nearest_distance(self): a = self.create_area() obj = _cartesiancomposite.new() obj.method = _cartesiancomposite.SelectionMethod_DISTANCE for f in self.DISTANCE_FIXTURES: ic = _raveio.open(f).object.getImage(0) obj.add(ic) obj.nodata = 255.0 obj.undetect = 0.0 result = obj.nearest(a) result.time = "120000" result.date = "20090501" result.source = "eua_gmaps" rio = _raveio.new() rio.object = result rio.save("cart_composite_distance.h5")
def generate(args): TIME = args.pop() DATE = args.pop() areaid = args.pop() files = args.pop() start = time.time() # For performance benchmarking comp = compositing.compositing() comp.filenames = files comp.detectors = "ropo,beamb,radvol-att,radvol-broad,distance,qi-total".split(",") comp.quantity = ["TH", "DBZH"] # comp.set_product_from_string("MAX") comp.set_product_from_string("PCAPPI") comp.range = 200000.0 comp.gain = rave_defines.GAIN comp.offset = rave_defines.OFFSET comp.prodpar = 1000.0 comp.set_method_from_string("HEIGHT_ABOVE_SEALEVEL") comp.qitotal_field = "pl.imgw.quality.qi_total" comp.pcsid = PCSID comp.xscale = AREA.xscale comp.yscale = AREA.yscale comp.zr_A = rave_defines.ZR_A comp.zr_b = rave_defines.ZR_b comp.applygapfilling = False comp.verbose = True comp.reprocess_quality_fields = False result = comp.generate(DATE, TIME, areaid) rio = _raveio.new() rio.object = result # Create temporary file in a place with write access rio.filename = rave_tempfile.mktemp(suffix='_%s_%sT%sZ.h5' % (areaid, DATE, TIME), close="True")[1] rio.save() end = time.time() return (rio.filename, end-start)
def main(options): comp = compositing() comp.filenames = options.infiles.split(",") comp.detectors = options.qc.split(",") comp.quantity = options.quantity comp.set_product_from_string(options.product) comp.range = options.range comp.gain = options.gain comp.offset = options.offset comp.prodpar = options.prodpar comp.minvalue = options.minvalue comp.set_method_from_string(options.method) comp.qitotal_field = options.qitotal_field comp.pcsid = options.pcsid comp.xscale = options.scale comp.yscale = options.scale comp.zr_A = options.zr_A comp.zr_b = options.zr_b if options.gf: comp.applygapfilling = True if options.ctfilter: comp.applyctfilter = True if options.grafilter: comp.applygra = True if options.ignore_malfunc: comp.ignore_malfunc = True if options.verbose: comp.verbose = True result = comp.generate(options.date, options.time, options.area) rio = _raveio.new() rio.object = result rio.filename = options.outfile if comp.verbose: logger.info("Saving %s" % rio.filename) rio.save()
def readLevelII(filename, return_l2=False): l2 = nexrad_level2.NEXRADLevel2File(filename) rio = _raveio.new() # Always assume we're reading volumes, not individual scans. if l2.nscans > 1: rio.object = _polarvolume.new() else: rio.object = _polarscan.new() obj = rio.object source = odim_source.SOURCE['us' + l2.volume_header['icao'][1:].lower()] obj.source = source.encode(UTF8) # Site location lat, lon, alt = l2.location() obj.longitude = lon * dr obj.latitude = lat * dr obj.height = float(alt) # Date and time conversions # obj.date, obj.time = makeDateTime(l2.volume_header['date'], l2.volume_header['time']) obj.date, obj.time = get_times(l2) getTopLevelHowAttrs(l2, obj) if rio.objectType == _rave.Rave_ObjectType_SCAN: populateScan(l2, obj) elif rio.objectType == _rave.Rave_ObjectType_PVOL: for i in range(l2.nscans): scan = _polarscan.new() populateScan(l2, scan, index=i) if len(scan.getParameterNames() ) > 0: # Don't add the scan if there are no moments in it obj.addScan(scan) if return_l2: return rio, l2 else: return rio