def plotwithopts(ifile, method, vars, options = defaultoption): from PseudoNetCDF.sci_var import getvarpnc from PseudoNetCDF.pncgen import pncgen exec(options.pre_txt) for varkey in vars: figpath = eval(method)(ifile = ifile, varkey = varkey, options = options, before = options.before_txt, after = options.after_txt) pncgen(getvarpnc(ifile, list(vars) + ['TFLAG', 'time', 'latitude', 'longitude', 'latitude_bounds', 'longitude_bounds']), figpath + '.nc', verbose = False) exec(options.post_txt)
def testNCF2BPCH(self): bpchfile=bpch2(self.bpchpath, noscale = True) from PseudoNetCDF.pncgen import pncgen pncgen(bpchfile,self.bpchpath + '.check', inmode = 'r', outmode = 'w', format = 'bpch', verbose = False) orig = open(self.bpchpath, 'rb').read() new = open(self.bpchpath + '.check', 'rb').read() assert(orig == new) os.remove(self.bpchpath+'.check')
def testNCF2BPCH(self): bpchfile=bpch2(self.bpchpath, noscale = True) from PseudoNetCDF.pncgen import pncgen pncgen(bpchfile,self.bpchpath + '.check', inmode = 'r', outmode = 'w', format = 'bpch', verbose = 0) orig = open(self.bpchpath, 'rb').read() new = open(self.bpchpath + '.check', 'rb').read() assert(orig == new) os.remove(self.bpchpath+'.check')
def testNCF2CSV(self): from PseudoNetCDF.pncgen import pncgen from io import BytesIO import tempfile out = BytesIO() out = tempfile.TemporaryFile(mode = 'w+t') pncgen(self.testfile, out, inmode = 'r', outmode = 'w', format = 'csv', verbose = False) out.seek(0,0) outval = out.read() assert(outval == self.checkval)
def testNCF2CSV(self): from PseudoNetCDF.pncgen import pncgen from io import BytesIO import tempfile out = BytesIO() out = tempfile.TemporaryFile(mode = 'w+t') pncgen(self.testfile, out, inmode = 'r', outmode = 'w', format = 'csv', verbose = 0) out.seek(0,0) outval = out.read() assert(outval == self.checkval)
def plotwithopts(ifile, method, vars, options=defaultoption): from PseudoNetCDF.sci_var import getvarpnc from PseudoNetCDF.pncgen import pncgen exec(options.pre_txt) for varkey in vars: figpath = eval(method)(ifile=ifile, varkey=varkey, options=options, before=options.before_txt, after=options.after_txt) pncgen(getvarpnc(ifile, list(vars) + _coordkeys), figpath + '.nc', verbose=0) exec(options.post_txt)
def testNCF2HUM(self): import PseudoNetCDF.testcase from PseudoNetCDF.pncgen import pncgen import os inpath = PseudoNetCDF.testcase.camxfiles_paths['humidity'] outpath = PseudoNetCDF.testcase.camxfiles_paths['humidity'] + '.check' infile = humidity(inpath, 4, 5) pncgen(infile, outpath, format='camxfiles.humidity') orig = open(inpath, 'rb').read() new = open(outpath, 'rb').read() assert (orig == new) os.remove(outpath)
def testNCF2CR(self): import PseudoNetCDF.testcase from PseudoNetCDF.pncgen import pncgen import os inpath = PseudoNetCDF.testcase.camxfiles_paths['cloud_rain'] outpath = inpath + '.check' infile = cloud_rain(inpath, 4, 5) pncgen(infile, outpath, format='camxfiles.cloud_rain') orig = open(inpath, 'rb').read() new = open(outpath, 'rb').read() assert (orig == new) os.remove(outpath)
def testNCF2KV(self): import PseudoNetCDF.testcase from PseudoNetCDF.pncgen import pncgen import os inpath = PseudoNetCDF.testcase.camxfiles_paths['vertical_diffusivity'] outpath = inpath + '.check' infile = vertical_diffusivity(inpath, 4, 5) pncgen(infile, outpath, format='camxfiles.vertical_diffusivity') orig = open(inpath, 'rb').read() new = open(outpath, 'rb').read() assert(orig == new) os.remove(outpath)
def testNCF2HP(self): import PseudoNetCDF.testcase from PseudoNetCDF.pncgen import pncgen import os inpath = PseudoNetCDF.testcase.camxfiles_paths['height_pressure'] outpath = inpath + '.check' infile = height_pressure(inpath, 4, 5) pncgen(infile, outpath, format='camxfiles.height_pressure') orig = open(inpath, 'rb').read() new = open(outpath, 'rb').read() assert (orig == new) os.remove(outpath)
def testNCF2TEMP(self): import PseudoNetCDF.testcase from PseudoNetCDF.pncgen import pncgen import os inpath = PseudoNetCDF.testcase.camxfiles_paths['temperature'] outpath=PseudoNetCDF.testcase.camxfiles_paths['temperature'] + '.check' infile=temperature(inpath,4,5) pncgen(infile, outpath, format = 'camxfiles.temperature') orig = open(inpath, 'rb').read() new = open(outpath, 'rb').read() assert(orig == new) os.remove(outpath)
def testNCF2KV(self): import PseudoNetCDF.testcase from PseudoNetCDF.pncgen import pncgen import os inpath = PseudoNetCDF.testcase.camxfiles_paths['vertical_diffusivity'] outpath=PseudoNetCDF.testcase.camxfiles_paths['vertical_diffusivity'] + '.check' infile=one3d(inpath,4,5) pncgen(infile, outpath, format = 'camxfiles.vertical_diffusivity') orig = open(inpath, 'rb').read() new = open(outpath, 'rb').read() assert(orig == new) os.remove(outpath)
def testNCF2UAMIV(self): from PseudoNetCDF.camxfiles.Memmaps import uamiv uamivfile=uamiv(self.uamivpath) from PseudoNetCDF.pncgen import pncgen pncgen(uamivfile,self.uamivpath + '.check', inmode = 'r', outmode = 'w', format = 'uamiv', verbose = False) check = True uamivfile2=uamiv(self.uamivpath + '.check') for k, v in uamivfile.variables.items(): nv = uamivfile2.variables[k] check = check & bool((nv[...] == v[...]).all()) assert(check) os.remove(self.uamivpath+'.check')
def testNCF2HUM(self): import PseudoNetCDF.testcase from PseudoNetCDF.pncgen import pncgen import os inpath = PseudoNetCDF.testcase.camxfiles_paths['humidity'] outpath=PseudoNetCDF.testcase.camxfiles_paths['humidity'] + '.check' infile=humidity(inpath, 4, 5) pncgen(infile, outpath, format = 'camxfiles.humidity') orig = open(inpath, 'rb').read() new = open(outpath, 'rb').read() assert(orig == new) os.remove(outpath)
def testNCF2LB(self): import PseudoNetCDF.testcase from PseudoNetCDF.pncgen import pncgen import os inpath = PseudoNetCDF.testcase.camxfiles_paths['lateral_boundary'] outpath = inpath + '.check' infile = lateral_boundary(inpath) pncgen(infile, outpath, format='camxfiles.lateral_boundary') orig = open(inpath, 'rb').read() new = open(outpath, 'rb').read() assert (orig == new) os.remove(outpath)
def testNCF2LB(self): import PseudoNetCDF.testcase from PseudoNetCDF.camxfiles.Writers import ncf2lateral_boundary from PseudoNetCDF.pncgen import pncgen import os inpath = PseudoNetCDF.testcase.camxfiles_paths['lateral_boundary'] outpath=PseudoNetCDF.testcase.camxfiles_paths['lateral_boundary'] + '.check' infile=lateral_boundary(inpath) pncgen(infile, outpath, format = 'camxfiles.lateral_boundary') orig = open(inpath, 'rb').read() new = open(outpath, 'rb').read() assert(orig == new) os.remove(outpath)
def testNCF2WD(self): import PseudoNetCDF.testcase from PseudoNetCDF.pncgen import pncgen from PseudoNetCDF.camxfiles.Writers import ncf2wind import os inpath = PseudoNetCDF.testcase.camxfiles_paths['wind'] outpath = PseudoNetCDF.testcase.camxfiles_paths['wind'] + '.check' infile = wind(inpath, 4, 5) pncgen(infile, outpath, format='camxfiles.wind') orig = open(inpath, 'rb').read() new = open(outpath, 'rb').read() assert (orig == new) os.remove(outpath)
def testNCF2WD(self): import PseudoNetCDF.testcase from PseudoNetCDF.pncgen import pncgen from PseudoNetCDF.camxfiles.Writers import ncf2wind import os inpath = PseudoNetCDF.testcase.camxfiles_paths['wind'] outpath=PseudoNetCDF.testcase.camxfiles_paths['wind'] + '.check' infile=wind(inpath,4,5) pncgen(infile, outpath, format = 'camxfiles.wind') orig = open(inpath, 'rb').read() new = open(outpath, 'rb').read() assert(orig == new) os.remove(outpath)
def testNCF2BPCH(self): import warnings with warnings.catch_warnings(): warnings.filterwarnings( 'ignore', 'Not scaling variables; good for direct writing' ) bpchfile = bpch(self.bpchpath, noscale=True) outpath = self.bpchpath + '.check' from PseudoNetCDF.pncgen import pncgen pncgen(bpchfile, outpath, inmode='r', outmode='w', format='bpch', verbose=0) orig = open(self.bpchpath, 'rb').read() new = open(outpath, 'rb').read() assert(orig == new) os.remove(outpath) from PseudoNetCDF.sci_var import reduce_dim, slice_dim ALD2 = bpchfile.variables['IJ-AVG-$_ALD2'] ALD2_check = np.array( [1.60520077e-02, 1.82803553e-02, 2.00258084e-02, 2.01461259e-02, 1.84865110e-02, 2.49667447e-02, 2.73083989e-02, 2.87465211e-02, 2.89694592e-02, 2.87686456e-02, 2.87277419e-02, 3.08121163e-02, 3.22086290e-02, 3.35262120e-02, 3.41329686e-02, 3.05218045e-02, 3.30278911e-02, 3.58164124e-02, 3.93186994e-02, 4.15412188e-02, 1.60520077e-02, 1.82803553e-02, 2.00258084e-02, 2.01461259e-02, 1.84865110e-02, 2.49667447e-02, 2.73083989e-02, 2.87465211e-02, 2.89694592e-02, 2.87686456e-02, 2.87277419e-02, 3.08121163e-02, 3.22086290e-02, 3.35262120e-02, 3.41329686e-02, 3.05218045e-02, 3.30278911e-02, 3.58164124e-02, 3.93186994e-02, 4.15412188e-02, 1.60520077e-02, 1.82803553e-02, 2.00258084e-02, 2.01461259e-02, 1.84865110e-02, 2.49667447e-02, 2.73083989e-02, 2.87465211e-02, 2.89694592e-02, 2.87686456e-02, 2.87277419e-02, 3.08121163e-02, 3.22086290e-02, 3.35262120e-02, 3.41329686e-02, 3.05218045e-02, 3.30278911e-02, 3.58164124e-02, 3.93186994e-02, 4.15412188e-02])\ .reshape(ALD2.shape) slided_reduced_bpchfile = slice_dim( reduce_dim(bpchfile, 'layer,mean'), 'time,0') pncgen(slided_reduced_bpchfile, outpath, inmode='r', outmode='w', format='bpch', verbose=0) ALD2_check_slided_reduced = ALD2_check[0].mean(0)[None, None] ALD2 = slided_reduced_bpchfile.variables['IJ-AVG-$_ALD2'] np.testing.assert_allclose(ALD2, ALD2_check_slided_reduced * 1e-9) with warnings.catch_warnings(): warnings.filterwarnings( 'ignore', 'Not scaling variables; good for direct writing' ) bpchfile = bpch(outpath) ALD2 = bpchfile.variables['IJ-AVG-$_ALD2'] np.testing.assert_allclose(ALD2, ALD2_check_slided_reduced)
def testNCF2PT(self): import PseudoNetCDF.testcase from PseudoNetCDF.pncgen import pncgen import os inpath = PseudoNetCDF.testcase.camxfiles_paths['point_source'] outpath = inpath + '.check' infile = point_source(inpath) pncgen(infile, outpath, format='camxfiles.point_source') orig = open(inpath, 'rb').read() new = open(outpath, 'rb').read() # outfile=point_source(outpath) # for k, v0 in infile.variables.items(): # v1 = outfile.variables[k] # print(k, (v0[:] == v1[:]).all()) assert(orig == new) os.remove(outpath)
def testNCF2PT(self): import PseudoNetCDF.testcase from PseudoNetCDF.pncgen import pncgen import os inpath = PseudoNetCDF.testcase.camxfiles_paths['point_source'] outpath=PseudoNetCDF.testcase.camxfiles_paths['point_source'] + '.check' infile=point_source(inpath) pncgen(infile, outpath, format = 'camxfiles.point_source') orig = open(inpath, 'rb').read() new = open(outpath, 'rb').read() #outfile=point_source(outpath) #for k, v0 in infile.variables.items(): # v1 = outfile.variables[k] # print(k, (v0[:] == v1[:]).all()) assert(orig == new) os.remove(outpath)
def testNCF2UAMIV(self): from PseudoNetCDF.camxfiles.Memmaps import uamiv uamivfile = uamiv(self.uamivpath) from PseudoNetCDF.pncgen import pncgen pncgen(uamivfile, self.uamivpath + '.check', inmode='r', outmode='w', format='uamiv', verbose=False) check = True uamivfile2 = uamiv(self.uamivpath + '.check') for k, v in uamivfile.variables.items(): nv = uamivfile2.variables[k] check = check & bool((nv[...] == v[...]).all()) assert (check) os.remove(self.uamivpath + '.check')
def plotwithopts(ifile, method, vars, options=defaultoption): from PseudoNetCDF.sci_var import getvarpnc from PseudoNetCDF.pncgen import pncgen exec(options.pre_txt) for varkey in vars: figpath = eval(method)(ifile=ifile, varkey=varkey, options=options, before=options.before_txt, after=options.after_txt) pncgen(getvarpnc( ifile, list(vars) + [ 'TFLAG', 'time', 'latitude', 'longitude', 'latitude_bounds', 'longitude_bounds' ]), figpath + '.nc', verbose=False) exec(options.post_txt)
def testNCF2BPCH(self): bpchfile=bpch(self.bpchpath, noscale = True) outpath = self.bpchpath + '.check' from PseudoNetCDF.pncgen import pncgen pncgen(bpchfile,outpath, inmode = 'r', outmode = 'w', format = 'bpch', verbose = False) orig = open(self.bpchpath, 'rb').read() new = open(outpath, 'rb').read() assert(orig == new) os.remove(outpath) from PseudoNetCDF.sci_var import reduce_dim, slice_dim ALD2 = bpchfile.variables['IJ-AVG-$_ALD2'] ALD2_check = np.array([1.60520077e-02, 1.82803553e-02, 2.00258084e-02, 2.01461259e-02, 1.84865110e-02, 2.49667447e-02, 2.73083989e-02, 2.87465211e-02, 2.89694592e-02, 2.87686456e-02, 2.87277419e-02, 3.08121163e-02, 3.22086290e-02, 3.35262120e-02, 3.41329686e-02, 3.05218045e-02, 3.30278911e-02, 3.58164124e-02, 3.93186994e-02, 4.15412188e-02, 1.60520077e-02, 1.82803553e-02, 2.00258084e-02, 2.01461259e-02, 1.84865110e-02, 2.49667447e-02, 2.73083989e-02, 2.87465211e-02, 2.89694592e-02, 2.87686456e-02, 2.87277419e-02, 3.08121163e-02, 3.22086290e-02, 3.35262120e-02, 3.41329686e-02, 3.05218045e-02, 3.30278911e-02, 3.58164124e-02, 3.93186994e-02, 4.15412188e-02, 1.60520077e-02, 1.82803553e-02, 2.00258084e-02, 2.01461259e-02, 1.84865110e-02, 2.49667447e-02, 2.73083989e-02, 2.87465211e-02, 2.89694592e-02, 2.87686456e-02, 2.87277419e-02, 3.08121163e-02, 3.22086290e-02, 3.35262120e-02, 3.41329686e-02, 3.05218045e-02, 3.30278911e-02, 3.58164124e-02, 3.93186994e-02, 4.15412188e-02]).reshape(ALD2.shape) bpchfile = slice_dim(reduce_dim(bpchfile, 'layer,mean'), 'time,0') pncgen(bpchfile,outpath, inmode = 'r', outmode = 'w', format = 'bpch', verbose = False) ALD2_check = ALD2_check[0].mean(0)[None, None] ALD2 = bpchfile.variables['IJ-AVG-$_ALD2'] np.testing.assert_allclose(ALD2, ALD2_check * 1e-9) bpchfile = bpch(outpath) ALD2 = bpchfile.variables['IJ-AVG-$_ALD2'] np.testing.assert_allclose(ALD2, ALD2_check)
def plotwithopts(ifile, method, vars, options=defaultoption): from PseudoNetCDF.sci_var import getvarpnc from PseudoNetCDF.pncgen import pncgen import matplotlib.pyplot as plt # dummy assignment so that flake8 sees plt as used # plt is loaded into the environment for exec varkey = plt exec(options.pre_txt) for varkey in vars: figpath = eval(method)(ifile=ifile, varkey=varkey, options=options, before=options.before_txt, after=options.after_txt) pncgen(getvarpnc(ifile, list(vars) + list(_coordkeys)), figpath + '.nc', verbose=0) exec(options.post_txt)
def pncwrite(*args, **kwds): """See PseudoNetCDF.pncgen.pncgen *args : iterable **kwds : keywords keywords for pncgen help : boolean without format, returns help of pncopen and with format keyword, returns help of the function that writes that format. """ from PseudoNetCDF.pncgen import pncgen formathelp = kwds.pop('help', False) if formathelp: if 'format' in kwds: format = kwds['format'] writer = getwriterdict()[format] return help(writer) else: return help(pncgen) return pncgen(*args, **kwds)
aggfunc=np.sum) oldpolname = None outfile = getvarpnc(Dataset(templatepath, 'r+'), ['TFLAG'] + unique_vars) add_ioapi_from_ioapi(outfile) # outfile.variables['TFLAG'][:, :, 0] = np.arange(0, 25)[:, None] # outfile.variables['TFLAG'][:, :, 0] = 2014001 print('Working on: ' + csvpath) for (polname, rowi, coli), groupdata in aggdata.iterrows(): if args.verbose > 2: print(polname) if polname != oldpolname: if not oldpolname is None: if args.verbose > 0: print('Writing out ' + polname + ' to ' + var.long_name) var[0:26, 0, :, :] = temp[0:26, 0, :, :] if args.verbose > 0: print('Starting ' + polname) if polname == 'BEN': var = outfile.variables['BENZENE'] elif polname != 'CO2': var = outfile.variables[polname] else: warn('WARNING: skipping %s in %s' % (polname, csvpath)) oldpolname = None continue temp = np.zeros_like(var[:]) oldpolname = polname temp[0:26, 0, rowi, coli] += np.array( [groupdata[hours[k]].sum() for k in range(0, 26)]) # catch the last pollutant var[0:26, 0, :, :] = temp[0:26, 0, :, :] pncgen(outfile, outpath, verbose=args.verbose)
zipout = open(filepath, 'wb') zipout.write(data) zipout.flush() zipout.close() import zipfile zf = zipfile.ZipFile(filepath) zf.extract(yearpath) yearpaths.append(yearpath) from PseudoNetCDF.epafiles import aqsraw from PseudoNetCDF.sci_var import stack_files from PseudoNetCDF.pncgen import pncgen infiles = [] for yearpath in yearpaths: infile = aqsraw(yearpath, timeresolution=args.timeresolution, param=args.param, bdate=args.bdate, edate=args.edate, rdate=args.rdate, wktpolygon=args.wktpolygon, sampleval=args.sampleval, verbose=args.verbose) infiles.append(infile) outfile = stack_files(infiles, 'time') persisted = pncgen(outfile, args.outpath) persisted.sync() print('Successful')
def pncwrite(*args, **kwds): """See PseudoNetCDF.pncgen.pncgen""" from PseudoNetCDF.pncgen import pncgen return pncgen(*args, **kwds)
def wrt_uamiv(fout_path, fin, *, lsurf = False, lapp = False, lemis = False, lcmaq = False, ounits = None): """ Create an UAMIV file using fin class. Arguments: fout_path - The IOAPI file name including path fin - A class which include all the contents, data arrays, attributes, variable names. The data array shape is (nspc,nsteps,nz,ny,nx). lsurf - If True, output file has only 1 layer regardless of shape of data array. lapp - If True, output file must already exist. Skip writing file dimensions and attributes. It simply appends data arrays into the existing output file. lemis - If True, it is emission file, and variable units have rates instead of concentration lcmaq - If True, the file is for CMAQ, and varnames are checked against CMAQ PM species so that PM unit is assigned, microgram/m**3 instead of moles/m**3. ounits - A list which has units of variables """ # Handling arguments lounit = True if ounits == None: lounit = False ounits = [] # Include modules import datetime from PseudoNetCDF import PseudoNetCDFFile from PseudoNetCDF.pncgen import pncgen from PseudoNetCDF import PNC from PseudoNetCDF.sci_var import stack_files from CAMxtools.write.wrt_ioapi import find_unit from pathlib import Path import os #prepare file attributes novars = getattr(fin,'NVARS') varnames = getattr(fin,'VAR-LIST').split() var0 = varnames[0] nsteps = fin.variables[var0].shape[0] nz = getattr(fin,'NLAYS') ny = getattr(fin,'NROWS') nx = getattr(fin,'NCOLS') lgrdded = False if getattr(fin,'FTYPE') == 1: lgrdded = True if lgrdded: # GRIDDED assert len(fin.variables[var0].shape) == 4, "len(fin.variables[var0].shape) MUST be 4" else: # BOUNDARY CONDITION assert len(fin.variables[var0].shape) == 3, "len(fin.variables[var0].shape) MUST be 3" ncells = fin.variables[var0].shape[2] assert ncells == 2*(nx+ny)+4, "ncells MUST be 2*(nx+ny)+4" #open output file if not lgrdded: #BOUNDARY ftype = 'lateral_boundary' else: ftype = 'uamiv' newf = PseudoNetCDFFile() # Set tstep tstep = getattr(fin,'TSTEP') #copy dimensions dimensions_keys = "TSTEP DATE-TIME LAY VAR".split() # These dimension keys must be in the file to be used by m3tools if lgrdded: # GRIDDED dimensions_keys.append("ROW"); dimensions_keys.append("COL") else: dimensions_keys.append("PERIM") for i in dimensions_keys: if i == 'VAR': size = novars elif i == 'TSTEP': size = nsteps elif i == 'DATE-TIME': size = 2 elif i == 'LAY': if lsurf: size = 1 else: size = nz elif i == 'VAR': size = novars elif i == 'ROW': size = ny elif i == 'COL': size = nx else : size = ncells # i == 'PERIM' newf.createDimension(i,size) #copy global attributes attribs = "XORIG YORIG XCELL YCELL PLON PLAT TLAT1 TLAT2 IUTM ISTAG CPROJ GDTYP XCENT YCENT P_ALP P_BET P_GAM NLAYS NROWS NCOLS NVARS VAR-LIST NAME NOTE ITZON FTYPE VGTYP VGTOP VGLVLS GDNAM UPNAM FILEDESC SDATE STIME TSTEP Conventions history".split() # These attributes must be in the file for pncgen to uamiv cdate = int(datetime.date.today().strftime("%Y%j")) ctime = int(datetime.datetime.now().strftime("%H%M%S")) for i in attribs: try: val = getattr(fin,i) except: val = "" if 'numpy.float32' in str(type(val)): val = val.item() if i == 'PLON': val = getattr(fin,'XCENT') if i == 'PLAT': val = getattr(fin,'YCENT') if i == 'TLAT1': val = getattr(fin,'P_ALP') if i == 'TLAT2': val = getattr(fin,'P_GAM') if i == 'IUTM': val = 0 if getattr(fin,'GDTYP') == 5: val = 1 if i == 'ISTAG': val = 0 if i == 'CPROJ': if getattr(fin,'GDTYP') == 1: #LATLON val = 0 elif getattr(fin,'GDTYP') == 5: #UTM val = 1 elif getattr(fin,'GDTYP') == 2: #LCP val = 2 elif getattr(fin,'GDTYP') == 6: #PSP val = 4 elif getattr(fin,'GDTYP') == 7: #Equatorial Mercator val = 5 else: print("GDTYP = {}".format(GDTYP)) exit("Not relevant projection") if i == 'NLAYS': if lsurf: val = 1 else: size = nz if i == 'NROWS': val = ny if i == 'NCOLS': val = nx if i == 'NVARS': val = novars if i == 'NSTEPS': val = nsteps if i == 'NAME': name_str = 'AVERAGE' if lemis: name_str = 'EMISSIONS' if not lgrdded: name_str = 'BOUNDARY' val = '{:<10s}'.format(name_str) if i == 'NOTE': val = '{:<60s}'.format("wrt_uamiv in CAMxtools") if i == 'ITZON': val = 0 if i == 'VGTYP': if lsurf or nz == 1: val = -9999 else: size = 2 # VGSGPN3 non-h sigma-p if i == 'VGTOP': if lsurf or nz == 1: val = -9.999E36 else: size = getattr(fin,'VGTOP') if i == 'GDNAM' or i == 'UPNAM' or i == 'FILEDESC': val = '{:<16s}'.format("CAMx") if i == 'Conventions': val = "CF-1.6" if i == 'history': val = '{:<250s}'.format("unspecified") setattr(newf,i,val) #copy variables extra = "TFLAG".split() for i in extra+varnames: if i == 'TFLAG': newf.createVariable(i,('int32'),(u'TSTEP', u'VAR', u'DATE-TIME')) newf.variables[i].setncattr("units","<YYYYDDD,HHMMSS>") newf.variables[i].setncattr("long_name","TFLAG") newf.variables[i].setncattr("var_desc",'{:<80s}'.format("Timestep-valid flags: (1) YYYYDDD or (2) HHMMSS")) idate = getattr(fin,'SDATE') itime = getattr(fin,'STIME') for istep in range(nsteps): newf.variables['TFLAG'][istep,:,0] = [idate for ivar in range(len(varnames))] newf.variables['TFLAG'][istep,:,1] = [itime for ivar in range(len(varnames))] itime += tstep if itime == 240000: itime = 0 idate = int((datetime.datetime.strptime(str(idate),"%Y%j") + datetime.timedelta(days=1)).strftime("%Y%j")) else: if lgrdded: # GRIDDED var_type = str(type(fin.variables[var0][0,0,0,0])) else: var_type = str(type(fin.variables[var0][0,0,0])) if 'numpy.float' in var_type: dtype = 'float32' else: dtype = 'int32' if lgrdded: # GRIDDED newf.createVariable(i, (dtype), ('TSTEP', 'LAY', 'ROW', 'COL')) else: newf.createVariable(i, (dtype), ('TSTEP', 'LAY', 'PERIM')) #Find a relevant unit if lounit: s = varnames.index(i) unit = ounits[s] else: unit = find_unit(i,lemis=lemis,lcmaq=lcmaq) newf.variables[i].setncattr("long_name",'{:<16s}'.format(i)) newf.variables[i].setncattr("units",'{:<16s}'.format(unit)) newf.variables[i].setncattr("var_desc",'{:<80s}'.format("".join(["VARIABLE ",i]))) newf.variables[i]=fin.variables[i] # Create UAMIV file if lapp: # Save the original fout_path to fout_old_path fout_old_path = fout_path + ".old" try: os.rename(fout_path,fout_old_path) # Move the original output to fout_old_path #pncargs = '--format=' + ftype + ',mode="r+"' #pncargs = '--format=' + ftype except: print("Output file is {}".format(fout_path)) exit("Output file does not exist while lapp is True") pncargs = '--format=' + ftype oldfile = PNC(pncargs, fout_old_path).ifiles[0] fout_tmp_path = fout_path + ".tmp" pncgen(newf, fout_tmp_path, format = ftype) tmpfile = PNC(pncargs, fout_tmp_path).ifiles[0] newf_app = stack_files([oldfile, tmpfile], 'TSTEP') pncgen(newf_app, fout_path, format = ftype) if Path(fout_old_path).exists(): os.remove(fout_old_path) if Path(fout_tmp_path).exists(): os.remove(fout_tmp_path) else: pncgen(newf, fout_path, format = ftype) # close files if 'newfile' in locals(): del newfile del fin if 'newf' in locals(): newf.close() print ('*** SUCCESS writing UAMIV file') return
def output(out, outpath, config, verbose = 0): """ Make final unit conversions, add log information and save output to disk out - PseudoNetCDFFile with output data outpath - Path for output config - configuration dictionary """ from repair_ae import repair_ae from PseudoNetCDF.pncgen import pncgen tmp = defaultdict(lambda: 1) tmp['np'] = np for k, v in config['unitconversions']['metadefs'].iteritems(): eval(v, None, tmp) for k in tmp.keys(): if k != 'np': tmp[k] = out.variables[k] for k, v in config['unitconversions']['metadefs'].iteritems(): exec('%s = %s' % (k, v), tmp, out.variables) if verbose > 0: status('Vertical profile Low -> High') np.set_printoptions(precision = 2) for vark, varo in out.variables.items(): if hasattr(varo, 'unitnow'): if varo.unitnow.strip() != varo.units.strip(): expr = config['unitconversions']['%s->%s' % (varo.unitnow.strip(), varo.units.strip())].replace('<value>', 'varo') exec('varo[:] = %s' % expr, dict(varo = varo), out.variables) varo.history += ';' + expr.replace('varo', 'RESULT') del varo.unitnow if vark not in ('time', 'TFLAG') and 'latitude' not in vark and 'longitude' not in vark and (varo[...] <= 0).any(): if config['zero_negs']: have_zeros = varo[...] <= 0 varo[np.where(have_zeros)] = 1.e-30 warn(vark + ' %d (of %d; %.2f%%) negative or zero values were set to 1e-30' % (have_zeros.sum(), have_zeros.size, have_zeros.mean() * 100), stacklevel = 1) else: warn(vark + ' has negative or zero values', stacklevel = 1) if verbose > 0 and 'TSTEP' in varo.dimensions and 'PERIM' in varo.dimensions: status('%s: %s' % (vark, str(varo[:, :].mean(0).mean(1)))) np.set_printoptions(precision = None) out.geos2cmaq_warnings = myioo.getwarnings() out.geos2cmaq_errors = myioo.geterrors() out.geos2cmaq_status = myioo.getstatus() varnames = [k.ljust(16) for k in out.variables.keys() if k[:1] in uppercase and k != 'TFLAG'] out.createDimension('VAR', len(varnames)) tflag = out.variables['TFLAG'] var = out.createVariable('TFLAG', 'i', ('TSTEP', 'VAR', 'DATE-TIME')) var.long_name = 'TFLAG'.ljust(16); var.var_desc = "Timestep-valid flags: (1) YYYYDDD or (2) HHMMSS".ljust(80) var.units = "<YYYYDDD,HHMMSS>" var[:, :, :] = tflag[:, [0], :].repeat(var.shape[1], 1) f = pncgen(out, outpath, inmode = 'r', outmode = 'w', format = 'NETCDF3_CLASSIC', verbose = False) setattr(f, 'VAR-LIST', ''.join(varnames)) setattr(f, 'NVARS', len(varnames)) f.sync() f.close() if config['repair_aero']: f = Dataset(outpath, 'r+') repair_ae(f, myioo) f.sync() f.close()
p2p.addGlobalProperties(obstimefile, nomtimefile) for vark in varkeys: p2p.addVariable(obstimefile, nomtimefile, vark, data=False) outvar = nomtimefile.variables[vark] var = obstimefile.variables[vark] timem = (var[:] / var[:]) * time[:][:, None] dt = np.ma.MaskedArray(unomtimes[:, None, None]) - timem[None, :, :] nomtime_idx, points_idx = np.indices(dt[:, 0].shape) time_idx = np.abs(dt).argmin(1) outvals = var[:][time_idx, points_idx] min_dt = dt[nomtime_idx, time_idx, points_idx] dt_mask = (min_dt < dt_min) | (min_dt > dt_max) outvals = np.ma.masked_where(dt_mask, outvals) outvar[:] = outvals[:] pncgen(nomtimefile, args.hourlypath, verbose=0) # :DD_long_name = "QC data descriptor model: QC summary values" ; # :DD_reference = "AWIPS Technique Specification Package (TSP) 88-21-R2" ; # :DD_values = "Z,C,S,V,X,Q,K,k,G, or B" ; # :DD_value_Z = "No QC applied" ; # :DD_value_C = "Passed QC stage 1" ; # :DD_value_S = "Passed QC stages 1 and 2" ; # :DD_value_V = "Passed QC stages 1, 2 and 3" ; # :DD_value_X = "Failed QC stage 1" ; # :DD_value_Q = "Passed QC stage 1, but failed stages 2 or 3 " ; # :DD_value_K = "Passed QC stages 1, 2, 3, and 4" ; # :DD_value_k = "Passed QC stage 1,2, and 3, failed stage 4 " ; # :DD_value_G = "Included in accept list" ; # :DD_value_B = "Included in reject list" ;