def plotwithopts(ifile, method, vars, options = defaultoption):
from PseudoNetCDF.sci_var import getvarpnc
from PseudoNetCDF.pncgen import pncgen
exec(options.pre_txt)
for varkey in vars:
figpath = eval(method)(ifile = ifile, varkey = varkey, options = options, before = options.before_txt, after = options.after_txt)
pncgen(getvarpnc(ifile, list(vars) + ['TFLAG', 'time', 'latitude', 'longitude', 'latitude_bounds', 'longitude_bounds']), figpath + '.nc', verbose = False)
exec(options.post_txt)
def testNCF2BPCH(self):
bpchfile=bpch2(self.bpchpath, noscale = True)
from PseudoNetCDF.pncgen import pncgen
pncgen(bpchfile,self.bpchpath + '.check', inmode = 'r', outmode = 'w', format = 'bpch', verbose = False)
orig = open(self.bpchpath, 'rb').read()
new = open(self.bpchpath + '.check', 'rb').read()
assert(orig == new)
os.remove(self.bpchpath+'.check')
def testNCF2BPCH(self):
bpchfile=bpch2(self.bpchpath, noscale = True)
from PseudoNetCDF.pncgen import pncgen
pncgen(bpchfile,self.bpchpath + '.check', inmode = 'r', outmode = 'w', format = 'bpch', verbose = 0)
orig = open(self.bpchpath, 'rb').read()
new = open(self.bpchpath + '.check', 'rb').read()
assert(orig == new)
os.remove(self.bpchpath+'.check')
def testNCF2CSV(self):
from PseudoNetCDF.pncgen import pncgen
from io import BytesIO
import tempfile
out = BytesIO()
out = tempfile.TemporaryFile(mode = 'w+t')
pncgen(self.testfile, out, inmode = 'r', outmode = 'w', format = 'csv', verbose = False)
out.seek(0,0)
outval = out.read()
assert(outval == self.checkval)
def testNCF2CSV(self):
from PseudoNetCDF.pncgen import pncgen
from io import BytesIO
import tempfile
out = BytesIO()
out = tempfile.TemporaryFile(mode = 'w+t')
pncgen(self.testfile, out, inmode = 'r', outmode = 'w', format = 'csv', verbose = 0)
out.seek(0,0)
outval = out.read()
assert(outval == self.checkval)
def plotwithopts(ifile, method, vars, options=defaultoption):
from PseudoNetCDF.sci_var import getvarpnc
from PseudoNetCDF.pncgen import pncgen
exec(options.pre_txt)
for varkey in vars:
figpath = eval(method)(ifile=ifile, varkey=varkey, options=options,
before=options.before_txt,
after=options.after_txt)
pncgen(getvarpnc(ifile, list(vars) + _coordkeys),
figpath + '.nc', verbose=0)
exec(options.post_txt)
def testNCF2HUM(self):
import PseudoNetCDF.testcase
from PseudoNetCDF.pncgen import pncgen
import os
inpath = PseudoNetCDF.testcase.camxfiles_paths['humidity']
outpath = PseudoNetCDF.testcase.camxfiles_paths['humidity'] + '.check'
infile = humidity(inpath, 4, 5)
pncgen(infile, outpath, format='camxfiles.humidity')
orig = open(inpath, 'rb').read()
new = open(outpath, 'rb').read()
assert (orig == new)
os.remove(outpath)
def testNCF2CR(self):
import PseudoNetCDF.testcase
from PseudoNetCDF.pncgen import pncgen
import os
inpath = PseudoNetCDF.testcase.camxfiles_paths['cloud_rain']
outpath = inpath + '.check'
infile = cloud_rain(inpath, 4, 5)
pncgen(infile, outpath, format='camxfiles.cloud_rain')
orig = open(inpath, 'rb').read()
new = open(outpath, 'rb').read()
assert (orig == new)
os.remove(outpath)
def testNCF2KV(self):
import PseudoNetCDF.testcase
from PseudoNetCDF.pncgen import pncgen
import os
inpath = PseudoNetCDF.testcase.camxfiles_paths['vertical_diffusivity']
outpath = inpath + '.check'
infile = vertical_diffusivity(inpath, 4, 5)
pncgen(infile, outpath, format='camxfiles.vertical_diffusivity')
orig = open(inpath, 'rb').read()
new = open(outpath, 'rb').read()
assert(orig == new)
os.remove(outpath)
def testNCF2HP(self):
import PseudoNetCDF.testcase
from PseudoNetCDF.pncgen import pncgen
import os
inpath = PseudoNetCDF.testcase.camxfiles_paths['height_pressure']
outpath = inpath + '.check'
infile = height_pressure(inpath, 4, 5)
pncgen(infile, outpath, format='camxfiles.height_pressure')
orig = open(inpath, 'rb').read()
new = open(outpath, 'rb').read()
assert (orig == new)
os.remove(outpath)
def testNCF2TEMP(self):
import PseudoNetCDF.testcase
from PseudoNetCDF.pncgen import pncgen
import os
inpath = PseudoNetCDF.testcase.camxfiles_paths['temperature']
outpath=PseudoNetCDF.testcase.camxfiles_paths['temperature'] + '.check'
infile=temperature(inpath,4,5)
pncgen(infile, outpath, format = 'camxfiles.temperature')
orig = open(inpath, 'rb').read()
new = open(outpath, 'rb').read()
assert(orig == new)
os.remove(outpath)
def testNCF2KV(self):
import PseudoNetCDF.testcase
from PseudoNetCDF.pncgen import pncgen
import os
inpath = PseudoNetCDF.testcase.camxfiles_paths['vertical_diffusivity']
outpath=PseudoNetCDF.testcase.camxfiles_paths['vertical_diffusivity'] + '.check'
infile=one3d(inpath,4,5)
pncgen(infile, outpath, format = 'camxfiles.vertical_diffusivity')
orig = open(inpath, 'rb').read()
new = open(outpath, 'rb').read()
assert(orig == new)
os.remove(outpath)
def testNCF2UAMIV(self):
from PseudoNetCDF.camxfiles.Memmaps import uamiv
uamivfile=uamiv(self.uamivpath)
from PseudoNetCDF.pncgen import pncgen
pncgen(uamivfile,self.uamivpath + '.check', inmode = 'r', outmode = 'w', format = 'uamiv', verbose = False)
check = True
uamivfile2=uamiv(self.uamivpath + '.check')
for k, v in uamivfile.variables.items():
nv = uamivfile2.variables[k]
check = check & bool((nv[...] == v[...]).all())
assert(check)
os.remove(self.uamivpath+'.check')
def testNCF2HUM(self):
import PseudoNetCDF.testcase
from PseudoNetCDF.pncgen import pncgen
import os
inpath = PseudoNetCDF.testcase.camxfiles_paths['humidity']
outpath=PseudoNetCDF.testcase.camxfiles_paths['humidity'] + '.check'
infile=humidity(inpath, 4, 5)
pncgen(infile, outpath, format = 'camxfiles.humidity')
orig = open(inpath, 'rb').read()
new = open(outpath, 'rb').read()
assert(orig == new)
os.remove(outpath)
def testNCF2LB(self):
import PseudoNetCDF.testcase
from PseudoNetCDF.pncgen import pncgen
import os
inpath = PseudoNetCDF.testcase.camxfiles_paths['lateral_boundary']
outpath = inpath + '.check'
infile = lateral_boundary(inpath)
pncgen(infile, outpath, format='camxfiles.lateral_boundary')
orig = open(inpath, 'rb').read()
new = open(outpath, 'rb').read()
assert (orig == new)
os.remove(outpath)
def testNCF2LB(self):
import PseudoNetCDF.testcase
from PseudoNetCDF.camxfiles.Writers import ncf2lateral_boundary
from PseudoNetCDF.pncgen import pncgen
import os
inpath = PseudoNetCDF.testcase.camxfiles_paths['lateral_boundary']
outpath=PseudoNetCDF.testcase.camxfiles_paths['lateral_boundary'] + '.check'
infile=lateral_boundary(inpath)
pncgen(infile, outpath, format = 'camxfiles.lateral_boundary')
orig = open(inpath, 'rb').read()
new = open(outpath, 'rb').read()
assert(orig == new)
os.remove(outpath)
def testNCF2WD(self):
import PseudoNetCDF.testcase
from PseudoNetCDF.pncgen import pncgen
from PseudoNetCDF.camxfiles.Writers import ncf2wind
import os
inpath = PseudoNetCDF.testcase.camxfiles_paths['wind']
outpath = PseudoNetCDF.testcase.camxfiles_paths['wind'] + '.check'
infile = wind(inpath, 4, 5)
pncgen(infile, outpath, format='camxfiles.wind')
orig = open(inpath, 'rb').read()
new = open(outpath, 'rb').read()
assert (orig == new)
os.remove(outpath)
def testNCF2WD(self):
import PseudoNetCDF.testcase
from PseudoNetCDF.pncgen import pncgen
from PseudoNetCDF.camxfiles.Writers import ncf2wind
import os
inpath = PseudoNetCDF.testcase.camxfiles_paths['wind']
outpath=PseudoNetCDF.testcase.camxfiles_paths['wind'] + '.check'
infile=wind(inpath,4,5)
pncgen(infile, outpath, format = 'camxfiles.wind')
orig = open(inpath, 'rb').read()
new = open(outpath, 'rb').read()
assert(orig == new)
os.remove(outpath)
def testNCF2BPCH(self):
import warnings
with warnings.catch_warnings():
warnings.filterwarnings(
'ignore', 'Not scaling variables; good for direct writing'
)
bpchfile = bpch(self.bpchpath, noscale=True)
outpath = self.bpchpath + '.check'
from PseudoNetCDF.pncgen import pncgen
pncgen(bpchfile, outpath, inmode='r',
outmode='w', format='bpch', verbose=0)
orig = open(self.bpchpath, 'rb').read()
new = open(outpath, 'rb').read()
assert(orig == new)
os.remove(outpath)
from PseudoNetCDF.sci_var import reduce_dim, slice_dim
ALD2 = bpchfile.variables['IJ-AVG-$_ALD2']
ALD2_check = np.array(
[1.60520077e-02, 1.82803553e-02, 2.00258084e-02, 2.01461259e-02,
1.84865110e-02, 2.49667447e-02, 2.73083989e-02, 2.87465211e-02,
2.89694592e-02, 2.87686456e-02, 2.87277419e-02, 3.08121163e-02,
3.22086290e-02, 3.35262120e-02, 3.41329686e-02, 3.05218045e-02,
3.30278911e-02, 3.58164124e-02, 3.93186994e-02, 4.15412188e-02,
1.60520077e-02, 1.82803553e-02, 2.00258084e-02, 2.01461259e-02,
1.84865110e-02, 2.49667447e-02, 2.73083989e-02, 2.87465211e-02,
2.89694592e-02, 2.87686456e-02, 2.87277419e-02, 3.08121163e-02,
3.22086290e-02, 3.35262120e-02, 3.41329686e-02, 3.05218045e-02,
3.30278911e-02, 3.58164124e-02, 3.93186994e-02, 4.15412188e-02,
1.60520077e-02, 1.82803553e-02, 2.00258084e-02, 2.01461259e-02,
1.84865110e-02, 2.49667447e-02, 2.73083989e-02, 2.87465211e-02,
2.89694592e-02, 2.87686456e-02, 2.87277419e-02, 3.08121163e-02,
3.22086290e-02, 3.35262120e-02, 3.41329686e-02, 3.05218045e-02,
3.30278911e-02, 3.58164124e-02, 3.93186994e-02, 4.15412188e-02])\
.reshape(ALD2.shape)
slided_reduced_bpchfile = slice_dim(
reduce_dim(bpchfile, 'layer,mean'), 'time,0')
pncgen(slided_reduced_bpchfile, outpath, inmode='r',
outmode='w', format='bpch', verbose=0)
ALD2_check_slided_reduced = ALD2_check[0].mean(0)[None, None]
ALD2 = slided_reduced_bpchfile.variables['IJ-AVG-$_ALD2']
np.testing.assert_allclose(ALD2, ALD2_check_slided_reduced * 1e-9)
with warnings.catch_warnings():
warnings.filterwarnings(
'ignore', 'Not scaling variables; good for direct writing'
)
bpchfile = bpch(outpath)
ALD2 = bpchfile.variables['IJ-AVG-$_ALD2']
np.testing.assert_allclose(ALD2, ALD2_check_slided_reduced)
def testNCF2PT(self):
import PseudoNetCDF.testcase
from PseudoNetCDF.pncgen import pncgen
import os
inpath = PseudoNetCDF.testcase.camxfiles_paths['point_source']
outpath = inpath + '.check'
infile = point_source(inpath)
pncgen(infile, outpath, format='camxfiles.point_source')
orig = open(inpath, 'rb').read()
new = open(outpath, 'rb').read()
# outfile=point_source(outpath)
# for k, v0 in infile.variables.items():
# v1 = outfile.variables[k]
# print(k, (v0[:] == v1[:]).all())
assert(orig == new)
os.remove(outpath)
def testNCF2PT(self):
import PseudoNetCDF.testcase
from PseudoNetCDF.pncgen import pncgen
import os
inpath = PseudoNetCDF.testcase.camxfiles_paths['point_source']
outpath=PseudoNetCDF.testcase.camxfiles_paths['point_source'] + '.check'
infile=point_source(inpath)
pncgen(infile, outpath, format = 'camxfiles.point_source')
orig = open(inpath, 'rb').read()
new = open(outpath, 'rb').read()
#outfile=point_source(outpath)
#for k, v0 in infile.variables.items():
# v1 = outfile.variables[k]
# print(k, (v0[:] == v1[:]).all())
assert(orig == new)
os.remove(outpath)
def testNCF2UAMIV(self):
from PseudoNetCDF.camxfiles.Memmaps import uamiv
uamivfile = uamiv(self.uamivpath)
from PseudoNetCDF.pncgen import pncgen
pncgen(uamivfile,
self.uamivpath + '.check',
inmode='r',
outmode='w',
format='uamiv',
verbose=False)
check = True
uamivfile2 = uamiv(self.uamivpath + '.check')
for k, v in uamivfile.variables.items():
nv = uamivfile2.variables[k]
check = check & bool((nv[...] == v[...]).all())
assert (check)
os.remove(self.uamivpath + '.check')
def plotwithopts(ifile, method, vars, options=defaultoption):
from PseudoNetCDF.sci_var import getvarpnc
from PseudoNetCDF.pncgen import pncgen
exec(options.pre_txt)
for varkey in vars:
figpath = eval(method)(ifile=ifile,
varkey=varkey,
options=options,
before=options.before_txt,
after=options.after_txt)
pncgen(getvarpnc(
ifile,
list(vars) + [
'TFLAG', 'time', 'latitude', 'longitude', 'latitude_bounds',
'longitude_bounds'
]),
figpath + '.nc',
verbose=False)
exec(options.post_txt)
def testNCF2BPCH(self):
bpchfile=bpch(self.bpchpath, noscale = True)
outpath = self.bpchpath + '.check'
from PseudoNetCDF.pncgen import pncgen
pncgen(bpchfile,outpath, inmode = 'r', outmode = 'w', format = 'bpch', verbose = False)
orig = open(self.bpchpath, 'rb').read()
new = open(outpath, 'rb').read()
assert(orig == new)
os.remove(outpath)
from PseudoNetCDF.sci_var import reduce_dim, slice_dim
ALD2 = bpchfile.variables['IJ-AVG-$_ALD2']
ALD2_check = np.array([1.60520077e-02, 1.82803553e-02, 2.00258084e-02, 2.01461259e-02, 1.84865110e-02, 2.49667447e-02, 2.73083989e-02, 2.87465211e-02, 2.89694592e-02, 2.87686456e-02, 2.87277419e-02, 3.08121163e-02, 3.22086290e-02, 3.35262120e-02, 3.41329686e-02, 3.05218045e-02, 3.30278911e-02, 3.58164124e-02, 3.93186994e-02, 4.15412188e-02, 1.60520077e-02, 1.82803553e-02, 2.00258084e-02, 2.01461259e-02, 1.84865110e-02, 2.49667447e-02, 2.73083989e-02, 2.87465211e-02, 2.89694592e-02, 2.87686456e-02, 2.87277419e-02, 3.08121163e-02, 3.22086290e-02, 3.35262120e-02, 3.41329686e-02, 3.05218045e-02, 3.30278911e-02, 3.58164124e-02, 3.93186994e-02, 4.15412188e-02, 1.60520077e-02, 1.82803553e-02, 2.00258084e-02, 2.01461259e-02, 1.84865110e-02, 2.49667447e-02, 2.73083989e-02, 2.87465211e-02, 2.89694592e-02, 2.87686456e-02, 2.87277419e-02, 3.08121163e-02, 3.22086290e-02, 3.35262120e-02, 3.41329686e-02, 3.05218045e-02, 3.30278911e-02, 3.58164124e-02, 3.93186994e-02, 4.15412188e-02]).reshape(ALD2.shape)
bpchfile = slice_dim(reduce_dim(bpchfile, 'layer,mean'), 'time,0')
pncgen(bpchfile,outpath, inmode = 'r', outmode = 'w', format = 'bpch', verbose = False)
ALD2_check = ALD2_check[0].mean(0)[None, None]
ALD2 = bpchfile.variables['IJ-AVG-$_ALD2']
np.testing.assert_allclose(ALD2, ALD2_check * 1e-9)
bpchfile = bpch(outpath)
ALD2 = bpchfile.variables['IJ-AVG-$_ALD2']
np.testing.assert_allclose(ALD2, ALD2_check)
def plotwithopts(ifile, method, vars, options=defaultoption):
from PseudoNetCDF.sci_var import getvarpnc
from PseudoNetCDF.pncgen import pncgen
import matplotlib.pyplot as plt
# dummy assignment so that flake8 sees plt as used
# plt is loaded into the environment for exec
varkey = plt
exec(options.pre_txt)
for varkey in vars:
figpath = eval(method)(ifile=ifile,
varkey=varkey,
options=options,
before=options.before_txt,
after=options.after_txt)
pncgen(getvarpnc(ifile,
list(vars) + list(_coordkeys)),
figpath + '.nc',
verbose=0)
exec(options.post_txt)
def pncwrite(*args, **kwds):
"""See PseudoNetCDF.pncgen.pncgen
*args : iterable
**kwds : keywords
keywords for pncgen
help : boolean
without format, returns help of pncopen and with format keyword,
returns help of the function that writes that format.
"""
from PseudoNetCDF.pncgen import pncgen
formathelp = kwds.pop('help', False)
if formathelp:
if 'format' in kwds:
format = kwds['format']
writer = getwriterdict()[format]
return help(writer)
else:
return help(pncgen)
return pncgen(*args, **kwds)
aggfunc=np.sum)
oldpolname = None
outfile = getvarpnc(Dataset(templatepath, 'r+'), ['TFLAG'] + unique_vars)
add_ioapi_from_ioapi(outfile)
# outfile.variables['TFLAG'][:, :, 0] = np.arange(0, 25)[:, None]
# outfile.variables['TFLAG'][:, :, 0] = 2014001
print('Working on: ' + csvpath)
for (polname, rowi, coli), groupdata in aggdata.iterrows():
if args.verbose > 2: print(polname)
if polname != oldpolname:
if not oldpolname is None:
if args.verbose > 0:
print('Writing out ' + polname + ' to ' + var.long_name)
var[0:26, 0, :, :] = temp[0:26, 0, :, :]
if args.verbose > 0: print('Starting ' + polname)
if polname == 'BEN':
var = outfile.variables['BENZENE']
elif polname != 'CO2':
var = outfile.variables[polname]
else:
warn('WARNING: skipping %s in %s' % (polname, csvpath))
oldpolname = None
continue
temp = np.zeros_like(var[:])
oldpolname = polname
temp[0:26, 0, rowi, coli] += np.array(
[groupdata[hours[k]].sum() for k in range(0, 26)])
# catch the last pollutant
var[0:26, 0, :, :] = temp[0:26, 0, :, :]
pncgen(outfile, outpath, verbose=args.verbose)
zipout = open(filepath, 'wb')
zipout.write(data)
zipout.flush()
zipout.close()
import zipfile
zf = zipfile.ZipFile(filepath)
zf.extract(yearpath)
yearpaths.append(yearpath)
from PseudoNetCDF.epafiles import aqsraw
from PseudoNetCDF.sci_var import stack_files
from PseudoNetCDF.pncgen import pncgen
infiles = []
for yearpath in yearpaths:
infile = aqsraw(yearpath,
timeresolution=args.timeresolution,
param=args.param,
bdate=args.bdate,
edate=args.edate,
rdate=args.rdate,
wktpolygon=args.wktpolygon,
sampleval=args.sampleval,
verbose=args.verbose)
infiles.append(infile)
outfile = stack_files(infiles, 'time')
persisted = pncgen(outfile, args.outpath)
persisted.sync()
print('Successful')
def pncwrite(*args, **kwds):
"""See PseudoNetCDF.pncgen.pncgen"""
from PseudoNetCDF.pncgen import pncgen
return pncgen(*args, **kwds)
def wrt_uamiv(fout_path, fin, *, lsurf = False, lapp = False, lemis = False, lcmaq = False, ounits = None):
"""
Create an UAMIV file using fin class.
Arguments:
fout_path - The IOAPI file name including path
fin - A class which include all the contents, data arrays, attributes, variable names.
The data array shape is (nspc,nsteps,nz,ny,nx).
lsurf - If True, output file has only 1 layer regardless of shape of data array.
lapp - If True, output file must already exist. Skip writing file dimensions and attributes.
It simply appends data arrays into the existing output file.
lemis - If True, it is emission file, and variable units have rates instead of concentration
lcmaq - If True, the file is for CMAQ, and varnames are checked against CMAQ PM species
so that PM unit is assigned, microgram/m**3 instead of moles/m**3.
ounits - A list which has units of variables
"""
# Handling arguments
lounit = True
if ounits == None:
lounit = False
ounits = []
# Include modules
import datetime
from PseudoNetCDF import PseudoNetCDFFile
from PseudoNetCDF.pncgen import pncgen
from PseudoNetCDF import PNC
from PseudoNetCDF.sci_var import stack_files
from CAMxtools.write.wrt_ioapi import find_unit
from pathlib import Path
import os
#prepare file attributes
novars = getattr(fin,'NVARS')
varnames = getattr(fin,'VAR-LIST').split()
var0 = varnames[0]
nsteps = fin.variables[var0].shape[0]
nz = getattr(fin,'NLAYS')
ny = getattr(fin,'NROWS')
nx = getattr(fin,'NCOLS')
lgrdded = False
if getattr(fin,'FTYPE') == 1: lgrdded = True
if lgrdded: # GRIDDED
assert len(fin.variables[var0].shape) == 4, "len(fin.variables[var0].shape) MUST be 4"
else: # BOUNDARY CONDITION
assert len(fin.variables[var0].shape) == 3, "len(fin.variables[var0].shape) MUST be 3"
ncells = fin.variables[var0].shape[2]
assert ncells == 2*(nx+ny)+4, "ncells MUST be 2*(nx+ny)+4"
#open output file
if not lgrdded: #BOUNDARY
ftype = 'lateral_boundary'
else:
ftype = 'uamiv'
newf = PseudoNetCDFFile()
# Set tstep
tstep = getattr(fin,'TSTEP')
#copy dimensions
dimensions_keys = "TSTEP DATE-TIME LAY VAR".split() # These dimension keys must be in the file to be used by m3tools
if lgrdded: # GRIDDED
dimensions_keys.append("ROW"); dimensions_keys.append("COL")
else:
dimensions_keys.append("PERIM")
for i in dimensions_keys:
if i == 'VAR': size = novars
elif i == 'TSTEP': size = nsteps
elif i == 'DATE-TIME': size = 2
elif i == 'LAY':
if lsurf:
size = 1
else:
size = nz
elif i == 'VAR': size = novars
elif i == 'ROW': size = ny
elif i == 'COL': size = nx
else : size = ncells # i == 'PERIM'
newf.createDimension(i,size)
#copy global attributes
attribs = "XORIG YORIG XCELL YCELL PLON PLAT TLAT1 TLAT2 IUTM ISTAG CPROJ GDTYP XCENT YCENT P_ALP P_BET P_GAM NLAYS NROWS NCOLS NVARS VAR-LIST NAME NOTE ITZON FTYPE VGTYP VGTOP VGLVLS GDNAM UPNAM FILEDESC SDATE STIME TSTEP Conventions history".split() # These attributes must be in the file for pncgen to uamiv
cdate = int(datetime.date.today().strftime("%Y%j"))
ctime = int(datetime.datetime.now().strftime("%H%M%S"))
for i in attribs:
try: val = getattr(fin,i)
except: val = ""
if 'numpy.float32' in str(type(val)):
val = val.item()
if i == 'PLON': val = getattr(fin,'XCENT')
if i == 'PLAT': val = getattr(fin,'YCENT')
if i == 'TLAT1': val = getattr(fin,'P_ALP')
if i == 'TLAT2': val = getattr(fin,'P_GAM')
if i == 'IUTM':
val = 0
if getattr(fin,'GDTYP') == 5: val = 1
if i == 'ISTAG': val = 0
if i == 'CPROJ':
if getattr(fin,'GDTYP') == 1: #LATLON
val = 0
elif getattr(fin,'GDTYP') == 5: #UTM
val = 1
elif getattr(fin,'GDTYP') == 2: #LCP
val = 2
elif getattr(fin,'GDTYP') == 6: #PSP
val = 4
elif getattr(fin,'GDTYP') == 7: #Equatorial Mercator
val = 5
else:
print("GDTYP = {}".format(GDTYP))
exit("Not relevant projection")
if i == 'NLAYS':
if lsurf:
val = 1
else:
size = nz
if i == 'NROWS': val = ny
if i == 'NCOLS': val = nx
if i == 'NVARS': val = novars
if i == 'NSTEPS': val = nsteps
if i == 'NAME':
name_str = 'AVERAGE'
if lemis: name_str = 'EMISSIONS'
if not lgrdded: name_str = 'BOUNDARY'
val = '{:<10s}'.format(name_str)
if i == 'NOTE': val = '{:<60s}'.format("wrt_uamiv in CAMxtools")
if i == 'ITZON': val = 0
if i == 'VGTYP':
if lsurf or nz == 1:
val = -9999
else:
size = 2 # VGSGPN3 non-h sigma-p
if i == 'VGTOP':
if lsurf or nz == 1:
val = -9.999E36
else:
size = getattr(fin,'VGTOP')
if i == 'GDNAM' or i == 'UPNAM' or i == 'FILEDESC':
val = '{:<16s}'.format("CAMx")
if i == 'Conventions': val = "CF-1.6"
if i == 'history': val = '{:<250s}'.format("unspecified")
setattr(newf,i,val)
#copy variables
extra = "TFLAG".split()
for i in extra+varnames:
if i == 'TFLAG':
newf.createVariable(i,('int32'),(u'TSTEP', u'VAR', u'DATE-TIME'))
newf.variables[i].setncattr("units","<YYYYDDD,HHMMSS>")
newf.variables[i].setncattr("long_name","TFLAG")
newf.variables[i].setncattr("var_desc",'{:<80s}'.format("Timestep-valid flags: (1) YYYYDDD or (2) HHMMSS"))
idate = getattr(fin,'SDATE')
itime = getattr(fin,'STIME')
for istep in range(nsteps):
newf.variables['TFLAG'][istep,:,0] = [idate for ivar in range(len(varnames))]
newf.variables['TFLAG'][istep,:,1] = [itime for ivar in range(len(varnames))]
itime += tstep
if itime == 240000:
itime = 0
idate = int((datetime.datetime.strptime(str(idate),"%Y%j") + datetime.timedelta(days=1)).strftime("%Y%j"))
else:
if lgrdded: # GRIDDED
var_type = str(type(fin.variables[var0][0,0,0,0]))
else:
var_type = str(type(fin.variables[var0][0,0,0]))
if 'numpy.float' in var_type:
dtype = 'float32'
else:
dtype = 'int32'
if lgrdded: # GRIDDED
newf.createVariable(i, (dtype), ('TSTEP', 'LAY', 'ROW', 'COL'))
else:
newf.createVariable(i, (dtype), ('TSTEP', 'LAY', 'PERIM'))
#Find a relevant unit
if lounit:
s = varnames.index(i)
unit = ounits[s]
else:
unit = find_unit(i,lemis=lemis,lcmaq=lcmaq)
newf.variables[i].setncattr("long_name",'{:<16s}'.format(i))
newf.variables[i].setncattr("units",'{:<16s}'.format(unit))
newf.variables[i].setncattr("var_desc",'{:<80s}'.format("".join(["VARIABLE ",i])))
newf.variables[i]=fin.variables[i]
# Create UAMIV file
if lapp: # Save the original fout_path to fout_old_path
fout_old_path = fout_path + ".old"
try:
os.rename(fout_path,fout_old_path) # Move the original output to fout_old_path
#pncargs = '--format=' + ftype + ',mode="r+"'
#pncargs = '--format=' + ftype
except:
print("Output file is {}".format(fout_path))
exit("Output file does not exist while lapp is True")
pncargs = '--format=' + ftype
oldfile = PNC(pncargs, fout_old_path).ifiles[0]
fout_tmp_path = fout_path + ".tmp"
pncgen(newf, fout_tmp_path, format = ftype)
tmpfile = PNC(pncargs, fout_tmp_path).ifiles[0]
newf_app = stack_files([oldfile, tmpfile], 'TSTEP')
pncgen(newf_app, fout_path, format = ftype)
if Path(fout_old_path).exists(): os.remove(fout_old_path)
if Path(fout_tmp_path).exists(): os.remove(fout_tmp_path)
else:
pncgen(newf, fout_path, format = ftype)
# close files
if 'newfile' in locals(): del newfile
del fin
if 'newf' in locals(): newf.close()
print ('*** SUCCESS writing UAMIV file')
return
def output(out, outpath, config, verbose = 0):
"""
Make final unit conversions, add log information
and save output to disk
out - PseudoNetCDFFile with output data
outpath - Path for output
config - configuration dictionary
"""
from repair_ae import repair_ae
from PseudoNetCDF.pncgen import pncgen
tmp = defaultdict(lambda: 1)
tmp['np'] = np
for k, v in config['unitconversions']['metadefs'].iteritems():
eval(v, None, tmp)
for k in tmp.keys():
if k != 'np':
tmp[k] = out.variables[k]
for k, v in config['unitconversions']['metadefs'].iteritems():
exec('%s = %s' % (k, v), tmp, out.variables)
if verbose > 0: status('Vertical profile Low -> High')
np.set_printoptions(precision = 2)
for vark, varo in out.variables.items():
if hasattr(varo, 'unitnow'):
if varo.unitnow.strip() != varo.units.strip():
expr = config['unitconversions']['%s->%s' % (varo.unitnow.strip(), varo.units.strip())].replace('<value>', 'varo')
exec('varo[:] = %s' % expr, dict(varo = varo), out.variables)
varo.history += ';' + expr.replace('varo', 'RESULT')
del varo.unitnow
if vark not in ('time', 'TFLAG') and 'latitude' not in vark and 'longitude' not in vark and (varo[...] <= 0).any():
if config['zero_negs']:
have_zeros = varo[...] <= 0
varo[np.where(have_zeros)] = 1.e-30
warn(vark + ' %d (of %d; %.2f%%) negative or zero values were set to 1e-30' % (have_zeros.sum(), have_zeros.size, have_zeros.mean() * 100), stacklevel = 1)
else:
warn(vark + ' has negative or zero values', stacklevel = 1)
if verbose > 0 and 'TSTEP' in varo.dimensions and 'PERIM' in varo.dimensions:
status('%s: %s' % (vark, str(varo[:, :].mean(0).mean(1))))
np.set_printoptions(precision = None)
out.geos2cmaq_warnings = myioo.getwarnings()
out.geos2cmaq_errors = myioo.geterrors()
out.geos2cmaq_status = myioo.getstatus()
varnames = [k.ljust(16) for k in out.variables.keys() if k[:1] in uppercase and k != 'TFLAG']
out.createDimension('VAR', len(varnames))
tflag = out.variables['TFLAG']
var = out.createVariable('TFLAG', 'i', ('TSTEP', 'VAR', 'DATE-TIME'))
var.long_name = 'TFLAG'.ljust(16);
var.var_desc = "Timestep-valid flags: (1) YYYYDDD or (2) HHMMSS".ljust(80)
var.units = "<YYYYDDD,HHMMSS>"
var[:, :, :] = tflag[:, [0], :].repeat(var.shape[1], 1)
f = pncgen(out, outpath, inmode = 'r', outmode = 'w', format = 'NETCDF3_CLASSIC', verbose = False)
setattr(f, 'VAR-LIST', ''.join(varnames))
setattr(f, 'NVARS', len(varnames))
f.sync()
f.close()
if config['repair_aero']:
f = Dataset(outpath, 'r+')
repair_ae(f, myioo)
f.sync()
f.close()
p2p.addGlobalProperties(obstimefile, nomtimefile)
for vark in varkeys:
p2p.addVariable(obstimefile, nomtimefile, vark, data=False)
outvar = nomtimefile.variables[vark]
var = obstimefile.variables[vark]
timem = (var[:] / var[:]) * time[:][:, None]
dt = np.ma.MaskedArray(unomtimes[:, None, None]) - timem[None, :, :]
nomtime_idx, points_idx = np.indices(dt[:, 0].shape)
time_idx = np.abs(dt).argmin(1)
outvals = var[:][time_idx, points_idx]
min_dt = dt[nomtime_idx, time_idx, points_idx]
dt_mask = (min_dt < dt_min) | (min_dt > dt_max)
outvals = np.ma.masked_where(dt_mask, outvals)
outvar[:] = outvals[:]
pncgen(nomtimefile, args.hourlypath, verbose=0)
# :DD_long_name = "QC data descriptor model: QC summary values" ;
# :DD_reference = "AWIPS Technique Specification Package (TSP) 88-21-R2" ;
# :DD_values = "Z,C,S,V,X,Q,K,k,G, or B" ;
# :DD_value_Z = "No QC applied" ;
# :DD_value_C = "Passed QC stage 1" ;
# :DD_value_S = "Passed QC stages 1 and 2" ;
# :DD_value_V = "Passed QC stages 1, 2 and 3" ;
# :DD_value_X = "Failed QC stage 1" ;
# :DD_value_Q = "Passed QC stage 1, but failed stages 2 or 3 " ;
# :DD_value_K = "Passed QC stages 1, 2, 3, and 4" ;
# :DD_value_k = "Passed QC stage 1,2, and 3, failed stage 4 " ;
# :DD_value_G = "Included in accept list" ;
# :DD_value_B = "Included in reject list" ;
