def __str__(self):
return """{0}
Creation Status: {1}
Source Seed: {2}\tSource NN: {3}\tSource NB: {4}\t Source Lp: {5}
Box Matrix: {6}
Box String: {7}
Radis: {8}
Number of Nodes: {9}\tNumber of Surface Nodes: {10}\tNumber of Bonds {11}
Source Z: {12}\tSource Delta Z: {13}
Z: {14}\tDelta Z: {15}
""".format(
self.data,
chartostring(self.get("creation_status", 0)),
self.get("seed_src", 0),
self.get("NN_src", 0),
self.get("NB_src", 0),
self.get("Lp_src", 0),
self.get("box_mat", 0),
chartostring(self.get("box_str", 0)),
self.get("radius", 0),
self.get("NN", 0),
self.get("NSN", 0),
self.get("NB", 0),
self.get("Z_src", 0),
self.get("DeltaZ_src", 0),
self.get("Z", 0),
self.get("DeltaZ", 0),
)
def drifter_meta():
imei = {}
wmo = {}
deployment = {}
with Dataset(app.config['DRIFTER_AGG_URL'], 'r') as ds:
for idx, b in enumerate(ds.variables['buoy'][:]):
bid = str(chartostring(b))[:-3]
for data, key in [
[imei, 'imei'],
[wmo, 'wmo'],
[deployment, 'deployment']
]:
d = str(chartostring(ds.variables[key][idx][0]))
if data.get(d) is None:
data[d] = [bid]
else:
data[d].append(bid)
return {
'imei': imei,
'wmo': wmo,
'deployment': deployment,
}
def observation_meta():
with Dataset(app.config["OBSERVATION_AGG_URL"], 'r') as ds:
ship = chartostring(ds['ship'][:])
trip = chartostring(ds['trip'][:])
data = {
'ship': sorted(np.unique(ship).tolist()),
'trip': sorted(np.unique(trip).tolist()),
}
return data
def main():
global ncfile, prefix
parser = argparse.ArgumentParser(description="A commandline tool to convert WRF timeseries files to netCDF4")
parser.add_argument('netcdf', metavar="netCDF4 file", type=str, nargs=1, help="NetCDF file base name. Create it first with ts_make_ncs.sh")
parser.add_argument('-d', '--domain', metavar="domain", type=int, nargs=1, help="WRF domain number", default=0)
args = parser.parse_args()
for varname in ['TS']: # ,'UU','VV','TH','QV','PH']:
logging.info( "{}".format(varname) )
ncfile = cdf.Dataset( args.netcdf[0] + "." + varname + ".nc", "r+" )
do_tslist()
prefix = ncfile.variables['prefix']
for stationi in range(nstations):
filename = "{}.d{:02d}.".format( cdf.chartostring( prefix[stationi] ), args.domain[0] )
if not os.path.isfile( filename + "TS" ):
continue
if varname in ['TS',]:
if not ntimes:
simplecount(filename + "TS", args.domain[0])
do_tsfile ( filename + "TS", stationi )
else:
do_profile( filename + varname, stationi, varname )
if varname in ['TS',]:
flush_tsfile()
else:
ncfile.variables[varname][:,:,:] = profile [:,:,:]
ncfile.close()
def class4(class4_id, projection, resolution, extent):
dataset_url = app.config["CLASS4_URL"] % class4_id
proj = pyproj.Proj(init=projection)
view = _get_view(extent)
rmse = []
lat = []
lon = []
point_id = []
identifiers = []
with Dataset(dataset_url, 'r') as ds:
lat_in = ds['latitude'][:]
lon_in = ds['longitude'][:]
ids = map(str.strip, chartostring(ds['id'][:]))
for i in range(0, lat_in.shape[0]):
x, y = proj(lon_in[i], lat_in[i])
p = Point(y, x)
if view.envelope.intersects(p):
lat.append(float(lat_in[i]))
lon.append(float(lon_in[i]))
identifiers.append(ids[i])
best = ds['best_estimate'][i, 0, :]
obsv = ds['observation'][i, 0, :]
point_id.append(i)
rmse.append(np.ma.sqrt(((best - obsv) ** 2).mean()))
rmse = np.ma.hstack(rmse)
rmse_norm = np.clip(rmse / 1.5, 0, 1)
loc = zip(lon, lat)
points = []
for idx, ll in enumerate(loc):
if np.ma.is_masked(rmse[idx]):
continue
points.append({
'type': "Feature",
'geometry': {
'type': "Point",
'coordinates': ll,
},
'properties': {
'name': "%s" % identifiers[idx],
'id': "%s/%d" % (class4_id, point_id[idx]),
'error': float(rmse[idx]),
'error_norm': float(rmse_norm[idx]),
'type': 'class4',
'resolution': 0,
},
})
result = {
'type': "FeatureCollection",
'features': points,
}
return result
def list_class4(d):
dataset_url = app.config["CLASS4_URL"] % d
with Dataset(dataset_url, 'r') as ds:
lat = ds['latitude'][:]
lon = ds['longitude'][:]
ids = map(str.strip, chartostring(ds['id'][:]))
rmse = []
for i in range(0, lat.shape[0]):
best = ds['best_estimate'][i, 0, :]
obsv = ds['observation'][i, 0, :]
rmse.append(np.ma.sqrt(((best - obsv) ** 2).mean()))
rmse = np.ma.hstack(rmse)
maxval = rmse.mean() + 2 * rmse.std()
rmse_norm = rmse / maxval
loc = zip(lat, lon)
points = []
for idx, ll in enumerate(loc):
if np.ma.is_masked(rmse[idx]):
continue
points.append({
'name': "%s" % ids[idx],
'loc': "%f,%f" % ll,
'id': "%s/%d" % (d, idx),
'rmse': float(rmse[idx]),
'rmse_norm': float(rmse_norm[idx]),
})
points = sorted(points, key=lambda k: k['id'])
return points
def runTest(self):
"""testing functions for converting arrays of chars to fixed-len strings"""
nc = Dataset(FILE_NAME)
assert nc.dimensions['n1'].isunlimited() == True
v = nc.variables['strings']
v2 = nc.variables['strings2']
v3 = nc.variables['strings3']
assert v.dtype.str[1:] in ['S1','U1']
assert v.shape == (nrecs,n2,nchar)
for nrec in range(nrecs):
data2 = chartostring(v[nrec],encoding='ascii')
assert_array_equal(data2,datau[nrec])
data2 = v2[:]
data2[0] = v2[0]
data2[0,1] = v2[0,1]
assert_array_equal(data2,datau)
data3 = v3[:]
assert_array_equal(data3,datau)
# these slices should return a char array, not a string array
data4 = v2[:,:,0]
assert(data4.dtype.itemsize == 1)
assert_array_equal(data4, datac[:,:,0])
data5 = v2[0,0:nchar,0]
assert(data5.dtype.itemsize == 1)
assert_array_equal(data5, datac[0,0:nchar,0])
# test turning auto-conversion off.
v2.set_auto_chartostring(False)
data6 = v2[:]
assert(data6.dtype.itemsize == 1)
assert_array_equal(data6, datac)
nc.set_auto_chartostring(False)
data7 = v3[:]
assert(data7.dtype.itemsize == 1)
assert_array_equal(data7, datac)
nc.close()
def maketransectdf(transect, dt_from=None, dt_to=None):
"""Read some transect data"""
transectidx, filter, vardict, ds = prepare_vardict(transect, 'transect', False, dt_from, dt_to)
if transectidx is None:
alongshore = np.zeros(0)
areaname = np.zeros(0)
mean_high_water = np.zeros(0)
mean_low_water = np.zeros(0)
else:
alongshore = ds.variables['alongshore'][transectidx]
areaname = str(netCDF4.chartostring(ds.variables['areaname'][transectidx])).strip()
mean_high_water = ds.variables['mean_high_water'][transectidx]
mean_low_water = ds.variables['mean_low_water'][transectidx]
ds.close()
transectdf = pandas.DataFrame(
index=[transect],
data=dict(
transect=transect,
areaname=areaname,
mean_high_water=mean_high_water,
mean_low_water=mean_low_water
)
)
return transectdf
def get_data_from_file(path, field_name = 'water_discharge'):
"""
returns streamflow, times, x_indices, y_indices
"""
date_format = '%Y_%m_%d_%H_%M'
print path
ds = nc.Dataset(path)
#dims: time, cell_index
discharge = ds.variables[field_name][:]
times = ds.variables['time'][:]
times = chartostring(times)
if ds.variables.has_key('x_index'):
x_indices = ds.variables['x_index'][:]
y_indices = ds.variables['y_index'][:]
else:
x_indices = ds.variables['x-index'][:]
y_indices = ds.variables['y-index'][:]
date_times = map(lambda x: datetime.strptime( x , date_format ), times)
return discharge, date_times, x_indices, y_indices
def do_tsfile(filename, stationi):
if not os.path.isfile( filename):
logging.info( "Skipping station %s : TS", cdf.chartostring( prefix[stationi] ) )
return
logging.debug( "{} starting".format( filename ) )
logging.debug( "parsing ts file" )
# Parse TS file, see the README.tslist in the WRF run directory for details
fileTS = open( filename, 'r' )
# Header
# NZCM McMurdo 2 7 mcm (-77.850, 166.710) ( 153, 207) (-77.768, 166.500) 81.8 meters
#
# Those are name of the station, grid ID, time-series ID, station lat/lon, grid indices (nearest grid point to
# the station location), grid lat/lon, elevation.
line = fileTS.next()
fields = re.search(r"\( *(\d+) *, *(\d+) *\) *\( *(-?\d+\.\d+) *, *(-?\d+\.\d+) *\) * (-?\d+(\.\d+)?) *meters", line).groups()
ncfile.variables['gj' ][stationi] = fields[0]
ncfile.variables['gi' ][stationi] = fields[1]
ncfile.variables['glat'][stationi] = fields[2]
ncfile.variables['glon'][stationi] = fields[3]
ncfile.variables['elevation'][stationi] = fields[4]
# Body
# 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
# id, ts_hour, id_tsloc, ix, iy, t, q, u, v, psfc, glw, gsw, hfx, lh, tsk, tslb(1), rainc, rainnc, clw
timei = -1
for line in fileTS:
timei += 1
fields = line.split()
time [ timei ] = fields[ 1]
T2m [ timei,stationi ] = fields[ 5]
Q2m [ timei,stationi ] = fields[ 6]
U10m [ timei,stationi ] = fields[ 7]
V10m [ timei,stationi ] = fields[ 8]
psfc [ timei,stationi ] = fields[ 9]
glw [ timei,stationi ] = fields[10]
gsw [ timei,stationi ] = fields[11]
hfx [ timei,stationi ] = fields[12]
lh [ timei,stationi ] = fields[13]
tsk [ timei,stationi ] = fields[14]
tslb1 [ timei,stationi ] = fields[15]
rainc [ timei,stationi ] = fields[16]
rainnc[ timei,stationi ] = fields[17]
clw [ timei,stationi ] = fields[18]
tc2m [ timei,stationi ] = fields[19]
tp2m [ timei,stationi ] = fields[20]
fileTS.close()
logging.info( "{} done".format( filename ) )
def xtimeGetYear(xtime):
"""Get an array of years from an xtime array, ignoring any partial year information"""
# First parse the xtime character array into a string
xtimestr = netCDF4.chartostring(xtime) # convert from the character array to an array of strings using the netCDF4 module's function
years = np.zeros( (len(xtimestr),) )
for i in range(len(xtimestr)):
years[i] = ( int(xtimestr[i].split('-')[0]) ) # Get the year part and make it an integer
return years
def test_sweep_mode():
assert 'standard_name' in radar.sweep_mode
assert radar.sweep_mode['data'].shape == (1, 32)
assert radar.sweep_mode['data'].dtype.char == 'S'
str_array = netCDF4.chartostring(radar.sweep_mode['data'])
try:
assert np.all(str_array == ['azimuth_surveillance'])
except AssertionError:
assert np.all(str_array == [b'azimuth_surveillance'])
def read_json(open_file, var_name):
qp = nc.chartostring((open_file.variables[var_name][:]))
try:
parsed_json = json.loads(qp[0])
return parsed_json
except:
parsed_json = 'n/a'
return parsed_json
def check_valid_time_format(section, dataset, text, var_name):
""" Check that a variable is a valid UTC time formatted string. """
if var_name in dataset.variables:
s = str(netCDF4.chartostring(dataset.variables[var_name][:]))
try:
time.strptime(s, '%Y-%m-%dT%H:%M:%SZ')
except:
tup = (text, var_name, s, 'yyyy-mm-ddThh:mm:ssZ')
t = "%s '%s' has an invalid format: %s should be %s" % (tup)
log_error(section, t)
def insert_station(sesh, cf, i, name, x, y):
station = Station(
name=str(chartostring(name[i])),
x=x[i],
x_units=x.units,
y=y[i],
y_units=y.units,
)
sesh.add(station)
return station
def runTest(self):
"""testing functions for converting arrays of chars to fixed-len strings"""
nc = Dataset(FILE_NAME)
assert nc.dimensions['n1'].isunlimited() == True
v = nc.variables['strings']
assert v.dtype.str[1:] == 'S1'
assert v.shape == (nrecs,n2,nchar)
for nrec in range(nrecs):
data2 = chartostring(v[nrec])
assert_array_equal(data2,data[nrec])
nc.close()
def check_dataset_metadata_in_storage_unit(nco, dataset_dir):
assert len(nco.variables['dataset']) == 1 # 1 time slice
stored_metadata = nco.variables['dataset'][0]
if not isinstance(stored_metadata, str):
stored_metadata = netCDF4.chartostring(stored_metadata)
stored_metadata = str(np.char.decode(stored_metadata))
ds_filename = dataset_dir / 'agdc-metadata.yaml'
stored = yaml.safe_load(stored_metadata)
[(_, original)] = read_documents(ds_filename)
assert len(stored['lineage']['source_datasets']) == 1
assert next(iter(stored['lineage']['source_datasets'].values())) == original
def find_station(sesh, cf, i, name, x, y):
return (
sesh.query(Station)
.filter_by(
name=str(chartostring(name[i])),
x=x[i],
x_units=x.units,
y=y[i],
y_units=y.units,
)
.first()
)
def check_dataset_metadata_in_storage_unit(nco, dataset_dir):
assert len(nco.variables['dataset']) == 1 # 1 time slice
stored_metadata = nco.variables['dataset'][0]
if not isinstance(stored_metadata, str):
stored_metadata = netCDF4.chartostring(stored_metadata)
stored_metadata = str(np.char.decode(stored_metadata))
ds_filename = dataset_dir / 'agdc-metadata.yaml'
with ds_filename.open() as f:
orig_metadata = f.read()
stored = make_pgsqljson_match_yaml_load(yaml.safe_load(stored_metadata))
original = make_pgsqljson_match_yaml_load(yaml.safe_load(orig_metadata))
assert len(stored['lineage']['source_datasets']) == 1
assert next(iter(stored['lineage']['source_datasets'].values())) == original
def xtime2numtime(xtime):
# First parse the xtime character array into a string
xtimestr = netCDF4.chartostring(xtime) # convert from the character array to an array of strings using the netCDF4 module's function
dt = []
for stritem in xtimestr:
itemarray = stritem.strip().replace('_', '-').replace(':', '-').split('-') # Get an array of strings that are Y,M,D,h,m,s
results = [int(i) for i in itemarray]
if (results[0] < 1900): # datetime has a bug where years less than 1900 are invalid on some systems
results[0] += 1900
dt.append( datetime.datetime(*results) ) # * notation passes in the array as arguments
numtime = netCDF4.date2num(dt, units='seconds since '+str(dt[0])) # use the netCDF4 module's function for converting a datetime to a time number
return numtime
def test_create_string_variable(tmpnetcdf_filename):
nco = create_netcdf(tmpnetcdf_filename)
coord = create_coordinate(nco, 'greg', numpy.array([1.0, 3.0, 9.0]), 'cubic gregs')
dtype = numpy.dtype('S100')
data = numpy.array(["test-str1", "test-str2", "test-str3"], dtype=dtype)
var = create_variable(nco, 'str_var', Variable(dtype, None, ('greg',), None))
var[:] = netcdfy_data(data)
nco.close()
with netCDF4.Dataset(tmpnetcdf_filename) as nco:
assert 'str_var' in nco.variables
assert netCDF4.chartostring(nco['str_var'][0]) == data[0]
def runTest(self):
"""testing compound attributes"""
f = Dataset(self.file, 'r')
v = f.variables[VAR_NAME]
g = f.groups[GROUP_NAME]
vv = g.variables[VAR_NAME2]
assert_array_almost_equal(v.missing_values['speed'], missvals['speed'])
assert_array_almost_equal(v.missing_values['direction'],\
missvals['direction'])
assert_array_almost_equal(vv.missing_values['speed'], missvals['speed'])
assert_array_almost_equal(vv.missing_values['direction'],\
missvals['direction'])
assert_array_equal(v.units['speed'], windunits['speed'].squeeze())
assert_array_equal(v.units['direction'],\
windunits['direction'].squeeze())
assert_array_equal(vv.units['speed'], windunits['speed'].squeeze())
assert_array_equal(vv.units['direction'],\
windunits['direction'].squeeze())
assert(chartostring(v.units['speed']).item().rstrip().decode(default_encoding) == 'm/s')
assert(chartostring(v.units['direction']).item().rstrip().decode(default_encoding) == 'degrees')
assert(chartostring(vv.units['speed']).item().rstrip().decode(default_encoding) == 'm/s')
assert(chartostring(vv.units['direction']).item().rstrip().decode(default_encoding) == 'degrees')
f.close()
def xtime2numtimeMy(xtime):
"""Define a function to convert xtime character array to numeric time values using local arithmetic"""
# First parse the xtime character array into a string
xtimestr = netCDF4.chartostring(xtime) # convert from the character array to an array of strings using the netCDF4 module's function
numtime = np.zeros( (len(xtimestr),) )
ii = 0
dayOfMonthStart = [1, 32, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335]
for stritem in xtimestr:
itemarray = stritem.strip().replace('_', '-').replace(':', '-').split('-') # Get an array of strings that are Y,M,D,h,m,s
results = [int(i) for i in itemarray]
numtime[ii] = results[0] + (dayOfMonthStart[results[1]-1]-1 + results[2]) / 365.0 # decimal year
ii += 1
return numtime
def test_non_scalar_variables(self) :
var = self.dataset.variables['regcodes']
self.assertTrue(var.long_name == "region codes")
self.assertTrue(var.shape == (3,))
data = var[:]
expected = np.array([1,2,3], dtype=np.int32)
expected.shape = (3,)
self.assertTrue(np.array_equal(data, expected))
var = self.dataset.variables['regions']
self.assertTrue(var.long_name == "region names")
self.assertTrue(var.shape == (3,10))
data = var[:]
data = nc4.chartostring(data)
self.assertTrue(data[0].startswith(b"Europe"))
self.assertTrue(data[1].startswith(b"Americas"))
self.assertTrue(data[2].startswith(b"Asia"))
var = self.dataset.variables['digits']
self.assertTrue(var.long_name == "decimal digits")
self.assertTrue(var.shape == (10,))
data = var[:]
data = nc4.chartostring(data)
self.assertTrue(data == b"0123456789")
var = self.dataset.variables['dna_code']
self.assertTrue(var.long_name == "DNA code")
self.assertTrue(var.shape == (2,3,4))
data = var[:]
sample = nc4.chartostring(data[0])
self.assertTrue(sample[0] == b"ACTG")
self.assertTrue(sample[1] == b"ACGG")
self.assertTrue(sample[2] == b"ATGC")
sample = nc4.chartostring(data[1])
self.assertTrue(sample[0] == b"CTGA")
self.assertTrue(sample[1] == b"GCTA")
self.assertTrue(sample[2] == b"TGCA")
def maketransectdf(transect, url='http://opendap.deltares.nl/thredds/dodsC/opendap/rijkswaterstaat/jarkus/profiles/transect.nc'):
"""Read some transect data"""
ds = netCDF4.Dataset(url)
transectidx = bisect.bisect_left(ds.variables['id'],transect)
if ds.variables['id'][transectidx] != transect:
idfound = ds.variables['id'][transectidx]
ds.close()
raise ValueError("Could not find transect data for transect {}, closest is {}".format(transect, idfound))
alongshore = ds.variables['alongshore'][transectidx]
areaname = netCDF4.chartostring(ds.variables['areaname'][transectidx])
mean_high_water = ds.variables['mean_high_water'][transectidx]
mean_low_water = ds.variables['mean_low_water'][transectidx]
ds.close()
transectdf = pandas.DataFrame(index=[transect], data=dict(transect=transect, areaname=areaname, mean_high_water=mean_high_water, mean_low_water=mean_low_water))
return transectdf
def xtime2numtime(xtime):
"""Define a function to convert xtime character array to numeric time values using datetime objects"""
# First parse the xtime character array into a string
xtimestr = netCDF4.chartostring(xtime) # convert from the character array to an array of strings using the netCDF4 module's function
dt = []
for stritem in xtimestr:
itemarray = stritem.strip().replace('_', '-').replace(':', '-').split('-') # Get an array of strings that are Y,M,D,h,m,s
results = [int(i) for i in itemarray]
if (results[0] < 1900): # datetime has a bug where years less than 1900 are invalid on some systems
results[0] += 1900
dt.append( datetime.datetime(*results) ) # * notation passes in the array as arguments
# use the netCDF4 module's function for converting a datetime to a time number
numtime = netCDF4.date2num(dt, units='seconds since '+str(dt[0]))
numtime /= (3600.0 * 24.0 * 365.0)
numtime -= numtime[0] # return years from start
return numtime
def normalize_array(var):
"""
Returns a normalized data array from a NetCDF4 variable. This is mostly
used to normalize string types between py2 and py3. It has no effect on types
other than chars/strings
"""
if np.issubdtype(var.dtype, 'S1'):
if var.dtype == str:
# Python 2 on netCDF4 'string' variables needs this.
# Python 3 returns false for np.issubdtype(var.dtype, 'S1')
return var[:]
def decoder(x):
return str(x.decode('utf-8'))
vfunc = np.vectorize(decoder)
return vfunc(nc4.chartostring(var[:]))
else:
return var[:]
def test_insert_station(test_session_with_empty_db, tiny_dsg_dataset):
var_name = tiny_dsg_dataset.dependent_varnames()[0]
instance_dim = tiny_dsg_dataset.instance_dim(var_name)
name = tiny_dsg_dataset.id_instance_var(var_name)
lat = tiny_dsg_dataset.spatial_instance_var(var_name, 'X')
lon = tiny_dsg_dataset.spatial_instance_var(var_name, 'Y')
i = 0
assert instance_dim.size > 0
check_insert(
insert_station,
test_session_with_empty_db,
tiny_dsg_dataset, i, name, lon, lat,
name=str(chartostring(name[i])),
x=lon[i],
x_units=lon.units,
y=lat[i],
y_units=lat.units
)
def GetFileDateTime(file_obj, time, string=True):
""" Returns a string of the date time from model time index"""
tindex = GetTimeIndex(file_obj, time)
print(tindex)
try:
valid_time = netCDF4.chartostring(file_obj.variables['Times'][tindex])
valid_DT = datetime.datetime.strptime(
valid_time, '%Y-%m-%d_%H:%M:%S') + datetime.datetime(
1900, 1, 1, 0, 0, 0)
except:
return 'Error Parsing netcdf file time variable'
if string:
valid_DT
else:
return valid_time
def xtime2numtimeMy(xtime):
"""Define a function to convert xtime character array to numeric time values using local arithmetic"""
# First parse the xtime character array into a string
xtimestr = netCDF4.chartostring(
xtime
) # convert from the character array to an array of strings using the netCDF4 module's function
numtime = np.zeros((len(xtimestr), ))
ii = 0
dayOfMonthStart = [1, 32, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335]
for stritem in xtimestr:
itemarray = stritem.strip().replace('_', '-').replace(':', '-').split(
'-') # Get an array of strings that are Y,M,D,h,m,s
results = [int(i) for i in itemarray]
numtime[ii] = results[0] + (dayOfMonthStart[results[1] - 1] - 1 +
results[2]) / 365.0 # decimal year
ii += 1
return numtime
def render(self, paths, variable, metric, region):
xs, ys = [], []
for path in paths:
with netCDF4.Dataset(path) as dataset:
mi = index(dataset.variables["metric_names"][:], metric)
ai = index(dataset.variables["area_names"][:], region)
names = np.char.strip(
netCDF4.chartostring(
dataset.variables["forecast_names"][:]))
fi = names == "forecast"
y = dataset.variables[variable][:, fi, :, mi, ai]
y = y.mean(axis=(0, 2))
x = dataset.variables["forecasts"][fi]
xs.append(x)
ys.append(y)
x = np.mean(xs, axis=0)
y = np.mean(ys, axis=0)
self.source.data = {"x": x, "y": y}
def readTimeSeries (ncFile, ncVar = 'zeta', verbose=1):
"""
Reads fort.61.nc-like file
"""
if verbose:
msg( 'i','Reading [' + ncVar + '] from ' + ncFile)
if not os.path.exists (ncFile):
msg( 'e','File ' + ncFile + ' does not exist.')
return
nc = netCDF4.Dataset( ncFile )
fld = nc.variables[ncVar][:]
missingVal = nc.variables[ncVar]._FillValue
try:
fld.unshare_mask()
except:
pass
fld [np.where(fld == missingVal)] = np.nan
lon = nc.variables['x'][:]
lat = nc.variables['y'][:]
tim = nc.variables['time'][:]
nam = nc.variables['station_name'][:]
stations = netCDF4.chartostring(nam) # Python3 requirement?
ncTitle = nc.getncattr('title')
try:
baseDate = datetime.strptime(nc.variables['time'].base_date[0:19],
'%Y-%m-%d %H:%M:%S')
except: # when 00 sec is not written at all
baseDate = datetime.strptime(nc.variables['time'].base_date[0:19],
'%Y-%m-%d %H:%M ')
realtime = np.array([baseDate +
timedelta(seconds=int(tim[i]))
for i in range(len(tim))])
return {'lat' : lat,
'lon' : lon,
'time' : realtime,
'base_date' : baseDate,
'zeta' : fld,
'stations' : stations,
'title' : ncTitle}
def read_point_files(data_files, variables, start_date):
for j, files in enumerate(data_files):
for i, name in enumerate(files):
print(name.split('/')[-1])
nc_file = netCDF4.Dataset(name, 'r')
# Initializations for first iteration of run
if j == 0 and i == 0:
# Station information
stations = {}
stations['name'] = netCDF4.chartostring(
nc_file.variables['station_name'][:, :]).tolist()
stations['lon'] = np.squeeze(nc_file.variables['longitude'][:])
stations['lat'] = np.squeeze(nc_file.variables['latitude'][:])
nstations = len(stations['name'])
# Model output data (nfiles = number of timesnaps, 1 timesnap per file)
nfiles = len(files)
data = {}
for var in variables:
data[var] = np.empty((nfiles, nstations))
# Time information
data['time'] = np.empty(nfiles)
data['ref_date'] = nc_file.variables['time'].getncattr(
'units').replace('days since ', '')
data['ref_date'] = datetime.datetime.strptime(
data['ref_date'], '%Y-%m-%d %H:%M:%S')
# Get time and output variables
if j == 0:
data['time'][i] = nc_file.variables['time'][:]
for var in variables:
if var in nc_file.variables:
print(var)
data[var][i][:] = nc_file.variables[var][:]
data['datetime'], data['date'] = output_time_to_date(
data['time'], data['ref_date'], start_date)
return data, stations
def timestamps(self):
if self.__timestamp_cache.get("timestamps") is None:
for v in ['Times']:
if v in self._dataset.variables:
var = self._dataset.variables[v]
break
tz = pytz.timezone(var.time_zone)
timestamps = np.array(
map(
lambda datestr:
dateutil.parser.parse(datestr).replace(tzinfo=tz),
chartostring(var[:])
)
)
timestamps.flags.writeable = False
self.__timestamp_cache["timestamps"] = timestamps
return self.__timestamp_cache.get("timestamps")
def timestamps(self):
""" Loads, caches, and returns the time dimension from a dataset.
"""
if self.__timestamp_cache.get("timestamps") is None:
for v in ['Times']:
if v in self.nc_data.dataset.variables:
var = self.nc_data.get_dataset_variable(v)
break
tz = pytz.timezone(var.time_zone)
time_list = list(
map(
lambda dateStr: dateutil.parser.parse(dateStr).replace(
tzinfo=tz), netcdf.chartostring(var[:])))
timestamps = np.array(time_list)
timestamps.setflags(write=False) # Make immutable
self.__timestamp_cache["timestamps"] = timestamps
return self.__timestamp_cache.get("timestamps")
def openfile(self):
# takes care of opening files
print self.path+self.filename
if (os.path.isfile(self.path+self.filename)):
# file exists
self.rootcdf = Dataset(self.path+self.filename, 'a') # open a dataset
if self.rtype=='full':
self.stns = chartostring(self.rootcdf.variables['station_name'][:])
self.stns = (np.ma.masked_array(self.stns, self.stns=='')).compressed()
self.stntoid = dict(zip(self.stns,np.arange(0,len(self.stns))))
self.instodid.clear()
self.obstoid.clear()
else:
self.rootcdf = Dataset(self.path+self.filename, 'w') # open a dataset
self.newfile()
self.stntoid.clear()
self.instodid.clear()
self.obstoid.clear()
def read_mmd(sensor, pathname):
"""
:type sensor: str
:type pathname: str
:rtype : tuple
"""
mmd = Dataset(pathname)
mmd.set_auto_maskandscale(True)
nx = len(mmd.dimensions['atsr.nx'])
ny = len(mmd.dimensions['atsr.ny'])
m_ids = mmd.variables['matchup.id'][:]
mmd.variables[sensor + '.matchup_elem'].set_auto_maskandscale(False)
mmd.variables[sensor + '.matchup_line'].set_auto_maskandscale(False)
m_elems = mmd.variables['atsr.3.matchup_elem'][:]
m_lines = mmd.variables['atsr.3.matchup_line'][:]
m_source_filenames = chartostring(mmd.variables[sensor + '.l1b_filename'][:])
return mmd, nx, ny, m_ids, m_source_filenames, m_elems, m_lines
def getProfiles(full_path,SearchLimits,StartDate,EndDate,FillValue):
s_lim, n_lim, w_lim, e_lim = SearchLimits[:] # Unpack SearchLimits.
filename_list = glob.glob(full_path) # Search for files in full_path.
matching_file_list = [] # Make an empty list to hold filenames.
Argo_Dataframe = pd.DataFrame(columns=["PRES","PSAL","TEMPR","LAT","LON","JULD","ID"]) # Make an empty dataframe to hold profile data.
for filename in filename_list: # For each file in full_path...
nc = netCDF4.Dataset(filename) # Call the Dataset constructor.
try: # Try to read the file.
LAT = nc.variables['LATITUDE'][:]
LON = nc.variables['LONGITUDE'][:]
JULD = nc.variables['JULD'][:]
# See which profiles have the correct lat,lon,date.
LAT_good_loci = np.where((s_lim <= LAT) & (LAT <= n_lim))
LON_good_loci = np.where((w_lim <= LON) & (LON <= e_lim))
JULD_good_loci = np.where((StartDate <= JULD) & (JULD < (EndDate+1))) # (The "+1" is to include the EndDate.)
loci_of_good_profiles = np.intersect1d(np.intersect1d(LAT_good_loci,LON_good_loci),JULD_good_loci)
if (len(loci_of_good_profiles) > 0): # If there is at least one good profile in this file...
matching_file_list.append(filename) # Record filename to list.
TEMPR = nc.variables['TEMP_ADJUSTED'][:] # Read all profiles in the file.
PSAL = nc.variables['PSAL_ADJUSTED'][:]
PRES = nc.variables['PRES_ADJUSTED'][:]
ID = netCDF4.chartostring(nc.variables['PLATFORM_NUMBER'][:])
ID = ID.astype(np.int)
for locus in loci_of_good_profiles: # Write each good profile into temporary cell array...
if not all(x == FillValue for x in PRES[locus]): # As long is profile is not just fill values.
Profile_Dataframe = pd.DataFrame({"PRES":[PRES[locus]],"PSAL":[PSAL[locus]],"TEMPR":[TEMPR[locus]],"LAT":LAT[locus],"LON":LON[locus],"JULD":JULD[locus],"ID":ID[locus]},index=[0])
Argo_Dataframe= pd.concat([Argo_Dataframe, Profile_Dataframe],axis=0,sort=False) # Combine temporary with general dataframe.
except Exception as error_message: # Throw an exception if unable to read file.
print(error_message)
nc.close() # Close the file.
print('\nMatching files:\n'+'\n'.join(matching_file_list)) # Print a list of matching files.
print(Argo_Dataframe) # Print the dataframe.
print(Argo_Dataframe.shape)
return(matching_file_list,Argo_Dataframe) # Return the list and the dataframe.
def __init__(self, data_file, errors_file=None, batch_size=32,
tile_shape=(128,128), random_seed=None):
self.data_file = data_file
self.errors_file = errors_file
self.batch_size = batch_size
self.tile_shape = tile_shape
self.img_shape = tile_shape # for compatibility
self.ds = netCDF4.Dataset(data_file, 'r')
self.N = self.ds["image"].shape[0]
self.n_channels = self.ds["image"].shape[-1]
self.image_shape = self.ds["image"].shape[1:3]
self.timestamps = [str(ts) for ts in
netCDF4.chartostring(self.ds["timestamp"][:])]
with open(errors_file) as f:
self.errors = json.load(f)
self.prng = np.random.RandomState(seed=random_seed)
def read_mmd(sensor, pathname):
"""
:type sensor: str
:type pathname: str
:rtype : tuple
"""
mmd = Dataset(pathname)
mmd.set_auto_maskandscale(True)
nx = len(mmd.dimensions['atsr.nx'])
ny = len(mmd.dimensions['atsr.ny'])
m_ids = mmd.variables['matchup.id'][:]
mmd.variables[sensor + '.matchup_elem'].set_auto_maskandscale(False)
mmd.variables[sensor + '.matchup_line'].set_auto_maskandscale(False)
m_elems = mmd.variables['atsr.3.matchup_elem'][:]
m_lines = mmd.variables['atsr.3.matchup_line'][:]
m_source_filenames = chartostring(mmd.variables[sensor +
'.l1b_filename'][:])
return mmd, nx, ny, m_ids, m_source_filenames, m_elems, m_lines
def read_data(self):
with nc4.Dataset(self.filename) as f:
grp_resp = f.groups['response_data']
resp = {}
pts = {}
skipvar = ["x", "y", "z", "time", "points"]
t = grp_resp.variables['time'][:]
pts_list = []
for point in nc4.chartostring(grp_resp.variables['points'][:]):
pts_list.append(point)
for ptid, x in enumerate(grp_resp.variables['x'][:]):
y = grp_resp.variables['y'][:][ptid]
z = grp_resp.variables['z'][:][ptid]
pts[pts_list[ptid]] = (x, y, z)
resp[pts_list[ptid]] = {}
for var in grp_resp.variables:
if not (var in skipvar):
resp[pts_list[ptid]][var] = grp_resp.variables[
var][:, ptid]
return (resp, t, pts)
def test_find_or_insert_stations(
test_session_with_empty_db, tiny_dsg_dataset):
var_name = tiny_dsg_dataset.dependent_varnames()[0]
stations = find_or_insert_stations(
test_session_with_empty_db, tiny_dsg_dataset, var_name)
instance_dim = tiny_dsg_dataset.instance_dim(var_name)
name = tiny_dsg_dataset.id_instance_var(var_name)
lat = tiny_dsg_dataset.spatial_instance_var(var_name, 'X')
lon = tiny_dsg_dataset.spatial_instance_var(var_name, 'Y')
assert len(stations) == instance_dim.size
for i, station in enumerate(stations):
check_properties(
station,
name=str(chartostring(name[i])),
x=lon[i],
x_units=lon.units,
y=lat[i],
y_units=lat.units
)
def read_line_data_header_variables(self, L):
"""
Reads the header data from a NetCDF file.
Args:
L (object): Line object
Returns:
none
"""
L.bank = self.nc.variables['Header.LineData.Bank'][0]
L.v_source = self.nc.variables['Header.LineData.VSource'][0]
L.v_system = self.nc.variables['Header.LineData.VSystem'][0]
L.z_source = self.nc.variables['Header.LineData.ZSource'][0]
L.frest = self.nc.variables['Header.LineData.LineRestFrequency'][0]
L.dop_track = self.nc.variables['Header.LineData.DopplerTrack'][0]
L.vbary = self.nc.variables['Header.LineData.VBarycenter'][0]
L.vobs = self.nc.variables['Header.LineData.VObservatory'][0]
L.fsky = self.nc.variables['Header.LineData.SkyFrequency'][0]
L.flo1 = self.nc.variables['Header.LineData.LO1Frequency'][0]
L.flo2 = self.nc.variables['Header.LineData.LO2Frequency'][0]
L.fif1 = self.nc.variables['Header.LineData.IF1Frequency'][0]
L.fif2 = self.nc.variables['Header.LineData.IF2Frequency'][0]
L.fsyn = self.nc.variables['Header.LineData.SynthesizerFrequency'][0]
L.foff = self.nc.variables['Header.LineData.FrequencyOffset'][0]
L.voff = self.nc.variables['Header.LineData.VelocityOffset'][0]
L.sideband = self.nc.variables['Header.LineData.Sideband'][0]
L.nchan = self.nc.variables['Header.LineData.NChannels'][0]
L.bw = self.nc.variables['Header.LineData.Bandwidth'][0]
L.dfdc = self.nc.variables['Header.LineData.DeltaFrequency'][0]
L.dvdc = self.nc.variables['Header.LineData.DeltaVelocity'][0]
L.v0_lsr = self.nc.variables['Header.LineData.LSRFrameV0'][0]
L.v0_bary = self.nc.variables['Header.LineData.BarycenterFrameV0'][0]
L.v0_sky = self.nc.variables['Header.LineData.SkyFrameV0'][0]
L.xtype = self.nc.variables['Header.LineData.XType'][0]
L.iarray = self.nc.variables['Header.LineData.ChannelNumber'][:]
# the following are the generic spectrum axis parameters
L.ch0 = self.nc.variables['Header.SpectrumAxis.CRPIX'][0]
L.xv0 = self.nc.variables['Header.SpectrumAxis.CRVAL'][0]
L.dxdc = self.nc.variables['Header.SpectrumAxis.CDELT'][0]
L.xname = netCDF4.chartostring(
self.nc.variables['Header.SpectrumAxis.CTYPE'][:])
L.xarray = self.nc.variables['Header.SpectrumAxis.CAXIS'][:]
def test_hourly_aggregator_with_nonqc(self):
output_file, bad_files = hourly_aggregator(
files_to_aggregate=INPUT_FILES,
site_code='NRSROT',
qcflags=(0, 1, 2),
input_dir=TEST_ROOT,
output_dir='/tmp',
download_url_prefix='http://test.download.url',
opendap_url_prefix='http://test.opendap.url')
self.assertRegex(
output_file,
r'IMOS_ANMN-NRS_BOSTUZ_20181213_NRSROT_FV02_hourly-timeseries-including-non-QC'
r'_END-20190523_C-\d{8}\.nc')
dataset = Dataset(output_file)
self.assertEqual(dataset['source_file'].download_url_prefix,
'http://test.download.url')
self.assertEqual(dataset['source_file'].opendap_url_prefix,
'http://test.opendap.url')
for f in chartostring(dataset['source_file'][:]):
self.assertIn(f, INPUT_FILES)
def readDSGfile(netCDFfiles):
ds = Dataset(netCDFfiles[1], 'a')
vs = ds.get_variables_by_attributes(
standard_name='sea_water_pressure_due_to_sea_water')
vs = ds.get_variables_by_attributes(long_name='actual depth')
pres_var = vs[0]
var_temp = ds.variables["TEMP"]
plot_var = var_temp[:]
plt.plot(plot_var)
plt.show()
time_var = ds.variables["TIME"]
time = num2date(time_var[:],
units=time_var.units,
calendar=time_var.calendar)
index_var = ds.variables["instrument_index"]
idx = index_var[:]
instrument_id_var = ds.variables["instrument_id"]
#print(idx)
i = 0
for x in chartostring(instrument_id_var[:]):
print(i, x, time[idx == 1], plot_var[idx == i])
plt.plot(time[idx == i], plot_var[idx == i]) # , marker='.'
i += 1
#plt.gca().invert_yaxis()
plt.grid(True)
# close the netCDF file
ds.close()
plt.show()
def fromNetCDF(self, ncfilename):
nc_fid = netCDF4.Dataset(ncfilename, 'r')
timestamp = datetime.strptime( \
nc_fid.getncattr( 'sliceCenterTimeUTC' ), \
"%Y-%m-%d_%H:%M:00" )
#print timestamp.isoformat()
time_resol = timedelta( minutes = \
int( nc_fid.getncattr( 'sliceTimeResolutionMinutes' ) ) )
stations = netCDF4.chartostring( \
nc_fid.variables[ 'stationId'][ : ] )
discharge = nc_fid.variables['discharge'][:]
queryTime = nc_fid.variables['queryTime'][:]
quality = nc_fid.variables['discharge_quality'][:]
# for s, d, q in zip( stations, discharge, queryTime ):
# print 'USGS.' + s.rstrip(), d, \
# datetime.utcfromtimestamp( q ).isoformat(), \
# datetime.utcfromtimestamp( q ).tzname(), \
# q
# self.centralTimeStamp = timestamp
# self.sliceTimeResolution = time_resol
# self.obvStationTimeValue = []
stationTimeValue = []
for s, d, q, qual in zip(stations, discharge, queryTime, quality):
stationTimeValue.append( \
( 'USGS.' + s.strip(), \
datetime.utcfromtimestamp( q ), d, qual ) )
nc_fid.close()
return self(timestamp, time_resol, stationTimeValue)
def read_file(netcdf_file_name):
"""Reads class-activation maps from NetCDF file.
:param netcdf_file_name: Path to input file.
:return: gradcam_dict: Dictionary with the following keys.
gradcam_dict['class_activation_matrix']: See doc for `write_file`.
gradcam_dict['example_id_strings']: Same.
gradcam_dict['model_file_name']: Same.
gradcam_dict['activation_layer_name']: Same.
gradcam_dict['vector_output_layer_name']: Same.
gradcam_dict['output_neuron_indices']: Same.
gradcam_dict['ideal_activation']: Same.
"""
dataset_object = netCDF4.Dataset(netcdf_file_name)
gradcam_dict = {
CLASS_ACTIVATIONS_KEY:
dataset_object.variables[CLASS_ACTIVATIONS_KEY][:],
EXAMPLE_IDS_KEY: [
str(id) for id in netCDF4.chartostring(
dataset_object.variables[EXAMPLE_IDS_KEY][:])
],
MODEL_FILE_KEY:
str(getattr(dataset_object, MODEL_FILE_KEY)),
ACTIVATION_LAYER_KEY:
str(getattr(dataset_object, ACTIVATION_LAYER_KEY)),
VECTOR_OUT_LAYER_KEY:
str(getattr(dataset_object, VECTOR_OUT_LAYER_KEY)),
OUTPUT_NEURONS_KEY:
numpy.array(getattr(dataset_object, OUTPUT_NEURONS_KEY), dtype=int),
IDEAL_ACTIVATION_KEY:
getattr(dataset_object, IDEAL_ACTIVATION_KEY)
}
dataset_object.close()
return gradcam_dict
def openfile(self):
# takes care of opening files
print self.path + self.filename
if (os.path.isfile(self.path + self.filename)):
# file exists
self.rootcdf = Dataset(self.path + self.filename,
'a') # open a dataset
if self.rtype == 'full':
self.stns = chartostring(
self.rootcdf.variables['station_name'][:])
self.stns = (np.ma.masked_array(self.stns,
self.stns == '')).compressed()
self.stntoid = dict(
zip(self.stns, np.arange(0, len(self.stns))))
self.instodid.clear()
self.obstoid.clear()
else:
self.rootcdf = Dataset(self.path + self.filename,
'w') # open a dataset
self.newfile()
self.stntoid.clear()
self.instodid.clear()
self.obstoid.clear()
def process(ncfn):
"""Process this file """
pgconn = get_dbconn('iem')
icursor = pgconn.cursor()
xref = {}
icursor.execute("""SELECT id, network from stations where
network ~* 'ASOS' or network = 'AWOS' and country = 'US'""")
for row in icursor:
xref[row[0]] = row[1]
icursor.close()
nc = ncopen(ncfn)
data = {}
for vname in [
'stationId',
'observationTime',
'temperature',
'dewpoint',
'altimeter', # Pa
'windDir',
'windSpeed', # mps
'windGust',
'visibility', # m
'precipAccum',
'presWx',
'skyCvr',
'skyCovLayerBase',
'autoRemark',
'operatorRemark'
]:
data[vname] = nc.variables[vname][:]
for qc in ['QCR', 'QCD']:
vname2 = vname + qc
if vname2 in nc.variables:
data[vname2] = nc.variables[vname2][:]
for vname in ['temperature', 'dewpoint']:
data[vname + "C"] = temperature(data[vname], 'K').value('C')
data[vname] = temperature(data[vname], 'K').value('F')
for vname in ['windSpeed', 'windGust']:
data[vname] = speed(data[vname], 'MPS').value('KT')
data['altimeter'] = pressure(data['altimeter'], 'PA').value("IN")
data['skyCovLayerBase'] = distance(data['skyCovLayerBase'],
'M').value("FT")
data['visibility'] = distance(data['visibility'], 'M').value("MI")
data['precipAccum'] = distance(data['precipAccum'], 'MM').value("IN")
stations = chartostring(data['stationId'][:])
presentwxs = chartostring(data['presWx'][:])
skycs = chartostring(data['skyCvr'][:])
autoremarks = chartostring(data['autoRemark'][:])
opremarks = chartostring(data['operatorRemark'][:])
def decision(i, fieldname, tolerance):
"""Our decision if we are going to take a HFMETAR value or not"""
if data[fieldname][i] is np.ma.masked:
return None
if data["%sQCR" % (fieldname, )][i] == 0:
return data[fieldname][i]
# Now we have work to do
departure = np.ma.max(np.ma.abs(data['%sQCD' % (fieldname, )][i, :]))
# print("departure: %s tolerance: %s" % (departure, tolerance))
if departure <= tolerance:
return data[fieldname][i]
return None
for i, sid in tqdm(enumerate(stations),
total=len(stations),
disable=(not sys.stdout.isatty())):
sid3 = sid[1:] if sid[0] == 'K' else sid
ts = datetime.datetime(1970, 1, 1) + datetime.timedelta(
seconds=data['observationTime'][i])
ts = ts.replace(tzinfo=pytz.utc)
mtr = "%s %sZ AUTO " % (sid, ts.strftime("%d%H%M"))
network = xref.get(sid3, 'ASOS')
iem = Observation(sid3, network, ts)
# 06019G23KT
val = decision(i, 'windDir', 15)
if val is not None:
iem.data['drct'] = int(val)
mtr += "%03i" % (iem.data['drct'], )
else:
mtr += "///"
val = decision(i, 'windSpeed', 10)
if val is not None:
iem.data['sknt'] = int(val)
mtr += "%02i" % (iem.data['sknt'], )
else:
mtr += "//"
val = decision(i, 'windGust', 10)
if val is not None and val > 0:
iem.data['gust'] = int(val)
mtr += "G%02i" % (iem.data['gust'], )
mtr += "KT "
val = decision(i, 'visibility', 4)
if val is not None:
iem.data['vsby'] = float(val)
mtr += "%sSM " % (vsbyfmt(iem.data['vsby']), )
presentwx = presentwxs[i]
if presentwx != '':
# database storage is comma delimited
iem.data['wxcodes'] = presentwx.split(" ")
mtr += "%s " % (presentwx, )
for _i, (skyc,
_l) in enumerate(zip(skycs[i], data['skyCovLayerBase'][i])):
if skyc != '':
iem.data['skyc%s' % (_i + 1, )] = skyc
if skyc != 'CLR':
iem.data['skyl%s' % (_i + 1, )] = int(_l)
mtr += "%s%03i " % (skyc, int(_l) / 100)
else:
mtr += "CLR "
t = ""
tgroup = "T"
val = decision(i, 'temperature', 10)
if val is not None:
# Recall the pain enabling this
# iem.data['tmpf'] = float(data['temperature'][i])
tmpc = float(data['temperatureC'][i])
t = "%s%02i/" % ("M" if tmpc < 0 else "", tmpc if tmpc > 0 else
(0 - tmpc))
tgroup += "%s%03i" % ("1" if tmpc < 0 else "0",
(tmpc if tmpc > 0 else (0 - tmpc)) * 10.)
val = decision(i, 'dewpoint', 10)
if val is not None:
# iem.data['dwpf'] = float(data['dewpoint'][i])
tmpc = float(data['dewpointC'][i])
if t != "":
t = "%s%s%02i " % (t, "M" if tmpc < 0 else "",
tmpc if tmpc > 0 else 0 - tmpc)
tgroup += "%s%03i" % ("1" if tmpc < 0 else "0",
(tmpc if tmpc > 0 else (0 - tmpc)) * 10.)
if len(t) > 4:
mtr += t
val = decision(i, 'altimeter', 20)
if val is not None:
iem.data['alti'] = float(round(val, 2))
mtr += "A%4i " % (iem.data['alti'] * 100., )
mtr += "RMK "
val = decision(i, 'precipAccum', 25)
if val is not None:
if val >= 0.01:
iem.data['phour'] = float(round(val, 2))
mtr += "P%04i " % (iem.data['phour'] * 100., )
elif val < 0.01 and val > 0:
# Trace
mtr += "P0000 "
iem.data['phour'] = TRACE_VALUE
if tgroup != "T":
mtr += "%s " % (tgroup, )
if autoremarks[i] != '' or opremarks[i] != '':
mtr += "%s %s " % (autoremarks[i], opremarks[i])
mtr += "MADISHF"
# Eat our own dogfood
try:
Metar.Metar(mtr)
iem.data['raw'] = mtr
except Exception as exp:
print("dogfooding extract_hfmetar %s resulted in %s" % (mtr, exp))
continue
for key in iem.data:
if isinstance(iem.data[key], np.float32):
print("key: %s type: %s" % (key, type(iem.data[key])))
icursor = pgconn.cursor()
if not iem.save(icursor, force_current_log=True, skip_current=True):
print(("extract_hfmetar: unknown station? %s %s %s\n%s") %
(sid3, network, ts, mtr))
icursor.close()
pgconn.commit()
stationobs_units['temp_sounding'] = stringtoarr('Kelvin',NUMCHARS)
stationobs_units['press_sounding'] = stringtoarr('hPa',NUMCHARS)
statdat.units = stationobs_units
# close and reopen the file.
f.close()
f = Dataset('compound_example.nc')
print f
statdat = f.variables['station_obs']
# print out data in variable.
print 'data in a variable of compound type:'
print '----'
for data in statdat[:]:
for name in statdat.dtype.names:
if data[name].dtype.kind == 'S': # a string
# convert array of characters back to a string for display.
print name,': value =',chartostring(data[name]),\
': units=',chartostring(statdat.units[name])
elif data[name].dtype.kind == 'V': # a nested compound type
print name,data[name].dtype.names,': value=',data[name],': units=',\
tuple([chartostring(s) for s in tuple(statdat.units[name])])
else: # a numeric type.
print name,': value=',data[name],': units=',chartostring(statdat.units[name])
print '----'
f.close()
f = Dataset('tst_vlen.nc','w')
vlen_t = f.createVLType(numpy.int32, 'phony_vlen')
x = f.createDimension('x',3)
y = f.createDimension('y',4)
vlvar = f.createVariable('phony_vlen_var', vlen_t, ('y','x'))
import random
def process(ncfn):
"""Process this file """
pgconn = get_dbconn("iem")
icursor = pgconn.cursor()
xref = {}
icursor.execute("SELECT id, network from stations where "
"network ~* 'ASOS' or network = 'AWOS' and country = 'US'")
for row in icursor:
xref[row[0]] = row[1]
icursor.close()
nc = ncopen(ncfn)
data = {}
for vname in [
"stationId",
"observationTime",
"temperature",
"dewpoint",
"altimeter", # Pa
"windDir",
"windSpeed", # mps
"windGust", # mps
"visibility", # m
"precipAccum",
"presWx",
"skyCvr",
"skyCovLayerBase",
"autoRemark",
"operatorRemark",
]:
data[vname] = nc.variables[vname][:]
for qc in ["QCR", "QCD"]:
vname2 = vname + qc
if vname2 in nc.variables:
data[vname2] = nc.variables[vname2][:]
for vname in ["temperature", "dewpoint"]:
data[vname + "C"] = temperature(data[vname], "K").value("C")
data[vname] = temperature(data[vname], "K").value("F")
for vname in ["windSpeed", "windGust"]:
data[vname] = (masked_array(data[vname], units("meter / second")).to(
units("knots")).magnitude)
data["altimeter"] = pressure(data["altimeter"], "PA").value("IN")
data["skyCovLayerBase"] = distance(data["skyCovLayerBase"],
"M").value("FT")
data["visibility"] = distance(data["visibility"], "M").value("MI")
data["precipAccum"] = distance(data["precipAccum"], "MM").value("IN")
stations = chartostring(data["stationId"][:])
presentwxs = chartostring(data["presWx"][:])
skycs = chartostring(data["skyCvr"][:])
autoremarks = chartostring(data["autoRemark"][:])
opremarks = chartostring(data["operatorRemark"][:])
def decision(i, fieldname, tolerance):
"""Our decision if we are going to take a HFMETAR value or not"""
if data[fieldname][i] is np.ma.masked:
return None
if data["%sQCR" % (fieldname, )][i] == 0:
return data[fieldname][i]
# Now we have work to do
departure = np.ma.max(np.ma.abs(data["%sQCD" % (fieldname, )][i, :]))
# print("departure: %s tolerance: %s" % (departure, tolerance))
if departure <= tolerance:
return data[fieldname][i]
return None
for i, sid in tqdm(
enumerate(stations),
total=len(stations),
disable=(not sys.stdout.isatty()),
):
if len(sid) < 3:
continue
sid3 = sid[1:] if sid.startswith("K") else sid
ts = datetime.datetime(1970, 1, 1) + datetime.timedelta(
seconds=data["observationTime"][i])
ts = ts.replace(tzinfo=pytz.UTC)
mtr = "%s %sZ AUTO " % (sid, ts.strftime("%d%H%M"))
network = xref.get(sid3, "ASOS")
iem = Observation(sid3, network, ts)
# 06019G23KT
val = decision(i, "windDir", 15)
if val is not None:
iem.data["drct"] = int(val)
mtr += "%03i" % (iem.data["drct"], )
else:
mtr += "///"
val = decision(i, "windSpeed", 10)
if val is not None:
iem.data["sknt"] = int(val)
mtr += "%02i" % (iem.data["sknt"], )
else:
mtr += "//"
val = decision(i, "windGust", 10)
if val is not None and val > 0:
iem.data["gust"] = int(val)
mtr += "G%02i" % (iem.data["gust"], )
mtr += "KT "
val = decision(i, "visibility", 4)
if val is not None:
iem.data["vsby"] = float(val)
mtr += "%sSM " % (vsbyfmt(iem.data["vsby"]), )
presentwx = presentwxs[i]
if presentwx != "":
# database storage is comma delimited
iem.data["wxcodes"] = presentwx.split(" ")
mtr += "%s " % (presentwx, )
for _i, (skyc,
_l) in enumerate(zip(skycs[i], data["skyCovLayerBase"][i])):
if skyc != "":
iem.data["skyc%s" % (_i + 1, )] = skyc
if skyc != "CLR":
iem.data["skyl%s" % (_i + 1, )] = int(_l)
mtr += "%s%03i " % (skyc, int(_l) / 100)
else:
mtr += "CLR "
t = ""
tgroup = "T"
val = decision(i, "temperature", 10)
if val is not None:
# Recall the pain enabling this
# iem.data['tmpf'] = float(data['temperature'][i])
tmpc = float(data["temperatureC"][i])
t = "%s%02i/" % (
"M" if tmpc < 0 else "",
tmpc if tmpc > 0 else (0 - tmpc),
)
tgroup += "%s%03i" % (
"1" if tmpc < 0 else "0",
(tmpc if tmpc > 0 else (0 - tmpc)) * 10.0,
)
val = decision(i, "dewpoint", 10)
if val is not None:
# iem.data['dwpf'] = float(data['dewpoint'][i])
tmpc = float(data["dewpointC"][i])
if t != "":
t = "%s%s%02i " % (
t,
"M" if tmpc < 0 else "",
tmpc if tmpc > 0 else 0 - tmpc,
)
tgroup += "%s%03i" % (
"1" if tmpc < 0 else "0",
(tmpc if tmpc > 0 else (0 - tmpc)) * 10.0,
)
if len(t) > 4:
mtr += t
val = decision(i, "altimeter", 20)
if val is not None:
iem.data["alti"] = float(round(val, 2))
mtr += "A%4i " % (iem.data["alti"] * 100.0, )
mtr += "RMK "
val = decision(i, "precipAccum", 25)
if val is not None:
if val > 0.009:
iem.data["phour"] = float(round(val, 2))
mtr += "P%04i " % (iem.data["phour"] * 100.0, )
elif val > 0:
# Trace
mtr += "P0000 "
iem.data["phour"] = TRACE_VALUE
if tgroup != "T":
mtr += "%s " % (tgroup, )
if autoremarks[i] != "" or opremarks[i] != "":
mtr += "%s %s " % (autoremarks[i], opremarks[i])
mtr += "MADISHF"
# Eat our own dogfood
try:
Metar.Metar(mtr)
iem.data["raw"] = mtr
except Exception as exp:
print("dogfooding extract_hfmetar %s resulted in %s" % (mtr, exp))
continue
for key in iem.data:
if isinstance(iem.data[key], np.float32):
print("key: %s type: %s" % (key, type(iem.data[key])))
icursor = pgconn.cursor()
if not iem.save(icursor, force_current_log=True, skip_current=True):
print(("extract_hfmetar: unknown station? %s %s %s\n%s") %
(sid3, network, ts, mtr))
icursor.close()
pgconn.commit()
print 'Basal homologous temperature at divide (deg C) = {}'.format(basalTemp[ind][0]-basalPmpTemp[ind][0])
print 'EISMINT models basal temperature at divide (m):'
print ' 3d models (6 of them): -13.34 +/- 0.56'
print '===================================='
print ''
# Plot the results
fig = plt.figure(1, facecolor='w')
markersize = 30.0
fig.add_subplot(1,1,1)
plt.scatter(xCell,yCell,markersize,thk[:], marker='h', edgecolors='none')
plt.colorbar()
plt.axis('equal')
plt.title('Modeled thickness (m) \n at time ' + netCDF4.chartostring(xtime)[timelev].strip() )
plt.draw()
if options.saveimage:
plotname = 'halfar-results.png'
plt.savefig(plotname, dpi=150)
print 'Saved plot as ' + plotname
if options.hidefigs:
print "Plot display disabled with -n argument."
else:
print 'Showing plot... Close plot window to exit.'
plt.show()
def read_cfradial(filename,
field_names=None,
additional_metadata=None,
file_field_names=False,
exclude_fields=None,
include_fields=None,
delay_field_loading=False,
**kwargs):
"""
Read a Cfradial netCDF file.
Parameters
----------
filename : str
Name of CF/Radial netCDF file to read data from.
field_names : dict, optional
Dictionary mapping field names in the file names to radar field names.
Unlike other read functions, fields not in this dictionary or having a
value of None are still included in the radar.fields dictionary, to
exclude them use the `exclude_fields` parameter. Fields which are
mapped by this dictionary will be renamed from key to value.
additional_metadata : dict of dicts, optional
This parameter is not used, it is included for uniformity.
file_field_names : bool, optional
True to force the use of the field names from the file in which
case the `field_names` parameter is ignored. False will use to
`field_names` parameter to rename fields.
exclude_fields : list or None, optional
List of fields to exclude from the radar object. This is applied
after the `file_field_names` and `field_names` parameters. Set
to None to include all fields specified by include_fields.
include_fields : list or None, optional
List of fields to include from the radar object. This is applied
after the `file_field_names` and `field_names` parameters. Set
to None to include all fields not specified by exclude_fields.
delay_field_loading : bool
True to delay loading of field data from the file until the 'data'
key in a particular field dictionary is accessed. In this case
the field attribute of the returned Radar object will contain
LazyLoadDict objects not dict objects. Delayed field loading will not
provide any speedup in file where the number of gates vary between
rays (ngates_vary=True) and is not recommended.
Returns
-------
radar : Radar
Radar object.
Notes
-----
This function has not been tested on "stream" Cfradial files.
"""
# test for non empty kwargs
_test_arguments(kwargs)
# create metadata retrieval object
filemetadata = FileMetadata('cfradial', field_names, additional_metadata,
file_field_names, exclude_fields)
# read the data
ncobj = netCDF4.Dataset(filename)
ncvars = ncobj.variables
# 4.1 Global attribute -> move to metadata dictionary
metadata = dict([(k, getattr(ncobj, k)) for k in ncobj.ncattrs()])
if 'n_gates_vary' in metadata:
metadata['n_gates_vary'] = 'false' # corrected below
# 4.2 Dimensions (do nothing)
# 4.3 Global variable -> move to metadata dictionary
if 'volume_number' in ncvars:
metadata['volume_number'] = int(ncvars['volume_number'][:])
else:
metadata['volume_number'] = 0
global_vars = {
'platform_type': 'fixed',
'instrument_type': 'radar',
'primary_axis': 'axis_z'
}
# ignore time_* global variables, these are calculated from the time
# variable when the file is written.
for var, default_value in global_vars.items():
if var in ncvars:
metadata[var] = str(netCDF4.chartostring(ncvars[var][:]))
else:
metadata[var] = default_value
# 4.4 coordinate variables -> create attribute dictionaries
time = _ncvar_to_dict(ncvars['time'])
_range = _ncvar_to_dict(ncvars['range'])
# 4.5 Ray dimension variables
# 4.6 Location variables -> create attribute dictionaries
latitude = _ncvar_to_dict(ncvars['latitude'])
longitude = _ncvar_to_dict(ncvars['longitude'])
altitude = _ncvar_to_dict(ncvars['altitude'])
if 'altitude_agl' in ncvars:
altitude_agl = _ncvar_to_dict(ncvars['altitude_agl'])
else:
altitude_agl = None
# 4.7 Sweep variables -> create atrribute dictionaries
sweep_mode = _ncvar_to_dict(ncvars['sweep_mode'])
fixed_angle = _ncvar_to_dict(ncvars['fixed_angle'])
sweep_start_ray_index = _ncvar_to_dict(ncvars['sweep_start_ray_index'])
sweep_end_ray_index = _ncvar_to_dict(ncvars['sweep_end_ray_index'])
if 'sweep_number' in ncvars:
sweep_number = _ncvar_to_dict(ncvars['sweep_number'])
else:
nsweeps = len(sweep_start_ray_index['data'])
sweep_number = filemetadata('sweep_number')
sweep_number['data'] = np.arange(nsweeps, dtype='float32')
warnings.warn("Warning: File violates CF/Radial convention. " +
"Missing sweep_number variable")
if 'target_scan_rate' in ncvars:
target_scan_rate = _ncvar_to_dict(ncvars['target_scan_rate'])
else:
target_scan_rate = None
if 'rays_are_indexed' in ncvars:
rays_are_indexed = _ncvar_to_dict(ncvars['rays_are_indexed'])
else:
rays_are_indexed = None
if 'ray_angle_res' in ncvars:
ray_angle_res = _ncvar_to_dict(ncvars['ray_angle_res'])
else:
ray_angle_res = None
# first sweep mode determines scan_type
try:
mode = netCDF4.chartostring(sweep_mode['data'][0])[()].decode('utf-8')
except AttributeError:
# Python 3, all strings are already unicode.
mode = netCDF4.chartostring(sweep_mode['data'][0])[()]
# options specified in the CF/Radial standard
if mode == 'rhi':
scan_type = 'rhi'
elif mode == 'vertical_pointing':
scan_type = 'vpt'
elif mode == 'azimuth_surveillance':
scan_type = 'ppi'
elif mode == 'elevation_surveillance':
scan_type = 'rhi'
elif mode == 'manual_ppi':
scan_type = 'ppi'
elif mode == 'manual_rhi':
scan_type = 'rhi'
# fallback types
elif 'sur' in mode:
scan_type = 'ppi'
elif 'sec' in mode:
scan_type = 'sector'
elif 'rhi' in mode:
scan_type = 'rhi'
elif 'ppi' in mode:
scan_type = 'ppi'
else:
scan_type = 'other'
# 4.8 Sensor pointing variables -> create attribute dictionaries
azimuth = _ncvar_to_dict(ncvars['azimuth'])
elevation = _ncvar_to_dict(ncvars['elevation'])
if 'scan_rate' in ncvars:
scan_rate = _ncvar_to_dict(ncvars['scan_rate'])
else:
scan_rate = None
if 'antenna_transition' in ncvars:
antenna_transition = _ncvar_to_dict(ncvars['antenna_transition'])
else:
antenna_transition = None
# 4.9 Moving platform geo-reference variables
# Aircraft specific varaibles
if 'rotation' in ncvars:
rotation = _ncvar_to_dict(ncvars['rotation'])
else:
rotation = None
if 'tilt' in ncvars:
tilt = _ncvar_to_dict(ncvars['tilt'])
else:
tilt = None
if 'roll' in ncvars:
roll = _ncvar_to_dict(ncvars['roll'])
else:
roll = None
if 'drift' in ncvars:
drift = _ncvar_to_dict(ncvars['drift'])
else:
drift = None
if 'heading' in ncvars:
heading = _ncvar_to_dict(ncvars['heading'])
else:
heading = None
if 'pitch' in ncvars:
pitch = _ncvar_to_dict(ncvars['pitch'])
else:
pitch = None
if 'georefs_applied' in ncvars:
georefs_applied = _ncvar_to_dict(ncvars['georefs_applied'])
else:
georefs_applied = None
# 4.10 Moments field data variables -> field attribute dictionary
if 'ray_n_gates' in ncvars:
# all variables with dimensions of n_points are fields.
keys = [k for k, v in ncvars.items() if v.dimensions == ('n_points', )]
else:
# all variables with dimensions of 'time', 'range' are fields
keys = [
k for k, v in ncvars.items() if v.dimensions == ('time', 'range')
]
fields = {}
for key in keys:
field_name = filemetadata.get_field_name(key)
if field_name is None:
if exclude_fields is not None and key in exclude_fields:
if key not in include_fields:
continue
if include_fields is None or key in include_fields:
field_name = key
else:
continue
fields[field_name] = _ncvar_to_dict(ncvars[key], delay_field_loading)
if 'ray_n_gates' in ncvars:
shape = (len(ncvars['time']), len(ncvars['range']))
ray_n_gates = ncvars['ray_n_gates'][:]
ray_start_index = ncvars['ray_start_index'][:]
for dic in fields.values():
_unpack_variable_gate_field_dic(dic, shape, ray_n_gates,
ray_start_index)
# 4.5 instrument_parameters sub-convention -> instrument_parameters dict
# 4.6 radar_parameters sub-convention -> instrument_parameters dict
keys = [k for k in _INSTRUMENT_PARAMS_DIMS.keys() if k in ncvars]
instrument_parameters = dict((k, _ncvar_to_dict(ncvars[k])) for k in keys)
if instrument_parameters == {}: # if no parameters set to None
instrument_parameters = None
# 4.7 lidar_parameters sub-convention -> skip
# 4.8 radar_calibration sub-convention -> radar_calibration
keys = _find_all_meta_group_vars(ncvars, 'radar_calibration')
radar_calibration = dict((k, _ncvar_to_dict(ncvars[k])) for k in keys)
if radar_calibration == {}:
radar_calibration = None
# do not close file is field loading is delayed
if not delay_field_loading:
ncobj.close()
return Radar(time,
_range,
fields,
metadata,
scan_type,
latitude,
longitude,
altitude,
sweep_number,
sweep_mode,
fixed_angle,
sweep_start_ray_index,
sweep_end_ray_index,
azimuth,
elevation,
instrument_parameters=instrument_parameters,
radar_calibration=radar_calibration,
altitude_agl=altitude_agl,
scan_rate=scan_rate,
antenna_transition=antenna_transition,
target_scan_rate=target_scan_rate,
rays_are_indexed=rays_are_indexed,
ray_angle_res=ray_angle_res,
rotation=rotation,
tilt=tilt,
roll=roll,
drift=drift,
heading=heading,
pitch=pitch,
georefs_applied=georefs_applied)
def ohc_timeseries(config, streamMap=None, variableMap=None):
"""
Performs analysis of ocean heat content (OHC) from time-series output.
config is an instance of an MpasAnalysisConfigParser containing
configuration options.
config is an instance of MpasAnalysisConfigParser containing configuration
options.
If present, streamMap is a dictionary of MPAS-O stream names that map to
their mpas_analysis counterparts.
If present, variableMap is a dictionary of MPAS-O variable names that map
to their mpas_analysis counterparts.
Author: Xylar Asay-Davis, Milena Veneziani
Last Modified: 01/07/2017
"""
# read parameters from config file
casename = config.get('case', 'casename')
ref_casename_v0 = config.get('case', 'ref_casename_v0')
indir_v0data = config.get('paths', 'ref_archive_v0_ocndir')
compare_with_obs = config.getboolean('ohc_timeseries', 'compare_with_obs')
plots_dir = config.get('paths', 'plots_dir')
yr_offset = config.getint('time', 'yr_offset')
N_movavg = config.getint('ohc_timeseries', 'N_movavg')
regions = config.getExpression('regions', 'regions')
plot_titles = config.getExpression('regions', 'plot_titles')
iregions = config.getExpression('ohc_timeseries', 'regionIndicesToPlot')
indir = config.get('paths', 'archive_dir_ocn')
namelist_filename = config.get('input', 'ocean_namelist_filename')
namelist = NameList(namelist_filename, path=indir)
streams_filename = config.get('input', 'ocean_streams_filename')
streams = StreamsFile(streams_filename, streamsdir=indir)
# Note: input file, not a mesh file because we need dycore specific fields
# such as refBottomDepth and namelist fields such as config_density0, as
# well as simulationStartTime, that are not guaranteed to be in the mesh file.
try:
inputfile = streams.readpath('restart')[0]
except ValueError:
raise IOError(
'No MPAS-O restart file found: need at least one restart file for OHC calculation'
)
# get a list of timeSeriesStats output files from the streams file,
# reading only those that are between the start and end dates
startDate = config.get('time', 'timeseries_start_date')
endDate = config.get('time', 'timeseries_end_date')
streamName = streams.find_stream(streamMap['timeSeriesStats'])
infiles = streams.readpath(streamName,
startDate=startDate,
endDate=endDate)
print 'Reading files {} through {}'.format(infiles[0], infiles[-1])
# Define/read in general variables
print ' Read in depth and compute specific depth indexes...'
f = netcdf_dataset(inputfile, mode='r')
# reference depth [m]
depth = f.variables['refBottomDepth'][:]
# simulation start time
simStartTime = netCDF4.chartostring(f.variables['simulationStartTime'][:])
simStartTime = str(simStartTime)
f.close()
# specific heat [J/(kg*degC)]
cp = namelist.getfloat('config_specific_heat_sea_water')
# [kg/m3]
rho = namelist.getfloat('config_density0')
fac = 1e-22 * rho * cp
k700m = np.where(depth > 700.)[0][0] - 1
k2000m = np.where(depth > 2000.)[0][0] - 1
kbtm = len(depth) - 1
# Load data
print ' Load ocean data...'
varList = [
'avgLayerTemperature', 'sumLayerMaskValue', 'avgLayerArea',
'avgLayerThickness'
]
ds = xr.open_mfdataset(
infiles,
preprocess=lambda x: preprocess_mpas(x,
yearoffset=yr_offset,
timestr='Time',
onlyvars=varList,
varmap=variableMap))
ds = remove_repeated_time_index(ds)
# convert the start and end dates to datetime objects using
# the Date class, which ensures the results are within the
# supported range
time_start = Date(startDate).to_datetime(yr_offset)
time_end = Date(endDate).to_datetime(yr_offset)
# select only the data in the specified range of years
ds = ds.sel(Time=slice(time_start, time_end))
# Select year-1 data and average it (for later computing anomalies)
time_start_yr1 = Date(simStartTime).to_datetime(yr_offset)
if time_start_yr1 < time_start:
startDate_yr1 = simStartTime
endDate_yr1 = startDate_yr1[0:5] + '12-31' + startDate_yr1[10:]
infiles_yr1 = streams.readpath(streamName,
startDate=startDate_yr1,
endDate=endDate_yr1)
ds_yr1 = xr.open_mfdataset(
infiles_yr1,
preprocess=lambda x: preprocess_mpas(x,
yearoffset=yr_offset,
timestr='Time',
onlyvars=varList,
varmap=variableMap))
ds_yr1 = remove_repeated_time_index(ds_yr1)
else:
time_start = datetime.datetime(time_start.year, 1, 1)
time_end = datetime.datetime(time_start.year, 12, 31)
ds_yr1 = ds.sel(Time=slice(time_start, time_end))
mean_yr1 = ds_yr1.mean('Time')
print ' Compute temperature anomalies...'
avgLayerTemperature = ds.avgLayerTemperature
avgLayerTemperature_yr1 = mean_yr1.avgLayerTemperature
avgLayTemp_anomaly = avgLayerTemperature - avgLayerTemperature_yr1
year_start = (pd.to_datetime(ds.Time.min().values)).year
year_end = (pd.to_datetime(ds.Time.max().values)).year
time_start = datetime.datetime(year_start, 1, 1)
time_end = datetime.datetime(year_end, 12, 31)
if ref_casename_v0 != 'None':
print ' Load in OHC for ACMEv0 case...'
infiles_v0data = '{}/OHC.{}.year*.nc'.format(indir_v0data,
ref_casename_v0)
ds_v0 = xr.open_mfdataset(
infiles_v0data,
preprocess=lambda x: preprocess_mpas(x, yearoffset=yr_offset))
ds_v0 = remove_repeated_time_index(ds_v0)
year_end_v0 = (pd.to_datetime(ds_v0.Time.max().values)).year
if year_start <= year_end_v0:
ds_v0_tslice = ds_v0.sel(Time=slice(time_start, time_end))
else:
print ' Warning: v0 time series lies outside current bounds of v1 time series. Skipping it.'
ref_casename_v0 = 'None'
sumLayerMaskValue = ds.sumLayerMaskValue
avgLayerArea = ds.avgLayerArea
avgLayerThickness = ds.avgLayerThickness
print ' Compute OHC and make plots...'
for index in range(len(iregions)):
iregion = iregions[index]
# Compute volume of each layer in the region:
layerArea = sumLayerMaskValue[:, iregion, :] * \
avgLayerArea[:, iregion, :]
layerVolume = layerArea * avgLayerThickness[:, iregion, :]
# Compute OHC:
ohc = layerVolume * avgLayTemp_anomaly[:, iregion, :]
# OHC over 0-bottom depth range:
ohc_tot = ohc.sum('nVertLevels')
ohc_tot = fac * ohc_tot
# OHC over 0-700m depth range:
ohc_700m = fac * ohc[:, 0:k700m].sum('nVertLevels')
# OHC over 700m-2000m depth range:
ohc_2000m = fac * ohc[:, k700m + 1:k2000m].sum('nVertLevels')
# OHC over 2000m-bottom depth range:
ohc_btm = ohc[:, k2000m + 1:kbtm].sum('nVertLevels')
ohc_btm = fac * ohc_btm
title = 'OHC, {}, 0-bottom (thick-), 0-700m (thin-), 700-2000m (--),' \
' 2000m-bottom (-.) \n {}'.format(plot_titles[iregion], casename)
xlabel = 'Time [years]'
ylabel = '[x$10^{22}$ J]'
if ref_casename_v0 != 'None':
figname = '{}/ohc_{}_{}_{}.png'.format(plots_dir, regions[iregion],
casename, ref_casename_v0)
ohc_v0_tot = ds_v0_tslice.ohc_tot
ohc_v0_700m = ds_v0_tslice.ohc_700m
ohc_v0_2000m = ds_v0_tslice.ohc_2000m
ohc_v0_btm = ds_v0_tslice.ohc_btm
title = '{} (r), {} (b)'.format(title, ref_casename_v0)
timeseries_analysis_plot(
config, [
ohc_tot, ohc_700m, ohc_2000m, ohc_btm, ohc_v0_tot,
ohc_v0_700m, ohc_v0_2000m, ohc_v0_btm
],
N_movavg,
title,
xlabel,
ylabel,
figname,
lineStyles=[
'r-', 'r-', 'r--', 'r-.', 'b-', 'b-', 'b--', 'b-.'
],
lineWidths=[2, 1, 1.5, 1.5, 2, 1, 1.5, 1.5])
if not compare_with_obs and ref_casename_v0 == 'None':
figname = '{}/ohc_{}_{}.png'.format(plots_dir, regions[iregion],
casename)
timeseries_analysis_plot(config,
[ohc_tot, ohc_700m, ohc_2000m, ohc_btm],
N_movavg,
title,
xlabel,
ylabel,
figname,
lineStyles=['r-', 'r-', 'r--', 'r-.'],
lineWidths=[2, 1, 1.5, 1.5])
def main():
"""Go Main Go"""
pgconn = get_dbconn('iem')
icursor = pgconn.cursor()
now = datetime.datetime.utcnow() - datetime.timedelta(hours=3)
fn = "/mesonet/data/madis/metar/%s.nc" % (now.strftime("%Y%m%d_%H00"), )
table = "current_qc"
if not os.path.isfile(fn):
sys.exit()
nc = ncopen(fn)
ids = chartostring(nc.variables["stationName"][:])
nc_tmpk = nc.variables["temperature"]
nc_dwpk = nc.variables["dewpoint"]
nc_alti = nc.variables["altimeter"]
tmpkqcd = nc.variables["temperatureQCD"]
dwpkqcd = nc.variables["dewpointQCD"]
altiqcd = nc.variables["altimeterQCD"]
for j in range(ids.shape[0]):
sid = ids[j]
if len(sid) < 4:
continue
if sid[0] == "K":
ts = datetime.datetime(1970, 1, 1) + datetime.timedelta(
seconds=int(nc.variables["timeObs"][j]))
ts = ts.replace(tzinfo=pytz.utc)
(tmpf, tmpf_qc_av, tmpf_qc_sc) = ('Null', 'Null', 'Null')
(dwpf, dwpf_qc_av, dwpf_qc_sc) = ('Null', 'Null', 'Null')
(alti, alti_qc_av, alti_qc_sc) = ('Null', 'Null', 'Null')
if (not np.ma.is_masked(nc_tmpk[j]) and
not np.ma.is_masked(tmpkqcd[j, 0]) and
not np.ma.is_masked(tmpkqcd[j, 6])):
tmpf = check(temperature(nc_tmpk[j], 'K').value('F'))
tmpf_qc_av = figure(nc_tmpk[j], tmpkqcd[j, 0])
tmpf_qc_sc = figure(nc_tmpk[j], tmpkqcd[j, 6])
if (not np.ma.is_masked(nc_dwpk[j]) and
not np.ma.is_masked(dwpkqcd[j, 0]) and
not np.ma.is_masked(dwpkqcd[j, 6])):
dwpf = check(temperature(nc_dwpk[j], 'K').value('F'))
dwpf_qc_av = figure(nc_dwpk[j], dwpkqcd[j, 0])
dwpf_qc_sc = figure(nc_dwpk[j], dwpkqcd[j, 6])
if not np.ma.is_masked(nc_alti[j]):
alti = check(nc_alti[j] / 100.0 * 0.0295298)
alti_qc_av = figure_alti(alti, altiqcd[j, 0] * 0.0295298)
alti_qc_sc = figure_alti(alti, altiqcd[j, 6] * 0.0295298)
sql = """
UPDATE %s SET tmpf = %s, tmpf_qc_av = %s,
tmpf_qc_sc = %s, dwpf = %s, dwpf_qc_av = %s,
dwpf_qc_sc = %s, alti = %s, alti_qc_av = %s,
alti_qc_sc = %s, valid = '%s' WHERE
station = '%s'
""" % (table, tmpf,
tmpf_qc_av, tmpf_qc_sc, dwpf, dwpf_qc_av,
dwpf_qc_sc, alti, alti_qc_av, alti_qc_sc,
ts.strftime("%Y-%m-%d %H:%M+00"), sid[1:])
sql = sql.replace("--", "Null").replace("nan", "Null")
try:
icursor.execute(sql)
except Exception as exp:
print(exp)
print(sql)
nc.close()
icursor.close()
pgconn.commit()
pgconn.close()
def read_kazr(filename, field_names=None, additional_metadata=None,
file_field_names=False, exclude_fields=None,
include_fields=None):
"""
Read K-band ARM Zenith Radar (KAZR) NetCDF ingest data.
Parameters
----------
filename : str
Name of NetCDF file to read data from.
field_names : dict, optional
Dictionary mapping field names in the file names to radar field names.
Unlike other read functions, fields not in this dictionary or having a
value of None are still included in the radar.fields dictionary, to
exclude them use the `exclude_fields` parameter. Fields which are
mapped by this dictionary will be renamed from key to value.
additional_metadata : dict of dicts, optional
This parameter is not used, it is included for uniformity.
file_field_names : bool, optional
True to force the use of the field names from the file in which
case the `field_names` parameter is ignored. False will use to
`field_names` parameter to rename fields.
exclude_fields : list or None, optional
List of fields to exclude from the radar object. This is applied
after the `file_field_names` and `field_names` parameters. Set
to None to include all fields specified by include_fields.
include_fields : list or None, optional
List of fields to include from the radar object. This is applied
after the `file_field_names` and `field_names` parameters. Set
to None to include all fields not specified by exclude_fields.
Returns
-------
radar : Radar
Radar object.
"""
# create metadata retrieval object
filemetadata = FileMetadata(
'cfradial', field_names, additional_metadata, file_field_names,
exclude_fields, include_fields)
# read the data
ncobj = netCDF4.Dataset(filename)
ncvars = ncobj.variables
# 4.1 Global attribute -> move to metadata dictionary
metadata = dict([(k, getattr(ncobj, k)) for k in ncobj.ncattrs()])
metadata['n_gates_vary'] = 'false'
# 4.2 Dimensions (do nothing)
# 4.3 Global variable -> move to metadata dictionary
if 'volume_number' in ncvars:
metadata['volume_number'] = int(ncvars['volume_number'][:])
else:
metadata['volume_number'] = 0
global_vars = {'platform_type': 'fixed', 'instrument_type': 'radar',
'primary_axis': 'axis_z'}
# ignore time_* global variables, these are calculated from the time
# variable when the file is written.
for var, default_value in global_vars.items():
if var in ncvars:
metadata[var] = str(netCDF4.chartostring(ncvars[var][:]))
else:
metadata[var] = default_value
# 4.4 coordinate variables -> create attribute dictionaries
time = cfradial._ncvar_to_dict(ncvars['time'])
_range = cfradial._ncvar_to_dict(ncvars['range'])
# 4.5 Ray dimension variables
# 4.6 Location variables -> create attribute dictionaries
# the only difference in this section to cfradial.read_cfradial is the
# minor variable name differences:
# latitude -> lat
# longitude -> lon
# altitdue -> alt
latitude = cfradial._ncvar_to_dict(ncvars['lat'])
longitude = cfradial._ncvar_to_dict(ncvars['lon'])
altitude = cfradial._ncvar_to_dict(ncvars['alt'])
# 4.7 Sweep variables -> create atrribute dictionaries
# this is the section that needed the most work since the initial NetCDF
# file did not contain any sweep information
sweep_number = filemetadata('sweep_number')
sweep_number['data'] = np.array([0], dtype=np.int32)
sweep_mode = filemetadata('sweep_mode')
sweep_mode['data'] = np.array(['vertical_pointing'], dtype=np.str)
fixed_angle = filemetadata('fixed_angle')
fixed_angle['data'] = np.array([90.0], dtype=np.float32)
sweep_start_ray_index = filemetadata('sweep_start_ray_index')
sweep_start_ray_index['data'] = np.array([0], dtype=np.int32)
sweep_end_ray_index = filemetadata('sweep_end_ray_index')
sweep_end_ray_index['data'] = np.array(
[ncvars['time'].size - 1], dtype=np.int32)
# first sweep mode determines scan_type
# this module is specific to vertically-pointing data
scan_type = 'vpt'
# 4.8 Sensor pointing variables -> create attribute dictionaries
# this section also required some changes since the initial NetCDF did not
# contain any sensor pointing variables
azimuth = filemetadata('azimuth')
azimuth['data'] = 0.0 * np.ones(ncvars['time'].size, dtype=np.float32)
elevation = filemetadata('elevation')
elevation['data'] = 90.0 * np.ones(ncvars['time'].size, dtype=np.float32)
# 4.9 Moving platform geo-reference variables
# 4.10 Moments field data variables -> field attribute dictionary
# all variables with dimensions of 'time', 'range' are fields
keys = [k for k, v in ncvars.items() if
v.dimensions == ('time', 'range')]
fields = {}
for key in keys:
field_name = filemetadata.get_field_name(key)
if field_name is None:
if exclude_fields is not None and key in exclude_fields:
continue
if include_fields is not None and not key in include_fields:
continue
field_name = key
fields[field_name] = cfradial._ncvar_to_dict(ncvars[key])
# 4.5 instrument_parameters sub-convention -> instrument_parameters dict
# this section needed multiple changes and/or additions since the
# instrument parameters were primarily located in the global attributes
# this section is likely still incomplete
omega = float(ncobj.radar_operating_frequency.split()[0])
frequency = filemetadata('frequency')
frequency['data'] = np.array([omega / 1e9], dtype=np.float32)
prt_mode = filemetadata('prt_mode')
prt_mode['data'] = np.array(['fixed'], dtype=np.str)
prf = float(ncobj.pulse_repetition_frequency.split()[0])
prt = filemetadata('prt')
prt['data'] = (1.0 / prf) * np.ones(ncvars['time'].size, dtype=np.float32)
v_nq = float(ncobj.nyquist_velocity.split()[0])
nyquist_velocity = filemetadata('nyquist_velocity')
nyquist_velocity['data'] = v_nq * np.ones(ncvars['time'].size,
dtype=np.float32),
samples = int(ncobj.num_spectral_averages)
n_samples = filemetadata('n_samples')
n_samples['data'] = samples * np.ones(ncvars['time'].size, dtype=np.int32)
# 4.6 radar_parameters sub-convention -> instrument_parameters dict
# this section needed multiple changes and/or additions since the
# radar instrument parameters were primarily located in the global
# attributes
# this section is likely still incomplete
instrument_parameters = {
'frequency': frequency,
'prt_mode': prt_mode,
'prt': prt,
'nyquist_velocity': nyquist_velocity,
'n_samples': n_samples,
}
# 4.7 lidar_parameters sub-convention -> skip
# 4.8 radar_calibration sub-convention -> skip
# close NetCDF object
ncobj.close()
return Radar(
time, _range, fields, metadata, scan_type,
latitude, longitude, altitude,
sweep_number, sweep_mode, fixed_angle, sweep_start_ray_index,
sweep_end_ray_index,
azimuth, elevation,
instrument_parameters=instrument_parameters)
def test_write_two_datasets(self):
# Dictionaries representing data sets
catalogue = Catalogue(
identifier='bob',
datasets=[
CatalogueDataSet(
name='SomeData',
path='/where/is/it',
subsets=[
CatalogueDataSubset(layout=DataStorageFiles(patterns=[
'something_fancy-with % signs and everything.nc',
'and another array member'
]),
matches=[
CatalogueFileEntry(
name='myname.nc',
time=datetime(
2016, 05, 20, 13, 05),
tags=['onetag', 'twotags'],
size=1356,
checksum='ABC129549'),
CatalogueFileEntry(
name='othername.nc',
time=datetime(
2016, 05, 20, 14, 05),
tags=['hello'],
size=None,
checksum=None),
CatalogueFileEntry(
name='bob.nc',
time=datetime(
2016, 05, 20, 15, 05),
tags=None,
size=55552,
checksum='checkitout')
],
value_errors=[
'whatwentwrong?',
'we have a problem'
],
archive_unused=[
'why is this even here :-/'
])
],
non_matching=[
'/bob/is/not/here', '/neither/is/ermintrude'
]),
CatalogueDataSet(name='Other Stuff Here',
path='/somewhere/here',
subsets=[],
non_matching=['harrumph']),
])
# temporary file to use for output
testfile = tempfile.NamedTemporaryFile(suffix='.nc')
# instantiate writer and do output
writer = CatalogueWriterNetCDF()
writer.save(testfile.name, catalogue)
# TEMP: show netcdf contents
# print '\n' + subprocess.Popen(['ncdump', testfile.name], stdout=subprocess.PIPE).communicate()[0]
# TEMP: show cfchecks output
# print '\n' + subprocess.Popen(['cfchecks', testfile.name],
# stdout=subprocess.PIPE).communicate()[0]
# load with netCDF library
netcdf = Dataset(testfile.name)
# Check top level
self.assertEqual('bob', netcdf.identifier)
self.assertTrue('datasets' in netcdf.dimensions)
self.assertEqual(2, len(netcdf.dimensions['datasets']))
self.assertFalse(netcdf.dimensions['datasets'].isunlimited())
self.assertEqual(2, len(netcdf.groups))
# Groups for each data set
group0 = netcdf.groups['datasets_00000000']
group1 = netcdf.groups['datasets_00000001']
# Check dataset group attributes
self.assertEqual('SomeData', group0.getncattr('name'))
self.assertEqual('/where/is/it', group0.getncattr('path'))
# Check numbered subsets
self.assertEqual(1, len(group0.dimensions['subsets']))
self.assertFalse(group0.dimensions['subsets'].isunlimited())
subset0 = group0.groups['subsets_00000000']
# Storage spec
# self.assertEqual('eumopps.storage.DataStorageFiles', subset0.storage)
# Layout subgroup
layout = subset0.groups['layout']
self.assertEqual(2, len(layout.dimensions['patterns']))
self.assertFalse(layout.dimensions['patterns'].isunlimited())
self.assertEqual(2, len(layout.variables['patterns']))
self.assertEqual('something_fancy-with % signs and everything.nc',
chartostring(layout.variables['patterns'][0]))
self.assertEqual('and another array member',
chartostring(layout.variables['patterns'][1]))
# Matches arrays
matches = subset0.groups['matches']
self.assertEqual(3, matches.dimensions['list_count'].size)
self.assertEqual('seconds since 1850-01-01 00:00:00 UTC',
matches.variables['time'].units)
self.assertEqual(5250575100, matches.variables['time'][0])
self.assertEqual('myname.nc',
chartostring(matches.variables['name'][0]))
self.assertEqual('onetag',
chartostring(matches.variables['tags'][0][0]))
self.assertEqual('twotags',
chartostring(matches.variables['tags'][0][1]))
self.assertIsInstance(matches.variables['size'][0], numpy.int64)
self.assertEqual(1356, matches.variables['size'][0])
self.assertEqual('ABC129549',
chartostring(matches.variables['checksum'][0]))
self.assertEqual('othername.nc',
chartostring(matches.variables['name'][1]))
self.assertEqual(5250578700, matches.variables['time'][1])
self.assertEqual('hello',
chartostring(matches.variables['tags'][1][0]))
self.assertEqual('', chartostring(matches.variables['tags'][2][1]))
self.assertTrue(matches.variables['size'][1].mask)
self.assertEqual('', chartostring(matches.variables['checksum'][1]))
self.assertEqual('bob.nc', chartostring(matches.variables['name'][2]))
self.assertEqual(5250582300, matches.variables['time'][2])
self.assertEqual('', chartostring(matches.variables['tags'][2][0]))
self.assertEqual('', chartostring(matches.variables['tags'][2][1]))
self.assertTrue(isinstance(matches.variables['size'][2], numpy.int64))
self.assertEqual(55552, matches.variables['size'][2])
self.assertEqual('checkitout',
chartostring(matches.variables['checksum'][2]))
# Value errors array of strings
self.assertEqual(2, len(subset0.variables['value_errors']))
self.assertEqual('whatwentwrong?',
chartostring(subset0.variables['value_errors'][0]))
self.assertEqual('we have a problem',
chartostring(subset0.variables['value_errors'][1]))
# 'archive_unused' array of strings
self.assertEqual(1, len(subset0.variables['archive_unused']))
self.assertEqual('why is this even here :-/',
chartostring(subset0.variables['archive_unused'][0]))
# Second data set example
self.assertEqual(2, len(group0.dimensions['non_matching']))
self.assertFalse(group0.dimensions['non_matching'].isunlimited())
self.assertEqual('/bob/is/not/here',
chartostring(group0.variables['non_matching'][0]))
self.assertEqual('/neither/is/ermintrude',
chartostring(group0.variables['non_matching'][1]))
self.assertEqual('Other Stuff Here', group1.getncattr('name'))
self.assertEqual('/somewhere/here', group1.getncattr('path'))
self.assertEqual(1, len(group1.dimensions['non_matching']))
self.assertFalse(group1.dimensions['non_matching'].isunlimited())
self.assertEqual('harrumph',
chartostring(group1.variables['non_matching'][0]))
def load_data(self):
indices = self.class4[:, 1].astype(int)
# Expecting specific class4 ID format: "class4_YYYMMDD_*.nc"
with Dataset(
current_app.config["CLASS4_FNAME_PATTERN"] %
(self.class4[0][0][7:11], self.class4[0][0]), 'r') as ds:
self.latitude = ds['latitude'][indices]
self.longitude = ds['longitude'][indices]
self.ids = list(map(str.strip, chartostring(ds['id'][indices])))
self.variables = list(
map(str.strip, chartostring(ds['varname'][:])))
self.variable_units = list(
map(str.strip, chartostring(ds['unitname'][:])))
forecast_data = []
observed_data = []
climatology_data = []
depths = []
for i in indices:
f_data = []
o_data = []
c_data = []
for j in range(0, len(self.variables)):
if self.forecast == 'best':
f_data.append(ds['best_estimate'][i, j, :])
else:
f_data.append(ds['forecast'][i, j,
int(self.forecast), :])
o_data.append(ds['observation'][i, j, :])
c_data.append(ds['climatology'][i, j, :])
forecast_data.append(np.ma.vstack(f_data))
observed_data.append(np.ma.vstack(o_data))
climatology_data.append(np.ma.vstack(c_data))
depths.append(ds['depth'][i, :])
self.depth_unit = ds['depth'].units
self.forecast_data = np.ma.array(forecast_data)
self.observed_data = np.ma.array(observed_data)
self.climatology_data = np.ma.array(climatology_data)
self.depths = np.ma.vstack(depths)
additional_model_data = []
additional_model_names = []
for m in self.models:
additional_model_names.append(m.split("_")[2])
# Expecting specific class4 ID format: "class4_YYYMMDD_*.nc"
with Dataset(
current_app.config["CLASS4_FNAME_PATTERN"] % (m[7:11], m),
'r') as ds:
m_data = []
for i in indices:
data = []
for j in range(0, len(self.variables)):
data.append(ds['best_estimate'][i, j, :])
m_data.append(np.ma.vstack(data))
additional_model_data.append(np.ma.array(m_data))
self.additional_model_data = np.ma.array(additional_model_data)
self.additional_model_names = additional_model_names
def decode_strs(ncin, vname):
"""Reads a bunch of strings and returns as a list"""
return [
x.decode().strip()
for x in netCDF4.chartostring(ncin.variables[vname][:])
]
