def new_scaled_netcdf(ncfile_out, nc_interpol, times_out,
t_unit, station_names=None):
"""
Create netCDF file for scaled results (same for all reanalyses)
Returns the file object so that kernel functions can
successively write variables to it.
"""
if path.isfile(ncfile_out): # raise exception if file exists
raise FileExistsError("File already exists: {}".format(ncfile_out))
# make netCDF outfile, variables are written in kernels
rootgrp = nc_new_file(ncfile_out, fmt='NETCDF4')
rootgrp.source = 'Reanalysis data interpolated and scaled to stations'
# dimensions
n_station = len(nc_interpol.variables['station'][:])
station = rootgrp.createDimension('station', n_station)
time = rootgrp.createDimension('time', len(times_out))
# base variables
time = ncvar_add_time(rootgrp, units=t_unit,
calendar='gregorian', dimensions=('time'),
dtype='i8')
station = ncvar_add_station(rootgrp)
latitude = ncvar_add_latitude(rootgrp)
longitude = ncvar_add_longitude(rootgrp)
height = ncvar_add_ellipsoid_height(rootgrp)
crs = rootgrp.createVariable('crs','i4')
crs.long_name = 'coordinate system'
crs.grid_mapping_name = 'latitude_longitude'
crs.longitude_of_prime_meridian = 0.0
crs.semi_major_axis = 6378137
crs.inverse_flattening = 298.2572236
crs.wkt = 'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.01745329251994328,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]'
# assign base variables
time[:] = times_out
station[:] = nc_interpol.variables['station'][:]
latitude[:] = nc_interpol.variables['latitude'][:]
longitude[:] = nc_interpol.variables['longitude'][:]
height[:] = nc_interpol.variables['height'][:]
# add station names to netcdf
if station_names is not None:
# first convert to character array
names_out = nc.stringtochar(np.array(station_names, 'S32'))
# create space in the netcdf
_ = rootgrp.createDimension('name_strlen', 32)
st = rootgrp.createVariable('station_name', "S1", ('station', 'name_strlen'))
st.standard_name = 'platform_name'
st.units = ''
# add data
st[:] = names_out
return rootgrp
def stringarray_to_chararray(arr, numchars=None):
"""
Convert an string array to a character array with one extra dimension.
Parameters
----------
arr : array
Array with numpy dtype 'SN', where N is the number of characters
in the string.
numchars : int
Number of characters used to represent the string. If numchar > N
the results will be padded on the right with blanks. The default,
None will use N.
Returns
-------
chararr : array
Array with dtype 'S1' and shape = arr.shape + (numchars, ).
"""
carr = netCDF4.stringtochar(arr)
if numchars is None:
return carr
arr_numchars = carr.shape[-1]
if numchars <= arr_numchars:
raise ValueError('numchars must be >= %i' % (arr_numchars))
chararr = np.zeros(arr.shape + (numchars, ), dtype='S1')
chararr[..., :arr_numchars] = carr[:]
return chararr
def write_to_file(filename,data,var,xtime):
if os.path.isfile(filename):
data_nc = netCDF4.Dataset(filename,'a', format='NETCDF3_64BIT_OFFSET')
else:
data_nc = netCDF4.Dataset(filename,'w', format='NETCDF3_64BIT_OFFSET')
# Find dimesions
ncells = data.shape[1]
nsnaps = data.shape[0]
# Declare dimensions
data_nc.createDimension('nCells',ncells)
data_nc.createDimension('StrLen',64)
data_nc.createDimension('Time',None)
# Create time variable
time = data_nc.createVariable('xtime','S1',('Time','StrLen'))
time[:] = netCDF4.stringtochar(xtime)
# Declear variables
time = data_nc.dimensions['Time'].name
ncells = data_nc.dimensions['nCells'].name
data_var = data_nc.createVariable(var,np.float64,(time,ncells))
# Set variables
data_var[:,:] = data[:,:]
data_nc.close()
def write_collection(coll, nc_names, coll_properties, cra, nc):
nc_names.instance_dim = INSTANCE_DIM
coll.to_netcdf(nc, nc_names=nc_names, use_vlen=(not cra))
string_id = nc_names.x_var[:(nc_names.x_var.find('_') + 1)]
for pk, pv in coll_properties.items():
vname = '{}{}'.format(string_id, pk)
try:
dtype = pv.dtype
except AttributeError:
dtype = type(pv[0])
if dtype in (str, unicode):
dtype = 'S1'
if dtype == 'S1':
pv = ['' if v is None else v for v in pv]
max_char = len(max(pv, key=len))
dim_name = vname + '_strlen'
nc.createDimension(dim_name, max_char)
var = nc.createVariable(vname,
dtype,
dimensions=(INSTANCE_DIM, dim_name))
for i in range(len(var)):
var[i] = stringtochar(np.array(pv[i], 'S' + str(max_char)))
else:
var = nc.createVariable(vname, dtype, dimensions=(INSTANCE_DIM, ))
var[:] = pv
def define_forecasts(dataset, names, hours):
dataset.createDimension("forecasts", len(names))
var = dataset.createVariable("forecast_names", "c",
("forecasts", "string_length"))
var[:] = netCDF4.stringtochar(np.array(names, dtype="S64"))
var = dataset.createVariable("forecasts", "f", ("forecasts", ))
var[:] = hours
def writeNC_dims(fn, scalarv, midSoil, midToto, ifcToto, hrus, hru_type):
""" Write <vars>[hru] array in netCDF4 file,<fn> and variable of
<varname> """
print "writing output file"
nc_out = nc4.Dataset(fn, 'w', format='NETCDF4')
# Create dimensions
dim_hru = nc_out.createDimension('hru', len(hrus))
dim_scalarv = nc_out.createDimension('scalarv', scalarv)
dim_midSoil = nc_out.createDimension('midSoil', midSoil)
dim_midToto = nc_out.createDimension('midToto', midToto)
dim_ifcToto = nc_out.createDimension('ifcToto', ifcToto)
# --- Create HRU ID variable (can be either int or string)
if hru_type == 'str':
# string HRU (need to add string length)
max_strlen = 20 # EC
dim_str = nc_out.createDimension('strlen', max_strlen)
hruId = nc_out.createVariable('hruId', 'S1', ('hru', 'strlen'))
hruId[:] = nc4.stringtochar(
np.asarray(hrus, dtype='S{}'.format(max_strlen)))
else:
# integer HRU
hruId = nc_out.createVariable('hruId', 'i8', ('hru', )) # edited EC
hruId[:] = np.asarray(hrus, dtype='int')
# add attribute
hruId.long_name = 'USGS HUC12 ID'
return nc_out
def define(dataset, metrics, areas, times, units):
dataset.createDimension("time", len(times))
dataset.createDimension("surface", 1)
dataset.createDimension("metrics", len(metrics))
dataset.createDimension("areas", len(areas))
dataset.createDimension("string_length", 64)
var = dataset.createVariable("metric_names", "c",
("metrics", "string_length"))
var[:] = netCDF4.stringtochar(np.array(metrics, dtype="S64"))
var = dataset.createVariable("area_names", "c",
("areas", "string_length"))
var[:] = netCDF4.stringtochar(np.array(areas, dtype="S64"))
var = dataset.createVariable("time", "d", ("time", ))
var.units = units
if len(times) > 0:
var[:] = netCDF4.date2num(times, units=units)
def writeNC_dims(fn, hrus, hru_type):
""" Write <vars>[hru] array in netCDF4 file,<fn> and variable of
<varname> """
print "starting output file"
nc_out = nc4.Dataset(fn, 'w', format='NETCDF4')
# Create dimensions
#dim_hru = nc_out.createDimension('hru', len(hrus))
dim_gru = nc_out.createDimension('gru', len(hrus))
# --- Create HRU and GRU ID variables (can be either int or string)
# --- currently these are all the same
if hru_type == 'str':
# string HRU (need to add string length)
max_strlen = 20
dim_str = nc_out.createDimension('strlen', max_strlen)
gruId = nc_out.createVariable('gruId', 'S1', ('gru', 'strlen'))
gruId[:] = nc4.stringtochar(
np.asarray(hrus, dtype='S{}'.format(max_strlen)))
else:
gruId = nc_out.createVariable('gruId', 'i8', ('gru', ))
gruId[:] = np.asarray(hrus, dtype='int')
# add attributes
gruId.long_name = 'ID of group of response unit (GRU) -- USGS HUC12 ID'
return nc_out
def stringarray_to_chararray(arr, numchars=None):
"""
Convert an string array to a character array with one extra dimension.
Parameters
----------
arr : array
Array with numpy dtype 'SN', where N is the number of characters
in the string.
numchars : int
Number of characters used to represent the string. If numchar > N
the results will be padded on the right with blanks. The default,
None will use N.
Returns
-------
chararr : array
Array with dtype 'S1' and shape = arr.shape + (numchars, ).
"""
carr = netCDF4.stringtochar(arr)
if numchars is None:
return carr
arr_numchars = carr.shape[-1]
if numchars <= arr_numchars:
raise ValueError('numchars must be >= %i' % (arr_numchars))
chararr = np.zeros(arr.shape + (numchars, ), dtype='S1')
chararr[..., :arr_numchars] = carr[:]
return chararr
def write_out_fixed_forcing(filenameOut, nx, ny, ugx, ugy):
fileOut = Dataset(filenameOut, "w", format="NETCDF3_CLASSIC")
fileOut.climatology = 1
fileOut.createDimension("strLen", 64)
fileOut.createDimension("Time", 1)
fileOut.createDimension("nx", nx)
fileOut.createDimension("ny", ny)
var = fileOut.createVariable("Time", "c", dimensions=["Time", "strLen"])
year = 0
month = 1
day = 1
hour = 0
minute = 0
second = 0
timeStr = "%4.4i-%2.2i-%2.2i_%2.2i:%2.2i:%2.2i" % (year, month, day, hour,
minute, second)
var[0, 0:19] = netCDF4.stringtochar(np.array(timeStr, 'S19'))
varx = fileOut.createVariable("xAtmWind",
"d",
dimensions=["Time", "nx", "ny"])
vary = fileOut.createVariable("yAtmWind",
"d",
dimensions=["Time", "nx", "ny"])
varx[0, :, :] = ugx[:, :]
vary[0, :, :] = ugy[:, :]
fileOut.close()
def main():
# Command line arguments
parser = argparse.ArgumentParser(description=
'Changes the value of a named char variable in a netcdf file.',
epilog='Written by A.Adcroft, 2013.')
parser.add_argument('filename', type=str,
help='netcdf file to modify.')
parser.add_argument('variable', type=str,
help='Name of char variable to change.')
parser.add_argument('value', type=str,
help='Contents to change string to.')
optCmdLineArgs = parser.parse_args()
rg = Dataset(optCmdLineArgs.filename, 'a' );
if optCmdLineArgs.variable in rg.variables:
var = rg.variables[optCmdLineArgs.variable]
dat = np.empty(1,'S'+repr(len(var)))
dat[0] = optCmdLineArgs.value
dc = stringtochar(dat)
var[:] = dc
else: error('"'+optCmdLineArgs.variable+'" was not found in "'+optCmdLineArgs.filename+'".')
rg.close()
print 'File "%s" updated.'%(optCmdLineArgs.filename)
def main():
# Command line arguments
parser = argparse.ArgumentParser(
description=
'Changes the value of a named char variable in a netcdf file.',
epilog='Written by A.Adcroft, 2013.')
parser.add_argument('filename', type=str, help='netcdf file to modify.')
parser.add_argument('variable',
type=str,
help='Name of char variable to change.')
parser.add_argument('value',
type=str,
help='Contents to change string to.')
optCmdLineArgs = parser.parse_args()
rg = Dataset(optCmdLineArgs.filename, 'a')
if optCmdLineArgs.variable in rg.variables:
var = rg.variables[optCmdLineArgs.variable]
dat = np.empty(1, 'S' + repr(len(var)))
dat[0] = optCmdLineArgs.value
dc = stringtochar(dat)
var[:] = dc
else:
error('"' + optCmdLineArgs.variable + '" was not found in "' +
optCmdLineArgs.filename + '".')
rg.close()
print 'File "%s" updated.' % (optCmdLineArgs.filename)
def writeNetCDFData(fn, prate, times, hrus):
""" Write <vars>[time,hru] array in netCDF4 file,<fn> and variable of <varname> """
ncfile = nc4.Dataset(fn,'w',format='NETCDF4')
# Data should be 2D [time x hru]
dim2size = prate.shape[1]
dim_1 = ncfile.createDimension('time', None) # time axis - record dimension
dim_2 = ncfile.createDimension('hru', dim2size) # hru axis
max_strlen = 12 # EAC
dim_3 = ncfile.createDimension('strlen', max_strlen) # string length for hrus EAC adde
# Define dimensions variable to hold the var
t = ncfile.createVariable('time', 'i4', ('time', ))
hru = ncfile.createVariable('hru', 'S1', ('hru', 'strlen')) # edited EAC
# Define 2D variables to hold the data
# AWW: changed variable name to match use for cfsv2
map = ncfile.createVariable('prate','f4',('time','hru',))
# tavg_2m = ncfile.createVariable('tmp_2m','f4',('time','hru',))
# Populate netcdf file variables
t[:] = times # NM added
hru[:] = nc4.stringtochar(np.asarray(hrus, dtype='S{}'.format(max_strlen))) # EAC
map[:,:] = prate # for mean area precip (map), etc.
# tavg_2m[:,:] = tmp_2m
# Attributes (should take more from input netcdf)
# set time axis
t.time_origin = '1970-JAN-01 00:00:00'
t.title = 'Time'
t.long_name = 'Time axis'
t.units = 'seconds since 1970-01-01 00:00:00'
t.calendar = 'Gregorian'
t.axis = 'T'
# set hru axis
hru.title = 'HRU ID'
hru.long_name = 'HRU axis'
hru.axis = 'X'
# set variable attributes
# AW - commented out vars above and used temp_2m
map.associate = 'time hru'
map.units = 'kg/m^2/s'
map.setMissing = '1.e+20'
map.axis = 'TX'
#tavg_2m.associate = 'time hru'
#tavg_2m.units = 'degrees C'
#tavg_2m.setMissing = '1.e+20'
#tavg_2m.axis = 'TX'
# Write basic global attribute
ncfile.history = 'Created ' + time.ctime(time.time())
ncfile.source = os.path.dirname(os.path.abspath(__file__))+__file__[1:]
ncfile.close()
def str_list_to_char_arr(slist, maxlen) : #--------------------------------------------------------------------------------------------------- """ Convert a list of regular python strings to a numpy character array of type '|S1', which is what is required by the netCDF4 module. The maximum length of each string in the output netcdf array is defined by maxlen. It's usually the last dimension in the variable declaration. """ stype = 'S%d' % maxlen tarr = np.array(slist, dtype=stype) return nc4.stringtochar(tarr)
def str_list_to_char_arr(slist, maxlen):
#---------------------------------------------------------------------------------------------------
"""
Convert a list of regular python strings to a numpy character array of type '|S1', which is what
is required by the netCDF4 module. The maximum length of each string in the output netcdf array
is defined by maxlen. It's usually the last dimension in the variable declaration.
"""
stype = 'S%d' % maxlen
tarr = np.array(slist, dtype=stype)
return nc4.stringtochar(tarr)
def setUp(self):
self.file = FILE_NAME
nc = Dataset(FILE_NAME, 'w', format=FILE_FORMAT)
nc.createDimension('n1', None)
nc.createDimension('n2', n2)
nc.createDimension('nchar', nchar)
v = nc.createVariable('strings', 'S1', ('n1', 'n2', 'nchar'))
for nrec in range(nrecs):
datac = stringtochar(data)
v[nrec] = datac[nrec]
nc.close()
def setUp(self):
self.file = FILE_NAME
nc = Dataset(FILE_NAME,'w',format=FILE_FORMAT)
nc.createDimension('n1',None)
nc.createDimension('n2',n2)
nc.createDimension('nchar',nchar)
v = nc.createVariable('strings','S1',('n1','n2','nchar'))
for nrec in range(nrecs):
datac = stringtochar(data)
v[nrec] = datac[nrec]
nc.close()
def setUp(self):
self.file = FILE_NAME
nc = Dataset(FILE_NAME, "w", format=FILE_FORMAT)
nc.createDimension("n1", None)
nc.createDimension("n2", n2)
nc.createDimension("nchar", nchar)
v = nc.createVariable("strings", "S1", ("n1", "n2", "nchar"))
for nrec in range(nrecs):
datac = stringtochar(data)
v[nrec] = datac[nrec]
nc.close()
def __setitem__(self, idx, value):
attributes = self.attributes
type = attributes['ftype']
if type == "M": value = value.astype("datetime64[ms]").astype("f8")
if type == 'S1':
if not self.isChar:
value = stringtochar(
np.array(value).astype("S{}".format(
self.variable.shape[1])))
value = setT(self.tattributes, value)
self.variable.__setitem__(idx, value)
def setstr(self, k, s, dim=128):
name = self[k]
nc = self.nc_node
if not isinstance(dim, str) or dim is None:
if dim is None:
dim = len(s)
dim_name = f'{name}_slen'
nc.createDimension(dim_name, dim)
else:
dim_name = dim
v = nc.createVariable(name, 'S1', (dim_name, ))
v[:] = netCDF4.stringtochar(np.array([s], dtype=f'S{dim}'))
return v
def writeNC_trialParam(fn, var_out, varname, attNames, attContents, hrus, hru_type):
""" Write <vars>[hru] array in netCDF4 file,<fn> and variable of
<varname> """
print "writing output file"
ncfile = nc4.Dataset(fn, 'w', format='NETCDF4')
# Data should be 1D [hru]
dimSize = var_out.shape[0]
dim_1 = ncfile.createDimension('hru', dimSize) # hru axis
# ==== Populate netcdf file variables =====
# --- Create HRU ID variable (can be either int or string)
if hru_type == 'str':
# string HRU (need to add string length)
max_strlen = 20 # EC
dim_str = ncfile.createDimension('strlen', max_strlen)
hruId = ncfile.createVariable('hruID', 'S1', ('hru', 'strlen'))
hruId[:] = nc4.stringtochar(np.asarray(hrus,
dtype='S{}'.format(max_strlen)))
else:
# integer HRU
hruId = ncfile.createVariable('hruID', 'i8', ('hru', )) # edited EC
hruId[:] = np.asarray(hrus, dtype='int')
#hruId[:] = np.asarray(hrus, dtype='i8') # AW for HUCs
# add attribute
hruId.long_name = 'USGS HUC12 ID'
# geophysical var data and attributes from input file
print "adding data & attributes"
ncvar = ncfile.createVariable(varname,var_out.dtype,('hru',),fill_value='-999.0')
ncvar[:] = var_out # store mean areal data in netcdf file
# for i in range(len(attNames)):
# ncvar.setncatts({attNames[i]: attContents[i]})
# -- from another script
# Copy the variable attributes
# dsout.variables[v_name].setncatts({k: varin.getncattr(k) for k in varin.ncattrs()})
# Create the variables in the file
#for v_name, varin in dsin.variables.iteritems():
# dsout.createVariable(v_name, varin.datatype, varin.dimensions)
# Copy the variables values
#dsout.variables[v_name][:] = uuu[:]
# -- end ... could be useful syntax --
# Write basic global attribute
ncfile.history = 'Created ' + time.ctime(time.time())
ncfile.source = os.path.dirname(os.path.abspath(__file__))+__file__[1:]
ncfile.close()
def _create_ncvar(dic, dataset, name, dimensions):
"""
Create and fill a Variable in a netCDF Dataset object.
Parameters
----------
dic : dict
Radar dictionary to containing variable data and meta-data
dataset : Dataset
NetCDF dataset to create variable in.
name : str
Name of variable to create.
dimension : tuple of str
Dimension of variable.
"""
# create array from list, etc.
data = dic['data']
if isinstance(data, np.ndarray) is not True:
print "Warning, converting non-array to array:", name
data = np.array(data)
# convert string array to character arrays
if data.dtype.char is 'S' and data.dtype != 'S1':
import netCDF4
data = netCDF4.stringtochar(data)
# create the dataset variable
if 'least_significant_digit' in dic:
lsd = dic['least_significant_digit']
else:
lsd = None
if "_FillValue" in dic:
fill_value = dic['_FillValue']
else:
fill_value = None
ncvar = dataset.createVariable(name, data.dtype, dimensions,
zlib=True, least_significant_digit=lsd,
fill_value=fill_value)
# set all attributes
for key, value in dic.iteritems():
if key not in ['data', '_FillValue']:
ncvar.setncattr(key, value)
# set the data
if data.shape == ():
data.shape = (1,)
ncvar[:] = data[:]
def setUp(self):
self.file = FILE_NAME
nc = Dataset(FILE_NAME,'w',format=FILE_FORMAT)
nc.createDimension('n1',None)
nc.createDimension('n2',n2)
nc.createDimension('nchar',nchar)
v = nc.createVariable('strings','S1',('n1','n2','nchar'))
v2 = nc.createVariable('strings2','S1',('n1','n2','nchar'))
# if _Encoding set, string array should automatically be converted
# to a char array and vice-versan
v2._Encoding = 'ascii'
v3 = nc.createVariable('strings3','S1',('n1','n2','nchar'))
v3._Encoding = 'ascii'
for nrec in range(nrecs):
datac = stringtochar(data,encoding='ascii')
v[nrec] = datac[nrec]
v2[:-1] = data[:-1]
v2[-1] = data[-1]
v2[-1,-1] = data[-1,-1] # write single element
v2[-1,-1] = data[-1,-1].tostring() # write single python string
# _Encoding should be ignored if an array of characters is specified
v3[:] = stringtochar(data, encoding='ascii')
nc.close()
def setUp(self):
self.file = FILE_NAME
nc = Dataset(FILE_NAME, 'w', format=FILE_FORMAT)
nc.createDimension('n1', None)
nc.createDimension('n2', n2)
nc.createDimension('nchar', nchar)
v = nc.createVariable('strings', 'S1', ('n1', 'n2', 'nchar'))
v2 = nc.createVariable('strings2', 'S1', ('n1', 'n2', 'nchar'))
# if _Encoding set, string array should automatically be converted
# to a char array and vice-versan
v2._Encoding = 'ascii'
v3 = nc.createVariable('strings3', 'S1', ('n1', 'n2', 'nchar'))
v3._Encoding = 'ascii'
for nrec in range(nrecs):
datac = stringtochar(data, encoding='ascii')
v[nrec] = datac[nrec]
v2[:-1] = data[:-1]
v2[-1] = data[-1]
v2[-1, -1] = data[-1, -1] # write single element
v2[-1, -1] = data[-1, -1].tostring() # write single python string
# _Encoding should be ignored if an array of characters is specified
v3[:] = stringtochar(data, encoding='ascii')
nc.close()
def create_header(self, specobj):
for varname, attrname in header_variables.items():
#print varname
if varname == 'Header.Mode.Bitcode':
bcode = getattr(specobj.mode, attrname)
self.nc.variables[
'Header.Mode.Bitcode'][:len(bcode)] = netCDF4.stringtochar(
numpy.array([bcode]))
elif varname == 'Header.Mode.roach_num':
self.nc.variables[varname][:] = getattr(specobj, attrname)
elif varname == 'Header.Telescope.source_name':
sname = specobj.source_name
self.nc.variables['Header.Telescope.source_name'][:len(
sname)] = netCDF4.stringtochar(numpy.array([sname]))
elif varname == 'Header.Telescope.obspgm':
obspgm = specobj.obspgm
self.nc.variables['Header.Telescope.obspgm'][:len(
obspgm)] = netCDF4.stringtochar(numpy.array([obspgm]))
elif varname in ('Header.Telescope.ObsNum',
'Header.Telescope.SubObsNum',
'Header.Telescope.ScanNum'):
self.nc.variables[varname][:] = getattr(specobj, attrname)
else:
self.nc.variables[varname][:] = getattr(specobj.mode, attrname)
def addImage2netcdf(image, ncFile, granule, imgTime):
dataset = nc.Dataset(ncFile, 'a')
### Updating time
time = dataset.variables['time']
name = dataset.variables['granule']
data = dataset.variables['image']
numGranules = time.shape[0]
time[numGranules] = nc.date2num(imgTime,
units=time.units,
calendar=time.calendar)
name[numGranules] = nc.stringtochar(np.array(granule, 'S100'))
data[numGranules, :, :] = image
dataset.close()
def __setitem__(self, idx, value):
"""
Set data based on the query
"""
vname, idx = self.__checkVariable(idx)
if self.variables[vname]['type'] == "S":
value = stringtochar(
np.array(value).astype("S{}".format(self.shape[1])))
# if self.variables[vname]['type']!="S":
value = checkValue(value, idx, self.shape)
if isQuickSet(idx, self.ndata, self.master, self.child):
self._quickSet(vname, self._getPartIndex(idx), value)
else:
self.__subrun(vname, idx, value=value)
def strlist_to_netcdf(a, dim=None):
"""Convert a list of strings to a netCDF4 chararray.
Arguments:
a: a list of strings
dim (optionnal): netCDF4 dimensions
Returns:
netCDF4 chararray
"""
if dim is None:
dim = (len(a), max(len(s) for s in a))
else:
dim = tuple(len(d) for d in dim)
array = np.chararray(dim)
array[:] = [list(s.ljust(dim[1])) for s in a]
return netCDF4.stringtochar(array)
def OpenO(self, nc_file_name):
"""Opens NetCDF file for output.
Opens a NetCDF file of nc_file_name for writing (output).
Parameters
----------
nc_file_name : str
Name of netCDF file you wish to write to.
"""
if self.__openI:
print('ERROR: Cannot set input and output in same instance')
sys.exit(-1)
if self.__openO:
print('ERROR: Cannot open file twice!')
sys.exit(-1)
# Conventions
self.__nc_file = Dataset(nc_file_name, 'w', format='NETCDF4')
self.__nc_file.Conventions = 'AMBER'
self.__nc_file.ConventionVersion = '1.0'
self.__nc_file.program = 'LammpsPy.py'
self.__nc_file.programVersion = '1.0'
# Dimensions except number of particles
self.__nc_file.createDimension('frame', 0)
self.__nc_file.createDimension('spatial', 3)
self.__nc_file.createDimension('cell_spatial', 3)
self.__nc_file.createDimension('cell_angular', 3)
self.__nc_file.createDimension('label', 5)
# Preliminary variables
char_sp_var = self.__nc_file.createVariable('char spatial', 'S1',
('spatial'))
char_sp_var[:] = ['x', 'y', 'z']
char_csp_var = self.__nc_file.createVariable('char cell_spatial', 'S1',
('cell_spatial'))
char_csp_var[:] = ['a', 'b', 'c']
char_sp_var = self.__nc_file.createVariable('char cell_angular', 'S1',
('cell_angular', 'label'))
char_sp_var[:] = stringtochar(np.array(['alpha', 'beta', 'gamma']))
# notes OpenO has been run
self.__openO = True
def GetDim(NCData,var,shape,DimDict,i,istime):
output=[]
#grab type
try:
val_type=str(var.dtype)
except AttributeError:
val_type=type(var)
#grab dimension
for dim in range(0,i): #loop on the dimensions
if type(shape[0])==int:
try:
output=output+[str(DimDict[shape[dim]])] #test if the dimension allready exist
except KeyError: #if not create it
if (shape[dim])>1:
index=len(DimDict)+1
NewDim=NCData.createDimension('Dimension'+str(index),(shape[dim]))
DimDict[len(NewDim)]='Dimension'+str(index)
output=output+[str(DimDict[shape[dim]])]
print('Defining dimension ' +'Dimension'+str(index))
elif type(shape[0])==str:#dealling with a dictionnary
try:
output=[str(DimDict[numpy.shape(shape)[0]])]+['DictDim']
except KeyError:
index=len(DimDict)+1
NewDim=NCData.createDimension('Dimension'+str(index),numpy.shape(shape)[0])
DimDict[len(NewDim)]='Dimension'+str(index)
output=[str(DimDict[numpy.shape(dict.keys(var))[0]])]+['Dimension6']
print('Defining dimension ' +'Dimension'+str(index))
break
if istime:
output=output+['Dimension4']
#dealing with char and not string as we should so we need to had a string length
if val_type=='bool' or val_type==collections.OrderedDict or val_type==list:
charvar=stringtochar(numpy.array(var))
stringlength=charvar.shape[charvar.ndim-1]
try:
output=output+[str(DimDict[stringlength])] #test if the dimension allready exist
except KeyError: #if not create it
if (shape[dim])>1:
index=len(DimDict)+1
NewDim=NCData.createDimension('Dimension'+str(index),(stringlength))
DimDict[len(NewDim)]='Dimension'+str(index)
output=output+[str(DimDict[stringlength])]
print('Defining dimension ' +'Dimension'+str(index))
return tuple(output), DimDict
def create_netcdf_label():
execute_create_netcdf_label_start_time = datetime.datetime.now()
# extract one hot encoding column
labels = [col[0] for col in listOfDiacritizedCharacter]
# convert unicode to string
labels = [str(x) for x in labels]
# convert array of string to array of char to be compatible with NETCDF
# you will find strange values, but do not worry, it will exported correctly
labels = netcdf_helpers.stringtochar(np.array(labels))
global purified_labels
purified_labels = []
purified_labels = labels
execute_create_netcdf_label_end_time = datetime.datetime.now()
print "create_netcdf_label takes : ", execute_create_netcdf_label_end_time - execute_create_netcdf_label_start_time
def process(self):
"""
Run all relevant processes and save data. Each kernel processes one
variable and adds it to the netCDF file.
"""
if not path.isdir(path.dirname(self.output_file)):
makedirs(path.dirname(self.outfile))
self.rg = new_scaled_netcdf(self.output_file,
self.nc_pl,
self.times_out_nc,
t_unit=self.scaled_t_units)
# add station names to netcdf
# first convert to character array
names_out = nc.stringtochar(
np.array(self.stations['station_name'], 'S32'))
# create space in the netcdf
_ = self.rg.createDimension('name_strlen', 32)
st = self.rg.createVariable('station_name', "S1",
('station', 'name_strlen'))
st.standard_name = 'platform_name'
st.units = ''
# add data
st[:] = names_out
# iterate through kernels and start process
for kernel_name in self.kernels:
if hasattr(self, kernel_name):
print(kernel_name)
getattr(self, kernel_name)()
else:
logger.error(f"Missing kernel {kernel_name}")
# self.conv_geotop()
# close netCDF files
self.rg.close()
self.nc_pl.close()
self.nc_sf.close()
self.nc_sa.close()
def save_string_list(group, membername, stringarray, dimensionname):
# Compute length of strings
max_length = max([ len(element) if (element is not None) else 0 for element in stringarray ]) + 1
# Name used to store length of strings
lengthname = STRING_LENGTH_DIMENSION_FORMAT.format(membername)
# Build new variable
group.createDimension(lengthname, max_length)
group.createVariable(membername, 'S1', [dimensionname, lengthname])
# Get rid of the behaviour in which None elements map to the string 'None'
# (we want them just to be empty strings)
stringarray_filtered = [ s if (s is not None ) else '' for s in stringarray ]
# Convert to NumPy at max_length (which includes a null terminator)
stringrray_numpy = numpy.array(stringarray_filtered, dtype=('S' + str(max_length)))
# Set in output
group.variables[membername][:] = stringtochar(stringrray_numpy)
def AddVariable(self,
name,
dimnames,
values,
dtype='f4',
attr={},
indent=''):
"""
Add variable with data.
Optional arguments:
dtype : 'f4' | 'i4' | ...
if not defined, value.dtype is used
attr : optional attributes
indent : specifiy whitespace to indent logging messages
"""
# modules:
import logging
# info ...
logging.debug(indent + 'add variable "%s" ...' % name)
# data type:
if dtype == None: dtype = values.dtype
# create::
varid = self.ncid.createVariable(name, dtype, dimnames)
# fill data:
if values != None:
if dtype == 'S1':
varid[:] = netCDF4.stringtochar(values)
else:
varid[:] = values
#endif
#endif
# attributes:
for key in attr.keys():
varid.setncattr(key, attr[key])
def addvar(self, tag, typ, dim, data=None, attributes=None):
thistype = {'float': 'f', 'double': 'd', 'char': 'S1', 'int': 'i'}[typ]
if dim == None:
vlvar = self.nc.createVariable(tag, thistype)
else:
if type(dim) == type(()):
vlvar = self.nc.createVariable(tag, thistype, dim)
else:
vlvar = self.nc.createVariable(tag, thistype, (dim, ))
if attributes != None:
for k in attributes:
vlvar.setncattr(k, attributes[k])
if data != None:
if typ == 'char':
print dim
print data
x = numpy.chararray(self.dimensions[dim[0]],
itemsize=self.dimensions[dim[1]])
x[:] = data
self.nc.variables[tag][:] = netCDF4.stringtochar(x)
else:
self.nc.variables[tag][:] = data
def write_means_nc(configdict, RMSE, MAE, counts):
exists = False
# first check to see if file exists
exists = True if os.path.isfile(configdict['outfile']) else False
if not exists:
with nc.Dataset(configdict['outfile'], mode='w') as outfile:
# create dimensions and variables
time_dim = outfile.createDimension("time", None)
str_dim = outfile.createDimension("timestrlen", 10)
timestamp = outfile.createVariable("timestamp", "S1",
("time", "timestrlen"))
timestamp.long_name = "valid time in YYYYMMDDHH"
for vname in configdict['variables']:
varstr = vname + '_RMSE'
outvar1 = outfile.createVariable(varstr, "f4", ("time"))
varstr = vname + '_MAE'
outvar2 = outfile.createVariable(varstr, "f4", ("time"))
varstr = vname + '_counts'
outvar3 = outfile.createVariable(varstr, "i4", ("time"))
with nc.Dataset(configdict['outfile'], mode='a') as outfile:
# determine index in unlimited dimension
timestamp = outfile.variables['timestamp']
idx = timestamp.shape[0]
# write out data to file
tmpstr = np.array(configdict['timestamp'].strftime('%Y%m%d%H'),
dtype="S10")
timestamp[idx,:] = nc.stringtochar(tmpstr)
for vname in configdict['variables']:
varstr = vname + '_RMSE'
outvar = outfile.variables[varstr]
outvar[idx] = RMSE[vname]
varstr = vname + '_MAE'
outvar = outfile.variables[varstr]
outvar[idx] = MAE[vname]
varstr = vname + '_counts'
outvar = outfile.variables[varstr]
outvar[idx] = counts[vname]
def SaveFile(filename, y, lon, lat, names):
""" Saves y into netcdf file under filename in datadirectory """
datadir = GetDir()
savefile = datadir + filename + '.nc'
print(savefile)
# Copy dimension and information for meta data
dataset = netCDF4.Dataset(savefile, 'w', format='NETCDF4_CLASSIC')
print(dataset)
# create dimensions
lonlat = dataset.createDimension('lonlat', len(lon) * len(lat))
strlen = dataset.createDimension('strlen', None)
samples = dataset.createDimension('samples', len(names))
# create variables
samples_list = dataset.createVariable('samples_list', 'S1',
('samples', 'strlen'))
sample_str = netCDF4.stringtochar(np.array(names))
samples_list[:] = sample_str
y_pred = dataset.createVariable('y_pred', np.float64,
('samples', 'lonlat'))
y_pred[:, :] = y
def write_out_time_varying_forcing(filenameOut, nx, ny, ugx, ugy):
# time varying forcing
fileOut = Dataset(filenameOut, "w", format="NETCDF3_CLASSIC")
fileOut.climatology = 0
fileOut.createDimension("strLen", 64)
fileOut.createDimension("Time", 10)
fileOut.createDimension("nx", nx)
fileOut.createDimension("ny", ny)
var = fileOut.createVariable("Time", "c", dimensions=["Time", "strLen"])
year = 1
month = 1
hour = 0
minute = 0
second = 0
for day in range(0, 10):
timeStr = "%4.4i-%2.2i-%2.2i_%2.2i:%2.2i:%2.2i" % (
year, month, day + 1, hour, minute, second)
var[day, 0:19] = netCDF4.stringtochar(np.array(timeStr, 'S19'))
varx = fileOut.createVariable("xAtmWind",
"d",
dimensions=["Time", "nx", "ny"])
vary = fileOut.createVariable("yAtmWind",
"d",
dimensions=["Time", "nx", "ny"])
for day in range(0, 10):
varx[day, :, :] = ugx[:, :] * (float(2 - day) / 2.0)
vary[day, :, :] = ugy[:, :] * (float(2 - day) / 2.0)
fileOut.close()
def write_collection(coll, nc_names, coll_properties, cra, nc):
nc_names.instance_dim = INSTANCE_DIM
coll.to_netcdf(nc, nc_names=nc_names, use_vlen=(not cra))
string_id = nc_names.x_var[:(nc_names.x_var.find('_') + 1)]
for pk, pv in coll_properties.items():
vname = '{}{}'.format(string_id, pk)
try:
dtype = pv.dtype
except AttributeError:
dtype = type(pv[0])
if dtype in (str, unicode):
dtype = 'S1'
if dtype == 'S1':
pv = ['' if v is None else v for v in pv]
max_char = len(max(pv, key=len))
dim_name = vname + '_strlen'
nc.createDimension(dim_name, max_char)
var = nc.createVariable(
vname, dtype, dimensions=(INSTANCE_DIM, dim_name))
for i in range(len(var)):
var[i] = stringtochar(np.array(pv[i], 'S' + str(max_char)))
else:
var = nc.createVariable(vname, dtype, dimensions=(INSTANCE_DIM,))
var[:] = pv
def control_to_netcdf(controlFile, outputFileName, **kwargs):
"""
Converts the input file into netCDF format.
Args:
fileHandle: is the input file
"""
start = time.time()
index = 0
controlParameterNames = []
controlParameterDataTypes = []
# Initialize new dataset
ncfile = netCDF4.Dataset(outputFileName, mode='w')
fileHandle = open(controlFile, 'r')
for line in fileHandle:
if "####" in line:
nameOfControlParameters = fileHandle.next().strip()
name = nameOfControlParameters
controlParameterNames.append(nameOfControlParameters)
numberOfParameterValues = int(fileHandle.next().strip())
# Initialize dimension
ncfile.createDimension(nameOfControlParameters, numberOfParameterValues)
nameOfControlParameters = []
valueType = int(fileHandle.next().strip())
dataTypeOfParameterValues = _get_datatype(valueType)
controlParameterDataTypes.append(dataTypeOfParameterValues)
for i in range(numberOfParameterValues):
_store_parameter_values_in_a_list(fileHandle, valueType, nameOfControlParameters)
if valueType == 4:
lengthOfLongestWord = max(len(word) for word in nameOfControlParameters)
ncfile.createDimension('length_of_longest_string_in_'+name, lengthOfLongestWord)
# Define variable
var = ncfile.createVariable(controlParameterNames[index], controlParameterDataTypes[index],
(controlParameterNames[index], 'length_of_longest_string_in_'+name))
# Write data for parameters with string values
var[:] = netCDF4.stringtochar(numpy.array(nameOfControlParameters))
else:
var = ncfile.createVariable(controlParameterNames[index], controlParameterDataTypes[index],
(controlParameterNames[index],))
var[:] = nameOfControlParameters
index += 1
# Global attributes
fileHandle = open(controlFile, 'r')
ncfile.title = fileHandle.next().strip()
# Close the 'ncfile' object
ncfile.close()
def write_dsg(file,ofile,my_series):
nc = netCDF4.Dataset(file)
vars=nc.variables.keys()
coord_vars = ['time','time2','depth','lat','lon']
# find data variables by removing coordinate variables from the variable list
data_vars = [var for var in vars if var not in coord_vars]
nt = len(nc.dimensions['time'])
nz = len(nc.dimensions['depth'])
nco = netCDF4.Dataset(ofile,'w')
# create dimensions
nco.createDimension('time',nt)
if nz > 1:
nco.createDimension('depth',nz)
nchar=20
nco.createDimension('nchar',nchar)
# create coordinate variables
time_v = nco.createVariable('time', 'f8', ('time'))
lon_v = nco.createVariable('lon','f4')
lat_v = nco.createVariable('lat','f4')
if nz > 1:
depth_v = nco.createVariable('depth','f4',dimensions='depth')
else:
depth_v = nco.createVariable('depth','f4')
station_v = nco.createVariable('site','S1',('nchar'))
# write global attributes
g_attdict = nc.__dict__
g_attdict['Conventions'] = 'CF-1.6'
if nz>1:
g_attdict['featureType'] = 'timeSeriesProfile'
else:
g_attdict['featureType'] = 'timeSeries'
g_attdict['naming_authority'] = 'gov.usgs'
g_attdict['id'] = id
g_attdict['source'] = 'USGS'
g_attdict['institution'] = 'Woods Hole Coastal and Marine Science Center'
g_attdict['project'] = my_series.project_name.strip()
g_attdict['title'] = '%s/%s/%s' % (g_attdict['source'],g_attdict['id'],g_attdict['project'])
g_attdict['keywords']='Oceans > Ocean Pressure > Water Pressure, Oceans > Ocean Temperature > Water Temperature, Oceans > Salinity/Density > Conductivity, Oceans > Salinity/Density > Salinity'
g_attdict['keywords_vocabulary'] = 'GCMD Science Keywords'
g_attdict['standard_name_vocabulary'] = 'CF-1.6'
g_attdict['creator_email'] = '*****@*****.**'
g_attdict['creator_name'] = my_series.project_pi.strip()
g_attdict['publisher_name'] = 'Ellyn Montgomery'
g_attdict['summary'] = my_series.project_summary.strip()
nco.setncatts(g_attdict)
# write station variable
station_v.cf_role = 'timeseries_id'
station_v.standard_name = 'station_id'
data = numpy.empty((1,),'S'+repr(nchar))
data[0] = id
station_v[:] = netCDF4.stringtochar(data)
# write time variable
time_v.units = 'seconds since 1858-11-17 00:00:00 +0:00'
time_v.standard_name = 'time'
time_v.calendar = 'gregorian'
time_v[:] = (nc.variables['time'][:]-2400001.)*3600.*24. + nc.variables['time2'][:]/1000.
# write lon variable
lon_v.units = 'degree_east'
lon_v.standard_name = 'longitude'
lon_v[:] = nc.variables['lon'][:]
# write lat variable
lat_v.units = 'degree_north'
lat_v.standard_name = 'latitude'
lat_v[:] = nc.variables['lat'][:]
# write depth variable
depth_v.units = 'm'
depth_v.standard_name = 'depth'
depth_v.positive = 'down'
depth_v.axis = 'Z'
depth_v[:] = nc.variables['depth'][:]
# create the data variables
var_v=[]
for varname in data_vars:
ncvar = nc.variables[varname]
# if time series variable
if size(ncvar) == nt:
var = nco.createVariable(varname,ncvar.dtype,('time'))
elif size(ncvar) == nz:
var = nco.createVariable(varname,ncvar.dtype,('depth'))
else:
var = nco.createVariable(varname,ncvar.dtype,('time','depth'))
# load old variable attributes and modify if necessary
attdict = ncvar.__dict__
# if dounpackshort and 'scale_factor' in attdict: del attdict['scale_factor']
attdict['coordinates'] = 'time lon lat depth'
# assign standard_name if in dictionary
a =[k for (k, v) in d.iteritems() if attdict['long_name'].strip().lower() in v]
if len(a)==1: attdict['standard_name']=a[0]
# write variable attributes
var.setncatts(attdict)
# write the data
# print ncvar
var[:] = ncvar[:]
nco.close()
def generateString(length, alphabet=string.ascii_letters + string.digits + string.punctuation):
return(''.join([random.choice(alphabet) for i in range(length)]))
# test conversion of arrays of fixed-length strings
# to arrays of characters (with an extra dimension), and vice-versa.
FILE_NAME = 'tst_stringarr.nc'
FILE_FORMAT = 'NETCDF4_CLASSIC'
n2 = 20; nchar = 12; nrecs = 4
data = numpy.empty((nrecs,n2),'S'+repr(nchar))
for nrec in range(nrecs):
for n in range(n2):
data[nrec,n] = generateString(nchar)
datau = data.astype('U')
datac = stringtochar(data, encoding='ascii')
class StringArrayTestCase(unittest.TestCase):
def setUp(self):
self.file = FILE_NAME
nc = Dataset(FILE_NAME,'w',format=FILE_FORMAT)
nc.createDimension('n1',None)
nc.createDimension('n2',n2)
nc.createDimension('nchar',nchar)
v = nc.createVariable('strings','S1',('n1','n2','nchar'))
v2 = nc.createVariable('strings2','S1',('n1','n2','nchar'))
# if _Encoding set, string array should automatically be converted
# to a char array and vice-versan
v2._Encoding = 'ascii'
v3 = nc.createVariable('strings3','S1',('n1','n2','nchar'))
f.close()
f = Dataset('clouds.nc')
cloud_var = f.variables['primary_cloud']
print(cloud_var)
print(cloud_var.datatype.enum_dict)
print(cloud_var[:])
f.close()
# dealing with strings
from netCDF4 import stringtochar
nc = Dataset('stringtest.nc','w',format='NETCDF4_CLASSIC')
nc.createDimension('nchars',3)
nc.createDimension('nstrings',None)
v = nc.createVariable('strings','S1',('nstrings','nchars'))
datain = numpy.array(['foo','bar'],dtype='S3')
v[:] = stringtochar(datain) # manual conversion to char array
print(v[:]) # data returned as char array
v._Encoding = 'ascii' # this enables automatic conversion
v[:] = datain # conversion to char array done internally
print(v[:]) # data returned in numpy string array
nc.close()
# strings in compound types
nc = Dataset('compoundstring_example.nc','w')
dtype = numpy.dtype([('observation', 'f4'),
('station_name','S12')])
station_data_t = nc.createCompoundType(dtype,'station_data')
nc.createDimension('station',None)
statdat = nc.createVariable('station_obs', station_data_t, ('station',))
data = numpy.empty(2,station_data_t.dtype_view)
data['observation'][:] = (123.,3.14)
data['station_name'][:] = ('Boulder','New York')
def write_param_file(file_name,
nc_format='NETCDF3_CLASSIC',
glob_atts=NcGlobals(),
full_time_length=None,
subset_length=None,
unit_hydrograph_dt=None,
outlet_lon=None,
outlet_lat=None,
outlet_x_ind=None,
outlet_y_ind=None,
outlet_decomp_ind=None,
outlet_number=None,
outlet_mask=None,
outlet_name=None,
outlet_upstream_gridcells=None,
outlet_upstream_area=None,
source_lon=None,
source_lat=None,
source_x_ind=None,
source_y_ind=None,
source_decomp_ind=None,
source_time_offset=None,
source2outlet_ind=None,
source_tracer=None,
unit_hydrograph=None):
""" Write a standard RVIC Parameter file """
# ---------------------------------------------------------------- #
# Open file
f = Dataset(file_name, 'w', format=nc_format)
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# Time Variables
# Timesteps
timesteps = f.createDimension('timesteps', subset_length)
timesteps = f.createVariable('timesteps', NC_DOUBLE, ('timesteps',))
timesteps[:] = np.arange(subset_length)
for key, val in share.timesteps.__dict__.iteritems():
if val:
setattr(timesteps, key, val)
timesteps.timestep_length = 'unit_hydrograph_dt'
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# write global attributes
glob_atts.update()
for key, val in glob_atts.atts.iteritems():
if val:
setattr(f, key, val)
f.featureType = "timeSeries"
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# 0-D variables
# Full time length (size of ring)
ftl = f.createVariable('full_time_length', NC_INT, ())
ftl[:] = full_time_length
for key, val in share.full_time_length.__dict__.iteritems():
if val:
setattr(ftl, key, val)
# Subset Length
sl = f.createVariable('subset_length', NC_INT, ())
sl[:] = subset_length
for key, val in share.subset_length.__dict__.iteritems():
if val:
setattr(sl, key, val)
# UH timestep
uh_dt = f.createVariable('unit_hydrograph_dt', NC_DOUBLE, ())
uh_dt[:] = unit_hydrograph_dt
for key, val in share.unit_hydrograph_dt.__dict__.iteritems():
if val:
setattr(uh_dt, key, val)
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# Outlet Dimensions
if outlet_y_ind.ndim == 0:
numoutlets = 1
outlet_name = np.array([outlet_name])
else:
numoutlets = len(outlet_lon)
ocoords = ('outlets',)
outlets = f.createDimension(ocoords[0], numoutlets)
nocoords = ocoords + ('nc_chars',)
char_names = stringtochar(outlet_name)
chars = f.createDimension(nocoords[1], char_names.shape[1])
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# 1-D Outlet Variables
# Outlet Cell Longitudes
olon = f.createVariable('outlet_lon', NC_DOUBLE, ocoords)
olon[:] = outlet_lon
for key, val in share.outlet_lon.__dict__.iteritems():
if val:
setattr(olon, key, val)
# Outlet Cell Latitudes
olat = f.createVariable('outlet_lat', NC_DOUBLE, ocoords)
olat[:] = outlet_lat
for key, val in share.outlet_lat.__dict__.iteritems():
if val:
setattr(olat, key, val)
# Outlet Cell X Indicies
oxi = f.createVariable('outlet_x_ind', NC_INT, ocoords)
oxi[:] = outlet_x_ind
for key, val in share.outlet_x_ind.__dict__.iteritems():
if val:
setattr(oxi, key, val)
# Outlet Cell Y Indicies
oyi = f.createVariable('outlet_y_ind', NC_INT, ocoords)
oyi[:] = outlet_y_ind
for key, val in share.outlet_y_ind.__dict__.iteritems():
if val:
setattr(oyi, key, val)
# Outlet Cell Decomp IDs
odi = f.createVariable('outlet_decomp_ind', NC_INT, ocoords)
odi[:] = outlet_decomp_ind
for key, val in share.outlet_decomp_ind.__dict__.iteritems():
if val:
setattr(odi, key, val)
# Outlet Cell Number
on = f.createVariable('outlet_number', NC_INT, ocoords)
on[:] = outlet_number
for key, val in share.outlet_number.__dict__.iteritems():
if val:
setattr(on, key, val)
# Outlet Mask
om = f.createVariable('outlet_mask', NC_INT, ocoords)
om[:] = outlet_mask
for key, val in share.outlet_mask.__dict__.iteritems():
if val:
setattr(om, key, val)
# Outlet Upstream area
oua = f.createVariable('outlet_upstream_area', NC_DOUBLE, ocoords)
oua[:] = outlet_upstream_area
for key, val in share.outlet_upstream_area.__dict__.iteritems():
if val:
setattr(oua, key, val)
# Outlet Upstream grid cells
oug = f.createVariable('outlet_upstream_gridcells', NC_INT, ocoords)
oug[:] = outlet_upstream_gridcells
for key, val in share.outlet_upstream_gridcells.__dict__.iteritems():
if val:
setattr(oug, key, val)
# Outlet Names
onm = f.createVariable('outlet_name', NC_CHAR, nocoords)
onm[:, :] = char_names
for key, val in share.outlet_name.__dict__.iteritems():
if val:
setattr(onm, key, val)
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# Source Dimension
scoords = ('sources',)
sources = f.createDimension(scoords[0], len(source_lon))
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# 1D Source Variables
# Source Cell Longitudes
slon = f.createVariable('source_lon', NC_DOUBLE, scoords)
slon[:] = source_lon
for key, val in share.source_lon.__dict__.iteritems():
if val:
setattr(slon, key, val)
# Source Cell Latitudes
slat = f.createVariable('source_lat', NC_DOUBLE, scoords)
slat[:] = source_lat
for key, val in share.source_lat.__dict__.iteritems():
if val:
setattr(slat, key, val)
# Source Cell X Indicies
sxi = f.createVariable('source_x_ind', NC_INT, scoords)
sxi[:] = source_x_ind
for key, val in share.source_x_ind.__dict__.iteritems():
if val:
setattr(sxi, key, val)
# Source Cell Y Indicies
syi = f.createVariable('source_y_ind', NC_INT, scoords)
syi[:] = source_y_ind
for key, val in share.source_y_ind.__dict__.iteritems():
if val:
setattr(syi, key, val)
# Source Cell Decomp IDs
sdi = f.createVariable('source_decomp_ind', NC_INT, scoords)
sdi[:] = source_decomp_ind
for key, val in share.source_decomp_ind.__dict__.iteritems():
if val:
setattr(sdi, key, val)
# Source Cell Time Offset
sto = f.createVariable('source_time_offset', NC_INT, scoords)
sto[:] = source_time_offset
for key, val in share.source_time_offset.__dict__.iteritems():
if val:
setattr(sto, key, val)
# Source to Outlet Index Mapping
s2o = f.createVariable('source2outlet_ind', NC_INT, scoords)
s2o[:] = source2outlet_ind
for key, val in share.source2outlet_ind.__dict__.iteritems():
if val:
setattr(s2o, key, val)
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# 3-D Source Variables
uhcords = ('timesteps',) + scoords +('tracers',)
tracers = f.createDimension(uhcords[2], 1)
# Unit Hydrographs
uhs = f.createVariable('unit_hydrograph', NC_DOUBLE, uhcords)
uhs[:, :] = unit_hydrograph
for key, val in share.unit_hydrograph.__dict__.iteritems():
if val:
setattr(uhs, key, val)
# ---------------------------------------------------------------- #
f.close()
log.info('Finished writing %s' % file_name)
def create_pigment_tss_nc(metadata, data, output_folder):
""" create a netcdf file for pigment or TSS data """
netcdf_filepath = os.path.join(output_folder, "%s.nc" % create_filename_output(metadata, data))
output_netcdf_obj = Dataset(netcdf_filepath, "w", format="NETCDF4")
# read gatts from input, add them to output. Some gatts will be overwritten
input_gatts = metadata['gatts']
check_vessel_name(input_gatts['vessel_name']) # this raises a warning only
if input_gatts['vessel_name'].strip() == '':
input_gatts['vessel_name'] = 'UNKNOWN VESSEL'
gatt_to_dispose = ['geospatial_lat_min', 'geospatial_lat_max', 'geospatial_lon_min',
'geospatial_lon_max', 'geospatial_vertical_min', 'geospatial_vertical_max',
'conventions', 'local_time_zone']
for gatt in input_gatts.keys():
if gatt not in gatt_to_dispose:
if input_gatts[gatt] != '':
setattr(output_netcdf_obj, gatt, input_gatts[gatt])
setattr(output_netcdf_obj, 'input_xls_filename', os.path.basename(metadata['filename_input']))
if 'local_time_zone' in input_gatts.keys():
if input_gatts['local_time_zone'] != '':
setattr(output_netcdf_obj, 'local_time_zone', np.float(input_gatts['local_time_zone']))
output_netcdf_obj.date_created = datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ")
output_netcdf_obj.geospatial_vertical_min = data.Depth.min()
output_netcdf_obj.geospatial_vertical_max = data.Depth.max()
output_netcdf_obj.createDimension("obs", data.shape[0])
output_netcdf_obj.createDimension("station", len(data.Station_Code.unique()))
output_netcdf_obj.createDimension('name_strlen', 50)
# a profile is defined by a time station combo. 2 profiles at the same time
# but at a different location can exist. In order to find the unique
# profiles, the unique values of a string array of 'time-station' is counted
time_station_arr = ['%s_%s' % (a, b) for a, b in zip(data.index, data.Station_Code.values)]
len_prof = len(np.unique(time_station_arr))
output_netcdf_obj.createDimension("profile", len_prof)
var_time = output_netcdf_obj.createVariable("TIME", "d", "profile", fill_value=get_imos_parameter_info('TIME', '_FillValue'))
var_lat = output_netcdf_obj.createVariable("LATITUDE", "f4", "station", fill_value=get_imos_parameter_info('LATITUDE', '_FillValue'))
var_lon = output_netcdf_obj.createVariable("LONGITUDE", "f4", "station", fill_value=get_imos_parameter_info('LONGITUDE', '_FillValue'))
var_station_name = output_netcdf_obj.createVariable("station_name", "S1", (u'station', u'name_strlen'))
var_station_idx = output_netcdf_obj.createVariable("station_index", "i4", "profile")
var_profile = output_netcdf_obj.createVariable("profile", "i4", "profile")
var_rowsize = output_netcdf_obj.createVariable("row_size", "i4", "profile")
var_depth = output_netcdf_obj.createVariable("DEPTH", "f4", "obs", fill_value=get_imos_parameter_info('DEPTH', '_FillValue'))
var = 'DEPTH'
if metadata['varatts']['Depth']['Comments'] != '' or metadata['varatts']['Depth']['Comments'] != 'positive down':
setattr(output_netcdf_obj[var], 'comments', metadata['varatts']['Depth']['Comments'].replace('positive down', ''))
# creation of rest of variables
var_to_dispose = ['Latitude', 'Longitude', 'Depth', 'Time', 'Station_Code']
for var in data.columns:
if var not in var_to_dispose:
if metadata['varatts'][var]['Fill value'] == '':
fillvalue = -999
else:
fillvalue = metadata['varatts'][var]['Fill value']
output_netcdf_obj.createVariable(var, "d", "obs", fill_value=fillvalue)
if metadata['varatts'][var]['IMOS long_name'] != '':
setattr(output_netcdf_obj[var], 'long_name', metadata['varatts'][var]['IMOS long_name'])
if metadata['varatts'][var]['Units'] != '':
setattr(output_netcdf_obj[var], 'units', metadata['varatts'][var]['Units'])
if metadata['varatts'][var]['Comments'] != '':
setattr(output_netcdf_obj[var], 'comments', metadata['varatts'][var]['Comments'])
# SPM is set wrongly as a standard_name is original xls files
if 'SPM' not in var:
if metadata['varatts'][var]['CF standard_name'] != '':
setattr(output_netcdf_obj[var], 'standard_name', metadata['varatts'][var]['CF standard_name'])
if 'Sample_Number' in var:
setattr(output_netcdf_obj[var], 'units', 1)
if np.dtype(data[var]) == 'O':
os.remove(netcdf_filepath)
_error('Incorrect values for variable \"%s\"' % var)
output_netcdf_obj[var][:] = np.array(data[var].values).astype(np.double)
# Contigious ragged array representation of Stations netcdf 1.5
# add gatts and variable attributes as stored in config files
conf_file_generic = os.path.join(os.path.dirname(__file__), 'generate_nc_file_att')
generate_netcdf_att(output_netcdf_obj, conf_file_generic, conf_file_point_of_truth=True)
# lat lon depth
_, idx_station_uniq = np.unique(data.Station_Code, return_index=True)
idx_station_uniq.sort()
var_lat[:] = data.Latitude.values[idx_station_uniq].astype(np.float)
var_lon[:] = data.Longitude.values[idx_station_uniq].astype(np.float)
if np.dtype(data.Depth) == 'O':
try:
var_depth[:] = data.Depth.values.astype(np.float)
except ValueError:
os.remove(netcdf_filepath)
_error('Incorrect depth value')
else:
var_depth[:] = data.Depth.values.astype(np.float)
var_depth.positive = 'down'
# time
_, idx_time_station_uniq = np.unique(time_station_arr, return_index=True)
idx_time_station_uniq.sort()
time_values = (data.index[idx_time_station_uniq]).to_pydatetime()
time_val_dateobj = date2num(time_values, output_netcdf_obj['TIME'].units, output_netcdf_obj['TIME'].calendar)
var_time[:] = time_val_dateobj.astype(np.double)
# station
var_station_name[:] = stringtochar(np.array(data.Station_Code.values[idx_station_uniq], 'S50'))
# compute number of observations per profile
if len_prof == 1:
var_rowsize[:] = data.shape[0]
else:
n_obs_per_prof = []
for i in range(len_prof - 1):
n_obs_per_prof.append(idx_time_station_uniq[i + 1] - idx_time_station_uniq[i])
n_obs_per_prof.append(len(data.index.values) - idx_time_station_uniq[-1])
var_rowsize[:] = n_obs_per_prof
# compute association between profile number and station name
# which station this profile is for
aa = np.array(data.Station_Code)[idx_station_uniq].tolist()
bb = np.array(data.Station_Code)[idx_time_station_uniq].tolist()
var_station_idx[:] = [aa.index(b) + 1 for b in bb]
# profile
var_profile[:] = range(1, len_prof + 1)
output_netcdf_obj.geospatial_vertical_min = output_netcdf_obj['DEPTH'][:].min()
output_netcdf_obj.geospatial_vertical_max = output_netcdf_obj['DEPTH'][:].max()
output_netcdf_obj.geospatial_lat_min = output_netcdf_obj['LATITUDE'][:].min()
output_netcdf_obj.geospatial_lat_max = output_netcdf_obj['LATITUDE'][:].max()
output_netcdf_obj.geospatial_lon_min = output_netcdf_obj['LONGITUDE'][:].min()
output_netcdf_obj.geospatial_lon_max = output_netcdf_obj['LONGITUDE'][:].max()
output_netcdf_obj.time_coverage_start = min(time_values).strftime('%Y-%m-%dT%H:%M:%SZ')
output_netcdf_obj.time_coverage_end = max(time_values).strftime('%Y-%m-%dT%H:%M:%SZ')
output_netcdf_obj.close()
return netcdf_filepath
def create_absorption_nc(metadata, data, output_folder):
""" create a netcdf file for absorption data """
netcdf_filepath = os.path.join(output_folder, "%s.nc" % create_filename_output(metadata, data))
output_netcdf_obj = Dataset(netcdf_filepath, "w", format="NETCDF4")
data_dict = data[1]
data_df = data[0]
# read gatts from input, add them to output. Some gatts will be overwritten
input_gatts = metadata['gatts']
check_vessel_name(input_gatts['vessel_name']) # this raises a warning only
if input_gatts['vessel_name'].strip() == '':
input_gatts['vessel_name'] = 'UNKNOWN VESSEL'
gatt_to_dispose = ['geospatial_lat_min', 'geospatial_lat_max', 'geospatial_lon_min',
'geospatial_lon_max', 'geospatial_vertical_min', 'geospatial_vertical_max',
'conventions', 'local_time_zone']
for gatt in input_gatts.keys():
if gatt not in gatt_to_dispose:
if input_gatts[gatt] != '':
setattr(output_netcdf_obj, gatt, input_gatts[gatt])
setattr(output_netcdf_obj, 'input_xls_filename', os.path.basename(metadata['filename_input']))
if 'local_time_zone' in input_gatts.keys():
if input_gatts['local_time_zone'] != '':
setattr(output_netcdf_obj, 'local_time_zone', np.float(input_gatts['local_time_zone']))
output_netcdf_obj.date_created = datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ")
output_netcdf_obj.geospatial_vertical_min = min(data_dict['Depth'])
output_netcdf_obj.geospatial_vertical_max = max(data_dict['Depth'])
output_netcdf_obj.createDimension("obs", data_df.shape[1])
output_netcdf_obj.createDimension("station", len(np.unique(data_dict['Station_Code'])))
output_netcdf_obj.createDimension('name_strlen', 50)
output_netcdf_obj.createDimension('wavelength', data_df.shape[0])
# a profile is defined by a time station combo. 2 profiles at the same time
# but at a different location can exist. In order to find the unique
# profiles, the unique values of a string array of 'time-station' is counted
time_station_arr = ['%s_%s' % (a, b) for a, b in zip(data_dict['Dates'], data_dict['Station_Code'])]
len_prof = len(np.unique(time_station_arr))
output_netcdf_obj.createDimension("profile", len_prof)
var_time = output_netcdf_obj.createVariable("TIME", "d", "profile", fill_value=get_imos_parameter_info('TIME', '_FillValue'))
var_lat = output_netcdf_obj.createVariable("LATITUDE", "f", "station", fill_value=get_imos_parameter_info('LATITUDE', '_FillValue'))
var_lon = output_netcdf_obj.createVariable("LONGITUDE", "f", "station", fill_value=get_imos_parameter_info('LONGITUDE', '_FillValue'))
var_station_name = output_netcdf_obj.createVariable("station_name", "S1", (u'station', u'name_strlen'))
var_station_idx = output_netcdf_obj.createVariable("station_index", "i4", "profile")
var_profile = output_netcdf_obj.createVariable("profile", "i4", "profile")
var_rowsize = output_netcdf_obj.createVariable("row_size", "i4", "profile")
var_depth = output_netcdf_obj.createVariable("DEPTH", "f", "obs", fill_value=get_imos_parameter_info('DEPTH', '_FillValue'))
var_wavelength = output_netcdf_obj.createVariable("wavelength", "f", "wavelength")
var = data_dict['main_var_name'][0]
output_netcdf_obj.createVariable(var, "d", ("obs", "wavelength"), fill_value=metadata['varatts_col'][var]['Fill value'])
if metadata['varatts_col'][var]['IMOS long_name'] != '':
setattr(output_netcdf_obj[var], 'long_name', metadata['varatts_col'][var]['IMOS long_name'])
if metadata['varatts_col'][var]['Units'] != '':
setattr(output_netcdf_obj[var], 'units', metadata['varatts_col'][var]['Units'])
if metadata['varatts_col'][var]['Comments'] != '':
setattr(output_netcdf_obj[var], 'comments', metadata['varatts_col'][var]['Comments'])
if metadata['varatts_col'][var]['CF standard_name'] != '':
setattr(output_netcdf_obj[var], 'standard_name', metadata['varatts_col'][var]['CF standard_name'])
data_val = data_df.transpose()
output_netcdf_obj[var][:] = np.array(data_val.values)
# Contigious ragged array representation of Stations netcdf 1.5
# add gatts and variable attributes as stored in config files
conf_file_generic = os.path.join(os.path.dirname(__file__), 'generate_nc_file_att')
generate_netcdf_att(output_netcdf_obj, conf_file_generic, conf_file_point_of_truth=True)
# lat lon depth
_, idx_station_uniq = np.unique(data_dict['Station_Code'], return_index=True)
idx_station_uniq.sort()
var_lat[:] = np.array(data_dict['Latitude'])[idx_station_uniq]
var_lon[:] = np.array(data_dict['Longitude'])[idx_station_uniq]
var_depth[:] = data_dict['Depth']
var_depth.positive = 'down'
# time
_, idx_time_station_uniq = np.unique(time_station_arr, return_index=True)
idx_time_station_uniq.sort()
time_values = (data_dict['Dates'][idx_time_station_uniq]).to_pydatetime()
time_val_dateobj = date2num(time_values, output_netcdf_obj['TIME'].units, output_netcdf_obj['TIME'].calendar)
var_time[:] = time_val_dateobj
# wavelength
var = 'Wavelength'
var_wavelength[:] = data_dict['Wavelength']
if metadata['varatts_col'][var]['IMOS long_name'] != '':
setattr(var_wavelength, 'long_name', metadata['varatts_col'][var]['IMOS long_name'])
if metadata['varatts_col'][var]['Units'] != '':
setattr(var_wavelength, 'units', metadata['varatts_col'][var]['Units'])
if metadata['varatts_col'][var]['Comments'] != '':
setattr(var_wavelength, 'comments', metadata['varatts_col'][var]['Comments'])
if metadata['varatts_col'][var]['CF standard_name'] != '':
setattr(var_wavelength, 'standard_name', metadata['varatts_col'][var]['CF standard_name'])
# stationss
var_station_name[:] = stringtochar(np.array(data_dict['Station_Code'], 'S50')[np.sort(idx_station_uniq)])
# compute number of observations per profile
if len_prof == 1:
var_rowsize[:] = data.shape[1]
else:
n_obs_per_prof = []
for i in range(len_prof - 1):
n_obs_per_prof.append(idx_time_station_uniq[i + 1] - idx_time_station_uniq[i])
n_obs_per_prof.append(data_df.shape[1] - idx_time_station_uniq[-1])
var_rowsize[:] = n_obs_per_prof
# compute association between profile number and station name
# which station this profile is for
aa = np.array(data_dict['Station_Code'])[idx_station_uniq].tolist()
bb = np.array(data_dict['Station_Code'])[idx_time_station_uniq].tolist()
var_station_idx[:] = [aa.index(b) + 1 for b in bb]
# profile
var_profile[:] = range(1, len_prof + 1)
output_netcdf_obj.geospatial_vertical_min = output_netcdf_obj['DEPTH'][:].min()
output_netcdf_obj.geospatial_vertical_max = output_netcdf_obj['DEPTH'][:].max()
output_netcdf_obj.geospatial_lat_min = output_netcdf_obj['LATITUDE'][:].min()
output_netcdf_obj.geospatial_lat_max = output_netcdf_obj['LATITUDE'][:].max()
output_netcdf_obj.geospatial_lon_min = output_netcdf_obj['LONGITUDE'][:].min()
output_netcdf_obj.geospatial_lon_max = output_netcdf_obj['LONGITUDE'][:].max()
output_netcdf_obj.time_coverage_start = min(time_values).strftime('%Y-%m-%dT%H:%M:%SZ')
output_netcdf_obj.time_coverage_end = max(time_values).strftime('%Y-%m-%dT%H:%M:%SZ')
output_netcdf_obj.close()
return netcdf_filepath
def grid2dsg(ifile,ofile,coord_vars=['time','time2','depth','lat','lon'],
project_name=None,project_pi=None,project_summary=None):
nc = netCDF4.Dataset(ifile)
id = '%s/%s' % (project_name,ifile.split('.')[0])
#id = ifile.split('.')[0]
vars=nc.variables.keys()
data_vars = [var for var in vars if var not in coord_vars]
nt = len(nc.dimensions['time'])
nz = len(nc.dimensions['depth'])
# create dimensions of output file
nco = netCDF4.Dataset(ofile,'w',clobber=True)
nco.createDimension('time',nt)
if nz > 1:
nco.createDimension('depth',nz)
nchar=20
nco.createDimension('nchar',nchar)
# create coordinate variables
time_v = nco.createVariable('time', 'f8', ('time'))
lon_v = nco.createVariable('lon','f4')
lat_v = nco.createVariable('lat','f4')
if nz > 1:
depth_v = nco.createVariable('depth','f4',dimensions='depth')
else:
depth_v = nco.createVariable('depth','f4')
station_v = nco.createVariable('site','S1',('nchar'))
# write global attributes
g_attdict = nc.__dict__
g_attdict['Conventions'] = 'CF-1.6'
if nz>1:
g_attdict['featureType'] = 'timeSeriesProfile'
else:
g_attdict['featureType'] = 'timeSeries'
g_attdict['naming_authority'] = 'gov.usgs'
g_attdict['id'] = id
g_attdict['source'] = 'USGS'
g_attdict['institution'] = 'USGS Woods Hole Coastal and Marine Science Center'
g_attdict['project'] = 'Coastal and Marine Geology Program'
g_attdict['title'] = '%s/%s/%s' % (g_attdict['source'],project_name,g_attdict['id'])
g_attdict['keywords']='Oceans > Ocean Pressure > Water Pressure, Oceans > Ocean Temperature > Water Temperature, Oceans > Salinity/Density > Conductivity, Oceans > Salinity/Density > Salinity'
g_attdict['keywords_vocabulary']='GCMD Science Keywords'
g_attdict['standard_name_vocabulary'] = 'CF-1.6'
g_attdict['creator_email'] = '*****@*****.**'
g_attdict['creator_name'] = 'Rich Signell'
g_attdict['creator_phone'] = '+1 (508) 548-8700'
g_attdict['creator_url'] = 'http://www.usgs.gov'
g_attdict['publisher_email'] = '*****@*****.**'
g_attdict['publisher_name'] = 'Ellyn Montgomery'
g_attdict['publisher_phone'] = '+1 (508) 548-8700'
g_attdict['publisher_url'] = 'http://www.usgs.gov'
g_attdict['contributor_name'] = project_pi
g_attdict['contributor_role'] = 'principalInvestigator' #from esip ACDD
g_attdict['summary'] = project_summary
nco.setncatts(g_attdict)
# write station variable
station_v.cf_role = 'timeseries_id'
station_v.standard_name = 'station_id'
data = numpy.empty((1,),'S'+repr(nchar))
data[0] = ifile.split('.')[0]
station_v[:] = netCDF4.stringtochar(data)
# write time variable
time_v.units = 'milliseconds since 1858-11-17 00:00:00 +0:00'
time_v.standard_name = 'time'
time_v.calendar = 'gregorian'
time_v[:] = (np.int64(nc.variables['time'][:])-2400001)*3600*24*1000 + nc.variables['time2'][:]
# write lon variable
lon_v.units = 'degree_east'
lon_v.standard_name = 'longitude'
lon_v[:] = nc.variables['lon'][:]
# write lat variable
lat_v.units = 'degree_north'
lat_v.standard_name = 'latitude'
lat_v[:] = nc.variables['lat'][:]
# write depth variable
depth_v.units = 'm'
depth_v.standard_name = 'depth'
depth_v.positive = 'down'
depth_v.axis = 'Z'
depth_v[:] = nc.variables['depth'][:]
# create the data variables
var_v=[]
for varname in data_vars:
ncvar = nc.variables[varname]
# if time series variable
if size(ncvar) == nt:
var = nco.createVariable(varname,ncvar.dtype,('time'))
elif size(ncvar) == nz:
var = nco.createVariable(varname,ncvar.dtype,('depth'))
else:
var = nco.createVariable(varname,ncvar.dtype,('time','depth'))
# load old variable attributes and modify if necessary
attdict = ncvar.__dict__
# if dounpackshort and 'scale_factor' in attdict: del attdict['scale_factor']
attdict['coordinates'] = 'time lon lat depth'
# assign standard_name if in dictionary
a =[k for (k, v) in d.iteritems() if attdict['long_name'].strip().lower() in v]
if len(a)==1: attdict['standard_name']=a[0]
# write variable attributes
var.setncatts(attdict)
# write the data
# print ncvar
var[:] = ncvar[:]
nco.close()
def generate_qld_netcdf(resource_id, metadata, output_path):
"""
generate a netcdf file (wave or current) for a resource_id
:param resource_id:
:param metadata:
:param output_path:
:return:
"""
last_mod_date = get_last_modification_date_resource_id(metadata['package_name'], resource_id)
if last_mod_date == None:
# creating an epoch date
last_mod_date = datetime.datetime(1970, 1, 1, 0, 0)
wave_df = retrieve_json_data(resource_id)
if wave_df is None:
logger.error('No valid data to process for resource_id {resource_id}'.format(resource_id=resource_id))
return
if 'Current Speed' in wave_df.columns.values or 'Current Direction' in wave_df.columns.values:
logger.info('Processing Current data')
data_code = 'V'
else:
logger.info('Processing Wave data')
data_code = 'W'
var_mapping = param_mapping_parser(QLD_WAVE_PARAMETER_MAPPING)
date_start_str = wave_df.index.strftime('%Y%m%dT%H%M%SZ').values.min()
date_end_str = wave_df.index.strftime('%Y%m%dT%H%M%SZ').values.max()
nc_file_name = 'DES-QLD_{data_code}_{date_start}_{deployment_code}_WAVERIDER_FV01_END-{date_end}.nc'.format(
date_start=date_start_str,
data_code=data_code,
deployment_code=metadata['site_name'].replace(' ', '-'),
date_end=date_end_str)
nc_file_path = os.path.join(output_path, nc_file_name)
logger.info('Creating NetCDF {netcdf} from resource_id {resource_id}'.format(
netcdf=os.path.basename(nc_file_path),
resource_id=resource_id))
with Dataset(nc_file_path, 'w', format='NETCDF4') as nc_file_obj:
nc_file_obj.createDimension("TIME", wave_df.index.shape[0])
nc_file_obj.createDimension("station_id_strlen", 30)
nc_file_obj.createVariable("LATITUDE", "d", fill_value=FILLVALUE)
nc_file_obj.createVariable("LONGITUDE", "d", fill_value=FILLVALUE)
nc_file_obj.createVariable("STATION_ID", "S1", ("TIME", "station_id_strlen"))
nc_file_obj["LATITUDE"][:] = metadata['latitude']
nc_file_obj["LONGITUDE"][:] = metadata['longitude']
nc_file_obj["STATION_ID"][:] = [stringtochar(np.array(metadata['site_name'], 'S30'))] * \
wave_df.shape[0]
var_time = nc_file_obj.createVariable("TIME", "d", "TIME")
# add gatts and variable attributes as stored in config files
generate_netcdf_att(nc_file_obj, NC_ATT_CONFIG, conf_file_point_of_truth=True)
time_val_dateobj = date2num(wave_df.index.to_pydatetime(), var_time.units, var_time.calendar)
var_time[:] = time_val_dateobj
df_varname_ls = list(wave_df[wave_df.keys()].columns.values)
for df_varname in df_varname_ls:
df_varname_mapped_equivalent = df_varname
mapped_varname = var_mapping.loc[df_varname_mapped_equivalent]['VARNAME']
dtype = wave_df[df_varname].values.dtype
if dtype == np.dtype('int64'):
dtype = np.dtype('int16') # short
else:
dtype = np.dtype('f')
nc_file_obj.createVariable(mapped_varname, dtype, "TIME", fill_value=FILLVALUE)
set_var_attr(nc_file_obj, var_mapping, mapped_varname, df_varname_mapped_equivalent, dtype)
setattr(nc_file_obj[mapped_varname], 'coordinates', "TIME LATITUDE LONGITUDE")
try:
nc_file_obj[mapped_varname][:] = wave_df[df_varname].values
except ValueError:
pass
setattr(nc_file_obj, 'operator', metadata['owner'])
setattr(nc_file_obj, 'title', 'Delayed mode wave data measured at {site}'.format(site=metadata['site_name']))
setattr(nc_file_obj, 'site_code', metadata['site_code'])
setattr(nc_file_obj, 'site_name', metadata['site_name'])
if not np.isnan(metadata['wmo_id']):
setattr(nc_file_obj, 'wmo_id', int(metadata['wmo_id']))
setattr(nc_file_obj, 'geospatial_lat_min', metadata['latitude'])
setattr(nc_file_obj, 'geospatial_lat_max', metadata['latitude'])
setattr(nc_file_obj, 'geospatial_lon_min', metadata['longitude'])
setattr(nc_file_obj, 'geospatial_lon_max', metadata['longitude'])
setattr(nc_file_obj, 'time_coverage_start', wave_df.index.strftime('%Y-%m-%dT%H:%M:%SZ').values.min())
setattr(nc_file_obj, 'time_coverage_end', wave_df.index.strftime('%Y-%m-%dT%H:%M:%SZ').values.max())
setattr(nc_file_obj, 'date_created', pd.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"))
data_url = '{base_url_data}{id}&limit={limit}'.format(base_url_data=BASE_URL_DATA,
id=resource_id,
limit=LIMIT_VALUES)
setattr(nc_file_obj, 'data_original_url', data_url)
setattr(nc_file_obj, 'glossary', 'https://www.qld.gov.au/environment/coasts-waterways/beach/waves-glossary')
setattr(nc_file_obj, 'wave_monitoring_faq', 'https://www.qld.gov.au/environment/coasts-waterways/beach/waves')
setattr(nc_file_obj, 'first_deployment_date', metadata.first_deployment_date.strftime("%Y-%m-%dT%H:%M:%SZ"))
setattr(nc_file_obj, 'water_depth', metadata.water_depth)
setattr(nc_file_obj, 'water_depth_units', 'meters')
setattr(nc_file_obj, 'site_information_url', metadata.source_url)
setattr(nc_file_obj, 'owner', metadata.owner)
setattr(nc_file_obj, 'instrument_model', metadata.instrument_model)
setattr(nc_file_obj, 'instrument_maker', metadata.instrument_maker)
setattr(nc_file_obj, 'waverider_type', metadata.waverider_type)
github_comment = 'Product created with %s' % get_git_revision_script_url(os.path.realpath(__file__))
nc_file_obj.lineage = ('%s %s' % (getattr(nc_file_obj, 'lineage', ''), github_comment))
# save to pickle file the new last downloaded date for future run
pickle_file = os.path.join(WIP_DIR, 'last_downloaded_date_resource_id.pickle')
last_downloaded_date_resources = load_pickle_db(pickle_file)
if not last_downloaded_date_resources:
last_downloaded_date_resources = dict()
last_modification = last_mod_date
last_downloaded_date_resources[resource_id] = last_modification
with open(pickle_file, 'wb') as p_write:
pickle.dump(last_downloaded_date_resources, p_write)
return nc_file_path
def encode_cf_variable(array):
"""Converts an XArray into an XArray suitable for saving as a netCDF
variable
"""
dimensions = array.dimensions
data = array.data
attributes = array.attributes.copy()
encoding = array.encoding.copy()
if (np.issubdtype(data.dtype, np.datetime64)
or (data.dtype.kind == 'O'
and isinstance(data.reshape(-1)[0], datetime))):
# encode datetime arrays into numeric arrays
(data, units, calendar) = utils.encode_cf_datetime(
data, encoding.pop('units', None), encoding.pop('calendar', None))
attributes['units'] = units
attributes['calendar'] = calendar
elif data.dtype == np.dtype('O'):
# Occasionally, one will end up with variables with dtype=object
# (likely because they were created from pandas objects which don't
# maintain dtype careful). Thie code makes a best effort attempt to
# encode them into a dtype that NETCDF can handle by inspecting the
# dtype of the first element.
dtype = np.array(data.reshape(-1)[0]).dtype
# N.B. the "astype" call below will fail if data cannot be cast to the
# type of its first element (which is probably the only sensible thing
# to do).
data = np.asarray(data).astype(dtype)
def get_to(source, dest, k):
v = source.get(k)
dest[k] = v
return v
# encode strings as character arrays
if np.issubdtype(data.dtype, (str, unicode)):
data = nc4.stringtochar(data)
dimensions = dimensions + ('string%s' % data.shape[-1],)
# unscale/mask
if any(k in encoding for k in ['add_offset', 'scale_factor']):
data = np.array(data, dtype=float, copy=True)
if 'add_offset' in encoding:
data -= get_to(encoding, attributes, 'add_offset')
if 'scale_factor' in encoding:
data /= get_to(encoding, attributes, 'scale_factor')
# replace NaN with the fill value
if '_FillValue' in encoding:
if encoding['_FillValue'] is np.nan:
attributes['_FillValue'] = np.nan
else:
nans = np.isnan(data)
if nans.any():
data[nans] = get_to(encoding, attributes, '_FillValue')
# restore original dtype
if 'dtype' in encoding:
if np.issubdtype(encoding['dtype'], int):
data = data.round()
data = data.astype(encoding['dtype'])
return xarray.XArray(dimensions, data, attributes, encoding=encoding)
def __write_array(self):
""" Write history file """
# ------------------------------------------------------------ #
# Open file
f = Dataset(self.filename, 'w', self._file_format)
# ------------------------------------------------------------ #
# ------------------------------------------------------------ #
# Time Variable
time = f.createDimension('time', None)
time = f.createVariable('time', self._ncprec, ('time',),
**self.ncvaropts)
time[:] = self._out_times[:self._out_data_i+1]
for key, val in share.time.__dict__.iteritems():
if val:
setattr(time, key, val)
time.calendar = self._calendar
if self._avgflag != 'I':
nv = f.createDimension('nv', 2)
time.bounds = 'time_bnds'
time_bnds = f.createVariable('time_bnds', self._ncprec,
('time', 'nv',), **self.ncvaropts)
time_bnds[:, :] = self._out_time_bnds[:self._out_data_i+1]
# ------------------------------------------------------------ #
# ------------------------------------------------------------ #
# Setup Coordinate Variables
coords = ('outlets',)
outlets = f.createDimension('outlets', self._num_outlets)
nocoords = coords + ('nc_chars',)
char_names = stringtochar(self._outlet_name)
chars = f.createDimension(nocoords[1], char_names.shape[1])
# ------------------------------------------------------------ #
# ------------------------------------------------------------ #
# Variables
outlet_lon = f.createVariable('lon', self._ncprec, coords,
**self.ncvaropts)
outlet_lat = f.createVariable('lat', self._ncprec, coords,
**self.ncvaropts)
outlet_x_ind = f.createVariable('outlet_x_ind', NC_INT, coords,
**self.ncvaropts)
outlet_y_ind = f.createVariable('outlet_y_ind', NC_INT, coords,
**self.ncvaropts)
outlet_decomp_ind = f.createVariable('outlet_decomp_ind', NC_INT,
coords, **self.ncvaropts)
onm = f.createVariable('outlet_name', NC_CHAR, nocoords,
**self.ncvaropts)
outlet_lon[:] = self._outlet_lon
outlet_lat[:] = self._outlet_lat
outlet_x_ind[:] = self._outlet_x_ind
outlet_y_ind[:] = self._outlet_y_ind
outlet_decomp_ind[:] = self._outlet_decomp_ind
onm[:, :] = char_names
for key, val in share.outlet_lon.__dict__.iteritems():
if val:
setattr(outlet_lon, key, val)
for key, val in share.outlet_lat.__dict__.iteritems():
if val:
setattr(outlet_lat, key, val)
for key, val in share.outlet_y_ind.__dict__.iteritems():
if val:
setattr(outlet_y_ind, key, val)
for key, val in share.outlet_x_ind.__dict__.iteritems():
if val:
setattr(outlet_x_ind, key, val)
for key, val in share.outlet_decomp_ind.__dict__.iteritems():
if val:
setattr(outlet_decomp_ind, key, val)
for key, val in share.outlet_name.__dict__.iteritems():
if val:
setattr(onm, key, val)
# ------------------------------------------------------------ #
# ------------------------------------------------------------ #
# Write Fields
tcoords = ('time',) + coords
for field in self._fincl:
var = f.createVariable(field, self._ncprec, tcoords,
**self.ncvaropts)
var[:, :] = self._out_data[field][:self._out_data_i+1]
for key, val in getattr(share, field).__dict__.iteritems():
if val:
setattr(var, key, val)
var.units = self._units
# ------------------------------------------------------------ #
# ------------------------------------------------------------ #
# write global attributes
self._glob_ats.update()
for key, val in self._glob_ats.atts.iteritems():
if val:
setattr(f, key, val)
f.featureType = "timeSeries"
# ------------------------------------------------------------ #
f.close()
log.info('Finished writing %s', self.filename)
def add_var(dst, name, dims, data=None, shape=None, atts=None, dtype=None, zlib=True, fillValue=None,
lusestr=True, **kwargs):
''' Function to add a Variable to a NetCDF Dataset; returns the Variable reference. '''
# all remaining kwargs are passed on to dst.createVariable()
# use data array to infer dimensions and data type
if data is not None:
if not isinstance(data,np.ndarray): raise TypeError
if len(dims) != data.ndim: raise NCDataError, "Number of dimensions in '%s' does not match data array."%(name,)
if shape:
if shape != data.shape: raise NCDataError, "Shape of '%s' does not match data array."%(name,)
else: shape = data.shape
# get dtype
if dtype:
if dtype != data.dtype: data = data.astype(dtype)
# raise NCDataError, "Data type in '%s' does not match data array."%(name,)
else: dtype = data.dtype
if dtype is None: raise NCDataError, "Cannot construct a NetCDF Variable without a data array or an abstract data type."
dtype = np.dtype(dtype) # use numpy types
if dtype is np.dtype('bool_'): dtype = np.dtype('i1') # cast numpy bools as 8-bit integers
lstrvar = ( dtype.kind == 'S' and not lusestr )
# check/create dimensions
if shape is None: shape = [None,]*len(dims)
else: shape = list(shape)
if len(shape) != len(dims): raise NCAxisError
for i,dim in zip(xrange(len(dims)),dims):
if dim in dst.dimensions:
if shape[i] is None:
shape[i] = len(dst.dimensions[dim])
else:
if shape[i] != len(dst.dimensions[dim]):
raise NCAxisError, 'Size of dimension %s does not match records! %i != %i'%(dim,shape[i],len(dst.dimensions[dim]))
else:
if shape[i] is not None: dst.createDimension(dim, size=shape[i])
else: raise NCAxisError, "Cannot construct dimension '%s' without size information."%(dims,)
dims = tuple(dims); shape = tuple(shape)
# figure out parameters for variable
varargs = dict() # arguments to be passed to createVariable
if isinstance(zlib,dict): varargs.update(zlib)
elif zlib: varargs.update(zlib_default)
varargs.update(kwargs)
if fillValue is None:
if atts and '_FillValue' in atts: fillValue = atts['_FillValue'] # will be removed later
elif atts and 'missing_value' in atts: fillValue = atts['missing_value']
elif data is not None and isinstance(data,ma.MaskedArray): # defaults values for numpy masked arrays
fillValue = ma.default_fill_value(dtype)
# if isinstance(dtype,np.bool_): fillValue = True
# elif isinstance(dtype,np.integer): fillValue = 999999
# elif isinstance(dtype,np.floating): fillValue = 1.e20
# elif isinstance(dtype,np.complexfloating): fillValue = 1.e20+0j
# elif isinstance(dtype,np.flexible): fillValue = 'N/A'
# else: fillValue = None # for 'object'
else: pass # if it is not a masked array and no missing value information was passed, don't assign fillValue
else:
if data is not None and isinstance(data,ma.MaskedArray): data.set_fill_value(fillValue)
# make sure fillValue is OK (there have been problems...)
fillValue = checkFillValue(fillValue, dtype)
if fillValue is not None:
atts['missing_value'] = fillValue # I use fillValue and missing_value the same way
# add extra dimension for strings
if lstrvar and dtype.itemsize > 1:
# add extra dimension
shape = shape + (dtype.itemsize,)
dims = dims + ('str_dim_'+name,) # naming pattern for string dimensions
dst.createDimension(dims[-1], size=shape[-1])
# change dtype to single char string
dtype = np.dtype('|S1')
# convert string arrays to char arrays
if data is not None:
data = nc.stringtochar(data)
assert data.dtype == dtype, str(data.dtype)+', '+str(dtype)
# create netcdf variable
var = dst.createVariable(name, dtype, dims, fill_value=fillValue, **varargs)
# add attributes
if atts: var.setncatts(coerceAtts(atts))
# assign coordinate data if given
if data is not None: var[:] = data
# return var reference
return var
def writeToNetCDF(self, inputFilePath, outputFilePath, commandLine, format, flatten=False, _debug=True):
# weird, but useful to check whether the HeaderInfo id in the netCDF
# file
setattr(self, "header_info", None)
netCDFHandler = _file_existence_check(outputFilePath, format, self)
delattr(self, "header_info")
#### Replace the original isdigit function
camera_opt = "SWIR"
##### Write the data from metadata to netCDF #####
for key, data in self.__dict__.items():
tempGroup = netCDFHandler.createGroup(key) if not flatten else netCDFHandler
for subkey, subdata in data.items():
if not _IS_DIGIT(subdata): #Case for letter variables
if 'date' in subkey and subkey != "date of installation" and subkey != "date of handover":
assert subdata != "todo", '"todo" is not a legal value for the keys'
tempVariable = tempGroup.createVariable(_reformat_string(subkey), 'f8')
tempVariable[...] = translate_time(subdata)
setattr(tempVariable, "units", "days since 1970-01-01 00:00:00")
setattr(tempVariable, "calender", "gregorian")
setattr(tempGroup, _reformat_string(subkey), subdata)
else: #Case for digits variables
if "time" in data:
yearMonthDate = data["time"]
elif "Time" in data:
yearMonthDate = data["Time"]
setattr(tempGroup, _reformat_string(subkey), subdata)
short_name, attributes = _generate_attr(subkey)
tempVariable = tempGroup.createVariable(short_name, 'f8')
for name, value in attributes.items():
setattr(tempVariable, name, value)
tempVariable[...] = float(subdata)
##### Write data from header files to netCDF #####
wavelength = get_wavelength(inputFilePath)
netCDFHandler.createDimension("wavelength", len(wavelength))
# Check if the wavelength is correctly collected
assert len(wavelength) in (955, 272), "ERROR: Failed to get wavlength informations. Please check if you modified the *.hdr files"
camera_opt = 'VNIR' if len(wavelength) == 955 else 'SWIR' # Choose appropriate camera by counting the number of wavelengths.
tempWavelength = netCDFHandler.createVariable(
"wavelength", 'f8', 'wavelength')
setattr(tempWavelength, 'long_name', 'Hyperspectral Wavelength')
setattr(tempWavelength, 'units', 'nanometers')
tempWavelength[...] = wavelength
write_header_file(inputFilePath, netCDFHandler, flatten, _debug)
##### Write the data from frameIndex files to netCDF #####
tempFrameTime = frame_index_parser(''.join((inputFilePath.strip("raw"), "frameIndex.txt")), yearMonthDate)
netCDFHandler.createDimension("time", len(tempFrameTime))
# Check if the frame time information is correctly collected
assert len(tempFrameTime), "ERROR: Failed to collect frame time information from " + ''.join((inputFilePath.strip("raw"), "frameIndex.txt")) + ". Please check the file."
frameTime = netCDFHandler.createVariable("frametime", "f8", ("time",))
frameTime[...] = tempFrameTime
setattr(frameTime, "units", "days since 1970-01-01 00:00:00")
setattr(frameTime, "calender", "gregorian")
setattr(frameTime, "notes", "Each time of the scanline of the y taken")
########################### Adding geographic positions ###########################
xPixelsLocation, yPixelsLocation, boundingBox, googleMapAddress\
= pixel2Geographic("".join((inputFilePath[:-4],"_metadata.json")), "".join((inputFilePath,'.hdr')), camera_opt)
# Check if the image width and height are correctly collected.
assert len(xPixelsLocation) > 0 and len(yPixelsLocation) > 0, "ERROR: Failed to collect the image size metadata from " + "".join((inputFilePath,'.hdr')) + ". Please check the file."
netCDFHandler.createDimension("x", len(xPixelsLocation))
x = netCDFHandler.createVariable("x", "f8", ("x",))
x[...] = xPixelsLocation
setattr(netCDFHandler.variables["x"], "units", "meters")
setattr(netCDFHandler.variables['x'], 'reference_point', 'Southeast corner of field')
setattr(netCDFHandler.variables['x'], "long_name", "North-south offset from southeast corner of field")
netCDFHandler.createDimension("y", len(yPixelsLocation))
y = netCDFHandler.createVariable("y", "f8", ("y",))
y[...] = yPixelsLocation
setattr(netCDFHandler.variables["y"], "units", "meters")
setattr(netCDFHandler.variables['y'], 'reference_point', 'Southeast corner of field')
setattr(netCDFHandler.variables['y'], "long_name", "Distance west of the southeast corner of the field")
lat_pt, lon_pt = REFERENCE_POINT
lat_pt_var = netCDFHandler.createVariable("lat_reference_point", "f8")
lat_pt_var[...] = lat_pt
setattr(netCDFHandler.variables["lat_reference_point"], "units", "degrees_north")
setattr(netCDFHandler.variables["lat_reference_point"], "long_name", "Latitude of the master reference point at southeast corner of field")
setattr(netCDFHandler.variables["lat_reference_point"], "provenance", "https://github.com/terraref/reference-data/issues/32 by Dr. David LeBauer")
lon_pt_var = netCDFHandler.createVariable("lon_reference_point", "f8")
lon_pt_var[...] = lon_pt
setattr(netCDFHandler.variables["lon_reference_point"], "units", "degrees_east")
setattr(netCDFHandler.variables["lon_reference_point"], "long_name", "Longitude of the master reference point at southeast corner of field")
setattr(netCDFHandler.variables["lon_reference_point"], "provenance", "https://github.com/terraref/reference-data/issues/32 by Dr. David LeBauer")
x_ref_pt = netCDFHandler.createVariable("x_reference_point", "f8")
x_ref_pt[...] = 0
setattr(netCDFHandler.variables["x_reference_point"], "units", "meters")
setattr(netCDFHandler.variables["x_reference_point"], "long_name", "x of the master reference point at southeast corner of field")
setattr(netCDFHandler.variables["x_reference_point"], "provenance", "https://github.com/terraref/reference-data/issues/32 by Dr. David LeBauer")
y_ref_pt = netCDFHandler.createVariable("y_reference_point", "f8")
y_ref_pt[...] = 0
setattr(netCDFHandler.variables["y_reference_point"], "units", "meters")
setattr(netCDFHandler.variables["y_reference_point"], "long_name", "y of the master reference point at southeast corner of field")
setattr(netCDFHandler.variables["y_reference_point"], "provenance", "https://github.com/terraref/reference-data/issues/32 by Dr. David LeBauer")
# Write latitude and longitude of bounding box
SE, SW, NE, NW = boundingBox[0], boundingBox[1], boundingBox[2], boundingBox[3]
lat_se, lon_se = tuple(SE.split(", "))
lat_sw, lon_sw = tuple(SW.split(", "))
lat_ne, lon_ne = tuple(NE.split(", "))
lat_nw, lon_nw = tuple(NW.split(", "))
latSe = netCDFHandler.createVariable("lat_img_se", "f8")
latSe[...] = float(lat_se)
setattr(netCDFHandler.variables["lat_img_se"], "units", "degrees_north")
setattr(netCDFHandler.variables["lat_img_se"], "long_name", "Latitude of southeast corner of image")
# have a "x_y_img_se" in meters, double
lonSe = netCDFHandler.createVariable("lon_img_se", "f8")
lonSe[...] = float(lon_se)
setattr(netCDFHandler.variables["lon_img_se"], "units", "degrees_east")
setattr(netCDFHandler.variables["lon_img_se"], "long_name", "Longitude of southeast corner of image")
latSw = netCDFHandler.createVariable("lat_img_sw", "f8")
latSw[...] = float(lat_sw)
setattr(netCDFHandler.variables["lat_img_sw"], "units", "degrees_north")
setattr(netCDFHandler.variables["lat_img_sw"], "long_name", "Latitude of southwest corner of image")
lonSw = netCDFHandler.createVariable("lon_img_sw", "f8")
lonSw[...] = float(lon_sw)
setattr(netCDFHandler.variables["lon_img_sw"], "units", "degrees_east")
setattr(netCDFHandler.variables["lon_img_sw"], "long_name", "Longitude of southwest corner of image")
latNe = netCDFHandler.createVariable("lat_img_ne", "f8")
latNe[...] = float(lat_ne)
setattr(netCDFHandler.variables["lat_img_ne"], "units", "degrees_north")
setattr(netCDFHandler.variables["lat_img_ne"], "long_name", "Latitude of northeast corner of image")
lonNe = netCDFHandler.createVariable("lon_img_ne", "f8")
lonNe[...] = float(lon_ne)
setattr(netCDFHandler.variables["lon_img_ne"], "units", "degrees_east")
setattr(netCDFHandler.variables["lon_img_ne"], "long_name", "Longitude of northeast corner of image")
latNw = netCDFHandler.createVariable("lat_img_nw", "f8")
latNw[...] = float(lat_nw)
setattr(netCDFHandler.variables["lat_img_nw"], "units", "degrees_north")
setattr(netCDFHandler.variables["lat_img_nw"], "long_name", "Latitude of northwest corner of image")
lonNw = netCDFHandler.createVariable("lon_img_nw", "f8")
lonNw[...] = float(lon_nw)
setattr(netCDFHandler.variables["lon_img_nw"], "units", "degrees_east")
setattr(netCDFHandler.variables["lon_img_nw"], "long_name", "Longitude of northwest corner of image")
xSe = netCDFHandler.createVariable("x_img_se", "f8")
xSe[...] = float(x[-1])
setattr(netCDFHandler.variables["x_img_se"], "units", "meters")
setattr(netCDFHandler.variables["x_img_se"], "long_name", "Southeast corner of image, north distance to reference point")
# have a "x_y_img_se" in meters, double
ySe = netCDFHandler.createVariable("y_img_se", "f8")
ySe[...] = float(y[-1])
setattr(netCDFHandler.variables["y_img_se"], "units", "meters")
setattr(netCDFHandler.variables["y_img_se"], "long_name", "Southeast corner of image, west distance to reference point")
xSw = netCDFHandler.createVariable("x_img_sw", "f8")
xSw[...] = float(x[0])
setattr(netCDFHandler.variables["x_img_sw"], "units", "meters")
setattr(netCDFHandler.variables["x_img_sw"], "long_name", "Southwest corner of image, north distance to reference point")
ySw = netCDFHandler.createVariable("y_img_sw", "f8")
ySw[...] = float(y[-1])
setattr(netCDFHandler.variables["y_img_sw"], "units", "meters")
setattr(netCDFHandler.variables["y_img_sw"], "long_name", "Southwest corner of image, west distance to reference point")
xNe = netCDFHandler.createVariable("x_img_ne", "f8")
xNe[...] = float(x[-1])
setattr(netCDFHandler.variables["x_img_ne"], "units", "meters")
setattr(netCDFHandler.variables["x_img_ne"], "long_name", "Northeast corner of image, north distance to reference point")
yNe = netCDFHandler.createVariable("y_img_ne", "f8")
yNe[...] = float(y[0])
setattr(netCDFHandler.variables["y_img_ne"], "units", "meters")
setattr(netCDFHandler.variables["y_img_ne"], "long_name", "Northeast corner of image, west distance to reference point")
xNw = netCDFHandler.createVariable("x_img_nw", "f8")
xNw[...] = float(x[0])
setattr(netCDFHandler.variables["x_img_nw"], "units", "meters")
setattr(netCDFHandler.variables["x_img_nw"], "long_name", "Northwest corner of image, north distance to reference point")
yNw = netCDFHandler.createVariable("y_img_nw", "f8")
yNw[...] = float(y[0])
setattr(netCDFHandler.variables["y_img_nw"], "units", "meters")
setattr(netCDFHandler.variables["y_img_nw"], "long_name", "Northwest corner of image, west distance to reference point")
if format == "NETCDF3_CLASSIC":
netCDFHandler.createDimension("length of Google Map String", len(googleMapAddress))
googleMapView = netCDFHandler.createVariable("Google_Map_View", "S1", ("length of Google Map String",))
tempAddress = np.chararray((1, 1), itemsize=len(googleMapAddress))
tempAddress[:] = googleMapAddress
googleMapView[...] = stringtochar(tempAddress)[0]
else:
googleMapView = netCDFHandler.createVariable("Google_Map_View", str)
googleMapView[...] = googleMapAddress
setattr(netCDFHandler.variables["Google_Map_View"], "usage", "copy and paste to your web browser")
setattr(netCDFHandler.variables["Google_Map_View"], 'reference_point', 'Southeast corner of field')
y_pxl_sz = netCDFHandler.createVariable("y_pxl_sz", "f8")
y_pxl_sz[...] = 0.98526434004512529576754637665e-3
setattr(netCDFHandler.variables["y_pxl_sz"], "units", "meters")
setattr(netCDFHandler.variables["y_pxl_sz"], "notes", "y coordinate length of a single pixel in pictures captured by SWIR and VNIR camera")
if camera_opt == "SWIR":
x_pxl_sz = netCDFHandler.createVariable("x_pxl_sz", "f8")
x_pxl_sz[...] = 1.025e-3
setattr(netCDFHandler.variables["x_pxl_sz"], "units", "meters")
setattr(netCDFHandler.variables["x_pxl_sz"], "notes", "x coordinate length of a single pixel in SWIR images")
else:
x_pxl_sz = netCDFHandler.createVariable("x_pxl_sz", "f8")
x_pxl_sz[...] = 1.930615052e-3
setattr(netCDFHandler.variables["x_pxl_sz"], "units", "meters")
setattr(netCDFHandler.variables["x_pxl_sz"], "notes", "x coordinate length of a single pixel in VNIR images")
##### Write the history to netCDF #####
netCDFHandler.history = ''.join((_TIMESTAMP(), ': python ', commandLine))
netCDFHandler.close()
FILE_FORMAT = 'NETCDF4_CLASSIC'
chars = '1234567890aabcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
nc = Dataset(FILE_NAME,'w',format=FILE_FORMAT)
n2 = 10; nchar = 12; nrecs = 4
nc.createDimension('n1',None)
nc.createDimension('n2',n2)
nc.createDimension('nchar',nchar)
v = nc.createVariable('strings','S1',('n1','n2','nchar'))
for nrec in range(nrecs):
data = []
data = numpy.empty((n2,),'S'+repr(nchar))
# fill data with random nchar character strings
for n in range(n2):
data[n] = ''.join([random.choice(chars) for i in range(nchar)])
print nrec,data
# convert data to array of characters with an extra dimension
# (the number of characters per string) added to the right.
datac = stringtochar(data)
v[nrec] = datac
nc.close()
nc = Dataset(FILE_NAME)
v = nc.variables['strings']
print v.shape, v.dtype
for nrec in range(nrecs):
# read character array back, convert to an array of strings
# of length equal to the rightmost dimension.
print nrec, chartostring(v[nrec])
nc.close()
def write_restart(self, current_history_files, history_restart_files):
# ------------------------------------------------------------ #
# Open file
filename = self.timestamp.strftime(self.__fname_format)
f = Dataset(filename, 'w', self.file_format)
# ------------------------------------------------------------ #
# ------------------------------------------------------------ #
# Time Variables
# Current time
time = f.createDimension('time', 1)
time = f.createVariable('time', NC_DOUBLE, ('time',))
time[:] = date2num(self.timestamp, TIMEUNITS, calendar=self._calendar)
for key, val in share.time.__dict__.iteritems():
if val:
setattr(time, key, val)
time.calendar = self._calendar
# Timesteps
timesteps = f.createDimension('timesteps', self.full_time_length)
timesteps = f.createVariable('timesteps', NC_DOUBLE, ('timesteps',))
timesteps[:] = np.arange(self.full_time_length)
for key, val in share.timesteps.__dict__.iteritems():
if val:
setattr(timesteps, key, val)
timesteps.timestep_length = 'unit_hydrograph_dt'
# UH timestep
unit_hydrograph_dt = f.createVariable('unit_hydrograph_dt',
NC_DOUBLE, ())
unit_hydrograph_dt[:] = self.unit_hydrograph_dt
for key, val in share.unit_hydrograph_dt.__dict__.iteritems():
if val:
setattr(unit_hydrograph_dt, key, val)
timemgr_rst_type = f.createVariable('timemgr_rst_type', NC_DOUBLE, ())
timemgr_rst_type[:] = self._calendar_key
for key, val in share.timemgr_rst_type.__dict__.iteritems():
if val:
setattr(timemgr_rst_type, key, val)
timemgr_rst_step_sec = f.createVariable('timemgr_rst_step_sec',
NC_DOUBLE, ())
timemgr_rst_step_sec[:] = self.unit_hydrograph_dt
for key, val in share.timemgr_rst_step_sec.__dict__.iteritems():
if val:
setattr(timemgr_rst_step_sec, key, val)
timemgr_rst_start_ymd = f.createVariable('timemgr_rst_start_ymd',
NC_DOUBLE, ())
timemgr_rst_start_ymd[:] = self._start_date.year*10000 \
+ self._start_date.month*100 + self._start_date.day
for key, val in share.timemgr_rst_start_ymd.__dict__.iteritems():
if val:
setattr(timemgr_rst_start_ymd, key, val)
timemgr_rst_start_tod = f.createVariable('timemgr_rst_start_tod',
NC_DOUBLE, ())
timemgr_rst_start_tod[:] = (self._start_ord % 1) * SECSPERDAY
for key, val in share.timemgr_rst_start_tod.__dict__.iteritems():
if val:
setattr(timemgr_rst_start_tod, key, val)
timemgr_rst_ref_ymd = f.createVariable('timemgr_rst_ref_ymd',
NC_DOUBLE, ())
timemgr_rst_ref_ymd[:] = REFERENCE_DATE
for key, val in share.timemgr_rst_ref_ymd.__dict__.iteritems():
if val:
setattr(timemgr_rst_ref_ymd, key, val)
timemgr_rst_ref_tod = f.createVariable('timemgr_rst_ref_tod',
NC_DOUBLE, ())
timemgr_rst_ref_tod[:] = REFERENCE_TIME
for key, val in share.timemgr_rst_ref_tod.__dict__.iteritems():
if val:
setattr(timemgr_rst_ref_tod, key, val)
timemgr_rst_curr_ymd = f.createVariable('timemgr_rst_curr_ymd',
NC_DOUBLE, ())
timemgr_rst_curr_ymd[:] = self.timestamp.year*10000 + \
self.timestamp.month*100+self.timestamp.day
for key, val in share.timemgr_rst_curr_ymd.__dict__.iteritems():
if val:
setattr(timemgr_rst_curr_ymd, key, val)
timemgr_rst_curr_tod = f.createVariable('timemgr_rst_curr_tod',
NC_DOUBLE, ())
timemgr_rst_curr_tod[:] = (self.time_ord % 1)*SECSPERDAY
for key, val in share.timemgr_rst_curr_tod.__dict__.iteritems():
if val:
setattr(timemgr_rst_curr_tod, key, val)
# ------------------------------------------------------------ #
# Setup Tape Dimensions
coords = ('tapes', 'max_chars')
ntapes = f.createDimension(coords[0], len(history_restart_files))
ntapes = f.createDimension(coords[1], MAX_NC_CHARS)
# ------------------------------------------------------------ #
# ------------------------------------------------------------ #
# Write Fields
locfnh = f.createVariable('locfnh', NC_CHAR, coords)
for i, string in enumerate(current_history_files):
locfnh[i, :] = stringtochar(np.array(string.ljust(MAX_NC_CHARS)))
locfnh.long_name = 'History filename'
locfnh.comment = 'This variable NOT needed for startup or branch simulations'
locfnhr = f.createVariable('locfnhr', NC_CHAR, coords)
for i, string in enumerate(history_restart_files):
locfnh[i, :] = stringtochar(np.array(string.ljust(MAX_NC_CHARS)))
locfnhr.long_name = 'History restart filename'
locfnhr.comment = 'This variable NOT needed for startup or branch simulations'
# ------------------------------------------------------------ #
# ------------------------------------------------------------ #
# Setup Point Dimensions
coords = ('outlets', )
outlets = f.createDimension(coords[0], self.n_outlets)
# ------------------------------------------------------------ #
# ------------------------------------------------------------ #
# Write Fields
oyi = f.createVariable('outlet_y_ind', NC_INT, coords[0])
oyi[:] = self.outlet_y_ind
for key, val in share.outlet_y_ind.__dict__.iteritems():
if val:
setattr(oyi, key, val)
oxi = f.createVariable('outlet_x_ind', NC_INT, coords[0])
oxi[:] = self.outlet_x_ind
for key, val in share.outlet_x_ind.__dict__.iteritems():
if val:
setattr(oxi, key, val)
odi = f.createVariable('outlet_decomp_ind', NC_INT, coords[0])
odi[:] = self.outlet_decomp_ind
for key, val in share.outlet_decomp_ind.__dict__.iteritems():
if val:
setattr(odi, key, val)
tcoords = ('timesteps', ) + coords
for tracer in RVIC_TRACERS:
ring = f.createVariable('{0}_ring'.format(tracer),
NC_DOUBLE, tcoords)
ring[:, :] = self.ring[tracer][:, :]
for key, val in share.ring.__dict__.iteritems():
if val:
setattr(ring, key, val)
# ------------------------------------------------------------ #
# ------------------------------------------------------------ #
# write global attributes
self.glob_atts.update()
for key, val in self.glob_atts.atts.iteritems():
if val:
setattr(f, key, val)
# ------------------------------------------------------------ #
f.close()
log.info('Finished writing %s', filename)
return filename
nc_file_name_var.setncattr("long_name", "source file for this instrument")
nc_instrument_type_var = nc_out.createVariable("instrument_type", "S1", ("instrument", "strlen"))
nc_instrument_type_var.setncattr("long_name", "source instrument make, model, serial_number")
filen = 0
data = numpy.empty(len(files), dtype="S256")
instrument = numpy.empty(len(files), dtype="S256")
for path_file in files:
data[filen] = path_file
with Dataset(path_file, mode="r") as nc_type:
instrument[filen] = nc_type.instrument + '-' + nc_type.instrument_serial_number
filen += 1
nc_file_name_var[:] = stringtochar(data)
nc_instrument_type_var[:] = stringtochar(instrument)
#
# create a list of variables needed
#
filen = 0
# include the DEPTH variable
var_names_all = var_to_agg + ['DEPTH']
# add the ancillary variables for the ones requested
for v in var_names_all:
if hasattr(var_list[v], 'ancillary_variables'):
var_names_all += [var_list[v].ancillary_variables]
def write_param_file(
file_name,
nc_format="NETCDF3_CLASSIC",
glob_atts=NcGlobals(),
full_time_length=None,
subset_length=None,
unit_hydrograph_dt=None,
outlet_lon=None,
outlet_lat=None,
outlet_x_ind=None,
outlet_y_ind=None,
outlet_decomp_ind=None,
outlet_number=None,
outlet_mask=None,
outlet_name=None,
outlet_upstream_gridcells=None,
outlet_upstream_area=None,
source_lon=None,
source_lat=None,
source_x_ind=None,
source_y_ind=None,
source_decomp_ind=None,
source_time_offset=None,
source2outlet_ind=None,
unit_hydrograph=None,
zlib=True,
complevel=4,
least_significant_digit=None,
):
"""Write a standard RVIC Parameter file """
# ---------------------------------------------------------------- #
# netCDF variable options
ncvaropts = {"zlib": zlib, "complevel": complevel, "least_significant_digit": least_significant_digit}
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# Open file
f = Dataset(file_name, "w", format=nc_format)
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# Time Variables
# Timesteps
timesteps = f.createDimension("timesteps", subset_length)
timesteps = f.createVariable("timesteps", NC_DOUBLE, ("timesteps",), **ncvaropts)
timesteps[:] = np.arange(subset_length)
for key, val in iteritems(share.timesteps):
if val:
setattr(timesteps, key, val.encode())
timesteps.timestep_length = b"unit_hydrograph_dt"
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# write global attributes
glob_atts.update()
for key, val in iteritems(glob_atts.atts):
if val:
setattr(f, key, val.encode())
f.featureType = b"timeSeries"
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# 0-D variables
# Full time length (size of ring)
ftl = f.createVariable("full_time_length", NC_INT, (), **ncvaropts)
ftl[:] = full_time_length
for key, val in iteritems(share.full_time_length):
if val:
setattr(ftl, key, val.encode())
# Subset Length
sl = f.createVariable("subset_length", NC_INT, (), **ncvaropts)
sl[:] = subset_length
for key, val in iteritems(share.subset_length):
if val:
setattr(sl, key, val.encode())
# UH timestep
uh_dt = f.createVariable("unit_hydrograph_dt", NC_DOUBLE, (), **ncvaropts)
uh_dt[:] = unit_hydrograph_dt
for key, val in iteritems(share.unit_hydrograph_dt):
if val:
setattr(uh_dt, key, val.encode())
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# Outlet Dimensions
if outlet_y_ind.ndim == 0:
numoutlets = 1
outlet_name = np.array([outlet_name])
else:
numoutlets = len(outlet_lon)
ocoords = ("outlets",)
f.createDimension(ocoords[0], numoutlets)
nocoords = ocoords + ("nc_chars",)
char_names = stringtochar(outlet_name)
f.createDimension(nocoords[1], char_names.shape[1])
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# 1-D Outlet Variables
# Outlet Cell Longitudes
olon = f.createVariable("outlet_lon", NC_DOUBLE, ocoords, **ncvaropts)
olon[:] = outlet_lon
for key, val in iteritems(share.outlet_lon):
if val:
setattr(olon, key, val.encode())
# Outlet Cell Latitudes
olat = f.createVariable("outlet_lat", NC_DOUBLE, ocoords, **ncvaropts)
olat[:] = outlet_lat
for key, val in iteritems(share.outlet_lat):
if val:
setattr(olat, key, val.encode())
# Outlet Cell X Indicies
oxi = f.createVariable("outlet_x_ind", NC_INT, ocoords, **ncvaropts)
oxi[:] = outlet_x_ind
for key, val in iteritems(share.outlet_x_ind):
if val:
setattr(oxi, key, val.encode())
# Outlet Cell Y Indicies
oyi = f.createVariable("outlet_y_ind", NC_INT, ocoords, **ncvaropts)
oyi[:] = outlet_y_ind
for key, val in iteritems(share.outlet_y_ind):
if val:
setattr(oyi, key, val.encode())
# Outlet Cell Decomp IDs
odi = f.createVariable("outlet_decomp_ind", NC_INT, ocoords, **ncvaropts)
odi[:] = outlet_decomp_ind
for key, val in iteritems(share.outlet_decomp_ind):
if val:
setattr(odi, key, val.encode())
# Outlet Cell Number
on = f.createVariable("outlet_number", NC_INT, ocoords, **ncvaropts)
on[:] = outlet_number
for key, val in iteritems(share.outlet_number):
if val:
setattr(on, key, val.encode())
# Outlet Mask
om = f.createVariable("outlet_mask", NC_INT, ocoords, **ncvaropts)
om[:] = outlet_mask
for key, val in iteritems(share.outlet_mask):
if val:
setattr(om, key, val.encode())
# Outlet Upstream area
oua = f.createVariable("outlet_upstream_area", NC_DOUBLE, ocoords, **ncvaropts)
oua[:] = outlet_upstream_area
for key, val in iteritems(share.outlet_upstream_area):
if val:
setattr(oua, key, val.encode())
# Outlet Upstream grid cells
oug = f.createVariable("outlet_upstream_gridcells", NC_INT, ocoords, **ncvaropts)
oug[:] = outlet_upstream_gridcells
for key, val in iteritems(share.outlet_upstream_gridcells):
if val:
setattr(oug, key, val.encode())
# Outlet Names
onm = f.createVariable("outlet_name", NC_CHAR, nocoords)
onm[:, :] = char_names
for key, val in iteritems(share.outlet_name):
if val:
setattr(onm, key, val.encode())
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# Source Dimension
scoords = ("sources",)
f.createDimension(scoords[0], len(source_lon))
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# 1D Source Variables
# Source Cell Longitudes
slon = f.createVariable("source_lon", NC_DOUBLE, scoords, **ncvaropts)
slon[:] = source_lon
for key, val in iteritems(share.source_lon):
if val:
setattr(slon, key, val.encode())
# Source Cell Latitudes
slat = f.createVariable("source_lat", NC_DOUBLE, scoords, **ncvaropts)
slat[:] = source_lat
for key, val in iteritems(share.source_lat):
if val:
setattr(slat, key, val.encode())
# Source Cell X Indicies
sxi = f.createVariable("source_x_ind", NC_INT, scoords, **ncvaropts)
sxi[:] = source_x_ind
for key, val in iteritems(share.source_x_ind):
if val:
setattr(sxi, key, val.encode())
# Source Cell Y Indicies
syi = f.createVariable("source_y_ind", NC_INT, scoords, **ncvaropts)
syi[:] = source_y_ind
for key, val in iteritems(share.source_y_ind):
if val:
setattr(syi, key, val.encode())
# Source Cell Decomp IDs
sdi = f.createVariable("source_decomp_ind", NC_INT, scoords, **ncvaropts)
sdi[:] = source_decomp_ind
for key, val in iteritems(share.source_decomp_ind):
if val:
setattr(sdi, key, val.encode())
# Source Cell Time Offset
sto = f.createVariable("source_time_offset", NC_INT, scoords, **ncvaropts)
sto[:] = source_time_offset
for key, val in iteritems(share.source_time_offset):
if val:
setattr(sto, key, val.encode())
# Source to Outlet Index Mapping
s2o = f.createVariable("source2outlet_ind", NC_INT, scoords, **ncvaropts)
s2o[:] = source2outlet_ind
for key, val in iteritems(share.source2outlet_ind):
if val:
setattr(s2o, key, val.encode())
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# 3-D Source Variables
uhcords = ("timesteps",) + scoords + ("tracers",)
f.createDimension(uhcords[2], 1)
# Unit Hydrographs
uhs = f.createVariable("unit_hydrograph", NC_DOUBLE, uhcords, **ncvaropts)
uhs[:, :] = unit_hydrograph
for key, val in iteritems(share.unit_hydrograph):
if val:
setattr(uhs, key, val.encode())
# ---------------------------------------------------------------- #
f.close()
log.info("Finished writing %s", file_name)