forked from ledm/NetCDF_manip
/
convertToOneDNC.py
213 lines (177 loc) · 7.35 KB
/
convertToOneDNC.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
try: from netCDF4 import Dataset, default_fillvals
except: from netCDF4 import Dataset, _default_fillvals
from datetime import date
from getpass import getuser
from os.path import exists
from numpy import array, int64
from numpy.ma import array as marray, nonzero,masked_where,compressed
from pruneNC import todaystr
from operator import itemgetter
""" This routine takes a netcdf input and creates a new one with only 1 dimension. """
# list of variables to save, assuming some conventions
alwaysInclude = ['time', 'lat','lon', 'latbnd', 'lonbnd','LONGITUDE','LATITUDE','DEPTH','TIME','nav_lat','nav_lon','time_counter','deptht','depth', ]
#'crs',]'lat_bnds','lon_bnds',
def getCoordsToKeep(nc,variables,newMask='',debug = False):
"""This routine takes an ncdfView object, a list of varialbes, and tests which coordinates should be saved.
A Mask can be applied too.
"""
CoordsToKeep={}
for var in variables:
if var in alwaysInclude: continue
arr = nc.variables[var][:]
if len(newMask):
out = []
if newMask.shape != arr.shape:
if debug:'getCoordsToKeep:\t',var,'Wrong shape'
continue
try: # 1D arrays
for i,m in enumerate(newMask):
if not m: continue
out.append(arr[i])
arr= array(out).squeeze()
except:
# multi dimensional arrays
arr = masked_where(newMask,array(arr))
#nearlyZero = 1.E-6
nz = nonzero(arr)#+nearlyZero
#nz = compressed(arr)
#if debug:print "getCoordsToKeep:\tcompressed array:",len(nz),nz.shape,nz.min(),nz.max(),nz.mean()
if not len(nz):
variables.remove(var)
continue
nzdims = len(nz)
for i,a in enumerate(nz[0]):
if var in ['OBSERVATIONS']:
coords = tuple([nz[j][i] for j in xrange(nzdims)])
#if coords not in CoordsToKeep.keys():
# print "NEWS OBSERVATION LOCATION:",i,coords
CoordsToKeep[coords] = i
else:
coords = tuple([nz[j][i] for j in xrange(nzdims)])
CoordsToKeep[coords] = i
if debug: print "getCoordsToKeep:\t",var,"\tndims:", nzdims, len(nz[0]),"\tNumber of Coords:", len(CoordsToKeep.keys())
return CoordsToKeep,variables
class convertToOneDNC:
def __init__(self, filenameIn, filenameOut, newMask='', variables=[], debug=False,dictToKeep=''):
self.fni=filenameIn
self.fno=filenameOut
self.vars=variables
self.newMask=newMask
self.debug=debug
self.dictToKeep = dictToKeep
self.run()
def run(self):
if not exists(self.fni):
print 'convertToOneDNC:\tERROR:\tinputfile name does not exists:', self.fni
return
nci = Dataset(self.fni,'r')
if not self.vars:
self.vars = nci.variables.keys() # save all
#check that there are some overlap between input vars and nci:
for v in self.vars:
if v in nci.variables.keys():continue
print 'convertToOneDNC:\tERROR:\tvariable,' ,v,', not found in ',self.fni
return
#create dataset and header.
if self.debug: print 'convertToOneDNC:\tINFO:\tCreating a new dataset:\t', self.fno
nco = Dataset(self.fno,'w')
for a in nci.ncattrs():
if self.debug: print 'convertToOneDNC:\tINFO:\tcopying attribute: \t\"'+a+'\":\t', nci.getncattr(a)
nco.setncattr(a,nci.getncattr(a))
appendToDesc= 'Reprocessed on '+todaystr()+' by '+getuser()+' using convertToOneDNC.py'
try: nco.Notes = nci.Notes + '\n\t\t'+appendToDesc
except: nco.Notes = appendToDesc
save = list(set(nci.variables.keys()).intersection(set(alwaysInclude) ) )
save = sorted(list(set(sorted(save + self.vars))))
# test to find out which coordinates should be saved.
if not self.dictToKeep:
CoordsToKeep,save=getCoordsToKeep(nci,save,newMask=self.newMask,debug = self.debug)
else:
CoordsToKeep = self.dictToKeep
# create dimensions:
#for d in nci.dimensions.keys():
# if d in ['time',]: nco.createDimension(d, None)
# else: nco.createDimension(d, len(nci.dimensions[d]))
nco.createDimension('index', None)
# create Variables:
nco.createVariable('index', int64, ['index',],zlib=True,complevel=5)#,chunksizes=10000)
nco.createVariable('index_t', int64, ['index',],zlib=True,complevel=5)#,chunksizes=10000)
nco.createVariable('index_z', int64, ['index',],zlib=True,complevel=5)#,chunksizes=10000)
nco.createVariable('index_y', int64, ['index',],zlib=True,complevel=5)#,chunksizes=10000)
nco.createVariable('index_x', int64, ['index',],zlib=True,complevel=5)#,chunksizes=10000)
for var in save:
nco.createVariable(var, nci.variables[var].dtype, ['index',],zlib=True,complevel=5)#,chunksizes=10000)
# Long Names:
nco.variables['index'].long_name='index'
nco.variables['index_t'].long_name='index - time'
nco.variables['index_z'].long_name='index - depth'
nco.variables['index_y'].long_name='index - latitude'
nco.variables['index_x'].long_name='index - longitude'
for var in save:
try: long_name=nci.variables[var].long_name
except:
if self.debug: print 'convertToOneDNC:\tWarning:\tNo long_name for ', var
long_name = var
nco.variables[var].long_name=long_name
if self.debug: print 'convertToOneDNC:\t Adding long_name for ', var, long_name
# Units:
nco.variables['index'].units=''
nco.variables['index_t'].units=''
nco.variables['index_z'].units=''
nco.variables['index_y'].units=''
nco.variables['index_x'].units=''
for var in save:
try: nco.variables[var].units=nci.variables[var].units
except: print 'convertToOneDNC:\tWarning:\tNo units for ', var
# Fill Values:
sorted_Coords = sorted(CoordsToKeep.iteritems(), key=itemgetter(1))
data={}
if self.debug: print 'convertToOneDNC:\tINFO:\tCopying index ...' , len(sorted_Coords)
# nco.variables['index'][:] = [ int(a[1]) for a in sorted_Coords]
nco.variables['index'][:] = array([ a[1] for a in sorted_Coords])
nco.sync()
if self.debug: print 'convertToOneDNC:\tINFO:\tCopying index t ...'
nco.variables['index_t'][:] = array([a[0][0] for a in sorted_Coords])
nco.sync()
if self.debug: print 'convertToOneDNC:\tINFO:\tCopying index z ...'
nco.variables['index_z'][:] = array([a[0][1] for a in sorted_Coords])
nco.sync()
if self.debug: print 'convertToOneDNC:\tINFO:\tCopying index y ...'
nco.variables['index_y'][:] = array([a[0][2] for a in sorted_Coords])
nco.sync()
if self.debug: print 'convertToOneDNC:\tINFO:\tCopying index x ...'
nco.variables['index_x'][:] = array([a[0][3] for a in sorted_Coords])
nco.sync()
for var in save:
if self.debug: print 'convertToOneDNC:\tINFO:\tCopying ', var, ' ...'
arr = nci.variables[var][:]
outarr = []
if arr.ndim ==1:
if var.lower() in ['time','time_counter','t']: d = 0
if var.lower() in ['depth','deptht',]: d = 1
if var.lower() in ['latbnd','lat','latitude']: d = 2
if var.lower() in ['lonbnd','lon','longitude']:d = 3
#for c in (CoordsToKeep.keys()):
for c in sorted_Coords:
outarr.append(arr[c[0][d]])
try: print var, d
except: var, "not found"
elif arr.ndim ==2:
if var.lower() in ['nav_lat','nav_lon']: d = (2,3)
print var, 'lendth : 2', d
for c in sorted_Coords:
#for c in sorted(CoordsToKeep.keys()):
outarr.append(arr[(c[0][2:])])
else:
#for c in sorted(CoordsToKeep.keys()):
for c in sorted_Coords:
outarr.append(arr[c[0]])
outarr= marray(outarr)
if self.debug: print 'convertToOneDNC:\tINFO:\tSaving var:',var, arr.shape, '->', outarr.shape , 'coords:',len(sorted_Coords)
nco.variables[var][:] =outarr
nco.sync()
# Close netcdfs:
nco.close()
nci.close()
if self.debug: print 'convertToOneDNC:\tINFO:\tsuccessfully created:\t', self.fno
return