def read_data(self):
# open a new netCDF file for reading.
ncfile = NetCDFFile(self.file,'r')
dimNames = ncfile.dimensions.keys()
variableNames = ncfile.variables.keys()
#print dimNames
#print variableNames
# Get the geolocation data
intime = ncfile.variables['time']
inlatitude = ncfile.variables['latitude']
inlongitude = ncfile.variables['longitude']
print 'time =', N.array(intime)
self.intime_size = N.size(intime)
self.inlat_size = N.size(inlatitude)
self.inlon_size = N.size(inlongitude)
#print 'original time size = ', self.intime_size
#print 'original latitude size = ', self.inlat_size
#print 'original longitude size =', self.inlon_size
# Here we have to 'expand out' lat, lon and time so that the
# middle end is permitted to function as it does for irregular
# grids. This involves quite a bit of repititive values and
# increased memory consumption.
new_var_time = N.array([])
new_var_lat = N.array([])
new_var_lon = N.array([])
coloc_param_size = self.intime_size * self.inlat_size * self.inlon_size
new_var_time.resize(coloc_param_size)
new_var_lat.resize(coloc_param_size)
new_var_lon.resize(coloc_param_size)
#print 'new time size = ', new_time_size
#print 'new latitude size = ', new_lat_size
#print 'new longitude size =', new_lon_size
# Now that we have both the original representation of the geolocation params
# and a properly-sized location to put their expanded representations - do the
# expansion
i = j = k = m = 0
for i in range(0, self.intime_size):
# Convert hours since 01/01/1900 to unix time
tk = UT.hours_since_20th_century_to_unix_time(intime[i])
for j in range(0, self.inlat_size):
lat_tk = inlatitude[j]
for k in range(0, self.inlon_size):
lon_tk = inlongitude[k]
new_var_time[m] = tk
new_var_lat[m] = lat_tk
new_var_lon[m] = lon_tk
# Show leading edge of expanded colocation params
#if(m < 242):
#print m, ": ", tk, ", ", lat_tk, ", ", lon_tk
m += 1
self.time = new_var_time
self.latitude = new_var_lat
### convert to (-180,180) so it's consistent with CloudSat
### self.longitude = new_var_lon
self.longitude = N.where(new_var_lon > 180.0, new_var_lon - 360.0, new_var_lon)
# End of colocation parameter expansion for regular grids
#print 'new time size = ', N.size(self.time)
#print 'new latitude size = ', N.size(self.latitude)
#print 'new longitude size =', N.size(self.longitude)
for i in range(len(variableNames)):
if(variableNames[i]!='time' and variableNames[i]!='latitude' and variableNames[i]!='longitude'):
local_var = ncfile.variables[variableNames[i]]
# get data of the variable
local_data = N.array(local_var.getValue())
# get attributes of the variable
attList = dir(local_var)
attribute = {}
for j in attList:
if(j!='assignValue' and j!='getValue' and j!='typecode'):
attValue = getattr(local_var, j)
attribute[j]=attValue
#print j, attValue
#print variableNames[i], "attribute = ", attribute
# collect indices of data with missing values or filled value
N1 = local_var.shape[0]
N2 = local_var.shape[1]
N3 = local_var.shape[2]
#print 'n1,n2,n3 = ', N1, N2, N3
#print 'local data dimension', local_data.shape
if(attribute['_FillValue']!= attribute['missing_value']):
print "fill value differ from missing value"
print attribute['_FillValue']
print attribute['missing_value']
print attribute['_FillValue'].shape
# find all the indices of the local data whose values are invalid/missing
missing_value1 = N.where(local_data == attribute['_FillValue'])
missing_value2 = N.where(local_data == attribute['missing_value'])
print 'missing_value1=', missing_value1
# update data with scale_factor and add_offset
local_data = local_data*attribute['scale_factor']+attribute['add_offset']
# update missing data value
local_data[missing_value1] = UT.NAN
local_data[missing_value2] = UT.NAN
# remove unnecessary attributes since we already applied scale_factor, add_offset, missing_value, and fillvalue.
del attribute['_FillValue']
del attribute['scale_factor']
del attribute['add_offset']
del attribute['missing_value']
# Now we reshaped have to reshape to 1D for the middle end
oneD_data = N.reshape(local_data, local_data.shape[0]*local_data.shape[1]*local_data.shape[2])
# store (attribute, data) in the data dictionary
self.data[variableNames[i]]=(attribute, oneD_data)
#print "attribute = ", attribute
#print "data = ", oneD_data
# Dynamically spot check our flattening of the 3D array such that is is
# straight forwardly indexed into using time, lat and lon fields (after
# expansion).
if 0:
testx = 8 # Valid in range 0 to size of time in original data
testy = 42 # Valid in range 0 to size of lons in original data
testz = 56 # Valid in range 0 to size of lats in original data
if(local_data[testx][testy][testz] != oneD_data[testx*240*121 + testy*240 + testz]):
print 'Flattening of 3D array failed'
print 'sample point data3d[x][y][z] = ', local_data[testx][testy][testz]
print 'sample point data1d[x*240*121 + y*240 + z] = ', oneD_data[testx*240*121 + testy*240 + testz]
sys.exit(-1)
ncfile.close()
def read_data(self):
# open a new netCDF file for reading.
ncfile = NetCDFFile(self.file, 'r')
dimNames = ncfile.dimensions.keys()
variableNames = ncfile.variables.keys()
#print dimNames
#print variableNames
# Get the geolocation data
intime = ncfile.variables['time']
inlatitude = ncfile.variables['latitude']
inlongitude = ncfile.variables['longitude']
print 'time =', N.array(intime)
self.intime_size = N.size(intime)
self.inlat_size = N.size(inlatitude)
self.inlon_size = N.size(inlongitude)
#print 'original time size = ', self.intime_size
#print 'original latitude size = ', self.inlat_size
#print 'original longitude size =', self.inlon_size
# Here we have to 'expand out' lat, lon and time so that the
# middle end is permitted to function as it does for irregular
# grids. This involves quite a bit of repititive values and
# increased memory consumption.
new_var_time = N.array([])
new_var_lat = N.array([])
new_var_lon = N.array([])
coloc_param_size = self.intime_size * self.inlat_size * self.inlon_size
new_var_time.resize(coloc_param_size)
new_var_lat.resize(coloc_param_size)
new_var_lon.resize(coloc_param_size)
#print 'new time size = ', new_time_size
#print 'new latitude size = ', new_lat_size
#print 'new longitude size =', new_lon_size
# Now that we have both the original representation of the geolocation params
# and a properly-sized location to put their expanded representations - do the
# expansion
i = j = k = m = 0
for i in range(0, self.intime_size):
# Convert hours since 01/01/1900 to unix time
tk = UT.hours_since_20th_century_to_unix_time(intime[i])
for j in range(0, self.inlat_size):
lat_tk = inlatitude[j]
for k in range(0, self.inlon_size):
lon_tk = inlongitude[k]
new_var_time[m] = tk
new_var_lat[m] = lat_tk
new_var_lon[m] = lon_tk
# Show leading edge of expanded colocation params
#if(m < 242):
#print m, ": ", tk, ", ", lat_tk, ", ", lon_tk
m += 1
self.time = new_var_time
self.latitude = new_var_lat
### convert to (-180,180) so it's consistent with CloudSat
### self.longitude = new_var_lon
self.longitude = N.where(new_var_lon > 180.0, new_var_lon - 360.0,
new_var_lon)
# End of colocation parameter expansion for regular grids
#print 'new time size = ', N.size(self.time)
#print 'new latitude size = ', N.size(self.latitude)
#print 'new longitude size =', N.size(self.longitude)
for i in range(len(variableNames)):
if (variableNames[i] != 'time' and variableNames[i] != 'latitude'
and variableNames[i] != 'longitude'):
local_var = ncfile.variables[variableNames[i]]
# get data of the variable
local_data = N.array(local_var.getValue())
# get attributes of the variable
attList = dir(local_var)
attribute = {}
for j in attList:
if (j != 'assignValue' and j != 'getValue'
and j != 'typecode'):
attValue = getattr(local_var, j)
attribute[j] = attValue
#print j, attValue
#print variableNames[i], "attribute = ", attribute
# collect indices of data with missing values or filled value
N1 = local_var.shape[0]
N2 = local_var.shape[1]
N3 = local_var.shape[2]
#print 'n1,n2,n3 = ', N1, N2, N3
#print 'local data dimension', local_data.shape
if (attribute['_FillValue'] != attribute['missing_value']):
print "fill value differ from missing value"
print attribute['_FillValue']
print attribute['missing_value']
print attribute['_FillValue'].shape
# find all the indices of the local data whose values are invalid/missing
missing_value1 = N.where(local_data == attribute['_FillValue'])
missing_value2 = N.where(
local_data == attribute['missing_value'])
print 'missing_value1=', missing_value1
# update data with scale_factor and add_offset
local_data = local_data * attribute[
'scale_factor'] + attribute['add_offset']
# update missing data value
local_data[missing_value1] = UT.NAN
local_data[missing_value2] = UT.NAN
# remove unnecessary attributes since we already applied scale_factor, add_offset, missing_value, and fillvalue.
del attribute['_FillValue']
del attribute['scale_factor']
del attribute['add_offset']
del attribute['missing_value']
# Now we reshaped have to reshape to 1D for the middle end
oneD_data = N.reshape(
local_data, local_data.shape[0] * local_data.shape[1] *
local_data.shape[2])
# store (attribute, data) in the data dictionary
self.data[variableNames[i]] = (attribute, oneD_data)
#print "attribute = ", attribute
#print "data = ", oneD_data
# Dynamically spot check our flattening of the 3D array such that is is
# straight forwardly indexed into using time, lat and lon fields (after
# expansion).
if 0:
testx = 8 # Valid in range 0 to size of time in original data
testy = 42 # Valid in range 0 to size of lons in original data
testz = 56 # Valid in range 0 to size of lats in original data
if (local_data[testx][testy][testz] !=
oneD_data[testx * 240 * 121 + testy * 240 +
testz]):
print 'Flattening of 3D array failed'
print 'sample point data3d[x][y][z] = ', local_data[
testx][testy][testz]
print 'sample point data1d[x*240*121 + y*240 + z] = ', oneD_data[
testx * 240 * 121 + testy * 240 + testz]
sys.exit(-1)
ncfile.close()
def read_data(self):
# open a new netCDF file for reading.
ncfile = NetCDFFile(self.file,'r')
dimNames = ncfile.dimensions.keys()
variableNames = ncfile.variables.keys()
#print dimNames
#print variableNames
# Get the geolocation data
intime = ncfile.variables['time']
inlatitude = ncfile.variables['latitude']
inlongitude = ncfile.variables['longitude']
print 'time = ', N.array(intime)
self.intime_size = N.size(intime)
self.inlat_size = N.size(inlatitude)
self.inlon_size = N.size(inlongitude)
# print 'original time size = ', self.intime_size
# print 'original latitude size = ', self.inlat_size
# print 'original longitude size =', self.inlon_size
# Here we have to 'expand out' lat, lon and time so that the
# middle end is permitted to function as it does for irregular
# grids. This involves quite a bit of repititive values and
# increased memory consumption.
new_var_time = N.array([])
new_var_lat = N.array([])
new_var_lon = N.array([])
coloc_param_size = self.intime_size * self.inlat_size * self.inlon_size
new_var_time.resize(coloc_param_size)
new_var_lat.resize(coloc_param_size)
new_var_lon.resize(coloc_param_size)
#print 'new time size = ', new_time_size
#print 'new latitude size = ', new_lat_size
#print 'new longitude size =', new_lon_size
# Now that we have both the original representation of the geolocation params
# and a properly-sized location to put their expanded representations - do the
# expansion
i = j = k = m = 0
for i in range(0, self.intime_size):
# Convert hours since 01/01/1900 to unix time
tk = UT.hours_since_20th_century_to_unix_time(intime[i])
for j in range(0, self.inlat_size):
lat_tk = inlatitude[j]
for k in range(0, self.inlon_size):
lon_tk = inlongitude[k]
new_var_time[m] = tk
new_var_lat[m] = lat_tk
new_var_lon[m] = lon_tk
# Show leading edge of expanded colocation params
#if(m < 242):
#print m, ": ", tk, ", ", lat_tk, ", ", lon_tk
m += 1
self.time = new_var_time
self.latitude = new_var_lat
### convert to (-180,180) so it's consistent with CloudSat
### self.longitude = new_var_lon
self.longitude = N.where(new_var_lon > 180.0, new_var_lon - 360.0, new_var_lon)
# End of colocation parameter expansion for regular grids
level_name = ['levelist']
for i in range(len(level_name)):
local_level = ncfile.variables[level_name[i]]
level_data = local_level.getValue()
attList = dir(local_level)
attribute = {}
for j in attList:
if(j!='assignValue' and j!='getValue' and j!='typecode'):
attValue = getattr(local_level, j)
attribute[j] = attValue
# add dimension1 name in the attribute list
attribute['dimension1']='pressure_level'
# Store (attribute, data) in the level dictionary
self.levels[level_name[i]]=(attribute, level_data)
#print 'new time size = ', N.size(self.time)
#print 'new latitude size = ', N.size(self.latitude)
#print 'new longitude size =', N.size(self.longitude)
for i in range(len(variableNames)):
if(variableNames[i]!='time'
and variableNames[i]!='latitude'
and variableNames[i]!='longitude'
and variableNames[i]!='levelist'):
local_var = ncfile.variables[variableNames[i]]
# get data of the variable
local_data = N.array(local_var.getValue())
# get attributes of the variable
attList = dir(local_var)
attribute = {}
for j in attList:
if(j!='assignValue' and j!='getValue' and j!='typecode'):
attValue = getattr(local_var, j)
attribute[j]=attValue
#print j, attValue
#print variableNames[i], "attribute = ", attribute
# collect indices of data with missing values or filled value
N1 = local_var.shape[0]
N2 = local_var.shape[1]
N3 = local_var.shape[2]
N4 = local_var.shape[3]
#print 'n1,n2,n3.n4 = ', N1, N2, N3, N4
#print 'local data dimension', local_data.shape
if(attribute['_FillValue']!= attribute['missing_value']):
print "fill value differ from missing value"
print attribute['_FillValue']
print attribute['missing_value']
print attribute['_FillValue'].shape
# find all the indices of the local data whose values are invalid/missing
missing_value1 = N.where(local_data == attribute['_FillValue'])
missing_value2 = N.where(local_data == attribute['missing_value'])
# update data with scale_factor and add_offset
local_data = local_data*attribute['scale_factor']+attribute['add_offset']
# update missing data value
local_data[missing_value1] = UT.NAN
local_data[missing_value2] = UT.NAN
# remove unnecessary attributes since we already applied scale_factor, add_offset, missing_value, and fillvalue.
del attribute['_FillValue']
del attribute['scale_factor']
del attribute['add_offset']
del attribute['missing_value']
# add attribute for dimension name
attribute['dimension1']='pressure_level'
# Now we have to swap the axis of array to make the p-level to be the last axis
local_data = N.swapaxes(local_data, 1, 2) # swap the axes of p-level and latitude
local_data = N.swapaxes(local_data, 2, 3) # swap the axes of p-level and longitude
# Now the array has the axis in this order (time, lat, lon, p-level)
# Now we have to reshape the four-dimensional array to 2-dimensional array for middle end
print local_data.shape
twoD_data = N.reshape(local_data, (local_data.shape[0]*local_data.shape[1]*local_data.shape[2],local_data.shape[3]))
# store (attribute, data) in the data dictionary
self.data[variableNames[i]]=(attribute, twoD_data)
#print "attribute = ", attribute
#print "data = ", twoD_data
ncfile.close()
