def process(self):
if plotting:
debug(2, "Extracting...")
netcdf_file = netCDF.MFDataset(self.files, aggdim='time')
time_var = netcdf_file.variables['time']
data_var = netcdf_file.variables[self.variable]
times = time_var[:]
if time_var.units:
times = [
datetime.datetime.strptime(
netCDF.num2date(x, time_var.units,
calendar='standard').isoformat(),
"%Y-%m-%dT%H:%M:%S") for x in times
]
else:
# the time variable doesn't have units; this can be caused by a thredds aggregation that uses dateFormatMark
# to grab the date from the filename, in which case the date is an array of strings
times = [
datetime.datetime.strptime(x.tostring(), "%Y-%m-%dT%H:%M:%SZ")
for x in times
]
with open(self._csv, "rb") as csvfile:
csv_file = csvfile.read()
with open(self._csv, "rb") as csvfile:
self.numline = len(csvfile.readlines())
data = csv.DictReader(csv_file.splitlines(), delimiter=',')
ret = []
times_sorted_indexes = np.argsort(times)
times_sorted = np.sort(times)
average_time_interval = (times_sorted[-1] - times_sorted[0]) / len(
set(times_sorted))
lat_var = getCoordinateVariable(netcdf_file, "Lat")[:]
lon_var = getCoordinateVariable(netcdf_file, "Lon")[:]
if np.amax(lat_var) > 90:
for i, lat in enumerate(lat_var):
if lat > 90:
lat_var[i] = lat - 180
if np.amax(lon_var) > 180:
for i, lon in enumerate(lon_var):
if lon > 180:
lon_var[i] = lon - 360
lat_end = len(lat_var) - 1
lat_offset = (lat_var[lat_end] - lat_var[0]) / lat_end
lon_end = len(lon_var) - 1
lon_offset = (lon_var[lon_end] - lon_var[0]) / lon_end
# Calculate the distance from the centre of a pixel to a corner
offset_distance = calculateDistance(0, 0, lat_offset, lon_offset) / 2
self.start_time = time.clock()
self.last_time = time.clock()
for row in data:
if len(lat_var) <= 1:
lat_index = 0
else:
current_lat = float(row['Latitude'])
t_lat = current_lat - lat_var[0]
lat_index = int(round(abs(t_lat / lat_offset)))
if len(lon_var) <= 1:
lon_index = 0
else:
current_lon = float(row['Longitude'])
t_lon = current_lon - lon_var[0]
lon_index = int(round(abs(t_lon / lon_offset)))
try:
track_date = datetime.datetime.strptime(
row['Date'], "%d/%m/%Y %H:%M:%S")
except ValueError:
track_date = datetime.datetime.strptime(
row['Date'], "%d/%m/%Y %H:%M")
time_index = find_closest(times_sorted,
track_date,
arr_indexes=times_sorted_indexes,
time=True,
arr_sorted=True)
if lat_index > lat_end:
lat_index = lat_end
if lon_index > lon_end:
lon_index = lon_end
# Calculate the distance from the desired point to the centre of the chosen pixel
distance_from_desired = calculateDistance(current_lat, current_lon,
lat_var[lat_index],
lon_var[lon_index])
if distance_from_desired > offset_distance:
# If the distance is greater than the offset distance then something has gone wrong
# and the wrong pixel has been chosen.
# Set the value to NaN to avoid returning an incorrect result
data_value = float('nan')
if plotting:
debug(
0,
"Incorrect pixel selected! Pixel at {:+07.3f}, {:+08.3f} is further than {:6.2f}km from point at {:+07.3f}, {:+08.3f} ({:8.2f}km). Setting {} value to NaN."
.format(lat_var[lat_index], lon_var[lon_index],
offset_distance, current_lat, current_lon,
distance_from_desired, self.variable))
elif abs(times[time_index] - track_date) > (2 *
average_time_interval):
data_value = float('nan')
elif len(data_var.dimensions) == 4:
# If the file has a depth variable, use the first depth
data_value = data_var[time_index][0][lat_index][lon_index]
else:
data_value = data_var[time_index][lat_index][lon_index]
_ret = {}
_ret['track_date'] = track_date.isoformat()
if time_var.units:
_ret['data_date'] = netCDF.num2date(
time_var[time_index], time_var.units,
calendar='standard').isoformat()
else:
_ret['data_date'] = time_var[time_index].tostring()
if self.matchup:
_ret['match_value'] = row['data_point']
_ret['track_lat'] = row['Latitude']
_ret['track_lon'] = row['Longitude']
_ret['data_value'] = float(data_value) if not np.isnan(
float(data_value)) else "null"
ret.append(_ret)
if plotting and self.status_details:
self.update_status(len(ret))
return ret
def process(self):
#print "running basic processing on %s" % self.filename
# create three arrays, 1D lat 1D lon 2D data
#print "processing image"
netcdf_file = netCDF.Dataset(self.filename, "r")
#variable = np.ma.masked_array(netcdf_file.variables[self.variable])
variable = np.ma.array(netcdf_file.variables[self.variable][:])
lats = getCoordinateVariable(netcdf_file, "Lat")
lons = getCoordinateVariable(netcdf_file, "Lon")
time_dim_index = netcdf_file.variables[self.variable].dimensions.index(
'time')
if 'depth' in netcdf_file.variables[self.variable].dimensions:
depth_dim_index = netcdf_file.variables[
self.variable].dimensions.index('depth')
var_list = []
lat_list = []
lon_list = []
#print variable.shape
if (len(variable.shape) > 3):
#print "hmmmm"
#print variable.shape
#print variable
#print np.nanmean(variable, axis=time_dim_index).shape
var_list = [[
float(x) if not np.isinf(x) and not np.isnan(x) else None
for x in y
] for y in np.nanmean(variable, axis=time_dim_index)[0]]
#print var_list
lat_list = [float(x) for x in lats]
lon_list = [float(x) for x in lons]
elif (len(variable.shape) > 2):
#print variable.shape
#print variable
#print np.nanmean(variable, axis=time_dim_index)
var_list = [[
float(x) if not np.isinf(x) and not np.isnan(x) else None
for x in y
] for y in np.nanmean(variable, axis=time_dim_index)]
#var_list = [[float(x) for x in y] for y in variable[0]]
#print var_list
lat_list = [float(x) for x in lats]
lon_list = [float(x) for x in lons]
else:
var_list = [list(x) for x in variable]
lat_list = [float(x) for x in lats]
lon_list = [float(x) for x in lons]
#print len(lat_list)
#print len(lon_list)
#print len(var_list)
#print len(var_list[0])
#print lat_list
_ret = {}
_ret['vars'] = ['Data', 'Latitudes', 'Longitudes']
_ret['data'] = []
_ret['data'].append(var_list)
_ret['data'].append(lat_list)
_ret['data'].append(lon_list)
#print json.dumps(_ret )
return json.dumps(_ret)
def process(self):
if plotting:
debug(2, "Extracting...")
netcdf_file = netCDF.MFDataset(self.files, aggdim='time')
time_var = netcdf_file.variables['time']
data_var = netcdf_file.variables[self.variable]
times = time_var[:]
if time_var.units:
times = [datetime.datetime.strptime(netCDF.num2date(x, time_var.units, calendar='standard').isoformat(), "%Y-%m-%dT%H:%M:%S") for x in times]
else:
# the time variable doesn't have units; this can be caused by a thredds aggregation that uses dateFormatMark
# to grab the date from the filename, in which case the date is an array of strings
times = [datetime.datetime.strptime(x.tostring(), "%Y-%m-%dT%H:%M:%SZ") for x in times]
with open(self._csv, "rb") as csvfile:
csv_file = csvfile.read()
with open(self._csv, "rb") as csvfile:
self.numline = len(csvfile.readlines())
data = csv.DictReader(csv_file.splitlines(), delimiter=',')
ret = []
times_sorted_indexes = np.argsort(times)
times_sorted = np.sort(times)
average_time_interval = (times_sorted[-1] - times_sorted[0]) / len(set(times_sorted))
lat_var = getCoordinateVariable(netcdf_file, "Lat")[:]
lon_var = getCoordinateVariable(netcdf_file, "Lon")[:]
if np.amax(lat_var) > 90:
for i, lat in enumerate(lat_var):
if lat > 90:
lat_var[i] = lat - 180
if np.amax(lon_var) > 180:
for i, lon in enumerate(lon_var):
if lon > 180:
lon_var[i] = lon - 360
lat_end = len(lat_var) - 1
lat_offset = (lat_var[lat_end] - lat_var[0]) / lat_end
lon_end = len(lon_var) - 1
lon_offset = (lon_var[lon_end] - lon_var[0]) / lon_end
# Calculate the distance from the centre of a pixel to a corner
offset_distance = calculateDistance(0, 0, lat_offset, lon_offset) / 2
self.start_time = time.clock()
self.last_time = time.clock()
for row in data:
if len(lat_var) <= 1:
lat_index = 0
else:
current_lat = float(row['Latitude'])
t_lat = current_lat - lat_var[0]
lat_index = int(round(abs(t_lat / lat_offset)))
if len(lon_var) <= 1:
lon_index = 0
else:
current_lon = float(row['Longitude'])
t_lon = current_lon - lon_var[0]
lon_index = int(round(abs(t_lon / lon_offset)))
try:
track_date = datetime.datetime.strptime(row['Date'], "%d/%m/%Y %H:%M:%S")
except ValueError:
try:
track_date = datetime.datetime.strptime(row['Date'], "%d/%m/%Y %H:%M")
except ValueError:
track_date = datetime.datetime.strptime(row['Date'], "%d/%m/%Y")
time_index = find_closest(times_sorted, track_date, arr_indexes=times_sorted_indexes, time=True, arr_sorted=True)
if lat_index > lat_end:
lat_index = lat_end
if lon_index > lon_end:
lon_index = lon_end
# Calculate the distance from the desired point to the centre of the chosen pixel
distance_from_desired = calculateDistance(current_lat, current_lon, lat_var[lat_index], lon_var[lon_index])
if distance_from_desired > offset_distance:
# If the distance is greater than the offset distance then something has gone wrong
# and the wrong pixel has been chosen.
# Set the value to NaN to avoid returning an incorrect result
data_value = float('nan')
if plotting:
debug(0, "Incorrect pixel selected! Pixel at {:+07.3f}, {:+08.3f} is further than {:6.2f}km from point at {:+07.3f}, {:+08.3f} ({:8.2f}km). Setting {} value to NaN.".format(
lat_var[lat_index], lon_var[lon_index], offset_distance, current_lat, current_lon, distance_from_desired, self.variable))
elif abs(times[time_index] - track_date) > (2 * average_time_interval):
data_value = float('nan')
elif len(data_var.dimensions) == 4:
# If the file has a depth variable, use the first depth
data_value = data_var[time_index][0][lat_index][lon_index]
else:
data_value = data_var[time_index][lat_index][lon_index]
_ret = {}
_ret['track_date'] = track_date.isoformat()
if time_var.units:
_ret['data_date'] = netCDF.num2date(time_var[time_index], time_var.units, calendar='standard').isoformat()
else:
_ret['data_date'] = time_var[time_index].tostring()
if self.matchup:
_ret['match_value'] = row['data_point']
_ret['track_lat'] = row['Latitude']
_ret['track_lon'] = row['Longitude']
_ret['data_value'] = float(data_value) if not np.isnan(float(data_value)) else "null"
ret.append(_ret)
if plotting and self.status_details:
self.update_status(len(ret))
return ret
def process(self):
if plotting:
debug(2, "Extracting...")
netcdf_file = netCDF.MFDataset(self.files, aggdim='time')
time_var = netcdf_file.variables['time']
data_var = netcdf_file.variables[self.variable]
times = time_var[:]
if time_var.units:
times = [datetime.datetime.strptime(netCDF.num2date(x, time_var.units, calendar='standard').isoformat(), "%Y-%m-%dT%H:%M:%S") for x in times]
else:
# the time variable doesn't have units; this can be caused by a thredds aggregation that uses dateFormatMark
# to grab the date from the filename, in which case the date is an array of strings
times = [datetime.datetime.strptime(x.tostring(), "%Y-%m-%dT%H:%M:%SZ") for x in times]
with open(self._csv, "rb") as csvfile:
csv_file = csvfile.read()
with open(self._csv, "rb") as csvfile:
self.numline = len(csvfile.readlines())
data = csv.DictReader(csv_file.splitlines(), delimiter=',')
ret = []
times_sorted_indexes = np.argsort(times)
times_sorted = np.sort(times)
lat_var = getCoordinateVariable(netcdf_file, "Lat")[:]
lon_var = getCoordinateVariable(netcdf_file, "Lon")[:]
if np.amax(lat_var) > 90:
for i, lat in enumerate(lat_var):
if lat > 90:
lat_var[i] = lat - 180
if np.amax(lon_var) > 180:
for i, lon in enumerate(lon_var):
if lon > 180:
lon_var[i] = lon - 360
lat_end = len(lat_var) - 1
lat_offset = (lat_var[lat_end] - lat_var[0]) / lat_end
lon_end = len(lon_var) - 1
lon_offset = (lon_var[lon_end] - lon_var[0]) / lon_end
self.start_time = time.clock()
self.last_time = time.clock()
for row in data:
if len(lat_var) <= 1:
lat_index = 0
else:
current_lat = float(row['Latitude'])
t_lat = current_lat - lat_var[0]
lat_index = int(round(abs(t_lat / lat_offset)))
if len(lon_var) <= 1:
lon_index = 0
else:
current_lon = float(row['Longitude'])
t_lon = current_lon - lon_var[0]
lon_index = int(round(abs(t_lon / lon_offset)))
track_date = datetime.datetime.strptime(row['Date'], "%d/%m/%Y %H:%M")
time_index = find_closest(times_sorted, track_date, arr_indexes=times_sorted_indexes, time=True, arr_sorted=True)
if lat_index > lat_end:
lat_index = lat_end
if lon_index > lon_end:
lon_index = lat_end
if len(data_var.dimensions) == 4:
# If the file has a depth variable, use the first depth
data = data_var[time_index][0][lat_index][lon_index]
else:
data = data_var[time_index][lat_index][lon_index]
_ret = {}
_ret['track_date'] = track_date.isoformat()
if time_var.units:
_ret['data_date'] = netCDF.num2date(time_var[time_index], time_var.units, calendar='standard').isoformat()
else:
_ret['data_date'] = time_var[time_index].tostring()
if self.matchup:
_ret['match_value'] = row['data_point']
_ret['track_lat'] = row['Latitude']
_ret['track_lon'] = row['Longitude']
_ret['data_value'] = float(data) if not np.isnan(float(data)) else "null"
ret.append(_ret)
if plotting and self.status_details:
self.update_status(len(ret))
return ret
def process(self):
#print "running basic processing on %s" % self.filename
# create three arrays, 1D lat 1D lon 2D data
#print "processing image"
netcdf_file = netCDF.Dataset(self.filename, "r")
#variable = np.ma.masked_array(netcdf_file.variables[self.variable])
variable = np.ma.array(netcdf_file.variables[self.variable][:])
lats = getCoordinateVariable(netcdf_file, "Lat")
lons = getCoordinateVariable(netcdf_file, "Lon")
time_dim_index = netcdf_file.variables[self.variable].dimensions.index('time')
if 'depth' in netcdf_file.variables[self.variable].dimensions:
depth_dim_index = netcdf_file.variables[self.variable].dimensions.index('depth')
var_list = []
lat_list = []
lon_list = []
#print variable.shape
if(len(variable.shape) > 3 ):
#print "hmmmm"
#print variable.shape
#print variable
#print np.nanmean(variable, axis=time_dim_index).shape
var_list = [[float(x) if not np.isinf(x) and not np.isnan(x) else None for x in y ] for y in np.nanmean(variable, axis=time_dim_index)[0]]
#print var_list
lat_list = [float(x) for x in lats]
lon_list = [float(x) for x in lons]
elif(len(variable.shape) > 2 ):
#print variable.shape
#print variable
#print np.nanmean(variable, axis=time_dim_index)
var_list = [[float(x) if not np.isinf(x) and not np.isnan(x) else None for x in y ] for y in np.nanmean(variable, axis=time_dim_index)]
#var_list = [[float(x) for x in y] for y in variable[0]]
#print var_list
lat_list = [float(x) for x in lats]
lon_list = [float(x) for x in lons]
else:
var_list = [list(x) for x in variable]
lat_list = [float(x) for x in lats]
lon_list = [float(x) for x in lons]
#print len(lat_list)
#print len(lon_list)
#print len(var_list)
#print len(var_list[0])
#print lat_list
_ret = {}
_ret['vars'] = ['Data','Latitudes','Longitudes']
_ret['data'] = []
_ret['data'].append(var_list)
_ret['data'].append(lat_list)
_ret['data'].append(lon_list)
#print json.dumps(_ret )
return json.dumps(_ret )
#netcdf_variable = netcdf_file[variable]
