def get_netcdf_data(request):
errors = []
# Filter Month
filter_month = ""
if 'filter-month' in request.GET:
try:
filter_month = int(request.GET['filter-month'])
except:
errors.append(
"Enter an integer for the correct month. 0 is January 11 is December"
)
# Start date
start_date = ''
if 'start-date' in request.GET:
try:
start_date = str(request.GET['start-date'])
except:
errors.append("Enter a start-date in yyyy-mm-dd format")
try:
dt.datetime.strptime(start_date, "%Y-%m-%d")
except:
errors.append("Enter a start-date in yyyy-mm-dd format")
# End date
end_date = ''
if 'end-date' in request.GET:
try:
end_date = str(request.GET['end-date'])
except:
errors.append("Enter an end-date in yyyy-mm-dd format")
try:
dt.datetime.strptime(end_date, "%Y-%m-%d")
except:
errors.append("Enter a end-date in yyyy-mm-dd format")
if not start_date == '' or not end_date == '':
if start_date >= end_date:
return HttpResponse(
"start_date can not be greater than or equal to end_date")
# Lat
if 'lat' in request.GET:
try:
lat = float(request.GET['lat'])
if lat < -90 or lat > 90:
errors.append("Lat parameter needs to within -90 to 90 range")
except:
errors.append("Lat parameter needs to be a float")
else:
errors.append("You need to specify a lat parameter")
# Lon
if 'lon' in request.GET:
try:
lon = float(request.GET['lon'])
if lat < -180 or lat > 180:
errors.append(
"Lon parameter needs to within -180 to 180 range")
except:
errors.append("Lon parameter needs to be a float")
else:
errors.append("You need to specify a lon parameter")
positive_east_longitude = "True"
# Positive east longitude
if 'positive-east-longitude' in request.GET:
positive_east_longitude = request.GET['positive-east-longitude']
if positive_east_longitude == "True" or positive_east_longitude == "False":
pass
else:
errors.append(
"positive-east-longitude parameter should be either True or False"
)
# Variable
if 'variable' in request.GET:
variable_list = request.GET.getlist('variable')
#print "got the variables", variable_list
for variable in variable_list:
if str(variable).isdigit():
errors.append(
"variable parameters must be a a variable name not a number"
)
else:
str(variable)
else:
errors.append("You need to specify a variable parameter")
# Variable Name
if 'variable-name' in request.GET:
variable_name_list = request.GET.getlist('variable-name')
#print "got the variables names list", variable_name_list
for variable_name in variable_name_list:
if str(variable_name).isdigit():
errors.append(
"variable names must be a a variable name string not a number"
)
else:
str(variable_name)
else:
errors.append("You need to specify a variable-name parameter")
# Data Path
if 'data-path' in request.GET:
data_path_list = request.GET.getlist('data-path')
#print "got the data paths", data_path_list
for url in data_path_list:
if str(url).isdigit():
errors.append(
"data-path parameters must be a a url not a number")
else:
str(url)
else:
errors.append("You need to specify a data-path parameter")
# Request JSON parameter
request_JSON = "False"
if 'request-JSON' in request.GET:
request_JSON = request.GET['request-JSON']
if request_JSON == "True" or request_JSON == "False":
pass
else:
errors.append(
"You need to specify a request_JSON parameter as True or False"
)
# Decimal Precision
decimal_precision = 6
if 'decimal-precision' in request.GET:
decimal_precision = request.GET['decimal-precision']
try:
decimal_precision = int(decimal_precision)
if decimal_precision >= 0 and decimal_precision <= 10:
pass
else:
errors.append(
"You need to specify a decimal_precision parameter as an integer between 0-10"
)
except:
errors.append(
"You need to specify a decimal_precision parameter as an integer between 0-10"
)
# Precision string used to set the number of decimals places after the decimal dynamically
precision_string = "{0:.%sf}" % decimal_precision
# CSV Download
download_csv = "False"
if 'download-csv' in request.GET:
download_csv = request.GET["download-csv"]
if download_csv == "True" or download_csv == "False":
pass
else:
errors.append(
"You need to specify a download-csv parameter as True or False"
)
start_year = 1900
start_month = 1
start_day = 1
time_metric = "days"
time_units = 1
# Errors
if errors:
return HttpResponse(errors)
else:
# Set number of processes to the number of variables being called
p = Pool(len(variable_list))
# Set request lat lon variable
request_lat_lon = False
# List to hold function parameters as tuples
# Each set of parameters is for one URL call
function_parameters = []
# List for all returned netcdf data
netcdf_data_list = []
# Get the dates data
request_dates = "True"
netcdf_time_list, netcdf_time_index = models.get_netcdf_data(
lat,
lon,
positive_east_longitude,
variable_list[0],
request_dates,
start_year,
start_month,
start_day,
time_metric,
time_units,
data_path_list[0],
request_lat_lon=False,
start_date=start_date,
end_date=end_date,
start_date_index='',
end_date_index='')
#print netcdf_time_list
if netcdf_time_list[0] == 'error':
return HttpResponse("There was an error: " + netcdf_time_list[1])
# Index of times that have been filtered
netcdf_start_date_index = netcdf_time_index[0]
netcdf_end_date_index = netcdf_time_index[1]
#print netcdf_start_time_index
#print netcdf_end_time_index
# Set as false until request is made later for just the dates
request_dates = "False"
# List to hold Metadata items
metadata_list = []
metadata_column_list = []
# Process each variable from the variable list
#### for url in url LIST
#print variable_list
for i in range(len(variable_list)):
#print i
function_parameters.append(
(lat, lon, positive_east_longitude, variable_list[i],
request_dates, start_year, start_month, start_day,
time_metric, time_units, data_path_list[i], request_lat_lon,
start_date, end_date, netcdf_start_date_index,
netcdf_end_date_index))
# m returns variable long name, variable units
m = models.get_netcdf_metadata(lat, lon, positive_east_longitude,
variable_list[i], request_dates,
start_year, start_month, start_day,
time_metric, time_units,
data_path_list[i])
# Contains long names of variables
metadata_list.append(m[0])
# Contains user defined variable names and units from metadata
metadata_column_list.append(variable_name_list[i] + ' (' + m[1] +
')')
# Map to pool - this gets netcdf data into a workable list
netcdf_data_list.append(
p.map(allow_mulitple_parameters, function_parameters))
#print netcdf_data_list
# Close subprocess workers (open files)
p.terminate()
p.join()
#print netcdf_time_list.index("1960-01-01")
# After getting all data successfully set request dates false and request_lat_lon true
request_dates = "False"
request_lat_lon = "True"
actual_lat_lon = models.get_netcdf_data(lat,
lon,
positive_east_longitude,
variable_list[0],
request_dates,
start_year,
start_month,
start_day,
time_metric,
time_units,
data_path_list[0],
request_lat_lon,
start_date='',
end_date='',
start_date_index='',
end_date_index='')
# Converts from U'' to ''
metadata_column_list = [str(x) for x in metadata_column_list]
metadata_list = [str(x) for x in metadata_list]
# Convert metadata colum list to string and clean it up
metadata_columns_string = str(metadata_column_list)
metadata_columns_string = clean_list_string(metadata_columns_string)
#metadata_column_list = [str(x) for x in metadata_column_list]
# URL data was requested with
request_path = request.META['HTTP_HOST'] + request.get_full_path()
# List containing the clean string names of the NetCDF filenames
netcdf_filenames_list = [str(x.split('/')[-1]) for x in data_path_list]
netcdf_filenames_list = [
clean_list_string(x.split('/')[-1]) for x in netcdf_filenames_list
]
# String to list
metadata_columns_string_split = metadata_columns_string.split(',')
column_format_csv = [['yyyy-mm-dd']]
for i in metadata_columns_string_split:
#print i
column_format_csv[0].append(i)
#print column_format_csv
# Metadata header section
metadata_header = """#Variables:<br />"""
for i in range(len(metadata_list)):
#print i
metadata_header += "#" + metadata_columns_string_split[
i] + ":" + str(metadata_list[i]) + "<br />"
#print netcdf_filenames_list
# Add Lat/Lon to Metadata header
metadata_header += "#Data Extracted for Point Location: %.4f Latitude, %.4f Longitude (closest value to Latitude:%s, Longitude:%s)" % (
actual_lat_lon[0], actual_lat_lon[1], lat, lon)
metadata_variable_string = ""
for i in netcdf_filenames_list:
metadata_variable_string += "#%s <br />" % i
if download_csv == "False":
# Get metadata
metadata = "%s <br />#Original Data File(s):<br />%s#===============================================<br />yyyy-mm-dd,%s<br />" % (
metadata_header, metadata_variable_string,
metadata_columns_string)
else:
metadata = "%s <br />#Original Data File(s):<br />%s#===============================================<br />" % (
metadata_header, metadata_variable_string)
metadata_rows = metadata.split("<br />")
#for r in metadata_rows:
#print type(r), r
#print metadata_rows
# Write CSV style response
response_string = ""
response_rows = []
variable_columns = []
for variable_dataset in netcdf_data_list[0]:
# Convert to float and truncate to decimal precision variable
variable_dataset = [
precision_string.format(float(i)) for i in variable_dataset
]
variable_columns.append(variable_dataset)
# Create time and variable rows
for i in range(len(netcdf_time_list)):
new_row = []
new_row.append(netcdf_time_list[i])
for v in variable_columns:
new_row.append(v[i])
#for x in new_row:
response_rows.append(new_row)
# new_row = []
if filter_month in range(0, 12):
# Concatenate lists to stings
response_rows_strings = [[','.join(x)] for x in response_rows]
#print response_rows_strings
filtered_dates = filter_dates_by_month(dates=response_rows_strings,
months=[filter_month])
#print filtered_dates
response_rows = [
x[0].split(',') for x in filtered_dates[int(filter_month)]
]
response_metadata_rows = []
# Create metadata rows
for i in range(len(metadata_rows)):
new_row = []
new_row.append(metadata_rows[i])
response_metadata_rows.append(new_row)
def rows_to_JSON(column_names, row_data):
""" Process rows into dictionary objects (columns) to convert to JSON"""
#Dictionary of JSON rows
JSON_dictionary = {
"metadata": metadata_rows,
'lat_lon': actual_lat_lon
}
# Loop through each column and set the colunm name for JSON and assign data
for i in range(len(column_names)):
column_data = [row[i] for row in response_rows]
#print column_names[i]
JSON_dictionary[column_names[i].split(" ")[0]] = column_data
#print JSON_dictionary[column_names[i].split(" ")[0]]
object_for_JSON = {
"data": [
JSON_dictionary,
]
}
return object_for_JSON
# Get JSON Data
if request_JSON == "True":
# Convert the data to dictionary to get converted into JSON
#print column_format_csv[0]
object_for_JSON = rows_to_JSON(column_format_csv[0], response_rows)
#print JSON_dictionary
response = json.dumps(object_for_JSON)
return HttpResponse(response, content_type="application/json")
# Download CSV Data
if download_csv == "True":
# Create the HttpResponse object with the appropriate CSV header.
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="data.csv"'
writer = csv.writer(response)
# Get Metadata rows
for row in response_metadata_rows:
writer.writerow(row)
for row in column_format_csv:
writer.writerow(row)
# Get data rows
for row in response_rows:
#print row
writer.writerow(row)
return response
else:
# convert rows to string
for r in response_rows:
# Convert list to string
r = str(r)
# Remove square bracket
response_string += r
response_string += "<br //>"
response_string = clean_list_string(response_string)
#return HttpResponse([(response_string + "%s," % i) for i in netcdf_data_list])
return HttpResponse(metadata + response_string)
def get_netcdf_data(request):
errors = []
# Filter Month
filter_month = ""
if 'filter-month' in request.GET:
try:
filter_month = int(request.GET['filter-month'])
except:
errors.append("Enter an integer for the correct month. 0 is January 11 is December")
# Start date
start_date = ''
if 'start-date' in request.GET:
try:
start_date = str(request.GET['start-date'])
except:
errors.append("Enter a start-date in yyyy-mm-dd format")
try:
dt.datetime.strptime(start_date, "%Y-%m-%d")
except:
errors.append("Enter a start-date in yyyy-mm-dd format")
# End date
end_date = ''
if 'end-date' in request.GET:
try:
end_date = str(request.GET['end-date'])
except:
errors.append("Enter an end-date in yyyy-mm-dd format")
try:
dt.datetime.strptime(end_date, "%Y-%m-%d")
except:
errors.append("Enter a end-date in yyyy-mm-dd format")
if not start_date == '' or not end_date == '':
if start_date >= end_date:
return HttpResponse("start_date can not be greater than or equal to end_date")
# Lat
if 'lat' in request.GET:
try:
lat = float(request.GET['lat'])
if lat < -90 or lat > 90:
errors.append("Lat parameter needs to within -90 to 90 range")
except:
errors.append("Lat parameter needs to be a float")
else:
errors.append("You need to specify a lat parameter")
# Lon
if 'lon' in request.GET:
try:
lon = float(request.GET['lon'])
if lat < -180 or lat > 180:
errors.append("Lon parameter needs to within -180 to 180 range")
except:
errors.append("Lon parameter needs to be a float")
else:
errors.append("You need to specify a lon parameter")
positive_east_longitude = "True"
# Positive east longitude
if 'positive-east-longitude' in request.GET:
positive_east_longitude = request.GET['positive-east-longitude']
if positive_east_longitude == "True" or positive_east_longitude == "False":
pass
else:
errors.append("positive-east-longitude parameter should be either True or False")
# Variable
if 'variable' in request.GET:
variable_list = request.GET.getlist('variable')
#print "got the variables", variable_list
for variable in variable_list:
if str(variable).isdigit():
errors.append("variable parameters must be a a variable name not a number")
else:
str(variable)
else:
errors.append("You need to specify a variable parameter")
# Variable Name
if 'variable-name' in request.GET:
variable_name_list = request.GET.getlist('variable-name')
#print "got the variables names list", variable_name_list
for variable_name in variable_name_list:
if str(variable_name).isdigit():
errors.append("variable names must be a a variable name string not a number")
else:
str(variable_name)
else:
errors.append("You need to specify a variable-name parameter")
# Data Path
if 'data-path' in request.GET:
data_path_list = request.GET.getlist('data-path')
#print "got the data paths", data_path_list
for url in data_path_list:
if str(url).isdigit():
errors.append("data-path parameters must be a a url not a number")
else:
str(url)
else:
errors.append("You need to specify a data-path parameter")
# Request JSON parameter
request_JSON = "False"
if 'request-JSON' in request.GET:
request_JSON = request.GET['request-JSON']
if request_JSON == "True" or request_JSON == "False":
pass
else:
errors.append("You need to specify a request_JSON parameter as True or False")
# Decimal Precision
decimal_precision = 6
if 'decimal-precision' in request.GET:
decimal_precision = request.GET['decimal-precision']
try:
decimal_precision = int(decimal_precision)
if decimal_precision >= 0 and decimal_precision <= 10:
pass
else:
errors.append("You need to specify a decimal_precision parameter as an integer between 0-10")
except:
errors.append("You need to specify a decimal_precision parameter as an integer between 0-10")
# Precision string used to set the number of decimals places after the decimal dynamically
precision_string = "{0:.%sf}" % decimal_precision
# CSV Download
download_csv = "False"
if 'download-csv' in request.GET:
download_csv = request.GET["download-csv"]
if download_csv == "True" or download_csv == "False":
pass
else:
errors.append("You need to specify a download-csv parameter as True or False")
start_year=1900
start_month=1
start_day=1
time_metric="days"
time_units=1
# Errors
if errors:
return HttpResponse(errors)
else:
# Set number of processes to the number of variables being called
p = Pool(len(variable_list))
# Set request lat lon variable
request_lat_lon = False
# List to hold function parameters as tuples
# Each set of parameters is for one URL call
function_parameters = []
# List for all returned netcdf data
netcdf_data_list = []
# Get the dates data
request_dates = "True"
netcdf_time_list, netcdf_time_index = models.get_netcdf_data(lat, lon, positive_east_longitude, variable_list[0], request_dates, start_year, start_month, start_day, time_metric,time_units, data_path_list[0], request_lat_lon=False, start_date=start_date, end_date=end_date, start_date_index='', end_date_index='' )
#print netcdf_time_list
if netcdf_time_list[0] == 'error':
return HttpResponse("There was an error: " + netcdf_time_list[1] )
# Index of times that have been filtered
netcdf_start_date_index = netcdf_time_index[0]
netcdf_end_date_index = netcdf_time_index[1]
#print netcdf_start_time_index
#print netcdf_end_time_index
# Set as false until request is made later for just the dates
request_dates = "False"
# List to hold Metadata items
metadata_list = []
metadata_column_list = []
# Process each variable from the variable list
#### for url in url LIST
#print variable_list
for i in range(len(variable_list)):
#print i
function_parameters.append((lat,lon,positive_east_longitude,variable_list[i],request_dates, start_year, start_month, start_day, time_metric,time_units, data_path_list[i], request_lat_lon, start_date, end_date, netcdf_start_date_index, netcdf_end_date_index))
# m returns variable long name, variable units
m = models.get_netcdf_metadata(lat,lon,positive_east_longitude,variable_list[i],request_dates, start_year, start_month, start_day, time_metric,time_units, data_path_list[i])
# Contains long names of variables
metadata_list.append(m[0])
# Contains user defined variable names and units from metadata
metadata_column_list.append(variable_name_list[i] + ' (' + m[1] + ')')
# Map to pool - this gets netcdf data into a workable list
netcdf_data_list.append ( p.map(allow_mulitple_parameters, function_parameters) )
#print netcdf_data_list
# Close subprocess workers (open files)
p.terminate()
p.join()
#print netcdf_time_list.index("1960-01-01")
# After getting all data successfully set request dates false and request_lat_lon true
request_dates = "False"
request_lat_lon = "True"
actual_lat_lon = models.get_netcdf_data(lat, lon, positive_east_longitude, variable_list[0], request_dates, start_year, start_month, start_day, time_metric,time_units, data_path_list[0], request_lat_lon, start_date='', end_date='', start_date_index='', end_date_index='' )
# Converts from U'' to ''
metadata_column_list = [str(x) for x in metadata_column_list]
metadata_list = [str(x) for x in metadata_list]
# Convert metadata colum list to string and clean it up
metadata_columns_string = str(metadata_column_list)
metadata_columns_string = clean_list_string(metadata_columns_string)
#metadata_column_list = [str(x) for x in metadata_column_list]
# URL data was requested with
request_path = request.META['HTTP_HOST'] + request.get_full_path()
# List containing the clean string names of the NetCDF filenames
netcdf_filenames_list = [str(x.split('/')[-1]) for x in data_path_list]
netcdf_filenames_list = [clean_list_string(x.split('/')[-1]) for x in netcdf_filenames_list]
# String to list
metadata_columns_string_split = metadata_columns_string.split(',')
column_format_csv = [['yyyy-mm-dd']]
for i in metadata_columns_string_split:
#print i
column_format_csv[0].append(i)
#print column_format_csv
# Metadata header section
metadata_header = """#Variables:<br />"""
for i in range(len(metadata_list)):
#print i
metadata_header += "#" + metadata_columns_string_split[i] + ":" + str(metadata_list[i]) + "<br />"
#print netcdf_filenames_list
# Add Lat/Lon to Metadata header
metadata_header += "#Data Extracted for Point Location: %.4f Latitude, %.4f Longitude (closest value to Latitude:%s, Longitude:%s)" % (actual_lat_lon[0], actual_lat_lon[1], lat,lon )
metadata_variable_string = ""
for i in netcdf_filenames_list:
metadata_variable_string += "#%s <br />" % i
if download_csv == "False":
# Get metadata
metadata = "%s <br />#Original Data File(s):<br />%s#===============================================<br />yyyy-mm-dd,%s<br />" % (metadata_header, metadata_variable_string, metadata_columns_string)
else:
metadata = "%s <br />#Original Data File(s):<br />%s#===============================================<br />" % (metadata_header, metadata_variable_string)
metadata_rows = metadata.split("<br />")
#for r in metadata_rows:
#print type(r), r
#print metadata_rows
# Write CSV style response
response_string = ""
response_rows = []
variable_columns = []
for variable_dataset in netcdf_data_list[0]:
# Convert to float and truncate to decimal precision variable
variable_dataset = [precision_string.format(float(i)) for i in variable_dataset]
variable_columns.append(variable_dataset)
# Create time and variable rows
for i in range(len(netcdf_time_list)):
new_row = []
new_row.append(netcdf_time_list[i])
for v in variable_columns:
new_row.append(v[i])
#for x in new_row:
response_rows.append(new_row)
# new_row = []
if filter_month in range(0,12):
# Concatenate lists to stings
response_rows_strings = [[','.join(x)] for x in response_rows]
#print response_rows_strings
filtered_dates = filter_dates_by_month(dates=response_rows_strings, months=[filter_month])
#print filtered_dates
response_rows = [x[0].split(',') for x in filtered_dates[int(filter_month)]]
response_metadata_rows = []
# Create metadata rows
for i in range(len(metadata_rows)):
new_row = []
new_row.append(metadata_rows[i])
response_metadata_rows.append(new_row)
def rows_to_JSON(column_names, row_data):
""" Process rows into dictionary objects (columns) to convert to JSON"""
#Dictionary of JSON rows
JSON_dictionary = {"metadata": metadata_rows, 'lat_lon': actual_lat_lon
}
# Loop through each column and set the colunm name for JSON and assign data
for i in range(len(column_names)):
column_data = [row[i] for row in response_rows]
#print column_names[i]
JSON_dictionary[column_names[i].split(" ")[0]] = column_data
#print JSON_dictionary[column_names[i].split(" ")[0]]
object_for_JSON = {"data":
[
JSON_dictionary,
]
}
return object_for_JSON
# Get JSON Data
if request_JSON == "True":
# Convert the data to dictionary to get converted into JSON
#print column_format_csv[0]
object_for_JSON = rows_to_JSON(column_format_csv[0], response_rows)
#print JSON_dictionary
response = json.dumps(object_for_JSON)
return HttpResponse(response, content_type="application/json")
# Download CSV Data
if download_csv == "True":
# Create the HttpResponse object with the appropriate CSV header.
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="data.csv"'
writer = csv.writer(response)
# Get Metadata rows
for row in response_metadata_rows:
writer.writerow(row)
for row in column_format_csv:
writer.writerow(row)
# Get data rows
for row in response_rows:
#print row
writer.writerow(row)
return response
else:
# convert rows to string
for r in response_rows:
# Convert list to string
r = str(r)
# Remove square bracket
response_string += r
response_string += "<br //>"
response_string = clean_list_string(response_string)
#return HttpResponse([(response_string + "%s," % i) for i in netcdf_data_list])
return HttpResponse(metadata+response_string)
def allow_mulitple_parameters(args):
return models.get_netcdf_data(*args)
def allow_mulitple_parameters(args):
return models.get_netcdf_data(*args)
