def netcdf_histogram_visualization(variablename='', xaxis='', start='0', end='0', sectionNumber='1'):
global netcdf_file_local_path
# open netcdf file
netcdf_aim_file = Dataset(netcdf_file_local_path, "r")
element_num = len(netcdf_aim_file.dimensions[xaxis])
section_num = int(sectionNumber)
class_information_list = []
temp_dim_handle = netcdf_aim_file.variables[variablename]
start_frame = int(start)
end_frame = int(end)
# load_num means how many frames to visualize each time
load_number = 5
# if start==end==0 means this is the first step
if start_frame==0 and end_frame==0:
if element_num <= load_number:
# element_num-1 because the index starts from 0
end_frame = element_num - 1
else:
end_frame = load_number
# set up for loop to create a list of class
# for count in range(0,element_num):
for count in range(start_frame,end_frame):
# record time
# start_time = time.time()
# initialize these array and lists
section_results_number = []
tab_name_list = []
# TODO need to dynamically obtain data, not just first dimension
aim_array = temp_dim_handle[count].flatten()
units = ''
standard_name = ''
util.sort_section_number(aim_array, section_num, section_results_number, tab_name_list)
# .ncattrs() is used to list all the attributes of the chosen variable
if 'units' in temp_dim_handle.ncattrs():
units = temp_dim_handle.units
if 'standard_name' in temp_dim_handle.ncattrs():
standard_name = temp_dim_handle.standard_name
temp_object = FrequenceDistributionInformation(aim_array, section_results_number, tab_name_list, units, standard_name)
class_information_list.append(temp_object)
# print('--- %s second ---' % (time.time() - start_time))
return render_template('visualization/netcdf_visualization_histogram_results.html', \
variablename=variablename, class_information_list=class_information_list, \
load_number=load_number, element_num=element_num)
# if not the first step, we need return a json file contains all the information to visualize
'''
json file is like this:
{
"timestamp":[
{
"standard_name": "XXX",
"units": "YYY",
"section_results_number":[1,2,3],
"tab_name_list": ['aatobb','bbtocc']
},
{}
]
}
information is based on time, this means json[y] is the information for timestamp y
'''
# for this part, start and end should be all less than element_num
# and this is controlled in netcdf_histogram_visualization_results.js
# I don't know why elif does not work
# elif start_frame<element_num and end_frame<=element_num:
if start_frame<element_num and end_frame<=element_num:
json_final = {}
json_timestamp = []
# !!!!!!!!!!!!!!!!!!!!
# set up for loop to create a json file
for count in range(start_frame,end_frame):
# record time
# start_time = time.time()
# initialize these array and lists
section_results_number = []
tab_name_list = []
json_projects = {}
# TODO need to dynamically obtain data, not just first dimension
aim_array = temp_dim_handle[count].flatten()
units = ''
standard_name = ''
util.sort_section_number(aim_array, section_num, section_results_number, tab_name_list)
# .ncattrs() is used to list all the attributes of the chosen variable
if 'units' in temp_dim_handle.ncattrs():
units = temp_dim_handle.units
if 'standard_name' in temp_dim_handle.ncattrs():
standard_name = temp_dim_handle.standard_name
json_projects['units'] = units
json_projects['standard_name'] = standard_name
json_projects['section_results_number'] = section_results_number
json_projects['tab_name_list'] = tab_name_list
json_timestamp.append(json_projects)
json_final['timestamp'] = json_timestamp
json_final = json.dumps(json_final)
return json_final
else:
# we should return an error page is start and end is bigger than element_num
return 'error'
def relation(path):
# get the current app location
# app/visualization
app_root = os.path.dirname(os.path.abspath(__file__))
# the file extension could only be csv now
if path.endswith('.csv'):
# print app_root + "/tempData/" + path
filename_csv = path.rsplit("/",1)[1]
return send_from_directory(app_root + "/tempData/", filename_csv)
elif path.endswith('.nc'):
# print app_root + "/tempData/" + path
filename_nc = path.rsplit("/",1)[1]
return send_from_directory(app_root + "/tempData/", filename_nc)
elif path.endswith('.js'):
# this line is used to get the js filename
filename_js = path.rsplit("/",1)[1]
return send_from_directory(app_root + "/../templates/visualization/js", \
filename_js)
elif path.endswith('.png'):
# this line is used to get the png filename
filename_img = path.rsplit("/",1)[1]
return send_from_directory(app_root + "/../templates/visualization/img", \
filename_img)
# this used to obtain parameters for NetCDF line chart visualization
elif path.endswith('LineChartNetCDFParameters'):
# this line is used to get the nc parameters
vis_param = path.rsplit("/",1)[1]
vis_param = util.replace_last(vis_param,'LineChartNetCDFParameters','')
csv_string = line_chart_last_step(vis_param)
return render_template('visualization/netcdf_visualization_line_chart_results.html', \
csv_string=csv_string)
# this used to obtain parameters for NetCDF bar chart visualization
elif path.endswith('BarChartNetCDFParameters'):
# this line is used to get the nc parameters
vis_param = path.rsplit("/",1)[1]
vis_param = util.replace_last(vis_param,'BarChartNetCDFParameters','')
x_axis = []
x_name = ''
y_axis = []
y_name = ''
name_list = []
bar_chart_last_step(vis_param, x_axis, y_axis, name_list)
x_name = name_list[0]
y_name = name_list[1]
# print 'aaaaaaaaaaaaa'
# print x_axis
return render_template('visualization/netcdf_visualization_bar_chart_results.html', \
x_axis=x_axis, x_name=x_name, \
y_axis=y_axis, y_name=y_name)
elif path.endswith('3DApplicationResults/'):
filename = path.rsplit('/')[-3]
app_root = os.path.dirname(os.path.abspath(__file__))
download_dir = app_root + '/tempData/'
# item_name_list = util.csv_interface_obtain_labels(filename,download_dir)
item_name_list = util.csv_interface_obtain_labels(filename, download_dir)
input_filename = filename
# print item_name_list
# print input_filename
return render_template('visualization/csv_visualization_results.html', \
item_name_list=item_name_list, \
input_filename=input_filename)
elif path.endswith('3DApplicationResultsHistogram/'):
# grab data from file
filename = path.rsplit('/')[-3]
app_root = os.path.dirname(os.path.abspath(__file__))
download_dir = app_root + '/tempData/'
input_file = open(download_dir + filename,'r')
# the file just have one line
data = input_file.readline()
# print 'aaaaaaaaaaaaaaaaaa'
# print data
# -2 is because we need to remove section number and variable name
element_num = len(data.split('//'))-2
section_num = int(data.split('//')[1])
class_information_list = []
variablename = data.split('//')[0]
# print 'aaaaaaaaaaaaaaaaaa'
# print variablename
# set up for loop to create a list of class
for count in range(0,element_num):
# initialize these array and lists
section_results_number = []
tab_name_list = []
# get array
aim_array = data.split('//')[count+2].split(',')
# convert all the elements in the array into float
aim_array = [float(i) for i in aim_array]
# convert list into numpy array
aim_array = numpy.asarray(aim_array)
units = ''
# for this part only we used variable name as standard name
standard_name = variablename
util.sort_section_number(aim_array, section_num, section_results_number, tab_name_list)
# create the class object
temp_object = FrequenceDistributionInformation(aim_array, section_results_number, tab_name_list, units, standard_name)
class_information_list.append(temp_object)
# this is used to compromise with my own part updates
load_number = element_num
return render_template('visualization/netcdf_visualization_histogram_results.html', \
variablename=variablename, class_information_list=class_information_list, load_number=load_number)
# all the data about certain variable in a NetCDF file will be
# transfered as json type data
elif path.endswith('NetCDFMapData'):
# /filename:variablename:3:x_dimension:y_dimension:NetCDFMapData or
# /filename:variablename:4:x_dimension:y_dimension:NetCDFMapData
app_root = os.path.dirname(os.path.abspath(__file__))
download_dir = app_root + '/tempData/'
filename = path.split(':')[0]
variable_name = path.split(':')[1]
# open netcdf file
netcdf_file_local_path = download_dir + filename
netcdf_aim_file = Dataset(netcdf_file_local_path, "r")
# grab dimension name list
dimension_name_list = list(netcdf_aim_file.variables[variable_name].dimensions)
dimension_num = path.split(':')[2]
lat = path.split(':')[3]
lon = path.split(':')[4]
lon_length = len(netcdf_aim_file.dimensions[lon])
lat_length = len(netcdf_aim_file.dimensions[lat])
# create json to store the variable data
# and because we will visualize data on a map
# the visualization data will be 3D (use 2D as x and y and display the
# last dimension the data on a map)
# or will be 4D (use 2D data as x and y, when users click on a element,
# then display the the left 2D in a line chart)
# for 4D we can also use slider. e.g. temperature is a 4D variable
# with (lat, lon, time, temperture)
json_projects = []
# use different for loop if it is 4D, now only it works for 3D
# now the two loops are the same
if int(dimension_num) == 3:
# need to figure out which dimension is at first
if dimension_name_list.index(lat) == 0:
for count_lon in range(0,lon_length):
temp_array = ''
for count_lat in range(0,lat_length):
# python dictionary does not like array in it, so have to use string
temp_array = temp_array + str(netcdf_aim_file.variables[variable_name][count_lat][count_lon]) +","
# remove the last ,
# don't know why temp_array.rstrip(',') does not work
temp_change_str_into_list = temp_array.split(',')
temp_change_str_into_list.pop()
temp_array = ','.join(temp_change_str_into_list)
json_projects.append({"data" : temp_array})
else:
for count_lon in range(0,lon_length):
temp_array = ''
for count_lat in range(0,lat_length):
# python dictionary does not like array in it, so have to use string
temp_array = temp_array + str(netcdf_aim_file.variables[variable_name][count_lon][count_lat]) +","
# remove the last ,
# don't know why temp_array.rstrip(',') does not work
temp_change_str_into_list = temp_array.split(',')
temp_change_str_into_list.pop()
temp_array = ','.join(temp_change_str_into_list)
json_projects.append({"data" : temp_array})
json_projects = json.dumps(json_projects)
elif int(dimension_num) == 4:
# create executable string for 4D to find out the order of each dimension
# two of the dimensions are variable itself and "another one"
# for the "another one" we can use 0 to specify its value for now
exec_string='temp_array = temp_array + str(netcdf_aim_file.variables[variable_name]'
lat_dimension_index = dimension_name_list.index(lat)
lon_dimension_index = dimension_name_list.index(lon)
for count in range(3):
if count == lon_dimension_index:
exec_string = exec_string + '[count_lon]'
elif count == lat_dimension_index:
exec_string = exec_string + '[count_lat]'
else:
exec_string = exec_string + '[0]'
exec_string = exec_string + ')+' +"','"
# print exec_string
for count_lon in range(0,lon_length):
temp_array = ''
for count_lat in range(0,lat_length):
# python dictionary does not like array in it, so have to use string
# exec_string should be something like this
# temp_array = temp_array + str(netcdf_aim_file.variables[variable_name][0][count_lon][count_lat]) +","
exec exec_string
# remove the last ,
# don't know why temp_array.rstrip(',') does not work
temp_change_str_into_list = temp_array.split(',')
temp_change_str_into_list.pop()
temp_array = ','.join(temp_change_str_into_list)
json_projects.append({"data" : temp_array})
json_projects = json.dumps(json_projects)
return json_projects
# all the data about certain variable in a NetCDF file will be
# transfered as json type data, lat and lon as variables
# and they are decided by the same dimensions
elif path.endswith('NetCDFMapDataOther'):
# /filename:variablename:3:x_dimension:y_dimension:NetCDFMapDataOther or
# /filename:variablename:4:x_dimension:y_dimension:NetCDFMapDataOther
app_root = os.path.dirname(os.path.abspath(__file__))
download_dir = app_root + '/tempData/'
filename = path.split(':')[0]
variable_name = path.split(':')[1]
# print filename
# open netcdf file
netcdf_file_local_path = download_dir + filename
netcdf_aim_file = Dataset(netcdf_file_local_path, "r")
# grab dimension name list
dimension_name_list = list(netcdf_aim_file.variables[variable_name].dimensions)
dimension_num = path.split(':')[2]
lat = path.split(':')[3]
lon = path.split(':')[4]
lon_length = len(netcdf_aim_file.variables[lon])
lat_length = len(netcdf_aim_file.variables[lat])
lon_dimension_list = list(netcdf_aim_file.variables[lon].dimensions)
lat_dimension_list = list(netcdf_aim_file.variables[lat].dimensions)
# create json to store the variable data
# and because we will visualize data on a map
# the visualization data will be 3D (use 2D as x and y and display the
# last dimension the data on a map)
# or will be 4D (use 2D data as x and y, when users click on a element,
# then display the the left 2D in a line chart)
# for 4D we can also use slider. e.g. temperature is a 4D variable
# with (lat, lon, time, temperture)
# for 3D we won't use json_projects_final
# for 4D we will use json_projects_final for the final json outputs
# use different for loop if it is 4D, now only it works for 3D
# now the two loops are almost the same
if int(dimension_num) == 3:
json_projects = []
first_dimension_length = len(netcdf_aim_file.dimensions[dimension_name_list[0]])
second_dimension_length = len(netcdf_aim_file.dimensions[dimension_name_list[1]])
for count_y in range(0,first_dimension_length):
temp_array = ''
for count_x in range(0,second_dimension_length):
# python dictionary does not like array in it, so have to use string
temp_array = temp_array + str(netcdf_aim_file.variables[variable_name][count_y][count_x]) +","
# remove the last ,
# don't know why temp_array.rstrip(',') does not work
temp_change_str_into_list = temp_array.split(',')
temp_change_str_into_list.pop()
temp_array = ','.join(temp_change_str_into_list)
json_projects.append({"data" : temp_array})
json_projects = json.dumps(json_projects)
return json_projects
elif int(dimension_num) == 4:
'''
4D json file will be like this:
{
"min_value":xxx,
"max_value":xxx,
"x_num":xxx,
"y_num":xxx,
"max_lat":xxx,
"max_lon":xxx,
"min_lat":xxx,
"min_lon":xxx,
"time":['2014-1-1',...,],
"location":[{"lat":123,
"lon":234,
"x_location":xxx,
"y_location":xxx,
"data":[1,2,3,4]},
{...}
]
}
x_num means how many elements we have alone x-axis
y_num means how many elements we have alone y-axis
(x_location,y_location) is the coordinate in the 2D map
min_value is the minimum value of all location "data"
time should have as many elements as data
which means we can find data timestamp from time array
!!!!
max_lat,max_lon,min_lat,min_lon are used to locate the 2D map
in google map. I know this is not right if the data is not rectangular
!!!!
'''
# NOTICE!!!!!
# for this version the lat and lon should be decided by the same two dimensions
# and lat_lon_first_dimension_length is for 2D map y-axis
# and lat_lon_second_dimension_length is for 2D map x-axis
lat_lon_first_dimension_length = len(netcdf_aim_file.dimensions[lon_dimension_list[0]])
lat_lon_second_dimension_length = len(netcdf_aim_file.dimensions[lon_dimension_list[1]])
# other_dimension_name is for the dimension that is not decided lat and lon
other_dimension_name = ''
for item in dimension_name_list:
if item not in lon_dimension_list:
other_dimension_name = item
other_dimension_length = len(netcdf_aim_file.dimensions[other_dimension_name])
# create executable string for 4D to find out the order of each dimension
# two of the dimensions are variable itself and "another one"
# for the "another one" we can use 0 to specify its value for now
exec_string='temp_array = temp_total_array['
lat_lon_first_dimension_index = dimension_name_list.index(lon_dimension_list[0])
lat_lon_second_dimension_index = dimension_name_list.index(lon_dimension_list[1])
# dynamically create execuation string
# exec_string should be something like:
# temp_array = temp_total_array[:,count_lat_lon_first,count_lat_lon_second].tolist()
for count in range(3):
if count == lat_lon_first_dimension_index:
exec_string = exec_string + 'count_lat_lon_first,'
elif count == lat_lon_second_dimension_index:
exec_string = exec_string + 'count_lat_lon_second,'
else:
exec_string = exec_string + ':,'
# .item() is used to convert numpy.float into float
exec_string = exec_string + '].tolist()'
# print exec_string
json_projects = {}
json_projects_location = []
json_projects_final = {}
# get all the data into an numpy array
temp_total_array = netcdf_aim_file.variables[variable_name][:][:][:]
# get the min and max lat and lon
min_lat = netcdf_aim_file.variables[lat][:][:].min().item()
max_lat = netcdf_aim_file.variables[lat][:][:].max().item()
min_lon = netcdf_aim_file.variables[lon][:][:].min().item()
max_lon = netcdf_aim_file.variables[lon][:][:].max().item()
# get the max and min value of the chosen variable
min_value = netcdf_aim_file.variables[variable_name][:][:][:].min().item()
max_value = netcdf_aim_file.variables[variable_name][:][:][:].max().item()
# record time
start_time = time.time()
# create json file
temp_lat_array = netcdf_aim_file.variables[lat][:][:]
temp_lon_array = netcdf_aim_file.variables[lon][:][:]
for count_lat_lon_first in range(0,lat_lon_first_dimension_length):
for count_lat_lon_second in range(0,lat_lon_second_dimension_length):
# numpy_array[:,1,1] means obtain all data from numpy_array[all][1][1]
# temp_array = temp_total_array[:,count_lat_lon_first,count_lat_lon_second].tolist()
exec exec_string
json_projects = {}
json_projects['data'] = temp_array
# lat and lon should be decided by two dimensions in this version application
# .item() is used to convert numpy.float into float
json_projects['lat'] = temp_lat_array[count_lat_lon_first][count_lat_lon_second].item()
json_projects['lon'] = temp_lon_array[count_lat_lon_first][count_lat_lon_second].item()
json_projects['x_location'] = count_lat_lon_second
json_projects['y_location'] = count_lat_lon_first
json_projects_location.append(json_projects)
print('--- %s second for extration and creating json---' % (time.time() - start_time))
start_time = time.time()
# this is used to store other dimension information
temp_array = []
for count_other in range(0,other_dimension_length):
temp_array.append(netcdf_aim_file.variables[other_dimension_name][count_other].item())
# add location
json_projects_final['location'] = json_projects_location
json_projects_final['y_num'] = lat_lon_first_dimension_length
json_projects_final['x_num'] = lat_lon_second_dimension_length
json_projects_final['min_value'] = min_value
json_projects_final['max_value'] = max_value
# record the min and max of lat and lon
json_projects_final['min_lon'] = min_lon
json_projects_final['min_lat'] = min_lat
json_projects_final['max_lon'] = max_lon
json_projects_final['max_lat'] = max_lat
# add another dimension
json_projects_final[other_dimension_name] = temp_array
json_projects_final = json.dumps(json_projects_final)
# print the total time
print('--- %s second for final json---' % (time.time() - start_time))
return json_projects_final