Example #1
0
def netcdf_histogram_visualization(variablename='', xaxis='', start='0', end='0', sectionNumber='1'):
    global netcdf_file_local_path
    # open netcdf file
    netcdf_aim_file = Dataset(netcdf_file_local_path, "r")
    
    element_num = len(netcdf_aim_file.dimensions[xaxis])
    section_num = int(sectionNumber)
    class_information_list = []
    temp_dim_handle = netcdf_aim_file.variables[variablename]
    start_frame = int(start)
    end_frame = int(end)
    # load_num means how many frames to visualize each time
    load_number = 5
    # if start==end==0 means this is the first step
    if start_frame==0 and end_frame==0:
        if element_num <= load_number:
            # element_num-1 because the index starts from 0
            end_frame = element_num - 1
        else:
            end_frame = load_number
        # set up for loop to create a list of class
        # for count in range(0,element_num):
        for count in range(start_frame,end_frame):
            # record time
            # start_time = time.time()
            # initialize these array and lists
            section_results_number = []
            tab_name_list = []
            # TODO need to dynamically obtain data, not just first dimension
            aim_array = temp_dim_handle[count].flatten()
            units = ''
            standard_name = ''
            util.sort_section_number(aim_array, section_num, section_results_number, tab_name_list)
            # .ncattrs() is used to list all the attributes of the chosen variable
            if  'units' in temp_dim_handle.ncattrs():
                units = temp_dim_handle.units
            if  'standard_name' in temp_dim_handle.ncattrs():
                standard_name = temp_dim_handle.standard_name
            temp_object = FrequenceDistributionInformation(aim_array, section_results_number, tab_name_list, units, standard_name)
            class_information_list.append(temp_object)
            # print('--- %s second ---' % (time.time() - start_time))
    
        return render_template('visualization/netcdf_visualization_histogram_results.html', \
               variablename=variablename, class_information_list=class_information_list, \
               load_number=load_number, element_num=element_num)
    # if not the first step, we need return a json file contains all the information to visualize
    '''
    json file is like this:
    {
        "timestamp":[
             {
                 "standard_name": "XXX",
                 "units": "YYY",
                 "section_results_number":[1,2,3],
                 "tab_name_list": ['aatobb','bbtocc']
             },
             {}
        ]
    }
    information is based on time, this means json[y] is the information for timestamp y
    '''    
    # for this part, start and end should be all less than element_num
    # and this is controlled in netcdf_histogram_visualization_results.js
    # I don't know why elif does not work
    # elif start_frame<element_num and end_frame<=element_num:
    if start_frame<element_num and end_frame<=element_num:
        json_final = {}
        json_timestamp = []
        # !!!!!!!!!!!!!!!!!!!!
        # set up for loop to create a json file
        for count in range(start_frame,end_frame):
            # record time
            # start_time = time.time()
            # initialize these array and lists
            section_results_number = []
            tab_name_list = []
            json_projects = {}
            # TODO need to dynamically obtain data, not just first dimension
            aim_array = temp_dim_handle[count].flatten()
            units = ''
            standard_name = ''
            util.sort_section_number(aim_array, section_num, section_results_number, tab_name_list)
            # .ncattrs() is used to list all the attributes of the chosen variable
            if  'units' in temp_dim_handle.ncattrs():
                units = temp_dim_handle.units
            if  'standard_name' in temp_dim_handle.ncattrs():
                standard_name = temp_dim_handle.standard_name

            json_projects['units'] = units
            json_projects['standard_name'] = standard_name
            json_projects['section_results_number'] = section_results_number
            json_projects['tab_name_list'] = tab_name_list
            json_timestamp.append(json_projects)

        json_final['timestamp'] = json_timestamp
        json_final = json.dumps(json_final)
        return json_final
    else:
        # we should return an error page is start and end is bigger than element_num
        return 'error'
Example #2
0
def relation(path):
    # get the current app location
    # app/visualization
    app_root = os.path.dirname(os.path.abspath(__file__))
    # the file extension could only be csv now
    if path.endswith('.csv'):
        # print app_root + "/tempData/" + path
        filename_csv = path.rsplit("/",1)[1]
        return send_from_directory(app_root + "/tempData/", filename_csv)
    elif path.endswith('.nc'):
        # print app_root + "/tempData/" + path
        filename_nc = path.rsplit("/",1)[1]
        return send_from_directory(app_root + "/tempData/", filename_nc)
    elif path.endswith('.js'):
        # this line is used to get the js filename
		filename_js = path.rsplit("/",1)[1]
		return send_from_directory(app_root + "/../templates/visualization/js", \
                                   filename_js)
    elif path.endswith('.png'):
        # this line is used to get the png filename
		filename_img = path.rsplit("/",1)[1]
		return send_from_directory(app_root + "/../templates/visualization/img", \
                                   filename_img)
    # this used to obtain parameters for NetCDF line chart visualization
    elif path.endswith('LineChartNetCDFParameters'):
        # this line is used to get the nc parameters
        vis_param = path.rsplit("/",1)[1]
        vis_param = util.replace_last(vis_param,'LineChartNetCDFParameters','')
        csv_string = line_chart_last_step(vis_param)
        return render_template('visualization/netcdf_visualization_line_chart_results.html', \
                               csv_string=csv_string)
    # this used to obtain parameters for NetCDF bar chart visualization
    elif path.endswith('BarChartNetCDFParameters'):
        # this line is used to get the nc parameters
        vis_param = path.rsplit("/",1)[1]
        vis_param = util.replace_last(vis_param,'BarChartNetCDFParameters','')
        x_axis = []
        x_name = ''
        y_axis = []
        y_name = ''
        name_list = []
        bar_chart_last_step(vis_param, x_axis, y_axis, name_list)
        x_name = name_list[0]
        y_name = name_list[1]
        # print 'aaaaaaaaaaaaa'
        # print x_axis
        return render_template('visualization/netcdf_visualization_bar_chart_results.html', \
                               x_axis=x_axis, x_name=x_name, \
                               y_axis=y_axis, y_name=y_name)
    elif path.endswith('3DApplicationResults/'):
        filename = path.rsplit('/')[-3]
        app_root = os.path.dirname(os.path.abspath(__file__))
        download_dir = app_root + '/tempData/'
        # item_name_list = util.csv_interface_obtain_labels(filename,download_dir)
        item_name_list = util.csv_interface_obtain_labels(filename, download_dir)
        input_filename = filename
        # print item_name_list
        # print input_filename
        return render_template('visualization/csv_visualization_results.html', \
                               item_name_list=item_name_list, \
                               input_filename=input_filename)

    elif path.endswith('3DApplicationResultsHistogram/'):
        # grab data from file
        filename = path.rsplit('/')[-3]
        app_root = os.path.dirname(os.path.abspath(__file__))
        download_dir = app_root + '/tempData/'
        input_file = open(download_dir + filename,'r')
        # the file just have one line
        data = input_file.readline()
        # print 'aaaaaaaaaaaaaaaaaa'
        # print data

        # -2 is because we need to remove section number and variable name
        element_num = len(data.split('//'))-2
        section_num = int(data.split('//')[1])
        class_information_list = []
        variablename = data.split('//')[0]

        # print 'aaaaaaaaaaaaaaaaaa'
        # print variablename

    
        # set up for loop to create a list of class
        for count in range(0,element_num):
            # initialize these array and lists
            section_results_number = []
            tab_name_list = []
            # get array
            aim_array = data.split('//')[count+2].split(',')
            # convert all the elements in the array into float
            aim_array = [float(i) for i in aim_array]
            # convert list into numpy array
            aim_array = numpy.asarray(aim_array)
            units = ''
            # for this part only we used variable name as standard name
            standard_name = variablename
            util.sort_section_number(aim_array, section_num, section_results_number, tab_name_list)
            # create the class object
            temp_object = FrequenceDistributionInformation(aim_array, section_results_number, tab_name_list, units, standard_name)
            class_information_list.append(temp_object)
        # this is used to compromise with my own part updates
        load_number = element_num
        return render_template('visualization/netcdf_visualization_histogram_results.html', \
               variablename=variablename, class_information_list=class_information_list, load_number=load_number)

    # all the data about certain variable in a NetCDF file will be
    # transfered as json type data
    elif path.endswith('NetCDFMapData'):
        # /filename:variablename:3:x_dimension:y_dimension:NetCDFMapData or
        # /filename:variablename:4:x_dimension:y_dimension:NetCDFMapData
        
        app_root = os.path.dirname(os.path.abspath(__file__))
        download_dir = app_root + '/tempData/'

        filename = path.split(':')[0]
        variable_name = path.split(':')[1]
        # open netcdf file
        netcdf_file_local_path = download_dir + filename
        netcdf_aim_file = Dataset(netcdf_file_local_path, "r")
        # grab dimension name list
        dimension_name_list = list(netcdf_aim_file.variables[variable_name].dimensions)
        dimension_num = path.split(':')[2]
        lat = path.split(':')[3]
        lon = path.split(':')[4]
        lon_length = len(netcdf_aim_file.dimensions[lon])
        lat_length = len(netcdf_aim_file.dimensions[lat])
        # create json to store the variable data
        # and because we will visualize data on a map
        # the visualization data will be 3D (use 2D as x and y and display the
        # last dimension the data on a map)
        # or will be 4D (use 2D data as x and y, when users click on a element,
        # then display the the left 2D in a line chart)
        # for 4D we can also use slider. e.g. temperature is a 4D variable
        # with (lat, lon, time, temperture)
        json_projects = []

        # use different for loop if it is 4D, now only it works for 3D
        # now the two loops are the same
        if int(dimension_num) == 3:
            # need to figure out which dimension is at first
            if dimension_name_list.index(lat) == 0:
                for count_lon in range(0,lon_length):
                    temp_array = ''
                    for count_lat in range(0,lat_length):
                        # python dictionary does not like array in it, so have to use string
                        temp_array = temp_array + str(netcdf_aim_file.variables[variable_name][count_lat][count_lon]) +","
                    # remove the last ,
                    # don't know why temp_array.rstrip(',') does not work
                    temp_change_str_into_list = temp_array.split(',')
                    temp_change_str_into_list.pop()
                    temp_array = ','.join(temp_change_str_into_list)
                    json_projects.append({"data" : temp_array})
            else:
                for count_lon in range(0,lon_length):
                    temp_array = ''
                    for count_lat in range(0,lat_length):
                        # python dictionary does not like array in it, so have to use string
                        temp_array = temp_array + str(netcdf_aim_file.variables[variable_name][count_lon][count_lat]) +","
                    # remove the last ,
                    # don't know why temp_array.rstrip(',') does not work
                    temp_change_str_into_list = temp_array.split(',')
                    temp_change_str_into_list.pop()
                    temp_array = ','.join(temp_change_str_into_list)
                    json_projects.append({"data" : temp_array})

            json_projects = json.dumps(json_projects)
        elif int(dimension_num) == 4:
            # create executable string for 4D to find out the order of each dimension
            # two of the dimensions are variable itself and "another one"
            # for the "another one" we can use 0 to specify its value for now
            exec_string='temp_array = temp_array + str(netcdf_aim_file.variables[variable_name]'
            lat_dimension_index = dimension_name_list.index(lat)
            lon_dimension_index = dimension_name_list.index(lon)
            for count in range(3):
                if count == lon_dimension_index:
                    exec_string  = exec_string + '[count_lon]'
                elif count == lat_dimension_index:
                    exec_string  = exec_string + '[count_lat]'
                else:
                    exec_string = exec_string + '[0]'

            exec_string = exec_string + ')+' +"','"
            # print exec_string

            for count_lon in range(0,lon_length):
                temp_array = ''
                for count_lat in range(0,lat_length):
                    # python dictionary does not like array in it, so have to use string
                    # exec_string should be something like this
                    # temp_array = temp_array + str(netcdf_aim_file.variables[variable_name][0][count_lon][count_lat]) +","
                    exec exec_string
                # remove the last ,
                # don't know why temp_array.rstrip(',') does not work
                temp_change_str_into_list = temp_array.split(',')
                temp_change_str_into_list.pop()
                temp_array = ','.join(temp_change_str_into_list)
                json_projects.append({"data" : temp_array})

            json_projects = json.dumps(json_projects)
        return json_projects
    # all the data about certain variable in a NetCDF file will be
    # transfered as json type data, lat and lon as variables
    # and they are decided by the same dimensions
    elif path.endswith('NetCDFMapDataOther'):
        # /filename:variablename:3:x_dimension:y_dimension:NetCDFMapDataOther or
        # /filename:variablename:4:x_dimension:y_dimension:NetCDFMapDataOther
        
        app_root = os.path.dirname(os.path.abspath(__file__))
        download_dir = app_root + '/tempData/'

        filename = path.split(':')[0]
        variable_name = path.split(':')[1]
        # print filename
        # open netcdf file
        netcdf_file_local_path = download_dir + filename
        netcdf_aim_file = Dataset(netcdf_file_local_path, "r")
        # grab dimension name list
        dimension_name_list = list(netcdf_aim_file.variables[variable_name].dimensions)
        dimension_num = path.split(':')[2]
        lat = path.split(':')[3]
        lon = path.split(':')[4]
        lon_length = len(netcdf_aim_file.variables[lon])
        lat_length = len(netcdf_aim_file.variables[lat])
        lon_dimension_list = list(netcdf_aim_file.variables[lon].dimensions)
        lat_dimension_list = list(netcdf_aim_file.variables[lat].dimensions)
        # create json to store the variable data
        # and because we will visualize data on a map
        # the visualization data will be 3D (use 2D as x and y and display the
        # last dimension the data on a map)
        # or will be 4D (use 2D data as x and y, when users click on a element,
        # then display the the left 2D in a line chart)
        # for 4D we can also use slider. e.g. temperature is a 4D variable
        # with (lat, lon, time, temperture)
        # for 3D we won't use json_projects_final
        # for 4D we will use json_projects_final for the final json outputs


        # use different for loop if it is 4D, now only it works for 3D
        # now the two loops are almost the same
        if int(dimension_num) == 3:
            json_projects = []
            first_dimension_length =  len(netcdf_aim_file.dimensions[dimension_name_list[0]])
            second_dimension_length =  len(netcdf_aim_file.dimensions[dimension_name_list[1]])

            for count_y in range(0,first_dimension_length):
                temp_array = ''
                for count_x in range(0,second_dimension_length):
                    # python dictionary does not like array in it, so have to use string
                    temp_array = temp_array + str(netcdf_aim_file.variables[variable_name][count_y][count_x]) +","
                # remove the last ,
                # don't know why temp_array.rstrip(',') does not work
                temp_change_str_into_list = temp_array.split(',')
                temp_change_str_into_list.pop()
                temp_array = ','.join(temp_change_str_into_list)
                json_projects.append({"data" : temp_array})

            json_projects = json.dumps(json_projects)

            return json_projects

        elif int(dimension_num) == 4:
            '''
            4D json file will be like this:
            {
                "min_value":xxx,
                "max_value":xxx,
                "x_num":xxx,
                "y_num":xxx,
                "max_lat":xxx,
                "max_lon":xxx,
                "min_lat":xxx,
                "min_lon":xxx,
                "time":['2014-1-1',...,],
                "location":[{"lat":123,
                            "lon":234,
                            "x_location":xxx,
                            "y_location":xxx,
                            "data":[1,2,3,4]},
                             {...}
                           ]
            }
            x_num means how many elements we have alone x-axis
            y_num means how many elements we have alone y-axis
            (x_location,y_location) is the coordinate in the 2D map
            min_value is the minimum value of all location "data"
            time should have as many elements as data
            which means we can find data timestamp from time array
            !!!!
            max_lat,max_lon,min_lat,min_lon are used to locate the 2D map
            in google map. I know this is not right if the data is not rectangular
            !!!!
            '''
            # NOTICE!!!!!
            # for this version the lat and lon should be decided by the same two dimensions
            # and lat_lon_first_dimension_length is for 2D map y-axis
            # and lat_lon_second_dimension_length is for 2D map x-axis
            lat_lon_first_dimension_length = len(netcdf_aim_file.dimensions[lon_dimension_list[0]])
            lat_lon_second_dimension_length = len(netcdf_aim_file.dimensions[lon_dimension_list[1]])
            # other_dimension_name is for the dimension that is not decided lat and lon
            other_dimension_name = ''
            for item in dimension_name_list:
                if item not in lon_dimension_list:
                    other_dimension_name = item
            
            other_dimension_length =  len(netcdf_aim_file.dimensions[other_dimension_name])

            # create executable string for 4D to find out the order of each dimension
            # two of the dimensions are variable itself and "another one"
            # for the "another one" we can use 0 to specify its value for now
            exec_string='temp_array = temp_total_array['
            lat_lon_first_dimension_index = dimension_name_list.index(lon_dimension_list[0])
            lat_lon_second_dimension_index = dimension_name_list.index(lon_dimension_list[1])
            # dynamically create execuation string
            # exec_string should be something like:
            # temp_array = temp_total_array[:,count_lat_lon_first,count_lat_lon_second].tolist()
            for count in range(3):
                if count == lat_lon_first_dimension_index:
                    exec_string  = exec_string + 'count_lat_lon_first,'
                elif count == lat_lon_second_dimension_index:
                    exec_string  = exec_string + 'count_lat_lon_second,'
                else:
                    exec_string = exec_string + ':,'
            # .item() is used to convert numpy.float into float
            exec_string = exec_string + '].tolist()'

            # print exec_string
            json_projects = {}
            json_projects_location = []
            json_projects_final = {}
            
            # get all the data into an numpy array
            temp_total_array = netcdf_aim_file.variables[variable_name][:][:][:]
            # get the min and max lat and lon
            min_lat = netcdf_aim_file.variables[lat][:][:].min().item()
            max_lat = netcdf_aim_file.variables[lat][:][:].max().item()
            min_lon = netcdf_aim_file.variables[lon][:][:].min().item()
            max_lon = netcdf_aim_file.variables[lon][:][:].max().item()
            # get the max and min value of the chosen variable
            min_value = netcdf_aim_file.variables[variable_name][:][:][:].min().item()
            max_value = netcdf_aim_file.variables[variable_name][:][:][:].max().item()
            # record time
            start_time = time.time()
            
            # create json file
            temp_lat_array = netcdf_aim_file.variables[lat][:][:]
            temp_lon_array = netcdf_aim_file.variables[lon][:][:]
            for count_lat_lon_first in range(0,lat_lon_first_dimension_length):
                for count_lat_lon_second in range(0,lat_lon_second_dimension_length):
                    # numpy_array[:,1,1] means obtain all data from numpy_array[all][1][1]
                    # temp_array = temp_total_array[:,count_lat_lon_first,count_lat_lon_second].tolist()
                    exec exec_string
                    json_projects = {}
                    json_projects['data'] = temp_array
                    # lat and lon should be decided by two dimensions in this version application
                    # .item() is used to convert numpy.float into float
                    json_projects['lat'] = temp_lat_array[count_lat_lon_first][count_lat_lon_second].item()
                    json_projects['lon'] = temp_lon_array[count_lat_lon_first][count_lat_lon_second].item()
                    json_projects['x_location'] = count_lat_lon_second
                    json_projects['y_location'] = count_lat_lon_first
                    json_projects_location.append(json_projects)

            print('--- %s second for extration and creating json---' % (time.time() - start_time))
            start_time = time.time()
            
            # this is used to store other dimension information
            temp_array = []
            for count_other in range(0,other_dimension_length):
                temp_array.append(netcdf_aim_file.variables[other_dimension_name][count_other].item())
            
            # add location
            json_projects_final['location'] = json_projects_location
            
            json_projects_final['y_num'] = lat_lon_first_dimension_length
            json_projects_final['x_num'] = lat_lon_second_dimension_length
            json_projects_final['min_value'] = min_value
            json_projects_final['max_value'] = max_value
            # record the min and max of lat and lon
            json_projects_final['min_lon'] = min_lon
            json_projects_final['min_lat'] = min_lat
            json_projects_final['max_lon'] = max_lon
            json_projects_final['max_lat'] = max_lat
            
            # add another dimension
            json_projects_final[other_dimension_name] = temp_array
            json_projects_final = json.dumps(json_projects_final)
            
            # print the total time
            print('--- %s second for final json---' % (time.time() - start_time))

            return json_projects_final