def upload_file_handler(request):
    if(request.method == 'POST'):
        # upload file
        form = UploadFileForm(request.POST, request.FILES)
        if form.is_valid():
            data_file = request.FILES['data_file']
            column_header = form.cleaned_data['column_header']
            
            # filename = fs.save_file(data_file)
 
            column_header_idx = None
            if column_header == "on":
                column_header_idx = 0
            df = DataFrameUtil.file_to_dataframe(data_file, header=column_header_idx)
            # file_json_data, columns_value = DataFrameUtil.convert_csv_to_json(file_full_path, header_row=column_header_idx, orient='values')  # values, records
            # analyze_results = analyze_data(file_full_path)
            analyze_results = DataFrameUtil.analyze_dataframe(df)
            file_json_data, columns_name = DataFrameUtil.dataframe_to_json(df)
            
            resp_data = {  # msg.SUCCESS:'The file has been uploaded successfully.', \
                    'table_data': file_json_data, \
                    'table_columns': columns_name, \
                    'analysis': analyze_results}

            return JsonResponse(resp_data)
        else:
            # Form validation error
            resp_data = {msg.ERROR: escape(form._errors)}
            return JsonResponse(resp_data)
    else:
        resp_data = {msg.ERROR: "request is not POST."}
        return JsonResponse(resp_data)
def save_data_handler(request):
    """
    Clean up data
    """
    form = SaveFileForm(request.POST, request.FILES)
    if form.is_valid():
        file = request.FILES["data_file"]
        choice_cleanup = form.cleaned_data["choice_cleanup"]
        column_header = form.cleaned_data["column_header"]
        exclude_columns = form.cleaned_data["exclude_columns"]
        remain_columns = form.cleaned_data["remain_columns"]
        split_row_from = form.cleaned_data["split_row_from"]
        split_row_to = form.cleaned_data["split_row_to"]
        save_as_name = form.cleaned_data["save_as_name"]
    
        if save_as_name:
            # When column header is check, set to row 0 (zero based index) 
            column_header_idx = None
            if column_header == "on":
                column_header_idx = 0
                
            # df = read_file_to_dataframe(file_name, column_header_idx)
            df = DataFrameUtil.file_to_dataframe(file, header=column_header_idx)
            # Split row from - to
            if split_row_from and split_row_from:
                # To zero based index.
                split_row_from_idx = int(split_row_from) - 1
                split_row_to_idx = int(split_row_to)
                df = df.iloc[split_row_from_idx:split_row_to_idx, :]
                
            # Delete NaN row
            if choice_cleanup == "delete":
                df = DataFrameUtil.drop_na_row(df)
                
            # Drop columns and store to new df.
            if exclude_columns:
                df = dataframe_exclude_columns(df, exclude_columns)
                
            if remain_columns:
                df = dataframe_remain_columns(df, remain_columns) 
            
            # Don't forget to add '.csv' at the end of the path
            header = False
            if column_header_idx != None:
                header = True
                
            df.to_csv(fs.get_base_location() + save_as_name, index=None, header=header) 
        
            columns_value = df.columns.tolist()
            file_json_data = df.to_json(orient='values') 
            analyze_results = DataFrameUtil.analyze_dataframe(df)
            
            resp_data = {msg.SUCCESS:'The file has been save as ' + save_as_name, \
            'table_data': file_json_data, \
            'table_columns': columns_value, \
            'analysis': analyze_results} 
    else:
        resp_data = {msg.ERROR:'[ERROR] Invalid parameter.'}
    return JsonResponse(resp_data)
def analyze_data(file_full_path, header_row=None):
    
    # Read data from file by panda dataframe
    # TODO header should be specified by user
    
    # Check NaN
    df = DataFrameUtil.convert_file_to_dataframe(file_full_path, header=header_row)
    results = DataFrameUtil.analyze_dataframe(df, header_row)
    
    return results
def matched_keys_handler(request):
    form = ExtractMatchedKeysForm(request.POST, request.FILES)
    resp_data = dict()
    if form.is_valid():
        key_file = request.FILES["key_file"]
        data_file = request.FILES["data_file"]
        
        df_result = extract_matched_key(key_file, data_file)
        
        file_json_data = df_result.to_json(orient='values')
        analyze_results = DataFrameUtil.analyze_dataframe(df_result)
        resp_data['table_data'] = file_json_data  # df_result.values
        resp_data['table_columns'] = df_result.columns.tolist()
        resp_data['analysis'] = analyze_results
    else:
        resp_data[msg.ERROR] = escape(form._errors)
    
    return JsonResponse(resp_data) 
def process_clean_up_data_handler(request):
    """
    Clean up data by removing NaN rows, drop columns
    """
    form = ProcessFileForm(request.POST, request.FILES)
    if form.is_valid():
        file_name = request.FILES["data_file"]
        choice_cleanup = form.cleaned_data["choice_cleanup"]
        column_header = form.cleaned_data["column_header"]
        exclude_columns = form.cleaned_data["exclude_columns"]
        remain_columns = form.cleaned_data["remain_columns"]
        split_row_from = form.cleaned_data["split_row_from"]
        split_row_to = form.cleaned_data["split_row_to"]
        
        df = None
        if file_name:
            
            # When column header is check, set to row 0 (zero based index) 
            column_header_idx = None
            if column_header == "on":
                column_header_idx = 0
        
            df = DataFrameUtil.file_to_dataframe(file_name, header=column_header_idx)
            # df = read_file_to_dataframe(file_name, column_header_idx)
            
            # Split row from - to
            if split_row_from and split_row_from:
                # To zero based index.
                split_row_from_idx = split_row_from - 1
                split_row_to_idx = split_row_to
                df = df.iloc[split_row_from_idx:split_row_to_idx, :]
            
            # TODO file with mean, median
            # Delete NaN row
            if choice_cleanup == "delete":
                df = DataFrameUtil.drop_na_row(df)
                  
            # Drop columns and store to new df.
            if exclude_columns:
                df = dataframe_exclude_columns(df, exclude_columns)
            
            # Drop other columns except those specified by user.
            if remain_columns:
                df = dataframe_remain_columns(df, remain_columns) 
    
            file_json_data = df.to_json(orient='values')
            columns_value = df.columns.tolist()
                
            analyze_results = DataFrameUtil.analyze_dataframe(df)
            
            resp_data = {  # msg.SUCCESS:'The file has been uploaded successfully.', \
            'table_data': file_json_data, \
            'table_columns': columns_value, \
            'analysis': analyze_results}  
        else:
            resp_data = {msg.ERROR:'[ERROR] Invalid request parameters.'}
    else:
        # Form validation error
        resp_data = {msg.ERROR: escape(form._errors)}
        return JsonResponse(resp_data)
    return JsonResponse(resp_data)