def upload_file_handler(request): if(request.method == 'POST'): # upload file form = UploadFileForm(request.POST, request.FILES) if form.is_valid(): data_file = request.FILES['data_file'] column_header = form.cleaned_data['column_header'] # filename = fs.save_file(data_file) column_header_idx = None if column_header == "on": column_header_idx = 0 df = DataFrameUtil.file_to_dataframe(data_file, header=column_header_idx) # file_json_data, columns_value = DataFrameUtil.convert_csv_to_json(file_full_path, header_row=column_header_idx, orient='values') # values, records # analyze_results = analyze_data(file_full_path) analyze_results = DataFrameUtil.analyze_dataframe(df) file_json_data, columns_name = DataFrameUtil.dataframe_to_json(df) resp_data = { # msg.SUCCESS:'The file has been uploaded successfully.', \ 'table_data': file_json_data, \ 'table_columns': columns_name, \ 'analysis': analyze_results} return JsonResponse(resp_data) else: # Form validation error resp_data = {msg.ERROR: escape(form._errors)} return JsonResponse(resp_data) else: resp_data = {msg.ERROR: "request is not POST."} return JsonResponse(resp_data)
def save_data_handler(request): """ Clean up data """ form = SaveFileForm(request.POST, request.FILES) if form.is_valid(): file = request.FILES["data_file"] choice_cleanup = form.cleaned_data["choice_cleanup"] column_header = form.cleaned_data["column_header"] exclude_columns = form.cleaned_data["exclude_columns"] remain_columns = form.cleaned_data["remain_columns"] split_row_from = form.cleaned_data["split_row_from"] split_row_to = form.cleaned_data["split_row_to"] save_as_name = form.cleaned_data["save_as_name"] if save_as_name: # When column header is check, set to row 0 (zero based index) column_header_idx = None if column_header == "on": column_header_idx = 0 # df = read_file_to_dataframe(file_name, column_header_idx) df = DataFrameUtil.file_to_dataframe(file, header=column_header_idx) # Split row from - to if split_row_from and split_row_from: # To zero based index. split_row_from_idx = int(split_row_from) - 1 split_row_to_idx = int(split_row_to) df = df.iloc[split_row_from_idx:split_row_to_idx, :] # Delete NaN row if choice_cleanup == "delete": df = DataFrameUtil.drop_na_row(df) # Drop columns and store to new df. if exclude_columns: df = dataframe_exclude_columns(df, exclude_columns) if remain_columns: df = dataframe_remain_columns(df, remain_columns) # Don't forget to add '.csv' at the end of the path header = False if column_header_idx != None: header = True df.to_csv(fs.get_base_location() + save_as_name, index=None, header=header) columns_value = df.columns.tolist() file_json_data = df.to_json(orient='values') analyze_results = DataFrameUtil.analyze_dataframe(df) resp_data = {msg.SUCCESS:'The file has been save as ' + save_as_name, \ 'table_data': file_json_data, \ 'table_columns': columns_value, \ 'analysis': analyze_results} else: resp_data = {msg.ERROR:'[ERROR] Invalid parameter.'} return JsonResponse(resp_data)
def analyze_data(file_full_path, header_row=None): # Read data from file by panda dataframe # TODO header should be specified by user # Check NaN df = DataFrameUtil.convert_file_to_dataframe(file_full_path, header=header_row) results = DataFrameUtil.analyze_dataframe(df, header_row) return results
def matched_keys_handler(request): form = ExtractMatchedKeysForm(request.POST, request.FILES) resp_data = dict() if form.is_valid(): key_file = request.FILES["key_file"] data_file = request.FILES["data_file"] df_result = extract_matched_key(key_file, data_file) file_json_data = df_result.to_json(orient='values') analyze_results = DataFrameUtil.analyze_dataframe(df_result) resp_data['table_data'] = file_json_data # df_result.values resp_data['table_columns'] = df_result.columns.tolist() resp_data['analysis'] = analyze_results else: resp_data[msg.ERROR] = escape(form._errors) return JsonResponse(resp_data)
def process_clean_up_data_handler(request): """ Clean up data by removing NaN rows, drop columns """ form = ProcessFileForm(request.POST, request.FILES) if form.is_valid(): file_name = request.FILES["data_file"] choice_cleanup = form.cleaned_data["choice_cleanup"] column_header = form.cleaned_data["column_header"] exclude_columns = form.cleaned_data["exclude_columns"] remain_columns = form.cleaned_data["remain_columns"] split_row_from = form.cleaned_data["split_row_from"] split_row_to = form.cleaned_data["split_row_to"] df = None if file_name: # When column header is check, set to row 0 (zero based index) column_header_idx = None if column_header == "on": column_header_idx = 0 df = DataFrameUtil.file_to_dataframe(file_name, header=column_header_idx) # df = read_file_to_dataframe(file_name, column_header_idx) # Split row from - to if split_row_from and split_row_from: # To zero based index. split_row_from_idx = split_row_from - 1 split_row_to_idx = split_row_to df = df.iloc[split_row_from_idx:split_row_to_idx, :] # TODO file with mean, median # Delete NaN row if choice_cleanup == "delete": df = DataFrameUtil.drop_na_row(df) # Drop columns and store to new df. if exclude_columns: df = dataframe_exclude_columns(df, exclude_columns) # Drop other columns except those specified by user. if remain_columns: df = dataframe_remain_columns(df, remain_columns) file_json_data = df.to_json(orient='values') columns_value = df.columns.tolist() analyze_results = DataFrameUtil.analyze_dataframe(df) resp_data = { # msg.SUCCESS:'The file has been uploaded successfully.', \ 'table_data': file_json_data, \ 'table_columns': columns_value, \ 'analysis': analyze_results} else: resp_data = {msg.ERROR:'[ERROR] Invalid request parameters.'} else: # Form validation error resp_data = {msg.ERROR: escape(form._errors)} return JsonResponse(resp_data) return JsonResponse(resp_data)