def save_data_handler(request): """ Clean up data """ form = SaveFileForm(request.POST, request.FILES) if form.is_valid(): file = request.FILES["data_file"] choice_cleanup = form.cleaned_data["choice_cleanup"] column_header = form.cleaned_data["column_header"] exclude_columns = form.cleaned_data["exclude_columns"] remain_columns = form.cleaned_data["remain_columns"] split_row_from = form.cleaned_data["split_row_from"] split_row_to = form.cleaned_data["split_row_to"] save_as_name = form.cleaned_data["save_as_name"] if save_as_name: # When column header is check, set to row 0 (zero based index) column_header_idx = None if column_header == "on": column_header_idx = 0 # df = read_file_to_dataframe(file_name, column_header_idx) df = DataFrameUtil.file_to_dataframe(file, header=column_header_idx) # Split row from - to if split_row_from and split_row_from: # To zero based index. split_row_from_idx = int(split_row_from) - 1 split_row_to_idx = int(split_row_to) df = df.iloc[split_row_from_idx:split_row_to_idx, :] # Delete NaN row if choice_cleanup == "delete": df = DataFrameUtil.drop_na_row(df) # Drop columns and store to new df. if exclude_columns: df = dataframe_exclude_columns(df, exclude_columns) if remain_columns: df = dataframe_remain_columns(df, remain_columns) # Don't forget to add '.csv' at the end of the path header = False if column_header_idx != None: header = True df.to_csv(fs.get_base_location() + save_as_name, index=None, header=header) columns_value = df.columns.tolist() file_json_data = df.to_json(orient='values') analyze_results = DataFrameUtil.analyze_dataframe(df) resp_data = {msg.SUCCESS:'The file has been save as ' + save_as_name, \ 'table_data': file_json_data, \ 'table_columns': columns_value, \ 'analysis': analyze_results} else: resp_data = {msg.ERROR:'[ERROR] Invalid parameter.'} return JsonResponse(resp_data)
def process_clean_up_data_handler(request): """ Clean up data by removing NaN rows, drop columns """ form = ProcessFileForm(request.POST, request.FILES) if form.is_valid(): file_name = request.FILES["data_file"] choice_cleanup = form.cleaned_data["choice_cleanup"] column_header = form.cleaned_data["column_header"] exclude_columns = form.cleaned_data["exclude_columns"] remain_columns = form.cleaned_data["remain_columns"] split_row_from = form.cleaned_data["split_row_from"] split_row_to = form.cleaned_data["split_row_to"] df = None if file_name: # When column header is check, set to row 0 (zero based index) column_header_idx = None if column_header == "on": column_header_idx = 0 df = DataFrameUtil.file_to_dataframe(file_name, header=column_header_idx) # df = read_file_to_dataframe(file_name, column_header_idx) # Split row from - to if split_row_from and split_row_from: # To zero based index. split_row_from_idx = split_row_from - 1 split_row_to_idx = split_row_to df = df.iloc[split_row_from_idx:split_row_to_idx, :] # TODO file with mean, median # Delete NaN row if choice_cleanup == "delete": df = DataFrameUtil.drop_na_row(df) # Drop columns and store to new df. if exclude_columns: df = dataframe_exclude_columns(df, exclude_columns) # Drop other columns except those specified by user. if remain_columns: df = dataframe_remain_columns(df, remain_columns) file_json_data = df.to_json(orient='values') columns_value = df.columns.tolist() analyze_results = DataFrameUtil.analyze_dataframe(df) resp_data = { # msg.SUCCESS:'The file has been uploaded successfully.', \ 'table_data': file_json_data, \ 'table_columns': columns_value, \ 'analysis': analyze_results} else: resp_data = {msg.ERROR:'[ERROR] Invalid request parameters.'} else: # Form validation error resp_data = {msg.ERROR: escape(form._errors)} return JsonResponse(resp_data) return JsonResponse(resp_data)