def save_data_handler(request):
"""
Clean up data
"""
form = SaveFileForm(request.POST, request.FILES)
if form.is_valid():
file = request.FILES["data_file"]
choice_cleanup = form.cleaned_data["choice_cleanup"]
column_header = form.cleaned_data["column_header"]
exclude_columns = form.cleaned_data["exclude_columns"]
remain_columns = form.cleaned_data["remain_columns"]
split_row_from = form.cleaned_data["split_row_from"]
split_row_to = form.cleaned_data["split_row_to"]
save_as_name = form.cleaned_data["save_as_name"]
if save_as_name:
# When column header is check, set to row 0 (zero based index)
column_header_idx = None
if column_header == "on":
column_header_idx = 0
# df = read_file_to_dataframe(file_name, column_header_idx)
df = DataFrameUtil.file_to_dataframe(file, header=column_header_idx)
# Split row from - to
if split_row_from and split_row_from:
# To zero based index.
split_row_from_idx = int(split_row_from) - 1
split_row_to_idx = int(split_row_to)
df = df.iloc[split_row_from_idx:split_row_to_idx, :]
# Delete NaN row
if choice_cleanup == "delete":
df = DataFrameUtil.drop_na_row(df)
# Drop columns and store to new df.
if exclude_columns:
df = dataframe_exclude_columns(df, exclude_columns)
if remain_columns:
df = dataframe_remain_columns(df, remain_columns)
# Don't forget to add '.csv' at the end of the path
header = False
if column_header_idx != None:
header = True
df.to_csv(fs.get_base_location() + save_as_name, index=None, header=header)
columns_value = df.columns.tolist()
file_json_data = df.to_json(orient='values')
analyze_results = DataFrameUtil.analyze_dataframe(df)
resp_data = {msg.SUCCESS:'The file has been save as ' + save_as_name, \
'table_data': file_json_data, \
'table_columns': columns_value, \
'analysis': analyze_results}
else:
resp_data = {msg.ERROR:'[ERROR] Invalid parameter.'}
return JsonResponse(resp_data)
def process_clean_up_data_handler(request):
"""
Clean up data by removing NaN rows, drop columns
"""
form = ProcessFileForm(request.POST, request.FILES)
if form.is_valid():
file_name = request.FILES["data_file"]
choice_cleanup = form.cleaned_data["choice_cleanup"]
column_header = form.cleaned_data["column_header"]
exclude_columns = form.cleaned_data["exclude_columns"]
remain_columns = form.cleaned_data["remain_columns"]
split_row_from = form.cleaned_data["split_row_from"]
split_row_to = form.cleaned_data["split_row_to"]
df = None
if file_name:
# When column header is check, set to row 0 (zero based index)
column_header_idx = None
if column_header == "on":
column_header_idx = 0
df = DataFrameUtil.file_to_dataframe(file_name, header=column_header_idx)
# df = read_file_to_dataframe(file_name, column_header_idx)
# Split row from - to
if split_row_from and split_row_from:
# To zero based index.
split_row_from_idx = split_row_from - 1
split_row_to_idx = split_row_to
df = df.iloc[split_row_from_idx:split_row_to_idx, :]
# TODO file with mean, median
# Delete NaN row
if choice_cleanup == "delete":
df = DataFrameUtil.drop_na_row(df)
# Drop columns and store to new df.
if exclude_columns:
df = dataframe_exclude_columns(df, exclude_columns)
# Drop other columns except those specified by user.
if remain_columns:
df = dataframe_remain_columns(df, remain_columns)
file_json_data = df.to_json(orient='values')
columns_value = df.columns.tolist()
analyze_results = DataFrameUtil.analyze_dataframe(df)
resp_data = { # msg.SUCCESS:'The file has been uploaded successfully.', \
'table_data': file_json_data, \
'table_columns': columns_value, \
'analysis': analyze_results}
else:
resp_data = {msg.ERROR:'[ERROR] Invalid request parameters.'}
else:
# Form validation error
resp_data = {msg.ERROR: escape(form._errors)}
return JsonResponse(resp_data)
return JsonResponse(resp_data)
