def upload_file_handler(request):
if(request.method == 'POST'):
# upload file
form = UploadFileForm(request.POST, request.FILES)
if form.is_valid():
data_file = request.FILES['data_file']
column_header = form.cleaned_data['column_header']
# filename = fs.save_file(data_file)
column_header_idx = None
if column_header == "on":
column_header_idx = 0
df = DataFrameUtil.file_to_dataframe(data_file, header=column_header_idx)
# file_json_data, columns_value = DataFrameUtil.convert_csv_to_json(file_full_path, header_row=column_header_idx, orient='values') # values, records
# analyze_results = analyze_data(file_full_path)
analyze_results = DataFrameUtil.analyze_dataframe(df)
file_json_data, columns_name = DataFrameUtil.dataframe_to_json(df)
resp_data = { # msg.SUCCESS:'The file has been uploaded successfully.', \
'table_data': file_json_data, \
'table_columns': columns_name, \
'analysis': analyze_results}
return JsonResponse(resp_data)
else:
# Form validation error
resp_data = {msg.ERROR: escape(form._errors)}
return JsonResponse(resp_data)
else:
resp_data = {msg.ERROR: "request is not POST."}
return JsonResponse(resp_data)
def save_data_handler(request):
"""
Clean up data
"""
form = SaveFileForm(request.POST, request.FILES)
if form.is_valid():
file = request.FILES["data_file"]
choice_cleanup = form.cleaned_data["choice_cleanup"]
column_header = form.cleaned_data["column_header"]
exclude_columns = form.cleaned_data["exclude_columns"]
remain_columns = form.cleaned_data["remain_columns"]
split_row_from = form.cleaned_data["split_row_from"]
split_row_to = form.cleaned_data["split_row_to"]
save_as_name = form.cleaned_data["save_as_name"]
if save_as_name:
# When column header is check, set to row 0 (zero based index)
column_header_idx = None
if column_header == "on":
column_header_idx = 0
# df = read_file_to_dataframe(file_name, column_header_idx)
df = DataFrameUtil.file_to_dataframe(file, header=column_header_idx)
# Split row from - to
if split_row_from and split_row_from:
# To zero based index.
split_row_from_idx = int(split_row_from) - 1
split_row_to_idx = int(split_row_to)
df = df.iloc[split_row_from_idx:split_row_to_idx, :]
# Delete NaN row
if choice_cleanup == "delete":
df = DataFrameUtil.drop_na_row(df)
# Drop columns and store to new df.
if exclude_columns:
df = dataframe_exclude_columns(df, exclude_columns)
if remain_columns:
df = dataframe_remain_columns(df, remain_columns)
# Don't forget to add '.csv' at the end of the path
header = False
if column_header_idx != None:
header = True
df.to_csv(fs.get_base_location() + save_as_name, index=None, header=header)
columns_value = df.columns.tolist()
file_json_data = df.to_json(orient='values')
analyze_results = DataFrameUtil.analyze_dataframe(df)
resp_data = {msg.SUCCESS:'The file has been save as ' + save_as_name, \
'table_data': file_json_data, \
'table_columns': columns_value, \
'analysis': analyze_results}
else:
resp_data = {msg.ERROR:'[ERROR] Invalid parameter.'}
return JsonResponse(resp_data)
def analyze_data(file_full_path, header_row=None):
# Read data from file by panda dataframe
# TODO header should be specified by user
# Check NaN
df = DataFrameUtil.convert_file_to_dataframe(file_full_path, header=header_row)
results = DataFrameUtil.analyze_dataframe(df, header_row)
return results
def matched_keys_handler(request):
form = ExtractMatchedKeysForm(request.POST, request.FILES)
resp_data = dict()
if form.is_valid():
key_file = request.FILES["key_file"]
data_file = request.FILES["data_file"]
df_result = extract_matched_key(key_file, data_file)
file_json_data = df_result.to_json(orient='values')
analyze_results = DataFrameUtil.analyze_dataframe(df_result)
resp_data['table_data'] = file_json_data # df_result.values
resp_data['table_columns'] = df_result.columns.tolist()
resp_data['analysis'] = analyze_results
else:
resp_data[msg.ERROR] = escape(form._errors)
return JsonResponse(resp_data)
def process_clean_up_data_handler(request):
"""
Clean up data by removing NaN rows, drop columns
"""
form = ProcessFileForm(request.POST, request.FILES)
if form.is_valid():
file_name = request.FILES["data_file"]
choice_cleanup = form.cleaned_data["choice_cleanup"]
column_header = form.cleaned_data["column_header"]
exclude_columns = form.cleaned_data["exclude_columns"]
remain_columns = form.cleaned_data["remain_columns"]
split_row_from = form.cleaned_data["split_row_from"]
split_row_to = form.cleaned_data["split_row_to"]
df = None
if file_name:
# When column header is check, set to row 0 (zero based index)
column_header_idx = None
if column_header == "on":
column_header_idx = 0
df = DataFrameUtil.file_to_dataframe(file_name, header=column_header_idx)
# df = read_file_to_dataframe(file_name, column_header_idx)
# Split row from - to
if split_row_from and split_row_from:
# To zero based index.
split_row_from_idx = split_row_from - 1
split_row_to_idx = split_row_to
df = df.iloc[split_row_from_idx:split_row_to_idx, :]
# TODO file with mean, median
# Delete NaN row
if choice_cleanup == "delete":
df = DataFrameUtil.drop_na_row(df)
# Drop columns and store to new df.
if exclude_columns:
df = dataframe_exclude_columns(df, exclude_columns)
# Drop other columns except those specified by user.
if remain_columns:
df = dataframe_remain_columns(df, remain_columns)
file_json_data = df.to_json(orient='values')
columns_value = df.columns.tolist()
analyze_results = DataFrameUtil.analyze_dataframe(df)
resp_data = { # msg.SUCCESS:'The file has been uploaded successfully.', \
'table_data': file_json_data, \
'table_columns': columns_value, \
'analysis': analyze_results}
else:
resp_data = {msg.ERROR:'[ERROR] Invalid request parameters.'}
else:
# Form validation error
resp_data = {msg.ERROR: escape(form._errors)}
return JsonResponse(resp_data)
return JsonResponse(resp_data)