def post(self, request):
serializer = DataSetProcessingSerializer(data=request.data)
data_set = DataSet.objects.get(id=request.data['data_set_id'])
dataProcessing.process(open_path=data_set.step3).drop(request.data['drop_fields'], data_set.stepX1)
data_set.step3 = data_set.step3.replace(".csv", "d.csv")
data_set.save()
return Response({"message": "已成功批量删除字段"}, status=status.HTTP_200_OK)
def reset_columns(request):
data_set = DataSet.objects.get(id=request.data['data_set_id'])
dataProcessing.process(open_path=data_set.step3).reset_columns(request.data['reset'], data_set.stepX1)
# data_set.step3 = data_set.step3.replace(".csv", "r.csv")
data_set.save()
return Response({"message": "批量修改字段名已经完成"})
def set_index(request):
data_set = DataSet.objects.get(id=request.data['data_set_id'])
dataProcessing.process(open_path=data_set.step3).set_index()
data_set.step3 = data_set.step3.replace(".csv", "i.csv")
data_set.save()
return Response({"message": "序号列添加已完成"})
def test_changeType(request):
data_set = DataSet.objects.get(id=request.data['data_set_id'])
result = dataProcessing.process(open_path=data_set.step3).test_changeType(request.data['key'], data_set.stepX1)
print(result)
if not result:
dataProcessing.process(open_path=data_set.step3).changeType(request.data['key'], data_set.stepX1)
return Response({"message": "类型转换成功"})
else:
return Response({"违规率": str(result)})
def filters(request):
logger = logging.getLogger('django')
logger.debug("req_data is " + str(request.data['filter']))
data_set = DataSet.objects.get(id=request.data['data_set_id'])
dataProcessing.process(open_path=data_set.step3).filter_processing(request.data['logical_type'],
request.data['filter'])
data_set.save()
return Response({"message": "过滤处理已完成"})
def sorting(request):
data_set = DataSet.objects.get(id=request.data['data_set_id'])
dataProcessing.process(open_path=data_set.step3).sorting(request.data['col_name'], request.data['ascending'])
data_set.save()
path = "/home/ZoomInDataSet/test1.json" # 服务器路径
# path = "/Users/cribbee/Downloads/test1.json" # 本机的路径
# path = "D:\\Test\\test1.json" # windos 路径
transformer.trans(json_path=path, csv_path=data_set.step3).csv2json()
ds = codecs.open(path, 'r', 'utf-8')
ls = json.load(ds)
os.remove(path)
return Response({"message": "字段排序已完成", "data": ls, "code": "200"}, status=status.HTTP_200_OK)
def resetColumns_name_type_desc(request):
data_set = DataSet.objects.get(id=request.data['data_set_id'])
result = dataProcessing.process(open_path=data_set.step3).test_changeType(request.data['type_field'],
data_set.stepX1)
if not result:
dataProcessing.process(open_path=data_set.stepX1).changeDesc(request.data['desc_field'])
dataProcessing.process(open_path=data_set.step3).changeType(request.data['type_field'], data_set.stepX1)
dataProcessing.process(open_path=data_set.step3).reset_columns(request.data['reset_field'], data_set.stepX1)
return Response({"message": "修改成功"})
else:
dataProcessing.process(open_path=data_set.step3).reset_columns(request.data['reset_field'], data_set.stepX1)
return Response({"message": "类型修改失败", "违规率": str(result)})
def create(self, request, *args, **kwargs):
logger = logging.getLogger('django')
data_set = request.data['data_set']
row_num = request.data['row_num']
req_data = {}
req_data.__setitem__('task', request.data['task'])
req_data.__setitem__('title', request.data['title'])
req_data.__setitem__('step1', request.data['step1'])
req_data.__setitem__('step2', request.data['step2'])
req_data.__setitem__('step3', request.data['step3'])
req_data.__setitem__('stepX1', request.data['stepX1'])
taskinfo = TaskInfo.objects.get(id=req_data['task'])
# 每增加一个数据集，TaskInfo.data_num +1
taskinfo.data_num += 1
taskinfo.save()
# step1、2分别是存储的json文件名与最初始的csv文件名,并存储step3以备数据预处理使用,stepX1文件是数据集的总结性文件
req_data['step1'] = (str(taskinfo.task_folder) + "/Data/" + str(req_data['task']) + str(taskinfo.user_id) + str(
taskinfo.data_num) + "1.json")
req_data['step2'] = (str(taskinfo.task_folder) + "/Data/" + str(req_data['task']) + str(taskinfo.user_id) + str(
taskinfo.data_num) + "2.csv")
req_data['step3'] = (str(taskinfo.task_folder) + "/Data/" + str(req_data['task']) + str(taskinfo.user_id) + str(
taskinfo.data_num) + "3.csv")
req_data['stepX1'] = (
str(taskinfo.task_folder) + "/Data/" + str(req_data['task']) + str(taskinfo.user_id) + str(
taskinfo.data_num) + "sum_up.csv")
logger.debug("data_set_name is " + str(req_data['step1']))
serializer = self.get_serializer(data=req_data)
serializer.is_valid(raise_exception=True)
self.perform_create(serializer)
headers = self.get_success_headers(serializer.data)
# 初始json文件保存
dataProcessing.process(open_path=req_data['step1']).original_save(data_set)
# 对data_set做json2csv转换
transformer.trans(json_path=req_data['step1'], csv_path=req_data['step2']).json2csv()
# 对data_set进行csv文件格式下的祛除表头操作
dataProcessing.process(open_path=req_data['step2']).step2_save(int(row_num))
# 对保存后的文件复制保存以备数据处理使用
dataProcessing.process(open_path=req_data['step2']).step3_save(req_data['step3'])
# 对保存后的文件做总结表创建操作
dataProcessing.process(open_path=req_data['step2']).stepX1_save(req_data['step3'], req_data['stepX1'])
return Response(data=({"message": "数据上传已完成", "data": serializer.data, "code": "201"}),
status=status.HTTP_201_CREATED, headers=headers)
def dropnan(request):
data_set = DataSet.objects.get(id=request.data['data_set_id'])
dataProcessing.process(open_path=data_set.step3).dropnan()
return Response({"message": "去除空值处理完毕"})
def force_changeType(request):
data_set = DataSet.objects.get(id=request.data['data_set_id'])
dataProcessing.process(open_path=data_set.step3).dropRow(request.data['key'])
dataProcessing.process(open_path=data_set.step3).changeType(request.data['key'], data_set.stepX1)
data_set.save()
return Response({"message": "类型转换完成"})
def changeDesc(request):
data_set = DataSet.objects.get(id=request.data['data_set_id'])
dataProcessing.process(open_path=data_set.stepX1).changeDesc(request.data['key'])
data_set.save()
return Response({"message": "字段描述修改完成"})
def variance(request):
data_set = DataSet.objects.get(id=request.data['data_set_id'])
var1 = dataProcessing.process(open_path=data_set.step3).var(request.data['key'], data_set.stepX1)
data_set.save()
return Response({"message": "求方差完成", "data": str(var1)})
def standardDeviation(request):
data_set = DataSet.objects.get(id=request.data['data_set_id'])
std1 = dataProcessing.process(open_path=data_set.step3).std(request.data['key'], data_set.stepX1)
data_set.save()
return Response({"message": "求标准差完成", "data": str(std1)})
def missing_value(request):
data_set = DataSet.objects.get(id=request.data['id'])
dataProcessing.process(open_path=data_set.step3).missing_value(axis=request.data['axis'], how=request.data['how'])
data_set.step3 = data_set.step3.replace(".csv", "m.csv")
data_set.save()
return Response({"message": "缺失值处理已完成"})
def show_OriginColumnsName(request):
data_set = DataSet.objects.get(id=request.data['data_set_id'])
Columns_name = dataProcessing.process(open_path=data_set.stepX1).show_OriginColumnsName()
return Response({"message": "展示源文件列名", "data": str(dict(Columns_name)), "code": "200"}, status=status.HTTP_200_OK)
def show_desc(request):
data_set = DataSet.objects.get(id=request.data['data_set_id'])
desc = dataProcessing.process(open_path=data_set.stepX1).show_desc()
return Response({"message": "展示每列数据类型描述", "data": str(dict(desc)), "code": "200"}, status=status.HTTP_200_OK)
def zoomin_eval(request):
data_set = DataSet.objects.get(id=request.data['data_set_id'])
result = dataProcessing.process(open_path=data_set.step3, stepX_path=data_set.stepX1, newColumnName=request.data['newColumnName']).zoomin_eval(request.data['function'])
return Response({"message": str(result)})
def average(request):
data_set = DataSet.objects.get(id=request.data['data_set_id'])
average1 = dataProcessing.process(open_path=data_set.step3).average(request.data['key'], data_set.stepX1)
data_set.save()
return Response({"message": "求平均值完成", "data": str(average1)})
