def post(self, request): serializer = DataSetProcessingSerializer(data=request.data) data_set = DataSet.objects.get(id=request.data['data_set_id']) dataProcessing.process(open_path=data_set.step3).drop(request.data['drop_fields'], data_set.stepX1) data_set.step3 = data_set.step3.replace(".csv", "d.csv") data_set.save() return Response({"message": "已成功批量删除字段"}, status=status.HTTP_200_OK)
def reset_columns(request): data_set = DataSet.objects.get(id=request.data['data_set_id']) dataProcessing.process(open_path=data_set.step3).reset_columns(request.data['reset'], data_set.stepX1) # data_set.step3 = data_set.step3.replace(".csv", "r.csv") data_set.save() return Response({"message": "批量修改字段名已经完成"})
def set_index(request): data_set = DataSet.objects.get(id=request.data['data_set_id']) dataProcessing.process(open_path=data_set.step3).set_index() data_set.step3 = data_set.step3.replace(".csv", "i.csv") data_set.save() return Response({"message": "序号列添加已完成"})
def test_changeType(request): data_set = DataSet.objects.get(id=request.data['data_set_id']) result = dataProcessing.process(open_path=data_set.step3).test_changeType(request.data['key'], data_set.stepX1) print(result) if not result: dataProcessing.process(open_path=data_set.step3).changeType(request.data['key'], data_set.stepX1) return Response({"message": "类型转换成功"}) else: return Response({"违规率": str(result)})
def filters(request): logger = logging.getLogger('django') logger.debug("req_data is " + str(request.data['filter'])) data_set = DataSet.objects.get(id=request.data['data_set_id']) dataProcessing.process(open_path=data_set.step3).filter_processing(request.data['logical_type'], request.data['filter']) data_set.save() return Response({"message": "过滤处理已完成"})
def sorting(request): data_set = DataSet.objects.get(id=request.data['data_set_id']) dataProcessing.process(open_path=data_set.step3).sorting(request.data['col_name'], request.data['ascending']) data_set.save() path = "/home/ZoomInDataSet/test1.json" # 服务器路径 # path = "/Users/cribbee/Downloads/test1.json" # 本机的路径 # path = "D:\\Test\\test1.json" # windos 路径 transformer.trans(json_path=path, csv_path=data_set.step3).csv2json() ds = codecs.open(path, 'r', 'utf-8') ls = json.load(ds) os.remove(path) return Response({"message": "字段排序已完成", "data": ls, "code": "200"}, status=status.HTTP_200_OK)
def resetColumns_name_type_desc(request): data_set = DataSet.objects.get(id=request.data['data_set_id']) result = dataProcessing.process(open_path=data_set.step3).test_changeType(request.data['type_field'], data_set.stepX1) if not result: dataProcessing.process(open_path=data_set.stepX1).changeDesc(request.data['desc_field']) dataProcessing.process(open_path=data_set.step3).changeType(request.data['type_field'], data_set.stepX1) dataProcessing.process(open_path=data_set.step3).reset_columns(request.data['reset_field'], data_set.stepX1) return Response({"message": "修改成功"}) else: dataProcessing.process(open_path=data_set.step3).reset_columns(request.data['reset_field'], data_set.stepX1) return Response({"message": "类型修改失败", "违规率": str(result)})
def create(self, request, *args, **kwargs): logger = logging.getLogger('django') data_set = request.data['data_set'] row_num = request.data['row_num'] req_data = {} req_data.__setitem__('task', request.data['task']) req_data.__setitem__('title', request.data['title']) req_data.__setitem__('step1', request.data['step1']) req_data.__setitem__('step2', request.data['step2']) req_data.__setitem__('step3', request.data['step3']) req_data.__setitem__('stepX1', request.data['stepX1']) taskinfo = TaskInfo.objects.get(id=req_data['task']) # 每增加一个数据集,TaskInfo.data_num +1 taskinfo.data_num += 1 taskinfo.save() # step1、2分别是存储的json文件名与最初始的csv文件名,并存储step3以备数据预处理使用,stepX1文件是数据集的总结性文件 req_data['step1'] = (str(taskinfo.task_folder) + "/Data/" + str(req_data['task']) + str(taskinfo.user_id) + str( taskinfo.data_num) + "1.json") req_data['step2'] = (str(taskinfo.task_folder) + "/Data/" + str(req_data['task']) + str(taskinfo.user_id) + str( taskinfo.data_num) + "2.csv") req_data['step3'] = (str(taskinfo.task_folder) + "/Data/" + str(req_data['task']) + str(taskinfo.user_id) + str( taskinfo.data_num) + "3.csv") req_data['stepX1'] = ( str(taskinfo.task_folder) + "/Data/" + str(req_data['task']) + str(taskinfo.user_id) + str( taskinfo.data_num) + "sum_up.csv") logger.debug("data_set_name is " + str(req_data['step1'])) serializer = self.get_serializer(data=req_data) serializer.is_valid(raise_exception=True) self.perform_create(serializer) headers = self.get_success_headers(serializer.data) # 初始json文件保存 dataProcessing.process(open_path=req_data['step1']).original_save(data_set) # 对data_set做json2csv转换 transformer.trans(json_path=req_data['step1'], csv_path=req_data['step2']).json2csv() # 对data_set进行csv文件格式下的祛除表头操作 dataProcessing.process(open_path=req_data['step2']).step2_save(int(row_num)) # 对保存后的文件复制保存以备数据处理使用 dataProcessing.process(open_path=req_data['step2']).step3_save(req_data['step3']) # 对保存后的文件做总结表创建操作 dataProcessing.process(open_path=req_data['step2']).stepX1_save(req_data['step3'], req_data['stepX1']) return Response(data=({"message": "数据上传已完成", "data": serializer.data, "code": "201"}), status=status.HTTP_201_CREATED, headers=headers)
def dropnan(request): data_set = DataSet.objects.get(id=request.data['data_set_id']) dataProcessing.process(open_path=data_set.step3).dropnan() return Response({"message": "去除空值处理完毕"})
def force_changeType(request): data_set = DataSet.objects.get(id=request.data['data_set_id']) dataProcessing.process(open_path=data_set.step3).dropRow(request.data['key']) dataProcessing.process(open_path=data_set.step3).changeType(request.data['key'], data_set.stepX1) data_set.save() return Response({"message": "类型转换完成"})
def changeDesc(request): data_set = DataSet.objects.get(id=request.data['data_set_id']) dataProcessing.process(open_path=data_set.stepX1).changeDesc(request.data['key']) data_set.save() return Response({"message": "字段描述修改完成"})
def variance(request): data_set = DataSet.objects.get(id=request.data['data_set_id']) var1 = dataProcessing.process(open_path=data_set.step3).var(request.data['key'], data_set.stepX1) data_set.save() return Response({"message": "求方差完成", "data": str(var1)})
def standardDeviation(request): data_set = DataSet.objects.get(id=request.data['data_set_id']) std1 = dataProcessing.process(open_path=data_set.step3).std(request.data['key'], data_set.stepX1) data_set.save() return Response({"message": "求标准差完成", "data": str(std1)})
def missing_value(request): data_set = DataSet.objects.get(id=request.data['id']) dataProcessing.process(open_path=data_set.step3).missing_value(axis=request.data['axis'], how=request.data['how']) data_set.step3 = data_set.step3.replace(".csv", "m.csv") data_set.save() return Response({"message": "缺失值处理已完成"})
def show_OriginColumnsName(request): data_set = DataSet.objects.get(id=request.data['data_set_id']) Columns_name = dataProcessing.process(open_path=data_set.stepX1).show_OriginColumnsName() return Response({"message": "展示源文件列名", "data": str(dict(Columns_name)), "code": "200"}, status=status.HTTP_200_OK)
def show_desc(request): data_set = DataSet.objects.get(id=request.data['data_set_id']) desc = dataProcessing.process(open_path=data_set.stepX1).show_desc() return Response({"message": "展示每列数据类型描述", "data": str(dict(desc)), "code": "200"}, status=status.HTTP_200_OK)
def zoomin_eval(request): data_set = DataSet.objects.get(id=request.data['data_set_id']) result = dataProcessing.process(open_path=data_set.step3, stepX_path=data_set.stepX1, newColumnName=request.data['newColumnName']).zoomin_eval(request.data['function']) return Response({"message": str(result)})
def average(request): data_set = DataSet.objects.get(id=request.data['data_set_id']) average1 = dataProcessing.process(open_path=data_set.step3).average(request.data['key'], data_set.stepX1) data_set.save() return Response({"message": "求平均值完成", "data": str(average1)})