def save(request, project_id, component_id, atom_act_id, input_comp_id, feature_id, feature_target): if feature_target == "" or feature_id == "": return Response.fail(ERRORS.PARAMS_NOT_IS_NULL) if check_target(project_id, input_comp_id, feature_target): return Response.fail(ERRORS.TARGET_FIELD_SELECT_ERROR) atom_acts = AtomAct.objects.filter(project_id=project_id, component_id=atom_act_id) if len(atom_acts) == 0: return Response.fail(ERRORS.ATOM_ACT_NOT_CONFIGURED, None) atom_act = atom_acts[0] assert isinstance(atom_act, AtomAct) # learn_input_type = extract_component_type(atom_act.input_comp_id) # test_input_type = extract_component_type(input_comp_id) # feature_id = atom_act.feature_id # feature_target = atom_act.feature_target csv_readers = CsvReaderInfo.objects.filter(project_id=project_id, component_id=input_comp_id) if len(csv_readers) == 0: return Response.fail(ERRORS.COMPONENT_NOT_EXIST, None) csv_reader = csv_readers[0] assert isinstance(csv_reader, CsvReaderInfo) AtomTest.objects.filter(project_id=project_id, component_id=component_id).delete() AtomTest(project_id=project_id, component_id=component_id, atom_act_id=atom_act_id, input_comp_id=input_comp_id, feature_id=feature_id, feature_target=feature_target).save() return Response.success('')
def save(request, project_id, component_id, atom_learn_id, input_comp_id): atom_learn = AtomLearn.objects.filter(project_id=project_id, component_id=atom_learn_id) if len(atom_learn) == 0: return HttpResponse(Response.fail(ERRORS.ATOM_LEARN_NOT_CONFIGURED, None).to_json()) atom_learn = atom_learn[0] assert isinstance(atom_learn, AtomLearn) learn_input_type = extract_component_type(atom_learn.input_comp_id) act_input_type = extract_component_type(input_comp_id) feature_id = atom_learn.feature_id if act_input_type == COMPONENTS.HIVE_READER: fields = IOFieldType.objects.filter(project_id=project_id, component_id=input_comp_id, field__in=[feature_id]) elif act_input_type == COMPONENTS.ROBOTX_SPARK: fields = list(IOFieldType.objects.raw(robotx_field_in_query.format( project_id=project_id, component_id=input_comp_id, id=feature_id, target='' ))) elif act_input_type == COMPONENTS.FEATURE_COMBINE: fields = list(IOFieldType.objects.raw(combine_field_in_query.format( project_id=project_id, component_id=input_comp_id, id=feature_id, target='' ))) if len(fields)!=1: return HttpResponse(Response.fail(ERRORS.INPUT_NOT_SAME_AS_LEARN, None).to_json()) AtomAct.objects.filter(project_id=project_id,component_id=component_id).delete() AtomAct(project_id=project_id,component_id=component_id,atom_learn_id=atom_learn_id,input_comp_id=input_comp_id).save() if learn_input_type != act_input_type: return HttpResponse(Response.success(ERRORS.COMPONENT_NOT_SAME_AS_LEARN).to_json()) return HttpResponse(Response.success().to_json())
def downLoadReportZip(request): project_id = request.GET.get('project_id') component_id = request.GET.get('component_id') error_msg = "参数缺失" if project_id is None or project_id == "": return Response.fail(error_msg) zipfiles = None file_objs = None if component_id == "" or component_id == None: return Response.fail(error_msg) zipfiles = "_".join([project_id,getDate()]) file_objs = fileList(project_id) else: zipfiles = "_".join([project_id,component_id,getDate()]) file_objs = fileList(project_id, component_id) dir = getDir(project_id,component_id) print(zipfiles) utilities = ZipUtilities() for file_obj in file_objs: tmp_dl_path = os.path.join(dir, file_obj) utilities.toZip(tmp_dl_path, zipfiles) # utilities.close() response = StreamingHttpResponse(utilities.zip_file, content_type='application/zip') response['Content-Disposition'] = 'attachment;filename="{0}"'.format(zipfiles+".zip") return response
def saveInfo(request, project_id, atom_learn_id, input_comp_id, algorithm, params: List[Param]): # __ALGORITHM_PARAMS = ALGORITHM_PARAMS if algorithm not in ALGORITHM_PARAMS: return Response.fail(ERRORS.ALGORITHM_NOT_SUPPORTED, None) algorithm_params = ALGORITHM_PARAMS[algorithm] db_params = list() checking_results = list() for param in params: values = param.values param_name = param.name # 参数检查 param_limit = algorithm_params[param_name] checking_result = param_checking(param_name, values, param_limit) if checking_result is not None: checking_results.append(checking_result) else: db_params.append( AtomLearnParam(project_id=project_id, component_id=atom_learn_id, param_name=param_name, param_value=values)) # 参数有错 if len(checking_results) > 0: return Response.fail(ERRORS.ALGORITHM_PARAM_ERROR, checking_results) AtomLearn.objects.update_or_create(project_id=project_id, component_id=atom_learn_id, defaults=dict( input_comp_id=input_comp_id, algorithm=algorithm)) AtomLearnParam.objects.filter(project_id=project_id, component_id=atom_learn_id).delete() AtomLearnParam.objects.bulk_create(db_params) result = {"data": "保存成功", "status": True, "error": ""} return Response.success(result)
def get_log(request, project_id, component_id): task = Task.objects.filter(project_id=project_id, component_id=component_id) if len(task) == 0: return Response.fail(ERRORS.NO_SUCH_TASK) task = task[0] if not task.has_log: return Response.fail(ERRORS.TASK_HAS_NO_LOG) return Response.success(Component.fetch_log(project_id, component_id))
def execute(request, project_id, xml, execution_type, execute_id=None): execution = CurrentExecution.objects.filter(project_id=project_id) if len(execution) != 0: execution = execution[0] if execution.current_execution is not None: return HttpResponse( Response.fail(ERRORS.TASK_IS_EXECUTING).to_json()) save_xml(project_id, xml) root = et.fromstring(xml) flows = root.findall("./sequenceFlow") topology = Toplogy() for flow in flows: start = flow.get('sourceRefs') end = flow.get('targetRefs') topology.add_line(start, end) topology.sort() # 根据不同执行类型,构造不同的需要执行的组件 try: if execution_type == FULL_EXECUTION: levels = topology.levels need_execute = OrderedDict() # 获取需要执行的组件 for level in levels: for point in level: if point.type in EXECUTABLE: # 生成需要执行 配置文件 executor_class = eval(point.type) executor = executor_class(project_id, point.id) executor.need_execution(force=True) need_execute[point.id] = point task_id = execute_components(need_execute, project_id) elif execution_type == CONT_EXECUTION: levels = topology.levels # 记录需要执行的组件 need_execute = get_need_execute(levels, project_id) task_id = execute_components(need_execute, project_id) elif execution_type == SING_EXECUTION: levels, flat_points = topology.get_previous_component(execute_id) need_execute = get_need_execute(levels[:-1], project_id, flat_points) execute_point = list(levels[-1])[0] need_execute[execute_id] = execute_point executor_class = eval(execute_point.type) executor = executor_class(project_id, execute_point.id) executor.need_execution(force=True) task_id = execute_components(need_execute, project_id) except Exception as e: error_code = str(e) return HttpResponse(Response.fail(error_code, None).to_json()) return HttpResponse(Response.success(task_id).to_json())
def structure(request, table_name): # 检查 table_name 为非空 check = VALIDATE.not_null_validate(table_name, 'table_name') if check is not None: response = Response.fail(ERRORS.PARAMETER_VALUE_ERROR, check) return HttpResponse(response.to_json()) result = py4j_common_hive_util('checkExist', table_name) if isinstance(result, HttpResponse): return result if not result: return HttpResponse( Response.fail(ERRORS.HIVE_TABLE_NOT_EXIST, None).to_json()) result = py4j_common_hive_util('describeAndSample', table_name) result = list(result) result_trans = list() # 从数据库类型映射到建模类型 # 1. 不支持的数据类型,标记 ignore为 true # 2. 日期类型,标记前端,可选的最小粒度,记录在字段 date_format中 for field_desc in result: field = field_desc.getName() database_type_trans = field_desc.getType() ignore = True field_type = None date_format = None date_size = None if database_type_trans in DATABASE_MAPPING: ignore = False field_type = DATABASE_MAPPING[database_type_trans] sample_data = field_desc.getSampleData() if field_type == FIELDTYPE.FACTOR: if sample_data is not None: sample_data = list(sample_data) date_, size_ = is_date(sample_data) if date_: date_format = size_ date_size = size_ field_type = FIELDTYPE.DATE if database_type_trans == 'TIMESTAMP': date_format = 'second' date_size = 'second' elif database_type_trans == 'DATE': date_format = 'day' date_size = 'day' struct = StructureClass(field, field_type, database_type_trans, date_format, date_size, ignore) result_trans.append(struct) # result_trans.sort(key=lambda x: x.field) response = Response.success(result_trans) return HttpResponse(response.to_json())
def save_relation(request, project_id, component_id, robotx_spark_id, self_defined_feature_id, connections: List[Connection]): # 检查robotx objs = Container.objects.filter(project_id=project_id, component_id=robotx_spark_id) if len(objs) == 0: response = Response.fail(ERRORS.ROBOTX_NOT_CONFIGURED, None) return HttpResponse(response.to_json()) connection_of_robotx = set() connection_of_self_defined = set() feature_combine_relations = list() for connection in connections: connection_of_robotx.add(connection.robotx_field) connection_of_self_defined.add(connection.self_defined_field) feature_combine_relations.append( FeatureCombineRelation( project_id=project_id, component_id=component_id, robotx_field=connection.robotx_field, self_defined_field=connection.self_defined_field)) # 检查连接字段是否在robotx中 container = objs[0] table_name = container.table_name key_fields = set(container.key_fields.split(",")) if not key_fields.issuperset(connection_of_robotx): response = Response.fail(ERRORS.FIELD_NOT_FOUND_IN_ROBOTX, None) return HttpResponse(response.to_json()) # 检查 self_defined_feature objs = SelfDefinedFeatureType.objects.filter( field__in=connection_of_self_defined, project_id=project_id, component_id=self_defined_feature_id) if len(objs) != len(connection_of_self_defined): response = Response.fail(ERRORS.FIELD_NOT_FOUND_IN_SELF_DEFINED, None) return HttpResponse(response.to_json()) # 检查通过,保存 FeatureCombine.objects.filter(project_id=project_id, component_id=component_id).delete() FeatureCombine(project_id=project_id, component_id=component_id, robotx_table_name=table_name, robotx_spark_id=robotx_spark_id, self_defined_feature_id=self_defined_feature_id).save() FeatureCombineRelation.objects.filter(project_id=project_id, component_id=component_id).delete() FeatureCombineRelation.objects.bulk_create(feature_combine_relations) return HttpResponse(Response.success().to_json())
def view_table(request, project_id, component_id): robotx_task = Task.objects.filter(project_id=project_id, component_id=component_id) if len(robotx_task)==0: return HttpResponse(Response.fail(ERRORS.ROBOTX_NOT_SUCCESS).to_json()) robotx_task = robotx_task[0] assert isinstance(robotx_task, Task) if robotx_task.task_status != TASK_STATUS.SUCCEEDED: return HttpResponse(Response.fail(ERRORS.ROBOTX_NOT_SUCCESS).to_json()) result_table = RobotXSpark.output_table(project_id, component_id) result = py4j_common_hive_util('viewTable', result_table, 10) if isinstance(result, HttpResponse): return result return HttpResponse(Response.success([dict(name=k,value=list(v)) for k,v in result.items()]).to_json())
def delete(request, project_id, component_id): input_type = extract_component_type(component_id) if input_type == COMPONENTS.CSV_READER: CsvReaderInfo.objects.filter(project_id=project_id, component_id=component_id).delete() CsvReaderInfotype.objects.filter(project_id=project_id, component_id=component_id).delete() elif input_type == COMPONENTS.ATOM_ACT: AtomActModel.objects.filter(project_id=project_id, component_id=component_id).delete() elif input_type == COMPONENTS.ATOM_LEARN: AtomLearnModel.objects.filter(project_id=project_id, component_id=component_id).delete() AtomLearnParam.objects.filter(project_id=project_id, component_id=component_id).delete() elif input_type == COMPONENTS.ATOM_TEST: AtomTestModel.objects.filter(project_id=project_id, component_id=component_id).delete() elif input_type == COMPONENTS.ATOM_EXPLORE: AtomExploreModel.objects.filter(project_id=project_id, component_id=component_id).delete() AtomExploreParam.objects.filter(project_id=project_id, component_id=component_id).delete() elif input_type == COMPONENTS.ROBOTX: Container.objects.filter(project_id=project_id, component_id=component_id).delete() Relation.objects.filter(project_id=project_id, component_id=component_id).delete() else: return Response.fail(ERRORS.CSV_TYPE_ERROR, None) return Response.success()
def save_with_default(request, project_id, atom_learn_id, input_comp_id, id, target, algorithm): """ 保存,算法的高级参数使用默认 :param request: :param project_id: :param atom_learn_id: :param input_comp_id: :param id: :param target: :param algorithm: :return: """ if algorithm not in ALGORITHM_PARAMS: return HttpResponse( Response.fail(ERRORS.ALGORITHM_NOT_SUPPORTED, None).to_json()) AtomLearn.objects.update_or_create(project_id=project_id, component_id=atom_learn_id, defaults=dict( input_comp_id=input_comp_id, feature_id=id, feature_target=target, algorithm=algorithm)) default_params = ALGORITHM_PARAMS[algorithm] params = list() for param in default_params: params.append( AtomLearnParam(project_id=project_id, component_id=atom_learn_id, param_name=param, param_value=str(default_params[param]['default']))) AtomLearnParam.objects.filter(project_id=project_id, component_id=atom_learn_id).delete() AtomLearnParam.objects.bulk_create(params) return HttpResponse(Response.success().to_json())
def upload(request, file, user_pk): """ file upload :param request: :param file: :param user_pk: :return: """ if file.size > MY_DATA_LOCAL_FILE_MAX_SIZE_IN_BYTE: return Response.fail(ERRORS.MY_DATA_UPLOAD_SIZE_EXCEED) file_name = md5(user_pk, datetime.now()) # saving the file file_saving_path = os.path.join(MY_DATA_LOCAL_FILE_TMP_DIR, file_name) with open(file_saving_path, 'wb') as destination: if file.multiple_chunks(): for chunk in file.chunks(): destination.write(chunk) else: destination.write(file.read()) # record the data LocalFile(file_name=file_name, status=STATUS.UPLOADED).save() return Response.success(file_name)
def update(request, project_id, component_id, field_types: List[FieldType]): # 修改资料 db_field_types = [] for field in field_types: db_field_types.append(field.to_db_type(project_id, component_id)) # 检查数据类型 response = None # field_types = None # type: dict[str,FieldType] try: # 保存类型 for db_field_type in db_field_types: field = db_field_type.field field_type = db_field_type.field_type selected = db_field_type.selected date_format = db_field_type.date_format CsvReaderInfotype.objects.filter(project_id=project_id, component_id=component_id, field=field).update( field_type=field_type, date_format=date_format, selected=selected, ) response = Response.success({ "status": True, "data": "修改成功", "error": None }) return response except UnicodeDecodeError as e: response = Response.fail(ERRORS.CSV_UTF8_ERROR, None) return response
def login(request, user, password): user = auth.authenticate(username=user, password=password) if user is not None: auth.login(request, user) return NO_DETAIL_SUCCESS else: return Response.fail(LOGIN_ERROR)
def execution_status(request, project_id, task_id): execution = Execution.objects.filter(project_id=project_id, task_id=task_id) if len(execution) == 0: return HttpResponse(Response.fail(ERRORS.NO_SUCH_TASK).to_json()) execution = execution[0] tasks = Task.objects.order_by('record_time').filter(project_id=project_id, task_id=task_id) query = ExecutionQuery(execution.status, execution.start_time, execution.end_time) for task in tasks: assert isinstance(task, Task) component_id = task.component_id task_status = task.task_status detail = task.detail has_log = task.has_log application_id = task.application_id error_code = task.error_code tracking_url = task.tracking_url start_time = task.start_time end_time = task.end_time query.add_detail(component_id, task_status, detail, error_code, application_id, tracking_url, has_log, start_time, end_time) if execution.status != ExecutionStatus.RUNNING: CurrentExecution.objects.filter(project_id=project_id).update( current_execution=None) return HttpResponse(Response.success(query).to_json())
def data_import(request, db_type, ip, port: int, db, user, password, table, data_name, fields: List[DataType]): """ import database data to hdfs 1. record information of the data, including name, fields 2. submit a task to celery, record the information of task :param request: request object :param db_type: database type :param ip: database server ip :param port: database server port :param db: database name :param user: user name :param password: password :param table: name of table need import :param data_name: data name :param fields: filed description :return: """ user_id = request.user.id unique = DataBaseImport.check_name_uniqueness(user_id, data_name) if not unique: return Response.fail(DATA_NAME_HAS_BEEN_USED) data_id = DataBaseImport.initialize(user_id, data_name, fields) DataBaseImport.import_(data_id, DataBaseImport.TYPE.DATABASE, db_type, ip, port, db, user, password, table, fields) return NO_DETAIL_SUCCESS
def load_info(request, project_id, component_id): objs = CsvReaderInfo.objects.filter(project_id=project_id, component_id=component_id) if len(objs) == 0: response = Response.fail(ERRORS.COMPONENT_NOT_EXIST, None) return response response = Response.success(objs[0].magic_name) return response
def chain_validate(validators, params): messages = [] for validator, param in zip(validators, params): res = validator(param) if res is not None: messages.append(res) if len(messages) == 0: return None return Response.fail(ERRORS.PARAMETER_VALUE_ERROR, "\n".join(messages))
def saveInfo(request, project_id, atom_explore_id, input_comp_id, id, target, params: List[Param]): # __COMM_PARAMS = COMM_PARAMS # if target != "Target": # return Response.fail(ERRORS.TARGET_FIELD_SELECT_ERROR) if check_target(project_id, input_comp_id, target): return Response.fail(ERRORS.TARGET_FIELD_SELECT_ERROR) params_list = set([param_.name for param_ in params]) for name in params_list: if name not in COMM_PARAMS: return Response.fail(ERRORS.EXPLORE_PARAM_ERROR, None) db_params = list() checking_results = list() for param in params: values = param.values param_name = param.name # 参数检查 param_limit = COMM_PARAMS[param_name] checking_result = param_checking(param_name, values, param_limit) if checking_result is not None: checking_results.append(checking_result) else: db_params.append( AtomExploreParam(project_id=project_id, component_id=atom_explore_id, param_name=param_name, param_value=values)) # 参数有错 if len(checking_results) > 0: return Response.fail(ERRORS.EXPLORE_PARAM_ERROR, checking_results) AtomExplore.objects.update_or_create(project_id=project_id, component_id=atom_explore_id, defaults=dict( input_comp_id=input_comp_id, feature_id=id, feature_target=target)) AtomExploreParam.objects.filter(project_id=project_id, component_id=atom_explore_id).delete() AtomExploreParam.objects.bulk_create(db_params) result = {"data": "保存成功", "status": True, "error": ""} return Response.success(result)
def view_table(request, project_id, component_id): feature_combine_task = Task.objects.filter(project_id=project_id, component_id=component_id) if len(feature_combine_task) == 0: return HttpResponse( Response.fail(ERRORS.FEATURE_COMBINE_NOT_SUCCESS).to_json()) feature_combine_task = feature_combine_task[0] assert isinstance(feature_combine_task, Task) if feature_combine_task.task_status != TASK_STATUS.SUCCEEDED: return HttpResponse( Response.fail(ERRORS.FEATURE_COMBINE_NOT_SUCCESS).to_json()) result_table = FeatureCombineComp.output_table(project_id, component_id) result = py4j_common_hive_util('viewTable', result_table, 10) if isinstance(result, HttpResponse): return result return HttpResponse( Response.success([ dict(name=k, value=list(v)) for k, v in result.items() ]).to_json())
def robotx_spark_key_fields(request, project_id, component_id): objs = Container.objects.filter(project_id=project_id, component_id=component_id) if len(objs) == 0: response = Response.fail(ERRORS.ROBOTX_NOT_CONFIGURED, None) return HttpResponse(response.to_json()) container = objs[0] assert isinstance(container, Container) key_fields = container.key_fields.split(",") return HttpResponse(Response.success(key_fields).to_json())
def save(request, project_id, atom_learn_id, input_comp_id, id, target, algorithm, params: List[Param]): if algorithm not in ALGORITHM_PARAMS: return HttpResponse( Response.fail(ERRORS.ALGORITHM_NOT_SUPPORTED, None).to_json()) algorithm_params = ALGORITHM_PARAMS[algorithm] db_params = list() checking_results = list() for param in params: values = param.values param_name = param.name # 参数检查 param_limit = algorithm_params[param_name] checking_result = param_checking(param_name, values, param_limit) if checking_result is not None: checking_results.append(checking_result) else: db_params.append( AtomLearnParam(project_id=project_id, component_id=atom_learn_id, param_name=param_name, param_value=values)) # 参数有错 if len(checking_results) > 0: return HttpResponse( Response.fail(ERRORS.ALGORITHM_PARAM_ERROR, checking_results).to_json()) AtomLearn.objects.update_or_create(project_id=project_id, component_id=atom_learn_id, defaults=dict( input_comp_id=input_comp_id, feature_id=id, feature_target=target, algorithm=algorithm)) AtomLearnParam.objects.filter(project_id=project_id, component_id=atom_learn_id).delete() AtomLearnParam.objects.bulk_create(db_params) return HttpResponse(Response.success().to_json())
def saveInfo(request, project_id, component_id, atom_learn_id, input_comp_id, reason_code_nvars, ncores): try: if reason_code_nvars == "": return Response.fail(ERRORS.PARAMS_NOT_IS_NULL, None) ncores = 0 if ncores == "" else ncores atom_learn = AtomLearn.objects.filter(project_id=project_id, component_id=atom_learn_id) if len(atom_learn) == 0: return Response.fail(ERRORS.ATOM_LEARN_NOT_CONFIGURED, None) atom_learn = atom_learn[0] assert isinstance(atom_learn, AtomLearn) csv_reader = CsvReaderInfo.objects.filter(project_id=project_id, component_id=input_comp_id) if len(csv_reader) == 0: return Response.fail(ERRORS.CSV_READER_NOT_CONFIGURED, None) csv_reader = csv_reader[0] assert isinstance(csv_reader, CsvReaderInfo) # learn_input_type = extract_component_type(atom_learn.input_comp_id) # act_input_type = extract_component_type(input_comp_id) # feature_id = atom_learn.feature_id # if learn_input_type != act_input_type: # return Response.success(ERRORS.COMPONENT_NOT_SAME_AS_ACT) AtomAct.objects.filter(project_id=project_id, component_id=component_id).delete() AtomAct(project_id=project_id, component_id=component_id, atom_learn_id=atom_learn_id, input_comp_id=input_comp_id, reason_code_nvars=reason_code_nvars, ncores=ncores).save() return Response.success() except UnicodeDecodeError as e: response = Response.fail(ERRORS.COMPONENT_NOT_SAME_AS_ACT, None) return response
def save_with_default(request, project_id, atom_learn_id, input_comp_id, algorithm): """ 保存,算法的高级参数使用默认 :param request: :param project_id: :param atom_learn_id: :param input_comp_id: :param algorithm: :return: """ try: if algorithm not in algorithms.ALGORITHM_PARAMS: return Response.fail(ERRORS.ALGORITHM_NOT_SUPPORTED, None) AtomLearn.objects.update_or_create(project_id=project_id, component_id=atom_learn_id, defaults=dict( input_comp_id=input_comp_id, algorithm=algorithm)) params = list() if algorithm not in ["naivebayes", "lr"]: default_params = ALGORITHM_PARAMS[algorithm] for param in default_params: params.append( AtomLearnParam(project_id=project_id, component_id=atom_learn_id, param_name=param, param_value=str( default_params[param]['default']))) AtomLearnParam.objects.filter(project_id=project_id, component_id=atom_learn_id).delete() AtomLearnParam.objects.bulk_create(params) result = {"data": "保存成功", "status": True, "error": ""} return Response.success(result) except UnicodeDecodeError as e: response = Response.fail(ERRORS.SAVE_PARAM_ERROR, None) return response
def report(request): project_id = request.GET.get('project_id') component_id = request.GET.get('component_id') fileName = request.GET.get('fileName') error_msg = "参数缺失" if project_id is None or project_id == "": return Response.fail(error_msg) if component_id is None or component_id == "": return Response.fail(error_msg) component_type = re.sub('Atom', '', extract_component_type(component_id)) if fileName is None or fileName == "": if component_type in apps.COMPONENTS.ROBOTX: fileName = "tmp/full.sql" else: fileName =component_type+".txt" print(setting.WORKING_DIRECTORY, project_id, component_type,fileName) file = os.path.join(setting.WORKING_DIRECTORY, project_id, component_type,fileName) # file = os.path.join(setting.WORKING_DIRECTORY, project_id, component_id, component_type,fileName) try: with codecs.open(file, 'r+') as get: content = get.read() except FileNotFoundError: content = "File is not found. or You don't have permission to access this file." return Response.success({"data":content})
def preview(result, project_id, component_id): reader = HiveReader.objects.filter(project_id=project_id, component_id=component_id) if len(reader) == 0: return HttpResponse( Response.fail(ERRORS.HIVE_TABLE_NOT_EXIST).to_json()) reader = reader[0] table_name = reader.table_name result = py4j_common_hive_util('viewTable', table_name, 10) if isinstance(result, HttpResponse): return result return HttpResponse( Response.success([ dict(name=k, value=list(v)) for k, v in result.items() ]).to_json())
def saveInfo(request, project_id, component_id, magic_name, file_name): try: # 保存组件 CsvReaderInfo.objects.filter(project_id=project_id, component_id=component_id).delete() CsvReaderInfo(project_id=project_id, component_id=component_id, magic_name=magic_name, file_name=file_name).save() result = {"data": {}, "status": True, "error": ""} response = Response.success(result) return response except UnicodeDecodeError as e: response = Response.fail(ERRORS.CSV_SAVE_ERROR, None) return response
def load_field_type(request, project_id, component_id): objs = SelfDefinedFeature.objects.filter(project_id=project_id, component_id=component_id) if len(objs) != 1: response = Response.fail(ERRORS.COMPONENT_NOT_EXIST, None) return HttpResponse(response.to_json()) db_field_types = SelfDefinedFeatureType.objects.filter( project_id=project_id, component_id=component_id) field_types = list() for db_field_type in db_field_types: field_types.append( FieldType(db_field_type.field, db_field_type.field_type, db_field_type.ori_type, db_field_type.date_format, db_field_type.date_size, db_field_type.sample_data, db_field_type.selected)) response = Response.success(field_types) return HttpResponse(response.to_json())
def report(request, project_id, component_id): res = dict() for model_obj in MODEL_OBJECTS: prop = model_obj.name values = model_obj.cls.objects.filter(project_id=project_id, component_id=component_id) if len(values) == 0: if model_obj == ModelPredictionBIns: break else: continue value_lst = list() for val in values: value_lst.append({ p: val.__getattribute__(p) for p in model_obj.props }) res[prop] = value_lst if len(res) == 0: return HttpResponse(Response.fail(ERRORS.NO_REPORT).to_json()) return HttpResponse(Response.success(res).to_json())
def perview(request, project_id, component_id): self_defined_feature = CsvReaderInfo.objects.filter( project_id=project_id, component_id=component_id) if len(self_defined_feature) == 0: return Response.fail(ERRORS.NOT_INITED) data_saving_path = mk_working_directory(project_id, component_id, 'data.csv') result = list() with (open(data_saving_path, 'r', encoding='utf-8')) as f: csv_reader = csv.reader(f) for row_num, row in enumerate(csv_reader): if row_num > 10: break if len(result) == 0: for col in row: result.append(dict(name=col, value=list())) else: for column, sample in zip(result, row): column['value'].append(sample) return Response.success(result)