def save(request, project_id, component_id, atom_act_id, input_comp_id,
feature_id, feature_target):
if feature_target == "" or feature_id == "":
return Response.fail(ERRORS.PARAMS_NOT_IS_NULL)
if check_target(project_id, input_comp_id, feature_target):
return Response.fail(ERRORS.TARGET_FIELD_SELECT_ERROR)
atom_acts = AtomAct.objects.filter(project_id=project_id,
component_id=atom_act_id)
if len(atom_acts) == 0:
return Response.fail(ERRORS.ATOM_ACT_NOT_CONFIGURED, None)
atom_act = atom_acts[0]
assert isinstance(atom_act, AtomAct)
# learn_input_type = extract_component_type(atom_act.input_comp_id)
# test_input_type = extract_component_type(input_comp_id)
# feature_id = atom_act.feature_id
# feature_target = atom_act.feature_target
csv_readers = CsvReaderInfo.objects.filter(project_id=project_id,
component_id=input_comp_id)
if len(csv_readers) == 0:
return Response.fail(ERRORS.COMPONENT_NOT_EXIST, None)
csv_reader = csv_readers[0]
assert isinstance(csv_reader, CsvReaderInfo)
AtomTest.objects.filter(project_id=project_id,
component_id=component_id).delete()
AtomTest(project_id=project_id,
component_id=component_id,
atom_act_id=atom_act_id,
input_comp_id=input_comp_id,
feature_id=feature_id,
feature_target=feature_target).save()
return Response.success('')
def save(request, project_id, component_id, atom_learn_id, input_comp_id):
atom_learn = AtomLearn.objects.filter(project_id=project_id, component_id=atom_learn_id)
if len(atom_learn) == 0:
return HttpResponse(Response.fail(ERRORS.ATOM_LEARN_NOT_CONFIGURED, None).to_json())
atom_learn = atom_learn[0]
assert isinstance(atom_learn, AtomLearn)
learn_input_type = extract_component_type(atom_learn.input_comp_id)
act_input_type = extract_component_type(input_comp_id)
feature_id = atom_learn.feature_id
if act_input_type == COMPONENTS.HIVE_READER:
fields = IOFieldType.objects.filter(project_id=project_id, component_id=input_comp_id,
field__in=[feature_id])
elif act_input_type == COMPONENTS.ROBOTX_SPARK:
fields = list(IOFieldType.objects.raw(robotx_field_in_query.format(
project_id=project_id,
component_id=input_comp_id,
id=feature_id,
target=''
)))
elif act_input_type == COMPONENTS.FEATURE_COMBINE:
fields = list(IOFieldType.objects.raw(combine_field_in_query.format(
project_id=project_id,
component_id=input_comp_id,
id=feature_id,
target=''
)))
if len(fields)!=1:
return HttpResponse(Response.fail(ERRORS.INPUT_NOT_SAME_AS_LEARN, None).to_json())
AtomAct.objects.filter(project_id=project_id,component_id=component_id).delete()
AtomAct(project_id=project_id,component_id=component_id,atom_learn_id=atom_learn_id,input_comp_id=input_comp_id).save()
if learn_input_type != act_input_type:
return HttpResponse(Response.success(ERRORS.COMPONENT_NOT_SAME_AS_LEARN).to_json())
return HttpResponse(Response.success().to_json())
def downLoadReportZip(request):
project_id = request.GET.get('project_id')
component_id = request.GET.get('component_id')
error_msg = "参数缺失"
if project_id is None or project_id == "":
return Response.fail(error_msg)
zipfiles = None
file_objs = None
if component_id == "" or component_id == None:
return Response.fail(error_msg)
zipfiles = "_".join([project_id,getDate()])
file_objs = fileList(project_id)
else:
zipfiles = "_".join([project_id,component_id,getDate()])
file_objs = fileList(project_id, component_id)
dir = getDir(project_id,component_id)
print(zipfiles)
utilities = ZipUtilities()
for file_obj in file_objs:
tmp_dl_path = os.path.join(dir, file_obj)
utilities.toZip(tmp_dl_path, zipfiles)
# utilities.close()
response = StreamingHttpResponse(utilities.zip_file, content_type='application/zip')
response['Content-Disposition'] = 'attachment;filename="{0}"'.format(zipfiles+".zip")
return response
def saveInfo(request, project_id, atom_learn_id, input_comp_id, algorithm,
params: List[Param]):
# __ALGORITHM_PARAMS = ALGORITHM_PARAMS
if algorithm not in ALGORITHM_PARAMS:
return Response.fail(ERRORS.ALGORITHM_NOT_SUPPORTED, None)
algorithm_params = ALGORITHM_PARAMS[algorithm]
db_params = list()
checking_results = list()
for param in params:
values = param.values
param_name = param.name
# 参数检查
param_limit = algorithm_params[param_name]
checking_result = param_checking(param_name, values, param_limit)
if checking_result is not None:
checking_results.append(checking_result)
else:
db_params.append(
AtomLearnParam(project_id=project_id,
component_id=atom_learn_id,
param_name=param_name,
param_value=values))
# 参数有错
if len(checking_results) > 0:
return Response.fail(ERRORS.ALGORITHM_PARAM_ERROR, checking_results)
AtomLearn.objects.update_or_create(project_id=project_id,
component_id=atom_learn_id,
defaults=dict(
input_comp_id=input_comp_id,
algorithm=algorithm))
AtomLearnParam.objects.filter(project_id=project_id,
component_id=atom_learn_id).delete()
AtomLearnParam.objects.bulk_create(db_params)
result = {"data": "保存成功", "status": True, "error": ""}
return Response.success(result)
def get_log(request, project_id, component_id):
task = Task.objects.filter(project_id=project_id,
component_id=component_id)
if len(task) == 0:
return Response.fail(ERRORS.NO_SUCH_TASK)
task = task[0]
if not task.has_log:
return Response.fail(ERRORS.TASK_HAS_NO_LOG)
return Response.success(Component.fetch_log(project_id, component_id))
def execute(request, project_id, xml, execution_type, execute_id=None):
execution = CurrentExecution.objects.filter(project_id=project_id)
if len(execution) != 0:
execution = execution[0]
if execution.current_execution is not None:
return HttpResponse(
Response.fail(ERRORS.TASK_IS_EXECUTING).to_json())
save_xml(project_id, xml)
root = et.fromstring(xml)
flows = root.findall("./sequenceFlow")
topology = Toplogy()
for flow in flows:
start = flow.get('sourceRefs')
end = flow.get('targetRefs')
topology.add_line(start, end)
topology.sort()
# 根据不同执行类型，构造不同的需要执行的组件
try:
if execution_type == FULL_EXECUTION:
levels = topology.levels
need_execute = OrderedDict()
# 获取需要执行的组件
for level in levels:
for point in level:
if point.type in EXECUTABLE:
# 生成需要执行 配置文件
executor_class = eval(point.type)
executor = executor_class(project_id, point.id)
executor.need_execution(force=True)
need_execute[point.id] = point
task_id = execute_components(need_execute, project_id)
elif execution_type == CONT_EXECUTION:
levels = topology.levels
# 记录需要执行的组件
need_execute = get_need_execute(levels, project_id)
task_id = execute_components(need_execute, project_id)
elif execution_type == SING_EXECUTION:
levels, flat_points = topology.get_previous_component(execute_id)
need_execute = get_need_execute(levels[:-1], project_id,
flat_points)
execute_point = list(levels[-1])[0]
need_execute[execute_id] = execute_point
executor_class = eval(execute_point.type)
executor = executor_class(project_id, execute_point.id)
executor.need_execution(force=True)
task_id = execute_components(need_execute, project_id)
except Exception as e:
error_code = str(e)
return HttpResponse(Response.fail(error_code, None).to_json())
return HttpResponse(Response.success(task_id).to_json())
def structure(request, table_name):
# 检查 table_name 为非空
check = VALIDATE.not_null_validate(table_name, 'table_name')
if check is not None:
response = Response.fail(ERRORS.PARAMETER_VALUE_ERROR, check)
return HttpResponse(response.to_json())
result = py4j_common_hive_util('checkExist', table_name)
if isinstance(result, HttpResponse):
return result
if not result:
return HttpResponse(
Response.fail(ERRORS.HIVE_TABLE_NOT_EXIST, None).to_json())
result = py4j_common_hive_util('describeAndSample', table_name)
result = list(result)
result_trans = list()
# 从数据库类型映射到建模类型
# 1. 不支持的数据类型，标记 ignore为 true
# 2. 日期类型，标记前端，可选的最小粒度，记录在字段 date_format中
for field_desc in result:
field = field_desc.getName()
database_type_trans = field_desc.getType()
ignore = True
field_type = None
date_format = None
date_size = None
if database_type_trans in DATABASE_MAPPING:
ignore = False
field_type = DATABASE_MAPPING[database_type_trans]
sample_data = field_desc.getSampleData()
if field_type == FIELDTYPE.FACTOR:
if sample_data is not None:
sample_data = list(sample_data)
date_, size_ = is_date(sample_data)
if date_:
date_format = size_
date_size = size_
field_type = FIELDTYPE.DATE
if database_type_trans == 'TIMESTAMP':
date_format = 'second'
date_size = 'second'
elif database_type_trans == 'DATE':
date_format = 'day'
date_size = 'day'
struct = StructureClass(field, field_type, database_type_trans,
date_format, date_size, ignore)
result_trans.append(struct)
# result_trans.sort(key=lambda x: x.field)
response = Response.success(result_trans)
return HttpResponse(response.to_json())
def save_relation(request, project_id, component_id, robotx_spark_id,
self_defined_feature_id, connections: List[Connection]):
# 检查robotx
objs = Container.objects.filter(project_id=project_id,
component_id=robotx_spark_id)
if len(objs) == 0:
response = Response.fail(ERRORS.ROBOTX_NOT_CONFIGURED, None)
return HttpResponse(response.to_json())
connection_of_robotx = set()
connection_of_self_defined = set()
feature_combine_relations = list()
for connection in connections:
connection_of_robotx.add(connection.robotx_field)
connection_of_self_defined.add(connection.self_defined_field)
feature_combine_relations.append(
FeatureCombineRelation(
project_id=project_id,
component_id=component_id,
robotx_field=connection.robotx_field,
self_defined_field=connection.self_defined_field))
# 检查连接字段是否在robotx中
container = objs[0]
table_name = container.table_name
key_fields = set(container.key_fields.split(","))
if not key_fields.issuperset(connection_of_robotx):
response = Response.fail(ERRORS.FIELD_NOT_FOUND_IN_ROBOTX, None)
return HttpResponse(response.to_json())
# 检查 self_defined_feature
objs = SelfDefinedFeatureType.objects.filter(
field__in=connection_of_self_defined,
project_id=project_id,
component_id=self_defined_feature_id)
if len(objs) != len(connection_of_self_defined):
response = Response.fail(ERRORS.FIELD_NOT_FOUND_IN_SELF_DEFINED, None)
return HttpResponse(response.to_json())
# 检查通过，保存
FeatureCombine.objects.filter(project_id=project_id,
component_id=component_id).delete()
FeatureCombine(project_id=project_id,
component_id=component_id,
robotx_table_name=table_name,
robotx_spark_id=robotx_spark_id,
self_defined_feature_id=self_defined_feature_id).save()
FeatureCombineRelation.objects.filter(project_id=project_id,
component_id=component_id).delete()
FeatureCombineRelation.objects.bulk_create(feature_combine_relations)
return HttpResponse(Response.success().to_json())
def view_table(request, project_id, component_id):
robotx_task = Task.objects.filter(project_id=project_id, component_id=component_id)
if len(robotx_task)==0:
return HttpResponse(Response.fail(ERRORS.ROBOTX_NOT_SUCCESS).to_json())
robotx_task = robotx_task[0]
assert isinstance(robotx_task, Task)
if robotx_task.task_status != TASK_STATUS.SUCCEEDED:
return HttpResponse(Response.fail(ERRORS.ROBOTX_NOT_SUCCESS).to_json())
result_table = RobotXSpark.output_table(project_id, component_id)
result = py4j_common_hive_util('viewTable', result_table, 10)
if isinstance(result, HttpResponse):
return result
return HttpResponse(Response.success([dict(name=k,value=list(v)) for k,v in result.items()]).to_json())
def delete(request, project_id, component_id):
input_type = extract_component_type(component_id)
if input_type == COMPONENTS.CSV_READER:
CsvReaderInfo.objects.filter(project_id=project_id,
component_id=component_id).delete()
CsvReaderInfotype.objects.filter(project_id=project_id,
component_id=component_id).delete()
elif input_type == COMPONENTS.ATOM_ACT:
AtomActModel.objects.filter(project_id=project_id,
component_id=component_id).delete()
elif input_type == COMPONENTS.ATOM_LEARN:
AtomLearnModel.objects.filter(project_id=project_id,
component_id=component_id).delete()
AtomLearnParam.objects.filter(project_id=project_id,
component_id=component_id).delete()
elif input_type == COMPONENTS.ATOM_TEST:
AtomTestModel.objects.filter(project_id=project_id,
component_id=component_id).delete()
elif input_type == COMPONENTS.ATOM_EXPLORE:
AtomExploreModel.objects.filter(project_id=project_id,
component_id=component_id).delete()
AtomExploreParam.objects.filter(project_id=project_id,
component_id=component_id).delete()
elif input_type == COMPONENTS.ROBOTX:
Container.objects.filter(project_id=project_id,
component_id=component_id).delete()
Relation.objects.filter(project_id=project_id,
component_id=component_id).delete()
else:
return Response.fail(ERRORS.CSV_TYPE_ERROR, None)
return Response.success()
def save_with_default(request, project_id, atom_learn_id, input_comp_id, id,
target, algorithm):
"""
保存，算法的高级参数使用默认
:param request:
:param project_id:
:param atom_learn_id:
:param input_comp_id:
:param id:
:param target:
:param algorithm:
:return:
"""
if algorithm not in ALGORITHM_PARAMS:
return HttpResponse(
Response.fail(ERRORS.ALGORITHM_NOT_SUPPORTED, None).to_json())
AtomLearn.objects.update_or_create(project_id=project_id,
component_id=atom_learn_id,
defaults=dict(
input_comp_id=input_comp_id,
feature_id=id,
feature_target=target,
algorithm=algorithm))
default_params = ALGORITHM_PARAMS[algorithm]
params = list()
for param in default_params:
params.append(
AtomLearnParam(project_id=project_id,
component_id=atom_learn_id,
param_name=param,
param_value=str(default_params[param]['default'])))
AtomLearnParam.objects.filter(project_id=project_id,
component_id=atom_learn_id).delete()
AtomLearnParam.objects.bulk_create(params)
return HttpResponse(Response.success().to_json())
def upload(request, file, user_pk):
"""
file upload
:param request:
:param file:
:param user_pk:
:return:
"""
if file.size > MY_DATA_LOCAL_FILE_MAX_SIZE_IN_BYTE:
return Response.fail(ERRORS.MY_DATA_UPLOAD_SIZE_EXCEED)
file_name = md5(user_pk, datetime.now())
# saving the file
file_saving_path = os.path.join(MY_DATA_LOCAL_FILE_TMP_DIR, file_name)
with open(file_saving_path, 'wb') as destination:
if file.multiple_chunks():
for chunk in file.chunks():
destination.write(chunk)
else:
destination.write(file.read())
# record the data
LocalFile(file_name=file_name, status=STATUS.UPLOADED).save()
return Response.success(file_name)
def update(request, project_id, component_id, field_types: List[FieldType]):
# 修改资料
db_field_types = []
for field in field_types:
db_field_types.append(field.to_db_type(project_id, component_id))
# 检查数据类型
response = None
# field_types = None # type: dict[str,FieldType]
try:
# 保存类型
for db_field_type in db_field_types:
field = db_field_type.field
field_type = db_field_type.field_type
selected = db_field_type.selected
date_format = db_field_type.date_format
CsvReaderInfotype.objects.filter(project_id=project_id,
component_id=component_id,
field=field).update(
field_type=field_type,
date_format=date_format,
selected=selected,
)
response = Response.success({
"status": True,
"data": "修改成功",
"error": None
})
return response
except UnicodeDecodeError as e:
response = Response.fail(ERRORS.CSV_UTF8_ERROR, None)
return response
def login(request, user, password):
user = auth.authenticate(username=user, password=password)
if user is not None:
auth.login(request, user)
return NO_DETAIL_SUCCESS
else:
return Response.fail(LOGIN_ERROR)
def execution_status(request, project_id, task_id):
execution = Execution.objects.filter(project_id=project_id,
task_id=task_id)
if len(execution) == 0:
return HttpResponse(Response.fail(ERRORS.NO_SUCH_TASK).to_json())
execution = execution[0]
tasks = Task.objects.order_by('record_time').filter(project_id=project_id,
task_id=task_id)
query = ExecutionQuery(execution.status, execution.start_time,
execution.end_time)
for task in tasks:
assert isinstance(task, Task)
component_id = task.component_id
task_status = task.task_status
detail = task.detail
has_log = task.has_log
application_id = task.application_id
error_code = task.error_code
tracking_url = task.tracking_url
start_time = task.start_time
end_time = task.end_time
query.add_detail(component_id, task_status, detail, error_code,
application_id, tracking_url, has_log, start_time,
end_time)
if execution.status != ExecutionStatus.RUNNING:
CurrentExecution.objects.filter(project_id=project_id).update(
current_execution=None)
return HttpResponse(Response.success(query).to_json())
def data_import(request, db_type, ip, port: int, db, user, password, table,
data_name, fields: List[DataType]):
"""
import database data to hdfs
1. record information of the data, including name, fields
2. submit a task to celery, record the information of task
:param request: request object
:param db_type: database type
:param ip: database server ip
:param port: database server port
:param db: database name
:param user: user name
:param password: password
:param table: name of table need import
:param data_name: data name
:param fields: filed description
:return:
"""
user_id = request.user.id
unique = DataBaseImport.check_name_uniqueness(user_id, data_name)
if not unique:
return Response.fail(DATA_NAME_HAS_BEEN_USED)
data_id = DataBaseImport.initialize(user_id, data_name, fields)
DataBaseImport.import_(data_id, DataBaseImport.TYPE.DATABASE, db_type, ip,
port, db, user, password, table, fields)
return NO_DETAIL_SUCCESS
def load_info(request, project_id, component_id):
objs = CsvReaderInfo.objects.filter(project_id=project_id,
component_id=component_id)
if len(objs) == 0:
response = Response.fail(ERRORS.COMPONENT_NOT_EXIST, None)
return response
response = Response.success(objs[0].magic_name)
return response
def chain_validate(validators, params):
messages = []
for validator, param in zip(validators, params):
res = validator(param)
if res is not None:
messages.append(res)
if len(messages) == 0:
return None
return Response.fail(ERRORS.PARAMETER_VALUE_ERROR, "\n".join(messages))
def saveInfo(request, project_id, atom_explore_id, input_comp_id, id, target,
params: List[Param]):
# __COMM_PARAMS = COMM_PARAMS
# if target != "Target":
# return Response.fail(ERRORS.TARGET_FIELD_SELECT_ERROR)
if check_target(project_id, input_comp_id, target):
return Response.fail(ERRORS.TARGET_FIELD_SELECT_ERROR)
params_list = set([param_.name for param_ in params])
for name in params_list:
if name not in COMM_PARAMS:
return Response.fail(ERRORS.EXPLORE_PARAM_ERROR, None)
db_params = list()
checking_results = list()
for param in params:
values = param.values
param_name = param.name
# 参数检查
param_limit = COMM_PARAMS[param_name]
checking_result = param_checking(param_name, values, param_limit)
if checking_result is not None:
checking_results.append(checking_result)
else:
db_params.append(
AtomExploreParam(project_id=project_id,
component_id=atom_explore_id,
param_name=param_name,
param_value=values))
# 参数有错
if len(checking_results) > 0:
return Response.fail(ERRORS.EXPLORE_PARAM_ERROR, checking_results)
AtomExplore.objects.update_or_create(project_id=project_id,
component_id=atom_explore_id,
defaults=dict(
input_comp_id=input_comp_id,
feature_id=id,
feature_target=target))
AtomExploreParam.objects.filter(project_id=project_id,
component_id=atom_explore_id).delete()
AtomExploreParam.objects.bulk_create(db_params)
result = {"data": "保存成功", "status": True, "error": ""}
return Response.success(result)
def view_table(request, project_id, component_id):
feature_combine_task = Task.objects.filter(project_id=project_id,
component_id=component_id)
if len(feature_combine_task) == 0:
return HttpResponse(
Response.fail(ERRORS.FEATURE_COMBINE_NOT_SUCCESS).to_json())
feature_combine_task = feature_combine_task[0]
assert isinstance(feature_combine_task, Task)
if feature_combine_task.task_status != TASK_STATUS.SUCCEEDED:
return HttpResponse(
Response.fail(ERRORS.FEATURE_COMBINE_NOT_SUCCESS).to_json())
result_table = FeatureCombineComp.output_table(project_id, component_id)
result = py4j_common_hive_util('viewTable', result_table, 10)
if isinstance(result, HttpResponse):
return result
return HttpResponse(
Response.success([
dict(name=k, value=list(v)) for k, v in result.items()
]).to_json())
def robotx_spark_key_fields(request, project_id, component_id):
objs = Container.objects.filter(project_id=project_id,
component_id=component_id)
if len(objs) == 0:
response = Response.fail(ERRORS.ROBOTX_NOT_CONFIGURED, None)
return HttpResponse(response.to_json())
container = objs[0]
assert isinstance(container, Container)
key_fields = container.key_fields.split(",")
return HttpResponse(Response.success(key_fields).to_json())
def save(request, project_id, atom_learn_id, input_comp_id, id, target,
algorithm, params: List[Param]):
if algorithm not in ALGORITHM_PARAMS:
return HttpResponse(
Response.fail(ERRORS.ALGORITHM_NOT_SUPPORTED, None).to_json())
algorithm_params = ALGORITHM_PARAMS[algorithm]
db_params = list()
checking_results = list()
for param in params:
values = param.values
param_name = param.name
# 参数检查
param_limit = algorithm_params[param_name]
checking_result = param_checking(param_name, values, param_limit)
if checking_result is not None:
checking_results.append(checking_result)
else:
db_params.append(
AtomLearnParam(project_id=project_id,
component_id=atom_learn_id,
param_name=param_name,
param_value=values))
# 参数有错
if len(checking_results) > 0:
return HttpResponse(
Response.fail(ERRORS.ALGORITHM_PARAM_ERROR,
checking_results).to_json())
AtomLearn.objects.update_or_create(project_id=project_id,
component_id=atom_learn_id,
defaults=dict(
input_comp_id=input_comp_id,
feature_id=id,
feature_target=target,
algorithm=algorithm))
AtomLearnParam.objects.filter(project_id=project_id,
component_id=atom_learn_id).delete()
AtomLearnParam.objects.bulk_create(db_params)
return HttpResponse(Response.success().to_json())
def saveInfo(request, project_id, component_id, atom_learn_id, input_comp_id,
reason_code_nvars, ncores):
try:
if reason_code_nvars == "":
return Response.fail(ERRORS.PARAMS_NOT_IS_NULL, None)
ncores = 0 if ncores == "" else ncores
atom_learn = AtomLearn.objects.filter(project_id=project_id,
component_id=atom_learn_id)
if len(atom_learn) == 0:
return Response.fail(ERRORS.ATOM_LEARN_NOT_CONFIGURED, None)
atom_learn = atom_learn[0]
assert isinstance(atom_learn, AtomLearn)
csv_reader = CsvReaderInfo.objects.filter(project_id=project_id,
component_id=input_comp_id)
if len(csv_reader) == 0:
return Response.fail(ERRORS.CSV_READER_NOT_CONFIGURED, None)
csv_reader = csv_reader[0]
assert isinstance(csv_reader, CsvReaderInfo)
# learn_input_type = extract_component_type(atom_learn.input_comp_id)
# act_input_type = extract_component_type(input_comp_id)
# feature_id = atom_learn.feature_id
# if learn_input_type != act_input_type:
# return Response.success(ERRORS.COMPONENT_NOT_SAME_AS_ACT)
AtomAct.objects.filter(project_id=project_id,
component_id=component_id).delete()
AtomAct(project_id=project_id,
component_id=component_id,
atom_learn_id=atom_learn_id,
input_comp_id=input_comp_id,
reason_code_nvars=reason_code_nvars,
ncores=ncores).save()
return Response.success()
except UnicodeDecodeError as e:
response = Response.fail(ERRORS.COMPONENT_NOT_SAME_AS_ACT, None)
return response
def save_with_default(request, project_id, atom_learn_id, input_comp_id,
algorithm):
"""
保存，算法的高级参数使用默认
:param request:
:param project_id:
:param atom_learn_id:
:param input_comp_id:
:param algorithm:
:return:
"""
try:
if algorithm not in algorithms.ALGORITHM_PARAMS:
return Response.fail(ERRORS.ALGORITHM_NOT_SUPPORTED, None)
AtomLearn.objects.update_or_create(project_id=project_id,
component_id=atom_learn_id,
defaults=dict(
input_comp_id=input_comp_id,
algorithm=algorithm))
params = list()
if algorithm not in ["naivebayes", "lr"]:
default_params = ALGORITHM_PARAMS[algorithm]
for param in default_params:
params.append(
AtomLearnParam(project_id=project_id,
component_id=atom_learn_id,
param_name=param,
param_value=str(
default_params[param]['default'])))
AtomLearnParam.objects.filter(project_id=project_id,
component_id=atom_learn_id).delete()
AtomLearnParam.objects.bulk_create(params)
result = {"data": "保存成功", "status": True, "error": ""}
return Response.success(result)
except UnicodeDecodeError as e:
response = Response.fail(ERRORS.SAVE_PARAM_ERROR, None)
return response
def report(request):
project_id = request.GET.get('project_id')
component_id = request.GET.get('component_id')
fileName = request.GET.get('fileName')
error_msg = "参数缺失"
if project_id is None or project_id == "":
return Response.fail(error_msg)
if component_id is None or component_id == "":
return Response.fail(error_msg)
component_type = re.sub('Atom', '', extract_component_type(component_id))
if fileName is None or fileName == "":
if component_type in apps.COMPONENTS.ROBOTX:
fileName = "tmp/full.sql"
else:
fileName =component_type+".txt"
print(setting.WORKING_DIRECTORY, project_id, component_type,fileName)
file = os.path.join(setting.WORKING_DIRECTORY, project_id, component_type,fileName)
# file = os.path.join(setting.WORKING_DIRECTORY, project_id, component_id, component_type,fileName)
try:
with codecs.open(file, 'r+') as get:
content = get.read()
except FileNotFoundError:
content = "File is not found. or You don't have permission to access this file."
return Response.success({"data":content})
def preview(result, project_id, component_id):
reader = HiveReader.objects.filter(project_id=project_id,
component_id=component_id)
if len(reader) == 0:
return HttpResponse(
Response.fail(ERRORS.HIVE_TABLE_NOT_EXIST).to_json())
reader = reader[0]
table_name = reader.table_name
result = py4j_common_hive_util('viewTable', table_name, 10)
if isinstance(result, HttpResponse):
return result
return HttpResponse(
Response.success([
dict(name=k, value=list(v)) for k, v in result.items()
]).to_json())
def saveInfo(request, project_id, component_id, magic_name, file_name):
try:
# 保存组件
CsvReaderInfo.objects.filter(project_id=project_id,
component_id=component_id).delete()
CsvReaderInfo(project_id=project_id,
component_id=component_id,
magic_name=magic_name,
file_name=file_name).save()
result = {"data": {}, "status": True, "error": ""}
response = Response.success(result)
return response
except UnicodeDecodeError as e:
response = Response.fail(ERRORS.CSV_SAVE_ERROR, None)
return response
def load_field_type(request, project_id, component_id):
objs = SelfDefinedFeature.objects.filter(project_id=project_id,
component_id=component_id)
if len(objs) != 1:
response = Response.fail(ERRORS.COMPONENT_NOT_EXIST, None)
return HttpResponse(response.to_json())
db_field_types = SelfDefinedFeatureType.objects.filter(
project_id=project_id, component_id=component_id)
field_types = list()
for db_field_type in db_field_types:
field_types.append(
FieldType(db_field_type.field, db_field_type.field_type,
db_field_type.ori_type, db_field_type.date_format,
db_field_type.date_size, db_field_type.sample_data,
db_field_type.selected))
response = Response.success(field_types)
return HttpResponse(response.to_json())
def report(request, project_id, component_id):
res = dict()
for model_obj in MODEL_OBJECTS:
prop = model_obj.name
values = model_obj.cls.objects.filter(project_id=project_id, component_id=component_id)
if len(values) == 0:
if model_obj == ModelPredictionBIns:
break
else:
continue
value_lst = list()
for val in values:
value_lst.append({
p: val.__getattribute__(p)
for p in model_obj.props
})
res[prop] = value_lst
if len(res) == 0:
return HttpResponse(Response.fail(ERRORS.NO_REPORT).to_json())
return HttpResponse(Response.success(res).to_json())
def perview(request, project_id, component_id):
self_defined_feature = CsvReaderInfo.objects.filter(
project_id=project_id, component_id=component_id)
if len(self_defined_feature) == 0:
return Response.fail(ERRORS.NOT_INITED)
data_saving_path = mk_working_directory(project_id, component_id,
'data.csv')
result = list()
with (open(data_saving_path, 'r', encoding='utf-8')) as f:
csv_reader = csv.reader(f)
for row_num, row in enumerate(csv_reader):
if row_num > 10:
break
if len(result) == 0:
for col in row:
result.append(dict(name=col, value=list()))
else:
for column, sample in zip(result, row):
column['value'].append(sample)
return Response.success(result)