def saveInfo(request, project_id, atom_learn_id, input_comp_id, algorithm,
params: List[Param]):
# __ALGORITHM_PARAMS = ALGORITHM_PARAMS
if algorithm not in ALGORITHM_PARAMS:
return Response.fail(ERRORS.ALGORITHM_NOT_SUPPORTED, None)
algorithm_params = ALGORITHM_PARAMS[algorithm]
db_params = list()
checking_results = list()
for param in params:
values = param.values
param_name = param.name
# 参数检查
param_limit = algorithm_params[param_name]
checking_result = param_checking(param_name, values, param_limit)
if checking_result is not None:
checking_results.append(checking_result)
else:
db_params.append(
AtomLearnParam(project_id=project_id,
component_id=atom_learn_id,
param_name=param_name,
param_value=values))
# 参数有错
if len(checking_results) > 0:
return Response.fail(ERRORS.ALGORITHM_PARAM_ERROR, checking_results)
AtomLearn.objects.update_or_create(project_id=project_id,
component_id=atom_learn_id,
defaults=dict(
input_comp_id=input_comp_id,
algorithm=algorithm))
AtomLearnParam.objects.filter(project_id=project_id,
component_id=atom_learn_id).delete()
AtomLearnParam.objects.bulk_create(db_params)
result = {"data": "保存成功", "status": True, "error": ""}
return Response.success(result)
def save_with_default(request, project_id, atom_learn_id, input_comp_id, id,
target, algorithm):
"""
保存，算法的高级参数使用默认
:param request:
:param project_id:
:param atom_learn_id:
:param input_comp_id:
:param id:
:param target:
:param algorithm:
:return:
"""
if algorithm not in ALGORITHM_PARAMS:
return HttpResponse(
Response.fail(ERRORS.ALGORITHM_NOT_SUPPORTED, None).to_json())
AtomLearn.objects.update_or_create(project_id=project_id,
component_id=atom_learn_id,
defaults=dict(
input_comp_id=input_comp_id,
feature_id=id,
feature_target=target,
algorithm=algorithm))
default_params = ALGORITHM_PARAMS[algorithm]
params = list()
for param in default_params:
params.append(
AtomLearnParam(project_id=project_id,
component_id=atom_learn_id,
param_name=param,
param_value=str(default_params[param]['default'])))
AtomLearnParam.objects.filter(project_id=project_id,
component_id=atom_learn_id).delete()
AtomLearnParam.objects.bulk_create(params)
return HttpResponse(Response.success().to_json())
def downLoadReportZip(request):
project_id = request.GET.get('project_id')
component_id = request.GET.get('component_id')
error_msg = "参数缺失"
if project_id is None or project_id == "":
return Response.fail(error_msg)
zipfiles = None
file_objs = None
if component_id == "" or component_id == None:
return Response.fail(error_msg)
zipfiles = "_".join([project_id,getDate()])
file_objs = fileList(project_id)
else:
zipfiles = "_".join([project_id,component_id,getDate()])
file_objs = fileList(project_id, component_id)
dir = getDir(project_id,component_id)
print(zipfiles)
utilities = ZipUtilities()
for file_obj in file_objs:
tmp_dl_path = os.path.join(dir, file_obj)
utilities.toZip(tmp_dl_path, zipfiles)
# utilities.close()
response = StreamingHttpResponse(utilities.zip_file, content_type='application/zip')
response['Content-Disposition'] = 'attachment;filename="{0}"'.format(zipfiles+".zip")
return response
def update(request, project_id, component_id, field_types: List[FieldType]):
# 修改资料
db_field_types = []
for field in field_types:
db_field_types.append(field.to_db_type(project_id, component_id))
# 检查数据类型
response = None
# field_types = None # type: dict[str,FieldType]
try:
# 保存类型
for db_field_type in db_field_types:
field = db_field_type.field
field_type = db_field_type.field_type
selected = db_field_type.selected
date_format = db_field_type.date_format
CsvReaderInfotype.objects.filter(project_id=project_id,
component_id=component_id,
field=field).update(
field_type=field_type,
date_format=date_format,
selected=selected,
)
response = Response.success({
"status": True,
"data": "修改成功",
"error": None
})
return response
except UnicodeDecodeError as e:
response = Response.fail(ERRORS.CSV_UTF8_ERROR, None)
return response
def upload(request, file, user_pk):
"""
file upload
:param request:
:param file:
:param user_pk:
:return:
"""
if file.size > MY_DATA_LOCAL_FILE_MAX_SIZE_IN_BYTE:
return Response.fail(ERRORS.MY_DATA_UPLOAD_SIZE_EXCEED)
file_name = md5(user_pk, datetime.now())
# saving the file
file_saving_path = os.path.join(MY_DATA_LOCAL_FILE_TMP_DIR, file_name)
with open(file_saving_path, 'wb') as destination:
if file.multiple_chunks():
for chunk in file.chunks():
destination.write(chunk)
else:
destination.write(file.read())
# record the data
LocalFile(file_name=file_name, status=STATUS.UPLOADED).save()
return Response.success(file_name)
def save(request, project_id, component_id, atom_learn_id, input_comp_id):
atom_learn = AtomLearn.objects.filter(project_id=project_id, component_id=atom_learn_id)
if len(atom_learn) == 0:
return HttpResponse(Response.fail(ERRORS.ATOM_LEARN_NOT_CONFIGURED, None).to_json())
atom_learn = atom_learn[0]
assert isinstance(atom_learn, AtomLearn)
learn_input_type = extract_component_type(atom_learn.input_comp_id)
act_input_type = extract_component_type(input_comp_id)
feature_id = atom_learn.feature_id
if act_input_type == COMPONENTS.HIVE_READER:
fields = IOFieldType.objects.filter(project_id=project_id, component_id=input_comp_id,
field__in=[feature_id])
elif act_input_type == COMPONENTS.ROBOTX_SPARK:
fields = list(IOFieldType.objects.raw(robotx_field_in_query.format(
project_id=project_id,
component_id=input_comp_id,
id=feature_id,
target=''
)))
elif act_input_type == COMPONENTS.FEATURE_COMBINE:
fields = list(IOFieldType.objects.raw(combine_field_in_query.format(
project_id=project_id,
component_id=input_comp_id,
id=feature_id,
target=''
)))
if len(fields)!=1:
return HttpResponse(Response.fail(ERRORS.INPUT_NOT_SAME_AS_LEARN, None).to_json())
AtomAct.objects.filter(project_id=project_id,component_id=component_id).delete()
AtomAct(project_id=project_id,component_id=component_id,atom_learn_id=atom_learn_id,input_comp_id=input_comp_id).save()
if learn_input_type != act_input_type:
return HttpResponse(Response.success(ERRORS.COMPONENT_NOT_SAME_AS_LEARN).to_json())
return HttpResponse(Response.success().to_json())
def current_execution(request, project_id):
execution = CurrentExecution.objects.filter(project_id=project_id)
if len(execution) == 0:
return HttpResponse(Response.success().to_json())
execution = execution[0] # type: CurrentExecution
if execution.current_execution is None:
task_status_list = list()
tasks = Task.objects.order_by('record_time').filter(
project_id=project_id)
for task in tasks:
assert isinstance(task, Task)
component_id = task.component_id
task_status = task.task_status
detail = task.detail
has_log = task.has_log
application_id = task.application_id
error_code = task.error_code
tracking_url = task.tracking_url
start_time = task.start_time
end_time = task.end_time
task_status_list.append(
dict(component_id=component_id,
task_status=task_status,
error_code=error_code,
application_id=application_id,
tracking_url=tracking_url,
detail=detail,
has_log=has_log,
start_time=start_time,
end_time=end_time))
return HttpResponse(Response.success(task_status_list).to_json())
return HttpResponse(
Response.success(execution.current_execution).to_json())
def search_table(request, filename, index, page_num):
object = MyData.objects.filter(file_name__icontains= filename).order_by('creat_time').reverse() \
[(int(index) - 1) * int(page_num):int(index) * int(page_num)]
sum_data = MyData.objects.filter(file_name__icontains=filename)
print(len(sum_data))
sum_index = len(sum_data) / int(page_num)
remainder = len(sum_data) % int(page_num)
if remainder > 0:
sum_index_page = int(sum_index + 1)
print(sum_index_page)
else:
sum_index_page = math.floor(sum_index)
print(sum_index_page)
sum_index_x = dict()
sum_index_x['sum_index'] = sum_index_page
if len(object) == 0:
return HttpResponse(Response.success().to_json())
json_data = []
for row_obj in object:
result = dict() # temp store one jsonObject
result['file_name'] = row_obj.file_name
result['field_num'] = row_obj.field_num
result['file_size'] = row_obj.file_size
result['craet_time'] = row_obj.creat_time
result['creat_user'] = row_obj.creat_user
json_data.append(result)
json_data.append(sum_index_x)
return HttpResponse(Response.success(json_data).to_json())
def execution_status(request, project_id, task_id):
execution = Execution.objects.filter(project_id=project_id,
task_id=task_id)
if len(execution) == 0:
return HttpResponse(Response.fail(ERRORS.NO_SUCH_TASK).to_json())
execution = execution[0]
tasks = Task.objects.order_by('record_time').filter(project_id=project_id,
task_id=task_id)
query = ExecutionQuery(execution.status, execution.start_time,
execution.end_time)
for task in tasks:
assert isinstance(task, Task)
component_id = task.component_id
task_status = task.task_status
detail = task.detail
has_log = task.has_log
application_id = task.application_id
error_code = task.error_code
tracking_url = task.tracking_url
start_time = task.start_time
end_time = task.end_time
query.add_detail(component_id, task_status, detail, error_code,
application_id, tracking_url, has_log, start_time,
end_time)
if execution.status != ExecutionStatus.RUNNING:
CurrentExecution.objects.filter(project_id=project_id).update(
current_execution=None)
return HttpResponse(Response.success(query).to_json())
def delete(request, project_id, component_id):
input_type = extract_component_type(component_id)
if input_type == COMPONENTS.CSV_READER:
CsvReaderInfo.objects.filter(project_id=project_id,
component_id=component_id).delete()
CsvReaderInfotype.objects.filter(project_id=project_id,
component_id=component_id).delete()
elif input_type == COMPONENTS.ATOM_ACT:
AtomActModel.objects.filter(project_id=project_id,
component_id=component_id).delete()
elif input_type == COMPONENTS.ATOM_LEARN:
AtomLearnModel.objects.filter(project_id=project_id,
component_id=component_id).delete()
AtomLearnParam.objects.filter(project_id=project_id,
component_id=component_id).delete()
elif input_type == COMPONENTS.ATOM_TEST:
AtomTestModel.objects.filter(project_id=project_id,
component_id=component_id).delete()
elif input_type == COMPONENTS.ATOM_EXPLORE:
AtomExploreModel.objects.filter(project_id=project_id,
component_id=component_id).delete()
AtomExploreParam.objects.filter(project_id=project_id,
component_id=component_id).delete()
elif input_type == COMPONENTS.ROBOTX:
Container.objects.filter(project_id=project_id,
component_id=component_id).delete()
Relation.objects.filter(project_id=project_id,
component_id=component_id).delete()
else:
return Response.fail(ERRORS.CSV_TYPE_ERROR, None)
return Response.success()
def queryLog(request, project_id, component_id, task_id):
tasksLogs = Task.objects.filter(project_id=project_id,
component_id=component_id,
task_id=task_id)
if len(tasksLogs) == 0:
return Response.success()
params = list()
for tasksLog in tasksLogs:
params.append(
dict(
component_id=component_id,
project_id=project_id,
task_id=task_id,
component_type=tasksLog.component_type,
error_code=tasksLog.error_code,
application_id=tasksLog.application_id,
detail=tasksLog.detail,
has_log=tasksLog.has_log,
task_status=tasksLog.task_status,
relies=tasksLog.relies,
submit_time=tasksLog.submit_time,
record_time=tasksLog.record_time,
start_time=tasksLog.start_time,
end_time=tasksLog.end_time,
))
return Response.success(params)
def save(request, project_id, component_id, atom_act_id, input_comp_id,
feature_id, feature_target):
if feature_target == "" or feature_id == "":
return Response.fail(ERRORS.PARAMS_NOT_IS_NULL)
if check_target(project_id, input_comp_id, feature_target):
return Response.fail(ERRORS.TARGET_FIELD_SELECT_ERROR)
atom_acts = AtomAct.objects.filter(project_id=project_id,
component_id=atom_act_id)
if len(atom_acts) == 0:
return Response.fail(ERRORS.ATOM_ACT_NOT_CONFIGURED, None)
atom_act = atom_acts[0]
assert isinstance(atom_act, AtomAct)
# learn_input_type = extract_component_type(atom_act.input_comp_id)
# test_input_type = extract_component_type(input_comp_id)
# feature_id = atom_act.feature_id
# feature_target = atom_act.feature_target
csv_readers = CsvReaderInfo.objects.filter(project_id=project_id,
component_id=input_comp_id)
if len(csv_readers) == 0:
return Response.fail(ERRORS.COMPONENT_NOT_EXIST, None)
csv_reader = csv_readers[0]
assert isinstance(csv_reader, CsvReaderInfo)
AtomTest.objects.filter(project_id=project_id,
component_id=component_id).delete()
AtomTest(project_id=project_id,
component_id=component_id,
atom_act_id=atom_act_id,
input_comp_id=input_comp_id,
feature_id=feature_id,
feature_target=feature_target).save()
return Response.success('')
def stop_all(request, project_id):
current = CurrentExecution.objects.filter(
project_id=project_id) # type:list[CurrentExecution]
if len(current) == 0:
return Response.success("CURRENT_PROJECT_IS_NOT_EXECUTING")
task_id = current[0].current_execution
execution = Execution.objects.filter(project_id=project_id,
task_id=task_id)
if execution[0].status != ExecutionStatus.RUNNING:
return Response.success("CURRENT_PROJECT_IS_NOT_EXECUTING")
tasks = Task.objects.filter(
project_id=project_id,
task_id=task_id,
task_status__in=[apps.PENDING, apps.SUBMITTED,
apps.RUNNING]) # type: list[Task]
running_without_app = list()
for task in tasks:
if task.task_status == apps.PENDING:
task.relies = task.relies + 1
task.save()
logger.info("task[%s-%s-%s] PENDING CANCEL" %
(project_id, task.component_id, task_id))
elif task.task_status == apps.SUBMITTED:
celery_app.control.revoke(task.celery_id, terminate=True)
Task.objects.filter(
project_id=project_id,
task_id=task_id,
task_status=apps.SUBMITTED).update(task_status=apps.KILLED)
logger.info("task[%s-%s-%s] SUBMMITED REVOKE" %
(project_id, task.component_id, task_id))
elif task.task_status == apps.RUNNING:
# if task.application_id is not None:
celery_app.control.revoke(task.celery_id, terminate=True)
logger.info("task[%s-%s-%s] SUBMMITED REVOKE" %
(project_id, task.component_id, task_id))
Task.objects.filter(
project_id=project_id,
task_id=task_id,
task_status=apps.RUNNING).update(task_status=apps.KILLED)
# else:
# running_without_app.append(task.component_id)
while len(running_without_app) != 0:
time.sleep(3)
tasks = Task.objects.filter(project_id=project_id,
task_id=task_id,
task_status=apps.RUNNING,
component_id__in=running_without_app)
if len(tasks) == 0:
break
running_without_app = list()
for task in tasks:
celery_app.control.revoke(task.celery_id, terminate=True)
logger.info("task[%s-%s-%s] SUBMMITED REVOKE" %
(project_id, task.component_id, task_id))
# CurrentExecution.objects.filter(project_id=project_id).delete()
Execution.objects.filter(project_id=project_id, task_id=task_id).update(
status=apps.KILLED, end_time=datetime.datetime.now(), task_count=0)
return Response.success()
def load(request, project_id, atom_learn_id, input_comp_id):
atom_learn_db = AtomLearn.objects.filter(project_id=project_id,
component_id=atom_learn_id)
if len(atom_learn_db) == 0:
# 刚新建组件
return HttpResponse(Response.success().to_json())
data_changed = HttpResponse(Response.success("changed").to_json())
atom_learn = atom_learn_db[0]
# 检查 input_comp_id 是否一样
if atom_learn.input_comp_id != input_comp_id:
atom_learn_db.delete()
return data_changed
# todo 检查 id， target 是否在其中，还缺少 robotx和自定义特征组合
fields = list()
if input_comp_id.startswith(COMPONENTS.HIVE_READER):
# hive reader
# hive reader 中是否包含这两个字段
fields = IOFieldType.objects.filter(
project_id=project_id,
component_id=input_comp_id,
field__in=[atom_learn.feature_id, atom_learn.feature_target])
elif input_comp_id.startswith(COMPONENTS.ROBOTX_SPARK):
# RobotXSpark
fields = list(
IOFieldType.objects.raw(
robotx_field_in_query.format(
project_id=project_id,
component_id=input_comp_id,
id=atom_learn.feature_id,
target=atom_learn.feature_target)))
elif input_comp_id.startswith(COMPONENTS.FEATURE_COMBINE):
# feature combine
fields = list(
IOFieldType.objects.raw(
combine_field_in_query.format(
project_id=project_id,
component_id=input_comp_id,
id=atom_learn.feature_id,
target=atom_learn.feature_target)))
# id target 不在字段中
if len(fields) != 2:
atom_learn_db.delete()
return data_changed
# 检查通过，返回需要初始化的内容
algorithm_params = ALGORITHM_PARAMS[atom_learn.algorithm]
atom_learn_params = AtomLearnParam.objects.filter(
project_id=project_id, component_id=atom_learn_id)
params = list()
for atom_learn_param in atom_learn_params:
algorithm_param = copy.copy(
algorithm_params[atom_learn_param.param_name])
algorithm_param['value'] = atom_learn_param.param_value
params.append(algorithm_param)
result = dict(id=atom_learn.feature_id,
target=atom_learn.feature_target,
algorithm=atom_learn.algorithm,
params=params)
return HttpResponse(Response.success(result).to_json())
def load_xml(request, project_id, component_id):
config_path = os.path.join(WORKING_DIRECTORY, project_id, component_id, "RobotXSpark.xml")
if not os.path.exists(config_path):
return HttpResponse(Response.success('').to_json())
with open(config_path, 'r') as f:
xml = "".join(f.readlines())
return HttpResponse(Response.success(xml).to_json())
def load_info(request, project_id, component_id):
objs = CsvReaderInfo.objects.filter(project_id=project_id,
component_id=component_id)
if len(objs) == 0:
response = Response.fail(ERRORS.COMPONENT_NOT_EXIST, None)
return response
response = Response.success(objs[0].magic_name)
return response
def load(request, project_id):
config_path = os.path.join(WORKING_DIRECTORY, project_id, "config.xml")
if not os.path.exists(config_path):
return Response.success('')
with open(config_path, 'r') as f:
xml = "".join(f.readlines())
return Response.success(xml)
def get_log(request, project_id, component_id):
task = Task.objects.filter(project_id=project_id,
component_id=component_id)
if len(task) == 0:
return Response.fail(ERRORS.NO_SUCH_TASK)
task = task[0]
if not task.has_log:
return Response.fail(ERRORS.TASK_HAS_NO_LOG)
return Response.success(Component.fetch_log(project_id, component_id))
def load(request, project_id, component_id):
db_field_types = AtomAct.objects.filter(project_id=project_id,
component_id=component_id)
if len(db_field_types) == 0:
return Response.success()
db_field_type = db_field_types[0]
# return Response.fail(ERRORS.ATOM_ACT_NOT_CONFIGURED, None)
result = dict(reason_code_nvars=db_field_type.reason_code_nvars,
ncores=db_field_type.ncores)
return Response.success(result)
def load(request, project_id, atom_learn_id, input_comp_id):
atom_learn_db = AtomLearn.objects.filter(project_id=project_id,
component_id=atom_learn_id)
if len(atom_learn_db) == 0:
# 刚新建组件
return Response.success()
data_changed = Response.success("changed")
atom_learn = atom_learn_db[0]
# 检查 input_comp_id 是否一样
if atom_learn.input_comp_id != input_comp_id:
atom_learn_db.delete()
return data_changed
# todo 检查 id， target 是否在其中，还缺少 robotx和自定义特征组合
fields = list()
if input_comp_id.startswith(COMPONENTS.CSV_READER):
fields = CsvReaderInfotype.objects.filter(project_id=project_id,
component_id=input_comp_id)
# ,field__in=[atom_learn.feature_id, atom_learn.feature_target])
elif input_comp_id.startswith(COMPONENTS.ROBOTX):
# RobotX
fields = list(
CsvReaderInfotype.objects.raw(
robotx_field_in_query.format(
project_id=project_id,
component_id=input_comp_id,
id=atom_learn.feature_id,
target=atom_learn.feature_target)))
elif input_comp_id.startswith(COMPONENTS.ATOM_EXPLORE):
fields = AtomLearnParam.objects.filter(project_id=project_id,
component_id=atom_learn_id)
# id target 不在字段中
# if len(fields) == 0:
# atom_learn_db.delete()
# return data_changed
# 检查通过，返回需要初始化的内容
algorithm_params = ALGORITHM_PARAMS[atom_learn.algorithm]
atom_learn_params = AtomLearnParam.objects.filter(
project_id=project_id, component_id=atom_learn_id)
params = list()
for atom_learn_param in atom_learn_params:
if atom_learn_param.param_name in algorithm_params:
algorithm_param = copy.copy(
algorithm_params[atom_learn_param.param_name])
algorithm_param['value'] = atom_learn_param.param_value
params.append(algorithm_param)
result = dict(
# id=atom_learn.feature_id,
# target=atom_learn.feature_target,
algorithm=atom_learn.algorithm,
params=params)
return Response.success(result)
def robotx_spark_key_fields(request, project_id, component_id):
objs = Container.objects.filter(project_id=project_id,
component_id=component_id)
if len(objs) == 0:
response = Response.fail(ERRORS.ROBOTX_NOT_CONFIGURED, None)
return HttpResponse(response.to_json())
container = objs[0]
assert isinstance(container, Container)
key_fields = container.key_fields.split(",")
return HttpResponse(Response.success(key_fields).to_json())
def execute(request, project_id, xml, execution_type, execute_id=None):
execution = CurrentExecution.objects.filter(project_id=project_id)
if len(execution) != 0:
execution = execution[0]
if execution.current_execution is not None:
return HttpResponse(
Response.fail(ERRORS.TASK_IS_EXECUTING).to_json())
save_xml(project_id, xml)
root = et.fromstring(xml)
flows = root.findall("./sequenceFlow")
topology = Toplogy()
for flow in flows:
start = flow.get('sourceRefs')
end = flow.get('targetRefs')
topology.add_line(start, end)
topology.sort()
# 根据不同执行类型，构造不同的需要执行的组件
try:
if execution_type == FULL_EXECUTION:
levels = topology.levels
need_execute = OrderedDict()
# 获取需要执行的组件
for level in levels:
for point in level:
if point.type in EXECUTABLE:
# 生成需要执行 配置文件
executor_class = eval(point.type)
executor = executor_class(project_id, point.id)
executor.need_execution(force=True)
need_execute[point.id] = point
task_id = execute_components(need_execute, project_id)
elif execution_type == CONT_EXECUTION:
levels = topology.levels
# 记录需要执行的组件
need_execute = get_need_execute(levels, project_id)
task_id = execute_components(need_execute, project_id)
elif execution_type == SING_EXECUTION:
levels, flat_points = topology.get_previous_component(execute_id)
need_execute = get_need_execute(levels[:-1], project_id,
flat_points)
execute_point = list(levels[-1])[0]
need_execute[execute_id] = execute_point
executor_class = eval(execute_point.type)
executor = executor_class(project_id, execute_point.id)
executor.need_execution(force=True)
task_id = execute_components(need_execute, project_id)
except Exception as e:
error_code = str(e)
return HttpResponse(Response.fail(error_code, None).to_json())
return HttpResponse(Response.success(task_id).to_json())
def structure(request, table_name):
# 检查 table_name 为非空
check = VALIDATE.not_null_validate(table_name, 'table_name')
if check is not None:
response = Response.fail(ERRORS.PARAMETER_VALUE_ERROR, check)
return HttpResponse(response.to_json())
result = py4j_common_hive_util('checkExist', table_name)
if isinstance(result, HttpResponse):
return result
if not result:
return HttpResponse(
Response.fail(ERRORS.HIVE_TABLE_NOT_EXIST, None).to_json())
result = py4j_common_hive_util('describeAndSample', table_name)
result = list(result)
result_trans = list()
# 从数据库类型映射到建模类型
# 1. 不支持的数据类型，标记 ignore为 true
# 2. 日期类型，标记前端，可选的最小粒度，记录在字段 date_format中
for field_desc in result:
field = field_desc.getName()
database_type_trans = field_desc.getType()
ignore = True
field_type = None
date_format = None
date_size = None
if database_type_trans in DATABASE_MAPPING:
ignore = False
field_type = DATABASE_MAPPING[database_type_trans]
sample_data = field_desc.getSampleData()
if field_type == FIELDTYPE.FACTOR:
if sample_data is not None:
sample_data = list(sample_data)
date_, size_ = is_date(sample_data)
if date_:
date_format = size_
date_size = size_
field_type = FIELDTYPE.DATE
if database_type_trans == 'TIMESTAMP':
date_format = 'second'
date_size = 'second'
elif database_type_trans == 'DATE':
date_format = 'day'
date_size = 'day'
struct = StructureClass(field, field_type, database_type_trans,
date_format, date_size, ignore)
result_trans.append(struct)
# result_trans.sort(key=lambda x: x.field)
response = Response.success(result_trans)
return HttpResponse(response.to_json())
def save_relation(request, project_id, component_id, robotx_spark_id,
self_defined_feature_id, connections: List[Connection]):
# 检查robotx
objs = Container.objects.filter(project_id=project_id,
component_id=robotx_spark_id)
if len(objs) == 0:
response = Response.fail(ERRORS.ROBOTX_NOT_CONFIGURED, None)
return HttpResponse(response.to_json())
connection_of_robotx = set()
connection_of_self_defined = set()
feature_combine_relations = list()
for connection in connections:
connection_of_robotx.add(connection.robotx_field)
connection_of_self_defined.add(connection.self_defined_field)
feature_combine_relations.append(
FeatureCombineRelation(
project_id=project_id,
component_id=component_id,
robotx_field=connection.robotx_field,
self_defined_field=connection.self_defined_field))
# 检查连接字段是否在robotx中
container = objs[0]
table_name = container.table_name
key_fields = set(container.key_fields.split(","))
if not key_fields.issuperset(connection_of_robotx):
response = Response.fail(ERRORS.FIELD_NOT_FOUND_IN_ROBOTX, None)
return HttpResponse(response.to_json())
# 检查 self_defined_feature
objs = SelfDefinedFeatureType.objects.filter(
field__in=connection_of_self_defined,
project_id=project_id,
component_id=self_defined_feature_id)
if len(objs) != len(connection_of_self_defined):
response = Response.fail(ERRORS.FIELD_NOT_FOUND_IN_SELF_DEFINED, None)
return HttpResponse(response.to_json())
# 检查通过，保存
FeatureCombine.objects.filter(project_id=project_id,
component_id=component_id).delete()
FeatureCombine(project_id=project_id,
component_id=component_id,
robotx_table_name=table_name,
robotx_spark_id=robotx_spark_id,
self_defined_feature_id=self_defined_feature_id).save()
FeatureCombineRelation.objects.filter(project_id=project_id,
component_id=component_id).delete()
FeatureCombineRelation.objects.bulk_create(feature_combine_relations)
return HttpResponse(Response.success().to_json())
def container_fields(request, project_id, component_id):
query_sql = container_fields_sql.format(project_id=project_id,
component_id=component_id)
field_types = list(IOFieldType.objects.raw(query_sql))
if len(field_types) == 0:
return HttpResponse(Response.success().to_json())
structures = []
for field_type in field_types:
structure = StructureClass(field_type.field, field_type.field_type,
field_type.database_type,
field_type.date_format,
field_type.date_size, field_type.ignore)
structures.append(structure)
return HttpResponse(Response.success(structures).to_json())
def view_table(request, project_id, component_id):
robotx_task = Task.objects.filter(project_id=project_id, component_id=component_id)
if len(robotx_task)==0:
return HttpResponse(Response.fail(ERRORS.ROBOTX_NOT_SUCCESS).to_json())
robotx_task = robotx_task[0]
assert isinstance(robotx_task, Task)
if robotx_task.task_status != TASK_STATUS.SUCCEEDED:
return HttpResponse(Response.fail(ERRORS.ROBOTX_NOT_SUCCESS).to_json())
result_table = RobotXSpark.output_table(project_id, component_id)
result = py4j_common_hive_util('viewTable', result_table, 10)
if isinstance(result, HttpResponse):
return result
return HttpResponse(Response.success([dict(name=k,value=list(v)) for k,v in result.items()]).to_json())
def load_hive_reader(request, project_id, component_id):
component_id_validate = VALIDATE.component_id_validate(component_id, COMPONENTS.HIVE_READER)
if component_id_validate is not None:
return HttpResponse(component_id_validate.to_json())
hive_readers = HiveReader.objects.filter(project_id=project_id, component_id=component_id)
if len(hive_readers)==0:
# 组件不存在
response = Response.success()
return HttpResponse(response.to_json())
hive_reader = hive_readers[0]
response = Response.success(dict(
table_name = hive_reader.table_name,
logic_name = hive_reader.logic_name
))
return HttpResponse(response.to_json())
def kill_task(request, project_id, task_id):
executions = Execution.objects.filter(project_id=project_id,
task_id=task_id)
if len(executions) == 0:
return Response.success("NO_SUCH_TASK")
else:
try:
celery_app.control.revoke(task_id, terminate=True)
execution = executions[0]
execution.task_count = 0
execution.status = apps.KILLED
execution.save()
except RuntimeError:
return Response.success(apps.FAILED)
return Response.success(apps.KILLED)
def saveInfo(request, project_id, component_id, magic_name, file_name):
try:
# 保存组件
CsvReaderInfo.objects.filter(project_id=project_id,
component_id=component_id).delete()
CsvReaderInfo(project_id=project_id,
component_id=component_id,
magic_name=magic_name,
file_name=file_name).save()
result = {"data": {}, "status": True, "error": ""}
response = Response.success(result)
return response
except UnicodeDecodeError as e:
response = Response.fail(ERRORS.CSV_SAVE_ERROR, None)
return response
def preview(result, project_id, component_id):
reader = HiveReader.objects.filter(project_id=project_id,
component_id=component_id)
if len(reader) == 0:
return HttpResponse(
Response.fail(ERRORS.HIVE_TABLE_NOT_EXIST).to_json())
reader = reader[0]
table_name = reader.table_name
result = py4j_common_hive_util('viewTable', table_name, 10)
if isinstance(result, HttpResponse):
return result
return HttpResponse(
Response.success([
dict(name=k, value=list(v)) for k, v in result.items()
]).to_json())