def pre_update(self, item):
if item.expand:
core.validate_json(item.expand)
item.expand = json.dumps(json.loads(item.expand),
indent=4,
ensure_ascii=False)
user_roles = [role.name.lower() for role in list(get_user_roles())]
if "admin" in user_roles:
return
if not g.user.username in item.get_creators():
raise MyappException('just creator can add/edit')
def pre_add(self, item):
if item.job_type is None:
raise MyappException("Job type is mandatory")
core.validate_json(item.parameters)
item.parameters = self.validate_parameters(item.parameters,item.algorithm_name)
item.resource_memory=core.check_resource_memory(item.resource_memory,self.src_item_json.get('resource_memory',None) if self.src_item_json else None)
item.resource_cpu = core.check_resource_cpu(item.resource_cpu,self.src_item_json.get('resource_cpu',None) if self.src_item_json else None)
self.merge_trial_spec(item)
self.make_experiment(item)
def task_args_check(self, item):
core.validate_str(item.name, 'name')
core.validate_json(item.args)
task_args = json.loads(item.args)
job_args = json.loads(item.job_template.args)
item.args = json.dumps(core.validate_task_args(task_args, job_args),
indent=4,
ensure_ascii=False)
if item.volume_mount and ":" not in item.volume_mount:
raise MyappException(
'volume_mount is not valid, must contain : or null')
def service_pipeline_args_check(self, item):
core.validate_str(item.name, 'name')
if not item.dag_json:
item.dag_json = '{}'
core.validate_json(item.dag_json)
# 校验task的关系，没有闭环，并且顺序要对。没有配置的，自动没有上游，独立
# @pysnooper.snoop()
def order_by_upstream(dag_json):
order_dag = {}
tasks_name = list(dag_json.keys()) # 如果没有配全的话，可能只有局部的task
i = 0
while tasks_name:
i += 1
if i > 100: # 不会有100个依赖关系
break
for task_name in tasks_name:
# 没有上游的情况
if not dag_json[task_name]:
order_dag[task_name] = dag_json[task_name]
tasks_name.remove(task_name)
continue
# 没有上游的情况
elif 'upstream' not in dag_json[
task_name] or not dag_json[task_name]['upstream']:
order_dag[task_name] = dag_json[task_name]
tasks_name.remove(task_name)
continue
# 如果有上游依赖的话，先看上游任务是否已经加到里面了。
upstream_all_ready = True
for upstream_task_name in dag_json[task_name]['upstream']:
if upstream_task_name not in order_dag:
upstream_all_ready = False
if upstream_all_ready:
order_dag[task_name] = dag_json[task_name]
tasks_name.remove(task_name)
else:
dag_json[task_name]['upstream'] = []
order_dag[task_name] = dag_json[task_name]
tasks_name.remove(task_name)
if list(dag_json.keys()).sort() != list(order_dag.keys()).sort():
flash('dag service pipeline 存在循环或未知上游', category='warning')
raise MyappException('dag service pipeline 存在循环或未知上游')
return order_dag
# 配置上缺少的默认上游
dag_json = json.loads(item.dag_json)
item.dag_json = json.dumps(order_by_upstream(copy.deepcopy(dag_json)),
ensure_ascii=False,
indent=4)
def pre_add(self, item):
if not item.env:
item.env = ''
envs = item.env.strip().split('\n')
envs = [env.strip() for env in envs if env.strip() and '=' in env]
item.env = '\n'.join(envs)
if not item.args:
item.args = '{}'
item.args = core.validate_job_args(item)
if not item.expand or not item.expand.strip():
item.expand = '{}'
core.validate_json(item.expand)
item.expand = json.dumps(json.loads(item.expand),
indent=4,
ensure_ascii=False)
if not item.demo or not item.demo.strip():
item.demo = '{}'
core.validate_json(item.demo)
if item.hostAliases:
# if not item.images.entrypoint:
# raise MyappException('images entrypoint not exist')
all_host = {}
all_rows = re.split('\r|\n', item.hostAliases)
all_rows = [
all_row.strip() for all_row in all_rows if all_row.strip()
]
for row in all_rows:
hosts = row.split(' ')
hosts = [host for host in hosts if host]
if len(hosts) > 1:
if hosts[0] in all_host:
all_host[hosts[0]] = all_host[hosts[0]] + hosts[1:]
else:
all_host[hosts[0]] = hosts[1:]
hostAliases = ''
for ip in all_host:
hostAliases += ip + " " + " ".join(all_host[ip])
hostAliases += '\n'
item.hostAliases = hostAliases.strip()
task_args = json.loads(item.demo)
job_args = json.loads(item.args)
item.demo = json.dumps(core.validate_task_args(task_args, job_args),
indent=4,
ensure_ascii=False)
def pre_update(self, item):
if item.expand:
core.validate_json(item.expand)
item.expand = json.dumps(json.loads(item.expand),
indent=4,
ensure_ascii=False)
else:
item.expand = '{}'
item.name = item.name.replace('_', '-')[0:54].lower()
item.alert_status = ','.join(item.alert_status)
# self.service_pipeline_args_check(item)
item.change_datetime = datetime.datetime.now()
item.parameter = json.dumps(
json.loads(item.parameter), indent=4,
ensure_ascii=False) if item.parameter else '{}'
item.dag_json = json.dumps(
json.loads(item.dag_json), indent=4,
ensure_ascii=False) if item.dag_json else '{}'
item.volume_mount = item.project.volume_mount + ",%s(configmap):/config/" % item.name
def pre_update(self, item):
item.name = item.name.replace('_', '-')[0:54].lower()
if item.job_template is None:
raise MyappException("Job Template 为必选")
# if item.job_template.volume_mount and item.job_template.volume_mount not in item.volume_mount:
# if item.volume_mount:
# item.volume_mount += ","+item.job_template.volume_mount
# else:
# item.volume_mount = item.job_template.volume_mount
if item.outputs:
core.validate_json(item.outputs)
item.outputs = json.dumps(json.loads(item.outputs),
indent=4,
ensure_ascii=False)
if item.expand:
core.validate_json(item.expand)
item.expand = json.dumps(json.loads(item.expand),
indent=4,
ensure_ascii=False)
item.resource_memory = core.check_resource_memory(
item.resource_memory,
self.src_item_json.get('resource_memory', None))
item.resource_cpu = core.check_resource_cpu(
item.resource_cpu, self.src_item_json.get('resource_cpu', None))
# item.resource_memory=core.check_resource_memory(item.resource_memory,self.src_resource_memory)
# item.resource_cpu = core.check_resource_cpu(item.resource_cpu,self.src_resource_cpu)
self.merge_args(item, 'update')
self.task_args_check(item)
item.change_datetime = datetime.datetime.now()
if core.get_gpu(item.resource_gpu)[0]:
item.node_selector = item.node_selector.replace(
'cpu=true', 'gpu=true')
else:
item.node_selector = item.node_selector.replace(
'gpu=true', 'cpu=true')
def pre_add(self, item):
if item.expand:
core.validate_json(item.expand)
item.expand = json.dumps(json.loads(item.expand),
indent=4,
ensure_ascii=False)