def validate_parameters(self,parameters,algorithm):
try:
parameters = json.loads(parameters)
for parameter_name in parameters:
parameter = parameters[parameter_name]
if parameter['type'] == 'int' and 'min' in parameter and 'max' in parameter:
parameter['min'] = int(parameter['min'])
parameter['max'] = int(parameter['max'])
if not parameter['max']>parameter['min']:
raise Exception('min must lower than max')
continue
if parameter['type'] == 'double' and 'min' in parameter and 'max' in parameter:
parameter['min'] = float(parameter['min'])
parameter['max'] = float(parameter['max'])
if not parameter['max']>parameter['min']:
raise Exception('min must lower than max')
if algorithm=='grid':
parameter['step'] = float(parameter['step'])
continue
if parameter['type']=='categorical' and 'list' in parameter and type(parameter['list'])==list:
continue
raise MyappException('parameters type must in [int,double,categorical], and min\max\step\list should exist, and min must lower than max ')
return json.dumps(parameters,indent=4,ensure_ascii=False)
except Exception as e:
print(e)
raise MyappException('parameters not valid:'+str(e))
def pre_add(self, item):
item.name = item.name.replace('_', '-')[0:54].lower()
if item.job_template is None:
raise MyappException("Job Template 为必选")
item.volume_mount = item.pipeline.project.volume_mount # 默认使用项目的配置
if item.job_template.volume_mount and item.job_template.volume_mount not in item.volume_mount:
if item.volume_mount:
item.volume_mount += "," + item.job_template.volume_mount
else:
item.volume_mount = item.job_template.volume_mount
item.resource_memory = core.check_resource_memory(item.resource_memory)
item.resource_cpu = core.check_resource_cpu(item.resource_cpu)
self.merge_args(item, 'add')
self.task_args_check(item)
item.create_datetime = datetime.datetime.now()
item.change_datetime = datetime.datetime.now()
if core.get_gpu(item.resource_gpu)[0]:
item.node_selector = item.node_selector.replace(
'cpu=true', 'gpu=true')
else:
item.node_selector = item.node_selector.replace(
'gpu=true', 'cpu=true')
def pre_update_get(self, item):
self.pre_add_get()
user_roles = [role.name.lower() for role in list(get_user_roles())]
if "admin" in user_roles:
return
if not g.user.username in item.get_creators():
flash('just creator can add/edit user', 'warning')
raise MyappException('just creator can add/edit user')
def pre_update(self, item):
if item.expand:
core.validate_json(item.expand)
item.expand = json.dumps(json.loads(item.expand),
indent=4,
ensure_ascii=False)
user_roles = [role.name.lower() for role in list(get_user_roles())]
if "admin" in user_roles:
return
if not g.user.username in item.get_creators():
raise MyappException('just creator can add/edit')
def pre_add(self, item):
if item.job_type is None:
raise MyappException("Job type is mandatory")
core.validate_json(item.parameters)
item.parameters = self.validate_parameters(item.parameters,item.algorithm_name)
item.resource_memory=core.check_resource_memory(item.resource_memory,self.src_item_json.get('resource_memory',None) if self.src_item_json else None)
item.resource_cpu = core.check_resource_cpu(item.resource_cpu,self.src_item_json.get('resource_cpu',None) if self.src_item_json else None)
self.merge_trial_spec(item)
self.make_experiment(item)
def task_args_check(self, item):
core.validate_str(item.name, 'name')
core.validate_json(item.args)
task_args = json.loads(item.args)
job_args = json.loads(item.job_template.args)
item.args = json.dumps(core.validate_task_args(task_args, job_args),
indent=4,
ensure_ascii=False)
if item.volume_mount and ":" not in item.volume_mount:
raise MyappException(
'volume_mount is not valid, must contain : or null')
def pre_update(self, item):
item.name = item.name.replace('_', '-')[0:54].lower()
if item.job_template is None:
raise MyappException("Job Template 为必选")
# if item.job_template.volume_mount and item.job_template.volume_mount not in item.volume_mount:
# if item.volume_mount:
# item.volume_mount += ","+item.job_template.volume_mount
# else:
# item.volume_mount = item.job_template.volume_mount
if item.outputs:
core.validate_json(item.outputs)
item.outputs = json.dumps(json.loads(item.outputs),
indent=4,
ensure_ascii=False)
if item.expand:
core.validate_json(item.expand)
item.expand = json.dumps(json.loads(item.expand),
indent=4,
ensure_ascii=False)
item.resource_memory = core.check_resource_memory(
item.resource_memory,
self.src_item_json.get('resource_memory', None))
item.resource_cpu = core.check_resource_cpu(
item.resource_cpu, self.src_item_json.get('resource_cpu', None))
# item.resource_memory=core.check_resource_memory(item.resource_memory,self.src_resource_memory)
# item.resource_cpu = core.check_resource_cpu(item.resource_cpu,self.src_resource_cpu)
self.merge_args(item, 'update')
self.task_args_check(item)
item.change_datetime = datetime.datetime.now()
if core.get_gpu(item.resource_gpu)[0]:
item.node_selector = item.node_selector.replace(
'cpu=true', 'gpu=true')
else:
item.node_selector = item.node_selector.replace(
'gpu=true', 'cpu=true')
def order_by_upstream(dag_json):
order_dag = {}
tasks_name = list(dag_json.keys()) # 如果没有配全的话，可能只有局部的task
i = 0
while tasks_name:
i += 1
if i > 100: # 不会有100个依赖关系
break
for task_name in tasks_name:
# 没有上游的情况
if not dag_json[task_name]:
order_dag[task_name] = dag_json[task_name]
tasks_name.remove(task_name)
continue
# 没有上游的情况
elif 'upstream' not in dag_json[
task_name] or not dag_json[task_name]['upstream']:
order_dag[task_name] = dag_json[task_name]
tasks_name.remove(task_name)
continue
# 如果有上游依赖的话，先看上游任务是否已经加到里面了。
upstream_all_ready = True
for upstream_task_name in dag_json[task_name]['upstream']:
if upstream_task_name not in order_dag:
upstream_all_ready = False
if upstream_all_ready:
order_dag[task_name] = dag_json[task_name]
tasks_name.remove(task_name)
else:
dag_json[task_name]['upstream'] = []
order_dag[task_name] = dag_json[task_name]
tasks_name.remove(task_name)
if list(dag_json.keys()).sort() != list(order_dag.keys()).sort():
flash('dag service pipeline 存在循环或未知上游', category='warning')
raise MyappException('dag service pipeline 存在循环或未知上游')
return order_dag
def pre_update(self, item):
user_roles = [role.name.lower() for role in list(get_user_roles())]
if "admin" in user_roles:
return
if not g.user.username in item.project.get_creators():
raise MyappException('just creator can add/edit user')