def create_job(cls, project_id, type, user_ID, source_file_path,
run_args=None, running_module=None,
running_code=None):
business_mapper = {
'app': AppBusiness,
'module': ModuleBusiness,
'dataset': DatasetBusiness,
}
project = business_mapper[type].get_by_id(project_id)
options = {}
if running_module:
[modole_user_ID, module_name,
module_version] = running_module.split('/')
module_identity = f'{modole_user_ID}+{module_name}'
running_module = ModuleBusiness.get_by_identity(module_identity)
options = {
'running_module': running_module,
'module_version': module_version
}
user = UserBusiness.get_by_user_ID(user_ID)
new_job = JobBusiness.create_job(project=project, type=type, user=user, source_file_path=source_file_path,
run_args=run_args, running_code=running_code, **options)
# add job to project
project.jobs.append(new_job)
project.save()
return new_job
def update_job_log(job_id):
data = request.get_json()
log_type = data.get('log_type')
message = data.get('message')
result = JobBusiness.update_log(job_id, log_type, message)
result = json_utility.convert_to_json(result.to_mongo())
return jsonify({"response": result}), 200
def get_by_project(cls, project_type, project_id):
business_mapper = {
'app': AppBusiness,
'module': ModuleBusiness,
'dataset': DatasetBusiness,
}
project = business_mapper[project_type].get_by_id(project_id)
return JobBusiness.get_by_project(project_type, project)
def restart(cls, job_id, user_token, auto_restart_num=0, auto=False):
job = JobBusiness.get_by_id(job_id)
project = job[job.project_type]
project_path = f'{project.user.user_ID}/{project.name}'
# K8sBusiness.create_k8s_job(job, job.script_path, project_path,
# env=job.env, restart=True, args=job.args)
wait = False
if not auto:
if K8sBusiness.get_queuing_or_running_gpu_job_num() + \
JobBusiness.get_pending_job_num() > 1:
wait = True
print('restart need wait')
# print('auto restart: ', auto_restart_num, (str(job.id),
# job.script_path,
# project_path, job.env,
# user_token, wait))
celery.send_task('celery_conf.create_job',
(str(job.id),
job.script_path,
project_path, job.env,
user_token),
priority=auto_restart_num,
kwargs={
'wait': wait,
'restart': True,
'auto_restart_num': auto_restart_num},
queue=job.env
)
EventBusiness.create_event(
user=project.user,
target=job,
target_type='job',
action="restart",
)
return job
def all_running_pending_jobs_num(cls):
return K8sBusiness.get_queuing_or_running_gpu_job_num() + \
JobBusiness.get_pending_job_num()
def create_job(cls, project_id, type, user_ID, script_path, env,
user_token, task=None, function=None, display_name=None,
args='', katib_args={}):
project = cls.business_mapper[type].get_by_id(project_id)
user = project.user
path = os.path.join(project.path, script_path)
if not os.path.exists(path):
script_path = f'src/{script_path}'
new_job = JobBusiness.create_job(project=project, type=type, user=user,
script_path=script_path, env=env,
task=task, function=function,
display_name=display_name, args=args,
katib_args=katib_args)
def send_job():
# copy job to staging folder
staging_path = new_job.staging_path
staging_project_path = '/'.join(staging_path.split('/')[:-1])
if not os.path.exists(staging_project_path):
os.makedirs(staging_project_path)
time.sleep(2)
ProjectBusiness.copy_to_ignore_env(new_job.project,
staging_path,
no_site_packages=False,
no_remove=True)
project_path = f'{user.user_ID}/{project.name}'
# check if running or pending jobs which makes this job to wait
wait = False
if cls.all_running_pending_jobs_num() > 1 and env == 'gpu':
wait = True
print('need wait')
# send job to celery
celery.send_task('celery_conf.create_job',
args=(str(new_job.id),
new_job.script_path,
project_path, env,
user_token, function,
str(task.id) if task else None,
False, PY_SERVER, args, wait,
0),
queue=env)
# 如果是 katib ，状态改为 Queuing
# 如果不是，执行以前的流程
if katib_args:
new_job.status_bak = 'Queuing'
else:
eventlet.spawn_n(send_job)
EventBusiness.create_event(
user=user,
target=new_job,
target_type='job',
action="create",
)
# add job to project
project.jobs.append(new_job)
project.save()
return new_job, False
def success_job(job_id):
result = JobBusiness.update_job_status(job_id, 'success')
result = json_utility.convert_to_json(result.to_mongo())
return jsonify({"response": result}), 200