def _schedule_job(self, job_id):
job = Job.query.get(job_id)
assert job is not None, f'Job {job_id} not found'
if job.state != JobState.WAITING:
return job.state
with get_session(self._db_engine) as session:
job_service = JobService(session)
if not job_service.is_ready(job):
return job.state
config = job.get_config()
if config.is_federated:
if not job_service.is_peer_ready(job):
return job.state
try:
yaml = generate_job_run_yaml(job)
k8s_client.create_flapp(yaml)
except Exception as e:
logging.error(f'Start job {job_id} has error msg: {e.args}')
job.error_message = str(e)
db.session.commit()
return job.state
job.error_message = None
job.start()
db.session.commit()
return job.state
def _schedule_job(self, job_id):
job = Job.query.get(job_id)
assert job is not None, 'Job %d not found' % job_id
if job.state != JobState.WAITING:
return job.state
deps = JobDependency.query.filter(
JobDependency.dst_job_id == job.id).all()
for dep in deps:
src_job = Job.query.get(dep.src_job_id)
assert src_job is not None, 'Job %d not found' % dep.src_job_id
if not src_job.is_complete():
return job.state
k8s_client = get_client()
yaml = generate_job_run_yaml(job)
try:
k8s_client.create_or_replace_custom_object(CrdKind.FLAPP, yaml)
except RuntimeError as e:
logging.error('Start job %d has Runtime error msg: %s', job_id,
e.args)
return job.state
job.start()
db.session.commit()
return job.state
def get_checkpoint_path(job):
try:
yaml = generate_job_run_yaml(job)
# ToDo : OUTPUT_BASE_DIR may not be accurate
output_base_dir = yaml['OUTPUT_BASE_DIR']
except Exception as e:
logging.warning('Error building metrics: %s', repr(e))
output_base_dir = ''
return output_base_dir
def patch(self, workflow_id):
parser = reqparse.RequestParser()
parser.add_argument('target_state', type=str, required=False,
default=None, help='target_state is empty')
parser.add_argument('state', type=str, required=False,
default=None, help='state is empty')
parser.add_argument('forkable', type=bool)
parser.add_argument('metric_is_public', type=bool)
parser.add_argument('config', type=dict, required=False,
default=None, help='updated config')
data = parser.parse_args()
workflow = _get_workflow(workflow_id)
forkable = data['forkable']
if forkable is not None:
workflow.forkable = forkable
db.session.flush()
metric_is_public = data['metric_is_public']
if metric_is_public is not None:
workflow.metric_is_public = metric_is_public
db.session.flush()
target_state = data['target_state']
if target_state:
try:
if WorkflowState[target_state] == WorkflowState.RUNNING:
for job in workflow.owned_jobs:
try:
generate_job_run_yaml(job)
# TODO: check if peer variables is valid
except RuntimeError as e:
raise ValueError(
f'Invalid Variable when try '
f'to format the job {job.name}:{str(e)}')
workflow.update_target_state(WorkflowState[target_state])
db.session.flush()
logging.info('updated workflow %d target_state to %s',
workflow.id, workflow.target_state)
scheduler.wakeup(workflow.id)
except ValueError as e:
raise InvalidArgumentException(details=str(e)) from e
state = data['state']
if state:
try:
assert state == 'INVALID', \
'Can only set state to INVALID for invalidation'
workflow.invalidate()
db.session.flush()
logging.info('invalidate workflow %d', workflow.id)
except ValueError as e:
raise InvalidArgumentException(details=str(e)) from e
config = data['config']
if config:
try:
if workflow.target_state != WorkflowState.INVALID or \
workflow.state not in \
[WorkflowState.READY, WorkflowState.STOPPED]:
raise NoAccessException('Cannot edit running workflow')
config_proto = dict_to_workflow_definition(data['config'])
workflow.set_config(config_proto)
db.session.flush()
except ValueError as e:
raise InvalidArgumentException(details=str(e)) from e
db.session.commit()
return {'data': workflow.to_dict()}, HTTPStatus.OK
def patch(self, workflow_id):
parser = reqparse.RequestParser()
parser.add_argument('target_state', type=str, required=False,
default=None, help='target_state is empty')
parser.add_argument('state',
type=str,
required=False,
help='state is empty')
parser.add_argument('forkable', type=bool)
parser.add_argument('metric_is_public', type=bool)
parser.add_argument('config',
type=dict,
required=False,
help='updated config')
parser.add_argument('create_job_flags', type=list, required=False,
location='json',
help='flags in common.CreateJobFlag')
parser.add_argument('batch_update_interval',
type=int,
required=False,
help='interval for restart workflow in minute')
data = parser.parse_args()
workflow = _get_workflow(workflow_id)
# start workflow every interval time
batch_update_interval = data['batch_update_interval']
if batch_update_interval:
start_or_stop_cronjob(batch_update_interval, workflow)
forkable = data['forkable']
if forkable is not None:
workflow.forkable = forkable
db.session.flush()
metric_is_public = data['metric_is_public']
if metric_is_public is not None:
workflow.metric_is_public = metric_is_public
db.session.flush()
target_state = data['target_state']
if target_state:
try:
if WorkflowState[target_state] == WorkflowState.RUNNING:
for job in workflow.owned_jobs:
try:
generate_job_run_yaml(job)
# TODO: check if peer variables is valid
except Exception as e: # pylint: disable=broad-except
raise ValueError(
f'Invalid Variable when try '
f'to format the job {job.name}:{str(e)}')
workflow.update_target_state(WorkflowState[target_state])
db.session.flush()
logging.info('updated workflow %d target_state to %s',
workflow.id, workflow.target_state)
except ValueError as e:
raise InvalidArgumentException(details=str(e)) from e
state = data['state']
if state:
try:
assert state == 'INVALID', \
'Can only set state to INVALID for invalidation'
workflow.invalidate()
db.session.flush()
logging.info('invalidate workflow %d', workflow.id)
except ValueError as e:
raise InvalidArgumentException(details=str(e)) from e
config = data['config']
if config:
try:
if workflow.target_state != WorkflowState.INVALID or \
workflow.state not in \
[WorkflowState.READY, WorkflowState.STOPPED]:
raise NoAccessException('Cannot edit running workflow')
config_proto = dict_to_workflow_definition(data['config'])
workflow.set_config(config_proto)
db.session.flush()
except ValueError as e:
raise InvalidArgumentException(details=str(e)) from e
create_job_flags = data['create_job_flags']
if create_job_flags:
jobs = workflow.get_jobs()
if len(create_job_flags) != len(jobs):
raise InvalidArgumentException(
details='Number of job defs does not match number '
f'of create_job_flags {len(jobs)} '
f'vs {len(create_job_flags)}')
workflow.set_create_job_flags(create_job_flags)
flags = workflow.get_create_job_flags()
for i, job in enumerate(jobs):
if job.workflow_id == workflow.id:
job.is_disabled = flags[i] == \
common_pb2.CreateJobFlag.DISABLED
db.session.commit()
scheduler.wakeup(workflow.id)
return {'data': workflow.to_dict()}, HTTPStatus.OK