def put(self, job_uuid, run_uuid):
"""Set the status of a pipeline run."""
status_update = request.get_json()
# The pipeline run has reached a final state, thus we can update
# the job "completed_pipeline_runs" attribute.
if status_update["status"] in ["SUCCESS", "FAILURE"]:
job = models.Job.query.get_or_404(
job_uuid,
description="Job not found",
)
job.completed_pipeline_runs += 1
filter_by = {
"job_uuid": job_uuid,
"run_uuid": run_uuid,
}
try:
update_status_db(
status_update,
model=models.NonInteractivePipelineRun,
filter_by=filter_by,
)
db.session.commit()
except Exception:
db.session.rollback()
return {"message": "Failed update operation."}, 500
return {"message": "Status was updated successfully"}, 200
def abort_environment_build(environment_build_uuid, is_running=False):
"""Aborts an environment build.
Aborts an environment build by setting its state to ABORTED and
sending a REVOKE and ABORT command to celery.
Args:
is_running:
environment_build_uuid: uuid of the environment build to abort
Returns:
"""
filter_by = {
"build_uuid": environment_build_uuid,
}
status_update = {"status": "ABORTED"}
celery_app = make_celery(current_app)
# Make use of both constructs (revoke, abort) so we cover both a
# task that is pending and a task which is running.
celery_app.control.revoke(environment_build_uuid, timeout=1.0)
if is_running:
res = AbortableAsyncResult(environment_build_uuid, app=celery_app)
# It is responsibility of the task to terminate by reading it's
# aborted status.
res.abort()
update_status_db(
status_update,
model=models.EnvironmentBuild,
filter_by=filter_by,
)
def put(self, project_uuid, environment_uuid, image_tag):
"""Set the status of a environment build."""
status_update = request.get_json()
filter_by = {
"project_uuid": project_uuid,
"environment_uuid": environment_uuid,
"image_tag": int(image_tag),
}
try:
update_status_db(
status_update,
model=models.EnvironmentImageBuild,
filter_by=filter_by,
)
if status_update["status"] == "SUCCESS":
digest = registry.get_manifest_digest(
_config.ENVIRONMENT_IMAGE_NAME.format(
project_uuid=project_uuid,
environment_uuid=environment_uuid),
image_tag,
)
db.session.add(
models.EnvironmentImage(
project_uuid=project_uuid,
environment_uuid=environment_uuid,
tag=int(image_tag),
digest=digest,
))
db.session.commit()
except Exception:
db.session.rollback()
return {"message": "Failed update operation."}, 500
return {"message": "Status was updated successfully."}, 200
def _transaction(self, run_uuid):
"""Abort a pipeline level at the db level.
Args:
run_uuid:
Returns:
True if the run state was set to ABORTED, false if the run
did not exist or was not PENDING/STARTED.
"""
# If the run is not abortable return false, abortion failed.
# Set the state and return.
filter_by = {"uuid": run_uuid}
status_update = {"status": "ABORTED"}
# _can_abort is set to True if any row was affected, that is,
# the run was in a PENDING or STARTED state, since
# update_status_db won't update it otherwise.
can_abort = update_status_db(status_update,
model=models.PipelineRun,
filter_by=filter_by)
# Do not attempt to update the status of the steps if the status
# of the pipeline could not be updated.
if can_abort:
filter_by = {"run_uuid": run_uuid}
update_status_db(status_update,
model=models.PipelineRunStep,
filter_by=filter_by)
self.collateral_kwargs["run_uuid"] = run_uuid if can_abort else None
return can_abort
def put(self, experiment_uuid, run_uuid):
"""Set the status of a pipeline run."""
status_update = request.get_json()
filter_by = {
'experiment_uuid': experiment_uuid,
'run_uuid': run_uuid,
}
update_status_db(status_update,
model=models.NonInteractiveRun,
filter_by=filter_by)
return {'message': 'Status was updated successfully'}, 200
def put(self, job_uuid, run_uuid):
"""Set the status of a pipeline run."""
status_update = request.get_json()
try:
filter_by = {
"job_uuid": job_uuid,
"uuid": run_uuid,
}
update_status_db(
status_update,
model=models.NonInteractivePipelineRun,
filter_by=filter_by,
)
# See if the job is done running (all its runs are done).
if status_update["status"] in ["SUCCESS", "FAILURE"]:
# The job has 1 run for every parameters set.
job = (db.session.query(
models.Job.schedule,
func.jsonb_array_length(models.Job.parameters),
).filter_by(uuid=job_uuid).one())
# Only non recurring jobs terminate to SUCCESS.
if job.schedule is None:
completed_runs = (
models.NonInteractivePipelineRun.query.filter_by(
job_uuid=job_uuid).filter(
models.NonInteractivePipelineRun.status.in_(
["SUCCESS", "FAILURE"])).count())
current_app.logger.info(
(f"Non recurring job {job_uuid} has completed "
f"{completed_runs}/{job[1]} runs."))
if completed_runs == job[1]:
models.Job.query.filter_by(uuid=job_uuid).update(
{"status": "SUCCESS"})
db.session.commit()
except Exception as e:
db.session.rollback()
current_app.logger.error(e)
return {"message": "Failed update operation."}, 500
return {"message": "Status was updated successfully"}, 200
def put(self, environment_build_uuid):
"""Set the status of a environment build."""
status_update = request.get_json()
filter_by = {
"build_uuid": environment_build_uuid,
}
update_status_db(
status_update,
model=models.EnvironmentBuild,
filter_by=filter_by,
)
return {"message": "Status was updated successfully"}, 200
def _transaction(self, job_uuid: str):
# To be later used by the collateral function.
run_uuids = []
# Assign asap since the function will return if there is nothing
# to do.
self.collateral_kwargs["run_uuids"] = run_uuids
self.collateral_kwargs["job_uuid"] = job_uuid
job = (models.Job.query.options(joinedload(
models.Job.pipeline_runs)).filter_by(uuid=job_uuid).one_or_none())
if job is None:
return False
# No op if the job is already in an end state.
if job.status in ["SUCCESS", "FAILURE", "ABORTED"]:
return
job.status = "ABORTED"
# This way a recurring job or a job which is scheduled to run
# once in the future will not be scheduled anymore.
job.next_scheduled_time = None
# Store each uuid of runs that can still be aborted. These uuid
# are the celery task uuid as well.
for run in job.pipeline_runs:
if run.status in ["PENDING", "STARTED"]:
run_uuids.append(run.uuid)
# Set the state of each run and related steps to ABORTED. Note
# that the status of steps that have already been completed will
# not be modified.
for run_uuid in run_uuids:
filter_by = {"uuid": run_uuid}
status_update = {"status": "ABORTED"}
update_status_db(
status_update,
model=models.NonInteractivePipelineRun,
filter_by=filter_by,
)
filter_by = {"run_uuid": run_uuid}
status_update = {"status": "ABORTED"}
update_status_db(status_update,
model=models.PipelineRunStep,
filter_by=filter_by)
return True
def put(self, experiment_uuid, run_uuid, step_uuid):
"""Set the status of a pipeline step of a pipeline run."""
status_update = request.get_json()
filter_by = {
"run_uuid": run_uuid,
"step_uuid": step_uuid,
}
update_status_db(
status_update,
model=models.PipelineRunStep,
filter_by=filter_by,
)
return {"message": "Status was updated successfully"}, 200
def put(self, run_uuid):
"""Sets the status of a pipeline run."""
filter_by = {"uuid": run_uuid}
status_update = request.get_json()
try:
update_status_db(status_update,
model=models.PipelineRun,
filter_by=filter_by)
db.session.commit()
except Exception:
db.session.rollback()
return {"message": "Failed update operation."}, 500
return {"message": "Status was updated successfully."}, 200
def stop_experiment(experiment_uuid) -> bool:
"""Stop an experiment.
Args:
experiment_uuid:
Returns:
True if the experiment exists and was stopped, false
if it did not exist or if it was already completed.
"""
experiment = models.Experiment.query.filter_by(
experiment_uuid=experiment_uuid).one_or_none()
if experiment is None:
return False
run_uuids = [
run.run_uuid for run in experiment.pipeline_runs
if run.status in ["PENDING", "STARTED"]
]
if len(run_uuids) == 0:
return False
# Aborts and revokes all pipeline runs and waits for a
# reply for 1.0s.
celery = make_celery(current_app)
celery.control.revoke(run_uuids, timeout=1.0)
# TODO: possibly set status of steps and Run to "ABORTED"
# note that a race condition would be present since the task
# will try to set the status as well
for run_uuid in run_uuids:
res = AbortableAsyncResult(run_uuid, app=celery)
# it is responsibility of the task to terminate by reading \
# it's aborted status
res.abort()
filter_by = {"run_uuid": run_uuid}
status_update = {"status": "ABORTED"}
update_status_db(status_update,
model=models.NonInteractivePipelineRun,
filter_by=filter_by)
update_status_db(status_update,
model=models.PipelineRunStep,
filter_by=filter_by)
db.session.commit()
return True
def put(self, run_uuid, step_uuid):
"""Sets the status of a pipeline step."""
status_update = request.get_json()
# TODO: first check the status and make sure it says PENDING or
# whatever. Because if is empty then this would write it
# and then get overwritten afterwards with "PENDING".
filter_by = {"run_uuid": run_uuid, "step_uuid": step_uuid}
try:
update_status_db(status_update,
model=models.PipelineRunStep,
filter_by=filter_by)
db.session.commit()
except Exception:
db.session.rollback()
return {"message": "Failed update operation."}, 500
return {"message": "Status was updated successfully."}, 200
def put(self, environment_build_uuid):
"""Set the status of a environment build."""
status_update = request.get_json()
filter_by = {
"build_uuid": environment_build_uuid,
}
try:
update_status_db(
status_update,
model=models.EnvironmentBuild,
filter_by=filter_by,
)
db.session.commit()
except Exception:
db.session.rollback()
return {"message": "Failed update operation."}, 500
return {"message": "Status was updated successfully."}, 200
def put(self, run_uuid, step_uuid):
"""Sets the status of a pipeline step."""
status_update = request.get_json()
# TODO: don't we want to do this async? Since otherwise the API
# call might be blocking another since they both execute
# on the database? SQLite can only have one process write
# to the db. If this becomes an issue than we could also
# use an in-memory db (since that is a lot faster than
# disk). Otherwise we might have to use PostgreSQL.
# TODO: first check the status and make sure it says PENDING or
# whatever. Because if is empty then this would write it
# and then get overwritten afterwards with "PENDING".
filter_by = {"run_uuid": run_uuid, "step_uuid": step_uuid}
update_status_db(status_update,
model=models.InteractiveRunPipelineStep,
filter_by=filter_by)
return {"message": "Status was updated successfully"}, 200
def put(self, experiment_uuid, run_uuid):
"""Set the status of a pipeline run."""
status_update = request.get_json()
# The pipeline run has reached a final state, thus we can update
# the experiment "completed_pipeline_runs" attribute.
if status_update["status"] in ["SUCCESS", "FAILURE"]:
experiment = models.Experiment.query.get_or_404(
experiment_uuid,
description="Experiment not found",
)
experiment.completed_pipeline_runs += 1
filter_by = {
"experiment_uuid": experiment_uuid,
"run_uuid": run_uuid,
}
update_status_db(status_update,
model=models.NonInteractivePipelineRun,
filter_by=filter_by)
return {"message": "Status was updated successfully"}, 200
def _transaction(self, job_uuid: str):
# To be later used by the collateral function.
run_uuids = []
# Assign asap since the function will return if there is nothing
# to do.
self.collateral_kwargs["run_uuids"] = run_uuids
job = models.Job.query.filter_by(job_uuid=job_uuid).one_or_none()
if job is None:
return False
# Store each uuid of runs that can still be aborted. These uuid
# are the celery task uuid as well.
for run in job.pipeline_runs:
if run.status in ["PENDING", "STARTED"]:
run_uuids.append(run.run_uuid)
if len(run_uuids) == 0:
return False
# Set the state of each run and related steps to ABORTED. Note
# that the status of steps that have already been completed will
# not be modified.
for run_uuid in run_uuids:
filter_by = {"run_uuid": run_uuid}
status_update = {"status": "ABORTED"}
update_status_db(
status_update,
model=models.NonInteractivePipelineRun,
filter_by=filter_by,
)
update_status_db(status_update,
model=models.PipelineRunStep,
filter_by=filter_by)
return True
def put(self, experiment_uuid, run_uuid):
"""Set the status of a pipeline run."""
status_update = request.get_json()
# The pipeline run has reached a final state, thus we can update
# the experiment "completed_pipeline_runs" attribute.
if status_update['status'] in ['SUCCESS', 'FAILURE']:
experiment = models.Experiment.query.get_or_404(
experiment_uuid,
description='Experiment not found',
)
experiment.completed_pipeline_runs += 1
db.session.commit()
filter_by = {
'experiment_uuid': experiment_uuid,
'run_uuid': run_uuid,
}
update_status_db(status_update,
model=models.NonInteractiveRun,
filter_by=filter_by)
return {'message': 'Status was updated successfully'}, 200
def _transaction(self, environment_build_uuid: str):
filter_by = {
"build_uuid": environment_build_uuid,
}
status_update = {"status": "ABORTED"}
# Will return true if any row is affected, meaning that the
# environment build was actually PENDING or STARTED.
abortable = update_status_db(
status_update,
model=models.EnvironmentBuild,
filter_by=filter_by,
)
self.collateral_kwargs["environment_build_uuid"] = (
environment_build_uuid if abortable else None)
return abortable
def _transaction(self, project_uuid: str, environment_uuid: str,
image_tag: str):
filter_by = {
"project_uuid": project_uuid,
"environment_uuid": environment_uuid,
"image_tag": int(image_tag),
}
status_update = {"status": "ABORTED"}
# Will return true if any row is affected, meaning that the
# environment build was actually PENDING or STARTED.
abortable = update_status_db(
status_update,
model=models.EnvironmentImageBuild,
filter_by=filter_by,
)
self.collateral_kwargs["celery_task_uuid"] = None
if abortable:
env_build = models.EnvironmentImageBuild.query.filter_by(
**filter_by).one()
self.collateral_kwargs[
"celery_task_uuid"] = env_build.celery_task_uuid
return abortable
