class Pipeline(BaseModel):
__tablename__ = "pipelines"
project_uuid = db.Column(
db.String(36),
db.ForeignKey("projects.uuid", ondelete="CASCADE"),
primary_key=True,
)
uuid = db.Column(db.String(36), primary_key=True, nullable=False)
env_variables = deferred(
db.Column(JSONB, nullable=False, server_default="{}"))
# Note that all relationships are lazy=select.
interactive_sessions = db.relationship("InteractiveSession",
lazy="select",
passive_deletes=True,
cascade="all, delete")
jobs = db.relationship("Job",
lazy="select",
passive_deletes=True,
cascade="all, delete")
pipeline_runs = db.relationship("PipelineRun",
lazy="select",
passive_deletes=True,
cascade="all, delete")
class Project(BaseModel):
__tablename__ = "projects"
uuid = db.Column(db.String(36), primary_key=True, nullable=False)
env_variables = deferred(
db.Column(JSONB, nullable=False, server_default="{}"))
# Note that all relationships are lazy=select.
pipelines = db.relationship("Pipeline",
lazy="select",
passive_deletes=True,
cascade="all, delete")
environment_builds = db.relationship("EnvironmentBuild",
lazy="select",
passive_deletes=True,
cascade="all, delete")
interactive_sessions = db.relationship("InteractiveSession",
lazy="select",
passive_deletes=True,
cascade="all, delete")
jobs = db.relationship("Job",
lazy="select",
passive_deletes=True,
cascade="all, delete")
pipeline_runs = db.relationship("PipelineRun",
lazy="select",
passive_deletes=True,
cascade="all, delete")
class InteractiveRun(PipelineRun):
__tablename__ = "interactive_runs"
run_uuid = db.Column(db.String(36), primary_key=True)
# https://docs.sqlalchemy.org/en/14/orm/cascades.html#using-foreign-key-on-delete-cascade-with-orm-relationships
# In order to use ON DELETE foreign key cascades in conjunction
# with relationship(), it’s important to note first and foremost
# that the relationship.cascade setting must still be configured
# to match the desired “delete” or “set null” behavior
# Essentially, the specifed behaviour in the FK column
# and the one specified in the relationship must match.
pipeline_steps = db.relationship(
"InteractiveRunPipelineStep",
lazy="joined",
# do not rely on the db to delete
# TODO: can be set to true after we move away from sqllite
passive_deletes=False,
cascade="all, delete",
)
image_mappings = db.relationship(
"InteractiveRunImageMapping",
lazy="joined",
passive_deletes=False,
cascade="all, delete",
)
class InteractiveRun(PipelineRun):
__tablename__ = "interactive_runs"
run_uuid = db.Column(db.String(36), primary_key=True)
pipeline_steps = db.relationship("InteractiveRunPipelineStep",
lazy="joined")
class Experiment(BaseModel):
__tablename__ = "experiments"
__bind_key__ = "persistent_db"
experiment_uuid = db.Column(db.String(36), primary_key=True)
project_uuid = db.Column(db.String(36), )
pipeline_uuid = db.Column(db.String(36), primary_key=False)
total_number_of_pipeline_runs = db.Column(
db.Integer,
unique=False,
nullable=False,
)
scheduled_start = db.Column(db.DateTime, nullable=False)
completed_pipeline_runs = db.Column(
db.Integer,
unique=False,
default=0,
)
pipeline_runs = db.relationship("NonInteractiveRun",
lazy="joined",
passive_deletes=False,
cascade="all, delete")
def __repr__(self):
return f"<Experiment: {self.experiment_uuid}>"
class InteractiveRun(PipelineRun):
__tablename__ = 'interactive_runs'
run_uuid = db.Column(db.String(36), primary_key=True)
pipeline_steps = db.relationship('InteractiveRunPipelineStep',
lazy='joined')
class Experiment(BaseModel):
__tablename__ = 'experiments'
__bind_key__ = 'persistent_db'
experiment_uuid = db.Column(
db.String(36),
primary_key=True
)
pipeline_uuid = db.Column(
db.String(36),
primary_key=False
)
total_number_of_pipeline_runs = db.Column(
db.Integer,
unique=False,
nullable=False,
)
scheduled_start = db.Column(
db.DateTime,
nullable=False
)
completed_pipeline_runs = db.Column(
db.Integer,
unique=False,
default=0,
)
pipeline_runs = db.relationship('NonInteractiveRun', lazy='joined')
def __repr__(self):
return f'<Experiment: {self.experiment_uuid}>'
class NonInteractiveRun(PipelineRun):
__tablename__ = 'non_interactive_runs'
__bind_key__ = 'persistent_db'
experiment_uuid = db.Column(
db.String(36),
db.ForeignKey('experiments.experiment_uuid'),
primary_key=True
)
run_uuid = db.Column(
db.String(36),
primary_key=True
)
# This run_id is used to identify the pipeline run within the
# experiment and maintain a consistent ordering.
pipeline_run_id = db.Column(
db.Integer,
unique=False,
nullable=False,
)
started_time = db.Column(
db.DateTime,
unique=False,
nullable=True
)
finished_time = db.Column(
db.DateTime,
unique=False,
nullable=True
)
pipeline_steps = db.relationship('NonInteractiveRunPipelineStep', lazy='joined')
class PipelineRun(BaseModel):
__tablename__ = "pipeline_runs"
__table_args__ = (Index(
"ix_pipeline_runs_project_uuid_pipeline_uuid",
"project_uuid",
"pipeline_uuid",
), )
project_uuid = db.Column(
db.String(36),
db.ForeignKey("projects.uuid", ondelete="CASCADE"),
index=True,
nullable=False,
)
pipeline_uuid = db.Column(db.String(36),
index=True,
unique=False,
nullable=False)
uuid = db.Column(db.String(36), primary_key=True)
status = db.Column(db.String(15), unique=False, nullable=True)
started_time = db.Column(db.DateTime, unique=False, nullable=True)
finished_time = db.Column(db.DateTime, unique=False, nullable=True)
type = db.Column(db.String(50))
pipeline_steps = db.relationship(
"PipelineRunStep",
lazy="joined",
passive_deletes=True,
cascade="all, delete",
)
image_mappings = db.relationship(
"PipelineRunImageMapping",
lazy="joined",
passive_deletes=True,
cascade="all, delete",
)
# related to inheritance, the "type" column will be used to
# differentiate the different classes of entities
__mapper_args__ = {
"polymorphic_identity": "PipelineRun",
"polymorphic_on": type,
}
def __repr__(self):
return f"<{self.__class__.__name__}: {self.uuid}>"
class Run(BaseModel, db.Model):
__tablename__ = 'runs'
run_uuid = db.Column(db.String(36), primary_key=True)
pipeline_uuid = db.Column(db.String(36), unique=False, nullable=False)
status = db.Column(db.String(15), unique=False, nullable=True)
step_statuses = db.relationship('StepStatus', lazy='joined')
def __repr__(self):
return f'<Run {self.run_uuid}>'
class Project(db.Model):
__tablename__ = "project"
uuid = db.Column(db.String(255), nullable=False, primary_key=True)
path = db.Column(db.String(255), nullable=False)
__table_args__ = (UniqueConstraint("uuid", "path"), )
experiments = db.relationship("Experiment",
lazy="joined",
passive_deletes=False,
cascade="all, delete")
class NonInteractiveRun(PipelineRun):
__tablename__ = "non_interactive_runs"
__bind_key__ = "persistent_db"
# TODO: verify why the experiment_uuid should be part of the
# primary key
experiment_uuid = db.Column(
db.String(36),
db.ForeignKey("experiments.experiment_uuid", ondelete="CASCADE"),
primary_key=True,
)
# needs to be unique to be a FK constraint for images mappings
# that can delete on cascade
run_uuid = db.Column(db.String(36), primary_key=True, unique=True)
# This run_id is used to identify the pipeline run within the
# experiment and maintain a consistent ordering.
pipeline_run_id = db.Column(
db.Integer,
unique=False,
nullable=False,
)
started_time = db.Column(db.DateTime, unique=False, nullable=True)
finished_time = db.Column(db.DateTime, unique=False, nullable=True)
pipeline_steps = db.relationship(
"NonInteractiveRunPipelineStep",
lazy="joined",
passive_deletes=False,
cascade="all, delete",
)
image_mappings = db.relationship(
"NonInteractiveRunImageMapping",
lazy="joined",
passive_deletes=False,
cascade="all, delete",
)
class Job(BaseModel):
__tablename__ = "jobs"
job_uuid = db.Column(db.String(36), primary_key=True)
project_uuid = db.Column(db.String(36), )
pipeline_uuid = db.Column(db.String(36), primary_key=False)
total_number_of_pipeline_runs = db.Column(
db.Integer,
unique=False,
nullable=False,
)
scheduled_start = db.Column(db.DateTime, nullable=False)
completed_pipeline_runs = db.Column(
db.Integer,
unique=False,
server_default=text("0"),
)
pipeline_runs = db.relationship(
"NonInteractivePipelineRun",
lazy="joined",
# let the db take care of cascading deletions
# https://docs.sqlalchemy.org/en/13/orm/relationship_api.html#sqlalchemy.orm.relationship.params.passive_deletes
# A value of True indicates that unloaded child items should not
# be loaded during a delete operation on the parent. Normally,
# when a parent item is deleted, all child items are loaded so
# that they can either be marked as deleted, or have their
# foreign key to the parent set to NULL. Marking this flag as
# True usually implies an ON DELETE <CASCADE|SET NULL> rule is
# in place which will handle updating/deleting child rows on the
# database side.
passive_deletes=True,
# https://docs.sqlalchemy.org/en/14/orm/cascades.html#using-foreign-key-on-delete-cascade-with-orm-relationships
# In order to use ON DELETE foreign key cascades in conjunction
# with relationship(), it’s important to note first and foremost
# that the relationship.cascade setting must still be configured
# to match the desired “delete” or “set null” behavior
# Essentially, the specified behaviour in the FK column
# and the one specified in the relationship must match.
cascade="all, delete",
)
def __repr__(self):
return f"<Job: {self.job_uuid}>"
class Job(db.Model):
__tablename__ = "jobs"
name = db.Column(db.String(255), unique=False, nullable=False)
uuid = db.Column(db.String(255), unique=True, nullable=False, primary_key=True)
pipeline_uuid = db.Column(db.String(255), unique=False, nullable=False)
project_uuid = db.Column(
db.ForeignKey("project.uuid", ondelete="CASCADE"), unique=False, nullable=False
)
pipeline_name = db.Column(db.String(255), unique=False, nullable=False)
pipeline_path = db.Column(db.String(255), unique=False, nullable=False)
created = db.Column(
db.DateTime, nullable=False, server_default=text("timezone('utc', now())")
)
strategy_json = db.Column(db.Text, nullable=False)
draft = db.Column(db.Boolean())
pipeline_runs = db.relationship(
"PipelineRun", lazy="joined", passive_deletes=False, cascade="all, delete"
)
class Project(db.Model):
__tablename__ = "project"
uuid = db.Column(db.String(255), nullable=False, primary_key=True)
path = db.Column(db.String(255), nullable=False, unique=True)
# Can be: INITIALIZING, READY, DELETING. The status is used to avoid
# race conditions and inconsistencies when discovering new projects
# or projects that were deleted through the filesystem, given that
# discovery can be concurrent to project deletion or creation.
status = db.Column(
db.String(15),
unique=False,
nullable=False,
# The default value is rather important, so that people having
# their db automatically migrated will have projects in a valid
# state.
server_default=text("'READY'"),
)
__table_args__ = (UniqueConstraint("uuid", "path"),)
jobs = db.relationship(
"Job", lazy="joined", passive_deletes=False, cascade="all, delete"
)
class Job(BaseModel):
__tablename__ = "jobs"
__table_args__ = (Index("ix_jobs_project_uuid_pipeline_uuid",
"project_uuid", "pipeline_uuid"), )
name = db.Column(
db.String(255),
unique=False,
nullable=False,
# For migrating users.
server_default=text("'job'"),
)
pipeline_name = db.Column(
db.String(255),
unique=False,
nullable=False,
# For migrating users.
server_default=text("''"),
)
uuid = db.Column(db.String(36), primary_key=True)
project_uuid = db.Column(
db.String(36),
db.ForeignKey("projects.uuid", ondelete="CASCADE"),
index=True,
nullable=False,
)
pipeline_uuid = db.Column(db.String(36), index=True, nullable=False)
# Jobs that are to be schedule once (right now) or once in the
# future will have no schedule (null).
schedule = db.Column(db.String(100), nullable=True)
# A list of dictionaries. The length of the list is the number of
# non interactive runs that will be run, one for each parameters
# dictinary. A parameter dictionary maps step uuids to a dictionary,
# containing the parameters of that step for that particular run.
# [{ <step_uuid>: {"a": 1}, ...}, ...GG]
parameters = db.Column(
JSONB,
nullable=False,
# This way migrated entries that did not have this column will
# still be valid. Note that the entries will be stored as a list
# of dicts.
server_default="[]",
)
# Note that this column also contains the parameters that were
# stored within the pipeline definition file. These are not the job
# parameters, but the original ones.
pipeline_definition = db.Column(
JSONB,
nullable=False,
# This way migrated entries that did not have this column will
# still be valid.
server_default="{}",
)
pipeline_run_spec = db.Column(
JSONB,
nullable=False,
# This way migrated entries that did not have this column will
# still be valid.
server_default="{}",
)
# So that we can efficiently look for jobs to run.
next_scheduled_time = db.Column(TIMESTAMP(timezone=True), index=True)
# So that we can show the user the last time it was scheduled/run.
last_scheduled_time = db.Column(TIMESTAMP(timezone=True), index=True)
# So that we can "stamp" every non interactive run with the
# execution number it belongs to, e.g. the first time a job runs it
# will be batch 1, then 2, etc.
total_scheduled_executions = db.Column(
db.Integer,
unique=False,
server_default=text("0"),
)
pipeline_runs = db.relationship(
"NonInteractivePipelineRun",
lazy="select",
# let the db take care of cascading deletions
# https://docs.sqlalchemy.org/en/13/orm/relationship_api.html#sqlalchemy.orm.relationship.params.passive_deletes
# A value of True indicates that unloaded child items should not
# be loaded during a delete operation on the parent. Normally,
# when a parent item is deleted, all child items are loaded so
# that they can either be marked as deleted, or have their
# foreign key to the parent set to NULL. Marking this flag as
# True usually implies an ON DELETE <CASCADE|SET NULL> rule is
# in place which will handle updating/deleting child rows on the
# database side.
passive_deletes=True,
# https://docs.sqlalchemy.org/en/14/orm/cascades.html#using-foreign-key-on-delete-cascade-with-orm-relationships
# In order to use ON DELETE foreign key cascades in conjunction
# with relationship(), it’s important to note first and foremost
# that the relationship.cascade setting must still be configured
# to match the desired “delete” or “set null” behavior
# Essentially, the specified behaviour in the FK column
# and the one specified in the relationship must match.
cascade="all, delete",
# When querying a job and its runs the runs will be sorted by
# job schedule number and the index of the pipeline in that job.
order_by=(
"[desc(NonInteractivePipelineRun.job_run_index), "
"desc(NonInteractivePipelineRun.job_run_pipeline_run_index)]"),
)
# The status of a job can be DRAFT, PENDING, STARTED, SUCCESS,
# ABORTED, FAILURE. Jobs start as DRAFT, this indicates that the job
# has been created but that has not been started by the user. Once a
# job is started by the user, what happens depends on the type of
# job. One time jobs become PENDING, and become STARTED once they
# are run by the scheduler and their pipeline runs are added to the
# queue. Once they are completed, their status will be SUCCESS, if
# they are aborted, their status will be set to ABORTED. Recurring
# jobs, characterized by having a schedule, become STARTED, and can
# only move to the ABORTED state in case they get cancelled, which
# implies that the job will not be scheduled anymore. One time jobs
# which fail to run (the related pipeline runs scheduling fails) are
# set to FAILURE, this is not related to a failure at the pipeline
# run level.
status = db.Column(
db.String(15),
unique=False,
nullable=False,
# Pre-existing Jobs of migrating users will be set to SUCCESS.
server_default=text("'SUCCESS'"),
)
strategy_json = db.Column(
JSONB,
nullable=False,
server_default="{}",
)
env_variables = deferred(
db.Column(
JSONB,
nullable=False,
server_default="{}",
))
created_time = db.Column(
db.DateTime,
unique=False,
nullable=False,
index=True,
# For migrating users.
server_default=text("timezone('utc', now())"),
)
def __repr__(self):
return f"<Job: {self.uuid}>"
