class Pipeline(BaseModel): __tablename__ = "pipelines" project_uuid = db.Column( db.String(36), db.ForeignKey("projects.uuid", ondelete="CASCADE"), primary_key=True, ) uuid = db.Column(db.String(36), primary_key=True, nullable=False) env_variables = deferred( db.Column(JSONB, nullable=False, server_default="{}")) # Note that all relationships are lazy=select. interactive_sessions = db.relationship("InteractiveSession", lazy="select", passive_deletes=True, cascade="all, delete") jobs = db.relationship("Job", lazy="select", passive_deletes=True, cascade="all, delete") pipeline_runs = db.relationship("PipelineRun", lazy="select", passive_deletes=True, cascade="all, delete")
class Project(BaseModel): __tablename__ = "projects" uuid = db.Column(db.String(36), primary_key=True, nullable=False) env_variables = deferred( db.Column(JSONB, nullable=False, server_default="{}")) # Note that all relationships are lazy=select. pipelines = db.relationship("Pipeline", lazy="select", passive_deletes=True, cascade="all, delete") environment_builds = db.relationship("EnvironmentBuild", lazy="select", passive_deletes=True, cascade="all, delete") interactive_sessions = db.relationship("InteractiveSession", lazy="select", passive_deletes=True, cascade="all, delete") jobs = db.relationship("Job", lazy="select", passive_deletes=True, cascade="all, delete") pipeline_runs = db.relationship("PipelineRun", lazy="select", passive_deletes=True, cascade="all, delete")
class InteractiveRun(PipelineRun): __tablename__ = "interactive_runs" run_uuid = db.Column(db.String(36), primary_key=True) # https://docs.sqlalchemy.org/en/14/orm/cascades.html#using-foreign-key-on-delete-cascade-with-orm-relationships # In order to use ON DELETE foreign key cascades in conjunction # with relationship(), it’s important to note first and foremost # that the relationship.cascade setting must still be configured # to match the desired “delete” or “set null” behavior # Essentially, the specifed behaviour in the FK column # and the one specified in the relationship must match. pipeline_steps = db.relationship( "InteractiveRunPipelineStep", lazy="joined", # do not rely on the db to delete # TODO: can be set to true after we move away from sqllite passive_deletes=False, cascade="all, delete", ) image_mappings = db.relationship( "InteractiveRunImageMapping", lazy="joined", passive_deletes=False, cascade="all, delete", )
class InteractiveRun(PipelineRun): __tablename__ = "interactive_runs" run_uuid = db.Column(db.String(36), primary_key=True) pipeline_steps = db.relationship("InteractiveRunPipelineStep", lazy="joined")
class Experiment(BaseModel): __tablename__ = "experiments" __bind_key__ = "persistent_db" experiment_uuid = db.Column(db.String(36), primary_key=True) project_uuid = db.Column(db.String(36), ) pipeline_uuid = db.Column(db.String(36), primary_key=False) total_number_of_pipeline_runs = db.Column( db.Integer, unique=False, nullable=False, ) scheduled_start = db.Column(db.DateTime, nullable=False) completed_pipeline_runs = db.Column( db.Integer, unique=False, default=0, ) pipeline_runs = db.relationship("NonInteractiveRun", lazy="joined", passive_deletes=False, cascade="all, delete") def __repr__(self): return f"<Experiment: {self.experiment_uuid}>"
class InteractiveRun(PipelineRun): __tablename__ = 'interactive_runs' run_uuid = db.Column(db.String(36), primary_key=True) pipeline_steps = db.relationship('InteractiveRunPipelineStep', lazy='joined')
class Experiment(BaseModel): __tablename__ = 'experiments' __bind_key__ = 'persistent_db' experiment_uuid = db.Column( db.String(36), primary_key=True ) pipeline_uuid = db.Column( db.String(36), primary_key=False ) total_number_of_pipeline_runs = db.Column( db.Integer, unique=False, nullable=False, ) scheduled_start = db.Column( db.DateTime, nullable=False ) completed_pipeline_runs = db.Column( db.Integer, unique=False, default=0, ) pipeline_runs = db.relationship('NonInteractiveRun', lazy='joined') def __repr__(self): return f'<Experiment: {self.experiment_uuid}>'
class NonInteractiveRun(PipelineRun): __tablename__ = 'non_interactive_runs' __bind_key__ = 'persistent_db' experiment_uuid = db.Column( db.String(36), db.ForeignKey('experiments.experiment_uuid'), primary_key=True ) run_uuid = db.Column( db.String(36), primary_key=True ) # This run_id is used to identify the pipeline run within the # experiment and maintain a consistent ordering. pipeline_run_id = db.Column( db.Integer, unique=False, nullable=False, ) started_time = db.Column( db.DateTime, unique=False, nullable=True ) finished_time = db.Column( db.DateTime, unique=False, nullable=True ) pipeline_steps = db.relationship('NonInteractiveRunPipelineStep', lazy='joined')
class PipelineRun(BaseModel): __tablename__ = "pipeline_runs" __table_args__ = (Index( "ix_pipeline_runs_project_uuid_pipeline_uuid", "project_uuid", "pipeline_uuid", ), ) project_uuid = db.Column( db.String(36), db.ForeignKey("projects.uuid", ondelete="CASCADE"), index=True, nullable=False, ) pipeline_uuid = db.Column(db.String(36), index=True, unique=False, nullable=False) uuid = db.Column(db.String(36), primary_key=True) status = db.Column(db.String(15), unique=False, nullable=True) started_time = db.Column(db.DateTime, unique=False, nullable=True) finished_time = db.Column(db.DateTime, unique=False, nullable=True) type = db.Column(db.String(50)) pipeline_steps = db.relationship( "PipelineRunStep", lazy="joined", passive_deletes=True, cascade="all, delete", ) image_mappings = db.relationship( "PipelineRunImageMapping", lazy="joined", passive_deletes=True, cascade="all, delete", ) # related to inheritance, the "type" column will be used to # differentiate the different classes of entities __mapper_args__ = { "polymorphic_identity": "PipelineRun", "polymorphic_on": type, } def __repr__(self): return f"<{self.__class__.__name__}: {self.uuid}>"
class Run(BaseModel, db.Model): __tablename__ = 'runs' run_uuid = db.Column(db.String(36), primary_key=True) pipeline_uuid = db.Column(db.String(36), unique=False, nullable=False) status = db.Column(db.String(15), unique=False, nullable=True) step_statuses = db.relationship('StepStatus', lazy='joined') def __repr__(self): return f'<Run {self.run_uuid}>'
class Project(db.Model): __tablename__ = "project" uuid = db.Column(db.String(255), nullable=False, primary_key=True) path = db.Column(db.String(255), nullable=False) __table_args__ = (UniqueConstraint("uuid", "path"), ) experiments = db.relationship("Experiment", lazy="joined", passive_deletes=False, cascade="all, delete")
class NonInteractiveRun(PipelineRun): __tablename__ = "non_interactive_runs" __bind_key__ = "persistent_db" # TODO: verify why the experiment_uuid should be part of the # primary key experiment_uuid = db.Column( db.String(36), db.ForeignKey("experiments.experiment_uuid", ondelete="CASCADE"), primary_key=True, ) # needs to be unique to be a FK constraint for images mappings # that can delete on cascade run_uuid = db.Column(db.String(36), primary_key=True, unique=True) # This run_id is used to identify the pipeline run within the # experiment and maintain a consistent ordering. pipeline_run_id = db.Column( db.Integer, unique=False, nullable=False, ) started_time = db.Column(db.DateTime, unique=False, nullable=True) finished_time = db.Column(db.DateTime, unique=False, nullable=True) pipeline_steps = db.relationship( "NonInteractiveRunPipelineStep", lazy="joined", passive_deletes=False, cascade="all, delete", ) image_mappings = db.relationship( "NonInteractiveRunImageMapping", lazy="joined", passive_deletes=False, cascade="all, delete", )
class Job(BaseModel): __tablename__ = "jobs" job_uuid = db.Column(db.String(36), primary_key=True) project_uuid = db.Column(db.String(36), ) pipeline_uuid = db.Column(db.String(36), primary_key=False) total_number_of_pipeline_runs = db.Column( db.Integer, unique=False, nullable=False, ) scheduled_start = db.Column(db.DateTime, nullable=False) completed_pipeline_runs = db.Column( db.Integer, unique=False, server_default=text("0"), ) pipeline_runs = db.relationship( "NonInteractivePipelineRun", lazy="joined", # let the db take care of cascading deletions # https://docs.sqlalchemy.org/en/13/orm/relationship_api.html#sqlalchemy.orm.relationship.params.passive_deletes # A value of True indicates that unloaded child items should not # be loaded during a delete operation on the parent. Normally, # when a parent item is deleted, all child items are loaded so # that they can either be marked as deleted, or have their # foreign key to the parent set to NULL. Marking this flag as # True usually implies an ON DELETE <CASCADE|SET NULL> rule is # in place which will handle updating/deleting child rows on the # database side. passive_deletes=True, # https://docs.sqlalchemy.org/en/14/orm/cascades.html#using-foreign-key-on-delete-cascade-with-orm-relationships # In order to use ON DELETE foreign key cascades in conjunction # with relationship(), it’s important to note first and foremost # that the relationship.cascade setting must still be configured # to match the desired “delete” or “set null” behavior # Essentially, the specified behaviour in the FK column # and the one specified in the relationship must match. cascade="all, delete", ) def __repr__(self): return f"<Job: {self.job_uuid}>"
class Job(db.Model): __tablename__ = "jobs" name = db.Column(db.String(255), unique=False, nullable=False) uuid = db.Column(db.String(255), unique=True, nullable=False, primary_key=True) pipeline_uuid = db.Column(db.String(255), unique=False, nullable=False) project_uuid = db.Column( db.ForeignKey("project.uuid", ondelete="CASCADE"), unique=False, nullable=False ) pipeline_name = db.Column(db.String(255), unique=False, nullable=False) pipeline_path = db.Column(db.String(255), unique=False, nullable=False) created = db.Column( db.DateTime, nullable=False, server_default=text("timezone('utc', now())") ) strategy_json = db.Column(db.Text, nullable=False) draft = db.Column(db.Boolean()) pipeline_runs = db.relationship( "PipelineRun", lazy="joined", passive_deletes=False, cascade="all, delete" )
class Project(db.Model): __tablename__ = "project" uuid = db.Column(db.String(255), nullable=False, primary_key=True) path = db.Column(db.String(255), nullable=False, unique=True) # Can be: INITIALIZING, READY, DELETING. The status is used to avoid # race conditions and inconsistencies when discovering new projects # or projects that were deleted through the filesystem, given that # discovery can be concurrent to project deletion or creation. status = db.Column( db.String(15), unique=False, nullable=False, # The default value is rather important, so that people having # their db automatically migrated will have projects in a valid # state. server_default=text("'READY'"), ) __table_args__ = (UniqueConstraint("uuid", "path"),) jobs = db.relationship( "Job", lazy="joined", passive_deletes=False, cascade="all, delete" )
class Job(BaseModel): __tablename__ = "jobs" __table_args__ = (Index("ix_jobs_project_uuid_pipeline_uuid", "project_uuid", "pipeline_uuid"), ) name = db.Column( db.String(255), unique=False, nullable=False, # For migrating users. server_default=text("'job'"), ) pipeline_name = db.Column( db.String(255), unique=False, nullable=False, # For migrating users. server_default=text("''"), ) uuid = db.Column(db.String(36), primary_key=True) project_uuid = db.Column( db.String(36), db.ForeignKey("projects.uuid", ondelete="CASCADE"), index=True, nullable=False, ) pipeline_uuid = db.Column(db.String(36), index=True, nullable=False) # Jobs that are to be schedule once (right now) or once in the # future will have no schedule (null). schedule = db.Column(db.String(100), nullable=True) # A list of dictionaries. The length of the list is the number of # non interactive runs that will be run, one for each parameters # dictinary. A parameter dictionary maps step uuids to a dictionary, # containing the parameters of that step for that particular run. # [{ <step_uuid>: {"a": 1}, ...}, ...GG] parameters = db.Column( JSONB, nullable=False, # This way migrated entries that did not have this column will # still be valid. Note that the entries will be stored as a list # of dicts. server_default="[]", ) # Note that this column also contains the parameters that were # stored within the pipeline definition file. These are not the job # parameters, but the original ones. pipeline_definition = db.Column( JSONB, nullable=False, # This way migrated entries that did not have this column will # still be valid. server_default="{}", ) pipeline_run_spec = db.Column( JSONB, nullable=False, # This way migrated entries that did not have this column will # still be valid. server_default="{}", ) # So that we can efficiently look for jobs to run. next_scheduled_time = db.Column(TIMESTAMP(timezone=True), index=True) # So that we can show the user the last time it was scheduled/run. last_scheduled_time = db.Column(TIMESTAMP(timezone=True), index=True) # So that we can "stamp" every non interactive run with the # execution number it belongs to, e.g. the first time a job runs it # will be batch 1, then 2, etc. total_scheduled_executions = db.Column( db.Integer, unique=False, server_default=text("0"), ) pipeline_runs = db.relationship( "NonInteractivePipelineRun", lazy="select", # let the db take care of cascading deletions # https://docs.sqlalchemy.org/en/13/orm/relationship_api.html#sqlalchemy.orm.relationship.params.passive_deletes # A value of True indicates that unloaded child items should not # be loaded during a delete operation on the parent. Normally, # when a parent item is deleted, all child items are loaded so # that they can either be marked as deleted, or have their # foreign key to the parent set to NULL. Marking this flag as # True usually implies an ON DELETE <CASCADE|SET NULL> rule is # in place which will handle updating/deleting child rows on the # database side. passive_deletes=True, # https://docs.sqlalchemy.org/en/14/orm/cascades.html#using-foreign-key-on-delete-cascade-with-orm-relationships # In order to use ON DELETE foreign key cascades in conjunction # with relationship(), it’s important to note first and foremost # that the relationship.cascade setting must still be configured # to match the desired “delete” or “set null” behavior # Essentially, the specified behaviour in the FK column # and the one specified in the relationship must match. cascade="all, delete", # When querying a job and its runs the runs will be sorted by # job schedule number and the index of the pipeline in that job. order_by=( "[desc(NonInteractivePipelineRun.job_run_index), " "desc(NonInteractivePipelineRun.job_run_pipeline_run_index)]"), ) # The status of a job can be DRAFT, PENDING, STARTED, SUCCESS, # ABORTED, FAILURE. Jobs start as DRAFT, this indicates that the job # has been created but that has not been started by the user. Once a # job is started by the user, what happens depends on the type of # job. One time jobs become PENDING, and become STARTED once they # are run by the scheduler and their pipeline runs are added to the # queue. Once they are completed, their status will be SUCCESS, if # they are aborted, their status will be set to ABORTED. Recurring # jobs, characterized by having a schedule, become STARTED, and can # only move to the ABORTED state in case they get cancelled, which # implies that the job will not be scheduled anymore. One time jobs # which fail to run (the related pipeline runs scheduling fails) are # set to FAILURE, this is not related to a failure at the pipeline # run level. status = db.Column( db.String(15), unique=False, nullable=False, # Pre-existing Jobs of migrating users will be set to SUCCESS. server_default=text("'SUCCESS'"), ) strategy_json = db.Column( JSONB, nullable=False, server_default="{}", ) env_variables = deferred( db.Column( JSONB, nullable=False, server_default="{}", )) created_time = db.Column( db.DateTime, unique=False, nullable=False, index=True, # For migrating users. server_default=text("timezone('utc', now())"), ) def __repr__(self): return f"<Job: {self.uuid}>"