class User(db.Model): __tablename__ = "users" uuid = db.Column( db.String(36), primary_key=True, # required to be referenced as a foreign key by the Token table, # since postgres does not accept foreign keys referencing non # unique fields unique=True, ) username = db.Column( db.String(255), primary_key=True, ) password_hash = db.Column( db.String(255), primary_key=True, ) created = db.Column( db.DateTime, unique=False, nullable=False, server_default=text("timezone('utc', now())"), )
class InteractiveRunImageMapping(BaseModel): """Stores mappings between an interactive run and the environment images it uses. Used to understand if an image can be removed from the docker environment if it's not used by a run which is PENDING or STARTED. """ __tablename__ = "interactive_run_image_mapping" __table_args__ = ( UniqueConstraint("run_uuid", "orchest_environment_uuid"), UniqueConstraint("run_uuid", "docker_img_id"), ) run_uuid = db.Column( db.ForeignKey(InteractiveRun.run_uuid, ondelete="CASCADE"), unique=False, nullable=False, index=True, primary_key=True, ) orchest_environment_uuid = db.Column(db.String(36), unique=False, nullable=False, primary_key=True) docker_img_id = db.Column(db.String(), unique=False, nullable=False, primary_key=True) def __repr__(self): return (f"<InteractiveRunImageMapping: {self.run_uuid} | " f"{self.orchest_environment_uuid} | " f"{self.docker_img_id}>")
class Pipeline(BaseModel): __tablename__ = "pipelines" project_uuid = db.Column( db.String(36), db.ForeignKey("projects.uuid", ondelete="CASCADE"), primary_key=True, ) uuid = db.Column(db.String(36), primary_key=True, nullable=False) env_variables = deferred( db.Column(JSONB, nullable=False, server_default="{}")) # Note that all relationships are lazy=select. interactive_sessions = db.relationship("InteractiveSession", lazy="select", passive_deletes=True, cascade="all, delete") jobs = db.relationship("Job", lazy="select", passive_deletes=True, cascade="all, delete") pipeline_runs = db.relationship("PipelineRun", lazy="select", passive_deletes=True, cascade="all, delete")
class EnvironmentBuild(BaseModel): """State of environment builds. Table meant to store the state of the build task of an environment, i.e. when we need to build an image starting from a base image plus optional sh code. This is not related to keeping track of environments or images to decide if a project or pipeline can be run. """ __tablename__ = "environment_builds" __table_args__ = (Index("uuid_proj_env_index", "project_uuid", "environment_uuid"), ) # https://stackoverflow.com/questions/63164261/celery-task-id-max-length uuid = db.Column(db.String(36), primary_key=True, nullable=False) project_uuid = db.Column( db.String(36), db.ForeignKey("projects.uuid", ondelete="CASCADE"), primary_key=True, index=True, ) environment_uuid = db.Column(db.String(36), nullable=False, index=True) project_path = db.Column(db.String(4096), nullable=False, index=True) requested_time = db.Column(db.DateTime, unique=False, nullable=False) started_time = db.Column(db.DateTime, unique=False, nullable=True) finished_time = db.Column(db.DateTime, unique=False, nullable=True) status = db.Column(db.String(15), unique=False, nullable=True) def __repr__(self): return f"<EnvironmentBuildTask: {self.uuid}>"
class Experiment(BaseModel): __tablename__ = "experiments" __bind_key__ = "persistent_db" experiment_uuid = db.Column(db.String(36), primary_key=True) project_uuid = db.Column(db.String(36), ) pipeline_uuid = db.Column(db.String(36), primary_key=False) total_number_of_pipeline_runs = db.Column( db.Integer, unique=False, nullable=False, ) scheduled_start = db.Column(db.DateTime, nullable=False) completed_pipeline_runs = db.Column( db.Integer, unique=False, default=0, ) pipeline_runs = db.relationship("NonInteractiveRun", lazy="joined", passive_deletes=False, cascade="all, delete") def __repr__(self): return f"<Experiment: {self.experiment_uuid}>"
class InteractiveSession(BaseModel): __tablename__ = 'interactive_sessions' pipeline_uuid = db.Column( db.String(36), primary_key=True, ) status = db.Column( db.String(10), primary_key=False, ) # Used to connect to Jupyter notebook server. jupyter_server_ip = db.Column( db.String(15), unique=True, nullable=True, ) # IPv4 # Used to connect to Jupyter notebook server. notebook_server_info = db.Column( db.JSON, unique=True, nullable=True, ) # Docker container IDs. Used internally to identify the resources of # a specific session. container_ids = db.Column( db.JSON, unique=False, nullable=True, ) def __repr__(self): return f'<Launch {self.pipeline_uuid}>'
class PipelineRunPipelineStep(BaseModel): __abstract__ = True step_uuid = db.Column( db.String(36), primary_key=True ) status = db.Column( db.String(15), unique=False, nullable=True ) started_time = db.Column( db.DateTime, unique=False, nullable=True ) finished_time = db.Column( db.DateTime, unique=False, nullable=True ) def __repr__(self): return f'<{self.__class__.__name__}: {self.run_uuid}.{self.step_uuid}>'
class Project(db.Model): __tablename__ = "project" uuid = db.Column(db.String(255), nullable=False, primary_key=True) path = db.Column(db.String(255), nullable=False) __table_args__ = (UniqueConstraint("uuid", "path"), )
class Experiment(BaseModel): __tablename__ = 'experiments' __bind_key__ = 'persistent_db' experiment_uuid = db.Column( db.String(36), primary_key=True ) pipeline_uuid = db.Column( db.String(36), primary_key=False ) total_number_of_pipeline_runs = db.Column( db.Integer, unique=False, nullable=False, ) scheduled_start = db.Column( db.DateTime, nullable=False ) completed_pipeline_runs = db.Column( db.Integer, unique=False, default=0, ) pipeline_runs = db.relationship('NonInteractiveRun', lazy='joined') def __repr__(self): return f'<Experiment: {self.experiment_uuid}>'
class NonInteractiveRun(PipelineRun): __tablename__ = 'non_interactive_runs' __bind_key__ = 'persistent_db' experiment_uuid = db.Column( db.String(36), db.ForeignKey('experiments.experiment_uuid'), primary_key=True ) run_uuid = db.Column( db.String(36), primary_key=True ) # This run_id is used to identify the pipeline run within the # experiment and maintain a consistent ordering. pipeline_run_id = db.Column( db.Integer, unique=False, nullable=False, ) started_time = db.Column( db.DateTime, unique=False, nullable=True ) finished_time = db.Column( db.DateTime, unique=False, nullable=True ) pipeline_steps = db.relationship('NonInteractiveRunPipelineStep', lazy='joined')
class PipelineRun(BaseModel): __abstract__ = True pipeline_uuid = db.Column(db.String(36), unique=False, nullable=False) status = db.Column(db.String(15), unique=False, nullable=True) def __repr__(self): return f'<{self.__class__.__name__}: {self.run_uuid}>'
class Launch(BaseModel, db.Model): __tablename__ = 'launches' pipeline_uuid = db.Column(db.String(36), primary_key=True) server_ip = db.Column(db.String(15), unique=True, nullable=False) # IPv4 server_info = db.Column(db.JSON, unique=True, nullable=False) def __repr__(self): return f'<Launch {self.pipeline_uuid}>'
class Pipeline(db.Model): __tablename__ = "pipelines" uuid = db.Column(db.String(255), nullable=False, primary_key=True) project_uuid = db.Column(db.ForeignKey("projects.uuid", ondelete="CASCADE"), primary_key=True) path = db.Column(db.String(255), nullable=False)
class Pipeline(db.Model): __tablename__ = "pipeline" uuid = db.Column(db.String(255), nullable=False, primary_key=True) project_uuid = db.Column(db.ForeignKey("project.uuid"), primary_key=True) path = db.Column(db.String(255), nullable=False) __table_args__ = (UniqueConstraint("uuid", "project_uuid"), )
class Run(BaseModel, db.Model): __tablename__ = 'runs' run_uuid = db.Column(db.String(36), primary_key=True) pipeline_uuid = db.Column(db.String(36), unique=False, nullable=False) status = db.Column(db.String(15), unique=False, nullable=True) step_statuses = db.relationship('StepStatus', lazy='joined') def __repr__(self): return f'<Run {self.run_uuid}>'
class Image(db.Model): __tablename__ = 'images' name = db.Column(db.String(255), unique=True, nullable=False, primary_key=True) language = db.Column(db.String(255), nullable=False) created = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow) def __repr__(self): return f'<Images {self.name}:{self.language}>'
class Experiment(db.Model): __tablename__ = 'experiments' name = db.Column(db.String(255), unique=False, nullable=False) uuid = db.Column(db.String(255), unique=True, nullable=False, primary_key=True) pipeline_uuid = db.Column(db.String(255), unique=False, nullable=False) pipeline_name = db.Column(db.String(255), unique=False, nullable=False) created = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow) strategy_json = db.Column(db.Text, nullable=False) draft = db.Column(db.Boolean())
class DataSource(db.Model): __tablename__ = 'datasources' name = db.Column(db.String(255), unique=True, nullable=False, primary_key=True) source_type = db.Column(db.String(100), nullable=False) connection_details = db.Column(db.JSON, nullable=False) created = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow) def __repr__(self): return f'<DataSource {self.name}:{self.source_type}>'
class NonInteractiveRunPipelineStep(PipelineRunPipelineStep): __tablename__ = "non_interactive_run_pipeline_steps" __bind_key__ = "persistent_db" experiment_uuid = db.Column(db.String(36), db.ForeignKey("experiments.experiment_uuid"), primary_key=True) run_uuid = db.Column(db.String(36), db.ForeignKey("non_interactive_runs.run_uuid"), primary_key=True)
class Token(db.Model): __tablename__ = "tokens" token = db.Column(db.String(255)) user = db.Column(db.String(36), db.ForeignKey("users.uuid"), primary_key=True) created = db.Column( db.DateTime, unique=False, nullable=False, default=datetime.datetime.utcnow )
class Project(db.Model): __tablename__ = "project" uuid = db.Column(db.String(255), nullable=False, primary_key=True) path = db.Column(db.String(255), nullable=False) __table_args__ = (UniqueConstraint("uuid", "path"), ) experiments = db.relationship("Experiment", lazy="joined", passive_deletes=False, cascade="all, delete")
class DataSource(db.Model): __tablename__ = "datasources" name = db.Column(db.String(255), unique=True, nullable=False, primary_key=True) source_type = db.Column(db.String(100), nullable=False) connection_details = db.Column(db.JSON, nullable=False) created = db.Column( db.DateTime, nullable=False, server_default=text("timezone('utc', now())") ) def __repr__(self): return f"<DataSource {self.name}:{self.source_type}>"
class Commit(db.Model): __tablename__ = 'commits' uuid = db.Column(db.String(255), unique=True, nullable=False, primary_key=True) tag = db.Column(db.String(255), unique=False, nullable=False) name = db.Column(db.String(255), unique=False, nullable=False) base_image = db.Column(db.ForeignKey("images.name")) created = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow) building = db.Column(db.Boolean, default=False) def __repr__(self): return f'<Commit {self.name}:{self.base_image}:{self.uuid}>'
class StepStatus(BaseModel, db.Model): __tablename__ = 'stepstatus' run_uuid = db.Column(db.String(36), db.ForeignKey('runs.run_uuid'), primary_key=True) step_uuid = db.Column(db.String(36), primary_key=True) status = db.Column(db.String(15), unique=False, nullable=True) started_time = db.Column(db.DateTime, unique=False, nullable=True) ended_time = db.Column(db.DateTime, unique=False, nullable=True) def __repr__(self): return f'<StepStatus {self.run_uuid}.{self.step_uuid}>'
class BackgroundTask(db.Model): """BackgroundTasks, models all tasks to be run in the background.""" __tablename__ = "background_tasks" task_uuid = db.Column(db.String(36), primary_key=True, unique=True, nullable=False) # see background_task_executor types task_type = db.Column(db.String(50), unique=False, nullable=True) status = db.Column(db.String(15), unique=False, nullable=False) code = db.Column(db.String(15), unique=False, nullable=True) result = db.Column(db.String(), unique=False, nullable=True) def __repr__(self): return f"<BackgroundTask: {self.task_uuid}>"
class PipelineRun(db.Model): __tablename__ = 'pipelineruns' uuid = db.Column(db.String(255), unique=True, nullable=False, primary_key=True) id = db.Column(db.Integer(), unique=False) experiment = db.Column(db.ForeignKey("experiments.uuid")) parameter_json = db.Column(db.JSON, nullable=False)
class PipelineRunStep(BaseModel): __tablename__ = "pipeline_run_steps" run_uuid = db.Column( db.String(36), db.ForeignKey("pipeline_runs.uuid", ondelete="CASCADE"), primary_key=True, ) step_uuid = db.Column(db.String(36), primary_key=True) status = db.Column(db.String(15), unique=False, nullable=True) started_time = db.Column(db.DateTime, unique=False, nullable=True) finished_time = db.Column(db.DateTime, unique=False, nullable=True) def __repr__(self): return f"<{self.__class__.__name__}: {self.run_uuid}.{self.step_uuid}>"
class Project(BaseModel): __tablename__ = "projects" uuid = db.Column(db.String(36), primary_key=True, nullable=False) env_variables = deferred( db.Column(JSONB, nullable=False, server_default="{}")) # Note that all relationships are lazy=select. pipelines = db.relationship("Pipeline", lazy="select", passive_deletes=True, cascade="all, delete") environment_builds = db.relationship("EnvironmentBuild", lazy="select", passive_deletes=True, cascade="all, delete") interactive_sessions = db.relationship("InteractiveSession", lazy="select", passive_deletes=True, cascade="all, delete") jobs = db.relationship("Job", lazy="select", passive_deletes=True, cascade="all, delete") pipeline_runs = db.relationship("PipelineRun", lazy="select", passive_deletes=True, cascade="all, delete")
class PipelineRun(db.Model): __tablename__ = "pipelineruns" uuid = db.Column(db.String(255), unique=True, nullable=False, primary_key=True) id = db.Column(db.Integer(), unique=False) job = db.Column(db.ForeignKey("jobs.uuid", ondelete="CASCADE")) parameter_json = db.Column(db.JSON, nullable=False)
class Token(db.Model): __tablename__ = "tokens" token = db.Column(db.String(255)) user = db.Column(db.String(36), db.ForeignKey("users.uuid"), primary_key=True) created = db.Column( db.DateTime, unique=False, nullable=False, server_default=text("timezone('utc', now())"), )