class Setting(db.Model): __tablename__ = 'settings_v2' __table_args__ = (UniqueConstraint('key', name='uniq_key'), default_table_args('this is webconsole settings table')) id = db.Column(db.Integer, primary_key=True, comment='id') key = db.Column(db.String(255), nullable=False, comment='key') value = db.Column(db.Text, comment='value')
class SchedulerItem(db.Model): __tablename__ = 'scheduler_item_v2' __table_args__ = (UniqueConstraint('name', name='uniq_name'), default_table_args('scheduler items')) id = db.Column(db.Integer, comment='id', primary_key=True, autoincrement=True) name = db.Column(db.String(255), comment='item name', nullable=False) pipeline = db.Column(db.Text, comment='pipeline', nullable=False, default='{}') status = db.Column(db.Integer, comment='item status', nullable=False, default=ItemStatus.ON.value) interval_time = db.Column(db.Integer, comment='item run interval in second', nullable=False, default=-1) last_run_at = db.Column(db.DateTime(timezone=True), comment='last runner time') retry_cnt = db.Column(db.Integer, comment='retry count when item is failed', nullable=False, default=0) extra = db.Column(db.Text(), comment='extra info') created_at = db.Column(db.DateTime(timezone=True), comment='created at', server_default=func.now()) updated_at = db.Column(db.DateTime(timezone=True), comment='updated at', server_default=func.now(), onupdate=func.now()) deleted_at = db.Column(db.DateTime(timezone=True), comment='deleted at') def need_run(self) -> bool: # job runs one time if self.interval_time == -1 and self.last_run_at is None: return True if self.interval_time > 0: # cronjob if self.last_run_at is None: # never run return True # compare datetime in utc next_run_at = self.last_run_at.replace( tzinfo=datetime.timezone.utc) + datetime.timedelta( seconds=self.interval_time) utc_now = datetime.datetime.now(datetime.timezone.utc) logging.info(f'[composer] item id: {self.id}, ' f'next_run_at: {next_run_at.timestamp()}, ' f'utc_now: {utc_now.timestamp()}') if next_run_at.timestamp() < utc_now.timestamp(): return True return False
class Session(db.Model): __tablename__ = 'session_v2' __table_args__ = (Index('idx_jti', 'jti'), default_table_args('This is webconsole session table')) id = db.Column(db.Integer, primary_key=True, autoincrement=True, comment='session id') jti = db.Column(db.String(64), comment='JWT jti') expired_at = db.Column(db.DateTime(timezone=True), comment='expired time, for db automatically clear') created_at = db.Column(db.DateTime(timezone=True), server_default=func.now(), comment='created at')
class ModelGroup(db.Model): __tablename__ = 'model_groups_v2' __table_args__ = (default_table_args('model_groups_v2')) id = db.Column(db.Integer, primary_key=True, comment='id') name = db.Column(db.String(255), comment='name') # can be modified by end-user created_at = db.Column(db.DateTime(timezone=True), comment='created_at', server_default=func.now()) updated_at = db.Column(db.DateTime(timezone=True), comment='updated_at', server_default=func.now(), onupdate=func.now()) deleted_at = db.Column(db.DateTime(timezone=True), comment='deleted_at') # TODO https://code.byted.org/data/fedlearner_web_console_v2/issues/289 extra = db.Column(db.Text(), comment='extra') # json string
class OptimisticLock(db.Model): __tablename__ = 'optimistic_lock_v2' __table_args__ = ( UniqueConstraint('name', name='uniq_name'), default_table_args('optimistic lock'), ) id = db.Column(db.Integer, comment='id', primary_key=True, autoincrement=True) name = db.Column(db.String(255), comment='lock name', nullable=False) version = db.Column(db.BIGINT, comment='lock version', nullable=False) created_at = db.Column(db.DateTime(timezone=True), comment='created at', server_default=func.now()) updated_at = db.Column(db.DateTime(timezone=True), comment='updated at', server_default=func.now(), onupdate=func.now()) deleted_at = db.Column(db.DateTime(timezone=True), comment='deleted at')
class WorkflowTemplate(db.Model): __tablename__ = 'template_v2' __table_args__ = (UniqueConstraint('name', name='uniq_name'), Index('idx_group_alias', 'group_alias'), default_table_args('workflow template')) id = db.Column(db.Integer, primary_key=True, comment='id') name = db.Column(db.String(255), comment='name') comment = db.Column('cmt', db.String(255), key='comment', comment='comment') group_alias = db.Column(db.String(255), nullable=False, comment='group_alias') # max store 16777215 bytes (16 MB) config = db.Column(db.LargeBinary(16777215), nullable=False, comment='config') is_left = db.Column(db.Boolean, comment='is_left') editor_info = db.Column(db.LargeBinary(16777215), comment='editor_info', default=b'') kind = db.Column(db.Integer, comment='template kind') # WorkflowTemplateKind enum def set_config(self, proto): self.config = proto.SerializeToString() def set_editor_info(self, proto): self.editor_info = proto.SerializeToString() def get_config(self): proto = workflow_definition_pb2.WorkflowDefinition() proto.ParseFromString(self.config) return proto def get_editor_info(self): proto = workflow_definition_pb2.WorkflowTemplateEditorInfo() if self.editor_info is not None: proto.ParseFromString(self.editor_info) return proto
class Model(db.Model): __tablename__ = 'models_v2' __table_args__ = (default_table_args('model')) id = db.Column(db.Integer, primary_key=True, comment='id') name = db.Column(db.String(255), comment='model_name') version = db.Column(db.Integer, comment='model_version') parent_id = db.Column(db.String(255), comment='parent_id') job_name = db.Column(db.String(255), comment='job_name') type = db.Column(db.Enum(ModelType, native_enum=False), comment='type') state = db.Column(db.Enum(ModelState, native_enum=False), comment='state') create_time = db.Column(db.DateTime(timezone=True), comment='create_time') params = db.Column(db.Text(), comment='params') metrics = db.Column(db.Text(), comment='metrics') output_base_dir = db.Column(db.String(255), comment='model checkpoint/export path') parent = db.relationship('Model', primaryjoin=remote(id) == foreign(parent_id), backref='children') job = db.relationship('Job', primaryjoin=Job.name == foreign(job_name)) def __init__(self): self.create_time = datetime.now() self.version = 0 def commit(self): db.session.add(self) db.session.commit() def get_eval_model(self): """ Get the evaluation model inherited model Returns: a list of evaluation model """ eval_model = [ child for child in self.children if child.type in [ModelType.NN_EVALUATION, ModelType.TREE_EVALUATION] ] return eval_model
class User(db.Model): __tablename__ = 'users_v2' __table_args__ = (UniqueConstraint('username', name='uniq_username'), default_table_args('This is webconsole user table')) id = db.Column(db.Integer, primary_key=True, comment='user id') username = db.Column(db.String(255), comment='unique name of user') password = db.Column(db.String(255), comment='user password after encode') role = db.Column(db.Enum(Role, native_enum=False), default=Role.USER, comment='role of user') name = db.Column(db.String(255), comment='name of user') email = db.Column(db.String(255), comment='email of user') state = db.Column(db.Enum(State, native_enum=False), default=State.ACTIVE, comment='state of user') def set_password(self, password): self.password = pwd_context.hash(password) def verify_password(self, password): return pwd_context.verify(password, self.password)
class Model(db.Model): __tablename__ = 'models_v2' __table_args__ = (Index('idx_job_name', 'job_name'), UniqueConstraint('job_name', name='uniq_job_name'), default_table_args('model')) id = db.Column(db.Integer, primary_key=True, comment='id') name = db.Column(db.String(255), comment='name') # can be modified by end-user version = db.Column(db.Integer, default=0, comment='version') type = db.Column(db.Integer, comment='type') state = db.Column(db.Integer, comment='state') job_name = db.Column(db.String(255), comment='job_name') parent_id = db.Column(db.Integer, comment='parent_id') params = db.Column(db.Text(), comment='params') metrics = db.Column(db.Text(), comment='metrics') created_at = db.Column(db.DateTime(timezone=True), comment='created_at', server_default=func.now()) updated_at = db.Column(db.DateTime(timezone=True), comment='updated_at', server_default=func.now(), onupdate=func.now()) deleted_at = db.Column(db.DateTime(timezone=True), comment='deleted_at') group_id = db.Column(db.Integer, default=0, comment='group_id') # TODO https://code.byted.org/data/fedlearner_web_console_v2/issues/289 extra = db.Column(db.Text(), comment='extra') # json string parent = db.relationship('Model', primaryjoin=remote(id) == foreign(parent_id), backref='children') job = db.relationship('Job', primaryjoin=Job.name == foreign(job_name)) def get_eval_model(self): return [ child for child in self.children if child.type in [ModelType.NN_EVALUATION.value, ModelType.TREE_EVALUATION.value] ]
class SchedulerRunner(db.Model): __tablename__ = 'scheduler_runner_v2' __table_args__ = (default_table_args('scheduler runners')) id = db.Column(db.Integer, comment='id', primary_key=True, autoincrement=True) item_id = db.Column(db.Integer, comment='item id', nullable=False) status = db.Column(db.Integer, comment='runner status', nullable=False, default=RunnerStatus.INIT.value) start_at = db.Column(db.DateTime(timezone=True), comment='runner start time') end_at = db.Column(db.DateTime(timezone=True), comment='runner end time') pipeline = db.Column(db.Text(), comment='pipeline from scheduler item', nullable=False, default='{}') output = db.Column(db.Text(), comment='output', nullable=False, default='{}') context = db.Column(db.Text(), comment='context', nullable=False, default='{}') extra = db.Column(db.Text(), comment='extra info') created_at = db.Column(db.DateTime(timezone=True), comment='created at', server_default=func.now()) updated_at = db.Column(db.DateTime(timezone=True), comment='updated at', server_default=func.now(), onupdate=func.now()) deleted_at = db.Column(db.DateTime(timezone=True), comment='deleted at')