class Setting(db.Model):
__tablename__ = 'settings_v2'
__table_args__ = (UniqueConstraint('key', name='uniq_key'),
default_table_args('this is webconsole settings table'))
id = db.Column(db.Integer, primary_key=True, comment='id')
key = db.Column(db.String(255), nullable=False, comment='key')
value = db.Column(db.Text, comment='value')
class SchedulerItem(db.Model):
__tablename__ = 'scheduler_item_v2'
__table_args__ = (UniqueConstraint('name', name='uniq_name'),
default_table_args('scheduler items'))
id = db.Column(db.Integer,
comment='id',
primary_key=True,
autoincrement=True)
name = db.Column(db.String(255), comment='item name', nullable=False)
pipeline = db.Column(db.Text,
comment='pipeline',
nullable=False,
default='{}')
status = db.Column(db.Integer,
comment='item status',
nullable=False,
default=ItemStatus.ON.value)
interval_time = db.Column(db.Integer,
comment='item run interval in second',
nullable=False,
default=-1)
last_run_at = db.Column(db.DateTime(timezone=True),
comment='last runner time')
retry_cnt = db.Column(db.Integer,
comment='retry count when item is failed',
nullable=False,
default=0)
extra = db.Column(db.Text(), comment='extra info')
created_at = db.Column(db.DateTime(timezone=True),
comment='created at',
server_default=func.now())
updated_at = db.Column(db.DateTime(timezone=True),
comment='updated at',
server_default=func.now(),
onupdate=func.now())
deleted_at = db.Column(db.DateTime(timezone=True), comment='deleted at')
def need_run(self) -> bool:
# job runs one time
if self.interval_time == -1 and self.last_run_at is None:
return True
if self.interval_time > 0: # cronjob
if self.last_run_at is None: # never run
return True
# compare datetime in utc
next_run_at = self.last_run_at.replace(
tzinfo=datetime.timezone.utc) + datetime.timedelta(
seconds=self.interval_time)
utc_now = datetime.datetime.now(datetime.timezone.utc)
logging.info(f'[composer] item id: {self.id}, '
f'next_run_at: {next_run_at.timestamp()}, '
f'utc_now: {utc_now.timestamp()}')
if next_run_at.timestamp() < utc_now.timestamp():
return True
return False
class Session(db.Model):
__tablename__ = 'session_v2'
__table_args__ = (Index('idx_jti', 'jti'),
default_table_args('This is webconsole session table'))
id = db.Column(db.Integer,
primary_key=True,
autoincrement=True,
comment='session id')
jti = db.Column(db.String(64), comment='JWT jti')
expired_at = db.Column(db.DateTime(timezone=True),
comment='expired time, for db automatically clear')
created_at = db.Column(db.DateTime(timezone=True),
server_default=func.now(),
comment='created at')
class ModelGroup(db.Model):
__tablename__ = 'model_groups_v2'
__table_args__ = (default_table_args('model_groups_v2'))
id = db.Column(db.Integer, primary_key=True, comment='id')
name = db.Column(db.String(255),
comment='name') # can be modified by end-user
created_at = db.Column(db.DateTime(timezone=True),
comment='created_at',
server_default=func.now())
updated_at = db.Column(db.DateTime(timezone=True),
comment='updated_at',
server_default=func.now(),
onupdate=func.now())
deleted_at = db.Column(db.DateTime(timezone=True), comment='deleted_at')
# TODO https://code.byted.org/data/fedlearner_web_console_v2/issues/289
extra = db.Column(db.Text(), comment='extra') # json string
class OptimisticLock(db.Model):
__tablename__ = 'optimistic_lock_v2'
__table_args__ = (
UniqueConstraint('name', name='uniq_name'),
default_table_args('optimistic lock'),
)
id = db.Column(db.Integer,
comment='id',
primary_key=True,
autoincrement=True)
name = db.Column(db.String(255), comment='lock name', nullable=False)
version = db.Column(db.BIGINT, comment='lock version', nullable=False)
created_at = db.Column(db.DateTime(timezone=True),
comment='created at',
server_default=func.now())
updated_at = db.Column(db.DateTime(timezone=True),
comment='updated at',
server_default=func.now(),
onupdate=func.now())
deleted_at = db.Column(db.DateTime(timezone=True), comment='deleted at')
class WorkflowTemplate(db.Model):
__tablename__ = 'template_v2'
__table_args__ = (UniqueConstraint('name', name='uniq_name'),
Index('idx_group_alias', 'group_alias'),
default_table_args('workflow template'))
id = db.Column(db.Integer, primary_key=True, comment='id')
name = db.Column(db.String(255), comment='name')
comment = db.Column('cmt',
db.String(255),
key='comment',
comment='comment')
group_alias = db.Column(db.String(255),
nullable=False,
comment='group_alias')
# max store 16777215 bytes (16 MB)
config = db.Column(db.LargeBinary(16777215),
nullable=False,
comment='config')
is_left = db.Column(db.Boolean, comment='is_left')
editor_info = db.Column(db.LargeBinary(16777215),
comment='editor_info',
default=b'')
kind = db.Column(db.Integer,
comment='template kind') # WorkflowTemplateKind enum
def set_config(self, proto):
self.config = proto.SerializeToString()
def set_editor_info(self, proto):
self.editor_info = proto.SerializeToString()
def get_config(self):
proto = workflow_definition_pb2.WorkflowDefinition()
proto.ParseFromString(self.config)
return proto
def get_editor_info(self):
proto = workflow_definition_pb2.WorkflowTemplateEditorInfo()
if self.editor_info is not None:
proto.ParseFromString(self.editor_info)
return proto
class Model(db.Model):
__tablename__ = 'models_v2'
__table_args__ = (default_table_args('model'))
id = db.Column(db.Integer, primary_key=True, comment='id')
name = db.Column(db.String(255), comment='model_name')
version = db.Column(db.Integer, comment='model_version')
parent_id = db.Column(db.String(255), comment='parent_id')
job_name = db.Column(db.String(255), comment='job_name')
type = db.Column(db.Enum(ModelType, native_enum=False), comment='type')
state = db.Column(db.Enum(ModelState, native_enum=False), comment='state')
create_time = db.Column(db.DateTime(timezone=True), comment='create_time')
params = db.Column(db.Text(), comment='params')
metrics = db.Column(db.Text(), comment='metrics')
output_base_dir = db.Column(db.String(255),
comment='model checkpoint/export path')
parent = db.relationship('Model',
primaryjoin=remote(id) == foreign(parent_id),
backref='children')
job = db.relationship('Job', primaryjoin=Job.name == foreign(job_name))
def __init__(self):
self.create_time = datetime.now()
self.version = 0
def commit(self):
db.session.add(self)
db.session.commit()
def get_eval_model(self):
"""
Get the evaluation model inherited model
Returns:
a list of evaluation model
"""
eval_model = [
child for child in self.children if child.type in
[ModelType.NN_EVALUATION, ModelType.TREE_EVALUATION]
]
return eval_model
class User(db.Model):
__tablename__ = 'users_v2'
__table_args__ = (UniqueConstraint('username', name='uniq_username'),
default_table_args('This is webconsole user table'))
id = db.Column(db.Integer, primary_key=True, comment='user id')
username = db.Column(db.String(255), comment='unique name of user')
password = db.Column(db.String(255), comment='user password after encode')
role = db.Column(db.Enum(Role, native_enum=False),
default=Role.USER,
comment='role of user')
name = db.Column(db.String(255), comment='name of user')
email = db.Column(db.String(255), comment='email of user')
state = db.Column(db.Enum(State, native_enum=False),
default=State.ACTIVE,
comment='state of user')
def set_password(self, password):
self.password = pwd_context.hash(password)
def verify_password(self, password):
return pwd_context.verify(password, self.password)
class Model(db.Model):
__tablename__ = 'models_v2'
__table_args__ = (Index('idx_job_name', 'job_name'),
UniqueConstraint('job_name', name='uniq_job_name'),
default_table_args('model'))
id = db.Column(db.Integer, primary_key=True, comment='id')
name = db.Column(db.String(255),
comment='name') # can be modified by end-user
version = db.Column(db.Integer, default=0, comment='version')
type = db.Column(db.Integer, comment='type')
state = db.Column(db.Integer, comment='state')
job_name = db.Column(db.String(255), comment='job_name')
parent_id = db.Column(db.Integer, comment='parent_id')
params = db.Column(db.Text(), comment='params')
metrics = db.Column(db.Text(), comment='metrics')
created_at = db.Column(db.DateTime(timezone=True),
comment='created_at',
server_default=func.now())
updated_at = db.Column(db.DateTime(timezone=True),
comment='updated_at',
server_default=func.now(),
onupdate=func.now())
deleted_at = db.Column(db.DateTime(timezone=True), comment='deleted_at')
group_id = db.Column(db.Integer, default=0, comment='group_id')
# TODO https://code.byted.org/data/fedlearner_web_console_v2/issues/289
extra = db.Column(db.Text(), comment='extra') # json string
parent = db.relationship('Model',
primaryjoin=remote(id) == foreign(parent_id),
backref='children')
job = db.relationship('Job', primaryjoin=Job.name == foreign(job_name))
def get_eval_model(self):
return [
child for child in self.children if child.type in
[ModelType.NN_EVALUATION.value, ModelType.TREE_EVALUATION.value]
]
class SchedulerRunner(db.Model):
__tablename__ = 'scheduler_runner_v2'
__table_args__ = (default_table_args('scheduler runners'))
id = db.Column(db.Integer,
comment='id',
primary_key=True,
autoincrement=True)
item_id = db.Column(db.Integer, comment='item id', nullable=False)
status = db.Column(db.Integer,
comment='runner status',
nullable=False,
default=RunnerStatus.INIT.value)
start_at = db.Column(db.DateTime(timezone=True),
comment='runner start time')
end_at = db.Column(db.DateTime(timezone=True), comment='runner end time')
pipeline = db.Column(db.Text(),
comment='pipeline from scheduler item',
nullable=False,
default='{}')
output = db.Column(db.Text(),
comment='output',
nullable=False,
default='{}')
context = db.Column(db.Text(),
comment='context',
nullable=False,
default='{}')
extra = db.Column(db.Text(), comment='extra info')
created_at = db.Column(db.DateTime(timezone=True),
comment='created at',
server_default=func.now())
updated_at = db.Column(db.DateTime(timezone=True),
comment='updated at',
server_default=func.now(),
onupdate=func.now())
deleted_at = db.Column(db.DateTime(timezone=True), comment='deleted at')