class UserRoleBinding(BaseModelMixin):
__tablename__ = "user_role_binding"
__table_args__ = (db.UniqueConstraint('username',
'role_id',
name='unique_user_role'), )
username = db.Column(db.CHAR(128), nullable=False)
role_id = db.Column(db.Integer,
db.ForeignKey('role.id', ondelete='CASCADE'),
nullable=False)
@classmethod
def create(cls, username, role):
ur = cls(username=username, role_id=role.id)
db.session.add(ur)
db.session.commit()
return ur
def __str__(self):
return "UserRoleBinding: {} -> {}".format(self.username, self.role)
@classmethod
def get_roles_by_name(cls, username):
l = cls.query.filter_by(username=username)
return [binding.role for binding in l]
class Release(BaseModelMixin):
__table_args__ = (
db.UniqueConstraint('app_id', 'tag'),
)
# git tag
tag = db.Column(db.CHAR(64), nullable=False, index=True)
# TODO use ForeignKey
app_id = db.Column(db.Integer, nullable=False)
image = db.Column(db.CHAR(255), nullable=False, default='')
build_status = db.Column(db.Boolean, nullable=False, default=False)
specs_text = db.Column(db.Text)
# store trivial info like branch, author, git tag, commit messages
misc = db.Column(db.Text)
def __str__(self):
return '<{r.appname}:{r.tag}>'.format(r=self)
@classmethod
def create(cls, app, tag, specs_text, image=None, build_status=False, branch='', author='', commit_message=''):
"""app must be an App instance"""
appname = app.name
# check the format of specs text(ignore the result)
app_specs_schema.load(yaml.safe_load(specs_text))
misc = {
'author': author,
'commit_message': commit_message,
'git': app.git,
}
try:
new_release = cls(tag=tag, app_id=app.id, image=image, build_status=build_status, specs_text=specs_text, misc=json.dumps(misc))
db.session.add(new_release)
db.session.commit()
except IntegrityError:
logger.warn('Fail to create Release %s %s, duplicate', appname, tag)
db.session.rollback()
raise
return new_release
def update(self, specs_text, image=None, build_status=False, branch='', author='', commit_message=''):
"""app must be an App instance"""
# check the format of specs text(ignore the result)
app_specs_schema.load(yaml.safe_load(specs_text))
misc = {
'author': author,
'commit_message': commit_message,
'git': self.git,
}
try:
# self.specs_text = specs_text
super(Release, self).update(specs_text=specs_text, image=image, build_status=build_status, misc=json.dumps(misc))
except:
logger.warn('Fail to update Release %s %s', self.appname, self.tag)
db.session.rollback()
# raise
return self
def delete(self):
logger.warn('Deleting release %s', self)
return super(Release, self).delete()
@classmethod
def get(cls, id):
r = super(Release, cls).get(id)
if r and r.app:
return r
return None
@classmethod
def get_by_app(cls, name, start=0, limit=100):
app = App.get_by_name(name)
if not app:
return []
q = cls.query.filter_by(app_id=app.id).order_by(cls.id.desc())
return q[start:start + limit]
@classmethod
def get_by_app_and_tag(cls, name, tag):
app = App.get_by_name(name)
if not app:
raise ValueError('app {} not found'.format(name))
return cls.query.filter_by(app_id=app.id, tag=tag).first()
@property
def raw(self):
"""if no builds clause in app.yaml, this release is considered raw"""
return not self.specs.builds
@property
def app(self):
return App.get(self.app_id)
@property
def appname(self):
return self.app.name
@property
def commit_message(self):
misc = json.loads(self.misc)
return misc.get('commit_message')
@property
def author(self):
misc = json.loads(self.misc)
return misc.get('author')
@property
def git(self):
misc = json.loads(self.misc)
return misc.get('git')
@cached_property
def specs(self):
dic = yaml.safe_load(self.specs_text)
unmarshal_result = app_specs_schema.load(dic)
return unmarshal_result.data
@cached_property
def specs_dict(self):
return yaml.safe_load(self.specs_text)
@property
def service(self):
return self.specs.service
def update_build_status(self, status):
try:
self.build_status = status
db.session.add(self)
db.session.commit()
except StaleDataError:
db.session.rollback()
class DeployVersion(BaseModelMixin):
# git tag
tag = db.Column(db.CHAR(64), nullable=False, index=True)
app_id = db.Column(db.Integer, nullable=False)
parent_id = db.Column(db.Integer, nullable=False)
cluster = db.Column(db.CHAR(64), nullable=False)
config_id = db.Column(db.Integer)
specs_text = db.Column(db.Text)
yaml_name = db.Column(db.CHAR(128))
def __str__(self):
return 'DeployVersion <{r.appname}:{r.tag}:{r.id}>'.format(r=self)
@classmethod
def create(cls, app, tag, yaml_name, specs_text, parent_id, cluster, config_id=None):
"""app must be an App instance"""
if isinstance(specs_text, Dict):
specs_text = yaml.dump(specs_text.to_dict())
elif isinstance(specs_text, dict):
specs_text = yaml.dump(specs_text)
else:
# check the format of specs text(ignore the result)
app_specs_schema.load(yaml.safe_load(specs_text))
try:
ver = cls(tag=tag, app_id=app.id, parent_id=parent_id, cluster=cluster,
config_id=config_id, yaml_name=yaml_name, specs_text=specs_text)
db.session.add(ver)
db.session.commit()
except IntegrityError:
logger.warn('Fail to create SpecVersion %s %s, duplicate', app.name, tag)
db.session.rollback()
raise
return ver
def delete(self):
logger.warn('Deleting DeployVersion %s', self)
return super(DeployVersion, self).delete()
@classmethod
def get(cls, id):
if isinstance(id, str):
id = int(id)
r = cls.query.get(id)
if r and r.app:
return r
return None
@classmethod
def get_by_app(cls, app, start=0, limit=None):
q = cls.query.filter_by(app_id=app.id).order_by(cls.id.desc())
if limit is None:
return q[start:]
else:
return q[start:start + limit]
@classmethod
def get_previous_version(cls, cur_id, revision):
while revision >= 0:
ver = cls.get(id=cur_id)
cur_id = ver.parent_id
revision -= 1
return cls.get(id=cur_id)
@property
def release(self):
return Release.query.filter_by(app_id=self.app_id, tag=self.tag).first()
@property
def app(self):
return App.get(self.app_id)
@property
def appname(self):
return self.app.name
@cached_property
def specs(self):
dic = yaml.safe_load(self.specs_text)
unmarshal_result = app_specs_schema.load(dic)
return unmarshal_result.data
@cached_property
def app_config(self):
if self.config_id is None:
return None
return AppConfig.get(self.config_id)
def to_k8s_annotation(self):
return {
'deploy_id': self.id,
'app_id': self.app_id,
'release_tag': self.tag,
'config_id': self.config_id,
'cluster': self.cluster,
'yaml_name': self.yaml_name,
}
class App(BaseModelMixin):
__tablename__ = "app"
name = db.Column(db.CHAR(64), nullable=False, unique=True)
git = db.Column(db.String(255), nullable=False)
type = db.Column(db.CHAR(64), nullable=False)
subscribers = db.Column(db.Text())
def __str__(self):
return self.name
@classmethod
def get_or_create(cls, name, git, apptype, subscribers=None):
app = cls.get_by_name(name)
if app:
return app
return cls.create(name, git, apptype, subscribers)
@classmethod
def create(cls, name, git, apptype, subscribers=None):
subscriber_names = None
if subscribers is not None:
subscriber_names = json.dumps([u.username for u in subscribers])
app = cls(name=name, git=git, type=apptype, subscribers=subscriber_names)
db.session.add(app)
db.session.commit()
return app
@classmethod
def get_by_name(cls, name):
return cls.query.filter_by(name=name).first()
@property
def latest_release(self):
return Release.query.filter_by(app_id=self.id).order_by(Release.id.desc()).limit(1).first()
def get_release_by_tag(self, tag):
return Release.query.filter_by(app_id=self.id, tag=tag).first()
@property
def service(self):
release = self.latest_release
return release and release.service
@property
def specs(self):
r = self.latest_release
return r and r.specs
@property
def subscriber_list(self):
if not self.subscribers:
return []
try:
username_list = json.loads(self.subscribers)
except json.JSONDecodeError as e:
logger.exception(f"subscribers of app {self.name} is invalid({self.subscribers})")
return []
from console.models.user import User
users = []
for username in username_list:
user = User.get_by_username(username)
if user is not None:
users.append(user)
return users
def delete(self):
"""
the caller must ensure the all kubernetes objects have been deleted
:return:
"""
# delete all releases
Release.query.filter_by(app_id=self.id).delete()
DeployVersion.query.filter_by(app_id=self.id).delete()
# delete all op log
from console.models.oplog import OPLog
OPLog.delete_by_app_id(self.id)
return super(App, self).delete()
class Role(BaseModelMixin):
__tablename__ = "role"
name = db.Column(db.CHAR(64), nullable=False, unique=True)
# if apps is empty, it means all app
apps = db.relationship('App',
secondary=role_app_association,
backref=db.backref('roles', lazy='dynamic'),
lazy='dynamic')
# actions is a json with the following format:
# ["get", "deploy", "get_config"],
actions = db.Column(db.Text, nullable=False)
# clusters is a json list with the following format:
# ["cluster1", "cluster2", "cluster3"]
# if clusters is an empty list, it mains allows all clusters
clusters = db.Column(db.Text)
users = db.relationship('UserRoleBinding',
cascade="all,delete",
backref='role',
lazy='dynamic')
groups = db.relationship('GroupRoleBinding',
cascade="all,delete",
backref='role',
lazy='dynamic')
def __str__(self):
return self.name
@classmethod
def create(cls, name, apps, actions, clusters=None):
actions_txt, clusters_txt = None, None
if actions:
action_vals = [act.value for act in actions]
actions_txt = json.dumps(action_vals)
if clusters:
clusters_txt = json.dumps(clusters)
r = cls(name=name,
apps=apps,
actions=actions_txt,
clusters=clusters_txt)
try:
db.session.add(r)
db.session.commit()
except IntegrityError:
logger.warn('Fail to create role %s', name)
db.session.rollback()
raise
return r
@classmethod
def get_by_name(cls, name):
return cls.query.filter_by(name=name).first()
@property
def app_names(self):
return [app.name for app in self.apps]
@property
def action_list(self):
try:
actions = str2actions(self.actions)
except AttributeError:
logger.error("invalid action text", self.actions)
return []
if len(actions) == 0:
actions = _all_action_list
return actions
@property
def cluster_list(self):
if not self.clusters:
return get_cluster_names()
clusters = json.loads(self.clusters)
if len(clusters) == 0:
return get_cluster_names()
else:
return clusters
@property
def app_list(self):
apps = self.apps.all()
if len(apps) == 0:
from console.models.app import App
return App.get_all()
else:
return apps
def to_dict(self):
d = {
"name": self.name,
"apps": self.app_names,
"actions": json.loads(self.actions),
"clusters": self.cluster_list,
}
return d
class OPLog(BaseModelMixin):
__tablename__ = 'operation_log'
user_id = db.Column(db.Integer, nullable=False, default=0, index=True)
app_id = db.Column(db.Integer, nullable=False, default=0, index=True)
appname = db.Column(db.CHAR(64), nullable=False, default='', index=True)
tag = db.Column(db.CHAR(64), nullable=False, default='', index=True)
cluster = db.Column(db.CHAR(64), nullable=False, default='')
action = db.Column(Enum34(OPType))
content = db.Column(db.Text)
@classmethod
def get_by(cls, **kwargs):
'''
query operation logs, all fields could be used as query parameters
'''
purge_none_val_from_dict(kwargs)
limit = kwargs.pop('limit', 100)
time_window = kwargs.pop('time_window', None)
filters = [getattr(cls, k) == v for k, v in kwargs.items()]
if time_window:
left, right = time_window
left = left or datetime.min
right = right or datetime.now()
filters.extend([cls.created >= left, cls.created <= right])
return cls.query.filter(sqlalchemy.and_(*filters)).order_by(
cls.id.desc()).limit(limit).all()
@classmethod
def create(cls,
user_id=None,
app_id=None,
appname=None,
tag=None,
action=None,
content=None,
cluster=''):
op_log = cls(user_id=user_id,
app_id=app_id,
cluster=cluster,
appname=appname,
tag=tag,
action=action,
content=content)
db.session.add(op_log)
db.session.commit()
return op_log
@property
def verbose_action(self):
return self.action.name
def to_dict(self):
dic = {
c.key: getattr(self, c.key)
for c in inspect(self).mapper.column_attrs
if c.key not in ('user_id', 'app_id')
}
user = User.get_by_id(self.user_id)
dic['username'] = user.nickname
dic['action'] = self.action.name
return dic
class User(BaseModelMixin):
__tablename__ = "user"
username = db.Column(db.CHAR(50), nullable=False, unique=True)
email = db.Column(db.String(100), nullable=False, unique=True, index=True)
nickname = db.Column(db.CHAR(50), nullable=False)
avatar = db.Column(db.String(2000), nullable=False)
privileged = db.Column(db.Integer, default=0)
data = db.Column(db.Text)
@classmethod
def create(cls, username=None, email=None, nickname=None, avatar=None,
privileged=0, data=None):
if isinstance(data, dict):
data = json.dumps(data)
user = cls(username=username, email=email, nickname=nickname,
avatar=avatar, data=data)
try:
db.session.add(user)
db.session.commit()
except IntegrityError:
db.session.rollback()
raise
return user
def __str__(self):
return '{class_} {u.name} {u.email}'.format(
class_=self.__class__,
u=self,
)
@classmethod
def get_by_username(cls, username):
return cls.query.filter_by(username=username).first()
@classmethod
def get_by_email(cls, email):
return cls.query.filter_by(email=email).first()
@classmethod
def get_by_id(cls, user_id):
return cls.query.filter_by(id=user_id).first()
@classmethod
def set_authlib_user(cls, auth_user):
user = cls.query.filter_by(email=auth_user.email).first()
username = auth_user.preferred_username
nickname = auth_user.nickname
if username is None:
username = auth_user.name
if nickname is None:
nickname = auth_user.name
avatar = auth_user.picture
data = json.dumps(dict(auth_user))
if not user:
user = cls.create(username=username, email=auth_user.email, nickname=nickname, avatar=avatar, data=data)
else:
user.update(username=username, email=auth_user.email, nickname=nickname, avatar=avatar,
data=data)
return user
def granted_to_app(self, app):
if self.privileged:
return True
from console.models.app import App
return self.apps.filter(App.id == app.id).first() is not None
def list_app(self, start=0, limit=500):
from console.models.app import App
if self.privileged:
return App.get_all()[start: start+limit]
return self.apps.all()[start: start+limit]
def list_job(self):
from console.models.job import Job
if self.privileged:
return Job.get_all()
return self.jobs.all()
def granted_to_job(self, job):
if self.privileged:
return True
from console.models.job import Job
return self.jobs.filter(Job.id == job.id).first() is not None
def elevate_privilege(self):
self.privileged = 1
db.session.add(self)
db.session.commit()
def __str__(self):
return self.nickname
class SpecVersion(BaseModelMixin):
# TODO use ForeignKey
# git tag
tag = db.Column(db.CHAR(64), nullable=False, index=True)
app_id = db.Column(db.Integer, nullable=False)
specs_text = db.Column(db.Text)
def __str__(self):
return 'SpecVersion <{r.appname}:{r.tag}:{r.id}>'.format(r=self)
@classmethod
def create(cls, app, tag, specs_text):
"""app must be an App instance"""
if isinstance(specs_text, Dict):
specs_text = yaml.dump(specs_text.to_dict())
elif isinstance(specs_text, dict):
specs_text = yaml.dump(specs_text)
else:
# check the format of specs text(ignore the result)
app_specs_schema.load(yaml.load(specs_text))
try:
new_release = cls(tag=tag, app_id=app.id, specs_text=specs_text)
db.session.add(new_release)
db.session.commit()
except IntegrityError:
logger.warn('Fail to create SpecVersion %s %s, duplicate', app.name, tag)
db.session.rollback()
raise
return new_release
def delete(self):
logger.warn('Deleting release %s', self)
return super(SpecVersion, self).delete()
@classmethod
def get(cls, id):
r = super(SpecVersion, cls).get(id)
if r and r.app:
return r
return None
@classmethod
def get_by_app(cls, app, start=0, limit=None):
q = cls.query.filter_by(app_id=app.id).order_by(cls.id.desc())
if limit is None:
return q[start:]
else:
return q[start:start + limit]
@classmethod
def get_newest_version_by_tag_app(cls, app_id, tag):
return SpecVersion.query.filter_by(app_id=app_id, tag=tag).order_by(SpecVersion.id.desc()).first()
@property
def release(self):
return Release.query.filter_by(app_id=self.app_id, tag=self.tag).first()
@property
def app(self):
return App.get(self.app_id)
@property
def appname(self):
return self.app.name
@cached_property
def specs(self):
dic = yaml.load(self.specs_text)
unmarshal_result = app_specs_schema.load(dic)
return unmarshal_result.data
class App(BaseModelMixin):
__tablename__ = "app"
name = db.Column(db.CHAR(64), nullable=False, unique=True)
git = db.Column(db.String(255), nullable=False)
type = db.Column(db.CHAR(64), nullable=False)
users = db.relationship('User', secondary=app_user_association,
backref=db.backref('apps', lazy='dynamic'), lazy='dynamic')
def __str__(self):
return self.name
@classmethod
def get_or_create(cls, name, git, apptype):
app = cls.get_by_name(name)
if app:
return app
app = cls(name=name, git=git, type=apptype)
db.session.add(app)
db.session.commit()
return app
@classmethod
def get_by_name(cls, name):
return cls.query.filter_by(name=name).first()
def grant_user(self, user):
from console.models.user import User
if self.users.filter(User.id == user.id).first() is None:
self.users.append(user)
db.session.add(self)
db.session.commit()
def revoke_user(self, user):
self.users.remove(user)
db.session.add(self)
db.session.commit()
def list_users(self):
return self.users.all()
@property
def latest_release(self):
return Release.query.filter_by(app_id=self.id).order_by(Release.id.desc()).limit(1).first()
def get_release_by_tag(self, tag):
return Release.query.filter_by(app_id=self.id, tag=tag).first()
@property
def service(self):
release = self.latest_release
return release and release.service
@property
def specs(self):
r = self.latest_release
return r and r.specs
@property
def subscribers(self):
specs = self.specs
return specs and specs.subscribers
def delete(self):
"""
the caller must ensure the all kubernetes objects have been deleted
:return:
"""
appname = self.name
# delete all releases
Release.query.filter_by(app_id=self.id).delete()
SpecVersion.query.filter_by(app_id=self.id).delete()
return super(App, self).delete()
class Job(BaseModelMixin):
__tablename__ = "job"
name = db.Column(db.CHAR(64), nullable=False, unique=True)
git = db.Column(db.String(255), nullable=False, default='')
branch = db.Column(db.String(255), nullable=False, default='')
commit = db.Column(db.String(255), nullable=False, default='')
specs_text = db.Column(db.Text)
nickname = db.Column(db.String(64), nullable=False)
comment = db.Column(db.Text)
version = db.Column(db.Integer, nullable=False, default=0)
status = db.Column(db.String(64), nullable=False, default='')
users = db.relationship('User',
secondary=job_user_association,
backref=db.backref('jobs', lazy='dynamic'),
lazy='dynamic')
def __str__(self):
return '<{}:{}>'.format(self.name, self.git)
@classmethod
def get_or_create(cls,
name,
git=None,
branch=None,
commit=None,
specs_text=None,
comment=None,
status=None):
job = cls.get_by_name(name)
if job:
return job
return cls.create(name=name,
git=git,
branch=branch,
commit=commit,
specs_text=specs_text,
comment=comment,
status=status)
@classmethod
def create(cls,
name,
git=None,
branch=None,
commit=None,
specs_text=None,
comment=None,
status=None):
try:
job = cls(name=name,
git=git,
branch=branch,
commit=commit,
specs_text=specs_text,
nickname=g.user.nickname,
comment=comment,
status=status)
db.session.add(job)
db.session.commit()
except IntegrityError as e:
logger.warn('Fail to create Job %s %s, duplicate', name)
db.session.rollback()
raise e
return job
def update_status(self, status):
try:
self.status = status
db.session.add(self)
db.session.commit()
except StaleDataError:
db.session.rollback()
except Exception:
db.session.rollback()
def inc_version(self):
self.version += 1
try:
db.session.add(self)
db.session.commit()
except StaleDataError:
db.session.rollback()
except Exception:
db.session.rollback()
@classmethod
def get_by_name(cls, name):
return cls.query.filter_by(name=name).first()
def grant_user(self, user):
if user.granted_to_job(self):
return
self.users.append(user)
db.session.add(self)
db.session.commit()
def revoke_user(self, user):
self.users.remove(user)
db.session.add(self)
db.session.commit()
def list_users(self):
return self.users.all()
@cached_property
def specs(self):
dic = yaml.load(self.specs_text)
return load_job_specs(dic)
def delete(self):
"""
the caller must ensure the all kubernetes objects have been deleted
:return:
"""
return super(Job, self).delete()
