class OPLog(BaseModelMixin):
__tablename__ = 'operation_log'
container_id = db.Column(db.CHAR(64), nullable=False, default='', index=True)
zone = db.Column(db.CHAR(64), nullable=False, default='', index=True)
user_id = db.Column(db.Integer, nullable=False, default=0, index=True)
appname = db.Column(db.CHAR(64), nullable=False, default='', index=True)
sha = db.Column(db.CHAR(64), nullable=False, default='', index=True)
action = db.Column(Enum34(OPType))
content = db.Column(db.JSON)
@classmethod
def get_by(cls, **kwargs):
'''
query operation logs, all fields could be used as query parameters
'''
purge_none_val_from_dict(kwargs)
container_id = kwargs.pop('container_id', None)
sha = kwargs.pop('sha', None)
limit = kwargs.pop('limit', 200)
time_window = kwargs.pop('time_window', None)
filters = [getattr(cls, k)==v for k, v in kwargs.items()]
if container_id:
if len(container_id) < 7:
raise ValueError('minimum container_id length is 7')
filters.append(cls.container_id.like('{}%'.format(container_id)))
if sha:
if len(sha) < 7:
raise ValueError('minimum sha length is 7')
filters.append(cls.sha.like('{}%'.format(sha)))
if time_window:
left, right = time_window
left = left or datetime.min
right = right or datetime.now()
filters.extend([cls.created >= left, cls.created <= right])
return cls.query.filter(sqlalchemy.and_(*filters)).order_by(cls.id.desc()).limit(limit).all()
@classmethod
def create(cls, zone=None, container_id=None, user_id=None, appname=None,
sha=None, action=None, content=None):
op_log = cls(container_id=container_id, zone=zone, user_id=user_id,
appname=appname, sha=sha, action=action, content=content)
db.session.add(op_log)
db.session.commit()
return op_log
@property
def verbose_action(self):
return self.action.name
@property
def short_sha(self):
return self.sha and self.sha[:7]
class ELBInstance(BaseModelMixin, Jsonized):
"""name 相同的 ELBInstance 组成一个 ELB, ELB 是一个虚拟的概念"""
__tablename__ = 'elb'
addr = db.Column(db.String(128), nullable=False)
container_id = db.Column(db.CHAR(64), nullable=False, index=True)
name = db.Column(db.String(64))
zone = db.Column(db.String(50), nullable=False)
@classmethod
def create(cls, addr, container_id, name):
container = Container.get_by_container_id(container_id)
ins = super(ELBInstance, cls).create(addr=addr,
container_id=container_id,
zone=container.zone, name=name)
return ins
@classmethod
def get_by_zone(cls, zone):
return cls.query.filter_by(zone=zone).order_by(cls.id.desc()).all()
def clear_rules(self):
"""clear rules in the whole ELB"""
rules = ELBRuleSet.query.filter_by(elbname=self.name).all()
domains = [r.domain for r in rules]
for rule in rules:
elbset = rule.get_elbset()
elbset.delete_domain_rules(domains)
for rule in rules:
rule.delete()
@classmethod
def get_by(cls, **kwargs):
container_id = kwargs.pop('container_id', None)
query_set = cls.query.filter_by(**purge_none_val_from_dict(kwargs))
if container_id:
if len(container_id) < 7:
raise ValueError('Container ID too short: {}'.format(container_id))
query_set = query_set.filter(cls.container_id.like('{}%'.format(container_id)))
res = query_set.order_by(cls.id.desc()).all()
return res
@cached_property
def elb(self):
return get_elb_client(self.name, self.zone)
@property
def container(self):
return Container.get_by_container_id(self.container_id)
@property
def address(self):
_, address = self.container.publish.popitem()
return address
def is_alive(self):
return self.container and self.container.is_healthy()
class AppUserRelation(BaseModelMixin):
__table_args__ = (db.UniqueConstraint('user_id', 'appname'), )
appname = db.Column(db.CHAR(64), nullable=False, index=True)
user_id = db.Column(db.Integer, nullable=False)
@classmethod
def create(cls, app, user):
relation = cls(appname=app.name, user_id=user.id)
try:
db.session.add(relation)
db.session.commit()
return relation
except IntegrityError:
db.session.rollback()
raise
class Combo(BaseModelMixin):
__table_args__ = (db.UniqueConstraint('appname', 'name'), )
appname = db.Column(db.CHAR(64), nullable=False, index=True)
name = db.Column(db.CHAR(64), nullable=False, index=True)
entrypoint_name = db.Column(db.CHAR(64), nullable=False)
podname = db.Column(db.CHAR(64), nullable=False)
nodename = db.Column(db.CHAR(64))
extra_args = db.Column(db.String(100))
networks = db.Column(db.JSON) # List of network names
cpu_quota = db.Column(db.Float, nullable=False)
memory = db.Column(db.Integer, nullable=False)
count = db.Column(db.Integer, default=1)
envname = db.Column(db.CHAR(64))
debug = db.Column(db.Integer, default=0)
def __str__(self):
return '<{} combo:{}>'.format(self.appname, self.name)
@classmethod
def create(cls,
appname=None,
name=None,
entrypoint_name=None,
podname=None,
nodename=None,
extra_args=None,
networks=None,
cpu_quota=None,
memory=None,
count=None,
envname=None,
debug=None):
try:
combo = cls(appname=appname,
name=name,
entrypoint_name=entrypoint_name,
podname=podname,
nodename=nodename,
extra_args=extra_args,
networks=networks,
cpu_quota=cpu_quota,
memory=memory,
count=count,
envname=envname,
debug=debug)
db.session.add(combo)
db.session.commit()
return combo
except IntegrityError:
db.session.rollback()
raise
class ELBRuleSet(BaseModelMixin, Jsonized):
"""(app, pod, entrypoint)的容器对elbname应用规则,
规则包括(domain, arguments)"""
__tablename__ = 'elb_rule_set'
__table_args__ = (
Index('idx_app_pod_entry', 'appname', 'podname', 'entrypoint'),
Index('idx_elb_zone', 'elbname', 'zone'),
)
appname = db.Column(db.CHAR(64), default='')
podname = db.Column(db.String(50), default='')
entrypoint = db.Column(db.String(50), default='')
elbname = db.Column(db.String(100), default='')
zone = db.Column(db.String(50), default='')
domain = db.Column(db.String(100), default='')
arguments = db.Column(db.JSON)
@classmethod
def create(cls, appname, podname, entrypoint, elbname, zone, domain, arguments):
# 检查下这个arguments合法不
build_elb_ruleset(arguments)
return super(ELBRuleSet, cls).create(appname=appname, podname=podname,
entrypoint=entrypoint,
elbname=elbname, zone=zone,
domain=domain,
arguments=arguments)
def to_elbruleset(self):
return build_elb_ruleset(self.arguments)
def get_backend_rule(self):
elbruleset = self.to_elbruleset()
return [r for r in elbruleset.rules if isinstance(r, BackendRule)]
def get_elbset(self):
return get_elb_client(self.elbname, self.zone)
class User(BaseModelMixin):
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.CHAR(50), nullable=False)
email = db.Column(db.String(100), nullable=False)
access_token = db.Column(db.CHAR(60), nullable=True, index=True)
privileged = db.Column(db.Integer, default=0)
data = db.Column(db.JSON)
@classmethod
def create(cls,
id=None,
name=None,
email=None,
access_token=None,
privileged=0,
data=None):
user = cls(id=id,
name=name,
email=email,
data=data,
access_token=access_token)
try:
db.session.add(user)
db.session.commit()
except IntegrityError:
db.session.rollback()
raise
return user
def __str__(self):
return '{class_} {u.id} {u.name}'.format(
class_=self.__class__,
u=self,
)
@classmethod
def get_by_access_token(cls, access_token):
if not access_token:
# access_token could be missing so this method is expected to be
# called with access_token=None a lot, better check this before
# initiating database query
return None
return cls.query.filter_by(access_token=access_token).first()
@classmethod
def get_by_name(cls, name):
return cls.query.filter_by(name=name).first()
@classmethod
def set_authlib_user(cls, auth_user):
user = cls.query.filter_by(id=auth_user.sub).first()
token = fetch_token(OAUTH_APP_NAME)
access_token = token.get('access_token')
if not user:
user = cls.create(auth_user.sub,
auth_user.name,
auth_user.email,
access_token,
data=dict(auth_user))
else:
user.update(name=auth_user.name,
email=auth_user.email,
data=dict(auth_user),
access_token=access_token)
return user
def granted_to_app(self, app):
if self.privileged:
return True
from citadel.models.app import AppUserRelation
r = AppUserRelation.query.filter_by(appname=app.name,
user_id=self.id).all()
return bool(r)
def list_app(self):
from citadel.models.app import AppUserRelation, App
if self.privileged:
return App.get_all()
rs = AppUserRelation.query.filter_by(user_id=self.id)
return [App.get_by_name(r.appname) for r in rs]
def to_dict(self):
return {
c.key: getattr(self, c.key)
for c in inspect(self).mapper.column_attrs
if c.key != 'access_token'
}
def elevate_privilege(self):
self.privileged = 1
db.session.add(self)
db.session.commit()
class App(BaseModelMixin):
name = db.Column(db.CHAR(64), nullable=False, unique=True)
# 形如 [email protected]:platform/apollo.git
git = db.Column(db.String(255), nullable=False)
tackle_rule = db.Column(db.JSON)
# {'prod': {'PASSWORD': '******'}, 'test': {'PASSWORD': '******'}}
env_sets = db.Column(db.JSON, default={})
def __str__(self):
return '<{}:{}>'.format(self.name, self.git)
@classmethod
def get_or_create(cls, name, git=None, tackle_rule=None):
app = cls.get_by_name(name)
if app:
return app
tackle_rule = tackle_rule if tackle_rule else {}
app = cls(name=name, git=git, tackle_rule=tackle_rule)
db.session.add(app)
db.session.commit()
return app
@classmethod
def get_by_name(cls, name):
return cls.query.filter_by(name=name).first()
@classmethod
def get_by_user(cls, user_id):
"""拿这个user可以有的app, 跟app自己的user_id没关系."""
names = AppUserRelation.get_appname_by_user_id(user_id)
return [cls.get_by_name(n) for n in names]
@classmethod
def get_apps_with_tackle_rule(cls):
return cls.query.filter(cls.tackle_rule != {}).all()
def get_combos(self):
return Combo.query.filter_by(appname=self.name).all()
def create_combo(self, **kwargs):
kwargs['appname'] = self.name
return Combo.create(**kwargs)
def get_combo(self, combo_name):
return Combo.query.filter_by(appname=self.name,
name=combo_name).first()
def delete_combo(self, combo_name):
return Combo.query.filter_by(appname=self.name,
name=combo_name).delete()
def grant_user(self, user):
AppUserRelation.create(self, user)
def revoke_user(self, user):
AppUserRelation.query.filter_by(appname=self.name,
user_id=user.id).delete()
db.session.commit()
def list_users(self):
from citadel.models.user import User
user_ids = [
r.user_id
for r in AppUserRelation.filter_by(appname=self.name).all()
]
users = [User.get(id_) for id_ in user_ids]
return users
def get_env_sets(self):
return self.env_sets
def get_env_set(self, envname):
env_sets = self.env_sets
return EnvSet(env_sets.get(envname, {}))
def add_env_set(self, envname, data):
if envname in self.env_sets:
raise ValueError(
'{} already exists, use update API'.format(envname))
self.update_env_set(envname, data)
def update_env_set(self, envname, data):
env_set = EnvSet(**data)
env_sets = self.env_sets.copy()
env_sets[envname] = env_set
self.env_sets = env_sets
logger.debug('Update env set %s for %s, full env_sets: %s', envname,
self.name, env_sets)
db.session.add(self)
db.session.commit()
def remove_env_set(self, envname):
env_sets = self.env_sets.copy()
env = env_sets.pop(envname, None)
if env:
self.env_sets = env_sets
db.session.add(self)
db.session.commit()
return bool(env)
@property
def latest_release(self):
return Release.query.filter_by(app_id=self.id).order_by(
Release.id.desc()).limit(1).first()
@property
def entrypoints(self):
release = self.latest_release
return release and release.entrypoints
@property
def specs(self):
r = self.latest_release
return r and r.specs
@property
def subscribers(self):
specs = self.specs
return specs and specs.subscribers
@property
def cronjob_entrypoints(self):
# FIXME
specs = self.specs
return tuple(t[1] for t in specs.crontab)
def get_container_list(self, zone=None):
from .container import Container
return Container.get_by(appname=self.name, zone=zone)
def has_problematic_container(self, zone=None):
containers = self.get_container_list(zone)
if not containers or {c.status() for c in containers} == {'running'}:
return False
return True
@property
def app_status_assembler(self):
return AppStatusAssembler(self.name)
def update_tackle_rule(self, rule):
"""
{
"container_tackle_rule": [
{
"strategy": "respawn",
"situations": ["(healthy == 0) * 2m"],
"kwargs": {
"floor": 2,
"celling": 8
}
}
]
}
"""
if isinstance(rule, str):
rule = json.loads(rule)
self.tackle_rule = rule
db.session.add(self)
db.session.commit()
def get_release(self, sha):
return Release.get_by_app_and_sha(self.name, sha)
def delete(self):
appname = self.name
containers = self.get_container_list(None)
if containers:
raise ModelDeleteError(
'App {} is still running, containers {}, remove them before deleting app'
.format(appname, containers))
# delete all releases
Release.query.filter_by(app_id=self.id).delete()
# delete all permissions
AppUserRelation.query.filter_by(appname=appname).delete()
# TODO: delete all ELB rules
return super(App, self).delete()
def get_associated_elb_rules(self, zone=DEFAULT_ZONE):
# TODO
pass
class Release(BaseModelMixin):
__table_args__ = (db.UniqueConstraint('app_id', 'sha'), )
sha = db.Column(db.CHAR(64), nullable=False, index=True)
app_id = db.Column(db.Integer, nullable=False)
image = db.Column(db.String(255), nullable=False, default='')
specs_text = db.Column(db.JSON)
# store trivial info like branch, author, git tag, commit messages
misc = db.Column(db.JSON)
def __str__(self):
return '<{r.appname}:{r.short_sha}>'.format(r=self)
@classmethod
def create(cls,
app,
sha,
specs_text=None,
branch='',
git_tag='',
author='',
commit_message='',
git=''):
"""app must be an App instance"""
appname = app.name
unmarshal_result = specs_schema.load(yaml.load(specs_text))
misc = {
'git_tag': git_tag,
'author': author,
'commit_message': commit_message,
}
try:
new_release = cls(sha=sha,
app_id=app.id,
specs_text=specs_text,
misc=misc)
db.session.add(new_release)
db.session.commit()
except IntegrityError:
logger.warn('Fail to create Release %s %s, duplicate', appname,
sha)
db.session.rollback()
raise
return new_release
def delete(self):
container_list = self.get_container_list()
if container_list:
raise ModelDeleteError(
'Release {} is still running, delete containers {} before deleting this release'
.format(self.short_sha, container_list))
logger.warn('Deleting release %s', self)
return super(Release, self).delete()
@classmethod
def get(cls, id):
r = super(Release, cls).get(id)
# 要检查下 app 还在不在, 不在就失败吧
if r and r.app:
return r
return None
@classmethod
def get_by_app(cls, name, start=0, limit=None):
app = App.get_by_name(name)
if not app:
return []
q = cls.query.filter_by(app_id=app.id).order_by(cls.id.desc())
return q[start:start + limit]
@classmethod
def get_by_app_and_sha(cls, name, sha):
app = App.get_by_name(name)
if not app:
raise ValueError('app {} not found'.format(name))
if len(sha) < 7:
raise ValueError('minimum sha length is 7')
return cls.query.filter(cls.app_id == app.id,
cls.sha.like('{}%'.format(sha))).first()
@property
def raw(self):
"""if no builds clause in app.yaml, this release is considered raw"""
return not self.specs.stages
@property
def short_sha(self):
return self.sha[:7]
@property
def app(self):
return App.get(self.app_id)
@property
def appname(self):
return self.app.name
def get_container_list(self, zone=None):
from .container import Container
return Container.get_by(appname=self.appname, sha=self.sha, zone=zone)
@property
def git_tag(self):
return self.misc.get('git_tag')
@property
def commit_message(self):
return self.misc.get('commit_message')
@property
def author(self):
return self.misc.get('author')
@property
def git(self):
return self.misc.get('git')
@cached_property
def specs(self):
dic = yaml.load(self.specs_text)
unmarshal_result = specs_schema.load(dic)
return unmarshal_result.data
@property
def entrypoints(self):
return self.specs.entrypoints
def update_image(self, image):
try:
self.image = image
logger.debug('Set image %s for release %s', image, self.sha)
db.session.add(self)
db.session.commit()
except StaleDataError:
db.session.rollback()
def make_core_deploy_options(self, combo_name):
combo = Combo.query.filter_by(appname=self.appname,
name=combo_name).first()
entrypoint_name = combo.entrypoint_name
specs = self.specs
entrypoint = specs.entrypoints[entrypoint_name]
# TODO: extra hosts support
# TODO: wtf is meta
# TODO: wtf is nodelabels
hook = entrypoint.hook
if hook:
hook_opt = pb.HookOptions(after_start=hook.after_start,
before_stop=hook.before_stop,
force=hook.force)
else:
hook_opt = None
healthcheck = entrypoint.healthcheck
healthcheck_opt = pb.HealthCheckOptions(
tcp_ports=[str(p) for p in healthcheck.tcp_ports],
http_port=str(healthcheck.http_port),
url=healthcheck.http_url,
code=healthcheck.http_code)
entrypoint_opt = pb.EntrypointOptions(
name=entrypoint_name,
command=entrypoint.command,
privileged=entrypoint.privileged,
dir=entrypoint.working_dir,
log_config=entrypoint.log_config,
publish=entrypoint.publish,
healthcheck=healthcheck_opt,
hook=hook_opt,
restart_policy=entrypoint.restart)
app = self.app
env_set = app.get_env_set(combo.envname)
networks = {network_name: '' for network_name in combo.networks}
deploy_opt = pb.DeployOptions(name=specs.name,
entrypoint=entrypoint_opt,
podname=combo.podname,
nodename=combo.nodename,
image=self.image,
extra_args=combo.extra_args,
cpu_quota=combo.cpu_quota,
memory=combo.memory,
count=combo.count,
env=env_set.to_env_vars(),
dns=specs.dns,
extra_hosts=specs.hosts,
volumes=specs.volumes,
networks=networks,
networkmode=entrypoint.network_mode,
user=specs.container_user,
debug=combo.debug)
return deploy_opt
def make_core_build_options(self):
specs = self.specs
app = self.app
builds_map = {
stage_name: pb.Build(**build)
for stage_name, build in specs.builds.items()
}
core_builds = pb.Builds(stages=specs.stages, builds=builds_map)
container_user = specs.container_user if self.raw else app.name
opts = pb.BuildImageOptions(name=app.name,
user=container_user,
uid=app.id,
tag=self.short_sha,
builds=core_builds)
return opts
class Container(BaseModelMixin):
__table_args__ = (
db.Index('appname_sha', 'appname', 'sha'),
)
appname = db.Column(db.CHAR(64), nullable=False)
sha = db.Column(db.CHAR(64), nullable=False)
container_id = db.Column(db.CHAR(64), nullable=False, index=True)
container_name = db.Column(db.CHAR(64), nullable=False, index=True)
combo_name = db.Column(db.CHAR(64), nullable=False)
entrypoint_name = db.Column(db.String(50), nullable=False)
envname = db.Column(db.String(50))
cpu_quota = db.Column(db.Numeric(12, 3), nullable=False)
memory = db.Column(db.BigInteger, nullable=False)
zone = db.Column(db.String(50), nullable=False)
podname = db.Column(db.String(50), nullable=False)
nodename = db.Column(db.String(50), nullable=False)
deploy_info = db.Column(db.JSON, default={})
override_status = db.Column(db.Integer, default=ContainerOverrideStatus.NONE)
initialized = db.Column(db.Integer, default=0)
def __str__(self):
return '<Container {c.zone}:{c.appname}:{c.short_sha}:{c.entrypoint_name}:{c.short_id}>'.format(c=self)
@classmethod
def create(cls, appname=None, sha=None, container_id=None,
container_name=None, combo_name=None, entrypoint_name=None,
envname=None, cpu_quota=None, memory=None, zone=None,
podname=None, nodename=None,
override_status=ContainerOverrideStatus.NONE):
try:
c = cls(appname=appname, sha=sha, container_id=container_id,
container_name=container_name, combo_name=combo_name,
entrypoint_name=entrypoint_name, envname=envname,
cpu_quota=cpu_quota, memory=memory, zone=zone,
podname=podname, nodename=nodename,
override_status=override_status)
db.session.add(c)
db.session.commit()
except IntegrityError:
db.session.rollback()
# TODO: This must not go wrong!
raise
return c
@classmethod
def get_by_container_id(cls, container_id):
"""get by container_id, prefix can be used in container_id"""
if len(container_id or '') < 7:
raise ValueError('Must provide full container ID, got {}'.format(container_id))
c = cls.query.filter(cls.container_id.like('{}%'.format(container_id))).first()
return c
@classmethod
def get_by_container_ids(cls, container_ids):
containers = [cls.get_by_container_id(cid) for cid in container_ids]
return [c for c in containers if c]
@property
def core_deploy_key(self):
return '{prefix}/{c.appname}/{c.entrypoint_name}/{c.nodename}/{c.container_id}'.format(c=self, prefix=CORE_DEPLOY_INFO_PATH)
def is_healthy(self):
return self.deploy_info.get('Healthy')
@property
def app(self):
from .app import App
return App.get_by_name(self.appname)
@property
def release(self):
from .app import Release
return Release.get_by_app_and_sha(self.appname, self.sha)
@classmethod
def get_by(cls, **kwargs):
sha = kwargs.pop('sha', '')
container_id = kwargs.pop('container_id', '')
query_set = cls.query.filter_by(**purge_none_val_from_dict(kwargs))
if sha:
query_set = query_set.filter(cls.sha.like('{}%'.format(sha)))
if container_id:
query_set = query_set.filter(cls.container_id.like('{}%'.format(container_id)))
return query_set.order_by(cls.id.desc()).all()
@property
def specs_entrypoint(self):
return self.release.specs.entrypoints[self.entrypoint_name]
@property
def backup_path(self):
return self.specs_entrypoint.backup_path
@property
def publish(self):
return self.deploy_info.get('Publish', {})
@property
def ident(self):
return self.container_name.rsplit('_', 2)[-1]
@property
def short_id(self):
return self.container_id[:7]
@property
def short_sha(self):
return self.sha[:7]
def is_removing(self):
return self.override_status == ContainerOverrideStatus.REMOVING
def is_cronjob(self):
return self.entrypoint_name in self.app.cronjob_entrypoints
def is_debug(self):
return self.override_status == ContainerOverrideStatus.DEBUG
def mark_debug(self):
self.override_status = ContainerOverrideStatus.DEBUG
try:
db.session.add(self)
db.session.commit()
except StaleDataError:
db.session.rollback()
def mark_removing(self):
self.override_status = ContainerOverrideStatus.REMOVING
try:
db.session.add(self)
db.session.commit()
except StaleDataError:
db.session.rollback()
def mark_initialized(self):
self.initialized = 1
db.session.add(self)
db.session.commit()
def update_deploy_info(self, deploy_info):
logger.debug('Update deploy_info for %s: %s', self, deploy_info)
self.deploy_info = deploy_info
db.session.add(self)
db.session.commit()
def wait_for_erection(self, timeout=None):
"""wait until this container is healthy, timeout can be timedelta or
seconds, timeout default to erection_timeout in specs, if timeout is 0,
don't even wait and just report healthy"""
if not timeout:
timeout = timedelta(seconds=self.release.specs.erection_timeout)
elif isinstance(timeout, Number):
timeout = timedelta(seconds=timeout)
if not timeout:
return True
must_end = datetime.now() + timeout
logger.debug('Waiting for container %s to become healthy...', self)
while datetime.now() < must_end:
if self.is_healthy():
return True
sleep(2)
# deploy_info is written by watch-etcd services, so it's very
# important to constantly query database, without refresh we'll be
# constantly hitting sqlalchemy cache
db.session.refresh(self, attribute_names=['deploy_info'])
db.session.commit()
return False
def status(self):
if self.is_debug():
return 'debug'
if self.is_removing():
return 'removing'
running = self.deploy_info.get('Running')
healthy = self.deploy_info.get('Healthy')
if running:
if healthy:
return 'running'
else:
return 'sick'
else:
return 'dead'
def get_node(self):
return get_core(self.zone).get_node(self.podname, self.nodename)
