def wait_for_erection(self, timeout=None):
"""wait until this container is healthy, timeout can be timedelta or
seconds, timeout default to erection_timeout in specs, if timeout is 0,
don't even wait and just report healthy"""
if not timeout:
timeout = timedelta(seconds=self.release.specs.erection_timeout)
elif isinstance(timeout, Number):
timeout = timedelta(seconds=timeout)
if not timeout:
return True
must_end = datetime.now() + timeout
logger.debug('Waiting for container %s to become healthy...', self)
while datetime.now() < must_end:
if self.is_healthy():
return True
sleep(2)
# deploy_info is written by watch-etcd services, so it's very
# important to constantly query database, without refresh we'll be
# constantly hitting sqlalchemy cache
db.session.refresh(self, attribute_names=['deploy_info'])
db.session.commit()
return False
def deploy_elb(socket):
"""Remove the specified containers
:<json string name: required, ELB cluster name
:<json string zone: required
:<json string sha: required, minimum length is 7
:<json string combo_name: required, the combo used to create ELB containers
:<json string nodename: optional
"""
payload = None
while True:
message = socket.receive()
try:
payload = deploy_elb_schema.loads(message)
break
except ValidationError as e:
socket.send(json.dumps(e.messages))
except JSONDecodeError as e:
socket.send(json.dumps({'error': str(e)}))
args = payload.data
async_result = create_elb_instance.delay(user_id=session.get('user_id'),
**args)
for m in celery_task_stream_response(async_result.task_id):
logger.debug(m)
socket.send(m)
def _last_for(self, lval, threshold):
comparison_expression_key = self._make_comparison_expression_key(*lval)
eval_result = self._eval_expression(lval)
logger.debug('Expression %s evaluated to be %s', lval, eval_result)
if eval_result is True:
now = datetime.now()
if comparison_expression_key in rds:
# redis 里标记了，拿出来比较一下，达到了阈值就返回True
ts = float(rds.get(comparison_expression_key))
marked_at = datetime.fromtimestamp(ts)
delta = now - marked_at
if delta >= threshold:
rds.delete(comparison_expression_key)
return True
else:
# redis 里没标记，那就是第一次出现，记下时间
now_ts = time.mktime(now.utctimetuple())
logger.debug(
'Marking comparison_expression_key: %s, value %s, expire in %s',
comparison_expression_key, now_ts, threshold)
rds.setex(comparison_expression_key, now_ts, threshold * 2)
else:
# 发现一次不满足条件，前功尽弃
rds.delete(comparison_expression_key)
return False
def create_container(self, zone=None, user_id=None, appname=None, sha=None,
combo_name=None, debug=False, task_id=None):
release = Release.get_by_app_and_sha(appname, sha)
app = release.app
combo = app.get_combo(combo_name)
deploy_opt = release.make_core_deploy_options(combo_name)
ms = get_core(zone).create_container(deploy_opt)
bad_news = []
deploy_messages = []
for m in ms:
self.stream_output(m, task_id=task_id)
deploy_messages.append(m)
if m.success:
logger.debug('Creating container %s:%s got grpc message %s', appname, combo.entrypoint_name, m)
override_status = ContainerOverrideStatus.DEBUG if debug else ContainerOverrideStatus.NONE
Container.create(appname,
sha,
m.id,
m.name,
combo_name,
combo.entrypoint_name,
combo.envname,
combo.cpu_quota,
combo.memory,
zone,
m.podname,
m.nodename,
override_status=override_status)
op_content = {'entrypoint': combo.entrypoint_name,
'envname': combo.envname,
'networks': combo.networks,
'container_id': m.id}
OPLog.create(container_id=m.id,
user_id=user_id,
action=OPType.CREATE_CONTAINER,
appname=appname,
sha=sha,
zone=zone,
content=op_content)
else:
bad_news.append(m)
if bad_news:
msg = 'Deploy {}\n*BAD NEWS*:\n```\n{}\n```\n'.format(
appname,
json.dumps(bad_news, cls=VersatileEncoder),
)
notbot_sendmsg(app.subscribers, msg)
# if called synchronously, the caller will not be able to receive the task
# output by redis pubsub, so the task function will return the
# deploy_messages
# if called asynchronously, grpc message cannot be pickled easily, that's
# why this task only return when called synchronously
if not self.request.id:
return deploy_messages
def update_image(self, image):
try:
self.image = image
logger.debug('Set image %s for release %s', image, self.sha)
db.session.add(self)
db.session.commit()
except StaleDataError:
db.session.rollback()
def update_env_set(self, envname, data):
env_set = EnvSet(**data)
env_sets = self.env_sets.copy()
env_sets[envname] = env_set
self.env_sets = env_sets
logger.debug('Update env set %s for %s, full env_sets: %s', envname,
self.name, env_sets)
db.session.add(self)
db.session.commit()
def set_node_availability(self, podname, nodename, is_available=True):
stub = self._get_stub()
logger.debug('Set node %s:%s available: %s', podname, nodename,
is_available)
opts = NodeAvailable(podname=podname,
nodename=nodename,
available=is_available)
n = stub.SetNodeAvailable(opts, _UNARY_TIMEOUT)
return n
def __call__(self, smart_status, dangers, **kwargs):
"""yeah, TackleTask has fixed args, and custom kwargs"""
cooldown = int(parse_timespan(kwargs.get('cooldown', '1m')))
strategy = self.name
key = CITADEL_TACKLE_TASK_THROTTLING_KEY.format(id_=smart_status.name, strategy=strategy)
if key in rds:
logger.debug('Skip tackle strategy {}'.format(strategy))
return
logger.debug('Mark {} with ttl {}'.format(key, cooldown))
rds.setex(key, 'true', cooldown)
super(TackleTask, self).__call__(smart_status, dangers, **kwargs)
def celery_task_stream_response(celery_task_ids):
if isinstance(celery_task_ids, str):
celery_task_ids = celery_task_ids,
task_progress_channels = [TASK_PUBSUB_CHANNEL.format(task_id=id_) for id_ in celery_task_ids]
pubsub = rds.pubsub()
pubsub.subscribe(task_progress_channels)
for item in pubsub.listen():
# each content is a single JSON encoded grpc message
raw_content = item['data']
# omit the initial message where item['data'] is 1L
if not isinstance(raw_content, (bytes, str)):
continue
content = raw_content.decode('utf-8')
logger.debug('Got pubsub message: %s', content)
# task will publish TASK_PUBSUB_EOF at success or failure
if content.startswith('CELERY_TASK_DONE'):
finished_task_id = content[content.find(':') + 1:]
finished_task_channel = TASK_PUBSUB_CHANNEL.format(task_id=finished_task_id)
logger.debug('Task %s finished, break celery_task_stream_response', finished_task_id)
pubsub.unsubscribe(finished_task_channel)
else:
yield content
def remove(socket):
"""Remove the specified containers
:<json list container_ids: required, a list of container_id
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
"id": "9c91d06cb165e829e8e0ad5d5b5484c47d4596af04122489e4ead677113cccb4",
"success": true,
"message": "hook output",
"__class__": "RemoveContainerMessage"
}
"""
payload = None
while True:
message = socket.receive()
try:
payload = remove_container_schema.loads(message)
break
except ValidationError as e:
socket.send(json.dumps(e.messages))
except JSONDecodeError as e:
socket.send(json.dumps({'error': str(e)}))
args = payload.data
async_result = remove_container.delay(user_id=session.get('user_id'),
**args)
for m in celery_task_stream_response(async_result.task_id):
logger.debug(m)
socket.send(m)
def update_deploy_info(self, deploy_info):
logger.debug('Update deploy_info for %s: %s', self, deploy_info)
self.deploy_info = deploy_info
db.session.add(self)
db.session.commit()
def build(socket):
"""Build an image for the specified release, the API will return all docker
build messages, key frames as shown in the example responses
:<json string appname: required, the app name
:<json string sha: required, minimum length is 7
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
"id": "",
"status": "",
"progress": "",
"error": "",
"stream": "Step 1/7 : FROM python:latest as make-artifacts",
"error_detail": {
"code": 0,
"message": "",
"__class__": "ErrorDetail"
},
"__class__": "BuildImageMessage"
}
{
"id": "0179a75e26fe",
"status": "Pushing",
"progress": "[==================================================>] 6.656kB",
"error": "",
"stream": "",
"error_detail": {
"code": 0,
"message": "",
"__class__": "ErrorDetail"
},
"__class__": "BuildImageMessage"
}
{
"id": "",
"status": "finished",
"progress": "hub.ricebook.net/projecteru2/test-app:3641aca",
"error": "",
"stream": "finished hub.ricebook.net/projecteru2/test-app:3641aca",
"error_detail": {
"code": 0,
"message": "",
"__class__": "ErrorDetail"
},
"__class__": "BuildImageMessage"
}
"""
payload = None
while True:
message = socket.receive()
try:
payload = build_args_schema.loads(message)
break
except ValidationError as e:
socket.send(json.dumps(e.messages))
except JSONDecodeError as e:
socket.send(json.dumps({'error': str(e)}))
args = payload.data
async_result = build_image.delay(args['appname'], args['sha'])
for m in celery_task_stream_response(async_result.task_id):
logger.debug(m)
socket.send(m)
def renew(socket):
"""Create a new container to substitute the old one, this API can be used
to upgrade a app to a specified version, or simply re-create a container
using the same combo.
Things can go wrong at any step, the example response
was the output generated by ``renew("1aa8a638a153b393ee423c0a8c158757b13ab74591ade036b6e73ac33a4bdeac", "3641aca")``
which failed because the newly created container didn't become healthy
within the given time limit.
:<json list container_ids: required, a list of container_id
:<json string sha: required, minimum length is 7
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
"podname": "eru",
"nodename": "c1-eru-2.ricebook.link",
"id": "2f123f1abcdfc8208b298c89e10bcd8f48f9fdb25c9eb7874ea5cc7199825e6e",
"name": "test-app_web_rvrhPg",
"error": "",
"success": true,
"cpu": {"0": 20},
"quota": 0.2,
"memory": 134217728,
"publish": {"bridge": "172.17.0.5:6789"},
"hook": "hook output",
"__class__": "CreateContainerMessage"
}
{
"id": "2f123f1abcdfc8208b298c89e10bcd8f48f9fdb25c9eb7874ea5cc7199825e6e",
"success": true,
"message": "hook output",
"__class__": "RemoveContainerMessage"
}
{
"error": "New container <Container test-zone:test-app:3641aca:web:2f123f1 did't became healthy, remove result: id: 2f123f1abcdfc8208b298c89e10bcd8f48f9fdb25c9eb7874ea5cc7199825e6e success: true",
"args": ["1aa8a638a153b393ee423c0a8c158757b13ab74591ade036b6e73ac33a4bdeac", "3641aca"],
"kwargs": {"user_id": null}
}
"""
payload = None
while True:
message = socket.receive()
try:
payload = renew_schema.loads(message)
break
except ValidationError as e:
socket.send(json.dumps(e.messages))
except JSONDecodeError as e:
socket.send(json.dumps({'error': str(e)}))
args = payload.data
containers = [
Container.get_by_container_id(id_) for id_ in args['container_ids']
]
appnames = {c.appname for c in containers}
sha = args['sha']
if len(appnames) > 1 and sha:
socket.send(
json.dumps({
'error':
'cannot provide sha when renewing containers of multiple apps: {}'
.format(appnames)
}))
socket.close()
appname = appnames.pop()
app = App.get_by_name(appname)
if not app:
socket.send(json.dumps({'error': 'app {} not found'.format(appname)}))
socket.close()
task_ids = []
for c in containers:
async_result = renew_container.delay(c.container_id,
sha,
user_id=session.get('user_id'))
task_ids.append(async_result.task_id)
for m in celery_task_stream_response(task_ids):
logger.debug(m)
socket.send(m)
def deploy(socket):
"""Create containers for the specified release
:<json string appname: required
:<json string zone: required
:<json string sha: required, minimum length is 7
:<json string combo_name: required, specify the combo to use, you can
update combo value using this API, so all parameters in the
:http:post:`/api/app/(appname)/combo` are supported
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
"podname": "eru",
"nodename": "c1-eru-2.ricebook.link",
"id": "9c91d06cb165e829e8e0ad5d5b5484c47d4596af04122489e4ead677113cccb4",
"name": "test-app_web_kMqYFQ",
"error": "",
"success": true,
"cpu": {"0": 20},
"quota": 0.2,
"memory": 134217728,
"publish": {"bridge": "172.17.0.5:6789"},
"hook": "I am the hook output",
"__class__": "CreateContainerMessage"
}
"""
payload = None
while True:
message = socket.receive()
try:
payload = deploy_schema.loads(message)
break
except ValidationError as e:
socket.send(json.dumps(e.messages))
except JSONDecodeError as e:
socket.send(json.dumps({'error': str(e)}))
args = payload.data
appname = args['appname']
app = App.get_by_name(appname)
if not app:
socket.send(json.dumps({'error': 'app {} not found'.format(appname)}))
socket.close()
combo_name = args['combo_name']
combo = app.get_combo(combo_name)
if not combo:
socket.send(
json.dumps({
'error':
'combo {} for app {} not found'.format(combo_name, app)
}))
socket.close()
combo.update(**{k: v for k, v in args.items() if hasattr(combo, k)})
async_result = create_container.delay(user_id=session.get('user_id'),
**args)
for m in celery_task_stream_response(async_result.task_id):
logger.debug(m)
socket.send(m)