def __init__(self, job, *args, **kwargs):
self.job = job
self.configs = kwargs.get("config")
self.region_api = RegionAPI(conf=self.configs['region'])
image_config = self.configs["publish"]["image"]
self.region_client = RegionBackAPI()
self.region_registry = RegistryAPI(
host=image_config.get('curr_registry'))
# self.region_registry.set_log_topic('mq_work.image_manual')
self.oss_registry = RegistryAPI(host=image_config.get('all_registry'))
self.oss_registry.set_log_topic('mq_work.image_manual')
self.locker = TaskLocker(conf=self.configs['etcd'])
self.api = ACPAPI(conf=self.configs['region'])
self.namespace = image_config.get('oss_namespace')
self.user_cs_client = UserConsoleAPI(conf=self.configs['userconsole'])
def __init__(self, job, *args, **kwargs):
self.job = job
self.configs = kwargs.get("config")
self.region_api = RegionAPI(conf=self.configs['region'])
image_config = self.configs["publish"]["image"]
self.region_client = RegionBackAPI()
self.region_registry = RegistryAPI(
host=image_config.get('curr_registry'))
# self.region_registry.set_log_topic('mq_work.image_manual')
self.oss_registry = RegistryAPI(host=image_config.get('all_registry'))
self.oss_registry.set_log_topic('mq_work.image_manual')
self.locker = TaskLocker(conf=self.configs['etcd'])
self.api = ACPAPI(conf=self.configs['region'])
self.namespace = image_config.get('oss_namespace')
self.user_cs_client = UserConsoleAPI(conf=self.configs['userconsole'])
def __init__(self, job, *args, **kwargs):
self.job = job
self.configs = kwargs.get("config")
self.region_api = RegionAPI(conf=self.configs["region"])
self.region_client = RegionBackAPI()
self.api = ACPAPI(conf=self.configs['region'])
image_config = self.configs["publish"]["image"]
self.region_registry = RegistryAPI(
host=image_config.get('curr_registry'))
self.oss_registry = RegistryAPI(host=image_config.get('all_registry'))
self.region_registry.set_log_topic('mq_work.app_image')
self.oss_registry.set_log_topic('mq_work.app_image')
self.locker = TaskLocker(conf=self.configs['etcd'])
self.namespace = image_config.get('oss_namespace')
self.user_cs_client = UserConsoleAPI(conf=self.configs["userconsole"])
self.hubclient = HubUtils(image_config)
# 是否配置oss.goodrain.me
self.is_region_image = image_config.get('all_region_image')
self.is_oss_image = image_config.get('oss_image')
class ImageManual():
def __init__(self, job, *args, **kwargs):
self.job = job
self.configs = kwargs.get("config")
self.region_api = RegionAPI(conf=self.configs['region'])
image_config = self.configs["publish"]["image"]
self.region_client = RegionBackAPI()
self.region_registry = RegistryAPI(
host=image_config.get('curr_registry'))
# self.region_registry.set_log_topic('mq_work.image_manual')
self.oss_registry = RegistryAPI(host=image_config.get('all_registry'))
self.oss_registry.set_log_topic('mq_work.image_manual')
self.locker = TaskLocker(conf=self.configs['etcd'])
self.api = ACPAPI(conf=self.configs['region'])
self.namespace = image_config.get('oss_namespace')
self.user_cs_client = UserConsoleAPI(conf=self.configs['userconsole'])
def do_work(self):
try:
task = json.loads(self.job.body)
self.task = task
if "event_id" in self.task:
self.event_id = task["event_id"]
self.log = EventLog().bind(event_id=self.event_id,
step="image_manual")
else:
self.event_id = ""
self.log = EventLog().bind(event_id="", step="image_manual")
logger.info("mq_work.image_manual",
"new image_manual task: {}".format(task))
if task['action'] == 'create_new_version':
self.log.info("开始升级应用。")
self.create_new_version()
elif task['action'] == 'download_and_deploy':
self.log.info("开始下载镜像并部署应用。")
self.download_and_deploy()
elif task['action'] == 'delete_old_version':
self.log.info("开始删除旧版本。")
self.delete_old_version()
except Exception as e:
if self.log:
self.log.error("从自定义镜像部署应用失败。{}".format(e.message),
step="callback",
status="failure")
logger.exception('mq_work.image_manual', e)
def create_new_version(self):
logger.debug("mq_work.image_manual",
"now create new version and upload image")
def delete_old_version(self):
logger.debug("mq_work.image_manual", "now delete old version")
def download_and_deploy(self):
image = self.task['image']
# namespace = self.task['namespace']
tenant_name = self.task['tenant_name']
service_alias = self.task['service_alias']
event_id = self.task['event_id']
service_alias = self.task.get("service_alias", None)
has_download = False
inner_image = self.oss_registry.rename_image(image)
inner_image = "{0}_{1}".format(inner_image, service_alias)
local_image = self.region_registry.rename_image(image)
local_image = "{0}_{1}".format(local_image, service_alias)
# 直接下载docker image
try:
self.log.info("开始下载镜像:{0}".format(image))
pull_result = self._pull(image)
if pull_result:
# image_id = self.get_image_property(image, 'Id')
self._tag(image, local_image)
self.log.info("修改镜像名为：{0}".format(local_image))
ok = self._push(local_image)
if not ok:
self.log.error("上传镜像发生错误，重试失败，退出。",
step="callback",
status="failure")
return
self.log.info("镜像推送到本地仓库完成。")
# self._tag(image, inner_image)
# self._push(inner_image)
has_download = True
else:
self.log.error("下载镜像发生错误。", step="callback", status="failure")
logger.error("mq_work.image_manual",
"download image failed! image:{}".format(image))
except Exception as e:
self.log.error("镜像操作发生错误。{0}".format(e.__str__()),
step="callback",
status="failure")
logger.exception("mq_work.image_manual", e)
version_status = {
"final_status": "failure",
}
if has_download:
self.log.info("应用同步完成。", step="app-image", status="success")
version_body = {
"type": 'image',
"path": local_image,
"event_id": self.event_id
}
version_status['final_status'] = "success"
try:
self.region_client.update_version_region(
json.dumps(version_body))
self.region_client.update_version_event(
self.event_id, json.dumps(version_status))
except Exception as e:
pass
try:
self.api.update_iamge(tenant_name, service_alias, local_image)
self.log.info("应用信息更新完成，开始启动应用。",
step="app-image",
status="success")
self.api.start_service(tenant_name, service_alias, event_id)
except Exception as e:
logger.exception(e)
self.log.error("应用自动启动失败。请手动启动",
step="callback",
status="failure")
else:
try:
self.region_client.update_version_event(
self.event_id, json.dumps(version_status))
except Exception as e:
pass
self.log.error("应用同步失败。", step="callback", status="failure")
def queryServiceStatus(self, service_id):
try:
res, body = self.region_api.is_service_running(service_id)
logger.info(
'mq_work.image_manual',
"service_id=" + service_id + ";body=" + json.dumps(body))
status = body.get(service_id, "closed")
if status == "running":
self.log.debug("依赖的应用状态已经为运行中。", step="worker")
return True
except:
pass
self.log.debug("依赖的应用状态不是运行中，本应用稍后启动。", step="worker")
return False
def get_image_property(self, image, name):
query_format = '{{.%s}}' % name
try:
output = shell.call("{2} inspect -f '{0}' {1}".format(
query_format, image, DOCKER_BIN))
if output == '<no value>':
return None
else:
return output[0].rstrip('\n')
except shell.ExecException, e:
logger.exception("mq_work.image_manual", e)
return None
class AppImage():
def __init__(self, job, *args, **kwargs):
self.job = job
self.configs = kwargs.get("config")
self.region_api = RegionAPI(conf=self.configs["region"])
self.region_client = RegionBackAPI()
self.api = ACPAPI(conf=self.configs['region'])
image_config = self.configs["publish"]["image"]
self.region_registry = RegistryAPI(
host=image_config.get('curr_registry'))
self.oss_registry = RegistryAPI(host=image_config.get('all_registry'))
self.region_registry.set_log_topic('mq_work.app_image')
self.oss_registry.set_log_topic('mq_work.app_image')
self.locker = TaskLocker(conf=self.configs['etcd'])
self.namespace = image_config.get('oss_namespace')
self.user_cs_client = UserConsoleAPI(conf=self.configs["userconsole"])
self.hubclient = HubUtils(image_config)
# 是否配置oss.goodrain.me
self.is_region_image = image_config.get('all_region_image')
self.is_oss_image = image_config.get('oss_image')
def do_work(self):
try:
task = json.loads(self.job.body)
self.task = task
if "event_id" in self.task:
self.event_id = task["event_id"]
self.log = EventLog().bind(event_id=self.event_id)
else:
self.event_id = ""
self.log = EventLog().bind(event_id="")
if task['action'] == 'create_new_version':
self.log.info("开始发布升级应用。", step="app-image")
self.create_new_version()
elif task['action'] == 'download_and_deploy':
self.log.info("开始同步和部署应用。", step="app-image")
self.download_and_deploy()
elif task['action'] == 'delete_old_version':
self.log.info("开始删除旧版本应用。", step="app-image")
self.delete_old_version()
except Exception as e:
if self.log:
self.log.error(
"从云市部署应用失败。{}".format(e.message),
step="callback",
status="failure")
logger.exception('mq_work.app_image', e)
def create_new_version(self):
image = self.task['image']
service_key = self.task['service_key']
app_version = self.task['app_version']
oss_image = self.oss_registry.rename_image(image)
dest = self.task['dest']
share_id = self.task.get("share_id", None)
if dest == "yb":
if self.region_registry.exist_image(image):
logger.debug("mq_work.app_image",
"now local exists, oss doesnot exists")
data = {
'service_key': service_key,
'app_version': app_version,
'image': image,
'dest_yb': True,
'dest_ys': False,
'slug': ""
}
if share_id is not None:
data["share_id"] = share_id
try:
self.region_client.service_publish_new_region(data)
except Exception as e:
self.region_client.service_publish_failure_region(data)
self.log.error("云帮应用本地发布失败,保存publish 失败。{0}".format(e.message),step="callback",status="failure")
pass
if self.is_region_image and not self.oss_registry.exist_image(
oss_image):
try:
self.log.info("开始拉取镜像。")
ok = self._pull(image)
if not ok:
self.log.error(
"拉取镜像发生错误，构建退出。",
step="callback",
status="failure")
self.region_client.service_publish_failure_region(data)
return
image_id = self.get_image_property(image, 'Id')
self.log.info("拉取镜像完成。")
self._tag(image_id, oss_image)
self.log.info("镜像更改tag完成。开始上传镜像到云帮")
ok = self._push(oss_image)
if not ok:
self.log.error(
"拉取镜像发生错误，构建退出。",
step="callback",
status="failure")
self.region_client.service_publish_failure_region(data)
return
self.log.info("上传镜像到云帮完成")
# 发送通知到web
self.user_cs_client.service_publish_success(
json.dumps(data))
try:
self.region_client.service_publish_success_region(data)
except Exception as e:
logger.exception(e)
self.region_client.service_publish_failure_region(data)
pass
self.log.info(
"云帮应用发布完毕", step="last", status="success")
except (shell.ExecException, Exception), e:
logger.exception("mq_work.app_image", e)
logger.error("mq_work.app_image", e)
self.region_client.service_publish_failure_region(data)
self.log.error(
"云帮应用发布失败 {}".format(e.message),
step="callback",
status="failure")
else:
# 发送通知到web
self.user_cs_client.service_publish_success(
json.dumps(data))
try:
self.region_client.service_publish_success_region(data)
except Exception as e:
self.region_client.service_publish_failure_region(data)
logger.exception(e)
pass
self.log.info("云帮应用发布完毕", step="last", status="success")
else:
self.log.info("镜像不存在，发布失败", step="callback", status="failure")
elif dest == "ys":
# 当前有镜像并且云市的image数据中心开启
if self.region_registry.exist_image(image) and self.is_oss_image:
req = {
'service_key': service_key,
'app_version': app_version,
'image': image,
'slug': "",
'dest_ys': True,
'dest_yb': False
}
if share_id is not None:
req["share_id"] = share_id
try:
self.region_client.service_publish_new_region(data)
except Exception as e:
self.region_client.service_publish_failure_region(data)
self.log.info("云帮应用本地发布失败,保存publish 失败。{0}".format(e.message),step="callback",status="failure")
pass
# self.region_client.service_publish_new_region(req)
self.log.info("开始上传镜像到云市")
# 修改image name
hub_image = self.hubclient.rename_image(image)
logger.info("mq_work.app_image",
'hub_image={}'.format(hub_image))
# 检查是否存在
data = self.hubclient.parse_image(image)
logger.info("mq_work.app_image", 'data={}'.format(data))
# 判断tag是否存在,
tag_exists = self.hubclient.check(data.name, data.tag)
logger.info("mq_work.app_image",
'tag_exists={}'.format(tag_exists))
try:
self.log.info("开始从云帮拉取镜像。")
ok = self._pull(image)
if not ok:
self.log.error(
"拉取镜像发生错误，构建退出。",
step="callback",
status="failure")
self.region_client.service_publish_failure_region(req)
return
image_id = self.get_image_property(image, 'Id')
self.log.info("从云帮拉取镜像完成,更改镜像TAG")
self._tag(image_id, hub_image)
self.log.info("更改镜像TAG完成，开始上传镜像到云市")
ok = self._push(hub_image)
if not ok:
self.log.error(
"拉取镜像发生错误，构建退出。",
step="callback",
status="failure")
self.region_client.service_publish_failure_region(req)
return
self.log.info("上传镜像到云市完成。")
# 发送通知到web
self.user_cs_client.service_publish_success(
json.dumps(req))
try:
self.region_client.service_publish_success_region(req)
except Exception as e:
self.region_client.service_publish_failure_region(req)
logger.exception(e)
pass
self.log.info("云市应用发布完毕", step="last", status="success")
except (shell.ExecException, Exception), e:
logger.exception("mq_work.app_image", e)
logger.error("mq_work.app_image", e)
self.region_client.service_publish_failure_region(req)
self.log.error(
"云市应用发布失败 {}".format(e.message),
step="callback",
status="failure")
class ImageManual():
def __init__(self, job, *args, **kwargs):
self.job = job
self.configs = kwargs.get("config")
self.region_api = RegionAPI(conf=self.configs['region'])
image_config = self.configs["publish"]["image"]
self.region_client = RegionBackAPI()
self.region_registry = RegistryAPI(
host=image_config.get('curr_registry'))
# self.region_registry.set_log_topic('mq_work.image_manual')
self.oss_registry = RegistryAPI(host=image_config.get('all_registry'))
self.oss_registry.set_log_topic('mq_work.image_manual')
self.locker = TaskLocker(conf=self.configs['etcd'])
self.api = ACPAPI(conf=self.configs['region'])
self.namespace = image_config.get('oss_namespace')
self.user_cs_client = UserConsoleAPI(conf=self.configs['userconsole'])
def do_work(self):
try:
task = json.loads(self.job.body)
self.task = task
if "event_id" in self.task:
self.event_id = task["event_id"]
self.log = EventLog().bind(
event_id=self.event_id, step="image_manual")
else:
self.event_id = ""
self.log = EventLog().bind(event_id="", step="image_manual")
logger.info("mq_work.image_manual",
"new image_manual task: {}".format(task))
if task['action'] == 'create_new_version':
self.log.info("开始升级应用。")
self.create_new_version()
elif task['action'] == 'download_and_deploy':
self.log.info("开始下载镜像并部署应用。")
self.download_and_deploy()
elif task['action'] == 'delete_old_version':
self.log.info("开始删除旧版本。")
self.delete_old_version()
except Exception as e:
if self.log:
self.log.error(
"从自定义镜像部署应用失败。{}".format(e.message),
step="callback",
status="failure")
logger.exception('mq_work.image_manual', e)
def create_new_version(self):
logger.debug("mq_work.image_manual",
"now create new version and upload image")
def delete_old_version(self):
logger.debug("mq_work.image_manual", "now delete old version")
def download_and_deploy(self):
image = self.task['image']
# namespace = self.task['namespace']
tenant_name = self.task['tenant_name']
service_alias = self.task['service_alias']
event_id = self.task['event_id']
service_alias = self.task.get("service_alias", None)
has_download = False
inner_image = self.oss_registry.rename_image(image)
inner_image = "{0}_{1}".format(inner_image, service_alias)
local_image = self.region_registry.rename_image(image)
local_image = "{0}_{1}".format(local_image, service_alias)
# 直接下载docker image
try:
self.log.info("开始下载镜像:{0}".format(image))
pull_result = self._pull(image)
if pull_result:
# image_id = self.get_image_property(image, 'Id')
self._tag(image, local_image)
self.log.info("修改镜像名为：{0}".format(local_image))
ok = self._push(local_image)
if not ok:
self.log.error(
"上传镜像发生错误，重试失败，退出。", step="callback", status="failure")
return
self.log.info("镜像推送到本地仓库完成。")
# self._tag(image, inner_image)
# self._push(inner_image)
has_download = True
else:
self.log.error("下载镜像发生错误。", step="callback", status="failure")
logger.error("mq_work.image_manual",
"download image failed! image:{}".format(image))
except Exception as e:
self.log.error(
"镜像操作发生错误。{0}".format(e.__str__()),
step="callback",
status="failure")
logger.exception("mq_work.image_manual", e)
version_status = {
"final_status":"failure",
}
if has_download:
self.log.info("应用同步完成。", step="app-image", status="success")
version_body = {
"type": 'image',
"path": local_image,
"event_id": self.event_id
}
body = {
"deploy_version": self.task['deploy_version'],
"event_id": self.event_id,
}
version_status['final_status'] = "success"
try:
self.region_client.update_version_region(json.dumps(version_body))
self.region_client.update_version_event(self.event_id,json.dumps(version_status))
except Exception as e:
pass
try:
self.api.update_iamge(tenant_name, service_alias, local_image)
version = self.task['deploy_version']
self.log.info("应用信息更新完成，开始启动应用。", step="app-image", status="success")
self.api.upgrade_service(self.task['tenant_name'], self.task['service_alias'], json.dumps(body))
# self.api.start_service(tenant_name, service_alias, event_id)
except Exception as e:
logger.exception(e)
self.log.error(
"应用自动启动失败。请手动启动", step="callback", status="failure")
else:
try:
self.region_client.update_version_event(self.event_id,json.dumps(version_status))
except Exception as e:
pass
self.log.error("应用同步失败。", step="callback", status="failure")
def queryServiceStatus(self, service_id):
try:
res, body = self.region_api.is_service_running(service_id)
logger.info(
'mq_work.image_manual',
"service_id=" + service_id + ";body=" + json.dumps(body))
status = body.get(service_id, "closed")
if status == "running":
self.log.debug("依赖的应用状态已经为运行中。", step="worker")
return True
except:
pass
self.log.debug("依赖的应用状态不是运行中，本应用稍后启动。", step="worker")
return False
def get_image_property(self, image, name):
query_format = '{{.%s}}' % name
try:
output = shell.call("{2} inspect -f '{0}' {1}".format(
query_format, image, DOCKER_BIN))
if output == '<no value>':
return None
else:
return output[0].rstrip('\n')
except shell.ExecException, e:
logger.exception("mq_work.image_manual", e)
return None