def clone(self):
self.log.info("开始拉取代码。。", step="build-worker")
# code, output = shell.runsingle("git clone --branch master --depth 1 {0} {1}".format(self.repo_url, self.source_dir))
result = False
num = 0
while num < 2:
try:
shell.call("timeout -k 9 {2} git clone {0} {1}".format(
self.repo_url, self.source_dir, CLONE_TIMEOUT))
result = True
break
except shell.ExecException, e:
num = num + 1
self.prepare()
if num < 2:
self.log.error(
"拉取代码发生错误,开始重试 {}".format(e.message),
status="failure",
step="worker-clone")
else:
self.log.error(
"拉取代码发生错误,部署停止 {}".format(e.message),
status="failure",
step="callback")
logger.exception('build_work.main', e)
result = False
def clone(self):
self.log.info("开始拉取代码。。", step="build-worker")
# code, output = shell.runsingle("git clone --branch master --depth 1 {0} {1}".format(self.repo_url, self.source_dir))
result = False
num = 0
while num < 2:
try:
cmdstr = "timeout -k 9 {2} git clone {0} {1}".format(
self.repo_url, self.source_dir, CLONE_TIMEOUT)
if "github.com" in self.repo_url:
cmdstr = "timeout -k 9 {2} git clone -c http.proxy=http://127.0.0.1:18888 {0} {1}".format(
self.repo_url, self.source_dir, CLONE_TIMEOUT)
shell.call(cmdstr)
result = True
break
except shell.ExecException, e:
num = num + 1
self.prepare()
if num < 2:
self.log.error("拉取代码发生错误,开始重试 {}".format(e.message),
status="failure",
step="worker-clone")
else:
self.log.error("拉取代码发生错误,部署停止 {}".format(e.message),
status="failure",
step="callback")
logger.exception('build_work.main', e)
result = False
def _push(self, image):
cmd = "{0} push {1}".format(DOCKER_BIN, image)
logger.info("mq_work.image_manual", cmd)
retry = 2
while retry:
try:
p = shell.start(cmd)
while p.is_running():
line = p.readline()
self.log.debug(line.rstrip('\n').lstrip('\x1b[1G'),
step="push-image")
for line in p.unread_lines:
self.log.debug(line, step="push-image")
if p.exit_with_err():
self.log.error("上传镜像失败。" +
("开始进行重试." if retry > 0 else ""),
step="push-image",
status="failure")
retry -= 1
continue
return True
except shell.ExecException, e:
self.log.error("上传镜像发生错误。{}" + (
"开始进行重试." if retry > 0 else "").format(e.message))
logger.error(e)
retry -= 1
def _push(self, image):
cmd = "{} push {}".format(DOCKER_BIN, image)
logger.info("mq_work.app_image", cmd)
retry = 2
while retry:
try:
p = shell.start(cmd)
while p.is_running():
line = p.readline()
self.log.debug(
line.rstrip('\n').lstrip('\x1b[1G'), step="push-image")
for line in p.unread_lines:
self.log.debug(line, step="push-image")
if p.exit_with_err():
self.log.error(
"上传镜像失败。" + ("开始进行重试." if retry > 0 else ""),
step="push-image",
status="failure")
retry -= 1
continue
return True
except shell.ExecException, e:
self.log.error("上传镜像发生错误。{0}" + ("开始进行重试." if retry > 0 else
"").format(e.message))
logger.error(e)
retry -= 1
def get_commit_info(self):
try:
output = shell.call(
"""git log -n 1 --pretty --format='{"hash":"%H","author":"%an","timestamp":%at}'""",
self.source_dir)
if type(output) is list:
output = output[0]
jdata = json.loads(output)
output2 = shell.call("""git log -n 1 --pretty --format=%s""",
self.source_dir)
if type(output2) is list:
subject = output2[0]
jdata['subject'] = subject
else:
jdata['subject'] = 'unknown'
return jdata
except shell.ExecException, e:
logger.exception('build_work.main', e)
return "{}"
def get_commit_info(self):
try:
output = shell.call(
"""git log -n 1 --pretty --format='{"hash":"%H","author":"%an","timestamp":%at}'""",
self.source_dir)
if type(output) is list:
output = output[0]
jdata = json.loads(output)
output2 = shell.call("""git log -n 1 --pretty --format=%s""",
self.source_dir)
if type(output2) is list:
subject = output2[0]
jdata['subject'] = subject
else:
jdata['subject'] = 'unknown'
return jdata
except shell.ExecException, e:
logger.exception('build_work.main', e)
return "{}"
def get_image_property(self, image, name):
query_format = '{{.%s}}' % name
try:
output = shell.call("{2} inspect -f '{0}' {1}".format(
query_format, image, DOCKER_BIN))
if output == '<no value>':
return None
else:
return output[0].rstrip('\n')
except shell.ExecException, e:
logger.exception("mq_work.image_manual", e)
return None
def get_image_property(self, image, name):
query_format = '{{.%s}}' % name
try:
output = shell.call("{2} inspect -f '{0}' {1}".format(
query_format, image, DOCKER_BIN))
if output == '<no value>':
return None
else:
return output[0].rstrip('\n')
except shell.ExecException, e:
logger.exception("mq_work.app_image", e)
return None
def build_code(self):
self.log.info("开始编译代码包", step="build_code")
package_name = '{0}/{1}.tgz'.format(self.tgz_dir, self.deploy_version)
self.logfile = '{0}/{1}.log'.format(self.tgz_dir, self.deploy_version)
repos = self.repo_url.split(" ")
self.log.debug("repos=" + repos[1], step="build_code")
#master
no_cache = self.build_envs.pop('NO_CACHE', False)
if no_cache:
try:
shutil.rmtree(self.cache_dir)
os.makedirs(self.cache_dir)
os.chown(self.cache_dir, 200, 200)
self.log.debug("清理缓存目录{0}".format(self.cache_dir),
step="build_code")
except Exception as e:
self.log.error("清理缓存目录{0}失败{1}".format(self.cache_dir,
e.message),
step="build_code")
pass
try:
cmd = "perl {0} -b {1} -s {2} -c {3} -d {4} -v {5} -l {6} -tid {7} -sid {8} --name {9}".format(
self.build_cmd, repos[1], self.source_dir, self.cache_dir,
self.tgz_dir, self.deploy_version, self.logfile,
self.tenant_id, self.service_id, self.build_name)
if self.build_envs:
build_env_string = ':::'.join(
map(lambda x: "{}='{}'".format(x, self.build_envs[x]),
self.build_envs.keys()))
cmd += " -e {}".format(build_env_string)
p = shell.start(cmd)
while p.is_running():
line = p.readline()
self.log.debug(line.rstrip('\n').lstrip('\x1b[1G'),
step="build_code")
for line in p.unread_lines:
self.log.debug(line, step="build_code")
if p.exit_with_err():
self.log.error("编译代码包失败。", step="build_code", status="failure")
return False
self.log.debug("编译代码包完成。", step="build_code", status="success")
except shell.ExecException, e:
self.log.error("编译代码包过程遇到异常，{}".format(e.message),
step="build_code",
status="failure")
return False
def build_code(self):
self.log.info("开始编译代码包", step="build_code")
package_name = '{0}/{1}.tgz'.format(self.tgz_dir, self.deploy_version)
self.logfile = '{0}/{1}.log'.format(self.tgz_dir, self.deploy_version)
repos = self.repo_url.split(" ")
self.log.debug("repos=" + repos[1], step="build_code")
#master
no_cache = self.build_envs.pop('NO_CACHE', False)
if no_cache:
try:
shutil.rmtree(self.cache_dir)
os.makedirs(self.cache_dir)
self.log.debug(
"清理缓存目录{0}".format(self.cache_dir), step="build_code")
except Exception as e:
self.log.error(
"清理缓存目录{0}失败{1}".format(self.cache_dir, e.message),
step="build_code")
pass
try:
cmd = "perl {0} -b {1} -s {2} -c {3} -d {4} -v {5} -l {6} -tid {7} -sid {8} --name {9}".format(
self.build_cmd, repos[1], self.source_dir, self.cache_dir,
self.tgz_dir, self.deploy_version, self.logfile,
self.tenant_id, self.service_id, self.build_name)
if self.build_envs:
build_env_string = ':::'.join(
map(lambda x: "{}='{}'".format(x, self.build_envs[x]),
self.build_envs.keys()))
cmd += " -e {}".format(build_env_string)
p = shell.start(cmd)
while p.is_running():
line = p.readline()
self.log.debug(
line.rstrip('\n').lstrip('\x1b[1G'), step="build_code")
for line in p.unread_lines:
self.log.debug(line, step="build_code")
if p.exit_with_err():
self.log.error("编译代码包失败。", step="build_code", status="failure")
return False
self.log.debug("编译代码包完成。", step="build_code", status="success")
except shell.ExecException, e:
self.log.error(
"编译代码包过程遇到异常，{}".format(e.message),
step="build_code",
status="failure")
return False
def rewrite_files(self, dockerfile, insert_lines, cmd, entrypoint):
extend_lines = map(lambda x: x + '\n', insert_lines)
try:
f = open(dockerfile, 'r')
lines = f.readlines()
for line in lines:
if line.startswith('ENTRYPOINT') or line.startswith('CMD'):
lines.remove(line)
lines.extend(extend_lines)
f.close()
f = open(dockerfile, 'w')
f.writelines(lines)
f.close()
shutil.copytree('./lib/.goodrain',
'{0}/.goodrain'.format(self.source_dir))
if entrypoint is not None:
entrypoint_cmd = ' '.join(entrypoint)
shell.call(
'''sed -i -e 's#_type_#ENTRYPOINT#' -e 's#^_entrypoint_#'{0}'#' .goodrain/init'''
.format(pipes.quote(entrypoint_cmd)),
cwd=self.source_dir)
if cmd is not None:
shell.call(
'''sed -i -e 's#^_cmd_#'{0}'#' .goodrain/init'''.
format(pipes.quote(cmd)),
cwd=self.source_dir)
else:
shell.call(
'''sed -i -e 's#_type_#CMD#' -e 's#^_cmd_#'{0}'#' .goodrain/init'''
.format(pipes.quote(cmd)),
cwd=self.source_dir)
return True
except (shell.ExecException, OSError), e:
logger.exception('build_work.main', e)
return False
def rewrite_files(self, dockerfile, insert_lines, cmd, entrypoint):
extend_lines = map(lambda x: x + '\n', insert_lines)
try:
f = open(dockerfile, 'r')
lines = f.readlines()
for line in lines:
if line.startswith('ENTRYPOINT') or line.startswith('CMD'):
lines.remove(line)
lines.extend(extend_lines)
f.close()
f = open(dockerfile, 'w')
f.writelines(lines)
f.close()
shutil.copytree('./lib/.goodrain',
'{0}/.goodrain'.format(self.source_dir))
if entrypoint is not None:
entrypoint_cmd = ' '.join(entrypoint)
shell.call(
'''sed -i -e 's#_type_#ENTRYPOINT#' -e 's#^_entrypoint_#'{0}'#' .goodrain/init'''.
format(pipes.quote(entrypoint_cmd)),
cwd=self.source_dir)
if cmd is not None:
shell.call(
'''sed -i -e 's#^_cmd_#'{0}'#' .goodrain/init'''.
format(pipes.quote(cmd)),
cwd=self.source_dir)
else:
shell.call(
'''sed -i -e 's#_type_#CMD#' -e 's#^_cmd_#'{0}'#' .goodrain/init'''.
format(pipes.quote(cmd)),
cwd=self.source_dir)
return True
except (shell.ExecException, OSError), e:
logger.exception('build_work.main', e)
return False
def build_image(self):
# self.write_build_log(u"开始编译Dockerfile")
'''
insert_lines = [
'RUN which wget || (apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y wget) || (yum install -y wget)',
'RUN which curl || (apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y curl) || (yum install -y curl)',
'RUN mkdir -pv /opt/bin',
'ADD ./.goodrain/init /opt/bin/init',
'',
'RUN wget http://lang.goodrain.me/public/gr-listener -O /opt/bin/gr-listener -q && \\',
' chmod 755 /opt/bin/*',
'',
'RUN rm -rf /var/lib/dpkg/* /var/lib/apt/*',
'ENTRYPOINT ["/opt/bin/init"]',
]
'''
dockerfile = '{0}/{1}'.format(self.source_dir, 'Dockerfile')
update_items, entrypoint, cmd = self.get_dockerfile_items(dockerfile)
# 重新解析dockerfile
pdf = None
try:
self.log.info("开始解析Dockerfile", step="build_image")
pdf = ParseDockerFile(dockerfile)
except ValueError as e:
self.log.error("用户自定义的volume路径包含相对路径,必须为绝对路径!",
step="build_image",
status="failure")
logger.exception(e)
return False
except Exception as e:
self.log.error("解析Dockerfile发生异常",
step="build_image",
status="failure")
logger.exception(e)
s = self.repo_url
regex = re.compile(r'.*(?:\:|\/)([\w\-\.]+)/([\w\-\.]+)\.git')
m = regex.match(s)
account, project = m.groups()
_name = '_'.join([self.service_id[:12], account, project])
_tag = self.deploy_version
build_image_name = '{0}/{1}:{2}'.format(REGISTRY_DOMAIN, _name, _tag)
# image name must be lower
build_image_name = build_image_name.lower()
self.log.debug("构建镜像名称为{0}".format(build_image_name),
step="build_image")
#build_image_name=""
no_cache = self.build_envs.pop('NO_CACHE', False)
if no_cache:
build_cmd = "{0} build -t {1} --no-cache .".format(
DOCKER_BIN, build_image_name)
else:
build_cmd = "{0} build -t {1} .".format(DOCKER_BIN,
build_image_name)
p = shell.start(build_cmd, cwd=self.source_dir)
while p.is_running():
line = p.readline()
self.log.debug(line, step="build_image")
for line in p.unread_lines:
self.log.debug(line, step="build_image")
if p.exit_with_err():
self.log.error("构建失败，请检查Debug日志排查！",
step="build_image",
status="failure")
return False
self.log.debug("镜像构建成功。开始推送", step="build_image", status="success")
try:
shell.call("{0} push {1}".format(DOCKER_BIN, build_image_name))
except shell.ExecException, e:
self.log.error("镜像推送失败。{}".format(e.message),
step="push_image",
status="failure")
return False
def _tag(self, image_id, image):
cmd = "{2} tag {0} {1}".format(image_id, image, DOCKER_BIN)
logger.info("mq_work.app_image", cmd)
shell.call(cmd)
class CodeCheck():
watching_topics = ('service_event_msg', )
required_configs = ('region', 'userconsole', 'etcd.lock')
def __init__(self, job, *args, **kwargs):
self.job = job
self.configs = kwargs.get("config")
self.user_cs_client = UserConsoleAPI(conf=self.configs['userconsole'])
self.region_client = RegionBackAPI()
task = json.loads(self.job.body)
self.base_dir = kwargs.get('base_dir')
for k in ('tenant_id', 'service_id', 'action'):
setattr(self, k, task[k])
if 'event_id' in task:
self.event_id = task["event_id"]
self.log = EventLog().bind(event_id=self.event_id)
else:
self.event_id = "system"
self.log = EventLog().bind(event_id=self.event_id)
self.task = task
self.locker = TaskLocker(conf=self.configs['etcd'])
# self.waittime = int(task['wait_time'])
self.log.info(u"worker已收到异步任务。", step="worker")
def do_work(self):
logger.info('mq_work.service_event',
"plugin %s execute start" % __name__)
self.log.debug(u"代码检查异步处理开始。", step="worker", status="start")
self.code_check()
logger.info('mq_work.service_event',
"plugin %s execute finished" % __name__)
def code_check(self):
git_url = self.task['git_url']
check_type = self.task['check_type']
code_version = self.task['code_version']
git_project_id = self.task['git_project_id']
code_from = self.task['code_from']
url_repos = self.task['url_repos']
lock_id = 'code_check.' + self.service_id
logger.info(
'mq_work.code_check',
"git_url {0},check_type {1}, code_version {2},git_project_id {3},code_from {4},url_repos {5} "
.format(git_url, check_type, code_version, git_project_id,
code_from, url_repos))
try:
if self.locker.exists(lock_id):
logger.info('mq_work.code_check',
"lock_id {} exists, do nothing".format(lock_id))
self.log.info('lock_id {} exists, do nothing'.format(lock_id),
step="check_exist")
return
self.locker.add_lock(lock_id, bytes(git_url))
logger.info('add lock_id {}'.format(lock_id), step="check-exist")
except Exception, e:
pass
logger.info('mq_work.code_check',
'added lock <{}> for [{}]'.format(lock_id, git_url))
logger.info(
'mq_work.code_check',
self.tenant_id + "=" + self.service_id + " start code check")
if self.event_id:
self.log.info("代码检测{0},{1} 开始".format(self.tenant_id,
self.service_id),
step="check-start")
cmd = '/bin/bash {0}/scripts/detect.sh {1} {2} "{3}" {4}'.format(
self.base_dir, self.tenant_id, self.service_id, git_url,
self.base_dir)
try:
output = Executer.call(cmd)
self.requestConsole(self.service_id, output[0].rstrip('\n'),
check_type, git_url, code_version,
git_project_id, code_from, url_repos)
if self.event_id:
self.log.info("代码检测完成,请重新部署", step="last", status="success")
except Executer.ExecException, e:
logger.info('mq_work.code_check', 'code check failed')
logger.info('mq_work.code_check', e)
logger.info('mq_work.code_check', e.output)
self.log.error("代码检测异常 {}".format(e),
step="callback",
status="failure")
def _tag(self, image_id, image):
cmd = "{2} tag {0} {1}".format(image_id, image, DOCKER_BIN)
logger.info("mq_work.image_manual", cmd)
shell.call(cmd)
def build_image(self):
# self.write_build_log(u"开始编译Dockerfile")
'''
insert_lines = [
'RUN which wget || (apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y wget) || (yum install -y wget)',
'RUN which curl || (apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y curl) || (yum install -y curl)',
'RUN mkdir -pv /opt/bin',
'ADD ./.goodrain/init /opt/bin/init',
'',
'RUN wget http://lang.goodrain.me/public/gr-listener -O /opt/bin/gr-listener -q && \\',
' chmod 755 /opt/bin/*',
'',
'RUN rm -rf /var/lib/dpkg/* /var/lib/apt/*',
'ENTRYPOINT ["/opt/bin/init"]',
]
'''
dockerfile = '{0}/{1}'.format(self.source_dir, 'Dockerfile')
update_items, entrypoint, cmd = self.get_dockerfile_items(dockerfile)
# 重新解析dockerfile
pdf = None
try:
self.log.info("开始解析Dockerfile", step="build_image")
pdf = ParseDockerFile(dockerfile)
except ValueError as e:
self.log.error(
"用户自定义的volume路径包含相对路径,必须为绝对路径!",
step="build_image",
status="failure")
logger.exception(e)
return False
except Exception as e:
self.log.error(
"解析Dockerfile发生异常", step="build_image", status="failure")
logger.exception(e)
s = self.repo_url
regex = re.compile(r'.*(?:\:|\/)([\w\-\.]+)/([\w\-\.]+)\.git')
m = regex.match(s)
account, project = m.groups()
_name = '_'.join([self.service_id[:12], account, project])
_tag = self.deploy_version
build_image_name = '{0}/{1}:{2}'.format(REGISTRY_DOMAIN, _name, _tag)
# image name must be lower
build_image_name = build_image_name.lower()
self.log.debug(
"构建镜像名称为{0}".format(build_image_name), step="build_image")
#build_image_name=""
no_cache = self.build_envs.pop('NO_CACHE', False)
if no_cache:
build_cmd = "{0} build -t {1} --no-cache .".format(
DOCKER_BIN, build_image_name)
else:
build_cmd = "{0} build -t {1} .".format(DOCKER_BIN,
build_image_name)
p = shell.start(build_cmd, cwd=self.source_dir)
while p.is_running():
line = p.readline()
self.log.debug(line, step="build_image")
for line in p.unread_lines:
self.log.debug(line, step="build_image")
if p.exit_with_err():
self.log.error(
"构建失败，请检查Debug日志排查！", step="build_image", status="failure")
return False
self.log.debug("镜像构建成功。开始推送", step="build_image", status="success")
try:
shell.call("{0} push {1}".format(DOCKER_BIN, build_image_name))
except shell.ExecException, e:
self.log.error(
"镜像推送失败。{}".format(e.message),
step="push_image",
status="failure")
return False
