def clone(self): self.log.info("开始拉取代码。。", step="build-worker") # code, output = shell.runsingle("git clone --branch master --depth 1 {0} {1}".format(self.repo_url, self.source_dir)) result = False num = 0 while num < 2: try: cmdstr = "timeout -k 9 {2} git clone {0} {1}".format( self.repo_url, self.source_dir, CLONE_TIMEOUT) if "github.com" in self.repo_url: cmdstr = "timeout -k 9 {2} git clone -c http.proxy=http://127.0.0.1:18888 {0} {1}".format( self.repo_url, self.source_dir, CLONE_TIMEOUT) shell.call(cmdstr) result = True break except shell.ExecException, e: num = num + 1 self.prepare() if num < 2: self.log.error("拉取代码发生错误,开始重试 {}".format(e.message), status="failure", step="worker-clone") else: self.log.error("拉取代码发生错误,部署停止 {}".format(e.message), status="failure", step="callback") logger.exception('build_work.main', e) result = False
def clone(self): self.log.info("开始拉取代码。。", step="build-worker") # code, output = shell.runsingle("git clone --branch master --depth 1 {0} {1}".format(self.repo_url, self.source_dir)) result = False num = 0 while num < 2: try: shell.call("timeout -k 9 {2} git clone {0} {1}".format( self.repo_url, self.source_dir, CLONE_TIMEOUT)) result = True break except shell.ExecException, e: num = num + 1 self.prepare() if num < 2: self.log.error( "拉取代码发生错误,开始重试 {}".format(e.message), status="failure", step="worker-clone") else: self.log.error( "拉取代码发生错误,部署停止 {}".format(e.message), status="failure", step="callback") logger.exception('build_work.main', e) result = False
def get_commit_info(self): try: output = shell.call( """git log -n 1 --pretty --format='{"hash":"%H","author":"%an","timestamp":%at}'""", self.source_dir) if type(output) is list: output = output[0] jdata = json.loads(output) output2 = shell.call("""git log -n 1 --pretty --format=%s""", self.source_dir) if type(output2) is list: subject = output2[0] jdata['subject'] = subject else: jdata['subject'] = 'unknown' return jdata except shell.ExecException, e: logger.exception('build_work.main', e) return "{}"
def get_commit_info(self): try: output = shell.call( """git log -n 1 --pretty --format='{"hash":"%H","author":"%an","timestamp":%at}'""", self.source_dir) if type(output) is list: output = output[0] jdata = json.loads(output) output2 = shell.call("""git log -n 1 --pretty --format=%s""", self.source_dir) if type(output2) is list: subject = output2[0] jdata['subject'] = subject else: jdata['subject'] = 'unknown' return jdata except shell.ExecException, e: logger.exception('build_work.main', e) return "{}"
def get_image_property(self, image, name): query_format = '{{.%s}}' % name try: output = shell.call("{2} inspect -f '{0}' {1}".format( query_format, image, DOCKER_BIN)) if output == '<no value>': return None else: return output[0].rstrip('\n') except shell.ExecException, e: logger.exception("mq_work.image_manual", e) return None
def get_image_property(self, image, name): query_format = '{{.%s}}' % name try: output = shell.call("{2} inspect -f '{0}' {1}".format( query_format, image, DOCKER_BIN)) if output == '<no value>': return None else: return output[0].rstrip('\n') except shell.ExecException, e: logger.exception("mq_work.app_image", e) return None
def rewrite_files(self, dockerfile, insert_lines, cmd, entrypoint): extend_lines = map(lambda x: x + '\n', insert_lines) try: f = open(dockerfile, 'r') lines = f.readlines() for line in lines: if line.startswith('ENTRYPOINT') or line.startswith('CMD'): lines.remove(line) lines.extend(extend_lines) f.close() f = open(dockerfile, 'w') f.writelines(lines) f.close() shutil.copytree('./lib/.goodrain', '{0}/.goodrain'.format(self.source_dir)) if entrypoint is not None: entrypoint_cmd = ' '.join(entrypoint) shell.call( '''sed -i -e 's#_type_#ENTRYPOINT#' -e 's#^_entrypoint_#'{0}'#' .goodrain/init''' .format(pipes.quote(entrypoint_cmd)), cwd=self.source_dir) if cmd is not None: shell.call( '''sed -i -e 's#^_cmd_#'{0}'#' .goodrain/init'''. format(pipes.quote(cmd)), cwd=self.source_dir) else: shell.call( '''sed -i -e 's#_type_#CMD#' -e 's#^_cmd_#'{0}'#' .goodrain/init''' .format(pipes.quote(cmd)), cwd=self.source_dir) return True except (shell.ExecException, OSError), e: logger.exception('build_work.main', e) return False
def rewrite_files(self, dockerfile, insert_lines, cmd, entrypoint): extend_lines = map(lambda x: x + '\n', insert_lines) try: f = open(dockerfile, 'r') lines = f.readlines() for line in lines: if line.startswith('ENTRYPOINT') or line.startswith('CMD'): lines.remove(line) lines.extend(extend_lines) f.close() f = open(dockerfile, 'w') f.writelines(lines) f.close() shutil.copytree('./lib/.goodrain', '{0}/.goodrain'.format(self.source_dir)) if entrypoint is not None: entrypoint_cmd = ' '.join(entrypoint) shell.call( '''sed -i -e 's#_type_#ENTRYPOINT#' -e 's#^_entrypoint_#'{0}'#' .goodrain/init'''. format(pipes.quote(entrypoint_cmd)), cwd=self.source_dir) if cmd is not None: shell.call( '''sed -i -e 's#^_cmd_#'{0}'#' .goodrain/init'''. format(pipes.quote(cmd)), cwd=self.source_dir) else: shell.call( '''sed -i -e 's#_type_#CMD#' -e 's#^_cmd_#'{0}'#' .goodrain/init'''. format(pipes.quote(cmd)), cwd=self.source_dir) return True except (shell.ExecException, OSError), e: logger.exception('build_work.main', e) return False
def build_image(self): # self.write_build_log(u"开始编译Dockerfile") ''' insert_lines = [ 'RUN which wget || (apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y wget) || (yum install -y wget)', 'RUN which curl || (apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y curl) || (yum install -y curl)', 'RUN mkdir -pv /opt/bin', 'ADD ./.goodrain/init /opt/bin/init', '', 'RUN wget http://lang.goodrain.me/public/gr-listener -O /opt/bin/gr-listener -q && \\', ' chmod 755 /opt/bin/*', '', 'RUN rm -rf /var/lib/dpkg/* /var/lib/apt/*', 'ENTRYPOINT ["/opt/bin/init"]', ] ''' dockerfile = '{0}/{1}'.format(self.source_dir, 'Dockerfile') update_items, entrypoint, cmd = self.get_dockerfile_items(dockerfile) # 重新解析dockerfile pdf = None try: self.log.info("开始解析Dockerfile", step="build_image") pdf = ParseDockerFile(dockerfile) except ValueError as e: self.log.error("用户自定义的volume路径包含相对路径,必须为绝对路径!", step="build_image", status="failure") logger.exception(e) return False except Exception as e: self.log.error("解析Dockerfile发生异常", step="build_image", status="failure") logger.exception(e) s = self.repo_url regex = re.compile(r'.*(?:\:|\/)([\w\-\.]+)/([\w\-\.]+)\.git') m = regex.match(s) account, project = m.groups() _name = '_'.join([self.service_id[:12], account, project]) _tag = self.deploy_version build_image_name = '{0}/{1}:{2}'.format(REGISTRY_DOMAIN, _name, _tag) # image name must be lower build_image_name = build_image_name.lower() self.log.debug("构建镜像名称为{0}".format(build_image_name), step="build_image") #build_image_name="" no_cache = self.build_envs.pop('NO_CACHE', False) if no_cache: build_cmd = "{0} build -t {1} --no-cache .".format( DOCKER_BIN, build_image_name) else: build_cmd = "{0} build -t {1} .".format(DOCKER_BIN, build_image_name) p = shell.start(build_cmd, cwd=self.source_dir) while p.is_running(): line = p.readline() self.log.debug(line, step="build_image") for line in p.unread_lines: self.log.debug(line, step="build_image") if p.exit_with_err(): self.log.error("构建失败,请检查Debug日志排查!", step="build_image", status="failure") return False self.log.debug("镜像构建成功。开始推送", step="build_image", status="success") try: shell.call("{0} push {1}".format(DOCKER_BIN, build_image_name)) except shell.ExecException, e: self.log.error("镜像推送失败。{}".format(e.message), step="push_image", status="failure") return False
def _tag(self, image_id, image): cmd = "{2} tag {0} {1}".format(image_id, image, DOCKER_BIN) logger.info("mq_work.image_manual", cmd) shell.call(cmd)
def build_image(self): # self.write_build_log(u"开始编译Dockerfile") ''' insert_lines = [ 'RUN which wget || (apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y wget) || (yum install -y wget)', 'RUN which curl || (apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y curl) || (yum install -y curl)', 'RUN mkdir -pv /opt/bin', 'ADD ./.goodrain/init /opt/bin/init', '', 'RUN wget http://lang.goodrain.me/public/gr-listener -O /opt/bin/gr-listener -q && \\', ' chmod 755 /opt/bin/*', '', 'RUN rm -rf /var/lib/dpkg/* /var/lib/apt/*', 'ENTRYPOINT ["/opt/bin/init"]', ] ''' dockerfile = '{0}/{1}'.format(self.source_dir, 'Dockerfile') update_items, entrypoint, cmd = self.get_dockerfile_items(dockerfile) # 重新解析dockerfile pdf = None try: self.log.info("开始解析Dockerfile", step="build_image") pdf = ParseDockerFile(dockerfile) except ValueError as e: self.log.error( "用户自定义的volume路径包含相对路径,必须为绝对路径!", step="build_image", status="failure") logger.exception(e) return False except Exception as e: self.log.error( "解析Dockerfile发生异常", step="build_image", status="failure") logger.exception(e) s = self.repo_url regex = re.compile(r'.*(?:\:|\/)([\w\-\.]+)/([\w\-\.]+)\.git') m = regex.match(s) account, project = m.groups() _name = '_'.join([self.service_id[:12], account, project]) _tag = self.deploy_version build_image_name = '{0}/{1}:{2}'.format(REGISTRY_DOMAIN, _name, _tag) # image name must be lower build_image_name = build_image_name.lower() self.log.debug( "构建镜像名称为{0}".format(build_image_name), step="build_image") #build_image_name="" no_cache = self.build_envs.pop('NO_CACHE', False) if no_cache: build_cmd = "{0} build -t {1} --no-cache .".format( DOCKER_BIN, build_image_name) else: build_cmd = "{0} build -t {1} .".format(DOCKER_BIN, build_image_name) p = shell.start(build_cmd, cwd=self.source_dir) while p.is_running(): line = p.readline() self.log.debug(line, step="build_image") for line in p.unread_lines: self.log.debug(line, step="build_image") if p.exit_with_err(): self.log.error( "构建失败,请检查Debug日志排查!", step="build_image", status="failure") return False self.log.debug("镜像构建成功。开始推送", step="build_image", status="success") try: shell.call("{0} push {1}".format(DOCKER_BIN, build_image_name)) except shell.ExecException, e: self.log.error( "镜像推送失败。{}".format(e.message), step="push_image", status="failure") return False
class CodeCheck(): watching_topics = ('service_event_msg', ) required_configs = ('region', 'userconsole', 'etcd.lock') def __init__(self, job, *args, **kwargs): self.job = job self.configs = kwargs.get("config") self.user_cs_client = UserConsoleAPI(conf=self.configs['userconsole']) self.region_client = RegionBackAPI() task = json.loads(self.job.body) self.base_dir = kwargs.get('base_dir') for k in ('tenant_id', 'service_id', 'action'): setattr(self, k, task[k]) if 'event_id' in task: self.event_id = task["event_id"] self.log = EventLog().bind(event_id=self.event_id) else: self.event_id = "system" self.log = EventLog().bind(event_id=self.event_id) self.task = task self.locker = TaskLocker(conf=self.configs['etcd']) # self.waittime = int(task['wait_time']) self.log.info(u"worker已收到异步任务。", step="worker") def do_work(self): logger.info('mq_work.service_event', "plugin %s execute start" % __name__) self.log.debug(u"代码检查异步处理开始。", step="worker", status="start") self.code_check() logger.info('mq_work.service_event', "plugin %s execute finished" % __name__) def code_check(self): git_url = self.task['git_url'] check_type = self.task['check_type'] code_version = self.task['code_version'] git_project_id = self.task['git_project_id'] code_from = self.task['code_from'] url_repos = self.task['url_repos'] lock_id = 'code_check.' + self.service_id logger.info( 'mq_work.code_check', "git_url {0},check_type {1}, code_version {2},git_project_id {3},code_from {4},url_repos {5} " .format(git_url, check_type, code_version, git_project_id, code_from, url_repos)) try: if self.locker.exists(lock_id): logger.info('mq_work.code_check', "lock_id {} exists, do nothing".format(lock_id)) self.log.info('lock_id {} exists, do nothing'.format(lock_id), step="check_exist") return self.locker.add_lock(lock_id, bytes(git_url)) logger.info('add lock_id {}'.format(lock_id), step="check-exist") except Exception, e: pass logger.info('mq_work.code_check', 'added lock <{}> for [{}]'.format(lock_id, git_url)) logger.info( 'mq_work.code_check', self.tenant_id + "=" + self.service_id + " start code check") if self.event_id: self.log.info("代码检测{0},{1} 开始".format(self.tenant_id, self.service_id), step="check-start") cmd = '/bin/bash {0}/scripts/detect.sh {1} {2} "{3}" {4}'.format( self.base_dir, self.tenant_id, self.service_id, git_url, self.base_dir) try: output = Executer.call(cmd) self.requestConsole(self.service_id, output[0].rstrip('\n'), check_type, git_url, code_version, git_project_id, code_from, url_repos) if self.event_id: self.log.info("代码检测完成,请重新部署", step="last", status="success") except Executer.ExecException, e: logger.info('mq_work.code_check', 'code check failed') logger.info('mq_work.code_check', e) logger.info('mq_work.code_check', e.output) self.log.error("代码检测异常 {}".format(e), step="callback", status="failure")
def _tag(self, image_id, image): cmd = "{2} tag {0} {1}".format(image_id, image, DOCKER_BIN) logger.info("mq_work.app_image", cmd) shell.call(cmd)