def modify_modules(self, commits):
LOG.big_log_start("Start parser module")
result = set()
self.modules = set()
for path, dirs, file_names in self.g:
#根目录全局打包
# if self.project_path == path:
# for file_name in file_names:
# if file_name in commits:
# LOG.debug(path)
# result.add(path)
# return result
if (ABYSSYAML in file_names
or ABYSSYML in file_names) and path not in self.modules:
LOG.debug(path)
self.modules.add(path)
# commits为空时，比如打tag
if commits is None or len(commits) < 1:
LOG.debug("tag 编译所有模块")
return self.modules
for commit in commits:
module_path = self.match_module(commit)
if module_path == self.project_path:
#如果是依赖模块就打包全部模块
return self.modules
if module_path not in result:
result.add(module_path)
return result
def login_aws(self, name):
result = subprocess.call(LOG.debug(
"$(aws ecr get-login --no-include-email --region ap-southeast-1 --profile {name})".format(name=name)), shell=True)
if result != 0:
LOG.error("login failed")
return False
return True
def docker_process(self, module):
if module == self.file_manager.WORKSPACE_BUILD:
short_module_name = 'All'
else:
short_module_name = module.replace(self.file_manager.WORKSPACE_BUILD+'/', '')
LOG.big_log_start("[{m}] Start Build".format(m=short_module_name))
self.abyss_config = ConfigParser(module, self.pipe)
# 真正的build ================================================================================================
for command in self.abyss_config.build():
# build_project = subprocess.Popen(['/bin/bash', '-c', LOG.debug(command)])
build_project = subprocess.call(LOG.debug(command), shell=True,
cwd=self.file_manager.WORKSPACE_BUILD, env=self.new_env)
if build_project != 0:
raise Exception("[{m}] Module build failed".format(m=short_module_name))
LOG.big_log_end("[{m}] Build Module Success".format(m=short_module_name))
self.release = self.abyss_config.deploy_release()
if isinstance(config[self.pipe], dict):
registry_config = config[self.pipe][self.release]
else:
registry_config = config[self.pipe]
LOG.debug("[{m}] Deploy release: {release}".format(m=short_module_name, release=self.release))
# 处理镜像 ================================================================================================
docker_worker = DockerWorker(
registry=registry_config.DOCKER_REGISTRY,
image=self.abyss_config.image(),
module_name=short_module_name
)
self.login_docker_repository(docker_worker, registry_config)
repo_name = registry_config.DOCKER_REGISTRY + "/" + self.abyss_config.repo()
if not docker_worker.tag(repo_name, self.git_worker.TAG):
raise Exception("[{m}] tag failed".format(m=short_module_name))
if not docker_worker.push(repo_name, self.git_worker.TAG):
raise Exception("[{m}] push failed".format(m=short_module_name))
if not docker_worker.tag(repo_name, 'latest'):
raise Exception("[{m}] tag latest failed".format(m=short_module_name))
if not docker_worker.push(repo_name, 'latest'):
raise Exception("[{m}] push latest failed".format(m=short_module_name))
def tag(self, repo, tag):
LOG.big_log_start('[{m}] Start TAG Docker Image'.format(m=self.module_name))
imageID = \
subprocess.check_output(LOG.debug('docker images -q {image}'.format(image=self.IMAGE)),
shell=True).decode('utf-8').split('\n')[0]
if imageID == "":
LOG.error("[{m}] Image not fond: ".format(m=self.module_name) + self.IMAGE)
return False
result = subprocess.call(LOG.debug(
'docker tag {imageID} {repo}:{tag}'.format(
imageID=imageID, repo=repo, tag=tag)), shell=True)
if result != 0:
LOG.error("Docker Tag failed for: ")
return False
LOG.big_log_end('Tag Success')
return True
def pull_code(self):
"""
从gitSource 拉取code
:return:
true: 代码拉取成功
false: 代码拉取失败
"""
LOG.big_log_start("Start Pull Code")
# clean
LOG.debug('Start clean workplace: ' + self.WORKPLACE)
shutil.rmtree(self.WORKPLACE)
os.mkdir(self.WORKPLACE)
LOG.debug('Start Downloading Source Code')
# clone
LOG.debug('git clone {repo}'.format(repo=self.GIT_URL))
clone = subprocess.call('git clone {repo}'.format(repo=self.GIT_URL),
shell=True,
cwd=self.WORKPLACE)
if clone != 0:
LOG.error("git clone failed")
return False
# find project
self.PROJECT_PATH = os.path.join(
self.WORKPLACE,
subprocess.check_output(
'ls -d [!_]*', shell=True,
cwd=self.WORKPLACE).decode('utf-8').split('\n')[0])
LOG.debug("Project path: " + self.PROJECT_PATH)
# checkout
LOG.debug('git checkout {coPoint}'.format(coPoint=self.BRANCH))
checkout = subprocess.call(
'git checkout {coPoint}'.format(coPoint=self.BRANCH),
shell=True,
cwd=self.PROJECT_PATH)
if checkout != 0:
LOG.error("git checkout failed")
return False
LOG.big_log_end("Pull Success")
return True
def pre_env(self):
# 准备环境 ============================================================================================
LOG.big_log_start("Start Build")
new_env = {}
new_env['env'] = self.pipe
new_env['pipe'] = self.pipe
new_env['pipe'] = self.pipe
commit = self.git_worker.get_commit()
new_env['version'] = self.git_worker.BRANCH.replace("/", "-") + "-" + commit[0]
new_env['commitId'] = commit[0]
new_env['commitTime'] = commit[1]
new_env['commitTimeFormat'] = commit[2]
new_env['commitMessage'] = commit[3]
for k in new_env:
LOG.debug("new_env[{key}] = {value}".format(key=k, value=new_env[k]))
new_env.update(os.environ.copy())
self.new_env = new_env
def login(self,account, password):
LOG.big_log_start('Start Login Docker Registry')
result = subprocess.call(LOG.debug(
'docker login -u {account} -p {password} {registry}'.format(
account=account,
password=password,
registry=self.REGISTRY)), shell=True)
if result != 0:
LOG.error("login failed")
return False
LOG.big_log_end('Login Success')
return True
def push(self, repo, tag):
"""
上传镜像阶段
:return: True or False PUSH是否成功
"""
LOG.big_log_start('[{m}] Start Push Docker Image'.format(m=self.module_name))
push_latest = subprocess.call(LOG.debug(
'docker push {repo}:{tag}'.format(
repo=repo, tag=tag)), shell=True)
if push_latest != 0:
LOG.error("[{m}] Docker push failed".format(m=self.module_name))
return False
LOG.big_log_end('[{m}] Push Success'.format(m=self.module_name))
return True
def copy_project(self, target):
"""
复制到指定目录
:param target: 指定目录
:return:
true: 复制成功
false: 复制失败
"""
LOG.big_log_start("Start copy project")
# clean
shutil.rmtree(target)
os.mkdir(target)
# copy
cp = subprocess.call(LOG.debug('cp -a . ' + target),
shell=True,
cwd=self.PROJECT_PATH)
if cp != 0:
LOG.error("code copy failed")
return False
LOG.big_log_end("Copy Success")
return True
def __init__(self, project_path, pipe):
config_path = os.path.join(project_path, "abyss.yaml")
LOG.debug("find abyss.yaml in "+config_path)
if not os.path.exists(config_path):
config_path = os.path.join(project_path, "abyss.yml")
LOG.debug("find abyss.yml in " + config_path)
if not os.path.exists(config_path):
LOG.error("abyss.yaml nonfound")
with open(config_path, 'r') as f:
for line in f.readlines():
LOG.debug(line.strip())
f = open(config_path, 'r')
self.CONFIG = yaml.load(f)
f.close()
self.pipe = pipe
if "WORKSPACE" in os.environ:
workplace = os.environ['WORKSPACE']
else:
LOG.error("Missing workplace")
sys.exit(1)
if "git_ssh_url" in os.environ:
git_url = os.environ['git_ssh_url']
else:
LOG.error("Missing git_ssh_url")
sys.exit(1)
if "ref" in os.environ:
git_ref = os.environ['ref']
else:
LOG.error("Missing git_ref")
sys.exit(1)
pipe = transfer(git_ref)
LOG.debug(pipe)
if not CIDocker(workplace=workplace,
git_url=git_url,
git_ref=git_ref,
pipe=pipe,
commits=commits).ci_process():
LOG.big_log_start("Jenkins Job Failed!")
sys.exit(1)
LOG.big_log_start("Jenkins Job Success!")