def notify_all_respon_person(self):
"""
notice all packages to person
"""
status_list = ["failed", "unresolvable"]
for status in status_list:
for proj in self.proj.split(','):
self._get_proj_status(proj, status)
if self.failed_pkglist:
with ThreadPoolExecutor(10) as executor:
for pkg in self.failed_pkglist:
executor.submit(self._get_pkg_owner_email, proj,
pkg, status)
if self.to_addr_list:
message = self._edit_email_content()
for x in range(5):
ret = self._send_email(message)
if ret == 0:
log.info("send email succeed !")
sys.exit(0)
else:
log.error("send email failed, Error:%s" % ret)
raise SystemExit("Failed to send email!")
else:
log.info("No build failed or unresolvable packages.")
def _deal_some_param(self, file_content):
"""
deal with some data and relation
return a dict
"""
pattern_string = ['.meta', '.prjconf', '/_service', '/_meta']
proj_list = os.listdir(os.path.join(self.obs_meta_path, "master"))
proj_list.remove("openEuler:Mainline:RISC-V")
for pattern in pattern_string:
cmd = 'echo "%s" | grep "%s$"' % (file_content.strip(), pattern)
if os.popen(cmd).read():
tmp = {}
log_type, branch_name, proj, pkg, new_proj, new_pkg, \
multi_version_dir = self._parse_git_log(file_content)
tmp["log_type"] = log_type
tmp["branch_name"] = branch_name
tmp["proj"] = proj
tmp["pkg"] = pkg
tmp["new_proj"] = new_proj
tmp["new_pkg"] = new_pkg
tmp["exist_flag"] = 0
tmp["multi_version_dir"] = multi_version_dir
for p in proj_list:
cmd = "osc ls %s 2>&1 | grep -q -Fx %s" % (p, pkg)
if os.system(cmd) == 0:
if p == proj:
log.info(
"package %s hava existed in obs project %s" %
(pkg, p))
tmp["exist_flag"] = 1
return tmp
else:
continue
def _clean(self):
"""
Cleanup the current directory of obs_meta
"""
cmd = "if [ -d obs_meta ];then rm -rf obs_meta && echo 'Finish clean the obs_meta';fi"
rm_result = os.popen(cmd).readlines()
log.info(rm_result)
def _check_service_meta(self, change_list):
"""
check the _service or _meta file in commit
"""
flag_list = []
for change in change_list:
if "_service" in change:
error_flag = self._distinguish_check_service(change)
flag_list.append(error_flag)
elif "OBS_PRJ_meta" in change and "prjconf" not in change:
error_flag1 = None
if "multi" in change or "Multi" in change:
error_flag1 = self._multi_name_check(change)
error_flag2 = self._check_pro_meta(change)
error_flag = error_flag1 or error_flag2
flag_list.append(error_flag)
elif "OBS_PRJ_meta" in change and "prjconf" in change:
continue
else:
log.info("There are no _service or _meta need to check")
os.chdir(self.current_path)
self._clean()
if "yes" in flag_list:
print("error_check_list:")
for error_service in self.check_error:
print(error_service)
raise SystemExit("*******PLEASE CHECK YOUR PR*******")
def __init__(self, **kwargs):
"""
obs_project: obs project name
repo: obs project repo where store all packages
arch:
pkgs:
"""
self.kwargs = kwargs
par = ParserConfigIni()
self.obs_project_repo_dict = par.get_obs_repos_dict()
self.obs_project_root_path = par.get_obs_prj_root_path()
self.obs_pkg_rpms_url = par.get_repos_dict()["obs_pkg_rpms"]
self.obs_project = self.kwargs["project"]
self.rpms_to_repo_path = None
self.old_pkg_rpms = None
self.repo = self.kwargs["repo"]
self.arch = self.kwargs["arch"]
self.pkgs = self.kwargs["pkglist"]
self.compare = self.kwargs["compare"]
self.old_pkg_rpms = {}
self._set_rpms_to_repo()
self.pex = Pexpect(self.kwargs["repo_server_user"],
self.kwargs["repo_server_ip"],
self.kwargs["repo_server_pwd"],
self.kwargs["repo_server_port"])
log.info(self.rpms_to_repo_path)
self.obs_pkg_rpms_files_dir = None
self._download_obs_pkg_rpms_file(self.obs_pkg_rpms_url, self.kwargs["gitee_user"], \
self.kwargs["gitee_pwd"])
self.obs_pkg_rpms_file = os.path.join(self.obs_pkg_rpms_files_dir, "repo_files", \
"%s_%s.yaml" % (self.obs_project.replace(":", "-"), self.arch))
self.get_old_rpms_list_from_file(self.obs_pkg_rpms_file)
def _modify_pkg_service(self, proj, pkg, branch_name):
"""
change the service file for the package
return 0 or -1
"""
service_file_path = os.path.join(self.obs_meta_path,
self.multi_version_dir, branch_name,
proj, pkg, "_service")
_tmpdir = os.popen("mktemp -d").read().strip('\n')
pkg_tmpdir = os.path.join(_tmpdir, proj, pkg)
cmd = "cd %s && osc co %s %s &>/dev/null && cd -" % (_tmpdir, proj,
pkg)
if os.system(cmd) == 0:
cmd = "cd %s && cp -f %s %s && osc add _service && osc ci -m 'modify by %s' && cd -" \
% (pkg_tmpdir, service_file_path, pkg_tmpdir, self.giteeUserName)
if os.system(cmd) == 0:
log.info("modify %s %s _service success!" % (proj, pkg))
shutil.rmtree(_tmpdir)
return 0
else:
log.error("modify %s %s _service failed!" % (proj, pkg))
shutil.rmtree(_tmpdir)
return -1
else:
log.warning("%s %s not found" % (proj, pkg))
shutil.rmtree(_tmpdir)
return -1
def _ensure_delete_tags(self, change_msg, yaml_old_msg, yaml_all_msg):
"""
verify the rpm added to delete tag exists in the previous file
"""
del_tag_rpm = {}
all_msg_rpm = {}
error_flag = False
change_file = change_msg.keys()
for change in change_file:
del_tag_rpm[change] = []
all_msg_rpm[change] = []
for msg in change_msg[change]:
if self._check_delete_tag(msg, yaml_all_msg[change]):
del_tag_rpm[change].append(msg['name'])
if not del_tag_rpm[change]:
del del_tag_rpm[change]
for msg in yaml_old_msg[change]:
all_msg_rpm[change].append(msg['name'])
if not del_tag_rpm:
return
log.info("The Pr contain the rpm change in delete tag for {0}".format(del_tag_rpm))
info_dict = {}
for change in del_tag_rpm.keys():
info_dict[change] = []
for rpm in del_tag_rpm[change]:
if rpm not in all_msg_rpm[change]:
error_flag = True
info_dict[change].append(rpm)
if not info_dict[change]:
del info_dict[change]
if error_flag:
log.error("Check the delete group in the {0}!!!".format(info_dict))
raise SystemExit("ERROR:Please check your PR")
def sync_yaml_meta(self):
"""
integration of functions
"""
change_file = self._get_change_file()
yaml_dict = {}
for line in change_file:
log.info("line:%s" % line)
name = list(line.split())[1]
branch = name.split('/')[0]
file_path = os.path.join(self.release_management_path, name)
yaml_dict = self._get_yaml_file_msg(file_path)
if not yaml_dict:
log.info("%s file content is empty!" % name)
else:
if "everything" in yaml_dict['packages'].keys():
msg, del_msg, prj_pkg = self._parse_yaml_msg(
yaml_dict, "new")
self._add_prj_meta_pkgs_service(msg)
else:
msg, del_msg, prj_pkg = self._parse_yaml_msg(
yaml_dict, "old")
self._add_prj_meta_pkgs_service(msg)
for tmp in del_msg:
self._del_pkg(tmp)
self._verify_meta_file(prj_pkg)
ret = self._push_code()
return ret
def _add_pkg_service(self, tmp):
"""
add obs_meta packages _service file
"""
from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'],
tmp['obs_from'], tmp['pkgname'])
pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'],
tmp['obs_to'], tmp['pkgname'])
pkg_service_path = os.path.join(pkg_path, "_service")
if tmp['branch_from'] == "master":
branch = "openEuler"
else:
branch = tmp['branch_from']
if not os.path.exists(pkg_path):
os.makedirs(pkg_path)
if not os.path.exists(pkg_service_path):
cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path)
if os.system(cmd) == 0:
cmd = "sed -i 's/%s\//%s\//g' %s/_service" % (
branch, tmp['branch_to'], pkg_path)
if os.system(cmd) == 0:
log.info("add %s %s %s _service succeed!" %
(tmp['branch_to'], tmp['obs_to'], tmp['pkgname']))
else:
log.info("add %s %s %s _service failed!" %
(tmp['branch_to'], tmp['obs_to'], tmp['pkgname']))
else:
log.error("copy %s service file failed!" % tmp['pkgname'])
def _parse_git_log(self, line):
"""
deal diff_patch line mesg
"""
log.info("line:%s" % line)
new_file_path = ''
complete_new_file_path = ''
log_list = list(line.split())
temp_log_type = log_list[0]
if len(log_list) == 3:
if log_list[1].split('/')[0] != log_list[2].split('/')[0] and \
log_list[1].split('/')[-2] == log_list[2].split('/')[-2]:
log.error("ERROR_COMMIT: %s" % line)
log.error("FAILED:PR contains the movement of same package \
between branches")
raise SystemExit("*******PLEASE CHECK YOUR PR*******")
else:
new_file_path = list(line.split())[2]
complete_new_file_path = list(line.split())[2]
elif len(log_list) == 2:
if temp_log_type != "D":
new_file_path = list(line.split())[1]
complete_new_file_path = list(line.split())[1]
log.info(new_file_path)
return new_file_path, complete_new_file_path
def _check_pkg_services(self, pkg_path):
"""
Check the format of the service file and the correctness of its URL
"""
service_name = []
service_branch = []
service_next = []
error_flag2 = ""
error_flag3 = ""
os.chdir("%s/obs_meta" % self.current_path)
error_flag1 = self._check_file_format(pkg_path)
if error_flag1 != "yes":
url_list = self._get_url_info(pkg_path)
for all_url in url_list:
service_next.append(all_url.split('/')[0])
service_name.append(all_url.split('/')[-1])
service_branch.append(all_url.split('/')[-2])
path_info = self._get_path_info(pkg_path)
log.info("Url_Next:%s" % service_next)
log.info("Pkgname_in_Service:%s" % service_name)
log.info("pkgbranch_in_Service:%s" % service_branch)
log.info("Pkgname_in_Obe_meta_path:%s" % path_info[0])
log.info("Pkgbranch_in_Obs_meta_path:%s" % path_info[1])
all_info = {'ser_branch':service_branch, 'pkg_url':path_info[1],
'ser_next':service_next, 'pkg_name':path_info[0],
'ser_name':service_name}
error_flag2 = self._check_correspond_from_service(**all_info)
if self.token:
error_flag3 = self._detect_protect_branch(pkg_path)
error_flag4 = self._one_pkg_one_branch(pkg_path)
error_flag = error_flag1 or error_flag2 or error_flag3 or error_flag4
if error_flag:
self.check_error.append(pkg_path)
return error_flag
def _clean(self, pkgname):
"""
remove the useless pkg dir
"""
cmd = "if [ -d {0} ];then rm -rf {0} && echo 'Finish clean the {0}';fi".format(pkgname)
rm_result = os.popen(cmd).readlines()
log.info(rm_result)
def _get_repo_change_file(self, owner=None, pkgname=None, repo_path=None):
"""
Obtain the change for latest commit
"""
changed_file_cmd = "git diff --name-status HEAD~1 HEAD~0"
fetch_cmd = "git fetch origin pull/%s/head:thispr" % self.prid
checkout_cmd = "git checkout thispr"
get_fetch = None
if not repo_path:
self._clean(pkgname)
release_path = self._get_latest_git_repo(owner, pkgname)
self.manage_path = release_path
os.chdir(release_path)
else:
os.chdir(repo_path)
for x in range(5):
fetch_result = os.system(fetch_cmd)
checkout_result = os.system(checkout_cmd)
log.debug("STATUS:{0} and {1}".format(fetch_result, checkout_result))
if fetch_result == 0 and checkout_result == 0:
get_fetch = True
break
else:
os.chdir(self.current_path)
self._clean(pkgname)
path_release = self._get_latest_git_repo(owner, pkgname)
os.chdir(path_release)
self.manage_path = os.path.join(self.current_path, pkgname)
changed_file = os.popen(changed_file_cmd).readlines()
if get_fetch and changed_file:
log.info(changed_file)
return changed_file
else:
raise SystemExit("Error:can not obtain the content for this commit")
def _get_obs_project(self):
"""
Get the obs project from gitee_branch
"""
log.info("Start get the obs_project")
if "Multi-Version" in self.gitee_branch:
path = self.meta_path + '/' + 'multi_version/' + self.gitee_branch
log.info(path)
cmd = "find %s -name %s | awk -F '/' '{print $5}'" % (path, self.repository)
else:
path = self.meta_path + '/' + self.gitee_branch
log.info(path)
cmd = "find %s -name %s | awk -F '/' '{print $4}'" % (path, self.repository)
all_project = os.popen(cmd).readlines()
log.info(all_project)
obs_project = None
for project in all_project:
obs_project = project.replace('\n', '')
if ":Bak" in obs_project:
continue
else:
break
if obs_project and ":Bak" not in obs_project:
log.info("The %s obs_project for gitee_%s is <<<<%s>>>>" % (
self.repository, self.gitee_branch, obs_project))
return obs_project
else:
log.error("Failed !!! The rpm is not exist in %s branch !!!"
% self.gitee_branch)
return False
def _check_branch_msg(self, change_msg, yaml_path_list, manage_path):
"""
check the branch msg in your commit
"""
error_msg = {}
branch_msg_path = os.path.join(manage_path, "valid_release_branches.yaml")
with open(branch_msg_path, 'r', encoding='utf-8') as f:
branch_result = yaml.load(f, Loader=yaml.FullLoader)
for yaml_path in yaml_path_list:
log.info("{0} branch check".format(yaml_path))
error_msg[yaml_path] = []
yaml_branch = yaml_path.split("/")[-2]
for msg in change_msg[yaml_path]:
if msg['branch_to'] == yaml_branch and \
msg['branch_from'] in branch_result['branch'].keys() and \
msg['branch_to'] in branch_result['branch'][msg['branch_from']]:
continue
else:
error_msg[yaml_path].append(msg)
if not error_msg[yaml_path]:
del error_msg[yaml_path]
else:
log.error("Wrong branch msg in there:")
for msg in error_msg[yaml_path]:
log.error(msg)
if error_msg:
return True
else:
return False
def _check_rpms_integrity(self, old_pack_msg, new_pack_msg, yaml_path_list):
"""
ensure the rpms exist in all the tags
"""
old_pkg = {}
new_pkg = {}
error_pkg = {}
log.info("rpms exists check")
for change_file in yaml_path_list:
old_pkg[change_file] = []
new_pkg[change_file] = []
for msg in old_pack_msg[change_file]:
old_pkg[change_file].append(msg['name'])
for msg in new_pack_msg[change_file]:
new_pkg[change_file].append(msg['name'])
for change_file in yaml_path_list:
error_pkg[change_file] = []
for pkg in old_pkg[change_file]:
if pkg not in new_pkg[change_file]:
error_pkg[change_file].append(pkg)
if not error_pkg[change_file]:
del error_pkg[change_file]
if error_pkg:
log.error("May be {0} should not be delete".format(error_pkg))
raise SystemExit("ERROR: Please check your PR")
def sync_code(self):
"""
sync not_same_packages code
"""
if self.not_same_packages:
log.info("Start synchronization code...")
self.kwargs['pkglist'] = self.not_same_packages
sy = SYNCCode(**self.kwargs)
sy.sync_code_to_obs()
def _del_pkg(self, tmp):
"""
delete obs_meta packages
"""
pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'],
tmp['obs_to'], tmp['pkgname'])
if os.path.exists(pkg_path):
shutil.rmtree(pkg_path)
log.info("delete %s %s %s succeed!" %
(tmp['branch_to'], tmp['obs_to'], tmp['pkgname']))
def _rollback_get_msg(self, repo_path):
"""
rollback to last commit
"""
os.chdir(repo_path)
roll = os.system("git reset --hard HEAD^")
if roll == 0:
log.info("Already rollback to last commit")
else:
raise SystemExit("Error: fail to rollback to last commit")
def _get_proj_status(self, proj, status):
"""
get proj build results
"""
cmd = "osc r --csv %s 2>/dev/null | grep %s | awk -F ';' '{print $1}'" % (
proj, status)
self.failed_pkglist = [
x for x in os.popen(cmd).read().split('\n') if x != ''
]
log.info("%s build %s packages:%s" %
(proj, status, self.failed_pkglist))
def _echo_to_obs_pkg_rpms(self, all_time):
"""
echo the time str to a package
all_time: the list for rpm name and timestr
"""
echo_cmd = "echo %s > ./obs_pkg_rpms/%s/%s" % (all_time[1], \
self.branch, all_time[0])
log.info(echo_cmd)
cmd_result = os.system(echo_cmd)
if cmd_result != 0:
raise SystemExit("This %s echo error to %s" % (all_time[0], self.branch))
def _sync_pkg_code(self, proj, pkg, branch_name):
"""
when adding a new package to a project, sync code
"""
os.chdir(self.kwargs["init_path"])
log.info("Start synchronization code...")
self.kwargs['project'] = proj
self.kwargs['repository'] = pkg
self.kwargs['branch'] = branch_name
sy = SYNCCode(**self.kwargs)
sy.sync_code_to_obs()
def check(self, package):
"""
check one package
"""
gitee_spec = self.get_gitee_spec(self.branch, package)
obs_spec = self.get_obs_spec(self.project, package)
if self.same_or_not(gitee_spec, obs_spec, package):
log.info("codes of %s are same between gitee and obs" % package)
else:
self.not_same_packages.append(package)
log.info("codes of %s are not same between gitee and obs" % package)
def _pkgname_check(self, pkg_name, service_name_list):
"""
check the pkg_name from change_path and service_name_list
"""
if pkg_name in service_name_list:
log.info("SUCCESS_CHECK:The %s in _service url is correct" % pkg_name)
return
else:
log.error("**************_Service URL ERROR*****************")
log.error("FAILED:The %s in _service pkgname is not same" % pkg_name)
error_flag = "yes"
return error_flag
def _get_latest_git_repo(self, owner, pkgname):
"""
get the latest git repo
"""
os.chdir(self.current_path)
rpm_url = "https://gitee.com/{0}/{1}".format(owner, pkgname)
pkg_path = git_repo_src(rpm_url, self.giteeuser, self.giteeuserpwd)
if pkg_path:
log.info("{0}:{1}".format(pkgname, pkg_path))
return pkg_path
else:
raise SystemExit("Error:{0} Clone-error,Please check yournet.".format(pkgname))
def check_all(self):
"""
check all packages
"""
if self.packages:
pool = threadpool.ThreadPool(30)
reqs = threadpool.makeRequests(self.check, self.packages)
for req in reqs:
pool.putRequest(req)
pool.wait()
log.info("codes not same between gitee and obs:%s" % self.not_same_packages)
self.sync_code()
def _get_multi_branch_and_project(self, multi_change):
"""
get all the multi_version branch and one of the corresponding projects
"""
pro_dir = "multi_version/%s" % multi_change.split('/')[1]
branchs = os.listdir(os.path.join(self.current_path, \
"obs_meta", "OBS_PRJ_meta/multi_version"))
projects = os.listdir(os.path.join(self.current_path, \
"obs_meta", "OBS_PRJ_meta", pro_dir))
log.info("multi_branch_list:%s" % branchs)
log.info("multi_project_list:%s" % projects)
multi_info = {'branchs':branchs, 'projects':projects}
return multi_info
def _del_pkg(self, proj, pkg):
"""
delete the project package
return 0 or -1
"""
cmd = "osc api -X DELETE /source/%s/%s" % (proj, pkg)
ret = os.popen(cmd).read()
if "<summary>Ok</summary>" in ret:
log.info("delete %s %s success!" % (proj, pkg))
return 0
else:
log.error("delete %s %s failed!" % (proj, pkg))
return -1
def _del_obs_pkg_service(self, proj, pkg):
"""
delete the obs project package service file
return 0 or -1
"""
cmd = "osc api -X DELETE /source/%s/%s/_service" % (proj, pkg)
ret = os.popen(cmd).read()
if "<summary>Ok</summary>" in ret:
log.info("delete %s %s _service by %s successful!" %
(proj, pkg, self.giteeUserName))
return 0
else:
log.error("delete %s %s _service failed!" % (proj, pkg))
return -1
def same_or_not(self, gitee_spec, obs_spec, package):
"""
check spec file between gitee spec file and obs spec file
return: False - not same; True - same
"""
if gitee_spec and obs_spec:
for ospec in obs_spec:
sf = ospec.split(":")[-1]
index = gitee_spec.index(sf)
log.info("gitee spec:%s, obs spec:%s" % (gitee_spec[index], ospec))
cmd = "diff %s %s" % (gitee_spec[index], ospec)
log.info(cmd)
ret = os.popen(cmd).read()
cmd = "rm -rf %s %s" % (gitee_spec[index], ospec)
os.system(cmd)
if ret:
log.info("diff-" + ret)
return False
return True
elif not gitee_spec and not obs_spec:
log.info("%s has no script" % package)
return True
elif not gitee_spec:
log.error("%s no spec from gitee" % package)
return False
elif not obs_spec:
log.error("%s no spec from obs" % package)
return False