def _ensure_delete_tags(self, change_msg, yaml_old_msg, yaml_all_msg):
"""
verify the rpm added to delete tag exists in the previous file
"""
del_tag_rpm = {}
all_msg_rpm = {}
error_flag = False
change_file = change_msg.keys()
for change in change_file:
del_tag_rpm[change] = []
all_msg_rpm[change] = []
for msg in change_msg[change]:
if self._check_delete_tag(msg, yaml_all_msg[change]):
del_tag_rpm[change].append(msg['name'])
if not del_tag_rpm[change]:
del del_tag_rpm[change]
for msg in yaml_old_msg[change]:
all_msg_rpm[change].append(msg['name'])
if not del_tag_rpm:
return
log.info("The Pr contain the rpm change in delete tag for {0}".format(del_tag_rpm))
info_dict = {}
for change in del_tag_rpm.keys():
info_dict[change] = []
for rpm in del_tag_rpm[change]:
if rpm not in all_msg_rpm[change]:
error_flag = True
info_dict[change].append(rpm)
if not info_dict[change]:
del info_dict[change]
if error_flag:
log.error("Check the delete group in the {0}!!!".format(info_dict))
raise SystemExit("ERROR:Please check your PR")
def _parse_git_log(self, line):
"""
deal diff_patch line mesg
"""
log.info("line:%s" % line)
new_file_path = ''
complete_new_file_path = ''
log_list = list(line.split())
temp_log_type = log_list[0]
if len(log_list) == 3:
if log_list[1].split('/')[0] != log_list[2].split('/')[0] and \
log_list[1].split('/')[-2] == log_list[2].split('/')[-2]:
log.error("ERROR_COMMIT: %s" % line)
log.error("FAILED:PR contains the movement of same package \
between branches")
raise SystemExit("*******PLEASE CHECK YOUR PR*******")
else:
new_file_path = list(line.split())[2]
complete_new_file_path = list(line.split())[2]
elif len(log_list) == 2:
if temp_log_type != "D":
new_file_path = list(line.split())[1]
complete_new_file_path = list(line.split())[1]
log.info(new_file_path)
return new_file_path, complete_new_file_path
def notify_all_respon_person(self):
"""
notice all packages to person
"""
status_list = ["failed", "unresolvable"]
for status in status_list:
for proj in self.proj.split(','):
self._get_proj_status(proj, status)
if self.failed_pkglist:
with ThreadPoolExecutor(10) as executor:
for pkg in self.failed_pkglist:
executor.submit(self._get_pkg_owner_email, proj,
pkg, status)
if self.to_addr_list:
message = self._edit_email_content()
for x in range(5):
ret = self._send_email(message)
if ret == 0:
log.info("send email succeed !")
sys.exit(0)
else:
log.error("send email failed, Error:%s" % ret)
raise SystemExit("Failed to send email!")
else:
log.info("No build failed or unresolvable packages.")
def _get_obs_project(self):
"""
Get the obs project from gitee_branch
"""
log.info("Start get the obs_project")
if "Multi-Version" in self.gitee_branch:
path = self.meta_path + '/' + 'multi_version/' + self.gitee_branch
log.info(path)
cmd = "find %s -name %s | awk -F '/' '{print $5}'" % (path, self.repository)
else:
path = self.meta_path + '/' + self.gitee_branch
log.info(path)
cmd = "find %s -name %s | awk -F '/' '{print $4}'" % (path, self.repository)
all_project = os.popen(cmd).readlines()
log.info(all_project)
obs_project = None
for project in all_project:
obs_project = project.replace('\n', '')
if ":Bak" in obs_project:
continue
else:
break
if obs_project and ":Bak" not in obs_project:
log.info("The %s obs_project for gitee_%s is <<<<%s>>>>" % (
self.repository, self.gitee_branch, obs_project))
return obs_project
else:
log.error("Failed !!! The rpm is not exist in %s branch !!!"
% self.gitee_branch)
return False
def _check_branch_msg(self, change_msg, yaml_path_list, manage_path):
"""
check the branch msg in your commit
"""
error_msg = {}
branch_msg_path = os.path.join(manage_path, "valid_release_branches.yaml")
with open(branch_msg_path, 'r', encoding='utf-8') as f:
branch_result = yaml.load(f, Loader=yaml.FullLoader)
for yaml_path in yaml_path_list:
log.info("{0} branch check".format(yaml_path))
error_msg[yaml_path] = []
yaml_branch = yaml_path.split("/")[-2]
for msg in change_msg[yaml_path]:
if msg['branch_to'] == yaml_branch and \
msg['branch_from'] in branch_result['branch'].keys() and \
msg['branch_to'] in branch_result['branch'][msg['branch_from']]:
continue
else:
error_msg[yaml_path].append(msg)
if not error_msg[yaml_path]:
del error_msg[yaml_path]
else:
log.error("Wrong branch msg in there:")
for msg in error_msg[yaml_path]:
log.error(msg)
if error_msg:
return True
else:
return False
def same_or_not(self, gitee_spec, obs_spec, package):
"""
check spec file between gitee spec file and obs spec file
return: False - not same; True - same
"""
if gitee_spec and obs_spec:
for ospec in obs_spec:
sf = ospec.split(":")[-1]
index = gitee_spec.index(sf)
log.info("gitee spec:%s, obs spec:%s" % (gitee_spec[index], ospec))
cmd = "diff %s %s" % (gitee_spec[index], ospec)
log.info(cmd)
ret = os.popen(cmd).read()
cmd = "rm -rf %s %s" % (gitee_spec[index], ospec)
os.system(cmd)
if ret:
log.info("diff-" + ret)
return False
return True
elif not gitee_spec and not obs_spec:
log.info("%s has no script" % package)
return True
elif not gitee_spec:
log.error("%s no spec from gitee" % package)
return False
elif not obs_spec:
log.error("%s no spec from obs" % package)
return False
def _get_yaml_msg(self, yaml_path_list, manage_path, rollback=None):
"""
get the pkg msg in pckg-mgmt.yaml
"""
error_pkg = {}
error_flag = False
if rollback == True:
self._rollback_get_msg(manage_path)
all_pack_msg = {}
for yaml_path in yaml_path_list:
file_path = os.path.join(manage_path, yaml_path)
if os.path.exists(file_path):
with open(file_path, 'r', encoding='utf-8')as f:
result = yaml.load(f, Loader=yaml.FullLoader)
if "natural" in result['packages'].keys():
all_pack_msg[yaml_path] = result['packages']['natural'] + \
result['packages']['recycle'] + result['packages']['delete']
else:
all_pack_msg[yaml_path] = result['packages']['everything']['baseos'] + \
result['packages']['everything']['other'] + result['packages']['epol'] + \
result['packages']['recycle'] + result['packages']['delete']
else:
all_pack_msg[yaml_path] = []
error_pkg[yaml_path] = []
for pkg_info in all_pack_msg[yaml_path]:
if ' ' in pkg_info['name']:
error_name = pkg_info['name']
error_flag = True
error_pkg[yaml_path].append(error_name)
if error_flag:
log.error("as follows pkgs {0} space in name".format(error_pkg))
raise SystemExit("ERROR: Please check yaml pkg name")
return all_pack_msg
def _check_rpms_integrity(self, old_pack_msg, new_pack_msg, yaml_path_list):
"""
ensure the rpms exist in all the tags
"""
old_pkg = {}
new_pkg = {}
error_pkg = {}
log.info("rpms exists check")
for change_file in yaml_path_list:
old_pkg[change_file] = []
new_pkg[change_file] = []
for msg in old_pack_msg[change_file]:
old_pkg[change_file].append(msg['name'])
for msg in new_pack_msg[change_file]:
new_pkg[change_file].append(msg['name'])
for change_file in yaml_path_list:
error_pkg[change_file] = []
for pkg in old_pkg[change_file]:
if pkg not in new_pkg[change_file]:
error_pkg[change_file].append(pkg)
if not error_pkg[change_file]:
del error_pkg[change_file]
if error_pkg:
log.error("May be {0} should not be delete".format(error_pkg))
raise SystemExit("ERROR: Please check your PR")
def rpms_exists(self, rpms_list):
"""
check rpms exists
rpms_list:
"""
try:
tmp_list = [
rpms_list[i:i + 500] for i in range(0, len(rpms_list), 500)
]
for l in tmp_list:
cmd = """
for r in %s
do
ls %s/%s/%s/:full/ | grep "$r"
if [ $? -ne 0 ];then
echo "notfind"
break
else
echo "find"
fi
done
""" % (' '.join(l), self.obs_project_root_path, self.rpms_to_repo_path,
self.arch)
ret = self.pex.ssh_cmd(cmd)
if "notfind" in str(ret) or "find" not in str(ret):
return False
except Exception as e:
log.error(e)
return False
return True
def _modify_pkg_service(self, proj, pkg, branch_name):
"""
change the service file for the package
return 0 or -1
"""
service_file_path = os.path.join(self.obs_meta_path,
self.multi_version_dir, branch_name,
proj, pkg, "_service")
_tmpdir = os.popen("mktemp -d").read().strip('\n')
pkg_tmpdir = os.path.join(_tmpdir, proj, pkg)
cmd = "cd %s && osc co %s %s &>/dev/null && cd -" % (_tmpdir, proj,
pkg)
if os.system(cmd) == 0:
cmd = "cd %s && cp -f %s %s && osc add _service && osc ci -m 'modify by %s' && cd -" \
% (pkg_tmpdir, service_file_path, pkg_tmpdir, self.giteeUserName)
if os.system(cmd) == 0:
log.info("modify %s %s _service success!" % (proj, pkg))
shutil.rmtree(_tmpdir)
return 0
else:
log.error("modify %s %s _service failed!" % (proj, pkg))
shutil.rmtree(_tmpdir)
return -1
else:
log.warning("%s %s not found" % (proj, pkg))
shutil.rmtree(_tmpdir)
return -1
def _add_pkg_service(self, tmp):
"""
add obs_meta packages _service file
"""
from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'],
tmp['obs_from'], tmp['pkgname'])
pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'],
tmp['obs_to'], tmp['pkgname'])
pkg_service_path = os.path.join(pkg_path, "_service")
if tmp['branch_from'] == "master":
branch = "openEuler"
else:
branch = tmp['branch_from']
if not os.path.exists(pkg_path):
os.makedirs(pkg_path)
if not os.path.exists(pkg_service_path):
cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path)
if os.system(cmd) == 0:
cmd = "sed -i 's/%s\//%s\//g' %s/_service" % (
branch, tmp['branch_to'], pkg_path)
if os.system(cmd) == 0:
log.info("add %s %s %s _service succeed!" %
(tmp['branch_to'], tmp['obs_to'], tmp['pkgname']))
else:
log.info("add %s %s %s _service failed!" %
(tmp['branch_to'], tmp['obs_to'], tmp['pkgname']))
else:
log.error("copy %s service file failed!" % tmp['pkgname'])
def get_old_rpms_list_from_file(self, file_path):
"""
get old pkg rpms dict
file_path: yaml file which store all packages's rpms
"""
if os.path.exists(file_path):
with open(file_path, "r") as f:
self.old_pkg_rpms = yaml.load(f, Loader=yaml.FullLoader)
else:
log.error("no file for get rpms")
def _pkgname_check(self, pkg_name, service_name_list):
"""
check the pkg_name from change_path and service_name_list
"""
if pkg_name in service_name_list:
log.info("SUCCESS_CHECK:The %s in _service url is correct" % pkg_name)
return
else:
log.error("**************_Service URL ERROR*****************")
log.error("FAILED:The %s in _service pkgname is not same" % pkg_name)
error_flag = "yes"
return error_flag
def _get_change_file(self):
"""
get release-managemnet change file
"""
if os.path.exists(self.release_management_path):
os.chdir(self.release_management_path)
cmd = "git diff --name-status HEAD~1 HEAD~0 | grep pckg-mgmt.yaml"
result = os.popen(cmd).read().split('\n')
change_file = [x for x in result if x != '']
return change_file
else:
log.error("%s not exist!" % self.release_management_path)
sys.exit(1)
def _del_pkg(self, proj, pkg):
"""
delete the project package
return 0 or -1
"""
cmd = "osc api -X DELETE /source/%s/%s" % (proj, pkg)
ret = os.popen(cmd).read()
if "<summary>Ok</summary>" in ret:
log.info("delete %s %s success!" % (proj, pkg))
return 0
else:
log.error("delete %s %s failed!" % (proj, pkg))
return -1
def _obs_meta_action(self):
"""
action basis on change of obs_meta
return:
"""
log.debug("obs_meta change")
if not self.kwargs["obs_meta_path"]:
log.error("can not find obs_meta path")
else:
obs_prjm = OBSPrjManager(self.kwargs["obs_meta_path"])
obs_prjm.manager_action()
obs_pkgm = OBSPkgManager(**self.kwargs)
obs_pkgm.obs_pkg_admc()
def _check_yaml_format(self, yaml_path_list, manage_path):
"""
check the format for the yaml file
"""
for yaml_path in yaml_path_list:
manage_yaml_path = os.path.join(manage_path, yaml_path)
try:
with open(manage_yaml_path, 'r', encoding='utf-8') as f:
result = yaml.load(f, Loader=yaml.FullLoader)
log.info("{0} format check".format(yaml_path))
except Exception as e:
log.error("**********FORMAT ERROR***********")
log.error("%s format bad Because:%s" % (yaml_path, e))
raise SystemExit("May be %s has a bad format" % yaml_path)
def backup_old_rpms_by_pkg(self, pkg, rpms_list):
"""
backup old rpms by package name
pkg: name of packages
rpms_list: all rpms of package, type is list
"""
try:
t = time.strftime("%Y-%m-%d-%H-%M", time.localtime())
backup_dir = os.path.join(self.obs_project_root_path,
self.rpms_to_repo_path, "backup")
pkg_bak = os.path.join(backup_dir, "%s-%s" % (pkg, t))
self.backup_old_rpm(backup_dir, pkg, pkg_bak, rpms_list)
except ValueError as e:
log.error(e)
return False
except SystemError as e:
log.error(e)
return False
except TypeError as e:
log.error(e)
return False
except KeyError as e:
log.error(e)
return False
return True
def _exist_for_pro_and_repo(self, project, repository):
"""
check the project and repository is or not exist in obs
"""
error_flag = None
exit_cmd = "osc r %s --xml 2>/dev/null | grep %s" % (project, repository)
exit_result = os.popen(exit_cmd).read()
log.info(exit_result)
if not exit_result:
error_flag = "yes"
log.error("Failed:The %s Not exit in %s" % (repository, project))
else:
log.info("Success:The %s exit in %s" % (repository, project))
return error_flag
def _gitee_pr_to_obs(self, obs_pro):
"""
obs_pro:The obs project that gitee branch corresponds
"""
osc_service = os.popen("osc service remoterun %s %s" % (obs_pro, self.repository)).readlines()
for results in osc_service:
if "ok" in results:
log.info(results)
log.info("Success for osc service remoterun the %s" % self.repository)
return True
else:
continue
log.error("Failed !!! fail to service remoterun !!!")
return False
def _del_obs_pkg_service(self, proj, pkg):
"""
delete the obs project package service file
return 0 or -1
"""
cmd = "osc api -X DELETE /source/%s/%s/_service" % (proj, pkg)
ret = os.popen(cmd).read()
if "<summary>Ok</summary>" in ret:
log.info("delete %s %s _service by %s successful!" %
(proj, pkg, self.giteeUserName))
return 0
else:
log.error("delete %s %s _service failed!" % (proj, pkg))
return -1
def _modify_pkg_meta(self, proj, pkg, branch_name):
"""
change the package of the meta
return 0 or -1
"""
file_path = os.path.join(self.obs_meta_path, self.multi_version_dir,
branch_name, proj, pkg, ".osc/_meta")
_tmpdir = os.popen("mktemp -d").read().strip('\n')
cmd = "cd %s && osc meta pkg %s %s --file=%s | grep ^Done. && cd -" % (
_tmpdir, proj, pkg, file_path)
if os.system(cmd) != 0:
log.error("modify %s %s _meta failed!" % (proj, pkg))
shutil.rmtree(_tmpdir)
return -1
shutil.rmtree(_tmpdir)
def do_all(self):
"""
make the get obs_meta change fuction and check fuction doing
"""
if self.prid and self.token:
release_management_data = self._release_management_tree()
change_result, complete_change_result = self._get_all_change_file()
pr_check_result = self._check_pr_rule(release_management_data, complete_change_result)
check_result = self._check_service_meta(change_result)
elif not self.prid and self.branch:
self._clean()
self._get_latest_obs_meta()
self._check_branch_service(self.branch)
else:
log.error("ERROR_INPUT:PLEASE CHECK YOU INPUT")
def _pre_sync_code(self, project=None):
"""
The way to offer that make all fuction runing
"""
if not project:
obs_project = self._get_obs_project()
else:
obs_project = project
log.info("The %s in %s" % (self.repository, obs_project))
pull_result = self._get_latest_gitee_pull()
remoterun_result = self._gitee_pr_to_obs(obs_project)
if obs_project and pull_result and remoterun_result:
log.info("SYNC %s in %s SUCCESS" % (self.repository, obs_project))
return True
else:
log.error("SYNC %s in %s ERROR: Please check the log.error" % (self.repository, obs_project))
return False
def _check_pckg_yaml(self, file_msg, branch):
"""
check pckg-mgmt.yaml file
"""
proj = branch.replace("-", ":")
flag = False
for title in file_msg['packages']:
for msg in file_msg['packages'][title]:
if msg['branch_to'] != branch:
flag = True
log.error("%s branch_to is error, please check yaml." %
msg['name'])
if not msg['obs_to'].startswith(proj):
flag = True
log.error("%s obs_to is error, please check yaml." %
msg['name'])
return flag
def update_pkg(self, pkg):
"""
update one package
pkg: name of package
"""
if pkg in self.old_pkg_rpms.keys():
old_rpms_list = self.old_pkg_rpms[pkg]
old_rpms_list.sort()
else:
old_rpms_list = None
new_rpms_list = self.get_new_rpms_by_pkg(pkg)
new_rpms_list.sort()
if self.kwargs["all"] == "True" and new_rpms_list:
self.backup_old_rpms_by_pkg(pkg, old_rpms_list)
self.copy_new_rpms_to_repo(pkg, new_rpms_list)
elif old_rpms_list != new_rpms_list and new_rpms_list:
try:
self.backup_old_rpms_by_pkg(pkg, old_rpms_list)
self.copy_new_rpms_to_repo(pkg, new_rpms_list)
except Exception as e:
self.backup_old_rpms_by_pkg(pkg, new_rpms_list)
elif self.kwargs["compare"] == "True":
if not self.compare_rpms(pkg):
self.backup_old_rpms_by_pkg(pkg, old_rpms_list)
self.copy_new_rpms_to_repo(pkg, new_rpms_list)
else:
log.debug("%s all rpms are latest should do nothing" % pkg)
if new_rpms_list:
for i in range(5):
try:
if not self.rpms_exists(new_rpms_list):
log.debug(
"check %s rpms not exists, shoud copy again" % pkg)
self.copy_new_rpms_to_repo(pkg, new_rpms_list)
else:
log.debug("check %s all rpms exists" % pkg)
break
except Exception as e:
log.error(e)
else:
cmd = "osc list %s | grep ^%s" % (self.obs_project, pkg)
ret = os.popen(cmd).read().split("\n")
if pkg not in ret:
self.backup_old_rpms_by_pkg(pkg, old_rpms_list)
self.old_pkg_rpms[pkg] = new_rpms_list
def _check_pkg_from(self, meta_path, yaml_msg, change_file, yaml_all_msg):
"""
Detects the existence of file contents
"""
error_flag = False
for change in change_file:
log.info("{0} pkg_from check".format(change))
for msg in yaml_msg[change]:
delete_tag = self._check_delete_tag(msg, yaml_all_msg[change])
if delete_tag:
continue
msg_path = os.path.join(meta_path, msg['branch_from'],
msg['obs_from'], msg['name'])
if not os.path.exists(msg_path):
yaml_key = os.path.join(msg['branch_from'],
msg['obs_from'], msg['name'])
log.error("The {0} not exist in obs_meta".format(yaml_key))
error_flag = True
return error_flag
def sync_code_to_obs(self):
"""
The only way to offer that make all fuction runing
"""
if not self.pkgs:
if self.repository and self.gitee_branch:
if not self._pre_sync_code(self.project):
raise SystemExit("SYNC %s ERROR" % self.repository)
else:
raise SystemExit('please check you arguments')
else:
if "broken" == self.pkgs[0] and len(self.pkgs) == 1:
project_flag = "yes"
broken_cmd = "osc r --csv %s -r standard_aarch64 -a aarch64 2>/dev/null | \
grep broken | awk -F ';' '{print $1}'" % self.project
log.info("Get the broken rpm:%s" % broken_cmd)
broken_result = os.popen(broken_cmd).read()
log.debug(broken_result)
broken_list = broken_result.split('\n')
self.pkgs = [x for x in broken_list if x != '']
log.info("broken_rpmlist: %s" % self.pkgs)
if not self.pkgs:
log.info("There are no broken pkgs in %s" % self.project)
if "All" == self.pkgs[0] and len(self.pkgs) == 1:
project_flag = "yes"
if not self.repository and self.project:
cmd = "osc ls %s 2>/dev/null" % self.project
pkgs = os.popen(cmd).read().split('\n')
self.pkgs = [x for x in pkgs if x != '']
log.info("project_rpmlist: %s" % self.pkgs)
if not self.pkgs:
log.info("There are no pkgs in %s" % self.project)
for pkg in self.pkgs:
if "\n" in pkg:
self.repository = pkg.replace('\n', '')
else:
self.repository = pkg
sync_result = self._pre_sync_code(self.project)
if not sync_result:
self.sync_failed_rpms.append(self.repository)
if self.sync_failed_rpms:
log.error("SYNC ERROR LIST: %s" % ",".join(self.sync_failed_rpms))
raise SystemExit("Failed, There are some pkgs sync failure")
def _distinguish_check_service(self, change):
"""
Distingguish the handing of Multi branch from that of Service
"""
if "multi" in change.split('/')[0] or "Multi" in change.split('/')[0] and \
len(change.split('/')) == 5:
multi_info = self._get_multi_branch_and_project(change)
if change.split('/')[1] in multi_info['branchs'] and \
change.split('/')[2] in multi_info['projects']:
error_flag = self._check_pkg_services(change)
else:
log.error("Please check your commit dir %s!!!" % change)
error_flag = "yes"
elif len(change.split('/')) == 4:
error_flag = self._check_pkg_services(change)
else:
log.error("Please check your commit dir %s!!!" % change)
error_flag = "yes"
return error_flag
def _check_same_pckg(self, change_file_path, yaml_msg):
"""
check the repeat pkg for the yaml file
"""
all_pkg_name = {}
for change_file in change_file_path:
pkg_name = []
log.info("{0} repeat pkg check".format(change_file))
for msg in yaml_msg[change_file]:
pkg_name.append(msg['name'])
dict_pkg_name = dict(Counter(pkg_name))
repeat_pkg = [key for key, value in dict_pkg_name.items() if value > 1]
if repeat_pkg:
all_pkg_name[change_file] = repeat_pkg
if all_pkg_name:
log.error("The following packages are duplicated in the YAML files")
log.error(all_pkg_name)
return True
else:
return False
