def _ensure_delete_tags(self, change_msg, yaml_old_msg, yaml_all_msg): """ verify the rpm added to delete tag exists in the previous file """ del_tag_rpm = {} all_msg_rpm = {} error_flag = False change_file = change_msg.keys() for change in change_file: del_tag_rpm[change] = [] all_msg_rpm[change] = [] for msg in change_msg[change]: if self._check_delete_tag(msg, yaml_all_msg[change]): del_tag_rpm[change].append(msg['name']) if not del_tag_rpm[change]: del del_tag_rpm[change] for msg in yaml_old_msg[change]: all_msg_rpm[change].append(msg['name']) if not del_tag_rpm: return log.info("The Pr contain the rpm change in delete tag for {0}".format(del_tag_rpm)) info_dict = {} for change in del_tag_rpm.keys(): info_dict[change] = [] for rpm in del_tag_rpm[change]: if rpm not in all_msg_rpm[change]: error_flag = True info_dict[change].append(rpm) if not info_dict[change]: del info_dict[change] if error_flag: log.error("Check the delete group in the {0}!!!".format(info_dict)) raise SystemExit("ERROR:Please check your PR")
def _parse_git_log(self, line): """ deal diff_patch line mesg """ log.info("line:%s" % line) new_file_path = '' complete_new_file_path = '' log_list = list(line.split()) temp_log_type = log_list[0] if len(log_list) == 3: if log_list[1].split('/')[0] != log_list[2].split('/')[0] and \ log_list[1].split('/')[-2] == log_list[2].split('/')[-2]: log.error("ERROR_COMMIT: %s" % line) log.error("FAILED:PR contains the movement of same package \ between branches") raise SystemExit("*******PLEASE CHECK YOUR PR*******") else: new_file_path = list(line.split())[2] complete_new_file_path = list(line.split())[2] elif len(log_list) == 2: if temp_log_type != "D": new_file_path = list(line.split())[1] complete_new_file_path = list(line.split())[1] log.info(new_file_path) return new_file_path, complete_new_file_path
def notify_all_respon_person(self): """ notice all packages to person """ status_list = ["failed", "unresolvable"] for status in status_list: for proj in self.proj.split(','): self._get_proj_status(proj, status) if self.failed_pkglist: with ThreadPoolExecutor(10) as executor: for pkg in self.failed_pkglist: executor.submit(self._get_pkg_owner_email, proj, pkg, status) if self.to_addr_list: message = self._edit_email_content() for x in range(5): ret = self._send_email(message) if ret == 0: log.info("send email succeed !") sys.exit(0) else: log.error("send email failed, Error:%s" % ret) raise SystemExit("Failed to send email!") else: log.info("No build failed or unresolvable packages.")
def _get_obs_project(self): """ Get the obs project from gitee_branch """ log.info("Start get the obs_project") if "Multi-Version" in self.gitee_branch: path = self.meta_path + '/' + 'multi_version/' + self.gitee_branch log.info(path) cmd = "find %s -name %s | awk -F '/' '{print $5}'" % (path, self.repository) else: path = self.meta_path + '/' + self.gitee_branch log.info(path) cmd = "find %s -name %s | awk -F '/' '{print $4}'" % (path, self.repository) all_project = os.popen(cmd).readlines() log.info(all_project) obs_project = None for project in all_project: obs_project = project.replace('\n', '') if ":Bak" in obs_project: continue else: break if obs_project and ":Bak" not in obs_project: log.info("The %s obs_project for gitee_%s is <<<<%s>>>>" % ( self.repository, self.gitee_branch, obs_project)) return obs_project else: log.error("Failed !!! The rpm is not exist in %s branch !!!" % self.gitee_branch) return False
def _check_branch_msg(self, change_msg, yaml_path_list, manage_path): """ check the branch msg in your commit """ error_msg = {} branch_msg_path = os.path.join(manage_path, "valid_release_branches.yaml") with open(branch_msg_path, 'r', encoding='utf-8') as f: branch_result = yaml.load(f, Loader=yaml.FullLoader) for yaml_path in yaml_path_list: log.info("{0} branch check".format(yaml_path)) error_msg[yaml_path] = [] yaml_branch = yaml_path.split("/")[-2] for msg in change_msg[yaml_path]: if msg['branch_to'] == yaml_branch and \ msg['branch_from'] in branch_result['branch'].keys() and \ msg['branch_to'] in branch_result['branch'][msg['branch_from']]: continue else: error_msg[yaml_path].append(msg) if not error_msg[yaml_path]: del error_msg[yaml_path] else: log.error("Wrong branch msg in there:") for msg in error_msg[yaml_path]: log.error(msg) if error_msg: return True else: return False
def same_or_not(self, gitee_spec, obs_spec, package): """ check spec file between gitee spec file and obs spec file return: False - not same; True - same """ if gitee_spec and obs_spec: for ospec in obs_spec: sf = ospec.split(":")[-1] index = gitee_spec.index(sf) log.info("gitee spec:%s, obs spec:%s" % (gitee_spec[index], ospec)) cmd = "diff %s %s" % (gitee_spec[index], ospec) log.info(cmd) ret = os.popen(cmd).read() cmd = "rm -rf %s %s" % (gitee_spec[index], ospec) os.system(cmd) if ret: log.info("diff-" + ret) return False return True elif not gitee_spec and not obs_spec: log.info("%s has no script" % package) return True elif not gitee_spec: log.error("%s no spec from gitee" % package) return False elif not obs_spec: log.error("%s no spec from obs" % package) return False
def _get_yaml_msg(self, yaml_path_list, manage_path, rollback=None): """ get the pkg msg in pckg-mgmt.yaml """ error_pkg = {} error_flag = False if rollback == True: self._rollback_get_msg(manage_path) all_pack_msg = {} for yaml_path in yaml_path_list: file_path = os.path.join(manage_path, yaml_path) if os.path.exists(file_path): with open(file_path, 'r', encoding='utf-8')as f: result = yaml.load(f, Loader=yaml.FullLoader) if "natural" in result['packages'].keys(): all_pack_msg[yaml_path] = result['packages']['natural'] + \ result['packages']['recycle'] + result['packages']['delete'] else: all_pack_msg[yaml_path] = result['packages']['everything']['baseos'] + \ result['packages']['everything']['other'] + result['packages']['epol'] + \ result['packages']['recycle'] + result['packages']['delete'] else: all_pack_msg[yaml_path] = [] error_pkg[yaml_path] = [] for pkg_info in all_pack_msg[yaml_path]: if ' ' in pkg_info['name']: error_name = pkg_info['name'] error_flag = True error_pkg[yaml_path].append(error_name) if error_flag: log.error("as follows pkgs {0} space in name".format(error_pkg)) raise SystemExit("ERROR: Please check yaml pkg name") return all_pack_msg
def _check_rpms_integrity(self, old_pack_msg, new_pack_msg, yaml_path_list): """ ensure the rpms exist in all the tags """ old_pkg = {} new_pkg = {} error_pkg = {} log.info("rpms exists check") for change_file in yaml_path_list: old_pkg[change_file] = [] new_pkg[change_file] = [] for msg in old_pack_msg[change_file]: old_pkg[change_file].append(msg['name']) for msg in new_pack_msg[change_file]: new_pkg[change_file].append(msg['name']) for change_file in yaml_path_list: error_pkg[change_file] = [] for pkg in old_pkg[change_file]: if pkg not in new_pkg[change_file]: error_pkg[change_file].append(pkg) if not error_pkg[change_file]: del error_pkg[change_file] if error_pkg: log.error("May be {0} should not be delete".format(error_pkg)) raise SystemExit("ERROR: Please check your PR")
def rpms_exists(self, rpms_list): """ check rpms exists rpms_list: """ try: tmp_list = [ rpms_list[i:i + 500] for i in range(0, len(rpms_list), 500) ] for l in tmp_list: cmd = """ for r in %s do ls %s/%s/%s/:full/ | grep "$r" if [ $? -ne 0 ];then echo "notfind" break else echo "find" fi done """ % (' '.join(l), self.obs_project_root_path, self.rpms_to_repo_path, self.arch) ret = self.pex.ssh_cmd(cmd) if "notfind" in str(ret) or "find" not in str(ret): return False except Exception as e: log.error(e) return False return True
def _modify_pkg_service(self, proj, pkg, branch_name): """ change the service file for the package return 0 or -1 """ service_file_path = os.path.join(self.obs_meta_path, self.multi_version_dir, branch_name, proj, pkg, "_service") _tmpdir = os.popen("mktemp -d").read().strip('\n') pkg_tmpdir = os.path.join(_tmpdir, proj, pkg) cmd = "cd %s && osc co %s %s &>/dev/null && cd -" % (_tmpdir, proj, pkg) if os.system(cmd) == 0: cmd = "cd %s && cp -f %s %s && osc add _service && osc ci -m 'modify by %s' && cd -" \ % (pkg_tmpdir, service_file_path, pkg_tmpdir, self.giteeUserName) if os.system(cmd) == 0: log.info("modify %s %s _service success!" % (proj, pkg)) shutil.rmtree(_tmpdir) return 0 else: log.error("modify %s %s _service failed!" % (proj, pkg)) shutil.rmtree(_tmpdir) return -1 else: log.warning("%s %s not found" % (proj, pkg)) shutil.rmtree(_tmpdir) return -1
def _add_pkg_service(self, tmp): """ add obs_meta packages _service file """ from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) pkg_service_path = os.path.join(pkg_path, "_service") if tmp['branch_from'] == "master": branch = "openEuler" else: branch = tmp['branch_from'] if not os.path.exists(pkg_path): os.makedirs(pkg_path) if not os.path.exists(pkg_service_path): cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path) if os.system(cmd) == 0: cmd = "sed -i 's/%s\//%s\//g' %s/_service" % ( branch, tmp['branch_to'], pkg_path) if os.system(cmd) == 0: log.info("add %s %s %s _service succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) else: log.info("add %s %s %s _service failed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) else: log.error("copy %s service file failed!" % tmp['pkgname'])
def get_old_rpms_list_from_file(self, file_path): """ get old pkg rpms dict file_path: yaml file which store all packages's rpms """ if os.path.exists(file_path): with open(file_path, "r") as f: self.old_pkg_rpms = yaml.load(f, Loader=yaml.FullLoader) else: log.error("no file for get rpms")
def _pkgname_check(self, pkg_name, service_name_list): """ check the pkg_name from change_path and service_name_list """ if pkg_name in service_name_list: log.info("SUCCESS_CHECK:The %s in _service url is correct" % pkg_name) return else: log.error("**************_Service URL ERROR*****************") log.error("FAILED:The %s in _service pkgname is not same" % pkg_name) error_flag = "yes" return error_flag
def _get_change_file(self): """ get release-managemnet change file """ if os.path.exists(self.release_management_path): os.chdir(self.release_management_path) cmd = "git diff --name-status HEAD~1 HEAD~0 | grep pckg-mgmt.yaml" result = os.popen(cmd).read().split('\n') change_file = [x for x in result if x != ''] return change_file else: log.error("%s not exist!" % self.release_management_path) sys.exit(1)
def _del_pkg(self, proj, pkg): """ delete the project package return 0 or -1 """ cmd = "osc api -X DELETE /source/%s/%s" % (proj, pkg) ret = os.popen(cmd).read() if "<summary>Ok</summary>" in ret: log.info("delete %s %s success!" % (proj, pkg)) return 0 else: log.error("delete %s %s failed!" % (proj, pkg)) return -1
def _obs_meta_action(self): """ action basis on change of obs_meta return: """ log.debug("obs_meta change") if not self.kwargs["obs_meta_path"]: log.error("can not find obs_meta path") else: obs_prjm = OBSPrjManager(self.kwargs["obs_meta_path"]) obs_prjm.manager_action() obs_pkgm = OBSPkgManager(**self.kwargs) obs_pkgm.obs_pkg_admc()
def _check_yaml_format(self, yaml_path_list, manage_path): """ check the format for the yaml file """ for yaml_path in yaml_path_list: manage_yaml_path = os.path.join(manage_path, yaml_path) try: with open(manage_yaml_path, 'r', encoding='utf-8') as f: result = yaml.load(f, Loader=yaml.FullLoader) log.info("{0} format check".format(yaml_path)) except Exception as e: log.error("**********FORMAT ERROR***********") log.error("%s format bad Because:%s" % (yaml_path, e)) raise SystemExit("May be %s has a bad format" % yaml_path)
def backup_old_rpms_by_pkg(self, pkg, rpms_list): """ backup old rpms by package name pkg: name of packages rpms_list: all rpms of package, type is list """ try: t = time.strftime("%Y-%m-%d-%H-%M", time.localtime()) backup_dir = os.path.join(self.obs_project_root_path, self.rpms_to_repo_path, "backup") pkg_bak = os.path.join(backup_dir, "%s-%s" % (pkg, t)) self.backup_old_rpm(backup_dir, pkg, pkg_bak, rpms_list) except ValueError as e: log.error(e) return False except SystemError as e: log.error(e) return False except TypeError as e: log.error(e) return False except KeyError as e: log.error(e) return False return True
def _exist_for_pro_and_repo(self, project, repository): """ check the project and repository is or not exist in obs """ error_flag = None exit_cmd = "osc r %s --xml 2>/dev/null | grep %s" % (project, repository) exit_result = os.popen(exit_cmd).read() log.info(exit_result) if not exit_result: error_flag = "yes" log.error("Failed:The %s Not exit in %s" % (repository, project)) else: log.info("Success:The %s exit in %s" % (repository, project)) return error_flag
def _gitee_pr_to_obs(self, obs_pro): """ obs_pro:The obs project that gitee branch corresponds """ osc_service = os.popen("osc service remoterun %s %s" % (obs_pro, self.repository)).readlines() for results in osc_service: if "ok" in results: log.info(results) log.info("Success for osc service remoterun the %s" % self.repository) return True else: continue log.error("Failed !!! fail to service remoterun !!!") return False
def _del_obs_pkg_service(self, proj, pkg): """ delete the obs project package service file return 0 or -1 """ cmd = "osc api -X DELETE /source/%s/%s/_service" % (proj, pkg) ret = os.popen(cmd).read() if "<summary>Ok</summary>" in ret: log.info("delete %s %s _service by %s successful!" % (proj, pkg, self.giteeUserName)) return 0 else: log.error("delete %s %s _service failed!" % (proj, pkg)) return -1
def _modify_pkg_meta(self, proj, pkg, branch_name): """ change the package of the meta return 0 or -1 """ file_path = os.path.join(self.obs_meta_path, self.multi_version_dir, branch_name, proj, pkg, ".osc/_meta") _tmpdir = os.popen("mktemp -d").read().strip('\n') cmd = "cd %s && osc meta pkg %s %s --file=%s | grep ^Done. && cd -" % ( _tmpdir, proj, pkg, file_path) if os.system(cmd) != 0: log.error("modify %s %s _meta failed!" % (proj, pkg)) shutil.rmtree(_tmpdir) return -1 shutil.rmtree(_tmpdir)
def do_all(self): """ make the get obs_meta change fuction and check fuction doing """ if self.prid and self.token: release_management_data = self._release_management_tree() change_result, complete_change_result = self._get_all_change_file() pr_check_result = self._check_pr_rule(release_management_data, complete_change_result) check_result = self._check_service_meta(change_result) elif not self.prid and self.branch: self._clean() self._get_latest_obs_meta() self._check_branch_service(self.branch) else: log.error("ERROR_INPUT:PLEASE CHECK YOU INPUT")
def _pre_sync_code(self, project=None): """ The way to offer that make all fuction runing """ if not project: obs_project = self._get_obs_project() else: obs_project = project log.info("The %s in %s" % (self.repository, obs_project)) pull_result = self._get_latest_gitee_pull() remoterun_result = self._gitee_pr_to_obs(obs_project) if obs_project and pull_result and remoterun_result: log.info("SYNC %s in %s SUCCESS" % (self.repository, obs_project)) return True else: log.error("SYNC %s in %s ERROR: Please check the log.error" % (self.repository, obs_project)) return False
def _check_pckg_yaml(self, file_msg, branch): """ check pckg-mgmt.yaml file """ proj = branch.replace("-", ":") flag = False for title in file_msg['packages']: for msg in file_msg['packages'][title]: if msg['branch_to'] != branch: flag = True log.error("%s branch_to is error, please check yaml." % msg['name']) if not msg['obs_to'].startswith(proj): flag = True log.error("%s obs_to is error, please check yaml." % msg['name']) return flag
def update_pkg(self, pkg): """ update one package pkg: name of package """ if pkg in self.old_pkg_rpms.keys(): old_rpms_list = self.old_pkg_rpms[pkg] old_rpms_list.sort() else: old_rpms_list = None new_rpms_list = self.get_new_rpms_by_pkg(pkg) new_rpms_list.sort() if self.kwargs["all"] == "True" and new_rpms_list: self.backup_old_rpms_by_pkg(pkg, old_rpms_list) self.copy_new_rpms_to_repo(pkg, new_rpms_list) elif old_rpms_list != new_rpms_list and new_rpms_list: try: self.backup_old_rpms_by_pkg(pkg, old_rpms_list) self.copy_new_rpms_to_repo(pkg, new_rpms_list) except Exception as e: self.backup_old_rpms_by_pkg(pkg, new_rpms_list) elif self.kwargs["compare"] == "True": if not self.compare_rpms(pkg): self.backup_old_rpms_by_pkg(pkg, old_rpms_list) self.copy_new_rpms_to_repo(pkg, new_rpms_list) else: log.debug("%s all rpms are latest should do nothing" % pkg) if new_rpms_list: for i in range(5): try: if not self.rpms_exists(new_rpms_list): log.debug( "check %s rpms not exists, shoud copy again" % pkg) self.copy_new_rpms_to_repo(pkg, new_rpms_list) else: log.debug("check %s all rpms exists" % pkg) break except Exception as e: log.error(e) else: cmd = "osc list %s | grep ^%s" % (self.obs_project, pkg) ret = os.popen(cmd).read().split("\n") if pkg not in ret: self.backup_old_rpms_by_pkg(pkg, old_rpms_list) self.old_pkg_rpms[pkg] = new_rpms_list
def _check_pkg_from(self, meta_path, yaml_msg, change_file, yaml_all_msg): """ Detects the existence of file contents """ error_flag = False for change in change_file: log.info("{0} pkg_from check".format(change)) for msg in yaml_msg[change]: delete_tag = self._check_delete_tag(msg, yaml_all_msg[change]) if delete_tag: continue msg_path = os.path.join(meta_path, msg['branch_from'], msg['obs_from'], msg['name']) if not os.path.exists(msg_path): yaml_key = os.path.join(msg['branch_from'], msg['obs_from'], msg['name']) log.error("The {0} not exist in obs_meta".format(yaml_key)) error_flag = True return error_flag
def sync_code_to_obs(self): """ The only way to offer that make all fuction runing """ if not self.pkgs: if self.repository and self.gitee_branch: if not self._pre_sync_code(self.project): raise SystemExit("SYNC %s ERROR" % self.repository) else: raise SystemExit('please check you arguments') else: if "broken" == self.pkgs[0] and len(self.pkgs) == 1: project_flag = "yes" broken_cmd = "osc r --csv %s -r standard_aarch64 -a aarch64 2>/dev/null | \ grep broken | awk -F ';' '{print $1}'" % self.project log.info("Get the broken rpm:%s" % broken_cmd) broken_result = os.popen(broken_cmd).read() log.debug(broken_result) broken_list = broken_result.split('\n') self.pkgs = [x for x in broken_list if x != ''] log.info("broken_rpmlist: %s" % self.pkgs) if not self.pkgs: log.info("There are no broken pkgs in %s" % self.project) if "All" == self.pkgs[0] and len(self.pkgs) == 1: project_flag = "yes" if not self.repository and self.project: cmd = "osc ls %s 2>/dev/null" % self.project pkgs = os.popen(cmd).read().split('\n') self.pkgs = [x for x in pkgs if x != ''] log.info("project_rpmlist: %s" % self.pkgs) if not self.pkgs: log.info("There are no pkgs in %s" % self.project) for pkg in self.pkgs: if "\n" in pkg: self.repository = pkg.replace('\n', '') else: self.repository = pkg sync_result = self._pre_sync_code(self.project) if not sync_result: self.sync_failed_rpms.append(self.repository) if self.sync_failed_rpms: log.error("SYNC ERROR LIST: %s" % ",".join(self.sync_failed_rpms)) raise SystemExit("Failed, There are some pkgs sync failure")
def _distinguish_check_service(self, change): """ Distingguish the handing of Multi branch from that of Service """ if "multi" in change.split('/')[0] or "Multi" in change.split('/')[0] and \ len(change.split('/')) == 5: multi_info = self._get_multi_branch_and_project(change) if change.split('/')[1] in multi_info['branchs'] and \ change.split('/')[2] in multi_info['projects']: error_flag = self._check_pkg_services(change) else: log.error("Please check your commit dir %s!!!" % change) error_flag = "yes" elif len(change.split('/')) == 4: error_flag = self._check_pkg_services(change) else: log.error("Please check your commit dir %s!!!" % change) error_flag = "yes" return error_flag
def _check_same_pckg(self, change_file_path, yaml_msg): """ check the repeat pkg for the yaml file """ all_pkg_name = {} for change_file in change_file_path: pkg_name = [] log.info("{0} repeat pkg check".format(change_file)) for msg in yaml_msg[change_file]: pkg_name.append(msg['name']) dict_pkg_name = dict(Counter(pkg_name)) repeat_pkg = [key for key, value in dict_pkg_name.items() if value > 1] if repeat_pkg: all_pkg_name[change_file] = repeat_pkg if all_pkg_name: log.error("The following packages are duplicated in the YAML files") log.error(all_pkg_name) return True else: return False