def _update_obs_repo_rpms(self): """ update obs repo rpms """ log.debug("update repo rpms") update_repo = RPMManager(**self.kwargs) update_repo.update_pkgs()
def run(self): """ run main return: """ if self.kwargs["check_codes"]: self._check_codes() elif self.kwargs["repo_rpms_update"]: self._update_obs_repo_rpms() elif self.kwargs["latest_info"]: self._save_latest_info() elif self.kwargs["repository"] == "obs_meta": self._obs_meta_action() elif self.kwargs["sync_gitee_to_obs"] == "true": if not self.update_enabled_flag[self.kwargs["branch"]]: if self.kwargs["repository"] and self.kwargs[ "repository"] not in self.ignore_list: log.debug( "can not update branch:%s, package: %s" % (self.kwargs["branch"], self.kwargs["repository"])) self._save_unsync_info() else: self._update_package() elif self.kwargs["check_pkg_service"] == "true": print("check_pkg_service") check = CheckMetaPull(**self.kwargs) check.do_all() elif self.kwargs["get_latest_date"] == "true" and self.kwargs["branch"]: self._get_latest_date() elif self.kwargs["obs_mail_notice"] == "true": self._mail_notice() elif self.kwargs["pckg_mgmt"] == "true": self._pckg_mgmt() elif self.kwargs["check_pckg_mgmt"] == "true": self._check_release_mgmt()
def _get_latest_date(self): """ get the latest git date to obs_pkgs_rpms """ log.debug("get latest date") get_date = GETDate(**self.kwargs) get_date.update_to_obs_pkg_rpms()
def _mail_notice(self): """ send mail to the package owner """ log.debug("OBS Mail Notice") mail_notice = ObsMailNotice(**self.kwargs) mail_notice.notify_all_respon_person()
def _get_repo_change_file(self, owner=None, pkgname=None, repo_path=None): """ Obtain the change for latest commit """ changed_file_cmd = "git diff --name-status HEAD~1 HEAD~0" fetch_cmd = "git fetch origin pull/%s/head:thispr" % self.prid checkout_cmd = "git checkout thispr" get_fetch = None if not repo_path: self._clean(pkgname) release_path = self._get_latest_git_repo(owner, pkgname) self.manage_path = release_path os.chdir(release_path) else: os.chdir(repo_path) for x in range(5): fetch_result = os.system(fetch_cmd) checkout_result = os.system(checkout_cmd) log.debug("STATUS:{0} and {1}".format(fetch_result, checkout_result)) if fetch_result == 0 and checkout_result == 0: get_fetch = True break else: os.chdir(self.current_path) self._clean(pkgname) path_release = self._get_latest_git_repo(owner, pkgname) os.chdir(path_release) self.manage_path = os.path.join(self.current_path, pkgname) changed_file = os.popen(changed_file_cmd).readlines() if get_fetch and changed_file: log.info(changed_file) return changed_file else: raise SystemExit("Error:can not obtain the content for this commit")
def _check_codes(self): """ check codes between gitee and obs """ log.debug("check codes") check_codes = CheckCode(**self.kwargs) check_codes.check_all()
def _pckg_mgmt(self): """ sync obs_meta according to the pckg-mgmt.yaml """ log.debug("Sync Pckg Mgmt") mgmt = SyncPckgMgmt(**self.kwargs) mgmt.sync_yaml_meta()
def _save_latest_info(self): """ save package latest info return: """ log.debug("save latest info") si = SaveInfo(self.kwargs["gitee_user"], self.kwargs["gitee_pwd"]) si.save_latest_info(self.kwargs["branch"])
def _update_package(self): """ update package code for obs return: """ log.debug("update package code") syc = SYNCCode(**self.kwargs) syc.sync_code_to_obs()
def _save_unsync_info(self): """ save unsync package info for manual operation later return: """ log.debug("save package info") si = SaveInfo(self.kwargs["gitee_user"], self.kwargs["gitee_pwd"]) si.save_unsync_package(self.kwargs["repository"], self.kwargs["branch"])
def update_repos_db(self): """ update obs repos db """ cmd = "chown -R obsrun:obsrun %s/%s/%s/:full; obs_admin --rescan-repository %s %s %s" \ % (self.obs_project_root_path, self.rpms_to_repo_path, self.arch, \ self.rpms_to_repo_path.split("/")[0], self.rpms_to_repo_path.split("/")[1], self.arch) log.debug(cmd) ret = self.pex.ssh_cmd(cmd)
def copy_new_rpm(self, pkg): """ copy rpm to repo pkg: name of package rpm: name of rpm file of pkg """ cmd = "cp -p %s/%s/%s/%s/%s/*.rpm %s/%s/%s/:full/ && rm -rf %s/%s/%s/:full/*.src.rpm" \ % (self.obs_project_root_path, self.obs_project, self.repo, \ self.arch, pkg, self.obs_project_root_path, \ self.rpms_to_repo_path, self.arch, self.obs_project_root_path, self.rpms_to_repo_path, self.arch) log.debug("%s: %s" % (pkg, cmd)) self.pex.ssh_cmd(cmd)
def _obs_meta_action(self): """ action basis on change of obs_meta return: """ log.debug("obs_meta change") if not self.kwargs["obs_meta_path"]: log.error("can not find obs_meta path") else: obs_prjm = OBSPrjManager(self.kwargs["obs_meta_path"]) obs_prjm.manager_action() obs_pkgm = OBSPkgManager(**self.kwargs) obs_pkgm.obs_pkg_admc()
def _copy_package(self, pkg, from_path, to_path): """ copy package from from_path to to_path from_path: path of package that will be copied to_path: path fo package that will go """ if self.kwargs["branch"] == "master": cmd = "cp -r %s/%s %s && sed -i 's/openEuler/%s/g' %s/%s/_service" % \ (from_path, pkg, to_path, self.kwargs["branch2"], to_path, pkg) else: cmd = "cp -r %s/%s %s && sed -i 's/%s/%s/g' %s/%s/_service" % \ (from_path, pkg, to_path, self.kwargs["branch"], \ self.kwargs["branch2"], to_path, pkg) ret = os.popen(cmd).read() log.debug(ret)
def _git_clone(self, git_house): """ git clone function """ if git_house == "obs_meta": os.system( "cd %s && git diff --name-status HEAD~1 HEAD~0 > %s && cd -" % (self.obs_meta_path, self.patch_file_path)) if git_house == "community": community_path = os.path.join(self.init_path, "community") if os.path.exists(community_path): shutil.rmtree(community_path) git_url = "https://%s:%[email protected]/openeuler/community.git" % ( self.giteeUserName, self.giteeUserPwd) ret = os.popen("git lfs --depth 1 clone %s %s" % (git_url, community_path)).read() log.debug(ret)
def _save_latest_info_by_pkg(self, datestr_root_path, prj, pkg, file_list, f_csv): """ save latest info of package """ if pkg in file_list: cmd = "cat %s/%s" % (datestr_root_path, pkg) log.debug(cmd) timestr = os.popen(cmd).read().replace("\n", "") else: timestr = 0 cmd = "osc list -b %s %s|grep rpm" % (prj, pkg) log.debug(cmd) rpms = ' '.join( list( set(os.popen(cmd).read().replace(" ", "").split("\n")) - set(['']))) f_csv.writerow([timestr, pkg, rpms])
def _copy_packages(self): """ copy some packages from obs project A to project B """ if self.kwargs["branch2"] and self.kwargs["project2"]: os.chdir(self.kwargs["obs_meta_path"]) from_path = os.path.join(self.kwargs["branch"], self.kwargs["project"]) to_path = os.path.join(self.kwargs["branch2"], self.kwargs["project2"]) pkgs = self.kwargs["pkglist"] if not pkgs: pkgs = os.listdir(from_path) for pkg in pkgs: self._copy_package(pkg, from_path, to_path) cmd = "git add %s && git commit -m 'add pkgs to project %s' && git push" % \ (to_path, self.kwargs["project2"]) ret = os.popen(cmd).read() log.debug(ret)
def sync_code_to_obs(self): """ The only way to offer that make all fuction runing """ if not self.pkgs: if self.repository and self.gitee_branch: if not self._pre_sync_code(self.project): raise SystemExit("SYNC %s ERROR" % self.repository) else: raise SystemExit('please check you arguments') else: if "broken" == self.pkgs[0] and len(self.pkgs) == 1: project_flag = "yes" broken_cmd = "osc r --csv %s -r standard_aarch64 -a aarch64 2>/dev/null | \ grep broken | awk -F ';' '{print $1}'" % self.project log.info("Get the broken rpm:%s" % broken_cmd) broken_result = os.popen(broken_cmd).read() log.debug(broken_result) broken_list = broken_result.split('\n') self.pkgs = [x for x in broken_list if x != ''] log.info("broken_rpmlist: %s" % self.pkgs) if not self.pkgs: log.info("There are no broken pkgs in %s" % self.project) if "All" == self.pkgs[0] and len(self.pkgs) == 1: project_flag = "yes" if not self.repository and self.project: cmd = "osc ls %s 2>/dev/null" % self.project pkgs = os.popen(cmd).read().split('\n') self.pkgs = [x for x in pkgs if x != ''] log.info("project_rpmlist: %s" % self.pkgs) if not self.pkgs: log.info("There are no pkgs in %s" % self.project) for pkg in self.pkgs: if "\n" in pkg: self.repository = pkg.replace('\n', '') else: self.repository = pkg sync_result = self._pre_sync_code(self.project) if not sync_result: self.sync_failed_rpms.append(self.repository) if self.sync_failed_rpms: log.error("SYNC ERROR LIST: %s" % ",".join(self.sync_failed_rpms)) raise SystemExit("Failed, There are some pkgs sync failure")
def _push_to_pkg_rpms(self): """ push the latest obs_pkg_rpms to origin """ os.chdir("obs_pkg_rpms") status_cmd = "git status -s" commit_cmd = "git add -A && git commit -m 'update for package'" if os.popen(status_cmd).read(): if os.system(commit_cmd) == 0: for i in range(5): push_cmd = "git push -f" push_result = subprocess.getstatusoutput(push_cmd) log.info(push_result) if push_result[0] == 0: log.info("SUCCESS:Push success for latest date to obs_pkg_rpms") return else: log.debug("Try Push to obs_pkg_rpms: %s" % i) raise SystemExit("Failed: Push to obs_pkg_rpms failed") else: log.info("NO CHAGE,nothing to commit")
def _get_diff_msg(self, old_msg, new_msg, change_file_list): """ Get the content of this submission """ change_list = {} for change in change_file_list: change_list[change] = [] for new in new_msg[change]: if new not in old_msg[change]: change_list[change].append(new) for change in change_file_list: if change_list[change]: log.debug("Change in {0}:".format(change)) for msg in change_list[change]: log.debug(msg) else: del change_list[change] log.info("The are no new msg in {0}!!!".format(change)) if change_list: return change_list else: log.info("The are no new msg in your yaml!!!") sys.exit()
def save_latest_info(self, branch_name): """ save latest info of package, include update time of package code and rpms """ self._download() try: datestr_root_path = os.path.join(self.obs_pkg_rpms_files_dir, branch_name) if not os.path.exists(datestr_root_path): os.makedirs(datestr_root_path) latest_rpm_file = os.path.join(self.obs_pkg_rpms_files_dir, "latest_rpm", "%s.csv" % branch_name) with open(latest_rpm_file, "w") as f: f_csv = csv.writer(f) file_list = os.listdir(datestr_root_path) prj_list = self.branch_prj[branch_name].split(" ") log.debug(prj_list) var_list = [] for prj in prj_list: cmd = "osc list %s" % prj log.debug(cmd) pkgs = os.popen(cmd).read().split("\n") for pkg in pkgs: var_list.append( ([datestr_root_path, prj, pkg, file_list, f_csv], None)) try: pool = threadpool.ThreadPool(20) requests = threadpool.makeRequests( self._save_latest_info_by_pkg, var_list) for req in requests: pool.putRequest(req) pool.wait() except KeyboardInterrupt as e: log.error(e) cmd = "sort -u -r %s > tmp && cat tmp > %s" % (latest_rpm_file, latest_rpm_file) if os.system(cmd) != 0: log.error("sort file fail") cmd = "cd %s && git pull && git add * && git commit -m 'update file' && git push" % \ self.obs_pkg_rpms_files_dir log.debug(cmd) for i in range(5): if os.system(cmd) == 0: break except AttributeError as e: log.error(e) finally: self._delete()
def update_pkg(self, pkg): """ update one package pkg: name of package """ if pkg in self.old_pkg_rpms.keys(): old_rpms_list = self.old_pkg_rpms[pkg] old_rpms_list.sort() else: old_rpms_list = None new_rpms_list = self.get_new_rpms_by_pkg(pkg) new_rpms_list.sort() if self.kwargs["all"] == "True" and new_rpms_list: self.backup_old_rpms_by_pkg(pkg, old_rpms_list) self.copy_new_rpms_to_repo(pkg, new_rpms_list) elif old_rpms_list != new_rpms_list and new_rpms_list: try: self.backup_old_rpms_by_pkg(pkg, old_rpms_list) self.copy_new_rpms_to_repo(pkg, new_rpms_list) except Exception as e: self.backup_old_rpms_by_pkg(pkg, new_rpms_list) elif self.kwargs["compare"] == "True": if not self.compare_rpms(pkg): self.backup_old_rpms_by_pkg(pkg, old_rpms_list) self.copy_new_rpms_to_repo(pkg, new_rpms_list) else: log.debug("%s all rpms are latest should do nothing" % pkg) if new_rpms_list: for i in range(5): try: if not self.rpms_exists(new_rpms_list): log.debug( "check %s rpms not exists, shoud copy again" % pkg) self.copy_new_rpms_to_repo(pkg, new_rpms_list) else: log.debug("check %s all rpms exists" % pkg) break except Exception as e: log.error(e) else: cmd = "osc list %s | grep ^%s" % (self.obs_project, pkg) ret = os.popen(cmd).read().split("\n") if pkg not in ret: self.backup_old_rpms_by_pkg(pkg, old_rpms_list) self.old_pkg_rpms[pkg] = new_rpms_list