def _get_pkg_lib(self): """ Get package library. :param None: None :returns: library path list of package :raises: None """ try: user_gitee = gitee.Gitee() except NameError: sys.exit(1) spec_string = user_gitee.get_spec(self.repo) if not spec_string: print("WARNING: Spec of {pkg} can't be found on master".format(pkg=self.repo)) return None lib_list = [] spec_file = Spec.from_string(spec_string) with tempfile.TemporaryDirectory() as dir_name: for pkg_name in spec_file.packages_dict.keys(): pkg_path = self.download_pkg(pkg_name, dir_name) if not pkg_path: continue lib_path = self.get_lib_path(pkg_path) if lib_path: lib_list.extend(lib_path) self.delete_pkg(pkg_path) return list(set(lib_list))
def check_repo(repo, branch, batch_num): """ Check source url of multi-packages in repo like src-openeuler. batch_num is batch num of one thread """ try: user_gitee = gitee.Gitee() except NameError: sys.exit(1) repo_info = user_gitee.get_community(repo) if not repo_info: print("WARNING: {repo}.yaml can't be found in community.".format( repo=repo)) sys.exit(1) repo_yaml = yaml.load(repo_info, Loader=yaml.Loader) repo_list = repo_yaml.get("repositories") thread_num = math.ceil(len(repo_list) / batch_num) threads = [] lock = threading.Lock() for number in range(thread_num): thread = threading.Thread(target=check_batch, args=(repo_list, branch, number, batch_num, lock)) threads.append(thread) for thread in threads: thread.start() for thread in threads: thread.join() print("Ending check.")
def main_process(repo, push, check_file): """ Main process for command line """ try: my_gitee = gitee.Gitee() except NameError: sys.exit(1) if check_file == 'yaml': file = my_gitee.get_yaml(repo) elif check_file == 'spec': file = my_gitee.get_spec(repo) else: print("ERROR: Not support {file} check".format(file=check_file)) return 'NOK' if not file: need_push_issue = True print("no {file} file found for {repo} project".format(file=check_file, repo=repo)) if push: issues = my_gitee.get_issues(repo) for issue in issues: if "Submit {file} file into this repository".format( file=check_file) in issue["title"]: need_push_issue = False ages = datetime.now() - my_gitee.get_gitee_datetime( issue["created_at"]) if ages.days <= 10: print("Advise has been issues only %d days ago" % ages.days) print("Give developers more time to handle it.") break my_gitee.post_issue_comment( repo, issue["number"], NEW_COMMENT.format(repo=repo, days=ages.days)) break if need_push_issue: if check_file == 'spec': my_gitee.post_issue( repo, "Submit {file} file into this repository: {repo}". format(file=check_file, repo=repo), NEW_SPEC_ISSUE_BODY.format(repo=repo, file=check_file)) elif my_gitee.get_spec(repo): my_gitee.post_issue( repo, "Submit {file} file into this repository: {repo}". format(file=check_file, repo=repo), NEW_YMAL_ISSUE_BODY.format(repo=repo, file=check_file)) else: print("No spec file find in repo {}".format(repo)) return 'NOK' return 'OK'
def main(): """ Main entrance for command line """ pars = argparse.ArgumentParser() pars.add_argument("repo_pkg", type=str, help="The repository or package to be upgraded") pars.add_argument("branch", type=str, help="The branch that upgrade based") pars.add_argument("-u", "--update", type=str, help="Auto upgrade for packages in repository "\ "or single package", choices=["repo", "pkg"]) pars.add_argument("-n", "--new_version", type=str, help="The upgrade version of package.") pars.add_argument("-d", "--download", help="Download upstream source code", action="store_true") pars.add_argument("-s", "--create_spec", help="Create spec file", action="store_true") pars.add_argument("-fc", "--fork_then_clone", help="Fork src-openeuler repo, then "\ "clone to local", action="store_true") pars.add_argument("-b", "--build_pkg", help="Build package in local", action="store_true") pars.add_argument("-p", "--push_create_pr_issue", help="Push update repo, create "\ , action="store_true") pars.add_argument("-c", "--check_rpm_abi", help="Check ABI compatibility."\ "PR and issue", action="store_true") args = pars.parse_args() global __WORK_PATH __WORK_PATH = os.getcwd() try: user_gitee = gitee.Gitee() except NameError: print('Error: create gitee failed', NameError) exit() if args.update: if args.update == "repo": auto_update_repo(user_gitee, args.repo_pkg, args.branch) else: auto_update_pkg(user_gitee, args.repo_pkg, args.branch, args.new_version) else: __manual_operate(user_gitee, args)
def main(): """ Main process for command line """ pars = argparse.ArgumentParser() pars.add_argument("-s", "--sig", type=str, required=True, help="Sig name") pars.add_argument("-p", "--push", help="Push the advise to gitee.com/src-openeuler", action="store_true") pars.add_argument("-f", "--file", type=str, default='spec', help="File name want to check, spec or yaml?") args = pars.parse_args() if args.file not in ('spec', 'yaml'): print("ERROR: Not support {file} check".format(file=args.file)) return try: user_gitee = gitee.Gitee() except NameError: sys.exit(1) repos = user_gitee.get_repos_by_sig(args.sig) print(repos) fail_list = [] total = len(repos) index = 0 for check_repo in repos: index = index + 1 result = main_process(check_repo, args.push, args.file) if result != 'OK': fail_list.append(check_repo) print("INFO: {index} in {total} check {repo} {file} {result}".format( index=index, total=total, repo=check_repo, file=args.file, result=result)) if fail_list: print('The repos listed below check {file} failed:'.format( file=args.file)) for repo in fail_list: print(repo)
def main(): """ Main entrance of the functionality """ args = args_parser() if args.quiet: sys.stdout = open('/dev/null', 'w') sys.stderr = sys.stdout work_dir = os.path.realpath(args.workdir) params = extract_params(args) if not params: return 1 group = params[0] repo_name = params[1] pull_id = params[2] try: user_gitee = gitee.Gitee() except NameError: sys.exit(1) pull_request = user_gitee.get_pr(repo_name, pull_id, group) if not pull_request: print("Failed to get PR:%s of repository:%s/%s, make sure the PR is exist." % (pull_id, group, repo_name)) return 1 if args.edit: if edit_review_status(args.edit, user_gitee, group, repo_name, pull_id) != 0: return 1 else: checklist = load_checklist(args.local, user_gitee) if not checklist: return 1 branch = pull_request['base']['label'] ret = prepare_env(work_dir, args.reuse, params, branch) if ret != 0: user_gitee.create_pr_comment(repo_name, pull_id, FAILURE_COMMENT, group) return 1 review_comment = review(checklist, pull_request, repo_name, branch, group) user_gitee.create_pr_comment(repo_name, pull_id, review_comment, group) if args.clean: cleanup_env(work_dir, group, repo_name, pull_id) print("push review list finish.") return 0
def process_one_repo(repo): """ Main process of the functionality """ try: user_gitee = gitee.Gitee() except NameError: sys.exit(1) issues_list = user_gitee.get_issues(repo, state="all") if not issues_list: print("WARNING: Can't find any issues of {pkg}".format(pkg=repo)) return [] template = None for issue in issues_list: if issue["issue_type"] == u"开源软件变更管理": template = issue["body"] break if not template: return [] variance_details = mrkd2json(template) return variance_details
def get_oe_repo_dict(cwd_path, use_cache, sig): """ get oe repo list from sigs.yaml """ logging.debug("begin to query oe.") oe_repo_dict = {} last_record_dict = {} my_gitee = gitee.Gitee() data = [] if not sig: sigs = my_gitee.get_sigs() logging.info('start to get sigs info.') for sig_name in sigs.keys(): repo_list = my_gitee.get_repos_by_sig(sig_name) data.extend(repo_list) logging.info('sig: %s -> repo: %s', sig_name, repo_list) else: data = my_gitee.get_repos_by_sig(sig) logging.info("repo need to check: %s", data) if use_cache: last_record_dict = read_pkginfo_lasttime() if len(last_record_dict) == 0: logging.info("last recorder not exist.") for name in data: repo_url = last_record_dict.get(name, None) if repo_url: logging.info("%s has record.", name) else: pkginfo = get_pkg_info(my_gitee, name, cwd_path) if pkginfo: repo_url = yaml2url.yaml2url(pkginfo) if not repo_url: repo_url = 'none' oe_repo_dict.update({name: repo_url}) logging.info("total %d repositories in src-openeuler", len(oe_repo_dict)) record_pkginfo(oe_repo_dict) return oe_repo_dict
def main_process(push, default, repo): """ Main process of the functionality """ print("Checking", repo) try: user_gitee = gitee.Gitee() except NameError: sys.exit(1) spec_string = user_gitee.get_spec(repo) if not spec_string: print("WARNING: Spec of {pkg} can't be found on master".format(pkg=repo)) return None spec_file = Spec.from_string(spec_string) cur_version = replace_macros(spec_file.version, spec_file) if cur_version.startswith('v') or cur_version.startswith('V'): cur_version = cur_version[1:] print("Current version is", cur_version) pkg_tags = get_ver_tags(user_gitee, repo, cwd_path=default) print("known release tags:", pkg_tags) if not pkg_tags: return None if cur_version not in pkg_tags: print("WARNING: Current {ver} doesn't exist in upstream." \ "Please double check.".format(ver=cur_version)) ver_rec = version_recommend.VersionRecommend(pkg_tags, cur_version, 0) print("Latest version is", ver_rec.latest_version) print("Maintain version is", ver_rec.maintain_version) need_push_issue = True if cur_version != ver_rec.latest_version: if push: issues = user_gitee.get_issues(repo) for issue in issues: if "Upgrade to latest release" in issue["title"]: need_push_issue = False ages = datetime.now() - user_gitee.get_gitee_datetime(issue["created_at"]) if ages.days <= 30: print("Advise has been issues only %d days ago" % ages.days) print("Give developers more time to handle it.") break user_gitee.post_issue_comment(repo, issue["number"], NEW_COMMENT.format( repo=repo, days=ages.days)) break if need_push_issue: tile = """Upgrade to latest release [{repo}: {cur_ver} -> {ver}]""".format(repo=repo, ver=ver_rec.latest_version, cur_ver=cur_version) body = """Dear {repo} maintainer: We found the latest version of {repo} is {ver}, while the current version in openEuler mainline is {cur_ver}. Please consider upgrading. Yours openEuler Advisor. If you think this is not proper issue, Please visit https://gitee.com/openeuler/openEuler-Advisor. Issues and feedbacks are welcome.""".format(repo=repo, ver=ver_rec.latest_version, cur_ver=cur_version) user_gitee.post_issue(repo, tile, body) return [repo, cur_version, ver_rec.latest_version]
def main(): """ Main process of the functionality """ parameters = argparse.ArgumentParser() parameters.add_argument( "-p", "--push", action="store_true", help="Push the version bump as an issue to src-openeuler repository") parameters.add_argument( "-d", "--default", type=str, default=os.getcwd(), help="The fallback place to look for YAML information") parameters.add_argument("-s", "--sig", required=True, help="Check yaml by Sig") args = parameters.parse_args() sig = args.sig try: user_gitee = gitee.Gitee() except NameError: sys.exit(1) repos = user_gitee.get_repos_by_sig(sig) print(repos) total = len(repos) index = 0 upgrade_list = [] for check_repo in repos: # sleep 10 second, avoid limited by github\gitlab index = index + 1 time.sleep(10) result = main_process(args.push, args.default, check_repo) if result: if result[1] != result[2]: print('''INFO: {index} in {total} check {repo} need upgrade \ from {current} to {latest}'''.format(index=index, total=total, repo=result[0], current=result[1], latest=result[2])) result.append('Y') upgrade_list.append(result) else: result.append('N') upgrade_list.append(result) print('''INFO: {index} in {total} check {repo} not need \ upgrade'''.format(index=index, total=total, repo=check_repo)) else: upgrade_list.append([check_repo, '-', '-', '-']) print('''INFO: {index} in {total} check {repo} \ latest version failed.'''.format(index=index, total=total, repo=check_repo)) if upgrade_list: print("Repo upgrade check result:") for upgrade_repo in upgrade_list: print("{repo} {current} {latest} {upgrade}".format( repo=upgrade_repo[0], current=upgrade_repo[1], latest=upgrade_repo[2], upgrade=upgrade_repo[3]))
def check_pkg(pkg, branch, check_file, lock): """ Check source url of single package """ try: user_gitee = gitee.Gitee() except NameError: sys.exit(1) check_file.writelines( "\n-----------------------Checking {}-----------------------".format( pkg)) lock.acquire() spec_str = user_gitee.get_spec(pkg, branch) lock.acquire() if not spec_str: check_file.writelines( "WARNING: Spec of {repo} can't be found on {br}".format(repo=pkg, br=branch)) return False repo_spec = Spec.from_string(spec_str) if repo_spec.sources_dict: source = replace_macros(repo_spec.sources[0], repo_spec) else: title = "Source url can't be found in spec on {br}".format(br=branch) check_file.writelines("WARNING: {content}".format(content=title)) create_issue(user_gitee, pkg, title, BODY_SOURCE) return False if re.search(r"%{.*?}", source): title = "Source url can't be parsed with extra macros in spec on {}.".format( branch) check_file.writelines("WARNING: {content}".format(content=title)) create_issue(user_gitee, pkg, title, BODY_SOURCE) return False if source.startswith("http") or source.startswith("ftp"): file_name = os.path.basename(source) down_cnt = 0 lock.acquire() while down_cnt < 2: down_cnt += 1 if not subprocess.call([ "timeout 15m wget -c {url} -O {name}".format( url=source, name=file_name) ], shell=True): break lock.release() title = "Source url may be wrong in spec on {br}.".format(br=branch) if os.path.exists(file_name): if subprocess.call(["tar -tvf {} &>/dev/null".format(file_name)], shell=True): check_file.writelines( "WARNING: {content}".format(content=title)) create_issue(user_gitee, pkg, title, BODY_SOURCE) result = False else: check_file.writelines("Check successfully.") result = True subprocess.call(["rm -rf {}".format(file_name)], shell=True) else: check_file.writelines("WARNING: {content}".format(content=title)) create_issue(user_gitee, pkg, title, BODY_SOURCE) result = False return result title = "Source url is invalid in spec on {br}.".format(br=branch) check_file.writelines("WARNING: {content}".format(content=title)) create_issue(user_gitee, pkg, title, BODY_SOURCE) return False