def check_scanoss(self): """ scanoss工具检查代码片段引用 https://osskb.org https://github.com/scanoss/scanner.py :return: """ gp = GitProxy(self._work_dir) diff_files = gp.diff_files_between_commits("HEAD~1", "HEAD~0") logger.debug("diff files: {}".format(diff_files)) self.copy_diff_files_to_dest(diff_files) blacklist_sbom = os.path.realpath( os.path.join(os.path.realpath(__file__), "../../../../conf/deny_list.sbom")) scan = ScanOSS(self._scanoss_api_key, self._scanoss_api_url, blacklist_sbom) result = scan.scan(self._work_diff_dir) # 保存详细结果到web server if not result: self.save_scanoss_result(scan.html) logger.warning("click {} view scanoss detail".format( self._scanoss_result_repo_path)) return SUCCESS if result else FAILED
def __init__(self, workspace, repo, conf=None): super(CheckSpec, self).__init__(workspace, repo, conf) self._gp = GitProxy(self._work_dir) self._gr = GiteeRepo(self._repo, self._work_dir, None) # don't care about decompress fp = self._gp.get_content_of_file_with_commit(self._gr.spec_file) self._spec = RPMSpecAdapter(fp) self._latest_commit = self._gp.commit_id_of_reverse_head_index(0)
def __init__(self, workspace, repo, conf): super(CheckPackageYaml, self).__init__(workspace, repo, conf) self._gp = GitProxy(self._work_dir) self._gr = GiteeRepo(self._repo, self._work_dir, None) # don't care about decompress if self._gr.spec_file: self._spec = RPMSpecAdapter( os.path.join(self._work_dir, self._gr.spec_file)) else: self._spec = None self._yaml_content = None self._yaml_changed = True self._is_standard = True
def apply_patch(self, patch, max_leading=5): """ 尝试所有路径和leading :param patch: 补丁 :param max_leading: leading path """ logger.debug("apply patch {}".format(patch)) for patch_dir in [ filename for filename in os.listdir(self._decompress_dir) if os.path.isdir(os.path.join(self._decompress_dir, filename)) ] + ["."]: if patch_dir.startswith(".git"): continue for leading in range(max_leading + 1): logger.debug("try dir {} -p{}".format(patch_dir, leading)) if GitProxy.apply_patch_at_dir( os.path.join(self._decompress_dir, patch_dir), os.path.join(self._work_dir, patch), leading): logger.debug("patch success") self.patch_dir_mapping[patch] = os.path.join( self._decompress_dir, patch_dir) return True logger.info("apply patch {} failed".format(patch)) return False
def check_code_style(self): """ 检查代码风格 :return: """ gp = GitProxy(self._work_dir) diff_files = gp.diff_files_between_commits("HEAD~1", "HEAD~0") logger.debug("diff files: {}".format(diff_files)) diff_code_files = [] # 仓库中变更的代码文件 diff_patch_code_files = [] # patch内的代码文件 for diff_file in diff_files: if GiteeRepo.is_code_file(diff_file): diff_code_files.append(diff_file) elif GiteeRepo.is_patch_file(diff_file): patch_dir = self._gr.patch_dir_mapping.get(diff_file) logger.debug("diff patch {} apply at dir {}".format( diff_file, patch_dir)) if patch_dir is not None: files_in_patch = gp.extract_files_path_of_patch(diff_file) patch_code_files = [ os.path.join(patch_dir, file_in_patch) for file_in_patch in files_in_patch if GiteeRepo.is_code_file(file_in_patch) ] # care about deleted file in patch, filter with "git patch --summary" maybe better diff_patch_code_files.extend([ code_file for code_file in patch_code_files if os.path.exists(code_file) ]) logger.debug("diff code files: {}".format(diff_code_files)) logger.debug("diff patch code files: {}".format(diff_patch_code_files)) rs_1 = self.check_file_under_work_dir(diff_code_files) logger.debug("check_file_under_work_dir: {}".format(rs_1)) rs_2 = self.check_files_inner_patch(diff_patch_code_files) logger.debug("check_files_inner_patch: {}".format(rs_2)) return rs_1 + rs_2
def __init__(self, workspace, repo, conf=None): super(CheckLicense, self).__init__(workspace, repo, conf) self._gp = GitProxy(self._work_dir) self._work_tar_dir = os.path.join(workspace, "code") self._gr = GiteeRepo(self._repo, self._work_dir, self._work_tar_dir) if self._gr.spec_file: self._spec = RPMSpecAdapter( os.path.join(self._work_dir, self._gr.spec_file)) else: self._spec = None self._pkg_license = PkgLicense() self._license_in_spec = set() self._license_in_src = set()
dd = DistDataset() dd.set_attr_stime("spb.job.stime") dd.set_attr("spb.job.link", os.environ["BUILD_URL"]) dd.set_attr("spb.trigger.reason", os.environ["BUILD_CAUSE"]) # suppress python warning warnings.filterwarnings("ignore") logging.getLogger("elasticsearch").setLevel(logging.WARNING) logging.getLogger("kafka").setLevel(logging.WARNING) kp = KafkaProducerProxy(brokers=os.environ["KAFKAURL"].split(",")) # download repo dd.set_attr_stime("spb.scm.stime") gp = GitProxy.init_repository(args.repo, work_dir=args.workspace) repo_url = "https://{}@gitee.com/{}/{}.git".format(args.account, args.owner, args.repo) if not gp.fetch_pull_request(repo_url, args.pr, depth=1): logger.info("fetch finished -") dd.set_attr("spb.scm.result", "failed") dd.set_attr_etime("spb.scm.etime") dd.set_attr_etime("spb.job.etime") #dd.set_attr("spb.job.result", "failed") # upload to es query = {"term": {"id": args.comment_id}} script = { "lang": "painless", "source": "ctx._source.spb_{}=params.spb".format(args.arch),
class CheckSpec(BaseCheck): """ check spec file """ def __init__(self, workspace, repo, conf=None): super(CheckSpec, self).__init__(workspace, repo, conf) self._gp = GitProxy(self._work_dir) self._gr = GiteeRepo(self._repo, self._work_dir, None) # don't care about decompress fp = self._gp.get_content_of_file_with_commit(self._gr.spec_file) self._spec = RPMSpecAdapter(fp) self._latest_commit = self._gp.commit_id_of_reverse_head_index(0) def _only_change_package_yaml(self): """ 如果本次提交只变更yaml,则无需检查version :return: boolean """ diff_files = self._gp.diff_files_between_commits("HEAD~1", "HEAD~0") package_yaml = "{}.yaml".format(self._repo) # package yaml file name if len(diff_files) == 1 and diff_files[0] == package_yaml: logger.debug("diff files: {}".format(diff_files)) return True return False def _is_lts_branch(self): """ check if lts branch :return boolean """ if self._tbranch: if "lts" in self._tbranch.lower(): return True return False def check_version(self): """ 检查当前版本号是否比上一个commit新 :return: """ # need check version? if self._only_change_package_yaml(): logger.debug("only change package yaml") return SUCCESS self._gp.checkout_to_commit_force("HEAD~1") try: gr = GiteeRepo(self._repo, self._work_dir, None) # don't care about decompress fp = self._gp.get_content_of_file_with_commit(gr.spec_file) if fp is None: # last commit has no spec file return SUCCESS spec_o = RPMSpecAdapter(fp) finally: self._gp.checkout_to_commit_force( self._latest_commit) # recover whatever self._ex_pkgship(spec_o) # if lts branch, version update is forbidden if self._is_lts_branch(): logger.debug("lts branch {}".format(self._tbranch)) if RPMSpecAdapter.compare_version(self._spec.version, spec_o.version) == 1: logger.error("version update of lts branch is forbidden") return FAILED if self._spec > spec_o: return SUCCESS elif self._spec < spec_o: if self._gp.is_revert_commit( depth=5): # revert, version back, ignore logger.debug("revert commit") return SUCCESS logger.error("current version: {}-r{}, last version: {}-r{}".format( self._spec.version, self._spec.release, spec_o.version, spec_o.release)) return FAILED def check_homepage(self, timeout=30, retrying=3, interval=1): """ 检查主页是否可访问 :param timeout: 超时时间 :param retrying: 重试次数 :param interval: 重试间隔 :return: """ homepage = self._spec.url logger.debug("homepage: {}".format(homepage)) if not homepage: return SUCCESS for _ in range(retrying): if 0 == do_requests("get", homepage, timeout=timeout): return SUCCESS time.sleep(interval) return FAILED def check_patches(self): """ 检查spec中的patch是否存在 :return: """ patches_spec = set(self._spec.patches) patches_file = set(self._gr.patch_files_not_recursive()) logger.debug("spec patches: {}".format(patches_spec)) logger.debug("file patches: {}".format(patches_file)) result = SUCCESS for patch in patches_spec - patches_file: logger.error("patch {} lost".format(patch)) result = FAILED for patch in patches_file - patches_spec: logger.warning("patch {} redundant".format(patch)) return result def _ex_exclusive_arch(self): """ 保存spec中exclusive_arch信息 :return: """ aarch64 = self._spec.include_aarch64_arch() x86_64 = self._spec.include_x86_arch() content = None if aarch64 and not x86_64: # only build aarch64 content = "aarch64" elif not aarch64 and x86_64: # only build x86_64 content = "x86-64" if content is not None: logger.info("exclusive arch \"{}\"".format(content)) try: with open("exclusive_arch", "w") as f: f.write(content) except IOError: logger.exception("save exclusive arch exception") def _ex_pkgship(self, spec): """ pkgship需求 :param spec: 上一个版本spec对应的RPMSpecAdapter对象 :return: """ if not self._repo == "pkgship": return logger.debug("special repo \"pkgship\"") compare_version = RPMSpecAdapter.compare_version( self._spec.version, spec.version) compare_release = RPMSpecAdapter.compare_version( self._spec.release, spec.release) compare = self._spec.compare(spec) rs = { "repo": "pkgship", "curr_version": self._spec.version, "curr_release": self._spec.release, "last_version": spec.version, "last_release": spec.release, "compare_version": compare_version, "compare_release": compare_release, "compare": compare } logger.info("{}".format(rs)) try: with open("pkgship_notify", "w") as f: yaml.safe_dump(rs, f) except IOError: logger.exception("save pkgship exception") def __call__(self, *args, **kwargs): logger.info("check {} spec ...".format(self._repo)) self._ex_exclusive_arch() self._tbranch = kwargs.get("tbranch", None) # 因门禁系统限制外网访问权限,将涉及外网访问的检查功能check_homepage暂时关闭 return self.start_check_with_order("version", "patches")
class CheckPackageYaml(BaseCheck): """ check yaml file """ NOT_FOUND = "NA" PACKAGE_YAML_NEEDED_KEY = [ "version_control", "src_repo", "tag_prefix", "separator" ] VERSION_CTRL_TRANS = {"gitlab.gnome": "gnome", "pypi": "pythonhosted"} def __init__(self, workspace, repo, conf): super(CheckPackageYaml, self).__init__(workspace, repo, conf) self._gp = GitProxy(self._work_dir) self._gr = GiteeRepo(self._repo, self._work_dir, None) # don't care about decompress if self._gr.spec_file: self._spec = RPMSpecAdapter( os.path.join(self._work_dir, self._gr.spec_file)) else: self._spec = None self._yaml_content = None self._yaml_changed = True self._is_standard = True def is_change_package_yaml(self, base="HEAD~1", head="HEAD~0"): """ 如果本次提交变更了yaml,则对yaml进行检查 :param: base:作为对比的提交点 head:此次提交点 :return: boolean """ diff_files = self._gp.diff_files_between_commits(base, head) package_yaml = "{}.yaml".format(self._repo) # package yaml file name for change_file in diff_files: if change_file == package_yaml: logger.debug("diff files: {}".format(diff_files)) return True return False def check_fields(self): """ 检查yaml规定字段的完整性 :return: """ if not self._yaml_changed: return SUCCESS yaml_path = self._gr.yaml_file if yaml_path is None: self._is_standard = False logger.warning("yaml file missing") return WARNING try: with open(os.path.join(self._work_dir, yaml_path), 'r') as yaml_data: # load yaml data self._yaml_content = yaml.safe_load(yaml_data) except IOError as e: logging.warning("package yaml not exist. {}".format(str(e))) return WARNING except yaml.YAMLError as exc: logging.warning("Error parsering YAML: {}".format(str(exc))) return WARNING result = SUCCESS for keyword in self.PACKAGE_YAML_NEEDED_KEY: if keyword not in self._yaml_content: logger.error("yaml field {} missing".format(keyword)) self._is_standard = True result = WARNING return result def check_repo(self): """ 检查yaml的有效性,能否从上游社区获取版本信息 :return: """ if not self._yaml_changed: return SUCCESS if not self._is_standard: logger.warning("yaml does not comply with the rule") return SUCCESS # get value by key from yaml data vc = self._yaml_content[ self.PACKAGE_YAML_NEEDED_KEY[0]] # value of version_control sr = self._yaml_content[ self.PACKAGE_YAML_NEEDED_KEY[1]] # value of src_repo if vc == self.NOT_FOUND or sr == self.NOT_FOUND: logger.warning("no info for upsteam") return WARNING release_tags = ReleaseTagsFactory.get_release_tags(vc) tags = release_tags.get_tags(sr) if not tags: logger.warning( "failed to get version by yaml, version_control: {t1}, src_repo: {t2}" .format(t1=vc, t2=sr)) return WARNING return SUCCESS def check_repo_domain(self): """ 检查spec中source0域名是否包含yaml的version_control,仅做日志告警只返回SUCCESS(autoconf为特例) :return: """ if not self._yaml_changed: return SUCCESS if not self._is_standard: logger.warning("yaml does not comply with the rule") return SUCCESS if not self._spec: logger.warning("spec does not exist") return SUCCESS vc = self._yaml_content[self.PACKAGE_YAML_NEEDED_KEY[0]] if vc == self.NOT_FOUND: return SUCCESS src_url = self._spec.get_source("Source0") if not src_url: src_url = self._spec.get_source("Source") vc = self.VERSION_CTRL_TRANS.get(vc, vc) # 对特殊的版本控制对应的域名进行转换 logger.debug("version control: {vctrl} source url: {url}".format( vctrl=vc, url=src_url)) if vc not in src_url: # 通过判断版本控制字段是否在主页url中 判断一致性 logger.warning("{vc} is not in url: {url}".format(vc=vc, url=src_url)) return WARNING return SUCCESS def check_repo_name(self): """ 检查spec中是否包含yaml中src_repo字段的软件名,仅做日志告警只返回SUCCESS :return: """ if not self._yaml_changed: return SUCCESS if not self._is_standard: logger.warning("yaml does not comply with the rule") return SUCCESS if not self._spec: logger.warning("spec does not exist") return SUCCESS sr = self._yaml_content[self.PACKAGE_YAML_NEEDED_KEY[1]] if sr == self.NOT_FOUND: return SUCCESS software_name_list = list(filter(None, sr.split("/"))) def guess_real_pkgname(name_list): """ 解析yaml中src_repo字段对应的软件包名 :return: """ pkgname = name_list[-1] if len(name_list) > 1 and name_list[-1] == "svn": pkgname = name_list[-2] if pkgname.endswith(".git"): pkgname = os.path.splitext(pkgname)[0] return pkgname software_name = guess_real_pkgname(software_name_list) src_url = self._spec.get_source("Source0") if not src_url: src_url = self._spec.get_source("Source") logger.debug("software name: {name} source url: {url}".format( name=software_name, url=src_url)) if software_name not in src_url: logger.warning("{name} is not in source0: {url}".format( name=software_name, url=src_url)) return WARNING return SUCCESS def __call__(self, *args, **kwargs): logger.info("check {} yaml ...".format(self._repo)) self._yaml_changed = self.is_change_package_yaml() # yaml文件变更 进行检查 # 因门禁系统限制外网访问权限,将涉及外网访问的检查功能check_repo暂时关闭 return self.start_check_with_order("fields", "repo_domain", "repo_name")