def check_scanoss(self):
"""
scanoss工具检查代码片段引用
https://osskb.org
https://github.com/scanoss/scanner.py
:return:
"""
gp = GitProxy(self._work_dir)
diff_files = gp.diff_files_between_commits("HEAD~1", "HEAD~0")
logger.debug("diff files: {}".format(diff_files))
self.copy_diff_files_to_dest(diff_files)
blacklist_sbom = os.path.realpath(
os.path.join(os.path.realpath(__file__),
"../../../../conf/deny_list.sbom"))
scan = ScanOSS(self._scanoss_api_key, self._scanoss_api_url,
blacklist_sbom)
result = scan.scan(self._work_diff_dir)
# 保存详细结果到web server
if not result:
self.save_scanoss_result(scan.html)
logger.warning("click {} view scanoss detail".format(
self._scanoss_result_repo_path))
return SUCCESS if result else FAILED
def __init__(self, workspace, repo, conf=None):
super(CheckSpec, self).__init__(workspace, repo, conf)
self._gp = GitProxy(self._work_dir)
self._gr = GiteeRepo(self._repo, self._work_dir,
None) # don't care about decompress
fp = self._gp.get_content_of_file_with_commit(self._gr.spec_file)
self._spec = RPMSpecAdapter(fp)
self._latest_commit = self._gp.commit_id_of_reverse_head_index(0)
def __init__(self, workspace, repo, conf):
super(CheckPackageYaml, self).__init__(workspace, repo, conf)
self._gp = GitProxy(self._work_dir)
self._gr = GiteeRepo(self._repo, self._work_dir,
None) # don't care about decompress
if self._gr.spec_file:
self._spec = RPMSpecAdapter(
os.path.join(self._work_dir, self._gr.spec_file))
else:
self._spec = None
self._yaml_content = None
self._yaml_changed = True
self._is_standard = True
def apply_patch(self, patch, max_leading=5):
"""
尝试所有路径和leading
:param patch: 补丁
:param max_leading: leading path
"""
logger.debug("apply patch {}".format(patch))
for patch_dir in [
filename for filename in os.listdir(self._decompress_dir)
if os.path.isdir(os.path.join(self._decompress_dir, filename))
] + ["."]:
if patch_dir.startswith(".git"):
continue
for leading in range(max_leading + 1):
logger.debug("try dir {} -p{}".format(patch_dir, leading))
if GitProxy.apply_patch_at_dir(
os.path.join(self._decompress_dir, patch_dir),
os.path.join(self._work_dir, patch), leading):
logger.debug("patch success")
self.patch_dir_mapping[patch] = os.path.join(
self._decompress_dir, patch_dir)
return True
logger.info("apply patch {} failed".format(patch))
return False
def check_code_style(self):
"""
检查代码风格
:return:
"""
gp = GitProxy(self._work_dir)
diff_files = gp.diff_files_between_commits("HEAD~1", "HEAD~0")
logger.debug("diff files: {}".format(diff_files))
diff_code_files = [] # 仓库中变更的代码文件
diff_patch_code_files = [] # patch内的代码文件
for diff_file in diff_files:
if GiteeRepo.is_code_file(diff_file):
diff_code_files.append(diff_file)
elif GiteeRepo.is_patch_file(diff_file):
patch_dir = self._gr.patch_dir_mapping.get(diff_file)
logger.debug("diff patch {} apply at dir {}".format(
diff_file, patch_dir))
if patch_dir is not None:
files_in_patch = gp.extract_files_path_of_patch(diff_file)
patch_code_files = [
os.path.join(patch_dir, file_in_patch)
for file_in_patch in files_in_patch
if GiteeRepo.is_code_file(file_in_patch)
]
# care about deleted file in patch, filter with "git patch --summary" maybe better
diff_patch_code_files.extend([
code_file for code_file in patch_code_files
if os.path.exists(code_file)
])
logger.debug("diff code files: {}".format(diff_code_files))
logger.debug("diff patch code files: {}".format(diff_patch_code_files))
rs_1 = self.check_file_under_work_dir(diff_code_files)
logger.debug("check_file_under_work_dir: {}".format(rs_1))
rs_2 = self.check_files_inner_patch(diff_patch_code_files)
logger.debug("check_files_inner_patch: {}".format(rs_2))
return rs_1 + rs_2
def __init__(self, workspace, repo, conf=None):
super(CheckLicense, self).__init__(workspace, repo, conf)
self._gp = GitProxy(self._work_dir)
self._work_tar_dir = os.path.join(workspace, "code")
self._gr = GiteeRepo(self._repo, self._work_dir, self._work_tar_dir)
if self._gr.spec_file:
self._spec = RPMSpecAdapter(
os.path.join(self._work_dir, self._gr.spec_file))
else:
self._spec = None
self._pkg_license = PkgLicense()
self._license_in_spec = set()
self._license_in_src = set()
dd = DistDataset()
dd.set_attr_stime("spb.job.stime")
dd.set_attr("spb.job.link", os.environ["BUILD_URL"])
dd.set_attr("spb.trigger.reason", os.environ["BUILD_CAUSE"])
# suppress python warning
warnings.filterwarnings("ignore")
logging.getLogger("elasticsearch").setLevel(logging.WARNING)
logging.getLogger("kafka").setLevel(logging.WARNING)
kp = KafkaProducerProxy(brokers=os.environ["KAFKAURL"].split(","))
# download repo
dd.set_attr_stime("spb.scm.stime")
gp = GitProxy.init_repository(args.repo, work_dir=args.workspace)
repo_url = "https://{}@gitee.com/{}/{}.git".format(args.account,
args.owner, args.repo)
if not gp.fetch_pull_request(repo_url, args.pr, depth=1):
logger.info("fetch finished -")
dd.set_attr("spb.scm.result", "failed")
dd.set_attr_etime("spb.scm.etime")
dd.set_attr_etime("spb.job.etime")
#dd.set_attr("spb.job.result", "failed")
# upload to es
query = {"term": {"id": args.comment_id}}
script = {
"lang": "painless",
"source": "ctx._source.spb_{}=params.spb".format(args.arch),
class CheckSpec(BaseCheck):
"""
check spec file
"""
def __init__(self, workspace, repo, conf=None):
super(CheckSpec, self).__init__(workspace, repo, conf)
self._gp = GitProxy(self._work_dir)
self._gr = GiteeRepo(self._repo, self._work_dir,
None) # don't care about decompress
fp = self._gp.get_content_of_file_with_commit(self._gr.spec_file)
self._spec = RPMSpecAdapter(fp)
self._latest_commit = self._gp.commit_id_of_reverse_head_index(0)
def _only_change_package_yaml(self):
"""
如果本次提交只变更yaml，则无需检查version
:return: boolean
"""
diff_files = self._gp.diff_files_between_commits("HEAD~1", "HEAD~0")
package_yaml = "{}.yaml".format(self._repo) # package yaml file name
if len(diff_files) == 1 and diff_files[0] == package_yaml:
logger.debug("diff files: {}".format(diff_files))
return True
return False
def _is_lts_branch(self):
"""
check if lts branch
:return boolean
"""
if self._tbranch:
if "lts" in self._tbranch.lower():
return True
return False
def check_version(self):
"""
检查当前版本号是否比上一个commit新
:return:
"""
# need check version？
if self._only_change_package_yaml():
logger.debug("only change package yaml")
return SUCCESS
self._gp.checkout_to_commit_force("HEAD~1")
try:
gr = GiteeRepo(self._repo, self._work_dir,
None) # don't care about decompress
fp = self._gp.get_content_of_file_with_commit(gr.spec_file)
if fp is None:
# last commit has no spec file
return SUCCESS
spec_o = RPMSpecAdapter(fp)
finally:
self._gp.checkout_to_commit_force(
self._latest_commit) # recover whatever
self._ex_pkgship(spec_o)
# if lts branch, version update is forbidden
if self._is_lts_branch():
logger.debug("lts branch {}".format(self._tbranch))
if RPMSpecAdapter.compare_version(self._spec.version,
spec_o.version) == 1:
logger.error("version update of lts branch is forbidden")
return FAILED
if self._spec > spec_o:
return SUCCESS
elif self._spec < spec_o:
if self._gp.is_revert_commit(
depth=5): # revert, version back, ignore
logger.debug("revert commit")
return SUCCESS
logger.error("current version: {}-r{}, last version: {}-r{}".format(
self._spec.version, self._spec.release, spec_o.version,
spec_o.release))
return FAILED
def check_homepage(self, timeout=30, retrying=3, interval=1):
"""
检查主页是否可访问
:param timeout: 超时时间
:param retrying: 重试次数
:param interval: 重试间隔
:return:
"""
homepage = self._spec.url
logger.debug("homepage: {}".format(homepage))
if not homepage:
return SUCCESS
for _ in range(retrying):
if 0 == do_requests("get", homepage, timeout=timeout):
return SUCCESS
time.sleep(interval)
return FAILED
def check_patches(self):
"""
检查spec中的patch是否存在
:return:
"""
patches_spec = set(self._spec.patches)
patches_file = set(self._gr.patch_files_not_recursive())
logger.debug("spec patches: {}".format(patches_spec))
logger.debug("file patches: {}".format(patches_file))
result = SUCCESS
for patch in patches_spec - patches_file:
logger.error("patch {} lost".format(patch))
result = FAILED
for patch in patches_file - patches_spec:
logger.warning("patch {} redundant".format(patch))
return result
def _ex_exclusive_arch(self):
"""
保存spec中exclusive_arch信息
:return:
"""
aarch64 = self._spec.include_aarch64_arch()
x86_64 = self._spec.include_x86_arch()
content = None
if aarch64 and not x86_64: # only build aarch64
content = "aarch64"
elif not aarch64 and x86_64: # only build x86_64
content = "x86-64"
if content is not None:
logger.info("exclusive arch \"{}\"".format(content))
try:
with open("exclusive_arch", "w") as f:
f.write(content)
except IOError:
logger.exception("save exclusive arch exception")
def _ex_pkgship(self, spec):
"""
pkgship需求
:param spec: 上一个版本spec对应的RPMSpecAdapter对象
:return:
"""
if not self._repo == "pkgship":
return
logger.debug("special repo \"pkgship\"")
compare_version = RPMSpecAdapter.compare_version(
self._spec.version, spec.version)
compare_release = RPMSpecAdapter.compare_version(
self._spec.release, spec.release)
compare = self._spec.compare(spec)
rs = {
"repo": "pkgship",
"curr_version": self._spec.version,
"curr_release": self._spec.release,
"last_version": spec.version,
"last_release": spec.release,
"compare_version": compare_version,
"compare_release": compare_release,
"compare": compare
}
logger.info("{}".format(rs))
try:
with open("pkgship_notify", "w") as f:
yaml.safe_dump(rs, f)
except IOError:
logger.exception("save pkgship exception")
def __call__(self, *args, **kwargs):
logger.info("check {} spec ...".format(self._repo))
self._ex_exclusive_arch()
self._tbranch = kwargs.get("tbranch", None)
# 因门禁系统限制外网访问权限，将涉及外网访问的检查功能check_homepage暂时关闭
return self.start_check_with_order("version", "patches")
class CheckPackageYaml(BaseCheck):
"""
check yaml file
"""
NOT_FOUND = "NA"
PACKAGE_YAML_NEEDED_KEY = [
"version_control", "src_repo", "tag_prefix", "separator"
]
VERSION_CTRL_TRANS = {"gitlab.gnome": "gnome", "pypi": "pythonhosted"}
def __init__(self, workspace, repo, conf):
super(CheckPackageYaml, self).__init__(workspace, repo, conf)
self._gp = GitProxy(self._work_dir)
self._gr = GiteeRepo(self._repo, self._work_dir,
None) # don't care about decompress
if self._gr.spec_file:
self._spec = RPMSpecAdapter(
os.path.join(self._work_dir, self._gr.spec_file))
else:
self._spec = None
self._yaml_content = None
self._yaml_changed = True
self._is_standard = True
def is_change_package_yaml(self, base="HEAD~1", head="HEAD~0"):
"""
如果本次提交变更了yaml，则对yaml进行检查
:param:
base:作为对比的提交点
head:此次提交点
:return: boolean
"""
diff_files = self._gp.diff_files_between_commits(base, head)
package_yaml = "{}.yaml".format(self._repo) # package yaml file name
for change_file in diff_files:
if change_file == package_yaml:
logger.debug("diff files: {}".format(diff_files))
return True
return False
def check_fields(self):
"""
检查yaml规定字段的完整性
:return:
"""
if not self._yaml_changed:
return SUCCESS
yaml_path = self._gr.yaml_file
if yaml_path is None:
self._is_standard = False
logger.warning("yaml file missing")
return WARNING
try:
with open(os.path.join(self._work_dir, yaml_path),
'r') as yaml_data: # load yaml data
self._yaml_content = yaml.safe_load(yaml_data)
except IOError as e:
logging.warning("package yaml not exist. {}".format(str(e)))
return WARNING
except yaml.YAMLError as exc:
logging.warning("Error parsering YAML: {}".format(str(exc)))
return WARNING
result = SUCCESS
for keyword in self.PACKAGE_YAML_NEEDED_KEY:
if keyword not in self._yaml_content:
logger.error("yaml field {} missing".format(keyword))
self._is_standard = True
result = WARNING
return result
def check_repo(self):
"""
检查yaml的有效性,能否从上游社区获取版本信息
:return:
"""
if not self._yaml_changed:
return SUCCESS
if not self._is_standard:
logger.warning("yaml does not comply with the rule")
return SUCCESS
# get value by key from yaml data
vc = self._yaml_content[
self.PACKAGE_YAML_NEEDED_KEY[0]] # value of version_control
sr = self._yaml_content[
self.PACKAGE_YAML_NEEDED_KEY[1]] # value of src_repo
if vc == self.NOT_FOUND or sr == self.NOT_FOUND:
logger.warning("no info for upsteam")
return WARNING
release_tags = ReleaseTagsFactory.get_release_tags(vc)
tags = release_tags.get_tags(sr)
if not tags:
logger.warning(
"failed to get version by yaml, version_control: {t1}, src_repo: {t2}"
.format(t1=vc, t2=sr))
return WARNING
return SUCCESS
def check_repo_domain(self):
"""
检查spec中source0域名是否包含yaml的version_control,仅做日志告警只返回SUCCESS(autoconf为特例)
:return:
"""
if not self._yaml_changed:
return SUCCESS
if not self._is_standard:
logger.warning("yaml does not comply with the rule")
return SUCCESS
if not self._spec:
logger.warning("spec does not exist")
return SUCCESS
vc = self._yaml_content[self.PACKAGE_YAML_NEEDED_KEY[0]]
if vc == self.NOT_FOUND:
return SUCCESS
src_url = self._spec.get_source("Source0")
if not src_url:
src_url = self._spec.get_source("Source")
vc = self.VERSION_CTRL_TRANS.get(vc, vc) # 对特殊的版本控制对应的域名进行转换
logger.debug("version control: {vctrl} source url: {url}".format(
vctrl=vc, url=src_url))
if vc not in src_url: # 通过判断版本控制字段是否在主页url中 判断一致性
logger.warning("{vc} is not in url: {url}".format(vc=vc,
url=src_url))
return WARNING
return SUCCESS
def check_repo_name(self):
"""
检查spec中是否包含yaml中src_repo字段的软件名,仅做日志告警只返回SUCCESS
:return:
"""
if not self._yaml_changed:
return SUCCESS
if not self._is_standard:
logger.warning("yaml does not comply with the rule")
return SUCCESS
if not self._spec:
logger.warning("spec does not exist")
return SUCCESS
sr = self._yaml_content[self.PACKAGE_YAML_NEEDED_KEY[1]]
if sr == self.NOT_FOUND:
return SUCCESS
software_name_list = list(filter(None, sr.split("/")))
def guess_real_pkgname(name_list):
"""
解析yaml中src_repo字段对应的软件包名
:return:
"""
pkgname = name_list[-1]
if len(name_list) > 1 and name_list[-1] == "svn":
pkgname = name_list[-2]
if pkgname.endswith(".git"):
pkgname = os.path.splitext(pkgname)[0]
return pkgname
software_name = guess_real_pkgname(software_name_list)
src_url = self._spec.get_source("Source0")
if not src_url:
src_url = self._spec.get_source("Source")
logger.debug("software name: {name} source url: {url}".format(
name=software_name, url=src_url))
if software_name not in src_url:
logger.warning("{name} is not in source0: {url}".format(
name=software_name, url=src_url))
return WARNING
return SUCCESS
def __call__(self, *args, **kwargs):
logger.info("check {} yaml ...".format(self._repo))
self._yaml_changed = self.is_change_package_yaml() # yaml文件变更 进行检查
# 因门禁系统限制外网访问权限，将涉及外网访问的检查功能check_repo暂时关闭
return self.start_check_with_order("fields", "repo_domain",
"repo_name")