def build_unzip(zip_path):
"""Return correct unzip command and the prefix folder name of the contents of zip file.
This function will run the zip file through a content list, parsing that list to get the
root folder name containing the zip file's contents.
The output of the unzip -q -l command has the following format:
***snip***
(optional) hash
Length Date Time Name
--------- ---------- ----- ----
0 01-01-2000 00:00 prefix-dir/sub_dir1/subdir2
***snip***
and this function gets the 'prefix-dir' portion from the start of the unzip -l output.
"""
prefix = ""
contents = subprocess.check_output(["unzip", "-q", "-l", zip_path], universal_newlines=True)
lines = contents.splitlines() if contents else []
# looking for directory prefix in unzip output as it may differ from default
if len(lines) > 2:
# In some cases there is a hash line so we detect this based on the header
# separator character '-'
prefix_line = 3 if len(lines) > 3 and lines[2][0] == '-' else 2
prefix = lines[prefix_line].split("/")[0].split()[-1]
else:
print_fatal("Zip file doesn't appear to have any content")
exit(1)
if not prefix:
print_fatal("Malformed zipfile, unable to determine zipfile prefix")
exit(1)
extract_cmd = "unzip -qq -d {0} {1}".format(build.base_path, zip_path)
return extract_cmd, prefix
def scan_for_licenses(srcdir, config, pkg_name):
"""Scan the project directory for things we can use to guess a description and summary."""
targets = [
"copyright", "copyright.txt", "apache-2.0", "artistic.txt",
"libcurllicense", "gpl.txt", "gpl2.txt", "gplv2.txt", "notice",
"copyrights", "about_bsd.txt"
]
# look for files that start with copying or licen[cs]e (but are
# not likely scripts) or end with licen[cs]e
target_pat = re.compile(
r"^((copying)|(licen[cs]e)|(e[dp]l-v\d+))|(licen[cs]e)(\.(txt|xml))?$")
for dirpath, dirnames, files in os.walk(srcdir):
for name in files:
if name.lower() in targets or target_pat.search(name.lower()):
license_from_copying_hash(os.path.join(dirpath, name), srcdir,
config, pkg_name)
# Also look for files that end with .txt and reside in a LICENSES
# directory. This is a convention that KDE is adopting.
if os.path.basename(dirpath) == "LICENSES" and re.search(
r'\.txt$', name):
license_from_copying_hash(os.path.join(dirpath, name), srcdir,
config, pkg_name)
if not licenses:
print_fatal(
" Cannot find any license or a valid {}.license file!\n".format(
pkg_name))
sys.exit(1)
print("Licenses : ", " ".join(sorted(licenses)))
def build_untar(tarball_path):
"""Determine extract command and tarball prefix from tar -tf output."""
prefix = None
if tarfile.is_tarfile(tarball_path):
with tarfile.open(tarball_path, 'r') as content:
lines = content.getnames()
# When tarball is not empty
if len(lines) == 0:
print_fatal("Tar file doesn't appear to have any content")
exit(1)
elif len(lines) > 1:
prefix = os.path.commonpath(lines)
else:
print_fatal("Not a valid tar file.")
exit(1)
# If we didn't find a common prefix, make a dir, based on the tar filename
if not prefix:
subdir = os.path.splitext(os.path.basename(tarball_path))[0]
extract_cmd = "tar --directory={0} --one-top-level={1} -xf {2}".format(
build.base_path, subdir, tarball_path)
else:
extract_cmd = "tar --directory={0} -xf {1}".format(
build.base_path, tarball_path)
return extract_cmd, prefix
def examine_abi_host(download_path, results_dir):
"""Make use of the hostside abireport tool."""
try:
util.call("abireport scan-packages \"{}\"".format(results_dir),
cwd=download_path)
except Exception as e:
util.print_fatal("Error invoking abireport: {}".format(e))
def save_system_pgo(self, mock_dir, content_name, config):
"""Copy chroot profiles to system pgo."""
root_dir_src = "/"
system_pgo_dir_src = "/var/tmp/pgo"
system_pgo_dir_dst = f"{config.download_path}/pgo.tar.gz"
system_gitignore = f"{config.download_path}/.gitignore"
tar_cmd = f"tar --directory={root_dir_src} --create --file=- var/tmp/pgo/ | pigz -9 -p 20 > {system_pgo_dir_dst}"
if os.path.isdir(system_pgo_dir_src):
if any(os.scandir(system_pgo_dir_src)):
if os.path.isfile(system_pgo_dir_dst):
os.remove(system_pgo_dir_dst)
try:
process = subprocess.run(
tar_cmd,
check=True,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
universal_newlines=True,
)
except subprocess.CalledProcessError as err:
print_fatal(
f"Unable to archive {system_pgo_dir_src} in {system_pgo_dir_dst} from {tar_cmd}: {err}"
)
sys.exit(1)
append_new_gitrule = True
with util.open_auto(system_gitignore, "r+") as gitignore:
for line in gitignore:
if "!pgo.tar.gz" in line:
append_new_gitrule = False
break
if append_new_gitrule:
gitignore.write("!pgo.tar.gz\n")
def git_ls_remote_custom_re(remote_url_cmd, clone_path, path, conf):
git_ls_remote_cmd1_result = ""
git_ls_remote_cmd1_re1_result_pre_sort = []
git_ls_remote_cmd1_re1_result_sorted = []
process = subprocess.run(
remote_url_cmd,
check=False,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
universal_newlines=True,
cwd=clone_path,
)
git_ls_remote_cmd1_result = process.stdout
git_ls_remote_cmd1_re1_result = []
if git_ls_remote_cmd1_result:
git_ls_remote_cmd1_re1 = re.compile(r"(?<=refs\/tags\/).*", re.MULTILINE)
git_ls_remote_cmd1_re1_result = git_ls_remote_cmd1_re1.findall(git_ls_remote_cmd1_result)
#if util.debugging:
#if git_ls_remote_cmd1_re1_result:
#for r in git_ls_remote_cmd1_re1_result:
#print_debug(r)
if git_ls_remote_cmd1_re1_result:
#if util.debugging:
#print_debug(f"conf.custom_git_re: {conf.custom_git_re2}")
git_ls_remote_cmd1_re4_pre = r"{0}".format(conf.custom_git_re2)
git_ls_remote_cmd1_re4 = re.Pattern
try:
git_ls_remote_cmd1_re4 = re.compile(git_ls_remote_cmd1_re4_pre, re.MULTILINE)
except re.error as err:
print_fatal(f"Custom git regex: {git_ls_remote_cmd1_re4.pattern}")
print_fatal(f"Unable to create custom git regex: {err}")
for r in git_ls_remote_cmd1_re1_result:
git_ls_remote_cmd1_re4_result = git_ls_remote_cmd1_re4.search(r)
if git_ls_remote_cmd1_re4_result:
#if util.debugging:
#print_debug(f"{git_ls_remote_cmd1_re4_result.group(1)}.{git_ls_remote_cmd1_re4_result.group(2)}.{git_ls_remote_cmd1_re4_result.group(3)}.{git_ls_remote_cmd1_re4_result.group(4)}.{git_ls_remote_cmd1_re4_result.group(5)}.{git_ls_remote_cmd1_re4_result.group(6)}.{git_ls_remote_cmd1_re4_result.group(7)}")
if git_ls_remote_cmd1_re4_result.group(1):
outputVersionPre = f"{git_ls_remote_cmd1_re4_result.group(1)}"
if git_ls_remote_cmd1_re4_result.group(2):
outputVersionPre = f"{outputVersionPre}.{git_ls_remote_cmd1_re4_result.group(2)}"
if git_ls_remote_cmd1_re4_result.group(3):
outputVersionPre = f"{outputVersionPre}.{git_ls_remote_cmd1_re4_result.group(3)}"
if git_ls_remote_cmd1_re4_result.group(4):
outputVersionPre = f"{outputVersionPre}.{git_ls_remote_cmd1_re4_result.group(4)}"
if git_ls_remote_cmd1_re4_result.group(5):
outputVersionPre = f"{outputVersionPre}{git_ls_remote_cmd1_re4_result.group(5)}"
#if util.debugging:
#print_debug(f"outputVersionPre: {outputVersionPre}")
git_ls_remote_cmd1_re1_result_pre_sort.append(outputVersionPre)
if git_ls_remote_cmd1_re1_result_pre_sort:
git_ls_remote_cmd1_re1_result_sorted = natsort.natsorted(git_ls_remote_cmd1_re1_result_pre_sort, key=lambda x: x.replace('.', '~')+'z')
if git_ls_remote_cmd1_re1_result_sorted:
return git_ls_remote_cmd1_re1_result_sorted[-1]
else:
return ""
def remove_clone_archive(path, clone_path, is_fatal):
"""Remove temporary clone_archive git folder."""
try:
call(f"rm -rf {clone_path}", cwd=path)
except subprocess.CalledProcessError as err:
if is_fatal:
print_fatal("Unable to remove {}: {}".format(clone_path, err))
def package(filemanager):
global round
round = round + 1
set_mock()
print("Building package " + tarball.name + " round", round)
# call(mock_cmd + " -q -r clear --scrub=cache")
# call(mock_cmd + " -q -r clear --scrub=all")
shutil.rmtree('{}/results'.format(download_path), ignore_errors=True)
os.makedirs('{}/results'.format(download_path))
util.call(
mock_cmd +
" -r clear --buildsrpm --sources=./ --spec={0}.spec --uniqueext={0} --result=results/ --no-cleanup-after"
.format(tarball.name),
logfile="%s/mock_srpm.log" % download_path,
cwd=download_path)
util.call("rm -f results/build.log", cwd=download_path)
srcrpm = "results/%s-%s-%s.src.rpm" % (tarball.name, tarball.version,
tarball.release)
returncode = util.call(
mock_cmd +
" -r clear --result=results/ %s --enable-plugin=ccache --uniqueext=%s --no-cleanup-after"
% (srcrpm, tarball.name),
logfile="%s/mock_build.log" % download_path,
check=False,
cwd=download_path)
# sanity check the build log
if not os.path.exists(download_path + "/results/build.log"):
util.print_fatal(
"Mock command failed, results log does not exist. User may not have correct permissions."
)
exit(1)
parse_build_results(download_path + "/results/build.log", returncode,
filemanager)
def scan_for_licenses(srcdir):
"""
Scan the project directory for things we can use to guess a description
and summary
"""
targets = ["copyright",
"copyright.txt",
"apache-2.0",
"libcurllicense",
"gpl.txt",
"gplv2.txt",
"notice",
"copyrights",
"about_bsd.txt"]
# look for files that start with copying or licen[cs]e (spelling errors)
# or end with licen[cs]e
target_pat = re.compile(r"^((copying)|(licen[cs]e))|(licen[cs]e)$")
for dirpath, dirnames, files in os.walk(srcdir):
for name in files:
if name.lower() in targets or target_pat.search(name.lower()):
license_from_copying_hash(os.path.join(dirpath, name), srcdir)
if not licenses:
print_fatal(" Cannot find any license or {}.license file!\n".format(tarball.name))
sys.exit(1)
print("Licenses : ", " ".join(sorted(licenses)))
def build_untar(tarball_path):
"""Determine extract command and tarball prefix from tar -tf output."""
tar_prefix = ""
try:
tarball_contents = subprocess.check_output(
["tar", "-tf", tarball_path], universal_newlines=True).split("\n")
except subprocess.CalledProcessError as cpe:
file_type = subprocess.check_output(["file", tarball_path
]).decode("utf-8").strip()
print_fatal(
"tarball inspection failed, unable to determine tarball contents:\n"
"{}\n{}\n".format(file_type, cpe))
exit(1)
extract_cmd = "tar --directory={0} -xf {1}".format(build.base_path,
tarball_path)
for line in tarball_contents:
# sometimes a leading ./ is prepended to the line, this is not the prefix
line = line.lstrip("./")
# skip this line, it does not contain the prefix or is not a directory
if not line or "/" not in line:
continue
tar_prefix = line.split("/")[0]
if tar_prefix:
break
if not tar_prefix:
print_fatal("malformed tarball, unable to determine tarball prefix")
exit(1)
return extract_cmd, tar_prefix
def build_unzip(zip_path):
"""Return correct unzip command and the prefix folder name of the contents of zip file.
This function will run the zip file through a content list, parsing that list to get the
root folder name containing the zip file's contents.
"""
prefix = None
if zipfile.is_zipfile(zip_path):
with zipfile.ZipFile(zip_path, 'r') as content:
lines = content.namelist()
# When zipfile is not empty
if len(lines) > 0:
prefix = os.path.commonpath(lines)
else:
print_fatal("Zip file doesn't appear to have any content")
exit(1)
else:
print_fatal("Not a valid zip file.")
exit(1)
# If we didn't find a common prefix, make a dir, based on the zip filename
if not prefix:
subdir = os.path.splitext(os.path.basename(zip_path))[0]
extract_cmd = "unzip -qq -d {0} {1}".format(
os.path.join(build.base_path, subdir), zip_path)
else:
extract_cmd = "unzip -qq -d {0} {1}".format(build.base_path, zip_path)
return extract_cmd, prefix
def purge_tree(tree):
if not os.path.exists(tree):
return
try:
shutil.rmtree(tree)
except Exception as e:
util.print_fatal("Cannot remove tree: {}".format(e))
sys.exit(1)
def purge_tree(tree):
if not os.path.exists(tree):
return
try:
shutil.rmtree(tree)
except Exception as e:
util.print_fatal("Cannot remove tree: {}".format(e))
sys.exit(1)
def logcheck(pkg_loc):
"""Try to discover configuration options that were automatically switched off."""
log = os.path.join(pkg_loc, 'results', 'build.log')
if not os.path.exists(log):
print('build log is missing, unable to perform logcheck.')
return
whitelist = []
file_dir = os.path.dirname(os.path.abspath(__file__))
file_path = os.path.join(file_dir, 'configure_whitelist')
with open(file_path, "r") as whitelistf:
for line in whitelistf:
if line.startswith("#"):
continue
whitelist.append(line.rstrip())
blacklist = []
file_dir = os.path.dirname(os.path.abspath(__file__))
file_path = os.path.join(file_dir, 'configure_blacklist')
with open(file_path, "r") as blacklistf:
for line in blacklistf:
if line.startswith("#"):
continue
blacklist.append(line.rstrip())
with open(log, 'r') as logf:
lines = logf.readlines()
pat = re.compile(r"^checking (?:for )?(.*?)\.\.\. no")
misses = []
for line in lines:
match = None
m = pat.search(line)
if m:
match = m.group(1)
if "none required" in line:
match = None
if "warning: format not a string literal" in line:
match = line
if not match or match in whitelist:
continue
if match in blacklist:
print_fatal("Blacklisted configure-miss is forbidden: " + match)
misses.append("Blacklisted configure-miss is forbidden: " + match)
write_misses(pkg_loc, misses)
exit(1)
print("Configure miss: " + match)
misses.append("Configure miss: " + match)
if not misses:
return
write_misses(pkg_loc, misses)
def package(filemanager, mockconfig, mockopts, cleanup=False):
"""Run main package build routine."""
global round
global uniqueext
global success
round = round + 1
success = 0
mock_cmd = get_mock_cmd()
print("Building package " + tarball.name + " round", round)
# determine uniqueext only once
if cleanup:
uniqueext = uniqueext or get_uniqueext("/var/lib/mock", "clear",
tarball.name)
cleanup_flag = "--cleanup-after"
else:
uniqueext = tarball.name
cleanup_flag = "--no-cleanup-after"
print("{} mock chroot at /var/lib/mock/clear-{}".format(
tarball.name, uniqueext))
if round == 1:
shutil.rmtree('{}/results'.format(download_path), ignore_errors=True)
os.makedirs('{}/results'.format(download_path))
util.call("{} -r {} --buildsrpm --sources=./ --spec={}.spec "
"--uniqueext={} --result=results/ {} {}".format(
mock_cmd, mockconfig, tarball.name, uniqueext, cleanup_flag,
mockopts),
logfile="%s/results/mock_srpm.log" % download_path,
cwd=download_path)
# back up srpm mock logs
util.call("mv results/root.log results/srpm-root.log", cwd=download_path)
util.call("mv results/build.log results/srpm-build.log", cwd=download_path)
srcrpm = "results/%s-%s-%s.src.rpm" % (tarball.name, tarball.version,
tarball.release)
returncode = util.call("{} -r {} --result=results/ {} "
"--enable-plugin=ccache --uniqueext={} {}".format(
mock_cmd, mockconfig, srcrpm, uniqueext,
cleanup_flag),
logfile="%s/results/mock_build.log" % download_path,
check=False,
cwd=download_path)
# sanity check the build log
if not os.path.exists(download_path + "/results/build.log"):
util.print_fatal(
"Mock command failed, results log does not exist. User may not have correct permissions."
)
exit(1)
is_clean = parse_buildroot_log(download_path + "/results/root.log",
returncode)
if is_clean:
parse_build_results(download_path + "/results/build.log", returncode,
filemanager)
def build_un7z(zip_path):
"""Return correct 7z command and the prefix folder name of the contents of 7z file.
This function will run the 7z file through a content list, parsing that list to get the
root folder name containing the 7z file's contents.
The output of the 7z l command has the following format:
***snip***
7-Zip [64] 16.02 : Copyright (c) 1999-2016 Igor Pavlov : 2016-05-21
p7zip Version 16.02 (locale=en_US.UTF-8,Utf16=on,HugeFiles=on,64 bits,4 CPUs Intel(R) Core(TM) i5-6260U CPU @ 1.80GHz (406E3),ASM,AES-NI)
Scanning the drive for archives:
1 file, 7933454 bytes (7748 KiB)
Listing archive: foo.7z
--
Path = foo.7z
Type = 7z
Physical Size = 7933454
Headers Size = 1526
Method = LZMA2:23
Solid = +
Blocks = 1
Date Time Attr Size Compressed Name
------------------- ----- ------------ ------------ ------------------------
2018-05-15 05:50:54 ....A 25095 7931928 prefix-dir/sub_dir1/subdir2
***snip***
and this function gets the 'prefix-dir' portion from the start of the unzip -l output.
"""
prefix = ""
contents = subprocess.check_output(["7z", "l", zip_path],
universal_newlines=True)
lines = contents.splitlines() if contents else []
# looking for directory prefix in unzip output as it may differ from default
past_header = False
for line in lines:
if past_header:
# This should be an archive entry; use it
prefix = line.split("/")[0].split()[-1]
break
if line.startswith('----------'):
# This is the header line; next line should be an archive entry
past_header = True
if not past_header:
print_fatal("Zip file doesn't appear to have any content")
exit(1)
if not prefix:
print_fatal("Malformed zipfile, unable to determine zipfile prefix")
exit(1)
extract_cmd = "7z x -o{0} {1}".format(build.base_path, zip_path)
return extract_cmd, prefix
def check_requirements(use_git):
"""Ensure all requirements are satisfied before continuing."""
required_bins = ["mock", "rpm2cpio", "nm", "objdump", "cpio", "readelf"]
if use_git:
required_bins.append("git")
missing = [x for x in required_bins if not binary_in_path(x)]
if missing:
print_fatal("Required programs are not installed: {}".format(", ".join(missing)))
sys.exit(1)
def license_from_copying_hash(copying, srcdir):
"""Add licenses based on the hash of the copying file"""
data = tarball.get_contents(copying)
if data.startswith(b'#!'):
# Not a license if this is a script
return
sh = hashlib.sha1()
sh.update(data)
hash_sum = sh.hexdigest()
if config.license_fetch:
values = {'hash': hash_sum, 'text': data, 'package': tarball.name}
data = urllib.parse.urlencode(values)
data = data.encode('utf-8')
buffer = BytesIO()
c = pycurl.Curl()
c.setopt(c.URL, config.license_fetch)
c.setopt(c.WRITEDATA, buffer)
c.setopt(c.POSTFIELDS, data)
c.setopt(c.FOLLOWLOCATION, 1)
try:
c.perform()
except Exception as excep:
print_fatal("Failed to fetch license from {}: {}".format(
config.license_fetch, excep))
c.close()
sys.exit(1)
c.close()
response = buffer.getvalue()
page = response.decode('utf-8').strip()
if page:
print("License : ", page, " (server) (", hash_sum, ")")
process_licenses(page)
if page != "none":
lic_path = copying[len(srcdir) + 1:]
license_files.append(shlex.quote(lic_path))
return
if hash_sum in config.license_hashes:
add_license(config.license_hashes[hash_sum])
else:
if not config.license_show:
return
print_warning("Unknown license {0} with hash {1}".format(
copying, hash_sum))
hash_url = config.license_show % {'HASH': hash_sum}
print_warning("Visit {0} to enter".format(hash_url))
def do_curl(url, dest=None, post=None, is_fatal=False):
"""
Perform a curl operation for `url`.
If `post` is set, a POST is performed for `url` with fields taken from the
specified value. Otherwise a GET is performed for `url`. If `dest` is set,
the curl response (if successful) is written to the specified path and the
path is returned. Otherwise a successful response is returned as a BytesIO
object. If `is_fatal` is `True` (`False` is the default), a GET failure,
POST failure, or a failure to write to the path specified for `dest`
results in the program exiting with an error. Otherwise, `None` is returned
for any of those error conditions.
"""
c = pycurl.Curl()
c.setopt(c.URL, url)
if post:
c.setopt(c.POSTFIELDS, post)
c.setopt(c.FOLLOWLOCATION, True)
c.setopt(c.FAILONERROR, True)
c.setopt(c.CONNECTTIMEOUT, 10)
c.setopt(c.TIMEOUT, 600)
c.setopt(c.LOW_SPEED_LIMIT, 1)
c.setopt(c.LOW_SPEED_TIME, 10)
buf = BytesIO()
c.setopt(c.WRITEDATA, buf)
try:
c.perform()
except pycurl.error as e:
if is_fatal:
print_fatal("Unable to fetch {}: {}".format(url, e))
sys.exit(1)
return None
finally:
c.close()
# write to dest if specified
if dest:
try:
with open(dest, 'wb') as fp:
fp.write(buf.getvalue())
except IOError as e:
if os.path.exists(dest):
os.unlink(dest)
if is_fatal:
print_fatal("Unable to write to {}: {}".format(dest, e))
sys.exit(1)
return None
if dest:
return dest
else:
return buf
def check_requirements(useGit):
""" Ensure all requirements are satisfied before continuing """
required_bins = ["mock", "rpm2cpio", "nm", "objdump", "cpio", "readelf"]
if useGit:
required_bins.append("git")
paths = os.getenv("PATH", default="/usr/bin:/bin").split(os.pathsep)
missing = [x for x in required_bins if not binary_in_path(x, paths)]
if len(missing) > 0:
print_fatal("Required programs are not installed: {}".format(", ".join(missing)))
sys.exit(1)
def set_zip_prefix(self):
"""Determine prefix folder name of zip file."""
if zipfile.is_zipfile(self.path):
with zipfile.ZipFile(self.path, 'r') as content:
lines = content.namelist()
# When zipfile is not empty
if len(lines) > 0:
self.prefix = os.path.commonpath(lines)
else:
print_fatal("Zip file doesn't appear to have any content")
exit(1)
else:
print_fatal("Not a valid zip file.")
exit(1)
def examine_abi(download_path, name):
"""Proxy the ABI reporting to the right function."""
download_path = os.path.abspath(download_path)
results_dir = os.path.abspath(os.path.join(download_path, "results"))
if not os.path.exists(results_dir):
util.print_fatal("Results directory does not exist, aborting")
sys.exit(1)
if util.binary_in_path("abireport"):
examine_abi_host(download_path, results_dir, name)
else:
util.print_warning("abireport is not installed. Using slow scanning")
examine_abi_fallback(download_path, results_dir, name)
def link_new_rpms_here():
make_cmd = f"make link-new-rpms-here"
try:
process = subprocess.run(
make_cmd,
check=True,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
universal_newlines=True,
)
except subprocess.CalledProcessError as err:
print_fatal(f"Error: {make_cmd}: {err}")
sys.exit(1)
def write_python_flags_fix(self, mock_dir, content_name, config):
"""Patch python to use custom flags."""
python_dir_dst = f"{mock_dir}/clear-{content_name}/root/usr/lib/python3.9"
python_dir_patched_file = f"{python_dir_dst}/patched"
patch_file = "/aot/build/clearlinux/projects/autospec/autospec/0001-Fix-PYTHON-flags.patch"
patch_cmd = f"sudo /usr/bin/patch --backup -p1 --fuzz=2 --input={patch_file}"
if not os.path.isfile(python_dir_patched_file):
try:
process = subprocess.run(
patch_cmd,
check=True,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
universal_newlines=True,
cwd=python_dir_dst,
)
except subprocess.CalledProcessError as err:
revert_patch = [
(f.path, f.path.replace(".orig", ""))
for f in scantree(python_dir_dst) if f.is_file()
and os.path.splitext(f.name)[1].lower() == ".orig"
]
for pcs in revert_patch:
process = subprocess.run(
f"sudo cp {pcs[0]} {pcs[1]}",
check=False,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
universal_newlines=True,
cwd=python_dir_dst,
)
print_fatal(
f"Unable to patch custom flags in {python_dir_dst}: {err}")
sys.exit(1)
process = subprocess.run(
f"echo patched | sudo tee patched",
check=False,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
universal_newlines=True,
cwd=python_dir_dst,
)
def really_download(upstream_url, destination):
"""
Ok, really download the tarball from url
"""
with open(destination, 'wb') as dfile:
c = pycurl.Curl()
c.setopt(c.URL, upstream_url)
c.setopt(c.WRITEDATA, dfile)
c.setopt(c.FOLLOWLOCATION, True)
try:
c.perform()
except pycurl.error:
print_fatal(f"unable to download {upstream_url}")
exit(1)
finally:
c.close()
def parse_buildroot_log(self, filename, returncode):
"""Handle buildroot log contents."""
if returncode == 0:
return True
self.must_restart = 0
is_clean = True
util.call("sync")
with util.open_auto(filename, "r") as rootlog:
loglines = rootlog.readlines()
missing_pat = re.compile(r"^.*No matching package to install: '(.*)'$")
for line in loglines:
match = missing_pat.match(line)
if match is not None:
util.print_fatal("Cannot resolve dependency name: {}".format(match.group(1)))
is_clean = False
return is_clean
def set_tar_prefix(self):
"""Determine prefix folder name of tar file."""
if tarfile.is_tarfile(self.path):
with tarfile.open(self.path, 'r') as content:
lines = content.getnames()
# When tarball is not empty
if len(lines) == 0:
print_fatal("Tar file doesn't appear to have any content")
exit(1)
elif len(lines) > 1:
if 'package.xml' in lines and self.pattern in ['phpize']:
lines.remove('package.xml')
self.prefix = os.path.commonpath(lines)
else:
print_fatal("Not a valid tar file.")
exit(1)
def license_from_copying_hash(copying, srcdir):
"""Add licenses based on the hash of the copying file"""
hash_sum = tarball.get_sha1sum(copying)
if config.license_fetch:
with open(copying, "r", encoding="latin-1") as myfile:
data = myfile.read()
values = {'hash': hash_sum, 'text': data, 'package': tarball.name}
data = urllib.parse.urlencode(values)
data = data.encode('utf-8')
buffer = BytesIO()
c = pycurl.Curl()
c.setopt(c.URL, config.license_fetch)
c.setopt(c.WRITEDATA, buffer)
c.setopt(c.POSTFIELDS, data)
c.setopt(c.FOLLOWLOCATION, 1)
try:
c.perform()
except Exception as excep:
print_fatal("Failed to fetch license from {}: {}"
.format(config.license_fetch, excep))
c.close()
sys.exit(1)
c.close()
response = buffer.getvalue()
page = response.decode('utf-8').strip()
if page:
print("License : ", page, " (server) (", hash_sum, ")")
add_license(page)
if page != "none":
license_files.append(copying[len(srcdir) + 1:])
return
if hash_sum in config.license_hashes:
add_license(config.license_hashes[hash_sum])
else:
if not config.license_show:
return
print_warning("Unknown license {0} with hash {1}".format(copying, hash_sum))
hash_url = config.license_show % {'HASH': hash_sum}
print_warning("Visit {0} to enter".format(hash_url))
def examine_abi_host(download_path, results_dir, name):
"""Make use of the hostside abireport tool."""
rpms = set()
for item in os.listdir(results_dir):
namelen = len(name)
if item.find("-extras-", namelen) >= namelen:
continue
if item.endswith(".rpm") and not item.endswith(".src.rpm"):
rpms.add("{}/{}".format(results_dir, item))
if len(rpms) == 0:
util.print_fatal("No usable rpms found, aborting")
sys.exit(1)
try:
util.call("abireport scan-packages {}".format(" ".join(rpms)),
cwd=download_path)
except Exception as e:
util.print_fatal("Error invoking abireport: {}".format(e))
def git_describe_custom_re(clone_path, conf):
outputVersion1 = ""
git_describe_cmd1 = f"git describe --abbrev=0 --tags"
git_describe_cmd1_result = ""
process = subprocess.run(
git_describe_cmd1,
check=False,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
universal_newlines=True,
cwd=clone_path,
)
git_describe_cmd1_result = process.stdout
if git_describe_cmd1_result:
#if util.debugging:
#print_debug(f"conf.custom_git_re2: {conf.custom_git_re2}")
git_describe_cmd2_re1_pre = r"{0}".format(conf.custom_git_re2)
git_describe_cmd2_re1 = re.Pattern
try:
git_describe_cmd2_re1 = re.compile(git_describe_cmd2_re1_pre, re.MULTILINE)
except re.error as err:
print_fatal(f"Custom git regex: {git_describe_cmd2_re1.pattern}")
print_fatal(f"Unable to create custom git regex: {err}")
print_info(f"Custom git regex 2: {git_describe_cmd2_re1.pattern}")
git_describe_cmd2_re1_result = git_describe_cmd2_re1.search(git_describe_cmd1_result)
if git_describe_cmd2_re1_result:
#if util.debugging:
#print_debug(f"{git_describe_cmd2_re1_result.group(1)}.{git_describe_cmd2_re1_result.group(2)}.{git_describe_cmd2_re1_result.group(3)}.{git_describe_cmd2_re1_result.group(4)}.{git_describe_cmd2_re1_result.group(5)}.{git_describe_cmd2_re1_result.group(6)}.{git_describe_cmd2_re1_result.group(7)}")
if git_describe_cmd2_re1_result.group(1):
outputVersion1 = f"{git_describe_cmd2_re1_result.group(1)}"
if git_describe_cmd2_re1_result.group(2):
outputVersion1 = f"{outputVersion1}.{git_describe_cmd2_re1_result.group(2)}"
if git_describe_cmd2_re1_result.group(3):
outputVersion1 = f"{outputVersion1}.{git_describe_cmd2_re1_result.group(3)}"
if git_describe_cmd2_re1_result.group(4):
outputVersion1 = f"{outputVersion1}.{git_describe_cmd2_re1_result.group(4)}"
if git_describe_cmd2_re1_result.group(5):
outputVersion1 = f"{outputVersion1}{git_describe_cmd2_re1_result.group(5)}"
return outputVersion1
def check_url_content(url):
"""
Check that the url to the infile file is raw or in plaintext.
This function checks the header content-type for a request to the infile
url. If it is html it converts eith a github url to point to the raw file,
a git url to point to the plaintext file, or returns None with a statement
that the file must be in plaintext or raw.
"""
if "text/html" in requests.head(url).headers['content-type']:
if re.match(r'^(http)s?://github.com(.*)', url):
url = url.replace("github",
"raw.githubusercontent").replace("blob/", "")
elif 'git' in url and "/tree/" in url:
url = url.replace("/tree/", "/plain/", 1)
else:
util.print_fatal("infile url has an html content-type, "
"please use plaintext.")
return None
return url
def git_clone(url, path, cmd_args, clone_path, force_module, force_fullclone, is_fatal=True):
try:
if force_module is True:
if force_fullclone is True:
print_info(f"git clone -j8 --branch={cmd_args}")
call(f"git clone -j8 --branch={cmd_args}", cwd=path)
else:
print_info(f"git clone --single-branch -j8 --branch={cmd_args}")
call(f"git clone --depth=1 --single-branch -j8 --branch={cmd_args}", cwd=path)
else:
if force_fullclone is True:
print_info(f"git clone --recurse-submodules -j8 --branch={cmd_args}")
call(f"git clone --recurse-submodules -j8 --branch={cmd_args}", cwd=path)
else:
print_info(f"git clone --single-branch --shallow-submodules --recurse-submodules -j8 --branch={cmd_args}")
call(f"git clone --depth=1 --single-branch --shallow-submodules --recurse-submodules -j8 --branch={cmd_args}", cwd=path)
except subprocess.CalledProcessError as err:
if is_fatal:
remove_clone_archive(path, clone_path, is_fatal)
print_fatal(f"Unable to clone {url} in {clone_path}: {err}")
sys.exit(1)
def examine_abi(download_path):
download_path = os.path.abspath(download_path)
results_dir = os.path.abspath(os.path.join(download_path, "results"))
if not os.path.exists(results_dir):
util.print_fatal("Results directory does not exist, aborting")
sys.exit(1)
old_dir = os.getcwd()
rpms = set()
for item in os.listdir(results_dir):
if item.endswith(".rpm") and not item.endswith(".src.rpm"):
rpms.add(os.path.basename(item))
if len(rpms) == 0:
util.print_fatal("No usable rpms found, aborting")
sys.exit(1)
extract_dir = os.path.abspath(os.path.join(download_path, "__extraction"))
purge_tree(extract_dir)
try:
os.makedirs(extract_dir)
except Exception as e:
util.print_fatal("Cannot create extraction tree: {}".format(e))
sys.exit(1)
os.chdir(extract_dir)
# Extract all those rpms to our current directory
try:
for rpm in rpms:
cmd = "rpm2cpio \"{}\" | cpio -imd 2>/dev/null".format(os.path.join(results_dir, rpm))
subprocess.check_call(cmd, shell=True)
except Exception as e:
util.print_fatal("Error extracting RPMS: {}".format(e))
os.chdir(download_path)
collected_files = set()
# Places we expect to find shared libraries
for check_path in valid_dirs:
if check_path[0] == '/':
check_path = check_path[1:]
dirn = os.path.join(extract_dir, check_path)
if not os.path.isdir(dirn):
continue
for file in os.listdir(dirn):
f = os.path.basename(file)
clean_path = os.path.abspath(os.path.join(dirn, f))
if not is_file_valid(clean_path):
continue
collected_files.add(clean_path)
abi_report = dict()
# Now examine these libraries
for library in sorted(collected_files):
soname = get_soname(library)
if not soname:
warn = "Failed to determine soname of: {}".format(library)
util.print_warning(warn)
soname = os.path.basename(library)
symbols = dump_symbols(library)
if symbols and len(symbols) > 0:
if soname not in abi_report:
abi_report[soname] = set()
abi_report[soname].update(symbols)
report_file = os.path.join(download_path, "symbols")
if len(abi_report) > 0:
# Finally, write the report
report = open(report_file, "w", encoding="utf-8")
for soname in sorted(abi_report.keys()):
for symbol in sorted(abi_report[soname]):
report.write("{}:{}\n".format(soname, symbol))
report.close()
else:
truncate_file(report_file)
# Write the library report
lib_deps = get_all_dependencies(extract_dir)
report_file = os.path.join(download_path, "used_libs")
if len(lib_deps) > 0:
report = open(report_file, "w", encoding="utf-8")
for soname in sorted(lib_deps):
report.write("{}\n".format(soname))
report.close()
else:
truncate_file(report_file)
os.chdir(old_dir)
purge_tree(extract_dir)
