def failed_pattern(line, pattern, verbose, buildtool=None):
global must_restart
pat = re.compile(pattern)
match = pat.search(line)
if not match:
return
s = match.group(1)
try:
if not buildtool:
req = config.failed_commands[s]
if req:
must_restart += buildreq.add_buildreq(req)
elif buildtool == 'pkgconfig':
must_restart += buildreq.add_pkgconfig_buildreq(s)
elif buildtool == 'R':
if buildreq.add_buildreq("R-" + s) > 0:
must_restart += 1
buildreq.add_requires("R-" + s)
elif buildtool == 'perl':
must_restart += buildreq.add_buildreq('perl(%s)' % s)
elif buildtool == 'pypi':
s = util.translate(s)
if not s:
return
must_restart += buildreq.add_buildreq(
util.translate('%s-python' % s))
elif buildtool == 'ruby':
if s in config.gems:
must_restart += buildreq.add_buildreq(config.gems[s])
else:
must_restart += buildreq.add_buildreq('rubygem-%s' % s)
elif buildtool == 'ruby table':
if s in config.gems:
must_restart += buildreq.add_buildreq(config.gems[s])
else:
print("Unknown ruby gem match", s)
elif buildtool == 'maven':
if s in config.maven_jars:
must_restart += buildreq.add_buildreq(config.maven_jars[s])
else:
must_restart += buildreq.add_buildreq('jdk-%s' % s)
elif buildtool == 'catkin':
must_restart += buildreq.add_pkgconfig_buildreq(s)
must_restart += buildreq.add_buildreq(s)
except:
if verbose > 0:
print("Unknown pattern match: ", s)
def test_add_requires(self):
"""
Test add_requires with unbanned new req already present in
buildreqs but not yet present in requires
"""
buildreq.add_buildreq('testreq')
self.assertTrue(buildreq.add_requires('testreq'))
self.assertIn('testreq', buildreq.requires)
def parse_config_files(path, bump, filemanager, version):
"""Parse the various configuration files that may exist in the package directory."""
global extra_configure
global extra_configure32
global extra_configure64
global extra_configure_avx2
global extra_configure_avx512
global config_files
global parallel_build
global license_fetch
global license_show
global git_uri
global os_packages
global urlban
global config_file
global profile_payload
global config_opts
global extra_make
global extra32_make
global extra_make_install
global extra_make32_install
global extra_cmake
global cmake_srcdir
global subdir
global install_macro
global disable_static
global prep_prepend
global build_prepend
global make_prepend
global install_prepend
global install_append
global patches
global autoreconf
global yum_conf
global custom_desc
global failed_pattern_dir
packages_file = None
# Require autospec.conf for additional features
if os.path.exists(config_file):
config = configparser.ConfigParser(interpolation=None)
config.read(config_file)
if "autospec" not in config.sections():
print("Missing autospec section..")
sys.exit(1)
git_uri = config['autospec'].get('git', None)
license_fetch = config['autospec'].get('license_fetch', None)
license_show = config['autospec'].get('license_show', None)
packages_file = config['autospec'].get('packages_file', None)
yum_conf = config['autospec'].get('yum_conf', None)
failed_pattern_dir = config['autospec'].get('failed_pattern_dir', None)
# support reading the local files relative to config_file
if packages_file and not os.path.isabs(packages_file):
packages_file = os.path.join(os.path.dirname(config_file),
packages_file)
if yum_conf and not os.path.isabs(yum_conf):
yum_conf = os.path.join(os.path.dirname(config_file), yum_conf)
if failed_pattern_dir and not os.path.isabs(failed_pattern_dir):
failed_pattern_dir = os.path.join(os.path.dirname(config_file),
failed_pattern_dir)
if not packages_file:
print(
"Warning: Set [autospec][packages_file] path to package list file for "
"requires validation")
packages_file = os.path.join(os.path.dirname(config_file),
"packages")
urlban = config['autospec'].get('urlban', None)
# Read values from options.conf (and deprecated files) and rewrite as necessary
read_config_opts(path)
if not git_uri:
print(
"Warning: Set [autospec][git] upstream template for remote git URI configuration"
)
if not license_fetch:
print(
"Warning: Set [autospec][license_fetch] uri for license fetch support"
)
if not license_show:
print(
"Warning: Set [autospec][license_show] uri for license link check support"
)
if not yum_conf:
print(
"Warning: Set [autospec][yum_conf] path to yum.conf file for whatrequires validation"
)
yum_conf = os.path.join(os.path.dirname(config_file),
"image-creator/yum.conf")
if packages_file:
os_packages = set(read_conf_file(packages_file))
else:
os_packages = set(read_conf_file("~/packages"))
wrapper = textwrap.TextWrapper()
wrapper.initial_indent = "# "
wrapper.subsequent_indent = "# "
def write_default_conf_file(name, description):
"""Write default configuration file with description to file name."""
config_files.add(name)
filename = os.path.join(path, name)
if os.path.isfile(filename):
return
write_out(filename, wrapper.fill(description) + "\n")
write_default_conf_file(
"buildreq_ban",
"This file contains build requirements that get picked up but are "
"undesirable. One entry per line, no whitespace.")
write_default_conf_file(
"pkgconfig_ban",
"This file contains pkgconfig build requirements that get picked up but"
" are undesirable. One entry per line, no whitespace.")
write_default_conf_file(
"requires_ban",
"This file contains runtime requirements that get picked up but are "
"undesirable. One entry per line, no whitespace.")
write_default_conf_file(
"buildreq_add",
"This file contains additional build requirements that did not get "
"picked up automatically. One name per line, no whitespace.")
write_default_conf_file(
"pkgconfig_add",
"This file contains additional pkgconfig build requirements that did "
"not get picked up automatically. One name per line, no whitespace.")
write_default_conf_file(
"requires_add",
"This file contains additional runtime requirements that did not get "
"picked up automatically. One name per line, no whitespace.")
write_default_conf_file(
"excludes",
"This file contains the output files that need %exclude. Full path "
"names, one per line.")
content = read_conf_file(os.path.join(path, "release"))
if content and content[0]:
r = int(content[0])
if bump:
r += 1
tarball.release = str(r)
print("Release :", tarball.release)
content = read_conf_file(os.path.join(path, "buildreq_ban"))
for banned in content:
print("Banning build requirement: %s." % banned)
buildreq.banned_buildreqs.add(banned)
content = read_conf_file(os.path.join(path, "pkgconfig_ban"))
for banned in content:
banned = "pkgconfig(%s)" % banned
print("Banning build requirement: %s." % banned)
buildreq.banned_buildreqs.add(banned)
content = read_conf_file(os.path.join(path, "requires_ban"))
for banned in content:
print("Banning runtime requirement: %s." % banned)
buildreq.banned_requires.add(banned)
content = read_conf_file(os.path.join(path, "buildreq_add"))
for extra in content:
print("Adding additional build requirement: %s." % extra)
buildreq.add_buildreq(extra)
content = read_conf_file(os.path.join(path, "buildreq_cache"))
if content and content[0] == version:
for extra in content[1:]:
print("Adding additional build (cache) requirement: %s." % extra)
buildreq.add_buildreq(extra)
content = read_conf_file(os.path.join(path, "pkgconfig_add"))
for extra in content:
extra = "pkgconfig(%s)" % extra
print("Adding additional build requirement: %s." % extra)
buildreq.add_buildreq(extra)
content = read_conf_file(os.path.join(path, "requires_add"))
for extra in content:
print("Adding additional runtime requirement: %s." % extra)
buildreq.add_requires(extra, override=True)
content = read_conf_file(os.path.join(path, "excludes"))
for exclude in content:
print("%%exclude for: %s." % exclude)
filemanager.excludes += content
content = read_conf_file(os.path.join(path, "extras"))
for extra in content:
print("extras for : %s." % extra)
filemanager.extras += content
content = read_conf_file(os.path.join(path, "dev_extras"))
for extra in content:
print("dev for : %s." % extra)
filemanager.dev_extras += content
content = read_conf_file(os.path.join(path, "setuid"))
for suid in content:
print("setuid for : %s." % suid)
filemanager.setuid += content
content = read_conf_file(os.path.join(path, "attrs"))
for line in content:
attr = re.split(r'\(|\)|,', line)
attr = [a.strip() for a in attr]
filename = attr.pop()
print("attr for: %s." % filename)
filemanager.attrs[filename] = attr
patches += read_conf_file(os.path.join(path, "series"))
pfiles = [("%s/%s" % (path, x.split(" ")[0])) for x in patches]
cmd = "egrep \"(\+\+\+|\-\-\-).*((Makefile.am)|(configure.ac|configure.in))\" %s" % " ".join(
pfiles) # noqa: W605
if patches and call(
cmd, check=False, stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL) == 0:
autoreconf = True
if any(p.lower().startswith('cve-') for p in patches):
config_opts['security_sensitive'] = True
rewrite_config_opts(path)
content = read_conf_file(os.path.join(path, "configure"))
extra_configure = " \\\n".join(content)
content = read_conf_file(os.path.join(path, "configure32"))
extra_configure32 = " \\\n".join(content)
content = read_conf_file(os.path.join(path, "configure64"))
extra_configure64 = " \\\n".join(content)
content = read_conf_file(os.path.join(path, "configure_avx2"))
extra_configure_avx2 = " \\\n".join(content)
content = read_conf_file(os.path.join(path, "configure_avx512"))
extra_configure_avx512 = " \\\n".join(content)
if config_opts["keepstatic"]:
disable_static = ""
if config_opts['broken_parallel_build']:
parallel_build = ""
content = read_conf_file(os.path.join(path, "make_args"))
if content:
extra_make = " \\\n".join(content)
content = read_conf_file(os.path.join(path, "make32_args"))
if content:
extra32_make = " \\\n".join(content)
content = read_conf_file(os.path.join(path, "make_install_args"))
if content:
extra_make_install = " \\\n".join(content)
content = read_conf_file(os.path.join(path, "make32_install_args"))
if content:
extra_make32_install = " \\\n".join(content)
content = read_conf_file(os.path.join(path, "install_macro"))
if content and content[0]:
install_macro = content[0]
content = read_conf_file(os.path.join(path, "cmake_args"))
if content:
extra_cmake = " \\\n".join(content)
content = read_conf_file(os.path.join(path, "cmake_srcdir"))
if content and content[0]:
cmake_srcdir = content[0]
content = read_conf_file(os.path.join(path, "subdir"))
if content and content[0]:
subdir = content[0]
content = read_conf_file(os.path.join(path, "build_pattern"))
if content and content[0]:
buildpattern.set_build_pattern(content[0], 20)
autoreconf = False
content = read_conf_file(os.path.join(path, "make_check_command"))
if content:
check.tests_config = '\n'.join(content)
content = read_conf_file(os.path.join(path, tarball.name + ".license"))
if content and content[0]:
words = content[0].split()
for word in words:
if word.find(":") < 0:
license.add_license(word)
content = read_conf_file(os.path.join(path, "golang_libpath"))
if content and content[0]:
tarball.golibpath = content[0]
print("golibpath : {}".format(tarball.golibpath))
if config_opts['use_clang']:
config_opts['funroll-loops'] = False
buildreq.add_buildreq("llvm")
if config_opts['32bit']:
buildreq.add_buildreq("glibc-libc32")
buildreq.add_buildreq("glibc-dev32")
buildreq.add_buildreq("gcc-dev32")
buildreq.add_buildreq("gcc-libgcc32")
buildreq.add_buildreq("gcc-libstdc++32")
prep_prepend = read_conf_file(os.path.join(path, "prep_prepend"))
if os.path.isfile(os.path.join(path, "prep_append")):
os.rename(os.path.join(path, "prep_append"),
os.path.join(path, "build_prepend"))
make_prepend = read_conf_file(os.path.join(path, "make_prepend"))
build_prepend = read_conf_file(os.path.join(path, "build_prepend"))
install_prepend = read_conf_file(os.path.join(path, "install_prepend"))
if os.path.isfile(os.path.join(path, "make_install_append")):
os.rename(os.path.join(path, "make_install_append"),
os.path.join(path, "install_append"))
install_append = read_conf_file(os.path.join(path, "install_append"))
profile_payload = read_conf_file(os.path.join(path, "profile_payload"))
custom_desc = read_conf_file(os.path.join(path, "description"))
def failed_pattern(line, pattern, verbose, buildtool=None):
"""Check against failed patterns to restart build as needed."""
global must_restart
global warned_about
pat = re.compile(pattern)
match = pat.search(line)
if not match:
return
s = match.group(1)
# standard configure cleanups
s = cleanup_req(s)
if s in config.ignored_commands:
return
try:
if not buildtool:
req = config.failed_commands[s]
if req:
must_restart += buildreq.add_buildreq(req, cache=True)
elif buildtool == 'pkgconfig':
must_restart += buildreq.add_pkgconfig_buildreq(s, cache=True)
elif buildtool == 'R':
if buildreq.add_buildreq("R-" + s, cache=True) > 0:
must_restart += 1
buildreq.add_requires("R-" + s)
elif buildtool == 'perl':
s = s.replace('inc::', '')
must_restart += buildreq.add_buildreq('perl(%s)' % s, cache=True)
elif buildtool == 'pypi':
s = util.translate(s)
if not s:
return
must_restart += buildreq.add_buildreq(util.translate('%s-python' %
s),
cache=True)
elif buildtool == 'ruby':
if s in config.gems:
must_restart += buildreq.add_buildreq(config.gems[s],
cache=True)
else:
must_restart += buildreq.add_buildreq('rubygem-%s' % s,
cache=True)
elif buildtool == 'ruby table':
if s in config.gems:
must_restart += buildreq.add_buildreq(config.gems[s],
cache=True)
else:
print("Unknown ruby gem match", s)
elif buildtool == 'maven':
if s in config.maven_jars:
must_restart += buildreq.add_buildreq(config.maven_jars[s],
cache=True)
else:
must_restart += buildreq.add_buildreq('jdk-%s' % s, cache=True)
elif buildtool == 'catkin':
must_restart += buildreq.add_pkgconfig_buildreq(s, cache=True)
must_restart += buildreq.add_buildreq(s, cache=True)
except Exception:
if s not in warned_about and s[:2] != '--':
print("Unknown pattern match: ", s)
warned_about.add(s)
def failed_pattern(line, pattern, verbose, buildtool=None):
"""Check against failed patterns to restart build as needed."""
global must_restart
global warned_about
pat = re.compile(pattern)
match = pat.search(line)
if not match:
return
s = match.group(1)
# standard configure cleanups
s = cleanup_req(s)
if s in config.ignored_commands:
return
try:
if not buildtool:
req = config.failed_commands[s]
if req:
must_restart += buildreq.add_buildreq(req, cache=True)
elif buildtool == 'pkgconfig':
must_restart += buildreq.add_pkgconfig_buildreq(s, cache=True)
elif buildtool == 'R':
if buildreq.add_buildreq("R-" + s, cache=True) > 0:
must_restart += 1
buildreq.add_requires("R-" + s)
elif buildtool == 'perl':
s = s.replace('inc::', '')
must_restart += buildreq.add_buildreq('perl(%s)' % s, cache=True)
elif buildtool == 'pypi':
s = util.translate(s)
if not s:
return
must_restart += buildreq.add_buildreq(util.translate('%s-python' % s), cache=True)
elif buildtool == 'ruby':
if s in config.gems:
must_restart += buildreq.add_buildreq(config.gems[s], cache=True)
else:
must_restart += buildreq.add_buildreq('rubygem-%s' % s, cache=True)
elif buildtool == 'ruby table':
if s in config.gems:
must_restart += buildreq.add_buildreq(config.gems[s], cache=True)
else:
print("Unknown ruby gem match", s)
elif buildtool == 'maven' or buildtool == 'gradle':
group_count = len(match.groups())
if group_count == 2:
# Add fully qualified versioned mvn() dependency
name = match.group(1)
# Hyphens are disallowed for version strings, so use dots instead
ver = match.group(2).replace('-', '.')
mvn_provide = f'mvn({name}) = {ver}'
must_restart += buildreq.add_buildreq(mvn_provide, cache=True)
elif s in config.maven_jars:
# Overrides for dependencies with custom grouping
must_restart += buildreq.add_buildreq(config.maven_jars[s], cache=True)
elif group_count == 3:
org = match.group(1)
name = match.group(2)
ver = match.group(3).replace('-', '.')
if re.search("-(parent|pom|bom)$", name):
mvn_provide = f'mvn({org}:{name}:pom) = {ver}'
else:
mvn_provide = f'mvn({org}:{name}:jar) = {ver}'
must_restart += buildreq.add_buildreq(mvn_provide, cache=True)
else:
# Fallback to mvn-ARTIFACTID package name
must_restart += buildreq.add_buildreq('mvn-%s' % s, cache=True)
elif buildtool == 'catkin':
must_restart += buildreq.add_pkgconfig_buildreq(s, cache=True)
must_restart += buildreq.add_buildreq(s, cache=True)
except Exception:
if s not in warned_about and s[:2] != '--':
print("Unknown pattern match: ", s)
warned_about.add(s)
def test_add_requires_not_in_buildreqs(self):
"""
Test add_requires with unbanned new req not present in buildreqs.
"""
self.assertFalse(buildreq.add_requires('testreq'))
self.assertNotIn('testreq', buildreq.requires)
