def failed_pattern(line, pattern, verbose, buildtool=None): global must_restart pat = re.compile(pattern) match = pat.search(line) if not match: return s = match.group(1) try: if not buildtool: req = config.failed_commands[s] if req: must_restart += buildreq.add_buildreq(req) elif buildtool == 'pkgconfig': must_restart += buildreq.add_pkgconfig_buildreq(s) elif buildtool == 'R': if buildreq.add_buildreq("R-" + s) > 0: must_restart += 1 buildreq.add_requires("R-" + s) elif buildtool == 'perl': must_restart += buildreq.add_buildreq('perl(%s)' % s) elif buildtool == 'pypi': s = util.translate(s) if not s: return must_restart += buildreq.add_buildreq( util.translate('%s-python' % s)) elif buildtool == 'ruby': if s in config.gems: must_restart += buildreq.add_buildreq(config.gems[s]) else: must_restart += buildreq.add_buildreq('rubygem-%s' % s) elif buildtool == 'ruby table': if s in config.gems: must_restart += buildreq.add_buildreq(config.gems[s]) else: print("Unknown ruby gem match", s) elif buildtool == 'maven': if s in config.maven_jars: must_restart += buildreq.add_buildreq(config.maven_jars[s]) else: must_restart += buildreq.add_buildreq('jdk-%s' % s) elif buildtool == 'catkin': must_restart += buildreq.add_pkgconfig_buildreq(s) must_restart += buildreq.add_buildreq(s) except: if verbose > 0: print("Unknown pattern match: ", s)
def test_add_requires(self): """ Test add_requires with unbanned new req already present in buildreqs but not yet present in requires """ buildreq.add_buildreq('testreq') self.assertTrue(buildreq.add_requires('testreq')) self.assertIn('testreq', buildreq.requires)
def parse_config_files(path, bump, filemanager, version): """Parse the various configuration files that may exist in the package directory.""" global extra_configure global extra_configure32 global extra_configure64 global extra_configure_avx2 global extra_configure_avx512 global config_files global parallel_build global license_fetch global license_show global git_uri global os_packages global urlban global config_file global profile_payload global config_opts global extra_make global extra32_make global extra_make_install global extra_make32_install global extra_cmake global cmake_srcdir global subdir global install_macro global disable_static global prep_prepend global build_prepend global make_prepend global install_prepend global install_append global patches global autoreconf global yum_conf global custom_desc global failed_pattern_dir packages_file = None # Require autospec.conf for additional features if os.path.exists(config_file): config = configparser.ConfigParser(interpolation=None) config.read(config_file) if "autospec" not in config.sections(): print("Missing autospec section..") sys.exit(1) git_uri = config['autospec'].get('git', None) license_fetch = config['autospec'].get('license_fetch', None) license_show = config['autospec'].get('license_show', None) packages_file = config['autospec'].get('packages_file', None) yum_conf = config['autospec'].get('yum_conf', None) failed_pattern_dir = config['autospec'].get('failed_pattern_dir', None) # support reading the local files relative to config_file if packages_file and not os.path.isabs(packages_file): packages_file = os.path.join(os.path.dirname(config_file), packages_file) if yum_conf and not os.path.isabs(yum_conf): yum_conf = os.path.join(os.path.dirname(config_file), yum_conf) if failed_pattern_dir and not os.path.isabs(failed_pattern_dir): failed_pattern_dir = os.path.join(os.path.dirname(config_file), failed_pattern_dir) if not packages_file: print( "Warning: Set [autospec][packages_file] path to package list file for " "requires validation") packages_file = os.path.join(os.path.dirname(config_file), "packages") urlban = config['autospec'].get('urlban', None) # Read values from options.conf (and deprecated files) and rewrite as necessary read_config_opts(path) if not git_uri: print( "Warning: Set [autospec][git] upstream template for remote git URI configuration" ) if not license_fetch: print( "Warning: Set [autospec][license_fetch] uri for license fetch support" ) if not license_show: print( "Warning: Set [autospec][license_show] uri for license link check support" ) if not yum_conf: print( "Warning: Set [autospec][yum_conf] path to yum.conf file for whatrequires validation" ) yum_conf = os.path.join(os.path.dirname(config_file), "image-creator/yum.conf") if packages_file: os_packages = set(read_conf_file(packages_file)) else: os_packages = set(read_conf_file("~/packages")) wrapper = textwrap.TextWrapper() wrapper.initial_indent = "# " wrapper.subsequent_indent = "# " def write_default_conf_file(name, description): """Write default configuration file with description to file name.""" config_files.add(name) filename = os.path.join(path, name) if os.path.isfile(filename): return write_out(filename, wrapper.fill(description) + "\n") write_default_conf_file( "buildreq_ban", "This file contains build requirements that get picked up but are " "undesirable. One entry per line, no whitespace.") write_default_conf_file( "pkgconfig_ban", "This file contains pkgconfig build requirements that get picked up but" " are undesirable. One entry per line, no whitespace.") write_default_conf_file( "requires_ban", "This file contains runtime requirements that get picked up but are " "undesirable. One entry per line, no whitespace.") write_default_conf_file( "buildreq_add", "This file contains additional build requirements that did not get " "picked up automatically. One name per line, no whitespace.") write_default_conf_file( "pkgconfig_add", "This file contains additional pkgconfig build requirements that did " "not get picked up automatically. One name per line, no whitespace.") write_default_conf_file( "requires_add", "This file contains additional runtime requirements that did not get " "picked up automatically. One name per line, no whitespace.") write_default_conf_file( "excludes", "This file contains the output files that need %exclude. Full path " "names, one per line.") content = read_conf_file(os.path.join(path, "release")) if content and content[0]: r = int(content[0]) if bump: r += 1 tarball.release = str(r) print("Release :", tarball.release) content = read_conf_file(os.path.join(path, "buildreq_ban")) for banned in content: print("Banning build requirement: %s." % banned) buildreq.banned_buildreqs.add(banned) content = read_conf_file(os.path.join(path, "pkgconfig_ban")) for banned in content: banned = "pkgconfig(%s)" % banned print("Banning build requirement: %s." % banned) buildreq.banned_buildreqs.add(banned) content = read_conf_file(os.path.join(path, "requires_ban")) for banned in content: print("Banning runtime requirement: %s." % banned) buildreq.banned_requires.add(banned) content = read_conf_file(os.path.join(path, "buildreq_add")) for extra in content: print("Adding additional build requirement: %s." % extra) buildreq.add_buildreq(extra) content = read_conf_file(os.path.join(path, "buildreq_cache")) if content and content[0] == version: for extra in content[1:]: print("Adding additional build (cache) requirement: %s." % extra) buildreq.add_buildreq(extra) content = read_conf_file(os.path.join(path, "pkgconfig_add")) for extra in content: extra = "pkgconfig(%s)" % extra print("Adding additional build requirement: %s." % extra) buildreq.add_buildreq(extra) content = read_conf_file(os.path.join(path, "requires_add")) for extra in content: print("Adding additional runtime requirement: %s." % extra) buildreq.add_requires(extra, override=True) content = read_conf_file(os.path.join(path, "excludes")) for exclude in content: print("%%exclude for: %s." % exclude) filemanager.excludes += content content = read_conf_file(os.path.join(path, "extras")) for extra in content: print("extras for : %s." % extra) filemanager.extras += content content = read_conf_file(os.path.join(path, "dev_extras")) for extra in content: print("dev for : %s." % extra) filemanager.dev_extras += content content = read_conf_file(os.path.join(path, "setuid")) for suid in content: print("setuid for : %s." % suid) filemanager.setuid += content content = read_conf_file(os.path.join(path, "attrs")) for line in content: attr = re.split(r'\(|\)|,', line) attr = [a.strip() for a in attr] filename = attr.pop() print("attr for: %s." % filename) filemanager.attrs[filename] = attr patches += read_conf_file(os.path.join(path, "series")) pfiles = [("%s/%s" % (path, x.split(" ")[0])) for x in patches] cmd = "egrep \"(\+\+\+|\-\-\-).*((Makefile.am)|(configure.ac|configure.in))\" %s" % " ".join( pfiles) # noqa: W605 if patches and call( cmd, check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) == 0: autoreconf = True if any(p.lower().startswith('cve-') for p in patches): config_opts['security_sensitive'] = True rewrite_config_opts(path) content = read_conf_file(os.path.join(path, "configure")) extra_configure = " \\\n".join(content) content = read_conf_file(os.path.join(path, "configure32")) extra_configure32 = " \\\n".join(content) content = read_conf_file(os.path.join(path, "configure64")) extra_configure64 = " \\\n".join(content) content = read_conf_file(os.path.join(path, "configure_avx2")) extra_configure_avx2 = " \\\n".join(content) content = read_conf_file(os.path.join(path, "configure_avx512")) extra_configure_avx512 = " \\\n".join(content) if config_opts["keepstatic"]: disable_static = "" if config_opts['broken_parallel_build']: parallel_build = "" content = read_conf_file(os.path.join(path, "make_args")) if content: extra_make = " \\\n".join(content) content = read_conf_file(os.path.join(path, "make32_args")) if content: extra32_make = " \\\n".join(content) content = read_conf_file(os.path.join(path, "make_install_args")) if content: extra_make_install = " \\\n".join(content) content = read_conf_file(os.path.join(path, "make32_install_args")) if content: extra_make32_install = " \\\n".join(content) content = read_conf_file(os.path.join(path, "install_macro")) if content and content[0]: install_macro = content[0] content = read_conf_file(os.path.join(path, "cmake_args")) if content: extra_cmake = " \\\n".join(content) content = read_conf_file(os.path.join(path, "cmake_srcdir")) if content and content[0]: cmake_srcdir = content[0] content = read_conf_file(os.path.join(path, "subdir")) if content and content[0]: subdir = content[0] content = read_conf_file(os.path.join(path, "build_pattern")) if content and content[0]: buildpattern.set_build_pattern(content[0], 20) autoreconf = False content = read_conf_file(os.path.join(path, "make_check_command")) if content: check.tests_config = '\n'.join(content) content = read_conf_file(os.path.join(path, tarball.name + ".license")) if content and content[0]: words = content[0].split() for word in words: if word.find(":") < 0: license.add_license(word) content = read_conf_file(os.path.join(path, "golang_libpath")) if content and content[0]: tarball.golibpath = content[0] print("golibpath : {}".format(tarball.golibpath)) if config_opts['use_clang']: config_opts['funroll-loops'] = False buildreq.add_buildreq("llvm") if config_opts['32bit']: buildreq.add_buildreq("glibc-libc32") buildreq.add_buildreq("glibc-dev32") buildreq.add_buildreq("gcc-dev32") buildreq.add_buildreq("gcc-libgcc32") buildreq.add_buildreq("gcc-libstdc++32") prep_prepend = read_conf_file(os.path.join(path, "prep_prepend")) if os.path.isfile(os.path.join(path, "prep_append")): os.rename(os.path.join(path, "prep_append"), os.path.join(path, "build_prepend")) make_prepend = read_conf_file(os.path.join(path, "make_prepend")) build_prepend = read_conf_file(os.path.join(path, "build_prepend")) install_prepend = read_conf_file(os.path.join(path, "install_prepend")) if os.path.isfile(os.path.join(path, "make_install_append")): os.rename(os.path.join(path, "make_install_append"), os.path.join(path, "install_append")) install_append = read_conf_file(os.path.join(path, "install_append")) profile_payload = read_conf_file(os.path.join(path, "profile_payload")) custom_desc = read_conf_file(os.path.join(path, "description"))
def failed_pattern(line, pattern, verbose, buildtool=None): """Check against failed patterns to restart build as needed.""" global must_restart global warned_about pat = re.compile(pattern) match = pat.search(line) if not match: return s = match.group(1) # standard configure cleanups s = cleanup_req(s) if s in config.ignored_commands: return try: if not buildtool: req = config.failed_commands[s] if req: must_restart += buildreq.add_buildreq(req, cache=True) elif buildtool == 'pkgconfig': must_restart += buildreq.add_pkgconfig_buildreq(s, cache=True) elif buildtool == 'R': if buildreq.add_buildreq("R-" + s, cache=True) > 0: must_restart += 1 buildreq.add_requires("R-" + s) elif buildtool == 'perl': s = s.replace('inc::', '') must_restart += buildreq.add_buildreq('perl(%s)' % s, cache=True) elif buildtool == 'pypi': s = util.translate(s) if not s: return must_restart += buildreq.add_buildreq(util.translate('%s-python' % s), cache=True) elif buildtool == 'ruby': if s in config.gems: must_restart += buildreq.add_buildreq(config.gems[s], cache=True) else: must_restart += buildreq.add_buildreq('rubygem-%s' % s, cache=True) elif buildtool == 'ruby table': if s in config.gems: must_restart += buildreq.add_buildreq(config.gems[s], cache=True) else: print("Unknown ruby gem match", s) elif buildtool == 'maven': if s in config.maven_jars: must_restart += buildreq.add_buildreq(config.maven_jars[s], cache=True) else: must_restart += buildreq.add_buildreq('jdk-%s' % s, cache=True) elif buildtool == 'catkin': must_restart += buildreq.add_pkgconfig_buildreq(s, cache=True) must_restart += buildreq.add_buildreq(s, cache=True) except Exception: if s not in warned_about and s[:2] != '--': print("Unknown pattern match: ", s) warned_about.add(s)
def failed_pattern(line, pattern, verbose, buildtool=None): """Check against failed patterns to restart build as needed.""" global must_restart global warned_about pat = re.compile(pattern) match = pat.search(line) if not match: return s = match.group(1) # standard configure cleanups s = cleanup_req(s) if s in config.ignored_commands: return try: if not buildtool: req = config.failed_commands[s] if req: must_restart += buildreq.add_buildreq(req, cache=True) elif buildtool == 'pkgconfig': must_restart += buildreq.add_pkgconfig_buildreq(s, cache=True) elif buildtool == 'R': if buildreq.add_buildreq("R-" + s, cache=True) > 0: must_restart += 1 buildreq.add_requires("R-" + s) elif buildtool == 'perl': s = s.replace('inc::', '') must_restart += buildreq.add_buildreq('perl(%s)' % s, cache=True) elif buildtool == 'pypi': s = util.translate(s) if not s: return must_restart += buildreq.add_buildreq(util.translate('%s-python' % s), cache=True) elif buildtool == 'ruby': if s in config.gems: must_restart += buildreq.add_buildreq(config.gems[s], cache=True) else: must_restart += buildreq.add_buildreq('rubygem-%s' % s, cache=True) elif buildtool == 'ruby table': if s in config.gems: must_restart += buildreq.add_buildreq(config.gems[s], cache=True) else: print("Unknown ruby gem match", s) elif buildtool == 'maven' or buildtool == 'gradle': group_count = len(match.groups()) if group_count == 2: # Add fully qualified versioned mvn() dependency name = match.group(1) # Hyphens are disallowed for version strings, so use dots instead ver = match.group(2).replace('-', '.') mvn_provide = f'mvn({name}) = {ver}' must_restart += buildreq.add_buildreq(mvn_provide, cache=True) elif s in config.maven_jars: # Overrides for dependencies with custom grouping must_restart += buildreq.add_buildreq(config.maven_jars[s], cache=True) elif group_count == 3: org = match.group(1) name = match.group(2) ver = match.group(3).replace('-', '.') if re.search("-(parent|pom|bom)$", name): mvn_provide = f'mvn({org}:{name}:pom) = {ver}' else: mvn_provide = f'mvn({org}:{name}:jar) = {ver}' must_restart += buildreq.add_buildreq(mvn_provide, cache=True) else: # Fallback to mvn-ARTIFACTID package name must_restart += buildreq.add_buildreq('mvn-%s' % s, cache=True) elif buildtool == 'catkin': must_restart += buildreq.add_pkgconfig_buildreq(s, cache=True) must_restart += buildreq.add_buildreq(s, cache=True) except Exception: if s not in warned_about and s[:2] != '--': print("Unknown pattern match: ", s) warned_about.add(s)
def test_add_requires_not_in_buildreqs(self): """ Test add_requires with unbanned new req not present in buildreqs. """ self.assertFalse(buildreq.add_requires('testreq')) self.assertNotIn('testreq', buildreq.requires)