def detect_build_from_url(url): """ Detect build patterns and build requirements from the patterns detected in the url. """ # R package if "cran.r-project.org" in url or "cran.rstudio.com" in url: buildpattern.set_build_pattern("R", 10) buildreq.add_buildreq("clr-R-helpers") # python if "pypi.python.org" in url or "pypi.debian.net" in url: buildpattern.set_build_pattern("distutils3", 10) buildreq.add_buildreq("python3-dev") # cpan if ".cpan.org/" in url or ".metacpan.org/" in url: buildpattern.set_build_pattern("cpan", 10) # ruby if "rubygems.org/" in url: buildpattern.set_build_pattern("ruby", 10) # maven if ".maven." in url: buildpattern.set_build_pattern("maven", 10) # rust crate if "crates.io" in url: buildpattern.set_build_pattern("cargo", 10)
def test_add_buildreq(self): """ Test add_buildreq with unbanned new req. Follow up by asserting that trying to add the same req a second time results in a False return. """ self.assertTrue(buildreq.add_buildreq('testreq')) self.assertIn('testreq', buildreq.buildreqs) self.assertFalse(buildreq.add_buildreq('testreq'))
def test_add_requires(self): """ Test add_requires with unbanned new req already present in buildreqs but not yet present in requires """ buildreq.add_buildreq('testreq') self.assertTrue(buildreq.add_requires('testreq')) self.assertIn('testreq', buildreq.requires)
def failed_pattern(line, pattern, verbose, buildtool=None): global must_restart pat = re.compile(pattern) match = pat.search(line) if not match: return s = match.group(1) try: if not buildtool: req = config.failed_commands[s] if req: must_restart += buildreq.add_buildreq(req) elif buildtool == 'pkgconfig': must_restart += buildreq.add_pkgconfig_buildreq(s) elif buildtool == 'R': if buildreq.add_buildreq("R-" + s) > 0: must_restart += 1 buildreq.add_requires("R-" + s) elif buildtool == 'perl': must_restart += buildreq.add_buildreq('perl(%s)' % s) elif buildtool == 'pypi': s = util.translate(s) if not s: return must_restart += buildreq.add_buildreq( util.translate('%s-python' % s)) elif buildtool == 'ruby': if s in config.gems: must_restart += buildreq.add_buildreq(config.gems[s]) else: must_restart += buildreq.add_buildreq('rubygem-%s' % s) elif buildtool == 'ruby table': if s in config.gems: must_restart += buildreq.add_buildreq(config.gems[s]) else: print("Unknown ruby gem match", s) elif buildtool == 'maven': if s in config.maven_jars: must_restart += buildreq.add_buildreq(config.maven_jars[s]) else: must_restart += buildreq.add_buildreq('jdk-%s' % s) elif buildtool == 'catkin': must_restart += buildreq.add_pkgconfig_buildreq(s) must_restart += buildreq.add_buildreq(s) except: if verbose > 0: print("Unknown pattern match: ", s)
def failed_pattern_maven(line, pattern, verbose=0): global must_restart global failed_commands pat = re.compile(pattern) match = pat.search(line) if match: s = match.group(1) try: if s in buildreq.maven_jars: must_restart = must_restart + buildreq.add_buildreq(buildreq.maven_jars[s]) else: must_restart = must_restart + buildreq.add_buildreq('jdk-%s' % s) except: if verbose > 0: print("Unknown maven jar match: ", s)
def simple_pattern(line, pattern, req): """Check for simple patterns and restart the build as needed.""" global must_restart pat = re.compile(pattern) match = pat.search(line) if match: must_restart += buildreq.add_buildreq(req, cache=True)
def failed_pattern_ruby(line, pattern, verbose=0): global must_restart global failed_commands pat = re.compile(pattern) match = pat.search(line) if match: s = match.group(1) try: if s in buildreq.gems: must_restart = must_restart + buildreq.add_buildreq(buildreq.gems[s]) else: must_restart = must_restart + buildreq.add_buildreq('rubygem-%s' % s) except: if verbose > 0: print("Unknown pattern match: ", pattern, s)
def failed_pattern_perl(line, pattern, verbose=0): global must_restart global failed_commands pat = re.compile(pattern) match = pat.search(line) if match: s = match.group(1) try: must_restart = must_restart + buildreq.add_buildreq('perl(%s)' % s) except: if verbose > 0: print("Unknown pattern match: ", pattern, s)
def failed_pattern_pypi(line, pattern, verbose=0): global must_restart global failed_commands pat = re.compile(pattern) match = pat.search(line) if match: s = util.translate(match.group(1)) if s == "": return try: must_restart = must_restart + buildreq.add_buildreq(util.translate('%s-python' % s)) except: if verbose > 0: print("Unknown python pattern match: ", pattern, s, line)
def failed_pattern_R(line, pattern, verbose=0): global must_restart global failed_commands pat = re.compile(pattern) match = pat.search(line) if match: s = match.group(1) try: if buildreq.add_buildreq("R-" + s) > 0: must_restart = must_restart + 1 files.push_main_requires("R-" + s) except: if verbose > 0: print("Unknown pattern match: ", pattern, s)
def failed_pattern_go(line, pattern, verbose=0): global must_restart global failed_commands pat = re.compile(pattern) match = pat.search(line) if match: s = util.translate(match.group(1)) if s == "": return elif s == match.group(1): # the requirement it's also golang libpath format # (e.g: github.com/<project>/<repo> so transform into pkg name s = util.golang_name(s) try: must_restart = must_restart + buildreq.add_buildreq(s) except: if verbose > 0: print("Unknown golang pattern match: ", pattern, s, line)
def failed_pattern(line, pattern, verbose, buildtool=None): """Check against failed patterns to restart build as needed.""" global must_restart global warned_about pat = re.compile(pattern) match = pat.search(line) if not match: return s = match.group(1) # standard configure cleanups s = cleanup_req(s) if s in config.ignored_commands: return try: if not buildtool: req = config.failed_commands[s] if req: must_restart += buildreq.add_buildreq(req, cache=True) elif buildtool == 'pkgconfig': must_restart += buildreq.add_pkgconfig_buildreq(s, cache=True) elif buildtool == 'R': if buildreq.add_buildreq("R-" + s, cache=True) > 0: must_restart += 1 buildreq.add_requires("R-" + s) elif buildtool == 'perl': s = s.replace('inc::', '') must_restart += buildreq.add_buildreq('perl(%s)' % s, cache=True) elif buildtool == 'pypi': s = util.translate(s) if not s: return must_restart += buildreq.add_buildreq(util.translate('%s-python' % s), cache=True) elif buildtool == 'ruby': if s in config.gems: must_restart += buildreq.add_buildreq(config.gems[s], cache=True) else: must_restart += buildreq.add_buildreq('rubygem-%s' % s, cache=True) elif buildtool == 'ruby table': if s in config.gems: must_restart += buildreq.add_buildreq(config.gems[s], cache=True) else: print("Unknown ruby gem match", s) elif buildtool == 'maven' or buildtool == 'gradle': group_count = len(match.groups()) if group_count == 2: # Add fully qualified versioned mvn() dependency name = match.group(1) # Hyphens are disallowed for version strings, so use dots instead ver = match.group(2).replace('-', '.') mvn_provide = f'mvn({name}) = {ver}' must_restart += buildreq.add_buildreq(mvn_provide, cache=True) elif s in config.maven_jars: # Overrides for dependencies with custom grouping must_restart += buildreq.add_buildreq(config.maven_jars[s], cache=True) elif group_count == 3: org = match.group(1) name = match.group(2) ver = match.group(3).replace('-', '.') if re.search("-(parent|pom|bom)$", name): mvn_provide = f'mvn({org}:{name}:pom) = {ver}' else: mvn_provide = f'mvn({org}:{name}:jar) = {ver}' must_restart += buildreq.add_buildreq(mvn_provide, cache=True) else: # Fallback to mvn-ARTIFACTID package name must_restart += buildreq.add_buildreq('mvn-%s' % s, cache=True) elif buildtool == 'catkin': must_restart += buildreq.add_pkgconfig_buildreq(s, cache=True) must_restart += buildreq.add_buildreq(s, cache=True) except Exception: if s not in warned_about and s[:2] != '--': print("Unknown pattern match: ", s) warned_about.add(s)
def scan_for_tests(src_dir): """ Scan source directory for test files and set tests_config accordingly """ global tests_config if config.config_opts['skip_tests'] or tests_config: return makeflags = "%{?_smp_mflags} " if config.parallel_build else "" testsuites = { "makecheck": "make VERBOSE=1 V=1 {}check".format(makeflags), "perlcheck": "make TEST_VERBOSE=1 test", "setup.py": "PYTHONPATH=%{buildroot}/usr/lib/python3.6/site-packages " "python3 setup.py test", "cmake": "pushd clr-build ; make test ; popd", "rakefile": "pushd %{buildroot}%{gem_dir}/gems/" + tarball.tarball_prefix + "\nrake --trace test TESTOPTS=\"-v\"\npopd", "rspec": "pushd %{buildroot}%{gem_dir}/gems/" + tarball.tarball_prefix + "\nrspec -I.:lib spec/\npopd" } files = os.listdir(src_dir) if "CMakeLists.txt" in files: makefile_path = os.path.join(src_dir, "CMakeLists.txt") if not os.path.isfile(makefile_path): return if "enable_testing" in open(makefile_path, encoding='latin-1').read(): tests_config = testsuites["cmake"] if "Makefile.in" in files: makefile_path = os.path.join(src_dir, "Makefile.in") if not os.path.isfile(makefile_path): return with open(makefile_path, 'r', encoding="latin-1") as make_fp: lines = make_fp.readlines() for line in lines: if line.startswith("check:"): tests_config = testsuites["makecheck"] break if line.startswith("test:"): tests_config = testsuites["perlcheck"] break elif "Makefile.am" in files: tests_config = testsuites["makecheck"] elif tarball.name.startswith("rubygem"): if any(t in files for t in ["test", "tests"]): r_testdir = [f for f in files if "test" in f if "." not in f].pop() pre_test = glob.glob(os.path.join(src_dir, "test*/test_*.rb")) post_test = glob.glob(os.path.join(src_dir, "test*/*_test.rb")) tests_config = "pushd %{buildroot}%{gem_dir}/gems/" + tarball.tarball_prefix if pre_test: tests_config += "\nruby -v -I.:lib:" + r_testdir + " test*/test_*.rb" if post_test: tests_config += "\nruby -v -I.:lib:" + r_testdir + " test*/*_test.rb" tests_config += "\npopd" elif "spec" in files: buildreq.add_buildreq("rubygem-rspec") buildreq.add_buildreq("rubygem-rspec-core") buildreq.add_buildreq("rubygem-rspec-expectations") buildreq.add_buildreq("rubygem-rspec-support") buildreq.add_buildreq("rubygem-rspec-mocks") buildreq.add_buildreq("rubygem-devise") buildreq.add_buildreq("rubygem-diff-lcs") tests_config = testsuites["rspec"] elif "Rakefile" in files: tests_config = testsuites["rakefile"] buildreq.add_buildreq("ruby") buildreq.add_buildreq("rubygem-rake") buildreq.add_buildreq("rubygem-test-unit") buildreq.add_buildreq("rubygem-minitest") elif "Makefile.PL" in files: tests_config = testsuites["perlcheck"] elif "setup.py" in files: with open(os.path.join(src_dir, "setup.py"), 'r', encoding="ascii", errors="surrogateescape") as setup_fp: setup_contents = setup_fp.read() if "test_suite" in setup_contents or "pbr=True" in setup_contents: tests_config = testsuites["setup.py"] elif buildpattern.default_pattern == "R": tests_config = "export _R_CHECK_FORCE_SUGGESTS_=false\n" \ "R CMD check --no-manual --no-examples --no-codoc -l " \ "%{buildroot}/usr/lib64/R/library " \ + tarball.rawname + "|| : \ncp ~/.stash/* " \ "%{buildroot}/usr/lib64/R/library/*/libs/ || :" if "tox.ini" in files: buildreq.add_buildreq("tox") buildreq.add_buildreq("pytest") buildreq.add_buildreq("virtualenv") buildreq.add_buildreq("pluggy") buildreq.add_buildreq("py-python") if tests_config != "" and config.config_opts['allow_test_failures']: if tarball.name.startswith("rubygem"): config_elements = tests_config.split("\n") config_elements.pop() tests_config = "\n".join(config_elements) + " || :\npopd" elif tests_config == testsuites["cmake"]: tests_config = "pushd clr-build ; make test ||: ; popd" else: tests_config = tests_config + " || :"
def parse_config_files(path, bump): global extra_configure global keepstatic global asneeded global optimize_size global optimize_speed global insecure_build global config_files global config_path global parallel_build global license_fetch global license_show global git_uri global config_file config_path = path # Require autospec.conf for additional features if os.path.exists(config_file): config = configparser.ConfigParser(interpolation=None) config.read(config_file) if "autospec" not in config.sections(): print("Missing autospec section..") sys.exit(1) git_uri = config['autospec'].get('git', None) license_fetch = config['autospec'].get('license_fetch', None) license_show = config['autospec'].get('license_show', None) if not git_uri: print("Warning: Set [autospec][git] upstream template for git support") if not license_fetch: print("Warning: Set [autospec][license_fetch] uri for license fetch support") if not license_show: print("Warning: Set [autospec][license_show] uri for license link check support") wrapper = textwrap.TextWrapper() wrapper.initial_indent = "# " wrapper.subsequent_indent = "# " def write_default_conf_file(name, description): config_files.add(name) filename = path + "/" + name if os.path.isfile(filename): return with open(filename, "w") as f: f.write(wrapper.fill(description) + "\n") write_default_conf_file("buildreq_ban", "This file contains build requirements that get picked up but are undesirable. One entry per line, no whitespace.") write_default_conf_file("pkgconfig_ban", "This file contains pkgconfig build requirements that get picked up but are undesirable. One entry per line, no whitespace.") write_default_conf_file("buildreq_add", "This file contains additional build requirements that did not get picked up automatically. One name per line, no whitespace.") write_default_conf_file("pkgconfig_add", "This file contains additional pkgconfig build requirements that did not get picked up automatically. One name per line, no whitespace.") write_default_conf_file("excludes", "This file contains the output files that need %exclude. Full path names, one per line.") content = read_conf_file("release") if content and content[0]: r = int(content[0]) if bump: r += 1 tarball.release = str(r) print("Release :", tarball.release) content = read_conf_file("buildreq_ban") for banned in content: print("Banning build requirement: %s." % banned) buildreq.banned_buildreqs.add(banned) content = read_conf_file("pkgconfig_ban") for banned in content: banned = "pkgconfig(%s)" % banned print("Banning build requirement: %s." % banned) buildreq.banned_buildreqs.add(banned) content = read_conf_file("buildreq_add") for extra in content: print("Adding additional build requirement: %s." % extra) buildreq.add_buildreq(extra) content = read_conf_file("pkgconfig_add") for extra in content: extra = "pkgconfig(%s)" % extra print("Adding additional build requirement: %s." % extra) buildreq.add_buildreq(extra) content = read_conf_file("excludes") for exclude in content: print("%%exclude for: %s." % exclude) files.excludes += content content = read_conf_file("extras") for extra in content: print("extras for: %s." % extra) files.extras += content content = read_conf_file("setuid") for suid in content: print("setuid for: %s." % suid) files.setuid += content content = read_conf_file("attrs") for line in content: attr = re.split('\(|\)|,', line) attr = [a.strip() for a in attr] filename = attr.pop() print("attr for: %s." % filename) files.attrs[filename] = attr patches.patches += read_conf_file("series") pfiles = [("%s/%s" % (path, x.split(" ")[0])) for x in patches.patches] cmd = "egrep \"(\+\+\+|\-\-\-).*((Makefile.am|Makefile.in)|(configure.ac|configure.in))\" %s" % \ " ".join(pfiles) if len(patches.patches) > 0 and call(cmd, check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) == 0: patches.autoreconf = True content = read_conf_file("configure") extra_configure = " \\\n".join(content) if read_conf_file("keepstatic"): keepstatic = 1 buildpattern.disable_static = "" if read_conf_file("asneeded"): print("Disabling LD_AS_NEEDED\n") asneeded = 0 if read_conf_file("optimize_size"): optimize_size = True if read_conf_file("funroll-loops"): optimize_speed = True if read_conf_file("insecure_build"): insecure_build = True if read_conf_file("broken_parallel_build"): parallel_build = "" content = read_conf_file("make_args") if content and content[0]: buildpattern.extra_make = content[0] content = read_conf_file("make_install_args") if content and content[0]: buildpattern.extra_make_install = content[0] content = read_conf_file("install_macro") if content and content[0]: buildpattern.install_macro = content[0] content = read_conf_file("cmake_args") if content and content[0]: buildpattern.extra_cmake = content[0] content = read_conf_file("subdir") if content and content[0]: buildpattern.subdir = content[0] content = read_conf_file("build_pattern") if content and content[0]: buildpattern.set_build_pattern(content[0], 20) patches.autoreconf = False if read_conf_file("skip_test_suite"): test.skip_tests = True if read_conf_file("unit_tests_must_pass"): test.new_pkg = False content = read_conf_file("make_check_command") if content and content[0]: test.tests_config = content[0] content = read_conf_file("allow_test_failures") if content and content[0]: test.allow_test_failures = True content = read_conf_file(tarball.name + ".license") if content and content[0]: words = content[0].split() for word in words: if word.find(":") < 0: license.add_license(word) buildpattern.make_install_append = read_conf_file("make_install_append")
def name_and_version(url_argument, name_argument): global name global rawname global version global url url = url_argument tarfile = os.path.basename(url) # it is important for the more specific patterns to come first pattern_options = [ r"(.*?)[\-_](v*[0-9]+[a-zalpha\+_spbfourcesigedsvstableP0-9\.\-\~]*)\.orig\.tar", r"(.*?)[\-_](v*[0-9]+[alpha\+_spbfourcesigedsvstableP0-9\.\-\~]*)\.src\.(tgz|tar|zip)", r"(.*?)[\-_](v*[0-9]+[alpha\+_sbpfourcesigedsvstableP0-9\.\-\~]*)\.(tgz|tar|zip)", r"(.*?)[\-_](v*[0-9]+[\+_spbfourcesigedsvstableP0-9\.\~]*)(-.*?)?\.tar", ] for pattern in pattern_options: p = re.compile(pattern) m = p.search(tarfile) if m: name = m.group(1).strip() version = m.group(2).strip() b = version.find("-") if b >= 0: version = version[:b] break rawname = name # R package if url_argument.find("cran.r-project.org") > 0 or url_argument.find("cran.rstudio.com") > 0: buildpattern.set_build_pattern("R", 10) files.want_dev_split = 0 buildreq.add_buildreq("clr-R-helpers") p = re.compile(r"([A-Za-z0-9]+)_(v*[0-9]+[\+_spbfourcesigedsvstableP0-9\.\~\-]*)\.tar\.gz") m = p.search(tarfile) if m: name = "R-" + m.group(1).strip() rawname = m.group(1).strip() version = m.group(2).strip() b = version.find("-") if b >= 0: version = version[:b] if url_argument.find("pypi.python.org") > 0: buildpattern.set_build_pattern("distutils", 10) url_argument = "http://pypi.debian.net/" + name + "/" + tarfile if url_argument.find("pypi.debian.net") > 0: buildpattern.set_build_pattern("distutils", 10) if url_argument.find(".cpan.org/CPAN/") > 0: buildpattern.set_build_pattern("cpan", 10) if name: name = "perl-" + name if url_argument.find(".metacpan.org/") > 0: buildpattern.set_build_pattern("cpan", 10) if name: name = "perl-" + name if "github.com" in url_argument: # define regex accepted for valid packages, important for specific # patterns to come before general ones github_patterns = [r"https://github.com/.*/(.*?)/archive/(.*)-final.tar", r"https://github.com/.*/.*/archive/[0-9a-fA-F]{1,40}\/(.*)\-(.*).tar", r"https://github.com/.*/(.*?)/archive/v?(.*).orig.tar", r"https://github.com/.*/(.*?)/archive/(.*).zip", r"https://github.com/.*/(.*?)/archive/v?(.*).tar"] for pattern in github_patterns: p = re.compile(pattern) m = p.search(url_argument) if m: name = m.group(1).strip() # convert from 7_4_2 to 7.4.2 version = m.group(2).strip().replace('_', '.') # remove release candidate tag b = version.find("-rc") if b > 0: version = version[:b] b = version.find("-") if b > 0: version = version[b + 1:] break if url_argument.find("bitbucket.org") > 0: p = re.compile(r"https://bitbucket.org/.*/(.*?)/get/[a-zA-Z_-]*([0-9][0-9_.]*).tar") m = p.search(url_argument) if m: name = m.group(1).strip() # convert from 7_4_2 to 7.4.2 version = m.group(2).strip().replace('_', '.') else: version = "1" # ruby if url_argument.find("rubygems.org/") > 0: buildpattern.set_build_pattern("ruby", 10) p = re.compile(r"(.*?)[\-_](v*[0-9]+[alpha\+_spbfourcesigedsvstableP0-9\.\-\~]*)\.gem") m = p.search(tarfile) if m: name = "rubygem-" + m.group(1).strip() # remove release candidate tag b = name.find("-rc") if b > 0: name = name[:b] rawname = m.group(1).strip() version = m.group(2).strip() b = version.find("-") if b >= 0: version = version[:b] # maven if url_argument.find("maven.org") > 0: buildpattern.set_build_pattern("maven", 10) # override from commandline if name_argument and name_argument[0] != name: pattern = name_argument[0] + r"[\-]*(.*)\.(tgz|tar|zip)" p = re.compile(pattern) m = p.search(tarfile) if m: name = name_argument[0] rawname = name version = m.group(1).strip() b = version.find("-") if b >= 0 and version.find("-beta") < 0: version = version[:b] if version.startswith('.'): version = version[1:] else: name = name_argument[0] if not name: split = url_argument.split('/') if len(split) > 3 and split[-2] in ('archive', 'tarball'): name = split[-3] version = split[-1] if version.startswith('v'): version = version[1:] # remove extension version = '.'.join(version.split('.')[:-1]) if version.endswith('.tar'): version = '.'.join(version.split('.')[:-1]) b = version.find("-") if b >= 0 and version.find("-beta") < 0: b = b + 1 version = version[b:] if len(version) > 0 and version[0].lower() in ['v', 'r']: version = version[1:] # remove package name from beginning of version if version.lower().startswith(name.lower()): pat = re.compile(re.escape(name), re.IGNORECASE) version = pat.sub('', version) if version[0] in ['.', '-', '_']: version = version[1:] assert name != ""
def name_and_version(name_arg, version_arg, filemanager): """Parse the url for the package name and version.""" global rawname global url global giturl global repo tarfile = os.path.basename(url) # If both name and version overrides are set via commandline, set the name # and version variables to the overrides and bail. If only one override is # set, continue to auto detect both name and version since the URL parsing # handles both. In this case, wait until the end to perform the override of # the one that was set. An extra conditional, that version_arg is a string # is added to enable a package to have multiple versions at the same time # for some language ecosystems. if name_arg and version_arg: # rawname == name in this case name = name_arg rawname = name_arg version = set_multi_version(version_arg) return name, rawname, convert_version(version, name) name = name_arg rawname = name_arg version = "" # it is important for the more specific patterns to come first pattern_options = [ # handle font packages with names ending in -nnndpi r"(.*-[0-9]+dpi)[-_]([0-9]+[a-zA-Z0-9\+_\.\-\~]*)\.(tgz|tar|zip)", r"(.*?)[-_][vs]?([0-9]+[a-zA-Z0-9\+_\.\-\~]*)\.(tgz|tar|zip)", ] for pattern in pattern_options: m = re.search(pattern, tarfile) if m: name = m.group(1).strip() version = convert_version(m.group(2), name) break # R package if "cran.r-project.org" in url or "cran.rstudio.com" in url: filemanager.want_dev_split = False m = re.search( r"([A-Za-z0-9.]+)_(v*[0-9]+[a-zA-Z0-9\+_\.\~\-]*)\.tar\.gz", tarfile) if m: name = "R-" + m.group(1).strip() rawname = m.group(1).strip() version = convert_version(m.group(2), name) if ".cpan.org/" in url or ".metacpan.org/" in url and name: name = "perl-" + name if "github.com" in url: # define regex accepted for valid packages, important for specific # patterns to come before general ones github_patterns = [ r"https?://github.com/(.*)/(.*?)/archive/[v|r]?.*/(.*).tar", r"https?://github.com/(.*)/(.*?)/archive/[-a-zA-Z_]*-(.*).tar", r"https?://github.com/(.*)/(.*?)/archive/[vVrR]?(.*).tar", r"https?://github.com/(.*)/.*-downloads/releases/download/.*?/(.*)-(.*).tar", r"https?://github.com/(.*)/(.*?)/releases/download/(.*)/", r"https?://github.com/(.*)/(.*?)/files/.*?/(.*).tar" ] for pattern in github_patterns: m = re.search(pattern, url) if m: repo = m.group(2).strip() if repo not in name: # Only take the repo name as the package name if it's more descriptive name = repo elif name != repo: name = re.sub(r"release-", '', name) name = re.sub(r"\d*$", '', name) rawname = name version = m.group(3).replace(name, '') if "archive" not in pattern: version = re.sub(r"^[-_.a-zA-Z]+", "", version) version = convert_version(version, name) if not giturl: giturl = "https://github.com/" + m.group( 1).strip() + "/" + repo + ".git" break if "gnome.org" in url: buildreq.add_buildreq("buildreq-gnome") if "kde.org" in url or "https://github.com/KDE" in url: buildreq.add_buildreq("buildreq-kde") # SQLite tarballs use 7 digit versions, e.g 3290000 = 3.29.0, 3081002 = 3.8.10.2 if "sqlite.org" in url: major = version[0] minor = version[1:3].lstrip("0").zfill(1) patch = version[3:5].lstrip("0").zfill(1) build = version[5:7].lstrip("0") version = major + "." + minor + "." + patch + "." + build version = version.strip(".") # construct github giturl from gnome projects if not giturl and "download.gnome.org" in url: giturl = "https://github.com/GNOME/{}.git".format(name) if "mirrors.kernel.org" in url: m = re.search(r".*/sourceware/(.*?)/releases/(.*?).tgz", url) if m: name = m.group(1).strip() version = convert_version(m.group(2), name) if "sourceforge.net" in url: scf_pats = [ r"projects/.*/files/(.*?)/(.*?)/[^-]*(-src)?.tar.gz", r"downloads.sourceforge.net/.*/([a-zA-Z]+)([-0-9\.]*)(-src)?.tar.gz" ] for pat in scf_pats: m = re.search(pat, url) if m: name = m.group(1).strip() version = convert_version(m.group(2), name) break if "bitbucket.org" in url: bitbucket_pats = [ r"/.*/(.*?)/.*/.*v([-\.0-9a-zA-Z_]*?).(tar|zip)", r"/.*/(.*?)/.*/([-\.0-9a-zA-Z_]*?).(tar|zip)" ] for pat in bitbucket_pats: m = re.search(pat, url) version = 1 if m: name = m.group(1).strip() version = convert_version(m.group(2), name) break # ruby if "rubygems.org/" in url: m = re.search(r"(.*?)[\-_](v*[0-9]+[a-zA-Z0-9\+_\.\-\~]*)\.gem", tarfile) if m: name = "rubygem-" + m.group(1).strip() # remove release candidate tag from the package name # https://rubygems.org/downloads/ruby-rc4-0.1.5.gem b = name.find("-rc") if b > 0: name = name[:b] rawname = m.group(1).strip() version = convert_version(m.group(2), name) # maven if ".maven." in url: maven_pats = [ r"maven.org/maven2/[a-zA-Z\-\_]+/([a-zA-Z\-\_])+/([a-zA-Z-\_\d.]+)/[a-zA-Z-\_\d.]*\.(?:pom|jar)", r"maven.apache.org/maven2/[a-zA-Z\-\_]+/([a-zA-Z\-\_])+/([\d.]+)/[a-z-\_.\d]*\.(?:pom|jar)", r"maven.org/maven2/(?:[a-zA-Z-\_.\d/]+)/([a-zA-Z-\_.\d]*)/([a-zA-Z\d\.\_\-]+)/(?:[a-zA-Z-\_.\d]*)\.(?:pom|jar)" ] for pat in maven_pats: m = re.search(pat, url) if m: name = m.group(1).strip() version = convert_version(m.group(2), name) break # rust crate if "crates.io" in url: m = re.search( r"/crates.io/api/v[0-9]+/crates/(.*)/(.*)/download.*\.crate", url) if m: name = m.group(1).strip() version = convert_version(m.group(2), name) if "gitlab.com" in url: # https://gitlab.com/leanlabsio/kanban/-/archive/1.7.1/kanban-1.7.1.tar.gz m = re.search(r"gitlab\.com/.*/(.*)/-/archive/(.*)/", url) if m: name = m.group(1).strip() version = convert_version(m.group(2), name) if "git.sr.ht" in url: # https://git.sr.ht/~sircmpwn/scdoc/archive/1.9.4.tar.gz m = re.search(r"git\.sr\.ht/.*/(.*)/archive/(.*).tar.gz", url) if m: name = m.group(1).strip() version = convert_version(m.group(2), name) # override name and version from commandline name = name_arg if name_arg else name version = version_arg if version_arg else version version = set_multi_version(version) return name, rawname, convert_version(version, name)
def failed_pattern(line, pattern, verbose, buildtool=None): """Check against failed patterns to restart build as needed.""" global must_restart global warned_about pat = re.compile(pattern) match = pat.search(line) if not match: return s = match.group(1) # standard configure cleanups s = cleanup_req(s) if s in config.ignored_commands: return try: if not buildtool: req = config.failed_commands[s] if req: must_restart += buildreq.add_buildreq(req, cache=True) elif buildtool == 'pkgconfig': must_restart += buildreq.add_pkgconfig_buildreq(s, cache=True) elif buildtool == 'R': if buildreq.add_buildreq("R-" + s, cache=True) > 0: must_restart += 1 buildreq.add_requires("R-" + s) elif buildtool == 'perl': s = s.replace('inc::', '') must_restart += buildreq.add_buildreq('perl(%s)' % s, cache=True) elif buildtool == 'pypi': s = util.translate(s) if not s: return must_restart += buildreq.add_buildreq(util.translate('%s-python' % s), cache=True) elif buildtool == 'ruby': if s in config.gems: must_restart += buildreq.add_buildreq(config.gems[s], cache=True) else: must_restart += buildreq.add_buildreq('rubygem-%s' % s, cache=True) elif buildtool == 'ruby table': if s in config.gems: must_restart += buildreq.add_buildreq(config.gems[s], cache=True) else: print("Unknown ruby gem match", s) elif buildtool == 'maven': if s in config.maven_jars: must_restart += buildreq.add_buildreq(config.maven_jars[s], cache=True) else: must_restart += buildreq.add_buildreq('jdk-%s' % s, cache=True) elif buildtool == 'catkin': must_restart += buildreq.add_pkgconfig_buildreq(s, cache=True) must_restart += buildreq.add_buildreq(s, cache=True) except Exception: if s not in warned_about and s[:2] != '--': print("Unknown pattern match: ", s) warned_about.add(s)
def test_add_buildreq_banned(self): """ Test add_buildreq with banned new req """ self.assertFalse(buildreq.add_buildreq('bannedreq')) self.assertNotIn('bannedreq', buildreq.buildreqs)
def name_and_version(name_arg, version_arg, filemanager): """Parse the url for the package name and version.""" global name global rawname global version global url global giturl global repo tarfile = os.path.basename(url) # If both name and version overrides are set via commandline, set the name # and version variables to the overrides and bail. If only one override is # set, continue to auto detect both name and version since the URL parsing # handles both. In this case, wait until the end to perform the override of # the one that was set. if name_arg and version_arg: # rawname == name in this case name = name_arg rawname = name version = version_arg return # it is important for the more specific patterns to come first pattern_options = [ # handle font packages with names ending in -nnndpi r"(.*-[0-9]+dpi)[-_]([0-9]+[a-zA-Z0-9\+_\.\-\~]*)\.(tgz|tar|zip)", r"(.*?)[-_][vs]?([0-9]+[a-zA-Z0-9\+_\.\-\~]*)\.(tgz|tar|zip)", ] for pattern in pattern_options: m = re.search(pattern, tarfile) if m: name = m.group(1).strip() version = convert_version(m.group(2)) break rawname = name # R package if "cran.r-project.org" in url or "cran.rstudio.com" in url: filemanager.want_dev_split = False m = re.search( r"([A-Za-z0-9.]+)_(v*[0-9]+[a-zA-Z0-9\+_\.\~\-]*)\.tar\.gz", tarfile) if m: name = "R-" + m.group(1).strip() rawname = m.group(1).strip() version = convert_version(m.group(2)) if ".cpan.org/" in url or ".metacpan.org/" in url and name: name = "perl-" + name if "github.com" in url: # define regex accepted for valid packages, important for specific # patterns to come before general ones github_patterns = [ r"https?://github.com/(.*)/(.*?)/archive/[v|r]?.*/(.*).tar", r"https?://github.com/(.*)/(.*?)/archive/[-a-zA-Z_]*-(.*).tar", r"https?://github.com/(.*)/(.*?)/archive/[vVrR]?(.*).tar", r"https?://github.com/(.*)/.*-downloads/releases/download/.*?/(.*)-(.*).tar", r"https?://github.com/(.*)/(.*?)/releases/download/(.*)/", r"https?://github.com/(.*)/(.*?)/files/.*?/(.*).tar" ] for pattern in github_patterns: m = re.search(pattern, url) if m: repo = m.group(2).strip() if repo not in name: # Only take the repo name as the package name if it's more descriptive name = repo elif name != repo: name = re.sub(r"release-", '', name) name = re.sub(r"\d*$", '', name) rawname = name version = m.group(3).replace(name, '') if "archive" not in pattern: version = re.sub(r"^[-_.a-zA-Z]+", "", version) version = convert_version(version) giturl = "https://github.com/" + m.group( 1).strip() + "/" + repo + ".git" break if "gnome.org" in url: buildreq.add_buildreq("buildreq-gnome") if "kde.org" in url or "https://github.com/KDE" in url: buildreq.add_buildreq("buildreq-kde") # construct github giturl from gnome projects if not giturl and "download.gnome.org" in url: giturl = "https://github.com/GNOME/{}.git".format(name) if "mirrors.kernel.org" in url: m = re.search(r".*/sourceware/(.*?)/releases/(.*?).tgz", url) if m: name = m.group(1).strip() version = convert_version(m.group(2)) if "sourceforge.net" in url: scf_pats = [ r"projects/.*/files/(.*?)/(.*?)/[^-]*(-src)?.tar.gz", r"downloads.sourceforge.net/.*/([a-zA-Z]+)([-0-9\.]*)(-src)?.tar.gz" ] for pat in scf_pats: m = re.search(pat, url) if m: name = m.group(1).strip() version = convert_version(m.group(2)) break if "bitbucket.org" in url: bitbucket_pats = [ r"/.*/(.*?)/.*/.*v([-\.0-9a-zA-Z_]*?).(tar|zip)", r"/.*/(.*?)/.*/([-\.0-9a-zA-Z_]*?).(tar|zip)" ] for pat in bitbucket_pats: m = re.search(pat, url) version = 1 if m: name = m.group(1).strip() version = convert_version(m.group(2)) break # ruby if "rubygems.org/" in url: m = re.search(r"(.*?)[\-_](v*[0-9]+[a-zA-Z0-9\+_\.\-\~]*)\.gem", tarfile) if m: name = "rubygem-" + m.group(1).strip() # remove release candidate tag from the package name # https://rubygems.org/downloads/ruby-rc4-0.1.5.gem b = name.find("-rc") if b > 0: name = name[:b] rawname = m.group(1).strip() version = convert_version(m.group(2)) # maven if ".maven." in url: m = re.search( r"/maven.*/org/.*/(.*?)/.*/.*[\-_]([0-9]+[a-zA-Z0-9\+_\.\-\~]*)\.jar", url) if m: name = m.group(1).strip() version = convert_version(m.group(2)) # rust crate if "crates.io" in url: m = re.search( r"/crates.io/api/v[0-9]+/crates/(.*)/(.*)/download.*\.crate", url) if m: name = m.group(1).strip() version = convert_version(m.group(2)) if "gitlab.com" in url: # https://gitlab.com/leanlabsio/kanban/-/archive/1.7.1/kanban-1.7.1.tar.gz m = re.search(r"gitlab\.com/.*/(.*)/-/archive/(.*)/", url) if m: name = m.group(1).strip() version = convert_version(m.group(2)) # override name and version from commandline name = name_arg if name_arg else name version = version_arg if version_arg else version # sanity check to make sure we aren't using an empty version if version == "": version = "1"
def parse_config_files(path, bump): global extra_configure global extra_configure32 global config_files global config_path global parallel_build global license_fetch global license_show global git_uri global urlban global config_file global profile_payload global config_opts config_path = path read_config_opts() # Require autospec.conf for additional features if os.path.exists(config_file): config = configparser.ConfigParser(interpolation=None) config.read(config_file) if "autospec" not in config.sections(): print("Missing autospec section..") sys.exit(1) git_uri = config["autospec"].get("git", None) license_fetch = config["autospec"].get("license_fetch", None) license_show = config["autospec"].get("license_show", None) urlban = config["autospec"].get("urlban", None) if not git_uri: print("Warning: Set [autospec][git] upstream template for git support") if not license_fetch: print("Warning: Set [autospec][license_fetch] uri for license fetch support") if not license_show: print("Warning: Set [autospec][license_show] uri for license link check support") wrapper = textwrap.TextWrapper() wrapper.initial_indent = "# " wrapper.subsequent_indent = "# " def write_default_conf_file(name, description): config_files.add(name) filename = path + "/" + name if os.path.isfile(filename): return with open(filename, "w") as f: f.write(wrapper.fill(description) + "\n") write_default_conf_file( "buildreq_ban", "This file contains build requirements that get picked up but are undesirable. One entry per line, no whitespace.", ) write_default_conf_file( "pkgconfig_ban", "This file contains pkgconfig build requirements that get picked up but are undesirable. One entry per line, no whitespace.", ) write_default_conf_file( "buildreq_add", "This file contains additional build requirements that did not get picked up automatically. One name per line, no whitespace.", ) write_default_conf_file( "pkgconfig_add", "This file contains additional pkgconfig build requirements that did not get picked up automatically. One name per line, no whitespace.", ) write_default_conf_file( "excludes", "This file contains the output files that need %exclude. Full path names, one per line." ) content = read_conf_file("release") if content and content[0]: r = int(content[0]) if bump: r += 1 tarball.release = str(r) print("Release :", tarball.release) content = read_conf_file("buildreq_ban") for banned in content: print("Banning build requirement: %s." % banned) buildreq.banned_buildreqs.add(banned) content = read_conf_file("pkgconfig_ban") for banned in content: banned = "pkgconfig(%s)" % banned print("Banning build requirement: %s." % banned) buildreq.banned_buildreqs.add(banned) content = read_conf_file("buildreq_add") for extra in content: print("Adding additional build requirement: %s." % extra) buildreq.add_buildreq(extra) content = read_conf_file("pkgconfig_add") for extra in content: extra = "pkgconfig(%s)" % extra print("Adding additional build requirement: %s." % extra) buildreq.add_buildreq(extra) content = read_conf_file("excludes") for exclude in content: print("%%exclude for: %s." % exclude) files.excludes += content content = read_conf_file("extras") for extra in content: print("extras for: %s." % extra) files.extras += content content = read_conf_file("setuid") for suid in content: print("setuid for: %s." % suid) files.setuid += content content = read_conf_file("attrs") for line in content: attr = re.split("\(|\)|,", line) attr = [a.strip() for a in attr] filename = attr.pop() print("attr for: %s." % filename) files.attrs[filename] = attr patches.patches += read_conf_file("series") pfiles = [("%s/%s" % (path, x.split(" ")[0])) for x in patches.patches] cmd = 'egrep "(\+\+\+|\-\-\-).*((Makefile.am)|(configure.ac|configure.in))" %s' % " ".join(pfiles) if len(patches.patches) > 0 and call(cmd, check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) == 0: patches.autoreconf = True content = read_conf_file("configure") extra_configure = " \\\n".join(content) content = read_conf_file("configure32") extra_configure32 = " \\\n".join(content) if config_opts["keepstatic"]: buildpattern.disable_static = "" if config_opts["broken_parallel_build"]: parallel_build = "" content = read_conf_file("make_args") if content and content[0]: buildpattern.extra_make = content[0] content = read_conf_file("make_install_args") if content and content[0]: buildpattern.extra_make_install = content[0] content = read_conf_file("install_macro") if content and content[0]: buildpattern.install_macro = content[0] content = read_conf_file("cmake_args") if content and content[0]: buildpattern.extra_cmake = content[0] content = read_conf_file("subdir") if content and content[0]: buildpattern.subdir = content[0] content = read_conf_file("build_pattern") if content and content[0]: buildpattern.set_build_pattern(content[0], 20) patches.autoreconf = False content = read_conf_file("make_check_command") if content and content[0]: test.tests_config = content[0] content = read_conf_file(tarball.name + ".license") if content and content[0]: words = content[0].split() for word in words: if word.find(":") < 0: license.add_license(word) content = read_conf_file("golang_libpath") if content and content[0]: tarball.golibpath = content[0] print("golibpath : {}".format(tarball.golibpath)) if config_opts["use_clang"]: config_opts["funroll-loops"] = False buildreq.add_buildreq("llvm-dev") if config_opts["32bit"]: buildreq.add_buildreq("glibc-libc32") buildreq.add_buildreq("glibc-dev32") buildreq.add_buildreq("gcc-dev32") buildreq.add_buildreq("gcc-libgcc32") buildreq.add_buildreq("gcc-libstdc++32") buildpattern.make_install_append = read_conf_file("make_install_append") buildpattern.prep_append = read_conf_file("prep_append") profile_payload = read_conf_file("profile_payload")
def simple_pattern(line, pattern, req): global must_restart pat = re.compile(pattern) match = pat.search(line) if match: must_restart = must_restart + buildreq.add_buildreq(req)
def simple_pattern(line, pattern, req): global must_restart pat = re.compile(pattern) match = pat.search(line) if match: must_restart += buildreq.add_buildreq(req)
def scan_for_tests(src_dir): """Scan source directory for test files and set tests_config accordingly.""" global tests_config if config.config_opts['skip_tests'] or tests_config: return makeflags = "%{?_smp_mflags} " if config.parallel_build else "" make_check = "make VERBOSE=1 V=1 {}check".format(makeflags) cmake_check = "make test" perl_check = "make TEST_VERBOSE=1 test" setup_check = "PYTHONPATH=%{buildroot}/usr/lib/python3.7/site-packages python3 setup.py test" if config.config_opts['allow_test_failures']: make_check += " || :" cmake_check += " || :" perl_check += " || :" setup_check += " || :" testsuites = { "makecheck": make_check, "perlcheck": perl_check, "setup.py": setup_check, "cmake": "cd clr-build; " + cmake_check, "rakefile": "pushd %{buildroot}%{gem_dir}/gems/" + tarball.tarball_prefix + "\nrake --trace test TESTOPTS=\"-v\"\npopd", "rspec": "pushd %{buildroot}%{gem_dir}/gems/" + tarball.tarball_prefix + "\nrspec -I.:lib spec/\npopd" } if config.config_opts['32bit']: testsuites["makecheck"] += "\ncd ../build32;\n" + make_check + " || :" testsuites["cmake"] += "\ncd ../clr-build32;\n" + cmake_check + " || :" if config.config_opts['use_avx2']: testsuites[ "makecheck"] += "\ncd ../buildavx2;\n" + make_check + " || :" testsuites[ "cmake"] += "\ncd ../clr-build-avx2;\n" + cmake_check + " || :" if config.config_opts['use_avx512']: testsuites[ "makecheck"] += "\ncd ../buildavx512;\n" + make_check + " || :" testsuites[ "cmake"] += "\ncd ../clr-build-avx512;\n" + cmake_check + " || :" files = os.listdir(src_dir) if buildpattern.default_pattern == "cmake": makefile_path = os.path.join(src_dir, "CMakeLists.txt") if not os.path.isfile(makefile_path): return if "enable_testing" in open(makefile_path, encoding='latin-1').read(): tests_config = testsuites["cmake"] elif buildpattern.default_pattern in [ "cpan", "configure", "configure_ac", "autogen" ] and "Makefile.in" in files: makefile_path = os.path.join(src_dir, "Makefile.in") if os.path.isfile(makefile_path): with open(makefile_path, 'r', encoding="latin-1") as make_fp: lines = make_fp.readlines() for line in lines: if line.startswith("check:"): tests_config = testsuites["makecheck"] break if line.startswith("test:"): tests_config = testsuites["perlcheck"] break elif buildpattern.default_pattern in [ "configure", "configure_ac", "autogen" ] and "Makefile.am" in files: tests_config = testsuites["makecheck"] elif buildpattern.default_pattern in ["cpan"] and "Makefile.PL" in files: tests_config = testsuites["perlcheck"] elif buildpattern.default_pattern in ["distutils3", "distutils23" ] and "setup.py" in files: with open(os.path.join(src_dir, "setup.py"), 'r', encoding="ascii", errors="surrogateescape") as setup_fp: setup_contents = setup_fp.read() if "test_suite" in setup_contents or "pbr=True" in setup_contents: tests_config = testsuites["setup.py"] elif buildpattern.default_pattern == "R": tests_config = "export _R_CHECK_FORCE_SUGGESTS_=false\n" \ "R CMD check --no-manual --no-examples --no-codoc -l " \ "%{buildroot}/usr/lib64/R/library " \ + tarball.rawname + "|| : \ncp ~/.stash/* " \ "%{buildroot}/usr/lib64/R/library/*/libs/ || :" if "tox.ini" in files: buildreq.add_buildreq("tox") buildreq.add_buildreq("pytest") buildreq.add_buildreq("virtualenv") buildreq.add_buildreq("pluggy") buildreq.add_buildreq("py-python")
def download_tarball(url_argument, name_argument, archives): global name global rawname global version global url global path global tarball_prefix global gcov_file url = url_argument tarfile = os.path.basename(url) pattern_options = [ r"(.*?)[\-_](v*[0-9]+[alpha\+_spbfourcesigedsvstableP0-9\.\-\~]*)\.src\.(tgz|tar|zip)", r"(.*?)[\-_](v*[0-9]+[alpha\+_sbpfourcesigedsvstableP0-9\.\-\~]*)\.(tgz|tar|zip)", r"(.*?)[\-_](v*[0-9]+[a-zalpha\+_spbfourcesigedsvstableP0-9\.\-\~]*)\.orig\.tar", r"(.*?)[\-_](v*[0-9]+[\+_spbfourcesigedsvstableP0-9\.\~]*)(-.*?)?\.tar", ] for pattern in pattern_options: p = re.compile(pattern) m = p.search(tarfile) if m: name = m.group(1).strip() version = m.group(2).strip() b = version.find("-") if b >= 0: version = version[:b] break rawname = name # R package if url_argument.find("cran.r-project.org") > 0 or url_argument.find("cran.rstudio.com") > 0: buildpattern.set_build_pattern("R", 10) files.want_dev_split = 0 buildreq.add_buildreq("clr-R-helpers") p = re.compile(r"([A-Za-z0-9]+)_(v*[0-9]+[\+_spbfourcesigedsvstableP0-9\.\~\-]*)\.tar\.gz") m = p.search(tarfile) if m: name = "R-" + m.group(1).strip() rawname = m.group(1).strip() version = m.group(2).strip() b = version.find("-") if b >= 0: version = version[:b] if url_argument.find("pypi.python.org") > 0: buildpattern.set_build_pattern("distutils", 10) if url_argument.find(".cpan.org/CPAN/") > 0: buildpattern.set_build_pattern("cpan", 10) if name: name = "perl-" + name if url_argument.find(".metacpan.org/") > 0: buildpattern.set_build_pattern("cpan", 10) if name: name = "perl-" + name if url_argument.find("github.com") > 0: p = re.compile(r"https://github.com/.*/(.*?)/archive/v?(.*).tar") m = p.search(url_argument) if m: name = m.group(1).strip() version = m.group(2).strip() if url_argument.find("bitbucket.org") > 0: p = re.compile(r"https://bitbucket.org/.*/(.*?)/get/[a-zA-Z_-]*([0-9][0-9_.]*).tar") m = p.search(url_argument) if m: name = m.group(1).strip() version = m.group(2).strip().replace('_', '.') # ruby if url_argument.find("rubygems.org/") > 0: buildpattern.set_build_pattern("ruby", 10) p = re.compile(r"(.*?)[\-_](v*[0-9]+[alpha\+_spbfourcesigedsvstableP0-9\.\-\~]*)\.gem") m = p.search(tarfile) if m: buildreq.add_buildreq("ruby") buildreq.add_buildreq("rubygem-rdoc") name = "rubygem-" + m.group(1).strip() rawname = m.group(1).strip() version = m.group(2).strip() b = version.find("-") if b >= 0: version = version[:b] # override from commandline if name_argument and name_argument[0] != name: pattern = name_argument[0] + r"[\-]*(.*)\.(tgz|tar|zip)" p = re.compile(pattern) m = p.search(tarfile) if m: name = name_argument[0] rawname = name version = m.group(1).strip() b = version.find("-") if b >= 0: version = version[:b] if version.startswith('.'): version = version[1:] else: name = name_argument[0] if not name: split = url_argument.split('/') if len(split) > 3 and split[-2] in ('archive', 'tarball'): name = split[-3] version = split[-1] if version.startswith('v'): version = version[1:] # remove extension version = '.'.join(version.split('.')[:-1]) if version.endswith('.tar'): version = '.'.join(version.split('.')[:-1]) b = version.find("-") if b >= 0: b = b + 1 version = version[b:] if version[0] in ['v', 'r']: version = version[1:] assert name != "" build.download_path = os.getcwd() + "/" + name call("mkdir -p %s" % build.download_path) gcov_path = build.download_path + "/" + name + ".gcov" if os.path.isfile(gcov_path): gcov_file = name + ".gcov" tarball_path = check_or_get_file(url, tarfile) sha1 = get_sha1sum(tarball_path) with open(build.download_path + "/upstream", "w") as file: file.write(sha1 + "/" + tarfile + "\n") tarball_prefix = name + "-" + version if tarfile.lower().endswith('.zip'): tarball_contents = subprocess.check_output( ["unzip", "-l", tarball_path], universal_newlines=True) if tarball_contents and len(tarball_contents.splitlines()) > 3: tarball_prefix = tarball_contents.splitlines()[3].rsplit("/")[0].split()[-1] extract_cmd = "unzip -d {0} {1}".format(build.base_path, tarball_path) elif tarfile.lower().endswith('.gem'): tarball_contents = subprocess.check_output( ["gem", "unpack", "--verbose", tarball_path], universal_newlines=True) extract_cmd = "gem unpack --target={0} {1}".format(build.base_path, tarball_path) if tarball_contents: tarball_prefix = tarball_contents.splitlines()[-1].rsplit("/")[-1] if tarball_prefix.endswith("'"): tarball_prefix = tarball_prefix[:-1] else: extract_cmd, tarball_prefix = build_untar(tarball_path) print("\n") print("Processing", url_argument) print( "=============================================================================================") print("Name :", name) print("Version :", version) print("Prefix :", tarball_prefix) with open(build.download_path + "/Makefile", "w") as file: file.write("PKG_NAME := " + name + "\n") file.write("URL := " + url_argument + "\n") file.write("ARCHIVES :=") for archive in archives: file.write(" {}".format(archive)) file.write("\n") file.write("\n") file.write("include ../common/Makefile.common\n") shutil.rmtree("{}".format(build.base_path), ignore_errors=True) os.makedirs("{}".format(build.output_path)) call("mkdir -p %s" % build.download_path) call(extract_cmd) path = build.base_path + tarball_prefix for archive, destination in zip(archives[::2], archives[1::2]): source_tarball_path = check_or_get_file(archive, os.path.basename(archive)) if source_tarball_path.lower().endswith('.zip'): tarball_contents = subprocess.check_output( ["unzip", "-l", source_tarball_path], universal_newlines=True) if tarball_contents and len(tarball_contents.splitlines()) > 3: source_tarball_prefix = tarball_contents.splitlines()[3].rsplit("/")[0].split()[-1] extract_cmd = "unzip -d {0} {1}".format(build.base_path, source_tarball_path) else: extract_cmd, source_tarball_prefix = build_untar(source_tarball_path) buildpattern.archive_details[archive + "prefix"] = source_tarball_prefix call(extract_cmd) tar_files = glob.glob("{0}{1}/*".format(build.base_path, source_tarball_prefix)) move_cmd = "mv " for tar_file in tar_files: move_cmd += tar_file + " " move_cmd += '{0}/{1}'.format(path, destination) mkdir_cmd = "mkdir -p " mkdir_cmd += '{0}/{1}'.format(path, destination) print("mkdir " + mkdir_cmd) call(mkdir_cmd) call(move_cmd) sha1 = get_sha1sum(source_tarball_path) with open(build.download_path + "/upstream", "a") as file: file.write(sha1 + "/" + os.path.basename(archive) + "\n")
def scan_for_tests(dir): global tests_config global skip_tests global allow_test_failures if skip_tests: return if len(tests_config) > 0: return testsuites = { "makecheck": "make VERBOSE=1 V=1 %{?_smp_mflags} check", "perlcheck": "make TEST_VERBOSE=1 test", "setup.py": "PYTHONPATH=%{buildroot}/usr/lib/python2.7/site-packages python2 setup.py test", "cmake": "pushd clr-build ; make test ; popd", "rakefile": "pushd %{buildroot}%{gem_dir}/gems/" + tarball.tarball_prefix + "\nrake --trace test TESTOPTS=\"-v\"\npopd", # "rubygems": "pushd %{buildroot}%{gem_dir}/gems/" + tarball.tarball_prefix + "\nruby -I\"lib:test*\" test*/*_test.rb \nruby -I\"lib:test*\" test*/test_*.rb\npopd", "rspec": "pushd %{buildroot}%{gem_dir}/gems/" + tarball.tarball_prefix + "\nrspec -I.:lib spec/\npopd" } files = os.listdir(dir) if "CMakeLists.txt" in files: makefile_path = os.path.join(dir, "CMakeLists.txt") if not os.path.isfile(makefile_path): return with open(makefile_path, encoding="latin-1") as fp: lines = fp.readlines() for line in lines: if line.find("enable_testing") >= 0: tests_config = testsuites["cmake"] break if "Makefile.in" in files: makefile_path = os.path.join(dir, "Makefile.in") if not os.path.isfile(makefile_path): return with open(makefile_path) as fp: lines = fp.readlines() for line in lines: if line.startswith("check:"): tests_config = testsuites["makecheck"] break if line.startswith("test:"): tests_config = testsuites["perlcheck"] break elif "Makefile.am" in files: tests_config = testsuites["makecheck"] elif tarball.name.startswith("rubygem"): if "test" in files or "tests" in files: r_testdir = [f for f in files if "test" in f if "." not in f].pop() pre_test = glob.glob(os.path.join(dir, "test*/test_*.rb")) post_test = glob.glob(os.path.join(dir, "test*/*_test.rb")) tests_config = "pushd %{buildroot}%{gem_dir}/gems/" + tarball.tarball_prefix if len(pre_test) > 0: tests_config += "\nruby -v -I.:lib:" + r_testdir + " test*/test_*.rb" if len(post_test) > 0: tests_config += "\nruby -v -I.:lib:" + r_testdir + " test*/*_test.rb" tests_config += "\npopd" elif "spec" in files: buildreq.add_buildreq("rubygem-rspec") buildreq.add_buildreq("rubygem-rspec-core") buildreq.add_buildreq("rubygem-rspec-expectations") buildreq.add_buildreq("rubygem-rspec-support") buildreq.add_buildreq("rubygem-rspec-mocks") buildreq.add_buildreq("rubygem-devise") buildreq.add_buildreq("rubygem-diff-lcs") tests_config = testsuites["rspec"] elif "Rakefile" in files: with open(os.path.join(dir, "Rakefile"), encoding="ascii", errors="surrogateescape") as fp: setup_contents = fp.read() # if setup_contents.find("task :test") >= 0 or setup_contents.find("task 'test'") >= 0: tests_config = testsuites["rakefile"] buildreq.add_buildreq("ruby") buildreq.add_buildreq("rubygem-rake") buildreq.add_buildreq("rubygem-test-unit") buildreq.add_buildreq("rubygem-minitest") elif "Makefile.PL" in files: tests_config = testsuites["perlcheck"] elif "setup.py" in files: with open(os.path.join(dir, "setup.py"), encoding="ascii", errors="surrogateescape") as fp: setup_contents = fp.read() if "test_suite" in setup_contents or "pbr=True" in setup_contents: tests_config = testsuites["setup.py"] elif buildpattern.default_pattern == "R": tests_config = "export _R_CHECK_FORCE_SUGGESTS_=false\nR CMD check --no-manual --no-examples --no-codoc -l %{buildroot}/usr/lib64/R/library " + tarball.rawname if "tox.ini" in files: buildreq.add_buildreq("tox") buildreq.add_buildreq("pytest") buildreq.add_buildreq("virtualenv") buildreq.add_buildreq("pluggy") buildreq.add_buildreq("py-python") if tests_config != "" and allow_test_failures: if tarball.name.startswith("rubygem"): config_elements = tests_config.split("\n") config_elements.pop() tests_config = "\n".join(config_elements) + " || :\npopd" elif tests_config == testsuites["cmake"]: tests_config = "pushd clr-build ; make test ||: ; popd" else: tests_config = tests_config + " || :" print(tests_config)
def parse_config_files(path, bump, filemanager, version): """Parse the various configuration files that may exist in the package directory.""" global extra_configure global extra_configure32 global extra_configure64 global extra_configure_avx2 global extra_configure_avx512 global config_files global parallel_build global license_fetch global license_show global git_uri global os_packages global urlban global config_file global profile_payload global config_opts global extra_make global extra32_make global extra_make_install global extra_make32_install global extra_cmake global cmake_srcdir global subdir global install_macro global disable_static global prep_prepend global build_prepend global make_prepend global install_prepend global install_append global patches global autoreconf global yum_conf global custom_desc global failed_pattern_dir packages_file = None # Require autospec.conf for additional features if os.path.exists(config_file): config = configparser.ConfigParser(interpolation=None) config.read(config_file) if "autospec" not in config.sections(): print("Missing autospec section..") sys.exit(1) git_uri = config['autospec'].get('git', None) license_fetch = config['autospec'].get('license_fetch', None) license_show = config['autospec'].get('license_show', None) packages_file = config['autospec'].get('packages_file', None) yum_conf = config['autospec'].get('yum_conf', None) failed_pattern_dir = config['autospec'].get('failed_pattern_dir', None) # support reading the local files relative to config_file if packages_file and not os.path.isabs(packages_file): packages_file = os.path.join(os.path.dirname(config_file), packages_file) if yum_conf and not os.path.isabs(yum_conf): yum_conf = os.path.join(os.path.dirname(config_file), yum_conf) if failed_pattern_dir and not os.path.isabs(failed_pattern_dir): failed_pattern_dir = os.path.join(os.path.dirname(config_file), failed_pattern_dir) if not packages_file: print( "Warning: Set [autospec][packages_file] path to package list file for " "requires validation") packages_file = os.path.join(os.path.dirname(config_file), "packages") urlban = config['autospec'].get('urlban', None) # Read values from options.conf (and deprecated files) and rewrite as necessary read_config_opts(path) if not git_uri: print( "Warning: Set [autospec][git] upstream template for remote git URI configuration" ) if not license_fetch: print( "Warning: Set [autospec][license_fetch] uri for license fetch support" ) if not license_show: print( "Warning: Set [autospec][license_show] uri for license link check support" ) if not yum_conf: print( "Warning: Set [autospec][yum_conf] path to yum.conf file for whatrequires validation" ) yum_conf = os.path.join(os.path.dirname(config_file), "image-creator/yum.conf") if packages_file: os_packages = set(read_conf_file(packages_file)) else: os_packages = set(read_conf_file("~/packages")) wrapper = textwrap.TextWrapper() wrapper.initial_indent = "# " wrapper.subsequent_indent = "# " def write_default_conf_file(name, description): """Write default configuration file with description to file name.""" config_files.add(name) filename = os.path.join(path, name) if os.path.isfile(filename): return write_out(filename, wrapper.fill(description) + "\n") write_default_conf_file( "buildreq_ban", "This file contains build requirements that get picked up but are " "undesirable. One entry per line, no whitespace.") write_default_conf_file( "pkgconfig_ban", "This file contains pkgconfig build requirements that get picked up but" " are undesirable. One entry per line, no whitespace.") write_default_conf_file( "requires_ban", "This file contains runtime requirements that get picked up but are " "undesirable. One entry per line, no whitespace.") write_default_conf_file( "buildreq_add", "This file contains additional build requirements that did not get " "picked up automatically. One name per line, no whitespace.") write_default_conf_file( "pkgconfig_add", "This file contains additional pkgconfig build requirements that did " "not get picked up automatically. One name per line, no whitespace.") write_default_conf_file( "requires_add", "This file contains additional runtime requirements that did not get " "picked up automatically. One name per line, no whitespace.") write_default_conf_file( "excludes", "This file contains the output files that need %exclude. Full path " "names, one per line.") content = read_conf_file(os.path.join(path, "release")) if content and content[0]: r = int(content[0]) if bump: r += 1 tarball.release = str(r) print("Release :", tarball.release) content = read_conf_file(os.path.join(path, "buildreq_ban")) for banned in content: print("Banning build requirement: %s." % banned) buildreq.banned_buildreqs.add(banned) content = read_conf_file(os.path.join(path, "pkgconfig_ban")) for banned in content: banned = "pkgconfig(%s)" % banned print("Banning build requirement: %s." % banned) buildreq.banned_buildreqs.add(banned) content = read_conf_file(os.path.join(path, "requires_ban")) for banned in content: print("Banning runtime requirement: %s." % banned) buildreq.banned_requires.add(banned) content = read_conf_file(os.path.join(path, "buildreq_add")) for extra in content: print("Adding additional build requirement: %s." % extra) buildreq.add_buildreq(extra) content = read_conf_file(os.path.join(path, "buildreq_cache")) if content and content[0] == version: for extra in content[1:]: print("Adding additional build (cache) requirement: %s." % extra) buildreq.add_buildreq(extra) content = read_conf_file(os.path.join(path, "pkgconfig_add")) for extra in content: extra = "pkgconfig(%s)" % extra print("Adding additional build requirement: %s." % extra) buildreq.add_buildreq(extra) content = read_conf_file(os.path.join(path, "requires_add")) for extra in content: print("Adding additional runtime requirement: %s." % extra) buildreq.add_requires(extra, override=True) content = read_conf_file(os.path.join(path, "excludes")) for exclude in content: print("%%exclude for: %s." % exclude) filemanager.excludes += content content = read_conf_file(os.path.join(path, "extras")) for extra in content: print("extras for : %s." % extra) filemanager.extras += content content = read_conf_file(os.path.join(path, "dev_extras")) for extra in content: print("dev for : %s." % extra) filemanager.dev_extras += content content = read_conf_file(os.path.join(path, "setuid")) for suid in content: print("setuid for : %s." % suid) filemanager.setuid += content content = read_conf_file(os.path.join(path, "attrs")) for line in content: attr = re.split(r'\(|\)|,', line) attr = [a.strip() for a in attr] filename = attr.pop() print("attr for: %s." % filename) filemanager.attrs[filename] = attr patches += read_conf_file(os.path.join(path, "series")) pfiles = [("%s/%s" % (path, x.split(" ")[0])) for x in patches] cmd = "egrep \"(\+\+\+|\-\-\-).*((Makefile.am)|(configure.ac|configure.in))\" %s" % " ".join( pfiles) # noqa: W605 if patches and call( cmd, check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) == 0: autoreconf = True if any(p.lower().startswith('cve-') for p in patches): config_opts['security_sensitive'] = True rewrite_config_opts(path) content = read_conf_file(os.path.join(path, "configure")) extra_configure = " \\\n".join(content) content = read_conf_file(os.path.join(path, "configure32")) extra_configure32 = " \\\n".join(content) content = read_conf_file(os.path.join(path, "configure64")) extra_configure64 = " \\\n".join(content) content = read_conf_file(os.path.join(path, "configure_avx2")) extra_configure_avx2 = " \\\n".join(content) content = read_conf_file(os.path.join(path, "configure_avx512")) extra_configure_avx512 = " \\\n".join(content) if config_opts["keepstatic"]: disable_static = "" if config_opts['broken_parallel_build']: parallel_build = "" content = read_conf_file(os.path.join(path, "make_args")) if content: extra_make = " \\\n".join(content) content = read_conf_file(os.path.join(path, "make32_args")) if content: extra32_make = " \\\n".join(content) content = read_conf_file(os.path.join(path, "make_install_args")) if content: extra_make_install = " \\\n".join(content) content = read_conf_file(os.path.join(path, "make32_install_args")) if content: extra_make32_install = " \\\n".join(content) content = read_conf_file(os.path.join(path, "install_macro")) if content and content[0]: install_macro = content[0] content = read_conf_file(os.path.join(path, "cmake_args")) if content: extra_cmake = " \\\n".join(content) content = read_conf_file(os.path.join(path, "cmake_srcdir")) if content and content[0]: cmake_srcdir = content[0] content = read_conf_file(os.path.join(path, "subdir")) if content and content[0]: subdir = content[0] content = read_conf_file(os.path.join(path, "build_pattern")) if content and content[0]: buildpattern.set_build_pattern(content[0], 20) autoreconf = False content = read_conf_file(os.path.join(path, "make_check_command")) if content: check.tests_config = '\n'.join(content) content = read_conf_file(os.path.join(path, tarball.name + ".license")) if content and content[0]: words = content[0].split() for word in words: if word.find(":") < 0: license.add_license(word) content = read_conf_file(os.path.join(path, "golang_libpath")) if content and content[0]: tarball.golibpath = content[0] print("golibpath : {}".format(tarball.golibpath)) if config_opts['use_clang']: config_opts['funroll-loops'] = False buildreq.add_buildreq("llvm") if config_opts['32bit']: buildreq.add_buildreq("glibc-libc32") buildreq.add_buildreq("glibc-dev32") buildreq.add_buildreq("gcc-dev32") buildreq.add_buildreq("gcc-libgcc32") buildreq.add_buildreq("gcc-libstdc++32") prep_prepend = read_conf_file(os.path.join(path, "prep_prepend")) if os.path.isfile(os.path.join(path, "prep_append")): os.rename(os.path.join(path, "prep_append"), os.path.join(path, "build_prepend")) make_prepend = read_conf_file(os.path.join(path, "make_prepend")) build_prepend = read_conf_file(os.path.join(path, "build_prepend")) install_prepend = read_conf_file(os.path.join(path, "install_prepend")) if os.path.isfile(os.path.join(path, "make_install_append")): os.rename(os.path.join(path, "make_install_append"), os.path.join(path, "install_append")) install_append = read_conf_file(os.path.join(path, "install_append")) profile_payload = read_conf_file(os.path.join(path, "profile_payload")) custom_desc = read_conf_file(os.path.join(path, "description"))
def name_and_version(name_arg, version_arg, filemanager): """ Parse the url for the package name and version """ global name global rawname global version global url tarfile = os.path.basename(url) # If both name and version overrides are set via commandline, set the name # and version variables to the overrides and bail. If only one override is # set, continue to auto detect both name and version since the URL parsing # handles both. In this case, wait until the end to perform the override of # the one that was set. if name_arg and version_arg: # rawname == name in this case name = name_arg rawname = name version = version_arg return # it is important for the more specific patterns to come first pattern_options = [ r"(.*?)[\-_](v*[0-9]+[a-zalpha\+_spbfourcesigedsvstableP0-9\.\-\~]*)\.orig\.tar", r"(.*?)[\-_](v*[0-9]+[alpha\+_spbfourcesigedsvstableP0-9\.\-\~]*)\.src\.(tgz|tar|zip)", r"(.*?)[\-_](v*[0-9]+[alpha\+_sbpfourcesigedsvstableP0-9\.\-\~]*)\.(tgz|tar|zip)", r"(.*?)[\-_](v*[0-9]+[\+_spbfourcesigedsvstableP0-9\.\~]*)(-.*?)?\.tar", ] for pattern in pattern_options: p = re.compile(pattern) m = p.search(tarfile) if m: name = m.group(1).strip() version = convert_version(m.group(2)) break rawname = name # R package if url.find("cran.r-project.org") > 0 or url.find("cran.rstudio.com") > 0: buildpattern.set_build_pattern("R", 10) filemanager.want_dev_split = False buildreq.add_buildreq("clr-R-helpers") p = re.compile( r"([A-Za-z0-9.]+)_(v*[0-9]+[\+_spbfourcesigedsvstableP0-9\.\~\-]*)\.tar\.gz" ) m = p.search(tarfile) if m: name = "R-" + m.group(1).strip() rawname = m.group(1).strip() version = m.group(2).strip().replace('-', '.') if url.find("pypi.python.org") > 0: buildpattern.set_build_pattern("distutils23", 10) url = "http://pypi.debian.net/" + name + "/" + tarfile buildreq.add_buildreq("python3-dev") buildreq.add_buildreq("python-dev") if url.find("pypi.debian.net") > 0: buildpattern.set_build_pattern("distutils23", 10) buildreq.add_buildreq("python3-dev") buildreq.add_buildreq("python-dev") if url.find(".cpan.org/CPAN/") > 0: buildpattern.set_build_pattern("cpan", 10) if name: name = "perl-" + name if url.find(".metacpan.org/") > 0: buildpattern.set_build_pattern("cpan", 10) if name: name = "perl-" + name if "github.com" in url: # define regex accepted for valid packages, important for specific # patterns to come before general ones github_patterns = [ r"https?://github.com/.*/(.*?)/archive/(.*)-final.tar", r"https?://github.com/.*/.*/archive/[0-9a-fA-F]{1,40}\/(.*)\-(.*).tar", r"https?://github.com/.*/(.*?)/archive/v?(.*).orig.tar", r"https?://github.com/.*/(.*?)/archive/(.*).zip", r"https?://github.com/.*/(.*?)/archive/v?(.*).tar", r"https?://github.com/.*/(.*?)/releases/download/v.*/(.*).tar" ] for pattern in github_patterns: p = re.compile(pattern) m = p.search(url) if m: name = m.group(1).strip() rawname = name version = convert_version(m.group(2)) break if url.find("bitbucket.org") > 0: p = re.compile( r"https?://bitbucket.org/.*/(.*?)/.*/([.0-9a-zA-Z_-]*?).tar") m = p.search(url) if m: name = m.group(1).strip() version = convert_version(m.group(2)) else: version = "1" # ruby if url.find("rubygems.org/") > 0: buildpattern.set_build_pattern("ruby", 10) p = re.compile( r"(.*?)[\-_](v*[0-9]+[alpha\+_spbfourcesigedsvstableP0-9\.\-\~]*)\.gem" ) m = p.search(tarfile) if m: name = "rubygem-" + m.group(1).strip() # remove release candidate tag b = name.find("-rc") if b > 0: name = name[:b] rawname = m.group(1).strip() version = m.group(2).strip() b = version.find("-") if b >= 0: version = version[:b] # maven if url.find("maven.org") > 0: buildpattern.set_build_pattern("maven", 10) if not name: split = url.split('/') if len(split) > 3 and split[-2] in ('archive', 'tarball'): name = split[-3] version = split[-1] version = version.lstrip('v') # remove extension version = version.rsplit('.', 1)[0] if version.endswith('.tar'): version = version.replace('.tar', '') # override name and version from commandline name = name_arg if name_arg else name version = version_arg if version_arg else version # sanity check to make sure we aren't using an empty version if version == "": version = "1"