def detect_build_from_url(url):
"""
Detect build patterns and build requirements from the patterns detected
in the url.
"""
# R package
if "cran.r-project.org" in url or "cran.rstudio.com" in url:
buildpattern.set_build_pattern("R", 10)
buildreq.add_buildreq("clr-R-helpers")
# python
if "pypi.python.org" in url or "pypi.debian.net" in url:
buildpattern.set_build_pattern("distutils3", 10)
buildreq.add_buildreq("python3-dev")
# cpan
if ".cpan.org/" in url or ".metacpan.org/" in url:
buildpattern.set_build_pattern("cpan", 10)
# ruby
if "rubygems.org/" in url:
buildpattern.set_build_pattern("ruby", 10)
# maven
if ".maven." in url:
buildpattern.set_build_pattern("maven", 10)
# rust crate
if "crates.io" in url:
buildpattern.set_build_pattern("cargo", 10)
def test_add_buildreq(self):
"""
Test add_buildreq with unbanned new req. Follow up by asserting that
trying to add the same req a second time results in a False return.
"""
self.assertTrue(buildreq.add_buildreq('testreq'))
self.assertIn('testreq', buildreq.buildreqs)
self.assertFalse(buildreq.add_buildreq('testreq'))
def test_add_requires(self):
"""
Test add_requires with unbanned new req already present in
buildreqs but not yet present in requires
"""
buildreq.add_buildreq('testreq')
self.assertTrue(buildreq.add_requires('testreq'))
self.assertIn('testreq', buildreq.requires)
def failed_pattern(line, pattern, verbose, buildtool=None):
global must_restart
pat = re.compile(pattern)
match = pat.search(line)
if not match:
return
s = match.group(1)
try:
if not buildtool:
req = config.failed_commands[s]
if req:
must_restart += buildreq.add_buildreq(req)
elif buildtool == 'pkgconfig':
must_restart += buildreq.add_pkgconfig_buildreq(s)
elif buildtool == 'R':
if buildreq.add_buildreq("R-" + s) > 0:
must_restart += 1
buildreq.add_requires("R-" + s)
elif buildtool == 'perl':
must_restart += buildreq.add_buildreq('perl(%s)' % s)
elif buildtool == 'pypi':
s = util.translate(s)
if not s:
return
must_restart += buildreq.add_buildreq(
util.translate('%s-python' % s))
elif buildtool == 'ruby':
if s in config.gems:
must_restart += buildreq.add_buildreq(config.gems[s])
else:
must_restart += buildreq.add_buildreq('rubygem-%s' % s)
elif buildtool == 'ruby table':
if s in config.gems:
must_restart += buildreq.add_buildreq(config.gems[s])
else:
print("Unknown ruby gem match", s)
elif buildtool == 'maven':
if s in config.maven_jars:
must_restart += buildreq.add_buildreq(config.maven_jars[s])
else:
must_restart += buildreq.add_buildreq('jdk-%s' % s)
elif buildtool == 'catkin':
must_restart += buildreq.add_pkgconfig_buildreq(s)
must_restart += buildreq.add_buildreq(s)
except:
if verbose > 0:
print("Unknown pattern match: ", s)
def failed_pattern_maven(line, pattern, verbose=0):
global must_restart
global failed_commands
pat = re.compile(pattern)
match = pat.search(line)
if match:
s = match.group(1)
try:
if s in buildreq.maven_jars:
must_restart = must_restart + buildreq.add_buildreq(buildreq.maven_jars[s])
else:
must_restart = must_restart + buildreq.add_buildreq('jdk-%s' % s)
except:
if verbose > 0:
print("Unknown maven jar match: ", s)
def simple_pattern(line, pattern, req):
"""Check for simple patterns and restart the build as needed."""
global must_restart
pat = re.compile(pattern)
match = pat.search(line)
if match:
must_restart += buildreq.add_buildreq(req, cache=True)
def failed_pattern_ruby(line, pattern, verbose=0):
global must_restart
global failed_commands
pat = re.compile(pattern)
match = pat.search(line)
if match:
s = match.group(1)
try:
if s in buildreq.gems:
must_restart = must_restart + buildreq.add_buildreq(buildreq.gems[s])
else:
must_restart = must_restart + buildreq.add_buildreq('rubygem-%s' % s)
except:
if verbose > 0:
print("Unknown pattern match: ", pattern, s)
def failed_pattern_perl(line, pattern, verbose=0):
global must_restart
global failed_commands
pat = re.compile(pattern)
match = pat.search(line)
if match:
s = match.group(1)
try:
must_restart = must_restart + buildreq.add_buildreq('perl(%s)' % s)
except:
if verbose > 0:
print("Unknown pattern match: ", pattern, s)
def failed_pattern_pypi(line, pattern, verbose=0):
global must_restart
global failed_commands
pat = re.compile(pattern)
match = pat.search(line)
if match:
s = util.translate(match.group(1))
if s == "":
return
try:
must_restart = must_restart + buildreq.add_buildreq(util.translate('%s-python' % s))
except:
if verbose > 0:
print("Unknown python pattern match: ", pattern, s, line)
def failed_pattern_R(line, pattern, verbose=0):
global must_restart
global failed_commands
pat = re.compile(pattern)
match = pat.search(line)
if match:
s = match.group(1)
try:
if buildreq.add_buildreq("R-" + s) > 0:
must_restart = must_restart + 1
files.push_main_requires("R-" + s)
except:
if verbose > 0:
print("Unknown pattern match: ", pattern, s)
def failed_pattern_go(line, pattern, verbose=0):
global must_restart
global failed_commands
pat = re.compile(pattern)
match = pat.search(line)
if match:
s = util.translate(match.group(1))
if s == "":
return
elif s == match.group(1):
# the requirement it's also golang libpath format
# (e.g: github.com/<project>/<repo> so transform into pkg name
s = util.golang_name(s)
try:
must_restart = must_restart + buildreq.add_buildreq(s)
except:
if verbose > 0:
print("Unknown golang pattern match: ", pattern, s, line)
def failed_pattern(line, pattern, verbose, buildtool=None):
"""Check against failed patterns to restart build as needed."""
global must_restart
global warned_about
pat = re.compile(pattern)
match = pat.search(line)
if not match:
return
s = match.group(1)
# standard configure cleanups
s = cleanup_req(s)
if s in config.ignored_commands:
return
try:
if not buildtool:
req = config.failed_commands[s]
if req:
must_restart += buildreq.add_buildreq(req, cache=True)
elif buildtool == 'pkgconfig':
must_restart += buildreq.add_pkgconfig_buildreq(s, cache=True)
elif buildtool == 'R':
if buildreq.add_buildreq("R-" + s, cache=True) > 0:
must_restart += 1
buildreq.add_requires("R-" + s)
elif buildtool == 'perl':
s = s.replace('inc::', '')
must_restart += buildreq.add_buildreq('perl(%s)' % s, cache=True)
elif buildtool == 'pypi':
s = util.translate(s)
if not s:
return
must_restart += buildreq.add_buildreq(util.translate('%s-python' % s), cache=True)
elif buildtool == 'ruby':
if s in config.gems:
must_restart += buildreq.add_buildreq(config.gems[s], cache=True)
else:
must_restart += buildreq.add_buildreq('rubygem-%s' % s, cache=True)
elif buildtool == 'ruby table':
if s in config.gems:
must_restart += buildreq.add_buildreq(config.gems[s], cache=True)
else:
print("Unknown ruby gem match", s)
elif buildtool == 'maven' or buildtool == 'gradle':
group_count = len(match.groups())
if group_count == 2:
# Add fully qualified versioned mvn() dependency
name = match.group(1)
# Hyphens are disallowed for version strings, so use dots instead
ver = match.group(2).replace('-', '.')
mvn_provide = f'mvn({name}) = {ver}'
must_restart += buildreq.add_buildreq(mvn_provide, cache=True)
elif s in config.maven_jars:
# Overrides for dependencies with custom grouping
must_restart += buildreq.add_buildreq(config.maven_jars[s], cache=True)
elif group_count == 3:
org = match.group(1)
name = match.group(2)
ver = match.group(3).replace('-', '.')
if re.search("-(parent|pom|bom)$", name):
mvn_provide = f'mvn({org}:{name}:pom) = {ver}'
else:
mvn_provide = f'mvn({org}:{name}:jar) = {ver}'
must_restart += buildreq.add_buildreq(mvn_provide, cache=True)
else:
# Fallback to mvn-ARTIFACTID package name
must_restart += buildreq.add_buildreq('mvn-%s' % s, cache=True)
elif buildtool == 'catkin':
must_restart += buildreq.add_pkgconfig_buildreq(s, cache=True)
must_restart += buildreq.add_buildreq(s, cache=True)
except Exception:
if s not in warned_about and s[:2] != '--':
print("Unknown pattern match: ", s)
warned_about.add(s)
def scan_for_tests(src_dir):
"""
Scan source directory for test files and set tests_config accordingly
"""
global tests_config
if config.config_opts['skip_tests'] or tests_config:
return
makeflags = "%{?_smp_mflags} " if config.parallel_build else ""
testsuites = {
"makecheck": "make VERBOSE=1 V=1 {}check".format(makeflags),
"perlcheck": "make TEST_VERBOSE=1 test",
"setup.py": "PYTHONPATH=%{buildroot}/usr/lib/python3.6/site-packages "
"python3 setup.py test",
"cmake": "pushd clr-build ; make test ; popd",
"rakefile": "pushd %{buildroot}%{gem_dir}/gems/" +
tarball.tarball_prefix +
"\nrake --trace test TESTOPTS=\"-v\"\npopd",
"rspec": "pushd %{buildroot}%{gem_dir}/gems/" +
tarball.tarball_prefix + "\nrspec -I.:lib spec/\npopd"
}
files = os.listdir(src_dir)
if "CMakeLists.txt" in files:
makefile_path = os.path.join(src_dir, "CMakeLists.txt")
if not os.path.isfile(makefile_path):
return
if "enable_testing" in open(makefile_path, encoding='latin-1').read():
tests_config = testsuites["cmake"]
if "Makefile.in" in files:
makefile_path = os.path.join(src_dir, "Makefile.in")
if not os.path.isfile(makefile_path):
return
with open(makefile_path, 'r', encoding="latin-1") as make_fp:
lines = make_fp.readlines()
for line in lines:
if line.startswith("check:"):
tests_config = testsuites["makecheck"]
break
if line.startswith("test:"):
tests_config = testsuites["perlcheck"]
break
elif "Makefile.am" in files:
tests_config = testsuites["makecheck"]
elif tarball.name.startswith("rubygem"):
if any(t in files for t in ["test", "tests"]):
r_testdir = [f for f in files if "test" in f if "." not in f].pop()
pre_test = glob.glob(os.path.join(src_dir, "test*/test_*.rb"))
post_test = glob.glob(os.path.join(src_dir, "test*/*_test.rb"))
tests_config = "pushd %{buildroot}%{gem_dir}/gems/" + tarball.tarball_prefix
if pre_test:
tests_config += "\nruby -v -I.:lib:" + r_testdir + " test*/test_*.rb"
if post_test:
tests_config += "\nruby -v -I.:lib:" + r_testdir + " test*/*_test.rb"
tests_config += "\npopd"
elif "spec" in files:
buildreq.add_buildreq("rubygem-rspec")
buildreq.add_buildreq("rubygem-rspec-core")
buildreq.add_buildreq("rubygem-rspec-expectations")
buildreq.add_buildreq("rubygem-rspec-support")
buildreq.add_buildreq("rubygem-rspec-mocks")
buildreq.add_buildreq("rubygem-devise")
buildreq.add_buildreq("rubygem-diff-lcs")
tests_config = testsuites["rspec"]
elif "Rakefile" in files:
tests_config = testsuites["rakefile"]
buildreq.add_buildreq("ruby")
buildreq.add_buildreq("rubygem-rake")
buildreq.add_buildreq("rubygem-test-unit")
buildreq.add_buildreq("rubygem-minitest")
elif "Makefile.PL" in files:
tests_config = testsuites["perlcheck"]
elif "setup.py" in files:
with open(os.path.join(src_dir, "setup.py"), 'r',
encoding="ascii",
errors="surrogateescape") as setup_fp:
setup_contents = setup_fp.read()
if "test_suite" in setup_contents or "pbr=True" in setup_contents:
tests_config = testsuites["setup.py"]
elif buildpattern.default_pattern == "R":
tests_config = "export _R_CHECK_FORCE_SUGGESTS_=false\n" \
"R CMD check --no-manual --no-examples --no-codoc -l " \
"%{buildroot}/usr/lib64/R/library " \
+ tarball.rawname + "|| : \ncp ~/.stash/* " \
"%{buildroot}/usr/lib64/R/library/*/libs/ || :"
if "tox.ini" in files:
buildreq.add_buildreq("tox")
buildreq.add_buildreq("pytest")
buildreq.add_buildreq("virtualenv")
buildreq.add_buildreq("pluggy")
buildreq.add_buildreq("py-python")
if tests_config != "" and config.config_opts['allow_test_failures']:
if tarball.name.startswith("rubygem"):
config_elements = tests_config.split("\n")
config_elements.pop()
tests_config = "\n".join(config_elements) + " || :\npopd"
elif tests_config == testsuites["cmake"]:
tests_config = "pushd clr-build ; make test ||: ; popd"
else:
tests_config = tests_config + " || :"
def parse_config_files(path, bump):
global extra_configure
global keepstatic
global asneeded
global optimize_size
global optimize_speed
global insecure_build
global config_files
global config_path
global parallel_build
global license_fetch
global license_show
global git_uri
global config_file
config_path = path
# Require autospec.conf for additional features
if os.path.exists(config_file):
config = configparser.ConfigParser(interpolation=None)
config.read(config_file)
if "autospec" not in config.sections():
print("Missing autospec section..")
sys.exit(1)
git_uri = config['autospec'].get('git', None)
license_fetch = config['autospec'].get('license_fetch', None)
license_show = config['autospec'].get('license_show', None)
if not git_uri:
print("Warning: Set [autospec][git] upstream template for git support")
if not license_fetch:
print("Warning: Set [autospec][license_fetch] uri for license fetch support")
if not license_show:
print("Warning: Set [autospec][license_show] uri for license link check support")
wrapper = textwrap.TextWrapper()
wrapper.initial_indent = "# "
wrapper.subsequent_indent = "# "
def write_default_conf_file(name, description):
config_files.add(name)
filename = path + "/" + name
if os.path.isfile(filename):
return
with open(filename, "w") as f:
f.write(wrapper.fill(description) + "\n")
write_default_conf_file("buildreq_ban", "This file contains build requirements that get picked up but are undesirable. One entry per line, no whitespace.")
write_default_conf_file("pkgconfig_ban", "This file contains pkgconfig build requirements that get picked up but are undesirable. One entry per line, no whitespace.")
write_default_conf_file("buildreq_add", "This file contains additional build requirements that did not get picked up automatically. One name per line, no whitespace.")
write_default_conf_file("pkgconfig_add", "This file contains additional pkgconfig build requirements that did not get picked up automatically. One name per line, no whitespace.")
write_default_conf_file("excludes", "This file contains the output files that need %exclude. Full path names, one per line.")
content = read_conf_file("release")
if content and content[0]:
r = int(content[0])
if bump:
r += 1
tarball.release = str(r)
print("Release :", tarball.release)
content = read_conf_file("buildreq_ban")
for banned in content:
print("Banning build requirement: %s." % banned)
buildreq.banned_buildreqs.add(banned)
content = read_conf_file("pkgconfig_ban")
for banned in content:
banned = "pkgconfig(%s)" % banned
print("Banning build requirement: %s." % banned)
buildreq.banned_buildreqs.add(banned)
content = read_conf_file("buildreq_add")
for extra in content:
print("Adding additional build requirement: %s." % extra)
buildreq.add_buildreq(extra)
content = read_conf_file("pkgconfig_add")
for extra in content:
extra = "pkgconfig(%s)" % extra
print("Adding additional build requirement: %s." % extra)
buildreq.add_buildreq(extra)
content = read_conf_file("excludes")
for exclude in content:
print("%%exclude for: %s." % exclude)
files.excludes += content
content = read_conf_file("extras")
for extra in content:
print("extras for: %s." % extra)
files.extras += content
content = read_conf_file("setuid")
for suid in content:
print("setuid for: %s." % suid)
files.setuid += content
content = read_conf_file("attrs")
for line in content:
attr = re.split('\(|\)|,', line)
attr = [a.strip() for a in attr]
filename = attr.pop()
print("attr for: %s." % filename)
files.attrs[filename] = attr
patches.patches += read_conf_file("series")
pfiles = [("%s/%s" % (path, x.split(" ")[0])) for x in patches.patches]
cmd = "egrep \"(\+\+\+|\-\-\-).*((Makefile.am|Makefile.in)|(configure.ac|configure.in))\" %s" % \
" ".join(pfiles)
if len(patches.patches) > 0 and call(cmd, check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) == 0:
patches.autoreconf = True
content = read_conf_file("configure")
extra_configure = " \\\n".join(content)
if read_conf_file("keepstatic"):
keepstatic = 1
buildpattern.disable_static = ""
if read_conf_file("asneeded"):
print("Disabling LD_AS_NEEDED\n")
asneeded = 0
if read_conf_file("optimize_size"):
optimize_size = True
if read_conf_file("funroll-loops"):
optimize_speed = True
if read_conf_file("insecure_build"):
insecure_build = True
if read_conf_file("broken_parallel_build"):
parallel_build = ""
content = read_conf_file("make_args")
if content and content[0]:
buildpattern.extra_make = content[0]
content = read_conf_file("make_install_args")
if content and content[0]:
buildpattern.extra_make_install = content[0]
content = read_conf_file("install_macro")
if content and content[0]:
buildpattern.install_macro = content[0]
content = read_conf_file("cmake_args")
if content and content[0]:
buildpattern.extra_cmake = content[0]
content = read_conf_file("subdir")
if content and content[0]:
buildpattern.subdir = content[0]
content = read_conf_file("build_pattern")
if content and content[0]:
buildpattern.set_build_pattern(content[0], 20)
patches.autoreconf = False
if read_conf_file("skip_test_suite"):
test.skip_tests = True
if read_conf_file("unit_tests_must_pass"):
test.new_pkg = False
content = read_conf_file("make_check_command")
if content and content[0]:
test.tests_config = content[0]
content = read_conf_file("allow_test_failures")
if content and content[0]:
test.allow_test_failures = True
content = read_conf_file(tarball.name + ".license")
if content and content[0]:
words = content[0].split()
for word in words:
if word.find(":") < 0:
license.add_license(word)
buildpattern.make_install_append = read_conf_file("make_install_append")
def name_and_version(url_argument, name_argument):
global name
global rawname
global version
global url
url = url_argument
tarfile = os.path.basename(url)
# it is important for the more specific patterns to come first
pattern_options = [
r"(.*?)[\-_](v*[0-9]+[a-zalpha\+_spbfourcesigedsvstableP0-9\.\-\~]*)\.orig\.tar",
r"(.*?)[\-_](v*[0-9]+[alpha\+_spbfourcesigedsvstableP0-9\.\-\~]*)\.src\.(tgz|tar|zip)",
r"(.*?)[\-_](v*[0-9]+[alpha\+_sbpfourcesigedsvstableP0-9\.\-\~]*)\.(tgz|tar|zip)",
r"(.*?)[\-_](v*[0-9]+[\+_spbfourcesigedsvstableP0-9\.\~]*)(-.*?)?\.tar",
]
for pattern in pattern_options:
p = re.compile(pattern)
m = p.search(tarfile)
if m:
name = m.group(1).strip()
version = m.group(2).strip()
b = version.find("-")
if b >= 0:
version = version[:b]
break
rawname = name
# R package
if url_argument.find("cran.r-project.org") > 0 or url_argument.find("cran.rstudio.com") > 0:
buildpattern.set_build_pattern("R", 10)
files.want_dev_split = 0
buildreq.add_buildreq("clr-R-helpers")
p = re.compile(r"([A-Za-z0-9]+)_(v*[0-9]+[\+_spbfourcesigedsvstableP0-9\.\~\-]*)\.tar\.gz")
m = p.search(tarfile)
if m:
name = "R-" + m.group(1).strip()
rawname = m.group(1).strip()
version = m.group(2).strip()
b = version.find("-")
if b >= 0:
version = version[:b]
if url_argument.find("pypi.python.org") > 0:
buildpattern.set_build_pattern("distutils", 10)
url_argument = "http://pypi.debian.net/" + name + "/" + tarfile
if url_argument.find("pypi.debian.net") > 0:
buildpattern.set_build_pattern("distutils", 10)
if url_argument.find(".cpan.org/CPAN/") > 0:
buildpattern.set_build_pattern("cpan", 10)
if name:
name = "perl-" + name
if url_argument.find(".metacpan.org/") > 0:
buildpattern.set_build_pattern("cpan", 10)
if name:
name = "perl-" + name
if "github.com" in url_argument:
# define regex accepted for valid packages, important for specific
# patterns to come before general ones
github_patterns = [r"https://github.com/.*/(.*?)/archive/(.*)-final.tar",
r"https://github.com/.*/.*/archive/[0-9a-fA-F]{1,40}\/(.*)\-(.*).tar",
r"https://github.com/.*/(.*?)/archive/v?(.*).orig.tar",
r"https://github.com/.*/(.*?)/archive/(.*).zip",
r"https://github.com/.*/(.*?)/archive/v?(.*).tar"]
for pattern in github_patterns:
p = re.compile(pattern)
m = p.search(url_argument)
if m:
name = m.group(1).strip()
# convert from 7_4_2 to 7.4.2
version = m.group(2).strip().replace('_', '.')
# remove release candidate tag
b = version.find("-rc")
if b > 0:
version = version[:b]
b = version.find("-")
if b > 0:
version = version[b + 1:]
break
if url_argument.find("bitbucket.org") > 0:
p = re.compile(r"https://bitbucket.org/.*/(.*?)/get/[a-zA-Z_-]*([0-9][0-9_.]*).tar")
m = p.search(url_argument)
if m:
name = m.group(1).strip()
# convert from 7_4_2 to 7.4.2
version = m.group(2).strip().replace('_', '.')
else:
version = "1"
# ruby
if url_argument.find("rubygems.org/") > 0:
buildpattern.set_build_pattern("ruby", 10)
p = re.compile(r"(.*?)[\-_](v*[0-9]+[alpha\+_spbfourcesigedsvstableP0-9\.\-\~]*)\.gem")
m = p.search(tarfile)
if m:
name = "rubygem-" + m.group(1).strip()
# remove release candidate tag
b = name.find("-rc")
if b > 0:
name = name[:b]
rawname = m.group(1).strip()
version = m.group(2).strip()
b = version.find("-")
if b >= 0:
version = version[:b]
# maven
if url_argument.find("maven.org") > 0:
buildpattern.set_build_pattern("maven", 10)
# override from commandline
if name_argument and name_argument[0] != name:
pattern = name_argument[0] + r"[\-]*(.*)\.(tgz|tar|zip)"
p = re.compile(pattern)
m = p.search(tarfile)
if m:
name = name_argument[0]
rawname = name
version = m.group(1).strip()
b = version.find("-")
if b >= 0 and version.find("-beta") < 0:
version = version[:b]
if version.startswith('.'):
version = version[1:]
else:
name = name_argument[0]
if not name:
split = url_argument.split('/')
if len(split) > 3 and split[-2] in ('archive', 'tarball'):
name = split[-3]
version = split[-1]
if version.startswith('v'):
version = version[1:]
# remove extension
version = '.'.join(version.split('.')[:-1])
if version.endswith('.tar'):
version = '.'.join(version.split('.')[:-1])
b = version.find("-")
if b >= 0 and version.find("-beta") < 0:
b = b + 1
version = version[b:]
if len(version) > 0 and version[0].lower() in ['v', 'r']:
version = version[1:]
# remove package name from beginning of version
if version.lower().startswith(name.lower()):
pat = re.compile(re.escape(name), re.IGNORECASE)
version = pat.sub('', version)
if version[0] in ['.', '-', '_']:
version = version[1:]
assert name != ""
def name_and_version(name_arg, version_arg, filemanager):
"""Parse the url for the package name and version."""
global rawname
global url
global giturl
global repo
tarfile = os.path.basename(url)
# If both name and version overrides are set via commandline, set the name
# and version variables to the overrides and bail. If only one override is
# set, continue to auto detect both name and version since the URL parsing
# handles both. In this case, wait until the end to perform the override of
# the one that was set. An extra conditional, that version_arg is a string
# is added to enable a package to have multiple versions at the same time
# for some language ecosystems.
if name_arg and version_arg:
# rawname == name in this case
name = name_arg
rawname = name_arg
version = set_multi_version(version_arg)
return name, rawname, convert_version(version, name)
name = name_arg
rawname = name_arg
version = ""
# it is important for the more specific patterns to come first
pattern_options = [
# handle font packages with names ending in -nnndpi
r"(.*-[0-9]+dpi)[-_]([0-9]+[a-zA-Z0-9\+_\.\-\~]*)\.(tgz|tar|zip)",
r"(.*?)[-_][vs]?([0-9]+[a-zA-Z0-9\+_\.\-\~]*)\.(tgz|tar|zip)",
]
for pattern in pattern_options:
m = re.search(pattern, tarfile)
if m:
name = m.group(1).strip()
version = convert_version(m.group(2), name)
break
# R package
if "cran.r-project.org" in url or "cran.rstudio.com" in url:
filemanager.want_dev_split = False
m = re.search(
r"([A-Za-z0-9.]+)_(v*[0-9]+[a-zA-Z0-9\+_\.\~\-]*)\.tar\.gz",
tarfile)
if m:
name = "R-" + m.group(1).strip()
rawname = m.group(1).strip()
version = convert_version(m.group(2), name)
if ".cpan.org/" in url or ".metacpan.org/" in url and name:
name = "perl-" + name
if "github.com" in url:
# define regex accepted for valid packages, important for specific
# patterns to come before general ones
github_patterns = [
r"https?://github.com/(.*)/(.*?)/archive/[v|r]?.*/(.*).tar",
r"https?://github.com/(.*)/(.*?)/archive/[-a-zA-Z_]*-(.*).tar",
r"https?://github.com/(.*)/(.*?)/archive/[vVrR]?(.*).tar",
r"https?://github.com/(.*)/.*-downloads/releases/download/.*?/(.*)-(.*).tar",
r"https?://github.com/(.*)/(.*?)/releases/download/(.*)/",
r"https?://github.com/(.*)/(.*?)/files/.*?/(.*).tar"
]
for pattern in github_patterns:
m = re.search(pattern, url)
if m:
repo = m.group(2).strip()
if repo not in name:
# Only take the repo name as the package name if it's more descriptive
name = repo
elif name != repo:
name = re.sub(r"release-", '', name)
name = re.sub(r"\d*$", '', name)
rawname = name
version = m.group(3).replace(name, '')
if "archive" not in pattern:
version = re.sub(r"^[-_.a-zA-Z]+", "", version)
version = convert_version(version, name)
if not giturl:
giturl = "https://github.com/" + m.group(
1).strip() + "/" + repo + ".git"
break
if "gnome.org" in url:
buildreq.add_buildreq("buildreq-gnome")
if "kde.org" in url or "https://github.com/KDE" in url:
buildreq.add_buildreq("buildreq-kde")
# SQLite tarballs use 7 digit versions, e.g 3290000 = 3.29.0, 3081002 = 3.8.10.2
if "sqlite.org" in url:
major = version[0]
minor = version[1:3].lstrip("0").zfill(1)
patch = version[3:5].lstrip("0").zfill(1)
build = version[5:7].lstrip("0")
version = major + "." + minor + "." + patch + "." + build
version = version.strip(".")
# construct github giturl from gnome projects
if not giturl and "download.gnome.org" in url:
giturl = "https://github.com/GNOME/{}.git".format(name)
if "mirrors.kernel.org" in url:
m = re.search(r".*/sourceware/(.*?)/releases/(.*?).tgz", url)
if m:
name = m.group(1).strip()
version = convert_version(m.group(2), name)
if "sourceforge.net" in url:
scf_pats = [
r"projects/.*/files/(.*?)/(.*?)/[^-]*(-src)?.tar.gz",
r"downloads.sourceforge.net/.*/([a-zA-Z]+)([-0-9\.]*)(-src)?.tar.gz"
]
for pat in scf_pats:
m = re.search(pat, url)
if m:
name = m.group(1).strip()
version = convert_version(m.group(2), name)
break
if "bitbucket.org" in url:
bitbucket_pats = [
r"/.*/(.*?)/.*/.*v([-\.0-9a-zA-Z_]*?).(tar|zip)",
r"/.*/(.*?)/.*/([-\.0-9a-zA-Z_]*?).(tar|zip)"
]
for pat in bitbucket_pats:
m = re.search(pat, url)
version = 1
if m:
name = m.group(1).strip()
version = convert_version(m.group(2), name)
break
# ruby
if "rubygems.org/" in url:
m = re.search(r"(.*?)[\-_](v*[0-9]+[a-zA-Z0-9\+_\.\-\~]*)\.gem",
tarfile)
if m:
name = "rubygem-" + m.group(1).strip()
# remove release candidate tag from the package name
# https://rubygems.org/downloads/ruby-rc4-0.1.5.gem
b = name.find("-rc")
if b > 0:
name = name[:b]
rawname = m.group(1).strip()
version = convert_version(m.group(2), name)
# maven
if ".maven." in url:
maven_pats = [
r"maven.org/maven2/[a-zA-Z\-\_]+/([a-zA-Z\-\_])+/([a-zA-Z-\_\d.]+)/[a-zA-Z-\_\d.]*\.(?:pom|jar)",
r"maven.apache.org/maven2/[a-zA-Z\-\_]+/([a-zA-Z\-\_])+/([\d.]+)/[a-z-\_.\d]*\.(?:pom|jar)",
r"maven.org/maven2/(?:[a-zA-Z-\_.\d/]+)/([a-zA-Z-\_.\d]*)/([a-zA-Z\d\.\_\-]+)/(?:[a-zA-Z-\_.\d]*)\.(?:pom|jar)"
]
for pat in maven_pats:
m = re.search(pat, url)
if m:
name = m.group(1).strip()
version = convert_version(m.group(2), name)
break
# rust crate
if "crates.io" in url:
m = re.search(
r"/crates.io/api/v[0-9]+/crates/(.*)/(.*)/download.*\.crate", url)
if m:
name = m.group(1).strip()
version = convert_version(m.group(2), name)
if "gitlab.com" in url:
# https://gitlab.com/leanlabsio/kanban/-/archive/1.7.1/kanban-1.7.1.tar.gz
m = re.search(r"gitlab\.com/.*/(.*)/-/archive/(.*)/", url)
if m:
name = m.group(1).strip()
version = convert_version(m.group(2), name)
if "git.sr.ht" in url:
# https://git.sr.ht/~sircmpwn/scdoc/archive/1.9.4.tar.gz
m = re.search(r"git\.sr\.ht/.*/(.*)/archive/(.*).tar.gz", url)
if m:
name = m.group(1).strip()
version = convert_version(m.group(2), name)
# override name and version from commandline
name = name_arg if name_arg else name
version = version_arg if version_arg else version
version = set_multi_version(version)
return name, rawname, convert_version(version, name)
def failed_pattern(line, pattern, verbose, buildtool=None):
"""Check against failed patterns to restart build as needed."""
global must_restart
global warned_about
pat = re.compile(pattern)
match = pat.search(line)
if not match:
return
s = match.group(1)
# standard configure cleanups
s = cleanup_req(s)
if s in config.ignored_commands:
return
try:
if not buildtool:
req = config.failed_commands[s]
if req:
must_restart += buildreq.add_buildreq(req, cache=True)
elif buildtool == 'pkgconfig':
must_restart += buildreq.add_pkgconfig_buildreq(s, cache=True)
elif buildtool == 'R':
if buildreq.add_buildreq("R-" + s, cache=True) > 0:
must_restart += 1
buildreq.add_requires("R-" + s)
elif buildtool == 'perl':
s = s.replace('inc::', '')
must_restart += buildreq.add_buildreq('perl(%s)' % s, cache=True)
elif buildtool == 'pypi':
s = util.translate(s)
if not s:
return
must_restart += buildreq.add_buildreq(util.translate('%s-python' %
s),
cache=True)
elif buildtool == 'ruby':
if s in config.gems:
must_restart += buildreq.add_buildreq(config.gems[s],
cache=True)
else:
must_restart += buildreq.add_buildreq('rubygem-%s' % s,
cache=True)
elif buildtool == 'ruby table':
if s in config.gems:
must_restart += buildreq.add_buildreq(config.gems[s],
cache=True)
else:
print("Unknown ruby gem match", s)
elif buildtool == 'maven':
if s in config.maven_jars:
must_restart += buildreq.add_buildreq(config.maven_jars[s],
cache=True)
else:
must_restart += buildreq.add_buildreq('jdk-%s' % s, cache=True)
elif buildtool == 'catkin':
must_restart += buildreq.add_pkgconfig_buildreq(s, cache=True)
must_restart += buildreq.add_buildreq(s, cache=True)
except Exception:
if s not in warned_about and s[:2] != '--':
print("Unknown pattern match: ", s)
warned_about.add(s)
def test_add_buildreq_banned(self):
"""
Test add_buildreq with banned new req
"""
self.assertFalse(buildreq.add_buildreq('bannedreq'))
self.assertNotIn('bannedreq', buildreq.buildreqs)
def name_and_version(name_arg, version_arg, filemanager):
"""Parse the url for the package name and version."""
global name
global rawname
global version
global url
global giturl
global repo
tarfile = os.path.basename(url)
# If both name and version overrides are set via commandline, set the name
# and version variables to the overrides and bail. If only one override is
# set, continue to auto detect both name and version since the URL parsing
# handles both. In this case, wait until the end to perform the override of
# the one that was set.
if name_arg and version_arg:
# rawname == name in this case
name = name_arg
rawname = name
version = version_arg
return
# it is important for the more specific patterns to come first
pattern_options = [
# handle font packages with names ending in -nnndpi
r"(.*-[0-9]+dpi)[-_]([0-9]+[a-zA-Z0-9\+_\.\-\~]*)\.(tgz|tar|zip)",
r"(.*?)[-_][vs]?([0-9]+[a-zA-Z0-9\+_\.\-\~]*)\.(tgz|tar|zip)",
]
for pattern in pattern_options:
m = re.search(pattern, tarfile)
if m:
name = m.group(1).strip()
version = convert_version(m.group(2))
break
rawname = name
# R package
if "cran.r-project.org" in url or "cran.rstudio.com" in url:
filemanager.want_dev_split = False
m = re.search(
r"([A-Za-z0-9.]+)_(v*[0-9]+[a-zA-Z0-9\+_\.\~\-]*)\.tar\.gz",
tarfile)
if m:
name = "R-" + m.group(1).strip()
rawname = m.group(1).strip()
version = convert_version(m.group(2))
if ".cpan.org/" in url or ".metacpan.org/" in url and name:
name = "perl-" + name
if "github.com" in url:
# define regex accepted for valid packages, important for specific
# patterns to come before general ones
github_patterns = [
r"https?://github.com/(.*)/(.*?)/archive/[v|r]?.*/(.*).tar",
r"https?://github.com/(.*)/(.*?)/archive/[-a-zA-Z_]*-(.*).tar",
r"https?://github.com/(.*)/(.*?)/archive/[vVrR]?(.*).tar",
r"https?://github.com/(.*)/.*-downloads/releases/download/.*?/(.*)-(.*).tar",
r"https?://github.com/(.*)/(.*?)/releases/download/(.*)/",
r"https?://github.com/(.*)/(.*?)/files/.*?/(.*).tar"
]
for pattern in github_patterns:
m = re.search(pattern, url)
if m:
repo = m.group(2).strip()
if repo not in name:
# Only take the repo name as the package name if it's more descriptive
name = repo
elif name != repo:
name = re.sub(r"release-", '', name)
name = re.sub(r"\d*$", '', name)
rawname = name
version = m.group(3).replace(name, '')
if "archive" not in pattern:
version = re.sub(r"^[-_.a-zA-Z]+", "", version)
version = convert_version(version)
giturl = "https://github.com/" + m.group(
1).strip() + "/" + repo + ".git"
break
if "gnome.org" in url:
buildreq.add_buildreq("buildreq-gnome")
if "kde.org" in url or "https://github.com/KDE" in url:
buildreq.add_buildreq("buildreq-kde")
# construct github giturl from gnome projects
if not giturl and "download.gnome.org" in url:
giturl = "https://github.com/GNOME/{}.git".format(name)
if "mirrors.kernel.org" in url:
m = re.search(r".*/sourceware/(.*?)/releases/(.*?).tgz", url)
if m:
name = m.group(1).strip()
version = convert_version(m.group(2))
if "sourceforge.net" in url:
scf_pats = [
r"projects/.*/files/(.*?)/(.*?)/[^-]*(-src)?.tar.gz",
r"downloads.sourceforge.net/.*/([a-zA-Z]+)([-0-9\.]*)(-src)?.tar.gz"
]
for pat in scf_pats:
m = re.search(pat, url)
if m:
name = m.group(1).strip()
version = convert_version(m.group(2))
break
if "bitbucket.org" in url:
bitbucket_pats = [
r"/.*/(.*?)/.*/.*v([-\.0-9a-zA-Z_]*?).(tar|zip)",
r"/.*/(.*?)/.*/([-\.0-9a-zA-Z_]*?).(tar|zip)"
]
for pat in bitbucket_pats:
m = re.search(pat, url)
version = 1
if m:
name = m.group(1).strip()
version = convert_version(m.group(2))
break
# ruby
if "rubygems.org/" in url:
m = re.search(r"(.*?)[\-_](v*[0-9]+[a-zA-Z0-9\+_\.\-\~]*)\.gem",
tarfile)
if m:
name = "rubygem-" + m.group(1).strip()
# remove release candidate tag from the package name
# https://rubygems.org/downloads/ruby-rc4-0.1.5.gem
b = name.find("-rc")
if b > 0:
name = name[:b]
rawname = m.group(1).strip()
version = convert_version(m.group(2))
# maven
if ".maven." in url:
m = re.search(
r"/maven.*/org/.*/(.*?)/.*/.*[\-_]([0-9]+[a-zA-Z0-9\+_\.\-\~]*)\.jar",
url)
if m:
name = m.group(1).strip()
version = convert_version(m.group(2))
# rust crate
if "crates.io" in url:
m = re.search(
r"/crates.io/api/v[0-9]+/crates/(.*)/(.*)/download.*\.crate", url)
if m:
name = m.group(1).strip()
version = convert_version(m.group(2))
if "gitlab.com" in url:
# https://gitlab.com/leanlabsio/kanban/-/archive/1.7.1/kanban-1.7.1.tar.gz
m = re.search(r"gitlab\.com/.*/(.*)/-/archive/(.*)/", url)
if m:
name = m.group(1).strip()
version = convert_version(m.group(2))
# override name and version from commandline
name = name_arg if name_arg else name
version = version_arg if version_arg else version
# sanity check to make sure we aren't using an empty version
if version == "":
version = "1"
def parse_config_files(path, bump):
global extra_configure
global extra_configure32
global config_files
global config_path
global parallel_build
global license_fetch
global license_show
global git_uri
global urlban
global config_file
global profile_payload
global config_opts
config_path = path
read_config_opts()
# Require autospec.conf for additional features
if os.path.exists(config_file):
config = configparser.ConfigParser(interpolation=None)
config.read(config_file)
if "autospec" not in config.sections():
print("Missing autospec section..")
sys.exit(1)
git_uri = config["autospec"].get("git", None)
license_fetch = config["autospec"].get("license_fetch", None)
license_show = config["autospec"].get("license_show", None)
urlban = config["autospec"].get("urlban", None)
if not git_uri:
print("Warning: Set [autospec][git] upstream template for git support")
if not license_fetch:
print("Warning: Set [autospec][license_fetch] uri for license fetch support")
if not license_show:
print("Warning: Set [autospec][license_show] uri for license link check support")
wrapper = textwrap.TextWrapper()
wrapper.initial_indent = "# "
wrapper.subsequent_indent = "# "
def write_default_conf_file(name, description):
config_files.add(name)
filename = path + "/" + name
if os.path.isfile(filename):
return
with open(filename, "w") as f:
f.write(wrapper.fill(description) + "\n")
write_default_conf_file(
"buildreq_ban",
"This file contains build requirements that get picked up but are undesirable. One entry per line, no whitespace.",
)
write_default_conf_file(
"pkgconfig_ban",
"This file contains pkgconfig build requirements that get picked up but are undesirable. One entry per line, no whitespace.",
)
write_default_conf_file(
"buildreq_add",
"This file contains additional build requirements that did not get picked up automatically. One name per line, no whitespace.",
)
write_default_conf_file(
"pkgconfig_add",
"This file contains additional pkgconfig build requirements that did not get picked up automatically. One name per line, no whitespace.",
)
write_default_conf_file(
"excludes", "This file contains the output files that need %exclude. Full path names, one per line."
)
content = read_conf_file("release")
if content and content[0]:
r = int(content[0])
if bump:
r += 1
tarball.release = str(r)
print("Release :", tarball.release)
content = read_conf_file("buildreq_ban")
for banned in content:
print("Banning build requirement: %s." % banned)
buildreq.banned_buildreqs.add(banned)
content = read_conf_file("pkgconfig_ban")
for banned in content:
banned = "pkgconfig(%s)" % banned
print("Banning build requirement: %s." % banned)
buildreq.banned_buildreqs.add(banned)
content = read_conf_file("buildreq_add")
for extra in content:
print("Adding additional build requirement: %s." % extra)
buildreq.add_buildreq(extra)
content = read_conf_file("pkgconfig_add")
for extra in content:
extra = "pkgconfig(%s)" % extra
print("Adding additional build requirement: %s." % extra)
buildreq.add_buildreq(extra)
content = read_conf_file("excludes")
for exclude in content:
print("%%exclude for: %s." % exclude)
files.excludes += content
content = read_conf_file("extras")
for extra in content:
print("extras for: %s." % extra)
files.extras += content
content = read_conf_file("setuid")
for suid in content:
print("setuid for: %s." % suid)
files.setuid += content
content = read_conf_file("attrs")
for line in content:
attr = re.split("\(|\)|,", line)
attr = [a.strip() for a in attr]
filename = attr.pop()
print("attr for: %s." % filename)
files.attrs[filename] = attr
patches.patches += read_conf_file("series")
pfiles = [("%s/%s" % (path, x.split(" ")[0])) for x in patches.patches]
cmd = 'egrep "(\+\+\+|\-\-\-).*((Makefile.am)|(configure.ac|configure.in))" %s' % " ".join(pfiles)
if len(patches.patches) > 0 and call(cmd, check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) == 0:
patches.autoreconf = True
content = read_conf_file("configure")
extra_configure = " \\\n".join(content)
content = read_conf_file("configure32")
extra_configure32 = " \\\n".join(content)
if config_opts["keepstatic"]:
buildpattern.disable_static = ""
if config_opts["broken_parallel_build"]:
parallel_build = ""
content = read_conf_file("make_args")
if content and content[0]:
buildpattern.extra_make = content[0]
content = read_conf_file("make_install_args")
if content and content[0]:
buildpattern.extra_make_install = content[0]
content = read_conf_file("install_macro")
if content and content[0]:
buildpattern.install_macro = content[0]
content = read_conf_file("cmake_args")
if content and content[0]:
buildpattern.extra_cmake = content[0]
content = read_conf_file("subdir")
if content and content[0]:
buildpattern.subdir = content[0]
content = read_conf_file("build_pattern")
if content and content[0]:
buildpattern.set_build_pattern(content[0], 20)
patches.autoreconf = False
content = read_conf_file("make_check_command")
if content and content[0]:
test.tests_config = content[0]
content = read_conf_file(tarball.name + ".license")
if content and content[0]:
words = content[0].split()
for word in words:
if word.find(":") < 0:
license.add_license(word)
content = read_conf_file("golang_libpath")
if content and content[0]:
tarball.golibpath = content[0]
print("golibpath : {}".format(tarball.golibpath))
if config_opts["use_clang"]:
config_opts["funroll-loops"] = False
buildreq.add_buildreq("llvm-dev")
if config_opts["32bit"]:
buildreq.add_buildreq("glibc-libc32")
buildreq.add_buildreq("glibc-dev32")
buildreq.add_buildreq("gcc-dev32")
buildreq.add_buildreq("gcc-libgcc32")
buildreq.add_buildreq("gcc-libstdc++32")
buildpattern.make_install_append = read_conf_file("make_install_append")
buildpattern.prep_append = read_conf_file("prep_append")
profile_payload = read_conf_file("profile_payload")
def simple_pattern(line, pattern, req):
global must_restart
pat = re.compile(pattern)
match = pat.search(line)
if match:
must_restart = must_restart + buildreq.add_buildreq(req)
def simple_pattern(line, pattern, req):
global must_restart
pat = re.compile(pattern)
match = pat.search(line)
if match:
must_restart += buildreq.add_buildreq(req)
def scan_for_tests(src_dir):
"""Scan source directory for test files and set tests_config accordingly."""
global tests_config
if config.config_opts['skip_tests'] or tests_config:
return
makeflags = "%{?_smp_mflags} " if config.parallel_build else ""
make_check = "make VERBOSE=1 V=1 {}check".format(makeflags)
cmake_check = "make test"
perl_check = "make TEST_VERBOSE=1 test"
setup_check = "PYTHONPATH=%{buildroot}/usr/lib/python3.7/site-packages python3 setup.py test"
if config.config_opts['allow_test_failures']:
make_check += " || :"
cmake_check += " || :"
perl_check += " || :"
setup_check += " || :"
testsuites = {
"makecheck":
make_check,
"perlcheck":
perl_check,
"setup.py":
setup_check,
"cmake":
"cd clr-build; " + cmake_check,
"rakefile":
"pushd %{buildroot}%{gem_dir}/gems/" + tarball.tarball_prefix +
"\nrake --trace test TESTOPTS=\"-v\"\npopd",
"rspec":
"pushd %{buildroot}%{gem_dir}/gems/" + tarball.tarball_prefix +
"\nrspec -I.:lib spec/\npopd"
}
if config.config_opts['32bit']:
testsuites["makecheck"] += "\ncd ../build32;\n" + make_check + " || :"
testsuites["cmake"] += "\ncd ../clr-build32;\n" + cmake_check + " || :"
if config.config_opts['use_avx2']:
testsuites[
"makecheck"] += "\ncd ../buildavx2;\n" + make_check + " || :"
testsuites[
"cmake"] += "\ncd ../clr-build-avx2;\n" + cmake_check + " || :"
if config.config_opts['use_avx512']:
testsuites[
"makecheck"] += "\ncd ../buildavx512;\n" + make_check + " || :"
testsuites[
"cmake"] += "\ncd ../clr-build-avx512;\n" + cmake_check + " || :"
files = os.listdir(src_dir)
if buildpattern.default_pattern == "cmake":
makefile_path = os.path.join(src_dir, "CMakeLists.txt")
if not os.path.isfile(makefile_path):
return
if "enable_testing" in open(makefile_path, encoding='latin-1').read():
tests_config = testsuites["cmake"]
elif buildpattern.default_pattern in [
"cpan", "configure", "configure_ac", "autogen"
] and "Makefile.in" in files:
makefile_path = os.path.join(src_dir, "Makefile.in")
if os.path.isfile(makefile_path):
with open(makefile_path, 'r', encoding="latin-1") as make_fp:
lines = make_fp.readlines()
for line in lines:
if line.startswith("check:"):
tests_config = testsuites["makecheck"]
break
if line.startswith("test:"):
tests_config = testsuites["perlcheck"]
break
elif buildpattern.default_pattern in [
"configure", "configure_ac", "autogen"
] and "Makefile.am" in files:
tests_config = testsuites["makecheck"]
elif buildpattern.default_pattern in ["cpan"] and "Makefile.PL" in files:
tests_config = testsuites["perlcheck"]
elif buildpattern.default_pattern in ["distutils3", "distutils23"
] and "setup.py" in files:
with open(os.path.join(src_dir, "setup.py"),
'r',
encoding="ascii",
errors="surrogateescape") as setup_fp:
setup_contents = setup_fp.read()
if "test_suite" in setup_contents or "pbr=True" in setup_contents:
tests_config = testsuites["setup.py"]
elif buildpattern.default_pattern == "R":
tests_config = "export _R_CHECK_FORCE_SUGGESTS_=false\n" \
"R CMD check --no-manual --no-examples --no-codoc -l " \
"%{buildroot}/usr/lib64/R/library " \
+ tarball.rawname + "|| : \ncp ~/.stash/* " \
"%{buildroot}/usr/lib64/R/library/*/libs/ || :"
if "tox.ini" in files:
buildreq.add_buildreq("tox")
buildreq.add_buildreq("pytest")
buildreq.add_buildreq("virtualenv")
buildreq.add_buildreq("pluggy")
buildreq.add_buildreq("py-python")
def download_tarball(url_argument, name_argument, archives):
global name
global rawname
global version
global url
global path
global tarball_prefix
global gcov_file
url = url_argument
tarfile = os.path.basename(url)
pattern_options = [
r"(.*?)[\-_](v*[0-9]+[alpha\+_spbfourcesigedsvstableP0-9\.\-\~]*)\.src\.(tgz|tar|zip)",
r"(.*?)[\-_](v*[0-9]+[alpha\+_sbpfourcesigedsvstableP0-9\.\-\~]*)\.(tgz|tar|zip)",
r"(.*?)[\-_](v*[0-9]+[a-zalpha\+_spbfourcesigedsvstableP0-9\.\-\~]*)\.orig\.tar",
r"(.*?)[\-_](v*[0-9]+[\+_spbfourcesigedsvstableP0-9\.\~]*)(-.*?)?\.tar",
]
for pattern in pattern_options:
p = re.compile(pattern)
m = p.search(tarfile)
if m:
name = m.group(1).strip()
version = m.group(2).strip()
b = version.find("-")
if b >= 0:
version = version[:b]
break
rawname = name
# R package
if url_argument.find("cran.r-project.org") > 0 or url_argument.find("cran.rstudio.com") > 0:
buildpattern.set_build_pattern("R", 10)
files.want_dev_split = 0
buildreq.add_buildreq("clr-R-helpers")
p = re.compile(r"([A-Za-z0-9]+)_(v*[0-9]+[\+_spbfourcesigedsvstableP0-9\.\~\-]*)\.tar\.gz")
m = p.search(tarfile)
if m:
name = "R-" + m.group(1).strip()
rawname = m.group(1).strip()
version = m.group(2).strip()
b = version.find("-")
if b >= 0:
version = version[:b]
if url_argument.find("pypi.python.org") > 0:
buildpattern.set_build_pattern("distutils", 10)
if url_argument.find(".cpan.org/CPAN/") > 0:
buildpattern.set_build_pattern("cpan", 10)
if name:
name = "perl-" + name
if url_argument.find(".metacpan.org/") > 0:
buildpattern.set_build_pattern("cpan", 10)
if name:
name = "perl-" + name
if url_argument.find("github.com") > 0:
p = re.compile(r"https://github.com/.*/(.*?)/archive/v?(.*).tar")
m = p.search(url_argument)
if m:
name = m.group(1).strip()
version = m.group(2).strip()
if url_argument.find("bitbucket.org") > 0:
p = re.compile(r"https://bitbucket.org/.*/(.*?)/get/[a-zA-Z_-]*([0-9][0-9_.]*).tar")
m = p.search(url_argument)
if m:
name = m.group(1).strip()
version = m.group(2).strip().replace('_', '.')
# ruby
if url_argument.find("rubygems.org/") > 0:
buildpattern.set_build_pattern("ruby", 10)
p = re.compile(r"(.*?)[\-_](v*[0-9]+[alpha\+_spbfourcesigedsvstableP0-9\.\-\~]*)\.gem")
m = p.search(tarfile)
if m:
buildreq.add_buildreq("ruby")
buildreq.add_buildreq("rubygem-rdoc")
name = "rubygem-" + m.group(1).strip()
rawname = m.group(1).strip()
version = m.group(2).strip()
b = version.find("-")
if b >= 0:
version = version[:b]
# override from commandline
if name_argument and name_argument[0] != name:
pattern = name_argument[0] + r"[\-]*(.*)\.(tgz|tar|zip)"
p = re.compile(pattern)
m = p.search(tarfile)
if m:
name = name_argument[0]
rawname = name
version = m.group(1).strip()
b = version.find("-")
if b >= 0:
version = version[:b]
if version.startswith('.'):
version = version[1:]
else:
name = name_argument[0]
if not name:
split = url_argument.split('/')
if len(split) > 3 and split[-2] in ('archive', 'tarball'):
name = split[-3]
version = split[-1]
if version.startswith('v'):
version = version[1:]
# remove extension
version = '.'.join(version.split('.')[:-1])
if version.endswith('.tar'):
version = '.'.join(version.split('.')[:-1])
b = version.find("-")
if b >= 0:
b = b + 1
version = version[b:]
if version[0] in ['v', 'r']:
version = version[1:]
assert name != ""
build.download_path = os.getcwd() + "/" + name
call("mkdir -p %s" % build.download_path)
gcov_path = build.download_path + "/" + name + ".gcov"
if os.path.isfile(gcov_path):
gcov_file = name + ".gcov"
tarball_path = check_or_get_file(url, tarfile)
sha1 = get_sha1sum(tarball_path)
with open(build.download_path + "/upstream", "w") as file:
file.write(sha1 + "/" + tarfile + "\n")
tarball_prefix = name + "-" + version
if tarfile.lower().endswith('.zip'):
tarball_contents = subprocess.check_output(
["unzip", "-l", tarball_path], universal_newlines=True)
if tarball_contents and len(tarball_contents.splitlines()) > 3:
tarball_prefix = tarball_contents.splitlines()[3].rsplit("/")[0].split()[-1]
extract_cmd = "unzip -d {0} {1}".format(build.base_path, tarball_path)
elif tarfile.lower().endswith('.gem'):
tarball_contents = subprocess.check_output(
["gem", "unpack", "--verbose", tarball_path], universal_newlines=True)
extract_cmd = "gem unpack --target={0} {1}".format(build.base_path, tarball_path)
if tarball_contents:
tarball_prefix = tarball_contents.splitlines()[-1].rsplit("/")[-1]
if tarball_prefix.endswith("'"):
tarball_prefix = tarball_prefix[:-1]
else:
extract_cmd, tarball_prefix = build_untar(tarball_path)
print("\n")
print("Processing", url_argument)
print(
"=============================================================================================")
print("Name :", name)
print("Version :", version)
print("Prefix :", tarball_prefix)
with open(build.download_path + "/Makefile", "w") as file:
file.write("PKG_NAME := " + name + "\n")
file.write("URL := " + url_argument + "\n")
file.write("ARCHIVES :=")
for archive in archives:
file.write(" {}".format(archive))
file.write("\n")
file.write("\n")
file.write("include ../common/Makefile.common\n")
shutil.rmtree("{}".format(build.base_path), ignore_errors=True)
os.makedirs("{}".format(build.output_path))
call("mkdir -p %s" % build.download_path)
call(extract_cmd)
path = build.base_path + tarball_prefix
for archive, destination in zip(archives[::2], archives[1::2]):
source_tarball_path = check_or_get_file(archive, os.path.basename(archive))
if source_tarball_path.lower().endswith('.zip'):
tarball_contents = subprocess.check_output(
["unzip", "-l", source_tarball_path], universal_newlines=True)
if tarball_contents and len(tarball_contents.splitlines()) > 3:
source_tarball_prefix = tarball_contents.splitlines()[3].rsplit("/")[0].split()[-1]
extract_cmd = "unzip -d {0} {1}".format(build.base_path, source_tarball_path)
else:
extract_cmd, source_tarball_prefix = build_untar(source_tarball_path)
buildpattern.archive_details[archive + "prefix"] = source_tarball_prefix
call(extract_cmd)
tar_files = glob.glob("{0}{1}/*".format(build.base_path, source_tarball_prefix))
move_cmd = "mv "
for tar_file in tar_files:
move_cmd += tar_file + " "
move_cmd += '{0}/{1}'.format(path, destination)
mkdir_cmd = "mkdir -p "
mkdir_cmd += '{0}/{1}'.format(path, destination)
print("mkdir " + mkdir_cmd)
call(mkdir_cmd)
call(move_cmd)
sha1 = get_sha1sum(source_tarball_path)
with open(build.download_path + "/upstream", "a") as file:
file.write(sha1 + "/" + os.path.basename(archive) + "\n")
def scan_for_tests(dir):
global tests_config
global skip_tests
global allow_test_failures
if skip_tests:
return
if len(tests_config) > 0:
return
testsuites = {
"makecheck": "make VERBOSE=1 V=1 %{?_smp_mflags} check",
"perlcheck": "make TEST_VERBOSE=1 test",
"setup.py": "PYTHONPATH=%{buildroot}/usr/lib/python2.7/site-packages python2 setup.py test",
"cmake": "pushd clr-build ; make test ; popd",
"rakefile": "pushd %{buildroot}%{gem_dir}/gems/" + tarball.tarball_prefix + "\nrake --trace test TESTOPTS=\"-v\"\npopd",
# "rubygems": "pushd %{buildroot}%{gem_dir}/gems/" + tarball.tarball_prefix + "\nruby -I\"lib:test*\" test*/*_test.rb \nruby -I\"lib:test*\" test*/test_*.rb\npopd",
"rspec": "pushd %{buildroot}%{gem_dir}/gems/" + tarball.tarball_prefix + "\nrspec -I.:lib spec/\npopd"
}
files = os.listdir(dir)
if "CMakeLists.txt" in files:
makefile_path = os.path.join(dir, "CMakeLists.txt")
if not os.path.isfile(makefile_path):
return
with open(makefile_path, encoding="latin-1") as fp:
lines = fp.readlines()
for line in lines:
if line.find("enable_testing") >= 0:
tests_config = testsuites["cmake"]
break
if "Makefile.in" in files:
makefile_path = os.path.join(dir, "Makefile.in")
if not os.path.isfile(makefile_path):
return
with open(makefile_path) as fp:
lines = fp.readlines()
for line in lines:
if line.startswith("check:"):
tests_config = testsuites["makecheck"]
break
if line.startswith("test:"):
tests_config = testsuites["perlcheck"]
break
elif "Makefile.am" in files:
tests_config = testsuites["makecheck"]
elif tarball.name.startswith("rubygem"):
if "test" in files or "tests" in files:
r_testdir = [f for f in files if "test" in f if "." not in f].pop()
pre_test = glob.glob(os.path.join(dir, "test*/test_*.rb"))
post_test = glob.glob(os.path.join(dir, "test*/*_test.rb"))
tests_config = "pushd %{buildroot}%{gem_dir}/gems/" + tarball.tarball_prefix
if len(pre_test) > 0:
tests_config += "\nruby -v -I.:lib:" + r_testdir + " test*/test_*.rb"
if len(post_test) > 0:
tests_config += "\nruby -v -I.:lib:" + r_testdir + " test*/*_test.rb"
tests_config += "\npopd"
elif "spec" in files:
buildreq.add_buildreq("rubygem-rspec")
buildreq.add_buildreq("rubygem-rspec-core")
buildreq.add_buildreq("rubygem-rspec-expectations")
buildreq.add_buildreq("rubygem-rspec-support")
buildreq.add_buildreq("rubygem-rspec-mocks")
buildreq.add_buildreq("rubygem-devise")
buildreq.add_buildreq("rubygem-diff-lcs")
tests_config = testsuites["rspec"]
elif "Rakefile" in files:
with open(os.path.join(dir, "Rakefile"), encoding="ascii", errors="surrogateescape") as fp:
setup_contents = fp.read()
# if setup_contents.find("task :test") >= 0 or setup_contents.find("task 'test'") >= 0:
tests_config = testsuites["rakefile"]
buildreq.add_buildreq("ruby")
buildreq.add_buildreq("rubygem-rake")
buildreq.add_buildreq("rubygem-test-unit")
buildreq.add_buildreq("rubygem-minitest")
elif "Makefile.PL" in files:
tests_config = testsuites["perlcheck"]
elif "setup.py" in files:
with open(os.path.join(dir, "setup.py"), encoding="ascii", errors="surrogateescape") as fp:
setup_contents = fp.read()
if "test_suite" in setup_contents or "pbr=True" in setup_contents:
tests_config = testsuites["setup.py"]
elif buildpattern.default_pattern == "R":
tests_config = "export _R_CHECK_FORCE_SUGGESTS_=false\nR CMD check --no-manual --no-examples --no-codoc -l %{buildroot}/usr/lib64/R/library " + tarball.rawname
if "tox.ini" in files:
buildreq.add_buildreq("tox")
buildreq.add_buildreq("pytest")
buildreq.add_buildreq("virtualenv")
buildreq.add_buildreq("pluggy")
buildreq.add_buildreq("py-python")
if tests_config != "" and allow_test_failures:
if tarball.name.startswith("rubygem"):
config_elements = tests_config.split("\n")
config_elements.pop()
tests_config = "\n".join(config_elements) + " || :\npopd"
elif tests_config == testsuites["cmake"]:
tests_config = "pushd clr-build ; make test ||: ; popd"
else:
tests_config = tests_config + " || :"
print(tests_config)
def parse_config_files(path, bump, filemanager, version):
"""Parse the various configuration files that may exist in the package directory."""
global extra_configure
global extra_configure32
global extra_configure64
global extra_configure_avx2
global extra_configure_avx512
global config_files
global parallel_build
global license_fetch
global license_show
global git_uri
global os_packages
global urlban
global config_file
global profile_payload
global config_opts
global extra_make
global extra32_make
global extra_make_install
global extra_make32_install
global extra_cmake
global cmake_srcdir
global subdir
global install_macro
global disable_static
global prep_prepend
global build_prepend
global make_prepend
global install_prepend
global install_append
global patches
global autoreconf
global yum_conf
global custom_desc
global failed_pattern_dir
packages_file = None
# Require autospec.conf for additional features
if os.path.exists(config_file):
config = configparser.ConfigParser(interpolation=None)
config.read(config_file)
if "autospec" not in config.sections():
print("Missing autospec section..")
sys.exit(1)
git_uri = config['autospec'].get('git', None)
license_fetch = config['autospec'].get('license_fetch', None)
license_show = config['autospec'].get('license_show', None)
packages_file = config['autospec'].get('packages_file', None)
yum_conf = config['autospec'].get('yum_conf', None)
failed_pattern_dir = config['autospec'].get('failed_pattern_dir', None)
# support reading the local files relative to config_file
if packages_file and not os.path.isabs(packages_file):
packages_file = os.path.join(os.path.dirname(config_file),
packages_file)
if yum_conf and not os.path.isabs(yum_conf):
yum_conf = os.path.join(os.path.dirname(config_file), yum_conf)
if failed_pattern_dir and not os.path.isabs(failed_pattern_dir):
failed_pattern_dir = os.path.join(os.path.dirname(config_file),
failed_pattern_dir)
if not packages_file:
print(
"Warning: Set [autospec][packages_file] path to package list file for "
"requires validation")
packages_file = os.path.join(os.path.dirname(config_file),
"packages")
urlban = config['autospec'].get('urlban', None)
# Read values from options.conf (and deprecated files) and rewrite as necessary
read_config_opts(path)
if not git_uri:
print(
"Warning: Set [autospec][git] upstream template for remote git URI configuration"
)
if not license_fetch:
print(
"Warning: Set [autospec][license_fetch] uri for license fetch support"
)
if not license_show:
print(
"Warning: Set [autospec][license_show] uri for license link check support"
)
if not yum_conf:
print(
"Warning: Set [autospec][yum_conf] path to yum.conf file for whatrequires validation"
)
yum_conf = os.path.join(os.path.dirname(config_file),
"image-creator/yum.conf")
if packages_file:
os_packages = set(read_conf_file(packages_file))
else:
os_packages = set(read_conf_file("~/packages"))
wrapper = textwrap.TextWrapper()
wrapper.initial_indent = "# "
wrapper.subsequent_indent = "# "
def write_default_conf_file(name, description):
"""Write default configuration file with description to file name."""
config_files.add(name)
filename = os.path.join(path, name)
if os.path.isfile(filename):
return
write_out(filename, wrapper.fill(description) + "\n")
write_default_conf_file(
"buildreq_ban",
"This file contains build requirements that get picked up but are "
"undesirable. One entry per line, no whitespace.")
write_default_conf_file(
"pkgconfig_ban",
"This file contains pkgconfig build requirements that get picked up but"
" are undesirable. One entry per line, no whitespace.")
write_default_conf_file(
"requires_ban",
"This file contains runtime requirements that get picked up but are "
"undesirable. One entry per line, no whitespace.")
write_default_conf_file(
"buildreq_add",
"This file contains additional build requirements that did not get "
"picked up automatically. One name per line, no whitespace.")
write_default_conf_file(
"pkgconfig_add",
"This file contains additional pkgconfig build requirements that did "
"not get picked up automatically. One name per line, no whitespace.")
write_default_conf_file(
"requires_add",
"This file contains additional runtime requirements that did not get "
"picked up automatically. One name per line, no whitespace.")
write_default_conf_file(
"excludes",
"This file contains the output files that need %exclude. Full path "
"names, one per line.")
content = read_conf_file(os.path.join(path, "release"))
if content and content[0]:
r = int(content[0])
if bump:
r += 1
tarball.release = str(r)
print("Release :", tarball.release)
content = read_conf_file(os.path.join(path, "buildreq_ban"))
for banned in content:
print("Banning build requirement: %s." % banned)
buildreq.banned_buildreqs.add(banned)
content = read_conf_file(os.path.join(path, "pkgconfig_ban"))
for banned in content:
banned = "pkgconfig(%s)" % banned
print("Banning build requirement: %s." % banned)
buildreq.banned_buildreqs.add(banned)
content = read_conf_file(os.path.join(path, "requires_ban"))
for banned in content:
print("Banning runtime requirement: %s." % banned)
buildreq.banned_requires.add(banned)
content = read_conf_file(os.path.join(path, "buildreq_add"))
for extra in content:
print("Adding additional build requirement: %s." % extra)
buildreq.add_buildreq(extra)
content = read_conf_file(os.path.join(path, "buildreq_cache"))
if content and content[0] == version:
for extra in content[1:]:
print("Adding additional build (cache) requirement: %s." % extra)
buildreq.add_buildreq(extra)
content = read_conf_file(os.path.join(path, "pkgconfig_add"))
for extra in content:
extra = "pkgconfig(%s)" % extra
print("Adding additional build requirement: %s." % extra)
buildreq.add_buildreq(extra)
content = read_conf_file(os.path.join(path, "requires_add"))
for extra in content:
print("Adding additional runtime requirement: %s." % extra)
buildreq.add_requires(extra, override=True)
content = read_conf_file(os.path.join(path, "excludes"))
for exclude in content:
print("%%exclude for: %s." % exclude)
filemanager.excludes += content
content = read_conf_file(os.path.join(path, "extras"))
for extra in content:
print("extras for : %s." % extra)
filemanager.extras += content
content = read_conf_file(os.path.join(path, "dev_extras"))
for extra in content:
print("dev for : %s." % extra)
filemanager.dev_extras += content
content = read_conf_file(os.path.join(path, "setuid"))
for suid in content:
print("setuid for : %s." % suid)
filemanager.setuid += content
content = read_conf_file(os.path.join(path, "attrs"))
for line in content:
attr = re.split(r'\(|\)|,', line)
attr = [a.strip() for a in attr]
filename = attr.pop()
print("attr for: %s." % filename)
filemanager.attrs[filename] = attr
patches += read_conf_file(os.path.join(path, "series"))
pfiles = [("%s/%s" % (path, x.split(" ")[0])) for x in patches]
cmd = "egrep \"(\+\+\+|\-\-\-).*((Makefile.am)|(configure.ac|configure.in))\" %s" % " ".join(
pfiles) # noqa: W605
if patches and call(
cmd, check=False, stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL) == 0:
autoreconf = True
if any(p.lower().startswith('cve-') for p in patches):
config_opts['security_sensitive'] = True
rewrite_config_opts(path)
content = read_conf_file(os.path.join(path, "configure"))
extra_configure = " \\\n".join(content)
content = read_conf_file(os.path.join(path, "configure32"))
extra_configure32 = " \\\n".join(content)
content = read_conf_file(os.path.join(path, "configure64"))
extra_configure64 = " \\\n".join(content)
content = read_conf_file(os.path.join(path, "configure_avx2"))
extra_configure_avx2 = " \\\n".join(content)
content = read_conf_file(os.path.join(path, "configure_avx512"))
extra_configure_avx512 = " \\\n".join(content)
if config_opts["keepstatic"]:
disable_static = ""
if config_opts['broken_parallel_build']:
parallel_build = ""
content = read_conf_file(os.path.join(path, "make_args"))
if content:
extra_make = " \\\n".join(content)
content = read_conf_file(os.path.join(path, "make32_args"))
if content:
extra32_make = " \\\n".join(content)
content = read_conf_file(os.path.join(path, "make_install_args"))
if content:
extra_make_install = " \\\n".join(content)
content = read_conf_file(os.path.join(path, "make32_install_args"))
if content:
extra_make32_install = " \\\n".join(content)
content = read_conf_file(os.path.join(path, "install_macro"))
if content and content[0]:
install_macro = content[0]
content = read_conf_file(os.path.join(path, "cmake_args"))
if content:
extra_cmake = " \\\n".join(content)
content = read_conf_file(os.path.join(path, "cmake_srcdir"))
if content and content[0]:
cmake_srcdir = content[0]
content = read_conf_file(os.path.join(path, "subdir"))
if content and content[0]:
subdir = content[0]
content = read_conf_file(os.path.join(path, "build_pattern"))
if content and content[0]:
buildpattern.set_build_pattern(content[0], 20)
autoreconf = False
content = read_conf_file(os.path.join(path, "make_check_command"))
if content:
check.tests_config = '\n'.join(content)
content = read_conf_file(os.path.join(path, tarball.name + ".license"))
if content and content[0]:
words = content[0].split()
for word in words:
if word.find(":") < 0:
license.add_license(word)
content = read_conf_file(os.path.join(path, "golang_libpath"))
if content and content[0]:
tarball.golibpath = content[0]
print("golibpath : {}".format(tarball.golibpath))
if config_opts['use_clang']:
config_opts['funroll-loops'] = False
buildreq.add_buildreq("llvm")
if config_opts['32bit']:
buildreq.add_buildreq("glibc-libc32")
buildreq.add_buildreq("glibc-dev32")
buildreq.add_buildreq("gcc-dev32")
buildreq.add_buildreq("gcc-libgcc32")
buildreq.add_buildreq("gcc-libstdc++32")
prep_prepend = read_conf_file(os.path.join(path, "prep_prepend"))
if os.path.isfile(os.path.join(path, "prep_append")):
os.rename(os.path.join(path, "prep_append"),
os.path.join(path, "build_prepend"))
make_prepend = read_conf_file(os.path.join(path, "make_prepend"))
build_prepend = read_conf_file(os.path.join(path, "build_prepend"))
install_prepend = read_conf_file(os.path.join(path, "install_prepend"))
if os.path.isfile(os.path.join(path, "make_install_append")):
os.rename(os.path.join(path, "make_install_append"),
os.path.join(path, "install_append"))
install_append = read_conf_file(os.path.join(path, "install_append"))
profile_payload = read_conf_file(os.path.join(path, "profile_payload"))
custom_desc = read_conf_file(os.path.join(path, "description"))
def name_and_version(name_arg, version_arg, filemanager):
"""
Parse the url for the package name and version
"""
global name
global rawname
global version
global url
tarfile = os.path.basename(url)
# If both name and version overrides are set via commandline, set the name
# and version variables to the overrides and bail. If only one override is
# set, continue to auto detect both name and version since the URL parsing
# handles both. In this case, wait until the end to perform the override of
# the one that was set.
if name_arg and version_arg:
# rawname == name in this case
name = name_arg
rawname = name
version = version_arg
return
# it is important for the more specific patterns to come first
pattern_options = [
r"(.*?)[\-_](v*[0-9]+[a-zalpha\+_spbfourcesigedsvstableP0-9\.\-\~]*)\.orig\.tar",
r"(.*?)[\-_](v*[0-9]+[alpha\+_spbfourcesigedsvstableP0-9\.\-\~]*)\.src\.(tgz|tar|zip)",
r"(.*?)[\-_](v*[0-9]+[alpha\+_sbpfourcesigedsvstableP0-9\.\-\~]*)\.(tgz|tar|zip)",
r"(.*?)[\-_](v*[0-9]+[\+_spbfourcesigedsvstableP0-9\.\~]*)(-.*?)?\.tar",
]
for pattern in pattern_options:
p = re.compile(pattern)
m = p.search(tarfile)
if m:
name = m.group(1).strip()
version = convert_version(m.group(2))
break
rawname = name
# R package
if url.find("cran.r-project.org") > 0 or url.find("cran.rstudio.com") > 0:
buildpattern.set_build_pattern("R", 10)
filemanager.want_dev_split = False
buildreq.add_buildreq("clr-R-helpers")
p = re.compile(
r"([A-Za-z0-9.]+)_(v*[0-9]+[\+_spbfourcesigedsvstableP0-9\.\~\-]*)\.tar\.gz"
)
m = p.search(tarfile)
if m:
name = "R-" + m.group(1).strip()
rawname = m.group(1).strip()
version = m.group(2).strip().replace('-', '.')
if url.find("pypi.python.org") > 0:
buildpattern.set_build_pattern("distutils23", 10)
url = "http://pypi.debian.net/" + name + "/" + tarfile
buildreq.add_buildreq("python3-dev")
buildreq.add_buildreq("python-dev")
if url.find("pypi.debian.net") > 0:
buildpattern.set_build_pattern("distutils23", 10)
buildreq.add_buildreq("python3-dev")
buildreq.add_buildreq("python-dev")
if url.find(".cpan.org/CPAN/") > 0:
buildpattern.set_build_pattern("cpan", 10)
if name:
name = "perl-" + name
if url.find(".metacpan.org/") > 0:
buildpattern.set_build_pattern("cpan", 10)
if name:
name = "perl-" + name
if "github.com" in url:
# define regex accepted for valid packages, important for specific
# patterns to come before general ones
github_patterns = [
r"https?://github.com/.*/(.*?)/archive/(.*)-final.tar",
r"https?://github.com/.*/.*/archive/[0-9a-fA-F]{1,40}\/(.*)\-(.*).tar",
r"https?://github.com/.*/(.*?)/archive/v?(.*).orig.tar",
r"https?://github.com/.*/(.*?)/archive/(.*).zip",
r"https?://github.com/.*/(.*?)/archive/v?(.*).tar",
r"https?://github.com/.*/(.*?)/releases/download/v.*/(.*).tar"
]
for pattern in github_patterns:
p = re.compile(pattern)
m = p.search(url)
if m:
name = m.group(1).strip()
rawname = name
version = convert_version(m.group(2))
break
if url.find("bitbucket.org") > 0:
p = re.compile(
r"https?://bitbucket.org/.*/(.*?)/.*/([.0-9a-zA-Z_-]*?).tar")
m = p.search(url)
if m:
name = m.group(1).strip()
version = convert_version(m.group(2))
else:
version = "1"
# ruby
if url.find("rubygems.org/") > 0:
buildpattern.set_build_pattern("ruby", 10)
p = re.compile(
r"(.*?)[\-_](v*[0-9]+[alpha\+_spbfourcesigedsvstableP0-9\.\-\~]*)\.gem"
)
m = p.search(tarfile)
if m:
name = "rubygem-" + m.group(1).strip()
# remove release candidate tag
b = name.find("-rc")
if b > 0:
name = name[:b]
rawname = m.group(1).strip()
version = m.group(2).strip()
b = version.find("-")
if b >= 0:
version = version[:b]
# maven
if url.find("maven.org") > 0:
buildpattern.set_build_pattern("maven", 10)
if not name:
split = url.split('/')
if len(split) > 3 and split[-2] in ('archive', 'tarball'):
name = split[-3]
version = split[-1]
version = version.lstrip('v')
# remove extension
version = version.rsplit('.', 1)[0]
if version.endswith('.tar'):
version = version.replace('.tar', '')
# override name and version from commandline
name = name_arg if name_arg else name
version = version_arg if version_arg else version
# sanity check to make sure we aren't using an empty version
if version == "":
version = "1"
