def test_new_vs_previous_guesses_match(): """Test cases where new and deprecated functions match""" cens = "GPL (>= 3)" fam = guess_license_family(cens) assert fam == "GPL3" prev = deprecated_guess_license_family(cens) assert fam == prev, "new and deprecated guesses differ" cens = "GNU Lesser General Public License" fam = guess_license_family(cens) assert fam == "LGPL", "guess_license_family({}) is {}".format(cens, fam) prev = deprecated_guess_license_family(cens) assert fam == prev, "new and deprecated guesses differ" cens = "GNU General Public License some stuff then a 3 then stuff" fam = guess_license_family(cens) assert fam == "GPL3", "guess_license_family({}) is {}".format(cens, fam) prev = deprecated_guess_license_family(cens) assert fam == prev, "new and deprecated guesses differ" cens = "Affero GPL" fam = guess_license_family(cens) assert fam == "AGPL", "guess_license_family({}) is {}".format(cens, fam) prev = deprecated_guess_license_family(cens) assert fam == prev, "new and deprecated guesses differ"
def test_lgpl(): licenses = {'GNU Lesser General Public License (LGPL)', 'LGPL-2.1', 'LGPL-2', 'LGPL-3', 'LGPL (>= 2)', 'BSD License and GNU Library or Lesser General Public License (LGPL)'} for cens in licenses: fam = guess_license_family(cens) assert fam == 'LGPL'
def test_gpl3(): licenses = {u'GPL 3', u'GPL-3', u'GPL-3 | file LICENSE', u'GPL-2 | GPL-3 | file LICENSE', u'GPL (>= 3) | file LICENCE', u'GPL (>= 2)', u'GPL-2 | GPL-3', u'GPL (>= 2) | file LICENSE'} for cens in licenses: fam = guess_license_family(cens) assert fam == u'GPL3'
def test_old_warnings_no_longer_fail(): # the following previously threw warnings. Came from r/linux-64 warnings = { u'MIT License', u'GNU Lesser General Public License (LGPL)', u'GPL-2 | GPL-3 | file LICENSE', u'GPL (>= 3) | file LICENCE', u'BSL-1.0', u'GPL (>= 2)', u'file LICENSE (FOSS)', u'Open Source (http://www.libpng.org/pub/png/src/libpng-LICENSE.txt)', u'MIT + file LICENSE', u'GPL-2 | GPL-3', u'GPL (>= 2) | file LICENSE', u'Unlimited', u'GPL-3 | file LICENSE', u'GNU General Public License v2 or later (GPLv2+)', u'LGPL-2.1', u'LGPL-2', u'LGPL-3', u'GPL', u'zlib (http://zlib.net/zlib_license.html)', u'Free software (X11 License)', u'Custom free software license', u'Old MIT', u'GPL 3', u'Apache License (== 2.0)', u'GPL (>= 3)', None, u'LGPL (>= 2)', u'BSD_2_clause + file LICENSE', u'GPL-3', u'GPL-2', u'BSD License and GNU Library or Lesser General Public License (LGPL)', u'GPL-2 | file LICENSE', u'BSD_3_clause + file LICENSE', u'CC0', u'MIT + file LICENSE | Unlimited', u'Apache License 2.0', u'BSD License', u'Lucent Public License' } for cens in warnings: fam = guess_license_family(cens) print('{}:{}'.format(cens, fam)) assert fam in allowed_license_families
def test_unlimited(): """The following is an unfortunate case where MIT is in UNLIMITED We could add words to filter out, but it would be hard to keep track of... """ cens = u'Unlimited' assert guess_license_family(cens) == 'MIT'
def test_not_gpl2(): licenses = {'GPL (>= 2)', 'LGPL (>= 2)', 'GPL', 'LGPL-3', 'GPL 3', 'GPL (>= 3)', 'Apache License (== 2.0)'} for cens in licenses: fam = guess_license_family(cens) assert fam != 'GPL2'
def test_lgpl(): licenses = {u'GNU Lesser General Public License (LGPL)', u'LGPL-2.1', u'LGPL-2', u'LGPL-3', u'LGPL (>= 2)', u'BSD License and GNU Library or Lesser General Public License (LGPL)'} for cens in licenses: fam = guess_license_family(cens) assert fam == u'LGPL'
def test_gpl3(): licenses = {'GPL 3', 'GPL-3', 'GPL-3 | file LICENSE', 'GPL-2 | GPL-3 | file LICENSE', 'GPL (>= 3) | file LICENCE', 'GPL (>= 2)', 'GPL-2 | GPL-3', 'GPL (>= 2) | file LICENSE'} for cens in licenses: fam = guess_license_family(cens) assert fam == 'GPL3'
def test_not_gpl2(): licenses = {u'GPL (>= 2)', u'LGPL (>= 2)', u'GPL', u'LGPL-3', u'GPL 3', u'GPL (>= 3)', u'Apache License (== 2.0)'} for cens in licenses: fam = guess_license_family(cens) assert fam != u'GPL2'
def test_gpl2(): licenses = { u'GPL-2', u'GPL-2 | file LICENSE', u'GNU General Public License v2 or later (GPLv2+)' } for cens in licenses: fam = guess_license_family(cens) assert fam == u'GPL2'
def test_other(): licenses = {u'file LICENSE (FOSS)', u'Open Source (http://www.libpng.org/pub/png/src/libpng-LICENSE.txt)', u'zlib (http://zlib.net/zlib_license.html)', u'Free software (X11 License)', u'Custom free software license'} for cens in licenses: fam = guess_license_family(cens) assert fam == u'OTHER'
def test_other(): licenses = {'file LICENSE (FOSS)', 'Open Source (http://www.libpng.org/pub/png/src/libpng-LICENSE.txt)', 'zlib (http://zlib.net/zlib_license.html)', 'Free software (X11 License)', 'Custom free software license'} for cens in licenses: fam = guess_license_family(cens) assert fam == 'OTHER'
def test_new_vs_previous_guesses_match(): """Test cases where new and deprecated functions match""" cens = "GPL (>= 3)" fam = guess_license_family(cens) assert fam == 'GPL3' cens = 'GNU Lesser General Public License' fam = guess_license_family(cens) assert fam == 'LGPL', 'guess_license_family({}) is {}'.format(cens, fam) cens = 'GNU General Public License some stuff then a 3 then stuff' fam = guess_license_family(cens) assert fam == 'GPL3', 'guess_license_family({}) is {}'.format(cens, fam) cens = 'Affero GPL' fam = guess_license_family(cens) assert fam == 'AGPL', 'guess_license_family({}) is {}'.format(cens, fam)
def test_new_vs_previous_guess_differ_gpl(): """Test cases where new and deprecated functions differ license = 'GPL' New guess is GPL, which is an allowed family, hence the most accurate. Previously, GPL3 was chosen over GPL """ cens = "GPL" fam = guess_license_family(cens) assert fam == 'GPL'
def test_new_vs_previous_guess_differ_multiple_gpl(): """Test cases where new and deprecated functions differ license = 'GPL-2 | GPL-3 | file LICENSE' New guess is GPL-3, which is the most accurate. Previously, somehow Public-Domain is closer than GPL2 or GPL3! """ cens = 'GPL-2 | GPL-3 | file LICENSE' fam = guess_license_family(cens) assert fam == 'GPL3', f'guess_license_family_from_index({cens}) is {fam}'
def test_new_vs_previous_guess_differ_multiple_gpl(): """Test cases where new and deprecated functions differ license = 'GPL-2 | GPL-3 | file LICENSE' New guess is GPL-3, which is the most accurate. Previously, somehow Public-Domain is closer than GPL2 or GPL3! """ cens = u'GPL-2 | GPL-3 | file LICENSE' fam = guess_license_family(cens) assert fam == 'GPL3', 'guess_license_family_from_index({}) is {}'.format(cens, fam)
def get_package_metadata(package, metadata, data, output_dir, python_version, all_extras, recursive, created_recipes, noarch_python, no_prompt, packages, extra_specs, config, setup_options): print("Downloading %s" % package) print("PyPI URL: ", metadata['pypiurl']) pkginfo = get_pkginfo(package, filename=metadata['filename'], pypiurl=metadata['pypiurl'], digest=metadata['digest'], python_version=python_version, extra_specs=extra_specs, setup_options=setup_options, config=config) metadata.update(get_entry_points(pkginfo)) requires = get_requirements(package, pkginfo, all_extras=all_extras) if requires or is_setuptools_enabled(pkginfo): list_deps = get_dependencies(requires, is_setuptools_enabled(pkginfo)) metadata['build_depends'] = ['pip'] + list_deps # Never add setuptools to runtime dependencies. metadata['run_depends'] = list_deps if recursive: packages += get_recursive_deps(created_recipes, list_deps, output_dir) if 'packagename' not in metadata: metadata['packagename'] = pkginfo['name'].lower() if metadata['version'] == 'UNKNOWN': metadata['version'] = pkginfo['version'] metadata["import_tests"] = get_import_tests(pkginfo, metadata.get("import_tests")) metadata['tests_require'] = get_tests_require(pkginfo) metadata["home"] = get_home(pkginfo, data) if not metadata.get("summary"): metadata["summary"] = get_summary(pkginfo) metadata["summary"] = get_summary(pkginfo) license_name = get_license_name(package, pkginfo, no_prompt, data) metadata["license"] = clean_license_name(license_name) metadata['license_family'] = guess_license_family( license_name, allowed_license_families) if 'new_hash_value' in pkginfo: metadata['digest'] = pkginfo['new_hash_value']
def test_lgpl(): licenses = { u"GNU Lesser General Public License (LGPL)", u"LGPL-2.1", u"LGPL-2", u"LGPL-3", u"LGPL (>= 2)", u"BSD License and GNU Library or Lesser General Public License (LGPL)", } for cens in licenses: fam = guess_license_family(cens) assert fam == u"LGPL"
def test_other(): licenses = { u"file LICENSE (FOSS)", u"CC0", u"Open Source (http://www.libpng.org/pub/png/src/libpng-LICENSE.txt)", u"zlib (http://zlib.net/zlib_license.html)", u"Free software (X11 License)", u"Custom free software license", } for cens in licenses: fam = guess_license_family(cens) assert fam == u"OTHER"
def test_new_vs_previous_guess_differ_gpl(): """Test cases where new and deprecated functions differ license = 'GPL' New guess is GPL, which is an allowed family, hence the most accurate. Previously, GPL3 was chosen over GPL """ cens = "GPL" fam = guess_license_family(cens) assert fam == 'GPL' prev = deprecated_guess_license_family(cens) assert fam != prev, 'new and deprecated guesses are unexpectedly the same' assert prev == 'GPL3' # bizarre when GPL is an allowed license family
def test_new_vs_previous_guess_differ_gpl(): """Test cases where new and deprecated functions differ license = 'GPL' New guess is GPL, which is an allowed family, hence the most accurate. Previously, GPL3 was chosen over GPL """ cens = "GPL" fam = guess_license_family(cens) assert fam == "GPL" prev = deprecated_guess_license_family(cens) assert fam != prev, "new and deprecated guesses are unexpectedly the same" assert prev == "GPL3" # bizarre when GPL is an allowed license family
def test_new_vs_previous_guess_differ_multiple_gpl(): """Test cases where new and deprecated functions differ license = 'GPL-2 | GPL-3 | file LICENSE' New guess is GPL-3, which is the most accurate. Previously, somehow PUBLICDOMAIN is closer than GPL2 or GPL3! """ cens = u"GPL-2 | GPL-3 | file LICENSE" fam = guess_license_family(cens) assert fam == "GPL3", "guess_license_family_from_index({}) is {}".format(cens, fam) prev = deprecated_guess_license_family(cens) assert fam != prev, "new and deprecated guesses are unexpectedly the same" assert prev == "PUBLICDOMAIN"
def remap_license(rpm_license): mapping = {'lgplv2+': 'LGPL (>= 2)', 'gplv2+': 'GPL (>= 2)', 'public domain (uncopyrighted)': 'Public-Domain', 'public domain': 'Public-Domain', 'mit/x11': 'MIT', 'the open group license': 'The Open Group License'} l_rpm_license = rpm_license.lower() if l_rpm_license in mapping: license, family = mapping[l_rpm_license], guess_license_family(mapping[l_rpm_license]) else: license, family = rpm_license, guess_license_family(rpm_license) # Yuck: if family == 'APACHE': family = 'Apache' elif family == 'PUBLIC-DOMAIN': family = 'Public-Domain' elif family == 'PROPRIETARY': family = 'Proprietary' elif family == 'OTHER': family = 'Other' return license, family
def test_gpl3(): licenses = { u"GPL 3", u"GPL-3", u"GPL-3 | file LICENSE", u"GPL-2 | GPL-3 | file LICENSE", u"GPL (>= 3) | file LICENCE", u"GPL (>= 2)", u"GPL-2 | GPL-3", u"GPL (>= 2) | file LICENSE", } for cens in licenses: fam = guess_license_family(cens) assert fam == u"GPL3"
def test_new_vs_previous_guess_differ_multiple_gpl(): """Test cases where new and deprecated functions differ license = 'GPL-2 | GPL-3 | file LICENSE' New guess is GPL-3, which is the most accurate. Previously, somehow PUBLICDOMAIN is closer than GPL2 or GPL3! """ cens = u'GPL-2 | GPL-3 | file LICENSE' fam = guess_license_family(cens) assert fam == 'GPL3', 'guess_license_family_from_index({}) is {}'.format( cens, fam) prev = deprecated_guess_license_family(cens) assert fam != prev, 'new and deprecated guesses are unexpectedly the same' assert prev == 'PUBLICDOMAIN'
def test_old_warnings_no_longer_fail(): # the following previously threw warnings. Came from r/linux-64 warnings = { u"MIT License", u"GNU Lesser General Public License (LGPL)", u"GPL-2 | GPL-3 | file LICENSE", u"GPL (>= 3) | file LICENCE", u"BSL-1.0", u"GPL (>= 2)", u"file LICENSE (FOSS)", u"Open Source (http://www.libpng.org/pub/png/src/libpng-LICENSE.txt)", u"MIT + file LICENSE", u"GPL-2 | GPL-3", u"GPL (>= 2) | file LICENSE", u"Unlimited", u"GPL-3 | file LICENSE", u"GNU General Public License v2 or later (GPLv2+)", u"LGPL-2.1", u"LGPL-2", u"LGPL-3", u"GPL", u"zlib (http://zlib.net/zlib_license.html)", u"Free software (X11 License)", u"Custom free software license", u"Old MIT", u"GPL 3", u"Apache License (== 2.0)", u"GPL (>= 3)", None, u"LGPL (>= 2)", u"BSD_2_clause + file LICENSE", u"GPL-3", u"GPL-2", u"BSD License and GNU Library or Lesser General Public License (LGPL)", u"GPL-2 | file LICENSE", u"BSD_3_clause + file LICENSE", u"CC0", u"MIT + file LICENSE | Unlimited", u"Apache License 2.0", u"BSD License", u"Lucent Public License", } for cens in warnings: fam = guess_license_family(cens) print("{}:{}".format(cens, fam)) assert fam in allowed_license_families
def test_old_warnings_no_longer_fail(): # the following previously threw warnings. Came from r/linux-64 warnings = {u'MIT License', u'GNU Lesser General Public License (LGPL)', u'GPL-2 | GPL-3 | file LICENSE', u'GPL (>= 3) | file LICENCE', u'BSL-1.0', u'GPL (>= 2)', u'file LICENSE (FOSS)', u'Open Source (http://www.libpng.org/pub/png/src/libpng-LICENSE.txt)', u'MIT + file LICENSE', u'GPL-2 | GPL-3', u'GPL (>= 2) | file LICENSE', u'Unlimited', u'GPL-3 | file LICENSE', u'GNU General Public License v2 or later (GPLv2+)', u'LGPL-2.1', u'LGPL-2', u'LGPL-3', u'GPL', u'zlib (http://zlib.net/zlib_license.html)', u'Free software (X11 License)', u'Custom free software license', u'Old MIT', u'GPL 3', u'Apache License (== 2.0)', u'GPL (>= 3)', None, u'LGPL (>= 2)', u'BSD_2_clause + file LICENSE', u'GPL-3', u'GPL-2', u'BSD License and GNU Library or Lesser General Public License (LGPL)', u'GPL-2 | file LICENSE', u'BSD_3_clause + file LICENSE', u'CC0', u'MIT + file LICENSE | Unlimited', u'Apache License 2.0', u'BSD License', u'Lucent Public License'} for cens in warnings: fam = guess_license_family(cens) print('{}:{}'.format(cens, fam)) assert fam in allowed_license_families
def get_package_metadata(package, d, data, output_dir, python_version, all_extras, recursive, created_recipes, noarch_python, noprompt, packages, extra_specs, config, setup_options): print("Downloading %s" % package) print("PyPI URL: ", d['pypiurl']) pkginfo = get_pkginfo(package, filename=d['filename'], pypiurl=d['pypiurl'], digest=d['digest'], python_version=python_version, extra_specs=extra_specs, setup_options=setup_options, config=config) setuptools_run = False # Look at the entry_points and construct console_script and # gui_scripts entry_points for conda entry_points = pkginfo.get('entry_points', []) if entry_points: if isinstance(entry_points, str): # makes sure it is left-shifted newstr = "\n".join(x.strip() for x in entry_points.splitlines()) _config = configparser.ConfigParser() entry_points = {} try: _config.readfp(StringIO(newstr)) except Exception as err: print("WARNING: entry-points not understood: ", err) print("The string was", newstr) entry_points = pkginfo['entry_points'] else: setuptools_run = True for section in _config.sections(): if section in ['console_scripts', 'gui_scripts']: value = [ '%s=%s' % (option, _config.get(section, option)) for option in _config.options(section) ] entry_points[section] = value if not isinstance(entry_points, dict): print("WARNING: Could not add entry points. They were:") print(entry_points) else: cs = entry_points.get('console_scripts', []) gs = entry_points.get('gui_scripts', []) if isinstance(cs, string_types): cs = [cs] elif cs and isinstance(cs, list) and isinstance(cs[0], list): # We can have lists of lists here cs = [item for sublist in [s for s in cs] for item in sublist] if isinstance(gs, string_types): gs = [gs] elif gs and isinstance(gs, list) and isinstance(gs[0], list): gs = [item for sublist in [s for s in gs] for item in sublist] # We have *other* kinds of entry-points so we need # setuptools at run-time if set(entry_points.keys()) - {'console_scripts', 'gui_scripts'}: setuptools_run = True # TODO: Use pythonw for gui scripts entry_list = (cs + gs) if len(cs + gs) != 0: d['entry_points'] = entry_list d['test_commands'] = make_entry_tests(entry_list) requires = get_requirements(package, pkginfo, all_extras=all_extras) if requires or setuptools_run: deps = [] if setuptools_run: deps.append('setuptools') for deptext in requires: if isinstance(deptext, string_types): deptext = deptext.splitlines() # Every item may be a single requirement # or a multiline requirements string... for dep in deptext: # ... and may also contain comments... dep = dep.split('#')[0].strip() if dep: # ... and empty (or comment only) lines dep, marker = parse_dep_with_env_marker(dep) spec = spec_from_line(dep) if spec is None: sys.exit("Error: Could not parse: %s" % dep) if '~' in spec: version = spec.split()[-1] tilde_version = '~ {}'.format(version) pin_compatible = convert_version(version) spec = spec.replace(tilde_version, pin_compatible) if marker: spec = ' '.join((spec, marker)) deps.append(spec) if 'setuptools' in deps: setuptools_run = False d['build_depends'] = ['pip'] + deps # Never add setuptools to runtime dependencies. d['run_depends'] = deps if recursive: for dep in deps: dep = dep.split()[0] if not exists(join(output_dir, dep)): if dep not in created_recipes: packages.append(dep) if 'packagename' not in d: d['packagename'] = pkginfo['name'].lower() if d['version'] == 'UNKNOWN': d['version'] = pkginfo['version'] if pkginfo.get('packages'): deps = set(pkginfo['packages']) if d['import_tests']: if not d['import_tests'] or d['import_tests'] == 'PLACEHOLDER': olddeps = [] else: olddeps = [x for x in d['import_tests'].split() if x != '-'] deps = set(olddeps) | deps d['import_tests'] = sorted(deps) d['tests_require'] = sorted([ spec_from_line(pkg) for pkg in ensure_list(pkginfo['tests_require']) ]) if pkginfo.get('home'): d['home'] = pkginfo['home'] else: if data and 'home' in data: d['home'] = data['home'] else: d['home'] = "The package home page" if pkginfo.get('summary'): if 'summary' in d and not d['summary']: # Need something here, use what the package had d['summary'] = pkginfo['summary'] else: d['summary'] = "Summary of the package" license_classifier = "License :: OSI Approved :: " if pkginfo.get('classifiers'): licenses = [ classifier.split(license_classifier, 1)[1] for classifier in pkginfo['classifiers'] if classifier.startswith(license_classifier) ] elif data and 'classifiers' in data: licenses = [ classifier.split(license_classifier, 1)[1] for classifier in data['classifiers'] if classifier.startswith(license_classifier) ] else: licenses = [] if not licenses: if pkginfo.get('license'): license_name = pkginfo['license'] elif data and 'license' in data: license_name = data['license'] else: license_name = None if license_name: if noprompt: pass elif '\n' not in license_name: print('Using "%s" for the license' % license_name) else: # Some projects put the whole license text in this field print("This is the license for %s" % package) print() print(license_name) print() license_name = input("What license string should I use? ") else: if noprompt: license_name = "UNKNOWN" else: license_name = input( ("No license could be found for %s on " + "PyPI or in the source. What license should I use? ") % package) else: license_name = ' or '.join(licenses) # remove the word license from the license clean_license_name = re.subn('(.*)\s+license', r'\1', license_name, flags=re.IGNORECASE)[0] d['license'] = clean_license_name d['license_family'] = guess_license_family(license_name, allowed_license_families) if 'new_hash_value' in pkginfo: d['digest'] = pkginfo['new_hash_value']
def test_mit(): licenses = {u"MIT License", u"MIT + file LICENSE", u"Old MIT"} for cens in licenses: fam = guess_license_family(cens) assert fam == u"MIT"
def test_cc(): fam = guess_license_family(u'CC0') assert fam == u'CC'
def main(): is_github_url = False this_dir = os.getcwd() # Unpack config = Config() cran_metadata = {} # Some packages are missing on some systems. Need to mark them so they get skipped. to_be_packaged = set() with TemporaryDirectory() as merged_td: for platform, details in sources.items(): with TemporaryDirectory() as td: os.chdir(td) libdir = None # libarchive cannot handle the .exe, just skip it. Means we cannot figure out packages that are not available # for Windows. if platform == 'win_no': details['cached_as'], sha256 = cache_file( config.src_cache, details['url'], details['fn'], details['sha']) libarchive.extract_file(details['cached_as']) libdir = os.path.join(td, details['library']) library = os.listdir(libdir) print(library) details['to_be_packaged'] = set(library) - set( R_BASE_PACKAGE_NAMES) elif platform == 'linux': details['cached_as'], sha256 = cache_file( config.src_cache, details['url'], details['fn'], details['sha']) libarchive.extract_file(details['cached_as']) import glob for filename in glob.iglob('**/*.rpm', recursive=True): print(filename) libarchive.extract_file(filename) libdir = os.path.join(td, details['library']) library = os.listdir(libdir) print(library) details['to_be_packaged'] = set(library) - set( R_BASE_PACKAGE_NAMES) elif platform == 'mac': details['cached_as'], sha256 = cache_file( config.src_cache, details['url'], details['fn'], details['sha']) os.system("bsdtar -xf {}".format(details['cached_as'])) payloads = glob.glob('./**/Payload', recursive=True) print(payloads) for payload in payloads: libarchive.extract_file(payload) libdir = os.path.join(td, details['library']) library = os.listdir(libdir) print(library) details['to_be_packaged'] = set(library) - set( R_BASE_PACKAGE_NAMES) if libdir: distutils.dir_util.copy_tree(libdir, merged_td) os.chdir(merged_td) libdir = merged_td # Fudge until we can unpack the Windows installer .exe on Linux sources['win']['to_be_packaged'] = sources['linux']['to_be_packaged'] # Get the superset of all packages (note we will no longer have the DESCRIPTION?!) for platform, details in sources.items(): if 'to_be_packaged' in details: to_be_packaged.update(details['to_be_packaged']) package_dicts = {} for package in sorted(list(to_be_packaged)): p = os.path.join(libdir, package, "DESCRIPTION") with open(p) as cran_description: description_text = cran_description.read() d = dict_from_cran_lines( remove_package_line_continuations( description_text.splitlines())) d['orig_description'] = description_text package = d['Package'].lower() cran_metadata[package] = d # Make sure package always uses the CRAN capitalization package = cran_metadata[package.lower()]['Package'] package_dicts[package.lower()] = {} package_dicts[package.lower()]['osx'] = True if package in sources[ 'mac']['to_be_packaged'] else False package_dicts[package.lower()]['win'] = True if package in sources[ 'win']['to_be_packaged'] else False package_dicts[ package.lower()]['linux'] = True if package in sources[ 'linux']['to_be_packaged'] else False for package in sorted(list(to_be_packaged)): cran_package = cran_metadata[package.lower()] package_dicts[package.lower()].update({ 'cran_packagename': package, 'packagename': 'r-' + package.lower(), 'patches': '', 'build_number': 0, 'build_depends': '', 'run_depends': '', # CRAN doesn't seem to have this metadata :( 'home_comment': '#', 'homeurl': '', 'summary_comment': '#', 'summary': '', }) d = package_dicts[package.lower()] d['url_key'] = 'url:' d['fn_key'] = 'fn:' d['git_url_key'] = '' d['git_tag_key'] = '' d['git_url'] = '' d['git_tag'] = '' d['hash_entry'] = '' d['build'], d['skip'] = build_and_skip_olw[( package_dicts[package.lower()]['osx'], package_dicts[package.lower()]['linux'], package_dicts[package.lower()]['win'])] d['cran_version'] = cran_package['Version'] # Conda versions cannot have -. Conda (verlib) will treat _ as a . d['conda_version'] = d['cran_version'].replace('-', '_') patches = [] script_env = [] extra_recipe_maintainers = [] build_number = 0 if not len(patches): patches.append("# patches:\n") patches.append(" # List any patch files here\n") patches.append(" # - fix.patch") if len(extra_recipe_maintainers): extra_recipe_maintainers[1:].sort() extra_recipe_maintainers.insert(0, "extra:\n ") d['build_number'] = build_number cached_path = None d['cran_metadata'] = '\n'.join( ['# %s' % l for l in cran_package['orig_lines'] if l]) # XXX: We should maybe normalize these d['license'] = cran_package.get("License", "None") d['license_family'] = guess_license_family( d['license'], allowed_license_families) if 'License_is_FOSS' in cran_package: d['license'] += ' (FOSS)' if cran_package.get('License_restricts_use') == 'yes': d['license'] += ' (Restricts use)' if "URL" in cran_package: d['home_comment'] = '' d['homeurl'] = ' ' + yaml_quote_string(cran_package['URL']) else: # use CRAN page as homepage if nothing has been specified d['home_comment'] = '' d['homeurl'] = ' https://mran.microsoft.com/package/{}'.format( package) if 'Description' in cran_package: d['summary_comment'] = '' d['summary'] = ' ' + yaml_quote_string( cran_package['Description'], indent=6) if "Suggests" in cran_package: d['suggests'] = " # Suggests: %s" % cran_package['Suggests'] else: d['suggests'] = '' if package.lower() == 'revoutilsmath': d['always_include_files'] = " always_include_files:\n" \ " - lib/R/lib/libRblas.so # [linux]" else: d['always_include_files'] = '' # Every package depends on at least R. # I'm not sure what the difference between depends and imports is. depends = [ s.strip() for s in cran_package.get('Depends', '').split(',') if s.strip() ] imports = [ s.strip() for s in cran_package.get('Imports', '').split(',') if s.strip() ] links = [ s.strip() for s in cran_package.get("LinkingTo", '').split(',') if s.strip() ] dep_dict = {} seen = set() for s in list(chain(imports, depends, links)): match = VERSION_DEPENDENCY_REGEX.match(s) if not match: sys.exit( "Could not parse version from dependency of %s: %s" % (package, s)) name = match.group('name') if name in seen: continue seen.add(name) archs = match.group('archs') relop = match.group('relop') or '' ver = match.group('version') or '' ver = ver.replace('-', '_') # If there is a relop there should be a version assert not relop or ver if archs: sys.exit( "Don't know how to handle archs from dependency of " "package %s: %s" % (package, s)) dep_dict[name] = '{relop}{version}'.format(relop=relop, version=ver) if 'R' not in dep_dict: dep_dict['R'] = '' need_git = is_github_url if cran_package.get("NeedsCompilation", 'no') == 'yes' and False: with tarfile.open(cached_path) as tf: need_f = any([ f.name.lower().endswith(('.f', '.f90', '.f77')) for f in tf ]) # Fortran builds use CC to perform the link (they do not call the linker directly). need_c = True if need_f else \ any([f.name.lower().endswith('.c') for f in tf]) need_cxx = any([ f.name.lower().endswith( ('.cxx', '.cpp', '.cc', '.c++')) for f in tf ]) need_autotools = any( [f.name.lower().endswith('/configure') for f in tf]) need_make = True if any((need_autotools, need_f, need_cxx, need_c)) else \ any([f.name.lower().endswith(('/makefile', '/makevars')) for f in tf]) else: need_c = need_cxx = need_f = need_autotools = need_make = False for dep_type in ['build', 'run']: deps = [] # Put non-R dependencies first. if dep_type == 'build': if need_c: deps.append( "{indent}{{{{ compiler('c') }}}} # [not win]" .format(indent=INDENT)) if need_cxx: deps.append( "{indent}{{{{ compiler('cxx') }}}} # [not win]" .format(indent=INDENT)) if need_f: deps.append( "{indent}{{{{ compiler('fortran') }}}} # [not win]" .format(indent=INDENT)) if need_c or need_cxx or need_f: deps.append( "{indent}{{{{native}}}}toolchain # [win]". format(indent=INDENT)) if need_autotools or need_make or need_git: deps.append( "{indent}{{{{posix}}}}filesystem # [win]". format(indent=INDENT)) if need_git: deps.append( "{indent}{{{{posix}}}}git".format(indent=INDENT)) if need_autotools: deps.append( "{indent}{{{{posix}}}}sed # [win]". format(indent=INDENT)) deps.append( "{indent}{{{{posix}}}}grep # [win]". format(indent=INDENT)) deps.append("{indent}{{{{posix}}}}autoconf".format( indent=INDENT)) deps.append("{indent}{{{{posix}}}}automake".format( indent=INDENT)) deps.append("{indent}{{{{posix}}}}pkg-config".format( indent=INDENT)) if need_make: deps.append( "{indent}{{{{posix}}}}make".format(indent=INDENT)) elif dep_type == 'run': if need_c or need_cxx or need_f: deps.append( "{indent}{{{{native}}}}gcc-libs # [win]". format(indent=INDENT)) for name in sorted(dep_dict): if name in R_BASE_PACKAGE_NAMES: continue if name == 'R': # Put R first # Regardless of build or run, and whether this is a recommended package or not, # it can only depend on 'r-base' since anything else can and will cause cycles # in the dependency graph. The cran metadata lists all dependencies anyway, even # those packages that are in the recommended group. # r_name = 'r-base ' + VERSION # We don't include any R version restrictions because we # always build R packages against an exact R version # deps.insert(0, '{indent}{r_name}'.format(indent=INDENT, r_name=r_name)) # Bit of a hack since I added overlinking checking. r_name = 'mro-base ' + VERSION deps.insert( 0, '{indent}{r_name}'.format(indent=INDENT, r_name=r_name)) else: conda_name = 'r-' + name.lower() if dep_dict[name]: deps.append('{indent}{name} {version}'.format( name=conda_name, version=dep_dict[name], indent=INDENT)) else: deps.append('{indent}{name}'.format( name=conda_name, indent=INDENT)) d['dep_dict'] = dep_dict # We need this for (1) # Make pin_subpackage from the deps. Done separately so the above is the same as conda-build's # CRAN skeleton (so it is easy to refactor CRAN skeleton so it can be reused here later). for i, dep in enumerate(deps): groups = re.match('(\n.* - )([\w-]+) ?([>=\w0-9._]+)?', dep, re.MULTILINE) indent = groups.group(1) name = groups.group(2) pinning = groups.group(3) if pinning: if '>=' in pinning: deps[ i] = "{}{{{{ pin_subpackage('{}', min_pin='{}', max_pin=None) }}}}".format( indent, name, pinning.replace('>=', '')) else: if name == 'mro-base': # We end up with filenames with 'r34*' in them unless we specify the version here. # TODO :: Ask @msarahan about this. deps[i] = "{}{} {{{{ version }}}}".format( indent, name) else: deps[ i] = "{}{{{{ pin_subpackage('{}', min_pin='{}', max_pin='{}') }}}}".format( indent, name, pinning, pinning) else: deps[ i] = "{}{{{{ pin_subpackage('{}', min_pin='x.x.x.x.x.x', max_pin='x.x.x.x.x.x') }}}}".format( indent, name) # Add missing conda package dependencies. if dep_type == 'run': if d['packagename'] in extra_deps: for extra_dep in extra_deps[d['packagename']]: print("extra_dep is {}".format(extra_dep)) deps.append(extra_dep) print(deps) d['%s_depends' % dep_type] = ''.join(deps) template = { 'version': VERSION, 'win_url': sources['win']['url'], 'win_fn': sources['win']['fn'], 'win_sha': sources['win']['sha'], 'linux_url': sources['linux']['url'], 'linux_fn': sources['linux']['fn'], 'linux_sha': sources['linux']['sha'], 'mac_url': sources['mac']['url'], 'mac_fn': sources['mac']['fn'], 'mac_sha': sources['mac']['sha'] } with open(os.path.join(this_dir, 'meta.yaml'), 'w') as meta_yaml: meta_yaml.write(HEADER.format(**template)) meta_yaml.write(BASE_PACKAGE) for package in package_dicts: d = package_dicts[package] # Normalize the metadata values d = { k: unicodedata.normalize("NFKD", text_type(v)).encode( 'ascii', 'ignore').decode() for k, v in iteritems(d) } meta_yaml.write(PACKAGE.format(**d)) meta_yaml.write(MRO_BASICS_METAPACKAGE) meta_subs = [] for package in package_dicts: meta_subs.append(' - {}{}'.format( package_dicts[package]['packagename'], package_dicts[package]['build'])) meta_yaml.write('\n'.join(sorted(meta_subs))) with open(os.path.join(this_dir, 'build.sh'), 'w') as build_sh: build_sh.write(BUILD_SH.format(**template)) with open(os.path.join(this_dir, 'install-mro-base.sh'), 'w') as install_mro_base: install_mro_base.write(INSTALL_MRO_BASE_HEADER.format(**template)) for excluded in sorted(to_be_packaged, key=lambda s: s.lower()): install_mro_base.write('EXCLUDED_PACKAGES+=(' + excluded + ')\n') install_mro_base.write(INSTALL_MRO_BASE_FOOTER.format(**template)) with open(os.path.join(this_dir, 'install-r-package.sh'), 'w') as install_r_package: install_r_package.write(INSTALL_R_PACKAGE.format(**template))
def test_gpl2(): licenses = {u"GPL-2", u"GPL-2 | file LICENSE", u"GNU General Public License v2 or later (GPLv2+)"} for cens in licenses: fam = guess_license_family(cens) assert fam == u"GPL2"
def test_not_gpl2(): licenses = {u"GPL (>= 2)", u"LGPL (>= 2)", u"GPL", u"LGPL-3", u"GPL 3", u"GPL (>= 3)", u"Apache License (== 2.0)"} for cens in licenses: fam = guess_license_family(cens) assert fam != u"GPL2"
def skeletonize(in_packages, output_dir=".", output_suffix="", add_maintainer=None, version=None, git_tag=None, cran_url="https://cran.r-project.org", recursive=False, archive=True, version_compare=False, update_policy='', r_interp='r-base', use_binaries_ver=None, use_noarch_generic=False, use_rtools_win=False, config=None): output_dir = realpath(output_dir) if not config: config = Config() if len(in_packages) > 1 and version_compare: raise ValueError("--version-compare only works with one package at a time") if update_policy == 'error' and not in_packages: raise ValueError("At least one package must be supplied") package_dicts = {} package_list = [] cran_url = cran_url.rstrip('/') cran_metadata = get_cran_metadata(cran_url, output_dir) # r_recipes_in_output_dir = [] # recipes = listdir(output_dir) # for recipe in recipes: # if not recipe.startswith('r-') or not isdir(recipe): # continue # r_recipes_in_output_dir.append(recipe) for package in in_packages: inputs_dict = package_to_inputs_dict(output_dir, output_suffix, git_tag, package) if inputs_dict: package_dicts.update({inputs_dict['pkg-name']: {'inputs': inputs_dict}}) for package_name, package_dict in package_dicts.items(): package_list.append(package_name) while package_list: inputs = package_dicts[package_list.pop()]['inputs'] location = inputs['location'] pkg_name = inputs['pkg-name'] is_github_url = location and 'github.com' in location is_tarfile = location and isfile(location) and tarfile.is_tarfile(location) url = inputs['location'] dir_path = inputs['new-location'] print("Making/refreshing recipe for {}".format(pkg_name)) # Bodges GitHub packages into cran_metadata if is_github_url or is_tarfile: rm_rf(config.work_dir) if is_github_url: m = metadata.MetaData.fromdict({'source': {'git_url': location}}, config=config) source.git_source(m.get_section('source'), m.config.git_cache, m.config.work_dir) new_git_tag = git_tag if git_tag else get_latest_git_tag(config) p = subprocess.Popen(['git', 'checkout', new_git_tag], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=config.work_dir) stdout, stderr = p.communicate() stdout = stdout.decode('utf-8') stderr = stderr.decode('utf-8') if p.returncode: sys.exit("Error: 'git checkout %s' failed (%s).\nInvalid tag?" % (new_git_tag, stderr.strip())) if stdout: print(stdout, file=sys.stdout) if stderr: print(stderr, file=sys.stderr) else: m = metadata.MetaData.fromdict({'source': {'url': location}}, config=config) source.unpack(m.get_section('source'), m.config.work_dir, m.config.src_cache, output_dir, m.config.work_dir) DESCRIPTION = join(config.work_dir, "DESCRIPTION") if not isfile(DESCRIPTION): sub_description_pkg = join(config.work_dir, 'pkg', "DESCRIPTION") sub_description_name = join(config.work_dir, location.split('/')[-1], "DESCRIPTION") if isfile(sub_description_pkg): DESCRIPTION = sub_description_pkg elif isfile(sub_description_name): DESCRIPTION = sub_description_name else: sys.exit("%s does not appear to be a valid R package " "(no DESCRIPTION file in %s, %s)" % (location, sub_description_pkg, sub_description_name)) with open(DESCRIPTION) as f: description_text = clear_whitespace(f.read()) d = dict_from_cran_lines(remove_package_line_continuations( description_text.splitlines())) d['orig_description'] = description_text package = d['Package'].lower() cran_metadata[package] = d else: package = pkg_name if pkg_name not in cran_metadata: sys.exit("Package %s not found" % pkg_name) # Make sure package always uses the CRAN capitalization package = cran_metadata[package.lower()]['Package'] if not is_github_url and not is_tarfile: session = get_session(output_dir) cran_metadata[package.lower()].update(get_package_metadata(cran_url, package, session)) cran_package = cran_metadata[package.lower()] package_dicts[package.lower()].update( { 'cran_packagename': package, 'packagename': 'r-' + package.lower(), 'patches': '', 'build_number': 0, 'build_depends': '', 'host_depends': '', 'run_depends': '', # CRAN doesn't seem to have this metadata :( 'home_comment': '#', 'homeurl': '', 'summary_comment': '#', 'summary': '', }) d = package_dicts[package.lower()] d['binary1'] = '' d['binary2'] = '' if version: d['version'] = version raise NotImplementedError("Package versions from CRAN are not yet implemented") d['cran_version'] = cran_package['Version'] # Conda versions cannot have -. Conda (verlib) will treat _ as a . d['conda_version'] = d['cran_version'].replace('-', '_') if version_compare: sys.exit(not version_compare(dir_path, d['conda_version'])) patches = [] script_env = [] extra_recipe_maintainers = [] build_number = 0 if update_policy.startswith('merge') and inputs['old-metadata']: m = inputs['old-metadata'] patches = make_array(m, 'source/patches') script_env = make_array(m, 'build/script_env') extra_recipe_maintainers = make_array(m, 'extra/recipe-maintainers', add_maintainer) if m.version() == d['conda_version']: build_number = int(m.get_value('build/number', 0)) build_number += 1 if update_policy == 'merge-incr-build-num' else 0 if add_maintainer: new_maintainer = "{indent}{add_maintainer}".format(indent=INDENT, add_maintainer=add_maintainer) if new_maintainer not in extra_recipe_maintainers: if not len(extra_recipe_maintainers): # We hit this case when there is no existing recipe. extra_recipe_maintainers = make_array({}, 'extra/recipe-maintainers', True) extra_recipe_maintainers.append(new_maintainer) if len(extra_recipe_maintainers): extra_recipe_maintainers[1:].sort() extra_recipe_maintainers.insert(0, "extra:\n ") d['extra_recipe_maintainers'] = ''.join(extra_recipe_maintainers) d['patches'] = ''.join(patches) d['script_env'] = ''.join(script_env) d['build_number'] = build_number cached_path = None cran_layout = {'source': {'selector': '{others}', 'dir': 'src/contrib/', 'ext': '.tar.gz', # If we had platform filters we would change this to: # build_for_linux or is_github_url or is_tarfile 'use_this': True}, 'win-64': {'selector': 'win64', 'dir': 'bin/windows/contrib/{}/'.format(use_binaries_ver), 'ext': '.zip', 'use_this': True if use_binaries_ver else False}, 'osx-64': {'selector': 'osx', 'dir': 'bin/macosx/el-capitan/contrib/{}/'.format( use_binaries_ver), 'ext': '.tgz', 'use_this': True if use_binaries_ver else False}} available = {} for archive_type, archive_details in iteritems(cran_layout): contrib_url = '' if archive_details['use_this']: if is_tarfile: filename = basename(location) contrib_url = relpath(location, dir_path) contrib_url_rendered = package_url = contrib_url sha256 = hashlib.sha256() cached_path = location elif not is_github_url: filename = '{}_{}'.format(package, d['cran_version']) + archive_details['ext'] contrib_url = '{{{{ cran_mirror }}}}/{}'.format(archive_details['dir']) contrib_url_rendered = cran_url + '/{}'.format(archive_details['dir']) package_url = contrib_url_rendered + filename sha256 = hashlib.sha256() print("Downloading {} from {}".format(archive_type, package_url)) # We may need to inspect the file later to determine which compilers are needed. cached_path, _ = source.download_to_cache(config.src_cache, '', dict({'url': package_url, 'fn': archive_type + '-' + filename})) available_details = {} available_details['selector'] = archive_details['selector'] if cached_path: sha256.update(open(cached_path, 'rb').read()) available_details['filename'] = filename available_details['contrib_url'] = contrib_url available_details['contrib_url_rendered'] = contrib_url_rendered available_details['package_url'] = package_url available_details['hash_entry'] = 'sha256: {}'.format(sha256.hexdigest()) available_details['cached_path'] = cached_path # This is rubbish; d[] should be renamed global[] and should be # merged into source and binaryN. if archive_type == 'source': if is_github_url: available_details['url_key'] = '' available_details['fn_key'] = '' available_details['git_url_key'] = 'git_url:' available_details['git_tag_key'] = 'git_tag:' hash_msg = '# You can add a hash for the file here, (md5, sha1 or sha256)' available_details['hash_entry'] = hash_msg available_details['filename'] = '' available_details['cranurl'] = '' available_details['git_url'] = url available_details['git_tag'] = new_git_tag available_details['archive_keys'] = '' else: available_details['url_key'] = 'url:' available_details['fn_key'] = 'fn:' available_details['git_url_key'] = '' available_details['git_tag_key'] = '' available_details['cranurl'] = ' ' + contrib_url + filename available_details['git_url'] = '' available_details['git_tag'] = '' available_details['patches'] = d['patches'] available[archive_type] = available_details # Figure out the selectors according to what is available. _all = ['linux', 'win32', 'win64', 'osx'] from_source = _all[:] binary_id = 1 for archive_type, archive_details in iteritems(available): if archive_type != 'source': sel = archive_details['selector'] from_source.remove(sel) binary_id += 1 else: for k, v in iteritems(archive_details): d[k] = v if from_source == _all: sel_src = "" sel_src_and_win = ' # [win]' sel_src_not_win = ' # [not win]' else: sel_src = ' # [' + ' or '.join(from_source) + ']' sel_src_and_win = ' # [' + ' or '.join(fs for fs in from_source if fs.startswith('win')) + ']' sel_src_not_win = ' # [' + ' or '.join(fs for fs in from_source if not fs.startswith('win')) + ']' d['sel_src'] = sel_src d['sel_src_and_win'] = sel_src_and_win d['sel_src_not_win'] = sel_src_not_win if 'source' in available: available_details = available['source'] available_details['sel'] = sel_src filename = available_details['filename'] if 'contrib_url' in available_details: contrib_url = available_details['contrib_url'] if archive: if is_tarfile: available_details['cranurl'] = (INDENT + contrib_url) else: available_details['cranurl'] = (INDENT + contrib_url + filename + sel_src + INDENT + contrib_url + 'Archive/{}/'.format(package) + filename + sel_src) else: available_details['cranurl'] = ' ' + contrib_url + filename + sel_src if not is_github_url: available_details['archive_keys'] = '{fn_key} {filename}{sel}\n' \ ' {url_key}{sel}' \ ' {cranurl}\n' \ ' {hash_entry}{sel}'.format( **available_details) d['cran_metadata'] = '\n'.join(['# %s' % l for l in cran_package['orig_lines'] if l]) # Render the source and binaryN keys binary_id = 1 for archive_type, archive_details in iteritems(available): if archive_type == 'source': d['source'] = SOURCE_META.format(**archive_details) else: archive_details['sel'] = ' # [' + archive_details['selector'] + ']' d['binary' + str(binary_id)] = BINARY_META.format(**archive_details) binary_id += 1 # XXX: We should maybe normalize these d['license'] = cran_package.get("License", "None") d['license_family'] = guess_license_family(d['license'], allowed_license_families) if 'License_is_FOSS' in cran_package: d['license'] += ' (FOSS)' if cran_package.get('License_restricts_use') == 'yes': d['license'] += ' (Restricts use)' if "URL" in cran_package: d['home_comment'] = '' d['homeurl'] = ' ' + yaml_quote_string(cran_package['URL']) else: # use CRAN page as homepage if nothing has been specified d['home_comment'] = '' if is_github_url: d['homeurl'] = ' {}'.format(location) else: d['homeurl'] = ' https://CRAN.R-project.org/package={}'.format(package) if not use_noarch_generic or cran_package.get("NeedsCompilation", 'no') == 'yes': d['noarch_generic'] = '' else: d['noarch_generic'] = 'noarch: generic' if 'Description' in cran_package: d['summary_comment'] = '' d['summary'] = ' ' + yaml_quote_string(cran_package['Description']) if "Suggests" in cran_package: d['suggests'] = "# Suggests: %s" % cran_package['Suggests'] else: d['suggests'] = '' # Every package depends on at least R. # I'm not sure what the difference between depends and imports is. depends = [s.strip() for s in cran_package.get('Depends', '').split(',') if s.strip()] imports = [s.strip() for s in cran_package.get('Imports', '').split(',') if s.strip()] links = [s.strip() for s in cran_package.get("LinkingTo", '').split(',') if s.strip()] dep_dict = {} seen = set() for s in list(chain(imports, depends, links)): match = VERSION_DEPENDENCY_REGEX.match(s) if not match: sys.exit("Could not parse version from dependency of %s: %s" % (package, s)) name = match.group('name') if name in seen: continue seen.add(name) archs = match.group('archs') relop = match.group('relop') or '' ver = match.group('version') or '' ver = ver.replace('-', '_') # If there is a relop there should be a version assert not relop or ver if archs: sys.exit("Don't know how to handle archs from dependency of " "package %s: %s" % (package, s)) dep_dict[name] = '{relop}{version}'.format(relop=relop, version=ver) if 'R' not in dep_dict: dep_dict['R'] = '' need_git = is_github_url if cran_package.get("NeedsCompilation", 'no') == 'yes': with tarfile.open(available['source']['cached_path']) as tf: need_f = any([f.name.lower().endswith(('.f', '.f90', '.f77')) for f in tf]) # Fortran builds use CC to perform the link (they do not call the linker directly). need_c = True if need_f else \ any([f.name.lower().endswith('.c') for f in tf]) need_cxx = any([f.name.lower().endswith(('.cxx', '.cpp', '.cc', '.c++')) for f in tf]) need_autotools = any([f.name.lower().endswith('/configure') for f in tf]) need_make = True if any((need_autotools, need_f, need_cxx, need_c)) else \ any([f.name.lower().endswith(('/makefile', '/makevars')) for f in tf]) else: need_c = need_cxx = need_f = need_autotools = need_make = False if 'Rcpp' in dep_dict or 'RcppArmadillo' in dep_dict: need_cxx = True if need_cxx: need_c = True for dep_type in ['build', 'host', 'run']: deps = [] # Put non-R dependencies first. if dep_type == 'build': if need_c: deps.append("{indent}{{{{ compiler('c') }}}} {sel}".format( indent=INDENT, sel=sel_src_not_win)) if need_cxx: deps.append("{indent}{{{{ compiler('cxx') }}}} {sel}".format( indent=INDENT, sel=sel_src_not_win)) if need_f: deps.append("{indent}{{{{ compiler('fortran') }}}} {sel}".format( indent=INDENT, sel=sel_src_not_win)) if use_rtools_win: need_c = need_cxx = need_f = need_autotools = need_make = False deps.append("{indent}{{{{native}}}}rtools {sel}".format( indent=INDENT, sel=sel_src_and_win)) deps.append("{indent}{{{{native}}}}extsoft {sel}".format( indent=INDENT, sel=sel_src_and_win)) if need_c or need_cxx or need_f: deps.append("{indent}{{{{native}}}}toolchain {sel}".format( indent=INDENT, sel=sel_src_and_win)) if need_autotools or need_make or need_git: deps.append("{indent}{{{{posix}}}}filesystem {sel}".format( indent=INDENT, sel=sel_src_and_win)) if need_git: deps.append("{indent}{{{{posix}}}}git".format(indent=INDENT)) if need_autotools: deps.append("{indent}{{{{posix}}}}sed {sel}".format( indent=INDENT, sel=sel_src_and_win)) deps.append("{indent}{{{{posix}}}}grep {sel}".format( indent=INDENT, sel=sel_src_and_win)) deps.append("{indent}{{{{posix}}}}autoconf {sel}".format( indent=INDENT, sel=sel_src)) deps.append("{indent}{{{{posix}}}}automake-wrapper {sel}".format( indent=INDENT, sel=sel_src_and_win)) deps.append("{indent}{{{{posix}}}}automake {sel}".format( indent=INDENT, sel=sel_src_and_win)) deps.append("{indent}{{{{posix}}}}pkg-config".format(indent=INDENT)) if need_make: deps.append("{indent}{{{{posix}}}}make {sel}".format( indent=INDENT, sel=sel_src)) elif dep_type == 'run': if need_c or need_cxx or need_f: deps.append("{indent}{{{{native}}}}gcc-libs {sel}".format( indent=INDENT, sel=sel_src_and_win)) if dep_type == 'host' or dep_type == 'run': for name in sorted(dep_dict): if name in R_BASE_PACKAGE_NAMES: continue if name == 'R': # Put R first # Regarless of build or run, and whether this is a # recommended package or not, it can only depend on # r_interp since anything else can and will cause # cycles in the dependency graph. The cran metadata # lists all dependencies anyway, even those packages # that are in the recommended group. # We don't include any R version restrictions because # conda-build always pins r-base and mro-base version. deps.insert(0, '{indent}{r_name}'.format(indent=INDENT, r_name=r_interp)) else: conda_name = 'r-' + name.lower() if dep_dict[name]: deps.append('{indent}{name} {version}'.format(name=conda_name, version=dep_dict[name], indent=INDENT)) else: deps.append('{indent}{name}'.format(name=conda_name, indent=INDENT)) if recursive: lower_name = name.lower() if lower_name not in package_dicts: inputs_dict = package_to_inputs_dict(output_dir, output_suffix, git_tag, lower_name) assert lower_name == inputs_dict['pkg-name'], \ "name %s != inputs_dict['pkg-name'] %s" % ( name, inputs_dict['pkg-name']) assert lower_name not in package_list package_dicts.update({lower_name: {'inputs': inputs_dict}}) package_list.append(lower_name) d['%s_depends' % dep_type] = ''.join(deps) for package in package_dicts: d = package_dicts[package] dir_path = d['inputs']['new-location'] if exists(dir_path) and not version_compare: if update_policy == 'error': raise RuntimeError("directory already exists " "(and --update-policy is 'error'): %s" % dir_path) elif update_policy == 'overwrite': rm_rf(dir_path) elif update_policy == 'skip-up-to-date' and up_to_date(cran_metadata, d['inputs']['old-metadata']): continue elif update_policy == 'skip-existing' and d['inputs']['old-metadata']: continue # Normalize the metadata values d = {k: unicodedata.normalize("NFKD", text_type(v)).encode('ascii', 'ignore') .decode() for k, v in iteritems(d)} try: makedirs(join(dir_path)) except: pass print("Writing recipe for %s" % package.lower()) with open(join(dir_path, 'meta.yaml'), 'w') as f: f.write(clear_whitespace(CRAN_META.format(**d))) if not exists(join(dir_path, 'build.sh')) or update_policy == 'overwrite': with open(join(dir_path, 'build.sh'), 'w') as f: if from_source == all: f.write(CRAN_BUILD_SH_SOURCE.format(**d)) elif from_source == []: f.write(CRAN_BUILD_SH_BINARY.format(**d)) else: tpbt = [target_platform_bash_test_by_sel[t] for t in from_source] d['source_pf_bash'] = ' || '.join(['[[ $target_platform ' + s + ' ]]' for s in tpbt]) f.write(CRAN_BUILD_SH_MIXED.format(**d)) if not exists(join(dir_path, 'bld.bat')) or update_policy == 'overwrite': with open(join(dir_path, 'bld.bat'), 'w') as f: if len([fs for fs in from_source if fs.startswith('win')]) == 2: f.write(CRAN_BLD_BAT_SOURCE.format(**d)) else: f.write(CRAN_BLD_BAT_MIXED.format(**d))
def skeletonize(packages, output_dir=".", version=None, git_tag=None, cran_url="https://cran.r-project.org/", recursive=False, archive=True, version_compare=False, update_outdated=False, config=None): if not config: config = Config() if len(packages) > 1 and version_compare: raise ValueError("--version-compare only works with one package at a time") if not update_outdated and not packages: raise ValueError("At least one package must be supplied") package_dicts = {} cran_metadata = get_cran_metadata(cran_url, output_dir) if update_outdated: packages = get_outdated(output_dir, cran_metadata, packages) for pkg in packages: rm_rf(join(output_dir[0], 'r-' + pkg)) while packages: package = packages.pop() is_github_url = 'github.com' in package url = package if is_github_url: rm_rf(config.work_dir) m = metadata.MetaData.fromdict({'source': {'git_url': package}}, config=config) source.git_source(m.get_section('source'), m.config.git_cache, m.config.work_dir) git_tag = git_tag[0] if git_tag else get_latest_git_tag(config) p = subprocess.Popen(['git', 'checkout', git_tag], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=config.work_dir) stdout, stderr = p.communicate() stdout = stdout.decode('utf-8') stderr = stderr.decode('utf-8') if p.returncode: sys.exit("Error: 'git checkout %s' failed (%s).\nInvalid tag?" % (git_tag, stderr.strip())) if stdout: print(stdout, file=sys.stdout) if stderr: print(stderr, file=sys.stderr) DESCRIPTION = join(config.work_dir, "DESCRIPTION") if not isfile(DESCRIPTION): sub_description_pkg = join(config.work_dir, 'pkg', "DESCRIPTION") sub_description_name = join(config.work_dir, package.split('/')[-1], "DESCRIPTION") if isfile(sub_description_pkg): DESCRIPTION = sub_description_pkg elif isfile(sub_description_name): DESCRIPTION = sub_description_name else: sys.exit("%s does not appear to be a valid R package " "(no DESCRIPTION file in %s, %s)" % (package, sub_description_pkg, sub_description_name)) with open(DESCRIPTION) as f: description_text = clear_trailing_whitespace(f.read()) d = dict_from_cran_lines(remove_package_line_continuations( description_text.splitlines())) d['orig_description'] = description_text package = d['Package'].lower() cran_metadata[package] = d if package.startswith('r-'): package = package[2:] if package.endswith('/'): package = package[:-1] if package.lower() not in cran_metadata: sys.exit("Package %s not found" % package) # Make sure package is always uses the CRAN capitalization package = cran_metadata[package.lower()]['Package'] if not is_github_url: session = get_session(output_dir) cran_metadata[package.lower()].update(get_package_metadata(cran_url, package, session)) dir_path = join(output_dir, 'r-' + package.lower()) if exists(dir_path) and not version_compare: raise RuntimeError("directory already exists: %s" % dir_path) cran_package = cran_metadata[package.lower()] d = package_dicts.setdefault(package, { 'cran_packagename': package, 'packagename': 'r-' + package.lower(), 'build_depends': '', 'run_depends': '', # CRAN doesn't seem to have this metadata :( 'home_comment': '#', 'homeurl': '', 'summary_comment': '#', 'summary': '', }) if is_github_url: d['url_key'] = '' d['fn_key'] = '' d['git_url_key'] = 'git_url:' d['git_tag_key'] = 'git_tag:' d['hash_entry'] = '# You can add a hash for the file here, like md5, sha1 or sha256' d['filename'] = '' d['cranurl'] = '' d['git_url'] = url d['git_tag'] = git_tag else: d['url_key'] = 'url:' d['fn_key'] = 'fn:' d['git_url_key'] = '' d['git_tag_key'] = '' d['git_url'] = '' d['git_tag'] = '' d['hash_entry'] = '' if version: d['version'] = version raise NotImplementedError("Package versions from CRAN are not yet implemented") d['cran_version'] = cran_package['Version'] # Conda versions cannot have -. Conda (verlib) will treat _ as a . d['conda_version'] = d['cran_version'].replace('-', '_') if version_compare: sys.exit(not version_compare(dir_path, d['conda_version'])) if not is_github_url: filename = '{}_{}.tar.gz' contrib_url = cran_url + 'src/contrib/' package_url = contrib_url + filename.format(package, d['cran_version']) # calculate sha256 by downloading source sha256 = hashlib.sha256() print("Downloading source from {}".format(package_url)) sha256.update(urlopen(package_url).read()) d['hash_entry'] = 'sha256: {}'.format(sha256.hexdigest()) d['filename'] = filename.format(package, '{{ version }}') if archive: d['cranurl'] = (INDENT + contrib_url + d['filename'] + INDENT + contrib_url + 'Archive/{}/'.format(package) + d['filename']) else: d['cranurl'] = ' ' + cran_url + 'src/contrib/' + d['filename'] d['cran_metadata'] = '\n'.join(['# %s' % l for l in cran_package['orig_lines'] if l]) # XXX: We should maybe normalize these d['license'] = cran_package.get("License", "None") d['license_family'] = guess_license_family(d['license'], allowed_license_families) if 'License_is_FOSS' in cran_package: d['license'] += ' (FOSS)' if cran_package.get('License_restricts_use') == 'yes': d['license'] += ' (Restricts use)' if "URL" in cran_package: d['home_comment'] = '' d['homeurl'] = ' ' + yaml_quote_string(cran_package['URL']) else: # use CRAN page as homepage if nothing has been specified d['home_comment'] = '' d['homeurl'] = ' https://CRAN.R-project.org/package={}'.format(package) if 'Description' in cran_package: d['summary_comment'] = '' d['summary'] = ' ' + yaml_quote_string(cran_package['Description']) if "Suggests" in cran_package: d['suggests'] = "# Suggests: %s" % cran_package['Suggests'] else: d['suggests'] = '' # Every package depends on at least R. # I'm not sure what the difference between depends and imports is. depends = [s.strip() for s in cran_package.get('Depends', '').split(',') if s.strip()] imports = [s.strip() for s in cran_package.get('Imports', '').split(',') if s.strip()] links = [s.strip() for s in cran_package.get("LinkingTo", '').split(',') if s.strip()] dep_dict = {} for s in set(chain(depends, imports, links)): match = VERSION_DEPENDENCY_REGEX.match(s) if not match: sys.exit("Could not parse version from dependency of %s: %s" % (package, s)) name = match.group('name') archs = match.group('archs') relop = match.group('relop') or '' version = match.group('version') or '' version = version.replace('-', '_') # If there is a relop there should be a version assert not relop or version if archs: sys.exit("Don't know how to handle archs from dependency of " "package %s: %s" % (package, s)) dep_dict[name] = '{relop}{version}'.format(relop=relop, version=version) if 'R' not in dep_dict: dep_dict['R'] = '' for dep_type in ['build', 'run']: deps = [] for name in sorted(dep_dict): if name in R_BASE_PACKAGE_NAMES: continue if name == 'R': # Put R first # Regarless of build or run, and whether this is a recommended package or not, # it can only depend on 'r-base' since anything else can and will cause cycles # in the dependency graph. The cran metadata lists all dependencies anyway, even # those packages that are in the recommended group. r_name = 'r-base' # We don't include any R version restrictions because we # always build R packages against an exact R version deps.insert(0, '{indent}{r_name}'.format(indent=INDENT, r_name=r_name)) else: conda_name = 'r-' + name.lower() if dep_dict[name]: deps.append('{indent}{name} {version}'.format(name=conda_name, version=dep_dict[name], indent=INDENT)) else: deps.append('{indent}{name}'.format(name=conda_name, indent=INDENT)) if recursive: if not exists(join(output_dir, conda_name)): packages.append(name) if cran_package.get("NeedsCompilation", 'no') == 'yes': if dep_type == 'build': deps.append('{indent}posix # [win]'.format(indent=INDENT)) deps.append('{indent}{{{{native}}}}toolchain # [win]'.format(indent=INDENT)) deps.append('{indent}gcc # [not win]'.format(indent=INDENT)) elif dep_type == 'run': deps.append('{indent}{{{{native}}}}gcc-libs # [win]'.format(indent=INDENT)) deps.append('{indent}libgcc # [not win]'.format(indent=INDENT)) d['%s_depends' % dep_type] = ''.join(deps) for package in package_dicts: d = package_dicts[package] name = d['packagename'] # Normalize the metadata values d = {k: unicodedata.normalize("NFKD", text_type(v)).encode('ascii', 'ignore') .decode() for k, v in iteritems(d)} makedirs(join(output_dir, name)) print("Writing recipe for %s" % package.lower()) with open(join(output_dir, name, 'meta.yaml'), 'w') as f: f.write(clear_trailing_whitespace(CRAN_META.format(**d))) with open(join(output_dir, name, 'build.sh'), 'w') as f: f.write(CRAN_BUILD_SH.format(**d)) with open(join(output_dir, name, 'bld.bat'), 'w') as f: f.write(CRAN_BLD_BAT.format(**d)) print("Done")
def skeletonize(in_packages, output_dir=".", output_suffix="", add_maintainer=None, version=None, git_tag=None, cran_url=None, recursive=False, archive=True, version_compare=False, update_policy='', r_interp='r-base', use_binaries_ver=None, use_noarch_generic=False, use_rtools_win=False, config=None, variant_config_files=None): output_dir = realpath(output_dir) config = get_or_merge_config(config, variant_config_files=variant_config_files) if not cran_url: with TemporaryDirectory() as t: _variant = get_package_variants(t, config)[0] cran_url = ensure_list(_variant.get('cran_mirror', DEFAULT_VARIANTS['cran_mirror']))[0] if len(in_packages) > 1 and version_compare: raise ValueError("--version-compare only works with one package at a time") if update_policy == 'error' and not in_packages: raise ValueError("At least one package must be supplied") package_dicts = {} package_list = [] cran_url = cran_url.rstrip('/') cran_metadata = get_cran_metadata(cran_url, output_dir) # r_recipes_in_output_dir = [] # recipes = listdir(output_dir) # for recipe in recipes: # if not recipe.startswith('r-') or not isdir(recipe): # continue # r_recipes_in_output_dir.append(recipe) for package in in_packages: inputs_dict = package_to_inputs_dict(output_dir, output_suffix, git_tag, package) if inputs_dict: package_dicts.update({inputs_dict['pkg-name']: {'inputs': inputs_dict}}) for package_name, package_dict in package_dicts.items(): package_list.append(package_name) while package_list: inputs = package_dicts[package_list.pop()]['inputs'] location = inputs['location'] pkg_name = inputs['pkg-name'] is_github_url = location and 'github.com' in location is_tarfile = location and isfile(location) and tarfile.is_tarfile(location) url = inputs['location'] dir_path = inputs['new-location'] print("Making/refreshing recipe for {}".format(pkg_name)) # Bodges GitHub packages into cran_metadata if is_github_url or is_tarfile: rm_rf(config.work_dir) if is_github_url: m = metadata.MetaData.fromdict({'source': {'git_url': location}}, config=config) source.git_source(m.get_section('source'), m.config.git_cache, m.config.work_dir) new_git_tag = git_tag if git_tag else get_latest_git_tag(config) p = subprocess.Popen(['git', 'checkout', new_git_tag], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=config.work_dir) stdout, stderr = p.communicate() stdout = stdout.decode('utf-8') stderr = stderr.decode('utf-8') if p.returncode: sys.exit("Error: 'git checkout %s' failed (%s).\nInvalid tag?" % (new_git_tag, stderr.strip())) if stdout: print(stdout, file=sys.stdout) if stderr: print(stderr, file=sys.stderr) else: m = metadata.MetaData.fromdict({'source': {'url': location}}, config=config) source.unpack(m.get_section('source'), m.config.work_dir, m.config.src_cache, output_dir, m.config.work_dir) DESCRIPTION = join(config.work_dir, "DESCRIPTION") if not isfile(DESCRIPTION): sub_description_pkg = join(config.work_dir, 'pkg', "DESCRIPTION") sub_description_name = join(config.work_dir, location.split('/')[-1], "DESCRIPTION") if isfile(sub_description_pkg): DESCRIPTION = sub_description_pkg elif isfile(sub_description_name): DESCRIPTION = sub_description_name else: sys.exit("%s does not appear to be a valid R package " "(no DESCRIPTION file in %s, %s)" % (location, sub_description_pkg, sub_description_name)) with open(DESCRIPTION) as f: description_text = clear_whitespace(f.read()) d = dict_from_cran_lines(remove_package_line_continuations( description_text.splitlines())) d['orig_description'] = description_text package = d['Package'].lower() cran_metadata[package] = d else: package = pkg_name if pkg_name not in cran_metadata: sys.exit("Package %s not found" % pkg_name) # Make sure package always uses the CRAN capitalization package = cran_metadata[package.lower()]['Package'] if not is_github_url and not is_tarfile: session = get_session(output_dir) cran_metadata[package.lower()].update(get_package_metadata(cran_url, package, session)) cran_package = cran_metadata[package.lower()] package_dicts[package.lower()].update( { 'cran_packagename': package, 'packagename': 'r-' + package.lower(), 'patches': '', 'build_number': 0, 'build_depends': '', 'host_depends': '', 'run_depends': '', # CRAN doesn't seem to have this metadata :( 'home_comment': '#', 'homeurl': '', 'summary_comment': '#', 'summary': '', }) d = package_dicts[package.lower()] d['binary1'] = '' d['binary2'] = '' if version: d['version'] = version raise NotImplementedError("Package versions from CRAN are not yet implemented") d['cran_version'] = cran_package['Version'] # Conda versions cannot have -. Conda (verlib) will treat _ as a . d['conda_version'] = d['cran_version'].replace('-', '_') if version_compare: sys.exit(not version_compare(dir_path, d['conda_version'])) patches = [] script_env = [] extra_recipe_maintainers = [] build_number = 0 if update_policy.startswith('merge') and inputs['old-metadata']: m = inputs['old-metadata'] patches = make_array(m, 'source/patches') script_env = make_array(m, 'build/script_env') extra_recipe_maintainers = make_array(m, 'extra/recipe-maintainers', add_maintainer) if m.version() == d['conda_version']: build_number = int(m.get_value('build/number', 0)) build_number += 1 if update_policy == 'merge-incr-build-num' else 0 if add_maintainer: new_maintainer = "{indent}{add_maintainer}".format(indent=INDENT, add_maintainer=add_maintainer) if new_maintainer not in extra_recipe_maintainers: if not len(extra_recipe_maintainers): # We hit this case when there is no existing recipe. extra_recipe_maintainers = make_array({}, 'extra/recipe-maintainers', True) extra_recipe_maintainers.append(new_maintainer) if len(extra_recipe_maintainers): extra_recipe_maintainers[1:].sort() extra_recipe_maintainers.insert(0, "extra:\n ") d['extra_recipe_maintainers'] = ''.join(extra_recipe_maintainers) d['patches'] = ''.join(patches) d['script_env'] = ''.join(script_env) d['build_number'] = build_number cached_path = None cran_layout = {'source': {'selector': '{others}', 'dir': 'src/contrib/', 'ext': '.tar.gz', # If we had platform filters we would change this to: # build_for_linux or is_github_url or is_tarfile 'use_this': True}, 'win-64': {'selector': 'win64', 'dir': 'bin/windows/contrib/{}/'.format(use_binaries_ver), 'ext': '.zip', 'use_this': True if use_binaries_ver else False}, 'osx-64': {'selector': 'osx', 'dir': 'bin/macosx/el-capitan/contrib/{}/'.format( use_binaries_ver), 'ext': '.tgz', 'use_this': True if use_binaries_ver else False}} available = {} for archive_type, archive_details in iteritems(cran_layout): contrib_url = '' if archive_details['use_this']: if is_tarfile: filename = basename(location) contrib_url = relpath(location, dir_path) contrib_url_rendered = package_url = contrib_url sha256 = hashlib.sha256() cached_path = location elif not is_github_url: filename_rendered = '{}_{}{}'.format( package, d['cran_version'], archive_details['ext']) filename = '{}_{{{{ version }}}}'.format(package) + archive_details['ext'] contrib_url = '{{{{ cran_mirror }}}}/{}'.format(archive_details['dir']) contrib_url_rendered = cran_url + '/{}'.format(archive_details['dir']) package_url = contrib_url_rendered + filename_rendered sha256 = hashlib.sha256() print("Downloading {} from {}".format(archive_type, package_url)) # We may need to inspect the file later to determine which compilers are needed. cached_path, _ = source.download_to_cache( config.src_cache, '', {'url': package_url, 'fn': archive_type + '-' + filename_rendered}) available_details = {} available_details['selector'] = archive_details['selector'] if cached_path: sha256.update(open(cached_path, 'rb').read()) available_details['filename'] = filename available_details['contrib_url'] = contrib_url available_details['contrib_url_rendered'] = contrib_url_rendered available_details['cranurl'] = package_url available_details['hash_entry'] = 'sha256: {}'.format(sha256.hexdigest()) available_details['cached_path'] = cached_path # This is rubbish; d[] should be renamed global[] and should be # merged into source and binaryN. if archive_type == 'source': if is_github_url: available_details['url_key'] = '' available_details['fn_key'] = '' available_details['git_url_key'] = 'git_url:' available_details['git_tag_key'] = 'git_tag:' hash_msg = '# You can add a hash for the file here, (md5, sha1 or sha256)' available_details['hash_entry'] = hash_msg available_details['filename'] = '' available_details['cranurl'] = '' available_details['git_url'] = url available_details['git_tag'] = new_git_tag available_details['archive_keys'] = '' else: available_details['url_key'] = 'url:' available_details['fn_key'] = 'fn:' available_details['git_url_key'] = '' available_details['git_tag_key'] = '' available_details['cranurl'] = ' ' + contrib_url + filename available_details['git_url'] = '' available_details['git_tag'] = '' available_details['patches'] = d['patches'] available[archive_type] = available_details # Figure out the selectors according to what is available. _all = ['linux', 'win32', 'win64', 'osx'] from_source = _all[:] binary_id = 1 for archive_type, archive_details in iteritems(available): if archive_type != 'source': sel = archive_details['selector'] from_source.remove(sel) binary_id += 1 else: for k, v in iteritems(archive_details): d[k] = v if from_source == _all: sel_src = "" sel_src_and_win = ' # [win]' sel_src_not_win = ' # [not win]' else: sel_src = ' # [' + ' or '.join(from_source) + ']' sel_src_and_win = ' # [' + ' or '.join(fs for fs in from_source if fs.startswith('win')) + ']' sel_src_not_win = ' # [' + ' or '.join(fs for fs in from_source if not fs.startswith('win')) + ']' d['sel_src'] = sel_src d['sel_src_and_win'] = sel_src_and_win d['sel_src_not_win'] = sel_src_not_win if 'source' in available: available_details = available['source'] available_details['sel'] = sel_src filename = available_details['filename'] if 'contrib_url' in available_details: contrib_url = available_details['contrib_url'] if archive: if is_tarfile: available_details['cranurl'] = (INDENT + contrib_url) else: available_details['cranurl'] = (INDENT + contrib_url + filename + sel_src + INDENT + contrib_url + 'Archive/{}/'.format(package) + filename + sel_src) else: available_details['cranurl'] = ' ' + contrib_url + filename + sel_src if not is_github_url: available_details['archive_keys'] = '{fn_key} {filename} {sel}\n' \ ' {url_key}{sel}' \ ' {cranurl}\n' \ ' {hash_entry}{sel}'.format( **available_details) d['cran_metadata'] = '\n'.join(['# %s' % l for l in cran_package['orig_lines'] if l]) # Render the source and binaryN keys binary_id = 1 for archive_type, archive_details in iteritems(available): if archive_type == 'source': d['source'] = SOURCE_META.format(**archive_details) else: archive_details['sel'] = ' # [' + archive_details['selector'] + ']' d['binary' + str(binary_id)] = BINARY_META.format(**archive_details) binary_id += 1 # XXX: We should maybe normalize these d['license'] = cran_package.get("License", "None") d['license_family'] = guess_license_family(d['license'], allowed_license_families) if 'License_is_FOSS' in cran_package: d['license'] += ' (FOSS)' if cran_package.get('License_restricts_use') == 'yes': d['license'] += ' (Restricts use)' if "URL" in cran_package: d['home_comment'] = '' d['homeurl'] = ' ' + yaml_quote_string(cran_package['URL']) else: # use CRAN page as homepage if nothing has been specified d['home_comment'] = '' if is_github_url: d['homeurl'] = ' {}'.format(location) else: d['homeurl'] = ' https://CRAN.R-project.org/package={}'.format(package) if not use_noarch_generic or cran_package.get("NeedsCompilation", 'no') == 'yes': d['noarch_generic'] = '' else: d['noarch_generic'] = 'noarch: generic' if 'Description' in cran_package: d['summary_comment'] = '' d['summary'] = ' ' + yaml_quote_string(cran_package['Description']) if "Suggests" in cran_package: d['suggests'] = "# Suggests: %s" % cran_package['Suggests'] else: d['suggests'] = '' # Every package depends on at least R. # I'm not sure what the difference between depends and imports is. depends = [s.strip() for s in cran_package.get('Depends', '').split(',') if s.strip()] imports = [s.strip() for s in cran_package.get('Imports', '').split(',') if s.strip()] links = [s.strip() for s in cran_package.get("LinkingTo", '').split(',') if s.strip()] dep_dict = {} seen = set() for s in list(chain(imports, depends, links)): match = VERSION_DEPENDENCY_REGEX.match(s) if not match: sys.exit("Could not parse version from dependency of %s: %s" % (package, s)) name = match.group('name') if name in seen: continue seen.add(name) archs = match.group('archs') relop = match.group('relop') or '' ver = match.group('version') or '' ver = ver.replace('-', '_') # If there is a relop there should be a version assert not relop or ver if archs: sys.exit("Don't know how to handle archs from dependency of " "package %s: %s" % (package, s)) dep_dict[name] = '{relop}{version}'.format(relop=relop, version=ver) if 'R' not in dep_dict: dep_dict['R'] = '' need_git = is_github_url if cran_package.get("NeedsCompilation", 'no') == 'yes': with tarfile.open(available['source']['cached_path']) as tf: need_f = any([f.name.lower().endswith(('.f', '.f90', '.f77')) for f in tf]) # Fortran builds use CC to perform the link (they do not call the linker directly). need_c = True if need_f else \ any([f.name.lower().endswith('.c') for f in tf]) need_cxx = any([f.name.lower().endswith(('.cxx', '.cpp', '.cc', '.c++')) for f in tf]) need_autotools = any([f.name.lower().endswith('/configure') for f in tf]) need_make = True if any((need_autotools, need_f, need_cxx, need_c)) else \ any([f.name.lower().endswith(('/makefile', '/makevars')) for f in tf]) else: need_c = need_cxx = need_f = need_autotools = need_make = False if 'Rcpp' in dep_dict or 'RcppArmadillo' in dep_dict: need_cxx = True if need_cxx: need_c = True for dep_type in ['build', 'host', 'run']: deps = [] # Put non-R dependencies first. if dep_type == 'build': if need_c: deps.append("{indent}{{{{ compiler('c') }}}} {sel}".format( indent=INDENT, sel=sel_src_not_win)) if need_cxx: deps.append("{indent}{{{{ compiler('cxx') }}}} {sel}".format( indent=INDENT, sel=sel_src_not_win)) if need_f: deps.append("{indent}{{{{ compiler('fortran') }}}}{sel}".format( indent=INDENT, sel=sel_src_not_win)) if use_rtools_win: need_c = need_cxx = need_f = need_autotools = need_make = False deps.append("{indent}{{{{native}}}}rtools {sel}".format( indent=INDENT, sel=sel_src_and_win)) deps.append("{indent}{{{{native}}}}extsoft {sel}".format( indent=INDENT, sel=sel_src_and_win)) if need_c or need_cxx or need_f: deps.append("{indent}{{{{native}}}}toolchain {sel}".format( indent=INDENT, sel=sel_src_and_win)) if need_autotools or need_make or need_git: deps.append("{indent}{{{{posix}}}}filesystem {sel}".format( indent=INDENT, sel=sel_src_and_win)) if need_git: deps.append("{indent}{{{{posix}}}}git".format(indent=INDENT)) if need_autotools: deps.append("{indent}{{{{posix}}}}sed {sel}".format( indent=INDENT, sel=sel_src_and_win)) deps.append("{indent}{{{{posix}}}}grep {sel}".format( indent=INDENT, sel=sel_src_and_win)) deps.append("{indent}{{{{posix}}}}autoconf {sel}".format( indent=INDENT, sel=sel_src)) deps.append("{indent}{{{{posix}}}}automake-wrapper{sel}".format( indent=INDENT, sel=sel_src_and_win)) deps.append("{indent}{{{{posix}}}}automake {sel}".format( indent=INDENT, sel=sel_src_and_win)) deps.append("{indent}{{{{posix}}}}pkg-config".format(indent=INDENT)) if need_make: deps.append("{indent}{{{{posix}}}}make {sel}".format( indent=INDENT, sel=sel_src)) elif dep_type == 'run': if need_c or need_cxx or need_f: deps.append("{indent}{{{{native}}}}gcc-libs {sel}".format( indent=INDENT, sel=sel_src_and_win)) if dep_type == 'host' or dep_type == 'run': for name in sorted(dep_dict): if name in R_BASE_PACKAGE_NAMES: continue if name == 'R': # Put R first # Regarless of build or run, and whether this is a # recommended package or not, it can only depend on # r_interp since anything else can and will cause # cycles in the dependency graph. The cran metadata # lists all dependencies anyway, even those packages # that are in the recommended group. # We don't include any R version restrictions because # conda-build always pins r-base and mro-base version. deps.insert(0, '{indent}{r_name}'.format(indent=INDENT, r_name=r_interp)) else: conda_name = 'r-' + name.lower() if dep_dict[name]: deps.append('{indent}{name} {version}'.format(name=conda_name, version=dep_dict[name], indent=INDENT)) else: deps.append('{indent}{name}'.format(name=conda_name, indent=INDENT)) if recursive: lower_name = name.lower() if lower_name not in package_dicts: inputs_dict = package_to_inputs_dict(output_dir, output_suffix, git_tag, lower_name) assert lower_name == inputs_dict['pkg-name'], \ "name %s != inputs_dict['pkg-name'] %s" % ( name, inputs_dict['pkg-name']) assert lower_name not in package_list package_dicts.update({lower_name: {'inputs': inputs_dict}}) package_list.append(lower_name) d['%s_depends' % dep_type] = ''.join(deps) for package in package_dicts: d = package_dicts[package] dir_path = d['inputs']['new-location'] if exists(dir_path) and not version_compare: if update_policy == 'error': raise RuntimeError("directory already exists " "(and --update-policy is 'error'): %s" % dir_path) elif update_policy == 'overwrite': rm_rf(dir_path) elif update_policy == 'skip-up-to-date' and up_to_date(cran_metadata, d['inputs']['old-metadata']): continue elif update_policy == 'skip-existing' and d['inputs']['old-metadata']: continue # Normalize the metadata values d = {k: unicodedata.normalize("NFKD", text_type(v)).encode('ascii', 'ignore') .decode() for k, v in iteritems(d)} try: makedirs(join(dir_path)) except: pass print("Writing recipe for %s" % package.lower()) with open(join(dir_path, 'meta.yaml'), 'w') as f: f.write(clear_whitespace(CRAN_META.format(**d))) if not exists(join(dir_path, 'build.sh')) or update_policy == 'overwrite': with open(join(dir_path, 'build.sh'), 'w') as f: if from_source == all: f.write(CRAN_BUILD_SH_SOURCE.format(**d)) elif from_source == []: f.write(CRAN_BUILD_SH_BINARY.format(**d)) else: tpbt = [target_platform_bash_test_by_sel[t] for t in from_source] d['source_pf_bash'] = ' || '.join(['[[ $target_platform ' + s + ' ]]' for s in tpbt]) f.write(CRAN_BUILD_SH_MIXED.format(**d)) if not exists(join(dir_path, 'bld.bat')) or update_policy == 'overwrite': with open(join(dir_path, 'bld.bat'), 'w') as f: if len([fs for fs in from_source if fs.startswith('win')]) == 2: f.write(CRAN_BLD_BAT_SOURCE.format(**d)) else: f.write(CRAN_BLD_BAT_MIXED.format(**d))
def skeletonize(in_packages, output_dir=".", output_suffix="", add_maintainer=None, version=None, git_tag=None, cran_url="https://cran.r-project.org/", recursive=False, archive=True, version_compare=False, update_policy='', config=None): output_dir = realpath(output_dir) if not config: config = Config() if len(in_packages) > 1 and version_compare: raise ValueError( "--version-compare only works with one package at a time") if update_policy == 'error' and not in_packages: raise ValueError("At least one package must be supplied") package_dicts = {} package_list = [] cran_metadata = get_cran_metadata(cran_url, output_dir) # r_recipes_in_output_dir = [] # recipes = listdir(output_dir) # for recipe in recipes: # if not recipe.startswith('r-') or not isdir(recipe): # continue # r_recipes_in_output_dir.append(recipe) for package in in_packages: inputs_dict = package_to_inputs_dict(output_dir, output_suffix, git_tag, package) if inputs_dict: package_dicts.update( {inputs_dict['pkg-name']: { 'inputs': inputs_dict }}) for package_name, package_dict in package_dicts.items(): package_list.append(package_name) while package_list: inputs = package_dicts[package_list.pop()]['inputs'] location = inputs['location'] pkg_name = inputs['pkg-name'] is_github_url = location and 'github.com' in location url = inputs['location'] dir_path = inputs['new-location'] print("Making/refreshing recipe for {}".format(pkg_name)) # Bodges GitHub packages into cran_metadata if is_github_url: rm_rf(config.work_dir) m = metadata.MetaData.fromdict({'source': { 'git_url': location }}, config=config) source.git_source(m.get_section('source'), m.config.git_cache, m.config.work_dir) new_git_tag = git_tag if git_tag else get_latest_git_tag(config) p = subprocess.Popen(['git', 'checkout', new_git_tag], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=config.work_dir) stdout, stderr = p.communicate() stdout = stdout.decode('utf-8') stderr = stderr.decode('utf-8') if p.returncode: sys.exit( "Error: 'git checkout %s' failed (%s).\nInvalid tag?" % (new_git_tag, stderr.strip())) if stdout: print(stdout, file=sys.stdout) if stderr: print(stderr, file=sys.stderr) DESCRIPTION = join(config.work_dir, "DESCRIPTION") if not isfile(DESCRIPTION): sub_description_pkg = join(config.work_dir, 'pkg', "DESCRIPTION") sub_description_name = join(config.work_dir, location.split('/')[-1], "DESCRIPTION") if isfile(sub_description_pkg): DESCRIPTION = sub_description_pkg elif isfile(sub_description_name): DESCRIPTION = sub_description_name else: sys.exit( "%s does not appear to be a valid R package " "(no DESCRIPTION file in %s, %s)" % (location, sub_description_pkg, sub_description_name)) with open(DESCRIPTION) as f: description_text = clear_trailing_whitespace(f.read()) d = dict_from_cran_lines( remove_package_line_continuations( description_text.splitlines())) d['orig_description'] = description_text package = d['Package'].lower() cran_metadata[package] = d else: package = pkg_name if pkg_name not in cran_metadata: sys.exit("Package %s not found" % pkg_name) # Make sure package always uses the CRAN capitalization package = cran_metadata[package.lower()]['Package'] if not is_github_url: session = get_session(output_dir) cran_metadata[package.lower()].update( get_package_metadata(cran_url, package, session)) cran_package = cran_metadata[package.lower()] package_dicts[package.lower()].update({ 'cran_packagename': package, 'packagename': 'r-' + package.lower(), 'patches': '', 'build_number': 0, 'build_depends': '', 'run_depends': '', # CRAN doesn't seem to have this metadata :( 'home_comment': '#', 'homeurl': '', 'summary_comment': '#', 'summary': '', }) d = package_dicts[package.lower()] if is_github_url: d['url_key'] = '' d['fn_key'] = '' d['git_url_key'] = 'git_url:' d['git_tag_key'] = 'git_tag:' d['hash_entry'] = '# You can add a hash for the file here, like md5, sha1 or sha256' d['filename'] = '' d['cranurl'] = '' d['git_url'] = url d['git_tag'] = new_git_tag else: d['url_key'] = 'url:' d['fn_key'] = 'fn:' d['git_url_key'] = '' d['git_tag_key'] = '' d['git_url'] = '' d['git_tag'] = '' d['hash_entry'] = '' if version: d['version'] = version raise NotImplementedError( "Package versions from CRAN are not yet implemented") d['cran_version'] = cran_package['Version'] # Conda versions cannot have -. Conda (verlib) will treat _ as a . d['conda_version'] = d['cran_version'].replace('-', '_') if version_compare: sys.exit(not version_compare(dir_path, d['conda_version'])) patches = [] script_env = [] extra_recipe_maintainers = [] build_number = 0 if update_policy.startswith('merge') and inputs['old-metadata']: m = inputs['old-metadata'] patches = make_array(m, 'source/patches') script_env = make_array(m, 'build/script_env') extra_recipe_maintainers = make_array(m, 'extra/recipe-maintainers', add_maintainer) if m.version() == d['conda_version']: build_number = int(m.get_value('build/number', 0)) build_number += 1 if update_policy == 'merge-incr-build-num' else 0 if not len(patches): patches.append("# patches:\n") patches.append(" # List any patch files here\n") patches.append(" # - fix.patch") if add_maintainer: new_maintainer = "{indent}{add_maintainer}".format( indent=INDENT, add_maintainer=add_maintainer) if new_maintainer not in extra_recipe_maintainers: if not len(extra_recipe_maintainers): # We hit this case when there is no existing recipe. extra_recipe_maintainers = make_array( {}, 'extra/recipe-maintainers', True) extra_recipe_maintainers.append(new_maintainer) if len(extra_recipe_maintainers): extra_recipe_maintainers[1:].sort() extra_recipe_maintainers.insert(0, "extra:\n ") d['extra_recipe_maintainers'] = ''.join(extra_recipe_maintainers) d['patches'] = ''.join(patches) d['script_env'] = ''.join(script_env) d['build_number'] = build_number cached_path = None if not is_github_url: filename = '{}_{}.tar.gz' contrib_url = cran_url + 'src/contrib/' package_url = contrib_url + filename.format( package, d['cran_version']) # calculate sha256 by downloading source sha256 = hashlib.sha256() print("Downloading source from {}".format(package_url)) # We may need to inspect the file later to determine which compilers are needed. cached_path, _ = source.download_to_cache( config.src_cache, '', dict({'url': package_url})) sha256.update(open(cached_path, 'rb').read()) d['hash_entry'] = 'sha256: {}'.format(sha256.hexdigest()) d['filename'] = filename.format(package, '{{ version }}') if archive: d['cranurl'] = (INDENT + contrib_url + d['filename'] + INDENT + contrib_url + 'Archive/{}/'.format(package) + d['filename']) else: d['cranurl'] = ' ' + cran_url + 'src/contrib/' + d['filename'] d['cran_metadata'] = '\n'.join( ['# %s' % l for l in cran_package['orig_lines'] if l]) # XXX: We should maybe normalize these d['license'] = cran_package.get("License", "None") d['license_family'] = guess_license_family(d['license'], allowed_license_families) if 'License_is_FOSS' in cran_package: d['license'] += ' (FOSS)' if cran_package.get('License_restricts_use') == 'yes': d['license'] += ' (Restricts use)' if "URL" in cran_package: d['home_comment'] = '' d['homeurl'] = ' ' + yaml_quote_string(cran_package['URL']) else: # use CRAN page as homepage if nothing has been specified d['home_comment'] = '' if is_github_url: d['homeurl'] = ' {}'.format(location) else: d['homeurl'] = ' https://CRAN.R-project.org/package={}'.format( package) if 'Description' in cran_package: d['summary_comment'] = '' d['summary'] = ' ' + yaml_quote_string(cran_package['Description']) if "Suggests" in cran_package: d['suggests'] = "# Suggests: %s" % cran_package['Suggests'] else: d['suggests'] = '' # Every package depends on at least R. # I'm not sure what the difference between depends and imports is. depends = [ s.strip() for s in cran_package.get('Depends', '').split(',') if s.strip() ] imports = [ s.strip() for s in cran_package.get('Imports', '').split(',') if s.strip() ] links = [ s.strip() for s in cran_package.get("LinkingTo", '').split(',') if s.strip() ] dep_dict = {} seen = set() for s in list(chain(imports, depends, links)): match = VERSION_DEPENDENCY_REGEX.match(s) if not match: sys.exit("Could not parse version from dependency of %s: %s" % (package, s)) name = match.group('name') if name in seen: continue seen.add(name) archs = match.group('archs') relop = match.group('relop') or '' ver = match.group('version') or '' ver = ver.replace('-', '_') # If there is a relop there should be a version assert not relop or ver if archs: sys.exit("Don't know how to handle archs from dependency of " "package %s: %s" % (package, s)) dep_dict[name] = '{relop}{version}'.format(relop=relop, version=ver) if 'R' not in dep_dict: dep_dict['R'] = '' need_git = is_github_url if cran_package.get("NeedsCompilation", 'no') == 'yes': with tarfile.open(cached_path) as tf: need_f = any([ f.name.lower().endswith(('.f', '.f90', '.f77')) for f in tf ]) # Fortran builds use CC to perform the link (they do not call the linker directly). need_c = True if need_f else \ any([f.name.lower().endswith('.c') for f in tf]) need_cxx = any([ f.name.lower().endswith(('.cxx', '.cpp', '.cc', '.c++')) for f in tf ]) need_autotools = any( [f.name.lower().endswith('/configure') for f in tf]) need_make = True if any((need_autotools, need_f, need_cxx, need_c)) else \ any([f.name.lower().endswith(('/makefile', '/makevars')) for f in tf]) else: need_c = need_cxx = need_f = need_autotools = need_make = False for dep_type in ['build', 'run']: deps = [] # Put non-R dependencies first. if dep_type == 'build': if need_c: deps.append( "{indent}{{{{ compiler('c') }}}} # [not win]". format(indent=INDENT)) if need_cxx: deps.append( "{indent}{{{{ compiler('cxx') }}}} # [not win]". format(indent=INDENT)) if need_f: deps.append( "{indent}{{{{ compiler('fortran') }}}} # [not win]". format(indent=INDENT)) if need_c or need_cxx or need_f: deps.append( "{indent}{{{{native}}}}toolchain # [win]". format(indent=INDENT)) if need_autotools or need_make or need_git: deps.append( "{indent}{{{{posix}}}}filesystem # [win]". format(indent=INDENT)) if need_git: deps.append( "{indent}{{{{posix}}}}git".format(indent=INDENT)) if need_autotools: deps.append( "{indent}{{{{posix}}}}sed # [win]". format(indent=INDENT)) deps.append( "{indent}{{{{posix}}}}grep # [win]". format(indent=INDENT)) deps.append( "{indent}{{{{posix}}}}autoconf".format(indent=INDENT)) deps.append( "{indent}{{{{posix}}}}automake".format(indent=INDENT)) deps.append("{indent}{{{{posix}}}}pkg-config".format( indent=INDENT)) if need_make: deps.append( "{indent}{{{{posix}}}}make".format(indent=INDENT)) elif dep_type == 'run': if need_c or need_cxx or need_f: deps.append( "{indent}{{{{native}}}}gcc-libs # [win]". format(indent=INDENT)) for name in sorted(dep_dict): if name in R_BASE_PACKAGE_NAMES: continue if name == 'R': # Put R first # Regarless of build or run, and whether this is a recommended package or not, # it can only depend on 'r-base' since anything else can and will cause cycles # in the dependency graph. The cran metadata lists all dependencies anyway, even # those packages that are in the recommended group. r_name = 'r-base' # We don't include any R version restrictions because we # always build R packages against an exact R version deps.insert( 0, '{indent}{r_name}'.format(indent=INDENT, r_name=r_name)) else: conda_name = 'r-' + name.lower() if dep_dict[name]: deps.append('{indent}{name} {version}'.format( name=conda_name, version=dep_dict[name], indent=INDENT)) else: deps.append('{indent}{name}'.format(name=conda_name, indent=INDENT)) if recursive: lower_name = name.lower() if lower_name not in package_dicts: inputs_dict = package_to_inputs_dict( output_dir, output_suffix, git_tag, lower_name) assert lower_name == inputs_dict['pkg-name'], \ "name %s != inputs_dict['pkg-name'] %s" % (name, inputs_dict['pkg-name']) assert lower_name not in package_list package_dicts.update( {lower_name: { 'inputs': inputs_dict }}) package_list.append(lower_name) d['%s_depends' % dep_type] = ''.join(deps) for package in package_dicts: d = package_dicts[package] dir_path = d['inputs']['new-location'] if exists(dir_path) and not version_compare: if update_policy == 'error': raise RuntimeError("directory already exists " "(and --update-policy is 'error'): %s" % dir_path) elif update_policy == 'overwrite': rm_rf(dir_path) elif update_policy == 'skip-up-to-date' and up_to_date( cran_metadata, d['inputs']['old-metadata']): continue elif update_policy == 'skip-existing' and d['inputs']['old-metadata']: continue # Normalize the metadata values d = { k: unicodedata.normalize("NFKD", text_type(v)).encode('ascii', 'ignore').decode() for k, v in iteritems(d) } try: makedirs(join(dir_path)) except: pass print("Writing recipe for %s" % package.lower()) with open(join(dir_path, 'meta.yaml'), 'w') as f: f.write(clear_trailing_whitespace(CRAN_META.format(**d))) if not exists(join(dir_path, 'build.sh')) or update_policy == 'overwrite': with open(join(dir_path, 'build.sh'), 'w') as f: f.write(CRAN_BUILD_SH.format(**d)) if not exists(join(dir_path, 'bld.bat')) or update_policy == 'overwrite': with open(join(dir_path, 'bld.bat'), 'w') as f: f.write(CRAN_BLD_BAT.format(**d))
def get_package_metadata(package, d, data, output_dir, python_version, all_extras, recursive, created_recipes, noarch_python, noprompt, packages, extra_specs, config, setup_options): print("Downloading %s" % package) pkginfo = get_pkginfo(package, filename=d['filename'], pypiurl=d['pypiurl'], md5=d['md5'], python_version=python_version, extra_specs=extra_specs, setup_options=setup_options, config=config) setuptools_build = pkginfo.get('setuptools', False) setuptools_run = False # Look at the entry_points and construct console_script and # gui_scripts entry_points for conda entry_points = pkginfo.get('entry_points', []) if entry_points: if isinstance(entry_points, str): # makes sure it is left-shifted newstr = "\n".join(x.strip() for x in entry_points.splitlines()) _config = configparser.ConfigParser() entry_points = {} try: _config.readfp(StringIO(newstr)) except Exception as err: print("WARNING: entry-points not understood: ", err) print("The string was", newstr) entry_points = pkginfo['entry_points'] else: setuptools_run = True for section in _config.sections(): if section in ['console_scripts', 'gui_scripts']: value = ['%s=%s' % (option, _config.get(section, option)) for option in _config.options(section)] entry_points[section] = value if not isinstance(entry_points, dict): print("WARNING: Could not add entry points. They were:") print(entry_points) else: cs = entry_points.get('console_scripts', []) gs = entry_points.get('gui_scripts', []) if isinstance(cs, string_types): cs = [cs] if isinstance(gs, string_types): gs = [gs] # We have *other* kinds of entry-points so we need # setuptools at run-time if set(entry_points.keys()) - {'console_scripts', 'gui_scripts'}: setuptools_build = True setuptools_run = True # TODO: Use pythonw for gui scripts entry_list = (cs + gs) if len(cs + gs) != 0: d['entry_points'] = INDENT.join([''] + entry_list) d['entry_comment'] = '' d['build_comment'] = '' d['test_commands'] = INDENT.join([''] + make_entry_tests(entry_list)) requires = get_requirements(package, pkginfo, all_extras=all_extras) if requires or setuptools_build or setuptools_run: deps = [] if setuptools_run: deps.append('setuptools') for deptext in requires: if isinstance(deptext, string_types): deptext = deptext.splitlines() # Every item may be a single requirement # or a multiline requirements string... for dep in deptext: # ... and may also contain comments... dep = dep.split('#')[0].strip() if dep: # ... and empty (or comment only) lines spec = spec_from_line(dep) if spec is None: sys.exit("Error: Could not parse: %s" % dep) deps.append(spec) if 'setuptools' in deps: setuptools_build = False setuptools_run = False d['egg_comment'] = '' d['build_comment'] = '' d['build_depends'] = INDENT.join([''] + ['setuptools'] * setuptools_build + deps) d['run_depends'] = INDENT.join([''] + ['setuptools'] * setuptools_run + deps) if recursive: for dep in deps: dep = dep.split()[0] if not exists(join(output_dir, dep)): if dep not in created_recipes: packages.append(dep) if noarch_python: d['build_comment'] = '' d['noarch_python_comment'] = '' if 'packagename' not in d: d['packagename'] = pkginfo['name'].lower() if d['version'] == 'UNKNOWN': d['version'] = pkginfo['version'] if pkginfo.get('packages'): deps = set(pkginfo['packages']) if d['import_tests']: if not d['import_tests'] or d['import_tests'] == 'PLACEHOLDER': olddeps = [] else: olddeps = [x for x in d['import_tests'].split() if x != '-'] deps = set(olddeps) | deps d['import_tests'] = INDENT.join(sorted(deps)) d['import_comment'] = '' d['tests_require'] = INDENT.join(sorted([spec_from_line(pkg) for pkg in pkginfo['tests_require']])) if pkginfo.get('homeurl'): d['homeurl'] = pkginfo['homeurl'] else: if data and 'homeurl' in data: d['homeurl'] = data['homeurl'] else: d['homeurl'] = "The package home page" d['home_comment'] = '#' if pkginfo.get('summary'): d['summary'] = repr(pkginfo['summary']) else: if data: d['summary'] = repr(data['summary']) else: d['summary'] = "Summary of the package" d['summary_comment'] = '#' if d['summary'].startswith("u'") or d['summary'].startswith('u"'): d['summary'] = d['summary'][1:] license_classifier = "License :: OSI Approved :: " if pkginfo.get('classifiers'): licenses = [classifier.split(license_classifier, 1)[1] for classifier in pkginfo['classifiers'] if classifier.startswith(license_classifier)] elif data and 'classifiers' in data: licenses = [classifier.split(license_classifier, 1)[1] for classifier in data['classifiers'] if classifier.startswith(license_classifier)] else: licenses = [] if not licenses: if pkginfo.get('license'): license_name = pkginfo['license'] elif data and 'license' in data: license_name = data['license'] else: license_name = None if license_name: if noprompt: pass elif '\n' not in license_name: print('Using "%s" for the license' % license_name) else: # Some projects put the whole license text in this field print("This is the license for %s" % package) print() print(license_name) print() license_name = input("What license string should I use? ") else: if noprompt: license_name = "UNKNOWN" else: license_name = input(("No license could be found for %s on " + "PyPI or in the source. What license should I use? ") % package) else: license_name = ' or '.join(licenses) d['license'] = license_name d['license_family'] = guess_license_family(license_name, allowed_license_families)
def test_cc(): fam = guess_license_family('CC0') assert fam == 'CC'
def test_gpl2(): licenses = {u'GPL-2', u'GPL-2 | file LICENSE', u'GNU General Public License v2 or later (GPLv2+)'} for cens in licenses: fam = guess_license_family(cens) assert fam == u'GPL2'
def skeletonize(packages, output_dir=".", version=None, git_tag=None, cran_url="https://cran.r-project.org/", recursive=False, archive=True, version_compare=False, update_outdated=False, config=None): if not config: config = Config() if len(packages) > 1 and version_compare: raise ValueError( "--version-compare only works with one package at a time") if not update_outdated and not packages: raise ValueError("At least one package must be supplied") package_dicts = {} cran_metadata = get_cran_metadata(cran_url, output_dir) if update_outdated: packages = get_outdated(output_dir, cran_metadata, packages) for pkg in packages: rm_rf(join(output_dir[0], 'r-' + pkg)) while packages: package = packages.pop() is_github_url = 'github.com' in package url = package if is_github_url: rm_rf(config.work_dir) m = metadata.MetaData.fromdict({'source': { 'git_url': package }}, config=config) source.git_source(m.get_section('source'), m.config.git_cache, m.config.work_dir) git_tag = git_tag[0] if git_tag else get_latest_git_tag(config) p = subprocess.Popen(['git', 'checkout', git_tag], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=config.work_dir) stdout, stderr = p.communicate() stdout = stdout.decode('utf-8') stderr = stderr.decode('utf-8') if p.returncode: sys.exit( "Error: 'git checkout %s' failed (%s).\nInvalid tag?" % (git_tag, stderr.strip())) if stdout: print(stdout, file=sys.stdout) if stderr: print(stderr, file=sys.stderr) DESCRIPTION = join(config.work_dir, "DESCRIPTION") if not isfile(DESCRIPTION): sub_description_pkg = join(config.work_dir, 'pkg', "DESCRIPTION") sub_description_name = join(config.work_dir, package.split('/')[-1], "DESCRIPTION") if isfile(sub_description_pkg): DESCRIPTION = sub_description_pkg elif isfile(sub_description_name): DESCRIPTION = sub_description_name else: sys.exit( "%s does not appear to be a valid R package " "(no DESCRIPTION file in %s, %s)" % (package, sub_description_pkg, sub_description_name)) with open(DESCRIPTION) as f: description_text = clear_trailing_whitespace(f.read()) d = dict_from_cran_lines( remove_package_line_continuations( description_text.splitlines())) d['orig_description'] = description_text package = d['Package'].lower() cran_metadata[package] = d if package.startswith('r-'): package = package[2:] if package.endswith('/'): package = package[:-1] if package.lower() not in cran_metadata: sys.exit("Package %s not found" % package) # Make sure package is always uses the CRAN capitalization package = cran_metadata[package.lower()]['Package'] if not is_github_url: session = get_session(output_dir) cran_metadata[package.lower()].update( get_package_metadata(cran_url, package, session)) dir_path = join(output_dir, 'r-' + package.lower()) if exists(dir_path) and not version_compare: raise RuntimeError("directory already exists: %s" % dir_path) cran_package = cran_metadata[package.lower()] d = package_dicts.setdefault( package, { 'cran_packagename': package, 'packagename': 'r-' + package.lower(), 'build_depends': '', 'run_depends': '', # CRAN doesn't seem to have this metadata :( 'home_comment': '#', 'homeurl': '', 'summary_comment': '#', 'summary': '', }) if is_github_url: d['url_key'] = '' d['fn_key'] = '' d['git_url_key'] = 'git_url:' d['git_tag_key'] = 'git_tag:' d['hash_entry'] = '# You can add a hash for the file here, like md5, sha1 or sha256' d['filename'] = '' d['cranurl'] = '' d['git_url'] = url d['git_tag'] = git_tag else: d['url_key'] = 'url:' d['fn_key'] = 'fn:' d['git_url_key'] = '' d['git_tag_key'] = '' d['git_url'] = '' d['git_tag'] = '' d['hash_entry'] = '' if version: d['version'] = version raise NotImplementedError( "Package versions from CRAN are not yet implemented") d['cran_version'] = cran_package['Version'] # Conda versions cannot have -. Conda (verlib) will treat _ as a . d['conda_version'] = d['cran_version'].replace('-', '_') if version_compare: sys.exit(not version_compare(dir_path, d['conda_version'])) if not is_github_url: filename = '{}_{}.tar.gz' contrib_url = cran_url + 'src/contrib/' package_url = contrib_url + filename.format( package, d['cran_version']) # calculate sha256 by downloading source sha256 = hashlib.sha256() print("Downloading source from {}".format(package_url)) sha256.update(urlopen(package_url).read()) d['hash_entry'] = 'sha256: {}'.format(sha256.hexdigest()) d['filename'] = filename.format(package, '{{ version }}') if archive: d['cranurl'] = (INDENT + contrib_url + d['filename'] + INDENT + contrib_url + 'Archive/{}/'.format(package) + d['filename']) else: d['cranurl'] = ' ' + cran_url + 'src/contrib/' + d['filename'] d['cran_metadata'] = '\n'.join( ['# %s' % l for l in cran_package['orig_lines'] if l]) # XXX: We should maybe normalize these d['license'] = cran_package.get("License", "None") d['license_family'] = guess_license_family(d['license'], allowed_license_families) if 'License_is_FOSS' in cran_package: d['license'] += ' (FOSS)' if cran_package.get('License_restricts_use') == 'yes': d['license'] += ' (Restricts use)' if "URL" in cran_package: d['home_comment'] = '' d['homeurl'] = ' ' + yaml_quote_string(cran_package['URL']) else: # use CRAN page as homepage if nothing has been specified d['home_comment'] = '' d['homeurl'] = ' https://CRAN.R-project.org/package={}'.format( package) if 'Description' in cran_package: d['summary_comment'] = '' d['summary'] = ' ' + yaml_quote_string(cran_package['Description']) if "Suggests" in cran_package: d['suggests'] = "# Suggests: %s" % cran_package['Suggests'] else: d['suggests'] = '' # Every package depends on at least R. # I'm not sure what the difference between depends and imports is. depends = [ s.strip() for s in cran_package.get('Depends', '').split(',') if s.strip() ] imports = [ s.strip() for s in cran_package.get('Imports', '').split(',') if s.strip() ] links = [ s.strip() for s in cran_package.get("LinkingTo", '').split(',') if s.strip() ] dep_dict = {} for s in set(chain(depends, imports, links)): match = VERSION_DEPENDENCY_REGEX.match(s) if not match: sys.exit("Could not parse version from dependency of %s: %s" % (package, s)) name = match.group('name') archs = match.group('archs') relop = match.group('relop') or '' version = match.group('version') or '' version = version.replace('-', '_') # If there is a relop there should be a version assert not relop or version if archs: sys.exit("Don't know how to handle archs from dependency of " "package %s: %s" % (package, s)) dep_dict[name] = '{relop}{version}'.format(relop=relop, version=version) if 'R' not in dep_dict: dep_dict['R'] = '' for dep_type in ['build', 'run']: deps = [] for name in sorted(dep_dict): if name in R_BASE_PACKAGE_NAMES: continue if name == 'R': # Put R first # Regarless of build or run, and whether this is a recommended package or not, # it can only depend on 'r-base' since anything else can and will cause cycles # in the dependency graph. The cran metadata lists all dependencies anyway, even # those packages that are in the recommended group. r_name = 'r-base' # We don't include any R version restrictions because we # always build R packages against an exact R version deps.insert( 0, '{indent}{r_name}'.format(indent=INDENT, r_name=r_name)) else: conda_name = 'r-' + name.lower() if dep_dict[name]: deps.append('{indent}{name} {version}'.format( name=conda_name, version=dep_dict[name], indent=INDENT)) else: deps.append('{indent}{name}'.format(name=conda_name, indent=INDENT)) if recursive: if not exists(join(output_dir, conda_name)): packages.append(name) if cran_package.get("NeedsCompilation", 'no') == 'yes': if dep_type == 'build': deps.append('{indent}posix # [win]'.format( indent=INDENT)) deps.append( '{indent}{{{{native}}}}toolchain # [win]'.format( indent=INDENT)) deps.append( '{indent}gcc # [not win]'.format( indent=INDENT)) elif dep_type == 'run': deps.append( '{indent}{{{{native}}}}gcc-libs # [win]'.format( indent=INDENT)) deps.append( '{indent}libgcc # [not win]'.format( indent=INDENT)) d['%s_depends' % dep_type] = ''.join(deps) for package in package_dicts: d = package_dicts[package] name = d['packagename'] # Normalize the metadata values d = { k: unicodedata.normalize("NFKD", text_type(v)).encode('ascii', 'ignore').decode() for k, v in iteritems(d) } makedirs(join(output_dir, name)) print("Writing recipe for %s" % package.lower()) with open(join(output_dir, name, 'meta.yaml'), 'w') as f: f.write(clear_trailing_whitespace(CRAN_META.format(**d))) with open(join(output_dir, name, 'build.sh'), 'w') as f: f.write(CRAN_BUILD_SH.format(**d)) with open(join(output_dir, name, 'bld.bat'), 'w') as f: f.write(CRAN_BLD_BAT.format(**d)) print("Done")
def test_mit(): licenses = {u'MIT License', u'MIT + file LICENSE', u'Old MIT'} for cens in licenses: fam = guess_license_family(cens) assert fam == u'MIT'
def get_package_metadata(package, d, data, output_dir, python_version, all_extras, recursive, created_recipes, noarch_python, noprompt, packages, config, setup_options): print("Downloading %s" % package) pkginfo = get_pkginfo(package, filename=d['filename'], pypiurl=d['pypiurl'], md5=d['md5'], python_version=python_version, setup_options=setup_options, config=config) setuptools_build = pkginfo.get('setuptools', False) setuptools_run = False # Look at the entry_points and construct console_script and # gui_scripts entry_points for conda entry_points = pkginfo.get('entry_points', []) if entry_points: if isinstance(entry_points, str): # makes sure it is left-shifted newstr = "\n".join(x.strip() for x in entry_points.splitlines()) _config = configparser.ConfigParser() entry_points = {} try: _config.readfp(StringIO(newstr)) except Exception as err: print("WARNING: entry-points not understood: ", err) print("The string was", newstr) entry_points = pkginfo['entry_points'] else: setuptools_run = True for section in _config.sections(): if section in ['console_scripts', 'gui_scripts']: value = ['%s=%s' % (option, _config.get(section, option)) for option in _config.options(section)] entry_points[section] = value if not isinstance(entry_points, dict): print("WARNING: Could not add entry points. They were:") print(entry_points) else: cs = entry_points.get('console_scripts', []) gs = entry_points.get('gui_scripts', []) if isinstance(cs, string_types): cs = [cs] if isinstance(gs, string_types): gs = [gs] # We have *other* kinds of entry-points so we need # setuptools at run-time if set(entry_points.keys()) - {'console_scripts', 'gui_scripts'}: setuptools_build = True setuptools_run = True # TODO: Use pythonw for gui scripts entry_list = (cs + gs) if len(cs + gs) != 0: d['entry_points'] = INDENT.join([''] + entry_list) d['entry_comment'] = '' d['build_comment'] = '' d['test_commands'] = INDENT.join([''] + make_entry_tests(entry_list)) requires = get_requirements(package, pkginfo, all_extras=all_extras) if requires or setuptools_build or setuptools_run: deps = [] if setuptools_run: deps.append('setuptools') for deptext in requires: if isinstance(deptext, string_types): deptext = deptext.splitlines() # Every item may be a single requirement # or a multiline requirements string... for dep in deptext: # ... and may also contain comments... dep = dep.split('#')[0].strip() if dep: # ... and empty (or comment only) lines spec = spec_from_line(dep) if spec is None: sys.exit("Error: Could not parse: %s" % dep) deps.append(spec) if 'setuptools' in deps: setuptools_build = False setuptools_run = False d['egg_comment'] = '' d['build_comment'] = '' d['build_depends'] = INDENT.join([''] + ['setuptools'] * setuptools_build + deps) d['run_depends'] = INDENT.join([''] + ['setuptools'] * setuptools_run + deps) if recursive: for dep in deps: dep = dep.split()[0] if not exists(join(output_dir, dep)): if dep not in created_recipes: packages.append(dep) if noarch_python: d['build_comment'] = '' d['noarch_python_comment'] = '' if 'packagename' not in d: d['packagename'] = pkginfo['name'].lower() if d['version'] == 'UNKNOWN': d['version'] = pkginfo['version'] if pkginfo.get('packages'): deps = set(pkginfo['packages']) if d['import_tests']: if not d['import_tests'] or d['import_tests'] == 'PLACEHOLDER': olddeps = [] else: olddeps = [x for x in d['import_tests'].split() if x != '-'] deps = set(olddeps) | deps d['import_tests'] = INDENT.join(sorted(deps)) d['import_comment'] = '' d['tests_require'] = INDENT.join(sorted([spec_from_line(pkg) for pkg in pkginfo['tests_require']])) if pkginfo.get('homeurl'): d['homeurl'] = pkginfo['homeurl'] else: if data and 'homeurl' in data: d['homeurl'] = data['homeurl'] else: d['homeurl'] = "The package home page" d['home_comment'] = '#' if pkginfo.get('summary'): d['summary'] = repr(pkginfo['summary']) else: if data: d['summary'] = repr(data['summary']) else: d['summary'] = "Summary of the package" d['summary_comment'] = '#' if d['summary'].startswith("u'") or d['summary'].startswith('u"'): d['summary'] = d['summary'][1:] license_classifier = "License :: OSI Approved :: " if pkginfo.get('classifiers'): licenses = [classifier.split(license_classifier, 1)[1] for classifier in pkginfo['classifiers'] if classifier.startswith(license_classifier)] elif data and 'classifiers' in data: licenses = [classifier.split(license_classifier, 1)[1] for classifier in data['classifiers'] if classifier.startswith(license_classifier)] else: licenses = [] if not licenses: if pkginfo.get('license'): license_name = pkginfo['license'] elif data and 'license' in data: license_name = data['license'] else: license_name = None if license_name: if noprompt: pass elif '\n' not in license_name: print('Using "%s" for the license' % license_name) else: # Some projects put the whole license text in this field print("This is the license for %s" % package) print() print(license_name) print() license_name = input("What license string should I use? ") else: if noprompt: license_name = "UNKNOWN" else: license_name = input(("No license could be found for %s on " + "PyPI or in the source. What license should I use? ") % package) else: license_name = ' or '.join(licenses) d['license'] = license_name d['license_family'] = guess_license_family(license_name, allowed_license_families)
def skeletonize( packages, output_dir=".", version=None, git_tag=None, cran_url="http://cran.r-project.org/", recursive=False, archive=True, version_compare=False, update_outdated=False, config=None, ): if not config: config = Config() if len(packages) > 1 and version_compare: raise ValueError("--version-compare only works with one package at a time") if not update_outdated and not packages: raise ValueError("At least one package must be supplied") package_dicts = {} cran_metadata = get_cran_metadata(cran_url, output_dir) if update_outdated: packages = get_outdated(output_dir, cran_metadata, packages) for pkg in packages: rm_rf(join(output_dir[0], "r-" + pkg)) while packages: package = packages.pop() is_github_url = "github.com" in package url = package if is_github_url: rm_rf(config.work_dir) source.git_source({"git_url": package}, ".", config=config) git_tag = git_tag[0] if git_tag else get_latest_git_tag(config) p = subprocess.Popen( ["git", "checkout", git_tag], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=config.work_dir ) stdout, stderr = p.communicate() stdout = stdout.decode("utf-8") stderr = stderr.decode("utf-8") if p.returncode: sys.exit("Error: 'git checkout %s' failed (%s).\nInvalid tag?" % (git_tag, stderr.strip())) if stdout: print(stdout, file=sys.stdout) if stderr: print(stderr, file=sys.stderr) DESCRIPTION = join(config.work_dir, "DESCRIPTION") if not isfile(DESCRIPTION): sub_description_pkg = join(config.work_dir, "pkg", "DESCRIPTION") sub_description_name = join(config.work_dir, package.split("/")[-1], "DESCRIPTION") if isfile(sub_description_pkg): DESCRIPTION = sub_description_pkg elif isfile(sub_description_name): DESCRIPTION = sub_description_name else: sys.exit( "%s does not appear to be a valid R package " "(no DESCRIPTION file in %s, %s)" % (package, sub_description_pkg, sub_description_name) ) with open(DESCRIPTION) as f: description_text = clear_trailing_whitespace(f.read()) d = dict_from_cran_lines(remove_package_line_continuations(description_text.splitlines())) d["orig_description"] = description_text package = d["Package"].lower() cran_metadata[package] = d if package.startswith("r-"): package = package[2:] if package.endswith("/"): package = package[:-1] if package.lower() not in cran_metadata: sys.exit("Package %s not found" % package) # Make sure package is always uses the CRAN capitalization package = cran_metadata[package.lower()]["Package"] if not is_github_url: session = get_session(output_dir) cran_metadata[package.lower()].update(get_package_metadata(cran_url, package, session)) dir_path = join(output_dir, "r-" + package.lower()) if exists(dir_path) and not version_compare: raise RuntimeError("directory already exists: %s" % dir_path) cran_package = cran_metadata[package.lower()] d = package_dicts.setdefault( package, { "cran_packagename": package, "packagename": "r-" + package.lower(), "build_depends": "", "run_depends": "", # CRAN doesn't seem to have this metadata :( "home_comment": "#", "homeurl": "", "summary_comment": "#", "summary": "", }, ) if is_github_url: d["url_key"] = "" d["fn_key"] = "" d["git_url_key"] = "git_url:" d["git_tag_key"] = "git_tag:" d["filename"] = "" d["cranurl"] = "" d["git_url"] = url d["git_tag"] = git_tag else: d["url_key"] = "url:" d["fn_key"] = "fn:" d["git_url_key"] = "" d["git_tag_key"] = "" d["git_url"] = "" d["git_tag"] = "" if version: d["version"] = version raise NotImplementedError("Package versions from CRAN are not yet implemented") d["cran_version"] = cran_package["Version"] # Conda versions cannot have -. Conda (verlib) will treat _ as a . d["conda_version"] = d["cran_version"].replace("-", "_") if version_compare: sys.exit(not version_compare(dir_path, d["conda_version"])) if not is_github_url: d["filename"] = "{cran_packagename}_{cran_version}.tar.gz".format(**d) if archive: d["cranurl"] = ( INDENT + cran_url + "src/contrib/" + d["filename"] + INDENT + cran_url + "src/contrib/" + "Archive/" + d["cran_packagename"] + "/" + d["filename"] ) else: d["cranurl"] = " " + cran_url + "src/contrib/" + d["filename"] d["cran_metadata"] = "\n".join(["# %s" % l for l in cran_package["orig_lines"] if l]) # XXX: We should maybe normalize these d["license"] = cran_package.get("License", "None") d["license_family"] = guess_license_family(d["license"], allowed_license_families) if "License_is_FOSS" in cran_package: d["license"] += " (FOSS)" if cran_package.get("License_restricts_use") == "yes": d["license"] += " (Restricts use)" if "URL" in cran_package: d["home_comment"] = "" d["homeurl"] = " " + yaml_quote_string(cran_package["URL"]) if "Description" in cran_package: d["summary_comment"] = "" d["summary"] = " " + yaml_quote_string(cran_package["Description"]) if "Suggests" in cran_package: d["suggests"] = "# Suggests: %s" % cran_package["Suggests"] else: d["suggests"] = "" # Every package depends on at least R. # I'm not sure what the difference between depends and imports is. depends = [s.strip() for s in cran_package.get("Depends", "").split(",") if s.strip()] imports = [s.strip() for s in cran_package.get("Imports", "").split(",") if s.strip()] links = [s.strip() for s in cran_package.get("LinkingTo", "").split(",") if s.strip()] dep_dict = {} for s in set(chain(depends, imports, links)): match = VERSION_DEPENDENCY_REGEX.match(s) if not match: sys.exit("Could not parse version from dependency of %s: %s" % (package, s)) name = match.group("name") archs = match.group("archs") relop = match.group("relop") or "" version = match.group("version") or "" version = version.replace("-", "_") # If there is a relop there should be a version assert not relop or version if archs: sys.exit("Don't know how to handle archs from dependency of " "package %s: %s" % (package, s)) dep_dict[name] = "{relop}{version}".format(relop=relop, version=version) if "R" not in dep_dict: dep_dict["R"] = "" for dep_type in ["build", "run"]: deps = [] for name in sorted(dep_dict): if name in R_BASE_PACKAGE_NAMES: continue if name == "R": # Put R first # Regarless of build or run, and whether this is a recommended package or not, # it can only depend on 'r-base' since anything else can and will cause cycles # in the dependency graph. The cran metadata lists all dependencies anyway, even # those packages that are in the recommended group. r_name = "r-base" # We don't include any R version restrictions because we # always build R packages against an exact R version deps.insert(0, "{indent}{r_name}".format(indent=INDENT, r_name=r_name)) else: conda_name = "r-" + name.lower() if dep_dict[name]: deps.append( "{indent}{name} {version}".format(name=conda_name, version=dep_dict[name], indent=INDENT) ) else: deps.append("{indent}{name}".format(name=conda_name, indent=INDENT)) if recursive: if not exists(join(output_dir, conda_name)): packages.append(name) if cran_package.get("NeedsCompilation", "no") == "yes": if dep_type == "build": deps.append("{indent}posix # [win]".format(indent=INDENT)) deps.append("{indent}{{{{native}}}}toolchain # [win]".format(indent=INDENT)) deps.append("{indent}gcc # [not win]".format(indent=INDENT)) d["%s_depends" % dep_type] = "".join(deps) for package in package_dicts: d = package_dicts[package] name = d["packagename"] # Normalize the metadata values d = {k: unicodedata.normalize("NFKD", text_type(v)).encode("ascii", "ignore").decode() for k, v in iteritems(d)} makedirs(join(output_dir, name)) print("Writing recipe for %s" % package.lower()) with open(join(output_dir, name, "meta.yaml"), "w") as f: f.write(clear_trailing_whitespace(CRAN_META.format(**d))) with open(join(output_dir, name, "build.sh"), "w") as f: f.write(CRAN_BUILD_SH.format(**d)) with open(join(output_dir, name, "bld.bat"), "w") as f: f.write(CRAN_BLD_BAT.format(**d)) print("Done")