def skeletonize(packages,
output_dir=".",
version=None,
git_tag=None,
cran_url="http://cran.r-project.org/",
recursive=False,
archive=True,
version_compare=False,
update_outdated=False,
config=None):
if not config:
config = Config()
if len(packages) > 1 and version_compare:
raise ValueError(
"--version-compare only works with one package at a time")
if not update_outdated and not packages:
raise ValueError("At least one package must be supplied")
package_dicts = {}
cran_metadata = get_cran_metadata(cran_url, output_dir)
if update_outdated:
packages = get_outdated(output_dir, cran_metadata, packages)
for pkg in packages:
rm_rf(join(output_dir[0], 'r-' + pkg))
while packages:
package = packages.pop()
is_github_url = 'github.com' in package
url = package
if is_github_url:
rm_rf(config.work_dir)
source.git_source({'git_url': package}, '.', config=config)
git_tag = git_tag[0] if git_tag else get_latest_git_tag(config)
p = subprocess.Popen(['git', 'checkout', git_tag],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=config.work_dir)
stdout, stderr = p.communicate()
stdout = stdout.decode('utf-8')
stderr = stderr.decode('utf-8')
if p.returncode:
sys.exit(
"Error: 'git checkout %s' failed (%s).\nInvalid tag?" %
(git_tag, stderr.strip()))
if stdout:
print(stdout, file=sys.stdout)
if stderr:
print(stderr, file=sys.stderr)
DESCRIPTION = join(config.work_dir, "DESCRIPTION")
if not isfile(DESCRIPTION):
sub_description_pkg = join(config.work_dir, 'pkg',
"DESCRIPTION")
sub_description_name = join(config.work_dir,
package.split('/')[-1],
"DESCRIPTION")
if isfile(sub_description_pkg):
DESCRIPTION = sub_description_pkg
elif isfile(sub_description_name):
DESCRIPTION = sub_description_name
else:
sys.exit(
"%s does not appear to be a valid R package "
"(no DESCRIPTION file in %s, %s)" %
(package, sub_description_pkg, sub_description_name))
with open(DESCRIPTION) as f:
description_text = clear_trailing_whitespace(f.read())
d = dict_from_cran_lines(
remove_package_line_continuations(
description_text.splitlines()))
d['orig_description'] = description_text
package = d['Package'].lower()
cran_metadata[package] = d
if package.startswith('r-'):
package = package[2:]
if package.endswith('/'):
package = package[:-1]
if package.lower() not in cran_metadata:
sys.exit("Package %s not found" % package)
# Make sure package is always uses the CRAN capitalization
package = cran_metadata[package.lower()]['Package']
if not is_github_url:
session = get_session(output_dir)
cran_metadata[package.lower()].update(
get_package_metadata(cran_url, package, session))
dir_path = join(output_dir, 'r-' + package.lower())
if exists(dir_path) and not version_compare:
raise RuntimeError("directory already exists: %s" % dir_path)
cran_package = cran_metadata[package.lower()]
d = package_dicts.setdefault(
package,
{
'cran_packagename': package,
'packagename': 'r-' + package.lower(),
'build_depends': '',
'run_depends': '',
# CRAN doesn't seem to have this metadata :(
'home_comment': '#',
'homeurl': '',
'summary_comment': '#',
'summary': '',
})
if is_github_url:
d['url_key'] = ''
d['fn_key'] = ''
d['git_url_key'] = 'git_url:'
d['git_tag_key'] = 'git_tag:'
d['filename'] = ''
d['cranurl'] = ''
d['git_url'] = url
d['git_tag'] = git_tag
else:
d['url_key'] = 'url:'
d['fn_key'] = 'fn:'
d['git_url_key'] = ''
d['git_tag_key'] = ''
d['git_url'] = ''
d['git_tag'] = ''
if version:
d['version'] = version
raise NotImplementedError(
"Package versions from CRAN are not yet implemented")
d['cran_version'] = cran_package['Version']
# Conda versions cannot have -. Conda (verlib) will treat _ as a .
d['conda_version'] = d['cran_version'].replace('-', '_')
if version_compare:
sys.exit(not version_compare(dir_path, d['conda_version']))
if not is_github_url:
d['filename'] = "{cran_packagename}_{cran_version}.tar.gz".format(
**d)
if archive:
d['cranurl'] = (INDENT + cran_url + 'src/contrib/' +
d['filename'] + INDENT + cran_url +
'src/contrib/' + 'Archive/' +
d['cran_packagename'] + '/' + d['filename'])
else:
d['cranurl'] = ' ' + cran_url + 'src/contrib/' + d['filename']
d['cran_metadata'] = '\n'.join(
['# %s' % l for l in cran_package['orig_lines'] if l])
# XXX: We should maybe normalize these
d['license'] = cran_package.get("License", "None")
d['license_family'] = guess_license_family(
d['license'], metadata.allowed_license_families)
if 'License_is_FOSS' in cran_package:
d['license'] += ' (FOSS)'
if cran_package.get('License_restricts_use') == 'yes':
d['license'] += ' (Restricts use)'
if "URL" in cran_package:
d['home_comment'] = ''
d['homeurl'] = ' ' + yaml_quote_string(cran_package['URL'])
if 'Description' in cran_package:
d['summary_comment'] = ''
d['summary'] = ' ' + yaml_quote_string(cran_package['Description'])
if "Suggests" in cran_package:
d['suggests'] = "# Suggests: %s" % cran_package['Suggests']
else:
d['suggests'] = ''
# Every package depends on at least R.
# I'm not sure what the difference between depends and imports is.
depends = [
s.strip() for s in cran_package.get('Depends', '').split(',')
if s.strip()
]
imports = [
s.strip() for s in cran_package.get('Imports', '').split(',')
if s.strip()
]
links = [
s.strip() for s in cran_package.get("LinkingTo", '').split(',')
if s.strip()
]
dep_dict = {}
for s in set(chain(depends, imports, links)):
match = VERSION_DEPENDENCY_REGEX.match(s)
if not match:
sys.exit("Could not parse version from dependency of %s: %s" %
(package, s))
name = match.group('name')
archs = match.group('archs')
relop = match.group('relop') or ''
version = match.group('version') or ''
version = version.replace('-', '_')
# If there is a relop there should be a version
assert not relop or version
if archs:
sys.exit("Don't know how to handle archs from dependency of "
"package %s: %s" % (package, s))
dep_dict[name] = '{relop}{version}'.format(relop=relop,
version=version)
if 'R' not in dep_dict:
dep_dict['R'] = ''
for dep_type in ['build', 'run']:
deps = []
for name in sorted(dep_dict):
if name in R_BASE_PACKAGE_NAMES:
continue
if name == 'R':
# Put R first
# Regarless of build or run, and whether this is a recommended package or not,
# it can only depend on 'r-base' since anything else can and will cause cycles
# in the dependency graph. The cran metadata lists all dependencies anyway, even
# those packages that are in the recommended group.
r_name = 'r-base'
# We don't include any R version restrictions because we
# always build R packages against an exact R version
deps.insert(
0, '{indent}{r_name}'.format(indent=INDENT,
r_name=r_name))
else:
conda_name = 'r-' + name.lower()
if dep_dict[name]:
deps.append('{indent}{name} {version}'.format(
name=conda_name,
version=dep_dict[name],
indent=INDENT))
else:
deps.append('{indent}{name}'.format(name=conda_name,
indent=INDENT))
if recursive:
if not exists(join(output_dir, conda_name)):
packages.append(name)
if cran_package.get("NeedsCompilation", 'no') == 'yes':
if dep_type == 'build':
deps.append('{indent}posix # [win]'.format(
indent=INDENT))
deps.append(
'{indent}{{{{native}}}}toolchain # [win]'.format(
indent=INDENT))
deps.append(
'{indent}gcc # [not win]'.format(
indent=INDENT))
d['%s_depends' % dep_type] = ''.join(deps)
for package in package_dicts:
d = package_dicts[package]
name = d['packagename']
# Normalize the metadata values
d = {
k: unicodedata.normalize("NFKD",
text_type(v)).encode('ascii',
'ignore').decode()
for k, v in iteritems(d)
}
makedirs(join(output_dir, name))
print("Writing recipe for %s" % package.lower())
with open(join(output_dir, name, 'meta.yaml'), 'w') as f:
f.write(clear_trailing_whitespace(CRAN_META.format(**d)))
with open(join(output_dir, name, 'build.sh'), 'w') as f:
f.write(CRAN_BUILD_SH.format(**d))
with open(join(output_dir, name, 'bld.bat'), 'w') as f:
f.write(CRAN_BLD_BAT.format(**d))
print("Done")
def get_package_metadata(package, d, data, output_dir, python_version, all_extras,
recursive, created_recipes, noarch_python, noprompt, packages,
config, setup_options):
print("Downloading %s" % package)
pkginfo = get_pkginfo(package,
filename=d['filename'],
pypiurl=d['pypiurl'],
md5=d['md5'],
python_version=python_version,
setup_options=setup_options,
config=config)
setuptools_build = pkginfo['setuptools']
setuptools_run = False
# Look at the entry_points and construct console_script and
# gui_scripts entry_points for conda
entry_points = pkginfo['entry_points']
if entry_points:
if isinstance(entry_points, str):
# makes sure it is left-shifted
newstr = "\n".join(x.strip()
for x in entry_points.splitlines())
_config = configparser.ConfigParser()
entry_points = {}
try:
_config.readfp(StringIO(newstr))
except Exception as err:
print("WARNING: entry-points not understood: ",
err)
print("The string was", newstr)
entry_points = pkginfo['entry_points']
else:
setuptools_run = True
for section in config.sections():
if section in ['console_scripts', 'gui_scripts']:
value = ['%s=%s' % (option, _config.get(section, option))
for option in config.options(section)]
entry_points[section] = value
if not isinstance(entry_points, dict):
print("WARNING: Could not add entry points. They were:")
print(entry_points)
else:
cs = entry_points.get('console_scripts', [])
gs = entry_points.get('gui_scripts', [])
if isinstance(cs, string_types):
cs = [cs]
if isinstance(gs, string_types):
gs = [gs]
# We have *other* kinds of entry-points so we need
# setuptools at run-time
if set(entry_points.keys()) - {'console_scripts', 'gui_scripts'}:
setuptools_build = True
setuptools_run = True
# TODO: Use pythonw for gui scripts
entry_list = (cs + gs)
if len(cs + gs) != 0:
d['entry_points'] = INDENT.join([''] + entry_list)
d['entry_comment'] = ''
d['build_comment'] = ''
d['test_commands'] = INDENT.join([''] + make_entry_tests(entry_list))
requires = get_requirements(package, pkginfo, all_extras=all_extras)
if requires or setuptools_build or setuptools_run:
deps = []
if setuptools_run:
deps.append('setuptools')
for deptext in requires:
if isinstance(deptext, string_types):
deptext = deptext.splitlines()
# Every item may be a single requirement
# or a multiline requirements string...
for dep in deptext:
# ... and may also contain comments...
dep = dep.split('#')[0].strip()
if dep: # ... and empty (or comment only) lines
spec = spec_from_line(dep)
if spec is None:
sys.exit("Error: Could not parse: %s" % dep)
deps.append(spec)
if 'setuptools' in deps:
setuptools_build = False
setuptools_run = False
d['egg_comment'] = ''
d['build_comment'] = ''
d['build_depends'] = INDENT.join([''] +
['setuptools'] * setuptools_build +
deps)
d['run_depends'] = INDENT.join([''] +
['setuptools'] * setuptools_run +
deps)
if recursive:
for dep in deps:
dep = dep.split()[0]
if not exists(join(output_dir, dep)):
if dep not in created_recipes:
packages.append(dep)
if noarch_python:
d['build_comment'] = ''
d['noarch_python_comment'] = ''
if 'packagename' not in d:
d['packagename'] = pkginfo['name'].lower()
if d['version'] == 'UNKNOWN':
d['version'] = pkginfo['version']
if pkginfo['packages']:
deps = set(pkginfo['packages'])
if d['import_tests']:
if not d['import_tests'] or d['import_tests'] == 'PLACEHOLDER':
olddeps = []
else:
olddeps = [x for x in d['import_tests'].split()
if x != '-']
deps = set(olddeps) | deps
d['import_tests'] = INDENT.join(sorted(deps))
d['import_comment'] = ''
d['tests_require'] = INDENT.join(sorted([spec_from_line(pkg) for pkg
in pkginfo['tests_require']]))
if pkginfo['homeurl'] is not None:
d['homeurl'] = pkginfo['homeurl']
else:
if data and 'homeurl' in data:
d['homeurl'] = data['homeurl']
else:
d['homeurl'] = "The package home page"
d['home_comment'] = '#'
if pkginfo['summary']:
d['summary'] = repr(pkginfo['summary'])
else:
if data:
d['summary'] = repr(data['summary'])
else:
d['summary'] = "Summary of the package"
d['summary_comment'] = '#'
if d['summary'].startswith("u'") or d['summary'].startswith('u"'):
d['summary'] = d['summary'][1:]
license_classifier = "License :: OSI Approved :: "
if pkginfo['classifiers']:
licenses = [classifier.split(license_classifier, 1)[1] for
classifier in pkginfo['classifiers'] if classifier.startswith(license_classifier)]
elif data and 'classifiers' in data:
licenses = [classifier.split(license_classifier, 1)[1] for classifier in
data['classifiers'] if classifier.startswith(license_classifier)]
else:
licenses = []
if not licenses:
if pkginfo['license']:
license_name = pkginfo['license']
elif data and 'license' in data:
license_name = data['license']
else:
license_name = None
if license_name:
if noprompt:
pass
elif '\n' not in license_name:
print('Using "%s" for the license' % license_name)
else:
# Some projects put the whole license text in this field
print("This is the license for %s" % package)
print()
print(license_name)
print()
license_name = input("What license string should I use? ")
else:
if noprompt:
license_name = "UNKNOWN"
else:
license_name = input(("No license could be found for %s on " +
"PyPI or in the source. What license should I use? ") %
package)
else:
license_name = ' or '.join(licenses)
d['license'] = license_name
d['license_family'] = guess_license_family(license_name, allowed_license_families)
def skeletonize(packages, output_dir=".", version=None, git_tag=None,
cran_url="http://cran.r-project.org/", recursive=False, archive=True,
version_compare=False, update_outdated=False, config=None):
if not config:
config = Config()
if len(packages) > 1 and version_compare:
raise ValueError("--version-compare only works with one package at a time")
if not update_outdated and not packages:
raise ValueError("At least one package must be supplied")
package_dicts = {}
cran_metadata = get_cran_metadata(cran_url, output_dir)
if update_outdated:
packages = get_outdated(output_dir, cran_metadata, packages)
for pkg in packages:
rm_rf(join(output_dir[0], 'r-' + pkg))
while packages:
package = packages.pop()
is_github_url = 'github.com' in package
url = package
if is_github_url:
rm_rf(config.work_dir)
source.git_source({'git_url': package}, '.', config=config)
git_tag = git_tag[0] if git_tag else get_latest_git_tag(config)
p = subprocess.Popen(['git', 'checkout', git_tag], stdout=subprocess.PIPE,
stderr=subprocess.PIPE, cwd=config.work_dir)
stdout, stderr = p.communicate()
stdout = stdout.decode('utf-8')
stderr = stderr.decode('utf-8')
if p.returncode:
sys.exit("Error: 'git checkout %s' failed (%s).\nInvalid tag?" %
(git_tag, stderr.strip()))
if stdout:
print(stdout, file=sys.stdout)
if stderr:
print(stderr, file=sys.stderr)
DESCRIPTION = join(config.work_dir, "DESCRIPTION")
if not isfile(DESCRIPTION):
sub_description_pkg = join(config.work_dir, 'pkg', "DESCRIPTION")
sub_description_name = join(config.work_dir, package.split('/')[-1], "DESCRIPTION")
if isfile(sub_description_pkg):
DESCRIPTION = sub_description_pkg
elif isfile(sub_description_name):
DESCRIPTION = sub_description_name
else:
sys.exit("%s does not appear to be a valid R package "
"(no DESCRIPTION file in %s, %s)"
% (package, sub_description_pkg, sub_description_name))
with open(DESCRIPTION) as f:
description_text = clear_trailing_whitespace(f.read())
d = dict_from_cran_lines(remove_package_line_continuations(
description_text.splitlines()))
d['orig_description'] = description_text
package = d['Package'].lower()
cran_metadata[package] = d
if package.startswith('r-'):
package = package[2:]
if package.endswith('/'):
package = package[:-1]
if package.lower() not in cran_metadata:
sys.exit("Package %s not found" % package)
# Make sure package is always uses the CRAN capitalization
package = cran_metadata[package.lower()]['Package']
if not is_github_url:
session = get_session(output_dir)
cran_metadata[package.lower()].update(get_package_metadata(cran_url,
package, session))
dir_path = join(output_dir, 'r-' + package.lower())
if exists(dir_path) and not version_compare:
raise RuntimeError("directory already exists: %s" % dir_path)
cran_package = cran_metadata[package.lower()]
d = package_dicts.setdefault(package,
{
'cran_packagename': package,
'packagename': 'r-' + package.lower(),
'build_depends': '',
'run_depends': '',
# CRAN doesn't seem to have this metadata :(
'home_comment': '#',
'homeurl': '',
'summary_comment': '#',
'summary': '',
})
if is_github_url:
d['url_key'] = ''
d['fn_key'] = ''
d['git_url_key'] = 'git_url:'
d['git_tag_key'] = 'git_tag:'
d['filename'] = ''
d['cranurl'] = ''
d['git_url'] = url
d['git_tag'] = git_tag
else:
d['url_key'] = 'url:'
d['fn_key'] = 'fn:'
d['git_url_key'] = ''
d['git_tag_key'] = ''
d['git_url'] = ''
d['git_tag'] = ''
if version:
d['version'] = version
raise NotImplementedError("Package versions from CRAN are not yet implemented")
d['cran_version'] = cran_package['Version']
# Conda versions cannot have -. Conda (verlib) will treat _ as a .
d['conda_version'] = d['cran_version'].replace('-', '_')
if version_compare:
sys.exit(not version_compare(dir_path, d['conda_version']))
if not is_github_url:
d['filename'] = "{cran_packagename}_{cran_version}.tar.gz".format(**d)
if archive:
d['cranurl'] = (INDENT + cran_url + 'src/contrib/' +
d['filename'] + INDENT + cran_url + 'src/contrib/' +
'Archive/' + d['cran_packagename'] + '/' + d['filename'])
else:
d['cranurl'] = ' ' + cran_url + 'src/contrib/' + d['filename']
d['cran_metadata'] = '\n'.join(['# %s' % l for l in
cran_package['orig_lines'] if l])
# XXX: We should maybe normalize these
d['license'] = cran_package.get("License", "None")
d['license_family'] = guess_license_family(d['license'],
metadata.allowed_license_families)
if 'License_is_FOSS' in cran_package:
d['license'] += ' (FOSS)'
if cran_package.get('License_restricts_use') == 'yes':
d['license'] += ' (Restricts use)'
if "URL" in cran_package:
d['home_comment'] = ''
d['homeurl'] = ' ' + yaml_quote_string(cran_package['URL'])
if 'Description' in cran_package:
d['summary_comment'] = ''
d['summary'] = ' ' + yaml_quote_string(cran_package['Description'])
if "Suggests" in cran_package:
d['suggests'] = "# Suggests: %s" % cran_package['Suggests']
else:
d['suggests'] = ''
# Every package depends on at least R.
# I'm not sure what the difference between depends and imports is.
depends = [s.strip() for s in cran_package.get('Depends',
'').split(',') if s.strip()]
imports = [s.strip() for s in cran_package.get('Imports',
'').split(',') if s.strip()]
links = [s.strip() for s in cran_package.get("LinkingTo",
'').split(',') if s.strip()]
dep_dict = {}
for s in set(chain(depends, imports, links)):
match = VERSION_DEPENDENCY_REGEX.match(s)
if not match:
sys.exit("Could not parse version from dependency of %s: %s" %
(package, s))
name = match.group('name')
archs = match.group('archs')
relop = match.group('relop') or ''
version = match.group('version') or ''
version = version.replace('-', '_')
# If there is a relop there should be a version
assert not relop or version
if archs:
sys.exit("Don't know how to handle archs from dependency of "
"package %s: %s" % (package, s))
dep_dict[name] = '{relop}{version}'.format(relop=relop, version=version)
if 'R' not in dep_dict:
dep_dict['R'] = ''
for dep_type in ['build', 'run']:
deps = []
for name in sorted(dep_dict):
if name in R_BASE_PACKAGE_NAMES:
continue
if name == 'R':
# Put R first
# Regarless of build or run, and whether this is a recommended package or not,
# it can only depend on 'r-base' since anything else can and will cause cycles
# in the dependency graph. The cran metadata lists all dependencies anyway, even
# those packages that are in the recommended group.
r_name = 'r-base'
# We don't include any R version restrictions because we
# always build R packages against an exact R version
deps.insert(0, '{indent}{r_name}'.format(indent=INDENT, r_name=r_name))
else:
conda_name = 'r-' + name.lower()
if dep_dict[name]:
deps.append('{indent}{name} {version}'.format(name=conda_name,
version=dep_dict[name], indent=INDENT))
else:
deps.append('{indent}{name}'.format(name=conda_name,
indent=INDENT))
if recursive:
if not exists(join(output_dir, conda_name)):
packages.append(name)
if cran_package.get("NeedsCompilation", 'no') == 'yes':
if dep_type == 'build':
deps.append('{indent}posix # [win]'.format(indent=INDENT))
deps.append('{indent}{{{{native}}}}toolchain # [win]'.format(indent=INDENT))
deps.append('{indent}gcc # [not win]'.format(indent=INDENT))
d['%s_depends' % dep_type] = ''.join(deps)
for package in package_dicts:
d = package_dicts[package]
name = d['packagename']
# Normalize the metadata values
d = {k: unicodedata.normalize("NFKD", text_type(v)).encode('ascii', 'ignore')
.decode() for k, v in iteritems(d)}
makedirs(join(output_dir, name))
print("Writing recipe for %s" % package.lower())
with open(join(output_dir, name, 'meta.yaml'), 'w') as f:
f.write(clear_trailing_whitespace(CRAN_META.format(**d)))
with open(join(output_dir, name, 'build.sh'), 'w') as f:
f.write(CRAN_BUILD_SH.format(**d))
with open(join(output_dir, name, 'bld.bat'), 'w') as f:
f.write(CRAN_BLD_BAT.format(**d))
print("Done")
