def get_package_metadata(args, package, d, data):
print("Downloading %s" % package)
[output_dir] = args.output_dir
pkginfo = get_pkginfo(package,
filename=d['filename'],
pypiurl=d['pypiurl'],
md5=d['md5'],
python_version=args.python_version)
setuptools_build = pkginfo['setuptools']
setuptools_run = False
# Look at the entry_points and construct console_script and
# gui_scripts entry_points for conda
entry_points = pkginfo['entry_points']
if entry_points:
if isinstance(entry_points, str):
# makes sure it is left-shifted
newstr = "\n".join(x.strip() for x in entry_points.splitlines())
config = configparser.ConfigParser()
entry_points = {}
try:
config.readfp(StringIO(newstr))
except Exception as err:
print("WARNING: entry-points not understood: ", err)
print("The string was", newstr)
entry_points = pkginfo['entry_points']
else:
setuptools_run = True
for section in config.sections():
if section in ['console_scripts', 'gui_scripts']:
value = [
'%s=%s' % (option, config.get(section, option))
for option in config.options(section)
]
entry_points[section] = value
if not isinstance(entry_points, dict):
print("WARNING: Could not add entry points. They were:")
print(entry_points)
else:
cs = entry_points.get('console_scripts', [])
gs = entry_points.get('gui_scripts', [])
if isinstance(cs, string_types):
cs = [cs]
if isinstance(gs, string_types):
gs = [gs]
# We have *other* kinds of entry-points so we need
# setuptools at run-time
if set(entry_points.keys()) - {'console_scripts', 'gui_scripts'}:
setuptools_build = True
setuptools_run = True
# TODO: Use pythonw for gui scripts
entry_list = (cs + gs)
if len(cs + gs) != 0:
d['entry_points'] = INDENT.join([''] + entry_list)
d['entry_comment'] = ''
d['build_comment'] = ''
d['test_commands'] = INDENT.join([''] +
make_entry_tests(entry_list))
requires = get_requirements(package, pkginfo, all_extras=args.all_extras)
if requires or setuptools_build or setuptools_run:
deps = []
if setuptools_run:
deps.append('setuptools')
for deptext in requires:
if isinstance(deptext, string_types):
deptext = deptext.splitlines()
# Every item may be a single requirement
# or a multiline requirements string...
for dep in deptext:
# ... and may also contain comments...
dep = dep.split('#')[0].strip()
if dep: # ... and empty (or comment only) lines
spec = spec_from_line(dep)
if spec is None:
sys.exit("Error: Could not parse: %s" % dep)
deps.append(spec)
if 'setuptools' in deps:
setuptools_build = False
setuptools_run = False
d['egg_comment'] = ''
d['build_comment'] = ''
d['build_depends'] = INDENT.join([''] +
['setuptools'] * setuptools_build +
deps)
d['run_depends'] = INDENT.join([''] + ['setuptools'] * setuptools_run +
deps)
if args.recursive:
for dep in deps:
dep = dep.split()[0]
if not exists(join(output_dir, dep)):
if dep not in args.created_recipes:
args.packages.append(dep)
if args.noarch_python:
d['build_comment'] = ''
d['noarch_python_comment'] = ''
if 'packagename' not in d:
d['packagename'] = pkginfo['name'].lower()
if d['version'] == 'UNKNOWN':
d['version'] = pkginfo['version']
if pkginfo['packages']:
deps = set(pkginfo['packages'])
if d['import_tests']:
if not d['import_tests'] or d['import_tests'] == 'PLACEHOLDER':
olddeps = []
else:
olddeps = [x for x in d['import_tests'].split() if x != '-']
deps = set(olddeps) | deps
d['import_tests'] = INDENT.join(sorted(deps))
d['import_comment'] = ''
d['tests_require'] = INDENT.join(
sorted([spec_from_line(pkg) for pkg in pkginfo['tests_require']]))
if pkginfo['homeurl'] is not None:
d['homeurl'] = pkginfo['homeurl']
else:
if data and 'homeurl' in data:
d['homeurl'] = data['homeurl']
else:
d['homeurl'] = "The package home page"
d['home_comment'] = '#'
if pkginfo['summary']:
d['summary'] = repr(pkginfo['summary'])
else:
if data:
d['summary'] = repr(data['summary'])
else:
d['summary'] = "Summary of the package"
d['summary_comment'] = '#'
if d['summary'].startswith("u'") or d['summary'].startswith('u"'):
d['summary'] = d['summary'][1:]
license_classifier = "License :: OSI Approved :: "
if pkginfo['classifiers']:
licenses = [
classifier.split(license_classifier, 1)[1]
for classifier in pkginfo['classifiers']
if classifier.startswith(license_classifier)
]
elif data and 'classifiers' in data:
licenses = [
classifier.split(license_classifier, 1)[1]
for classifier in data['classifiers']
if classifier.startswith(license_classifier)
]
else:
licenses = []
if not licenses:
if pkginfo['license']:
license = pkginfo['license']
elif data and 'license' in data:
license = data['license']
else:
license = None
if license:
if args.noprompt:
pass
elif '\n' not in license:
print('Using "%s" for the license' % license)
else:
# Some projects put the whole license text in this field
print("This is the license for %s" % package)
print()
print(license)
print()
license = input("What license string should I use? ")
else:
if args.noprompt:
license = "UNKNOWN"
else:
license = input(
("No license could be found for %s on " +
"PyPI or in the source. What license should I use? ") %
package)
else:
license = ' or '.join(licenses)
d['license'] = license
def get_package_metadata(args, package, d, data):
# Unfortunately, two important pieces of metadata are only stored in
# the package itself: the dependencies, and the entry points (if the
# package uses distribute). Our strategy is to download the package
# and "fake" distribute/setuptools's setup() function to get this
# information from setup.py. If this sounds evil, keep in mind that
# distribute itself already works by monkeypatching distutils.
import yaml
print("Downloading %s (use --no-download to skip this step)" % package)
tempdir = mkdtemp('conda_skeleton_' + d['filename'])
[output_dir] = args.output_dir
if not isdir(SRC_CACHE):
makedirs(SRC_CACHE)
try:
# Download it to the build source cache. That way, you have
# it.
download_path = join(SRC_CACHE, d['filename'])
if not isfile(download_path) or hashsum_file(download_path,
'md5') != d['md5']:
download(d['pypiurl'], join(SRC_CACHE, d['filename']))
else:
print("Using cached download")
print("Unpacking %s..." % package)
unpack(join(SRC_CACHE, d['filename']), tempdir)
print("done")
print("working in %s" % tempdir)
src_dir = get_dir(tempdir)
run_setuppy(src_dir, tempdir, args)
with open(join(tempdir, 'pkginfo.yaml')) as fn:
pkginfo = yaml.load(fn)
setuptools_build = pkginfo['setuptools']
setuptools_run = False
# Look at the entry_points and construct console_script and
# gui_scripts entry_points for conda
entry_points = pkginfo['entry_points']
if entry_points:
if isinstance(entry_points, str):
# makes sure it is left-shifted
newstr = "\n".join(x.strip()
for x in entry_points.split('\n'))
config = configparser.ConfigParser()
entry_points = {}
try:
config.readfp(StringIO(newstr))
except Exception as err:
print("WARNING: entry-points not understood: ",
err)
print("The string was", newstr)
entry_points = pkginfo['entry_points']
else:
setuptools_run = True
for section in config.sections():
if section in ['console_scripts', 'gui_scripts']:
value = ['%s=%s' % (option, config.get(section, option))
for option in config.options(section)]
entry_points[section] = value
if not isinstance(entry_points, dict):
print("WARNING: Could not add entry points. They were:")
print(entry_points)
else:
cs = entry_points.get('console_scripts', [])
gs = entry_points.get('gui_scripts', [])
if isinstance(cs, string_types):
cs = [cs]
if isinstance(gs, string_types):
gs = [gs]
# We have *other* kinds of entry-points so we need
# setuptools at run-time
if set(entry_points.keys()) - {'console_scripts', 'gui_scripts'}:
setuptools_build = True
setuptools_run = True
entry_list = (
cs
# TODO: Use pythonw for these
+ gs)
if len(cs + gs) != 0:
d['entry_points'] = INDENT.join([''] + entry_list)
d['entry_comment'] = ''
d['build_comment'] = ''
d['test_commands'] = INDENT.join([''] + make_entry_tests(entry_list))
# Look for package[extra,...] features spec:
match_extras = re.match(r'^([^[]+)\[([^]]+)\]$', package)
if match_extras:
package, extras = match_extras.groups()
extras = extras.split(',')
else:
extras = []
# Extract requested extra feature requirements...
if args.all_extras:
extras_require = list(pkginfo['extras_require'].values())
else:
try:
extras_require = [pkginfo['extras_require'][x] for x in extras]
except KeyError:
sys.exit("Error: Invalid extra features: [%s]"
% ','.join(extras))
#... and collect all needed requirement specs in a single list:
requires = []
for specs in [pkginfo['install_requires']] + extras_require:
if isinstance(specs, string_types):
requires.append(specs)
else:
requires.extend(specs)
if requires or setuptools_build or setuptools_run:
deps = []
if setuptools_run:
deps.append('setuptools')
for deptext in requires:
# Every item may be a single requirement
# or a multiline requirements string...
for dep in deptext.split('\n'):
#... and may also contain comments...
dep = dep.split('#')[0].strip()
if dep: #... and empty (or comment only) lines
spec = spec_from_line(dep)
if spec is None:
sys.exit("Error: Could not parse: %s" % dep)
deps.append(spec)
if 'setuptools' in deps:
setuptools_build = False
setuptools_run = False
d['egg_comment'] = ''
d['build_comment'] = ''
d['build_depends'] = INDENT.join([''] +
['setuptools'] * setuptools_build +
deps)
d['run_depends'] = INDENT.join([''] +
['setuptools'] * setuptools_run +
deps)
if args.recursive:
for dep in deps:
dep = dep.split()[0]
if not exists(join(output_dir, dep)):
args.packages.append(dep)
if 'packagename' not in d:
d['packagename'] = pkginfo['name'].lower()
if d['version'] == 'UNKNOWN':
d['version'] = pkginfo['version']
if pkginfo['packages']:
deps = set(pkginfo['packages'])
if d['import_tests']:
if not d['import_tests'] or d['import_tests'] == 'PLACEHOLDER':
olddeps = []
else:
olddeps = [x for x in d['import_tests'].split()
if x != '-']
deps = set(olddeps) | deps
d['import_tests'] = INDENT.join(sorted(deps))
d['import_comment'] = ''
if pkginfo['homeurl'] is not None:
d['homeurl'] = pkginfo['homeurl']
else:
if data:
d['homeurl'] = data['homeurl']
else:
d['homeurl'] = "The package home page"
d['home_comment'] = '#'
if pkginfo['summary']:
d['summary'] = repr(pkginfo['summary'])
else:
if data:
d['summary'] = repr(data['summary'])
else:
d['summary'] = "Summary of the package"
d['summary_comment'] = '#'
license_classifier = "License :: OSI Approved :: "
if pkginfo['classifiers']:
licenses = [classifier.split(license_classifier, 1)[1] for
classifier in pkginfo['classifiers'] if classifier.startswith(license_classifier)]
elif data and 'classifiers' in data:
licenses = [classifier.split(license_classifier, 1)[1] for classifier in
data['classifiers'] if classifier.startswith(license_classifier)]
else:
licenses = []
if not licenses:
if pkginfo['license']:
license = pkginfo['license']
elif data and 'license' in data:
license = data['license']
else:
license = None
if license:
if args.noprompt:
pass
elif '\n' not in license:
print('Using "%s" for the license' % license)
else:
# Some projects put the whole license text in this field
print("This is the license for %s" % package)
print()
print(license)
print()
license = input("What license string should I use? ")
else:
if args.noprompt:
license = "UNKNOWN"
else:
license = input(("No license could be found for %s on " +
"PyPI or in the source. What license should I use? ") %
package)
else:
license = ' or '.join(licenses)
d['license'] = license
finally:
rm_rf(tempdir)
def get_package_metadata(args, package, d, data):
print("Downloading %s" % package)
[output_dir] = args.output_dir
pkginfo = get_pkginfo(package,
filename=d['filename'],
pypiurl=d['pypiurl'],
md5=d['md5'],
python_version=args.python_version)
setuptools_build = pkginfo['setuptools']
setuptools_run = False
# Look at the entry_points and construct console_script and
# gui_scripts entry_points for conda
entry_points = pkginfo['entry_points']
if entry_points:
if isinstance(entry_points, str):
# makes sure it is left-shifted
newstr = "\n".join(x.strip()
for x in entry_points.split('\n'))
config = configparser.ConfigParser()
entry_points = {}
try:
config.readfp(StringIO(newstr))
except Exception as err:
print("WARNING: entry-points not understood: ",
err)
print("The string was", newstr)
entry_points = pkginfo['entry_points']
else:
setuptools_run = True
for section in config.sections():
if section in ['console_scripts', 'gui_scripts']:
value = ['%s=%s' % (option, config.get(section, option))
for option in config.options(section)]
entry_points[section] = value
if not isinstance(entry_points, dict):
print("WARNING: Could not add entry points. They were:")
print(entry_points)
else:
cs = entry_points.get('console_scripts', [])
gs = entry_points.get('gui_scripts', [])
if isinstance(cs, string_types):
cs = [cs]
if isinstance(gs, string_types):
gs = [gs]
# We have *other* kinds of entry-points so we need
# setuptools at run-time
if set(entry_points.keys()) - {'console_scripts', 'gui_scripts'}:
setuptools_build = True
setuptools_run = True
entry_list = (
cs
# TODO: Use pythonw for these
+ gs)
if len(cs + gs) != 0:
d['entry_points'] = INDENT.join([''] + entry_list)
d['entry_comment'] = ''
d['build_comment'] = ''
d['test_commands'] = INDENT.join([''] + make_entry_tests(entry_list))
requires = get_requirements(package, pkginfo, all_extras=args.all_extras)
if requires or setuptools_build or setuptools_run:
deps = []
if setuptools_run:
deps.append('setuptools')
for deptext in requires:
if isinstance(deptext, string_types):
deptext = deptext.split('\n')
# Every item may be a single requirement
# or a multiline requirements string...
for dep in deptext:
#... and may also contain comments...
dep = dep.split('#')[0].strip()
if dep: #... and empty (or comment only) lines
spec = spec_from_line(dep)
if spec is None:
sys.exit("Error: Could not parse: %s" % dep)
deps.append(spec)
if 'setuptools' in deps:
setuptools_build = False
setuptools_run = False
d['egg_comment'] = ''
d['build_comment'] = ''
d['build_depends'] = INDENT.join([''] +
['setuptools'] * setuptools_build +
deps)
d['run_depends'] = INDENT.join([''] +
['setuptools'] * setuptools_run +
deps)
if args.recursive:
for dep in deps:
dep = dep.split()[0]
if not exists(join(output_dir, dep)):
if dep not in args.created_recipes:
args.packages.append(dep)
if d['build_comment'] == '':
if args.noarch_python:
d['noarch_python_comment'] = ''
if 'packagename' not in d:
d['packagename'] = pkginfo['name'].lower()
if d['version'] == 'UNKNOWN':
d['version'] = pkginfo['version']
if pkginfo['packages']:
deps = set(pkginfo['packages'])
if d['import_tests']:
if not d['import_tests'] or d['import_tests'] == 'PLACEHOLDER':
olddeps = []
else:
olddeps = [x for x in d['import_tests'].split()
if x != '-']
deps = set(olddeps) | deps
d['import_tests'] = INDENT.join(sorted(deps))
d['import_comment'] = ''
d['tests_require'] = INDENT.join(sorted([spec_from_line(pkg) for pkg
in pkginfo['tests_require']]))
if pkginfo['homeurl'] is not None:
d['homeurl'] = pkginfo['homeurl']
else:
if data and 'homeurl' in data:
d['homeurl'] = data['homeurl']
else:
d['homeurl'] = "The package home page"
d['home_comment'] = '#'
if pkginfo['summary']:
d['summary'] = repr(pkginfo['summary'])
else:
if data:
d['summary'] = repr(data['summary'])
else:
d['summary'] = "Summary of the package"
d['summary_comment'] = '#'
if d['summary'].startswith("u'"):
d['summary'] = d['summary'][1:]
license_classifier = "License :: OSI Approved :: "
if pkginfo['classifiers']:
licenses = [classifier.split(license_classifier, 1)[1] for
classifier in pkginfo['classifiers'] if classifier.startswith(license_classifier)]
elif data and 'classifiers' in data:
licenses = [classifier.split(license_classifier, 1)[1] for classifier in
data['classifiers'] if classifier.startswith(license_classifier)]
else:
licenses = []
if not licenses:
if pkginfo['license']:
license = pkginfo['license']
elif data and 'license' in data:
license = data['license']
else:
license = None
if license:
if args.noprompt:
pass
elif '\n' not in license:
print('Using "%s" for the license' % license)
else:
# Some projects put the whole license text in this field
print("This is the license for %s" % package)
print()
print(license)
print()
license = input("What license string should I use? ")
else:
if args.noprompt:
license = "UNKNOWN"
else:
license = input(("No license could be found for %s on " +
"PyPI or in the source. What license should I use? ") %
package)
else:
license = ' or '.join(licenses)
d['license'] = license
def main(args, parser):
if len(args.packages) > 1 and args.download:
# Because if a package's setup.py imports setuptools, it will make all
# future packages look like they depend on distribute. Also, who knows
# what kind of monkeypatching the setup.pys out there could be doing.
print("WARNING: building more than one recipe at once without "
"--no-download is not recommended")
proxies = get_proxy_servers()
if proxies:
transport = RequestsTransport()
else:
transport = None
client = ServerProxy(args.pypi_url, transport=transport)
package_dicts = {}
[output_dir] = args.output_dir
indent = '\n - '
all_packages = client.list_packages()
all_packages_lower = [i.lower() for i in all_packages]
while args.packages:
package = args.packages.pop()
# Look for package[extra,...] features spec:
match_extras = re.match(r'^([^[]+)\[([^]]+)\]$', package)
if match_extras:
package, extras = match_extras.groups()
extras = extras.split(',')
else:
extras = []
dir_path = join(output_dir, package.lower())
if exists(dir_path):
raise RuntimeError("directory already exists: %s" % dir_path)
d = package_dicts.setdefault(package, {'packagename': package.lower(),
'run_depends': '',
'build_depends': '',
'entry_points': '',
'build_comment': '# ',
'test_commands': '',
'usemd5': '',
'entry_comment': '#',
'egg_comment': '#'})
d['import_tests'] = valid(package).lower()
if d['import_tests'] == '':
d['import_comment'] = '# '
else:
d['import_comment'] = ''
d['import_tests'] = indent + d['import_tests']
if args.version:
[version] = args.version
versions = client.package_releases(package, True)
if version not in versions:
sys.exit("Error: Version %s of %s is not available on PyPI."
% (version, package))
d['version'] = version
else:
versions = client.package_releases(package)
if not versions:
# The xmlrpc interface is case sensitive, but the index itself
# is apparently not (the last time I checked,
# len(set(all_packages_lower)) == len(set(all_packages)))
if package.lower() in all_packages_lower:
print("%s not found, trying %s" % (package, package.capitalize()))
args.packages.append(all_packages[all_packages_lower.index(package.lower())])
del package_dicts[package]
continue
sys.exit("Error: Could not find any versions of package %s" %
package)
if len(versions) > 1:
print("Warning, the following versions were found for %s" %
package)
for ver in versions:
print(ver)
print("Using %s" % versions[0])
print("Use --version to specify a different version.")
d['version'] = versions[0]
data = client.release_data(package, d['version'])
urls = client.release_urls(package, d['version'])
if not args.all_urls:
# Try to find source urls
urls = [url for url in urls if url['python_version'] == 'source']
if not urls:
if 'download_url' in data:
urls = [defaultdict(str, {'url': data['download_url']})]
urls[0]['filename'] = urls[0]['url'].split('/')[-1]
d['usemd5'] = '#'
else:
sys.exit("Error: No source urls found for %s" % package)
if len(urls) > 1 and not args.noprompt:
print("More than one source version is available for %s:" %
package)
for i, url in enumerate(urls):
print("%d: %s (%s) %s" % (i, url['url'],
human_bytes(url['size']),
url['comment_text']))
n = int(input("Which version should I use? "))
else:
n = 0
print("Using url %s (%s) for %s." % (urls[n]['url'],
human_bytes(urls[n]['size'] or 0),
package))
d['pypiurl'] = urls[n]['url']
d['md5'] = urls[n]['md5_digest']
d['filename'] = urls[n]['filename']
d['homeurl'] = data['home_page']
d['summary'] = repr(data['summary'])
license_classifier = "License :: OSI Approved ::"
if 'classifiers' in data:
licenses = [classifier.split(license_classifier, 1)[1] for classifier in
data['classifiers'] if classifier.startswith(license_classifier)]
else:
licenses = []
if not licenses:
if data['license']:
if args.noprompt:
license = data['license']
elif '\n' not in data['license']:
print('Using "%s" for the license' % data['license'])
license = data['license']
else:
# Some projects put the whole license text in this field
print("This is the license for %s" % package)
print()
print(data['license'])
print()
license = input("What license string should I use? ")
else:
if args.noprompt:
license = "UNKNOWN"
else:
license = input(("No license could be found for %s on " +
"PyPI. What license should I use? ") %
package)
else:
license = ' or '.join(licenses)
d['license'] = license
# Unfortunately, two important pieces of metadata are only stored in
# the package itself: the dependencies, and the entry points (if the
# package uses distribute). Our strategy is to download the package
# and "fake" distribute/setuptools's setup() function to get this
# information from setup.py. If this sounds evil, keep in mind that
# distribute itself already works by monkeypatching distutils.
if args.download:
import yaml
print("Downloading %s (use --no-download to skip this step)" %
package)
tempdir = mkdtemp('conda_skeleton_' + package)
if not isdir(SRC_CACHE):
makedirs(SRC_CACHE)
try:
# Download it to the build source cache. That way, you have
# it.
download_path = join(SRC_CACHE, d['filename'])
if not isfile(download_path) or hashsum_file(download_path,
'md5') != d['md5']:
download(d['pypiurl'], join(SRC_CACHE, d['filename']))
else:
print("Using cached download")
print("Unpacking %s..." % package)
unpack(join(SRC_CACHE, d['filename']), tempdir)
print("done")
print("working in %s" % tempdir)
src_dir = get_dir(tempdir)
run_setuppy(src_dir, tempdir, args)
with open(join(tempdir, 'pkginfo.yaml')) as fn:
pkginfo = yaml.load(fn)
setuptools_build = pkginfo['setuptools']
setuptools_run = False
# Look at the entry_points and construct console_script and
# gui_scripts entry_points for conda
entry_points = pkginfo['entry_points']
if entry_points:
if isinstance(entry_points, str):
# makes sure it is left-shifted
newstr = "\n".join(x.strip()
for x in entry_points.split('\n'))
config = configparser.ConfigParser()
entry_points = {}
try:
config.readfp(StringIO(newstr))
except Exception as err:
print("WARNING: entry-points not understood: ",
err)
print("The string was", newstr)
entry_points = pkginfo['entry_points']
else:
setuptools_run = True
for section in config.sections():
if section in ['console_scripts', 'gui_scripts']:
value = ['%s=%s' % (option, config.get(section, option))
for option in config.options(section)]
entry_points[section] = value
if not isinstance(entry_points, dict):
print("WARNING: Could not add entry points. They were:")
print(entry_points)
else:
cs = entry_points.get('console_scripts', [])
gs = entry_points.get('gui_scripts', [])
# We have *other* kinds of entry-points so we need
# setuptools at run-time
if not cs and not gs and len(entry_points) > 1:
setuptools_build = True
setuptools_run = True
entry_list = (
cs
# TODO: Use pythonw for these
+ gs)
if len(cs + gs) != 0:
d['entry_points'] = indent.join([''] + entry_list)
d['entry_comment'] = ''
d['build_comment'] = ''
d['test_commands'] = indent.join([''] + make_entry_tests(entry_list))
# Extract requested extra feature requirements...
if args.all_extras:
extras_require = list(pkginfo['extras_require'].values())
else:
try:
extras_require = [pkginfo['extras_require'][x] for x in extras]
except KeyError:
sys.exit("Error: Invalid extra features: [%s]"
% ','.join(extras))
#... and collect all needed requirement specs in a single list:
requires = []
for specs in [pkginfo['install_requires']] + extras_require:
if isinstance(specs, string_types):
requires.append(specs)
else:
requires.extend(specs)
if requires or setuptools_build or setuptools_run:
deps = []
for deptext in requires:
# Every item may be a single requirement
# or a multiline requirements string...
for dep in deptext.split('\n'):
#... and may also contain comments...
dep = dep.split('#')[0].strip()
if dep: #... and empty (or comment only) lines
spec = spec_from_line(dep)
if spec is None:
sys.exit("Error: Could not parse: %s" % dep)
deps.append(spec)
if 'setuptools' in deps:
setuptools_build = False
setuptools_run = False
d['egg_comment'] = ''
d['build_comment'] = ''
d['build_depends'] = indent.join([''] +
['setuptools'] * setuptools_build +
deps)
d['run_depends'] = indent.join([''] +
['setuptools'] * setuptools_run +
deps)
if args.recursive:
for dep in deps:
dep = dep.split()[0]
if not exists(join(output_dir, dep)):
args.packages.append(dep)
if pkginfo['packages']:
deps = set(pkginfo['packages'])
if d['import_tests']:
olddeps = [x for x in d['import_tests'].split()
if x != '-']
deps = set(olddeps) | deps
d['import_tests'] = indent.join([''] + sorted(deps))
d['import_comment'] = ''
finally:
rm_rf(tempdir)
for package in package_dicts:
d = package_dicts[package]
makedirs(join(output_dir, package.lower()))
print("Writing recipe for %s" % package.lower())
with open(join(output_dir, package.lower(), 'meta.yaml'), 'w') as f:
f.write(PYPI_META.format(**d))
with open(join(output_dir, package.lower(), 'build.sh'), 'w') as f:
f.write(PYPI_BUILD_SH.format(**d))
with open(join(output_dir, package.lower(), 'bld.bat'), 'w') as f:
f.write(PYPI_BLD_BAT.format(**d))
print("Done")
def main(args, parser):
if len(args.packages) > 1 and args.download:
# Because if a package's setup.py imports setuptools, it will make all
# future packages look like they depend on distribute. Also, who knows
# what kind of monkeypatching the setup.pys out there could be doing.
print("WARNING: building more than one recipe at once without "
"--no-download is not recommended")
proxies = get_proxy_servers()
if proxies:
transport = RequestsTransport()
else:
transport = None
client = ServerProxy(args.pypi_url, transport=transport)
package_dicts = {}
[output_dir] = args.output_dir
indent = '\n - '
all_packages = client.list_packages()
all_packages_lower = [i.lower() for i in all_packages]
while args.packages:
package = args.packages.pop()
# Look for package[extra,...] features spec:
match_extras = re.match(r'^([^[]+)\[([^]]+)\]$', package)
if match_extras:
package, extras = match_extras.groups()
extras = extras.split(',')
else:
extras = []
dir_path = join(output_dir, package.lower())
if exists(dir_path):
raise RuntimeError("directory already exists: %s" % dir_path)
d = package_dicts.setdefault(package, {'packagename': package.lower(),
'run_depends': '',
'build_depends': '',
'entry_points': '',
'build_comment': '# ',
'test_commands': '',
'usemd5': '',
'entry_comment': '#',
'egg_comment': '#'})
d['import_tests'] = valid(package).lower()
if d['import_tests'] == '':
d['import_comment'] = '# '
else:
d['import_comment'] = ''
d['import_tests'] = indent + d['import_tests']
if args.version:
[version] = args.version
versions = client.package_releases(package, True)
if version not in versions:
sys.exit("Error: Version %s of %s is not available on PyPI."
% (version, package))
d['version'] = version
else:
versions = client.package_releases(package)
if not versions:
# The xmlrpc interface is case sensitive, but the index itself
# is apparently not (the last time I checked,
# len(set(all_packages_lower)) == len(set(all_packages)))
if package.lower() in all_packages_lower:
print("%s not found, trying %s" % (package, package.capitalize()))
args.packages.append(all_packages[all_packages_lower.index(package.lower())])
del package_dicts[package]
continue
sys.exit("Error: Could not find any versions of package %s" %
package)
if len(versions) > 1:
print("Warning, the following versions were found for %s" %
package)
for ver in versions:
print(ver)
print("Using %s" % versions[0])
print("Use --version to specify a different version.")
d['version'] = versions[0]
data = client.release_data(package, d['version'])
urls = client.release_urls(package, d['version'])
if not args.all_urls:
# Try to find source urls
urls = [url for url in urls if url['python_version'] == 'source']
if not urls:
if 'download_url' in data:
urls = [defaultdict(str, {'url': data['download_url']})]
urls[0]['filename'] = urls[0]['url'].split('/')[-1]
d['usemd5'] = '#'
else:
sys.exit("Error: No source urls found for %s" % package)
if len(urls) > 1 and not args.noprompt:
print("More than one source version is available for %s:" %
package)
for i, url in enumerate(urls):
print("%d: %s (%s) %s" % (i, url['url'],
human_bytes(url['size']),
url['comment_text']))
n = int(input("Which version should I use? "))
else:
n = 0
print("Using url %s (%s) for %s." % (urls[n]['url'],
human_bytes(urls[n]['size'] or 0),
package))
d['pypiurl'] = urls[n]['url']
d['md5'] = urls[n]['md5_digest']
d['filename'] = urls[n]['filename']
d['homeurl'] = data['home_page']
d['summary'] = repr(data['summary'])
license_classifier = "License :: OSI Approved ::"
if 'classifiers' in data:
licenses = [classifier.split(license_classifier, 1)[1] for classifier in
data['classifiers'] if classifier.startswith(license_classifier)]
else:
licenses = []
if not licenses:
if data['license']:
if args.noprompt:
license = data['license']
elif '\n' not in data['license']:
print('Using "%s" for the license' % data['license'])
license = data['license']
else:
# Some projects put the whole license text in this field
print("This is the license for %s" % package)
print()
print(data['license'])
print()
license = input("What license string should I use? ")
else:
if args.noprompt:
license = "UNKNOWN"
else:
license = input(("No license could be found for %s on " +
"PyPI. What license should I use? ") %
package)
else:
license = ' or '.join(licenses)
d['license'] = license
# Unfortunately, two important pieces of metadata are only stored in
# the package itself: the dependencies, and the entry points (if the
# package uses distribute). Our strategy is to download the package
# and "fake" distribute/setuptools's setup() function to get this
# information from setup.py. If this sounds evil, keep in mind that
# distribute itself already works by monkeypatching distutils.
if args.download:
import yaml
print("Downloading %s (use --no-download to skip this step)" %
package)
tempdir = mkdtemp('conda_skeleton_' + package)
if not isdir(SRC_CACHE):
makedirs(SRC_CACHE)
try:
# Download it to the build source cache. That way, you have
# it.
download_path = join(SRC_CACHE, d['filename'])
if not isfile(download_path) or hashsum_file(download_path,
'md5') != d['md5']:
download(d['pypiurl'], join(SRC_CACHE, d['filename']))
else:
print("Using cached download")
print("Unpacking %s..." % package)
unpack(join(SRC_CACHE, d['filename']), tempdir)
print("done")
print("working in %s" % tempdir)
src_dir = get_dir(tempdir)
run_setuppy(src_dir, tempdir, args)
with open(join(tempdir, 'pkginfo.yaml'), encoding='utf-8') as fn:
pkginfo = yaml.load(fn)
setuptools_build = pkginfo['setuptools']
setuptools_run = False
# Look at the entry_points and construct console_script and
# gui_scripts entry_points for conda
entry_points = pkginfo['entry_points']
if entry_points:
if isinstance(entry_points, str):
# makes sure it is left-shifted
newstr = "\n".join(x.strip()
for x in entry_points.split('\n'))
config = configparser.ConfigParser()
entry_points = {}
try:
config.readfp(StringIO(newstr))
except Exception as err:
print("WARNING: entry-points not understood: ",
err)
print("The string was", newstr)
entry_points = pkginfo['entry_points']
else:
setuptools_run = True
for section in config.sections():
if section in ['console_scripts', 'gui_scripts']:
value = ['%s=%s' % (option, config.get(section, option))
for option in config.options(section)]
entry_points[section] = value
if not isinstance(entry_points, dict):
print("WARNING: Could not add entry points. They were:")
print(entry_points)
else:
cs = entry_points.get('console_scripts', [])
gs = entry_points.get('gui_scripts', [])
# We have *other* kinds of entry-points so we need
# setuptools at run-time
if not cs and not gs and len(entry_points) > 1:
setuptools_build = True
setuptools_run = True
entry_list = (
cs
# TODO: Use pythonw for these
+ gs)
if len(cs + gs) != 0:
d['entry_points'] = indent.join([''] + entry_list)
d['entry_comment'] = ''
d['build_comment'] = ''
d['test_commands'] = indent.join([''] + make_entry_tests(entry_list))
# Extract requested extra feature requirements...
if args.all_extras:
extras_require = list(pkginfo['extras_require'].values())
else:
try:
extras_require = [pkginfo['extras_require'][x] for x in extras]
except KeyError:
sys.exit("Error: Invalid extra features: [%s]"
% ','.join(extras))
#... and collect all needed requirement specs in a single list:
requires = []
for specs in [pkginfo['install_requires']] + extras_require:
if isinstance(specs, string_types):
requires.append(specs)
else:
requires.extend(specs)
if requires or setuptools_build or setuptools_run:
deps = []
for deptext in requires:
# Every item may be a single requirement
# or a multiline requirements string...
for dep in deptext.split('\n'):
#... and may also contain comments...
dep = dep.split('#')[0].strip()
if dep: #... and empty (or comment only) lines
spec = spec_from_line(dep)
if spec is None:
sys.exit("Error: Could not parse: %s" % dep)
deps.append(spec)
if 'setuptools' in deps:
setuptools_build = False
setuptools_run = False
d['egg_comment'] = ''
d['build_comment'] = ''
d['build_depends'] = indent.join([''] +
['setuptools'] * setuptools_build +
deps)
d['run_depends'] = indent.join([''] +
['setuptools'] * setuptools_run +
deps)
if args.recursive:
for dep in deps:
dep = dep.split()[0]
if not exists(join(output_dir, dep)):
args.packages.append(dep)
if pkginfo['packages']:
deps = set(pkginfo['packages'])
if d['import_tests']:
olddeps = [x for x in d['import_tests'].split()
if x != '-']
deps = set(olddeps) | deps
d['import_tests'] = indent.join([''] + sorted(deps))
d['import_comment'] = ''
finally:
rm_rf(tempdir)
for package in package_dicts:
d = package_dicts[package]
makedirs(join(output_dir, package.lower()))
print("Writing recipe for %s" % package.lower())
with open(join(output_dir, package.lower(), 'meta.yaml'), 'w',
encoding='utf-8') as f:
f.write(PYPI_META.format(**d))
with open(join(output_dir, package.lower(), 'build.sh'), 'w',
encoding='utf-8') as f:
f.write(PYPI_BUILD_SH.format(**d))
with open(join(output_dir, package.lower(), 'bld.bat'), 'w',
encoding='utf-8') as f:
f.write(PYPI_BLD_BAT.format(**d))
print("Done")