def add_requirement(self, requirement, ctx, transaction):
req = util.parse_requirement(requirement)
# If it's a Pyodide package, use that instead of the one on PyPI
if req.name in self.builtin_packages:
transaction['pyodide_packages'].add(req.name)
return
if req.marker:
if not markers.evaluator.evaluate(req.marker, ctx):
return
matcher = self.version_scheme.matcher(req.requirement)
# If we already have something that will work, don't
# fetch again
for name, ver in transaction['locked'].items():
if name == req.name:
if matcher.match(ver):
break
else:
raise ValueError(f"Requested '{requirement}', "
f"but {name}=={ver} is already installed")
else:
metadata = _get_pypi_json(req.name)
wheel, ver = self.find_wheel(metadata, req)
transaction['locked'][req.name] = ver
recurs_reqs = metadata.get('info', {}).get('requires_dist') or []
for recurs_req in recurs_reqs:
self.add_requirement(recurs_req, ctx, transaction)
transaction['wheels'].append((req.name, wheel, ver))
def create_dependency_descriptor(requirement_string):
"""
Create a DependencyDescriptor from a PEP440 compliant string.
See https://www.python.org/dev/peps/pep-0440/#version-specifiers
:param str requirement_string: a PEP440 compliant requirement string
:return: A descriptor with version constraints from the requirement string
:rtype: DependencyDescriptor
"""
symbol_mapping = {
'==': 'version_eq',
'!=': 'version_neq',
'<=': 'version_lte',
'>=': 'version_gte',
'>': 'version_gt',
'<': 'version_lt',
}
requirement = parse_requirement(requirement_string)
metadata = {}
for symbol, version in (requirement.constraints or []):
if symbol in symbol_mapping:
metadata[symbol_mapping[symbol]] = version
elif symbol == '~=':
metadata['version_gte'] = version
metadata['version_lt'] = _next_incompatible_version(version)
else:
logger.warn(
"Ignoring unknown symbol '{symbol}' in '{requirement}'".format(
locals()))
return DependencyDescriptor(requirement.name, metadata=metadata)
def safe_constraints(spec):
if is_url(spec):
return None
constraints = util.parse_requirement(spec).constraints or ()
constraint = lambda k, v: \
('{0} {1}'.format(k, v)
.replace('== ', '')
.replace('==', ''))
return ', '.join(constraint(k, v) for k, v in constraints) or None
def safe_constraints(spec):
if is_url(spec):
return None
constraints = util.parse_requirement(spec).constraints or ()
constraint = lambda k, v: \
('{0} {1}'.format(k, v)
.replace('== ', '')
.replace('==', ''))
return ', '.join(constraint(k, v) for k, v in constraints) or None
def read_requirment_txt(requirment_txt_path):
with open(requirment_txt_path, 'r') as f:
content = f.read()
package_list = content.splitlines()
parsed_package_list = {}
for item in package_list:
result = parse_requirement(item)
parsed_package_list[result.name] = result
return parsed_package_list
def safe_requirement(requirement):
safe = requirement.lower().replace("_", "-")
parsed = util.parse_requirement(safe)
output = parsed.name
if parsed.extras:
output += "[{0}]".format(",".join(parsed.extras))
if parsed.constraints:
def c(operator, version):
return version if operator == "==" else "{0} {1}".format(operator, version)
output += " ({0})".format(", ".join(c(*i) for i in parsed.constraints))
return output
def parse_requirement(spec):
if not is_url(spec):
requirement = util.parse_requirement(spec)
requirement.name = safe_name(requirement.name)
requirement.requirement = safe_requirement(spec)
requirement.is_link = False
else:
requirement = Requirement()
requirement.name = spec
requirement.requirement = spec
requirement.constraints = ()
requirement.is_link = True
return requirement
def safe_requirement(requirement):
safe = requirement.lower().replace('_', '-')
parsed = util.parse_requirement(safe)
output = parsed.name
if parsed.extras:
output += '[{0}]'.format(','.join(parsed.extras))
if parsed.constraints:
def c(operator, version):
return version if operator == '==' \
else '{0} {1}'.format(operator, version)
output += ' ({0})'.format(
', '.join(c(*i) for i in parsed.constraints))
return output
def parse_requirement(spec):
if not is_url(spec):
requirement = util.parse_requirement(spec)
requirement.name = safe_name(requirement.name)
requirement.requirement = safe_requirement(spec)
requirement.is_link = False
else:
requirement = Requirement()
requirement.name = spec
requirement.requirement = spec
requirement.constraints = ()
requirement.is_link = True
return requirement
def safe_requirement(requirement):
safe = requirement.lower().replace('_', '-')
parsed = util.parse_requirement(safe)
output = parsed.name
if parsed.extras:
output += '[{0}]'.format(','.join(parsed.extras))
if parsed.constraints:
def c(operator, version):
return version if operator == '==' \
else '{0} {1}'.format(operator, version)
output += ' ({0})'.format(', '.join(c(*i) for i in parsed.constraints))
return output
def test_parse_requirement(self):
# Empty requirements
for empty in ('', '#this should be ignored'):
self.assertIsNone(parse_requirement(empty))
# Invalid requirements
for invalid in ('a (', 'a/', 'a$', 'a [', 'a () [],', 'a 1.2'):
self.assertRaises(SyntaxError, parse_requirement, invalid)
# Valid requirements
def validate(r, values):
self.assertEqual(r.name, values[0])
self.assertEqual(r.constraints, values[1])
self.assertEqual(r.extras, values[2])
self.assertEqual(r.requirement, values[3])
self.assertEqual(r.url, values[4])
r = parse_requirement('a')
validate(r, ('a', None, None, 'a', None))
r = parse_requirement('a >= 1.2, <2.0,!=1.7')
validate(r,
('a', [('>=', '1.2'), ('<', '2.0'),
('!=', '1.7')], None, 'a >= 1.2, < 2.0, != 1.7', None))
r = parse_requirement('a [ab,cd , ef] >= 1.2, <2.0')
validate(
r,
('a', [('>=', '1.2'),
('<', '2.0')], ['ab', 'cd', 'ef'], 'a >= 1.2, < 2.0', None))
r = parse_requirement('a[]')
validate(r, ('a', None, None, 'a', None))
r = parse_requirement('a (== 1.2.*, != 1.2.1.*)')
validate(
r,
('a', [('==', '1.2.*'),
('!=', '1.2.1.*')], None, 'a == 1.2.*, != 1.2.1.*', None))
r = parse_requirement('a @ http://domain.com/path#abc=def')
validate(r, ('a', None, None, 'a', 'http://domain.com/path#abc=def'))
if False: # TODO re-enable
for e in ('*', ':*:', ':meta:', '-', '-abc'):
r = parse_requirement('a [%s]' % e)
validate(r, ('a', None, [e], 'a', None))
def __init__(self, spec):
"""
Takes a line from a requirements file or a single argument from the
command line and parses, determining constraints. Also contains helper
to download the requirement.
"""
# needs to handle
# tarball input
# packagename
# packagename==xxx
# packagename>=xxx
# packagename<=xxx
# packagename>xxx
# packagename<xxx
# git+https://myvcs.com/some_dependency@sometag#egg=SomeDependency
# -i, --index-url
# --extra-index-url
# --no-index
# -f, --find-links
# --allow-external
# --allow-all-external
# --allow-unverified
# --no-binary
# --only-binary
# Check if the requriement is a tarball
tarballs = [pkg for pkg in args.packages if os.path.isfile(pkg) and acceptable_file_type(pkg)]
args.packages = [pkg for pkg in args.packages if pkg not in tarballs]
if not is_url(spec):
requirement = util.parse_requirement(spec)
requirement.name = safe_name(requirement.name)
requirement.requirement = safe_requirement(spec)
requirement.is_link = False
else:
requirement = Requirement()
requirement.name = spec
requirement.requirement = spec
requirement.constraints = ()
requirement.is_link = True
requirement.extras = ()
return requirement
def add_requirement(self, requirement: str, ctx, transaction):
if requirement.startswith(("http://", "https://")):
# custom download location
name, wheel, version = _parse_wheel_url(requirement)
transaction["wheels"].append((name, wheel, version))
return
req = util.parse_requirement(requirement)
# If it's a Pyodide package, use that instead of the one on PyPI
if req.name in self.builtin_packages:
transaction["pyodide_packages"].add(req.name)
return
if req.marker:
if not markers.evaluator.evaluate(req.marker, ctx):
return
matcher = self.version_scheme.matcher(req.requirement)
# If we already have something that will work, don't
# fetch again
for name, ver in transaction["locked"].items():
if name == req.name:
if matcher.match(ver):
break
else:
raise ValueError(
f"Requested '{requirement}', "
f"but {name}=={ver} is already installed"
)
else:
metadata = _get_pypi_json(req.name)
wheel, ver = self.find_wheel(metadata, req)
transaction["locked"][req.name] = ver
recurs_reqs = metadata.get("info", {}).get("requires_dist") or []
for recurs_req in recurs_reqs:
self.add_requirement(recurs_req, ctx, transaction)
transaction["wheels"].append((req.name, wheel, ver))
def build_wheel(distname, options):
result = None
r = parse_requirement(distname)
if not r:
print('Invalid requirement: %r' % distname)
else:
dist = INSTALLED_DISTS.get_distribution(r.name)
if dist:
print('Can\'t build a wheel from already-installed '
'distribution %s' % dist.name_and_version)
else:
workdir = tempfile.mkdtemp() # where the Wheel input files will live
try:
paths = install_dist(distname, workdir, options)
paths['prefix'] = workdir
wheel = Wheel()
wheel.name = paths.pop('name')
wheel.version = paths.pop('version')
wheel.dirname = options.destdir
wheel.build(paths)
result = wheel
finally:
shutil.rmtree(workdir)
return result
def test_parse_requirement(self):
#Invalid requirements
self.assertIsNone(parse_requirement(''))
self.assertIsNone(parse_requirement('a ('))
self.assertIsNone(parse_requirement('a/'))
self.assertIsNone(parse_requirement('a$'))
self.assertIsNone(parse_requirement('a ()'))
self.assertIsNone(parse_requirement('a ['))
self.assertIsNone(parse_requirement('a () []'))
# Valid requirements
def validate(r, values):
self.assertEqual(r.name, values[0])
self.assertEqual(r.constraints, values[1])
self.assertEqual(r.extras, values[2])
self.assertEqual(r.requirement, values[3])
self.assertEqual(r.url, values[4])
r = parse_requirement('a')
validate(r, ('a', None, None, 'a', None))
r = parse_requirement('a 1.2')
validate(r, ('a', [('~=', '1.2')], None, 'a (~= 1.2)', None))
r = parse_requirement('a >= 1.2, <2.0,!=1.7')
validate(
r, ('a', [('>=', '1.2'), ('<', '2.0'),
('!=', '1.7')], None, 'a (>= 1.2, < 2.0, != 1.7)', None))
r = parse_requirement('a [ab,cd , ef] >= 1.2, <2.0')
validate(r, ('a', [('>=', '1.2'), ('<', '2.0')
], ['ab', 'cd', 'ef'], 'a (>= 1.2, < 2.0)', None))
r = parse_requirement('a[]')
validate(r, ('a', None, None, 'a', None))
r = parse_requirement('a (== 1.2.*, != 1.2.1.*)')
validate(
r,
('a', [('==', '1.2.*'),
('!=', '1.2.1.*')], None, 'a (== 1.2.*, != 1.2.1.*)', None))
r = parse_requirement('a (from http://domain.com/path#abc=def )')
validate(r, ('a', None, None, 'a', 'http://domain.com/path#abc=def'))
for e in ('*', ':*:', ':meta:', '-', '-abc'):
r = parse_requirement('a [%s]' % e)
validate(r, ('a', None, [e], 'a', None))
def test_parse_requirement(self):
#Invalid requirements
self.assertIsNone(parse_requirement(''))
self.assertIsNone(parse_requirement('a ('))
self.assertIsNone(parse_requirement('a/'))
self.assertIsNone(parse_requirement('a$'))
self.assertIsNone(parse_requirement('a ()'))
self.assertIsNone(parse_requirement('a ['))
self.assertIsNone(parse_requirement('a () []'))
# Valid requirements
def validate(r, values):
self.assertEqual(r.name, values[0])
self.assertEqual(r.constraints, values[1])
self.assertEqual(r.extras, values[2])
self.assertEqual(r.requirement, values[3])
r = parse_requirement('a')
validate(r, ('a', None, None, 'a'))
r = parse_requirement('a 1.2')
validate(r, ('a', [('==', '1.2')], None, 'a (== 1.2)'))
r = parse_requirement('a >= 1.2, <2.0,!=1.7')
validate(r, ('a', [('>=', '1.2'), ('<', '2.0'), ('!=', '1.7')], None,
'a (>= 1.2, < 2.0, != 1.7)'))
r = parse_requirement('a [ab,cd , ef] >= 1.2, <2.0')
validate(r, ('a', [('>=', '1.2'), ('<', '2.0')], ['ab', 'cd', 'ef'],
'a (>= 1.2, < 2.0)'))
r = parse_requirement('a[]')
validate(r, ('a', None, None, 'a'))
def safe_requirement(requirement):
return (util.parse_requirement(requirement).requirement
.replace('== ', '')
.replace('==', ''))
def test_parse_requirement(self):
#Invalid requirements
self.assertIsNone(parse_requirement(''))
self.assertIsNone(parse_requirement('a ('))
self.assertIsNone(parse_requirement('a/'))
self.assertIsNone(parse_requirement('a$'))
self.assertIsNone(parse_requirement('a ()'))
self.assertIsNone(parse_requirement('a ['))
self.assertIsNone(parse_requirement('a () []'))
# Valid requirements
def validate(r, values):
self.assertEqual(r.name, values[0])
self.assertEqual(r.constraints, values[1])
self.assertEqual(r.extras, values[2])
self.assertEqual(r.requirement, values[3])
self.assertEqual(r.url, values[4])
r = parse_requirement('a')
validate(r, ('a', None, None, 'a', None))
r = parse_requirement('a 1.2')
validate(r, ('a', [('~=', '1.2')], None, 'a (~= 1.2)', None))
r = parse_requirement('a >= 1.2, <2.0,!=1.7')
validate(r, ('a', [('>=', '1.2'), ('<', '2.0'), ('!=', '1.7')], None,
'a (>= 1.2, < 2.0, != 1.7)', None))
r = parse_requirement('a [ab,cd , ef] >= 1.2, <2.0')
validate(r, ('a', [('>=', '1.2'), ('<', '2.0')], ['ab', 'cd', 'ef'],
'a (>= 1.2, < 2.0)', None))
r = parse_requirement('a[]')
validate(r, ('a', None, None, 'a', None))
r = parse_requirement('a (== 1.2.*, != 1.2.1.*)')
validate(r, ('a', [('==', '1.2.*'), ('!=', '1.2.1.*')], None,
'a (== 1.2.*, != 1.2.1.*)', None))
r = parse_requirement('a (from http://domain.com/path#abc=def )')
validate(r, ('a', None, None, 'a', 'http://domain.com/path#abc=def'))
for e in ('*', ':*:', ':meta:', '-', '-abc'):
r = parse_requirement('a [%s]' % e)
validate(r, ('a', None, [e], 'a', None))
def safe_requirement(requirement):
return (util.parse_requirement(requirement).requirement.replace(
'== ', '').replace('==', ''))
def safe_constraints(spec):
if is_url(spec):
return None
constraints = util.parse_requirement(spec).constraints or ()
constraint = lambda k, v: ("{0} {1}".format(k, v).replace("== ", "").replace("==", ""))
return ", ".join(constraint(k, v) for k, v in constraints) or None
def safe_requirement(requirement):
return util.parse_requirement(requirement).requirement.replace("== ", "").replace("==", "")
def main(args=None):
parser = optparse.OptionParser(usage='%prog [options] requirement [requirement ...]')
parser.add_option('-d', '--dest', dest='destdir', metavar='DESTDIR',
default=os.getcwd(), help='Where you want the wheels '
'to be put.')
parser.add_option('-n', '--no-deps', dest='deps', default=True,
action='store_false',
help='Don\'t build dependent wheels.')
options, args = parser.parse_args(args)
options.compatible = True # may add flag to turn off later
if not args:
parser.print_usage()
else:
# Check if pip is available; no point in continuing, otherwise
try:
with open(os.devnull, 'w') as f:
p = subprocess.call(['pip', '--version'], stdout=f, stderr=subprocess.STDOUT)
except Exception:
p = 1
if p:
print('pip appears not to be available. Wheeler needs pip to '
'build wheels.')
return 1
if options.deps:
# collect all the requirements, including dependencies
u = 'http://pypi.python.org/simple/'
locator = AggregatingLocator(JSONLocator(),
SimpleScrapingLocator(u, timeout=3.0),
scheme='legacy')
finder = DependencyFinder(locator)
wanted = set()
for arg in args:
r = parse_requirement(arg)
if not r.constraints:
dname = r.name
else:
dname = '%s (%s)' % (r.name, ', '.join(r.constraints))
print('Finding the dependencies of %s ...' % arg)
dists, problems = finder.find(dname)
if problems:
print('There were some problems resolving dependencies '
'for %r.' % arg)
for _, info in problems:
print(' Unsatisfied requirement %r' % info)
wanted |= dists
want_ordered = True # set to False to skip ordering
if not want_ordered:
wanted = list(wanted)
else:
graph = make_graph(wanted, scheme=locator.scheme)
slist, cycle = graph.topological_sort()
if cycle:
# Now sort the remainder on dependency count.
cycle = sorted(cycle, reverse=True,
key=lambda d: len(graph.reverse_list[d]))
wanted = slist + cycle
# get rid of any installed distributions from the list
for w in list(wanted):
dist = INSTALLED_DISTS.get_distribution(w.name)
if dist or w.name in ('setuptools', 'distribute'):
wanted.remove(w)
s = w.name_and_version
print('Skipped already-installed distribution %s' % s)
# converted wanted list to pip-style requirements
args = ['%s==%s' % (dist.name, dist.version) for dist in wanted]
# Now go build
built = []
for arg in args:
wheel = build_wheel(arg, options)
if wheel:
built.append(wheel)
if built:
if options.destdir == os.getcwd():
dest = ''
else:
dest = ' in %s' % options.destdir
print('The following wheels were built%s:' % dest)
for wheel in built:
print(' %s' % wheel.filename)
