def get_python(config: Config) -> Python:
"""Get prefered Python.
Lookup order:
1. Defined in config.
2. Defined in dependencies file.
3. Current active Python.
Use it when you want to create new venv.
"""
pythons = Pythons()
# defined in config
python = config.get('python')
if python:
return pythons.get_best(python)
# defined in dependencies file
if 'from' in config:
loader = CONVERTERS[config['from']['format']]
root = loader.load(path=config['from']['path'])
if root.python:
return pythons.get_by_spec(root.python)
return pythons.current
def dumps(self, reqs, project: RootDependency, content=None) -> str:
if content:
doc = tomlkit.parse(content)
else:
doc = tomlkit.document()
section = doc['source'] if 'source' in doc else tomlkit.aot()
added_repos = {repo['name'] for repo in section}
updated = False
for req in reqs:
if not isinstance(req.dep.repo, RepositoriesRegistry):
continue
for repo in req.dep.repo.repos:
if repo.name in added_repos:
continue
# https://github.com/pypa/pipenv/issues/2231
if isinstance(repo, WarehouseLocalRepo):
continue
added_repos.add(repo.name)
source = tomlkit.table()
source['name'] = repo.name
source['url'] = repo.pretty_url
source['verify_ssl'] = repo.pretty_url.startswith('https://')
section.append(source)
updated = True
if updated:
doc['source'] = section
if project.python:
python = Pythons(abstract=True).get_by_spec(project.python)
if 'requires' not in doc:
doc['requires'] = tomlkit.table()
doc['requires']['python_version'] = str(python.get_short_version())
for section, is_dev in [('packages', False), ('dev-packages', True)]:
# create section if doesn't exist
if section not in doc:
doc[section] = tomlkit.table()
continue
# clean packages from old packages
names = {req.name for req in reqs if is_dev is req.is_dev}
for name in doc[section]:
if name not in names:
del doc[section][name]
# write new packages
for section, is_dev in [('packages', False), ('dev-packages', True)]:
for req in reqs:
if is_dev is req.is_dev:
doc[section][req.raw_name] = self._format_req(req=req)
return tomlkit.dumps(doc).rstrip() + '\n'
def dumps(self, reqs, project: RootDependency, content=None) -> str:
if content:
doc = tomlkit.parse(content)
else:
doc = tomlkit.document()
if 'source' not in doc:
doc['source'] = tomlkit.aot()
added_repos = {repo['name'] for repo in doc['source']}
for req in reqs:
if not isinstance(req.dep.repo, WareHouseRepo):
continue
if req.dep.repo.name in added_repos:
continue
added_repos.add(req.dep.repo.name)
doc['source'].append(
OrderedDict([
('name', req.dep.repo.name),
('url', req.dep.repo.pretty_url),
('verify_ssl', True),
]))
if project.python:
python = Pythons(abstract=True).get_by_spec(project.python)
if 'requires' not in doc:
doc['requires'] = tomlkit.table()
doc['requires']['python_version'] = str(python.get_short_version())
for section, is_dev in [('packages', False), ('dev-packages', True)]:
# create section if doesn't exist
if section not in doc:
doc[section] = tomlkit.table()
continue
# clean packages from old packages
names = {req.name for req in reqs if is_dev is req.is_dev}
for name in doc[section]:
if name not in names:
del doc[section][name]
# write new packages
for section, is_dev in [('packages', False), ('dev-packages', True)]:
for req in reqs:
if is_dev is req.is_dev:
doc[section][req.name] = self._format_req(req=req)
return tomlkit.dumps(doc)
def dumps(self, reqs, project: RootDependency, content=None) -> str:
packages = {True: OrderedDict(), False: OrderedDict()}
for req in reqs:
packages[req.is_dev][req.name] = dict(self._format_req(req=req))
python = Pythons(abstract=True).get_by_spec(project.python)
data = OrderedDict([
('_meta',
OrderedDict([
('sources', [
OrderedDict([
('url', 'https://pypi.python.org/simple'),
('verify_ssl', True),
('name', 'pypi'),
])
]),
('requires', {
'python_version': str(python.version)
}),
])),
('default', packages[False]),
('develop', packages[True]),
])
data['_meta']['hash'] = {'sha256': self._get_hash(data)}
data['_meta']['pipfile-spec'] = 6
return json.dumps(data, indent=4, separators=(',', ': '))
def dumps(self, reqs, project: RootDependency, content=None) -> str:
packages = {True: OrderedDict(), False: OrderedDict()}
for req in reqs:
packages[req.is_dev][req.raw_name] = dict(
self._format_req(req=req))
# get repos
repos = []
added_repos = set()
for req in reqs:
if not isinstance(req.dep.repo, WarehouseBaseRepo):
continue
for repo in req.dep.repo.repos:
if repo.from_config:
continue
if repo.name in added_repos:
continue
# https://github.com/pypa/pipenv/issues/2231
if isinstance(repo, WarehouseLocalRepo):
continue
added_repos.add(repo.name)
repos.append(
OrderedDict([
('url', repo.pretty_url),
('verify_ssl', repo.pretty_url.startswith('https://')),
('name', repo.name),
]))
# pipenv doesn't work without explicit repo
if not repos:
repos.append(
OrderedDict([
('url', 'https://pypi.org/simple/'),
('verify_ssl', True),
('name', 'pypi'),
]))
python = Pythons(abstract=True).get_by_spec(project.python)
data = OrderedDict([
('_meta',
OrderedDict([
('sources', repos),
('requires', {
'python_version': str(python.version)
}),
])),
('default', packages[False]),
('develop', packages[True]),
])
data['_meta']['hash'] = {'sha256': self._get_hash(data)}
data['_meta']['pipfile-spec'] = 6
return json.dumps(data, indent=4, separators=(',', ': '))
def test_lib_paths():
p = Pythons().current
assert len(p.lib_paths) > 2
# appveyor makes some hacks for python path
libs_found = {
path
for path in sys.path if 'Scripts' not in path and 'dephell' not in path
}
libs_real = {
str(path)
for path in p.lib_paths
if 'Scripts' not in str(path) and 'dephell' not in str(path)
}
assert libs_found == libs_real
def __call__(self) -> bool:
# load project
loader = CONVERTERS[self.config['from']['format']]
loader = loader.copy(project_path=Path(self.config['project']))
resolver = loader.load_resolver(path=self.config['from']['path'])
if loader.lock:
self.logger.warning('do not build project from lockfile!')
# attach
merged = attach_deps(resolver=resolver, config=self.config, merge=True)
if not merged:
conflict = analyze_conflict(resolver=resolver)
self.logger.warning('conflict was found')
print(conflict)
return False
# dump
project_path = Path(self.config['project'])
reqs = Requirement.from_graph(resolver.graph, lock=False)
self.logger.info('creating wheel...')
dumper = WheelConverter()
project = resolver.graph.metainfo
dumper.dump(path=project_path / 'dist', reqs=reqs, project=project)
wheel_path = dumper._get_path(path=project_path / 'dist',
project=project)
# get command
command = self.args.name
if not command:
command = self.config.get('command')
if not command:
self.logger.error('command required')
return False
if isinstance(command, str):
command = shlex.split(command)
# choose pythons
self.logger.info('get interpreters')
pythons = Pythons()
choosen_pythons: Tuple[Python, ...]
if 'python' in self.config:
# get from config
choosen_pythons = (pythons.get_best(self.config['python']), )
else:
# get from project
pythons_by_version = dict() # type: Dict[str, Python]
python_constraint = resolver.graph.metainfo.python
for python in pythons:
version = str(python.get_short_version())
if version in pythons_by_version:
continue
if python.version not in python_constraint:
continue
pythons_by_version[version] = python
choosen_pythons = tuple(pythons_by_version.values())
for python in choosen_pythons:
with TemporaryDirectory(
) as root_path: # type: Path # type: ignore
root_path = Path(root_path)
# make venv
self.logger.info('create venv',
extra=dict(python=str(python.version)))
venv = VEnv(path=root_path / 'venv')
venv.create(python_path=python.path)
# copy tests
for path in self.config['tests']: # type: Path
self.logger.info('copy files', extra=dict(path=path))
path = Path(path)
if not path.exists():
raise FileNotFoundError(str(path))
# copy file
if path.is_file():
shutil.copyfile(str(path), str(root_path / path.name))
continue
# copy dir
for subpath in path.glob('**/*'):
if not subpath.is_file():
continue
if '__pycache__' in subpath.parts:
continue
new_path = subpath.resolve().relative_to(
self.config['project'])
new_path = root_path.joinpath(new_path)
self.logger.debug('copy',
extra=dict(old=str(subpath),
new=str(new_path)))
new_path.parent.mkdir(exist_ok=True, parents=True)
shutil.copyfile(str(subpath), str(new_path))
# install project
self.logger.info('install project',
extra=dict(path=str(wheel_path)))
dep_spec = str(wheel_path)
extras = set(self.config.get('envs', [])) - {'main'}
if extras:
dep_spec += '[{}]'.format(','.join(extras))
# we are using pip here to make it closer to the real installation
result = subprocess.run(
[str(venv.bin_path / 'pip'), 'install', dep_spec],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
if result.returncode != 0:
self.logger.error('failed to install project')
self.logger.error(result.stderr.decode())
return False
# install executable
executable = venv.bin_path / command[0]
if not executable.exists():
self.logger.info('executable not found, installing',
extra=dict(executable=command[0], ))
result = subprocess.run(
[str(venv.bin_path / 'pip'), 'install', command[0]],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
if result.returncode != 0:
self.logger.error('failed to install tests executable')
self.logger.error(result.stderr.decode())
return False
# run tests
self.logger.info('run tests', extra=dict(command=command))
result = subprocess.run(
[str(executable)] + command[1:],
cwd=str(root_path),
)
if result.returncode != 0:
self.logger.error('command failed, stopping')
return False
return True
def dumps(self, reqs, project: RootDependency, content=None) -> str:
if content:
doc = tomlkit.parse(content)
else:
doc = tomlkit.document()
# repositories
section = doc['source'] if 'source' in doc else tomlkit.aot()
added_repos = {repo['name'] for repo in section}
updated = False
for req in reqs:
if not isinstance(req.dep.repo, WarehouseBaseRepo):
continue
for repo in req.dep.repo.repos:
if repo.from_config:
continue
if repo.name in added_repos:
continue
# https://github.com/pypa/pipenv/issues/2231
if isinstance(repo, WarehouseLocalRepo):
continue
added_repos.add(repo.name)
source = tomlkit.table()
source['name'] = repo.name
source['url'] = repo.pretty_url
source['verify_ssl'] = repo.pretty_url.startswith('https://')
section.append(source)
updated = True
# pipenv doesn't work without explicit repo
if not added_repos:
source = tomlkit.table()
source['name'] = 'pypi'
source['url'] = 'https://pypi.org/simple/'
source['verify_ssl'] = True
section.append(source)
updated = True
if updated:
doc['source'] = section
# python version
if project.python:
python = Pythons(abstract=True).get_by_spec(project.python)
if 'requires' not in doc:
doc['requires'] = tomlkit.table()
doc['requires']['python_version'] = str(python.get_short_version())
# dependencies
names_mapping = dict()
for section, is_dev in [('packages', False), ('dev-packages', True)]:
# create section if doesn't exist
if section not in doc:
doc[section] = tomlkit.table()
continue
# clean file from outdated dependencies
names = {req.name for req in reqs if is_dev is req.is_dev}
for name in dict(doc[section]):
normalized_name = canonicalize_name(name)
names_mapping[normalized_name] = name
if normalized_name not in names:
del doc[section][name]
# write new packages
for section, is_dev in [('packages', False), ('dev-packages', True)]:
for req in reqs:
if is_dev is not req.is_dev:
continue
raw_name = names_mapping.get(req.name, req.raw_name)
old_spec = doc[section].get(raw_name)
# do not overwrite dep if nothing is changed
if old_spec:
old_dep = self._make_deps(
root=RootDependency(),
name=raw_name,
content=old_spec,
)[0]
if req.same_dep(old_dep):
continue
# overwrite
doc[section][raw_name] = self._format_req(req=req)
return tomlkit.dumps(doc).rstrip() + '\n'
def test_get_by_version(version):
manager = Pythons(abstract=True)
python = manager.get_by_version(Version(version))
assert str(python.version).startswith(version)
def test_abstract():
manager = Pythons(abstract=True)
pythons = {'.'.join(map(str, p.version.release[:2])): p for p in manager}
assert len({'2.7', '3.5', '3.7'} - set(pythons)) == 0
assert pythons['4.0'].abstract is True
assert not all(p.abstract for p in manager)
def test_lib_path():
p = Pythons().current
assert 'site-packages' in str(p.lib_path)