def pip_install(pkg,
ver,
pkgpath,
data,
prefix,
dlprefix,
pip="pip",
*args,
**kwargs):
ver = strip_version(ver)
if ver == LATEST_PACKAGE_ALIAS:
ver = latest_pypi_version(pkg)
cmd = [
pip,
"install {}=={}".format(pkg, strip_version(ver)),
"--root {}".format(kwargs["fakeroot"]),
"--prefix {}".format(prefix),
"--no-index",
"--no-deps",
"--ignore-installed",
"--cache-dir {}".format(dlprefix),
"--find-links {}".format(dlprefix),
kwargs.get("makeopts", ""),
]
print('Installing {} ({}) from pip'.format(pkg, ver))
shell(cmd)
def grab(path, filename=None, version=None, protocol=None, pip='pip'):
# guess protocol if it's obvious from the url (usually is)
if protocol is None:
protocol = path.split(':')[0]
if protocol in ('http', 'https', 'ftp'):
shell('wget --quiet {} -O {}'.format(path, filename))
elif protocol in ('git'):
shell("git clone "
"-b {version} "
"-q --recursive "
"-- {path} {filename}"
"".format(
version=strip_version(version),
path=path,
filename=filename,
))
elif protocol in ('nfs', 'fs-ln'):
shell('cp --recursive --symbolic-link {} {}'.format(path, filename))
elif protocol in ('fs-cp'):
shell('cp --recursive {} {}'.format(path, filename))
elif protocol in ('rsync'):
shell('rsync -a {}/ {}'.format(path, filename))
else:
raise NotImplementedError('Unknown protocol {}'.format(protocol))
def _main(args):
args.prefix = os.path.abspath(args.prefix)
data = Data(extra_data_dirs=args.extra_data_dirs)
if args.download or (not args.build and not args.install):
git_hashes = fetch(args.pkgs,
args.repo,
outdir=args.cache,
pip=args.pip)
if args.download and not args.build:
sys.exit(0)
# append root to the temporary build dir, as we want a named root/
# directory as the distribution root, organised under the distribution name
# (release)
tmp_prefix = os.path.join(os.path.join(args.prefix), args.release, "root")
fakeroot = os.path.abspath(args.release)
if args.build or not args.install:
make(
args.pkgs,
args.repo,
data,
prefix=tmp_prefix,
dlprefix=args.cache,
builddir=args.tmp,
jobs=args.jobs,
cmk=args.cmake,
pip=args.pip,
virtualenv=args.virtualenv,
fakeroot=fakeroot,
)
shell("mv {} {}".format(args.release + tmp_prefix, args.release))
shell("rmdir -p --ignore-fail-on-non-empty {}".format(
os.path.dirname(args.release + tmp_prefix)))
if args.build and not args.install:
sys.exit(0)
# create the enable script
for tmpl, target in [("enable.in", "enable"),
("enable.csh.in", "enable.csh")]:
# TODO should args.release be release_path?
with open("{}/{}".format(args.release, target), "w") as f:
f.write(
shell([
"m4 {}".format(data.get("enable.m4")),
"-D komodo_prefix={}".format(tmp_prefix),
"-D komodo_pyver={}".format(args.pyver),
"-D komodo_release={}".format(args.release),
data.get(tmpl),
]).decode("utf-8"))
with open(args.locations_config) as defs, open(
os.path.join(args.release, "local"), "w") as local_activator, open(
os.path.join(args.release, "local.csh"),
"w") as local_csh_activator:
defs = yml.safe_load(defs)
local.write_local_activators(data, defs, local_activator,
local_csh_activator)
releasedoc = os.path.join(args.release, args.release)
with open(releasedoc, "w") as y:
release = {}
for pkg, ver in args.pkgs.items():
entry = args.repo[pkg][ver]
maintainer = args.repo[pkg][ver]["maintainer"]
if ver == LATEST_PACKAGE_ALIAS:
ver = latest_pypi_version(entry.get("pypi_package_name", pkg))
elif args.repo[pkg][ver].get("fetch") == "git":
ver = git_hashes[pkg]
release[pkg] = {
"version": ver,
"maintainer": maintainer,
}
yml.dump(release, y, default_flow_style=False)
if args.dry_run:
return
print("Installing {} to {}".format(args.release, args.prefix))
install_root = os.path.join(args.prefix, args.release, "root")
shell("{1} {0} .{0} {0}".format(args.release, args.renamer))
shell("rsync -a .{} {}".format(args.release, args.prefix), sudo=args.sudo)
if os.path.exists("{1}/{0}".format(args.release, args.prefix)):
shell(
"{2} {0} {0}.delete {1}/{0}".format(args.release, args.prefix,
args.renamer),
sudo=args.sudo,
)
shell(
"{2} .{0} {0} {1}/.{0}".format(args.release, args.prefix,
args.renamer),
sudo=args.sudo,
)
shell("rm -rf {1}/{0}.delete".format(args.release, args.prefix),
sudo=args.sudo)
if args.tmp:
# Allows e.g. pip to use this folder as tmpfolder, instead of in some
# cases falling back to /tmp, which is undesired when building on nfs.
os.environ["TMPDIR"] = args.tmp
print('Fixup #! in pip-provided packages if bin exist')
release_path = os.path.join(args.prefix, args.release)
release_root = os.path.join(release_path, "root")
for pkg, ver in args.pkgs.items():
current = args.repo[pkg][ver]
if current["make"] != "pip":
continue
package_name = current.get("pypi_package_name", pkg)
if ver == LATEST_PACKAGE_ALIAS:
ver = latest_pypi_version(package_name)
shell_input = [
args.pip,
"install {}=={}".format(package_name, strip_version(ver)),
"--prefix",
release_root,
"--no-index",
"--no-deps",
"--ignore-installed",
"--cache-dir {}".format(args.cache),
"--find-links {}".format(args.cache),
]
shell_input.append(current.get("makeopts"))
print(shell(shell_input, sudo=args.sudo))
fixup_python_shebangs(args.prefix, args.release)
switch.create_activator_switch(data, args.prefix, args.release)
# run any post-install scripts on the release
if args.postinst:
shell([args.postinst, release_path])
print("running", "find {} -name '*.pyc' -delete".format(release_root))
shell("find {} -name '*.pyc' -delete".format(release_root))
print("Setting permissions",
[data.get("set_permissions.sh"), release_path])
shell([data.get("set_permissions.sh"), release_path])
def fetch(pkgs, repo, outdir=".", pip="pip", git="git"):
missingpkg = [pkg for pkg in pkgs if pkg not in repo]
missingver = [
pkg for pkg, ver in pkgs.items()
if pkg in repo and ver not in repo[pkg]
]
if missingpkg:
eprint('Packages requested, but not found in the repository:')
eprint('missingpkg: {}'.format(','.join(missingpkg)))
for pkg in missingver:
eprint('missingver: missing version for {}: {} requested, found: {}'.
format(pkg, pkgs[pkg], ','.join(repo[pkg].keys())))
if missingpkg or missingver:
return
if outdir and not os.path.exists(outdir):
os.mkdir(outdir)
pypi_packages = []
with pushd(outdir):
for pkg, ver in pkgs.items():
current = repo[pkg][ver]
if "pypi_package_name" in current and current["make"] != "pip":
raise ValueError(
"pypi_package_name is only valid when building with pip")
url = current.get("source")
protocol = current.get("fetch")
pkg_alias = current.get("pypi_package_name", pkg)
name = "{} ({}): {}".format(pkg_alias, ver, url)
pkgname = "{}-{}".format(pkg_alias, ver)
if url is None and protocol is None:
package_folder = os.path.abspath(pkgname)
print('Nothing to fetch for {}, but created folder {}'.format(
pkgname, package_folder))
os.mkdir(pkgname)
continue
dst = pkgname
spliturl = url.split('?')[0].split('.')
ext = spliturl[-1]
if len(spliturl) > 1 and spliturl[-2] == 'tar':
ext = 'tar.{}'.format(spliturl[-1])
if ext in [
'rpm', 'tar', 'gz', 'tgz', 'tar.gz', 'tar.bz2', 'tar.xz'
]:
dst = '{}.{}'.format(dst, ext)
if url == "pypi":
print("Defering download of {}".format(name))
pypi_packages.append(normalize_filename(strip_version(dst)))
continue
print('Downloading {}'.format(name))
grab(url,
filename=dst,
version=ver,
protocol=protocol,
pip=pip,
git=git)
if ext in ['tgz', 'tar.gz', 'tar.bz2', 'tar.xz']:
print('Extracting {} ...'.format(dst))
topdir = shell(
' tar -xvf {}'.format(dst)).decode("utf-8").split()[0]
normalised_dir = topdir.split('/')[0]
if not os.path.exists(pkgname):
print('Creating symlink {} -> {}'.format(
normalised_dir, pkgname))
os.symlink(normalised_dir, pkgname)
print('Downloading {} pypi packages'.format(len(pypi_packages)))
shell([
pip, 'download', '--no-deps', '--dest .', " ".join(pypi_packages)
])