def fetch(pkgfile, repofile, outdir=None, pip='pip', git='git'):
with open(pkgfile) as p, open(repofile) as r:
pkgs, repo = yml.load(p), yml.load(r)
missingpkg = [pkg for pkg in pkgs if pkg not in repo]
missingver = [
pkg for pkg, ver in pkgs.items()
if pkg in repo and ver not in repo[pkg]
]
if missingpkg:
eprint('Packages requested, but not found in the repository:')
eprint('missingpkg: {}'.format(','.join(missingpkg)))
for pkg in missingver:
eprint('missingver: missing version for {}: {} requested, found: {}'.
format(pkg, pkgs[pkg], ','.join(repo[pkg].keys())))
if missingpkg or missingver:
return
for pkg, ver in pkgs.items():
print(pkg, ver)
url = repo[pkg][ver]['source']
pkgname = '{}-{}'.format(pkg, ver)
dst = pkgname
spliturl = url.split('?')[0].split('.')
ext = spliturl[-1]
if len(spliturl) > 1 and spliturl[-2] == 'tar':
ext = 'tar.{}'.format(spliturl[-1])
if ext in ['rpm', 'tar', 'gz', 'tgz', 'tar.gz', 'tar.bz2', 'tar.xz']:
dst = '{}.{}'.format(dst, ext)
if outdir and not os.path.exists(outdir):
os.mkdir(outdir)
if not outdir: outdir = '.'
with pushd(outdir):
print('Downloading {} ({}): {}'.format(pkg, ver, url))
protocol = repo[pkg][ver].get('fetch')
grab(url,
filename=dst,
version=ver,
protocol=protocol,
pip=pip,
git=git)
if ext in ['tgz', 'tar.gz', 'tar.bz2', 'tar.xz']:
print('Extracting {} ...'.format(dst))
topdir = shell(' tar -xvf {}'.format(dst)).split()[0]
normalised_dir = topdir.split('/')[0]
if not os.path.exists(pkgname):
print('Creating symlink {} -> {}'.format(
normalised_dir, pkgname))
os.symlink(normalised_dir, pkgname)
def rpm(pkg, ver, path, prefix, *args, **kwargs):
# cpio always outputs to cwd, can't be overriden with switches
with pushd(prefix):
print('Installing {} ({}) from rpm'.format(pkg, ver))
shell('rpm2cpio {}.rpm | cpio -imd --quiet'.format(path))
shell('rsync -a usr/* .')
shell('rm -rf usr')
def cmake(pkg, ver, path, prefix, builddir,
makeopts,
jobs,
cmake = 'cmake',
*args, **kwargs):
bdir = '{}-{}-build'.format(pkg, ver)
if builddir is not None:
bdir = os.path.join(builddir, bdir)
fakeroot = kwargs['fakeroot']
fakeprefix = fakeroot + prefix
flags = ['-DCMAKE_BUILD_TYPE=Release',
'-DBOOST_ROOT={}'.format(fakeprefix),
'-DBUILD_SHARED_LIBS=ON',
'-DCMAKE_PREFIX_PATH={}'.format(fakeprefix),
'-DCMAKE_MODULE_PATH={}/share/cmake/Modules'.format(fakeprefix),
'-DCMAKE_INSTALL_PREFIX={}'.format(prefix)
]
mkpath(bdir)
with pushd(bdir):
print('Installing {} ({}) from source with cmake'.format(pkg, ver))
shell([cmake, path] + flags + [makeopts])
print(shell('make -j{}'.format(jobs)))
print(shell('make DESTDIR={} install'.format(fakeroot)))
def grab(path,
filename=None,
version=None,
protocol=None,
pip='pip',
git='git'):
# guess protocol if it's obvious from the url (usually is)
if protocol is None:
protocol = path.split(':')[0]
if protocol in ('http', 'https', 'ftp'):
shell('wget --quiet {} -O {}'.format(path, filename))
#return urlretrieve(path, filename = filename)
elif protocol in ('git'):
shell('{} clone -q --recursive -- {} {}'.format(git, path, filename))
with pushd(filename):
shell('{} fetch --tags'.format(git))
shell('{} checkout -q {}'.format(git, version))
elif protocol in ('nfs', 'fs-ln'):
shell('cp --recursive --symbolic-link {} {}'.format(path, filename))
elif protocol in ('fs-cp'):
shell('cp --recursive {} {}'.format(path, filename))
elif protocol in ('rsync'):
shell('rsync -a {}/ {}'.format(path, filename))
elif protocol in ('pypi'):
shell([pip, 'download', '--dest .', normalize_filename(filename)])
else:
raise NotImplementedError('Unknown protocol {}'.format(protocol))
def sh(pkg, ver, pkgpath, prefix, makefile, *args, **kwargs):
if os.path.isfile(makefile):
makefile = os.path.abspath(makefile)
with pushd(pkgpath):
makefile = os.path.abspath(makefile)
cmd = ['bash {} --prefix {}'.format(makefile, prefix),
'--fakeroot {}'.format(kwargs['fakeroot']),
'--python {}/bin/python'.format(prefix)]
if 'jobs' in kwargs:
cmd.append('--jobs {}'.format(kwargs['jobs']))
if 'cmake' in kwargs:
cmd.append('--cmake {}'.format(kwargs['cmake']))
cmd.append(kwargs.get('makeopts'))
print('Installing {} ({}) from sh'.format(pkg, ver))
shell(cmd)