def generate_index(self):
"""Generated the repository index file (`index.gz`)
index.gz is the compressed sqlite index containing all of the succeeded
packages in the repository pool.
Return the number of packages added to the repository index.
"""
assert exists(self.repository.path)
idx_path = xjoin(self.repository.path, 'index')
idx_gz_path = idx_path + '.gz'
sh.rm(idx_path)
db = RepoPackageDatabase(idx_path, touch=True)
# Tag BE packages; so client may use it to determine if a package is
# available only to BE customers or not.
# See also: RepoPackage.requires_be_license property
pkgtags = 'be' if self.repository.name == 'be' else ''
with closing(db):
LOG.debug('finding packages in %s', self.repository.path)
packages = self.repository.find_packages()
LOG.debug('processing %d packages', len(packages))
rpkg_list = [
RepoPackage.create_from(
BinaryPackage(**self._read_info_json(pkgfile)),
relpath=relpath(pkgfile, self.repository.path),
tags=pkgtags)
for pkgfile in textui.ProgressBar.iterate(packages, note="Package")
]
# Optimize index size by removing the "description" field.
# PyPI's descriptions are typically very long - see
# http://pypi.python.org/pypi/zc.buildout for example - hence we
# must remove them from the index.
for rpkg in rpkg_list:
rpkg.description = ''
# keep only the latest pkg_version in index
LOG.debug("pruning older pkg_version's")
rpkg_list = _prune_older_binary_releases(rpkg_list)
LOG.debug('.. resulting in %d packages', len(rpkg_list))
LOG.info(' writing index (please wait) ...')
with db.transaction() as session:
session.add_all(rpkg_list)
session.commit()
session.close()
LOG.info(' compressing index: ...%s%s',
os.path.basename(idx_gz_path),
(' (%d)' % len(rpkg_list)) if rpkg_list else '')
sh.rm(idx_gz_path)
with closing(gzip.open(idx_gz_path, 'wb')) as f:
f.write(open(idx_path, 'rb').read())
sh.rm(idx_path)
return len(rpkg_list)
def install_local_file(pypmenv, pypmfile): # nodeps = True (always!)
"""Install a local .pypm files
Dependency is not automatically resolved, although we should do this in
future
"""
assert pypmfile.endswith(BinaryPackage.EXTENSION) and P.exists(pypmfile)
LOG.debug('Installing local package "%s"', pypmfile)
bpkg = PackageFile(pypmfile).to_binary_package()
# sanity check
if bpkg.pyver != pypmenv.pyenv.pyver:
raise ValueError(
'cannot install a {0} package on python {1}; {2}'.format(
bpkg.pyver, pypmenv.pyenv.pyver, pypmfile))
if bpkg.osarch != PLATNAME:
raise ValueError(
'incompatible platform {0}; {1}'.format(
bpkg.osarch, pypmfile))
pkg = RepoPackage.create_from(bpkg, relpath=pypmfile, tags='local')
with pypmenv.locked():
installer.Installer(pypmenv)._install(pkg, pypmfile)
def _generate_index_gz(self):
"""Generate `index.gz` - the repository index
index.gz is the compressed form of `index` which is nothing but a Sqlite
database of succeeded packages available in the repository. See
``RepoPackageDatabase``
"""
assert exists(self.repository.path)
idx_path = xjoin(self.repository.path, 'index')
idx_gz_path = idx_path + '.gz'
console.rm(idx_path)
db = RepoPackageDatabase(idx_path, touch=True)
# Tag BE packages; so client may use it to determine if a package is
# available only to BE customers or not.
pkgtags = 'be' if self.repository.name == 'be' else ''
with closing(db):
LOG.info('finding packages')
packages = self.repository.find_packages()
LOG.info('processing %d packages', len(packages))
rpkg_list = [
RepoPackage.create_from(
BinaryPackage.from_json(self._read_info_json(pkgfile)),
relpath=relpath(pkgfile, self.repository.path),
tags=pkgtags)
for pkgfile in ProgressBar.iterate(packages)
]
# Optimize index size by removing the "description" field.
# PyPI's descriptions are typically very long - see
# http://pypi.python.org/pypi/zc.buildout for example - hence we
# must remove them from the index.
for rpkg in rpkg_list:
rpkg.description = 'N/A'
# keep only the latest version/pkg_version in index
LOG.info('pruning older versions')
rpkg_list = _prune_older_binary_releases(rpkg_list)
LOG.info('writing index')
with db.transaction() as session:
session.add_all(rpkg_list)
session.commit()
session.close()
LOG.info('compresing index (%s)', idx_gz_path)
console.rm(idx_gz_path)
with closing(gzip.open(idx_gz_path, 'wb')) as f:
f.write(open(idx_path, 'rb').read())
console.rm(idx_path)
return len(rpkg_list)
def generate_index(self):
"""Generated the repository index file (`index.gz`)
index.gz is the compressed sqlite index containing all of the succeeded
packages in the repository pool.
Return the number of packages added to the repository index.
"""
from pypm.grail.package import PackageShare
assert P.exists(self.repository.path)
idx_path = xjoin(self.repository.path, 'index')
idx_gz_path = idx_path + '.gz'
sh.rm(idx_path)
db = RepoPackageDatabase(idx_path, touch=True)
# Tag BE packages; so client may use it to determine if a package is
# available only to BE customers or not.
# See also: RepoPackage.requires_be_license property
pkgtags = 'be' if self.repository.name == 'be' else ''
# Load package-specific data from share/p/*
pkgdata = dict([(s.name, s) for s in PackageShare.all()])
with closing(db):
LOG.debug('finding packages in %s', self.repository.path)
packages = self.repository.find_packages()
LOG.debug('processing %d packages', len(packages))
rpkg_list = [
RepoPackage.create_from(
BinaryPackage(**self._read_info_json(pkgfile)),
relpath=P.relpath(pkgfile, self.repository.path),
tags=pkgtags)
for pkgfile in textui.ProgressBar.iterate(packages,
note="Package")
]
for rpkg in rpkg_list:
# Optimize index size by removing the "description" field.
# PyPI's descriptions are typically very long - see
# http://pypi.python.org/pypi/zc.buildout for example - hence we
# must remove them from the index.
rpkg.description = ''
if rpkg.name in pkgdata:
# Add package notes to the description^Wextra field
# See pypm.common.package.RepoPackage.FIELDS to understand
# why we are abusing this field.
notes = list(pkgdata[rpkg.name].get_notes_for(
pyver=rpkg.pyver, osarch=rpkg.osarch))
rpkg.description = json.dumps({'notes': notes})
LOG.debug('Patching "description" field for %s', rpkg)
# keep only the latest pkg_version in index
LOG.debug("pruning older pkg_version's")
rpkg_list = _prune_older_binary_releases(rpkg_list)
LOG.debug('.. resulting in %d packages', len(rpkg_list))
LOG.info(' writing index (please wait) ...')
with db.transaction() as session:
session.add_all(rpkg_list)
session.commit()
session.close()
LOG.info(' compressing index: ...%s%s', os.path.basename(idx_gz_path),
(' (%d)' % len(rpkg_list)) if rpkg_list else '')
sh.rm(idx_gz_path)
with closing(gzip.open(idx_gz_path, 'wb')) as f:
f.write(open(idx_path, 'rb').read())
sh.rm(idx_path)
return len(rpkg_list)
def generate_index(self):
"""Generated the repository index file (`index.gz`)
index.gz is the compressed sqlite index containing all of the succeeded
packages in the repository pool.
Return the number of packages added to the repository index.
"""
from pypm.grail.package import PackageShare
assert P.exists(self.repository.path)
idx_path = xjoin(self.repository.path, 'index')
idx_gz_path = idx_path + '.gz'
sh.rm(idx_path)
db = RepoPackageDatabase(idx_path, touch=True)
# Tag BE packages; so client may use it to determine if a package is
# available only to BE customers or not.
# See also: RepoPackage.requires_be_license property
pkgtags = 'be' if self.repository.name == 'be' else ''
# Load package-specific data from share/p/*
pkgdata = dict([(s.name, s) for s in PackageShare.all()])
with closing(db):
LOG.debug('finding packages in %s', self.repository.path)
packages = self.repository.find_packages()
LOG.debug('processing %d packages', len(packages))
rpkg_list = [
RepoPackage.create_from(
BinaryPackage(**self._read_info_json(pkgfile)),
relpath=P.relpath(pkgfile, self.repository.path),
tags=pkgtags)
for pkgfile in textui.ProgressBar.iterate(packages, note="Package")
]
for rpkg in rpkg_list:
# Optimize index size by removing the "description" field.
# PyPI's descriptions are typically very long - see
# http://pypi.python.org/pypi/zc.buildout for example - hence we
# must remove them from the index.
rpkg.description = ''
if rpkg.name in pkgdata:
# Add package notes to the description^Wextra field
# See pypm.common.package.RepoPackage.FIELDS to understand
# why we are abusing this field.
notes = list(pkgdata[rpkg.name].get_notes_for(
pyver=rpkg.pyver, osarch=rpkg.osarch))
rpkg.description = json.dumps({
'notes': notes
})
LOG.debug('Patching "description" field for %s', rpkg)
# keep only the latest pkg_version in index
LOG.debug("pruning older pkg_version's")
rpkg_list = _prune_older_binary_releases(rpkg_list)
LOG.debug('.. resulting in %d packages', len(rpkg_list))
LOG.info(' writing index (please wait) ...')
with db.transaction() as session:
session.add_all(rpkg_list)
session.commit()
session.close()
LOG.info(' compressing index: ...%s%s',
os.path.basename(idx_gz_path),
(' (%d)' % len(rpkg_list)) if rpkg_list else '')
sh.rm(idx_gz_path)
with closing(gzip.open(idx_gz_path, 'wb')) as f:
f.write(open(idx_path, 'rb').read())
sh.rm(idx_path)
return len(rpkg_list)