def render(self):
api = pypi.proxy
settings = self.request.registry.settings
sanitize = asbool(settings['pyshop.mirror.sanitize'])
package_name = self.request.matchdict['package_name']
pkg = Package.by_name(self.session, package_name)
if pkg is None:
pkg = Package.by_name(self.session, package_name.replace('-', '_'))
refresh = True
session_users = {}
if pkg:
if pkg.local:
refresh = False
else:
if pkg.update_at:
log.debug('validating cache interval')
current_td = datetime.now() - pkg.update_at
max_td = timedelta(hours=int(
settings.get('pyshop.mirror.cache.ttl', '24')))
refresh = current_td > max_td
log.debug('"%s" > "%s": %s', current_td, max_td, refresh)
if refresh:
log.info('refresh package %s', package_name)
pypi_versions = api.package_releases(package_name, True)
# XXX package_releases is case sensitive
# but dependencies declaration not...
if not pypi_versions:
pkg_info = self._search_package(package_name)
if not pkg_info and '-' in package_name:
tmp_name = package_name.replace('-', '_')
pkg_info = self._search_package(tmp_name)
if not pkg_info and '_' in package_name:
tmp_name = package_name.replace('_', '-')
pkg_info = self._search_package(tmp_name)
if pkg_info:
package_name, pypi_versions = pkg_info
pypi_versions = [to_unicode(ver) for ver in pypi_versions]
else:
pypi_versions = []
if not pkg:
if not pypi_versions:
log.info('package %s has no versions', package_name)
return {'package': None, 'package_name': package_name}
if sanitize:
re_sanitize = re.compile(settings['pyshop.mirror.'
'sanitize.regex'])
pypi_versions = [
v for v in pypi_versions if re_sanitize.match(v)
]
# mirror the package now
log.info('mirror package %s now', package_name)
pkg = Package.by_name(self.session, package_name)
if not pkg:
pkg = Package(name=package_name, local=False)
self.session.add(pkg)
self.session.flush()
roles = api.package_roles(package_name)
for role, login in roles:
login = to_unicode(login)
log.info('Looking for non local user %s', login)
if _sanitize(login) in session_users:
user = session_users[_sanitize(login)]
else:
user = User.by_login(self.session, login, local=False)
if not user:
log.info('Not found. creating user %s', login)
user = User(login=login, local=False)
self.session.add(user)
if role == 'Owner':
pkg.owners.append(user)
self.session.add(pkg)
elif role == 'Maintainer':
pkg.maintainers.append(user)
self.session.add(pkg)
session_users[_sanitize(login)] = user
self.session.flush()
self.session.flush()
if not pkg.local and refresh:
log.debug('refreshing %s package', package_name)
pkg_versions = set(pypi_versions).difference(pkg.versions)
if not pkg_versions:
log.info('No new version to mirror')
log.debug('pypi versions: %s', pypi_versions)
log.debug('mirrored versions: %s', pkg.versions)
for version in pkg_versions:
log.info('Mirroring version %s', version)
release_data = api.release_data(package_name, version)
release = self._create_release(pkg, release_data,
session_users)
release_files = api.release_urls(package_name, version)
for data in release_files:
data = self._to_unicode(data)
filename = data['filename']
rf = ReleaseFile.by_filename(self.session, release,
filename)
if not rf:
rf = self._create_release_file(release, data)
pkg.update_at = func.now()
self.session.add(pkg)
log.info('package %s mirrored', package_name)
return {
'package': pkg,
'whlify': asbool(settings.get('pyshop.mirror.wheelify', '0'))
}
def _create_release(self, package, data, session_users):
log.info('Create release %s for package %s', data.get('version'),
package.name)
data = self._to_unicode(data)
release = Release(
package=package,
summary=data.get('summary'),
version=data.get('version'),
stable_version=data.get('stable_version'),
home_page=data.get('home_page'),
license=data.get('license'),
description=data.get('description'),
keywords=data.get('keywords'),
platform=data.get('platform'),
download_url=data.get('download_url'),
bugtrack_url=data.get('bugtrack_url'),
docs_url=data.get('docs_url'),
)
if data.get('author'):
log.info('Looking for author %s', data['author'])
if _sanitize(data['author']) in session_users:
author = session_users[_sanitize(data['author'])]
else:
author = User.by_login(self.session,
data['author'],
local=False)
if not author:
log.info('Author %s not found, creating', data['author'])
author = User(login=data['author'],
local=False,
email=data.get('author_email'))
self.session.add(author)
session_users[_sanitize(data['author'])] = author
release.author = author
self.session.flush()
if data.get('maintainer'):
log.info('Looking for maintainer %s', data['maintainer'])
if _sanitize(data['maintainer']) in session_users:
maintainer = session_users[_sanitize(data['maintainer'])]
else:
maintainer = User.by_login(self.session,
data['maintainer'],
local=False)
if not maintainer:
log.info('Maintainer not found, creating user %s',
data['maintainer'])
maintainer = User(login=data['maintainer'],
local=False,
email=data.get('maintainer_email'))
self.session.add(maintainer)
session_users[_sanitize(data['maintainer'])] = maintainer
release.maintainer = maintainer
self.session.flush()
for name in data.get('classifiers', []):
classifier = Classifier.by_name(self.session,
to_unicode(name),
create_if_not_exists=True)
while classifier:
if classifier not in release.classifiers:
release.classifiers.append(classifier)
if classifier not in package.classifiers:
package.classifiers.append(classifier)
classifier = classifier.parent
self.session.flush()
return release
def _search_package(self, package_name):
package_name = pypi.resolve_name(package_name)
pypi_versions = pypi.proxy.package_releases(package_name, True)
return to_unicode(package_name), pypi_versions
def _to_unicode(self, data):
# xmlrpc use utf8 encoded string
return dict([(key, to_unicode(val)) for key, val in data.items()])
def render(self):
api = pypi.proxy
settings = self.request.registry.settings
sanitize = asbool(settings['pyshop.mirror.sanitize'])
package_name = self.request.matchdict['package_name']
pkg = Package.by_name(self.session, package_name)
if pkg is None:
pkg = Package.by_name(self.session, package_name.replace('-', '_'))
refresh = True
session_users = {}
if pkg:
if pkg.local:
refresh = False
else:
if pkg.update_at:
log.debug('validating cache interval')
current_td = datetime.now() - pkg.update_at
max_td = timedelta(
hours=int(settings.get('pyshop.mirror.cache.ttl',
'24')))
refresh = current_td > max_td
log.debug('"%s" > "%s": %s',
current_td, max_td, refresh)
if refresh:
log.info('refresh package %s', package_name)
pypi_versions = api.package_releases(package_name, True)
# XXX package_releases is case sensitive
# but dependencies declaration not...
if not pypi_versions:
pkg_info = self._search_package(package_name)
if not pkg_info and '-' in package_name:
tmp_name = package_name.replace('-', '_')
pkg_info = self._search_package(tmp_name)
if not pkg_info and '_' in package_name:
tmp_name = package_name.replace('_', '-')
pkg_info = self._search_package(tmp_name)
if pkg_info:
package_name, pypi_versions = pkg_info
pypi_versions = [to_unicode(ver) for ver in pypi_versions]
else:
pypi_versions = []
if not pkg:
if not pypi_versions:
log.info('package %s has no versions', package_name)
return {'package': None,
'package_name': package_name}
if sanitize:
re_sanitize = re.compile(settings['pyshop.mirror.'
'sanitize.regex'])
pypi_versions = [v for v in pypi_versions
if re_sanitize.match(v)]
# mirror the package now
log.info('mirror package %s now', package_name)
pkg = Package.by_name(self.session, package_name)
if not pkg:
pkg = Package(name=package_name, local=False)
self.session.add(pkg)
self.session.flush()
roles = api.package_roles(package_name)
for role, login in roles:
login = to_unicode(login)
log.info('Looking for non local user %s', login)
if _sanitize(login) in session_users:
user = session_users[_sanitize(login)]
else:
user = User.by_login(self.session, login, local=False)
if not user:
log.info('Not found. creating user %s', login)
user = User(login=login, local=False)
self.session.add(user)
if role == 'Owner':
pkg.owners.append(user)
self.session.add(pkg)
elif role == 'Maintainer':
pkg.maintainers.append(user)
self.session.add(pkg)
session_users[_sanitize(login)] = user
self.session.flush()
self.session.flush()
if not pkg.local and refresh:
log.debug('refreshing %s package', package_name)
pkg_versions = set(pypi_versions).difference(pkg.versions)
if not pkg_versions:
log.info('No new version to mirror')
log.debug('pypi versions: %s', pypi_versions)
log.debug('mirrored versions: %s', pkg.versions)
for version in pkg_versions:
log.info('Mirroring version %s', version)
release_data = api.release_data(package_name, version)
release = self._create_release(pkg, release_data,
session_users)
release_files = api.release_urls(package_name, version)
for data in release_files:
data = self._to_unicode(data)
filename = data['filename']
rf = ReleaseFile.by_filename(self.session, release,
filename)
if not rf:
rf = self._create_release_file(release, data)
pkg.update_at = func.now()
self.session.add(pkg)
log.info('package %s mirrored', package_name)
return {'package': pkg,
'whlify': asbool(settings.get('pyshop.mirror.wheelify', '0'))}
def _search_package(self, package_name):
package_name = pypi.resolve_name(package_name)
pypi_versions = pypi.proxy.package_releases(package_name, True)
return to_unicode(package_name), pypi_versions
def _create_release(self, package, data, session_users):
log.info('Create release %s for package %s',
data.get('version'), package.name)
data = self._to_unicode(data)
release = Release(package=package,
summary=data.get('summary'),
version=data.get('version'),
stable_version=data.get('stable_version'),
home_page=data.get('home_page'),
license=data.get('license'),
description=data.get('description'),
keywords=data.get('keywords'),
platform=data.get('platform'),
download_url=data.get('download_url'),
bugtrack_url=data.get('bugtrack_url'),
docs_url=data.get('docs_url'),
)
if data.get('author'):
log.info('Looking for author %s', data['author'])
if _sanitize(data['author']) in session_users:
author = session_users[_sanitize(data['author'])]
else:
author = User.by_login(self.session, data['author'],
local=False)
if not author:
log.info('Author %s not found, creating',
data['author'])
author = User(login=data['author'],
local=False,
email=data.get('author_email'))
self.session.add(author)
session_users[_sanitize(data['author'])] = author
release.author = author
self.session.flush()
if data.get('maintainer'):
log.info('Looking for maintainer %s', data['maintainer'])
if _sanitize(data['maintainer']) in session_users:
maintainer = session_users[_sanitize(data['maintainer'])]
else:
maintainer = User.by_login(self.session, data['maintainer'],
local=False)
if not maintainer:
log.info('Maintainer not found, creating user %s',
data['maintainer'])
maintainer = User(login=data['maintainer'],
local=False,
email=data.get('maintainer_email'))
self.session.add(maintainer)
session_users[_sanitize(data['maintainer'])] = maintainer
release.maintainer = maintainer
self.session.flush()
for name in data.get('classifiers', []):
classifier = Classifier.by_name(self.session, to_unicode(name),
create_if_not_exists=True)
while classifier:
if classifier not in release.classifiers:
release.classifiers.append(classifier)
if classifier not in package.classifiers:
package.classifiers.append(classifier)
classifier = classifier.parent
self.session.flush()
return release
def _to_unicode(self, data):
# xmlrpc use utf8 encoded string
return dict([(key, to_unicode(val))
for key, val in data.items()])