def render(self): api = pypi.proxy settings = self.request.registry.settings sanitize = asbool(settings['pyshop.mirror.sanitize']) package_name = self.request.matchdict['package_name'] pkg = Package.by_name(self.session, package_name) if pkg is None: pkg = Package.by_name(self.session, package_name.replace('-', '_')) refresh = True session_users = {} if pkg: if pkg.local: refresh = False else: if pkg.update_at: log.debug('validating cache interval') current_td = datetime.now() - pkg.update_at max_td = timedelta(hours=int( settings.get('pyshop.mirror.cache.ttl', '24'))) refresh = current_td > max_td log.debug('"%s" > "%s": %s', current_td, max_td, refresh) if refresh: log.info('refresh package %s', package_name) pypi_versions = api.package_releases(package_name, True) # XXX package_releases is case sensitive # but dependencies declaration not... if not pypi_versions: pkg_info = self._search_package(package_name) if not pkg_info and '-' in package_name: tmp_name = package_name.replace('-', '_') pkg_info = self._search_package(tmp_name) if not pkg_info and '_' in package_name: tmp_name = package_name.replace('_', '-') pkg_info = self._search_package(tmp_name) if pkg_info: package_name, pypi_versions = pkg_info pypi_versions = [to_unicode(ver) for ver in pypi_versions] else: pypi_versions = [] if not pkg: if not pypi_versions: log.info('package %s has no versions', package_name) return {'package': None, 'package_name': package_name} if sanitize: re_sanitize = re.compile(settings['pyshop.mirror.' 'sanitize.regex']) pypi_versions = [ v for v in pypi_versions if re_sanitize.match(v) ] # mirror the package now log.info('mirror package %s now', package_name) pkg = Package.by_name(self.session, package_name) if not pkg: pkg = Package(name=package_name, local=False) self.session.add(pkg) self.session.flush() roles = api.package_roles(package_name) for role, login in roles: login = to_unicode(login) log.info('Looking for non local user %s', login) if _sanitize(login) in session_users: user = session_users[_sanitize(login)] else: user = User.by_login(self.session, login, local=False) if not user: log.info('Not found. creating user %s', login) user = User(login=login, local=False) self.session.add(user) if role == 'Owner': pkg.owners.append(user) self.session.add(pkg) elif role == 'Maintainer': pkg.maintainers.append(user) self.session.add(pkg) session_users[_sanitize(login)] = user self.session.flush() self.session.flush() if not pkg.local and refresh: log.debug('refreshing %s package', package_name) pkg_versions = set(pypi_versions).difference(pkg.versions) if not pkg_versions: log.info('No new version to mirror') log.debug('pypi versions: %s', pypi_versions) log.debug('mirrored versions: %s', pkg.versions) for version in pkg_versions: log.info('Mirroring version %s', version) release_data = api.release_data(package_name, version) release = self._create_release(pkg, release_data, session_users) release_files = api.release_urls(package_name, version) for data in release_files: data = self._to_unicode(data) filename = data['filename'] rf = ReleaseFile.by_filename(self.session, release, filename) if not rf: rf = self._create_release_file(release, data) pkg.update_at = func.now() self.session.add(pkg) log.info('package %s mirrored', package_name) return { 'package': pkg, 'whlify': asbool(settings.get('pyshop.mirror.wheelify', '0')) }
def _create_release(self, package, data, session_users): log.info('Create release %s for package %s', data.get('version'), package.name) data = self._to_unicode(data) release = Release( package=package, summary=data.get('summary'), version=data.get('version'), stable_version=data.get('stable_version'), home_page=data.get('home_page'), license=data.get('license'), description=data.get('description'), keywords=data.get('keywords'), platform=data.get('platform'), download_url=data.get('download_url'), bugtrack_url=data.get('bugtrack_url'), docs_url=data.get('docs_url'), ) if data.get('author'): log.info('Looking for author %s', data['author']) if _sanitize(data['author']) in session_users: author = session_users[_sanitize(data['author'])] else: author = User.by_login(self.session, data['author'], local=False) if not author: log.info('Author %s not found, creating', data['author']) author = User(login=data['author'], local=False, email=data.get('author_email')) self.session.add(author) session_users[_sanitize(data['author'])] = author release.author = author self.session.flush() if data.get('maintainer'): log.info('Looking for maintainer %s', data['maintainer']) if _sanitize(data['maintainer']) in session_users: maintainer = session_users[_sanitize(data['maintainer'])] else: maintainer = User.by_login(self.session, data['maintainer'], local=False) if not maintainer: log.info('Maintainer not found, creating user %s', data['maintainer']) maintainer = User(login=data['maintainer'], local=False, email=data.get('maintainer_email')) self.session.add(maintainer) session_users[_sanitize(data['maintainer'])] = maintainer release.maintainer = maintainer self.session.flush() for name in data.get('classifiers', []): classifier = Classifier.by_name(self.session, to_unicode(name), create_if_not_exists=True) while classifier: if classifier not in release.classifiers: release.classifiers.append(classifier) if classifier not in package.classifiers: package.classifiers.append(classifier) classifier = classifier.parent self.session.flush() return release
def _search_package(self, package_name): package_name = pypi.resolve_name(package_name) pypi_versions = pypi.proxy.package_releases(package_name, True) return to_unicode(package_name), pypi_versions
def _to_unicode(self, data): # xmlrpc use utf8 encoded string return dict([(key, to_unicode(val)) for key, val in data.items()])
def render(self): api = pypi.proxy settings = self.request.registry.settings sanitize = asbool(settings['pyshop.mirror.sanitize']) package_name = self.request.matchdict['package_name'] pkg = Package.by_name(self.session, package_name) if pkg is None: pkg = Package.by_name(self.session, package_name.replace('-', '_')) refresh = True session_users = {} if pkg: if pkg.local: refresh = False else: if pkg.update_at: log.debug('validating cache interval') current_td = datetime.now() - pkg.update_at max_td = timedelta( hours=int(settings.get('pyshop.mirror.cache.ttl', '24'))) refresh = current_td > max_td log.debug('"%s" > "%s": %s', current_td, max_td, refresh) if refresh: log.info('refresh package %s', package_name) pypi_versions = api.package_releases(package_name, True) # XXX package_releases is case sensitive # but dependencies declaration not... if not pypi_versions: pkg_info = self._search_package(package_name) if not pkg_info and '-' in package_name: tmp_name = package_name.replace('-', '_') pkg_info = self._search_package(tmp_name) if not pkg_info and '_' in package_name: tmp_name = package_name.replace('_', '-') pkg_info = self._search_package(tmp_name) if pkg_info: package_name, pypi_versions = pkg_info pypi_versions = [to_unicode(ver) for ver in pypi_versions] else: pypi_versions = [] if not pkg: if not pypi_versions: log.info('package %s has no versions', package_name) return {'package': None, 'package_name': package_name} if sanitize: re_sanitize = re.compile(settings['pyshop.mirror.' 'sanitize.regex']) pypi_versions = [v for v in pypi_versions if re_sanitize.match(v)] # mirror the package now log.info('mirror package %s now', package_name) pkg = Package.by_name(self.session, package_name) if not pkg: pkg = Package(name=package_name, local=False) self.session.add(pkg) self.session.flush() roles = api.package_roles(package_name) for role, login in roles: login = to_unicode(login) log.info('Looking for non local user %s', login) if _sanitize(login) in session_users: user = session_users[_sanitize(login)] else: user = User.by_login(self.session, login, local=False) if not user: log.info('Not found. creating user %s', login) user = User(login=login, local=False) self.session.add(user) if role == 'Owner': pkg.owners.append(user) self.session.add(pkg) elif role == 'Maintainer': pkg.maintainers.append(user) self.session.add(pkg) session_users[_sanitize(login)] = user self.session.flush() self.session.flush() if not pkg.local and refresh: log.debug('refreshing %s package', package_name) pkg_versions = set(pypi_versions).difference(pkg.versions) if not pkg_versions: log.info('No new version to mirror') log.debug('pypi versions: %s', pypi_versions) log.debug('mirrored versions: %s', pkg.versions) for version in pkg_versions: log.info('Mirroring version %s', version) release_data = api.release_data(package_name, version) release = self._create_release(pkg, release_data, session_users) release_files = api.release_urls(package_name, version) for data in release_files: data = self._to_unicode(data) filename = data['filename'] rf = ReleaseFile.by_filename(self.session, release, filename) if not rf: rf = self._create_release_file(release, data) pkg.update_at = func.now() self.session.add(pkg) log.info('package %s mirrored', package_name) return {'package': pkg, 'whlify': asbool(settings.get('pyshop.mirror.wheelify', '0'))}
def _search_package(self, package_name): package_name = pypi.resolve_name(package_name) pypi_versions = pypi.proxy.package_releases(package_name, True) return to_unicode(package_name), pypi_versions
def _create_release(self, package, data, session_users): log.info('Create release %s for package %s', data.get('version'), package.name) data = self._to_unicode(data) release = Release(package=package, summary=data.get('summary'), version=data.get('version'), stable_version=data.get('stable_version'), home_page=data.get('home_page'), license=data.get('license'), description=data.get('description'), keywords=data.get('keywords'), platform=data.get('platform'), download_url=data.get('download_url'), bugtrack_url=data.get('bugtrack_url'), docs_url=data.get('docs_url'), ) if data.get('author'): log.info('Looking for author %s', data['author']) if _sanitize(data['author']) in session_users: author = session_users[_sanitize(data['author'])] else: author = User.by_login(self.session, data['author'], local=False) if not author: log.info('Author %s not found, creating', data['author']) author = User(login=data['author'], local=False, email=data.get('author_email')) self.session.add(author) session_users[_sanitize(data['author'])] = author release.author = author self.session.flush() if data.get('maintainer'): log.info('Looking for maintainer %s', data['maintainer']) if _sanitize(data['maintainer']) in session_users: maintainer = session_users[_sanitize(data['maintainer'])] else: maintainer = User.by_login(self.session, data['maintainer'], local=False) if not maintainer: log.info('Maintainer not found, creating user %s', data['maintainer']) maintainer = User(login=data['maintainer'], local=False, email=data.get('maintainer_email')) self.session.add(maintainer) session_users[_sanitize(data['maintainer'])] = maintainer release.maintainer = maintainer self.session.flush() for name in data.get('classifiers', []): classifier = Classifier.by_name(self.session, to_unicode(name), create_if_not_exists=True) while classifier: if classifier not in release.classifiers: release.classifiers.append(classifier) if classifier not in package.classifiers: package.classifiers.append(classifier) classifier = classifier.parent self.session.flush() return release
def _to_unicode(self, data): # xmlrpc use utf8 encoded string return dict([(key, to_unicode(val)) for key, val in data.items()])