def release_urls(self, package_name, version): """ Retrieve a list of download URLs for the given package release. Returns a list of dicts with the following keys: url packagetype ('sdist', 'bdist', etc) filename size md5_digest downloads has_sig python_version (required version, or 'source', or 'any') comment_text """ session = DBSession() release_files = ReleaseFile.by_release(session, package_name, version) return [{ 'url': f.url, 'packagetype': f.package_type, 'filename': f.filename, 'size': f.size, 'md5_digest': f.md5_digest, 'downloads': f.downloads, 'has_sig': f.has_sig, 'comment_text': f.comment_text, 'python_version': f.python_version } for f in release_files]
def show_release_file(root, request): """ Download a release file. Must be used with :func:`pyshop.helpers.download.renderer_factory` to download the release file. :return: download informations :rtype: dict """ settings = request.registry.settings whlify = asbool(settings.get('pyshop.mirror.wheelify', '0')) session = DBSession() f = ReleaseFile.by_id(session, int(request.matchdict['file_id'])) whlify = whlify and f.package_type == 'sdist' filename = f.filename_whlified if whlify else f.filename url = f.url if url and url.startswith('http://pypi.python.org'): url = 'https' + url[4:] rv = {'url': url, 'filename': filename, 'original': f.filename, 'whlify': whlify } f.downloads += 1 f.release.downloads += 1 f.release.package.downloads += 1 session.add(f.release.package) session.add(f.release) session.add(f) return rv
def test_by_release(self): from pyshop.models import ReleaseFile files = ReleaseFile.by_release(self.session, u'mirrored_package2', u'1.0') self.assertIsInstance(files, list) files = [f.filename for f in files] self.assertEqual(files, [u'mirrored_package2-1.0.tar.gz'])
def release_urls(self, package_name, version): """ Retrieve a list of download URLs for the given package release. Returns a list of dicts with the following keys: url packagetype ('sdist', 'bdist', etc) filename size md5_digest downloads has_sig python_version (required version, or 'source', or 'any') comment_text """ session = DBSession() release_files = ReleaseFile.by_release(session, package_name, version) return [{'url': f.url, 'packagetype': f.package_type, 'filename': f.filename, 'size': f.size, 'md5_digest': f.md5_digest, 'downloads': f.downloads, 'has_sig': f.has_sig, 'comment_text': f.comment_text } for f in release_files]
def show_release_file(root, request): """ Download a release file. Must be used with :func:`pyshop.helpers.download.renderer_factory` to download the release file. :return: download informations :rtype: dict """ settings = request.registry.settings whlify = asbool(settings.get('pyshop.mirror.wheelify', '0')) session = DBSession() f = ReleaseFile.by_id(session, int(request.matchdict['file_id'])) whlify = whlify and f.package_type == 'sdist' filename = f.filename_whlified if whlify else f.filename url = f.url if url and url.startswith('http://pypi.python.org'): url = 'https' + url[4:] rv = { 'url': url, 'filename': filename, 'original': f.filename, 'whlify': whlify } f.downloads += 1 f.release.downloads += 1 f.release.package.downloads += 1 session.add(f.release.package) session.add(f.release) session.add(f) return rv
def show_release_file(root, request): """ Download a release file. Must be used with :func:`pyshop.helpers.download.renderer_factory` to download the release file. :return: download informations :rtype: dict """ session = DBSession() f = ReleaseFile.by_id(session, int(request.matchdict['file_id'])) url = f.url if url and url.startswith('http://pypi.python.org'): url = 'https' + url[4:] rv = {'url': url, 'filename': f.filename, } f.downloads += 1 f.release.downloads += 1 f.release.package.downloads += 1 session.add(f.release.package) session.add(f.release) session.add(f) return rv
def by_filename(self): from pyshop.models import ReleaseFile file = ReleaseFile.by_filename(self.session, u'mirrored_package1', u'mirrored_package1-0.2.egg') self.assertIsInstance(file, ReleaseFile) self.assertEqual(file.release.package.name, u'mirrored_package1') self.assertEqual(file.release.version, u'0.2') self.assertEqual(file.package_type, u'bdist_egg')
def release_downloads(request, package_name, version): """ Retrieve a list of files and download count for a given package and release version. """ session = DBSession() release_files = ReleaseFile.by_release(session, package_name, version) if release_files: release_files = [(f.release.package.name, f.filename) for f in release_files] return release_files
def release_downloads(self, package_name, version): """ Retrieve a list of files and download count for a given package and release version. """ session = DBSession() release_files = ReleaseFile.by_release(session, package_name, version) if release_files: release_files = [(f.release.package.name, f.filename) for f in release_files] return release_files
def _create_release_file(self, release, data): data = self._to_unicode(data) return ReleaseFile(release=release, filename=data['filename'], md5_digest=data['md5_digest'], url=data['url'], size=data['size'], package_type=data['packagetype'], python_version=data['python_version'], has_sig=data.get('has_sig', False), comment_text=data.get('comment_text'), )
def get_release_file(root, request): session = DBSession() f = ReleaseFile.by_id(session, int(request.matchdict['file_id'])) url = f.url if url and url.startswith('http://pypi.python.org'): url = 'https' + url[4:] rv = {'id': f.id, 'url': url, 'filename': f.filename, } f.downloads += 1 f.release.downloads += 1 f.release.package.downloads += 1 session.add(f.release.package) session.add(f.release) session.add(f) return rv
def render(self): api = pypi.proxy settings = self.request.registry.settings sanitize = asbool(settings['pyshop.mirror.sanitize']) package_name = self.request.matchdict['package_name'] pkg = Package.by_name(self.session, package_name) refresh = True if pkg: if pkg.local: refresh = False else: if pkg.update_at: current_td = datetime.now() - pkg.update_at max_td = timedelta(hours=int( settings.get('pyshop.mirror.cache.ttl', '24'))) refresh = current_td > max_td log.debug('"{cdt}" > "{max}": {refr}'.format( cdt=current_td, max=max_td, refr=refresh)) if refresh: log.info('refresh package {pkg}'.format(pkg=package_name)) pypi_versions = api.package_releases(package_name, True) # XXX package_releases is case sensitive # but dependancies declaration not... if not pypi_versions: pkg_info = self._search_package(package_name) if not pkg_info and '-' in package_name: tmp_name = package_name.replace('-', '_') pkg_info = self._search_package(tmp_name) if not pkg_info and '_' in package_name: tmp_name = package_name.replace('_', '-') pkg_info = self._search_package(tmp_name) if pkg_info: package_name, pypi_versions = pkg_info else: pypi_versions = [] if not pkg: if not pypi_versions: log.info('package %s has no versions' % package_name) return {'package': None, 'package_name': package_name} if sanitize: re_sanitize = re.compile(settings['pyshop.mirror.' 'sanitize.regex']) pypi_versions = [ v for v in pypi_versions if re_sanitize.match(v) ] # mirror the package now log.info('mirror package %s now' % package_name) pkg = Package.by_name(self.session, package_name) if not pkg: pkg = Package(name=package_name, local=False) self.session.add(pkg) self.session.flush() roles = api.package_roles(package_name) for role, login in roles: login = login.decode('utf-8') # XMLRPC should return utf-8 user = User.by_login(self.session, login, local=False) if not user: user = User(login=login, local=False) self.session.add(user) if role == 'Owner': pkg.owners.append(user) elif role == 'Maintainer': pkg.maintainers.append(user) self.session.flush() refresh = True if not pkg.local and refresh: pkg_versions = pkg.versions for version in pypi_versions: if version not in pkg_versions: release_data = api.release_data(package_name, version) release = self._create_release(pkg, release_data) release_files = api.release_urls(package_name, version) for data in release_files: filename = data['filename'].decode('utf-8') rf = ReleaseFile.by_filename(self.session, release, filename) if not rf: rf = self._create_release_file(release, data) pkg.update_at = func.now() self.session.add(pkg) log.info('package %s mirrored' % package_name) return { 'package': pkg, 'whlify': asbool(settings.get('pyshop.mirror.wheelify', '0')) }
def render(self): api = pypi.proxy settings = self.request.registry.settings sanitize = asbool(settings['pyshop.mirror.sanitize']) package_name = self.request.matchdict['package_name'] pkg = Package.by_name(self.session, package_name) if pkg is None: pkg = Package.by_name(self.session, package_name.replace('-', '_')) refresh = True session_users = {} if pkg: if pkg.local: refresh = False else: if pkg.update_at: log.debug('validating cache interval') current_td = datetime.now() - pkg.update_at max_td = timedelta( hours=int(settings.get('pyshop.mirror.cache.ttl', '24'))) refresh = current_td > max_td log.debug('"%s" > "%s": %s', current_td, max_td, refresh) if refresh: log.info('refresh package %s', package_name) pypi_versions = api.package_releases(package_name, True) # XXX package_releases is case sensitive # but dependencies declaration not... if not pypi_versions: pkg_info = self._search_package(package_name) if not pkg_info and '-' in package_name: tmp_name = package_name.replace('-', '_') pkg_info = self._search_package(tmp_name) if not pkg_info and '_' in package_name: tmp_name = package_name.replace('_', '-') pkg_info = self._search_package(tmp_name) if pkg_info: package_name, pypi_versions = pkg_info pypi_versions = [ver.decode('utf-8') for ver in pypi_versions] else: pypi_versions = [] if not pkg: if not pypi_versions: log.info('package %s has no versions', package_name) return {'package': None, 'package_name': package_name} if sanitize: re_sanitize = re.compile(settings['pyshop.mirror.' 'sanitize.regex']) pypi_versions = [v for v in pypi_versions if re_sanitize.match(v)] # mirror the package now log.info('mirror package %s now', package_name) pkg = Package.by_name(self.session, package_name) if not pkg: pkg = Package(name=package_name, local=False) self.session.add(pkg) self.session.flush() roles = api.package_roles(package_name) for role, login in roles: login = login.decode('utf-8') # XMLRPC should return utf-8 log.info('Looking for non local user %s', login) if _sanitize(login) in session_users: user = session_users[_sanitize(login)] else: user = User.by_login(self.session, login, local=False) if not user: log.info('Not found. creating user {0}'.format(login)) user = User(login=login, local=False) self.session.add(user) if role == 'Owner': pkg.owners.append(user) self.session.add(pkg) elif role == 'Maintainer': pkg.maintainers.append(user) self.session.add(pkg) session_users[_sanitize(login)] = user self.session.flush() self.session.flush() if not pkg.local and refresh: log.debug('refreshing %s package', package_name) pkg_versions = set(pypi_versions).difference(pkg.versions) if not pkg_versions: log.info('No new version to mirror') log.debug('pypi versions: %s', pypi_versions) log.debug('mirrored versions: %s', pkg.versions) for version in pkg_versions: log.info('Mirroring version %s', version) release_data = api.release_data(package_name, version) release = self._create_release(pkg, release_data, session_users) release_files = api.release_urls(package_name, version) for data in release_files: filename = data['filename'].decode('utf-8') rf = ReleaseFile.by_filename(self.session, release, filename) if not rf: rf = self._create_release_file(release, data) pkg.update_at = func.now() self.session.add(pkg) log.info('package %s mirrored' % package_name) return {'package': pkg, 'whlify': asbool(settings.get('pyshop.mirror.wheelify', '0'))}
def render(self): settings = self.request.registry.settings if not self.user: raise exc.HTTPForbidden() params = self.request.params if (asbool(settings['pyshop.upload.sanitize']) and not re.match(settings['pyshop.upload.sanitize.regex'], params['version'] )): raise exc.HTTPForbidden( "Provided version ({}) should match ""regexp {}" .format(params['version'], settings['pyshop.upload.sanitize.regex'])) pkg = Package.by_name(self.session, params['name']) if pkg and pkg.local: auth = [user for user in pkg.owners + pkg.maintainers if user == self.user] if not auth: raise exc.HTTPForbidden() elif not pkg: pkg = Package(name=params['name'], local=True) pkg.owners.append(self.user) content = self.request.POST['content'] input_file = content.file if asbool(settings.get('pyshop.upload.rewrite_filename', '1')): # rewrite the filename, do not use the posted one for security filename = self._guess_filename(params, content.filename) else: filename = content.filename dir_ = os.path.join(settings['pyshop.repository'], filename[0].lower()) if not os.path.exists(dir_): os.makedirs(dir_, 0o750) filepath = os.path.join(dir_, filename) while os.path.exists(filepath): log.warning('File %s exists but new upload self.request, deleting', filepath) os.unlink(filepath) size = 0 with open(filepath, 'wb') as output_file: input_file.seek(0) while True: data = input_file.read(2 << 16) if not data: break size += len(data) output_file.write(data) release = Release.by_version(self.session, pkg.name, params['version']) if not release: release = Release(package=pkg, version=params['version'], summary=params.get('summary'), author=self.user, home_page=params.get('home_page'), license=params.get('license'), description=params.get('description'), keywords=params.get('keywords'), platform=params.get('platform'), download_url=params.get('download_url'), docs_url=params.get('docs_url'), ) classifiers = params.getall('classifiers') for name in classifiers: classifier = Classifier.by_name(self.session, name, create_if_not_exists=True) while classifier: if classifier not in release.classifiers: release.classifiers.append(classifier) if classifier not in pkg.classifiers: pkg.classifiers.append(classifier) classifier = classifier.parent rfile = ReleaseFile.by_filename(self.session, release, filename) if not rfile: rfile = ReleaseFile(release=release, filename=filename, size=size, md5_digest=params.get('md5_digest'), package_type=params['filetype'], python_version=params.get('pyversion'), comment_text=params.get('comment'), ) self.session.add(rfile) self.session.add(release) pkg.update_at = func.now() self.session.add(pkg) return {'release_file': rfile}
def render(self): api = pypi.proxy settings = self.request.registry.settings sanitize = asbool(settings['pyshop.mirror.sanitize']) package_name = self.request.matchdict['package_name'] pkg = Package.by_name(self.session, package_name) if pkg is None: pkg = Package.by_name(self.session, package_name.replace('-', '_')) if pkg is None: pkg = Package.by_name(self.session, package_name.replace('-', '.')) refresh = True session_users = {} if pkg: if pkg.local: refresh = False else: if pkg.update_at: log.debug('validating cache interval') current_td = datetime.now() - pkg.update_at max_td = timedelta( hours=int(settings.get('pyshop.mirror.cache.ttl', '24'))) refresh = current_td > max_td log.debug('"%s" > "%s": %s', current_td, max_td, refresh) if refresh: log.info('refresh package %s', package_name) pypi_versions = api.package_releases(package_name, True) # XXX package_releases is case sensitive # but dependencies declaration not... if not pypi_versions: pkg_info = self._search_package(package_name) if not pkg_info and '-' in package_name: tmp_name = package_name.replace('-', '_') pkg_info = self._search_package(tmp_name) if not pkg_info and '-' in package_name: tmp_name = package_name.replace('-', '.') pkg_info = self._search_package(tmp_name) if not pkg_info and '_' in package_name: tmp_name = package_name.replace('_', '-') pkg_info = self._search_package(tmp_name) if pkg_info: package_name, pypi_versions = pkg_info pypi_versions = [ver.decode('utf-8') for ver in pypi_versions] else: pypi_versions = [] if not pkg: if not pypi_versions: log.info('package %s has no versions', package_name) return {'package': None, 'package_name': package_name} if sanitize: re_sanitize = re.compile(settings['pyshop.mirror.' 'sanitize.regex']) pypi_versions = [v for v in pypi_versions if re_sanitize.match(v)] # mirror the package now log.info('mirror package %s now', package_name) pkg = Package.by_name(self.session, package_name) if not pkg: pkg = Package(name=package_name, local=False) self.session.add(pkg) self.session.flush() roles = api.package_roles(package_name) for role, login in roles: login = login.decode('utf-8') # XMLRPC should return utf-8 log.info('Looking for non local user %s', login) if _sanitize(login) in session_users: user = session_users[_sanitize(login)] else: user = User.by_login(self.session, login, local=False) if not user: log.info('Not found. creating user {0}'.format(login)) user = User(login=login, local=False) self.session.add(user) if role == 'Owner': pkg.owners.append(user) self.session.add(pkg) elif role == 'Maintainer': pkg.maintainers.append(user) self.session.add(pkg) session_users[_sanitize(login)] = user self.session.flush() self.session.flush() if not pkg.local and refresh: log.debug('refreshing %s package', package_name) pkg_versions = set(pypi_versions).difference(pkg.versions) if not pkg_versions: log.info('No new version to mirror') log.debug('pypi versions: %s', pypi_versions) log.debug('mirrored versions: %s', pkg.versions) for version in pkg_versions: log.info('Mirroring version %s', version) release_data = api.release_data(package_name, version) release = self._create_release(pkg, release_data, session_users) release_files = api.release_urls(package_name, version) for data in release_files: filename = data['filename'].decode('utf-8') rf = ReleaseFile.by_filename(self.session, release, filename) if not rf: rf = self._create_release_file(release, data) pkg.update_at = func.now() self.session.add(pkg) log.info('package %s mirrored' % package_name) return {'package': pkg, 'whlify': asbool(settings.get('pyshop.mirror.wheelify', '0'))}
def setUpModule(): engine = create_engine(settings) populate(engine, interactive=False) session = DBSession() admin_user = User.by_login(session, u'admin') local_user = User(login=u'local_user', password=u'secret', local=True, firstname=u'Local', lastname=u'User') local_user.groups.append(Group.by_name(session, u'developer')) jdo = User(login=u'johndo', local=False) jdoe = User(login=u'janedoe', local=False) session.add(jdo) session.add(jdoe) session.add(local_user) classifiers_names = [ u'Programming Language :: Python', u'Programming Language :: Python :: 2.6', u'Programming Language :: Python :: 2.7', u'Topic :: Software Development', u'Topic :: System :: Archiving :: Mirroring', u'Topic :: System :: Archiving :: Packaging', u'Intended Audience :: Developers', u'Intended Audience :: System Administrators' ] classifiers = [ Classifier.by_name(session, name=c, create_if_not_exists=True) for c in classifiers_names ] pack1 = Package(name=u'mirrored_package1') pack1.owners.append(jdo) pack1.owners.append(jdoe) pack1.downloads = 7 session.add(pack1) release1 = Release(package=pack1, version=u'0.1', summary=u'Common Usage Library', author=jdoe) for c in classifiers[:3]: release1.classifiers.append(c) session.add(release1) release1.files.append( ReleaseFile(filename=u'mirrored_package1-0.1.tar.gz', package_type=u'sdist')) session.add(release1) release2 = Release(package=pack1, version=u'0.2', summary=u'Common Usage Library') for c in classifiers[:5]: release2.classifiers.append(c) release2.files.append( ReleaseFile(filename=u'mirrored_package1-0.2.tar.gz', package_type=u'sdist')) release2.files.append( ReleaseFile(filename=u'mirrored_package1-0.2.egg', package_type=u'bdist_egg')) session.add(release2) pack2 = Package(name=u'mirrored_package2') pack2.owners.append(jdo) pack2.maintainers.append(jdoe) pack2.downloads = 1 session.add(pack2) release3 = Release(package=pack2, version=u'1.0', summary=u'Web Framework For Everybody') for c in classifiers[:3] + classifiers[-2:-2]: release3.classifiers.append(c) session.add(release3) release3.files.append( ReleaseFile(filename=u'mirrored_package2-1.0.tar.gz', package_type=u'sdist')) session.add(release3) pack3 = Package(name=u'local_package1', local=True) pack3.owners.append(local_user) pack3.owners.append(admin_user) session.add(pack3) release4 = Release(package=pack3, version=u'0.1', summary=u'Pet Shop Application') for c in classifiers: release4.classifiers.append(c) release4.files.append( ReleaseFile(filename=u'local_package1-0.1.tar.gz', package_type=u'sdist')) session.add(release4) session.commit()
def render(self): api = pypi.proxy settings = self.request.registry.settings sanitize = asbool(settings['pyshop.mirror.sanitize']) package_name = self.request.matchdict['package_name'] pkg = Package.by_name(self.session, package_name) refresh = True if pkg: if pkg.local: refresh = False else: if pkg.update_at: td = datetime.now() - pkg.update_at # refresh if the package has not been update today # XXX should be configurable refresh = td.days > 0 if refresh: pypi_versions = api.package_releases(package_name, True) # XXX package_releases is case sensitive # but dependancies declaration not... if not pypi_versions: search_result = api.search({'name': package_name}, True) search_count = len(search_result) search_result = [p for p in search_result if p['name'].lower() == package_name or p['name'].lower().replace('-', '_') == package_name] log.debug('Found {sc}, matched {mc}'.format(sc=search_count, mc=len(search_result))) if search_result: package_name = search_result[0]['name'] pypi_versions = api.package_releases(package_name, True) else: pypi_versions = [] if not pkg: if not pypi_versions: log.info('package %s has no versions' % package_name) return {'package': None, 'package_name': package_name} if sanitize: re_sanitize = re.compile(settings['pyshop.mirror.' 'sanitize.regex']) pypi_versions = [v for v in pypi_versions if re_sanitize.match(v)] # mirror the package now log.info('mirror package %s now' % package_name) pkg = Package(name=package_name, local=False) roles = api.package_roles(package_name) for role, login in roles: login = login.decode('utf-8') # XMLRPC should return utf-8 user = User.by_login(self.session, login, local=False) if not user: user = User(login=login, local=False) self.session.add(user) if role == 'Owner': pkg.owners.append(user) elif role == 'Maintainer': pkg.maintainers.append(user) self.session.flush() refresh = True if not pkg.local and refresh: pkg_versions = pkg.versions for version in pypi_versions: if version not in pkg_versions: release_data = api.release_data(package_name, version) release = self._create_release(pkg, release_data) release_files = api.release_urls(package_name, version) for data in release_files: filename = data['filename'].decode('utf-8') rf = ReleaseFile.by_filename(self.session, release, filename) if not rf: rf = self._create_release_file(release, data) pkg.update_at = func.now() self.session.add(pkg) log.info('package %s mirrored' % package_name) return {'package': pkg}
def render(self): settings = self.request.registry.settings if not self.user: raise exc.HTTPForbidden() params = self.request.params if (asbool(settings['pyshop.upload.sanitize']) and not re.match(settings['pyshop.upload.sanitize.regex'], params['version'] )): raise exc.HTTPForbidden() pkg = Package.by_name(self.session, params['name']) if pkg and pkg.local: auth = [user for user in pkg.owners + pkg.maintainers if user == self.user] if not auth: raise exc.HTTPForbidden() elif not pkg: pkg = Package(name=params['name'], local=True) pkg.owners.append(self.user) content = self.request.POST['content'] input_file = content.file if asbool(settings.get('pyshop.upload.rewrite_filename', '1')): # rewrite the filename, do not use the posted one for security filename = self._guess_filename(params, content.filename) else: filename = content.filename dir_ = os.path.join(settings['pyshop.repository'], filename[0].lower()) if not os.path.exists(dir_): os.makedirs(dir_, 0o750) filepath = os.path.join(dir_, filename) while os.path.exists(filepath): log.warning('File %s exists but new upload self.request, deleting' % filepath) os.unlink(filepath) size = 0 with open(filepath, 'wb') as output_file: input_file.seek(0) while True: data = input_file.read(2 << 16) if not data: break size += len(data) output_file.write(data) release = Release.by_version(self.session, pkg.name, params['version']) if not release: release = Release(package=pkg, version=params['version'], summary=params.get('summary'), author=self.user, home_page=params.get('home_page'), license=params.get('license'), description=params.get('description'), keywords=params.get('keywords'), platform=params.get('platform'), download_url=params.get('download_url'), docs_url=params.get('docs_url'), ) classifiers = params.getall('classifiers') for name in classifiers: classifier = Classifier.by_name(self.session, name) while classifier: if classifier not in release.classifiers: release.classifiers.append(classifier) if classifier not in pkg.classifiers: pkg.classifiers.append(classifier) classifier = classifier.parent rfile = ReleaseFile.by_filename(self.session, release, filename) if not rfile: rfile = ReleaseFile(release=release, filename=filename, size=size, md5_digest=params.get('md5_digest'), package_type=params['filetype'], python_version=params.get('pyversion'), comment_text=params.get('comment'), ) self.session.add(rfile) self.session.add(release) pkg.update_at = func.now() self.session.add(pkg) return {'release_file': rfile}
def render(self): api = pypi.proxy settings = self.request.registry.settings sanitize = asbool(settings['pyshop.mirror.sanitize']) package_name = self.request.matchdict['package_name'] pkg = Package.by_name(self.session, package_name) refresh = True if pkg: if pkg.local: refresh = False else: if pkg.update_at: current_td = datetime.now() - pkg.update_at max_td = timedelta(hours=int(settings.get('pyshop.mirror.cache.ttl', '24'))) refresh = current_td > max_td log.debug('"{cdt}" > "{max}": {refr}'.format(cdt=current_td, max=max_td, refr=refresh)) if refresh: log.info('refresh package {pkg}'.format(pkg=package_name)) pypi_versions = api.package_releases(package_name, True) # XXX package_releases is case sensitive # but dependancies declaration not... if not pypi_versions: pkg_info = self._search_package(package_name) if not pkg_info and '-' in package_name: tmp_name = package_name.replace('-', '_') pkg_info = self._search_package(tmp_name) if not pkg_info and '_' in package_name: tmp_name = package_name.replace('_', '-') pkg_info = self._search_package(tmp_name) if pkg_info: package_name, pypi_versions = pkg_info else: pypi_versions = [] if not pkg: if not pypi_versions: log.info('package %s has no versions' % package_name) return {'package': None, 'package_name': package_name} if sanitize: re_sanitize = re.compile(settings['pyshop.mirror.' 'sanitize.regex']) pypi_versions = [v for v in pypi_versions if re_sanitize.match(v)] # mirror the package now log.info('mirror package %s now' % package_name) pkg = Package.by_name(self.session, package_name) if not pkg: pkg = Package(name=package_name, local=False) self.session.add(pkg) self.session.flush() roles = api.package_roles(package_name) for role, login in roles: login = login.decode('utf-8') # XMLRPC should return utf-8 user = User.by_login(self.session, login, local=False) if not user: user = User(login=login, local=False) self.session.add(user) if role == 'Owner': pkg.owners.append(user) elif role == 'Maintainer': pkg.maintainers.append(user) self.session.flush() refresh = True if not pkg.local and refresh: pkg_versions = pkg.versions for version in pypi_versions: if version not in pkg_versions: release_data = api.release_data(package_name, version) release = self._create_release(pkg, release_data) release_files = api.release_urls(package_name, version) for data in release_files: filename = data['filename'].decode('utf-8') rf = ReleaseFile.by_filename(self.session, release, filename) if not rf: rf = self._create_release_file(release, data) pkg.update_at = func.now() self.session.add(pkg) log.info('package %s mirrored' % package_name) return {'package': pkg, 'whlify': asbool(settings.get('pyshop.mirror.wheelify', '0'))}
def render(self): api = pypi.proxy settings = self.request.registry.settings satanize = asbool(settings['pyshop.mirror.satanize']) package_name = self.request.matchdict['package_name'] pkg = Package.by_name(self.session, package_name) refresh = True if pkg: if pkg.local: refresh = False else: if pkg.update_at: td = datetime.now() - pkg.update_at refresh = td.days > 0 or td.seconds > 10800 if refresh: pypi_versions = api.package_releases(package_name, True) # XXX package_releases is case sensitive # but dependancies declaration not... if not pypi_versions: package_name = package_name.lower() search_result = api.search({'name': package_name}, True) search_result = [p for p in search_result if p['name'].lower() == package_name] if search_result: package_name = search_result[0]['name'] pypi_versions = api.package_releases(package_name, True) else: pypi_versions = [] if not pkg: if not pypi_versions: log.info('package %s has no versions' % package_name) return {'package': None, 'package_name': package_name} if satanize: re_satanize = re.compile(settings['pyshop.mirror.' 'satanize.regex']) pypi_versions = [v for v in pypi_versions if re_satanize.match(v)] # mirror the package now log.info('mirror package %s now' % package_name) pkg = Package(name=package_name, local=False) roles = api.package_roles(package_name) for role, login in roles: user = User.by_login(self.session, login, local=False) if not user: user = User(login=login, local=False) self.session.add(user) if role == 'Owner': pkg.owners.append(user) elif role == 'Maintainer': pkg.maintainers.append(user) self.session.flush() refresh = True if not pkg.local and refresh: pkg_versions = pkg.versions for version in pypi_versions: if version not in pkg_versions: release_data = api.release_data(package_name, version) release = self._create_release(pkg, release_data) release_files = api.release_urls(package_name, version) for data in release_files: rf = ReleaseFile.by_filename(self.session, release, data['filename']) if not rf: rf = self._create_release_file(release, data) pkg.update_at = func.now() self.session.add(pkg) log.info('package %s mirrored' % package_name) return {'package': pkg}
def render(self): settings = self.request.registry.settings username = authenticated_userid(self.request) if not username: raise exc.HTTPForbidden() remote_user = User.by_login(self.session, username) if not remote_user: raise exc.HTTPForbidden() params = self.request.params if (asbool(settings['pyshop.upload.satanize']) and not re.match(settings['pyshop.upload.satanize.regex'], params['version'] )): raise exc.HTTPForbidden() pkg = Package.by_name(self.session, params['name']) if pkg: auth = [user for user in pkg.owners + pkg.maintainers if user == remote_user] if not auth: raise exc.HTTPForbidden() else: pkg = Package(name=params['name'], local=True) pkg.owners.append(remote_user) content = self.request.POST['content'] input_file = content.file # rewrite the filename, do not use the posted one for security filename = u'%s-%s.%s' % (params['name'], params['version'], {u'sdist': u'tar.gz', u'bdist_egg': u'egg', u'bdist_msi': u'msi', u'bdist_dmg': u'zip', # XXX or gztar ? u'bdist_rpm': u'rpm', u'bdist_dumb': u'msi', u'bdist_wininst': u'exe', }[params['filetype']]) dir_ = os.path.join(settings['pyshop.repository'], filename[0].lower()) if not os.path.exists(dir_): os.mkdir(dir_, 0750) filepath = os.path.join(dir_, filename) while os.path.exists(filepath): log.warn('File %s exists but new upload self.request, deleting' % filepath) os.unlink(filepath) size = 0 with open(filepath, 'wb') as output_file: input_file.seek(0) while True: data = input_file.read(2<<16) if not data: break size += len(data) output_file.write(data) release = Release.by_version(self.session, pkg.name, params['version']) if not release: release = Release(package=pkg, version=params['version'], summary=params.get('summary'), author=remote_user, home_page=params.get('home_page'), license=params.get('license'), description=params.get('description'), keywords=params.get('keywords'), platform=params.get('platform'), download_url=params.get('download_url'), docs_url=params.get('docs_url'), ) classifiers = params.getall('classifiers') for name in classifiers: classifier = Classifier.by_name(self.session, name) while classifier: if classifier not in release.classifiers: release.classifiers.append(classifier) if classifier not in pkg.classifiers: pkg.classifiers.append(classifier) classifier = classifier.parent rfile = ReleaseFile.by_filename(self.session, release, filename) if not rfile: rfile = ReleaseFile(release=release, filename=filename, size=size, md5_digest=params.get('md5_digest'), package_type=params['filetype'], python_version=params.get('pyversion'), comment_text=params.get('comment'), ) self.session.add(rfile) self.session.add(release) pkg.update_at = func.now() self.session.add(pkg) return {'release_file': rfile}