def render(self): req = self.request page_no = 1 page_size = 20 if 'page_no' in req.matchdict: page_no = int(req.matchdict['page_no']) opts = {} if 'form.submitted' in req.params: opts['local_only'] = req.params.get('local_only', '0') == '1' else: opts['local_only'] = True opts['names'] = [] opts['classifiers'] = [] if 'form.submitted' in req.params or \ req.params.get('classifier.added'): classifiers = [ Classifier.by_id(self.session, id) for id in set(req.params.getall('classifiers')) ] names = req.params.getall('names') if req.params.get('classifier.added'): classifier = Classifier.by_name(self.session, req.params['classifier.added']) if classifier: classifiers.append(classifier) else: names.append(req.params['classifier.added']) opts['classifiers'] = classifiers opts['names'] = names package_count = Package.by_filter(self.session, opts, count='*') return { u'has_page': package_count > page_size, u'paging': { u'route': u'list_package_page', u'qs': self.request.query_string, u'kwargs': {}, u'max': int(math.ceil(float(package_count) / page_size)), u'no': page_no }, u'package_count': package_count, u'packages': Package.by_filter(self.session, opts, limit=page_size, offset=page_size * (page_no - 1), order_by=func.lower(Package.name)), u'filter': opts, u'classifiers': Classifier.all(self.session, order_by=Classifier.name) }
def user_packages(request, user): """ Retrieve a list of [role_name, package_name] for a given username. Role is either 'Maintainer' or 'Owner'. """ session = DBSession() owned = Package.by_owner(session, user) maintained = Package.by_maintainer(session, user) owned = [('Owner', p.name) for p in owned] maintained = [('Maintainer', p.name) for p in maintained] return owned + maintained
def user_packages(self, user): """ Retrieve a list of [role_name, package_name] for a given username. Role is either 'Maintainer' or 'Owner'. """ session = DBSession() owned = Package.by_owner(session, user) maintained = Package.by_maintainer(session, user) owned = [('Owner', p.name) for p in owned] maintained = [('Maintainer', p.name) for p in maintained] return owned + maintained
def test_by_name(self): from pyshop.models import Package pkg = Package.by_name(self.session, u'mirrored_package1') self.assertIsInstance(pkg, Package) self.assertEqual(pkg.id, 1) self.assertEqual(pkg.name, u'mirrored_package1') # Test case insensitivity pkg = Package.by_name(self.session, u'MirRored_pAckaGe1') self.assertIsInstance(pkg, Package) self.assertEqual(pkg.id, 1) self.assertEqual(pkg.name, u'mirrored_package1')
def render(self): req = self.request page_no = 1 page_size = 20 if 'page_no' in req.matchdict: page_no = int(req.matchdict['page_no']) opts = {} if 'form.submitted' in req.params: opts['local_only'] = req.params.get('local_only', '0') == '1' else: opts['local_only'] = True opts['names'] = [] opts['classifiers'] = [] if 'form.submitted' in req.params or \ req.params.get('classifier.added'): classifiers = [Classifier.by_id(self.session, id) for id in set(req.params.getall('classifiers'))] names = req.params.getall('names') if req.params.get('classifier.added'): classifier = Classifier.by_name(self.session, req.params['classifier.added']) if classifier: classifiers.append(classifier) else: names.append(req.params['classifier.added']) opts['classifiers'] = classifiers opts['names'] = names package_count = Package.by_filter(self.session, opts, count='*') return {u'has_page': package_count > page_size, u'paging': {u'route': u'list_package_page', u'qs': self.request.query_string, u'kwargs': {}, u'max': int( math.ceil(float(package_count) / page_size)), u'no': page_no}, u'package_count': package_count, u'packages': Package.by_filter( self.session, opts, limit=page_size, offset=page_size * (page_no - 1), order_by=func.lower(Package.name) ), u'filter': opts, u'classifiers': Classifier.all(self.session, order_by=Classifier.name) }
def render(self): req = self.request page_no = 1 page_size = 20 if "page_no" in req.matchdict: page_no = int(req.matchdict["page_no"]) opts = {} if "form.submitted" in req.params: opts["local_only"] = req.params.get("local_only", "0") == "1" else: opts["local_only"] = True opts["names"] = [] opts["classifiers"] = [] # TODO: set defaults in settings if "form.submitted" in req.params or req.params.get("classifier.added"): classifiers = [Classifier.by_id(self.session, id) for id in set(req.params.getall("classifiers"))] names = req.params.getall("names") if req.params.get("classifier.added"): classifier = Classifier.by_name(self.session, req.params["classifier.added"]) if classifier: log.info("!" * 80) log.info(classifier.__dict__) classifiers.append(classifier) else: names.append(req.params["classifier.added"]) opts["classifiers"] = classifiers opts["names"] = names package_count = Package.by_filter(self.session, opts, count="*") return { u"has_page": package_count > page_size, u"paging": { u"route": u"list_package_page", u"qs": self.request.query_string, u"kwargs": {}, u"max": int(math.ceil(float(package_count) / page_size)), u"no": page_no, }, u"package_count": package_count, u"packages": Package.by_filter( self.session, opts, limit=page_size, offset=page_size * (page_no - 1), order_by=func.lower(Package.name) ), u"filter": opts, u"classifiers": Classifier.all(self.session, order_by=Classifier.name), }
def package_releases(request, package_name, show_hidden=False): """ Retrieve a list of the releases registered for the given package_name. Returns a list with all version strings if show_hidden is True or only the non-hidden ones otherwise.""" session = DBSession() package = Package.by_name(session, package_name) return [rel.version for rel in package.sorted_releases]
def list_packages(self): """ Retrieve a list of the package names registered with the package index. Returns a list of name strings. """ session = DBSession() names = [p.name for p in Package.all(session, order_by=Package.name)] return names
def package_releases(self, package_name, show_hidden=False): """ Retrieve a list of the releases registered for the given package_name. Returns a list with all version strings if show_hidden is True or only the non-hidden ones otherwise.""" session = DBSession() package = Package.by_name(session, package_name) return [rel.version for rel in package.sorted_releases]
def list_packages(request): """ Retrieve a list of the package names registered with the package index. Returns a list of name strings. """ session = DBSession() names = [p.name for p in Package.all(session, order_by=Package.name)] return names
def render(self): package = Package.by_name(self.session, self.request.matchdict["package_name"]) if not package: raise HTTPNotFound() if "form.refresh_package" in self.request.params: package.update_at = None self.session.add(package) owners = dict((usr.login, usr) for usr in package.owners) can_edit_role = self.login in owners.keys() and package.local if "form.add_role" in self.request.params: if not can_edit_role: raise HTTPForbidden() user = User.by_login(self.session, self.request.params["login"]) if user and user.has_permission("upload_releasefile"): if self.request.params["role"] == "owner": if user.login not in owners: package.owners.append(user) else: maintainers = [usr.login for usr in package.owners] if user.login not in maintainers: package.maintainers.append(user) self.session.add(package) if "form.remove_maintainer" in self.request.params: if not can_edit_role: raise HTTPForbidden() user = User.by_login(self.session, self.request.params["login"]) if user: maintainers = dict((usr.login, usr) for usr in package.maintainers) if user.login in maintainers: package.maintainers.remove(maintainers[user.login]) self.session.add(package) if "form.remove_owner" in self.request.params: if not can_edit_role: raise HTTPForbidden() user = User.by_login(self.session, self.request.params["login"]) if user: if user.login in owners: package.owners.remove(owners[user.login]) self.session.add(package) if "release_version" in self.request.matchdict: release = Release.by_version(self.session, package.name, self.request.matchdict["release_version"]) else: release = package.sorted_releases[0] return {u"package": package, u"release": release, u"can_edit_role": can_edit_role}
def package_roles(self, package_name): """ Retrieve a list of users and their attributes roles for a given package_name. Role is either 'Maintainer' or 'Owner'. """ session = DBSession() package = Package.by_name(session, package_name) owners = [('Owner', o.name) for o in package.owners] maintainers = [('Maintainer', o.name) for o in package.maintainers] return owners + maintainers
def render(self): package = Package.by_name(self.session, self.request.matchdict["package_name"]) if not package: raise HTTPNotFound() if "release_version" in self.request.matchdict: release = Release.by_version(self.session, package.name, self.request.matchdict["release_version"]) else: release = package.sorted_releases[0] return {u"package": package, u"release": release}
def render(self): package = Package.by_name(self.session, self.request.matchdict['package_name']) if not package: raise HTTPNotFound() if 'release_version' in self.request.matchdict: release = Release.by_version(self.session, package.name, self.request.matchdict['release_version']) else: release = package.sorted_releases[0] return {u'package': package, u'release': release, }
def render(self): package = Package.by_name(self.session, self.request.matchdict['package_name']) if not package: raise HTTPNotFound() release = Release.by_version(self.session, package.name, self.request.matchdict['release_version']) filename = release.files[0].filename if not filename.endswith('.whl'): raise HTTPBadRequest() whl = WheelFile(os.path.join(os.environ['PYSHOP_DIR'], 'repository', filename[0].lower(), filename)) meta_name = next(n for n in whl.zipfile.namelist() if n.endswith('.dist-info/pydist.json')) return json.loads(whl.zipfile.read(meta_name))['run_requires'][0]['requires']
def render(self): package = Package.by_name(self.session, self.request.matchdict['package_name']) if not package: raise HTTPNotFound() if 'form.refresh_package' in self.request.params: package.update_at = None self.session.add(package) if 'release_version' in self.request.matchdict: release = Release.by_version(self.session, package.name, self.request.matchdict['release_version']) else: release = package.sorted_releases[0] return {u'package': package, u'release': release, }
def render(self): package = Package.by_name(self.session, self.request.matchdict['package_name']) if not package: raise HTTPNotFound() if 'form.refresh_package' in self.request.params: package.update_at = None self.session.add(package) if 'release_version' in self.request.matchdict: release = Release.by_version( self.session, package.name, self.request.matchdict['release_version']) else: release = package.sorted_releases[0] return { u'package': package, u'release': release, }
def render(self): package = Package.by_name(self.session, self.request.matchdict['package_name'])
def test_by_name(self): from pyshop.models import Package pkg = Package.by_name(self.session, u'mirrored_package1') self.assertIsInstance(pkg, Package) self.assertEqual(pkg.id, 1) self.assertEqual(pkg.name, u'mirrored_package1')
def test_get_mirrored(self): from pyshop.models import Package pkges = Package.get_mirrored(self.session) self.assertIsInstance(pkges, list) pkges = [pkg.name for pkg in pkges] self.assertEqual(pkges, [u'mirrored_package1', u'mirrored_package2'])
def render(self): return {'packages': Package.all(self.session, order_by=Package.name)}
def test_by_maintainer(self): from pyshop.models import Package pkges = Package.by_maintainer(self.session, u'janedoe') self.assertIsInstance(pkges, list) pkges = [pkg.name for pkg in pkges] self.assertEqual(pkges, [u'mirrored_package2'])
def render(self): api = pypi.proxy settings = self.request.registry.settings sanitize = asbool(settings['pyshop.mirror.sanitize']) package_name = self.request.matchdict['package_name'] pkg = Package.by_name(self.session, package_name) refresh = True if pkg: if pkg.local: refresh = False else: if pkg.update_at: td = datetime.now() - pkg.update_at # refresh if the package has not been update today # XXX should be configurable refresh = td.days > 0 if refresh: pypi_versions = api.package_releases(package_name, True) # XXX package_releases is case sensitive # but dependancies declaration not... if not pypi_versions: search_result = api.search({'name': package_name}, True) search_count = len(search_result) search_result = [p for p in search_result if p['name'].lower() == package_name or p['name'].lower().replace('-', '_') == package_name] log.debug('Found {sc}, matched {mc}'.format(sc=search_count, mc=len(search_result))) if search_result: package_name = search_result[0]['name'] pypi_versions = api.package_releases(package_name, True) else: pypi_versions = [] if not pkg: if not pypi_versions: log.info('package %s has no versions' % package_name) return {'package': None, 'package_name': package_name} if sanitize: re_sanitize = re.compile(settings['pyshop.mirror.' 'sanitize.regex']) pypi_versions = [v for v in pypi_versions if re_sanitize.match(v)] # mirror the package now log.info('mirror package %s now' % package_name) pkg = Package(name=package_name, local=False) roles = api.package_roles(package_name) for role, login in roles: login = login.decode('utf-8') # XMLRPC should return utf-8 user = User.by_login(self.session, login, local=False) if not user: user = User(login=login, local=False) self.session.add(user) if role == 'Owner': pkg.owners.append(user) elif role == 'Maintainer': pkg.maintainers.append(user) self.session.flush() refresh = True if not pkg.local and refresh: pkg_versions = pkg.versions for version in pypi_versions: if version not in pkg_versions: release_data = api.release_data(package_name, version) release = self._create_release(pkg, release_data) release_files = api.release_urls(package_name, version) for data in release_files: filename = data['filename'].decode('utf-8') rf = ReleaseFile.by_filename(self.session, release, filename) if not rf: rf = self._create_release_file(release, data) pkg.update_at = func.now() self.session.add(pkg) log.info('package %s mirrored' % package_name) return {'package': pkg}
def setUpModule(): engine = create_engine(settings) populate(engine, interactive=False) session = DBSession() admin_user = User.by_login(session, u'admin') local_user = User(login=u'local_user', password=u'secret', local=True, firstname=u'Local', lastname=u'User') local_user.groups.append(Group.by_name(session, u'developer')) jdo = User(login=u'johndo', local=False) jdoe = User(login=u'janedoe', local=False) session.add(jdo) session.add(jdoe) session.add(local_user) classifiers_names = [u'Programming Language :: Python', u'Programming Language :: Python :: 2.6', u'Programming Language :: Python :: 2.7', u'Topic :: Software Development', u'Topic :: System :: Archiving :: Mirroring', u'Topic :: System :: Archiving :: Packaging', u'Intended Audience :: Developers', u'Intended Audience :: System Administrators' ] classifiers = [Classifier.by_name(session, name=c, create_if_not_exists=True) for c in classifiers_names] pack1 = Package(name=u'mirrored_package1') pack1.owners.append(jdo) pack1.owners.append(jdoe) pack1.downloads = 7 session.add(pack1) release1 = Release(package=pack1, version=u'0.1', summary=u'Common Usage Library', author=jdoe) for c in classifiers[:3]: release1.classifiers.append(c) session.add(release1) release1.files.append(ReleaseFile(filename=u'mirrored_package1-0.1.tar.gz', package_type=u'sdist')) session.add(release1) release2 = Release(package=pack1, version=u'0.2', summary=u'Common Usage Library') for c in classifiers[:5]: release2.classifiers.append(c) release2.files.append(ReleaseFile(filename=u'mirrored_package1-0.2.tar.gz', package_type=u'sdist')) release2.files.append(ReleaseFile(filename=u'mirrored_package1-0.2.egg', package_type=u'bdist_egg')) session.add(release2) pack2 = Package(name=u'mirrored_package2') pack2.owners.append(jdo) pack2.maintainers.append(jdoe) pack2.downloads = 1 session.add(pack2) release3 = Release(package=pack2, version=u'1.0', summary=u'Web Framework For Everybody') for c in classifiers[:3] + classifiers[-2:-2]: release3.classifiers.append(c) session.add(release3) release3.files.append(ReleaseFile(filename=u'mirrored_package2-1.0.tar.gz', package_type=u'sdist')) session.add(release3) pack3 = Package(name=u'local_package1', local=True) pack3.owners.append(local_user) pack3.owners.append(admin_user) session.add(pack3) release4 = Release(package=pack3, version=u'0.1', summary=u'Pet Shop Application') for c in classifiers: release4.classifiers.append(c) release4.files.append(ReleaseFile(filename=u'local_package1-0.1.tar.gz', package_type=u'sdist')) session.add(release4) session.commit()
def render(self): settings = self.request.registry.settings if not self.user: raise exc.HTTPForbidden() params = self.request.params if (asbool(settings['pyshop.upload.sanitize']) and not re.match(settings['pyshop.upload.sanitize.regex'], params['version'] )): raise exc.HTTPForbidden() pkg = Package.by_name(self.session, params['name']) if pkg and pkg.local: auth = [user for user in pkg.owners + pkg.maintainers if user == self.user] if not auth: raise exc.HTTPForbidden() elif not pkg: pkg = Package(name=params['name'], local=True) pkg.owners.append(self.user) content = self.request.POST['content'] input_file = content.file if asbool(settings.get('pyshop.upload.rewrite_filename', '1')): # rewrite the filename, do not use the posted one for security filename = self._guess_filename(params, content.filename) else: filename = content.filename dir_ = os.path.join(settings['pyshop.repository'], filename[0].lower()) if not os.path.exists(dir_): os.makedirs(dir_, 0o750) filepath = os.path.join(dir_, filename) while os.path.exists(filepath): log.warning('File %s exists but new upload self.request, deleting' % filepath) os.unlink(filepath) size = 0 with open(filepath, 'wb') as output_file: input_file.seek(0) while True: data = input_file.read(2 << 16) if not data: break size += len(data) output_file.write(data) release = Release.by_version(self.session, pkg.name, params['version']) if not release: release = Release(package=pkg, version=params['version'], summary=params.get('summary'), author=self.user, home_page=params.get('home_page'), license=params.get('license'), description=params.get('description'), keywords=params.get('keywords'), platform=params.get('platform'), download_url=params.get('download_url'), docs_url=params.get('docs_url'), ) classifiers = params.getall('classifiers') for name in classifiers: classifier = Classifier.by_name(self.session, name) while classifier: if classifier not in release.classifiers: release.classifiers.append(classifier) if classifier not in pkg.classifiers: pkg.classifiers.append(classifier) classifier = classifier.parent rfile = ReleaseFile.by_filename(self.session, release, filename) if not rfile: rfile = ReleaseFile(release=release, filename=filename, size=size, md5_digest=params.get('md5_digest'), package_type=params['filetype'], python_version=params.get('pyversion'), comment_text=params.get('comment'), ) self.session.add(rfile) self.session.add(release) pkg.update_at = func.now() self.session.add(pkg) return {'release_file': rfile}
def render(self): api = pypi.proxy settings = self.request.registry.settings sanitize = asbool(settings['pyshop.mirror.sanitize']) package_name = self.request.matchdict['package_name'] pkg = Package.by_name(self.session, package_name) refresh = True if pkg: if pkg.local: refresh = False else: if pkg.update_at: current_td = datetime.now() - pkg.update_at max_td = timedelta(hours=int(settings.get('pyshop.mirror.cache.ttl', '24'))) refresh = current_td > max_td log.debug('"{cdt}" > "{max}": {refr}'.format(cdt=current_td, max=max_td, refr=refresh)) if refresh: log.info('refresh package {pkg}'.format(pkg=package_name)) pypi_versions = api.package_releases(package_name, True) # XXX package_releases is case sensitive # but dependancies declaration not... if not pypi_versions: pkg_info = self._search_package(package_name) if not pkg_info and '-' in package_name: tmp_name = package_name.replace('-', '_') pkg_info = self._search_package(tmp_name) if not pkg_info and '_' in package_name: tmp_name = package_name.replace('_', '-') pkg_info = self._search_package(tmp_name) if pkg_info: package_name, pypi_versions = pkg_info else: pypi_versions = [] if not pkg: if not pypi_versions: log.info('package %s has no versions' % package_name) return {'package': None, 'package_name': package_name} if sanitize: re_sanitize = re.compile(settings['pyshop.mirror.' 'sanitize.regex']) pypi_versions = [v for v in pypi_versions if re_sanitize.match(v)] # mirror the package now log.info('mirror package %s now' % package_name) pkg = Package.by_name(self.session, package_name) if not pkg: pkg = Package(name=package_name, local=False) self.session.add(pkg) self.session.flush() roles = api.package_roles(package_name) for role, login in roles: login = login.decode('utf-8') # XMLRPC should return utf-8 user = User.by_login(self.session, login, local=False) if not user: user = User(login=login, local=False) self.session.add(user) if role == 'Owner': pkg.owners.append(user) elif role == 'Maintainer': pkg.maintainers.append(user) self.session.flush() refresh = True if not pkg.local and refresh: pkg_versions = pkg.versions for version in pypi_versions: if version not in pkg_versions: release_data = api.release_data(package_name, version) release = self._create_release(pkg, release_data) release_files = api.release_urls(package_name, version) for data in release_files: filename = data['filename'].decode('utf-8') rf = ReleaseFile.by_filename(self.session, release, filename) if not rf: rf = self._create_release_file(release, data) pkg.update_at = func.now() self.session.add(pkg) log.info('package %s mirrored' % package_name) return {'package': pkg, 'whlify': asbool(settings.get('pyshop.mirror.wheelify', '0'))}
def render(self): api = pypi.proxy settings = self.request.registry.settings sanitize = asbool(settings['pyshop.mirror.sanitize']) package_name = self.request.matchdict['package_name'] pkg = Package.by_name(self.session, package_name) refresh = True if pkg: if pkg.local: refresh = False else: if pkg.update_at: current_td = datetime.now() - pkg.update_at max_td = timedelta(hours=int( settings.get('pyshop.mirror.cache.ttl', '24'))) refresh = current_td > max_td log.debug('"{cdt}" > "{max}": {refr}'.format( cdt=current_td, max=max_td, refr=refresh)) if refresh: log.info('refresh package {pkg}'.format(pkg=package_name)) pypi_versions = api.package_releases(package_name, True) # XXX package_releases is case sensitive # but dependancies declaration not... if not pypi_versions: pkg_info = self._search_package(package_name) if not pkg_info and '-' in package_name: tmp_name = package_name.replace('-', '_') pkg_info = self._search_package(tmp_name) if not pkg_info and '_' in package_name: tmp_name = package_name.replace('_', '-') pkg_info = self._search_package(tmp_name) if pkg_info: package_name, pypi_versions = pkg_info else: pypi_versions = [] if not pkg: if not pypi_versions: log.info('package %s has no versions' % package_name) return {'package': None, 'package_name': package_name} if sanitize: re_sanitize = re.compile(settings['pyshop.mirror.' 'sanitize.regex']) pypi_versions = [ v for v in pypi_versions if re_sanitize.match(v) ] # mirror the package now log.info('mirror package %s now' % package_name) pkg = Package.by_name(self.session, package_name) if not pkg: pkg = Package(name=package_name, local=False) self.session.add(pkg) self.session.flush() roles = api.package_roles(package_name) for role, login in roles: login = login.decode('utf-8') # XMLRPC should return utf-8 user = User.by_login(self.session, login, local=False) if not user: user = User(login=login, local=False) self.session.add(user) if role == 'Owner': pkg.owners.append(user) elif role == 'Maintainer': pkg.maintainers.append(user) self.session.flush() refresh = True if not pkg.local and refresh: pkg_versions = pkg.versions for version in pypi_versions: if version not in pkg_versions: release_data = api.release_data(package_name, version) release = self._create_release(pkg, release_data) release_files = api.release_urls(package_name, version) for data in release_files: filename = data['filename'].decode('utf-8') rf = ReleaseFile.by_filename(self.session, release, filename) if not rf: rf = self._create_release_file(release, data) pkg.update_at = func.now() self.session.add(pkg) log.info('package %s mirrored' % package_name) return { 'package': pkg, 'whlify': asbool(settings.get('pyshop.mirror.wheelify', '0')) }
def render(self): api = pypi.proxy settings = self.request.registry.settings sanitize = asbool(settings['pyshop.mirror.sanitize']) package_name = self.request.matchdict['package_name'] pkg = Package.by_name(self.session, package_name) if pkg is None: pkg = Package.by_name(self.session, package_name.replace('-', '_')) refresh = True session_users = {} if pkg: if pkg.local: refresh = False else: if pkg.update_at: log.debug('validating cache interval') current_td = datetime.now() - pkg.update_at max_td = timedelta( hours=int(settings.get('pyshop.mirror.cache.ttl', '24'))) refresh = current_td > max_td log.debug('"%s" > "%s": %s', current_td, max_td, refresh) if refresh: log.info('refresh package %s', package_name) pypi_versions = api.package_releases(package_name, True) # XXX package_releases is case sensitive # but dependencies declaration not... if not pypi_versions: pkg_info = self._search_package(package_name) if not pkg_info and '-' in package_name: tmp_name = package_name.replace('-', '_') pkg_info = self._search_package(tmp_name) if not pkg_info and '_' in package_name: tmp_name = package_name.replace('_', '-') pkg_info = self._search_package(tmp_name) if pkg_info: package_name, pypi_versions = pkg_info pypi_versions = [ver.decode('utf-8') for ver in pypi_versions] else: pypi_versions = [] if not pkg: if not pypi_versions: log.info('package %s has no versions', package_name) return {'package': None, 'package_name': package_name} if sanitize: re_sanitize = re.compile(settings['pyshop.mirror.' 'sanitize.regex']) pypi_versions = [v for v in pypi_versions if re_sanitize.match(v)] # mirror the package now log.info('mirror package %s now', package_name) pkg = Package.by_name(self.session, package_name) if not pkg: pkg = Package(name=package_name, local=False) self.session.add(pkg) self.session.flush() roles = api.package_roles(package_name) for role, login in roles: login = login.decode('utf-8') # XMLRPC should return utf-8 log.info('Looking for non local user %s', login) if _sanitize(login) in session_users: user = session_users[_sanitize(login)] else: user = User.by_login(self.session, login, local=False) if not user: log.info('Not found. creating user {0}'.format(login)) user = User(login=login, local=False) self.session.add(user) if role == 'Owner': pkg.owners.append(user) self.session.add(pkg) elif role == 'Maintainer': pkg.maintainers.append(user) self.session.add(pkg) session_users[_sanitize(login)] = user self.session.flush() self.session.flush() if not pkg.local and refresh: log.debug('refreshing %s package', package_name) pkg_versions = set(pypi_versions).difference(pkg.versions) if not pkg_versions: log.info('No new version to mirror') log.debug('pypi versions: %s', pypi_versions) log.debug('mirrored versions: %s', pkg.versions) for version in pkg_versions: log.info('Mirroring version %s', version) release_data = api.release_data(package_name, version) release = self._create_release(pkg, release_data, session_users) release_files = api.release_urls(package_name, version) for data in release_files: filename = data['filename'].decode('utf-8') rf = ReleaseFile.by_filename(self.session, release, filename) if not rf: rf = self._create_release_file(release, data) pkg.update_at = func.now() self.session.add(pkg) log.info('package %s mirrored' % package_name) return {'package': pkg, 'whlify': asbool(settings.get('pyshop.mirror.wheelify', '0'))}
def setUpModule(): engine = create_engine(settings) populate(engine, interactive=False) session = DBSession() admin_user = User.by_login(session, u"admin") local_user = User(login=u"local_user", password=u"secret", local=True, firstname=u"Local", lastname=u"User") local_user.groups.append(Group.by_name(session, u"developer")) jdo = User(login=u"johndo", local=False) jdoe = User(login=u"janedoe", local=False) session.add(jdo) session.add(jdoe) session.add(local_user) classifiers_names = [ u"Programming Language :: Python", u"Programming Language :: Python :: 2.6", u"Programming Language :: Python :: 2.7", u"Topic :: Software Development", u"Topic :: System :: Archiving :: Mirroring", u"Topic :: System :: Archiving :: Packaging", u"Intended Audience :: Developers", u"Intended Audience :: System Administrators", ] classifiers = [Classifier.by_name(session, name=c) for c in classifiers_names] pack1 = Package(name=u"mirrored_package1") pack1.owners.append(jdo) pack1.owners.append(jdoe) pack1.downloads = 7 session.add(pack1) release1 = Release(package=pack1, version=u"0.1", summary=u"Common Usage Library", author=jdoe) for c in classifiers[:3]: release1.classifiers.append(c) session.add(release1) release1.files.append(ReleaseFile(filename=u"mirrored_package1-0.1.tar.gz", package_type=u"sdist")) session.add(release1) release2 = Release(package=pack1, version=u"0.2", summary=u"Common Usage Library") for c in classifiers[:5]: release2.classifiers.append(c) release2.files.append(ReleaseFile(filename=u"mirrored_package1-0.2.tar.gz", package_type=u"sdist")) release2.files.append(ReleaseFile(filename=u"mirrored_package1-0.2.egg", package_type=u"bdist_egg")) session.add(release2) pack2 = Package(name=u"mirrored_package2") pack2.owners.append(jdo) pack2.maintainers.append(jdoe) pack2.downloads = 1 session.add(pack2) release3 = Release(package=pack2, version=u"1.0", summary=u"Web Framework For Everybody") for c in classifiers[:3] + classifiers[-2:-2]: release3.classifiers.append(c) session.add(release3) release3.files.append(ReleaseFile(filename=u"mirrored_package2-1.0.tar.gz", package_type=u"sdist")) session.add(release3) pack3 = Package(name=u"local_package1", local=True) pack3.owners.append(local_user) pack3.owners.append(admin_user) session.add(pack3) release4 = Release(package=pack3, version=u"0.1", summary=u"Pet Shop Application") for c in classifiers: release4.classifiers.append(c) release4.files.append(ReleaseFile(filename=u"local_package1-0.1.tar.gz", package_type=u"sdist")) session.add(release4) session.commit()
def render(self): api = pypi.proxy settings = self.request.registry.settings sanitize = asbool(settings['pyshop.mirror.sanitize']) package_name = self.request.matchdict['package_name'] pkg = Package.by_name(self.session, package_name) if pkg is None: pkg = Package.by_name(self.session, package_name.replace('-', '_')) if pkg is None: pkg = Package.by_name(self.session, package_name.replace('-', '.')) refresh = True session_users = {} if pkg: if pkg.local: refresh = False else: if pkg.update_at: log.debug('validating cache interval') current_td = datetime.now() - pkg.update_at max_td = timedelta( hours=int(settings.get('pyshop.mirror.cache.ttl', '24'))) refresh = current_td > max_td log.debug('"%s" > "%s": %s', current_td, max_td, refresh) if refresh: log.info('refresh package %s', package_name) pypi_versions = api.package_releases(package_name, True) # XXX package_releases is case sensitive # but dependencies declaration not... if not pypi_versions: pkg_info = self._search_package(package_name) if not pkg_info and '-' in package_name: tmp_name = package_name.replace('-', '_') pkg_info = self._search_package(tmp_name) if not pkg_info and '-' in package_name: tmp_name = package_name.replace('-', '.') pkg_info = self._search_package(tmp_name) if not pkg_info and '_' in package_name: tmp_name = package_name.replace('_', '-') pkg_info = self._search_package(tmp_name) if pkg_info: package_name, pypi_versions = pkg_info pypi_versions = [ver.decode('utf-8') for ver in pypi_versions] else: pypi_versions = [] if not pkg: if not pypi_versions: log.info('package %s has no versions', package_name) return {'package': None, 'package_name': package_name} if sanitize: re_sanitize = re.compile(settings['pyshop.mirror.' 'sanitize.regex']) pypi_versions = [v for v in pypi_versions if re_sanitize.match(v)] # mirror the package now log.info('mirror package %s now', package_name) pkg = Package.by_name(self.session, package_name) if not pkg: pkg = Package(name=package_name, local=False) self.session.add(pkg) self.session.flush() roles = api.package_roles(package_name) for role, login in roles: login = login.decode('utf-8') # XMLRPC should return utf-8 log.info('Looking for non local user %s', login) if _sanitize(login) in session_users: user = session_users[_sanitize(login)] else: user = User.by_login(self.session, login, local=False) if not user: log.info('Not found. creating user {0}'.format(login)) user = User(login=login, local=False) self.session.add(user) if role == 'Owner': pkg.owners.append(user) self.session.add(pkg) elif role == 'Maintainer': pkg.maintainers.append(user) self.session.add(pkg) session_users[_sanitize(login)] = user self.session.flush() self.session.flush() if not pkg.local and refresh: log.debug('refreshing %s package', package_name) pkg_versions = set(pypi_versions).difference(pkg.versions) if not pkg_versions: log.info('No new version to mirror') log.debug('pypi versions: %s', pypi_versions) log.debug('mirrored versions: %s', pkg.versions) for version in pkg_versions: log.info('Mirroring version %s', version) release_data = api.release_data(package_name, version) release = self._create_release(pkg, release_data, session_users) release_files = api.release_urls(package_name, version) for data in release_files: filename = data['filename'].decode('utf-8') rf = ReleaseFile.by_filename(self.session, release, filename) if not rf: rf = self._create_release_file(release, data) pkg.update_at = func.now() self.session.add(pkg) log.info('package %s mirrored' % package_name) return {'package': pkg, 'whlify': asbool(settings.get('pyshop.mirror.wheelify', '0'))}
def render(self): settings = self.request.registry.settings username = authenticated_userid(self.request) if not username: raise exc.HTTPForbidden() remote_user = User.by_login(self.session, username) if not remote_user: raise exc.HTTPForbidden() params = self.request.params if (asbool(settings['pyshop.upload.satanize']) and not re.match(settings['pyshop.upload.satanize.regex'], params['version'] )): raise exc.HTTPForbidden() pkg = Package.by_name(self.session, params['name']) if pkg: auth = [user for user in pkg.owners + pkg.maintainers if user == remote_user] if not auth: raise exc.HTTPForbidden() else: pkg = Package(name=params['name'], local=True) pkg.owners.append(remote_user) content = self.request.POST['content'] input_file = content.file # rewrite the filename, do not use the posted one for security filename = u'%s-%s.%s' % (params['name'], params['version'], {u'sdist': u'tar.gz', u'bdist_egg': u'egg', u'bdist_msi': u'msi', u'bdist_dmg': u'zip', # XXX or gztar ? u'bdist_rpm': u'rpm', u'bdist_dumb': u'msi', u'bdist_wininst': u'exe', }[params['filetype']]) dir_ = os.path.join(settings['pyshop.repository'], filename[0].lower()) if not os.path.exists(dir_): os.mkdir(dir_, 0750) filepath = os.path.join(dir_, filename) while os.path.exists(filepath): log.warn('File %s exists but new upload self.request, deleting' % filepath) os.unlink(filepath) size = 0 with open(filepath, 'wb') as output_file: input_file.seek(0) while True: data = input_file.read(2<<16) if not data: break size += len(data) output_file.write(data) release = Release.by_version(self.session, pkg.name, params['version']) if not release: release = Release(package=pkg, version=params['version'], summary=params.get('summary'), author=remote_user, home_page=params.get('home_page'), license=params.get('license'), description=params.get('description'), keywords=params.get('keywords'), platform=params.get('platform'), download_url=params.get('download_url'), docs_url=params.get('docs_url'), ) classifiers = params.getall('classifiers') for name in classifiers: classifier = Classifier.by_name(self.session, name) while classifier: if classifier not in release.classifiers: release.classifiers.append(classifier) if classifier not in pkg.classifiers: pkg.classifiers.append(classifier) classifier = classifier.parent rfile = ReleaseFile.by_filename(self.session, release, filename) if not rfile: rfile = ReleaseFile(release=release, filename=filename, size=size, md5_digest=params.get('md5_digest'), package_type=params['filetype'], python_version=params.get('pyversion'), comment_text=params.get('comment'), ) self.session.add(rfile) self.session.add(release) pkg.update_at = func.now() self.session.add(pkg) return {'release_file': rfile}
def render(self): settings = self.request.registry.settings if not self.user: raise exc.HTTPForbidden() params = self.request.params if (asbool(settings['pyshop.upload.sanitize']) and not re.match(settings['pyshop.upload.sanitize.regex'], params['version'] )): raise exc.HTTPForbidden( "Provided version ({}) should match ""regexp {}" .format(params['version'], settings['pyshop.upload.sanitize.regex'])) pkg = Package.by_name(self.session, params['name']) if pkg and pkg.local: auth = [user for user in pkg.owners + pkg.maintainers if user == self.user] if not auth: raise exc.HTTPForbidden() elif not pkg: pkg = Package(name=params['name'], local=True) pkg.owners.append(self.user) content = self.request.POST['content'] input_file = content.file if asbool(settings.get('pyshop.upload.rewrite_filename', '1')): # rewrite the filename, do not use the posted one for security filename = self._guess_filename(params, content.filename) else: filename = content.filename dir_ = os.path.join(settings['pyshop.repository'], filename[0].lower()) if not os.path.exists(dir_): os.makedirs(dir_, 0o750) filepath = os.path.join(dir_, filename) while os.path.exists(filepath): log.warning('File %s exists but new upload self.request, deleting', filepath) os.unlink(filepath) size = 0 with open(filepath, 'wb') as output_file: input_file.seek(0) while True: data = input_file.read(2 << 16) if not data: break size += len(data) output_file.write(data) release = Release.by_version(self.session, pkg.name, params['version']) if not release: release = Release(package=pkg, version=params['version'], summary=params.get('summary'), author=self.user, home_page=params.get('home_page'), license=params.get('license'), description=params.get('description'), keywords=params.get('keywords'), platform=params.get('platform'), download_url=params.get('download_url'), docs_url=params.get('docs_url'), ) classifiers = params.getall('classifiers') for name in classifiers: classifier = Classifier.by_name(self.session, name, create_if_not_exists=True) while classifier: if classifier not in release.classifiers: release.classifiers.append(classifier) if classifier not in pkg.classifiers: pkg.classifiers.append(classifier) classifier = classifier.parent rfile = ReleaseFile.by_filename(self.session, release, filename) if not rfile: rfile = ReleaseFile(release=release, filename=filename, size=size, md5_digest=params.get('md5_digest'), package_type=params['filetype'], python_version=params.get('pyversion'), comment_text=params.get('comment'), ) self.session.add(rfile) self.session.add(release) pkg.update_at = func.now() self.session.add(pkg) return {'release_file': rfile}
def test_get_locals(self): from pyshop.models import Package pkges = Package.get_locals(self.session) self.assertIsInstance(pkges, list) pkges = [pkg.name for pkg in pkges] self.assertEqual(pkges, [u'local_package1'])
def render(self): package = Package.by_name(self.session, self.request.matchdict['package_name']) if not package: raise HTTPNotFound() if 'form.refresh_package' in self.request.params: package.update_at = None self.session.add(package) owners = dict((usr.login, usr) for usr in package.owners) can_edit_role = self.login in owners.keys() and package.local if 'form.add_role' in self.request.params: if not can_edit_role: raise HTTPForbidden() user = User.by_login(self.session, self.request.params['login']) if user and user.has_permission('upload_releasefile'): if self.request.params['role'] == 'owner': if user.login not in owners: package.owners.append(user) else: maintainers = [usr.login for usr in package.owners] if user.login not in maintainers: package.maintainers.append(user) self.session.add(package) if 'form.remove_maintainer' in self.request.params: if not can_edit_role: raise HTTPForbidden() user = User.by_login(self.session, self.request.params['login']) if user: maintainers = dict((usr.login, usr) for usr in package.maintainers) if user.login in maintainers: package.maintainers.remove(maintainers[user.login]) self.session.add(package) if 'form.remove_owner' in self.request.params: if not can_edit_role: raise HTTPForbidden() user = User.by_login(self.session, self.request.params['login']) if user: if user.login in owners: package.owners.remove(owners[user.login]) self.session.add(package) if 'release_version' in self.request.matchdict: release = Release.by_version(self.session, package.name, self.request.matchdict['release_version']) else: release = package.sorted_releases[0] return {u'package': package, u'release': release, u'can_edit_role': can_edit_role, }
def test_sorted_releases(self): from pyshop.models import Package pkg = Package.by_name(self.session, u'mirrored_package1') self.assertEqual([release.version for release in pkg.sorted_releases], ['0.2', '0.1'])
def render(self): package = Package.by_name(self.session, self.request.matchdict['package_name']) if not package: raise HTTPNotFound() if 'form.refresh_package' in self.request.params: package.update_at = None self.session.add(package) owners = dict((usr.login, usr) for usr in package.owners) can_edit_role = self.login in owners.keys() and package.local if 'form.add_role' in self.request.params: if not can_edit_role: raise HTTPForbidden() user = User.by_login(self.session, self.request.params['login']) if user and user.has_permission('upload_releasefile'): if self.request.params['role'] == 'owner': if user.login not in owners: package.owners.append(user) else: maintainers = [usr.login for usr in package.owners] if user.login not in maintainers: package.maintainers.append(user) self.session.add(package) if 'form.remove_maintainer' in self.request.params: if not can_edit_role: raise HTTPForbidden() user = User.by_login(self.session, self.request.params['login']) if user: maintainers = dict((usr.login, usr) for usr in package.maintainers) if user.login in maintainers: package.maintainers.remove(maintainers[user.login]) self.session.add(package) if 'form.remove_owner' in self.request.params: if not can_edit_role: raise HTTPForbidden() user = User.by_login(self.session, self.request.params['login']) if user: if user.login in owners: package.owners.remove(owners[user.login]) self.session.add(package) if 'release_version' in self.request.matchdict: release = Release.by_version( self.session, package.name, self.request.matchdict['release_version']) else: release = package.sorted_releases[0] return {u'package': package, u'release': release, u'can_edit_role': can_edit_role, }
def render(self): api = pypi.proxy settings = self.request.registry.settings satanize = asbool(settings['pyshop.mirror.satanize']) package_name = self.request.matchdict['package_name'] pkg = Package.by_name(self.session, package_name) refresh = True if pkg: if pkg.local: refresh = False else: if pkg.update_at: td = datetime.now() - pkg.update_at refresh = td.days > 0 or td.seconds > 10800 if refresh: pypi_versions = api.package_releases(package_name, True) # XXX package_releases is case sensitive # but dependancies declaration not... if not pypi_versions: package_name = package_name.lower() search_result = api.search({'name': package_name}, True) search_result = [p for p in search_result if p['name'].lower() == package_name] if search_result: package_name = search_result[0]['name'] pypi_versions = api.package_releases(package_name, True) else: pypi_versions = [] if not pkg: if not pypi_versions: log.info('package %s has no versions' % package_name) return {'package': None, 'package_name': package_name} if satanize: re_satanize = re.compile(settings['pyshop.mirror.' 'satanize.regex']) pypi_versions = [v for v in pypi_versions if re_satanize.match(v)] # mirror the package now log.info('mirror package %s now' % package_name) pkg = Package(name=package_name, local=False) roles = api.package_roles(package_name) for role, login in roles: user = User.by_login(self.session, login, local=False) if not user: user = User(login=login, local=False) self.session.add(user) if role == 'Owner': pkg.owners.append(user) elif role == 'Maintainer': pkg.maintainers.append(user) self.session.flush() refresh = True if not pkg.local and refresh: pkg_versions = pkg.versions for version in pypi_versions: if version not in pkg_versions: release_data = api.release_data(package_name, version) release = self._create_release(pkg, release_data) release_files = api.release_urls(package_name, version) for data in release_files: rf = ReleaseFile.by_filename(self.session, release, data['filename']) if not rf: rf = self._create_release_file(release, data) pkg.update_at = func.now() self.session.add(pkg) log.info('package %s mirrored' % package_name) return {'package': pkg}
def setUpModule(): engine = create_engine(settings) populate(engine, interactive=False) session = DBSession() admin_user = User.by_login(session, u'admin') local_user = User(login=u'local_user', password=u'secret', local=True, firstname=u'Local', lastname=u'User') local_user.groups.append(Group.by_name(session, u'developer')) jdo = User(login=u'johndo', local=False) jdoe = User(login=u'janedoe', local=False) session.add(jdo) session.add(jdoe) session.add(local_user) classifiers_names = [ u'Programming Language :: Python', u'Programming Language :: Python :: 2.6', u'Programming Language :: Python :: 2.7', u'Topic :: Software Development', u'Topic :: System :: Archiving :: Mirroring', u'Topic :: System :: Archiving :: Packaging', u'Intended Audience :: Developers', u'Intended Audience :: System Administrators' ] classifiers = [ Classifier.by_name(session, name=c, create_if_not_exists=True) for c in classifiers_names ] pack1 = Package(name=u'mirrored_package1') pack1.owners.append(jdo) pack1.owners.append(jdoe) pack1.downloads = 7 session.add(pack1) release1 = Release(package=pack1, version=u'0.1', summary=u'Common Usage Library', author=jdoe) for c in classifiers[:3]: release1.classifiers.append(c) session.add(release1) release1.files.append( ReleaseFile(filename=u'mirrored_package1-0.1.tar.gz', package_type=u'sdist')) session.add(release1) release2 = Release(package=pack1, version=u'0.2', summary=u'Common Usage Library') for c in classifiers[:5]: release2.classifiers.append(c) release2.files.append( ReleaseFile(filename=u'mirrored_package1-0.2.tar.gz', package_type=u'sdist')) release2.files.append( ReleaseFile(filename=u'mirrored_package1-0.2.egg', package_type=u'bdist_egg')) session.add(release2) pack2 = Package(name=u'mirrored_package2') pack2.owners.append(jdo) pack2.maintainers.append(jdoe) pack2.downloads = 1 session.add(pack2) release3 = Release(package=pack2, version=u'1.0', summary=u'Web Framework For Everybody') for c in classifiers[:3] + classifiers[-2:-2]: release3.classifiers.append(c) session.add(release3) release3.files.append( ReleaseFile(filename=u'mirrored_package2-1.0.tar.gz', package_type=u'sdist')) session.add(release3) pack3 = Package(name=u'local_package1', local=True) pack3.owners.append(local_user) pack3.owners.append(admin_user) session.add(pack3) release4 = Release(package=pack3, version=u'0.1', summary=u'Pet Shop Application') for c in classifiers: release4.classifiers.append(c) release4.files.append( ReleaseFile(filename=u'local_package1-0.1.tar.gz', package_type=u'sdist')) session.add(release4) session.commit()