def test_by_name(self):
from pyshop.models import Package
pkg = Package.by_name(self.session, u'mirrored_package1')
self.assertIsInstance(pkg, Package)
self.assertEqual(pkg.id, 1)
self.assertEqual(pkg.name, u'mirrored_package1')
# Test case insensitivity
pkg = Package.by_name(self.session, u'MirRored_pAckaGe1')
self.assertIsInstance(pkg, Package)
self.assertEqual(pkg.id, 1)
self.assertEqual(pkg.name, u'mirrored_package1')
def package_releases(self, package_name, show_hidden=False):
"""
Retrieve a list of the releases registered for the given package_name.
Returns a list with all version strings if show_hidden is True or
only the non-hidden ones otherwise."""
session = DBSession()
package = Package.by_name(session, package_name)
return [rel.version for rel in package.sorted_releases]
def package_releases(request, package_name, show_hidden=False):
"""
Retrieve a list of the releases registered for the given package_name.
Returns a list with all version strings if show_hidden is True or
only the non-hidden ones otherwise."""
session = DBSession()
package = Package.by_name(session, package_name)
return [rel.version for rel in package.sorted_releases]
def render(self):
package = Package.by_name(self.session, self.request.matchdict["package_name"])
if not package:
raise HTTPNotFound()
if "form.refresh_package" in self.request.params:
package.update_at = None
self.session.add(package)
owners = dict((usr.login, usr) for usr in package.owners)
can_edit_role = self.login in owners.keys() and package.local
if "form.add_role" in self.request.params:
if not can_edit_role:
raise HTTPForbidden()
user = User.by_login(self.session, self.request.params["login"])
if user and user.has_permission("upload_releasefile"):
if self.request.params["role"] == "owner":
if user.login not in owners:
package.owners.append(user)
else:
maintainers = [usr.login for usr in package.owners]
if user.login not in maintainers:
package.maintainers.append(user)
self.session.add(package)
if "form.remove_maintainer" in self.request.params:
if not can_edit_role:
raise HTTPForbidden()
user = User.by_login(self.session, self.request.params["login"])
if user:
maintainers = dict((usr.login, usr) for usr in package.maintainers)
if user.login in maintainers:
package.maintainers.remove(maintainers[user.login])
self.session.add(package)
if "form.remove_owner" in self.request.params:
if not can_edit_role:
raise HTTPForbidden()
user = User.by_login(self.session, self.request.params["login"])
if user:
if user.login in owners:
package.owners.remove(owners[user.login])
self.session.add(package)
if "release_version" in self.request.matchdict:
release = Release.by_version(self.session, package.name, self.request.matchdict["release_version"])
else:
release = package.sorted_releases[0]
return {u"package": package, u"release": release, u"can_edit_role": can_edit_role}
def package_roles(self, package_name):
"""
Retrieve a list of users and their attributes roles for a given
package_name. Role is either 'Maintainer' or 'Owner'.
"""
session = DBSession()
package = Package.by_name(session, package_name)
owners = [('Owner', o.name) for o in package.owners]
maintainers = [('Maintainer', o.name) for o in package.maintainers]
return owners + maintainers
def package_roles(self, package_name):
"""
Retrieve a list of users and their attributes roles for a given
package_name. Role is either 'Maintainer' or 'Owner'.
"""
session = DBSession()
package = Package.by_name(session, package_name)
owners = [('Owner', o.name) for o in package.owners]
maintainers = [('Maintainer', o.name) for o in package.maintainers]
return owners + maintainers
def render(self):
package = Package.by_name(self.session, self.request.matchdict["package_name"])
if not package:
raise HTTPNotFound()
if "release_version" in self.request.matchdict:
release = Release.by_version(self.session, package.name, self.request.matchdict["release_version"])
else:
release = package.sorted_releases[0]
return {u"package": package, u"release": release}
def render(self):
package = Package.by_name(self.session,
self.request.matchdict['package_name'])
if not package:
raise HTTPNotFound()
if 'release_version' in self.request.matchdict:
release = Release.by_version(self.session, package.name,
self.request.matchdict['release_version'])
else:
release = package.sorted_releases[0]
return {u'package': package,
u'release': release,
}
def render(self):
package = Package.by_name(self.session,
self.request.matchdict['package_name'])
if not package:
raise HTTPNotFound()
if 'form.refresh_package' in self.request.params:
package.update_at = None
self.session.add(package)
if 'release_version' in self.request.matchdict:
release = Release.by_version(self.session, package.name,
self.request.matchdict['release_version'])
else:
release = package.sorted_releases[0]
return {u'package': package,
u'release': release,
}
def render(self):
package = Package.by_name(self.session,
self.request.matchdict['package_name'])
if not package:
raise HTTPNotFound()
release = Release.by_version(self.session, package.name,
self.request.matchdict['release_version'])
filename = release.files[0].filename
if not filename.endswith('.whl'):
raise HTTPBadRequest()
whl = WheelFile(os.path.join(os.environ['PYSHOP_DIR'],
'repository',
filename[0].lower(),
filename))
meta_name = next(n for n in whl.zipfile.namelist()
if n.endswith('.dist-info/pydist.json'))
return json.loads(whl.zipfile.read(meta_name))['run_requires'][0]['requires']
def render(self):
package = Package.by_name(self.session,
self.request.matchdict['package_name'])
if not package:
raise HTTPNotFound()
if 'form.refresh_package' in self.request.params:
package.update_at = None
self.session.add(package)
if 'release_version' in self.request.matchdict:
release = Release.by_version(
self.session, package.name,
self.request.matchdict['release_version'])
else:
release = package.sorted_releases[0]
return {
u'package': package,
u'release': release,
}
def render(self):
package = Package.by_name(self.session,
self.request.matchdict['package_name'])
if not package:
raise HTTPNotFound()
if 'form.refresh_package' in self.request.params:
package.update_at = None
self.session.add(package)
owners = dict((usr.login, usr) for usr in package.owners)
can_edit_role = self.login in owners.keys() and package.local
if 'form.add_role' in self.request.params:
if not can_edit_role:
raise HTTPForbidden()
user = User.by_login(self.session, self.request.params['login'])
if user and user.has_permission('upload_releasefile'):
if self.request.params['role'] == 'owner':
if user.login not in owners:
package.owners.append(user)
else:
maintainers = [usr.login for usr in package.owners]
if user.login not in maintainers:
package.maintainers.append(user)
self.session.add(package)
if 'form.remove_maintainer' in self.request.params:
if not can_edit_role:
raise HTTPForbidden()
user = User.by_login(self.session, self.request.params['login'])
if user:
maintainers = dict((usr.login, usr)
for usr in package.maintainers)
if user.login in maintainers:
package.maintainers.remove(maintainers[user.login])
self.session.add(package)
if 'form.remove_owner' in self.request.params:
if not can_edit_role:
raise HTTPForbidden()
user = User.by_login(self.session, self.request.params['login'])
if user:
if user.login in owners:
package.owners.remove(owners[user.login])
self.session.add(package)
if 'release_version' in self.request.matchdict:
release = Release.by_version(
self.session, package.name,
self.request.matchdict['release_version'])
else:
release = package.sorted_releases[0]
return {u'package': package,
u'release': release,
u'can_edit_role': can_edit_role,
}
def render(self):
settings = self.request.registry.settings
if not self.user:
raise exc.HTTPForbidden()
params = self.request.params
if (asbool(settings['pyshop.upload.sanitize']) and
not re.match(settings['pyshop.upload.sanitize.regex'],
params['version']
)):
raise exc.HTTPForbidden(
"Provided version ({}) should match ""regexp {}"
.format(params['version'],
settings['pyshop.upload.sanitize.regex']))
pkg = Package.by_name(self.session, params['name'])
if pkg and pkg.local:
auth = [user for user in pkg.owners + pkg.maintainers
if user == self.user]
if not auth:
raise exc.HTTPForbidden()
elif not pkg:
pkg = Package(name=params['name'], local=True)
pkg.owners.append(self.user)
content = self.request.POST['content']
input_file = content.file
if asbool(settings.get('pyshop.upload.rewrite_filename', '1')):
# rewrite the filename, do not use the posted one for security
filename = self._guess_filename(params, content.filename)
else:
filename = content.filename
dir_ = os.path.join(settings['pyshop.repository'],
filename[0].lower())
if not os.path.exists(dir_):
os.makedirs(dir_, 0o750)
filepath = os.path.join(dir_, filename)
while os.path.exists(filepath):
log.warning('File %s exists but new upload self.request, deleting',
filepath)
os.unlink(filepath)
size = 0
with open(filepath, 'wb') as output_file:
input_file.seek(0)
while True:
data = input_file.read(2 << 16)
if not data:
break
size += len(data)
output_file.write(data)
release = Release.by_version(self.session, pkg.name,
params['version'])
if not release:
release = Release(package=pkg,
version=params['version'],
summary=params.get('summary'),
author=self.user,
home_page=params.get('home_page'),
license=params.get('license'),
description=params.get('description'),
keywords=params.get('keywords'),
platform=params.get('platform'),
download_url=params.get('download_url'),
docs_url=params.get('docs_url'),
)
classifiers = params.getall('classifiers')
for name in classifiers:
classifier = Classifier.by_name(self.session, name,
create_if_not_exists=True)
while classifier:
if classifier not in release.classifiers:
release.classifiers.append(classifier)
if classifier not in pkg.classifiers:
pkg.classifiers.append(classifier)
classifier = classifier.parent
rfile = ReleaseFile.by_filename(self.session, release, filename)
if not rfile:
rfile = ReleaseFile(release=release,
filename=filename,
size=size,
md5_digest=params.get('md5_digest'),
package_type=params['filetype'],
python_version=params.get('pyversion'),
comment_text=params.get('comment'),
)
self.session.add(rfile)
self.session.add(release)
pkg.update_at = func.now()
self.session.add(pkg)
return {'release_file': rfile}
def render(self):
api = pypi.proxy
settings = self.request.registry.settings
sanitize = asbool(settings['pyshop.mirror.sanitize'])
package_name = self.request.matchdict['package_name']
pkg = Package.by_name(self.session, package_name)
if pkg is None:
pkg = Package.by_name(self.session, package_name.replace('-', '_'))
if pkg is None:
pkg = Package.by_name(self.session, package_name.replace('-', '.'))
refresh = True
session_users = {}
if pkg:
if pkg.local:
refresh = False
else:
if pkg.update_at:
log.debug('validating cache interval')
current_td = datetime.now() - pkg.update_at
max_td = timedelta(
hours=int(settings.get('pyshop.mirror.cache.ttl',
'24')))
refresh = current_td > max_td
log.debug('"%s" > "%s": %s',
current_td, max_td, refresh)
if refresh:
log.info('refresh package %s', package_name)
pypi_versions = api.package_releases(package_name, True)
# XXX package_releases is case sensitive
# but dependencies declaration not...
if not pypi_versions:
pkg_info = self._search_package(package_name)
if not pkg_info and '-' in package_name:
tmp_name = package_name.replace('-', '_')
pkg_info = self._search_package(tmp_name)
if not pkg_info and '-' in package_name:
tmp_name = package_name.replace('-', '.')
pkg_info = self._search_package(tmp_name)
if not pkg_info and '_' in package_name:
tmp_name = package_name.replace('_', '-')
pkg_info = self._search_package(tmp_name)
if pkg_info:
package_name, pypi_versions = pkg_info
pypi_versions = [ver.decode('utf-8') for ver in pypi_versions]
else:
pypi_versions = []
if not pkg:
if not pypi_versions:
log.info('package %s has no versions', package_name)
return {'package': None,
'package_name': package_name}
if sanitize:
re_sanitize = re.compile(settings['pyshop.mirror.'
'sanitize.regex'])
pypi_versions = [v for v in pypi_versions
if re_sanitize.match(v)]
# mirror the package now
log.info('mirror package %s now', package_name)
pkg = Package.by_name(self.session, package_name)
if not pkg:
pkg = Package(name=package_name, local=False)
self.session.add(pkg)
self.session.flush()
roles = api.package_roles(package_name)
for role, login in roles:
login = login.decode('utf-8') # XMLRPC should return utf-8
log.info('Looking for non local user %s', login)
if _sanitize(login) in session_users:
user = session_users[_sanitize(login)]
else:
user = User.by_login(self.session, login, local=False)
if not user:
log.info('Not found. creating user {0}'.format(login))
user = User(login=login, local=False)
self.session.add(user)
if role == 'Owner':
pkg.owners.append(user)
self.session.add(pkg)
elif role == 'Maintainer':
pkg.maintainers.append(user)
self.session.add(pkg)
session_users[_sanitize(login)] = user
self.session.flush()
self.session.flush()
if not pkg.local and refresh:
log.debug('refreshing %s package', package_name)
pkg_versions = set(pypi_versions).difference(pkg.versions)
if not pkg_versions:
log.info('No new version to mirror')
log.debug('pypi versions: %s', pypi_versions)
log.debug('mirrored versions: %s', pkg.versions)
for version in pkg_versions:
log.info('Mirroring version %s', version)
release_data = api.release_data(package_name, version)
release = self._create_release(pkg, release_data,
session_users)
release_files = api.release_urls(package_name, version)
for data in release_files:
filename = data['filename'].decode('utf-8')
rf = ReleaseFile.by_filename(self.session, release,
filename)
if not rf:
rf = self._create_release_file(release, data)
pkg.update_at = func.now()
self.session.add(pkg)
log.info('package %s mirrored' % package_name)
return {'package': pkg,
'whlify': asbool(settings.get('pyshop.mirror.wheelify', '0'))}
def test_by_name(self):
from pyshop.models import Package
pkg = Package.by_name(self.session, u'mirrored_package1')
self.assertIsInstance(pkg, Package)
self.assertEqual(pkg.id, 1)
self.assertEqual(pkg.name, u'mirrored_package1')
def test_sorted_releases(self):
from pyshop.models import Package
pkg = Package.by_name(self.session, u'mirrored_package1')
self.assertEqual([release.version for release in pkg.sorted_releases],
['0.2', '0.1'])
def render(self):
api = pypi.proxy
settings = self.request.registry.settings
sanitize = asbool(settings['pyshop.mirror.sanitize'])
package_name = self.request.matchdict['package_name']
pkg = Package.by_name(self.session, package_name)
refresh = True
if pkg:
if pkg.local:
refresh = False
else:
if pkg.update_at:
current_td = datetime.now() - pkg.update_at
max_td = timedelta(hours=int(
settings.get('pyshop.mirror.cache.ttl', '24')))
refresh = current_td > max_td
log.debug('"{cdt}" > "{max}": {refr}'.format(
cdt=current_td, max=max_td, refr=refresh))
if refresh:
log.info('refresh package {pkg}'.format(pkg=package_name))
pypi_versions = api.package_releases(package_name, True)
# XXX package_releases is case sensitive
# but dependancies declaration not...
if not pypi_versions:
pkg_info = self._search_package(package_name)
if not pkg_info and '-' in package_name:
tmp_name = package_name.replace('-', '_')
pkg_info = self._search_package(tmp_name)
if not pkg_info and '_' in package_name:
tmp_name = package_name.replace('_', '-')
pkg_info = self._search_package(tmp_name)
if pkg_info:
package_name, pypi_versions = pkg_info
else:
pypi_versions = []
if not pkg:
if not pypi_versions:
log.info('package %s has no versions' % package_name)
return {'package': None, 'package_name': package_name}
if sanitize:
re_sanitize = re.compile(settings['pyshop.mirror.'
'sanitize.regex'])
pypi_versions = [
v for v in pypi_versions if re_sanitize.match(v)
]
# mirror the package now
log.info('mirror package %s now' % package_name)
pkg = Package.by_name(self.session, package_name)
if not pkg:
pkg = Package(name=package_name, local=False)
self.session.add(pkg)
self.session.flush()
roles = api.package_roles(package_name)
for role, login in roles:
login = login.decode('utf-8') # XMLRPC should return utf-8
user = User.by_login(self.session, login, local=False)
if not user:
user = User(login=login, local=False)
self.session.add(user)
if role == 'Owner':
pkg.owners.append(user)
elif role == 'Maintainer':
pkg.maintainers.append(user)
self.session.flush()
refresh = True
if not pkg.local and refresh:
pkg_versions = pkg.versions
for version in pypi_versions:
if version not in pkg_versions:
release_data = api.release_data(package_name, version)
release = self._create_release(pkg, release_data)
release_files = api.release_urls(package_name, version)
for data in release_files:
filename = data['filename'].decode('utf-8')
rf = ReleaseFile.by_filename(self.session, release,
filename)
if not rf:
rf = self._create_release_file(release, data)
pkg.update_at = func.now()
self.session.add(pkg)
log.info('package %s mirrored' % package_name)
return {
'package': pkg,
'whlify': asbool(settings.get('pyshop.mirror.wheelify', '0'))
}
def render(self):
settings = self.request.registry.settings
username = authenticated_userid(self.request)
if not username:
raise exc.HTTPForbidden()
remote_user = User.by_login(self.session, username)
if not remote_user:
raise exc.HTTPForbidden()
params = self.request.params
if (asbool(settings['pyshop.upload.satanize'])
and not re.match(settings['pyshop.upload.satanize.regex'],
params['version']
)):
raise exc.HTTPForbidden()
pkg = Package.by_name(self.session, params['name'])
if pkg:
auth = [user for user in pkg.owners + pkg.maintainers
if user == remote_user]
if not auth:
raise exc.HTTPForbidden()
else:
pkg = Package(name=params['name'], local=True)
pkg.owners.append(remote_user)
content = self.request.POST['content']
input_file = content.file
# rewrite the filename, do not use the posted one for security
filename = u'%s-%s.%s' % (params['name'], params['version'],
{u'sdist': u'tar.gz',
u'bdist_egg': u'egg',
u'bdist_msi': u'msi',
u'bdist_dmg': u'zip', # XXX or gztar ?
u'bdist_rpm': u'rpm',
u'bdist_dumb': u'msi',
u'bdist_wininst': u'exe',
}[params['filetype']])
dir_ = os.path.join(settings['pyshop.repository'],
filename[0].lower())
if not os.path.exists(dir_):
os.mkdir(dir_, 0750)
filepath = os.path.join(dir_, filename)
while os.path.exists(filepath):
log.warn('File %s exists but new upload self.request, deleting'
% filepath)
os.unlink(filepath)
size = 0
with open(filepath, 'wb') as output_file:
input_file.seek(0)
while True:
data = input_file.read(2<<16)
if not data:
break
size += len(data)
output_file.write(data)
release = Release.by_version(self.session, pkg.name,
params['version'])
if not release:
release = Release(package=pkg,
version=params['version'],
summary=params.get('summary'),
author=remote_user,
home_page=params.get('home_page'),
license=params.get('license'),
description=params.get('description'),
keywords=params.get('keywords'),
platform=params.get('platform'),
download_url=params.get('download_url'),
docs_url=params.get('docs_url'),
)
classifiers = params.getall('classifiers')
for name in classifiers:
classifier = Classifier.by_name(self.session, name)
while classifier:
if classifier not in release.classifiers:
release.classifiers.append(classifier)
if classifier not in pkg.classifiers:
pkg.classifiers.append(classifier)
classifier = classifier.parent
rfile = ReleaseFile.by_filename(self.session, release, filename)
if not rfile:
rfile = ReleaseFile(release=release,
filename=filename,
size=size,
md5_digest=params.get('md5_digest'),
package_type=params['filetype'],
python_version=params.get('pyversion'),
comment_text=params.get('comment'),
)
self.session.add(rfile)
self.session.add(release)
pkg.update_at = func.now()
self.session.add(pkg)
return {'release_file': rfile}
def render(self):
api = pypi.proxy
settings = self.request.registry.settings
satanize = asbool(settings['pyshop.mirror.satanize'])
package_name = self.request.matchdict['package_name']
pkg = Package.by_name(self.session, package_name)
refresh = True
if pkg:
if pkg.local:
refresh = False
else:
if pkg.update_at:
td = datetime.now() - pkg.update_at
refresh = td.days > 0 or td.seconds > 10800
if refresh:
pypi_versions = api.package_releases(package_name, True)
# XXX package_releases is case sensitive
# but dependancies declaration not...
if not pypi_versions:
package_name = package_name.lower()
search_result = api.search({'name': package_name}, True)
search_result = [p for p in search_result
if p['name'].lower() == package_name]
if search_result:
package_name = search_result[0]['name']
pypi_versions = api.package_releases(package_name, True)
else:
pypi_versions = []
if not pkg:
if not pypi_versions:
log.info('package %s has no versions' % package_name)
return {'package': None,
'package_name': package_name}
if satanize:
re_satanize = re.compile(settings['pyshop.mirror.'
'satanize.regex'])
pypi_versions = [v for v in pypi_versions
if re_satanize.match(v)]
# mirror the package now
log.info('mirror package %s now' % package_name)
pkg = Package(name=package_name, local=False)
roles = api.package_roles(package_name)
for role, login in roles:
user = User.by_login(self.session, login, local=False)
if not user:
user = User(login=login, local=False)
self.session.add(user)
if role == 'Owner':
pkg.owners.append(user)
elif role == 'Maintainer':
pkg.maintainers.append(user)
self.session.flush()
refresh = True
if not pkg.local and refresh:
pkg_versions = pkg.versions
for version in pypi_versions:
if version not in pkg_versions:
release_data = api.release_data(package_name, version)
release = self._create_release(pkg, release_data)
release_files = api.release_urls(package_name, version)
for data in release_files:
rf = ReleaseFile.by_filename(self.session, release,
data['filename'])
if not rf:
rf = self._create_release_file(release, data)
pkg.update_at = func.now()
self.session.add(pkg)
log.info('package %s mirrored' % package_name)
return {'package': pkg}
def render(self):
package = Package.by_name(self.session,
self.request.matchdict['package_name'])
def render(self):
settings = self.request.registry.settings
if not self.user:
raise exc.HTTPForbidden()
params = self.request.params
if (asbool(settings['pyshop.upload.sanitize'])
and not re.match(settings['pyshop.upload.sanitize.regex'],
params['version']
)):
raise exc.HTTPForbidden()
pkg = Package.by_name(self.session, params['name'])
if pkg and pkg.local:
auth = [user for user in pkg.owners + pkg.maintainers
if user == self.user]
if not auth:
raise exc.HTTPForbidden()
elif not pkg:
pkg = Package(name=params['name'], local=True)
pkg.owners.append(self.user)
content = self.request.POST['content']
input_file = content.file
if asbool(settings.get('pyshop.upload.rewrite_filename', '1')):
# rewrite the filename, do not use the posted one for security
filename = self._guess_filename(params, content.filename)
else:
filename = content.filename
dir_ = os.path.join(settings['pyshop.repository'],
filename[0].lower())
if not os.path.exists(dir_):
os.makedirs(dir_, 0o750)
filepath = os.path.join(dir_, filename)
while os.path.exists(filepath):
log.warning('File %s exists but new upload self.request, deleting'
% filepath)
os.unlink(filepath)
size = 0
with open(filepath, 'wb') as output_file:
input_file.seek(0)
while True:
data = input_file.read(2 << 16)
if not data:
break
size += len(data)
output_file.write(data)
release = Release.by_version(self.session, pkg.name,
params['version'])
if not release:
release = Release(package=pkg,
version=params['version'],
summary=params.get('summary'),
author=self.user,
home_page=params.get('home_page'),
license=params.get('license'),
description=params.get('description'),
keywords=params.get('keywords'),
platform=params.get('platform'),
download_url=params.get('download_url'),
docs_url=params.get('docs_url'),
)
classifiers = params.getall('classifiers')
for name in classifiers:
classifier = Classifier.by_name(self.session, name)
while classifier:
if classifier not in release.classifiers:
release.classifiers.append(classifier)
if classifier not in pkg.classifiers:
pkg.classifiers.append(classifier)
classifier = classifier.parent
rfile = ReleaseFile.by_filename(self.session, release, filename)
if not rfile:
rfile = ReleaseFile(release=release,
filename=filename,
size=size,
md5_digest=params.get('md5_digest'),
package_type=params['filetype'],
python_version=params.get('pyversion'),
comment_text=params.get('comment'),
)
self.session.add(rfile)
self.session.add(release)
pkg.update_at = func.now()
self.session.add(pkg)
return {'release_file': rfile}
def render(self):
api = pypi.proxy
settings = self.request.registry.settings
sanitize = asbool(settings['pyshop.mirror.sanitize'])
package_name = self.request.matchdict['package_name']
pkg = Package.by_name(self.session, package_name)
refresh = True
if pkg:
if pkg.local:
refresh = False
else:
if pkg.update_at:
current_td = datetime.now() - pkg.update_at
max_td = timedelta(hours=int(settings.get('pyshop.mirror.cache.ttl',
'24')))
refresh = current_td > max_td
log.debug('"{cdt}" > "{max}": {refr}'.format(cdt=current_td,
max=max_td,
refr=refresh))
if refresh:
log.info('refresh package {pkg}'.format(pkg=package_name))
pypi_versions = api.package_releases(package_name, True)
# XXX package_releases is case sensitive
# but dependancies declaration not...
if not pypi_versions:
pkg_info = self._search_package(package_name)
if not pkg_info and '-' in package_name:
tmp_name = package_name.replace('-', '_')
pkg_info = self._search_package(tmp_name)
if not pkg_info and '_' in package_name:
tmp_name = package_name.replace('_', '-')
pkg_info = self._search_package(tmp_name)
if pkg_info:
package_name, pypi_versions = pkg_info
else:
pypi_versions = []
if not pkg:
if not pypi_versions:
log.info('package %s has no versions' % package_name)
return {'package': None,
'package_name': package_name}
if sanitize:
re_sanitize = re.compile(settings['pyshop.mirror.'
'sanitize.regex'])
pypi_versions = [v for v in pypi_versions
if re_sanitize.match(v)]
# mirror the package now
log.info('mirror package %s now' % package_name)
pkg = Package.by_name(self.session, package_name)
if not pkg:
pkg = Package(name=package_name, local=False)
self.session.add(pkg)
self.session.flush()
roles = api.package_roles(package_name)
for role, login in roles:
login = login.decode('utf-8') # XMLRPC should return utf-8
user = User.by_login(self.session, login, local=False)
if not user:
user = User(login=login, local=False)
self.session.add(user)
if role == 'Owner':
pkg.owners.append(user)
elif role == 'Maintainer':
pkg.maintainers.append(user)
self.session.flush()
refresh = True
if not pkg.local and refresh:
pkg_versions = pkg.versions
for version in pypi_versions:
if version not in pkg_versions:
release_data = api.release_data(package_name, version)
release = self._create_release(pkg, release_data)
release_files = api.release_urls(package_name, version)
for data in release_files:
filename = data['filename'].decode('utf-8')
rf = ReleaseFile.by_filename(self.session, release,
filename)
if not rf:
rf = self._create_release_file(release, data)
pkg.update_at = func.now()
self.session.add(pkg)
log.info('package %s mirrored' % package_name)
return {'package': pkg,
'whlify': asbool(settings.get('pyshop.mirror.wheelify', '0'))}
def render(self):
api = pypi.proxy
settings = self.request.registry.settings
sanitize = asbool(settings['pyshop.mirror.sanitize'])
package_name = self.request.matchdict['package_name']
pkg = Package.by_name(self.session, package_name)
if pkg is None:
pkg = Package.by_name(self.session, package_name.replace('-', '_'))
refresh = True
session_users = {}
if pkg:
if pkg.local:
refresh = False
else:
if pkg.update_at:
log.debug('validating cache interval')
current_td = datetime.now() - pkg.update_at
max_td = timedelta(
hours=int(settings.get('pyshop.mirror.cache.ttl',
'24')))
refresh = current_td > max_td
log.debug('"%s" > "%s": %s',
current_td, max_td, refresh)
if refresh:
log.info('refresh package %s', package_name)
pypi_versions = api.package_releases(package_name, True)
# XXX package_releases is case sensitive
# but dependencies declaration not...
if not pypi_versions:
pkg_info = self._search_package(package_name)
if not pkg_info and '-' in package_name:
tmp_name = package_name.replace('-', '_')
pkg_info = self._search_package(tmp_name)
if not pkg_info and '_' in package_name:
tmp_name = package_name.replace('_', '-')
pkg_info = self._search_package(tmp_name)
if pkg_info:
package_name, pypi_versions = pkg_info
pypi_versions = [ver.decode('utf-8') for ver in pypi_versions]
else:
pypi_versions = []
if not pkg:
if not pypi_versions:
log.info('package %s has no versions', package_name)
return {'package': None,
'package_name': package_name}
if sanitize:
re_sanitize = re.compile(settings['pyshop.mirror.'
'sanitize.regex'])
pypi_versions = [v for v in pypi_versions
if re_sanitize.match(v)]
# mirror the package now
log.info('mirror package %s now', package_name)
pkg = Package.by_name(self.session, package_name)
if not pkg:
pkg = Package(name=package_name, local=False)
self.session.add(pkg)
self.session.flush()
roles = api.package_roles(package_name)
for role, login in roles:
login = login.decode('utf-8') # XMLRPC should return utf-8
log.info('Looking for non local user %s', login)
if _sanitize(login) in session_users:
user = session_users[_sanitize(login)]
else:
user = User.by_login(self.session, login, local=False)
if not user:
log.info('Not found. creating user {0}'.format(login))
user = User(login=login, local=False)
self.session.add(user)
if role == 'Owner':
pkg.owners.append(user)
self.session.add(pkg)
elif role == 'Maintainer':
pkg.maintainers.append(user)
self.session.add(pkg)
session_users[_sanitize(login)] = user
self.session.flush()
self.session.flush()
if not pkg.local and refresh:
log.debug('refreshing %s package', package_name)
pkg_versions = set(pypi_versions).difference(pkg.versions)
if not pkg_versions:
log.info('No new version to mirror')
log.debug('pypi versions: %s', pypi_versions)
log.debug('mirrored versions: %s', pkg.versions)
for version in pkg_versions:
log.info('Mirroring version %s', version)
release_data = api.release_data(package_name, version)
release = self._create_release(pkg, release_data,
session_users)
release_files = api.release_urls(package_name, version)
for data in release_files:
filename = data['filename'].decode('utf-8')
rf = ReleaseFile.by_filename(self.session, release,
filename)
if not rf:
rf = self._create_release_file(release, data)
pkg.update_at = func.now()
self.session.add(pkg)
log.info('package %s mirrored' % package_name)
return {'package': pkg,
'whlify': asbool(settings.get('pyshop.mirror.wheelify', '0'))}
def test_sorted_releases(self):
from pyshop.models import Package
pkg = Package.by_name(self.session, u'mirrored_package1')
self.assertEqual([release.version for release in pkg.sorted_releases],
['0.2', '0.1'])
def test_by_name(self):
from pyshop.models import Package
pkg = Package.by_name(self.session, u'mirrored_package1')
self.assertIsInstance(pkg, Package)
self.assertEqual(pkg.id, 1)
self.assertEqual(pkg.name, u'mirrored_package1')
def render(self):
package = Package.by_name(self.session,
self.request.matchdict['package_name'])
def render(self):
api = pypi.proxy
settings = self.request.registry.settings
sanitize = asbool(settings['pyshop.mirror.sanitize'])
package_name = self.request.matchdict['package_name']
pkg = Package.by_name(self.session, package_name)
refresh = True
if pkg:
if pkg.local:
refresh = False
else:
if pkg.update_at:
td = datetime.now() - pkg.update_at
# refresh if the package has not been update today
# XXX should be configurable
refresh = td.days > 0
if refresh:
pypi_versions = api.package_releases(package_name, True)
# XXX package_releases is case sensitive
# but dependancies declaration not...
if not pypi_versions:
search_result = api.search({'name': package_name}, True)
search_count = len(search_result)
search_result = [p for p in search_result
if p['name'].lower() == package_name
or p['name'].lower().replace('-', '_') == package_name]
log.debug('Found {sc}, matched {mc}'.format(sc=search_count, mc=len(search_result)))
if search_result:
package_name = search_result[0]['name']
pypi_versions = api.package_releases(package_name, True)
else:
pypi_versions = []
if not pkg:
if not pypi_versions:
log.info('package %s has no versions' % package_name)
return {'package': None,
'package_name': package_name}
if sanitize:
re_sanitize = re.compile(settings['pyshop.mirror.'
'sanitize.regex'])
pypi_versions = [v for v in pypi_versions
if re_sanitize.match(v)]
# mirror the package now
log.info('mirror package %s now' % package_name)
pkg = Package(name=package_name, local=False)
roles = api.package_roles(package_name)
for role, login in roles:
login = login.decode('utf-8') # XMLRPC should return utf-8
user = User.by_login(self.session, login, local=False)
if not user:
user = User(login=login, local=False)
self.session.add(user)
if role == 'Owner':
pkg.owners.append(user)
elif role == 'Maintainer':
pkg.maintainers.append(user)
self.session.flush()
refresh = True
if not pkg.local and refresh:
pkg_versions = pkg.versions
for version in pypi_versions:
if version not in pkg_versions:
release_data = api.release_data(package_name, version)
release = self._create_release(pkg, release_data)
release_files = api.release_urls(package_name, version)
for data in release_files:
filename = data['filename'].decode('utf-8')
rf = ReleaseFile.by_filename(self.session, release,
filename)
if not rf:
rf = self._create_release_file(release, data)
pkg.update_at = func.now()
self.session.add(pkg)
log.info('package %s mirrored' % package_name)
return {'package': pkg}
def render(self):
package = Package.by_name(self.session,
self.request.matchdict['package_name'])
if not package:
raise HTTPNotFound()
if 'form.refresh_package' in self.request.params:
package.update_at = None
self.session.add(package)
owners = dict((usr.login, usr) for usr in package.owners)
can_edit_role = self.login in owners.keys() and package.local
if 'form.add_role' in self.request.params:
if not can_edit_role:
raise HTTPForbidden()
user = User.by_login(self.session, self.request.params['login'])
if user and user.has_permission('upload_releasefile'):
if self.request.params['role'] == 'owner':
if user.login not in owners:
package.owners.append(user)
else:
maintainers = [usr.login for usr in package.owners]
if user.login not in maintainers:
package.maintainers.append(user)
self.session.add(package)
if 'form.remove_maintainer' in self.request.params:
if not can_edit_role:
raise HTTPForbidden()
user = User.by_login(self.session, self.request.params['login'])
if user:
maintainers = dict((usr.login, usr)
for usr in package.maintainers)
if user.login in maintainers:
package.maintainers.remove(maintainers[user.login])
self.session.add(package)
if 'form.remove_owner' in self.request.params:
if not can_edit_role:
raise HTTPForbidden()
user = User.by_login(self.session, self.request.params['login'])
if user:
if user.login in owners:
package.owners.remove(owners[user.login])
self.session.add(package)
if 'release_version' in self.request.matchdict:
release = Release.by_version(self.session, package.name,
self.request.matchdict['release_version'])
else:
release = package.sorted_releases[0]
return {u'package': package,
u'release': release,
u'can_edit_role': can_edit_role,
}
