def test_download_file(self):
package = models.Package(name='localshop')
package.save()
release = models.Release(
package=package, version='0.1')
release.save()
release_file = models.ReleaseFile(
release=release,
python_version='source',
url='http://pypi.python.org/packages/source/l/localshop/' \
'localshop-0.1.tar.gz'
)
release_file.save()
with mock.patch('requests.get') as mock_obj:
mock_obj.return_value = mock.Mock()
mock_obj.return_value.raw = HTTPResponse()
mock_obj.return_value.raw._fp = StringIO('test')
tasks.download_file(release_file.pk)
release_file = models.ReleaseFile.objects.get(pk=release_file.pk)
self.assertEqual(release_file.distribution.read(), 'test')
self.assertEqual(release_file.distribution.name,
'source/l/localshop/localshop-0.1.tar.gz')
def get_package_data(name, package=None):
"""Retrieve metadata information for the given package name"""
if not package:
package = models.Package(name=name)
releases = {}
else:
releases = package.get_all_releases()
client = xmlrpclib.ServerProxy('http://pypi.python.org/pypi')
versions = client.package_releases(package.name, True)
# package_releases() method is case-sensitive, if nothing found
# then we search for it
# XXX: Ask pypi to make it case-insensitive?
if not versions:
for item in client.search({'name': name}):
if name.lower() == item['name'].lower():
package.name = name = item['name']
break
else:
logger.info("No packages found matching %r", name)
return
# Retry retrieving the versions with the new/correct name
versions = client.package_releases(package.name, True)
# Save the package if it is new
if not package.pk:
package.save()
for version in versions:
release, files = releases.get(version, (None, {}))
if not release:
release = models.Release(package=package, version=version)
release.save()
data = client.release_data(package.name, release.version)
release_form = forms.PypiReleaseDataForm(data, instance=release)
if release_form.is_valid():
release_form.save()
release_files = client.package_urls(package.name, release.version)
for info in release_files:
release_file = files.get(info['filename'])
if not release_file:
release_file = models.ReleaseFile(release=release,
filename=info['filename'])
release_file.python_version = info['python_version']
release_file.filetype = info['packagetype']
release_file.url = info['url']
release_file.size = info['size']
release_file.md5_digest = info['md5_digest']
release_file.save()
package.update_timestamp = now()
package.save()
return package
def fetch_package(repository_pk, slug):
repository = models.Repository.objects.get(pk=repository_pk)
logging.info('start fetch_package: %s', slug)
package_data = pypi.get_package_information(
repository.upstream_pypi_url_api, slug)
if not package_data:
return
name = package_data['info']['name']
try:
package = repository.packages.get(name=name)
releases = package.get_all_releases()
except models.Package.DoesNotExist:
package = repository.packages.create(name=name)
releases = {}
for version, release_list in package_data['releases'].items():
release, files = releases.get(version, (None, {}))
if not release:
release = package.releases.create(version=version)
release_data = {
'author': package_data['info']['author'],
'author_email': package_data['info']['author_email'],
'description': package_data['info']['description'],
'download_url': package_data['info']['download_url'],
'home_page': package_data['info']['home_page'],
'license': package_data['info']['license'],
'summary': package_data['info']['summary'],
'version': version,
}
release_form = forms.PypiReleaseDataForm(release_data,
instance=release)
if release_form.is_valid():
release_form.save()
for data in release_list:
release_file = files.get(data['filename'])
if not release_file:
release_file = models.ReleaseFile(release=release,
filename=data['filename'])
release_file.python_version = data['python_version']
release_file.filetype = data['packagetype']
release_file.url = data['url']
release_file.size = data['size']
release_file.md5_digest = data['md5_digest']
release_file.save()
package.update_timestamp = now()
package.save()
logging.info('done fetch_package: %s', slug)
def handle_register_or_upload(post_data, files, user, repository):
"""Process a `register` or `upload` comment issued via distutils.
This method is called with the authenticated user.
"""
name = post_data.get('name')
version = post_data.get('version')
if settings.LOCALSHOP_VERSIONING_TYPE:
scheme = get_versio_versioning_scheme(
settings.LOCALSHOP_VERSIONING_TYPE)
try:
Version(version, scheme=scheme)
except AttributeError:
response = HttpResponseBadRequest(
reason="Invalid version supplied '{!s}' for '{!s}' scheme.".
format(version, settings.LOCALSHOP_VERSIONING_TYPE))
return response
if not name or not version:
logger.info("Missing name or version for package")
return HttpResponseBadRequest('No name or version given')
try:
condition = Q()
for search_name in get_search_names(name):
condition |= Q(name__iexact=search_name)
package = repository.packages.get(condition)
# Error out when we try to override a mirror'ed package for now
# not sure what the best thing is
if not package.is_local:
return HttpResponseBadRequest('%s is a pypi package!' %
package.name)
try:
release = package.releases.get(version=version)
except ObjectDoesNotExist:
release = None
except ObjectDoesNotExist:
package = None
release = None
# Validate the data
form = forms.ReleaseForm(post_data, instance=release)
if not form.is_valid():
return HttpResponseBadRequest(reason=form.errors.values()[0][0])
if not package:
pkg_form = forms.PackageForm(post_data, repository=repository)
if not pkg_form.is_valid():
return HttpResponseBadRequest(
reason=six.next(six.itervalues(pkg_form.errors))[0])
package = pkg_form.save()
release = form.save(commit=False)
release.package = package
release.save()
# If this is an upload action then process the uploaded file
if files:
filename = files['distribution']._name
try:
release_file = release.files.get(filename=filename)
if settings.LOCALSHOP_RELEASE_OVERWRITE is False:
message = 'That it already released, please bump version.'
return HttpResponseBadRequest(message)
except ObjectDoesNotExist:
release_file = models.ReleaseFile(release=release,
filename=filename)
form_file = forms.ReleaseFileForm(post_data,
files,
instance=release_file)
if not form_file.is_valid():
return HttpResponseBadRequest('ERRORS %s' % form_file.errors)
release_file = form_file.save(commit=False)
release_file.save()
return HttpResponse()
def get_package_data(name, package=None):
"""Retrieve metadata information for the given package name"""
if not package:
package = models.Package(name=name)
releases = {}
else:
releases = package.get_all_releases()
client = get_xmlrpc_client()
versions = client.package_releases(package.name, True)
# package_releases() method is case-sensitive, if nothing found
# then we search for it
# XXX: Ask pypi to make it case-insensitive?
names = get_search_names(name)
if not versions:
for item in client.search({'name': names}):
if item['name'].lower() in [n.lower() for n in names]:
package.name = name = item['name']
break
else:
logger.info("No packages found matching %r", name)
return
# Retry retrieving the versions with the new/correct name
versions = client.package_releases(package.name, True)
# If the matched package differs from the name we tried to retrieve then
# retry to fetch the package from the database.
if package.name != name:
try:
package = models.Package.objects.get(name=package.name)
except models.Package.objects.DoesNotExist:
pass
# Save the package if it is new
if not package.pk:
package.save()
for version in versions:
release, files = releases.get(version, (None, {}))
if not release:
release = models.Release(package=package, version=version)
release.save()
data = client.release_data(package.name, release.version)
release_form = forms.PypiReleaseDataForm(data, instance=release)
if release_form.is_valid():
release_form.save()
release_files = client.package_urls(package.name, release.version)
for info in release_files:
release_file = files.get(info['filename'])
if not release_file:
release_file = models.ReleaseFile(
release=release, filename=info['filename'])
release_file.python_version = info['python_version']
release_file.filetype = info['packagetype']
release_file.url = info['url']
release_file.size = info['size']
release_file.md5_digest = info['md5_digest']
release_file.save()
package.update_timestamp = now()
package.save()
return package
def handle_register_or_upload(post_data, files, user):
"""Process a `register` or `upload` comment issued via distutils.
This method is called with the authenticated user.
"""
name = post_data.get('name')
version = post_data.get('version')
if not name or not version:
logger.info("Missing name or version for package")
return HttpResponseBadRequest('No name or version given')
try:
package = models.Package.objects.get(name=name)
# Error out when we try to override a mirror'ed package for now
# not sure what the best thing is
if not package.is_local:
return HttpResponseBadRequest(
'%s is a pypi package!' % package.name)
# Ensure that the user is one of the owners
if not package.owners.filter(pk=user.pk).exists():
if not user.is_superuser:
return HttpResponseForbidden('No permission for this package')
# User is a superuser, add him to the owners
package.owners.add(user)
try:
release = package.releases.get(version=version)
except ObjectDoesNotExist:
release = None
except ObjectDoesNotExist:
package = None
release = None
# Validate the data
form = forms.ReleaseForm(post_data, instance=release)
if not form.is_valid():
return HttpResponseBadRequest('ERRORS %s' % form.errors)
if not package:
package = models.Package.objects.create(name=name, is_local=True)
package.owners.add(user)
package.save()
release = form.save(commit=False)
release.package = package
release.user = user
release.save()
# If this is an upload action then process the uploaded file
if files:
filename = files['distribution']._name
try:
release_file = release.files.get(filename=filename)
except ObjectDoesNotExist:
release_file = models.ReleaseFile(
release=release, filename=filename, user=user)
form_file = forms.ReleaseFileForm(
post_data, files, instance=release_file)
if not form_file.is_valid():
return HttpResponseBadRequest('ERRORS %s' % form_file.errors)
release_file = form_file.save(commit=False)
release_file.user = user
release_file.save()
return HttpResponse()
def fetch_package(self, slug):
"""
"""
logging.info('start fetch_package: %s', slug)
response = requests.get(settings.LOCALSHOP_PYPI_URL +
'/{}/json'.format(slug))
if response.status_code == 404:
return
if response.status_code != 200:
return
package_data = response.json()
name = package_data['info']['name']
try:
package = models.Package.objects.get(name=name)
releases = package.get_all_releases()
except models.Package.DoesNotExist:
package = models.Package(name=name)
releases = {}
# Save the package if it is new
if not package.pk:
package.save()
for version, release_list in package_data['releases'].items():
release, files = releases.get(version, (None, {}))
if not release:
release = models.Release(package=package, version=version)
release.save()
release_data = {
'author': package_data['info']['author'],
'author_email': package_data['info']['author_email'],
'description': package_data['info']['description'],
'download_url': package_data['info']['download_url'],
'home_page': package_data['info']['home_page'],
'license': package_data['info']['license'],
'summary': package_data['info']['summary'],
'version': version,
}
release_form = forms.PypiReleaseDataForm(release_data,
instance=release)
if release_form.is_valid():
release_form.save()
for data in release_list:
release_file = files.get(data['filename'])
if not release_file:
release_file = models.ReleaseFile(release=release,
filename=data['filename'])
release_file.python_version = data['python_version']
release_file.filetype = data['packagetype']
release_file.url = data['url']
release_file.size = data['size']
release_file.md5_digest = data['md5_digest']
release_file.save()
package.update_timestamp = now()
package.save()
logging.info('done fetch_package: %s', slug)
