def post(self):
"""POST
Parameters:
filename: filename of package e.g. 'Firefox-1.0.dmg'
"""
session = gaeserver.DoMunkiAuth(require_level=gaeserver.LEVEL_UPLOADPKG)
filename = self.request.get('filename')
pkginfo = models.PackageInfo.get_by_key_name(filename)
if not pkginfo:
self.response.set_status(404)
self.response.out.write('Pkginfo does not exist: %s' % filename)
return
plist = pkginfo.plist
catalogs = pkginfo.catalogs
install_types = pkginfo.install_types
#logging.info('Deleting package: %s', filename)
blobstore_key = pkginfo.blobstore_key
# Delete the PackageInfo entity, and then the package Blobstore entity.
pkginfo.delete()
gae_util.SafeBlobDel(blobstore_key)
# Recreate catalogs so references to this package don't exist anywhere.
for catalog in catalogs:
models.Catalog.Generate(catalog)
# Log admin delete to Datastore.
user = session.uuid
admin_log = models.AdminPackageLog(
user=user, action='deletepkg', filename=filename, catalogs=catalogs,
install_types=install_types, plist=plist)
admin_log.put()
class UploadPackage(admin.AdminHandler,
blobstore_handlers.BlobstoreUploadHandler):
"""Handler for /admin/uploadpkg."""
XSRF_PROTECTION = False
def get(self):
"""GET Handler.
With no parameters, return the URL to use to submit uploads via
multipart/form-data.
With mode and key parameters, return status of the previous uploadpkg
operation to the calling client.
This method actually acts as a helper to the starting and finishing
of the uploadpkg post() method.
Parameters:
mode: optionally, 'success' or 'error'
key: optionally, blobstore key that was uploaded
"""
if not auth.HasPermission(auth.UPLOAD):
self.error(httplib.FORBIDDEN)
return
mode = self.request.get('mode')
msg = self.request.get('msg', None)
if mode == 'success':
filename = self.request.get('filename')
msg = '%s successfully uploaded and is ready for deployment.' % filename
self.redirect('/admin/package/%s?msg=%s' % (filename, msg))
elif mode == 'error':
self.response.set_status(httplib.BAD_REQUEST)
self.response.out.write(msg)
else:
filename = self.request.get('filename')
if not filename:
self.response.set_status(httplib.NOT_FOUND)
self.response.out.write('Filename required')
return
p = models.PackageInfo.get_by_key_name(filename)
if not p:
self.response.set_status(httplib.BAD_REQUEST)
self.response.out.write(
'You must first upload a pkginfo for %s' % filename)
return
elif p.blob_info:
self.response.set_status(httplib.BAD_REQUEST)
self.response.out.write('This file already exists.')
return
upload_url = blobstore.create_upload_url(
'/admin/uploadpkg', gs_bucket_name=util.GetBlobstoreGSBucket())
values = {
'upload_url': upload_url,
'filename': filename,
'file_size_kbytes': p.plist['installer_item_size'],
}
self.Render('upload_pkg_form.html', values)
def _RedirectWithErrorMsg(self, msg):
logging.warning(msg)
self.redirect('/admin/uploadpkg?mode=error&msg=%s' % msg)
def post(self):
"""POST Handler.
This method behaves a little strangely. BlobstoreUploadHandler
only allows returns statuses of 301, 302, 303 (not even 200), so
one must redirect away to return more information to the caller.
Parameters:
file: package file contents
pkginfo: packageinfo file contents
name: filename of package e.g. 'Firefox-1.0.dmg'
"""
# Only blobstore/upload service/scotty requests should be
# invoking this handler.
if not handlers.IsBlobstore():
logging.error('POST to uploadpkg not from Blobstore: %s',
self.request.headers)
self.redirect('/admin/packages')
return
if not self.get_uploads('file'):
logging.error('Upload package does not exist.')
return
blob_info = self.get_uploads('file')[0]
blobstore_key = str(blob_info.key())
# Obtain a lock on the PackageInfo entity for this package.
lock = models.GetLockForPackage(blob_info.filename)
try:
lock.Acquire(timeout=30, max_acquire_attempts=5)
except datastore_locks.AcquireLockError:
gae_util.SafeBlobDel(blobstore_key)
self._RedirectWithErrorMsg('PackageInfo is locked')
return
p = models.PackageInfo.get_by_key_name(blob_info.filename)
if not p:
lock.Release()
gae_util.SafeBlobDel(blobstore_key)
self._RedirectWithErrorMsg('PackageInfo not found')
return
if not p.IsSafeToModify():
lock.Release()
gae_util.SafeBlobDel(blobstore_key)
self._RedirectWithErrorMsg('PackageInfo is not modifiable')
return
installer_item_size = p.plist['installer_item_size']
size_difference = int(blob_info.size / 1024) - installer_item_size
if abs(size_difference) > 1:
lock.Release()
gae_util.SafeBlobDel(blobstore_key)
msg = 'Blob size (%s) does not match PackageInfo plist size (%s)' % (
blob_info.size, installer_item_size)
self._RedirectWithErrorMsg(msg)
return
old_blobstore_key = None
if p.blobstore_key:
# a previous blob exists. delete it when the update has succeeded.
old_blobstore_key = p.blobstore_key
p.blob_info = blob_info
# update the PackageInfo model with the new plist string and blobstore key.
try:
p.put()
error = None
except db.Error, e:
logging.exception('error on PackageInfo.put()')
error = 'pkginfo.put() failed with: %s' % str(e)
# if it failed, delete the blob that was just uploaded -- it's
# an orphan.
if error is not None:
gae_util.SafeBlobDel(blobstore_key)
lock.Release()
self._RedirectWithErrorMsg(error)
return
# if an old blob was associated with this Package, delete it.
# the new blob that was just uploaded has replaced it.
if old_blobstore_key:
gae_util.SafeBlobDel(old_blobstore_key)
lock.Release()
user = users.get_current_user().email()
# Log admin upload to Datastore.
admin_log = models.AdminPackageLog(user=user,
action='uploadpkg',
filename=blob_info.filename)
admin_log.put()
self.redirect('/admin/uploadpkg?mode=success&filename=%s' %
blob_info.filename)
def post(self):
"""POST method.
This method behaves a little strangely. BlobstoreUploadHandler
only allows returns statuses of 301, 302, 303 (not even 200), so
one must redirect away to return more information to the caller.
Parameters:
file: package file contents
pkginfo: packageinfo file contents
name: filename of package e.g. 'Firefox-1.0.dmg'
"""
# Only blobstore/upload service/scotty requests should be
# invoking this handler.
if not handlers.IsBlobstore():
logging.critical('POST to /uploadpkg not from Blobstore: %s',
self.request.headers)
self.redirect('/')
gaeserver.DoMunkiAuth(require_level=gaeserver.LEVEL_UPLOADPKG)
user = self.request.get('user')
filename = self.request.get('name')
install_types = self.request.get('install_types')
catalogs = self.request.get('catalogs', None)
manifests = self.request.get('manifests', None)
if catalogs is None or not install_types or not user or not filename:
msg = 'uploadpkg POST required parameters missing'
logging.error(msg)
self.redirect('/uploadpkg?mode=error&msg=%s' % msg)
return
if catalogs == '':
catalogs = []
else:
catalogs = catalogs.split(',')
if manifests in ['', None]:
manifests = []
else:
manifests = manifests.split(',')
install_types = install_types.split(',')
upload_files = self.get_uploads('file')
upload_pkginfo_files = self.get_uploads('pkginfo')
if not len(upload_pkginfo_files) and not self.request.get('pkginfo'):
self.redirect('/uploadpkg?mode=error&msg=No%20file%20received')
return
if len(upload_pkginfo_files):
# obtain the pkginfo from a blob, and then throw it away. this is
# a necessary hack because the upload handler grabbed it, but we don't
# intend to keep it in blobstore.
pkginfo_str = gae_util.GetBlobAndDel(upload_pkginfo_files[0].key())
else:
# otherwise, grab the form parameter.
pkginfo_str = self.request.get('pkginfo')
blob_info = upload_files[0]
blobstore_key = str(blob_info.key())
# Parse, validate, and encode the pkginfo plist.
plist = plist_lib.MunkiPackageInfoPlist(pkginfo_str)
try:
plist.Parse()
except plist_lib.PlistError:
logging.exception('Invalid pkginfo plist uploaded:\n%s\n',
pkginfo_str)
gae_util.SafeBlobDel(blobstore_key)
self.redirect(
'/uploadpkg?mode=error&msg=No%20valid%20pkginfo%20received')
return
filename = plist['installer_item_location']
pkgdata_sha256 = plist['installer_item_hash']
# verify the blob was actually written; in case Blobstore failed to write
# the blob but still POSTed to this handler (very, very rare).
blob_info = blobstore.BlobInfo.get(blobstore_key)
if not blob_info:
logging.critical(
'Blobstore returned a key for %s that does not exist: %s',
filename, blobstore_key)
self.redirect('/uploadpkg?mode=error&msg=Blobstore%20failure')
return
# Obtain a lock on the PackageInfo entity for this package.
lock = 'pkgsinfo_%s' % filename
if not gae_util.ObtainLock(lock, timeout=5.0):
gae_util.SafeBlobDel(blobstore_key)
self.redirect(
'/uploadpkg?mode=error&msg=Could%20not%20lock%20pkgsinfo')
return
old_blobstore_key = None
pkg = models.PackageInfo.get_or_insert(filename)
if not pkg.IsSafeToModify():
gae_util.ReleaseLock(lock)
gae_util.SafeBlobDel(blobstore_key)
self.redirect(
'/uploadpkg?mode=error&msg=Package%20is%20not%20modifiable')
return
if pkg.blobstore_key:
# a previous blob exists. delete it when the update has succeeded.
old_blobstore_key = pkg.blobstore_key
pkg.blobstore_key = blobstore_key
pkg.name = plist.GetPackageName()
pkg.filename = filename
pkg.user = user
pkg.catalogs = catalogs
pkg.manifests = manifests
pkg.install_types = install_types
pkg.plist = plist
pkg.pkgdata_sha256 = pkgdata_sha256
# update the PackageInfo model with the new plist string and blobstore key.
try:
pkg.put()
success = True
except db.Error:
logging.exception('error on PackageInfo.put()')
success = False
# if it failed, delete the blob that was just uploaded -- it's
# an orphan.
if not success:
gae_util.SafeBlobDel(blobstore_key)
# if this is a new entity (get_or_insert puts), attempt to delete it.
if not old_blobstore_key:
gae_util.SafeEntityDel(pkg)
gae_util.ReleaseLock(lock)
self.redirect('/uploadpkg?mode=error')
return
# if an old blob was associated with this Package, delete it.
# the new blob that was just uploaded has replaced it.
if old_blobstore_key:
gae_util.SafeBlobDel(old_blobstore_key)
gae_util.ReleaseLock(lock)
# Generate catalogs for newly uploaded pkginfo plist.
for catalog in pkg.catalogs:
models.Catalog.Generate(catalog, delay=1)
# Log admin upload to Datastore.
admin_log = models.AdminPackageLog(user=user,
action='uploadpkg',
filename=filename,
catalogs=catalogs,
manifests=manifests,
install_types=install_types,
plist=pkg.plist.GetXml())
admin_log.put()
self.redirect('/uploadpkg?mode=success&key=%s' % blobstore_key)
class PackagesInfo(handlers.AuthenticationHandler):
"""Handler for /pkgsinfo/"""
def get(self, filename=None):
"""GET
Args:
filename: string like Firefox-1.0.dmg
"""
auth_return = auth.DoAnyAuth()
if hasattr(auth_return, 'email'):
email = auth_return.email()
if not any((
auth.IsAdminUser(email),
auth.IsSupportUser(email),
)):
raise auth.IsAdminMismatch
if filename:
filename = urllib.unquote(filename)
hash_str = self.request.get('hash')
if hash_str:
lock = models.GetLockForPackage(filename)
try:
lock.Acquire(timeout=30, max_acquire_attempts=5)
except datastore_locks.AcquireLockError:
self.response.set_status(httplib.FORBIDDEN)
self.response.out.write('Could not lock pkgsinfo')
return
pkginfo = models.PackageInfo.get_by_key_name(filename)
if pkginfo:
self.response.headers[
'Content-Type'] = 'text/xml; charset=utf-8'
if hash_str:
self.response.headers['X-Pkgsinfo-Hash'] = self._Hash(
pkginfo.plist)
self.response.out.write(pkginfo.plist)
else:
if hash_str:
lock.Release()
self.response.set_status(httplib.NOT_FOUND)
return
if hash_str:
lock.Release()
else:
query = models.PackageInfo.all()
filename = self.request.get('filename')
if filename:
query.filter('filename', filename)
install_types = self.request.get_all('install_types')
for install_type in install_types:
query.filter('install_types =', install_type)
catalogs = self.request.get_all('catalogs')
for catalog in catalogs:
query.filter('catalogs =', catalog)
pkgs = []
for p in query:
pkg = {}
for k in p.properties():
if k != '_plist':
pkg[k] = getattr(p, k)
pkgs.append(pkg)
self.response.out.write('<?xml version="1.0" encoding="UTF-8"?>\n')
self.response.out.write(plist.GetXmlStr(pkgs))
self.response.headers['Content-Type'] = 'text/xml; charset=utf-8'
def _Hash(self, s):
"""Return a sha256 hash for a string.
Args:
s: str
Returns:
str, sha256 digest
"""
h = hashlib.sha256(str(s))
return h.hexdigest()
def put(self, filename):
"""PUT
Args:
filename: string like Firefox-1.0.dmg
"""
session = gaeserver.DoMunkiAuth(
require_level=gaeserver.LEVEL_UPLOADPKG)
filename = urllib.unquote(filename)
hash_str = self.request.get('hash')
catalogs = self.request.get('catalogs', None)
manifests = self.request.get('manifests', None)
install_types = self.request.get('install_types')
if catalogs == '':
catalogs = []
elif catalogs:
catalogs = catalogs.split(',')
if manifests == '':
manifests = []
elif manifests:
manifests = manifests.split(',')
if install_types:
install_types = install_types.split(',')
mpl = MunkiPackageInfoPlistStrict(self.request.body)
try:
mpl.Parse()
except plist.PlistError, e:
logging.exception('Invalid pkginfo plist PUT: \n%s\n',
self.request.body)
self.response.set_status(httplib.BAD_REQUEST)
self.response.out.write(str(e))
return
lock_name = 'pkgsinfo_%s' % filename
lock = datastore_locks.DatastoreLock(lock_name)
try:
lock.Acquire(timeout=30, max_acquire_attempts=5)
except datastore_locks.AcquireLockError:
self.response.set_status(httplib.FORBIDDEN)
self.response.out.write('Could not lock pkgsinfo')
return
# To avoid pkginfo uploads without corresponding packages, only allow
# updates to existing PackageInfo entities, not creations of new ones.
pkginfo = models.PackageInfo.get_by_key_name(filename)
if pkginfo is None:
logging.warning(
'pkginfo "%s" does not exist; PUT only allows updates.',
filename)
self.response.set_status(httplib.FORBIDDEN)
self.response.out.write('Only updates supported')
lock.Release()
return
# If the pkginfo is not modifiable, ensure only manifests have changed.
if not pkginfo.IsSafeToModify():
if not mpl.EqualIgnoringManifestsAndCatalogs(pkginfo.plist):
logging.warning(
'pkginfo "%s" is in stable or testing; change prohibited.',
filename)
self.response.set_status(httplib.FORBIDDEN)
self.response.out.write('Changes to pkginfo not allowed')
lock.Release()
return
# If the update parameter asked for a careful update, by supplying
# a hash of the last known pkgsinfo, then compare the hash to help
# the client make a non destructive update.
if hash_str:
if self._Hash(pkginfo.plist) != hash_str:
self.response.set_status(httplib.CONFLICT)
self.response.out.write('Update hash does not match')
lock.Release()
return
# All verification has passed, so let's create the PackageInfo entity.
pkginfo.plist = mpl
pkginfo.name = mpl.GetPackageName()
if catalogs is not None:
pkginfo.catalogs = catalogs
if manifests is not None:
pkginfo.manifests = manifests
if install_types:
pkginfo.install_types = install_types
pkginfo.put()
lock.Release()
for track in pkginfo.catalogs:
models.Catalog.Generate(track, delay=1)
# Log admin pkginfo put to Datastore.
user = session.uuid
admin_log = models.AdminPackageLog(user=user,
action='pkginfo',
filename=filename,
catalogs=pkginfo.catalogs,
manifests=pkginfo.manifests,
install_types=pkginfo.install_types,
plist=pkginfo.plist.GetXml())
admin_log.put()
class UploadPackage(admin.AdminHandler,
blobstore_handlers.BlobstoreUploadHandler):
"""Handler for /admin/uploadpkg."""
def get(self):
"""GET Handler.
With no parameters, return the URL to use to submit uploads via
multipart/form-data.
With mode and key parameters, return status of the previous uploadpkg
operation to the calling client.
This method actually acts as a helper to the starting and finishing
of the uploadpkg post() method.
Parameters:
mode: optionally, 'success' or 'error'
key: optionally, blobstore key that was uploaded
"""
if not handlers.IsHttps(self):
# TODO(user): Does blobstore support https yet? If so, we can
# enforce security in app.yaml and not do this check here.
return
if not self.IsAdminUser():
self.error(403)
return
mode = self.request.get('mode')
msg = self.request.get('msg', None)
if mode == 'success':
filename = self.request.get('filename')
msg = '%s successfully uploaded and is ready for deployment.' % filename
self.redirect('/admin/package/%s?msg=%s' % (filename, msg))
elif mode == 'error':
self.response.set_status(400)
self.response.out.write(msg)
else:
filename = self.request.get('filename')
if not filename:
self.response.set_status(404)
self.response.out.write('Filename required')
return
p = models.PackageInfo.get_by_key_name(filename)
if not p:
self.response.set_status(400)
self.response.out.write(
'You must first upload a pkginfo for %s' % filename)
return
elif p.blob_info:
self.response.set_status(400)
self.response.out.write('This file already exists.')
return
values = {
'upload_url': blobstore.create_upload_url('/admin/uploadpkg'),
'filename': filename,
'file_size_kbytes': p.plist['installer_item_size'],
}
self.Render('upload_pkg_form.html', values)
def post(self):
"""POST Handler.
This method behaves a little strangely. BlobstoreUploadHandler
only allows returns statuses of 301, 302, 303 (not even 200), so
one must redirect away to return more information to the caller.
Parameters:
file: package file contents
pkginfo: packageinfo file contents
name: filename of package e.g. 'Firefox-1.0.dmg'
"""
# Only blobstore/upload service/scotty requests should be
# invoking this handler.
if not handlers.IsBlobstore():
logging.critical('POST to uploadpkg not from Blobstore: %s',
self.request.headers)
self.redirect('/admin/packages')
# TODO(user): do we check is admin?
if not self.get_uploads('file'):
logging.error('Upload package does not exist.')
return
blob_info = self.get_uploads('file')[0]
blobstore_key = str(blob_info.key())
# Obtain a lock on the PackageInfo entity for this package.
lock = 'pkgsinfo_%s' % blob_info.filename
if not gae_util.ObtainLock(lock, timeout=5.0):
gae_util.SafeBlobDel(blobstore_key)
self.redirect(
'/admin/uploadpkg?mode=error&msg=PackageInfo is locked')
return
p = models.PackageInfo.get_by_key_name(blob_info.filename)
if not p:
gae_util.ReleaseLock(lock)
gae_util.SafeBlobDel(blobstore_key)
self.redirect(
'/admin/uploadpkg?mode=error&msg=PackageInfo not found')
return
if not p.IsSafeToModify():
gae_util.ReleaseLock(lock)
gae_util.SafeBlobDel(blobstore_key)
self.redirect(
'/admin/uploadpkg?mode=error&msg=PackageInfo is not modifiable'
)
return
installer_item_size = p.plist['installer_item_size']
size_difference = int(blob_info.size / 1024) - installer_item_size
if abs(size_difference) > 1:
gae_util.ReleaseLock(lock)
gae_util.SafeBlobDel(blobstore_key)
msg = 'Blob size (%s) does not match PackageInfo plist size (%s)' % (
blob_info.size, installer_item_size)
self.redirect('/admin/uploadpkg?mode=error&msg=%s' % msg)
return
old_blobstore_key = None
if p.blob_info:
# a previous blob exists. delete it when the update has succeeded.
old_blobstore_key = p.blob_info.key()
p.blob_info = blob_info
# update the PackageInfo model with the new plist string and blobstore key.
try:
p.put()
error = None
except db.Error, e:
logging.exception('error on PackageInfo.put()')
error = 'pkginfo.put() failed with: %s' % str(e)
# if it failed, delete the blob that was just uploaded -- it's
# an orphan.
if error is not None:
gae_util.SafeBlobDel(blobstore_key)
gae_util.ReleaseLock(lock)
self.redirect('/admin/uploadpkg?mode=error&msg=%s' % error)
return
# if an old blob was associated with this Package, delete it.
# the new blob that was just uploaded has replaced it.
if old_blobstore_key:
gae_util.SafeBlobDel(old_blobstore_key)
gae_util.ReleaseLock(lock)
user = users.get_current_user().email()
# Log admin upload to Datastore.
admin_log = models.AdminPackageLog(user=user,
action='uploadpkg',
filename=blob_info.filename)
admin_log.put()
self.redirect('/admin/uploadpkg?mode=success&filename=%s' %
blob_info.filename)
