def testSafeBlobDel(self):
"""Test SafeBlobDel()."""
self.mox.StubOutWithMock(gae_util.blobstore, 'delete_async')
self.mox.StubOutWithMock(gae_util.logging, 'info')
blobstore_key = 'key'
gae_util.blobstore.delete_async(blobstore_key).AndReturn(None)
gae_util.blobstore.delete_async(blobstore_key).AndRaise(
gae_util.blobstore.Error)
exc = ''
gae_util.logging.warning(('blobstore.delete(%s) failed: %s. '
'this key is now probably orphaned.'),
blobstore_key, str(exc))
self.mox.ReplayAll()
gae_util.SafeBlobDel(blobstore_key)
gae_util.SafeBlobDel(blobstore_key)
self.mox.VerifyAll()
def post(self):
"""POST
Parameters:
filename: filename of package e.g. 'Firefox-1.0.dmg'
"""
session = gaeserver.DoMunkiAuth(require_level=gaeserver.LEVEL_UPLOADPKG)
filename = self.request.get('filename')
pkginfo = models.PackageInfo.get_by_key_name(filename)
if not pkginfo:
self.response.set_status(404)
self.response.out.write('Pkginfo does not exist: %s' % filename)
return
plist = pkginfo.plist
catalogs = pkginfo.catalogs
install_types = pkginfo.install_types
#logging.info('Deleting package: %s', filename)
blobstore_key = pkginfo.blobstore_key
# Delete the PackageInfo entity, and then the package Blobstore entity.
pkginfo.delete()
gae_util.SafeBlobDel(blobstore_key)
# Recreate catalogs so references to this package don't exist anywhere.
for catalog in catalogs:
models.Catalog.Generate(catalog)
# Log admin delete to Datastore.
user = session.uuid
admin_log = models.AdminPackageLog(
user=user, action='deletepkg', filename=filename, catalogs=catalogs,
install_types=install_types, plist=plist)
admin_log.put()
def delete(self, *args, **kwargs):
"""Deletes a PackageInfo and cleans up associated data in other models.
Any Blobstore blob associated with the PackageInfo is deleted, and all
Catalogs the PackageInfo was a member of are regenerated.
Returns:
return value from superlass delete()
"""
ret = super(PackageInfo, self).delete(*args, **kwargs)
for catalog in self.catalogs:
Catalog.Generate(catalog, delay=5)
if self.blobstore_key:
gae_util.SafeBlobDel(self.blobstore_key)
return ret
def testGetBlobAndDel(self):
"""Test GetBlobAndDel()."""
blobstore_key = 'key123'
blob_str = '10 print "hi" 20 goto 10'
mock_br = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(gae_util, 'blobstore',
self.mox.CreateMock(gae_util.blobstore))
self.mox.StubOutWithMock(gae_util, 'SafeBlobDel')
gae_util.blobstore.BlobReader(blobstore_key).AndReturn(mock_br)
mock_br.read(1024 * 1024).AndReturn(blob_str)
mock_br.close()
gae_util.SafeBlobDel(blobstore_key).AndReturn(None)
self.mox.ReplayAll()
self.assertEqual(blob_str, gae_util.GetBlobAndDel(blobstore_key))
self.mox.VerifyAll()
class UploadPackage(admin.AdminHandler,
blobstore_handlers.BlobstoreUploadHandler):
"""Handler for /admin/uploadpkg."""
XSRF_PROTECTION = False
def get(self):
"""GET Handler.
With no parameters, return the URL to use to submit uploads via
multipart/form-data.
With mode and key parameters, return status of the previous uploadpkg
operation to the calling client.
This method actually acts as a helper to the starting and finishing
of the uploadpkg post() method.
Parameters:
mode: optionally, 'success' or 'error'
key: optionally, blobstore key that was uploaded
"""
if not auth.HasPermission(auth.UPLOAD):
self.error(httplib.FORBIDDEN)
return
mode = self.request.get('mode')
msg = self.request.get('msg', None)
if mode == 'success':
filename = self.request.get('filename')
msg = '%s successfully uploaded and is ready for deployment.' % filename
self.redirect('/admin/package/%s?msg=%s' % (filename, msg))
elif mode == 'error':
self.response.set_status(httplib.BAD_REQUEST)
self.response.out.write(msg)
else:
filename = self.request.get('filename')
if not filename:
self.response.set_status(httplib.NOT_FOUND)
self.response.out.write('Filename required')
return
p = models.PackageInfo.get_by_key_name(filename)
if not p:
self.response.set_status(httplib.BAD_REQUEST)
self.response.out.write(
'You must first upload a pkginfo for %s' % filename)
return
elif p.blob_info:
self.response.set_status(httplib.BAD_REQUEST)
self.response.out.write('This file already exists.')
return
upload_url = blobstore.create_upload_url(
'/admin/uploadpkg', gs_bucket_name=util.GetBlobstoreGSBucket())
values = {
'upload_url': upload_url,
'filename': filename,
'file_size_kbytes': p.plist['installer_item_size'],
}
self.Render('upload_pkg_form.html', values)
def _RedirectWithErrorMsg(self, msg):
logging.warning(msg)
self.redirect('/admin/uploadpkg?mode=error&msg=%s' % msg)
def post(self):
"""POST Handler.
This method behaves a little strangely. BlobstoreUploadHandler
only allows returns statuses of 301, 302, 303 (not even 200), so
one must redirect away to return more information to the caller.
Parameters:
file: package file contents
pkginfo: packageinfo file contents
name: filename of package e.g. 'Firefox-1.0.dmg'
"""
# Only blobstore/upload service/scotty requests should be
# invoking this handler.
if not handlers.IsBlobstore():
logging.error('POST to uploadpkg not from Blobstore: %s',
self.request.headers)
self.redirect('/admin/packages')
return
if not self.get_uploads('file'):
logging.error('Upload package does not exist.')
return
blob_info = self.get_uploads('file')[0]
blobstore_key = str(blob_info.key())
# Obtain a lock on the PackageInfo entity for this package.
lock = models.GetLockForPackage(blob_info.filename)
try:
lock.Acquire(timeout=30, max_acquire_attempts=5)
except datastore_locks.AcquireLockError:
gae_util.SafeBlobDel(blobstore_key)
self._RedirectWithErrorMsg('PackageInfo is locked')
return
p = models.PackageInfo.get_by_key_name(blob_info.filename)
if not p:
lock.Release()
gae_util.SafeBlobDel(blobstore_key)
self._RedirectWithErrorMsg('PackageInfo not found')
return
if not p.IsSafeToModify():
lock.Release()
gae_util.SafeBlobDel(blobstore_key)
self._RedirectWithErrorMsg('PackageInfo is not modifiable')
return
installer_item_size = p.plist['installer_item_size']
size_difference = int(blob_info.size / 1024) - installer_item_size
if abs(size_difference) > 1:
lock.Release()
gae_util.SafeBlobDel(blobstore_key)
msg = 'Blob size (%s) does not match PackageInfo plist size (%s)' % (
blob_info.size, installer_item_size)
self._RedirectWithErrorMsg(msg)
return
old_blobstore_key = None
if p.blobstore_key:
# a previous blob exists. delete it when the update has succeeded.
old_blobstore_key = p.blobstore_key
p.blob_info = blob_info
# update the PackageInfo model with the new plist string and blobstore key.
try:
p.put()
error = None
except db.Error, e:
logging.exception('error on PackageInfo.put()')
error = 'pkginfo.put() failed with: %s' % str(e)
# if it failed, delete the blob that was just uploaded -- it's
# an orphan.
if error is not None:
gae_util.SafeBlobDel(blobstore_key)
lock.Release()
self._RedirectWithErrorMsg(error)
return
# if an old blob was associated with this Package, delete it.
# the new blob that was just uploaded has replaced it.
if old_blobstore_key:
gae_util.SafeBlobDel(old_blobstore_key)
lock.Release()
user = users.get_current_user().email()
# Log admin upload to Datastore.
admin_log = models.AdminPackageLog(user=user,
action='uploadpkg',
filename=blob_info.filename)
admin_log.put()
self.redirect('/admin/uploadpkg?mode=success&filename=%s' %
blob_info.filename)
def post(self):
"""POST Handler.
This method behaves a little strangely. BlobstoreUploadHandler
only allows returns statuses of 301, 302, 303 (not even 200), so
one must redirect away to return more information to the caller.
Parameters:
file: package file contents
pkginfo: packageinfo file contents
name: filename of package e.g. 'Firefox-1.0.dmg'
"""
# Only blobstore/upload service/scotty requests should be
# invoking this handler.
if not handlers.IsBlobstore():
logging.error('POST to uploadpkg not from Blobstore: %s',
self.request.headers)
self.redirect('/admin/packages')
return
if not self.get_uploads('file'):
logging.error('Upload package does not exist.')
return
blob_info = self.get_uploads('file')[0]
blobstore_key = str(blob_info.key())
# Obtain a lock on the PackageInfo entity for this package.
lock = models.GetLockForPackage(blob_info.filename)
try:
lock.Acquire(timeout=30, max_acquire_attempts=5)
except datastore_locks.AcquireLockError:
gae_util.SafeBlobDel(blobstore_key)
self._RedirectWithErrorMsg('PackageInfo is locked')
return
p = models.PackageInfo.get_by_key_name(blob_info.filename)
if not p:
lock.Release()
gae_util.SafeBlobDel(blobstore_key)
self._RedirectWithErrorMsg('PackageInfo not found')
return
if not p.IsSafeToModify():
lock.Release()
gae_util.SafeBlobDel(blobstore_key)
self._RedirectWithErrorMsg('PackageInfo is not modifiable')
return
installer_item_size = p.plist['installer_item_size']
size_difference = int(blob_info.size / 1024) - installer_item_size
if abs(size_difference) > 1:
lock.Release()
gae_util.SafeBlobDel(blobstore_key)
msg = 'Blob size (%s) does not match PackageInfo plist size (%s)' % (
blob_info.size, installer_item_size)
self._RedirectWithErrorMsg(msg)
return
old_blobstore_key = None
if p.blobstore_key:
# a previous blob exists. delete it when the update has succeeded.
old_blobstore_key = p.blobstore_key
p.blob_info = blob_info
# update the PackageInfo model with the new plist string and blobstore key.
try:
p.put()
error = None
except db.Error, e:
logging.exception('error on PackageInfo.put()')
error = 'pkginfo.put() failed with: %s' % str(e)
def post(self):
"""POST method.
This method behaves a little strangely. BlobstoreUploadHandler
only allows returns statuses of 301, 302, 303 (not even 200), so
one must redirect away to return more information to the caller.
Parameters:
file: package file contents
pkginfo: packageinfo file contents
name: filename of package e.g. 'Firefox-1.0.dmg'
"""
# Only blobstore/upload service/scotty requests should be
# invoking this handler.
if not handlers.IsBlobstore():
logging.critical('POST to /uploadpkg not from Blobstore: %s',
self.request.headers)
self.redirect('/')
gaeserver.DoMunkiAuth(require_level=gaeserver.LEVEL_UPLOADPKG)
user = self.request.get('user')
filename = self.request.get('name')
install_types = self.request.get('install_types')
catalogs = self.request.get('catalogs', None)
manifests = self.request.get('manifests', None)
if catalogs is None or not install_types or not user or not filename:
msg = 'uploadpkg POST required parameters missing'
logging.error(msg)
self.redirect('/uploadpkg?mode=error&msg=%s' % msg)
return
if catalogs == '':
catalogs = []
else:
catalogs = catalogs.split(',')
if manifests in ['', None]:
manifests = []
else:
manifests = manifests.split(',')
install_types = install_types.split(',')
upload_files = self.get_uploads('file')
upload_pkginfo_files = self.get_uploads('pkginfo')
if not len(upload_pkginfo_files) and not self.request.get('pkginfo'):
self.redirect('/uploadpkg?mode=error&msg=No%20file%20received')
return
if len(upload_pkginfo_files):
# obtain the pkginfo from a blob, and then throw it away. this is
# a necessary hack because the upload handler grabbed it, but we don't
# intend to keep it in blobstore.
pkginfo_str = gae_util.GetBlobAndDel(upload_pkginfo_files[0].key())
else:
# otherwise, grab the form parameter.
pkginfo_str = self.request.get('pkginfo')
blob_info = upload_files[0]
blobstore_key = str(blob_info.key())
# Parse, validate, and encode the pkginfo plist.
plist = plist_lib.MunkiPackageInfoPlist(pkginfo_str)
try:
plist.Parse()
except plist_lib.PlistError:
logging.exception('Invalid pkginfo plist uploaded:\n%s\n',
pkginfo_str)
gae_util.SafeBlobDel(blobstore_key)
self.redirect(
'/uploadpkg?mode=error&msg=No%20valid%20pkginfo%20received')
return
filename = plist['installer_item_location']
pkgdata_sha256 = plist['installer_item_hash']
# verify the blob was actually written; in case Blobstore failed to write
# the blob but still POSTed to this handler (very, very rare).
blob_info = blobstore.BlobInfo.get(blobstore_key)
if not blob_info:
logging.critical(
'Blobstore returned a key for %s that does not exist: %s',
filename, blobstore_key)
self.redirect('/uploadpkg?mode=error&msg=Blobstore%20failure')
return
# Obtain a lock on the PackageInfo entity for this package.
lock = 'pkgsinfo_%s' % filename
if not gae_util.ObtainLock(lock, timeout=5.0):
gae_util.SafeBlobDel(blobstore_key)
self.redirect(
'/uploadpkg?mode=error&msg=Could%20not%20lock%20pkgsinfo')
return
old_blobstore_key = None
pkg = models.PackageInfo.get_or_insert(filename)
if not pkg.IsSafeToModify():
gae_util.ReleaseLock(lock)
gae_util.SafeBlobDel(blobstore_key)
self.redirect(
'/uploadpkg?mode=error&msg=Package%20is%20not%20modifiable')
return
if pkg.blobstore_key:
# a previous blob exists. delete it when the update has succeeded.
old_blobstore_key = pkg.blobstore_key
pkg.blobstore_key = blobstore_key
pkg.name = plist.GetPackageName()
pkg.filename = filename
pkg.user = user
pkg.catalogs = catalogs
pkg.manifests = manifests
pkg.install_types = install_types
pkg.plist = plist
pkg.pkgdata_sha256 = pkgdata_sha256
# update the PackageInfo model with the new plist string and blobstore key.
try:
pkg.put()
success = True
except db.Error:
logging.exception('error on PackageInfo.put()')
success = False
# if it failed, delete the blob that was just uploaded -- it's
# an orphan.
if not success:
gae_util.SafeBlobDel(blobstore_key)
# if this is a new entity (get_or_insert puts), attempt to delete it.
if not old_blobstore_key:
gae_util.SafeEntityDel(pkg)
gae_util.ReleaseLock(lock)
self.redirect('/uploadpkg?mode=error')
return
# if an old blob was associated with this Package, delete it.
# the new blob that was just uploaded has replaced it.
if old_blobstore_key:
gae_util.SafeBlobDel(old_blobstore_key)
gae_util.ReleaseLock(lock)
# Generate catalogs for newly uploaded pkginfo plist.
for catalog in pkg.catalogs:
models.Catalog.Generate(catalog, delay=1)
# Log admin upload to Datastore.
admin_log = models.AdminPackageLog(user=user,
action='uploadpkg',
filename=filename,
catalogs=catalogs,
manifests=manifests,
install_types=install_types,
plist=pkg.plist.GetXml())
admin_log.put()
self.redirect('/uploadpkg?mode=success&key=%s' % blobstore_key)
class UploadPackage(admin.AdminHandler,
blobstore_handlers.BlobstoreUploadHandler):
"""Handler for /admin/uploadpkg."""
def get(self):
"""GET Handler.
With no parameters, return the URL to use to submit uploads via
multipart/form-data.
With mode and key parameters, return status of the previous uploadpkg
operation to the calling client.
This method actually acts as a helper to the starting and finishing
of the uploadpkg post() method.
Parameters:
mode: optionally, 'success' or 'error'
key: optionally, blobstore key that was uploaded
"""
if not handlers.IsHttps(self):
# TODO(user): Does blobstore support https yet? If so, we can
# enforce security in app.yaml and not do this check here.
return
if not self.IsAdminUser():
self.error(403)
return
mode = self.request.get('mode')
msg = self.request.get('msg', None)
if mode == 'success':
filename = self.request.get('filename')
msg = '%s successfully uploaded and is ready for deployment.' % filename
self.redirect('/admin/package/%s?msg=%s' % (filename, msg))
elif mode == 'error':
self.response.set_status(400)
self.response.out.write(msg)
else:
filename = self.request.get('filename')
if not filename:
self.response.set_status(404)
self.response.out.write('Filename required')
return
p = models.PackageInfo.get_by_key_name(filename)
if not p:
self.response.set_status(400)
self.response.out.write(
'You must first upload a pkginfo for %s' % filename)
return
elif p.blob_info:
self.response.set_status(400)
self.response.out.write('This file already exists.')
return
values = {
'upload_url': blobstore.create_upload_url('/admin/uploadpkg'),
'filename': filename,
'file_size_kbytes': p.plist['installer_item_size'],
}
self.Render('upload_pkg_form.html', values)
def post(self):
"""POST Handler.
This method behaves a little strangely. BlobstoreUploadHandler
only allows returns statuses of 301, 302, 303 (not even 200), so
one must redirect away to return more information to the caller.
Parameters:
file: package file contents
pkginfo: packageinfo file contents
name: filename of package e.g. 'Firefox-1.0.dmg'
"""
# Only blobstore/upload service/scotty requests should be
# invoking this handler.
if not handlers.IsBlobstore():
logging.critical('POST to uploadpkg not from Blobstore: %s',
self.request.headers)
self.redirect('/admin/packages')
# TODO(user): do we check is admin?
if not self.get_uploads('file'):
logging.error('Upload package does not exist.')
return
blob_info = self.get_uploads('file')[0]
blobstore_key = str(blob_info.key())
# Obtain a lock on the PackageInfo entity for this package.
lock = 'pkgsinfo_%s' % blob_info.filename
if not gae_util.ObtainLock(lock, timeout=5.0):
gae_util.SafeBlobDel(blobstore_key)
self.redirect(
'/admin/uploadpkg?mode=error&msg=PackageInfo is locked')
return
p = models.PackageInfo.get_by_key_name(blob_info.filename)
if not p:
gae_util.ReleaseLock(lock)
gae_util.SafeBlobDel(blobstore_key)
self.redirect(
'/admin/uploadpkg?mode=error&msg=PackageInfo not found')
return
if not p.IsSafeToModify():
gae_util.ReleaseLock(lock)
gae_util.SafeBlobDel(blobstore_key)
self.redirect(
'/admin/uploadpkg?mode=error&msg=PackageInfo is not modifiable'
)
return
installer_item_size = p.plist['installer_item_size']
size_difference = int(blob_info.size / 1024) - installer_item_size
if abs(size_difference) > 1:
gae_util.ReleaseLock(lock)
gae_util.SafeBlobDel(blobstore_key)
msg = 'Blob size (%s) does not match PackageInfo plist size (%s)' % (
blob_info.size, installer_item_size)
self.redirect('/admin/uploadpkg?mode=error&msg=%s' % msg)
return
old_blobstore_key = None
if p.blob_info:
# a previous blob exists. delete it when the update has succeeded.
old_blobstore_key = p.blob_info.key()
p.blob_info = blob_info
# update the PackageInfo model with the new plist string and blobstore key.
try:
p.put()
error = None
except db.Error, e:
logging.exception('error on PackageInfo.put()')
error = 'pkginfo.put() failed with: %s' % str(e)
# if it failed, delete the blob that was just uploaded -- it's
# an orphan.
if error is not None:
gae_util.SafeBlobDel(blobstore_key)
gae_util.ReleaseLock(lock)
self.redirect('/admin/uploadpkg?mode=error&msg=%s' % error)
return
# if an old blob was associated with this Package, delete it.
# the new blob that was just uploaded has replaced it.
if old_blobstore_key:
gae_util.SafeBlobDel(old_blobstore_key)
gae_util.ReleaseLock(lock)
user = users.get_current_user().email()
# Log admin upload to Datastore.
admin_log = models.AdminPackageLog(user=user,
action='uploadpkg',
filename=blob_info.filename)
admin_log.put()
self.redirect('/admin/uploadpkg?mode=success&filename=%s' %
blob_info.filename)
def post(self):
"""POST
This method behaves a little strangely. BlobstoreUploadHandler
only allows returns statuses of 301, 302, 303 (not even 200), so
one must redirect away to return more information to the caller.
Parameters:
file: package file contents
pkginfo: packageinfo file contents
name: filename of package e.g. 'Firefox-1.0.dmg'
"""
# Only blobstore/upload service/scotty requests should be
# invoking this handler.
if not handlers.IsBlobstore():
logging.critical('POST to /uploadpkg not from Blobstore: %s',
self.request.headers)
self.redirect('/')
gaeserver.DoMunkiAuth(require_level=gaeserver.LEVEL_UPLOADPKG)
user = self.request.get('user')
filename = self.request.get('name')
install_types = self.request.get('install_types')
catalogs = self.request.get('catalogs', None)
manifests = self.request.get('manifests', None)
if catalogs is None or not install_types or not user or not filename:
msg = 'uploadpkg POST required parameters missing'
logging.error(msg)
self.redirect('/uploadpkg?mode=error&msg=%s' % msg)
return
if catalogs == '':
catalogs = []
else:
catalogs = catalogs.split(',')
if manifests in ['', None]:
manifests = []
else:
manifests = manifests.split(',')
install_types = install_types.split(',')
upload_files = self.get_uploads('file')
upload_pkginfo_files = self.get_uploads('pkginfo')
if not len(upload_pkginfo_files) and not self.request.get('pkginfo'):
self.redirect('/uploadpkg?mode=error&msg=No%20file%20received')
return
if len(upload_pkginfo_files):
# obtain the pkginfo from a blob, and then throw it away. this is
# a necessary hack because the upload handler grabbed it, but we don't
# intend to keep it in blobstore.
pkginfo_str = gae_util.GetBlobAndDel(upload_pkginfo_files[0].key())
else:
# otherwise, grab the form parameter.
pkginfo_str = self.request.get('pkginfo')
blob_info = upload_files[0]
blobstore_key = str(blob_info.key())
# Parse, validate, and encode the pkginfo plist.
plist = plist_lib.MunkiPackageInfoPlist(pkginfo_str)
try:
plist.Parse()
except plist_lib.PlistError, e:
logging.exception('Invalid pkginfo plist uploaded:\n%s\n',
pkginfo_str)
gae_util.SafeBlobDel(blobstore_key)
self.redirect(
'/uploadpkg?mode=error&msg=No%20valid%20pkginfo%20received')
return
