def Generate(cls, name, delay=0):
"""Generates a Manifest plist and entity from matching PackageInfo entities.
Args:
name: str, manifest name. all PackageInfo entities with this name in the
"manifests" property will be included in the generated manifest.
delay: int. if > 0, Generate call is deferred this many seconds.
"""
if delay:
now = datetime.datetime.utcnow()
now_str = '%s-%d' % (now.strftime('%Y-%m-%d-%H-%M-%S'),
now.microsecond)
deferred_name = 'create-manifest-%s-%s' % (name, now_str)
deferred.defer(cls.Generate,
name,
_name=deferred_name,
_countdown=delay)
return
lock = 'manifest_lock_%s' % name
if not gae_util.ObtainLock(lock):
logging.debug('Manifest.Generate for %s is locked. Delaying....',
name)
cls.Generate(name, delay=5)
return
#logging.debug('Creating manifest: %s', name)
try:
package_infos = PackageInfo.all().filter('manifests =', name)
if not package_infos:
# TODO(user): if this happens we probably want to notify admins...
raise ManifestGenerateError('PackageInfo entities found: %s' %
name)
install_types = {}
for p in package_infos:
# Add all installs to their appropriate install type containers.
for install_type in p.install_types:
if install_type not in install_types:
install_types[install_type] = []
install_types[install_type].append(p.name)
# Generate a dictionary of the manifest data.
manifest_dict = {'catalogs': [name, 'apple_update_metadata']}
for k, v in install_types.iteritems():
manifest_dict[k] = v
# Save the new manifest to Datastore.
manifest_entity = cls.get_or_insert(name)
# Turn the manifest dictionary into XML.
manifest_entity.plist.SetContents(manifest_dict)
manifest_entity.put()
cls.DeleteMemcacheWrap(name)
#logging.debug(
# 'Manifest %s created successfully', name)
except (ManifestGenerateError, db.Error, plist_lib.Error):
logging.exception('Manifest.Generate failure: %s', name)
raise
finally:
gae_util.ReleaseLock(lock)
def testObtainLock(self):
"""Test ObtainLock()."""
lock = 'foo'
self.mox.StubOutWithMock(gae_util, 'memcache')
gae_util.memcache.incr('lock_%s' % lock, initial_value=0).AndReturn(1)
self.mox.ReplayAll()
self.assertTrue(gae_util.ObtainLock(lock))
self.mox.VerifyAll()
def Generate(cls, name, delay=0):
"""Generates a Catalog plist and entity from matching PackageInfo entities.
Args:
name: str, catalog name. all PackageInfo entities with this name in the
"catalogs" property will be included in the generated catalog.
delay: int, if > 0, Generate call is deferred this many seconds.
"""
if delay:
now = datetime.datetime.utcnow()
now_str = '%s-%d' % (now.strftime('%Y-%m-%d-%H-%M-%S'),
now.microsecond)
deferred_name = 'create-catalog-%s-%s' % (name, now_str)
deferred.defer(cls.Generate,
name,
_name=deferred_name,
_countdown=delay)
return
lock = 'catalog_lock_%s' % name
# Obtain a lock on the catalog name.
if not gae_util.ObtainLock(lock):
# If catalog creation for this name is already in progress then delay.
logging.debug('Catalog creation for %s is locked. Delaying....',
name)
cls.Generate(name, delay=10)
return
#logging.debug('Creating catalog: %s', name)
package_names = []
try:
pkgsinfo_dicts = []
package_infos = PackageInfo.all().filter('catalogs =', name)
if not package_infos:
# TODO(user): if this happens we probably want to notify admins...
raise CatalogGenerateError(
'No pkgsinfo found with catalog: %s' % name)
for p in package_infos:
package_names.append(p.name)
pkgsinfo_dicts.append(p.plist.GetXmlContent(indent_num=1))
catalog = constants.CATALOG_PLIST_XML % '\n'.join(pkgsinfo_dicts)
c = cls.get_or_insert(name)
c.package_names = package_names
c.name = name
c.plist = catalog
c.put()
cls.DeleteMemcacheWrap(name, prop_name='plist_xml')
#logging.debug('Generated catalog successfully: %s', name)
# Generate manifest for newly generated catalog.
Manifest.Generate(name, delay=1)
except (CatalogGenerateError, db.Error, plist_lib.Error):
logging.exception('Catalog.Generate failure for catalog: %s', name)
raise
finally:
gae_util.ReleaseLock(lock)
def testObtainLockWhenTimeoutTrue(self):
"""Test ObtainLock()."""
lock = 'foo'
self.mox.StubOutWithMock(gae_util, 'memcache')
self.mox.StubOutWithMock(gae_util.time, 'sleep')
gae_util.memcache.incr('lock_%s' % lock, initial_value=0).AndReturn(2)
gae_util.time.sleep(1).AndReturn(None)
gae_util.memcache.incr('lock_%s' % lock, initial_value=0).AndReturn(1)
self.mox.ReplayAll()
self.assertTrue(gae_util.ObtainLock(lock, timeout=1))
self.mox.VerifyAll()
def ApproveProposal(self):
"""Approve a pending proposal.
Raises:
PackageInfoProposalApprovalError: Approver not alloed to approve.
PackageInfoLockError: PackageInfo is locked.
PackageInfoUpdateError: Package is not eligible for catalogs.
"""
lock = 'pkgsinfo_%s' % self.filename
if not gae_util.ObtainLock(lock, timeout=5.0):
raise PackageInfoLockError
new_catalogs = [
c for c in self.catalogs if c not in self.pkginfo.catalogs
]
try:
self.pkginfo.VerifyPackageIsEligibleForNewCatalogs(new_catalogs)
except PackageInfoUpdateError:
gae_util.ReleaseLock(lock)
raise
approver = users.get_current_user().email()
if approver == self.user:
raise PackageInfoProposalApprovalError
self.approver = approver
self.status = 'approved'
self._PutAndLogPackageInfoProposalUpdate(self,
self.pkginfo.plist.GetXml(),
self.catalogs,
action='approve')
original_plist = self.pkginfo.plist.GetXml()
original_catalogs = self.pkginfo.catalogs
for key in self.COMMON_PROPERTIES:
setattr(self.pkginfo, key, getattr(self, key))
self.pkginfo.PutAndLogFromProposal(original_plist, original_catalogs)
gae_util.ReleaseLock(lock)
self.ProposalMailer('approval')
self.delete()
def get(self):
"""Handle GET."""
pkgs, unused_dt = models.ReportsCache.GetInstallCounts()
for p in gae_util.QueryIterator(models.PackageInfo.all()):
if not p.plist:
continue # skip over pkginfos without plists.
if p.munki_name not in pkgs:
# Skip pkginfos that ReportsCache lacks.
continue
elif not pkgs[p.munki_name].get('duration_seconds_avg', None):
# Skip pkginfos where there is no known average duration.
continue
# Obtain a lock on the PackageInfo entity for this package, or skip.
lock = 'pkgsinfo_%s' % p.filename
if not gae_util.ObtainLock(lock, timeout=5.0):
continue # Skip; it'll get updated next time around.
# Append the avg duration text to the description; in the future the
# avg duration time and overall install count will be added to they're
# own pkginfo keys so the information can be displayed independantly.
# This requires MSU changes to read and display such values, so for now
# simply append text to the description.
old_desc = p.plist['description']
avg_duration_text = models.PackageInfo.AVG_DURATION_TEXT % (
pkgs[p.munki_name]['duration_count'],
pkgs[p.munki_name]['duration_seconds_avg'])
p.description = '%s\n\n%s' % (p.description, avg_duration_text)
if p.plist['description'] != old_desc:
p.put(
) # Only bother putting the entity if the description changed.
gae_util.ReleaseLock(lock)
# Asyncronously regenerate all Catalogs to include updated pkginfo plists.
delay = 0
for track in common.TRACKS:
delay += 5
models.Catalog.Generate(track, delay=delay)
def post(self):
"""POST method.
This method behaves a little strangely. BlobstoreUploadHandler
only allows returns statuses of 301, 302, 303 (not even 200), so
one must redirect away to return more information to the caller.
Parameters:
file: package file contents
pkginfo: packageinfo file contents
name: filename of package e.g. 'Firefox-1.0.dmg'
"""
# Only blobstore/upload service/scotty requests should be
# invoking this handler.
if not handlers.IsBlobstore():
logging.critical('POST to /uploadpkg not from Blobstore: %s',
self.request.headers)
self.redirect('/')
gaeserver.DoMunkiAuth(require_level=gaeserver.LEVEL_UPLOADPKG)
user = self.request.get('user')
filename = self.request.get('name')
install_types = self.request.get('install_types')
catalogs = self.request.get('catalogs', None)
manifests = self.request.get('manifests', None)
if catalogs is None or not install_types or not user or not filename:
msg = 'uploadpkg POST required parameters missing'
logging.error(msg)
self.redirect('/uploadpkg?mode=error&msg=%s' % msg)
return
if catalogs == '':
catalogs = []
else:
catalogs = catalogs.split(',')
if manifests in ['', None]:
manifests = []
else:
manifests = manifests.split(',')
install_types = install_types.split(',')
upload_files = self.get_uploads('file')
upload_pkginfo_files = self.get_uploads('pkginfo')
if not len(upload_pkginfo_files) and not self.request.get('pkginfo'):
self.redirect('/uploadpkg?mode=error&msg=No%20file%20received')
return
if len(upload_pkginfo_files):
# obtain the pkginfo from a blob, and then throw it away. this is
# a necessary hack because the upload handler grabbed it, but we don't
# intend to keep it in blobstore.
pkginfo_str = gae_util.GetBlobAndDel(upload_pkginfo_files[0].key())
else:
# otherwise, grab the form parameter.
pkginfo_str = self.request.get('pkginfo')
blob_info = upload_files[0]
blobstore_key = str(blob_info.key())
# Parse, validate, and encode the pkginfo plist.
plist = plist_lib.MunkiPackageInfoPlist(pkginfo_str)
try:
plist.Parse()
except plist_lib.PlistError:
logging.exception('Invalid pkginfo plist uploaded:\n%s\n',
pkginfo_str)
gae_util.SafeBlobDel(blobstore_key)
self.redirect(
'/uploadpkg?mode=error&msg=No%20valid%20pkginfo%20received')
return
filename = plist['installer_item_location']
pkgdata_sha256 = plist['installer_item_hash']
# verify the blob was actually written; in case Blobstore failed to write
# the blob but still POSTed to this handler (very, very rare).
blob_info = blobstore.BlobInfo.get(blobstore_key)
if not blob_info:
logging.critical(
'Blobstore returned a key for %s that does not exist: %s',
filename, blobstore_key)
self.redirect('/uploadpkg?mode=error&msg=Blobstore%20failure')
return
# Obtain a lock on the PackageInfo entity for this package.
lock = 'pkgsinfo_%s' % filename
if not gae_util.ObtainLock(lock, timeout=5.0):
gae_util.SafeBlobDel(blobstore_key)
self.redirect(
'/uploadpkg?mode=error&msg=Could%20not%20lock%20pkgsinfo')
return
old_blobstore_key = None
pkg = models.PackageInfo.get_or_insert(filename)
if not pkg.IsSafeToModify():
gae_util.ReleaseLock(lock)
gae_util.SafeBlobDel(blobstore_key)
self.redirect(
'/uploadpkg?mode=error&msg=Package%20is%20not%20modifiable')
return
if pkg.blobstore_key:
# a previous blob exists. delete it when the update has succeeded.
old_blobstore_key = pkg.blobstore_key
pkg.blobstore_key = blobstore_key
pkg.name = plist.GetPackageName()
pkg.filename = filename
pkg.user = user
pkg.catalogs = catalogs
pkg.manifests = manifests
pkg.install_types = install_types
pkg.plist = plist
pkg.pkgdata_sha256 = pkgdata_sha256
# update the PackageInfo model with the new plist string and blobstore key.
try:
pkg.put()
success = True
except db.Error:
logging.exception('error on PackageInfo.put()')
success = False
# if it failed, delete the blob that was just uploaded -- it's
# an orphan.
if not success:
gae_util.SafeBlobDel(blobstore_key)
# if this is a new entity (get_or_insert puts), attempt to delete it.
if not old_blobstore_key:
gae_util.SafeEntityDel(pkg)
gae_util.ReleaseLock(lock)
self.redirect('/uploadpkg?mode=error')
return
# if an old blob was associated with this Package, delete it.
# the new blob that was just uploaded has replaced it.
if old_blobstore_key:
gae_util.SafeBlobDel(old_blobstore_key)
gae_util.ReleaseLock(lock)
# Generate catalogs for newly uploaded pkginfo plist.
for catalog in pkg.catalogs:
models.Catalog.Generate(catalog, delay=1)
# Log admin upload to Datastore.
admin_log = models.AdminPackageLog(user=user,
action='uploadpkg',
filename=filename,
catalogs=catalogs,
manifests=manifests,
install_types=install_types,
plist=pkg.plist.GetXml())
admin_log.put()
self.redirect('/uploadpkg?mode=success&key=%s' % blobstore_key)
class AppleSUS(handlers.AuthenticationHandler):
"""Handler for /applesus/"""
def get(self, client_id_str=''):
"""AppleSUS get handler.
Args:
name: str, optional, catalog name to get.
"""
session = auth.DoAnyAuth()
client_id = handlers.GetClientIdForRequest(
self.request, session=session, client_id_str=client_id_str)
# get only major.minor os_version, stripping miniscule versioning.
# i.e. 10.6.6 becomes 10.6, 10.23.6.x.x becomes 10.23
full_os_version = client_id.get('os_version', '') or ''
os_version = '.'.join(full_os_version.split('.', 2)[:2])
track = client_id.get('track', 'stable')
catalog_name = '%s_%s' % (os_version, track)
catalog = models.AppleSUSCatalog.MemcacheWrappedGet(catalog_name)
if not catalog:
logging.warning('Apple SUS catalog not found: %s', catalog_name)
self.response.set_status(404)
return
header_date_str = self.request.headers.get('If-Modified-Since', '')
catalog_date = catalog.mtime
if handlers.IsClientResourceExpired(catalog_date, header_date_str):
self.response.headers['Last-Modified'] = catalog_date.strftime(
handlers.HEADER_DATE_FORMAT)
self.response.headers['Content-Type'] = 'text/xml; charset=utf-8'
self.response.out.write(catalog.plist)
else:
self.response.set_status(304)
def put(self, name):
"""AppleSUS put handler.
Args:
name: str, catalog name to put.
"""
gaeserver.DoMunkiAuth(require_level=gaeserver.LEVEL_UPLOADPKG)
# try loading for validation's sake
c = plist.AppleSoftwareCatalogPlist(self.request.body)
try:
c.Parse()
except plist.PlistError, e:
logging.exception('Invalid Apple SUS catalog format: %s', str(e))
self.response.set_status(400)
self.response.out.write(str(e))
return
del(c)
lock = 'applesus_%s' % name
if not gae_util.ObtainLock(lock, timeout=5.0):
self.response.set_status(403)
self.response.out.write('Could not lock applesus')
return
try:
asucatalog = models.AppleSUSCatalog.get_or_insert(name)
asucatalog.plist = self.request.body # retain original appearance
asucatalog.put()
except (plist.PlistError, models.db.Error), e:
logging.exception('applesus: %s', str(e))
self.response.set_status(500)
self.response.out.write(str(e))
pass
def post(self):
"""POST Handler.
This method behaves a little strangely. BlobstoreUploadHandler
only allows returns statuses of 301, 302, 303 (not even 200), so
one must redirect away to return more information to the caller.
Parameters:
file: package file contents
pkginfo: packageinfo file contents
name: filename of package e.g. 'Firefox-1.0.dmg'
"""
# Only blobstore/upload service/scotty requests should be
# invoking this handler.
if not handlers.IsBlobstore():
logging.critical('POST to uploadpkg not from Blobstore: %s',
self.request.headers)
self.redirect('/admin/packages')
# TODO(user): do we check is admin?
if not self.get_uploads('file'):
logging.error('Upload package does not exist.')
return
blob_info = self.get_uploads('file')[0]
blobstore_key = str(blob_info.key())
# Obtain a lock on the PackageInfo entity for this package.
lock = 'pkgsinfo_%s' % blob_info.filename
if not gae_util.ObtainLock(lock, timeout=5.0):
gae_util.SafeBlobDel(blobstore_key)
self.redirect(
'/admin/uploadpkg?mode=error&msg=PackageInfo is locked')
return
p = models.PackageInfo.get_by_key_name(blob_info.filename)
if not p:
gae_util.ReleaseLock(lock)
gae_util.SafeBlobDel(blobstore_key)
self.redirect(
'/admin/uploadpkg?mode=error&msg=PackageInfo not found')
return
if not p.IsSafeToModify():
gae_util.ReleaseLock(lock)
gae_util.SafeBlobDel(blobstore_key)
self.redirect(
'/admin/uploadpkg?mode=error&msg=PackageInfo is not modifiable'
)
return
installer_item_size = p.plist['installer_item_size']
size_difference = int(blob_info.size / 1024) - installer_item_size
if abs(size_difference) > 1:
gae_util.ReleaseLock(lock)
gae_util.SafeBlobDel(blobstore_key)
msg = 'Blob size (%s) does not match PackageInfo plist size (%s)' % (
blob_info.size, installer_item_size)
self.redirect('/admin/uploadpkg?mode=error&msg=%s' % msg)
return
old_blobstore_key = None
if p.blob_info:
# a previous blob exists. delete it when the update has succeeded.
old_blobstore_key = p.blob_info.key()
p.blob_info = blob_info
# update the PackageInfo model with the new plist string and blobstore key.
try:
p.put()
error = None
except db.Error, e:
logging.exception('error on PackageInfo.put()')
error = 'pkginfo.put() failed with: %s' % str(e)
class PackagesInfo(handlers.AuthenticationHandler):
"""Handler for /pkgsinfo/"""
def get(self, filename=None):
"""GET
Args:
filename: string like Firefox-1.0.dmg
"""
auth_return = auth.DoAnyAuth()
if hasattr(auth_return, 'email'):
email = auth_return.email()
if not any((
auth.IsAdminUser(email),
auth.IsSupportUser(email),
)):
raise auth.IsAdminMismatch
if filename:
filename = urllib.unquote(filename)
hash_str = self.request.get('hash')
if hash_str:
lock = 'pkgsinfo_%s' % filename
if not gae_util.ObtainLock(lock, timeout=5.0):
self.response.set_status(httplib.FORBIDDEN)
self.response.out.write('Could not lock pkgsinfo')
return
pkginfo = models.PackageInfo.get_by_key_name(filename)
if pkginfo:
self.response.headers[
'Content-Type'] = 'text/xml; charset=utf-8'
if hash_str:
self.response.headers['X-Pkgsinfo-Hash'] = self._Hash(
pkginfo.plist)
self.response.out.write(pkginfo.plist)
else:
if hash_str:
gae_util.ReleaseLock(lock)
self.response.set_status(httplib.NOT_FOUND)
return
if hash_str:
gae_util.ReleaseLock(lock)
else:
query = models.PackageInfo.all()
filename = self.request.get('filename')
if filename:
query.filter('filename', filename)
install_types = self.request.get_all('install_types')
for install_type in install_types:
query.filter('install_types =', install_type)
catalogs = self.request.get_all('catalogs')
for catalog in catalogs:
query.filter('catalogs =', catalog)
pkgs = []
for p in query:
pkg = {}
for k in p.properties():
if k != '_plist':
pkg[k] = getattr(p, k)
pkgs.append(pkg)
self.response.out.write('<?xml version="1.0" encoding="UTF-8"?>\n')
self.response.out.write(plist.GetXmlStr(pkgs))
self.response.headers['Content-Type'] = 'text/xml; charset=utf-8'
def _Hash(self, s):
"""Return a sha256 hash for a string.
Args:
s: str
Returns:
str, sha256 digest
"""
h = hashlib.sha256(str(s))
return h.hexdigest()
def put(self, filename):
"""PUT
Args:
filename: string like Firefox-1.0.dmg
"""
session = gaeserver.DoMunkiAuth(
require_level=gaeserver.LEVEL_UPLOADPKG)
filename = urllib.unquote(filename)
hash_str = self.request.get('hash')
catalogs = self.request.get('catalogs', None)
manifests = self.request.get('manifests', None)
install_types = self.request.get('install_types')
if catalogs == '':
catalogs = []
elif catalogs:
catalogs = catalogs.split(',')
if manifests == '':
manifests = []
elif manifests:
manifests = manifests.split(',')
if install_types:
install_types = install_types.split(',')
mpl = MunkiPackageInfoPlistStrict(self.request.body)
try:
mpl.Parse()
except plist.PlistError, e:
logging.exception('Invalid pkginfo plist PUT: \n%s\n',
self.request.body)
self.response.set_status(httplib.BAD_REQUEST)
self.response.out.write(str(e))
return
lock = 'pkgsinfo_%s' % filename
if not gae_util.ObtainLock(lock, timeout=5.0):
self.response.set_status(httplib.FORBIDDEN)
self.response.out.write('Could not lock pkgsinfo')
return
# To avoid pkginfo uploads without corresponding packages, only allow
# updates to existing PackageInfo entities, not creations of new ones.
pkginfo = models.PackageInfo.get_by_key_name(filename)
if pkginfo is None:
logging.warning(
'pkginfo "%s" does not exist; PUT only allows updates.',
filename)
self.response.set_status(httplib.FORBIDDEN)
self.response.out.write('Only updates supported')
gae_util.ReleaseLock(lock)
return
# If the pkginfo is not modifiable, ensure only manifests have changed.
if not pkginfo.IsSafeToModify():
if not mpl.EqualIgnoringManifestsAndCatalogs(pkginfo.plist):
logging.warning(
'pkginfo "%s" is in stable or testing; change prohibited.',
filename)
self.response.set_status(httplib.FORBIDDEN)
self.response.out.write('Changes to pkginfo not allowed')
gae_util.ReleaseLock(lock)
return
# If the update parameter asked for a careful update, by supplying
# a hash of the last known pkgsinfo, then compare the hash to help
# the client make a non destructive update.
if hash_str:
if self._Hash(pkginfo.plist) != hash_str:
self.response.set_status(httplib.CONFLICT)
self.response.out.write('Update hash does not match')
gae_util.ReleaseLock(lock)
return
# All verification has passed, so let's create the PackageInfo entity.
pkginfo.plist = mpl
pkginfo.name = mpl.GetPackageName()
if catalogs is not None:
pkginfo.catalogs = catalogs
if manifests is not None:
pkginfo.manifests = manifests
if install_types:
pkginfo.install_types = install_types
pkginfo.put()
gae_util.ReleaseLock(lock)
for track in pkginfo.catalogs:
models.Catalog.Generate(track, delay=1)
# Log admin pkginfo put to Datastore.
user = session.uuid
admin_log = models.AdminPackageLog(user=user,
action='pkginfo',
filename=filename,
catalogs=pkginfo.catalogs,
manifests=pkginfo.manifests,
install_types=pkginfo.install_types,
plist=pkginfo.plist.GetXml())
admin_log.put()
def _ChangeProduct(self, product_id):
"""Method to change properties of a given Apple SUS product."""
user = users.get_current_user()
track = self.request.get('track')
enabled = self.request.get('enabled', None)
manual_override = self.request.get('manual_override', None)
unattended = self.request.get('unattended', None)
force_install_after_date = self.request.get('force_install_after_date',
None)
product = models.AppleSUSProduct.get_by_key_name(product_id)
if not product:
self.response.set_status(httplib.NOT_FOUND)
return
data = {
'product_id': product_id,
}
changed_tracks = set()
# set/unset manual_override property
if manual_override is not None:
manual_override = bool(int(manual_override))
product.manual_override = manual_override
product.put()
log_action = 'manual_override=%s' % manual_override
for track in common.TRACKS:
if track not in product.tracks:
prom_date = applesus.GetAutoPromoteDate(track, product)
if prom_date:
data['%s_promote_date' %
track] = prom_date.strftime('%b. %d, %Y')
data['manual_override'] = manual_override
# set/unset force_install_after_date property
elif force_install_after_date is not None:
if force_install_after_date:
try:
tomorrow = datetime.datetime.utcnow() + datetime.timedelta(
hours=12)
if datetime.datetime.strptime( # only allow future force install date
force_install_after_date,
'%Y-%m-%d %H:%M') > tomorrow:
product.force_install_after_date_str = force_install_after_date
else:
self.error(httplib.BAD_REQUEST)
return
except ValueError:
self.error(httplib.BAD_REQUEST)
return
else:
product.force_install_after_date = None
product.put()
data['force_install_after_date'] = force_install_after_date
log_action = 'force_install_after_date=%s' % force_install_after_date
changed_tracks.update(product.tracks)
# set/unset unattended property
elif unattended is not None:
unattended = bool(int(unattended))
product.unattended = unattended
product.put()
data['unattended'] = unattended
log_action = 'unattended=%s' % unattended
changed_tracks.update(product.tracks)
# add/remove track to product
elif enabled is not None:
enabled = bool(int(enabled))
if enabled:
if track not in product.tracks:
product.tracks.append(track)
product.put()
else:
if track in product.tracks:
product.tracks.remove(track)
product.put()
log_action = '%s=%s' % (track, enabled)
data.update({'track': track, 'enabled': enabled})
changed_tracks.add(track)
log = models.AdminAppleSUSProductLog(product_id=product_id,
action=log_action,
tracks=product.tracks,
user=user.email())
log.put()
# Send email notification to admins
if mail and settings.EMAIL_ON_EVERY_CHANGE:
display_name = '%s - %s' % (product.name, product.version)
subject = 'Apple SUS Update by %s - %s (%s)' % (user, display_name,
product_id)
body = '%s has set \'%s\' on %s.\n' % (user, log_action,
display_name)
body += '%s is now in %s track(s).\n' % (product_id, ', '.join(
map(str, product.tracks)))
mail.SendMail(settings.EMAIL_ADMIN_LIST, subject, body)
# Regenerate catalogs for any changed tracks, if a task isn't already
# queued to do so.
for track in changed_tracks:
if gae_util.ObtainLock(applesus.CATALOG_REGENERATION_LOCK_NAME %
track):
deferred.defer(applesus.GenerateAppleSUSCatalogs,
track=track,
delay=180)
# TODO(user): add a visual cue to UI so admins know a generation is pending.
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(json.dumps(data))
def Update(self, **kwargs):
"""Updates properties and/or plist of an existing PackageInfo entity.
Omitted properties are left unmodified on the PackageInfo entity.
Args:
**kwargs: many, below:
catalogs: list, optional, a subset of common.TRACKS.
manifests: list, optional, a subset of common.TRACKS.
install_types: list, optional, a subset of common.INSTALL_TYPES.
manifest_mod_access: list, optional, subset of
common.MANIFEST_MOD_GROUPS.
name: str, optional, pkginfo name value.
display_name: str, optional, pkginfo display_name value.
unattended_install: boolean, optional, True to set unattended_install.
unattended_uninstall: boolean, optional, True to set
unattended_uninstall.
description: str, optional, pkginfo description.
version: str, optional, pkginfo version.
minimum_os_version: str, optional, pkginfo minimum_os_version value.
maximum_os_version: str, optional, pkginfo maximum_os_version value.
force_install_after_date: datetime, optional, pkginfo
force_install_after_date value.
Raises:
PackageInfoLockError: if the package is already locked in the datastore.
PackageInfoUpdateError: there were validation problems with the pkginfo.
"""
catalogs = kwargs.get('catalogs')
manifests = kwargs.get('manifests')
install_types = kwargs.get('install_types')
manifest_mod_access = kwargs.get('manifest_mod_access')
name = kwargs.get('name')
display_name = kwargs.get('display_name')
unattended_install = kwargs.get('unattended_install')
unattended_uninstall = kwargs.get('unattended_uninstall')
description = kwargs.get('description')
version = kwargs.get('version')
minimum_os_version = kwargs.get('minimum_os_version')
maximum_os_version = kwargs.get('maximum_os_version')
category = kwargs.get('category')
developer = kwargs.get('developer')
force_install_after_date = kwargs.get('force_install_after_date')
original_plist = self.plist.GetXml()
lock = 'pkgsinfo_%s' % self.filename
if not gae_util.ObtainLock(lock, timeout=5.0):
raise PackageInfoLockError
if self.IsSafeToModify():
if name is not None:
self.plist['name'] = name
self.name = name
if description is not None:
self.description = description
if 'display_name' in self.plist and display_name == '':
self.plist.RemoveDisplayName()
elif display_name != '' and display_name is not None:
self.plist.SetDisplayName(display_name)
if install_types is not None:
self.install_types = install_types
if manifest_mod_access is not None:
self.manifest_mod_access = manifest_mod_access
if version is not None:
self.plist['version'] = version
if minimum_os_version is not None:
if not minimum_os_version and 'minimum_os_version' in self.plist:
del self.plist['minimum_os_version']
elif minimum_os_version:
self.plist['minimum_os_version'] = minimum_os_version
if maximum_os_version is not None:
if not maximum_os_version and 'maximum_os_version' in self.plist:
del self.plist['maximum_os_version']
elif maximum_os_version:
self.plist['maximum_os_version'] = maximum_os_version
if force_install_after_date is not None:
if force_install_after_date:
self.plist['force_install_after_date'] = force_install_after_date
else:
if 'force_install_after_date' in self.plist:
del self.plist['force_install_after_date']
self.plist.SetUnattendedInstall(unattended_install)
self.plist.SetUnattendedUninstall(unattended_uninstall)
self.plist['category'] = category
self.plist['developer'] = developer
else:
# If not safe to modify, only catalogs/manifests can be changed.
for k, v in kwargs.iteritems():
if v and k not in ['catalogs', 'manifests']:
if self.approval_required:
failure_message = ('PackageInfo is not safe to modify;'
' please remove from catalogs first.')
else:
failure_message = ('PackageInfo is not safe to modify;'
' please move to unstable first.')
gae_util.ReleaseLock(lock)
raise PackageInfoUpdateError(failure_message)
original_catalogs = self.catalogs
if self.approval_required and (
catalogs != self.catalogs or manifests != self.manifests):
self.proposal.Propose(catalogs=catalogs, manifests=manifests)
else:
if catalogs is not None:
self.catalogs = catalogs
self.plist['catalogs'] = catalogs
if manifests is not None:
self.manifests = manifests
try:
self._PutAndLogPackageInfoUpdate(self, original_plist, original_catalogs)
except PackageInfoUpdateError:
gae_util.ReleaseLock(lock)
raise
gae_util.ReleaseLock(lock)
def UpdateFromPlist(cls, plist, create_new=False):
"""Updates a PackageInfo entity from a plist_lib.ApplePlist object or str.
Args:
plist: str or plist_lib.ApplePlist object.
create_new: bool, optional, default False. If True, create a new
PackageInfo entity, only otherwise update an existing one.
Returns:
pkginfo: Returns updated PackageInfo object.
log: Returns AdminPackageLog record.
Raises:
PackageInfoLockError: if the package is already locked in the datastore.
PackageInfoNotFoundError: if the filename is not a key in the datastore.
PackageInfoUpdateError: there were validation problems with the pkginfo.
"""
if isinstance(plist, basestring) or isinstance(plist, unicode):
plist = plist_lib.MunkiPackageInfoPlist(plist)
plist.EncodeXml()
try:
plist.Parse()
except plist_lib.PlistError as e:
raise PackageInfoUpdateError(
'plist_lib.PlistError parsing plist XML: %s' % str(e))
filename = plist['installer_item_location']
lock = 'pkgsinfo_%s' % filename
if not gae_util.ObtainLock(lock, timeout=5.0):
raise PackageInfoLockError('This PackageInfo is locked.')
if create_new:
if cls.get_by_key_name(filename):
gae_util.ReleaseLock(lock)
raise PackageInfoUpdateError(
'An existing pkginfo exists for: %s' % filename)
pkginfo = cls._New(filename)
pkginfo.filename = filename
# If we're uploading a new pkginfo plist, wipe out catalogs.
plist['catalogs'] = []
original_plist = None
else:
pkginfo = cls.get_by_key_name(filename)
if not pkginfo:
gae_util.ReleaseLock(lock)
raise PackageInfoNotFoundError('pkginfo not found: %s' % filename)
original_plist = pkginfo.plist.GetXml()
if not pkginfo.IsSafeToModify():
gae_util.ReleaseLock(lock)
raise PackageInfoUpdateError(
'PackageInfo is not safe to modify; move to unstable first.')
pkginfo.plist = plist
pkginfo.name = plist['name']
original_catalogs = pkginfo.catalogs
pkginfo.catalogs = plist['catalogs']
pkginfo.pkgdata_sha256 = plist['installer_item_hash']
try:
log = cls._PutAndLogPackageInfoUpdate(
pkginfo, original_plist, original_catalogs)
except PackageInfoUpdateError:
gae_util.ReleaseLock(lock)
raise
gae_util.ReleaseLock(lock)
return pkginfo, log
class PackageInfo(BaseMunkiModel):
"""Munki pkginfo file, Blobstore key, etc., for the corresponding package.
_plist contents are generated offline by Munki tools and uploaded by admins.
name is something like: Adobe Flash, Mozilla Firefox, MS Office, etc.
"""
PLIST_LIB_CLASS = plist_lib.MunkiPackageInfoPlist
AVG_DURATION_TEXT = (
'%d users have installed this with an average duration of %d seconds.')
AVG_DURATION_REGEX = re.compile(
'\d+ users have installed this with an average duration of \d+ seconds\.'
)
# catalog names this pkginfo belongs to; unstable, testing, stable.
catalogs = db.StringListProperty()
# manifest names this pkginfo belongs to; unstable, testing, stable.
manifests = db.StringListProperty()
# install types for this pkg; managed_installs, managed_uninstalls,
# managed_updates, etc.
install_types = db.StringListProperty()
# admin username that uploaded pkginfo.
user = db.StringProperty()
# filename for the package data
filename = db.StringProperty()
# key to Blobstore for package data.
blobstore_key = db.StringProperty()
# sha256 hash of package data
pkgdata_sha256 = db.StringProperty()
# munki name in the form of pkginfo '%s-%s' % (display_name, version)
# this property is automatically updated on put()
munki_name = db.StringProperty()
# datetime when the PackageInfo was initially created.
created = db.DateTimeProperty(auto_now_add=True)
# str group name(s) in common.MANIFEST_MOD_GROUPS that have access to inject
# this package into manifests.
manifest_mod_access = db.StringListProperty()
def _GetDescription(self):
"""Returns only admin portion of the desc, omitting avg duration text."""
desc = self.plist.get('description', None)
if desc:
match = self.AVG_DURATION_REGEX.search(desc)
if match:
avg_duration_text = match.group(0)
return desc.replace(avg_duration_text, '').strip()
return desc
def _SetDescription(self, desc):
"""Sets the description to the plist, preserving any avg duration text."""
if self.AVG_DURATION_REGEX.search(desc):
# If the new description has the avg duration text, just keep it all.
self.plist['description'] = desc
else:
# Otherwise append the old avg duration text to the new description.
match = self.AVG_DURATION_REGEX.search(
self.plist.get('description', ''))
if match:
self.plist['description'] = '%s\n\n%s' % (desc, match.group(0))
else:
self.plist['description'] = desc
# Update the plist property with the new description.
self.plist = self.plist.GetXml()
description = property(_GetDescription, _SetDescription)
def _GetBlobInfo(self):
"""Returns the blobstore.BlobInfo object for the PackageInfo."""
if not self.blobstore_key:
return None
return blobstore.BlobInfo.get(self.blobstore_key)
def _SetBlobInfo(self, blob_info):
"""Sets the blobstore_key property from a given blobstore.BlobInfo object.
This mimics the new blobstore.BlobReferenceProperty() without requiring
a schema change, which isn't fun for external Simian customers.
"""
self.blobstore_key = str(blob_info.key())
blob_info = property(_GetBlobInfo, _SetBlobInfo)
def IsSafeToModify(self):
"""Returns True if the pkginfo is modifiable, False otherwise."""
if common.STABLE in self.manifests:
return False
elif common.TESTING in self.manifests:
return False
return True
def MakeSafeToModify(self):
"""Modifies a PackageInfo such that it is safe to modify."""
self.Update(catalogs=[], manifests=[])
def put(self, *args, **kwargs):
"""Put to Datastore, generating and setting the "munki_name" property.
Args:
args: list, optional, args to superclass put()
kwargs: dict, optional, keyword args to superclass put()
Returns:
return value from superclass put()
Raises:
PackageInfoUpdateError: pkginfo property validation failed.
"""
# Ensure any defined manifests have matching catalogs.
for manifest in self.manifests:
if manifest not in self.catalogs:
raise PackageInfoUpdateError(
'manifest did not have matching catalog: %s' % manifest)
# Always update the munki_name property with the latest Pkg-<Version> name
# for backwards compatibility.
try:
self.munki_name = self.plist.GetMunkiName()
except plist_lib.PlistNotParsedError:
self.munki_name = None
return super(PackageInfo, self).put(*args, **kwargs)
def delete(self, *args, **kwargs):
"""Deletes a PackageInfo and cleans up associated data in other models.
Any Blobstore blob associated with the PackageInfo is deleted, and all
Catalogs the PackageInfo was a member of are regenerated.
Returns:
return value from superlass delete()
"""
ret = super(PackageInfo, self).delete(*args, **kwargs)
for catalog in self.catalogs:
Catalog.Generate(catalog, delay=5)
if self.blobstore_key:
gae_util.SafeBlobDel(self.blobstore_key)
return ret
def VerifyPackageIsEligibleForNewCatalogs(self, new_catalogs):
"""Ensure a package with the same name does not exist in the new catalogs.
Args:
new_catalogs: list of str catalogs to verify the package name is not in.
Raises:
PackageInfoUpdateError: a new catalog contains a pkg with the same name.
"""
for catalog in new_catalogs:
if self.name in Catalog.get_by_key_name(catalog).package_names:
raise PackageInfoUpdateError(
'%r already exists in %r catalog' % (self.name, catalog))
@classmethod
def _PutAndLogPackageInfoUpdate(cls, pkginfo, original_plist,
original_catalogs):
"""Helper method called by Update or UpdateFromPlist to put/log the update.
Args:
pkginfo: a PackageInfo entity ready to be put to Datastore.
original_plist: str XML of the original pkginfo plist, before updates.
original_catalogs: list of catalog names the pkg was previously in.
Raises:
PackageInfoUpdateError: there were validation problems with the pkginfo.
"""
new_catalogs = [
c for c in pkginfo.catalogs if c not in original_catalogs
]
pkginfo.VerifyPackageIsEligibleForNewCatalogs(new_catalogs)
pkginfo.put()
delay = 0
changed_catalogs = set(original_catalogs + pkginfo.catalogs)
for track in sorted(changed_catalogs, reverse=True):
delay += 5
Catalog.Generate(track, delay=delay)
# Log admin pkginfo put to Datastore.
user = users.get_current_user().email()
log = base.AdminPackageLog(
user=user,
action='pkginfo',
filename=pkginfo.filename,
catalogs=pkginfo.catalogs,
manifests=pkginfo.manifests,
original_plist=original_plist,
install_types=pkginfo.install_types,
manifest_mod_access=pkginfo.manifest_mod_access)
# The plist property is a py property of _plist, and therefore cannot be
# set in the constructure, so set here.
log.plist = pkginfo.plist
log.put()
@classmethod
def _New(cls, key_name):
"""Returns a new PackageInfo entity with a given key name.
Only needed for unit test stubbing purposes.
Args:
key_name: str, key name for the entity.
Returns:
PackageInfo object isntance.
"""
return cls(key_name=key_name)
@classmethod
def UpdateFromPlist(cls, plist, create_new=False):
"""Updates a PackageInfo entity from a plist_lib.ApplePlist object or str.
Args:
plist: str or plist_lib.ApplePlist object.
create_new: bool, optional, default False. If True, create a new
PackageInfo entity, only otherwise update an existing one.
Raises:
PackageInfoLockError: if the package is already locked in the datastore.
PackageInfoNotFoundError: if the filename is not a key in the datastore.
PackageInfoUpdateError: there were validation problems with the pkginfo.
"""
if type(plist) is str or type(plist) is unicode:
plist = plist_lib.MunkiPackageInfoPlist(plist)
plist.EncodeXml()
try:
plist.Parse()
except plist_lib.PlistError, e:
raise PackageInfoUpdateError(
'plist_lib.PlistError parsing plist XML: %s', str(e))
filename = plist['installer_item_location']
lock = 'pkgsinfo_%s' % filename
if not gae_util.ObtainLock(lock, timeout=5.0):
raise PackageInfoLockError('This PackageInfo is locked.')
if create_new:
if cls.get_by_key_name(filename):
gae_util.ReleaseLock(lock)
raise PackageInfoUpdateError(
'An existing pkginfo exists for: %s' % filename)
pkginfo = cls._New(filename)
pkginfo.filename = filename
# If we're uploading a new pkginfo plist, wipe out catalogs.
plist['catalogs'] = []
original_plist = None
else:
pkginfo = cls.get_by_key_name(filename)
if not pkginfo:
gae_util.ReleaseLock(lock)
raise PackageInfoNotFoundError('pkginfo not found: %s' %
filename)
original_plist = pkginfo.plist.GetXml()
if not pkginfo.IsSafeToModify():
gae_util.ReleaseLock(lock)
raise PackageInfoUpdateError(
'PackageInfo is not safe to modify; move to unstable first.')
pkginfo.plist = plist
pkginfo.name = plist['name']
original_catalogs = pkginfo.catalogs
pkginfo.catalogs = plist['catalogs']
pkginfo.pkgdata_sha256 = plist['installer_item_hash']
try:
cls._PutAndLogPackageInfoUpdate(pkginfo, original_plist,
original_catalogs)
except PackageInfoUpdateError:
gae_util.ReleaseLock(lock)
raise
gae_util.ReleaseLock(lock)
return pkginfo
def _GenerateInstallCounts():
"""Generates a dictionary of all installs names and the count of each."""
# Obtain a lock.
lock_name = 'pkgs_list_cron_lock'
lock = gae_util.ObtainLock(lock_name)
if not lock:
logging.warning('GenerateInstallCounts: lock found; exiting.')
return
# Get a list of all packages that have previously been pushed.
pkgs, unused_dt = models.ReportsCache.GetInstallCounts()
# Generate a query of all InstallLog entites that haven't been read yet.
query = models.InstallLog.all().order('server_datetime')
cursor_obj = models.KeyValueCache.get_by_key_name('pkgs_list_cursor')
if cursor_obj:
query.with_cursor(cursor_obj.text_value)
# Loop over new InstallLog entries.
try:
installs = query.fetch(1000)
except:
installs = None
if not installs:
models.ReportsCache.SetInstallCounts(pkgs)
gae_util.ReleaseLock(lock_name)
return
for install in installs:
pkg_name = install.package
if pkg_name not in pkgs:
pkgs[pkg_name] = {
'install_count': 0,
'install_fail_count': 0,
'applesus': install.applesus,
}
if install.IsSuccess():
pkgs[pkg_name]['install_count'] = (
pkgs[pkg_name].setdefault('install_count', 0) + 1)
# (re)calculate avg_duration_seconds for this package.
if 'duration_seconds_avg' not in pkgs[pkg_name]:
pkgs[pkg_name]['duration_count'] = 0
pkgs[pkg_name]['duration_total_seconds'] = 0
pkgs[pkg_name]['duration_seconds_avg'] = None
# only proceed if entity has "duration_seconds" property != None.
if getattr(install, 'duration_seconds', None) is not None:
pkgs[pkg_name]['duration_count'] += 1
pkgs[pkg_name]['duration_total_seconds'] += (
install.duration_seconds)
pkgs[pkg_name]['duration_seconds_avg'] = int(
pkgs[pkg_name]['duration_total_seconds'] /
pkgs[pkg_name]['duration_count'])
else:
pkgs[pkg_name]['install_fail_count'] = (
pkgs[pkg_name].setdefault('install_fail_count', 0) + 1)
# Update any changed packages.
models.ReportsCache.SetInstallCounts(pkgs)
if not cursor_obj:
cursor_obj = models.KeyValueCache(key_name='pkgs_list_cursor')
cursor_txt = str(query.cursor())
cursor_obj.text_value = cursor_txt
cursor_obj.put()
# Delete the lock.
gae_util.ReleaseLock(lock_name)
deferred.defer(_GenerateInstallCounts)
def _GenerateMsuUserSummary(self, since_days=None, now=None):
"""Generate summary of MSU user data.
Args:
since_days: int, optional, only report on the last x days
now: datetime.datetime, optional, supply an alternative
value for the current date/time
"""
lock_name = 'msu_user_summary_lock'
cursor_name = 'msu_user_summary_cursor'
if since_days is None:
since = None
else:
since = '%dD' % since_days
lock_name = '%s_%s' % (lock_name, since)
cursor_name = '%s_%s' % (cursor_name, since)
lock = gae_util.ObtainLock(lock_name)
if not lock:
logging.warning('GenerateMsuUserSummary lock found; exiting.')
return
interested_events = self.USER_EVENTS
lquery = models.ComputerMSULog.all()
cursor = models.KeyValueCache.MemcacheWrappedGet(
cursor_name, 'text_value')
summary, unused_dt = models.ReportsCache.GetMsuUserSummary(since=since,
tmp=True)
if cursor and summary:
lquery.with_cursor(cursor)
else:
summary = {}
for event in interested_events:
summary[event] = 0
summary['total_events'] = 0
summary['total_users'] = 0
summary['total_uuids'] = 0
models.ReportsCache.SetMsuUserSummary(summary,
since=since,
tmp=True)
begin = time.time()
if now is None:
now = datetime.datetime.utcnow()
while True:
reports = lquery.fetch(self.FETCH_LIMIT)
if not reports:
break
userdata = {}
last_user = None
last_user_cursor = None
prev_user_cursor = None
n = 0
for report in reports:
userdata.setdefault(report.user, {})
userdata[report.user].setdefault(report.uuid, {}).update(
{report.event: report.mtime})
if last_user != report.user:
last_user = report.user
prev_user_cursor = last_user_cursor
last_user_cursor = str(lquery.cursor())
n += 1
if n == self.FETCH_LIMIT:
# full fetch, might not have finished this user -- rewind
del userdata[last_user]
last_user_cursor = prev_user_cursor
for user in userdata:
events = 0
for uuid in userdata[user]:
if 'launched' not in userdata[user][uuid]:
continue
for event in userdata[user][uuid]:
if since_days is None or IsTimeDelta(
userdata[user][uuid][event], now,
days=since_days):
summary.setdefault(event, 0)
summary[event] += 1
summary['total_events'] += 1
events += 1
if events:
summary['total_uuids'] += 1
if events:
summary['total_users'] += 1
summary.setdefault('total_users_%d_events' % events, 0)
summary['total_users_%d_events' % events] += 1
lquery = models.ComputerMSULog.all()
lquery.with_cursor(last_user_cursor)
end = time.time()
if (end - begin) > RUNTIME_MAX_SECS:
break
if reports:
models.ReportsCache.SetMsuUserSummary(summary,
since=since,
tmp=True)
models.KeyValueCache.MemcacheWrappedSet(cursor_name, 'text_value',
last_user_cursor)
if since_days:
args = '/%d' % since_days
else:
args = ''
taskqueue.add(url='/cron/reports_cache/msu_user_summary%s' % args,
method='GET',
countdown=5)
else:
models.ReportsCache.SetMsuUserSummary(summary, since=since)
models.KeyValueCache.DeleteMemcacheWrap(cursor_name,
prop_name='text_value')
models.ReportsCache.DeleteMsuUserSummary(since=since, tmp=True)
gae_util.ReleaseLock(lock_name)
def get(self, filename=None):
"""GET
Args:
filename: string like Firefox-1.0.dmg
"""
auth_return = auth.DoAnyAuth()
if hasattr(auth_return, 'email'):
email = auth_return.email()
if not any((
auth.IsAdminUser(email),
auth.IsSupportUser(email),
)):
raise auth.IsAdminMismatch
if filename:
filename = urllib.unquote(filename)
hash_str = self.request.get('hash')
if hash_str:
lock = 'pkgsinfo_%s' % filename
if not gae_util.ObtainLock(lock, timeout=5.0):
self.response.set_status(httplib.FORBIDDEN)
self.response.out.write('Could not lock pkgsinfo')
return
pkginfo = models.PackageInfo.get_by_key_name(filename)
if pkginfo:
self.response.headers[
'Content-Type'] = 'text/xml; charset=utf-8'
if hash_str:
self.response.headers['X-Pkgsinfo-Hash'] = self._Hash(
pkginfo.plist)
self.response.out.write(pkginfo.plist)
else:
if hash_str:
gae_util.ReleaseLock(lock)
self.response.set_status(httplib.NOT_FOUND)
return
if hash_str:
gae_util.ReleaseLock(lock)
else:
query = models.PackageInfo.all()
filename = self.request.get('filename')
if filename:
query.filter('filename', filename)
install_types = self.request.get_all('install_types')
for install_type in install_types:
query.filter('install_types =', install_type)
catalogs = self.request.get_all('catalogs')
for catalog in catalogs:
query.filter('catalogs =', catalog)
pkgs = []
for p in query:
pkg = {}
for k in p.properties():
if k != '_plist':
pkg[k] = getattr(p, k)
pkgs.append(pkg)
self.response.out.write('<?xml version="1.0" encoding="UTF-8"?>\n')
self.response.out.write(plist.GetXmlStr(pkgs))
self.response.headers['Content-Type'] = 'text/xml; charset=utf-8'