def _IsPackageUploadNecessary(self, filename, upload_pkginfo):
"""Returns True if the package file should be uploaded.
This method helps the client decide whether to upload the entire package
and package info, or just new package info. It compares the sha256 hash
of the existing package on the server with the one of the package file
which would potentially be uploaded. If the existing package info is
not obtainable or not parseable the hash value cannot be compared, so
True is returned to force an upload.
Args:
filename: str, package filename
upload_pkginfo: str, new pkginfo to upload.
Returns:
True if the package file and package info should be uploaded
False if the package file is same, so just upload package info
"""
try:
cur_pkginfo = self.GetPackageInfo(filename)
except client.SimianServerError:
return True
pkginfo_plist = plist.MunkiPackageInfoPlist(cur_pkginfo)
try:
pkginfo_plist.Parse()
except plist.PlistError:
return True
pkginfo = pkginfo_plist.GetContents()
upload_pkginfo = plist.MunkiPackageInfoPlist(upload_pkginfo)
upload_pkginfo.Parse()
if 'installer_item_hash' in pkginfo:
cur_sha256_hash = pkginfo['installer_item_hash']
elif 'uninstaller_item_hash' in pkginfo:
cur_sha256_hash = pkginfo['uninstaller_item_hash']
else:
cur_sha256_hash = None
upload_pkginfo_dict = upload_pkginfo.GetContents()
if 'installer_item_size' in upload_pkginfo_dict:
new_sha256_hash = upload_pkginfo_dict['installer_item_hash']
else:
new_sha256_hash = upload_pkginfo_dict['uninstaller_item_hash']
return cur_sha256_hash != new_sha256_hash
def PackageInfoEditHook(self, pkginfo):
"""Allow an interactive user to edit package info before uploading.
Args:
pkginfo: plist.MunkiPackageInfoPlist
Returns:
True if the package info is acceptable and no changes are necessary
False if the user has given up editing the package info and wishes
to abort the process
a plist.MunkiPackageInfoPlist if a new package info is being returned
"""
orig_pkginfo = pkginfo
(fd, filename) = tempfile.mkstemp('.plist')
os.write(fd, pkginfo.GetXml())
os.close(fd)
edit = True
while edit:
self._RunEditor(filename)
try:
fd = open(filename, 'r')
pkginfo = plist.MunkiPackageInfoPlist(fd.read())
fd.close()
pkginfo.Parse()
break
except IOError:
edit = False
except plist.Error, e:
print 'Error parsing plist: %s' % str(e)
yn = raw_input(
'Resulting plist contains errors. Re-edit? [y/n] ')
edit = yn.upper() in ['YES', 'Y']
def testEqualIgnoringManifestsAndCatalogsFalse(self):
"""Tests EqualIgnoringManifestsAndCatalogs() false."""
pkginfo = plist.MunkiPackageInfoPlist()
pkginfo._plist = {
'manifests': ['stable', 'testing'],
'catalogs': ['stable', 'testing'],
'foo': True,
}
other = plist.MunkiPackageInfoPlist()
other._plist = {
'manifests': ['unstable'],
'catalogs': ['unstable'],
'foo': False
}
self.assertFalse(pkginfo.EqualIgnoringManifestsAndCatalogs(other))
def NotifyAdminsOfPackageChangeFromPlist(self, plist_xml):
"""Notifies admins of changes to packages."""
plist = plist_lib.MunkiPackageInfoPlist(plist_xml)
plist.EncodeXml()
try:
plist.Parse()
except plist_lib.PlistError, e:
raise models.PackageInfoUpdateError(
'plist_lib.PlistError parsing plist XML: %s', str(e))
def testEq(self):
"""Test __eq__."""
other = plist.MunkiPackageInfoPlist()
other._plist = {'foo': 1}
self.munki._plist = {'foo': 1}
self.assertFalse(id(other._plist) == id(self.munki._plist))
self.assertTrue(self.munki == other)
self.assertFalse(self.munki == {'foo': 1})
self.assertFalse(self.munki == self)
other._plist = {'foo': 2}
self.assertFalse(self.munki == other)
def NotifyAdminsOfPackageChangeFromPlist(self, plist_xml):
"""Notifies admins of changes to packages."""
plist = plist_lib.MunkiPackageInfoPlist(plist_xml)
plist.EncodeXml()
try:
plist.Parse()
except plist_lib.PlistError as e:
raise models.PackageInfoUpdateError(
'plist_lib.PlistError parsing plist XML: %s', str(e))
subject_line = 'MSU Package Update by %s - %s' % (
users.get_current_user(), plist['installer_item_location'])
main_body = str(plist.GetXml(indent_num=2))
if mail:
mail.SendMail(settings.EMAIL_ADMIN_LIST, subject_line, main_body)
def UpdateFromPlist(cls, plist, create_new=False):
"""Updates a PackageInfo entity from a plist_lib.ApplePlist object or str.
Args:
plist: str or plist_lib.ApplePlist object.
create_new: bool, optional, default False. If True, create a new
PackageInfo entity, only otherwise update an existing one.
Raises:
PackageInfoLockError: if the package is already locked in the datastore.
PackageInfoNotFoundError: if the filename is not a key in the datastore.
PackageInfoUpdateError: there were validation problems with the pkginfo.
"""
if type(plist) is str or type(plist) is unicode:
plist = plist_lib.MunkiPackageInfoPlist(plist)
plist.EncodeXml()
try:
plist.Parse()
except plist_lib.PlistError, e:
raise PackageInfoUpdateError(
'plist_lib.PlistError parsing plist XML: %s', str(e))
def _DisplayPackagesList(self):
"""Displays list of all installs/removals/etc."""
installs, counts_mtime = models.ReportsCache.GetInstallCounts()
pending, pending_mtime = models.ReportsCache.GetPendingCounts()
packages = []
for p in models.PackageInfo.all():
pl = plist.MunkiPackageInfoPlist(p.plist)
pl.Parse()
pkg = {}
pkg['count'] = installs.get(p.munki_name,
{}).get('install_count', 'N/A')
pkg['fail_count'] = installs.get(p.munki_name,
{}).get('install_fail_count',
'N/A')
pkg['pending_count'] = pending.get(p.munki_name, 'N/A')
pkg['duration_seconds_avg'] = installs.get(p.munki_name, {}).get(
'duration_seconds_avg', None) or 'N/A'
pkg['unattended'] = pl.get('unattended_install', False)
force_install_after_date = pl.get('force_install_after_date', None)
if force_install_after_date:
pkg['force_install_after_date'] = force_install_after_date
pkg['catalogs'] = p.catalogs
pkg['manifests'] = p.manifests
pkg['munki_name'] = p.munki_name or pl.GetMunkiName()
pkg['filename'] = p.filename
pkg['install_types'] = p.install_types
pkg['description'] = pl['description']
packages.append(pkg)
packages.sort(key=lambda pkg: pkg['munki_name'].lower())
self.response.out.write(
RenderTemplate(
'templates/stats_installs.html', {
'packages': packages,
'counts_mtime': counts_mtime,
'pending_mtime': pending_mtime
}))
def post(self):
"""POST method.
This method behaves a little strangely. BlobstoreUploadHandler
only allows returns statuses of 301, 302, 303 (not even 200), so
one must redirect away to return more information to the caller.
Parameters:
file: package file contents
pkginfo: packageinfo file contents
name: filename of package e.g. 'Firefox-1.0.dmg'
"""
# Only blobstore/upload service/scotty requests should be
# invoking this handler.
if not handlers.IsBlobstore():
logging.critical('POST to /uploadpkg not from Blobstore: %s',
self.request.headers)
self.redirect('/')
gaeserver.DoMunkiAuth(require_level=gaeserver.LEVEL_UPLOADPKG)
user = self.request.get('user')
filename = self.request.get('name')
install_types = self.request.get('install_types')
catalogs = self.request.get('catalogs', None)
manifests = self.request.get('manifests', None)
if catalogs is None or not install_types or not user or not filename:
msg = 'uploadpkg POST required parameters missing'
logging.error(msg)
self.redirect('/uploadpkg?mode=error&msg=%s' % msg)
return
if catalogs == '':
catalogs = []
else:
catalogs = catalogs.split(',')
if manifests in ['', None]:
manifests = []
else:
manifests = manifests.split(',')
install_types = install_types.split(',')
upload_files = self.get_uploads('file')
upload_pkginfo_files = self.get_uploads('pkginfo')
if not len(upload_pkginfo_files) and not self.request.get('pkginfo'):
self.redirect('/uploadpkg?mode=error&msg=No%20file%20received')
return
if len(upload_pkginfo_files):
# obtain the pkginfo from a blob, and then throw it away. this is
# a necessary hack because the upload handler grabbed it, but we don't
# intend to keep it in blobstore.
pkginfo_str = gae_util.GetBlobAndDel(upload_pkginfo_files[0].key())
else:
# otherwise, grab the form parameter.
pkginfo_str = self.request.get('pkginfo')
blob_info = upload_files[0]
blobstore_key = str(blob_info.key())
# Parse, validate, and encode the pkginfo plist.
plist = plist_lib.MunkiPackageInfoPlist(pkginfo_str)
try:
plist.Parse()
except plist_lib.PlistError:
logging.exception('Invalid pkginfo plist uploaded:\n%s\n',
pkginfo_str)
gae_util.SafeBlobDel(blobstore_key)
self.redirect(
'/uploadpkg?mode=error&msg=No%20valid%20pkginfo%20received')
return
filename = plist['installer_item_location']
pkgdata_sha256 = plist['installer_item_hash']
# verify the blob was actually written; in case Blobstore failed to write
# the blob but still POSTed to this handler (very, very rare).
blob_info = blobstore.BlobInfo.get(blobstore_key)
if not blob_info:
logging.critical(
'Blobstore returned a key for %s that does not exist: %s',
filename, blobstore_key)
self.redirect('/uploadpkg?mode=error&msg=Blobstore%20failure')
return
# Obtain a lock on the PackageInfo entity for this package.
lock = 'pkgsinfo_%s' % filename
if not gae_util.ObtainLock(lock, timeout=5.0):
gae_util.SafeBlobDel(blobstore_key)
self.redirect(
'/uploadpkg?mode=error&msg=Could%20not%20lock%20pkgsinfo')
return
old_blobstore_key = None
pkg = models.PackageInfo.get_or_insert(filename)
if not pkg.IsSafeToModify():
gae_util.ReleaseLock(lock)
gae_util.SafeBlobDel(blobstore_key)
self.redirect(
'/uploadpkg?mode=error&msg=Package%20is%20not%20modifiable')
return
if pkg.blobstore_key:
# a previous blob exists. delete it when the update has succeeded.
old_blobstore_key = pkg.blobstore_key
pkg.blobstore_key = blobstore_key
pkg.name = plist.GetPackageName()
pkg.filename = filename
pkg.user = user
pkg.catalogs = catalogs
pkg.manifests = manifests
pkg.install_types = install_types
pkg.plist = plist
pkg.pkgdata_sha256 = pkgdata_sha256
# update the PackageInfo model with the new plist string and blobstore key.
try:
pkg.put()
success = True
except db.Error:
logging.exception('error on PackageInfo.put()')
success = False
# if it failed, delete the blob that was just uploaded -- it's
# an orphan.
if not success:
gae_util.SafeBlobDel(blobstore_key)
# if this is a new entity (get_or_insert puts), attempt to delete it.
if not old_blobstore_key:
gae_util.SafeEntityDel(pkg)
gae_util.ReleaseLock(lock)
self.redirect('/uploadpkg?mode=error')
return
# if an old blob was associated with this Package, delete it.
# the new blob that was just uploaded has replaced it.
if old_blobstore_key:
gae_util.SafeBlobDel(old_blobstore_key)
gae_util.ReleaseLock(lock)
# Generate catalogs for newly uploaded pkginfo plist.
for catalog in pkg.catalogs:
models.Catalog.Generate(catalog, delay=1)
# Log admin upload to Datastore.
admin_log = models.AdminPackageLog(user=user,
action='uploadpkg',
filename=filename,
catalogs=catalogs,
manifests=manifests,
install_types=install_types,
plist=pkg.plist.GetXml())
admin_log.put()
self.redirect('/uploadpkg?mode=success&key=%s' % blobstore_key)
def setUp(self):
mox.MoxTestBase.setUp(self)
self.stubs = stubout.StubOutForTesting()
self.munki = plist.MunkiPackageInfoPlist()
def UpdateFromPlist(cls, plist, create_new=False):
"""Updates a PackageInfo entity from a plist_lib.ApplePlist object or str.
Args:
plist: str or plist_lib.ApplePlist object.
create_new: bool, optional, default False. If True, create a new
PackageInfo entity, only otherwise update an existing one.
Returns:
pkginfo: Returns updated PackageInfo object.
log: Returns AdminPackageLog record.
Raises:
PackageInfoLockError: if the package is already locked in the datastore.
PackageInfoNotFoundError: if the filename is not a key in the datastore.
PackageInfoUpdateError: there were validation problems with the pkginfo.
"""
if isinstance(plist, basestring) or isinstance(plist, unicode):
plist = plist_lib.MunkiPackageInfoPlist(plist)
plist.EncodeXml()
try:
plist.Parse()
except plist_lib.PlistError as e:
raise PackageInfoUpdateError(
'plist_lib.PlistError parsing plist XML: %s' % str(e))
filename = plist['installer_item_location']
lock = GetLockForPackage(filename)
try:
lock.Acquire(timeout=600, max_acquire_attempts=5)
except datastore_locks.AcquireLockError:
raise PackageInfoLockError('This PackageInfo is locked.')
if create_new:
if cls.get_by_key_name(filename):
lock.Release()
raise PackageInfoUpdateError(
'An existing pkginfo exists for: %s' % filename)
pkginfo = cls._New(filename)
pkginfo.filename = filename
# If we're uploading a new pkginfo plist, wipe out catalogs.
plist['catalogs'] = []
original_plist = None
else:
pkginfo = cls.get_by_key_name(filename)
if not pkginfo:
lock.Release()
raise PackageInfoNotFoundError('pkginfo not found: %s' % filename)
original_plist = pkginfo.plist.GetXml()
if not pkginfo.IsSafeToModify():
lock.Release()
raise PackageInfoUpdateError(
'PackageInfo is not safe to modify; move to unstable first.')
pkginfo.plist = plist
pkginfo.name = plist['name']
original_catalogs = pkginfo.catalogs
pkginfo.catalogs = plist['catalogs']
pkginfo.pkgdata_sha256 = plist['installer_item_hash']
try:
log = cls._PutAndLogPackageInfoUpdate(
pkginfo, original_plist, original_catalogs)
except PackageInfoUpdateError:
lock.Release()
raise
lock.Release()
return pkginfo, log
class MunkiPackageInfo(object):
"""Class which handles Munki package info."""
REQUIRED_MUNKI_BINS = [MAKEPKGINFO]
def __init__(self):
self.filename = None
self.plist = None
self.munki_path = MUNKI_PATH
self.munki_install_verified = False
def IsOSX(self):
"""Returns True if the current OS is OS X, False if not."""
return os.uname()[0] == 'Darwin'
def _GetMunkiPath(self, filename):
"""Given a filename, return a full path including leading Munki path.
Args:
filename: str, like 'munkitool'
Returns:
str, like '/usr/local/munki/munkitool'
"""
return os.path.join(self.munki_path, filename)
def VerifyMunkiInstall(self):
"""Verify that Munki is installed and accessible on this system.
Raises:
MunkiInstallError: if a problem is detected
"""
if self.munki_install_verified:
return
for k in os.environ:
if k.startswith('PKGS_MUNKI_'):
return
if not self.IsOSX():
raise MunkiInstallError('Required Munki utilities only run on OS X')
if not os.path.isdir(self.munki_path):
raise MunkiInstallError(
'%s does not exist or is not a directory' % self.munki_path)
for f in self.REQUIRED_MUNKI_BINS:
if not os.path.isfile(self._GetMunkiPath(f)):
raise MunkiInstallError(
'%s/%s does not exist or is not a file' % self._GetMunkiPath(f))
self.munki_install_verified = True
def CreateFromPackage(self, filename, description, display_name, catalogs):
"""Create package info from a live package stored at filename.
Args:
filename: str
description: str, like "Security update for Foo Software"
display_name: str, like "Munki Client"
catalogs: list of str catalog names.
"""
self.VerifyMunkiInstall()
args = [self._GetMunkiPath(MAKEPKGINFO), filename]
args.append('--description=%s' % description)
if display_name:
args.append('--displayname=%s' % display_name)
for catalog in catalogs:
args.append('--catalog=%s' % catalog)
if 'PKGS_MUNKI_MAKEPKGINFO' in os.environ:
args[0] = os.environ['PKGS_MUNKI_MAKEPKGINFO']
try:
p = subprocess.Popen(
args,
stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
close_fds=True,
shell=False)
except OSError, e:
raise MunkiInstallError('Cannot execute %s: %s' % (' '.join(args), e))
(stdout, stderr) = p.communicate(None)
status = p.poll()
if status == 0 and stdout and not stderr:
self.filename = filename
else:
raise MunkiError(
'makepkginfo: exit status %d, stderr=%s' % (status, stderr))
self.plist = plist.MunkiPackageInfoPlist(stdout)
try:
self.plist.Parse()
except plist.Error, e:
raise Error(str(e))
def EditPackageInfo(self):
"""Edit package info on a package already on Simian.
Raises:
CliError: if package info is malformed
"""
(filepath, description, display_name, pkginfo_name, manifests,
catalogs, install_types, unattended_install,
unattended_uninstall) = (self.ValidatePackageConfig(defaults=False))
print 'Editing package info ...'
filename = os.path.basename(filepath)
(sha256_hash, pkginfo_xml) = self.client.GetPackageInfo(filename,
get_hash=True)
pkginfo = plist.MunkiPackageInfoPlist(pkginfo_xml)
pkginfo.Parse()
kwargs = {}
changes = []
# these values would be None if the user did not specify a value.
# note the change detection is a little weak, if we overwrite any
# value, even with the same values, we count it as a change.
if description is not None:
pkginfo.SetDescription(description)
changes.append('Description: %s' % description)
if display_name is not None:
pkginfo.SetDisplayName(display_name)
changes.append('Display name: %s' % display_name)
if pkginfo_name is not None:
pkginfo['name'] = pkginfo_name
changes.append('Pkginfo name: %s' % pkginfo_name)
if unattended_install is not None:
pkginfo.SetUnattendedInstall(unattended_install)
changes.append('Unattended install: %s' % unattended_install)
if unattended_uninstall is not None:
pkginfo.SetUnattendedUninstall(unattended_uninstall)
changes.append('Unattended uninstall: %s' % unattended_uninstall)
if catalogs is not None:
pkginfo.SetCatalogs(catalogs)
kwargs['catalogs'] = catalogs
changes.append('Catalogs: %s' % catalogs)
if manifests is not None:
kwargs['manifests'] = manifests
changes.append('Manifests: %s' % manifests)
if install_types is not None:
kwargs['install_types'] = install_types
changes.append('Install types: %s' % install_types)
edit_hooks = []
if self.config['template_pkginfo'] is not None:
edit_hooks.append(self.PackageInfoTemplateHook)
if self.config['edit_pkginfo'] is not None:
edit_hooks.append(self.PackageInfoEditHook)
# TODO(user): Refactor so that this code block and
# mac.client.UploadMunkiPackage share the same pkginfo_hooks iteration
# code.
for edit_hook in edit_hooks:
if edit_hook is not None:
new_pkginfo = edit_hook(pkginfo)
if new_pkginfo:
if new_pkginfo is not True:
pkginfo = new_pkginfo # changed and valid
pkginfo.SetChanged()
# populate catalogs value back into our properties
catalogs = pkginfo.GetContents().get('catalogs', [])
kwargs['catalogs'] = catalogs
else:
pass # no change at all
else:
raise CliError(
'Invalid package info') # changed but invalid now
if not kwargs and not pkginfo.HasChanged():
print 'Package info unchanged.'
return
# verify catalogs / manifests relationship
if manifests is not None:
intended_catalogs = (kwargs.get('catalogs', None)
or pkginfo.GetContents().get('catalogs', []))
for manifest in manifests:
if manifest not in intended_catalogs:
raise CliError('Manifest %s not in catalogs' % manifest)
# Parse pkginfo.GetXml() to ensure it's still valid before uploading.
new_xml = pkginfo.GetXml()
try:
tmp_pkginfo = plist.MunkiPackageInfoPlist(new_xml)
tmp_pkginfo.Parse()
except plist.Error, e:
raise CliError('Invalid package info: %s', str(e))
def post(self):
"""POST
This method behaves a little strangely. BlobstoreUploadHandler
only allows returns statuses of 301, 302, 303 (not even 200), so
one must redirect away to return more information to the caller.
Parameters:
file: package file contents
pkginfo: packageinfo file contents
name: filename of package e.g. 'Firefox-1.0.dmg'
"""
# Only blobstore/upload service/scotty requests should be
# invoking this handler.
if not handlers.IsBlobstore():
logging.critical('POST to /uploadpkg not from Blobstore: %s',
self.request.headers)
self.redirect('/')
gaeserver.DoMunkiAuth(require_level=gaeserver.LEVEL_UPLOADPKG)
user = self.request.get('user')
filename = self.request.get('name')
install_types = self.request.get('install_types')
catalogs = self.request.get('catalogs', None)
manifests = self.request.get('manifests', None)
if catalogs is None or not install_types or not user or not filename:
msg = 'uploadpkg POST required parameters missing'
logging.error(msg)
self.redirect('/uploadpkg?mode=error&msg=%s' % msg)
return
if catalogs == '':
catalogs = []
else:
catalogs = catalogs.split(',')
if manifests in ['', None]:
manifests = []
else:
manifests = manifests.split(',')
install_types = install_types.split(',')
upload_files = self.get_uploads('file')
upload_pkginfo_files = self.get_uploads('pkginfo')
if not len(upload_pkginfo_files) and not self.request.get('pkginfo'):
self.redirect('/uploadpkg?mode=error&msg=No%20file%20received')
return
if len(upload_pkginfo_files):
# obtain the pkginfo from a blob, and then throw it away. this is
# a necessary hack because the upload handler grabbed it, but we don't
# intend to keep it in blobstore.
pkginfo_str = gae_util.GetBlobAndDel(upload_pkginfo_files[0].key())
else:
# otherwise, grab the form parameter.
pkginfo_str = self.request.get('pkginfo')
blob_info = upload_files[0]
blobstore_key = str(blob_info.key())
# Parse, validate, and encode the pkginfo plist.
plist = plist_lib.MunkiPackageInfoPlist(pkginfo_str)
try:
plist.Parse()
except plist_lib.PlistError, e:
logging.exception('Invalid pkginfo plist uploaded:\n%s\n',
pkginfo_str)
gae_util.SafeBlobDel(blobstore_key)
self.redirect(
'/uploadpkg?mode=error&msg=No%20valid%20pkginfo%20received')
return
