def _IsPackageUploadNecessary(self, filename, upload_pkginfo): """Returns True if the package file should be uploaded. This method helps the client decide whether to upload the entire package and package info, or just new package info. It compares the sha256 hash of the existing package on the server with the one of the package file which would potentially be uploaded. If the existing package info is not obtainable or not parseable the hash value cannot be compared, so True is returned to force an upload. Args: filename: str, package filename upload_pkginfo: str, new pkginfo to upload. Returns: True if the package file and package info should be uploaded False if the package file is same, so just upload package info """ try: cur_pkginfo = self.GetPackageInfo(filename) except client.SimianServerError: return True pkginfo_plist = plist.MunkiPackageInfoPlist(cur_pkginfo) try: pkginfo_plist.Parse() except plist.PlistError: return True pkginfo = pkginfo_plist.GetContents() upload_pkginfo = plist.MunkiPackageInfoPlist(upload_pkginfo) upload_pkginfo.Parse() if 'installer_item_hash' in pkginfo: cur_sha256_hash = pkginfo['installer_item_hash'] elif 'uninstaller_item_hash' in pkginfo: cur_sha256_hash = pkginfo['uninstaller_item_hash'] else: cur_sha256_hash = None upload_pkginfo_dict = upload_pkginfo.GetContents() if 'installer_item_size' in upload_pkginfo_dict: new_sha256_hash = upload_pkginfo_dict['installer_item_hash'] else: new_sha256_hash = upload_pkginfo_dict['uninstaller_item_hash'] return cur_sha256_hash != new_sha256_hash
def PackageInfoEditHook(self, pkginfo): """Allow an interactive user to edit package info before uploading. Args: pkginfo: plist.MunkiPackageInfoPlist Returns: True if the package info is acceptable and no changes are necessary False if the user has given up editing the package info and wishes to abort the process a plist.MunkiPackageInfoPlist if a new package info is being returned """ orig_pkginfo = pkginfo (fd, filename) = tempfile.mkstemp('.plist') os.write(fd, pkginfo.GetXml()) os.close(fd) edit = True while edit: self._RunEditor(filename) try: fd = open(filename, 'r') pkginfo = plist.MunkiPackageInfoPlist(fd.read()) fd.close() pkginfo.Parse() break except IOError: edit = False except plist.Error, e: print 'Error parsing plist: %s' % str(e) yn = raw_input( 'Resulting plist contains errors. Re-edit? [y/n] ') edit = yn.upper() in ['YES', 'Y']
def testEqualIgnoringManifestsAndCatalogsFalse(self): """Tests EqualIgnoringManifestsAndCatalogs() false.""" pkginfo = plist.MunkiPackageInfoPlist() pkginfo._plist = { 'manifests': ['stable', 'testing'], 'catalogs': ['stable', 'testing'], 'foo': True, } other = plist.MunkiPackageInfoPlist() other._plist = { 'manifests': ['unstable'], 'catalogs': ['unstable'], 'foo': False } self.assertFalse(pkginfo.EqualIgnoringManifestsAndCatalogs(other))
def NotifyAdminsOfPackageChangeFromPlist(self, plist_xml): """Notifies admins of changes to packages.""" plist = plist_lib.MunkiPackageInfoPlist(plist_xml) plist.EncodeXml() try: plist.Parse() except plist_lib.PlistError, e: raise models.PackageInfoUpdateError( 'plist_lib.PlistError parsing plist XML: %s', str(e))
def testEq(self): """Test __eq__.""" other = plist.MunkiPackageInfoPlist() other._plist = {'foo': 1} self.munki._plist = {'foo': 1} self.assertFalse(id(other._plist) == id(self.munki._plist)) self.assertTrue(self.munki == other) self.assertFalse(self.munki == {'foo': 1}) self.assertFalse(self.munki == self) other._plist = {'foo': 2} self.assertFalse(self.munki == other)
def NotifyAdminsOfPackageChangeFromPlist(self, plist_xml): """Notifies admins of changes to packages.""" plist = plist_lib.MunkiPackageInfoPlist(plist_xml) plist.EncodeXml() try: plist.Parse() except plist_lib.PlistError as e: raise models.PackageInfoUpdateError( 'plist_lib.PlistError parsing plist XML: %s', str(e)) subject_line = 'MSU Package Update by %s - %s' % ( users.get_current_user(), plist['installer_item_location']) main_body = str(plist.GetXml(indent_num=2)) if mail: mail.SendMail(settings.EMAIL_ADMIN_LIST, subject_line, main_body)
def UpdateFromPlist(cls, plist, create_new=False): """Updates a PackageInfo entity from a plist_lib.ApplePlist object or str. Args: plist: str or plist_lib.ApplePlist object. create_new: bool, optional, default False. If True, create a new PackageInfo entity, only otherwise update an existing one. Raises: PackageInfoLockError: if the package is already locked in the datastore. PackageInfoNotFoundError: if the filename is not a key in the datastore. PackageInfoUpdateError: there were validation problems with the pkginfo. """ if type(plist) is str or type(plist) is unicode: plist = plist_lib.MunkiPackageInfoPlist(plist) plist.EncodeXml() try: plist.Parse() except plist_lib.PlistError, e: raise PackageInfoUpdateError( 'plist_lib.PlistError parsing plist XML: %s', str(e))
def _DisplayPackagesList(self): """Displays list of all installs/removals/etc.""" installs, counts_mtime = models.ReportsCache.GetInstallCounts() pending, pending_mtime = models.ReportsCache.GetPendingCounts() packages = [] for p in models.PackageInfo.all(): pl = plist.MunkiPackageInfoPlist(p.plist) pl.Parse() pkg = {} pkg['count'] = installs.get(p.munki_name, {}).get('install_count', 'N/A') pkg['fail_count'] = installs.get(p.munki_name, {}).get('install_fail_count', 'N/A') pkg['pending_count'] = pending.get(p.munki_name, 'N/A') pkg['duration_seconds_avg'] = installs.get(p.munki_name, {}).get( 'duration_seconds_avg', None) or 'N/A' pkg['unattended'] = pl.get('unattended_install', False) force_install_after_date = pl.get('force_install_after_date', None) if force_install_after_date: pkg['force_install_after_date'] = force_install_after_date pkg['catalogs'] = p.catalogs pkg['manifests'] = p.manifests pkg['munki_name'] = p.munki_name or pl.GetMunkiName() pkg['filename'] = p.filename pkg['install_types'] = p.install_types pkg['description'] = pl['description'] packages.append(pkg) packages.sort(key=lambda pkg: pkg['munki_name'].lower()) self.response.out.write( RenderTemplate( 'templates/stats_installs.html', { 'packages': packages, 'counts_mtime': counts_mtime, 'pending_mtime': pending_mtime }))
def post(self): """POST method. This method behaves a little strangely. BlobstoreUploadHandler only allows returns statuses of 301, 302, 303 (not even 200), so one must redirect away to return more information to the caller. Parameters: file: package file contents pkginfo: packageinfo file contents name: filename of package e.g. 'Firefox-1.0.dmg' """ # Only blobstore/upload service/scotty requests should be # invoking this handler. if not handlers.IsBlobstore(): logging.critical('POST to /uploadpkg not from Blobstore: %s', self.request.headers) self.redirect('/') gaeserver.DoMunkiAuth(require_level=gaeserver.LEVEL_UPLOADPKG) user = self.request.get('user') filename = self.request.get('name') install_types = self.request.get('install_types') catalogs = self.request.get('catalogs', None) manifests = self.request.get('manifests', None) if catalogs is None or not install_types or not user or not filename: msg = 'uploadpkg POST required parameters missing' logging.error(msg) self.redirect('/uploadpkg?mode=error&msg=%s' % msg) return if catalogs == '': catalogs = [] else: catalogs = catalogs.split(',') if manifests in ['', None]: manifests = [] else: manifests = manifests.split(',') install_types = install_types.split(',') upload_files = self.get_uploads('file') upload_pkginfo_files = self.get_uploads('pkginfo') if not len(upload_pkginfo_files) and not self.request.get('pkginfo'): self.redirect('/uploadpkg?mode=error&msg=No%20file%20received') return if len(upload_pkginfo_files): # obtain the pkginfo from a blob, and then throw it away. this is # a necessary hack because the upload handler grabbed it, but we don't # intend to keep it in blobstore. pkginfo_str = gae_util.GetBlobAndDel(upload_pkginfo_files[0].key()) else: # otherwise, grab the form parameter. pkginfo_str = self.request.get('pkginfo') blob_info = upload_files[0] blobstore_key = str(blob_info.key()) # Parse, validate, and encode the pkginfo plist. plist = plist_lib.MunkiPackageInfoPlist(pkginfo_str) try: plist.Parse() except plist_lib.PlistError: logging.exception('Invalid pkginfo plist uploaded:\n%s\n', pkginfo_str) gae_util.SafeBlobDel(blobstore_key) self.redirect( '/uploadpkg?mode=error&msg=No%20valid%20pkginfo%20received') return filename = plist['installer_item_location'] pkgdata_sha256 = plist['installer_item_hash'] # verify the blob was actually written; in case Blobstore failed to write # the blob but still POSTed to this handler (very, very rare). blob_info = blobstore.BlobInfo.get(blobstore_key) if not blob_info: logging.critical( 'Blobstore returned a key for %s that does not exist: %s', filename, blobstore_key) self.redirect('/uploadpkg?mode=error&msg=Blobstore%20failure') return # Obtain a lock on the PackageInfo entity for this package. lock = 'pkgsinfo_%s' % filename if not gae_util.ObtainLock(lock, timeout=5.0): gae_util.SafeBlobDel(blobstore_key) self.redirect( '/uploadpkg?mode=error&msg=Could%20not%20lock%20pkgsinfo') return old_blobstore_key = None pkg = models.PackageInfo.get_or_insert(filename) if not pkg.IsSafeToModify(): gae_util.ReleaseLock(lock) gae_util.SafeBlobDel(blobstore_key) self.redirect( '/uploadpkg?mode=error&msg=Package%20is%20not%20modifiable') return if pkg.blobstore_key: # a previous blob exists. delete it when the update has succeeded. old_blobstore_key = pkg.blobstore_key pkg.blobstore_key = blobstore_key pkg.name = plist.GetPackageName() pkg.filename = filename pkg.user = user pkg.catalogs = catalogs pkg.manifests = manifests pkg.install_types = install_types pkg.plist = plist pkg.pkgdata_sha256 = pkgdata_sha256 # update the PackageInfo model with the new plist string and blobstore key. try: pkg.put() success = True except db.Error: logging.exception('error on PackageInfo.put()') success = False # if it failed, delete the blob that was just uploaded -- it's # an orphan. if not success: gae_util.SafeBlobDel(blobstore_key) # if this is a new entity (get_or_insert puts), attempt to delete it. if not old_blobstore_key: gae_util.SafeEntityDel(pkg) gae_util.ReleaseLock(lock) self.redirect('/uploadpkg?mode=error') return # if an old blob was associated with this Package, delete it. # the new blob that was just uploaded has replaced it. if old_blobstore_key: gae_util.SafeBlobDel(old_blobstore_key) gae_util.ReleaseLock(lock) # Generate catalogs for newly uploaded pkginfo plist. for catalog in pkg.catalogs: models.Catalog.Generate(catalog, delay=1) # Log admin upload to Datastore. admin_log = models.AdminPackageLog(user=user, action='uploadpkg', filename=filename, catalogs=catalogs, manifests=manifests, install_types=install_types, plist=pkg.plist.GetXml()) admin_log.put() self.redirect('/uploadpkg?mode=success&key=%s' % blobstore_key)
def setUp(self): mox.MoxTestBase.setUp(self) self.stubs = stubout.StubOutForTesting() self.munki = plist.MunkiPackageInfoPlist()
def UpdateFromPlist(cls, plist, create_new=False): """Updates a PackageInfo entity from a plist_lib.ApplePlist object or str. Args: plist: str or plist_lib.ApplePlist object. create_new: bool, optional, default False. If True, create a new PackageInfo entity, only otherwise update an existing one. Returns: pkginfo: Returns updated PackageInfo object. log: Returns AdminPackageLog record. Raises: PackageInfoLockError: if the package is already locked in the datastore. PackageInfoNotFoundError: if the filename is not a key in the datastore. PackageInfoUpdateError: there were validation problems with the pkginfo. """ if isinstance(plist, basestring) or isinstance(plist, unicode): plist = plist_lib.MunkiPackageInfoPlist(plist) plist.EncodeXml() try: plist.Parse() except plist_lib.PlistError as e: raise PackageInfoUpdateError( 'plist_lib.PlistError parsing plist XML: %s' % str(e)) filename = plist['installer_item_location'] lock = GetLockForPackage(filename) try: lock.Acquire(timeout=600, max_acquire_attempts=5) except datastore_locks.AcquireLockError: raise PackageInfoLockError('This PackageInfo is locked.') if create_new: if cls.get_by_key_name(filename): lock.Release() raise PackageInfoUpdateError( 'An existing pkginfo exists for: %s' % filename) pkginfo = cls._New(filename) pkginfo.filename = filename # If we're uploading a new pkginfo plist, wipe out catalogs. plist['catalogs'] = [] original_plist = None else: pkginfo = cls.get_by_key_name(filename) if not pkginfo: lock.Release() raise PackageInfoNotFoundError('pkginfo not found: %s' % filename) original_plist = pkginfo.plist.GetXml() if not pkginfo.IsSafeToModify(): lock.Release() raise PackageInfoUpdateError( 'PackageInfo is not safe to modify; move to unstable first.') pkginfo.plist = plist pkginfo.name = plist['name'] original_catalogs = pkginfo.catalogs pkginfo.catalogs = plist['catalogs'] pkginfo.pkgdata_sha256 = plist['installer_item_hash'] try: log = cls._PutAndLogPackageInfoUpdate( pkginfo, original_plist, original_catalogs) except PackageInfoUpdateError: lock.Release() raise lock.Release() return pkginfo, log
class MunkiPackageInfo(object): """Class which handles Munki package info.""" REQUIRED_MUNKI_BINS = [MAKEPKGINFO] def __init__(self): self.filename = None self.plist = None self.munki_path = MUNKI_PATH self.munki_install_verified = False def IsOSX(self): """Returns True if the current OS is OS X, False if not.""" return os.uname()[0] == 'Darwin' def _GetMunkiPath(self, filename): """Given a filename, return a full path including leading Munki path. Args: filename: str, like 'munkitool' Returns: str, like '/usr/local/munki/munkitool' """ return os.path.join(self.munki_path, filename) def VerifyMunkiInstall(self): """Verify that Munki is installed and accessible on this system. Raises: MunkiInstallError: if a problem is detected """ if self.munki_install_verified: return for k in os.environ: if k.startswith('PKGS_MUNKI_'): return if not self.IsOSX(): raise MunkiInstallError('Required Munki utilities only run on OS X') if not os.path.isdir(self.munki_path): raise MunkiInstallError( '%s does not exist or is not a directory' % self.munki_path) for f in self.REQUIRED_MUNKI_BINS: if not os.path.isfile(self._GetMunkiPath(f)): raise MunkiInstallError( '%s/%s does not exist or is not a file' % self._GetMunkiPath(f)) self.munki_install_verified = True def CreateFromPackage(self, filename, description, display_name, catalogs): """Create package info from a live package stored at filename. Args: filename: str description: str, like "Security update for Foo Software" display_name: str, like "Munki Client" catalogs: list of str catalog names. """ self.VerifyMunkiInstall() args = [self._GetMunkiPath(MAKEPKGINFO), filename] args.append('--description=%s' % description) if display_name: args.append('--displayname=%s' % display_name) for catalog in catalogs: args.append('--catalog=%s' % catalog) if 'PKGS_MUNKI_MAKEPKGINFO' in os.environ: args[0] = os.environ['PKGS_MUNKI_MAKEPKGINFO'] try: p = subprocess.Popen( args, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True, shell=False) except OSError, e: raise MunkiInstallError('Cannot execute %s: %s' % (' '.join(args), e)) (stdout, stderr) = p.communicate(None) status = p.poll() if status == 0 and stdout and not stderr: self.filename = filename else: raise MunkiError( 'makepkginfo: exit status %d, stderr=%s' % (status, stderr)) self.plist = plist.MunkiPackageInfoPlist(stdout) try: self.plist.Parse() except plist.Error, e: raise Error(str(e))
def EditPackageInfo(self): """Edit package info on a package already on Simian. Raises: CliError: if package info is malformed """ (filepath, description, display_name, pkginfo_name, manifests, catalogs, install_types, unattended_install, unattended_uninstall) = (self.ValidatePackageConfig(defaults=False)) print 'Editing package info ...' filename = os.path.basename(filepath) (sha256_hash, pkginfo_xml) = self.client.GetPackageInfo(filename, get_hash=True) pkginfo = plist.MunkiPackageInfoPlist(pkginfo_xml) pkginfo.Parse() kwargs = {} changes = [] # these values would be None if the user did not specify a value. # note the change detection is a little weak, if we overwrite any # value, even with the same values, we count it as a change. if description is not None: pkginfo.SetDescription(description) changes.append('Description: %s' % description) if display_name is not None: pkginfo.SetDisplayName(display_name) changes.append('Display name: %s' % display_name) if pkginfo_name is not None: pkginfo['name'] = pkginfo_name changes.append('Pkginfo name: %s' % pkginfo_name) if unattended_install is not None: pkginfo.SetUnattendedInstall(unattended_install) changes.append('Unattended install: %s' % unattended_install) if unattended_uninstall is not None: pkginfo.SetUnattendedUninstall(unattended_uninstall) changes.append('Unattended uninstall: %s' % unattended_uninstall) if catalogs is not None: pkginfo.SetCatalogs(catalogs) kwargs['catalogs'] = catalogs changes.append('Catalogs: %s' % catalogs) if manifests is not None: kwargs['manifests'] = manifests changes.append('Manifests: %s' % manifests) if install_types is not None: kwargs['install_types'] = install_types changes.append('Install types: %s' % install_types) edit_hooks = [] if self.config['template_pkginfo'] is not None: edit_hooks.append(self.PackageInfoTemplateHook) if self.config['edit_pkginfo'] is not None: edit_hooks.append(self.PackageInfoEditHook) # TODO(user): Refactor so that this code block and # mac.client.UploadMunkiPackage share the same pkginfo_hooks iteration # code. for edit_hook in edit_hooks: if edit_hook is not None: new_pkginfo = edit_hook(pkginfo) if new_pkginfo: if new_pkginfo is not True: pkginfo = new_pkginfo # changed and valid pkginfo.SetChanged() # populate catalogs value back into our properties catalogs = pkginfo.GetContents().get('catalogs', []) kwargs['catalogs'] = catalogs else: pass # no change at all else: raise CliError( 'Invalid package info') # changed but invalid now if not kwargs and not pkginfo.HasChanged(): print 'Package info unchanged.' return # verify catalogs / manifests relationship if manifests is not None: intended_catalogs = (kwargs.get('catalogs', None) or pkginfo.GetContents().get('catalogs', [])) for manifest in manifests: if manifest not in intended_catalogs: raise CliError('Manifest %s not in catalogs' % manifest) # Parse pkginfo.GetXml() to ensure it's still valid before uploading. new_xml = pkginfo.GetXml() try: tmp_pkginfo = plist.MunkiPackageInfoPlist(new_xml) tmp_pkginfo.Parse() except plist.Error, e: raise CliError('Invalid package info: %s', str(e))
def post(self): """POST This method behaves a little strangely. BlobstoreUploadHandler only allows returns statuses of 301, 302, 303 (not even 200), so one must redirect away to return more information to the caller. Parameters: file: package file contents pkginfo: packageinfo file contents name: filename of package e.g. 'Firefox-1.0.dmg' """ # Only blobstore/upload service/scotty requests should be # invoking this handler. if not handlers.IsBlobstore(): logging.critical('POST to /uploadpkg not from Blobstore: %s', self.request.headers) self.redirect('/') gaeserver.DoMunkiAuth(require_level=gaeserver.LEVEL_UPLOADPKG) user = self.request.get('user') filename = self.request.get('name') install_types = self.request.get('install_types') catalogs = self.request.get('catalogs', None) manifests = self.request.get('manifests', None) if catalogs is None or not install_types or not user or not filename: msg = 'uploadpkg POST required parameters missing' logging.error(msg) self.redirect('/uploadpkg?mode=error&msg=%s' % msg) return if catalogs == '': catalogs = [] else: catalogs = catalogs.split(',') if manifests in ['', None]: manifests = [] else: manifests = manifests.split(',') install_types = install_types.split(',') upload_files = self.get_uploads('file') upload_pkginfo_files = self.get_uploads('pkginfo') if not len(upload_pkginfo_files) and not self.request.get('pkginfo'): self.redirect('/uploadpkg?mode=error&msg=No%20file%20received') return if len(upload_pkginfo_files): # obtain the pkginfo from a blob, and then throw it away. this is # a necessary hack because the upload handler grabbed it, but we don't # intend to keep it in blobstore. pkginfo_str = gae_util.GetBlobAndDel(upload_pkginfo_files[0].key()) else: # otherwise, grab the form parameter. pkginfo_str = self.request.get('pkginfo') blob_info = upload_files[0] blobstore_key = str(blob_info.key()) # Parse, validate, and encode the pkginfo plist. plist = plist_lib.MunkiPackageInfoPlist(pkginfo_str) try: plist.Parse() except plist_lib.PlistError, e: logging.exception('Invalid pkginfo plist uploaded:\n%s\n', pkginfo_str) gae_util.SafeBlobDel(blobstore_key) self.redirect( '/uploadpkg?mode=error&msg=No%20valid%20pkginfo%20received') return