class SetupTool(Folder): """ Profile-based site configuration manager. """ implements(ISetupTool) meta_type = 'Generic Setup Tool' _baseline_context_id = '' _profile_upgrade_versions = {} _exclude_global_steps = False security = ClassSecurityInfo() def __init__(self, id): self.id = str(id) self._import_registry = ImportStepRegistry() self._export_registry = ExportStepRegistry() self._toolset_registry = ToolsetRegistry() # # ISetupTool API # security.declareProtected(ManagePortal, 'getEncoding') def getEncoding(self): """ See ISetupTool. """ return 'utf-8' security.declareProtected(ManagePortal, 'getBaselineContextID') def getBaselineContextID(self): """ See ISetupTool. """ return self._baseline_context_id security.declareProtected(ManagePortal, 'setBaselineContext') def setBaselineContext(self, context_id, encoding=None): """ See ISetupTool. """ self._baseline_context_id = context_id self.applyContextById(context_id, encoding) security.declareProtected(ManagePortal, 'getExcludeGlobalSteps') def getExcludeGlobalSteps(self): """ See ISetupTool. """ return self._exclude_global_steps security.declareProtected(ManagePortal, 'setExcludeGlobalSteps') def setExcludeGlobalSteps(self, value): """ See ISetupTool. """ self._exclude_global_steps = value security.declareProtected(ManagePortal, 'applyContextById') def applyContextById(self, context_id, encoding=None): context = self._getImportContext(context_id) self.applyContext(context, encoding) security.declareProtected(ManagePortal, 'applyContext') def applyContext(self, context, encoding=None): self._updateImportStepsRegistry(context, encoding) self._updateExportStepsRegistry(context, encoding) security.declareProtected(ManagePortal, 'getImportStepRegistry') def getImportStepRegistry(self): """ See ISetupTool. """ return self._import_registry security.declareProtected(ManagePortal, 'getExportStepRegistry') def getExportStepRegistry(self): """ See ISetupTool. """ return self._export_registry security.declareProtected(ManagePortal, 'getImportStep') def getImportStep(self, step, default=None): """Simple wrapper to query both the global and local step registry.""" res = self._import_registry.getStep(step, self) if res is self and not self._exclude_global_steps: res = _import_step_registry.getStep(step, self) if res is not self: return res return default security.declareProtected(ManagePortal, 'getSortedImportSteps') def getSortedImportSteps(self): if self._exclude_global_steps: steps = set() else: steps = set(_import_step_registry.listSteps()) steps.update(set(self._import_registry.listSteps())) step_infos = [self.getImportStepMetadata(step) for step in steps] return tuple(_computeTopologicalSort(step_infos)) security.declareProtected(ManagePortal, 'getImportStepMetadata') def getImportStepMetadata(self, step, default=None): """Simple wrapper to query both the global and local step registry.""" res = self._import_registry.getStepMetadata(step, self) if res is self and not self._exclude_global_steps: res = _import_step_registry.getStepMetadata(step, default) if res is not self: return res return default security.declareProtected(ManagePortal, 'getExportStep') def getExportStep(self, step, default=None): """Simple wrapper to query both the global and local step registry.""" res = self._export_registry.getStep(step, self) if res is self and not self._exclude_global_steps: res = _export_step_registry.getStep(step, self) if res is not self: return res return default security.declareProtected(ManagePortal, 'listExportSteps') def listExportSteps(self): steps = set(self._export_registry.listSteps()) if not self._exclude_global_steps: steps.update(set(_export_step_registry.listSteps())) return tuple(steps) security.declareProtected(ManagePortal, 'getExportStepMetadata') def getExportStepMetadata(self, step, default=None): """Simple wrapper to query both the global and local step registry.""" res = self._export_registry.getStepMetadata(step, self) if res is self and not self._exclude_global_steps: res = _export_step_registry.getStepMetadata(step, default) if res is not self: return res return default security.declareProtected(ManagePortal, 'getToolsetRegistry') def getToolsetRegistry(self): """ See ISetupTool. """ return self._toolset_registry security.declareProtected(ManagePortal, 'runImportStepFromProfile') def runImportStepFromProfile(self, profile_id, step_id, run_dependencies=True, purge_old=None): """ See ISetupTool. """ context = self._getImportContext(profile_id, purge_old) self.applyContext(context) info = self.getImportStepMetadata(step_id) if info is None: raise ValueError('No such import step: %s' % step_id) dependencies = info.get('dependencies', ()) messages = {} steps = [] if run_dependencies: for dependency in dependencies: if dependency not in steps: steps.append(dependency) steps.append(step_id) full_import = (set(steps) == set(self.getSortedImportSteps())) event.notify( BeforeProfileImportEvent(self, profile_id, steps, full_import)) for step in steps: message = self._doRunImportStep(step, context) messages[step] = message or '' message_list = filter(None, [message]) message_list.extend([ '%s: %s' % x[1:] for x in context.listNotes() ]) messages[step_id] = '\n'.join(message_list) event.notify( ProfileImportedEvent(self, profile_id, steps, full_import)) return {'steps': steps, 'messages': messages} security.declareProtected(ManagePortal, 'runAllImportStepsFromProfile') def runAllImportStepsFromProfile(self, profile_id, purge_old=None, ignore_dependencies=False, archive=None): """ See ISetupTool. """ __traceback_info__ = profile_id result = self._runImportStepsFromContext( purge_old=purge_old, profile_id=profile_id, archive=archive, ignore_dependencies=ignore_dependencies) if profile_id is None: prefix = 'import-all-from-tar' else: prefix = 'import-all-%s' % profile_id.replace(':', '_') name = self._mangleTimestampName(prefix, 'log') self._createReport(name, result['steps'], result['messages']) return result security.declareProtected(ManagePortal, 'runExportStep') def runExportStep(self, step_id): """ See ISetupTool. """ return self._doRunExportSteps([step_id]) security.declareProtected(ManagePortal, 'runAllExportSteps') def runAllExportSteps(self): """ See ISetupTool. """ return self._doRunExportSteps(self.listExportSteps()) security.declareProtected(ManagePortal, 'createSnapshot') def createSnapshot(self, snapshot_id): """ See ISetupTool. """ context = SnapshotExportContext(self, snapshot_id) messages = {} steps = self.listExportSteps() for step_id in steps: handler = self.getExportStep(step_id) if handler is None: logger = logging.getLogger('GenericSetup') logger.error('Step %s has an invalid handler' % step_id) continue messages[step_id] = handler(context) return {'steps': steps, 'messages': messages, 'url': context.getSnapshotURL(), 'snapshot': context.getSnapshotFolder()} security.declareProtected(ManagePortal, 'compareConfigurations') def compareConfigurations(self, lhs_context, rhs_context, missing_as_empty=False, ignore_blanks=False, skip=SKIPPED_FILES, ): """ See ISetupTool. """ differ = ConfigDiff(lhs_context, rhs_context, missing_as_empty, ignore_blanks, skip, ) return differ.compare() security.declareProtected(ManagePortal, 'markupComparison') def markupComparison(self, lines): """ See ISetupTool. """ result = [] for line in lines.splitlines(): if line.startswith('** '): if line.find('File') > -1: if line.find('replaced') > -1: result.append(('file-to-dir', line)) elif line.find('added') > -1: result.append(('file-added', line)) else: result.append(('file-removed', line)) else: if line.find('replaced') > -1: result.append(('dir-to-file', line)) elif line.find('added') > -1: result.append(('dir-added', line)) else: result.append(('dir-removed', line)) elif line.startswith('@@'): result.append(('diff-range', line)) elif line.startswith(' '): result.append(('diff-context', line)) elif line.startswith('+'): result.append(('diff-added', line)) elif line.startswith('-'): result.append(('diff-removed', line)) elif line == '\ No newline at end of file': result.append(('diff-context', line)) else: result.append(('diff-header', line)) return '<pre>\n%s\n</pre>' % ( '\n'.join([('<span class="%s">%s</span>' % (cl, escape(l))) for cl, l in result])) # # ZMI # manage_options = ( Folder.manage_options[:1] + ({'label': 'Profiles', 'action': 'manage_tool'}, {'label': 'Import', 'action': 'manage_importSteps'}, {'label': 'Export', 'action': 'manage_exportSteps'}, {'label': 'Upgrades', 'action': 'manage_upgrades'}, {'label': 'Snapshots', 'action': 'manage_snapshots'}, {'label': 'Comparison', 'action': 'manage_showDiff'}, {'label': 'Manage', 'action': 'manage_stepRegistry'}) + Folder.manage_options[3:]) # skip "View", "Properties" security.declareProtected(ManagePortal, 'manage_tool') manage_tool = PageTemplateFile('sutProperties', _wwwdir) security.declareProtected(ManagePortal, 'manage_updateToolProperties') def manage_updateToolProperties(self, context_id, exclude_global_steps=False, RESPONSE=None): """ Update the tool's settings. """ self.setExcludeGlobalSteps(exclude_global_steps) self.setBaselineContext(context_id) if RESPONSE is not None: RESPONSE.redirect('%s/manage_tool?manage_tabs_message=%s' % (self.absolute_url(), 'Properties+updated.')) security.declareProtected(ManagePortal, 'manage_importSteps') manage_importSteps = PageTemplateFile('sutImportSteps', _wwwdir) security.declareProtected(ManagePortal, 'manage_importSelectedSteps') def manage_importSelectedSteps(self, ids, run_dependencies, context_id=None): """ Import the steps selected by the user. """ messages = {} if not ids: summary = 'No steps selected.' else: if context_id is None: context_id = self.getBaselineContextID() steps_run = [] for step_id in ids: result = self.runImportStepFromProfile(context_id, step_id, run_dependencies) steps_run.extend(result['steps']) messages.update(result['messages']) summary = 'Steps run: %s' % ', '.join(steps_run) name = self._mangleTimestampName('import-selected', 'log') self._createReport(name, result['steps'], result['messages']) return self.manage_importSteps(manage_tabs_message=summary, messages=messages) security.declareProtected(ManagePortal, 'manage_importAllSteps') def manage_importAllSteps(self, context_id=None): """ Import all steps. """ if context_id is None: context_id = self.getBaselineContextID() result = self.runAllImportStepsFromProfile(context_id, purge_old=None) steps_run = 'Steps run: %s' % ', '.join(result['steps']) return self.manage_importSteps(manage_tabs_message=steps_run, messages=result['messages']) security.declareProtected(ManagePortal, 'manage_importExtensions') def manage_importExtensions(self, RESPONSE, profile_ids=()): """ Import all steps for the selected extension profiles. """ detail = {} if len(profile_ids) == 0: message = 'Please select one or more extension profiles.' RESPONSE.redirect('%s/manage_tool?manage_tabs_message=%s' % (self.absolute_url(), message)) else: message = 'Imported profiles: %s' % ', '.join(profile_ids) for profile_id in profile_ids: result = self.runAllImportStepsFromProfile(profile_id) for k, v in result['messages'].items(): detail['%s:%s' % (profile_id, k)] = v return self.manage_importSteps(manage_tabs_message=message, messages=detail) security.declareProtected(ManagePortal, 'manage_importTarball') def manage_importTarball(self, tarball): """ Import steps from the uploaded tarball. """ if getattr(tarball, 'read', None) is not None: tarball = tarball.read() result = self.runAllImportStepsFromProfile(None, True, archive=tarball) steps_run = 'Steps run: %s' % ', '.join(result['steps']) return self.manage_importSteps(manage_tabs_message=steps_run, messages=result['messages']) security.declareProtected(ManagePortal, 'manage_exportSteps') manage_exportSteps = PageTemplateFile('sutExportSteps', _wwwdir) security.declareProtected(ManagePortal, 'manage_exportSelectedSteps') def manage_exportSelectedSteps(self, ids, RESPONSE): """ Export the steps selected by the user. """ if not ids: RESPONSE.redirect('%s/manage_exportSteps?manage_tabs_message=%s' % (self.absolute_url(), 'No+steps+selected.')) result = self._doRunExportSteps(ids) RESPONSE.setHeader('Content-type', 'application/x-gzip') RESPONSE.setHeader('Content-disposition', 'attachment; filename=%s' % result['filename']) return result['tarball'] security.declareProtected(ManagePortal, 'manage_exportAllSteps') def manage_exportAllSteps(self, RESPONSE): """ Export all steps. """ result = self.runAllExportSteps() RESPONSE.setHeader('Content-type', 'application/x-gzip') RESPONSE.setHeader('Content-disposition', 'attachment; filename=%s' % result['filename']) return result['tarball'] security.declareProtected(ManagePortal, 'manage_upgrades') manage_upgrades = PageTemplateFile('setup_upgrades', _wwwdir) security.declareProtected(ManagePortal, 'upgradeStepMacro') upgradeStepMacro = PageTemplateFile('upgradeStep', _wwwdir) security.declareProtected(ManagePortal, 'manage_snapshots') manage_snapshots = PageTemplateFile('sutSnapshots', _wwwdir) security.declareProtected(ManagePortal, 'listSnapshotInfo') def listSnapshotInfo(self): """ Return a list of mappings describing available snapshots. o Keys include: 'id' -- snapshot ID 'title' -- snapshot title or ID 'url' -- URL of the snapshot folder """ result = [] snapshots = self._getOb('snapshots', None) if snapshots: for id, folder in snapshots.objectItems('Folder'): result.append({'id': id, 'title': folder.title_or_id(), 'url': folder.absolute_url()}) return result security.declareProtected(ManagePortal, 'listProfileInfo') def listProfileInfo(self, for_=None): """ Return a list of mappings describing registered profiles. Base profile is listed first, extensions are sorted. o Keys include: 'id' -- profile ID 'title' -- profile title or ID 'description' -- description of the profile 'path' -- path to the profile within its product 'product' -- name of the registering product """ base = [] ext = [] for info in _profile_registry.listProfileInfo(for_): if info.get('type', BASE) == BASE: base.append(info) else: ext.append(info) ext.sort(lambda x, y: cmp(x['id'], y['id'])) return base + ext security.declareProtected(ManagePortal, 'listContextInfos') def listContextInfos(self): """ List registered profiles and snapshots. """ def readableType(x): if x is BASE: return 'base' elif x is EXTENSION: return 'extension' return 'unknown' s_infos = [{'id': 'snapshot-%s' % info['id'], 'title': info['title'], 'type': 'snapshot', } for info in self.listSnapshotInfo()] s_infos.sort(key=itemgetter('title')) p_infos = [{'id': 'profile-%s' % info['id'], 'title': info['title'], 'type': readableType(info['type']), } for info in self.listProfileInfo()] p_infos.sort(key=itemgetter('title')) return tuple(s_infos + p_infos) security.declareProtected(ManagePortal, 'getProfileImportDate') def getProfileImportDate(self, profile_id): """ See ISetupTool. """ prefix = ('import-all-%s-' % profile_id).replace(':', '_') candidates = [x for x in self.objectIds('File') if x[:-18] == prefix and x.endswith('.log')] if len(candidates) == 0: return None candidates.sort() last = candidates[-1] stamp = last[-18:-4] return '%s-%s-%sT%s:%s:%sZ' % (stamp[0:4], stamp[4:6], stamp[6:8], stamp[8:10], stamp[10:12], stamp[12:14], ) security.declareProtected(ManagePortal, 'manage_createSnapshot') def manage_createSnapshot(self, RESPONSE, snapshot_id=None): """ Create a snapshot with the given ID. o If no ID is passed, generate one. """ if snapshot_id is None: snapshot_id = self._mangleTimestampName('snapshot') self.createSnapshot(snapshot_id) return RESPONSE.redirect('%s/manage_snapshots?manage_tabs_message=%s' % (self.absolute_url(), 'Snapshot+created.')) security.declareProtected(ManagePortal, 'manage_showDiff') manage_showDiff = PageTemplateFile('sutCompare', _wwwdir) def manage_downloadDiff(self, lhs, rhs, missing_as_empty, ignore_blanks, RESPONSE, ): """ Crack request vars and call compareConfigurations. o Return the result as a 'text/plain' stream, suitable for framing. """ comparison = self.manage_compareConfigurations(lhs, rhs, missing_as_empty, ignore_blanks, ) RESPONSE.setHeader('Content-Type', 'text/plain') return _PLAINTEXT_DIFF_HEADER % (lhs, rhs, comparison) security.declareProtected(ManagePortal, 'manage_compareConfigurations') def manage_compareConfigurations(self, lhs, rhs, missing_as_empty, ignore_blanks, ): """ Crack request vars and call compareConfigurations. """ lhs_context = self._getImportContext(lhs) rhs_context = self._getImportContext(rhs) return self.compareConfigurations(lhs_context, rhs_context, missing_as_empty, ignore_blanks, ) security.declareProtected(ManagePortal, 'manage_stepRegistry') manage_stepRegistry = PageTemplateFile('sutManage', _wwwdir) security.declareProtected(ManagePortal, 'manage_deleteImportSteps') def manage_deleteImportSteps(self, ids, request=None): """ Delete selected import steps. """ if request is None: request = self.REQUEST for id in ids: self._import_registry.unregisterStep(id) self._p_changed = True url = self.absolute_url() request.RESPONSE.redirect("%s/manage_stepRegistry" % url) security.declareProtected(ManagePortal, 'manage_deleteExportSteps') def manage_deleteExportSteps(self, ids, request=None): """ Delete selected export steps. """ if request is None: request = self.REQUEST for id in ids: self._export_registry.unregisterStep(id) self._p_changed = True url = self.absolute_url() request.RESPONSE.redirect("%s/manage_stepRegistry" % url) # # Upgrades management # security.declareProtected(ManagePortal, 'getLastVersionForProfile') def getLastVersionForProfile(self, profile_id): """Return the last upgraded version for the specified profile. """ version = self._profile_upgrade_versions.get(profile_id, 'unknown') return version security.declareProtected(ManagePortal, 'setLastVersionForProfile') def setLastVersionForProfile(self, profile_id, version): """Set the last upgraded version for the specified profile. """ if isinstance(version, basestring): version = tuple(version.split('.')) prof_versions = self._profile_upgrade_versions.copy() prof_versions[profile_id] = version self._profile_upgrade_versions = prof_versions security.declareProtected(ManagePortal, 'getVersionForProfile') def getVersionForProfile(self, profile_id): """Return the registered filesystem version for the specified profile. """ return self.getProfileInfo(profile_id).get('version', 'unknown') security.declareProtected(ManagePortal, 'profileExists') def profileExists(self, profile_id): """Check if a profile exists.""" try: self.getProfileInfo(profile_id) except KeyError: return False else: return True security.declareProtected(ManagePortal, "getProfileInfo") def getProfileInfo(self, profile_id): if profile_id.startswith("profile-"): profile_id = profile_id[len('profile-'):] elif profile_id.startswith("snapshot-"): profile_id = profile_id[len('snapshot-'):] return _profile_registry.getProfileInfo(profile_id) security.declareProtected(ManagePortal, 'getDependenciesForProfile') def getDependenciesForProfile(self, profile_id): if profile_id.startswith("snapshot-"): return () if not self.profileExists(profile_id): raise KeyError(profile_id) try: return self.getProfileInfo(profile_id).get('dependencies', ()) except KeyError: return () security.declareProtected(ManagePortal, 'listProfilesWithUpgrades') def listProfilesWithUpgrades(self): return listProfilesWithUpgrades() security.declarePrivate('_massageUpgradeInfo') def _massageUpgradeInfo(self, info): """Add a couple of data points to the upgrade info dictionary. """ info = info.copy() info['haspath'] = info['source'] and info['dest'] info['ssource'] = '.'.join(info['source'] or ('all',)) info['sdest'] = '.'.join(info['dest'] or ('all',)) info['done'] = (not info['proposed'] and info['step'].checker is not None and not info['step'].checker(self)) return info security.declareProtected(ManagePortal, 'listUpgrades') def listUpgrades(self, profile_id, show_old=False): """Get the list of available upgrades. """ if show_old: source = None else: source = self.getLastVersionForProfile(profile_id) upgrades = listUpgradeSteps(self, profile_id, source) res = [] for info in upgrades: if type(info) == list: subset = [] for subinfo in info: subset.append(self._massageUpgradeInfo(subinfo)) res.append(subset) else: res.append(self._massageUpgradeInfo(info)) return res security.declareProtected(ManagePortal, 'manage_doUpgrades') def manage_doUpgrades(self, request=None): """Perform all selected upgrade steps. """ if request is None: request = self.REQUEST logger = logging.getLogger('GenericSetup') steps_to_run = request.form.get('upgrades', []) profile_id = request.get('profile_id', '') step = None for step_id in steps_to_run: step = _upgrade_registry.getUpgradeStep(profile_id, step_id) if step is not None: step.doStep(self) msg = "Ran upgrade step %s for profile %s" % (step.title, profile_id) logger.log(logging.INFO, msg) # We update the profile version to the last one we have reached # with running an upgrade step. if step and step.dest is not None and step.checker is None: self.setLastVersionForProfile(profile_id, step.dest) url = self.absolute_url() request.RESPONSE.redirect("%s/manage_upgrades?saved=%s" % (url, profile_id)) # # Helper methods # security.declarePrivate('_getImportContext') def _getImportContext(self, context_id, should_purge=None, archive=None): """ Crack ID and generate appropriate import context. """ encoding = self.getEncoding() if context_id is not None: if context_id.startswith('profile-'): context_id = context_id[len('profile-'):] info = _profile_registry.getProfileInfo(context_id) if info.get('product'): path = os.path.join(_getProductPath(info['product']), info['path']) else: path = info['path'] if should_purge is None: should_purge = (info.get('type') != EXTENSION) return DirectoryImportContext(self, path, should_purge, encoding) elif context_id.startswith('snapshot-'): context_id = context_id[len('snapshot-'):] if should_purge is None: should_purge = True return SnapshotImportContext(self, context_id, should_purge, encoding) if archive is not None: return TarballImportContext(tool=self, archive_bits=archive, encoding='UTF8', should_purge=should_purge, ) raise KeyError('Unknown context "%s"' % context_id) security.declarePrivate('_updateImportStepsRegistry') def _updateImportStepsRegistry(self, context, encoding): """ Update our import steps registry from our profile. """ xml = context.readDataFile(IMPORT_STEPS_XML) if xml is None: return info_list = self._import_registry.parseXML(xml, encoding) for step_info in info_list: id = step_info['id'] version = step_info.get('version') handler = step_info['handler'] dependencies = tuple(step_info.get('dependencies', ())) title = step_info.get('title', id) description = ''.join(step_info.get('description', [])) self._import_registry.registerStep(id=id, version=version, handler=handler, dependencies=dependencies, title=title, description=description, ) security.declarePrivate('_updateExportStepsRegistry') def _updateExportStepsRegistry(self, context, encoding): """ Update our export steps registry from our profile. """ xml = context.readDataFile(EXPORT_STEPS_XML) if xml is None: return info_list = self._export_registry.parseXML(xml, encoding) for step_info in info_list: id = step_info['id'] handler = step_info['handler'] title = step_info.get('title', id) description = ''.join(step_info.get('description', [])) self._export_registry.registerStep(id=id, handler=handler, title=title, description=description, ) security.declarePrivate('_doRunImportStep') def _doRunImportStep(self, step_id, context): """ Run a single import step, using a pre-built context. """ __traceback_info__ = step_id marker = object() handler = self.getImportStep(step_id) if handler is marker: raise ValueError('Invalid import step: %s' % step_id) if handler is None: msg = 'Step %s has an invalid import handler' % step_id logger = logging.getLogger('GenericSetup') logger.error(msg) return 'ERROR: ' + msg return handler(context) security.declarePrivate('_doRunExportSteps') def _doRunExportSteps(self, steps): """ See ISetupTool. """ context = TarballExportContext(self) messages = {} marker = object() for step_id in steps: handler = self.getExportStep(step_id, marker) if handler is marker: raise ValueError('Invalid export step: %s' % step_id) if handler is None: msg = 'Step %s has an invalid export handler' % step_id logger = logging.getLogger('GenericSetup') logger.error(msg) messages[step_id] = msg else: messages[step_id] = handler(context) return {'steps': steps, 'messages': messages, 'tarball': context.getArchive(), 'filename': context.getArchiveFilename()} security.declareProtected(ManagePortal, 'getProfileDependencyChain') def getProfileDependencyChain(self, profile_id, seen=None): if seen is None: seen = set() elif profile_id in seen: return [] # cycle break seen.add(profile_id) chain = [] dependencies = self.getDependenciesForProfile(profile_id) for dependency in dependencies: chain.extend(self.getProfileDependencyChain(dependency, seen)) chain.append(profile_id) return chain security.declarePrivate('_runImportStepsFromContext') def _runImportStepsFromContext(self, steps=None, purge_old=None, profile_id=None, archive=None, ignore_dependencies=False, seen=None): if profile_id is not None and not ignore_dependencies: try: chain = self.getProfileDependencyChain(profile_id) except KeyError, e: logger = logging.getLogger('GenericSetup') logger.error('Unknown step in dependency chain: %s' % str(e)) raise else:
class SetupTool(Folder): """ Profile-based site configuration manager. """ implements(ISetupTool) meta_type = 'Generic Setup Tool' _baseline_context_id = '' _profile_upgrade_versions = {} _exclude_global_steps = False security = ClassSecurityInfo() def __init__(self, id): self.id = str(id) self._import_registry = ImportStepRegistry() self._export_registry = ExportStepRegistry() self._toolset_registry = ToolsetRegistry() # # ISetupTool API # security.declareProtected(ManagePortal, 'getEncoding') def getEncoding(self): """ See ISetupTool. """ return 'utf-8' security.declareProtected(ManagePortal, 'getBaselineContextID') def getBaselineContextID(self): """ See ISetupTool. """ return self._baseline_context_id security.declareProtected(ManagePortal, 'setBaselineContext') def setBaselineContext(self, context_id, encoding=None): """ See ISetupTool. """ self._baseline_context_id = context_id self.applyContextById(context_id, encoding) security.declareProtected(ManagePortal, 'getExcludeGlobalSteps') def getExcludeGlobalSteps(self): """ See ISetupTool. """ return self._exclude_global_steps security.declareProtected(ManagePortal, 'setExcludeGlobalSteps') def setExcludeGlobalSteps(self, value): """ See ISetupTool. """ self._exclude_global_steps = value security.declareProtected(ManagePortal, 'applyContextById') def applyContextById(self, context_id, encoding=None): context = self._getImportContext(context_id) self.applyContext(context, encoding) security.declareProtected(ManagePortal, 'applyContext') def applyContext(self, context, encoding=None): self._updateImportStepsRegistry(context, encoding) self._updateExportStepsRegistry(context, encoding) security.declareProtected(ManagePortal, 'getImportStepRegistry') def getImportStepRegistry(self): """ See ISetupTool. """ return self._import_registry security.declareProtected(ManagePortal, 'getExportStepRegistry') def getExportStepRegistry(self): """ See ISetupTool. """ return self._export_registry security.declareProtected(ManagePortal, 'getImportStep') def getImportStep(self, step, default=None): """Simple wrapper to query both the global and local step registry.""" res = self._import_registry.getStep(step, self) if res is self and not self._exclude_global_steps: res = _import_step_registry.getStep(step, self) if res is not self: return res return default security.declareProtected(ManagePortal, 'getSortedImportSteps') def getSortedImportSteps(self): if self._exclude_global_steps: steps = set() else: steps = set(_import_step_registry.listSteps()) steps.update(set(self._import_registry.listSteps())) step_infos = [self.getImportStepMetadata(step) for step in steps] return tuple(_computeTopologicalSort(step_infos)) security.declareProtected(ManagePortal, 'getImportStepMetadata') def getImportStepMetadata(self, step, default=None): """Simple wrapper to query both the global and local step registry.""" res = self._import_registry.getStepMetadata(step, self) if res is self and not self._exclude_global_steps: res = _import_step_registry.getStepMetadata(step, default) if res is not self: return res return default security.declareProtected(ManagePortal, 'getExportStep') def getExportStep(self, step, default=None): """Simple wrapper to query both the global and local step registry.""" res = self._export_registry.getStep(step, self) if res is self and not self._exclude_global_steps: res = _export_step_registry.getStep(step, self) if res is not self: return res return default security.declareProtected(ManagePortal, 'listExportSteps') def listExportSteps(self): steps = set(self._export_registry.listSteps()) if not self._exclude_global_steps: steps.update(set(_export_step_registry.listSteps())) return tuple(steps) security.declareProtected(ManagePortal, 'listImportSteps') def listImportSteps(self): steps = set(self._import_registry.listSteps()) if not self._exclude_global_steps: steps.update(set(_import_step_registry.listSteps())) return tuple(steps) security.declareProtected(ManagePortal, 'getExportStepMetadata') def getExportStepMetadata(self, step, default=None): """Simple wrapper to query both the global and local step registry.""" res = self._export_registry.getStepMetadata(step, self) if res is self and not self._exclude_global_steps: res = _export_step_registry.getStepMetadata(step, default) if res is not self: return res return default security.declareProtected(ManagePortal, 'getToolsetRegistry') def getToolsetRegistry(self): """ See ISetupTool. """ return self._toolset_registry security.declareProtected(ManagePortal, 'runImportStepFromProfile') def runImportStepFromProfile(self, profile_id, step_id, run_dependencies=True, purge_old=None): """ See ISetupTool. """ context = self._getImportContext(profile_id, purge_old) self.applyContext(context) info = self.getImportStepMetadata(step_id) if info is None: generic_logger.error( "No such import step: '%s' Maybe you meant one of %s", step_id, str(self.listImportSteps())) raise ValueError('No such import step: %s' % step_id) dependencies = info.get('dependencies', ()) messages = {} steps = [] if run_dependencies: for dependency in dependencies: if dependency not in steps: steps.append(dependency) steps.append(step_id) full_import = (set(steps) == set(self.getSortedImportSteps())) event.notify( BeforeProfileImportEvent(self, profile_id, steps, full_import)) for step in steps: message = self._doRunImportStep(step, context) messages[step] = message or '' message_list = filter(None, [message]) message_list.extend(['%s: %s' % x[1:] for x in context.listNotes()]) messages[step_id] = '\n'.join(message_list) event.notify(ProfileImportedEvent(self, profile_id, steps, full_import)) return {'steps': steps, 'messages': messages} security.declareProtected(ManagePortal, 'runAllImportStepsFromProfile') def runAllImportStepsFromProfile(self, profile_id, purge_old=None, ignore_dependencies=False, archive=None, blacklisted_steps=None): """ See ISetupTool. """ __traceback_info__ = profile_id result = self._runImportStepsFromContext( purge_old=purge_old, profile_id=profile_id, archive=archive, ignore_dependencies=ignore_dependencies, blacklisted_steps=blacklisted_steps) if profile_id is None: prefix = 'import-all-from-tar' else: prefix = 'import-all-%s' % profile_id.replace(':', '_') name = self._mangleTimestampName(prefix, 'log') self._createReport(name, result['steps'], result['messages']) return result security.declareProtected(ManagePortal, 'runExportStep') def runExportStep(self, step_id): """ See ISetupTool. """ return self._doRunExportSteps([step_id]) security.declareProtected(ManagePortal, 'runAllExportSteps') def runAllExportSteps(self): """ See ISetupTool. """ return self._doRunExportSteps(self.listExportSteps()) security.declareProtected(ManagePortal, 'createSnapshot') def createSnapshot(self, snapshot_id): """ See ISetupTool. """ context = SnapshotExportContext(self, snapshot_id) messages = {} steps = self.listExportSteps() for step_id in steps: handler = self.getExportStep(step_id) if handler is None: logger = logging.getLogger('GenericSetup') logger.error('Step %s has an invalid handler' % step_id) continue messages[step_id] = handler(context) return { 'steps': steps, 'messages': messages, 'url': context.getSnapshotURL(), 'snapshot': context.getSnapshotFolder() } security.declareProtected(ManagePortal, 'compareConfigurations') def compareConfigurations( self, lhs_context, rhs_context, missing_as_empty=False, ignore_blanks=False, skip=SKIPPED_FILES, ): """ See ISetupTool. """ differ = ConfigDiff( lhs_context, rhs_context, missing_as_empty, ignore_blanks, skip, ) return differ.compare() security.declareProtected(ManagePortal, 'markupComparison') def markupComparison(self, lines): """ See ISetupTool. """ result = [] for line in lines.splitlines(): if line.startswith('** '): if line.find('File') > -1: if line.find('replaced') > -1: result.append(('file-to-dir', line)) elif line.find('added') > -1: result.append(('file-added', line)) else: result.append(('file-removed', line)) else: if line.find('replaced') > -1: result.append(('dir-to-file', line)) elif line.find('added') > -1: result.append(('dir-added', line)) else: result.append(('dir-removed', line)) elif line.startswith('@@'): result.append(('diff-range', line)) elif line.startswith(' '): result.append(('diff-context', line)) elif line.startswith('+'): result.append(('diff-added', line)) elif line.startswith('-'): result.append(('diff-removed', line)) elif line == '\ No newline at end of file': result.append(('diff-context', line)) else: result.append(('diff-header', line)) return '<pre>\n%s\n</pre>' % ('\n'.join( [('<span class="%s">%s</span>' % (cl, escape(l))) for cl, l in result])) # # ZMI # manage_options = ( Folder.manage_options[:1] + ({ 'label': 'Profiles', 'action': 'manage_tool' }, { 'label': 'Import', 'action': 'manage_importSteps' }, { 'label': 'Export', 'action': 'manage_exportSteps' }, { 'label': 'Upgrades', 'action': 'manage_upgrades' }, { 'label': 'Snapshots', 'action': 'manage_snapshots' }, { 'label': 'Comparison', 'action': 'manage_showDiff' }, { 'label': 'Manage', 'action': 'manage_stepRegistry' }) + Folder.manage_options[3:]) # skip "View", "Properties" security.declareProtected(ManagePortal, 'manage_tool') manage_tool = PageTemplateFile('sutProperties', _wwwdir) security.declareProtected(ManagePortal, 'manage_updateToolProperties') def manage_updateToolProperties(self, context_id, exclude_global_steps=False, RESPONSE=None): """ Update the tool's settings. """ self.setExcludeGlobalSteps(exclude_global_steps) self.setBaselineContext(context_id) if RESPONSE is not None: RESPONSE.redirect('%s/manage_tool?manage_tabs_message=%s' % (self.absolute_url(), 'Properties+updated.')) security.declareProtected(ManagePortal, 'manage_importSteps') manage_importSteps = PageTemplateFile('sutImportSteps', _wwwdir) security.declareProtected(ManagePortal, 'manage_importSelectedSteps') def manage_importSelectedSteps(self, ids, run_dependencies, context_id=None): """ Import the steps selected by the user. """ messages = {} if not ids: summary = 'No steps selected.' else: if context_id is None: context_id = self.getBaselineContextID() steps_run = [] for step_id in ids: result = self.runImportStepFromProfile(context_id, step_id, run_dependencies) steps_run.extend(result['steps']) messages.update(result['messages']) summary = 'Steps run: %s' % ', '.join(steps_run) name = self._mangleTimestampName('import-selected', 'log') self._createReport(name, result['steps'], result['messages']) return self.manage_importSteps(manage_tabs_message=summary, messages=messages) security.declareProtected(ManagePortal, 'manage_importAllSteps') def manage_importAllSteps(self, context_id=None): """ Import all steps. """ if context_id is None: context_id = self.getBaselineContextID() result = self.runAllImportStepsFromProfile(context_id, purge_old=None) steps_run = 'Steps run: %s' % ', '.join(result['steps']) return self.manage_importSteps(manage_tabs_message=steps_run, messages=result['messages']) security.declareProtected(ManagePortal, 'manage_importExtensions') def manage_importExtensions(self, RESPONSE, profile_ids=()): """ Import all steps for the selected extension profiles. """ detail = {} if len(profile_ids) == 0: message = 'Please select one or more extension profiles.' RESPONSE.redirect('%s/manage_tool?manage_tabs_message=%s' % (self.absolute_url(), message)) else: message = 'Imported profiles: %s' % ', '.join(profile_ids) for profile_id in profile_ids: result = self.runAllImportStepsFromProfile(profile_id) for k, v in result['messages'].items(): detail['%s:%s' % (profile_id, k)] = v return self.manage_importSteps(manage_tabs_message=message, messages=detail) security.declareProtected(ManagePortal, 'manage_importTarball') def manage_importTarball(self, tarball): """ Import steps from the uploaded tarball. """ if getattr(tarball, 'read', None) is not None: tarball = tarball.read() result = self.runAllImportStepsFromProfile(None, True, archive=tarball) steps_run = 'Steps run: %s' % ', '.join(result['steps']) return self.manage_importSteps(manage_tabs_message=steps_run, messages=result['messages']) security.declareProtected(ManagePortal, 'manage_exportSteps') manage_exportSteps = PageTemplateFile('sutExportSteps', _wwwdir) security.declareProtected(ManagePortal, 'manage_exportSelectedSteps') def manage_exportSelectedSteps(self, ids, RESPONSE): """ Export the steps selected by the user. """ if not ids: RESPONSE.redirect('%s/manage_exportSteps?manage_tabs_message=%s' % (self.absolute_url(), 'No+steps+selected.')) result = self._doRunExportSteps(ids) RESPONSE.setHeader('Content-type', 'application/x-gzip') RESPONSE.setHeader('Content-disposition', 'attachment; filename=%s' % result['filename']) return result['tarball'] security.declareProtected(ManagePortal, 'manage_exportAllSteps') def manage_exportAllSteps(self, RESPONSE): """ Export all steps. """ result = self.runAllExportSteps() RESPONSE.setHeader('Content-type', 'application/x-gzip') RESPONSE.setHeader('Content-disposition', 'attachment; filename=%s' % result['filename']) return result['tarball'] security.declareProtected(ManagePortal, 'manage_upgrades') manage_upgrades = PageTemplateFile('setup_upgrades', _wwwdir) security.declareProtected(ManagePortal, 'upgradeStepMacro') upgradeStepMacro = PageTemplateFile('upgradeStep', _wwwdir) security.declareProtected(ManagePortal, 'manage_snapshots') manage_snapshots = PageTemplateFile('sutSnapshots', _wwwdir) security.declareProtected(ManagePortal, 'listSnapshotInfo') def listSnapshotInfo(self): """ Return a list of mappings describing available snapshots. o Keys include: 'id' -- snapshot ID 'title' -- snapshot title or ID 'url' -- URL of the snapshot folder """ result = [] snapshots = self._getOb('snapshots', None) if snapshots: for id, folder in snapshots.objectItems('Folder'): result.append({ 'id': id, 'title': folder.title_or_id(), 'url': folder.absolute_url() }) return result security.declareProtected(ManagePortal, 'listProfileInfo') def listProfileInfo(self, for_=None): """ Return a list of mappings describing registered profiles. Base profile is listed first, extensions are sorted. o Keys include: 'id' -- profile ID 'title' -- profile title or ID 'description' -- description of the profile 'path' -- path to the profile within its product 'product' -- name of the registering product """ base = [] ext = [] for info in _profile_registry.listProfileInfo(for_): if info.get('type', BASE) == BASE: base.append(info) else: ext.append(info) ext.sort(lambda x, y: cmp(x['id'], y['id'])) return base + ext security.declareProtected(ManagePortal, 'listContextInfos') def listContextInfos(self): """ List registered profiles and snapshots. """ def readableType(x): if x is BASE: return 'base' elif x is EXTENSION: return 'extension' return 'unknown' s_infos = [{ 'id': 'snapshot-%s' % info['id'], 'title': info['title'], 'type': 'snapshot', } for info in self.listSnapshotInfo()] s_infos.sort(key=itemgetter('title')) p_infos = [{ 'id': 'profile-%s' % info['id'], 'title': info['title'], 'type': readableType(info['type']), } for info in self.listProfileInfo()] p_infos.sort(key=itemgetter('title')) return tuple(s_infos + p_infos) security.declareProtected(ManagePortal, 'getProfileImportDate') def getProfileImportDate(self, profile_id): """ See ISetupTool. """ prefix = ('import-all-%s-' % profile_id).replace(':', '_') candidates = [ x for x in self.objectIds('File') if x[:-18] == prefix and x.endswith('.log') ] if len(candidates) == 0: return None candidates.sort() last = candidates[-1] stamp = last[-18:-4] return '%s-%s-%sT%s:%s:%sZ' % ( stamp[0:4], stamp[4:6], stamp[6:8], stamp[8:10], stamp[10:12], stamp[12:14], ) security.declareProtected(ManagePortal, 'manage_createSnapshot') def manage_createSnapshot(self, RESPONSE, snapshot_id=None): """ Create a snapshot with the given ID. o If no ID is passed, generate one. """ if snapshot_id is None: snapshot_id = self._mangleTimestampName('snapshot') self.createSnapshot(snapshot_id) return RESPONSE.redirect('%s/manage_snapshots?manage_tabs_message=%s' % (self.absolute_url(), 'Snapshot+created.')) security.declareProtected(ManagePortal, 'manage_showDiff') manage_showDiff = PageTemplateFile('sutCompare', _wwwdir) def manage_downloadDiff( self, lhs, rhs, missing_as_empty, ignore_blanks, RESPONSE, ): """ Crack request vars and call compareConfigurations. o Return the result as a 'text/plain' stream, suitable for framing. """ comparison = self.manage_compareConfigurations( lhs, rhs, missing_as_empty, ignore_blanks, ) RESPONSE.setHeader('Content-Type', 'text/plain') return _PLAINTEXT_DIFF_HEADER % (lhs, rhs, comparison) security.declareProtected(ManagePortal, 'manage_compareConfigurations') def manage_compareConfigurations( self, lhs, rhs, missing_as_empty, ignore_blanks, ): """ Crack request vars and call compareConfigurations. """ lhs_context = self._getImportContext(lhs) rhs_context = self._getImportContext(rhs) return self.compareConfigurations( lhs_context, rhs_context, missing_as_empty, ignore_blanks, ) security.declareProtected(ManagePortal, 'manage_stepRegistry') manage_stepRegistry = PageTemplateFile('sutManage', _wwwdir) security.declareProtected(ManagePortal, 'manage_deleteImportSteps') def manage_deleteImportSteps(self, ids, request=None): """ Delete selected import steps. """ if request is None: request = self.REQUEST for id in ids: self._import_registry.unregisterStep(id) self._p_changed = True url = self.absolute_url() request.RESPONSE.redirect("%s/manage_stepRegistry" % url) security.declareProtected(ManagePortal, 'manage_deleteExportSteps') def manage_deleteExportSteps(self, ids, request=None): """ Delete selected export steps. """ if request is None: request = self.REQUEST for id in ids: self._export_registry.unregisterStep(id) self._p_changed = True url = self.absolute_url() request.RESPONSE.redirect("%s/manage_stepRegistry" % url) # # Upgrades management # security.declareProtected(ManagePortal, 'getLastVersionForProfile') def getLastVersionForProfile(self, profile_id): """Return the last upgraded version for the specified profile. """ version = self._profile_upgrade_versions.get(profile_id, 'unknown') return version security.declareProtected(ManagePortal, 'setLastVersionForProfile') def setLastVersionForProfile(self, profile_id, version): """Set the last upgraded version for the specified profile. """ if isinstance(version, basestring): version = tuple(version.split('.')) prof_versions = self._profile_upgrade_versions.copy() prof_versions[profile_id] = version self._profile_upgrade_versions = prof_versions security.declareProtected(ManagePortal, 'getVersionForProfile') def getVersionForProfile(self, profile_id): """Return the registered filesystem version for the specified profile. """ return self.getProfileInfo(profile_id).get('version', 'unknown') security.declareProtected(ManagePortal, 'profileExists') def profileExists(self, profile_id): """Check if a profile exists.""" try: self.getProfileInfo(profile_id) except KeyError: return False else: return True security.declareProtected(ManagePortal, "getProfileInfo") def getProfileInfo(self, profile_id): if profile_id.startswith("profile-"): profile_id = profile_id[len('profile-'):] elif profile_id.startswith("snapshot-"): profile_id = profile_id[len('snapshot-'):] return _profile_registry.getProfileInfo(profile_id) security.declareProtected(ManagePortal, 'getDependenciesForProfile') def getDependenciesForProfile(self, profile_id): if profile_id.startswith("snapshot-"): return () if not self.profileExists(profile_id): raise KeyError(profile_id) try: return self.getProfileInfo(profile_id).get('dependencies', ()) except KeyError: return () security.declareProtected(ManagePortal, 'listProfilesWithUpgrades') def listProfilesWithUpgrades(self): profiles = listProfilesWithUpgrades() profiles.sort() return profiles security.declarePrivate('_massageUpgradeInfo') def _massageUpgradeInfo(self, info): """Add a couple of data points to the upgrade info dictionary. """ info = info.copy() info['haspath'] = info['source'] and info['dest'] info['ssource'] = '.'.join(info['source'] or ('all', )) info['sdest'] = '.'.join(info['dest'] or ('all', )) info['done'] = (not info['proposed'] and info['step'].checker is not None and not info['step'].checker(self)) return info security.declareProtected(ManagePortal, 'listUpgrades') def listUpgrades(self, profile_id, show_old=False): """Get the list of available upgrades. """ if show_old: source = None else: source = self.getLastVersionForProfile(profile_id) upgrades = listUpgradeSteps(self, profile_id, source) res = [] for info in upgrades: if type(info) == list: subset = [] for subinfo in info: subset.append(self._massageUpgradeInfo(subinfo)) res.append(subset) else: res.append(self._massageUpgradeInfo(info)) return res security.declareProtected(ManagePortal, 'manage_doUpgrades') def manage_doUpgrades(self, request=None): """Perform all selected upgrade steps. """ if request is None: request = self.REQUEST logger = logging.getLogger('GenericSetup') steps_to_run = request.form.get('upgrades', []) profile_id = request.get('profile_id', '') step = None for step_id in steps_to_run: step = _upgrade_registry.getUpgradeStep(profile_id, step_id) if step is not None: step.doStep(self) msg = "Ran upgrade step %s for profile %s" % (step.title, profile_id) logger.log(logging.INFO, msg) # We update the profile version to the last one we have reached # with running an upgrade step. if step and step.dest is not None and step.checker is None: self.setLastVersionForProfile(profile_id, step.dest) url = self.absolute_url() request.RESPONSE.redirect("%s/manage_upgrades?saved=%s" % (url, profile_id)) # # Helper methods # security.declarePrivate('_getImportContext') def _getImportContext(self, context_id, should_purge=None, archive=None): """ Crack ID and generate appropriate import context. """ encoding = self.getEncoding() if context_id is not None: if context_id.startswith('profile-'): context_id = context_id[len('profile-'):] info = _profile_registry.getProfileInfo(context_id) if info.get('product'): path = os.path.join(_getProductPath(info['product']), info['path']) else: path = info['path'] if should_purge is None: should_purge = (info.get('type') != EXTENSION) return DirectoryImportContext(self, path, should_purge, encoding) elif context_id.startswith('snapshot-'): context_id = context_id[len('snapshot-'):] if should_purge is None: should_purge = True return SnapshotImportContext(self, context_id, should_purge, encoding) if archive is not None: return TarballImportContext( tool=self, archive_bits=archive, encoding='UTF8', should_purge=should_purge, ) raise KeyError('Unknown context "%s"' % context_id) security.declarePrivate('_updateImportStepsRegistry') def _updateImportStepsRegistry(self, context, encoding): """ Update our import steps registry from our profile. """ xml = context.readDataFile(IMPORT_STEPS_XML) if xml is None: return info_list = self._import_registry.parseXML(xml, encoding) for step_info in info_list: id = step_info['id'] version = step_info.get('version') handler = step_info['handler'] dependencies = tuple(step_info.get('dependencies', ())) title = step_info.get('title', id) description = ''.join(step_info.get('description', [])) self._import_registry.registerStep( id=id, version=version, handler=handler, dependencies=dependencies, title=title, description=description, ) security.declarePrivate('_updateExportStepsRegistry') def _updateExportStepsRegistry(self, context, encoding): """ Update our export steps registry from our profile. """ xml = context.readDataFile(EXPORT_STEPS_XML) if xml is None: return info_list = self._export_registry.parseXML(xml, encoding) for step_info in info_list: id = step_info['id'] handler = step_info['handler'] title = step_info.get('title', id) description = ''.join(step_info.get('description', [])) self._export_registry.registerStep( id=id, handler=handler, title=title, description=description, ) security.declarePrivate('_doRunImportStep') def _doRunImportStep(self, step_id, context): """ Run a single import step, using a pre-built context. """ __traceback_info__ = step_id marker = object() handler = self.getImportStep(step_id) if handler is marker: raise ValueError('Invalid import step: %s' % step_id) if handler is None: msg = 'Step %s has an invalid import handler' % step_id logger = logging.getLogger('GenericSetup') logger.error(msg) return 'ERROR: ' + msg return handler(context) security.declarePrivate('_doRunExportSteps') def _doRunExportSteps(self, steps): """ See ISetupTool. """ context = TarballExportContext(self) messages = {} marker = object() for step_id in steps: handler = self.getExportStep(step_id, marker) if handler is marker: raise ValueError('Invalid export step: %s' % step_id) if handler is None: msg = 'Step %s has an invalid export handler' % step_id logger = logging.getLogger('GenericSetup') logger.error(msg) messages[step_id] = msg else: messages[step_id] = handler(context) return { 'steps': steps, 'messages': messages, 'tarball': context.getArchive(), 'filename': context.getArchiveFilename() } security.declareProtected(ManagePortal, 'getProfileDependencyChain') def getProfileDependencyChain(self, profile_id, seen=None): if seen is None: seen = set() elif profile_id in seen: return [] # cycle break seen.add(profile_id) chain = [] dependencies = self.getDependenciesForProfile(profile_id) for dependency in dependencies: chain.extend(self.getProfileDependencyChain(dependency, seen)) chain.append(profile_id) return chain security.declarePrivate('_runImportStepsFromContext') def _runImportStepsFromContext(self, steps=None, purge_old=None, profile_id=None, archive=None, ignore_dependencies=False, seen=None, blacklisted_steps=None): if profile_id is not None and not ignore_dependencies: try: chain = self.getProfileDependencyChain(profile_id) except KeyError, e: logger = logging.getLogger('GenericSetup') logger.error('Unknown step in dependency chain: %s' % str(e)) raise else:
class SetupTool(Folder): """ Profile-based site configuration manager. """ implements(ISetupTool) meta_type = "Generic Setup Tool" _baseline_context_id = "" _profile_upgrade_versions = {} _exclude_global_steps = False security = ClassSecurityInfo() def __init__(self, id): self.id = str(id) self._import_registry = ImportStepRegistry() self._export_registry = ExportStepRegistry() self._toolset_registry = ToolsetRegistry() # # ISetupTool API # security.declareProtected(ManagePortal, "getEncoding") def getEncoding(self): """ See ISetupTool. """ return "utf-8" security.declareProtected(ManagePortal, "getBaselineContextID") def getBaselineContextID(self): """ See ISetupTool. """ return self._baseline_context_id security.declareProtected(ManagePortal, "setBaselineContext") def setBaselineContext(self, context_id, encoding=None): """ See ISetupTool. """ self._baseline_context_id = context_id self.applyContextById(context_id, encoding) security.declareProtected(ManagePortal, "getExcludeGlobalSteps") def getExcludeGlobalSteps(self): """ See ISetupTool. """ return self._exclude_global_steps security.declareProtected(ManagePortal, "setExcludeGlobalSteps") def setExcludeGlobalSteps(self, value): """ See ISetupTool. """ self._exclude_global_steps = value security.declareProtected(ManagePortal, "applyContextById") def applyContextById(self, context_id, encoding=None): context = self._getImportContext(context_id) self.applyContext(context, encoding) security.declareProtected(ManagePortal, "applyContext") def applyContext(self, context, encoding=None): self._updateImportStepsRegistry(context, encoding) self._updateExportStepsRegistry(context, encoding) security.declareProtected(ManagePortal, "getImportStepRegistry") def getImportStepRegistry(self): """ See ISetupTool. """ return self._import_registry security.declareProtected(ManagePortal, "getExportStepRegistry") def getExportStepRegistry(self): """ See ISetupTool. """ return self._export_registry security.declareProtected(ManagePortal, "getImportStep") def getImportStep(self, step, default=None): """Simple wrapper to query both the global and local step registry.""" res = self._import_registry.getStep(step, self) if res is self and not self._exclude_global_steps: res = _import_step_registry.getStep(step, self) if res is not self: return res return default security.declareProtected(ManagePortal, "getSortedImportSteps") def getSortedImportSteps(self): if self._exclude_global_steps: steps = set() else: steps = set(_import_step_registry.listSteps()) steps.update(set(self._import_registry.listSteps())) step_infos = [self.getImportStepMetadata(step) for step in steps] return tuple(_computeTopologicalSort(step_infos)) security.declareProtected(ManagePortal, "getImportStepMetadata") def getImportStepMetadata(self, step, default=None): """Simple wrapper to query both the global and local step registry.""" res = self._import_registry.getStepMetadata(step, self) if res is self and not self._exclude_global_steps: res = _import_step_registry.getStepMetadata(step, default) if res is not self: return res return default security.declareProtected(ManagePortal, "getExportStep") def getExportStep(self, step, default=None): """Simple wrapper to query both the global and local step registry.""" res = self._export_registry.getStep(step, self) if res is self and not self._exclude_global_steps: res = _export_step_registry.getStep(step, self) if res is not self: return res return default security.declareProtected(ManagePortal, "listExportSteps") def listExportSteps(self): steps = set(self._export_registry.listSteps()) if not self._exclude_global_steps: steps.update(set(_export_step_registry.listSteps())) return tuple(steps) security.declareProtected(ManagePortal, "listImportSteps") def listImportSteps(self): steps = set(self._import_registry.listSteps()) if not self._exclude_global_steps: steps.update(set(_import_step_registry.listSteps())) return tuple(steps) security.declareProtected(ManagePortal, "getExportStepMetadata") def getExportStepMetadata(self, step, default=None): """Simple wrapper to query both the global and local step registry.""" res = self._export_registry.getStepMetadata(step, self) if res is self and not self._exclude_global_steps: res = _export_step_registry.getStepMetadata(step, default) if res is not self: return res return default security.declareProtected(ManagePortal, "getToolsetRegistry") def getToolsetRegistry(self): """ See ISetupTool. """ return self._toolset_registry security.declareProtected(ManagePortal, "runImportStepFromProfile") def runImportStepFromProfile(self, profile_id, step_id, run_dependencies=True, purge_old=None): """ See ISetupTool. """ context = self._getImportContext(profile_id, purge_old) self.applyContext(context) info = self.getImportStepMetadata(step_id) if info is None: generic_logger.error( "No such import step: '%s' Maybe you meant one of %s", step_id, str(self.listImportSteps()) ) raise ValueError("No such import step: %s" % step_id) dependencies = info.get("dependencies", ()) messages = {} steps = [] if run_dependencies: for dependency in dependencies: if dependency not in steps: steps.append(dependency) steps.append(step_id) full_import = set(steps) == set(self.getSortedImportSteps()) event.notify(BeforeProfileImportEvent(self, profile_id, steps, full_import)) for step in steps: message = self._doRunImportStep(step, context) messages[step] = message or "" message_list = filter(None, [message]) message_list.extend(["%s: %s" % x[1:] for x in context.listNotes()]) messages[step_id] = "\n".join(message_list) event.notify(ProfileImportedEvent(self, profile_id, steps, full_import)) return {"steps": steps, "messages": messages} security.declareProtected(ManagePortal, "runAllImportStepsFromProfile") def runAllImportStepsFromProfile( self, profile_id, purge_old=None, ignore_dependencies=False, archive=None, blacklisted_steps=None, dependency_strategy=None, ): """ See ISetupTool. """ __traceback_info__ = profile_id result = self._runImportStepsFromContext( purge_old=purge_old, profile_id=profile_id, archive=archive, ignore_dependencies=ignore_dependencies, blacklisted_steps=blacklisted_steps, dependency_strategy=dependency_strategy, ) if profile_id is None: prefix = "import-all-from-tar" else: prefix = "import-all-%s" % profile_id.replace(":", "_") name = self._mangleTimestampName(prefix, "log") self._createReport(name, result["steps"], result["messages"]) return result security.declareProtected(ManagePortal, "runExportStep") def runExportStep(self, step_id): """ See ISetupTool. """ return self._doRunExportSteps([step_id]) security.declareProtected(ManagePortal, "runAllExportSteps") def runAllExportSteps(self): """ See ISetupTool. """ return self._doRunExportSteps(self.listExportSteps()) security.declareProtected(ManagePortal, "createSnapshot") def createSnapshot(self, snapshot_id): """ See ISetupTool. """ context = SnapshotExportContext(self, snapshot_id) messages = {} steps = self.listExportSteps() for step_id in steps: handler = self.getExportStep(step_id) if handler is None: logger = logging.getLogger("GenericSetup") logger.error("Step %s has an invalid handler" % step_id) continue messages[step_id] = handler(context) return { "steps": steps, "messages": messages, "url": context.getSnapshotURL(), "snapshot": context.getSnapshotFolder(), } security.declareProtected(ManagePortal, "compareConfigurations") def compareConfigurations( self, lhs_context, rhs_context, missing_as_empty=False, ignore_blanks=False, skip=SKIPPED_FILES ): """ See ISetupTool. """ differ = ConfigDiff(lhs_context, rhs_context, missing_as_empty, ignore_blanks, skip) return differ.compare() security.declareProtected(ManagePortal, "markupComparison") def markupComparison(self, lines): """ See ISetupTool. """ result = [] for line in lines.splitlines(): if line.startswith("** "): if line.find("File") > -1: if line.find("replaced") > -1: result.append(("file-to-dir", line)) elif line.find("added") > -1: result.append(("file-added", line)) else: result.append(("file-removed", line)) else: if line.find("replaced") > -1: result.append(("dir-to-file", line)) elif line.find("added") > -1: result.append(("dir-added", line)) else: result.append(("dir-removed", line)) elif line.startswith("@@"): result.append(("diff-range", line)) elif line.startswith(" "): result.append(("diff-context", line)) elif line.startswith("+"): result.append(("diff-added", line)) elif line.startswith("-"): result.append(("diff-removed", line)) elif line == "\ No newline at end of file": result.append(("diff-context", line)) else: result.append(("diff-header", line)) return "<pre>\n%s\n</pre>" % ("\n".join([('<span class="%s">%s</span>' % (cl, escape(l))) for cl, l in result])) # # ZMI # manage_options = ( Folder.manage_options[:1] + ( {"label": "Profiles", "action": "manage_tool"}, {"label": "Import", "action": "manage_importSteps"}, {"label": "Export", "action": "manage_exportSteps"}, {"label": "Upgrades", "action": "manage_upgrades"}, {"label": "Snapshots", "action": "manage_snapshots"}, {"label": "Comparison", "action": "manage_showDiff"}, {"label": "Manage", "action": "manage_stepRegistry"}, ) + Folder.manage_options[3:] ) # skip "View", "Properties" security.declareProtected(ManagePortal, "manage_tool") manage_tool = PageTemplateFile("sutProperties", _wwwdir) security.declareProtected(ManagePortal, "manage_updateToolProperties") def manage_updateToolProperties(self, context_id, exclude_global_steps=False, RESPONSE=None): """ Update the tool's settings. """ self.setExcludeGlobalSteps(exclude_global_steps) self.setBaselineContext(context_id) if RESPONSE is not None: RESPONSE.redirect("%s/manage_tool?manage_tabs_message=%s" % (self.absolute_url(), "Properties+updated.")) security.declareProtected(ManagePortal, "manage_importSteps") manage_importSteps = PageTemplateFile("sutImportSteps", _wwwdir) security.declareProtected(ManagePortal, "manage_importSelectedSteps") def manage_importSelectedSteps(self, ids, run_dependencies, context_id=None): """ Import the steps selected by the user. """ messages = {} if not ids: summary = "No steps selected." else: if context_id is None: context_id = self.getBaselineContextID() steps_run = [] for step_id in ids: result = self.runImportStepFromProfile(context_id, step_id, run_dependencies) steps_run.extend(result["steps"]) messages.update(result["messages"]) summary = "Steps run: %s" % ", ".join(steps_run) name = self._mangleTimestampName("import-selected", "log") self._createReport(name, result["steps"], result["messages"]) return self.manage_importSteps(manage_tabs_message=summary, messages=messages) security.declareProtected(ManagePortal, "manage_importAllSteps") def manage_importAllSteps(self, context_id=None, dependency_strategy=None): """ Import all steps. """ if context_id is None: context_id = self.getBaselineContextID() result = self.runAllImportStepsFromProfile(context_id, purge_old=None, dependency_strategy=dependency_strategy) steps_run = "Steps run: %s" % ", ".join(result["steps"]) return self.manage_importSteps(manage_tabs_message=steps_run, messages=result["messages"]) security.declareProtected(ManagePortal, "manage_importExtensions") def manage_importExtensions(self, RESPONSE, profile_ids=()): """ Import all steps for the selected extension profiles. """ detail = {} if len(profile_ids) == 0: message = "Please select one or more extension profiles." RESPONSE.redirect("%s/manage_tool?manage_tabs_message=%s" % (self.absolute_url(), message)) else: message = "Imported profiles: %s" % ", ".join(profile_ids) for profile_id in profile_ids: result = self.runAllImportStepsFromProfile(profile_id) for k, v in result["messages"].items(): detail["%s:%s" % (profile_id, k)] = v return self.manage_importSteps(manage_tabs_message=message, messages=detail) security.declareProtected(ManagePortal, "manage_importTarball") def manage_importTarball(self, tarball): """ Import steps from the uploaded tarball. """ if getattr(tarball, "read", None) is not None: tarball = tarball.read() result = self.runAllImportStepsFromProfile(None, True, archive=tarball) steps_run = "Steps run: %s" % ", ".join(result["steps"]) return self.manage_importSteps(manage_tabs_message=steps_run, messages=result["messages"]) security.declareProtected(ManagePortal, "manage_exportSteps") manage_exportSteps = PageTemplateFile("sutExportSteps", _wwwdir) security.declareProtected(ManagePortal, "manage_exportSelectedSteps") def manage_exportSelectedSteps(self, ids, RESPONSE): """ Export the steps selected by the user. """ if not ids: RESPONSE.redirect( "%s/manage_exportSteps?manage_tabs_message=%s" % (self.absolute_url(), "No+steps+selected.") ) result = self._doRunExportSteps(ids) RESPONSE.setHeader("Content-type", "application/x-gzip") RESPONSE.setHeader("Content-disposition", "attachment; filename=%s" % result["filename"]) return result["tarball"] security.declareProtected(ManagePortal, "manage_exportAllSteps") def manage_exportAllSteps(self, RESPONSE): """ Export all steps. """ result = self.runAllExportSteps() RESPONSE.setHeader("Content-type", "application/x-gzip") RESPONSE.setHeader("Content-disposition", "attachment; filename=%s" % result["filename"]) return result["tarball"] security.declareProtected(ManagePortal, "manage_upgrades") manage_upgrades = PageTemplateFile("setup_upgrades", _wwwdir) security.declareProtected(ManagePortal, "upgradeStepMacro") upgradeStepMacro = PageTemplateFile("upgradeStep", _wwwdir) security.declareProtected(ManagePortal, "manage_snapshots") manage_snapshots = PageTemplateFile("sutSnapshots", _wwwdir) security.declareProtected(ManagePortal, "listSnapshotInfo") def listSnapshotInfo(self): """ Return a list of mappings describing available snapshots. o Keys include: 'id' -- snapshot ID 'title' -- snapshot title or ID 'url' -- URL of the snapshot folder """ result = [] snapshots = self._getOb("snapshots", None) if snapshots: for id, folder in snapshots.objectItems("Folder"): result.append({"id": id, "title": folder.title_or_id(), "url": folder.absolute_url()}) return result security.declareProtected(ManagePortal, "listProfileInfo") def listProfileInfo(self, for_=None): """ Return a list of mappings describing registered profiles. Base profile is listed first, extensions are sorted. o Keys include: 'id' -- profile ID 'title' -- profile title or ID 'description' -- description of the profile 'path' -- path to the profile within its product 'product' -- name of the registering product """ base = [] ext = [] for info in _profile_registry.listProfileInfo(for_): if info.get("type", BASE) == BASE: base.append(info) else: ext.append(info) ext.sort(lambda x, y: cmp(x["id"], y["id"])) return base + ext security.declareProtected(ManagePortal, "listContextInfos") def listContextInfos(self): """ List registered profiles and snapshots. """ def readableType(x): if x is BASE: return "base" elif x is EXTENSION: return "extension" return "unknown" s_infos = [ {"id": "snapshot-%s" % info["id"], "title": info["title"], "type": "snapshot"} for info in self.listSnapshotInfo() ] s_infos.sort(key=itemgetter("title")) p_infos = [ {"id": "profile-%s" % info["id"], "title": info["title"], "type": readableType(info["type"])} for info in self.listProfileInfo() ] p_infos.sort(key=itemgetter("title")) return tuple(s_infos + p_infos) security.declareProtected(ManagePortal, "getProfileImportDate") def getProfileImportDate(self, profile_id): """ See ISetupTool. """ prefix = ("import-all-%s-" % profile_id).replace(":", "_") candidates = [x for x in self.objectIds("File") if x[:-18] == prefix and x.endswith(".log")] if len(candidates) == 0: return None candidates.sort() last = candidates[-1] stamp = last[-18:-4] return "%s-%s-%sT%s:%s:%sZ" % (stamp[0:4], stamp[4:6], stamp[6:8], stamp[8:10], stamp[10:12], stamp[12:14]) security.declareProtected(ManagePortal, "manage_createSnapshot") def manage_createSnapshot(self, RESPONSE, snapshot_id=None): """ Create a snapshot with the given ID. o If no ID is passed, generate one. """ if snapshot_id is None: snapshot_id = self._mangleTimestampName("snapshot") self.createSnapshot(snapshot_id) return RESPONSE.redirect( "%s/manage_snapshots?manage_tabs_message=%s" % (self.absolute_url(), "Snapshot+created.") ) security.declareProtected(ManagePortal, "manage_showDiff") manage_showDiff = PageTemplateFile("sutCompare", _wwwdir) def manage_downloadDiff(self, lhs, rhs, missing_as_empty, ignore_blanks, RESPONSE): """ Crack request vars and call compareConfigurations. o Return the result as a 'text/plain' stream, suitable for framing. """ comparison = self.manage_compareConfigurations(lhs, rhs, missing_as_empty, ignore_blanks) RESPONSE.setHeader("Content-Type", "text/plain") return _PLAINTEXT_DIFF_HEADER % (lhs, rhs, comparison) security.declareProtected(ManagePortal, "manage_compareConfigurations") def manage_compareConfigurations(self, lhs, rhs, missing_as_empty, ignore_blanks): """ Crack request vars and call compareConfigurations. """ lhs_context = self._getImportContext(lhs) rhs_context = self._getImportContext(rhs) return self.compareConfigurations(lhs_context, rhs_context, missing_as_empty, ignore_blanks) security.declareProtected(ManagePortal, "manage_stepRegistry") manage_stepRegistry = PageTemplateFile("sutManage", _wwwdir) security.declareProtected(ManagePortal, "manage_deleteImportSteps") def manage_deleteImportSteps(self, ids, request=None): """ Delete selected import steps. """ if request is None: request = self.REQUEST for id in ids: self._import_registry.unregisterStep(id) self._p_changed = True url = self.absolute_url() request.RESPONSE.redirect("%s/manage_stepRegistry" % url) security.declareProtected(ManagePortal, "manage_deleteExportSteps") def manage_deleteExportSteps(self, ids, request=None): """ Delete selected export steps. """ if request is None: request = self.REQUEST for id in ids: self._export_registry.unregisterStep(id) self._p_changed = True url = self.absolute_url() request.RESPONSE.redirect("%s/manage_stepRegistry" % url) # # Upgrades management # security.declareProtected(ManagePortal, "getLastVersionForProfile") def getLastVersionForProfile(self, profile_id): """Return the last upgraded version for the specified profile. """ prefix = "profile-" if profile_id.startswith(prefix): profile_id = profile_id[len(prefix) :] version = self._profile_upgrade_versions.get(profile_id, UNKNOWN) return version security.declareProtected(ManagePortal, "setLastVersionForProfile") def setLastVersionForProfile(self, profile_id, version): """Set the last upgraded version for the specified profile. """ if version == UNKNOWN: self.unsetLastVersionForProfile(profile_id) return prefix = "profile-" if profile_id.startswith(prefix): profile_id = profile_id[len(prefix) :] if isinstance(version, basestring): version = tuple(version.split(".")) # _profile_upgrade_versions is not persistent, so we must # force a safe by storing it fresh on self, instead of editing # the current dictionary. prof_versions = self._profile_upgrade_versions.copy() prof_versions[profile_id] = version self._profile_upgrade_versions = prof_versions security.declareProtected(ManagePortal, "unsetLastVersionForProfile") def unsetLastVersionForProfile(self, profile_id): """Unset the last upgraded version for the specified profile. """ prefix = "profile-" if profile_id.startswith(prefix): profile_id = profile_id[len(prefix) :] # _profile_upgrade_versions is not persistent, so we must # force a safe by storing it fresh on self, instead of editing # the current dictionary. prof_versions = self._profile_upgrade_versions.copy() if profile_id not in prof_versions: return del prof_versions[profile_id] self._profile_upgrade_versions = prof_versions security.declareProtected(ManagePortal, "getVersionForProfile") def getVersionForProfile(self, profile_id): """Return the registered filesystem version for the specified profile. """ return self.getProfileInfo(profile_id).get("version", UNKNOWN) security.declareProtected(ManagePortal, "purgeProfileVersions") def purgeProfileVersions(self): """Purge the profile upgrade versions. """ self._profile_upgrade_versions = {} generic_logger.info("Profile upgrade versions purged.") security.declareProtected(ManagePortal, "profileExists") def profileExists(self, profile_id): """Check if a profile exists.""" try: self.getProfileInfo(profile_id) except KeyError: return False else: return True security.declareProtected(ManagePortal, "getProfileInfo") def getProfileInfo(self, profile_id): return _profile_registry.getProfileInfo(profile_id) security.declareProtected(ManagePortal, "getDependenciesForProfile") def getDependenciesForProfile(self, profile_id): if profile_id.startswith("snapshot-"): return () if not self.profileExists(profile_id): raise KeyError(profile_id) try: return self.getProfileInfo(profile_id).get("dependencies", ()) except KeyError: return () security.declareProtected(ManagePortal, "listProfilesWithUpgrades") def listProfilesWithUpgrades(self): profiles = listProfilesWithUpgrades() profiles.sort() return profiles security.declarePrivate("_massageUpgradeInfo") def _massageUpgradeInfo(self, info): """Add a couple of data points to the upgrade info dictionary. """ info = info.copy() info["haspath"] = info["source"] and info["dest"] info["ssource"] = ".".join(info["source"] or ("all",)) info["sdest"] = ".".join(info["dest"] or ("all",)) info["done"] = not info["proposed"] and info["step"].checker is not None and not info["step"].checker(self) return info security.declareProtected(ManagePortal, "listUpgrades") def listUpgrades(self, profile_id, show_old=False): """Get the list of available upgrades. """ if show_old: source = None else: source = self.getLastVersionForProfile(profile_id) upgrades = listUpgradeSteps(self, profile_id, source) res = [] for info in upgrades: if type(info) == list: subset = [] for subinfo in info: subset.append(self._massageUpgradeInfo(subinfo)) res.append(subset) else: res.append(self._massageUpgradeInfo(info)) return res security.declareProtected(ManagePortal, "manage_doUpgrades") def manage_doUpgrades(self, request=None): """Perform all selected upgrade steps. """ if request is None: request = self.REQUEST logger = logging.getLogger("GenericSetup") steps_to_run = request.form.get("upgrades", []) profile_id = request.get("profile_id", "") step = None for step_id in steps_to_run: step = _upgrade_registry.getUpgradeStep(profile_id, step_id) if step is not None: step.doStep(self) msg = "Ran upgrade step %s for profile %s" % (step.title, profile_id) logger.log(logging.INFO, msg) # We update the profile version to the last one we have reached # with running an upgrade step. if step and step.dest is not None: self.setLastVersionForProfile(profile_id, step.dest) url = self.absolute_url() request.RESPONSE.redirect("%s/manage_upgrades?saved=%s" % (url, profile_id)) security.declareProtected(ManagePortal, "upgradeProfile") def upgradeProfile(self, profile_id, dest=None): """Upgrade a profile. Apply all upgrade steps. When 'dest' is given, only update to that version. If the version is not found, give a warning and do nothing. If the profile was not applied previously (last version for profile is unknown) we do nothing. """ if self.getLastVersionForProfile(profile_id) == UNKNOWN: generic_logger.warn("Version of profile %s is unknown, " "refusing to upgrade.", profile_id) return if dest is not None: # Upgrade to a specific destination version, if found. if isinstance(dest, basestring): dest = tuple(dest.split(".")) if self.getLastVersionForProfile(profile_id) == dest: generic_logger.warn("Profile %s is already at wanted " "destination %r.", profile_id, dest) return upgrades = self.listUpgrades(profile_id) # First get a list of single steps to apply. This may be # limited by the wanted destination version. to_apply = [] dest_found = False step = None for upgrade in upgrades: # An upgrade may be a single step (for a bare upgradeStep) # or a list of steps (for upgradeSteps containing upgradeStep # directives). if not isinstance(upgrade, list): upgrade = [upgrade] for upgradestep in upgrade: step = upgradestep["step"] to_apply.append(step) if dest is not None and step.dest == dest: dest_found = True if dest_found: break if dest is not None and not dest_found: generic_logger.warn( "No route found to destination version %r for profile %s. " "Profile stays at current version, %r", dest, profile_id, self.getLastVersionForProfile(profile_id), ) return if to_apply: for step in to_apply: step.doStep(self) # We update the profile version to the last one we have # reached with running an upgrade step. if step and step.dest is not None: self.setLastVersionForProfile(profile_id, step.dest) generic_logger.info( "Profile %s upgraded to version %r.", profile_id, self.getLastVersionForProfile(profile_id) ) else: generic_logger.info( "No upgrades available for profile %s. " "Profile stays at version %r.", profile_id, self.getLastVersionForProfile(profile_id), ) # # Helper methods # security.declarePrivate("_getImportContext") def _getImportContext(self, context_id, should_purge=None, archive=None): """ Crack ID and generate appropriate import context. Note: it seems context_id (profile id) and archive (tarball) are mutually exclusive. Exactly one of the two should be None. There seems to be no use case for a different combination. """ encoding = self.getEncoding() if context_id is not None: prefix = "snapshot-" if context_id.startswith(prefix): context_id = context_id[len(prefix) :] if should_purge is None: should_purge = True return SnapshotImportContext(self, context_id, should_purge, encoding) prefix = "profile-" if context_id.startswith(prefix): context_id = context_id[len(prefix) :] info = _profile_registry.getProfileInfo(context_id) if info.get("product"): path = os.path.join(_getProductPath(info["product"]), info["path"]) else: path = info["path"] if should_purge is None: should_purge = info.get("type") != EXTENSION return DirectoryImportContext(self, path, should_purge, encoding) if archive is not None: return TarballImportContext(tool=self, archive_bits=archive, encoding="UTF8", should_purge=should_purge) raise KeyError('Unknown context "%s"' % context_id) security.declarePrivate("_updateImportStepsRegistry") def _updateImportStepsRegistry(self, context, encoding): """ Update our import steps registry from our profile. """ xml = context.readDataFile(IMPORT_STEPS_XML) if xml is None: return info_list = self._import_registry.parseXML(xml, encoding) for step_info in info_list: id = step_info["id"] version = step_info.get("version") handler = step_info["handler"] dependencies = tuple(step_info.get("dependencies", ())) title = step_info.get("title", id) description = "".join(step_info.get("description", [])) self._import_registry.registerStep( id=id, version=version, handler=handler, dependencies=dependencies, title=title, description=description ) security.declarePrivate("_updateExportStepsRegistry") def _updateExportStepsRegistry(self, context, encoding): """ Update our export steps registry from our profile. """ xml = context.readDataFile(EXPORT_STEPS_XML) if xml is None: return info_list = self._export_registry.parseXML(xml, encoding) for step_info in info_list: id = step_info["id"] handler = step_info["handler"] title = step_info.get("title", id) description = "".join(step_info.get("description", [])) self._export_registry.registerStep(id=id, handler=handler, title=title, description=description) security.declarePrivate("_doRunImportStep") def _doRunImportStep(self, step_id, context): """ Run a single import step, using a pre-built context. """ __traceback_info__ = step_id marker = object() handler = self.getImportStep(step_id) if handler is marker: raise ValueError("Invalid import step: %s" % step_id) if handler is None: msg = "Step %s has an invalid import handler" % step_id logger = logging.getLogger("GenericSetup") logger.error(msg) return "ERROR: " + msg return handler(context) security.declarePrivate("_doRunExportSteps") def _doRunExportSteps(self, steps): """ See ISetupTool. """ context = TarballExportContext(self) messages = {} marker = object() for step_id in steps: handler = self.getExportStep(step_id, marker) if handler is marker: raise ValueError("Invalid export step: %s" % step_id) if handler is None: msg = "Step %s has an invalid export handler" % step_id logger = logging.getLogger("GenericSetup") logger.error(msg) messages[step_id] = msg else: messages[step_id] = handler(context) return { "steps": steps, "messages": messages, "tarball": context.getArchive(), "filename": context.getArchiveFilename(), } security.declareProtected(ManagePortal, "getProfileDependencyChain") def getProfileDependencyChain(self, profile_id, seen=None): if seen is None: seen = set() elif profile_id in seen: return [] # cycle break seen.add(profile_id) chain = [] dependencies = self.getDependenciesForProfile(profile_id) for dependency in dependencies: chain.extend(self.getProfileDependencyChain(dependency, seen)) chain.append(profile_id) return chain security.declarePrivate("_runImportStepsFromContext") def _runImportStepsFromContext( self, steps=None, purge_old=None, profile_id=None, archive=None, ignore_dependencies=False, blacklisted_steps=None, dependency_strategy=None, ): # 1. Determine upgrade strategy. # What do we do with already applied dependency profiles? # There are two ways to say you want to ignore all # dependencies. If one is enabled, we enable the other too. if dependency_strategy == DEPENDENCY_STRATEGY_IGNORE: ignore_dependencies = True elif ignore_dependencies: dependency_strategy = DEPENDENCY_STRATEGY_IGNORE # Turn None into the default: if dependency_strategy is None: dependency_strategy = DEFAULT_DEPENDENCY_STRATEGY # Determine the settings based on the strategy. if dependency_strategy == DEPENDENCY_STRATEGY_UPGRADE: apply_new_profiles = True reapply_old_profiles = False upgrade_old_profiles = True elif dependency_strategy == DEPENDENCY_STRATEGY_REAPPLY: apply_new_profiles = True reapply_old_profiles = True upgrade_old_profiles = False elif dependency_strategy == DEPENDENCY_STRATEGY_NEW: apply_new_profiles = True reapply_old_profiles = False upgrade_old_profiles = False elif dependency_strategy == DEPENDENCY_STRATEGY_IGNORE: apply_new_profiles = False reapply_old_profiles = False upgrade_old_profiles = False else: raise ValueError("Unknown dependency_strategy %r." % dependency_strategy) generic_logger.info("Importing profile %s with dependency strategy %s.", profile_id, dependency_strategy) # 2. Gather a list of profiles to handle. if profile_id is not None and not ignore_dependencies: try: chain = self.getProfileDependencyChain(profile_id) except KeyError, e: logger = logging.getLogger("GenericSetup") logger.error("Unknown step in dependency chain: %s" % str(e)) raise else: