def __init__(self, id):
self.id = str(id)
self._import_registry = ImportStepRegistry()
self._export_registry = ExportStepRegistry()
self._toolset_registry = ToolsetRegistry()
class SetupTool(Folder):
""" Profile-based site configuration manager.
"""
implements(ISetupTool)
meta_type = 'Generic Setup Tool'
_baseline_context_id = ''
_profile_upgrade_versions = {}
_exclude_global_steps = False
security = ClassSecurityInfo()
def __init__(self, id):
self.id = str(id)
self._import_registry = ImportStepRegistry()
self._export_registry = ExportStepRegistry()
self._toolset_registry = ToolsetRegistry()
#
# ISetupTool API
#
security.declareProtected(ManagePortal, 'getEncoding')
def getEncoding(self):
""" See ISetupTool.
"""
return 'utf-8'
security.declareProtected(ManagePortal, 'getBaselineContextID')
def getBaselineContextID(self):
""" See ISetupTool.
"""
return self._baseline_context_id
security.declareProtected(ManagePortal, 'setBaselineContext')
def setBaselineContext(self, context_id, encoding=None):
""" See ISetupTool.
"""
self._baseline_context_id = context_id
self.applyContextById(context_id, encoding)
security.declareProtected(ManagePortal, 'getExcludeGlobalSteps')
def getExcludeGlobalSteps(self):
""" See ISetupTool.
"""
return self._exclude_global_steps
security.declareProtected(ManagePortal, 'setExcludeGlobalSteps')
def setExcludeGlobalSteps(self, value):
""" See ISetupTool.
"""
self._exclude_global_steps = value
security.declareProtected(ManagePortal, 'applyContextById')
def applyContextById(self, context_id, encoding=None):
context = self._getImportContext(context_id)
self.applyContext(context, encoding)
security.declareProtected(ManagePortal, 'applyContext')
def applyContext(self, context, encoding=None):
self._updateImportStepsRegistry(context, encoding)
self._updateExportStepsRegistry(context, encoding)
security.declareProtected(ManagePortal, 'getImportStepRegistry')
def getImportStepRegistry(self):
""" See ISetupTool.
"""
return self._import_registry
security.declareProtected(ManagePortal, 'getExportStepRegistry')
def getExportStepRegistry(self):
""" See ISetupTool.
"""
return self._export_registry
security.declareProtected(ManagePortal, 'getImportStep')
def getImportStep(self, step, default=None):
"""Simple wrapper to query both the global and local step registry."""
res = self._import_registry.getStep(step, self)
if res is self and not self._exclude_global_steps:
res = _import_step_registry.getStep(step, self)
if res is not self:
return res
return default
security.declareProtected(ManagePortal, 'getSortedImportSteps')
def getSortedImportSteps(self):
if self._exclude_global_steps:
steps = set()
else:
steps = set(_import_step_registry.listSteps())
steps.update(set(self._import_registry.listSteps()))
step_infos = [self.getImportStepMetadata(step) for step in steps]
return tuple(_computeTopologicalSort(step_infos))
security.declareProtected(ManagePortal, 'getImportStepMetadata')
def getImportStepMetadata(self, step, default=None):
"""Simple wrapper to query both the global and local step registry."""
res = self._import_registry.getStepMetadata(step, self)
if res is self and not self._exclude_global_steps:
res = _import_step_registry.getStepMetadata(step, default)
if res is not self:
return res
return default
security.declareProtected(ManagePortal, 'getExportStep')
def getExportStep(self, step, default=None):
"""Simple wrapper to query both the global and local step registry."""
res = self._export_registry.getStep(step, self)
if res is self and not self._exclude_global_steps:
res = _export_step_registry.getStep(step, self)
if res is not self:
return res
return default
security.declareProtected(ManagePortal, 'listExportSteps')
def listExportSteps(self):
steps = set(self._export_registry.listSteps())
if not self._exclude_global_steps:
steps.update(set(_export_step_registry.listSteps()))
return tuple(steps)
security.declareProtected(ManagePortal, 'listImportSteps')
def listImportSteps(self):
steps = set(self._import_registry.listSteps())
if not self._exclude_global_steps:
steps.update(set(_import_step_registry.listSteps()))
return tuple(steps)
security.declareProtected(ManagePortal, 'getExportStepMetadata')
def getExportStepMetadata(self, step, default=None):
"""Simple wrapper to query both the global and local step registry."""
res = self._export_registry.getStepMetadata(step, self)
if res is self and not self._exclude_global_steps:
res = _export_step_registry.getStepMetadata(step, default)
if res is not self:
return res
return default
security.declareProtected(ManagePortal, 'getToolsetRegistry')
def getToolsetRegistry(self):
""" See ISetupTool.
"""
return self._toolset_registry
security.declareProtected(ManagePortal, 'runImportStepFromProfile')
def runImportStepFromProfile(self,
profile_id,
step_id,
run_dependencies=True,
purge_old=None):
""" See ISetupTool.
"""
context = self._getImportContext(profile_id, purge_old)
self.applyContext(context)
info = self.getImportStepMetadata(step_id)
if info is None:
generic_logger.error(
"No such import step: '%s' Maybe you meant one of %s", step_id,
str(self.listImportSteps()))
raise ValueError('No such import step: %s' % step_id)
dependencies = info.get('dependencies', ())
messages = {}
steps = []
if run_dependencies:
for dependency in dependencies:
if dependency not in steps:
steps.append(dependency)
steps.append(step_id)
full_import = (set(steps) == set(self.getSortedImportSteps()))
event.notify(
BeforeProfileImportEvent(self, profile_id, steps, full_import))
for step in steps:
message = self._doRunImportStep(step, context)
messages[step] = message or ''
message_list = filter(None, [message])
message_list.extend(['%s: %s' % x[1:] for x in context.listNotes()])
messages[step_id] = '\n'.join(message_list)
event.notify(ProfileImportedEvent(self, profile_id, steps,
full_import))
return {'steps': steps, 'messages': messages}
security.declareProtected(ManagePortal, 'runAllImportStepsFromProfile')
def runAllImportStepsFromProfile(self,
profile_id,
purge_old=None,
ignore_dependencies=False,
archive=None,
blacklisted_steps=None):
""" See ISetupTool.
"""
__traceback_info__ = profile_id
result = self._runImportStepsFromContext(
purge_old=purge_old,
profile_id=profile_id,
archive=archive,
ignore_dependencies=ignore_dependencies,
blacklisted_steps=blacklisted_steps)
if profile_id is None:
prefix = 'import-all-from-tar'
else:
prefix = 'import-all-%s' % profile_id.replace(':', '_')
name = self._mangleTimestampName(prefix, 'log')
self._createReport(name, result['steps'], result['messages'])
return result
security.declareProtected(ManagePortal, 'runExportStep')
def runExportStep(self, step_id):
""" See ISetupTool.
"""
return self._doRunExportSteps([step_id])
security.declareProtected(ManagePortal, 'runAllExportSteps')
def runAllExportSteps(self):
""" See ISetupTool.
"""
return self._doRunExportSteps(self.listExportSteps())
security.declareProtected(ManagePortal, 'createSnapshot')
def createSnapshot(self, snapshot_id):
""" See ISetupTool.
"""
context = SnapshotExportContext(self, snapshot_id)
messages = {}
steps = self.listExportSteps()
for step_id in steps:
handler = self.getExportStep(step_id)
if handler is None:
logger = logging.getLogger('GenericSetup')
logger.error('Step %s has an invalid handler' % step_id)
continue
messages[step_id] = handler(context)
return {
'steps': steps,
'messages': messages,
'url': context.getSnapshotURL(),
'snapshot': context.getSnapshotFolder()
}
security.declareProtected(ManagePortal, 'compareConfigurations')
def compareConfigurations(
self,
lhs_context,
rhs_context,
missing_as_empty=False,
ignore_blanks=False,
skip=SKIPPED_FILES,
):
""" See ISetupTool.
"""
differ = ConfigDiff(
lhs_context,
rhs_context,
missing_as_empty,
ignore_blanks,
skip,
)
return differ.compare()
security.declareProtected(ManagePortal, 'markupComparison')
def markupComparison(self, lines):
""" See ISetupTool.
"""
result = []
for line in lines.splitlines():
if line.startswith('** '):
if line.find('File') > -1:
if line.find('replaced') > -1:
result.append(('file-to-dir', line))
elif line.find('added') > -1:
result.append(('file-added', line))
else:
result.append(('file-removed', line))
else:
if line.find('replaced') > -1:
result.append(('dir-to-file', line))
elif line.find('added') > -1:
result.append(('dir-added', line))
else:
result.append(('dir-removed', line))
elif line.startswith('@@'):
result.append(('diff-range', line))
elif line.startswith(' '):
result.append(('diff-context', line))
elif line.startswith('+'):
result.append(('diff-added', line))
elif line.startswith('-'):
result.append(('diff-removed', line))
elif line == '\ No newline at end of file':
result.append(('diff-context', line))
else:
result.append(('diff-header', line))
return '<pre>\n%s\n</pre>' % ('\n'.join(
[('<span class="%s">%s</span>' % (cl, escape(l)))
for cl, l in result]))
#
# ZMI
#
manage_options = (
Folder.manage_options[:1] + ({
'label': 'Profiles',
'action': 'manage_tool'
}, {
'label': 'Import',
'action': 'manage_importSteps'
}, {
'label': 'Export',
'action': 'manage_exportSteps'
}, {
'label': 'Upgrades',
'action': 'manage_upgrades'
}, {
'label': 'Snapshots',
'action': 'manage_snapshots'
}, {
'label': 'Comparison',
'action': 'manage_showDiff'
}, {
'label': 'Manage',
'action': 'manage_stepRegistry'
}) + Folder.manage_options[3:]) # skip "View", "Properties"
security.declareProtected(ManagePortal, 'manage_tool')
manage_tool = PageTemplateFile('sutProperties', _wwwdir)
security.declareProtected(ManagePortal, 'manage_updateToolProperties')
def manage_updateToolProperties(self,
context_id,
exclude_global_steps=False,
RESPONSE=None):
""" Update the tool's settings.
"""
self.setExcludeGlobalSteps(exclude_global_steps)
self.setBaselineContext(context_id)
if RESPONSE is not None:
RESPONSE.redirect('%s/manage_tool?manage_tabs_message=%s' %
(self.absolute_url(), 'Properties+updated.'))
security.declareProtected(ManagePortal, 'manage_importSteps')
manage_importSteps = PageTemplateFile('sutImportSteps', _wwwdir)
security.declareProtected(ManagePortal, 'manage_importSelectedSteps')
def manage_importSelectedSteps(self,
ids,
run_dependencies,
context_id=None):
""" Import the steps selected by the user.
"""
messages = {}
if not ids:
summary = 'No steps selected.'
else:
if context_id is None:
context_id = self.getBaselineContextID()
steps_run = []
for step_id in ids:
result = self.runImportStepFromProfile(context_id, step_id,
run_dependencies)
steps_run.extend(result['steps'])
messages.update(result['messages'])
summary = 'Steps run: %s' % ', '.join(steps_run)
name = self._mangleTimestampName('import-selected', 'log')
self._createReport(name, result['steps'], result['messages'])
return self.manage_importSteps(manage_tabs_message=summary,
messages=messages)
security.declareProtected(ManagePortal, 'manage_importAllSteps')
def manage_importAllSteps(self, context_id=None):
""" Import all steps.
"""
if context_id is None:
context_id = self.getBaselineContextID()
result = self.runAllImportStepsFromProfile(context_id, purge_old=None)
steps_run = 'Steps run: %s' % ', '.join(result['steps'])
return self.manage_importSteps(manage_tabs_message=steps_run,
messages=result['messages'])
security.declareProtected(ManagePortal, 'manage_importExtensions')
def manage_importExtensions(self, RESPONSE, profile_ids=()):
""" Import all steps for the selected extension profiles.
"""
detail = {}
if len(profile_ids) == 0:
message = 'Please select one or more extension profiles.'
RESPONSE.redirect('%s/manage_tool?manage_tabs_message=%s' %
(self.absolute_url(), message))
else:
message = 'Imported profiles: %s' % ', '.join(profile_ids)
for profile_id in profile_ids:
result = self.runAllImportStepsFromProfile(profile_id)
for k, v in result['messages'].items():
detail['%s:%s' % (profile_id, k)] = v
return self.manage_importSteps(manage_tabs_message=message,
messages=detail)
security.declareProtected(ManagePortal, 'manage_importTarball')
def manage_importTarball(self, tarball):
""" Import steps from the uploaded tarball.
"""
if getattr(tarball, 'read', None) is not None:
tarball = tarball.read()
result = self.runAllImportStepsFromProfile(None, True, archive=tarball)
steps_run = 'Steps run: %s' % ', '.join(result['steps'])
return self.manage_importSteps(manage_tabs_message=steps_run,
messages=result['messages'])
security.declareProtected(ManagePortal, 'manage_exportSteps')
manage_exportSteps = PageTemplateFile('sutExportSteps', _wwwdir)
security.declareProtected(ManagePortal, 'manage_exportSelectedSteps')
def manage_exportSelectedSteps(self, ids, RESPONSE):
""" Export the steps selected by the user.
"""
if not ids:
RESPONSE.redirect('%s/manage_exportSteps?manage_tabs_message=%s' %
(self.absolute_url(), 'No+steps+selected.'))
result = self._doRunExportSteps(ids)
RESPONSE.setHeader('Content-type', 'application/x-gzip')
RESPONSE.setHeader('Content-disposition',
'attachment; filename=%s' % result['filename'])
return result['tarball']
security.declareProtected(ManagePortal, 'manage_exportAllSteps')
def manage_exportAllSteps(self, RESPONSE):
""" Export all steps.
"""
result = self.runAllExportSteps()
RESPONSE.setHeader('Content-type', 'application/x-gzip')
RESPONSE.setHeader('Content-disposition',
'attachment; filename=%s' % result['filename'])
return result['tarball']
security.declareProtected(ManagePortal, 'manage_upgrades')
manage_upgrades = PageTemplateFile('setup_upgrades', _wwwdir)
security.declareProtected(ManagePortal, 'upgradeStepMacro')
upgradeStepMacro = PageTemplateFile('upgradeStep', _wwwdir)
security.declareProtected(ManagePortal, 'manage_snapshots')
manage_snapshots = PageTemplateFile('sutSnapshots', _wwwdir)
security.declareProtected(ManagePortal, 'listSnapshotInfo')
def listSnapshotInfo(self):
""" Return a list of mappings describing available snapshots.
o Keys include:
'id' -- snapshot ID
'title' -- snapshot title or ID
'url' -- URL of the snapshot folder
"""
result = []
snapshots = self._getOb('snapshots', None)
if snapshots:
for id, folder in snapshots.objectItems('Folder'):
result.append({
'id': id,
'title': folder.title_or_id(),
'url': folder.absolute_url()
})
return result
security.declareProtected(ManagePortal, 'listProfileInfo')
def listProfileInfo(self, for_=None):
""" Return a list of mappings describing registered profiles.
Base profile is listed first, extensions are sorted.
o Keys include:
'id' -- profile ID
'title' -- profile title or ID
'description' -- description of the profile
'path' -- path to the profile within its product
'product' -- name of the registering product
"""
base = []
ext = []
for info in _profile_registry.listProfileInfo(for_):
if info.get('type', BASE) == BASE:
base.append(info)
else:
ext.append(info)
ext.sort(lambda x, y: cmp(x['id'], y['id']))
return base + ext
security.declareProtected(ManagePortal, 'listContextInfos')
def listContextInfos(self):
""" List registered profiles and snapshots.
"""
def readableType(x):
if x is BASE:
return 'base'
elif x is EXTENSION:
return 'extension'
return 'unknown'
s_infos = [{
'id': 'snapshot-%s' % info['id'],
'title': info['title'],
'type': 'snapshot',
} for info in self.listSnapshotInfo()]
s_infos.sort(key=itemgetter('title'))
p_infos = [{
'id': 'profile-%s' % info['id'],
'title': info['title'],
'type': readableType(info['type']),
} for info in self.listProfileInfo()]
p_infos.sort(key=itemgetter('title'))
return tuple(s_infos + p_infos)
security.declareProtected(ManagePortal, 'getProfileImportDate')
def getProfileImportDate(self, profile_id):
""" See ISetupTool.
"""
prefix = ('import-all-%s-' % profile_id).replace(':', '_')
candidates = [
x for x in self.objectIds('File')
if x[:-18] == prefix and x.endswith('.log')
]
if len(candidates) == 0:
return None
candidates.sort()
last = candidates[-1]
stamp = last[-18:-4]
return '%s-%s-%sT%s:%s:%sZ' % (
stamp[0:4],
stamp[4:6],
stamp[6:8],
stamp[8:10],
stamp[10:12],
stamp[12:14],
)
security.declareProtected(ManagePortal, 'manage_createSnapshot')
def manage_createSnapshot(self, RESPONSE, snapshot_id=None):
""" Create a snapshot with the given ID.
o If no ID is passed, generate one.
"""
if snapshot_id is None:
snapshot_id = self._mangleTimestampName('snapshot')
self.createSnapshot(snapshot_id)
return RESPONSE.redirect('%s/manage_snapshots?manage_tabs_message=%s' %
(self.absolute_url(), 'Snapshot+created.'))
security.declareProtected(ManagePortal, 'manage_showDiff')
manage_showDiff = PageTemplateFile('sutCompare', _wwwdir)
def manage_downloadDiff(
self,
lhs,
rhs,
missing_as_empty,
ignore_blanks,
RESPONSE,
):
""" Crack request vars and call compareConfigurations.
o Return the result as a 'text/plain' stream, suitable for framing.
"""
comparison = self.manage_compareConfigurations(
lhs,
rhs,
missing_as_empty,
ignore_blanks,
)
RESPONSE.setHeader('Content-Type', 'text/plain')
return _PLAINTEXT_DIFF_HEADER % (lhs, rhs, comparison)
security.declareProtected(ManagePortal, 'manage_compareConfigurations')
def manage_compareConfigurations(
self,
lhs,
rhs,
missing_as_empty,
ignore_blanks,
):
""" Crack request vars and call compareConfigurations.
"""
lhs_context = self._getImportContext(lhs)
rhs_context = self._getImportContext(rhs)
return self.compareConfigurations(
lhs_context,
rhs_context,
missing_as_empty,
ignore_blanks,
)
security.declareProtected(ManagePortal, 'manage_stepRegistry')
manage_stepRegistry = PageTemplateFile('sutManage', _wwwdir)
security.declareProtected(ManagePortal, 'manage_deleteImportSteps')
def manage_deleteImportSteps(self, ids, request=None):
""" Delete selected import steps.
"""
if request is None:
request = self.REQUEST
for id in ids:
self._import_registry.unregisterStep(id)
self._p_changed = True
url = self.absolute_url()
request.RESPONSE.redirect("%s/manage_stepRegistry" % url)
security.declareProtected(ManagePortal, 'manage_deleteExportSteps')
def manage_deleteExportSteps(self, ids, request=None):
""" Delete selected export steps.
"""
if request is None:
request = self.REQUEST
for id in ids:
self._export_registry.unregisterStep(id)
self._p_changed = True
url = self.absolute_url()
request.RESPONSE.redirect("%s/manage_stepRegistry" % url)
#
# Upgrades management
#
security.declareProtected(ManagePortal, 'getLastVersionForProfile')
def getLastVersionForProfile(self, profile_id):
"""Return the last upgraded version for the specified profile.
"""
version = self._profile_upgrade_versions.get(profile_id, 'unknown')
return version
security.declareProtected(ManagePortal, 'setLastVersionForProfile')
def setLastVersionForProfile(self, profile_id, version):
"""Set the last upgraded version for the specified profile.
"""
if isinstance(version, basestring):
version = tuple(version.split('.'))
prof_versions = self._profile_upgrade_versions.copy()
prof_versions[profile_id] = version
self._profile_upgrade_versions = prof_versions
security.declareProtected(ManagePortal, 'getVersionForProfile')
def getVersionForProfile(self, profile_id):
"""Return the registered filesystem version for the specified
profile.
"""
return self.getProfileInfo(profile_id).get('version', 'unknown')
security.declareProtected(ManagePortal, 'profileExists')
def profileExists(self, profile_id):
"""Check if a profile exists."""
try:
self.getProfileInfo(profile_id)
except KeyError:
return False
else:
return True
security.declareProtected(ManagePortal, "getProfileInfo")
def getProfileInfo(self, profile_id):
if profile_id.startswith("profile-"):
profile_id = profile_id[len('profile-'):]
elif profile_id.startswith("snapshot-"):
profile_id = profile_id[len('snapshot-'):]
return _profile_registry.getProfileInfo(profile_id)
security.declareProtected(ManagePortal, 'getDependenciesForProfile')
def getDependenciesForProfile(self, profile_id):
if profile_id.startswith("snapshot-"):
return ()
if not self.profileExists(profile_id):
raise KeyError(profile_id)
try:
return self.getProfileInfo(profile_id).get('dependencies', ())
except KeyError:
return ()
security.declareProtected(ManagePortal, 'listProfilesWithUpgrades')
def listProfilesWithUpgrades(self):
profiles = listProfilesWithUpgrades()
profiles.sort()
return profiles
security.declarePrivate('_massageUpgradeInfo')
def _massageUpgradeInfo(self, info):
"""Add a couple of data points to the upgrade info dictionary.
"""
info = info.copy()
info['haspath'] = info['source'] and info['dest']
info['ssource'] = '.'.join(info['source'] or ('all', ))
info['sdest'] = '.'.join(info['dest'] or ('all', ))
info['done'] = (not info['proposed']
and info['step'].checker is not None
and not info['step'].checker(self))
return info
security.declareProtected(ManagePortal, 'listUpgrades')
def listUpgrades(self, profile_id, show_old=False):
"""Get the list of available upgrades.
"""
if show_old:
source = None
else:
source = self.getLastVersionForProfile(profile_id)
upgrades = listUpgradeSteps(self, profile_id, source)
res = []
for info in upgrades:
if type(info) == list:
subset = []
for subinfo in info:
subset.append(self._massageUpgradeInfo(subinfo))
res.append(subset)
else:
res.append(self._massageUpgradeInfo(info))
return res
security.declareProtected(ManagePortal, 'manage_doUpgrades')
def manage_doUpgrades(self, request=None):
"""Perform all selected upgrade steps.
"""
if request is None:
request = self.REQUEST
logger = logging.getLogger('GenericSetup')
steps_to_run = request.form.get('upgrades', [])
profile_id = request.get('profile_id', '')
step = None
for step_id in steps_to_run:
step = _upgrade_registry.getUpgradeStep(profile_id, step_id)
if step is not None:
step.doStep(self)
msg = "Ran upgrade step %s for profile %s" % (step.title,
profile_id)
logger.log(logging.INFO, msg)
# We update the profile version to the last one we have reached
# with running an upgrade step.
if step and step.dest is not None and step.checker is None:
self.setLastVersionForProfile(profile_id, step.dest)
url = self.absolute_url()
request.RESPONSE.redirect("%s/manage_upgrades?saved=%s" %
(url, profile_id))
#
# Helper methods
#
security.declarePrivate('_getImportContext')
def _getImportContext(self, context_id, should_purge=None, archive=None):
""" Crack ID and generate appropriate import context.
"""
encoding = self.getEncoding()
if context_id is not None:
if context_id.startswith('profile-'):
context_id = context_id[len('profile-'):]
info = _profile_registry.getProfileInfo(context_id)
if info.get('product'):
path = os.path.join(_getProductPath(info['product']),
info['path'])
else:
path = info['path']
if should_purge is None:
should_purge = (info.get('type') != EXTENSION)
return DirectoryImportContext(self, path, should_purge,
encoding)
elif context_id.startswith('snapshot-'):
context_id = context_id[len('snapshot-'):]
if should_purge is None:
should_purge = True
return SnapshotImportContext(self, context_id, should_purge,
encoding)
if archive is not None:
return TarballImportContext(
tool=self,
archive_bits=archive,
encoding='UTF8',
should_purge=should_purge,
)
raise KeyError('Unknown context "%s"' % context_id)
security.declarePrivate('_updateImportStepsRegistry')
def _updateImportStepsRegistry(self, context, encoding):
""" Update our import steps registry from our profile.
"""
xml = context.readDataFile(IMPORT_STEPS_XML)
if xml is None:
return
info_list = self._import_registry.parseXML(xml, encoding)
for step_info in info_list:
id = step_info['id']
version = step_info.get('version')
handler = step_info['handler']
dependencies = tuple(step_info.get('dependencies', ()))
title = step_info.get('title', id)
description = ''.join(step_info.get('description', []))
self._import_registry.registerStep(
id=id,
version=version,
handler=handler,
dependencies=dependencies,
title=title,
description=description,
)
security.declarePrivate('_updateExportStepsRegistry')
def _updateExportStepsRegistry(self, context, encoding):
""" Update our export steps registry from our profile.
"""
xml = context.readDataFile(EXPORT_STEPS_XML)
if xml is None:
return
info_list = self._export_registry.parseXML(xml, encoding)
for step_info in info_list:
id = step_info['id']
handler = step_info['handler']
title = step_info.get('title', id)
description = ''.join(step_info.get('description', []))
self._export_registry.registerStep(
id=id,
handler=handler,
title=title,
description=description,
)
security.declarePrivate('_doRunImportStep')
def _doRunImportStep(self, step_id, context):
""" Run a single import step, using a pre-built context.
"""
__traceback_info__ = step_id
marker = object()
handler = self.getImportStep(step_id)
if handler is marker:
raise ValueError('Invalid import step: %s' % step_id)
if handler is None:
msg = 'Step %s has an invalid import handler' % step_id
logger = logging.getLogger('GenericSetup')
logger.error(msg)
return 'ERROR: ' + msg
return handler(context)
security.declarePrivate('_doRunExportSteps')
def _doRunExportSteps(self, steps):
""" See ISetupTool.
"""
context = TarballExportContext(self)
messages = {}
marker = object()
for step_id in steps:
handler = self.getExportStep(step_id, marker)
if handler is marker:
raise ValueError('Invalid export step: %s' % step_id)
if handler is None:
msg = 'Step %s has an invalid export handler' % step_id
logger = logging.getLogger('GenericSetup')
logger.error(msg)
messages[step_id] = msg
else:
messages[step_id] = handler(context)
return {
'steps': steps,
'messages': messages,
'tarball': context.getArchive(),
'filename': context.getArchiveFilename()
}
security.declareProtected(ManagePortal, 'getProfileDependencyChain')
def getProfileDependencyChain(self, profile_id, seen=None):
if seen is None:
seen = set()
elif profile_id in seen:
return [] # cycle break
seen.add(profile_id)
chain = []
dependencies = self.getDependenciesForProfile(profile_id)
for dependency in dependencies:
chain.extend(self.getProfileDependencyChain(dependency, seen))
chain.append(profile_id)
return chain
security.declarePrivate('_runImportStepsFromContext')
def _runImportStepsFromContext(self,
steps=None,
purge_old=None,
profile_id=None,
archive=None,
ignore_dependencies=False,
seen=None,
blacklisted_steps=None):
if profile_id is not None and not ignore_dependencies:
try:
chain = self.getProfileDependencyChain(profile_id)
except KeyError, e:
logger = logging.getLogger('GenericSetup')
logger.error('Unknown step in dependency chain: %s' % str(e))
raise
else:
class SetupTool(Folder):
""" Profile-based site configuration manager.
"""
implements(ISetupTool)
meta_type = 'Generic Setup Tool'
_baseline_context_id = ''
_profile_upgrade_versions = {}
_exclude_global_steps = False
security = ClassSecurityInfo()
def __init__(self, id):
self.id = str(id)
self._import_registry = ImportStepRegistry()
self._export_registry = ExportStepRegistry()
self._toolset_registry = ToolsetRegistry()
#
# ISetupTool API
#
security.declareProtected(ManagePortal, 'getEncoding')
def getEncoding(self):
""" See ISetupTool.
"""
return 'utf-8'
security.declareProtected(ManagePortal, 'getBaselineContextID')
def getBaselineContextID(self):
""" See ISetupTool.
"""
return self._baseline_context_id
security.declareProtected(ManagePortal, 'setBaselineContext')
def setBaselineContext(self, context_id, encoding=None):
""" See ISetupTool.
"""
self._baseline_context_id = context_id
self.applyContextById(context_id, encoding)
security.declareProtected(ManagePortal, 'getExcludeGlobalSteps')
def getExcludeGlobalSteps(self):
""" See ISetupTool.
"""
return self._exclude_global_steps
security.declareProtected(ManagePortal, 'setExcludeGlobalSteps')
def setExcludeGlobalSteps(self, value):
""" See ISetupTool.
"""
self._exclude_global_steps = value
security.declareProtected(ManagePortal, 'applyContextById')
def applyContextById(self, context_id, encoding=None):
context = self._getImportContext(context_id)
self.applyContext(context, encoding)
security.declareProtected(ManagePortal, 'applyContext')
def applyContext(self, context, encoding=None):
self._updateImportStepsRegistry(context, encoding)
self._updateExportStepsRegistry(context, encoding)
security.declareProtected(ManagePortal, 'getImportStepRegistry')
def getImportStepRegistry(self):
""" See ISetupTool.
"""
return self._import_registry
security.declareProtected(ManagePortal, 'getExportStepRegistry')
def getExportStepRegistry(self):
""" See ISetupTool.
"""
return self._export_registry
security.declareProtected(ManagePortal, 'getImportStep')
def getImportStep(self, step, default=None):
"""Simple wrapper to query both the global and local step registry."""
res = self._import_registry.getStep(step, self)
if res is self and not self._exclude_global_steps:
res = _import_step_registry.getStep(step, self)
if res is not self:
return res
return default
security.declareProtected(ManagePortal, 'getSortedImportSteps')
def getSortedImportSteps(self):
if self._exclude_global_steps:
steps = set()
else:
steps = set(_import_step_registry.listSteps())
steps.update(set(self._import_registry.listSteps()))
step_infos = [self.getImportStepMetadata(step) for step in steps]
return tuple(_computeTopologicalSort(step_infos))
security.declareProtected(ManagePortal, 'getImportStepMetadata')
def getImportStepMetadata(self, step, default=None):
"""Simple wrapper to query both the global and local step registry."""
res = self._import_registry.getStepMetadata(step, self)
if res is self and not self._exclude_global_steps:
res = _import_step_registry.getStepMetadata(step, default)
if res is not self:
return res
return default
security.declareProtected(ManagePortal, 'getExportStep')
def getExportStep(self, step, default=None):
"""Simple wrapper to query both the global and local step registry."""
res = self._export_registry.getStep(step, self)
if res is self and not self._exclude_global_steps:
res = _export_step_registry.getStep(step, self)
if res is not self:
return res
return default
security.declareProtected(ManagePortal, 'listExportSteps')
def listExportSteps(self):
steps = set(self._export_registry.listSteps())
if not self._exclude_global_steps:
steps.update(set(_export_step_registry.listSteps()))
return tuple(steps)
security.declareProtected(ManagePortal, 'getExportStepMetadata')
def getExportStepMetadata(self, step, default=None):
"""Simple wrapper to query both the global and local step registry."""
res = self._export_registry.getStepMetadata(step, self)
if res is self and not self._exclude_global_steps:
res = _export_step_registry.getStepMetadata(step, default)
if res is not self:
return res
return default
security.declareProtected(ManagePortal, 'getToolsetRegistry')
def getToolsetRegistry(self):
""" See ISetupTool.
"""
return self._toolset_registry
security.declareProtected(ManagePortal, 'runImportStepFromProfile')
def runImportStepFromProfile(self, profile_id, step_id,
run_dependencies=True, purge_old=None):
""" See ISetupTool.
"""
context = self._getImportContext(profile_id, purge_old)
self.applyContext(context)
info = self.getImportStepMetadata(step_id)
if info is None:
raise ValueError('No such import step: %s' % step_id)
dependencies = info.get('dependencies', ())
messages = {}
steps = []
if run_dependencies:
for dependency in dependencies:
if dependency not in steps:
steps.append(dependency)
steps.append(step_id)
full_import = (set(steps) == set(self.getSortedImportSteps()))
event.notify(
BeforeProfileImportEvent(self, profile_id, steps, full_import))
for step in steps:
message = self._doRunImportStep(step, context)
messages[step] = message or ''
message_list = filter(None, [message])
message_list.extend([ '%s: %s' % x[1:] for x in context.listNotes() ])
messages[step_id] = '\n'.join(message_list)
event.notify(
ProfileImportedEvent(self, profile_id, steps, full_import))
return {'steps': steps, 'messages': messages}
security.declareProtected(ManagePortal, 'runAllImportStepsFromProfile')
def runAllImportStepsFromProfile(self,
profile_id,
purge_old=None,
ignore_dependencies=False,
archive=None):
""" See ISetupTool.
"""
__traceback_info__ = profile_id
result = self._runImportStepsFromContext(
purge_old=purge_old,
profile_id=profile_id,
archive=archive,
ignore_dependencies=ignore_dependencies)
if profile_id is None:
prefix = 'import-all-from-tar'
else:
prefix = 'import-all-%s' % profile_id.replace(':', '_')
name = self._mangleTimestampName(prefix, 'log')
self._createReport(name, result['steps'], result['messages'])
return result
security.declareProtected(ManagePortal, 'runExportStep')
def runExportStep(self, step_id):
""" See ISetupTool.
"""
return self._doRunExportSteps([step_id])
security.declareProtected(ManagePortal, 'runAllExportSteps')
def runAllExportSteps(self):
""" See ISetupTool.
"""
return self._doRunExportSteps(self.listExportSteps())
security.declareProtected(ManagePortal, 'createSnapshot')
def createSnapshot(self, snapshot_id):
""" See ISetupTool.
"""
context = SnapshotExportContext(self, snapshot_id)
messages = {}
steps = self.listExportSteps()
for step_id in steps:
handler = self.getExportStep(step_id)
if handler is None:
logger = logging.getLogger('GenericSetup')
logger.error('Step %s has an invalid handler' % step_id)
continue
messages[step_id] = handler(context)
return {'steps': steps,
'messages': messages,
'url': context.getSnapshotURL(),
'snapshot': context.getSnapshotFolder()}
security.declareProtected(ManagePortal, 'compareConfigurations')
def compareConfigurations(self,
lhs_context,
rhs_context,
missing_as_empty=False,
ignore_blanks=False,
skip=SKIPPED_FILES,
):
""" See ISetupTool.
"""
differ = ConfigDiff(lhs_context,
rhs_context,
missing_as_empty,
ignore_blanks,
skip,
)
return differ.compare()
security.declareProtected(ManagePortal, 'markupComparison')
def markupComparison(self, lines):
""" See ISetupTool.
"""
result = []
for line in lines.splitlines():
if line.startswith('** '):
if line.find('File') > -1:
if line.find('replaced') > -1:
result.append(('file-to-dir', line))
elif line.find('added') > -1:
result.append(('file-added', line))
else:
result.append(('file-removed', line))
else:
if line.find('replaced') > -1:
result.append(('dir-to-file', line))
elif line.find('added') > -1:
result.append(('dir-added', line))
else:
result.append(('dir-removed', line))
elif line.startswith('@@'):
result.append(('diff-range', line))
elif line.startswith(' '):
result.append(('diff-context', line))
elif line.startswith('+'):
result.append(('diff-added', line))
elif line.startswith('-'):
result.append(('diff-removed', line))
elif line == '\ No newline at end of file':
result.append(('diff-context', line))
else:
result.append(('diff-header', line))
return '<pre>\n%s\n</pre>' % (
'\n'.join([('<span class="%s">%s</span>' % (cl, escape(l)))
for cl, l in result]))
#
# ZMI
#
manage_options = (
Folder.manage_options[:1] +
({'label': 'Profiles', 'action': 'manage_tool'},
{'label': 'Import', 'action': 'manage_importSteps'},
{'label': 'Export', 'action': 'manage_exportSteps'},
{'label': 'Upgrades', 'action': 'manage_upgrades'},
{'label': 'Snapshots', 'action': 'manage_snapshots'},
{'label': 'Comparison', 'action': 'manage_showDiff'},
{'label': 'Manage', 'action': 'manage_stepRegistry'}) +
Folder.manage_options[3:]) # skip "View", "Properties"
security.declareProtected(ManagePortal, 'manage_tool')
manage_tool = PageTemplateFile('sutProperties', _wwwdir)
security.declareProtected(ManagePortal, 'manage_updateToolProperties')
def manage_updateToolProperties(self, context_id,
exclude_global_steps=False,
RESPONSE=None):
""" Update the tool's settings.
"""
self.setExcludeGlobalSteps(exclude_global_steps)
self.setBaselineContext(context_id)
if RESPONSE is not None:
RESPONSE.redirect('%s/manage_tool?manage_tabs_message=%s'
% (self.absolute_url(), 'Properties+updated.'))
security.declareProtected(ManagePortal, 'manage_importSteps')
manage_importSteps = PageTemplateFile('sutImportSteps', _wwwdir)
security.declareProtected(ManagePortal, 'manage_importSelectedSteps')
def manage_importSelectedSteps(self, ids, run_dependencies,
context_id=None):
""" Import the steps selected by the user.
"""
messages = {}
if not ids:
summary = 'No steps selected.'
else:
if context_id is None:
context_id = self.getBaselineContextID()
steps_run = []
for step_id in ids:
result = self.runImportStepFromProfile(context_id,
step_id,
run_dependencies)
steps_run.extend(result['steps'])
messages.update(result['messages'])
summary = 'Steps run: %s' % ', '.join(steps_run)
name = self._mangleTimestampName('import-selected', 'log')
self._createReport(name, result['steps'], result['messages'])
return self.manage_importSteps(manage_tabs_message=summary,
messages=messages)
security.declareProtected(ManagePortal, 'manage_importAllSteps')
def manage_importAllSteps(self, context_id=None):
""" Import all steps.
"""
if context_id is None:
context_id = self.getBaselineContextID()
result = self.runAllImportStepsFromProfile(context_id, purge_old=None)
steps_run = 'Steps run: %s' % ', '.join(result['steps'])
return self.manage_importSteps(manage_tabs_message=steps_run,
messages=result['messages'])
security.declareProtected(ManagePortal, 'manage_importExtensions')
def manage_importExtensions(self, RESPONSE, profile_ids=()):
""" Import all steps for the selected extension profiles.
"""
detail = {}
if len(profile_ids) == 0:
message = 'Please select one or more extension profiles.'
RESPONSE.redirect('%s/manage_tool?manage_tabs_message=%s'
% (self.absolute_url(), message))
else:
message = 'Imported profiles: %s' % ', '.join(profile_ids)
for profile_id in profile_ids:
result = self.runAllImportStepsFromProfile(profile_id)
for k, v in result['messages'].items():
detail['%s:%s' % (profile_id, k)] = v
return self.manage_importSteps(manage_tabs_message=message,
messages=detail)
security.declareProtected(ManagePortal, 'manage_importTarball')
def manage_importTarball(self, tarball):
""" Import steps from the uploaded tarball.
"""
if getattr(tarball, 'read', None) is not None:
tarball = tarball.read()
result = self.runAllImportStepsFromProfile(None, True, archive=tarball)
steps_run = 'Steps run: %s' % ', '.join(result['steps'])
return self.manage_importSteps(manage_tabs_message=steps_run,
messages=result['messages'])
security.declareProtected(ManagePortal, 'manage_exportSteps')
manage_exportSteps = PageTemplateFile('sutExportSteps', _wwwdir)
security.declareProtected(ManagePortal, 'manage_exportSelectedSteps')
def manage_exportSelectedSteps(self, ids, RESPONSE):
""" Export the steps selected by the user.
"""
if not ids:
RESPONSE.redirect('%s/manage_exportSteps?manage_tabs_message=%s'
% (self.absolute_url(), 'No+steps+selected.'))
result = self._doRunExportSteps(ids)
RESPONSE.setHeader('Content-type', 'application/x-gzip')
RESPONSE.setHeader('Content-disposition',
'attachment; filename=%s' % result['filename'])
return result['tarball']
security.declareProtected(ManagePortal, 'manage_exportAllSteps')
def manage_exportAllSteps(self, RESPONSE):
""" Export all steps.
"""
result = self.runAllExportSteps()
RESPONSE.setHeader('Content-type', 'application/x-gzip')
RESPONSE.setHeader('Content-disposition',
'attachment; filename=%s' % result['filename'])
return result['tarball']
security.declareProtected(ManagePortal, 'manage_upgrades')
manage_upgrades = PageTemplateFile('setup_upgrades', _wwwdir)
security.declareProtected(ManagePortal, 'upgradeStepMacro')
upgradeStepMacro = PageTemplateFile('upgradeStep', _wwwdir)
security.declareProtected(ManagePortal, 'manage_snapshots')
manage_snapshots = PageTemplateFile('sutSnapshots', _wwwdir)
security.declareProtected(ManagePortal, 'listSnapshotInfo')
def listSnapshotInfo(self):
""" Return a list of mappings describing available snapshots.
o Keys include:
'id' -- snapshot ID
'title' -- snapshot title or ID
'url' -- URL of the snapshot folder
"""
result = []
snapshots = self._getOb('snapshots', None)
if snapshots:
for id, folder in snapshots.objectItems('Folder'):
result.append({'id': id,
'title': folder.title_or_id(),
'url': folder.absolute_url()})
return result
security.declareProtected(ManagePortal, 'listProfileInfo')
def listProfileInfo(self, for_=None):
""" Return a list of mappings describing registered profiles.
Base profile is listed first, extensions are sorted.
o Keys include:
'id' -- profile ID
'title' -- profile title or ID
'description' -- description of the profile
'path' -- path to the profile within its product
'product' -- name of the registering product
"""
base = []
ext = []
for info in _profile_registry.listProfileInfo(for_):
if info.get('type', BASE) == BASE:
base.append(info)
else:
ext.append(info)
ext.sort(lambda x, y: cmp(x['id'], y['id']))
return base + ext
security.declareProtected(ManagePortal, 'listContextInfos')
def listContextInfos(self):
""" List registered profiles and snapshots.
"""
def readableType(x):
if x is BASE:
return 'base'
elif x is EXTENSION:
return 'extension'
return 'unknown'
s_infos = [{'id': 'snapshot-%s' % info['id'],
'title': info['title'],
'type': 'snapshot',
}
for info in self.listSnapshotInfo()]
s_infos.sort(key=itemgetter('title'))
p_infos = [{'id': 'profile-%s' % info['id'],
'title': info['title'],
'type': readableType(info['type']),
}
for info in self.listProfileInfo()]
p_infos.sort(key=itemgetter('title'))
return tuple(s_infos + p_infos)
security.declareProtected(ManagePortal, 'getProfileImportDate')
def getProfileImportDate(self, profile_id):
""" See ISetupTool.
"""
prefix = ('import-all-%s-' % profile_id).replace(':', '_')
candidates = [x for x in self.objectIds('File')
if x[:-18] == prefix and x.endswith('.log')]
if len(candidates) == 0:
return None
candidates.sort()
last = candidates[-1]
stamp = last[-18:-4]
return '%s-%s-%sT%s:%s:%sZ' % (stamp[0:4],
stamp[4:6],
stamp[6:8],
stamp[8:10],
stamp[10:12],
stamp[12:14],
)
security.declareProtected(ManagePortal, 'manage_createSnapshot')
def manage_createSnapshot(self, RESPONSE, snapshot_id=None):
""" Create a snapshot with the given ID.
o If no ID is passed, generate one.
"""
if snapshot_id is None:
snapshot_id = self._mangleTimestampName('snapshot')
self.createSnapshot(snapshot_id)
return RESPONSE.redirect('%s/manage_snapshots?manage_tabs_message=%s'
% (self.absolute_url(), 'Snapshot+created.'))
security.declareProtected(ManagePortal, 'manage_showDiff')
manage_showDiff = PageTemplateFile('sutCompare', _wwwdir)
def manage_downloadDiff(self,
lhs,
rhs,
missing_as_empty,
ignore_blanks,
RESPONSE,
):
""" Crack request vars and call compareConfigurations.
o Return the result as a 'text/plain' stream, suitable for framing.
"""
comparison = self.manage_compareConfigurations(lhs,
rhs,
missing_as_empty,
ignore_blanks,
)
RESPONSE.setHeader('Content-Type', 'text/plain')
return _PLAINTEXT_DIFF_HEADER % (lhs, rhs, comparison)
security.declareProtected(ManagePortal, 'manage_compareConfigurations')
def manage_compareConfigurations(self,
lhs,
rhs,
missing_as_empty,
ignore_blanks,
):
""" Crack request vars and call compareConfigurations.
"""
lhs_context = self._getImportContext(lhs)
rhs_context = self._getImportContext(rhs)
return self.compareConfigurations(lhs_context,
rhs_context,
missing_as_empty,
ignore_blanks,
)
security.declareProtected(ManagePortal, 'manage_stepRegistry')
manage_stepRegistry = PageTemplateFile('sutManage', _wwwdir)
security.declareProtected(ManagePortal, 'manage_deleteImportSteps')
def manage_deleteImportSteps(self, ids, request=None):
""" Delete selected import steps.
"""
if request is None:
request = self.REQUEST
for id in ids:
self._import_registry.unregisterStep(id)
self._p_changed = True
url = self.absolute_url()
request.RESPONSE.redirect("%s/manage_stepRegistry" % url)
security.declareProtected(ManagePortal, 'manage_deleteExportSteps')
def manage_deleteExportSteps(self, ids, request=None):
""" Delete selected export steps.
"""
if request is None:
request = self.REQUEST
for id in ids:
self._export_registry.unregisterStep(id)
self._p_changed = True
url = self.absolute_url()
request.RESPONSE.redirect("%s/manage_stepRegistry" % url)
#
# Upgrades management
#
security.declareProtected(ManagePortal, 'getLastVersionForProfile')
def getLastVersionForProfile(self, profile_id):
"""Return the last upgraded version for the specified profile.
"""
version = self._profile_upgrade_versions.get(profile_id, 'unknown')
return version
security.declareProtected(ManagePortal, 'setLastVersionForProfile')
def setLastVersionForProfile(self, profile_id, version):
"""Set the last upgraded version for the specified profile.
"""
if isinstance(version, basestring):
version = tuple(version.split('.'))
prof_versions = self._profile_upgrade_versions.copy()
prof_versions[profile_id] = version
self._profile_upgrade_versions = prof_versions
security.declareProtected(ManagePortal, 'getVersionForProfile')
def getVersionForProfile(self, profile_id):
"""Return the registered filesystem version for the specified
profile.
"""
return self.getProfileInfo(profile_id).get('version', 'unknown')
security.declareProtected(ManagePortal, 'profileExists')
def profileExists(self, profile_id):
"""Check if a profile exists."""
try:
self.getProfileInfo(profile_id)
except KeyError:
return False
else:
return True
security.declareProtected(ManagePortal, "getProfileInfo")
def getProfileInfo(self, profile_id):
if profile_id.startswith("profile-"):
profile_id = profile_id[len('profile-'):]
elif profile_id.startswith("snapshot-"):
profile_id = profile_id[len('snapshot-'):]
return _profile_registry.getProfileInfo(profile_id)
security.declareProtected(ManagePortal, 'getDependenciesForProfile')
def getDependenciesForProfile(self, profile_id):
if profile_id.startswith("snapshot-"):
return ()
if not self.profileExists(profile_id):
raise KeyError(profile_id)
try:
return self.getProfileInfo(profile_id).get('dependencies', ())
except KeyError:
return ()
security.declareProtected(ManagePortal, 'listProfilesWithUpgrades')
def listProfilesWithUpgrades(self):
return listProfilesWithUpgrades()
security.declarePrivate('_massageUpgradeInfo')
def _massageUpgradeInfo(self, info):
"""Add a couple of data points to the upgrade info dictionary.
"""
info = info.copy()
info['haspath'] = info['source'] and info['dest']
info['ssource'] = '.'.join(info['source'] or ('all',))
info['sdest'] = '.'.join(info['dest'] or ('all',))
info['done'] = (not info['proposed'] and
info['step'].checker is not None and
not info['step'].checker(self))
return info
security.declareProtected(ManagePortal, 'listUpgrades')
def listUpgrades(self, profile_id, show_old=False):
"""Get the list of available upgrades.
"""
if show_old:
source = None
else:
source = self.getLastVersionForProfile(profile_id)
upgrades = listUpgradeSteps(self, profile_id, source)
res = []
for info in upgrades:
if type(info) == list:
subset = []
for subinfo in info:
subset.append(self._massageUpgradeInfo(subinfo))
res.append(subset)
else:
res.append(self._massageUpgradeInfo(info))
return res
security.declareProtected(ManagePortal, 'manage_doUpgrades')
def manage_doUpgrades(self, request=None):
"""Perform all selected upgrade steps.
"""
if request is None:
request = self.REQUEST
logger = logging.getLogger('GenericSetup')
steps_to_run = request.form.get('upgrades', [])
profile_id = request.get('profile_id', '')
step = None
for step_id in steps_to_run:
step = _upgrade_registry.getUpgradeStep(profile_id, step_id)
if step is not None:
step.doStep(self)
msg = "Ran upgrade step %s for profile %s" % (step.title,
profile_id)
logger.log(logging.INFO, msg)
# We update the profile version to the last one we have reached
# with running an upgrade step.
if step and step.dest is not None and step.checker is None:
self.setLastVersionForProfile(profile_id, step.dest)
url = self.absolute_url()
request.RESPONSE.redirect("%s/manage_upgrades?saved=%s"
% (url, profile_id))
#
# Helper methods
#
security.declarePrivate('_getImportContext')
def _getImportContext(self, context_id, should_purge=None, archive=None):
""" Crack ID and generate appropriate import context.
"""
encoding = self.getEncoding()
if context_id is not None:
if context_id.startswith('profile-'):
context_id = context_id[len('profile-'):]
info = _profile_registry.getProfileInfo(context_id)
if info.get('product'):
path = os.path.join(_getProductPath(info['product']),
info['path'])
else:
path = info['path']
if should_purge is None:
should_purge = (info.get('type') != EXTENSION)
return DirectoryImportContext(self, path, should_purge,
encoding)
elif context_id.startswith('snapshot-'):
context_id = context_id[len('snapshot-'):]
if should_purge is None:
should_purge = True
return SnapshotImportContext(self, context_id, should_purge,
encoding)
if archive is not None:
return TarballImportContext(tool=self,
archive_bits=archive,
encoding='UTF8',
should_purge=should_purge,
)
raise KeyError('Unknown context "%s"' % context_id)
security.declarePrivate('_updateImportStepsRegistry')
def _updateImportStepsRegistry(self, context, encoding):
""" Update our import steps registry from our profile.
"""
xml = context.readDataFile(IMPORT_STEPS_XML)
if xml is None:
return
info_list = self._import_registry.parseXML(xml, encoding)
for step_info in info_list:
id = step_info['id']
version = step_info.get('version')
handler = step_info['handler']
dependencies = tuple(step_info.get('dependencies', ()))
title = step_info.get('title', id)
description = ''.join(step_info.get('description', []))
self._import_registry.registerStep(id=id,
version=version,
handler=handler,
dependencies=dependencies,
title=title,
description=description,
)
security.declarePrivate('_updateExportStepsRegistry')
def _updateExportStepsRegistry(self, context, encoding):
""" Update our export steps registry from our profile.
"""
xml = context.readDataFile(EXPORT_STEPS_XML)
if xml is None:
return
info_list = self._export_registry.parseXML(xml, encoding)
for step_info in info_list:
id = step_info['id']
handler = step_info['handler']
title = step_info.get('title', id)
description = ''.join(step_info.get('description', []))
self._export_registry.registerStep(id=id,
handler=handler,
title=title,
description=description,
)
security.declarePrivate('_doRunImportStep')
def _doRunImportStep(self, step_id, context):
""" Run a single import step, using a pre-built context.
"""
__traceback_info__ = step_id
marker = object()
handler = self.getImportStep(step_id)
if handler is marker:
raise ValueError('Invalid import step: %s' % step_id)
if handler is None:
msg = 'Step %s has an invalid import handler' % step_id
logger = logging.getLogger('GenericSetup')
logger.error(msg)
return 'ERROR: ' + msg
return handler(context)
security.declarePrivate('_doRunExportSteps')
def _doRunExportSteps(self, steps):
""" See ISetupTool.
"""
context = TarballExportContext(self)
messages = {}
marker = object()
for step_id in steps:
handler = self.getExportStep(step_id, marker)
if handler is marker:
raise ValueError('Invalid export step: %s' % step_id)
if handler is None:
msg = 'Step %s has an invalid export handler' % step_id
logger = logging.getLogger('GenericSetup')
logger.error(msg)
messages[step_id] = msg
else:
messages[step_id] = handler(context)
return {'steps': steps,
'messages': messages,
'tarball': context.getArchive(),
'filename': context.getArchiveFilename()}
security.declareProtected(ManagePortal, 'getProfileDependencyChain')
def getProfileDependencyChain(self, profile_id, seen=None):
if seen is None:
seen = set()
elif profile_id in seen:
return [] # cycle break
seen.add(profile_id)
chain = []
dependencies = self.getDependenciesForProfile(profile_id)
for dependency in dependencies:
chain.extend(self.getProfileDependencyChain(dependency, seen))
chain.append(profile_id)
return chain
security.declarePrivate('_runImportStepsFromContext')
def _runImportStepsFromContext(self,
steps=None,
purge_old=None,
profile_id=None,
archive=None,
ignore_dependencies=False,
seen=None):
if profile_id is not None and not ignore_dependencies:
try:
chain = self.getProfileDependencyChain(profile_id)
except KeyError, e:
logger = logging.getLogger('GenericSetup')
logger.error('Unknown step in dependency chain: %s' % str(e))
raise
else:
def __init__(self, id):
self.id = str(id)
self._import_registry = ImportStepRegistry()
self._export_registry = ExportStepRegistry()
self._toolset_registry = ToolsetRegistry()
class SetupTool(Folder):
""" Profile-based site configuration manager.
"""
implements(ISetupTool)
meta_type = "Generic Setup Tool"
_baseline_context_id = ""
_profile_upgrade_versions = {}
_exclude_global_steps = False
security = ClassSecurityInfo()
def __init__(self, id):
self.id = str(id)
self._import_registry = ImportStepRegistry()
self._export_registry = ExportStepRegistry()
self._toolset_registry = ToolsetRegistry()
#
# ISetupTool API
#
security.declareProtected(ManagePortal, "getEncoding")
def getEncoding(self):
""" See ISetupTool.
"""
return "utf-8"
security.declareProtected(ManagePortal, "getBaselineContextID")
def getBaselineContextID(self):
""" See ISetupTool.
"""
return self._baseline_context_id
security.declareProtected(ManagePortal, "setBaselineContext")
def setBaselineContext(self, context_id, encoding=None):
""" See ISetupTool.
"""
self._baseline_context_id = context_id
self.applyContextById(context_id, encoding)
security.declareProtected(ManagePortal, "getExcludeGlobalSteps")
def getExcludeGlobalSteps(self):
""" See ISetupTool.
"""
return self._exclude_global_steps
security.declareProtected(ManagePortal, "setExcludeGlobalSteps")
def setExcludeGlobalSteps(self, value):
""" See ISetupTool.
"""
self._exclude_global_steps = value
security.declareProtected(ManagePortal, "applyContextById")
def applyContextById(self, context_id, encoding=None):
context = self._getImportContext(context_id)
self.applyContext(context, encoding)
security.declareProtected(ManagePortal, "applyContext")
def applyContext(self, context, encoding=None):
self._updateImportStepsRegistry(context, encoding)
self._updateExportStepsRegistry(context, encoding)
security.declareProtected(ManagePortal, "getImportStepRegistry")
def getImportStepRegistry(self):
""" See ISetupTool.
"""
return self._import_registry
security.declareProtected(ManagePortal, "getExportStepRegistry")
def getExportStepRegistry(self):
""" See ISetupTool.
"""
return self._export_registry
security.declareProtected(ManagePortal, "getImportStep")
def getImportStep(self, step, default=None):
"""Simple wrapper to query both the global and local step registry."""
res = self._import_registry.getStep(step, self)
if res is self and not self._exclude_global_steps:
res = _import_step_registry.getStep(step, self)
if res is not self:
return res
return default
security.declareProtected(ManagePortal, "getSortedImportSteps")
def getSortedImportSteps(self):
if self._exclude_global_steps:
steps = set()
else:
steps = set(_import_step_registry.listSteps())
steps.update(set(self._import_registry.listSteps()))
step_infos = [self.getImportStepMetadata(step) for step in steps]
return tuple(_computeTopologicalSort(step_infos))
security.declareProtected(ManagePortal, "getImportStepMetadata")
def getImportStepMetadata(self, step, default=None):
"""Simple wrapper to query both the global and local step registry."""
res = self._import_registry.getStepMetadata(step, self)
if res is self and not self._exclude_global_steps:
res = _import_step_registry.getStepMetadata(step, default)
if res is not self:
return res
return default
security.declareProtected(ManagePortal, "getExportStep")
def getExportStep(self, step, default=None):
"""Simple wrapper to query both the global and local step registry."""
res = self._export_registry.getStep(step, self)
if res is self and not self._exclude_global_steps:
res = _export_step_registry.getStep(step, self)
if res is not self:
return res
return default
security.declareProtected(ManagePortal, "listExportSteps")
def listExportSteps(self):
steps = set(self._export_registry.listSteps())
if not self._exclude_global_steps:
steps.update(set(_export_step_registry.listSteps()))
return tuple(steps)
security.declareProtected(ManagePortal, "listImportSteps")
def listImportSteps(self):
steps = set(self._import_registry.listSteps())
if not self._exclude_global_steps:
steps.update(set(_import_step_registry.listSteps()))
return tuple(steps)
security.declareProtected(ManagePortal, "getExportStepMetadata")
def getExportStepMetadata(self, step, default=None):
"""Simple wrapper to query both the global and local step registry."""
res = self._export_registry.getStepMetadata(step, self)
if res is self and not self._exclude_global_steps:
res = _export_step_registry.getStepMetadata(step, default)
if res is not self:
return res
return default
security.declareProtected(ManagePortal, "getToolsetRegistry")
def getToolsetRegistry(self):
""" See ISetupTool.
"""
return self._toolset_registry
security.declareProtected(ManagePortal, "runImportStepFromProfile")
def runImportStepFromProfile(self, profile_id, step_id, run_dependencies=True, purge_old=None):
""" See ISetupTool.
"""
context = self._getImportContext(profile_id, purge_old)
self.applyContext(context)
info = self.getImportStepMetadata(step_id)
if info is None:
generic_logger.error(
"No such import step: '%s' Maybe you meant one of %s", step_id, str(self.listImportSteps())
)
raise ValueError("No such import step: %s" % step_id)
dependencies = info.get("dependencies", ())
messages = {}
steps = []
if run_dependencies:
for dependency in dependencies:
if dependency not in steps:
steps.append(dependency)
steps.append(step_id)
full_import = set(steps) == set(self.getSortedImportSteps())
event.notify(BeforeProfileImportEvent(self, profile_id, steps, full_import))
for step in steps:
message = self._doRunImportStep(step, context)
messages[step] = message or ""
message_list = filter(None, [message])
message_list.extend(["%s: %s" % x[1:] for x in context.listNotes()])
messages[step_id] = "\n".join(message_list)
event.notify(ProfileImportedEvent(self, profile_id, steps, full_import))
return {"steps": steps, "messages": messages}
security.declareProtected(ManagePortal, "runAllImportStepsFromProfile")
def runAllImportStepsFromProfile(
self,
profile_id,
purge_old=None,
ignore_dependencies=False,
archive=None,
blacklisted_steps=None,
dependency_strategy=None,
):
""" See ISetupTool.
"""
__traceback_info__ = profile_id
result = self._runImportStepsFromContext(
purge_old=purge_old,
profile_id=profile_id,
archive=archive,
ignore_dependencies=ignore_dependencies,
blacklisted_steps=blacklisted_steps,
dependency_strategy=dependency_strategy,
)
if profile_id is None:
prefix = "import-all-from-tar"
else:
prefix = "import-all-%s" % profile_id.replace(":", "_")
name = self._mangleTimestampName(prefix, "log")
self._createReport(name, result["steps"], result["messages"])
return result
security.declareProtected(ManagePortal, "runExportStep")
def runExportStep(self, step_id):
""" See ISetupTool.
"""
return self._doRunExportSteps([step_id])
security.declareProtected(ManagePortal, "runAllExportSteps")
def runAllExportSteps(self):
""" See ISetupTool.
"""
return self._doRunExportSteps(self.listExportSteps())
security.declareProtected(ManagePortal, "createSnapshot")
def createSnapshot(self, snapshot_id):
""" See ISetupTool.
"""
context = SnapshotExportContext(self, snapshot_id)
messages = {}
steps = self.listExportSteps()
for step_id in steps:
handler = self.getExportStep(step_id)
if handler is None:
logger = logging.getLogger("GenericSetup")
logger.error("Step %s has an invalid handler" % step_id)
continue
messages[step_id] = handler(context)
return {
"steps": steps,
"messages": messages,
"url": context.getSnapshotURL(),
"snapshot": context.getSnapshotFolder(),
}
security.declareProtected(ManagePortal, "compareConfigurations")
def compareConfigurations(
self, lhs_context, rhs_context, missing_as_empty=False, ignore_blanks=False, skip=SKIPPED_FILES
):
""" See ISetupTool.
"""
differ = ConfigDiff(lhs_context, rhs_context, missing_as_empty, ignore_blanks, skip)
return differ.compare()
security.declareProtected(ManagePortal, "markupComparison")
def markupComparison(self, lines):
""" See ISetupTool.
"""
result = []
for line in lines.splitlines():
if line.startswith("** "):
if line.find("File") > -1:
if line.find("replaced") > -1:
result.append(("file-to-dir", line))
elif line.find("added") > -1:
result.append(("file-added", line))
else:
result.append(("file-removed", line))
else:
if line.find("replaced") > -1:
result.append(("dir-to-file", line))
elif line.find("added") > -1:
result.append(("dir-added", line))
else:
result.append(("dir-removed", line))
elif line.startswith("@@"):
result.append(("diff-range", line))
elif line.startswith(" "):
result.append(("diff-context", line))
elif line.startswith("+"):
result.append(("diff-added", line))
elif line.startswith("-"):
result.append(("diff-removed", line))
elif line == "\ No newline at end of file":
result.append(("diff-context", line))
else:
result.append(("diff-header", line))
return "<pre>\n%s\n</pre>" % ("\n".join([('<span class="%s">%s</span>' % (cl, escape(l))) for cl, l in result]))
#
# ZMI
#
manage_options = (
Folder.manage_options[:1]
+ (
{"label": "Profiles", "action": "manage_tool"},
{"label": "Import", "action": "manage_importSteps"},
{"label": "Export", "action": "manage_exportSteps"},
{"label": "Upgrades", "action": "manage_upgrades"},
{"label": "Snapshots", "action": "manage_snapshots"},
{"label": "Comparison", "action": "manage_showDiff"},
{"label": "Manage", "action": "manage_stepRegistry"},
)
+ Folder.manage_options[3:]
) # skip "View", "Properties"
security.declareProtected(ManagePortal, "manage_tool")
manage_tool = PageTemplateFile("sutProperties", _wwwdir)
security.declareProtected(ManagePortal, "manage_updateToolProperties")
def manage_updateToolProperties(self, context_id, exclude_global_steps=False, RESPONSE=None):
""" Update the tool's settings.
"""
self.setExcludeGlobalSteps(exclude_global_steps)
self.setBaselineContext(context_id)
if RESPONSE is not None:
RESPONSE.redirect("%s/manage_tool?manage_tabs_message=%s" % (self.absolute_url(), "Properties+updated."))
security.declareProtected(ManagePortal, "manage_importSteps")
manage_importSteps = PageTemplateFile("sutImportSteps", _wwwdir)
security.declareProtected(ManagePortal, "manage_importSelectedSteps")
def manage_importSelectedSteps(self, ids, run_dependencies, context_id=None):
""" Import the steps selected by the user.
"""
messages = {}
if not ids:
summary = "No steps selected."
else:
if context_id is None:
context_id = self.getBaselineContextID()
steps_run = []
for step_id in ids:
result = self.runImportStepFromProfile(context_id, step_id, run_dependencies)
steps_run.extend(result["steps"])
messages.update(result["messages"])
summary = "Steps run: %s" % ", ".join(steps_run)
name = self._mangleTimestampName("import-selected", "log")
self._createReport(name, result["steps"], result["messages"])
return self.manage_importSteps(manage_tabs_message=summary, messages=messages)
security.declareProtected(ManagePortal, "manage_importAllSteps")
def manage_importAllSteps(self, context_id=None, dependency_strategy=None):
""" Import all steps.
"""
if context_id is None:
context_id = self.getBaselineContextID()
result = self.runAllImportStepsFromProfile(context_id, purge_old=None, dependency_strategy=dependency_strategy)
steps_run = "Steps run: %s" % ", ".join(result["steps"])
return self.manage_importSteps(manage_tabs_message=steps_run, messages=result["messages"])
security.declareProtected(ManagePortal, "manage_importExtensions")
def manage_importExtensions(self, RESPONSE, profile_ids=()):
""" Import all steps for the selected extension profiles.
"""
detail = {}
if len(profile_ids) == 0:
message = "Please select one or more extension profiles."
RESPONSE.redirect("%s/manage_tool?manage_tabs_message=%s" % (self.absolute_url(), message))
else:
message = "Imported profiles: %s" % ", ".join(profile_ids)
for profile_id in profile_ids:
result = self.runAllImportStepsFromProfile(profile_id)
for k, v in result["messages"].items():
detail["%s:%s" % (profile_id, k)] = v
return self.manage_importSteps(manage_tabs_message=message, messages=detail)
security.declareProtected(ManagePortal, "manage_importTarball")
def manage_importTarball(self, tarball):
""" Import steps from the uploaded tarball.
"""
if getattr(tarball, "read", None) is not None:
tarball = tarball.read()
result = self.runAllImportStepsFromProfile(None, True, archive=tarball)
steps_run = "Steps run: %s" % ", ".join(result["steps"])
return self.manage_importSteps(manage_tabs_message=steps_run, messages=result["messages"])
security.declareProtected(ManagePortal, "manage_exportSteps")
manage_exportSteps = PageTemplateFile("sutExportSteps", _wwwdir)
security.declareProtected(ManagePortal, "manage_exportSelectedSteps")
def manage_exportSelectedSteps(self, ids, RESPONSE):
""" Export the steps selected by the user.
"""
if not ids:
RESPONSE.redirect(
"%s/manage_exportSteps?manage_tabs_message=%s" % (self.absolute_url(), "No+steps+selected.")
)
result = self._doRunExportSteps(ids)
RESPONSE.setHeader("Content-type", "application/x-gzip")
RESPONSE.setHeader("Content-disposition", "attachment; filename=%s" % result["filename"])
return result["tarball"]
security.declareProtected(ManagePortal, "manage_exportAllSteps")
def manage_exportAllSteps(self, RESPONSE):
""" Export all steps.
"""
result = self.runAllExportSteps()
RESPONSE.setHeader("Content-type", "application/x-gzip")
RESPONSE.setHeader("Content-disposition", "attachment; filename=%s" % result["filename"])
return result["tarball"]
security.declareProtected(ManagePortal, "manage_upgrades")
manage_upgrades = PageTemplateFile("setup_upgrades", _wwwdir)
security.declareProtected(ManagePortal, "upgradeStepMacro")
upgradeStepMacro = PageTemplateFile("upgradeStep", _wwwdir)
security.declareProtected(ManagePortal, "manage_snapshots")
manage_snapshots = PageTemplateFile("sutSnapshots", _wwwdir)
security.declareProtected(ManagePortal, "listSnapshotInfo")
def listSnapshotInfo(self):
""" Return a list of mappings describing available snapshots.
o Keys include:
'id' -- snapshot ID
'title' -- snapshot title or ID
'url' -- URL of the snapshot folder
"""
result = []
snapshots = self._getOb("snapshots", None)
if snapshots:
for id, folder in snapshots.objectItems("Folder"):
result.append({"id": id, "title": folder.title_or_id(), "url": folder.absolute_url()})
return result
security.declareProtected(ManagePortal, "listProfileInfo")
def listProfileInfo(self, for_=None):
""" Return a list of mappings describing registered profiles.
Base profile is listed first, extensions are sorted.
o Keys include:
'id' -- profile ID
'title' -- profile title or ID
'description' -- description of the profile
'path' -- path to the profile within its product
'product' -- name of the registering product
"""
base = []
ext = []
for info in _profile_registry.listProfileInfo(for_):
if info.get("type", BASE) == BASE:
base.append(info)
else:
ext.append(info)
ext.sort(lambda x, y: cmp(x["id"], y["id"]))
return base + ext
security.declareProtected(ManagePortal, "listContextInfos")
def listContextInfos(self):
""" List registered profiles and snapshots.
"""
def readableType(x):
if x is BASE:
return "base"
elif x is EXTENSION:
return "extension"
return "unknown"
s_infos = [
{"id": "snapshot-%s" % info["id"], "title": info["title"], "type": "snapshot"}
for info in self.listSnapshotInfo()
]
s_infos.sort(key=itemgetter("title"))
p_infos = [
{"id": "profile-%s" % info["id"], "title": info["title"], "type": readableType(info["type"])}
for info in self.listProfileInfo()
]
p_infos.sort(key=itemgetter("title"))
return tuple(s_infos + p_infos)
security.declareProtected(ManagePortal, "getProfileImportDate")
def getProfileImportDate(self, profile_id):
""" See ISetupTool.
"""
prefix = ("import-all-%s-" % profile_id).replace(":", "_")
candidates = [x for x in self.objectIds("File") if x[:-18] == prefix and x.endswith(".log")]
if len(candidates) == 0:
return None
candidates.sort()
last = candidates[-1]
stamp = last[-18:-4]
return "%s-%s-%sT%s:%s:%sZ" % (stamp[0:4], stamp[4:6], stamp[6:8], stamp[8:10], stamp[10:12], stamp[12:14])
security.declareProtected(ManagePortal, "manage_createSnapshot")
def manage_createSnapshot(self, RESPONSE, snapshot_id=None):
""" Create a snapshot with the given ID.
o If no ID is passed, generate one.
"""
if snapshot_id is None:
snapshot_id = self._mangleTimestampName("snapshot")
self.createSnapshot(snapshot_id)
return RESPONSE.redirect(
"%s/manage_snapshots?manage_tabs_message=%s" % (self.absolute_url(), "Snapshot+created.")
)
security.declareProtected(ManagePortal, "manage_showDiff")
manage_showDiff = PageTemplateFile("sutCompare", _wwwdir)
def manage_downloadDiff(self, lhs, rhs, missing_as_empty, ignore_blanks, RESPONSE):
""" Crack request vars and call compareConfigurations.
o Return the result as a 'text/plain' stream, suitable for framing.
"""
comparison = self.manage_compareConfigurations(lhs, rhs, missing_as_empty, ignore_blanks)
RESPONSE.setHeader("Content-Type", "text/plain")
return _PLAINTEXT_DIFF_HEADER % (lhs, rhs, comparison)
security.declareProtected(ManagePortal, "manage_compareConfigurations")
def manage_compareConfigurations(self, lhs, rhs, missing_as_empty, ignore_blanks):
""" Crack request vars and call compareConfigurations.
"""
lhs_context = self._getImportContext(lhs)
rhs_context = self._getImportContext(rhs)
return self.compareConfigurations(lhs_context, rhs_context, missing_as_empty, ignore_blanks)
security.declareProtected(ManagePortal, "manage_stepRegistry")
manage_stepRegistry = PageTemplateFile("sutManage", _wwwdir)
security.declareProtected(ManagePortal, "manage_deleteImportSteps")
def manage_deleteImportSteps(self, ids, request=None):
""" Delete selected import steps.
"""
if request is None:
request = self.REQUEST
for id in ids:
self._import_registry.unregisterStep(id)
self._p_changed = True
url = self.absolute_url()
request.RESPONSE.redirect("%s/manage_stepRegistry" % url)
security.declareProtected(ManagePortal, "manage_deleteExportSteps")
def manage_deleteExportSteps(self, ids, request=None):
""" Delete selected export steps.
"""
if request is None:
request = self.REQUEST
for id in ids:
self._export_registry.unregisterStep(id)
self._p_changed = True
url = self.absolute_url()
request.RESPONSE.redirect("%s/manage_stepRegistry" % url)
#
# Upgrades management
#
security.declareProtected(ManagePortal, "getLastVersionForProfile")
def getLastVersionForProfile(self, profile_id):
"""Return the last upgraded version for the specified profile.
"""
prefix = "profile-"
if profile_id.startswith(prefix):
profile_id = profile_id[len(prefix) :]
version = self._profile_upgrade_versions.get(profile_id, UNKNOWN)
return version
security.declareProtected(ManagePortal, "setLastVersionForProfile")
def setLastVersionForProfile(self, profile_id, version):
"""Set the last upgraded version for the specified profile.
"""
if version == UNKNOWN:
self.unsetLastVersionForProfile(profile_id)
return
prefix = "profile-"
if profile_id.startswith(prefix):
profile_id = profile_id[len(prefix) :]
if isinstance(version, basestring):
version = tuple(version.split("."))
# _profile_upgrade_versions is not persistent, so we must
# force a safe by storing it fresh on self, instead of editing
# the current dictionary.
prof_versions = self._profile_upgrade_versions.copy()
prof_versions[profile_id] = version
self._profile_upgrade_versions = prof_versions
security.declareProtected(ManagePortal, "unsetLastVersionForProfile")
def unsetLastVersionForProfile(self, profile_id):
"""Unset the last upgraded version for the specified profile.
"""
prefix = "profile-"
if profile_id.startswith(prefix):
profile_id = profile_id[len(prefix) :]
# _profile_upgrade_versions is not persistent, so we must
# force a safe by storing it fresh on self, instead of editing
# the current dictionary.
prof_versions = self._profile_upgrade_versions.copy()
if profile_id not in prof_versions:
return
del prof_versions[profile_id]
self._profile_upgrade_versions = prof_versions
security.declareProtected(ManagePortal, "getVersionForProfile")
def getVersionForProfile(self, profile_id):
"""Return the registered filesystem version for the specified
profile.
"""
return self.getProfileInfo(profile_id).get("version", UNKNOWN)
security.declareProtected(ManagePortal, "purgeProfileVersions")
def purgeProfileVersions(self):
"""Purge the profile upgrade versions.
"""
self._profile_upgrade_versions = {}
generic_logger.info("Profile upgrade versions purged.")
security.declareProtected(ManagePortal, "profileExists")
def profileExists(self, profile_id):
"""Check if a profile exists."""
try:
self.getProfileInfo(profile_id)
except KeyError:
return False
else:
return True
security.declareProtected(ManagePortal, "getProfileInfo")
def getProfileInfo(self, profile_id):
return _profile_registry.getProfileInfo(profile_id)
security.declareProtected(ManagePortal, "getDependenciesForProfile")
def getDependenciesForProfile(self, profile_id):
if profile_id.startswith("snapshot-"):
return ()
if not self.profileExists(profile_id):
raise KeyError(profile_id)
try:
return self.getProfileInfo(profile_id).get("dependencies", ())
except KeyError:
return ()
security.declareProtected(ManagePortal, "listProfilesWithUpgrades")
def listProfilesWithUpgrades(self):
profiles = listProfilesWithUpgrades()
profiles.sort()
return profiles
security.declarePrivate("_massageUpgradeInfo")
def _massageUpgradeInfo(self, info):
"""Add a couple of data points to the upgrade info dictionary.
"""
info = info.copy()
info["haspath"] = info["source"] and info["dest"]
info["ssource"] = ".".join(info["source"] or ("all",))
info["sdest"] = ".".join(info["dest"] or ("all",))
info["done"] = not info["proposed"] and info["step"].checker is not None and not info["step"].checker(self)
return info
security.declareProtected(ManagePortal, "listUpgrades")
def listUpgrades(self, profile_id, show_old=False):
"""Get the list of available upgrades.
"""
if show_old:
source = None
else:
source = self.getLastVersionForProfile(profile_id)
upgrades = listUpgradeSteps(self, profile_id, source)
res = []
for info in upgrades:
if type(info) == list:
subset = []
for subinfo in info:
subset.append(self._massageUpgradeInfo(subinfo))
res.append(subset)
else:
res.append(self._massageUpgradeInfo(info))
return res
security.declareProtected(ManagePortal, "manage_doUpgrades")
def manage_doUpgrades(self, request=None):
"""Perform all selected upgrade steps.
"""
if request is None:
request = self.REQUEST
logger = logging.getLogger("GenericSetup")
steps_to_run = request.form.get("upgrades", [])
profile_id = request.get("profile_id", "")
step = None
for step_id in steps_to_run:
step = _upgrade_registry.getUpgradeStep(profile_id, step_id)
if step is not None:
step.doStep(self)
msg = "Ran upgrade step %s for profile %s" % (step.title, profile_id)
logger.log(logging.INFO, msg)
# We update the profile version to the last one we have reached
# with running an upgrade step.
if step and step.dest is not None:
self.setLastVersionForProfile(profile_id, step.dest)
url = self.absolute_url()
request.RESPONSE.redirect("%s/manage_upgrades?saved=%s" % (url, profile_id))
security.declareProtected(ManagePortal, "upgradeProfile")
def upgradeProfile(self, profile_id, dest=None):
"""Upgrade a profile.
Apply all upgrade steps.
When 'dest' is given, only update to that version. If the
version is not found, give a warning and do nothing.
If the profile was not applied previously (last version for
profile is unknown) we do nothing.
"""
if self.getLastVersionForProfile(profile_id) == UNKNOWN:
generic_logger.warn("Version of profile %s is unknown, " "refusing to upgrade.", profile_id)
return
if dest is not None:
# Upgrade to a specific destination version, if found.
if isinstance(dest, basestring):
dest = tuple(dest.split("."))
if self.getLastVersionForProfile(profile_id) == dest:
generic_logger.warn("Profile %s is already at wanted " "destination %r.", profile_id, dest)
return
upgrades = self.listUpgrades(profile_id)
# First get a list of single steps to apply. This may be
# limited by the wanted destination version.
to_apply = []
dest_found = False
step = None
for upgrade in upgrades:
# An upgrade may be a single step (for a bare upgradeStep)
# or a list of steps (for upgradeSteps containing upgradeStep
# directives).
if not isinstance(upgrade, list):
upgrade = [upgrade]
for upgradestep in upgrade:
step = upgradestep["step"]
to_apply.append(step)
if dest is not None and step.dest == dest:
dest_found = True
if dest_found:
break
if dest is not None and not dest_found:
generic_logger.warn(
"No route found to destination version %r for profile %s. " "Profile stays at current version, %r",
dest,
profile_id,
self.getLastVersionForProfile(profile_id),
)
return
if to_apply:
for step in to_apply:
step.doStep(self)
# We update the profile version to the last one we have
# reached with running an upgrade step.
if step and step.dest is not None:
self.setLastVersionForProfile(profile_id, step.dest)
generic_logger.info(
"Profile %s upgraded to version %r.", profile_id, self.getLastVersionForProfile(profile_id)
)
else:
generic_logger.info(
"No upgrades available for profile %s. " "Profile stays at version %r.",
profile_id,
self.getLastVersionForProfile(profile_id),
)
#
# Helper methods
#
security.declarePrivate("_getImportContext")
def _getImportContext(self, context_id, should_purge=None, archive=None):
""" Crack ID and generate appropriate import context.
Note: it seems context_id (profile id) and archive (tarball)
are mutually exclusive. Exactly one of the two should be
None. There seems to be no use case for a different
combination.
"""
encoding = self.getEncoding()
if context_id is not None:
prefix = "snapshot-"
if context_id.startswith(prefix):
context_id = context_id[len(prefix) :]
if should_purge is None:
should_purge = True
return SnapshotImportContext(self, context_id, should_purge, encoding)
prefix = "profile-"
if context_id.startswith(prefix):
context_id = context_id[len(prefix) :]
info = _profile_registry.getProfileInfo(context_id)
if info.get("product"):
path = os.path.join(_getProductPath(info["product"]), info["path"])
else:
path = info["path"]
if should_purge is None:
should_purge = info.get("type") != EXTENSION
return DirectoryImportContext(self, path, should_purge, encoding)
if archive is not None:
return TarballImportContext(tool=self, archive_bits=archive, encoding="UTF8", should_purge=should_purge)
raise KeyError('Unknown context "%s"' % context_id)
security.declarePrivate("_updateImportStepsRegistry")
def _updateImportStepsRegistry(self, context, encoding):
""" Update our import steps registry from our profile.
"""
xml = context.readDataFile(IMPORT_STEPS_XML)
if xml is None:
return
info_list = self._import_registry.parseXML(xml, encoding)
for step_info in info_list:
id = step_info["id"]
version = step_info.get("version")
handler = step_info["handler"]
dependencies = tuple(step_info.get("dependencies", ()))
title = step_info.get("title", id)
description = "".join(step_info.get("description", []))
self._import_registry.registerStep(
id=id, version=version, handler=handler, dependencies=dependencies, title=title, description=description
)
security.declarePrivate("_updateExportStepsRegistry")
def _updateExportStepsRegistry(self, context, encoding):
""" Update our export steps registry from our profile.
"""
xml = context.readDataFile(EXPORT_STEPS_XML)
if xml is None:
return
info_list = self._export_registry.parseXML(xml, encoding)
for step_info in info_list:
id = step_info["id"]
handler = step_info["handler"]
title = step_info.get("title", id)
description = "".join(step_info.get("description", []))
self._export_registry.registerStep(id=id, handler=handler, title=title, description=description)
security.declarePrivate("_doRunImportStep")
def _doRunImportStep(self, step_id, context):
""" Run a single import step, using a pre-built context.
"""
__traceback_info__ = step_id
marker = object()
handler = self.getImportStep(step_id)
if handler is marker:
raise ValueError("Invalid import step: %s" % step_id)
if handler is None:
msg = "Step %s has an invalid import handler" % step_id
logger = logging.getLogger("GenericSetup")
logger.error(msg)
return "ERROR: " + msg
return handler(context)
security.declarePrivate("_doRunExportSteps")
def _doRunExportSteps(self, steps):
""" See ISetupTool.
"""
context = TarballExportContext(self)
messages = {}
marker = object()
for step_id in steps:
handler = self.getExportStep(step_id, marker)
if handler is marker:
raise ValueError("Invalid export step: %s" % step_id)
if handler is None:
msg = "Step %s has an invalid export handler" % step_id
logger = logging.getLogger("GenericSetup")
logger.error(msg)
messages[step_id] = msg
else:
messages[step_id] = handler(context)
return {
"steps": steps,
"messages": messages,
"tarball": context.getArchive(),
"filename": context.getArchiveFilename(),
}
security.declareProtected(ManagePortal, "getProfileDependencyChain")
def getProfileDependencyChain(self, profile_id, seen=None):
if seen is None:
seen = set()
elif profile_id in seen:
return [] # cycle break
seen.add(profile_id)
chain = []
dependencies = self.getDependenciesForProfile(profile_id)
for dependency in dependencies:
chain.extend(self.getProfileDependencyChain(dependency, seen))
chain.append(profile_id)
return chain
security.declarePrivate("_runImportStepsFromContext")
def _runImportStepsFromContext(
self,
steps=None,
purge_old=None,
profile_id=None,
archive=None,
ignore_dependencies=False,
blacklisted_steps=None,
dependency_strategy=None,
):
# 1. Determine upgrade strategy.
# What do we do with already applied dependency profiles?
# There are two ways to say you want to ignore all
# dependencies. If one is enabled, we enable the other too.
if dependency_strategy == DEPENDENCY_STRATEGY_IGNORE:
ignore_dependencies = True
elif ignore_dependencies:
dependency_strategy = DEPENDENCY_STRATEGY_IGNORE
# Turn None into the default:
if dependency_strategy is None:
dependency_strategy = DEFAULT_DEPENDENCY_STRATEGY
# Determine the settings based on the strategy.
if dependency_strategy == DEPENDENCY_STRATEGY_UPGRADE:
apply_new_profiles = True
reapply_old_profiles = False
upgrade_old_profiles = True
elif dependency_strategy == DEPENDENCY_STRATEGY_REAPPLY:
apply_new_profiles = True
reapply_old_profiles = True
upgrade_old_profiles = False
elif dependency_strategy == DEPENDENCY_STRATEGY_NEW:
apply_new_profiles = True
reapply_old_profiles = False
upgrade_old_profiles = False
elif dependency_strategy == DEPENDENCY_STRATEGY_IGNORE:
apply_new_profiles = False
reapply_old_profiles = False
upgrade_old_profiles = False
else:
raise ValueError("Unknown dependency_strategy %r." % dependency_strategy)
generic_logger.info("Importing profile %s with dependency strategy %s.", profile_id, dependency_strategy)
# 2. Gather a list of profiles to handle.
if profile_id is not None and not ignore_dependencies:
try:
chain = self.getProfileDependencyChain(profile_id)
except KeyError, e:
logger = logging.getLogger("GenericSetup")
logger.error("Unknown step in dependency chain: %s" % str(e))
raise
else:
