def _run(self, data):
logger.info("Start importing profile: " + data['config'])
Transmogrifier(self.context)(data['config'])
logger.info("Stop importing profile: " + data['config'])
IStatusMessage(self.request).addStatusMessage('Migrated: %s' %
data['config'],
type='info')
def import_oggbundle(app, args):
"""Handler for the 'bin/instance import' zopectl command.
"""
setup_logging()
# Discard the first three arguments, because they're not "actual" arguments
# but cruft that we get because of the way bin/instance [zopectl_cmd]
# scripts work.
args = parse_args(sys.argv[3:])
log.info("Importing OGGBundle %s" % args.bundle_path)
plone = setup_plone(get_first_plone_site(app))
# mark request with GEVER layer
alsoProvides(plone.REQUEST, IOpengeverBaseLayer)
# Don't use a separate ZODB connection to issue sequence numbers in
# order to avoid conflict errors during OGGBundle import
alsoProvides(plone.REQUEST, INoSeparateConnectionForSequenceNumbers)
# Add index to track imported GUIDs (if it doesn't exist yet)
add_guid_index()
transmogrifier = Transmogrifier(plone)
ann = IAnnotations(transmogrifier)
ann[BUNDLE_PATH_KEY] = args.bundle_path
ann[INTERMEDIATE_COMMITS_KEY] = not args.no_intermediate_commits
solr_enabled = api.portal.get_registry_record(
'opengever.base.interfaces.ISearchSettings.use_solr', default=False)
if solr_enabled:
# Check if solr is running
conn = getUtility(ISolrConnectionManager).connection
if conn.get('/schema').status == -1:
raise Exception(
"Solr isn't running, but solr reindexing is enabled. "
"Skipping solr reindexing via `--skip-solr`.")
else:
# Disable collective indexing as it can lead to too many
# subtransactions
unpatch_collective_indexing()
with DisabledLDAP(plone):
transmogrifier(u'opengever.bundle.oggbundle')
bundle = IAnnotations(transmogrifier)[BUNDLE_KEY]
timings = bundle.stats['timings']
if 'migration_finished' in timings:
duration = timings['migration_finished'] - timings['start_loading']
log.info("Duration: %.2fs" % duration.total_seconds())
log.info("Committing transaction...")
transaction.get().note("Finished import of OGGBundle %r" %
args.bundle_path)
transaction.commit()
log.info("Done.")
def test_oggbundle_transmogrifier(self):
# this is a bit hackish, but since builders currently don't work in
# layer setup/teardown and isolation of database content is ensured
# on a per test level we abuse just one test to setup the pipeline and
# test its data.
# load pipeline
# XXX move this to a layer
self.grant("Manager")
transmogrifier = Transmogrifier(api.portal.get())
IAnnotations(transmogrifier)[BUNDLE_PATH_KEY] = resource_filename(
'opengever.bundle.tests',
'assets/business_rule_violations.oggbundle')
# We need to add documents to dossiers that have already been created
# in the 'closed' state, which isn't allowed for anyone except users
# inheriting from `UnrestrictedUser` -> we need elevated privileges
with freeze(FROZEN_NOW), elevated_privileges():
transmogrifier(u'opengever.bundle.oggbundle')
# test content creation
# XXX use separate test-cases based on a layer
root = self.assert_repo_root_created()
self.assert_deeply_nested_repo_folder_created(root)
folder = root.restrictedTraverse(
'ordnungsposition-1/ordnungsposition-1.1')
self.assert_deeply_nested_subdossier_created(folder)
self.assert_resolved_dossier_with_violations_created(folder)
def runner(config, args={}):
from collective.transmogrifier.transmogrifier import Transmogrifier
# test.globs['transmogrifier'] = Transmogrifier(test.globs['plone'])
import zope.component
import zope.app.component
import collective.transmogrifier.sections
zcml.load_config('meta.zcml', zope.app.component)
zcml.load_config('meta.zcml', collective.transmogrifier)
zcml.load_config('configure.zcml', collective.transmogrifier.sections)
zcml.load_config('configure.zcml', transmogrify.htmltesting)
context = Context()
configuration_registry.registerConfiguration(
u'transmogrify.config.funnelweb', u"", u'', config)
transmogrifier = Transmogrifier(context)
overrides = {}
if type(args) == type(''):
for arg in args:
section, keyvalue = arg.split(':', 1)
key, value = keyvalue.split('=', 1)
overrides.setdefault('section', {})[key] = value
else:
overrides = args
transmogrifier(u'transmogrify.config.funnelweb', **overrides)
def test_oggbundle_transmogrifier(self):
# this is a bit hackish, but since builders currently don't work in
# layer setup/teardown and isolation of database content is ensured
# on a per test level we abuse just one test to setup the pipeline and
# test its data.
# load pipeline
# XXX move this to a layer
self.grant("Manager")
# Create the 'bundle_guid' index. In production, this will be done
# by the "bin/instance import" command in opengever.bundle.console
add_guid_index()
transmogrifier = Transmogrifier(api.portal.get())
IAnnotations(transmogrifier)[BUNDLE_PATH_KEY] = resource_filename(
'opengever.bundle.tests',
'assets/invalid_parent_reference.oggbundle')
with self.assertRaises(ReferenceNumberNotFound) as cm:
transmogrifier(u'opengever.bundle.oggbundle')
self.assertEqual(
"Couldn't find container with reference number Client1 7.7.7 "
"(referenced as parent by item by GUID GUID-dossier-A-7.7.7-1 )",
cm.exception.message)
def import_oggbundle(app, args):
"""Handler for the 'bin/instance import' zopectl command.
"""
setup_logging()
bundle_path = sys.argv[3]
log.info("Importing OGGBundle %s" % bundle_path)
plone = setup_plone(get_first_plone_site(app))
# mark request with GEVER layer
alsoProvides(plone.REQUEST, IOpengeverBaseLayer)
# Don't use a separate ZODB connection to issue sequence numbers in
# order to avoid conflict errors during OGGBundle import
alsoProvides(plone.REQUEST, INoSeparateConnectionForSequenceNumbers)
transmogrifier = Transmogrifier(plone)
IAnnotations(transmogrifier)[BUNDLE_PATH_KEY] = bundle_path
with DisabledLDAP(plone):
transmogrifier(u'opengever.bundle.oggbundle')
bundle = IAnnotations(transmogrifier)[BUNDLE_KEY]
timings = bundle.stats['timings']
duration = timings['done_post_processing'] - timings['start_loading']
log.info("Duration: %.2fs" % duration.total_seconds())
log.info("Committing transaction...")
transaction.get().note("Finished import of OGGBundle %r" % bundle_path)
transaction.commit()
log.info("Done.")
def test_oggbundle_transmogrifier(self):
# this is a bit hackish, but since builders currently don't work in
# layer setup/teardown and isolation of database content is ensured
# on a per test level we abuse just one test to setup the pipeline and
# test its data.
# load pipeline
# XXX move this to a layer
self.grant("Manager")
transmogrifier = Transmogrifier(api.portal.get())
annotations = IAnnotations(transmogrifier)
annotations[BUNDLE_PATH_KEY] = resource_filename(
'opengever.bundle.tests', 'assets/basic.oggbundle')
unc_share_asset_directory = resource_filename(
'opengever.bundle.tests', 'assets/files_outside_bundle')
annotations[BUNDLE_INGESTION_SETTINGS_KEY] = {
'unc_mounts': {u'\\\\host\\mount': unc_share_asset_directory},
}
# We need to add documents to dossiers that have already been created
# in the 'closed' state, which isn't allowed for anyone except users
# inheriting from `UnrestrictedUser` -> we need elevated privileges
with freeze(FROZEN_NOW), elevated_privileges():
transmogrifier(u'opengever.bundle.oggbundle')
bundle = annotations[BUNDLE_KEY]
# test content creation
# XXX use separate test-cases based on a layer
root = self.assert_repo_root_created()
folder_staff = self.assert_repo_folders_created(root)
dossier_peter = self.assert_dossiers_created(folder_staff)
self.assert_documents_created(dossier_peter)
self.assert_report_data_collected(bundle)
def handleRun(self, action):
data, errors = self.extractData()
if errors:
return False
logger.info("Start importing profile: " + data['config'])
Transmogrifier(self.context)(data['config'])
logger.info("Stop importing profile: " + data['config'])
def load_content(context):
if context.readDataFile('loadcontent_various.txt') is None:
return
portal = context.getSite()
if HAS_TRANSMOGRIFIER:
transmogrifier = Transmogrifier(portal)
transmogrifier(u'loadcontent')
transmogrifier(u'loadbookings')
return 'Imported content types...'
return 'Please install collective.transmogrifier to use this import step'
def start_import(context):
transmogrifier = Transmogrifier(context)
transmogrifier(u'opengever.examplecontent.repository')
transaction.commit()
transmogrifier(u'opengever.examplecontent.various')
transaction.commit()
transmogrifier(u'opengever.examplecontent.contacts')
transaction.commit()
def setUp(test):
baseSetUp(test)
from collective.transmogrifier.transmogrifier import Transmogrifier
test.globs['transmogrifier'] = Transmogrifier(test.globs['plone'])
zcml.load_config("meta.zcml", zope.security)
zcml.load_config("meta.zcml", zope.component)
zcml.load_config("meta.zcml", collective.transmogrifier)
zcml.load_config("configure.zcml", collective.transmogrifier)
zcml.load_config("configure.zcml", gu.transmogrifier)
zcml.load_config('testing.zcml', gu.transmogrifier.tests)
zcml.load_config('configure.zcml', org.bccvl.compute)
def test_transmogrify(self):
catalog = getToolByName(self.portal, 'portal_catalog')
results = catalog(portal_type='News Item')
self.assertEqual(len(results), 4)
results = catalog(portal_type='transmogrify.nitf.content')
self.assertEqual(len(results), 0)
transmogrifier = Transmogrifier(self.portal)
transmogrifier("nitfmigrator")
results = catalog({'portal_type': u'transmogrify.nitf.content', }, )
self.assertEqual(len(results), 4)
def setUp(test):
baseSetUp(test)
from collective.transmogrifier.transmogrifier import Transmogrifier
test.globs['transmogrifier'] = Transmogrifier(test.globs['plone'])
import zope.component
import collective.transmogrifier.sections
zcml.load_config('meta.zcml', zope.app.component)
zcml.load_config('configure.zcml', collective.transmogrifier.sections)
test.globs['plone'].portal_transforms = MockPortalTransforms()
provideUtility(PrettyPrinter,
name=u'collective.transmogrifier.sections.tests.pprinter')
provideUtility(WebCrawler, name=u'transmogrify.htmltesting.webcrawler')
provideUtility(TreeSerializer,
name=u'transmogrify.htmltesting.treeserializer')
provideUtility(TypeRecognitor,
name=u'transmogrify.htmltesting.typerecognitor')
provideUtility(TemplateFinder,
name=u'transmogrify.htmltesting.templatefinder')
provideUtility(urlnormalizer)
provideUtility(Relinker, name=u'transmogrify.htmltesting.relinker')
provideUtility(SimpleXPath, name=u'transmogrify.htmltesting.simplexpath')
provideUtility(SafePortalTransforms,
name=u'transmogrify.htmltesting.safeportaltransforms')
from backlinkstitle import BacklinksTitle
provideUtility(BacklinksTitle,
name=u'transmogrify.htmltesting.backlinkstitle')
from isindex import IsIndex
provideUtility(IsIndex, name=u'transmogrify.htmltesting.isindex')
from pathmover import PathMover
provideUtility(PathMover, name=u'transmogrify.htmltesting.pathmover')
from safeatschemaupdater import SafeATSchemaUpdaterSection
provideUtility(SafeATSchemaUpdaterSection,
name=u'transmogrify.htmltesting.safeatschemaupdater')
from constructor import SafeConstructorSection
provideUtility(SafeConstructorSection,
name=u'transmogrify.htmltesting.constructor')
from makeattachments import MakeAttachments
provideUtility(MakeAttachments,
name=u'transmogrify.htmltesting.makeattachments')
from debugsection import DebugSection
provideUtility(DebugSection, name=u'transmogrify.htmltesting.debugsection')
from staticcreator import StaticCreatorSection
provideUtility(StaticCreatorSection,
name=u'transmogrify.htmltesting.staticcreator')
provideUtility(HTMLSource,
name=u'transmogrify.htmltesting.test.htmlsource')
provideUtility(HTMLBacklinkSource,
name=u'transmogrify.htmltesting.test.htmlbacklinksource')
def execute_pipeline(portal, filepath):
try:
configuration_registry.getConfiguration(PIPELINE_ID)
except KeyError:
configuration_registry.registerConfiguration(PIPELINE_ID, u'', u'',
filepath)
try:
transmogrifier = Transmogrifier(portal)
transmogrifier(PIPELINE_ID)
except Exception as error:
error_msg = u"type: '{}', msg: '{}'".format(type(error), error)
to_send = [u'Critical error during pipeline: {}'.format(error_msg)]
send_report(portal, to_send)
raise error
def transmogrify(self, ignore_seen=False):
from collective.transmogrifier.transmogrifier import Transmogrifier
transmogrifier = Transmogrifier(self.portal)
transmogrifier(TEST_PIPELINE_CONFIG)
# RedirectorBlueprint logs the final count of seen and changed
# objects:
final_log = [
log_record for log_record in self.handler.buffer
# c.f.: RedirectorBlueprint.__iter__():
if getattr(log_record, 'changed_count', None) is not None
][-1]
if not ignore_seen:
self.assertEqual(final_log.seen_count, len(SOURCE_SAMPLE))
return final_log
def setUp(test):
baseSetUp(test)
from collective.transmogrifier.transmogrifier import Transmogrifier
test.globs['transmogrifier'] = Transmogrifier(test.globs['plone'])
import zope.component
import collective.transmogrifier.sections
zcml.load_config('meta.zcml', zope.app.component)
zcml.load_config('configure.zcml', collective.transmogrifier.sections)
zcml.load_config('configure.zcml', transmogrify.webcrawler)
provideUtility(PrettyPrinter,
name=u'collective.transmogrifier.sections.tests.pprinter')
def content_creation(setup):
"""Create content based on json definitions with a folder
"content_creation".
"""
data = setup.isDirectory(FOLDER_NAME)
if not data:
return
path = os.path.join(setup._profile_path, FOLDER_NAME).encode('utf-8')
assert os.path.isdir(path)
mogrifier = Transmogrifier(setup.getSite())
mogrifier(u'ftw.inflator.creation.content_creation_config',
jsonsource=dict(directory=path))
def start_import(context):
transmogrifier = Transmogrifier(context)
transmogrifier(u'opengever.setup.various')
transaction.commit()
transmogrifier(u'opengever.setup.templates')
transaction.commit()
# check if the setup should set local roles also one the repo root
if context.REQUEST.get('local_roles') == '1':
transmogrifier(u'opengever.setup.local_roles_with_repo')
else:
transmogrifier(u'opengever.setup.local_roles')
transaction.commit()
def import_oggbundle(app, args):
"""Handler for the 'bin/instance import' zopectl command.
"""
setup_logging()
# Discard the first three arguments, because they're not "actual" arguments
# but cruft that we get because of the way bin/instance [zopectl_cmd]
# scripts work.
args = parse_args(sys.argv[3:])
log.info("Importing OGGBundle %s" % args.bundle_path)
plone = setup_plone(get_first_plone_site(app))
# mark request with GEVER layer
alsoProvides(plone.REQUEST, IOpengeverBaseLayer)
# Don't use a separate ZODB connection to issue sequence numbers in
# order to avoid conflict errors during OGGBundle import
alsoProvides(plone.REQUEST, INoSeparateConnectionForSequenceNumbers)
# Add index to track imported GUIDs (if it doesn't exist yet)
add_guid_index()
transmogrifier = Transmogrifier(plone)
ann = IAnnotations(transmogrifier)
ann[BUNDLE_PATH_KEY] = args.bundle_path
ann[INTERMEDIATE_COMMITS_KEY] = not args.no_intermediate_commits
with DisabledLDAP(plone):
transmogrifier(u'opengever.bundle.oggbundle')
bundle = IAnnotations(transmogrifier)[BUNDLE_KEY]
timings = bundle.stats['timings']
if 'migration_finished' in timings:
duration = timings['migration_finished'] - timings['start_loading']
log.info("Duration: %.2fs" % duration.total_seconds())
log.info("Committing transaction...")
transaction.get().note("Finished import of OGGBundle %r" %
args.bundle_path)
transaction.commit()
log.info("Done.")
def sectionsSetUp(test):
setUp(test)
from collective.transmogrifier.transmogrifier import Transmogrifier
test.globs['transmogrifier'] = Transmogrifier(test.globs['plone'])
import zope.component
import collective.transmogrifier.sections
zcml.load_config('meta.zcml', zope.component)
zcml.load_config('configure.zcml', collective.transmogrifier.sections)
provideUtility(
SampleSource,
name=u'collective.transmogrifier.sections.tests.samplesource')
provideUtility(
RangeSource,
name=u'collective.transmogrifier.sections.tests.rangesource')
provideUtility(PrettyPrinter,
name=u'collective.transmogrifier.sections.tests.pprinter')
def setupPortalContent(p):
"""Cria conteudo de exemplo para este portal"""
# Importa conteudo
transmogrify = Transmogrifier(p)
transmogrify('brasil.gov.portal.conteudo')
# Pagina Inicial
capa_como_padrao(p)
# Pasta Assuntos
configura_assuntos(p)
# Pasta Acesso a Informacao
configura_acesso_informacao(p)
# Pasta de Menu de Relevancia
configura_menu_relevancia(p)
# Pasta Servicos
configura_servicos(p)
# Pasta Imagens
configura_imagens(p)
# Colecao de Ultimas noticias
configura_ultimas_noticias(p)
# Destaques
configura_destaques(p)
obj_ids = [
'acesso-a-informacao', 'assuntos', 'servicos', 'imagens', 'noticias',
'rodape', 'destaques', 'menu-de-relevancia', 'links-destaques', 'home',
'contato', 'acessibilidade', 'acesso-a-sistemas', 'area-de-imprensa',
'rss', 'eventos', 'videos', 'audios', 'links',
'pastas-com-exemplos-de-pecas'
]
publish_content(p, obj_ids)
# Remove conteúdo default do Doormat
remove_conteudo_doormat(p)
def sectionsSetUp(test):
setUp(test)
from collective.transmogrifier.transmogrifier import Transmogrifier
test.globs['transmogrifier'] = Transmogrifier(test.globs['plone'])
import collective.transmogrifier.sections
zcml.load_config('testing.zcml', collective.transmogrifier.sections)
provideUtility(
SampleSource,
name=u'collective.transmogrifier.sections.tests.samplesource')
provideUtility(
RangeSource,
name=u'collective.transmogrifier.sections.tests.rangesource')
import logging
from zope.testing import loggingsupport
test.globs['handler'] = loggingsupport.InstalledHandler('logger',
level=logging.INFO)
def run_migration(self, *args):
call = args[0]
if len(call) > 2:
pipeline_id = call[2]
for plonesite in self.objectValues("Plone Site"):
setSite(plonesite)
plonesite = spoofRequest(plonesite)
if pipeline_id in configuration_registry.listConfigurationIds():
transmogrifier = Transmogrifier(plonesite)
overrides = {}
if len(call) >= 4:
json_path = call[3]
overrides["jsonsource"] = {"path": json_path}
transmogrifier(pipeline_id, **overrides)
transaction.commit()
print("All set!")
else:
print("This migration id is invalid.")
else:
print("You need to tell me the migration id to run")
def handle_action(self, action):
""" Processes the form and runs the selected pipeline.
"""
data, errors = self.extractData()
if errors:
self.status = self.formErorrsMessage
return
pipeline_id = data.get('pipeline', None)
pipelines = configuration_registry.listConfigurationIds()
if pipeline_id is not None and pipeline_id in pipelines:
transmogrifier = Transmogrifier(self.context)
transmogrifier(pipeline_id)
pipeline = configuration_registry.getConfiguration(pipeline_id)
IStatusMessage(self.request).addStatusMessage(
'{0} {1}'.format(pipeline['title'],
_('successfully executed.')), 'info')
self.request.response.redirect(self.context.absolute_url())
else:
raise ActionExecutionError(
Invalid('{0}: {1}'.format(_(u'Invalid pipeline'),
pipeline_id)))
def handleRun(self, action):
data, errors = self.extractData()
if errors:
return False
overrides = {}
for key, value in data.items():
if '-' not in key:
continue
elif value is None:
continue
elif isinstance(value, list):
value = '\n'.join(value)
elif isinstance(value, bool):
value = value and 'True' or 'False'
else:
value = str(value)
section, key = key.split('-')
overrides.setdefault(section, {})[key] = value
logger.info("Start importing profile: " + data['config'])
Transmogrifier(self.context)(data['config'], **overrides)
logger.info("Stop importing profile: " + data['config'])
def setUp(test):
baseSetUp(test)
from collective.transmogrifier.transmogrifier import Transmogrifier
test.globs['transmogrifier'] = Transmogrifier(test.globs['plone'])
import zope.component
import collective.transmogrifier.sections
zcml.load_config('meta.zcml', zope.component)
zcml.load_config('configure.zcml', collective.transmogrifier.sections)
test.globs['plone'].portal_transforms = MockPortalTransforms()
provideUtility(PrettyPrinter,
name=u'collective.transmogrifier.sections.tests.pprinter')
#provideUtility(WebCrawler,
#name=u'transmogrify.webcrawler.webcrawler')
provideUtility(urlnormalizer)
#provideUtility(Relinker,
#name=u'transmogrify.webcrawler.relinker')
#provideUtility(SafePortalTransforms,
#name=u'transmogrify.webcrawler.safeportaltransforms')
#from backlinkstitle import BacklinksTitle
#provideUtility(BacklinksTitle,
#name=u'transmogrify.webcrawler.backlinkstitle')
from isindex import IsIndex
provideUtility(IsIndex, name=u'transmogrify.webcrawler.isindex')
#from makeattachments import MakeAttachments
#provideUtility(MakeAttachments,
#name=u'transmogrify.webcrawler.makeattachments')
#from debugsection import DebugSection
#provideUtility(DebugSection,
#name=u'transmogrify.webcrawler.debugsection')
provideUtility(HTMLSource, name=u'transmogrify.webcrawler.test.htmlsource')
provideUtility(HTMLBacklinkSource,
name=u'transmogrify.webcrawler.test.htmlbacklinksource')
def add(self, object):
# FIXME: this is a workaround, which is fine for small uploaded files.
# large uploads should go through another process anyway
# TODO: re implementing this method is the only way to know
# the full path of the object. We need the path to apply
# the transmogrifier chain.
#fti = getUtility(IDexterityFTI, name=self.portal_type)
container = aq_inner(self.context)
new_object = addContentToContainer(container, object)
# set data genre:
if self.datagenre:
IBCCVLMetadata(new_object)['genre'] = self.datagenre
# rdf commit should happens in transmogrifier step later on
# if fti.immediate_view:
# self.immediate_view = "%s/%s/%s" % (container.absolute_url(), new_object.id, fti.immediate_view,)
# else:
# self.immediate_view = "%s/%s" % (container.absolute_url(), new_object.id)
# run transmogrify md extraction here
# TODO: move this to an event listener?
tm = Transmogrifier(new_object)
tm('org.bccvl.site.add_file_metadata')
# We have to reindex after updating the object
new_object.reindexObject()
def import_bundle(self, bundle_name):
bundle_path = resource_filename('opengever.bundle.tests',
'assets/%s' % bundle_name)
transmogrifier = Transmogrifier(api.portal.get())
IAnnotations(transmogrifier)[BUNDLE_PATH_KEY] = bundle_path
transmogrifier(u'opengever.bundle.oggbundle')
def install_templates(self):
transmogrifier = Transmogrifier(self.setup.getSite())
transmogrifier(u'opengever.setup.content',
jsonsource=dict(directory=self.path))
def configure_local_roles(self):
transmogrifier = Transmogrifier(self.setup.getSite())
transmogrifier(u'opengever.setup.local_roles',
jsonsource=dict(directory=self.path))
def _loadOptions(self, opts):
from collective.transmogrifier.transmogrifier import Transmogrifier
tm = Transmogrifier(object())
tm._raw = opts
tm._data = {}
return tm
