def _run(self, data): logger.info("Start importing profile: " + data['config']) Transmogrifier(self.context)(data['config']) logger.info("Stop importing profile: " + data['config']) IStatusMessage(self.request).addStatusMessage('Migrated: %s' % data['config'], type='info')
def import_oggbundle(app, args): """Handler for the 'bin/instance import' zopectl command. """ setup_logging() # Discard the first three arguments, because they're not "actual" arguments # but cruft that we get because of the way bin/instance [zopectl_cmd] # scripts work. args = parse_args(sys.argv[3:]) log.info("Importing OGGBundle %s" % args.bundle_path) plone = setup_plone(get_first_plone_site(app)) # mark request with GEVER layer alsoProvides(plone.REQUEST, IOpengeverBaseLayer) # Don't use a separate ZODB connection to issue sequence numbers in # order to avoid conflict errors during OGGBundle import alsoProvides(plone.REQUEST, INoSeparateConnectionForSequenceNumbers) # Add index to track imported GUIDs (if it doesn't exist yet) add_guid_index() transmogrifier = Transmogrifier(plone) ann = IAnnotations(transmogrifier) ann[BUNDLE_PATH_KEY] = args.bundle_path ann[INTERMEDIATE_COMMITS_KEY] = not args.no_intermediate_commits solr_enabled = api.portal.get_registry_record( 'opengever.base.interfaces.ISearchSettings.use_solr', default=False) if solr_enabled: # Check if solr is running conn = getUtility(ISolrConnectionManager).connection if conn.get('/schema').status == -1: raise Exception( "Solr isn't running, but solr reindexing is enabled. " "Skipping solr reindexing via `--skip-solr`.") else: # Disable collective indexing as it can lead to too many # subtransactions unpatch_collective_indexing() with DisabledLDAP(plone): transmogrifier(u'opengever.bundle.oggbundle') bundle = IAnnotations(transmogrifier)[BUNDLE_KEY] timings = bundle.stats['timings'] if 'migration_finished' in timings: duration = timings['migration_finished'] - timings['start_loading'] log.info("Duration: %.2fs" % duration.total_seconds()) log.info("Committing transaction...") transaction.get().note("Finished import of OGGBundle %r" % args.bundle_path) transaction.commit() log.info("Done.")
def test_oggbundle_transmogrifier(self): # this is a bit hackish, but since builders currently don't work in # layer setup/teardown and isolation of database content is ensured # on a per test level we abuse just one test to setup the pipeline and # test its data. # load pipeline # XXX move this to a layer self.grant("Manager") transmogrifier = Transmogrifier(api.portal.get()) IAnnotations(transmogrifier)[BUNDLE_PATH_KEY] = resource_filename( 'opengever.bundle.tests', 'assets/business_rule_violations.oggbundle') # We need to add documents to dossiers that have already been created # in the 'closed' state, which isn't allowed for anyone except users # inheriting from `UnrestrictedUser` -> we need elevated privileges with freeze(FROZEN_NOW), elevated_privileges(): transmogrifier(u'opengever.bundle.oggbundle') # test content creation # XXX use separate test-cases based on a layer root = self.assert_repo_root_created() self.assert_deeply_nested_repo_folder_created(root) folder = root.restrictedTraverse( 'ordnungsposition-1/ordnungsposition-1.1') self.assert_deeply_nested_subdossier_created(folder) self.assert_resolved_dossier_with_violations_created(folder)
def runner(config, args={}): from collective.transmogrifier.transmogrifier import Transmogrifier # test.globs['transmogrifier'] = Transmogrifier(test.globs['plone']) import zope.component import zope.app.component import collective.transmogrifier.sections zcml.load_config('meta.zcml', zope.app.component) zcml.load_config('meta.zcml', collective.transmogrifier) zcml.load_config('configure.zcml', collective.transmogrifier.sections) zcml.load_config('configure.zcml', transmogrify.htmltesting) context = Context() configuration_registry.registerConfiguration( u'transmogrify.config.funnelweb', u"", u'', config) transmogrifier = Transmogrifier(context) overrides = {} if type(args) == type(''): for arg in args: section, keyvalue = arg.split(':', 1) key, value = keyvalue.split('=', 1) overrides.setdefault('section', {})[key] = value else: overrides = args transmogrifier(u'transmogrify.config.funnelweb', **overrides)
def test_oggbundle_transmogrifier(self): # this is a bit hackish, but since builders currently don't work in # layer setup/teardown and isolation of database content is ensured # on a per test level we abuse just one test to setup the pipeline and # test its data. # load pipeline # XXX move this to a layer self.grant("Manager") # Create the 'bundle_guid' index. In production, this will be done # by the "bin/instance import" command in opengever.bundle.console add_guid_index() transmogrifier = Transmogrifier(api.portal.get()) IAnnotations(transmogrifier)[BUNDLE_PATH_KEY] = resource_filename( 'opengever.bundle.tests', 'assets/invalid_parent_reference.oggbundle') with self.assertRaises(ReferenceNumberNotFound) as cm: transmogrifier(u'opengever.bundle.oggbundle') self.assertEqual( "Couldn't find container with reference number Client1 7.7.7 " "(referenced as parent by item by GUID GUID-dossier-A-7.7.7-1 )", cm.exception.message)
def import_oggbundle(app, args): """Handler for the 'bin/instance import' zopectl command. """ setup_logging() bundle_path = sys.argv[3] log.info("Importing OGGBundle %s" % bundle_path) plone = setup_plone(get_first_plone_site(app)) # mark request with GEVER layer alsoProvides(plone.REQUEST, IOpengeverBaseLayer) # Don't use a separate ZODB connection to issue sequence numbers in # order to avoid conflict errors during OGGBundle import alsoProvides(plone.REQUEST, INoSeparateConnectionForSequenceNumbers) transmogrifier = Transmogrifier(plone) IAnnotations(transmogrifier)[BUNDLE_PATH_KEY] = bundle_path with DisabledLDAP(plone): transmogrifier(u'opengever.bundle.oggbundle') bundle = IAnnotations(transmogrifier)[BUNDLE_KEY] timings = bundle.stats['timings'] duration = timings['done_post_processing'] - timings['start_loading'] log.info("Duration: %.2fs" % duration.total_seconds()) log.info("Committing transaction...") transaction.get().note("Finished import of OGGBundle %r" % bundle_path) transaction.commit() log.info("Done.")
def test_oggbundle_transmogrifier(self): # this is a bit hackish, but since builders currently don't work in # layer setup/teardown and isolation of database content is ensured # on a per test level we abuse just one test to setup the pipeline and # test its data. # load pipeline # XXX move this to a layer self.grant("Manager") transmogrifier = Transmogrifier(api.portal.get()) annotations = IAnnotations(transmogrifier) annotations[BUNDLE_PATH_KEY] = resource_filename( 'opengever.bundle.tests', 'assets/basic.oggbundle') unc_share_asset_directory = resource_filename( 'opengever.bundle.tests', 'assets/files_outside_bundle') annotations[BUNDLE_INGESTION_SETTINGS_KEY] = { 'unc_mounts': {u'\\\\host\\mount': unc_share_asset_directory}, } # We need to add documents to dossiers that have already been created # in the 'closed' state, which isn't allowed for anyone except users # inheriting from `UnrestrictedUser` -> we need elevated privileges with freeze(FROZEN_NOW), elevated_privileges(): transmogrifier(u'opengever.bundle.oggbundle') bundle = annotations[BUNDLE_KEY] # test content creation # XXX use separate test-cases based on a layer root = self.assert_repo_root_created() folder_staff = self.assert_repo_folders_created(root) dossier_peter = self.assert_dossiers_created(folder_staff) self.assert_documents_created(dossier_peter) self.assert_report_data_collected(bundle)
def handleRun(self, action): data, errors = self.extractData() if errors: return False logger.info("Start importing profile: " + data['config']) Transmogrifier(self.context)(data['config']) logger.info("Stop importing profile: " + data['config'])
def load_content(context): if context.readDataFile('loadcontent_various.txt') is None: return portal = context.getSite() if HAS_TRANSMOGRIFIER: transmogrifier = Transmogrifier(portal) transmogrifier(u'loadcontent') transmogrifier(u'loadbookings') return 'Imported content types...' return 'Please install collective.transmogrifier to use this import step'
def start_import(context): transmogrifier = Transmogrifier(context) transmogrifier(u'opengever.examplecontent.repository') transaction.commit() transmogrifier(u'opengever.examplecontent.various') transaction.commit() transmogrifier(u'opengever.examplecontent.contacts') transaction.commit()
def setUp(test): baseSetUp(test) from collective.transmogrifier.transmogrifier import Transmogrifier test.globs['transmogrifier'] = Transmogrifier(test.globs['plone']) zcml.load_config("meta.zcml", zope.security) zcml.load_config("meta.zcml", zope.component) zcml.load_config("meta.zcml", collective.transmogrifier) zcml.load_config("configure.zcml", collective.transmogrifier) zcml.load_config("configure.zcml", gu.transmogrifier) zcml.load_config('testing.zcml', gu.transmogrifier.tests) zcml.load_config('configure.zcml', org.bccvl.compute)
def test_transmogrify(self): catalog = getToolByName(self.portal, 'portal_catalog') results = catalog(portal_type='News Item') self.assertEqual(len(results), 4) results = catalog(portal_type='transmogrify.nitf.content') self.assertEqual(len(results), 0) transmogrifier = Transmogrifier(self.portal) transmogrifier("nitfmigrator") results = catalog({'portal_type': u'transmogrify.nitf.content', }, ) self.assertEqual(len(results), 4)
def setUp(test): baseSetUp(test) from collective.transmogrifier.transmogrifier import Transmogrifier test.globs['transmogrifier'] = Transmogrifier(test.globs['plone']) import zope.component import collective.transmogrifier.sections zcml.load_config('meta.zcml', zope.app.component) zcml.load_config('configure.zcml', collective.transmogrifier.sections) test.globs['plone'].portal_transforms = MockPortalTransforms() provideUtility(PrettyPrinter, name=u'collective.transmogrifier.sections.tests.pprinter') provideUtility(WebCrawler, name=u'transmogrify.htmltesting.webcrawler') provideUtility(TreeSerializer, name=u'transmogrify.htmltesting.treeserializer') provideUtility(TypeRecognitor, name=u'transmogrify.htmltesting.typerecognitor') provideUtility(TemplateFinder, name=u'transmogrify.htmltesting.templatefinder') provideUtility(urlnormalizer) provideUtility(Relinker, name=u'transmogrify.htmltesting.relinker') provideUtility(SimpleXPath, name=u'transmogrify.htmltesting.simplexpath') provideUtility(SafePortalTransforms, name=u'transmogrify.htmltesting.safeportaltransforms') from backlinkstitle import BacklinksTitle provideUtility(BacklinksTitle, name=u'transmogrify.htmltesting.backlinkstitle') from isindex import IsIndex provideUtility(IsIndex, name=u'transmogrify.htmltesting.isindex') from pathmover import PathMover provideUtility(PathMover, name=u'transmogrify.htmltesting.pathmover') from safeatschemaupdater import SafeATSchemaUpdaterSection provideUtility(SafeATSchemaUpdaterSection, name=u'transmogrify.htmltesting.safeatschemaupdater') from constructor import SafeConstructorSection provideUtility(SafeConstructorSection, name=u'transmogrify.htmltesting.constructor') from makeattachments import MakeAttachments provideUtility(MakeAttachments, name=u'transmogrify.htmltesting.makeattachments') from debugsection import DebugSection provideUtility(DebugSection, name=u'transmogrify.htmltesting.debugsection') from staticcreator import StaticCreatorSection provideUtility(StaticCreatorSection, name=u'transmogrify.htmltesting.staticcreator') provideUtility(HTMLSource, name=u'transmogrify.htmltesting.test.htmlsource') provideUtility(HTMLBacklinkSource, name=u'transmogrify.htmltesting.test.htmlbacklinksource')
def execute_pipeline(portal, filepath): try: configuration_registry.getConfiguration(PIPELINE_ID) except KeyError: configuration_registry.registerConfiguration(PIPELINE_ID, u'', u'', filepath) try: transmogrifier = Transmogrifier(portal) transmogrifier(PIPELINE_ID) except Exception as error: error_msg = u"type: '{}', msg: '{}'".format(type(error), error) to_send = [u'Critical error during pipeline: {}'.format(error_msg)] send_report(portal, to_send) raise error
def transmogrify(self, ignore_seen=False): from collective.transmogrifier.transmogrifier import Transmogrifier transmogrifier = Transmogrifier(self.portal) transmogrifier(TEST_PIPELINE_CONFIG) # RedirectorBlueprint logs the final count of seen and changed # objects: final_log = [ log_record for log_record in self.handler.buffer # c.f.: RedirectorBlueprint.__iter__(): if getattr(log_record, 'changed_count', None) is not None ][-1] if not ignore_seen: self.assertEqual(final_log.seen_count, len(SOURCE_SAMPLE)) return final_log
def setUp(test): baseSetUp(test) from collective.transmogrifier.transmogrifier import Transmogrifier test.globs['transmogrifier'] = Transmogrifier(test.globs['plone']) import zope.component import collective.transmogrifier.sections zcml.load_config('meta.zcml', zope.app.component) zcml.load_config('configure.zcml', collective.transmogrifier.sections) zcml.load_config('configure.zcml', transmogrify.webcrawler) provideUtility(PrettyPrinter, name=u'collective.transmogrifier.sections.tests.pprinter')
def content_creation(setup): """Create content based on json definitions with a folder "content_creation". """ data = setup.isDirectory(FOLDER_NAME) if not data: return path = os.path.join(setup._profile_path, FOLDER_NAME).encode('utf-8') assert os.path.isdir(path) mogrifier = Transmogrifier(setup.getSite()) mogrifier(u'ftw.inflator.creation.content_creation_config', jsonsource=dict(directory=path))
def start_import(context): transmogrifier = Transmogrifier(context) transmogrifier(u'opengever.setup.various') transaction.commit() transmogrifier(u'opengever.setup.templates') transaction.commit() # check if the setup should set local roles also one the repo root if context.REQUEST.get('local_roles') == '1': transmogrifier(u'opengever.setup.local_roles_with_repo') else: transmogrifier(u'opengever.setup.local_roles') transaction.commit()
def import_oggbundle(app, args): """Handler for the 'bin/instance import' zopectl command. """ setup_logging() # Discard the first three arguments, because they're not "actual" arguments # but cruft that we get because of the way bin/instance [zopectl_cmd] # scripts work. args = parse_args(sys.argv[3:]) log.info("Importing OGGBundle %s" % args.bundle_path) plone = setup_plone(get_first_plone_site(app)) # mark request with GEVER layer alsoProvides(plone.REQUEST, IOpengeverBaseLayer) # Don't use a separate ZODB connection to issue sequence numbers in # order to avoid conflict errors during OGGBundle import alsoProvides(plone.REQUEST, INoSeparateConnectionForSequenceNumbers) # Add index to track imported GUIDs (if it doesn't exist yet) add_guid_index() transmogrifier = Transmogrifier(plone) ann = IAnnotations(transmogrifier) ann[BUNDLE_PATH_KEY] = args.bundle_path ann[INTERMEDIATE_COMMITS_KEY] = not args.no_intermediate_commits with DisabledLDAP(plone): transmogrifier(u'opengever.bundle.oggbundle') bundle = IAnnotations(transmogrifier)[BUNDLE_KEY] timings = bundle.stats['timings'] if 'migration_finished' in timings: duration = timings['migration_finished'] - timings['start_loading'] log.info("Duration: %.2fs" % duration.total_seconds()) log.info("Committing transaction...") transaction.get().note("Finished import of OGGBundle %r" % args.bundle_path) transaction.commit() log.info("Done.")
def sectionsSetUp(test): setUp(test) from collective.transmogrifier.transmogrifier import Transmogrifier test.globs['transmogrifier'] = Transmogrifier(test.globs['plone']) import zope.component import collective.transmogrifier.sections zcml.load_config('meta.zcml', zope.component) zcml.load_config('configure.zcml', collective.transmogrifier.sections) provideUtility( SampleSource, name=u'collective.transmogrifier.sections.tests.samplesource') provideUtility( RangeSource, name=u'collective.transmogrifier.sections.tests.rangesource') provideUtility(PrettyPrinter, name=u'collective.transmogrifier.sections.tests.pprinter')
def setupPortalContent(p): """Cria conteudo de exemplo para este portal""" # Importa conteudo transmogrify = Transmogrifier(p) transmogrify('brasil.gov.portal.conteudo') # Pagina Inicial capa_como_padrao(p) # Pasta Assuntos configura_assuntos(p) # Pasta Acesso a Informacao configura_acesso_informacao(p) # Pasta de Menu de Relevancia configura_menu_relevancia(p) # Pasta Servicos configura_servicos(p) # Pasta Imagens configura_imagens(p) # Colecao de Ultimas noticias configura_ultimas_noticias(p) # Destaques configura_destaques(p) obj_ids = [ 'acesso-a-informacao', 'assuntos', 'servicos', 'imagens', 'noticias', 'rodape', 'destaques', 'menu-de-relevancia', 'links-destaques', 'home', 'contato', 'acessibilidade', 'acesso-a-sistemas', 'area-de-imprensa', 'rss', 'eventos', 'videos', 'audios', 'links', 'pastas-com-exemplos-de-pecas' ] publish_content(p, obj_ids) # Remove conteúdo default do Doormat remove_conteudo_doormat(p)
def sectionsSetUp(test): setUp(test) from collective.transmogrifier.transmogrifier import Transmogrifier test.globs['transmogrifier'] = Transmogrifier(test.globs['plone']) import collective.transmogrifier.sections zcml.load_config('testing.zcml', collective.transmogrifier.sections) provideUtility( SampleSource, name=u'collective.transmogrifier.sections.tests.samplesource') provideUtility( RangeSource, name=u'collective.transmogrifier.sections.tests.rangesource') import logging from zope.testing import loggingsupport test.globs['handler'] = loggingsupport.InstalledHandler('logger', level=logging.INFO)
def run_migration(self, *args): call = args[0] if len(call) > 2: pipeline_id = call[2] for plonesite in self.objectValues("Plone Site"): setSite(plonesite) plonesite = spoofRequest(plonesite) if pipeline_id in configuration_registry.listConfigurationIds(): transmogrifier = Transmogrifier(plonesite) overrides = {} if len(call) >= 4: json_path = call[3] overrides["jsonsource"] = {"path": json_path} transmogrifier(pipeline_id, **overrides) transaction.commit() print("All set!") else: print("This migration id is invalid.") else: print("You need to tell me the migration id to run")
def handle_action(self, action): """ Processes the form and runs the selected pipeline. """ data, errors = self.extractData() if errors: self.status = self.formErorrsMessage return pipeline_id = data.get('pipeline', None) pipelines = configuration_registry.listConfigurationIds() if pipeline_id is not None and pipeline_id in pipelines: transmogrifier = Transmogrifier(self.context) transmogrifier(pipeline_id) pipeline = configuration_registry.getConfiguration(pipeline_id) IStatusMessage(self.request).addStatusMessage( '{0} {1}'.format(pipeline['title'], _('successfully executed.')), 'info') self.request.response.redirect(self.context.absolute_url()) else: raise ActionExecutionError( Invalid('{0}: {1}'.format(_(u'Invalid pipeline'), pipeline_id)))
def handleRun(self, action): data, errors = self.extractData() if errors: return False overrides = {} for key, value in data.items(): if '-' not in key: continue elif value is None: continue elif isinstance(value, list): value = '\n'.join(value) elif isinstance(value, bool): value = value and 'True' or 'False' else: value = str(value) section, key = key.split('-') overrides.setdefault(section, {})[key] = value logger.info("Start importing profile: " + data['config']) Transmogrifier(self.context)(data['config'], **overrides) logger.info("Stop importing profile: " + data['config'])
def setUp(test): baseSetUp(test) from collective.transmogrifier.transmogrifier import Transmogrifier test.globs['transmogrifier'] = Transmogrifier(test.globs['plone']) import zope.component import collective.transmogrifier.sections zcml.load_config('meta.zcml', zope.component) zcml.load_config('configure.zcml', collective.transmogrifier.sections) test.globs['plone'].portal_transforms = MockPortalTransforms() provideUtility(PrettyPrinter, name=u'collective.transmogrifier.sections.tests.pprinter') #provideUtility(WebCrawler, #name=u'transmogrify.webcrawler.webcrawler') provideUtility(urlnormalizer) #provideUtility(Relinker, #name=u'transmogrify.webcrawler.relinker') #provideUtility(SafePortalTransforms, #name=u'transmogrify.webcrawler.safeportaltransforms') #from backlinkstitle import BacklinksTitle #provideUtility(BacklinksTitle, #name=u'transmogrify.webcrawler.backlinkstitle') from isindex import IsIndex provideUtility(IsIndex, name=u'transmogrify.webcrawler.isindex') #from makeattachments import MakeAttachments #provideUtility(MakeAttachments, #name=u'transmogrify.webcrawler.makeattachments') #from debugsection import DebugSection #provideUtility(DebugSection, #name=u'transmogrify.webcrawler.debugsection') provideUtility(HTMLSource, name=u'transmogrify.webcrawler.test.htmlsource') provideUtility(HTMLBacklinkSource, name=u'transmogrify.webcrawler.test.htmlbacklinksource')
def add(self, object): # FIXME: this is a workaround, which is fine for small uploaded files. # large uploads should go through another process anyway # TODO: re implementing this method is the only way to know # the full path of the object. We need the path to apply # the transmogrifier chain. #fti = getUtility(IDexterityFTI, name=self.portal_type) container = aq_inner(self.context) new_object = addContentToContainer(container, object) # set data genre: if self.datagenre: IBCCVLMetadata(new_object)['genre'] = self.datagenre # rdf commit should happens in transmogrifier step later on # if fti.immediate_view: # self.immediate_view = "%s/%s/%s" % (container.absolute_url(), new_object.id, fti.immediate_view,) # else: # self.immediate_view = "%s/%s" % (container.absolute_url(), new_object.id) # run transmogrify md extraction here # TODO: move this to an event listener? tm = Transmogrifier(new_object) tm('org.bccvl.site.add_file_metadata') # We have to reindex after updating the object new_object.reindexObject()
def import_bundle(self, bundle_name): bundle_path = resource_filename('opengever.bundle.tests', 'assets/%s' % bundle_name) transmogrifier = Transmogrifier(api.portal.get()) IAnnotations(transmogrifier)[BUNDLE_PATH_KEY] = bundle_path transmogrifier(u'opengever.bundle.oggbundle')
def install_templates(self): transmogrifier = Transmogrifier(self.setup.getSite()) transmogrifier(u'opengever.setup.content', jsonsource=dict(directory=self.path))
def configure_local_roles(self): transmogrifier = Transmogrifier(self.setup.getSite()) transmogrifier(u'opengever.setup.local_roles', jsonsource=dict(directory=self.path))
def _loadOptions(self, opts): from collective.transmogrifier.transmogrifier import Transmogrifier tm = Transmogrifier(object()) tm._raw = opts tm._data = {} return tm