def registerPersistentConfig(site, type_): """ Try to get persistent pipeline configuration of given type (export or import) and register it for use with transmogrifier. """ global CONFIGFILE anno = IAnnotations(site) key = '%s.%s' % (ANNOKEY, type_) config = anno.has_key(key) and anno[key] or None # unregister old config name = 'persitent-%s' % type_ if name in configuration_registry._config_ids: configuration_registry._config_ids.remove(name) del configuration_registry._config_info[name] # register new if config is not None: title = description = u'Persistent %s pipeline' tf = tempfile.NamedTemporaryFile('w+t', suffix='.cfg') tf.write(config) tf.seek(0) CONFIGFILE = tf configuration_registry.registerConfiguration(name, title, description, tf.name) return name else: return None
def runner(config, args={}): from collective.transmogrifier.transmogrifier import Transmogrifier # test.globs['transmogrifier'] = Transmogrifier(test.globs['plone']) import zope.component import zope.app.component import collective.transmogrifier.sections zcml.load_config('meta.zcml', zope.app.component) zcml.load_config('meta.zcml', collective.transmogrifier) zcml.load_config('configure.zcml', collective.transmogrifier.sections) zcml.load_config('configure.zcml', transmogrify.htmltesting) context = Context() configuration_registry.registerConfiguration( u'transmogrify.config.funnelweb', u"", u'', config) transmogrifier = Transmogrifier(context) overrides = {} if type(args) == type(''): for arg in args: section, keyvalue = arg.split(':', 1) key, value = keyvalue.split('=', 1) overrides.setdefault('section', {})[key] = value else: overrides = args transmogrifier(u'transmogrify.config.funnelweb', **overrides)
def runner(config, args={}): from collective.transmogrifier.transmogrifier import Transmogrifier # test.globs['transmogrifier'] = Transmogrifier(test.globs['plone']) import zope.component import zope.app.component import collective.transmogrifier.sections zcml.load_config('meta.zcml', zope.app.component) zcml.load_config('meta.zcml', collective.transmogrifier) zcml.load_config('configure.zcml', collective.transmogrifier.sections) zcml.load_config('configure.zcml', transmogrify.htmltesting) context = Context() configuration_registry.registerConfiguration( u'transmogrify.config.funnelweb', u"", u'', config) transmogrifier = Transmogrifier(context) overrides = {} if type(args) == type(''): for arg in args: section,keyvalue = arg.split(':',1) key,value = keyvalue.split('=',1) overrides.setdefault('section',{})[key] = value else: overrides = args transmogrifier(u'transmogrify.config.funnelweb', **overrides)
def registerConfig(name, configuration): filename = os.path.join(BASEDIR, '%s.cfg' % name) open(filename, 'w').write(configuration) configuration_registry.registerConfiguration( name, u"Pipeline configuration '%s' from " u"'collective.transmogrifier.tests'" % name, u'', filename)
def load_pipeline(config, parser): cparser = configparser.ConfigParser() try: config_info = configuration_registry.getConfiguration(config) fp = open(config_info['configuration']) pipelineid = config except: fp = open(config) configuration_registry.registerConfiguration( u'transmogrify.config.mr.migrator', u"", u'', config) pipelineid = 'transmogrify.config.mr.migrator' try: # configparser cparser.read_file(fp) except: # ConfigParser cparser.read(config) fp.close() if cparser.has_option('transmogrifier', 'include'): load_pipeline(cparser.get('transmogrifier', 'include'), parser) if cparser.has_option('transmogrifier', 'pipeline'): pipeline = [ p.strip() for p in cparser.get( 'transmogrifier', 'pipeline').split()] else: pipeline = [] for section in pipeline: if section == 'transmogrifier': continue if cparser.has_option(section, '@doc'): doc = cparser.get(section, '@doc') else: doc = '' group = OptionGroup(parser, section, doc) for key, value in cparser.items(section): if key.startswith('@'): if key == '@doc': continue metavar, _, help = value.partition(': ') if metavar.upper() == metavar: action = "store" else: action = "store_true" help = value arg = str("--%s:%s" % (section, key[1:])) group.add_option(arg, action=action, help=help, metavar=metavar) parser.add_option_group(group) return pipelineid, cparser
def load_pipeline(config, parser): cparser = configparser.ConfigParser() try: config_info = configuration_registry.getConfiguration(config) fp = open(config_info['configuration']) pipelineid = config except: fp = open(config) configuration_registry.registerConfiguration( u'transmogrify.config.mr.migrator', u"", u'', config) pipelineid = 'transmogrify.config.mr.migrator' try: # configparser cparser.read_file(fp) except: # ConfigParser cparser.read(config) fp.close() if cparser.has_option('transmogrifier', 'include'): load_pipeline(cparser.get('transmogrifier', 'include'), parser) if cparser.has_option('transmogrifier', 'pipeline'): pipeline = [ p.strip() for p in cparser.get('transmogrifier', 'pipeline').split() ] else: pipeline = [] for section in pipeline: if section == 'transmogrifier': continue if cparser.has_option(section, '@doc'): doc = cparser.get(section, '@doc') else: doc = '' group = OptionGroup(parser, section, doc) for key, value in cparser.items(section): if key.startswith('@'): if key == '@doc': continue metavar, _, help = value.partition(': ') if metavar.upper() == metavar: action = "store" else: action = "store_true" help = value arg = str("--%s:%s" % (section, key[1:])) group.add_option(arg, action=action, help=help, metavar=metavar) parser.add_option_group(group) return pipelineid, cparser
def execute_pipeline(portal, filepath): try: configuration_registry.getConfiguration(PIPELINE_ID) except KeyError: configuration_registry.registerConfiguration(PIPELINE_ID, u'', u'', filepath) try: transmogrifier = Transmogrifier(portal) transmogrifier(PIPELINE_ID) except Exception as error: error_msg = u"type: '{}', msg: '{}'".format(type(error), error) to_send = [u'Critical error during pipeline: {}'.format(error_msg)] send_report(portal, to_send) raise error
def registerDummyConfig(self, config, name): global CONFIGFILE # monkey in a config pipeline from a string if name in configuration_registry._config_ids: configuration_registry._config_ids.remove(name) del configuration_registry._config_info[name] # register new if config is not None: title = description = u'Persistent pipeline %s' % name tf = tempfile.NamedTemporaryFile('w+t', suffix='.cfg') tf.write(config) tf.seek(0) configuration_registry.registerConfiguration(name, title, description, tf.name) CONFIGFILE = tf return name else: return None
def runner(args={}): for k,_,v in [a.partition('=') for a in sys.argv[1:]]: k = k.lstrip('--') if ':' in k: part,_,key = k.partition(':') args.setdefault(part, {})[key] = v else: args[k] = v config = resource_filename(__name__,'pipeline.cfg') if args.get('pipeline') == '': f = open(config) print f.read() f.close() return else: config = args.get('pipeline', config) zcml.load_config('configure.zcml') context = Context() configuration_registry.registerConfiguration( u'transmogrify.config.funnelweb', u"", u'', config) transmogrifier = Transmogrifier(context) overrides = {} if type(args) == type(''): for arg in args: section,keyvalue = arg.split(':',1) key,value = keyvalue.split('=',1) overrides.setdefault('section',{})[key] = value else: overrides = args transmogrifier(u'transmogrify.config.funnelweb', **overrides)
def registerConfigExtParam(self): defconfig = self.getDefaultConfig() config = ConfigParser.RawConfigParser(allow_no_value=True) config.readfp(io.BytesIO(defconfig)) # unregister old config if EXT_PARAM_CONFIG in configuration_registry._config_ids: configuration_registry._config_ids.remove(EXT_PARAM_CONFIG) del configuration_registry._config_info[EXT_PARAM_CONFIG] if config is not None: dest_catalog_path = self.request.get('dest_catalog_path') dest_path_value = "python:item['_path'].replace('/temple',%s)"%dest_catalog_path config.set('catalogsource', 'remote-url', self.request.get('remote-url')) config.set('catalogsource', 'remote-username', self.request.get('remote-username')) config.set('catalogsource', 'remote-password', self.request.get('remote-password')) config.set('catalogsource', 'catalog-path', self.request.get('catalog-path')) config.set('catalogsource', 'catalog-query', self.request.get('catalog-query')) config.set('inserter', 'value', dest_path_value) title = description = u'Persistent %s pipeline' #tf = tempfile.NamedTemporaryFile('w+t', suffix='.cfg', dir='/tmp', delete=False) file_path = '/tmp/%s'%EXT_PARAM_CONFIG_FILE tf = open(file_path, 'w+t') config.write(tf) tf.seek(0) configuration_registry.registerConfiguration(EXT_PARAM_CONFIG, title, description, tf.name) return EXT_PARAM_CONFIG else: return None
def registerPersistentConfig(self): global CONFIGFILE site = self.context anno = IAnnotations(site) config = ANNOKEY in anno and anno[ANNOKEY] or None # unregister old config if PERSISTENT_CONFIG in configuration_registry._config_ids: configuration_registry._config_ids.remove(PERSISTENT_CONFIG) del configuration_registry._config_info[PERSISTENT_CONFIG] # register new if config is not None: title = description = u'Persistent %s pipeline' tf = tempfile.NamedTemporaryFile('w+t', suffix='.cfg', dir='/tmp') tf.write(config) tf.seek(0) CONFIGFILE = tf configuration_registry.registerConfiguration(PERSISTENT_CONFIG, title, description, tf.name) return PERSISTENT_CONFIG else: return None
def runner(args={}, pipeline=None): load_config('configure.zcml', mr.migrator) parser = OptionParser() parser.add_option("--pipeline", dest="pipeline", help="Transmogrifier pipeline.cfg to use", metavar="FILE" ) parser.add_option("--show-pipeline", dest="showpipeline", action = "store_true", help="Show contents of the pipeline" ) parser.add_option("--zcml", dest="zcml", action = "store", help="Load zcml" ) # Parse just the pipeline args ispipeline = lambda arg: [a for a in ['--pipeline','--show-pipeline'] if arg.startswith(a)] pargs = [arg for arg in sys.argv[1:] if ispipeline(arg)] (options, cargs) = parser.parse_args(pargs) if options.pipeline is not None: config = options.pipeline elif pipeline is not None: config = pipeline else: config = resource_filename(__name__,'pipeline.cfg') cparser = configparser.ConfigParser() context = Context() try: config_info = configuration_registry.getConfiguration(config) fp = open(config_info['configuration']) pipelineid = config except: fp = open(config) configuration_registry.registerConfiguration( u'transmogrify.config.mr.migrator', u"", u'', config) pipelineid = 'transmogrify.config.mr.migrator' try: # configparser cparser.read_file(fp) except: # ConfigParser cparser.read(config) fp.close() pipeline = [p.strip() for p in cparser.get('transmogrifier','pipeline').split()] for section in pipeline: if section == 'transmogrifier': continue if cparser.has_option(section,'@doc'): doc = cparser.get(section,'@doc') else: doc = '' group = OptionGroup(parser, section, doc) for key,value in cparser.items(section): if key.startswith('@'): if key == '@doc': continue metavar,_,help = value.partition(': ') if metavar.upper() == metavar: action = "store" else: action = "store_true" help = value group.add_option("--%s:%s"%(section,key[1:]), action=action, help=help, metavar=metavar) parser.add_option_group(group) pargs = [arg for arg in sys.argv[1:] if not arg.startswith('--template') ] (options, cargs) = parser.parse_args(pargs) cargs = {} for k,_,v in [a.partition('=') for a in sys.argv[1:]]: k = k.lstrip('--') if ':' in k: part,_,key = k.partition(':') if key.lower()=='debug': logger = logging.getLogger(part) logger.setLevel(logging.DEBUG) else: section = cargs.setdefault(part, {}) if key in section: section[key] = '%s\n%s' % (section[key],v) else: section[key] = v else: pass #cargs[k] = v for k,v in cargs.items(): args.setdefault(k, {}).update(v) #config = resource_filename(__name__,'pipeline.cfg') if options.showpipeline: f = open(config) print f.read() f.close() return if options.zcml: for zcml in options.zcml.split(): if not zcml.strip(): continue load_config('configure.zcml', __import__(zcml)) transmogrifier = Transmogrifier(context) overrides = {} if type(args) == type(''): for arg in args: section,keyvalue = arg.split(':',1) key,value = keyvalue.split('=',1) overrides.setdefault('section',{})[key] = value else: overrides = args transmogrifier(pipelineid, **overrides)
from collective.transmogrifier.transmogrifier import configuration_registry if __name__ == '__main__': """ Example: % ./bin/instance run import.py some_transmogrifier_configuration.cfg Plone """ parser = argparse.ArgumentParser() parser.add_argument('cfg') parser.add_argument('site') args_ = parser.parse_args() filepath = os.path.abspath(args_.cfg) filename = os.path.basename(filepath) configuration_registry.registerConfiguration( filename, filename, filepath, filepath) admin = UnrestrictedUser('admin', '', ['Manager'], '') newSecurityManager(None, admin) site = app[args_.site] Transmogrifier._raw = [] # TODO: bad bad boy, put this upstream Transmogrifier(site)(filename) transaction.commit()
def runner(args={}, pipeline=None): # Make sure GS ZCML is loaded before we load ours # XXX This delays loading a bit too long. Getting: # ConfigurationError: ('Unknown directive', # u'http://namespaces.zope.org/genericsetup', u'importStep') # again # # load_config('autoinclude.zcml', mr.migrator) load_config("meta.zcml", Products.GenericSetup) load_config("configure.zcml", Products.GenericSetup) load_config('configure.zcml', mr.migrator) parser = OptionParser() parser.add_option("--pipeline", dest="pipeline", help="Transmogrifier pipeline.cfg to use", metavar="FILE") parser.add_option("--show-pipeline", dest="showpipeline", action="store_true", help="Show contents of the pipeline") parser.add_option("--zcml", dest="zcml", action="store", help="Load zcml") # Parse just the pipeline args ispipeline = lambda arg: [ a for a in ['--pipeline', '--show-pipeline'] if arg.startswith(a)] pargs = [arg for arg in sys.argv[1:] if ispipeline(arg)] (options, cargs) = parser.parse_args(pargs) if options.pipeline is not None: config = options.pipeline elif pipeline is not None: config = pipeline else: # XXX How about if we look for pipeline.cfg in the cwd? # config = resource_filename(__name__, 'pipeline.cfg') config = 'pipeline.cfg' cparser = configparser.ConfigParser() context = Context() try: config_info = configuration_registry.getConfiguration(config) fp = open(config_info['configuration']) pipelineid = config except: fp = open(config) configuration_registry.registerConfiguration( u'transmogrify.config.mr.migrator', u"", u'', config) pipelineid = 'transmogrify.config.mr.migrator' try: # configparser cparser.read_file(fp) except: # ConfigParser cparser.read(config) fp.close() pipeline = [ p.strip() for p in cparser.get('transmogrifier', 'pipeline').split()] for section in pipeline: if section == 'transmogrifier': continue if cparser.has_option(section, '@doc'): doc = cparser.get(section, '@doc') else: doc = '' group = OptionGroup(parser, section, doc) for key, value in cparser.items(section): if key.startswith('@'): if key == '@doc': continue metavar, _, help = value.partition(': ') if metavar.upper() == metavar: action = "store" else: action = "store_true" help = value group.add_option("--%s:%s" % (section, key[1:]), action=action, help=help, metavar=metavar) parser.add_option_group(group) pargs = [arg for arg in sys.argv[1:] if not arg.startswith('--template')] (options, cargs) = parser.parse_args(pargs) cargs = {} for k, _, v in [a.partition('=') for a in sys.argv[1:]]: k = k.lstrip('--') if ':' in k: part, _, key = k.partition(':') if key.lower() == 'debug': logger = logging.getLogger(part) logger.setLevel(logging.DEBUG) else: section = cargs.setdefault(part, {}) if key in section: section[key] = '%s\n%s' % (section[key], v) else: section[key] = v else: pass #cargs[k] = v for k, v in cargs.items(): args.setdefault(k, {}).update(v) overrides = {} if type(args) == type(''): for arg in args: section, keyvalue = arg.split(':', 1) key, value = keyvalue.split('=', 1) overrides.setdefault('section', {})[key] = value else: overrides = args if options.showpipeline: for section, values in overrides.items(): for key,value in values.items(): cparser.set(section, key, value) cparser.write(sys.stdout) return # delay this so arg processing not so slow load_config("meta.zcml", Products.GenericSetup) load_config("configure.zcml", Products.GenericSetup) load_config('configure.zcml', mr.migrator) if options.zcml: for zcml in options.zcml.split(','): if not zcml.strip(): continue load_config('configure.zcml', __import__(zcml, fromlist=zcml.split('.'))) transmogrifier = Transmogrifier(context) transmogrifier(pipelineid, **overrides)