def registerPersistentConfig(site, type_):
""" Try to get persistent pipeline configuration of given type (export or import)
and register it for use with transmogrifier.
"""
global CONFIGFILE
anno = IAnnotations(site)
key = '%s.%s' % (ANNOKEY, type_)
config = anno.has_key(key) and anno[key] or None
# unregister old config
name = 'persitent-%s' % type_
if name in configuration_registry._config_ids:
configuration_registry._config_ids.remove(name)
del configuration_registry._config_info[name]
# register new
if config is not None:
title = description = u'Persistent %s pipeline'
tf = tempfile.NamedTemporaryFile('w+t', suffix='.cfg')
tf.write(config)
tf.seek(0)
CONFIGFILE = tf
configuration_registry.registerConfiguration(name, title, description, tf.name)
return name
else:
return None
def runner(config, args={}):
from collective.transmogrifier.transmogrifier import Transmogrifier
# test.globs['transmogrifier'] = Transmogrifier(test.globs['plone'])
import zope.component
import zope.app.component
import collective.transmogrifier.sections
zcml.load_config('meta.zcml', zope.app.component)
zcml.load_config('meta.zcml', collective.transmogrifier)
zcml.load_config('configure.zcml', collective.transmogrifier.sections)
zcml.load_config('configure.zcml', transmogrify.htmltesting)
context = Context()
configuration_registry.registerConfiguration(
u'transmogrify.config.funnelweb', u"", u'', config)
transmogrifier = Transmogrifier(context)
overrides = {}
if type(args) == type(''):
for arg in args:
section, keyvalue = arg.split(':', 1)
key, value = keyvalue.split('=', 1)
overrides.setdefault('section', {})[key] = value
else:
overrides = args
transmogrifier(u'transmogrify.config.funnelweb', **overrides)
def runner(config, args={}):
from collective.transmogrifier.transmogrifier import Transmogrifier
# test.globs['transmogrifier'] = Transmogrifier(test.globs['plone'])
import zope.component
import zope.app.component
import collective.transmogrifier.sections
zcml.load_config('meta.zcml', zope.app.component)
zcml.load_config('meta.zcml', collective.transmogrifier)
zcml.load_config('configure.zcml', collective.transmogrifier.sections)
zcml.load_config('configure.zcml', transmogrify.htmltesting)
context = Context()
configuration_registry.registerConfiguration(
u'transmogrify.config.funnelweb',
u"",
u'', config)
transmogrifier = Transmogrifier(context)
overrides = {}
if type(args) == type(''):
for arg in args:
section,keyvalue = arg.split(':',1)
key,value = keyvalue.split('=',1)
overrides.setdefault('section',{})[key] = value
else:
overrides = args
transmogrifier(u'transmogrify.config.funnelweb', **overrides)
def registerConfig(name, configuration):
filename = os.path.join(BASEDIR, '%s.cfg' % name)
open(filename, 'w').write(configuration)
configuration_registry.registerConfiguration(
name,
u"Pipeline configuration '%s' from "
u"'collective.transmogrifier.tests'" % name,
u'', filename)
def load_pipeline(config, parser):
cparser = configparser.ConfigParser()
try:
config_info = configuration_registry.getConfiguration(config)
fp = open(config_info['configuration'])
pipelineid = config
except:
fp = open(config)
configuration_registry.registerConfiguration(
u'transmogrify.config.mr.migrator',
u"",
u'', config)
pipelineid = 'transmogrify.config.mr.migrator'
try:
# configparser
cparser.read_file(fp)
except:
# ConfigParser
cparser.read(config)
fp.close()
if cparser.has_option('transmogrifier', 'include'):
load_pipeline(cparser.get('transmogrifier', 'include'), parser)
if cparser.has_option('transmogrifier', 'pipeline'):
pipeline = [
p.strip() for p in cparser.get(
'transmogrifier',
'pipeline').split()]
else:
pipeline = []
for section in pipeline:
if section == 'transmogrifier':
continue
if cparser.has_option(section, '@doc'):
doc = cparser.get(section, '@doc')
else:
doc = ''
group = OptionGroup(parser, section, doc)
for key, value in cparser.items(section):
if key.startswith('@'):
if key == '@doc':
continue
metavar, _, help = value.partition(': ')
if metavar.upper() == metavar:
action = "store"
else:
action = "store_true"
help = value
arg = str("--%s:%s" % (section, key[1:]))
group.add_option(arg, action=action,
help=help,
metavar=metavar)
parser.add_option_group(group)
return pipelineid, cparser
def load_pipeline(config, parser):
cparser = configparser.ConfigParser()
try:
config_info = configuration_registry.getConfiguration(config)
fp = open(config_info['configuration'])
pipelineid = config
except:
fp = open(config)
configuration_registry.registerConfiguration(
u'transmogrify.config.mr.migrator', u"", u'', config)
pipelineid = 'transmogrify.config.mr.migrator'
try:
# configparser
cparser.read_file(fp)
except:
# ConfigParser
cparser.read(config)
fp.close()
if cparser.has_option('transmogrifier', 'include'):
load_pipeline(cparser.get('transmogrifier', 'include'), parser)
if cparser.has_option('transmogrifier', 'pipeline'):
pipeline = [
p.strip()
for p in cparser.get('transmogrifier', 'pipeline').split()
]
else:
pipeline = []
for section in pipeline:
if section == 'transmogrifier':
continue
if cparser.has_option(section, '@doc'):
doc = cparser.get(section, '@doc')
else:
doc = ''
group = OptionGroup(parser, section, doc)
for key, value in cparser.items(section):
if key.startswith('@'):
if key == '@doc':
continue
metavar, _, help = value.partition(': ')
if metavar.upper() == metavar:
action = "store"
else:
action = "store_true"
help = value
arg = str("--%s:%s" % (section, key[1:]))
group.add_option(arg,
action=action,
help=help,
metavar=metavar)
parser.add_option_group(group)
return pipelineid, cparser
def execute_pipeline(portal, filepath):
try:
configuration_registry.getConfiguration(PIPELINE_ID)
except KeyError:
configuration_registry.registerConfiguration(PIPELINE_ID, u'', u'',
filepath)
try:
transmogrifier = Transmogrifier(portal)
transmogrifier(PIPELINE_ID)
except Exception as error:
error_msg = u"type: '{}', msg: '{}'".format(type(error), error)
to_send = [u'Critical error during pipeline: {}'.format(error_msg)]
send_report(portal, to_send)
raise error
def registerDummyConfig(self, config, name):
global CONFIGFILE
# monkey in a config pipeline from a string
if name in configuration_registry._config_ids:
configuration_registry._config_ids.remove(name)
del configuration_registry._config_info[name]
# register new
if config is not None:
title = description = u'Persistent pipeline %s' % name
tf = tempfile.NamedTemporaryFile('w+t', suffix='.cfg')
tf.write(config)
tf.seek(0)
configuration_registry.registerConfiguration(name, title, description, tf.name)
CONFIGFILE = tf
return name
else:
return None
def runner(args={}):
for k,_,v in [a.partition('=') for a in sys.argv[1:]]:
k = k.lstrip('--')
if ':' in k:
part,_,key = k.partition(':')
args.setdefault(part, {})[key] = v
else:
args[k] = v
config = resource_filename(__name__,'pipeline.cfg')
if args.get('pipeline') == '':
f = open(config)
print f.read()
f.close()
return
else:
config = args.get('pipeline', config)
zcml.load_config('configure.zcml')
context = Context()
configuration_registry.registerConfiguration(
u'transmogrify.config.funnelweb',
u"",
u'', config)
transmogrifier = Transmogrifier(context)
overrides = {}
if type(args) == type(''):
for arg in args:
section,keyvalue = arg.split(':',1)
key,value = keyvalue.split('=',1)
overrides.setdefault('section',{})[key] = value
else:
overrides = args
transmogrifier(u'transmogrify.config.funnelweb', **overrides)
def registerConfigExtParam(self):
defconfig = self.getDefaultConfig()
config = ConfigParser.RawConfigParser(allow_no_value=True)
config.readfp(io.BytesIO(defconfig))
# unregister old config
if EXT_PARAM_CONFIG in configuration_registry._config_ids:
configuration_registry._config_ids.remove(EXT_PARAM_CONFIG)
del configuration_registry._config_info[EXT_PARAM_CONFIG]
if config is not None:
dest_catalog_path = self.request.get('dest_catalog_path')
dest_path_value = "python:item['_path'].replace('/temple',%s)"%dest_catalog_path
config.set('catalogsource', 'remote-url', self.request.get('remote-url'))
config.set('catalogsource', 'remote-username', self.request.get('remote-username'))
config.set('catalogsource', 'remote-password', self.request.get('remote-password'))
config.set('catalogsource', 'catalog-path', self.request.get('catalog-path'))
config.set('catalogsource', 'catalog-query', self.request.get('catalog-query'))
config.set('inserter', 'value', dest_path_value)
title = description = u'Persistent %s pipeline'
#tf = tempfile.NamedTemporaryFile('w+t', suffix='.cfg', dir='/tmp', delete=False)
file_path = '/tmp/%s'%EXT_PARAM_CONFIG_FILE
tf = open(file_path, 'w+t')
config.write(tf)
tf.seek(0)
configuration_registry.registerConfiguration(EXT_PARAM_CONFIG,
title, description,
tf.name)
return EXT_PARAM_CONFIG
else:
return None
def registerPersistentConfig(self):
global CONFIGFILE
site = self.context
anno = IAnnotations(site)
config = ANNOKEY in anno and anno[ANNOKEY] or None
# unregister old config
if PERSISTENT_CONFIG in configuration_registry._config_ids:
configuration_registry._config_ids.remove(PERSISTENT_CONFIG)
del configuration_registry._config_info[PERSISTENT_CONFIG]
# register new
if config is not None:
title = description = u'Persistent %s pipeline'
tf = tempfile.NamedTemporaryFile('w+t', suffix='.cfg', dir='/tmp')
tf.write(config)
tf.seek(0)
CONFIGFILE = tf
configuration_registry.registerConfiguration(PERSISTENT_CONFIG,
title, description,
tf.name)
return PERSISTENT_CONFIG
else:
return None
def runner(args={}, pipeline=None):
load_config('configure.zcml', mr.migrator)
parser = OptionParser()
parser.add_option("--pipeline", dest="pipeline",
help="Transmogrifier pipeline.cfg to use",
metavar="FILE"
)
parser.add_option("--show-pipeline", dest="showpipeline",
action = "store_true",
help="Show contents of the pipeline"
)
parser.add_option("--zcml", dest="zcml",
action = "store",
help="Load zcml"
)
# Parse just the pipeline args
ispipeline = lambda arg: [a for a in ['--pipeline','--show-pipeline'] if arg.startswith(a)]
pargs = [arg for arg in sys.argv[1:] if ispipeline(arg)]
(options, cargs) = parser.parse_args(pargs)
if options.pipeline is not None:
config = options.pipeline
elif pipeline is not None:
config = pipeline
else:
config = resource_filename(__name__,'pipeline.cfg')
cparser = configparser.ConfigParser()
context = Context()
try:
config_info = configuration_registry.getConfiguration(config)
fp = open(config_info['configuration'])
pipelineid = config
except:
fp = open(config)
configuration_registry.registerConfiguration(
u'transmogrify.config.mr.migrator',
u"",
u'', config)
pipelineid = 'transmogrify.config.mr.migrator'
try:
# configparser
cparser.read_file(fp)
except:
# ConfigParser
cparser.read(config)
fp.close()
pipeline = [p.strip() for p in cparser.get('transmogrifier','pipeline').split()]
for section in pipeline:
if section == 'transmogrifier':
continue
if cparser.has_option(section,'@doc'):
doc = cparser.get(section,'@doc')
else:
doc = ''
group = OptionGroup(parser, section, doc)
for key,value in cparser.items(section):
if key.startswith('@'):
if key == '@doc':
continue
metavar,_,help = value.partition(': ')
if metavar.upper() == metavar:
action = "store"
else:
action = "store_true"
help = value
group.add_option("--%s:%s"%(section,key[1:]), action=action,
help=help,
metavar=metavar)
parser.add_option_group(group)
pargs = [arg for arg in sys.argv[1:] if not arg.startswith('--template') ]
(options, cargs) = parser.parse_args(pargs)
cargs = {}
for k,_,v in [a.partition('=') for a in sys.argv[1:]]:
k = k.lstrip('--')
if ':' in k:
part,_,key = k.partition(':')
if key.lower()=='debug':
logger = logging.getLogger(part)
logger.setLevel(logging.DEBUG)
else:
section = cargs.setdefault(part, {})
if key in section:
section[key] = '%s\n%s' % (section[key],v)
else:
section[key] = v
else:
pass
#cargs[k] = v
for k,v in cargs.items():
args.setdefault(k, {}).update(v)
#config = resource_filename(__name__,'pipeline.cfg')
if options.showpipeline:
f = open(config)
print f.read()
f.close()
return
if options.zcml:
for zcml in options.zcml.split():
if not zcml.strip():
continue
load_config('configure.zcml', __import__(zcml))
transmogrifier = Transmogrifier(context)
overrides = {}
if type(args) == type(''):
for arg in args:
section,keyvalue = arg.split(':',1)
key,value = keyvalue.split('=',1)
overrides.setdefault('section',{})[key] = value
else:
overrides = args
transmogrifier(pipelineid, **overrides)
from collective.transmogrifier.transmogrifier import configuration_registry
if __name__ == '__main__':
"""
Example:
% ./bin/instance run import.py some_transmogrifier_configuration.cfg Plone
"""
parser = argparse.ArgumentParser()
parser.add_argument('cfg')
parser.add_argument('site')
args_ = parser.parse_args()
filepath = os.path.abspath(args_.cfg)
filename = os.path.basename(filepath)
configuration_registry.registerConfiguration(
filename, filename, filepath, filepath)
admin = UnrestrictedUser('admin', '', ['Manager'], '')
newSecurityManager(None, admin)
site = app[args_.site]
Transmogrifier._raw = [] # TODO: bad bad boy, put this upstream
Transmogrifier(site)(filename)
transaction.commit()
def runner(args={}, pipeline=None):
# Make sure GS ZCML is loaded before we load ours
# XXX This delays loading a bit too long. Getting:
# ConfigurationError: ('Unknown directive',
# u'http://namespaces.zope.org/genericsetup', u'importStep')
# again
#
# load_config('autoinclude.zcml', mr.migrator)
load_config("meta.zcml", Products.GenericSetup)
load_config("configure.zcml", Products.GenericSetup)
load_config('configure.zcml', mr.migrator)
parser = OptionParser()
parser.add_option("--pipeline", dest="pipeline",
help="Transmogrifier pipeline.cfg to use",
metavar="FILE")
parser.add_option("--show-pipeline", dest="showpipeline",
action="store_true",
help="Show contents of the pipeline")
parser.add_option("--zcml", dest="zcml",
action="store",
help="Load zcml")
# Parse just the pipeline args
ispipeline = lambda arg: [
a for a in ['--pipeline', '--show-pipeline'] if arg.startswith(a)]
pargs = [arg for arg in sys.argv[1:] if ispipeline(arg)]
(options, cargs) = parser.parse_args(pargs)
if options.pipeline is not None:
config = options.pipeline
elif pipeline is not None:
config = pipeline
else:
# XXX How about if we look for pipeline.cfg in the cwd?
# config = resource_filename(__name__, 'pipeline.cfg')
config = 'pipeline.cfg'
cparser = configparser.ConfigParser()
context = Context()
try:
config_info = configuration_registry.getConfiguration(config)
fp = open(config_info['configuration'])
pipelineid = config
except:
fp = open(config)
configuration_registry.registerConfiguration(
u'transmogrify.config.mr.migrator',
u"",
u'', config)
pipelineid = 'transmogrify.config.mr.migrator'
try:
# configparser
cparser.read_file(fp)
except:
# ConfigParser
cparser.read(config)
fp.close()
pipeline = [
p.strip() for p in cparser.get('transmogrifier', 'pipeline').split()]
for section in pipeline:
if section == 'transmogrifier':
continue
if cparser.has_option(section, '@doc'):
doc = cparser.get(section, '@doc')
else:
doc = ''
group = OptionGroup(parser, section, doc)
for key, value in cparser.items(section):
if key.startswith('@'):
if key == '@doc':
continue
metavar, _, help = value.partition(': ')
if metavar.upper() == metavar:
action = "store"
else:
action = "store_true"
help = value
group.add_option("--%s:%s" % (section, key[1:]), action=action,
help=help,
metavar=metavar)
parser.add_option_group(group)
pargs = [arg for arg in sys.argv[1:] if not arg.startswith('--template')]
(options, cargs) = parser.parse_args(pargs)
cargs = {}
for k, _, v in [a.partition('=') for a in sys.argv[1:]]:
k = k.lstrip('--')
if ':' in k:
part, _, key = k.partition(':')
if key.lower() == 'debug':
logger = logging.getLogger(part)
logger.setLevel(logging.DEBUG)
else:
section = cargs.setdefault(part, {})
if key in section:
section[key] = '%s\n%s' % (section[key], v)
else:
section[key] = v
else:
pass
#cargs[k] = v
for k, v in cargs.items():
args.setdefault(k, {}).update(v)
overrides = {}
if type(args) == type(''):
for arg in args:
section, keyvalue = arg.split(':', 1)
key, value = keyvalue.split('=', 1)
overrides.setdefault('section', {})[key] = value
else:
overrides = args
if options.showpipeline:
for section, values in overrides.items():
for key,value in values.items():
cparser.set(section, key, value)
cparser.write(sys.stdout)
return
# delay this so arg processing not so slow
load_config("meta.zcml", Products.GenericSetup)
load_config("configure.zcml", Products.GenericSetup)
load_config('configure.zcml', mr.migrator)
if options.zcml:
for zcml in options.zcml.split(','):
if not zcml.strip():
continue
load_config('configure.zcml', __import__(zcml, fromlist=zcml.split('.')))
transmogrifier = Transmogrifier(context)
transmogrifier(pipelineid, **overrides)
