def execute_pipeline(portal, filepath):
try:
configuration_registry.getConfiguration(PIPELINE_ID)
except KeyError:
configuration_registry.registerConfiguration(PIPELINE_ID, u'', u'',
filepath)
try:
transmogrifier = Transmogrifier(portal)
transmogrifier(PIPELINE_ID)
except Exception as error:
error_msg = u"type: '{}', msg: '{}'".format(type(error), error)
to_send = [u'Critical error during pipeline: {}'.format(error_msg)]
send_report(portal, to_send)
raise error
def __call__(self, context):
terms = []
for conf_id in configuration_registry.listConfigurationIds():
conf = configuration_registry.getConfiguration(conf_id)
terms.append(
SimpleVocabulary.createTerm(conf_id, conf_id, conf['title']))
return SimpleVocabulary(terms)
def getConfig(self, type_):
key = "%s.%s" % (ANNOKEY, type_)
if key in self.anno:
return self.anno[key]
else:
fname = configuration_registry.getConfiguration(type_)["configuration"]
return file(fname).read()
def write_config(self, config):
config['remote-skip-paths'] = ' '.join(config['remote-skip-paths'].split())
dashboard = configuration_registry.getConfiguration('genweb.migrations.dashboard')
filename = dashboard['configuration']
config_file = open(filename, 'w')
config_file.write(TEMPLATE % config)
config_file.close()
def getConfig(self, type_):
key = '%s.%s' % (ANNOKEY, type_)
if key in self.anno:
return self.anno[key]
else:
fname = configuration_registry.getConfiguration(type_)['configuration']
return file(fname).read()
def pipelines_list(context):
items = []
pids = configuration_registry.listConfigurationIds()
for pid in pids:
pipeline = configuration_registry.getConfiguration(pid)
items.append((pipeline['title'], pipeline['id']))
return SimpleVocabulary.fromItems(items)
def pipelines_list(context):
items = []
pids = configuration_registry.listConfigurationIds()
for pid in pids:
pipeline = configuration_registry.getConfiguration(pid)
items.append((pipeline['title'], pipeline['id']))
return SimpleVocabulary.fromItems(items)
def __call__(self, context):
terms = []
for conf_id in configuration_registry.listConfigurationIds():
conf = configuration_registry.getConfiguration(conf_id)
terms.append(SimpleVocabulary.createTerm(
conf_id, conf_id, conf['title'])
)
return SimpleVocabulary(terms)
def load_pipeline(config, parser):
cparser = configparser.ConfigParser()
try:
config_info = configuration_registry.getConfiguration(config)
fp = open(config_info['configuration'])
pipelineid = config
except:
fp = open(config)
configuration_registry.registerConfiguration(
u'transmogrify.config.mr.migrator',
u"",
u'', config)
pipelineid = 'transmogrify.config.mr.migrator'
try:
# configparser
cparser.read_file(fp)
except:
# ConfigParser
cparser.read(config)
fp.close()
if cparser.has_option('transmogrifier', 'include'):
load_pipeline(cparser.get('transmogrifier', 'include'), parser)
if cparser.has_option('transmogrifier', 'pipeline'):
pipeline = [
p.strip() for p in cparser.get(
'transmogrifier',
'pipeline').split()]
else:
pipeline = []
for section in pipeline:
if section == 'transmogrifier':
continue
if cparser.has_option(section, '@doc'):
doc = cparser.get(section, '@doc')
else:
doc = ''
group = OptionGroup(parser, section, doc)
for key, value in cparser.items(section):
if key.startswith('@'):
if key == '@doc':
continue
metavar, _, help = value.partition(': ')
if metavar.upper() == metavar:
action = "store"
else:
action = "store_true"
help = value
arg = str("--%s:%s" % (section, key[1:]))
group.add_option(arg, action=action,
help=help,
metavar=metavar)
parser.add_option_group(group)
return pipelineid, cparser
def load_pipeline(config, parser):
cparser = configparser.ConfigParser()
try:
config_info = configuration_registry.getConfiguration(config)
fp = open(config_info['configuration'])
pipelineid = config
except:
fp = open(config)
configuration_registry.registerConfiguration(
u'transmogrify.config.mr.migrator', u"", u'', config)
pipelineid = 'transmogrify.config.mr.migrator'
try:
# configparser
cparser.read_file(fp)
except:
# ConfigParser
cparser.read(config)
fp.close()
if cparser.has_option('transmogrifier', 'include'):
load_pipeline(cparser.get('transmogrifier', 'include'), parser)
if cparser.has_option('transmogrifier', 'pipeline'):
pipeline = [
p.strip()
for p in cparser.get('transmogrifier', 'pipeline').split()
]
else:
pipeline = []
for section in pipeline:
if section == 'transmogrifier':
continue
if cparser.has_option(section, '@doc'):
doc = cparser.get(section, '@doc')
else:
doc = ''
group = OptionGroup(parser, section, doc)
for key, value in cparser.items(section):
if key.startswith('@'):
if key == '@doc':
continue
metavar, _, help = value.partition(': ')
if metavar.upper() == metavar:
action = "store"
else:
action = "store_true"
help = value
arg = str("--%s:%s" % (section, key[1:]))
group.add_option(arg,
action=action,
help=help,
metavar=metavar)
parser.add_option_group(group)
return pipelineid, cparser
def __call__(self):
pipeline_id = 'import-chm-terms'
pipelines = configuration_registry.listConfigurationIds()
if pipeline_id is not None and pipeline_id in pipelines:
transmogrifier = Transmogrifier(self.context)
transmogrifier(pipeline_id)
pipeline = configuration_registry.getConfiguration(pipeline_id)
return 'done'
return 'invalid pipeline'
def __call__(self, context):
terms = []
for conf_id in configuration_registry.listConfigurationIds():
conf_file = _load_config(conf_id)
for section_id in conf_file.keys():
section = conf_file[section_id]
if section.get("blueprint", "") in SOURCE_SECTIONS:
conf = configuration_registry.getConfiguration(conf_id)
terms.append(SimpleVocabulary.createTerm(conf_id, conf_id, conf["title"]))
break
return SimpleVocabulary(terms)
def __call__(self, context):
terms = []
for conf_id in configuration_registry.listConfigurationIds():
conf_file = _load_config(conf_id)
for section_id in conf_file.keys():
section = conf_file[section_id]
if section.get('blueprint', '') in SOURCE_SECTIONS:
conf = configuration_registry.getConfiguration(conf_id)
terms.append(SimpleVocabulary.createTerm(
conf_id, conf_id, conf['title']))
break
return SimpleVocabulary(terms)
def __call__(self, context):
terms = []
for conf_id in configuration_registry.listConfigurationIds():
print conf_id
conf_file = _load_config(conf_id)
for section_id in conf_file.keys():
section = conf_file[section_id]
if section.get('blueprint', '') == 'plone.app.transmogrifier.atschemaupdater':
conf = configuration_registry.getConfiguration(conf_id)
terms.append(SimpleVocabulary.createTerm(
conf_id, conf_id, conf['title']))
break
return SimpleVocabulary(terms)
def testConfigZCMLDefaults(self):
xmlconfig.string('''\
<configure
xmlns:transmogrifier="http://namespaces.plone.org/transmogrifier"
i18n_domain="collective.transmogrifier">
<transmogrifier:registerConfig
name="collective.transmogrifier.tests.configname"
configuration="filename.cfg"
/>
</configure>''', context=self.context)
self.context.execute_actions()
self.assertEqual(configuration_registry.listConfigurationIds(),
(u'collective.transmogrifier.tests.configname',))
self.assertEqual(
configuration_registry.getConfiguration(
u'collective.transmogrifier.tests.configname'),
dict(id=u'collective.transmogrifier.tests.configname',
title=u'Pipeline configuration '
u"'collective.transmogrifier.tests.configname'",
description=u'',
configuration=os.path.join(os.getcwd(), 'filename.cfg')))
def handle_action(self, action):
""" Processes the form and runs the selected pipeline.
"""
data, errors = self.extractData()
if errors:
self.status = self.formErorrsMessage
return
pipeline_id = data.get('pipeline', None)
pipelines = configuration_registry.listConfigurationIds()
if pipeline_id is not None and pipeline_id in pipelines:
transmogrifier = Transmogrifier(self.context)
transmogrifier(pipeline_id)
pipeline = configuration_registry.getConfiguration(pipeline_id)
IStatusMessage(self.request).addStatusMessage('{0} {1}'
.format(pipeline['title'], _('successfully executed.')), 'info')
self.request.response.redirect(self.context.absolute_url())
else:
raise ActionExecutionError(Invalid('{0}: {1}'
.format(_(u'Invalid pipeline'), pipeline_id)))
def testConfigZCMLDefaults(self):
zcml.load_string('''\
<configure
xmlns:transmogrifier="http://namespaces.plone.org/transmogrifier"
i18n_domain="collective.transmogrifier">
<transmogrifier:registerConfig
name="collective.transmogrifier.tests.configname"
configuration="filename.cfg"
/>
</configure>''')
self.assertEqual(configuration_registry.listConfigurationIds(),
(u'collective.transmogrifier.tests.configname',))
path = os.path.split(collective.transmogrifier.__file__)[0]
self.assertEqual(
configuration_registry.getConfiguration(
u'collective.transmogrifier.tests.configname'),
dict(id=u'collective.transmogrifier.tests.configname',
title=u'Pipeline configuration '
u"'collective.transmogrifier.tests.configname'",
description=u'',
configuration=os.path.join(path, 'filename.cfg')))
def testConfigZCMLDefaults(self):
zcml.load_string('''\
<configure
xmlns:transmogrifier="http://namespaces.plone.org/transmogrifier"
i18n_domain="collective.transmogrifier">
<transmogrifier:registerConfig
name="collective.transmogrifier.tests.configname"
configuration="filename.cfg"
/>
</configure>''')
self.assertEqual(configuration_registry.listConfigurationIds(),
(u'collective.transmogrifier.tests.configname', ))
path = os.path.split(collective.transmogrifier.__file__)[0]
self.assertEqual(
configuration_registry.getConfiguration(
u'collective.transmogrifier.tests.configname'),
dict(id=u'collective.transmogrifier.tests.configname',
title=u'Pipeline configuration '
u"'collective.transmogrifier.tests.configname'",
description=u'',
configuration=os.path.join(path, 'filename.cfg')))
def handle_action(self, action):
""" Processes the form and runs the selected pipeline.
"""
data, errors = self.extractData()
if errors:
self.status = self.formErorrsMessage
return
pipeline_id = data.get('pipeline', None)
pipelines = configuration_registry.listConfigurationIds()
if pipeline_id is not None and pipeline_id in pipelines:
transmogrifier = Transmogrifier(self.context)
transmogrifier(pipeline_id)
pipeline = configuration_registry.getConfiguration(pipeline_id)
IStatusMessage(self.request).addStatusMessage(
'{0} {1}'.format(pipeline['title'],
_('successfully executed.')), 'info')
self.request.response.redirect(self.context.absolute_url())
else:
raise ActionExecutionError(
Invalid('{0}: {1}'.format(_(u'Invalid pipeline'),
pipeline_id)))
def runner(args={}, pipeline=None):
load_config('configure.zcml', mr.migrator)
parser = OptionParser()
parser.add_option("--pipeline", dest="pipeline",
help="Transmogrifier pipeline.cfg to use",
metavar="FILE"
)
parser.add_option("--show-pipeline", dest="showpipeline",
action = "store_true",
help="Show contents of the pipeline"
)
parser.add_option("--zcml", dest="zcml",
action = "store",
help="Load zcml"
)
# Parse just the pipeline args
ispipeline = lambda arg: [a for a in ['--pipeline','--show-pipeline'] if arg.startswith(a)]
pargs = [arg for arg in sys.argv[1:] if ispipeline(arg)]
(options, cargs) = parser.parse_args(pargs)
if options.pipeline is not None:
config = options.pipeline
elif pipeline is not None:
config = pipeline
else:
config = resource_filename(__name__,'pipeline.cfg')
cparser = configparser.ConfigParser()
context = Context()
try:
config_info = configuration_registry.getConfiguration(config)
fp = open(config_info['configuration'])
pipelineid = config
except:
fp = open(config)
configuration_registry.registerConfiguration(
u'transmogrify.config.mr.migrator',
u"",
u'', config)
pipelineid = 'transmogrify.config.mr.migrator'
try:
# configparser
cparser.read_file(fp)
except:
# ConfigParser
cparser.read(config)
fp.close()
pipeline = [p.strip() for p in cparser.get('transmogrifier','pipeline').split()]
for section in pipeline:
if section == 'transmogrifier':
continue
if cparser.has_option(section,'@doc'):
doc = cparser.get(section,'@doc')
else:
doc = ''
group = OptionGroup(parser, section, doc)
for key,value in cparser.items(section):
if key.startswith('@'):
if key == '@doc':
continue
metavar,_,help = value.partition(': ')
if metavar.upper() == metavar:
action = "store"
else:
action = "store_true"
help = value
group.add_option("--%s:%s"%(section,key[1:]), action=action,
help=help,
metavar=metavar)
parser.add_option_group(group)
pargs = [arg for arg in sys.argv[1:] if not arg.startswith('--template') ]
(options, cargs) = parser.parse_args(pargs)
cargs = {}
for k,_,v in [a.partition('=') for a in sys.argv[1:]]:
k = k.lstrip('--')
if ':' in k:
part,_,key = k.partition(':')
if key.lower()=='debug':
logger = logging.getLogger(part)
logger.setLevel(logging.DEBUG)
else:
section = cargs.setdefault(part, {})
if key in section:
section[key] = '%s\n%s' % (section[key],v)
else:
section[key] = v
else:
pass
#cargs[k] = v
for k,v in cargs.items():
args.setdefault(k, {}).update(v)
#config = resource_filename(__name__,'pipeline.cfg')
if options.showpipeline:
f = open(config)
print f.read()
f.close()
return
if options.zcml:
for zcml in options.zcml.split():
if not zcml.strip():
continue
load_config('configure.zcml', __import__(zcml))
transmogrifier = Transmogrifier(context)
overrides = {}
if type(args) == type(''):
for arg in args:
section,keyvalue = arg.split(':',1)
key,value = keyvalue.split('=',1)
overrides.setdefault('section',{})[key] = value
else:
overrides = args
transmogrifier(pipelineid, **overrides)
def runner(args={}, pipeline=None):
# Make sure GS ZCML is loaded before we load ours
# XXX This delays loading a bit too long. Getting:
# ConfigurationError: ('Unknown directive',
# u'http://namespaces.zope.org/genericsetup', u'importStep')
# again
#
# load_config('autoinclude.zcml', mr.migrator)
load_config("meta.zcml", Products.GenericSetup)
load_config("configure.zcml", Products.GenericSetup)
load_config('configure.zcml', mr.migrator)
parser = OptionParser()
parser.add_option("--pipeline", dest="pipeline",
help="Transmogrifier pipeline.cfg to use",
metavar="FILE")
parser.add_option("--show-pipeline", dest="showpipeline",
action="store_true",
help="Show contents of the pipeline")
parser.add_option("--zcml", dest="zcml",
action="store",
help="Load zcml")
# Parse just the pipeline args
ispipeline = lambda arg: [
a for a in ['--pipeline', '--show-pipeline'] if arg.startswith(a)]
pargs = [arg for arg in sys.argv[1:] if ispipeline(arg)]
(options, cargs) = parser.parse_args(pargs)
if options.pipeline is not None:
config = options.pipeline
elif pipeline is not None:
config = pipeline
else:
# XXX How about if we look for pipeline.cfg in the cwd?
# config = resource_filename(__name__, 'pipeline.cfg')
config = 'pipeline.cfg'
cparser = configparser.ConfigParser()
context = Context()
try:
config_info = configuration_registry.getConfiguration(config)
fp = open(config_info['configuration'])
pipelineid = config
except:
fp = open(config)
configuration_registry.registerConfiguration(
u'transmogrify.config.mr.migrator',
u"",
u'', config)
pipelineid = 'transmogrify.config.mr.migrator'
try:
# configparser
cparser.read_file(fp)
except:
# ConfigParser
cparser.read(config)
fp.close()
pipeline = [
p.strip() for p in cparser.get('transmogrifier', 'pipeline').split()]
for section in pipeline:
if section == 'transmogrifier':
continue
if cparser.has_option(section, '@doc'):
doc = cparser.get(section, '@doc')
else:
doc = ''
group = OptionGroup(parser, section, doc)
for key, value in cparser.items(section):
if key.startswith('@'):
if key == '@doc':
continue
metavar, _, help = value.partition(': ')
if metavar.upper() == metavar:
action = "store"
else:
action = "store_true"
help = value
group.add_option("--%s:%s" % (section, key[1:]), action=action,
help=help,
metavar=metavar)
parser.add_option_group(group)
pargs = [arg for arg in sys.argv[1:] if not arg.startswith('--template')]
(options, cargs) = parser.parse_args(pargs)
cargs = {}
for k, _, v in [a.partition('=') for a in sys.argv[1:]]:
k = k.lstrip('--')
if ':' in k:
part, _, key = k.partition(':')
if key.lower() == 'debug':
logger = logging.getLogger(part)
logger.setLevel(logging.DEBUG)
else:
section = cargs.setdefault(part, {})
if key in section:
section[key] = '%s\n%s' % (section[key], v)
else:
section[key] = v
else:
pass
#cargs[k] = v
for k, v in cargs.items():
args.setdefault(k, {}).update(v)
overrides = {}
if type(args) == type(''):
for arg in args:
section, keyvalue = arg.split(':', 1)
key, value = keyvalue.split('=', 1)
overrides.setdefault('section', {})[key] = value
else:
overrides = args
if options.showpipeline:
for section, values in overrides.items():
for key,value in values.items():
cparser.set(section, key, value)
cparser.write(sys.stdout)
return
# delay this so arg processing not so slow
load_config("meta.zcml", Products.GenericSetup)
load_config("configure.zcml", Products.GenericSetup)
load_config('configure.zcml', mr.migrator)
if options.zcml:
for zcml in options.zcml.split(','):
if not zcml.strip():
continue
load_config('configure.zcml', __import__(zcml, fromlist=zcml.split('.')))
transmogrifier = Transmogrifier(context)
transmogrifier(pipelineid, **overrides)
