def getConfig(configFile = None, configDict = {}, section = None, additional = []):
if configDict and section:
configDict = {section: configDict}
config = createConfigFactory(configFile, configDict, additional = additional).getConfig()
if section:
return config.changeView(addSections = [section])
return config
def main():
configFactory = createConfigFactory(configFile = args[0], additional = [OptsConfigFiller(parser)])
config = configFactory.getConfig()
logging_setup(config.changeView(setSections = ['logging']))
# Check work dir validity (default work directory is the config file name)
if not os.path.exists(config.getWorkPath()):
if not config.getState('init'):
utils.vprint('Will force initialization of %s if continued!' % config.getWorkPath(), -1)
config.setState(True, 'init')
if config.getChoiceYesNo('workdir create', True,
interactive = 'Do you want to create the working directory %s?' % config.getWorkPath()):
utils.ensureDirExists(config.getWorkPath(), 'work directory')
# Create workflow and freeze config settings
globalConfig = config.changeView(setSections = ['global'])
workflow = globalConfig.getPlugin('workflow', 'Workflow:global', cls = Workflow).getInstance()
configFactory.freezeConfig(writeConfig = config.getState('init', detail = 'config'))
# Give config help
if opts.help_cfg or opts.help_scfg:
config.write(sys.stdout, printDefault = opts.help_cfg, printUnused = False,
printMinimal = opts.help_scfg, printSource = opts.help_cfg)
sys.exit(os.EX_OK)
# Check if user requested deletion / reset of jobs
if opts.delete:
workflow.jobManager.delete(workflow.wms, opts.delete)
sys.exit(os.EX_OK)
if opts.reset:
workflow.jobManager.reset(workflow.wms, opts.reset)
sys.exit(os.EX_OK)
# Run the configured workflow
workflow.run()
def __init__(self, config):
from grid_control.tasks import TaskModule
newVerbosity = utils.verbosity(utils.verbosity() - 3)
extConfigFN = config.getPath('source config')
extConfig = createConfigFactory(extConfigFN).getConfig(setSections = ['global'])
self.extWorkDir = extConfig.getWorkPath()
self.extTask = extConfig.getPlugin(['task', 'module'], cls = TaskModule).getInstance()
selector = config.get('source job selector', '')
extJobDB = JobDB(extConfig, jobSelector = lambda jobNum, jobObj: jobObj.state == Job.SUCCESS)
self.selected = sorted(extJobDB.getJobs(JobSelector.create(selector, task = self.extTask)))
utils.verbosity(newVerbosity + 3)
def getEntries(self, path, metadata, events, seList, objStore):
datacachePath = os.path.join(objStore.get('GC_WORKDIR', ''), 'datacache.dat')
source = utils.QM((self.source == '') and os.path.exists(datacachePath), datacachePath, self.source)
if source and (source not in self.lfnMap):
pSource = DataProvider.create(createConfigFactory().getConfig(), source, 'ListProvider')
for (n, fl) in map(lambda b: (b[DataProvider.Dataset], b[DataProvider.FileList]), pSource.getBlocks()):
self.lfnMap.setdefault(source, {}).update(dict(map(lambda fi: (self.lfnTrans(fi[DataProvider.URL]), n), fl)))
pList = set()
for key in filter(lambda k: k in metadata, self.parentKeys):
pList.update(map(lambda pPath: self.lfnMap.get(source, {}).get(self.lfnTrans(pPath)), metadata[key]))
metadata['PARENT_PATH'] = filter(lambda x: x, pList)
yield (path, metadata, events, seList, objStore)
def loadState(path, cfg = None):
src = DataSplitter._getIOHandler().loadState(path)
if cfg == None:
cfg = createConfigFactory(configDict = src.metadata).getConfig()
splitter = DataSplitter.getInstance(src.classname, cfg)
splitter.splitSource = src
# Transfer config protocol (in case no split function is called)
splitter._protocol = src.metadata['None']
for section in filter(lambda x: x, src.metadata):
meta2prot = lambda (k, v): ('[%s] %s' % (section.replace('None ', ''), k), v)
splitter._protocol.update(dict(map(meta2prot, src.metadata[section].items())))
return splitter
def loadState(path):
config = createConfigFactory(useDefaultFiles = False, configDict = {'dataset': {
'nickname check consistency': 'False', 'nickname check collision': 'False'}}).getConfig()
return DataProvider.getInstance('ListProvider', config, path, None, None)