def determine_globaltag(self, basedirs):
sys.argv = [os.path.basename(self.pset)] + self.arguments
with open(util.findpath(basedirs, self.pset), 'r') as f:
source = imp.load_source('cms_config_source', self.pset, f)
process = source.process
if hasattr(process, 'GlobalTag') and hasattr(process.GlobalTag.globaltag, 'value'):
self.globaltag = process.GlobalTag.globaltag.value()
def determine_globaltag(self, basedirs):
sys.argv = [os.path.basename(self.pset)] + self.arguments
with open(util.findpath(basedirs, self.pset), 'r') as f:
source = imp.load_source('cms_config_source', self.pset, f)
process = source.process
if hasattr(process, 'GlobalTag') and hasattr(process.GlobalTag.globaltag, 'value'):
self.globaltag = process.GlobalTag.globaltag.value()
def copy_inputs(self, basedirs, overwrite=False):
"""Make a copy of extra input files.
Includes CMSSW parameter set if specified. Already present files
will not be overwritten unless specified.
"""
if not os.path.exists(self.workdir):
os.makedirs(self.workdir)
if self.pset:
shutil.copy(util.findpath(basedirs, self.pset), os.path.join(self.workdir, os.path.basename(self.pset)))
if self.extra_inputs is None:
return []
def copy_file(fn):
source = os.path.abspath(util.findpath(basedirs, fn))
target = os.path.join(self.workdir, os.path.basename(fn))
if not os.path.exists(target) or overwrite:
if not os.path.exists(os.path.dirname(target)):
os.makedirs(os.path.dirname(target))
logger.debug("copying '{0}' to '{1}'".format(source, target))
if os.path.isfile(source):
shutil.copy(source, target)
elif os.path.isdir(source):
shutil.copytree(source, target)
else:
raise NotImplementedError
return target
files = map(copy_file, self.extra_inputs)
self.extra_inputs = files
def determine_outputs(self, basedirs):
"""Determine output files for CMSSW tasks.
"""
self.outputs = []
# To avoid problems loading configs that use the VarParsing module
sys.argv = [os.path.basename(self.pset)] + self.arguments
with open(util.findpath(basedirs, self.pset), 'r') as f:
source = imp.load_source('cms_config_source', self.pset, f)
process = source.process
for label, module in process.outputModules.items():
self.outputs.append(module.fileName.value().replace('file:', ''))
if 'TFileService' in process.services:
self.outputs.append(process.services['TFileService'].fileName.value().replace('file:', ''))
self.edm_output = False
logger.info("workflow {0}: adding output file(s) '{1}'".format(self.label, ', '.join(self.outputs)))
def copy_file(fn):
source = os.path.abspath(util.findpath(basedirs, fn))
target = os.path.join(self.workdir, os.path.basename(fn))
if not os.path.exists(target) or overwrite:
if not os.path.exists(os.path.dirname(target)):
os.makedirs(os.path.dirname(target))
logger.debug("copying '{0}' to '{1}'".format(source, target))
if os.path.isfile(source):
shutil.copy(source, target)
elif os.path.isdir(source):
shutil.copytree(source, target)
else:
raise NotImplementedError
return target
def get_info(self, cfg):
dataset = cfg['dataset']
if dataset not in self.__dsets:
instance = cfg.get('dbs instance', 'global')
mask = util.findpath(cfg['basedirs'], cfg['lumi mask']) if cfg.get('lumi mask') else None
file_based = cfg.get('file based', False)
res = self.query_database(dataset, instance, mask, file_based)
num = cfg.get('events per job')
if num:
res.jobsize = int(math.ceil(num / float(res.total_events) * res.total_lumis))
else:
res.jobsize = cfg.get('lumis per job', 25)
self.__dsets[dataset] = res
return self.__dsets[dataset]
def determine_outputs(self, basedirs):
"""Determine output files for CMSSW tasks.
"""
self.outputs = []
# To avoid problems loading configs that use the VarParsing module
sys.argv = [os.path.basename(self.pset)] + self.arguments
with open(util.findpath(basedirs, self.pset), 'r') as f:
source = imp.load_source('cms_config_source', self.pset, f)
process = source.process
for label, module in process.outputModules.items():
self.outputs.append(module.fileName.value().replace('file:', ''))
if 'TFileService' in process.services:
self.outputs.append(process.services['TFileService'].fileName.value().replace('file:', ''))
self.merge_command = 'hadd'
self.merge_args = ['@outputfiles', '@inputfiles']
logger.info("workflow {0}: adding output file(s) '{1}'".format(self.label, ', '.join(self.outputs)))
def copy_file(fn):
source = os.path.abspath(util.findpath(basedirs, fn))
target = os.path.join(self.workdir, os.path.basename(fn))
if not os.path.exists(target) or overwrite:
if not os.path.exists(os.path.dirname(target)):
os.makedirs(os.path.dirname(target))
logger.debug("copying '{0}' to '{1}'".format(source, target))
if os.path.isfile(source):
shutil.copy(source, target)
elif os.path.isdir(source):
shutil.copytree(source, target)
else:
logger.error("no such file or directory: {}".format(source))
sys.exit(1)
return target
def copy_file(fn):
source = os.path.abspath(util.findpath(basedirs, fn))
target = os.path.join(self.workdir, os.path.basename(fn))
if not os.path.exists(target) or overwrite:
if not os.path.exists(os.path.dirname(target)):
os.makedirs(os.path.dirname(target))
logger.debug("copying '{0}' to '{1}'".format(source, target))
if os.path.isfile(source):
shutil.copy(source, target)
elif os.path.isdir(source):
shutil.copytree(source, target)
else:
logger.error(
"no such file or directory: {}".format(source))
sys.exit(1)
return target
def copy_inputs(self, basedirs, overwrite=False):
"""Make a copy of extra input files.
Includes CMSSW parameter set if specified. Already present files
will not be overwritten unless specified.
"""
if not os.path.exists(self.workdir):
os.makedirs(self.workdir)
if self.pset:
shutil.copy(util.findpath(basedirs, self.pset), os.path.join(self.workdir, os.path.basename(self.pset)))
if self.extra_inputs is None:
return []
def copy_file(fn):
source = os.path.abspath(util.findpath(basedirs, fn))
target = os.path.join(self.workdir, os.path.basename(fn))
if not os.path.exists(target) or overwrite:
if not os.path.exists(os.path.dirname(target)):
os.makedirs(os.path.dirname(target))
logger.debug("copying '{0}' to '{1}'".format(source, target))
if os.path.isfile(source):
shutil.copy(source, target)
elif os.path.isdir(source):
shutil.copytree(source, target)
else:
logger.error("no such file or directory: {}".format(source))
sys.exit(1)
return target
files = map(copy_file, self.extra_inputs)
self.extra_inputs = files
def __init__(self, config):
super(JobProvider, self).__init__(config)
self.__chirp = self.config.get('stageout server', None)
self.__sandbox = os.path.join(self.workdir, 'sandbox')
self.__datasets = {}
self.__configs = {}
self.__jobhandlers = {}
self.__interface = MetaInterface()
self.__store = jobit.JobitStore(self.config)
self.__grid_files = [(os.path.join('/cvmfs/grid.cern.ch', x), os.path.join('grid', x)) for x in
['3.2.11-1/external/etc/profile.d/clean-grid-env-funcs.sh',
'3.2.11-1/external/etc/profile.d/grid-env-funcs.sh',
'3.2.11-1/external/etc/profile.d/grid-env.sh',
'3.2.11-1/etc/profile.d/grid-env.sh',
'3.2.11-1/glite/bin/voms-proxy-info',
'3.2.11-1/glite/lib64/libvomsapi_nog.so.0.0.0',
'3.2.11-1/glite/lib64/libvomsapi_nog.so.0',
'etc/grid-security/certificates'
]
]
if self.config.get('use dashboard', False):
logging.info("using dashboard with task id {0}".format(self.taskid))
self.__dash = dash.Monitor(self.taskid)
else:
self.__dash = dash.DummyMonitor(self.taskid)
if not util.checkpoint(self.workdir, 'sandbox'):
blacklist = self.config.get('sandbox blacklist', [])
sandbox.package(os.environ['LOCALRT'], self.__sandbox, blacklist, self.config.get('recycle sandbox'))
util.register_checkpoint(self.workdir, 'sandbox', 'CREATED')
self.__dash.register_run()
else:
for id in self.__store.reset_jobits():
self.__dash.update_job(id, dash.ABORTED)
for cfg in self.config['tasks']:
label = cfg['label']
cfg['basedirs'] = self.basedirs
cms_config = cfg.get('cmssw config')
if cms_config:
self.__configs[label] = os.path.basename(cms_config)
self.__datasets[label] = cfg.get('dataset', cfg.get('files', ''))
if cms_config and not cfg.has_key('outputs'):
sys.argv = [sys.argv[0]] #To avoid problems loading configs that use the VarParsing module
with open(cms_config, 'r') as f:
source = imp.load_source('cms_config_source', cms_config, f)
cfg_interface = CfgInterface(source.process)
if hasattr(cfg_interface.data.GlobalTag.globaltag, 'value'): #Possibility: make this mandatory?
cfg['global tag'] = cfg_interface.data.GlobalTag.globaltag.value()
for m in cfg_interface.data.outputModules:
self.outputs[label].append(getattr(cfg_interface.data, m).fileName._value)
taskdir = os.path.join(self.workdir, label)
if not util.checkpoint(self.workdir, label):
if cms_config:
shutil.copy(util.findpath(self.basedirs, cms_config), os.path.join(taskdir, os.path.basename(cms_config)))
logging.info("querying backend for {0}".format(label))
dataset_info = self.__interface.get_info(cfg)
logging.info("registering {0} in database".format(label))
self.__store.register(cfg, dataset_info)
util.register_checkpoint(self.workdir, label, 'REGISTERED')
elif os.path.exists(os.path.join(taskdir, 'running')):
for d in os.listdir(os.path.join(taskdir, 'running')):
shutil.move(os.path.join(taskdir, 'running', d), os.path.join(taskdir, 'failed'))