def submit(tar_ball):
name = tar_ball.rsplit('.')[0].replace('st_tch_top_', '').replace(
'_slc6_amd64_gcc700_CMSSW_9_3_0_tarball', '')
config = Configuration()
config.section_('General')
config.General.requestName = name
config.General.workArea = 'crab_projects'
config.General.transferOutputs = True
config.General.transferLogs = True
config.section_('JobType')
config.JobType.pluginName = 'Analysis'
config.JobType.psetName = 'TOP-RunIIFall18wmGS-00060_1_cfg.py'
config.section_('Data')
config.Data.inputDataset = "/ST_t-channel_4f_elmuDecays_13TeV-amcatnloFXFX-pythia8/dseith-GT_stEFT_dim2_930X11July-1109e3973be8fcd89b87655c5e215dad/USER"
config.Data.inputDBS = 'phys03'
config.Data.splitting = 'Automatic'
config.Data.outLFNDirBase = '/store/user/%s/' % (getUsername())
config.Data.publication = True
config.Data.outputDatasetTag = 'GT_stEFT_dim2_930X_GEN' + timestamp
config.JobType.maxMemoryMB = 5000
config.JobType.numCores = 4
config.section_('User')
config.User.voGroup = 'dcms'
config.section_('Site')
config.Site.storageSite = 'T1_DE_KIT_Disk'
crabCommand('submit', config=config)
def submit(tar_ball):
name = tar_ball.rsplit('.')[0].replace('st_tch_top_', '').replace(
'_slc6_amd64_gcc700_CMSSW_9_3_0_tarball', '')
config = Configuration()
config.section_('General')
config.General.requestName = name
config.General.workArea = 'crab_projects'
config.General.transferOutputs = True
config.General.transferLogs = True
config.section_('JobType')
config.JobType.pluginName = 'PrivateMC'
config.JobType.psetName = 'TOP-RunIIFall18wmLHEGS-00060_1_cfg.py'
config.JobType.inputFiles = [tar_ball]
config.section_('Data')
config.Data.outputPrimaryDataset = 'ST_t-channel_4f_elmuDecays_13TeV-amcatnloFXFX-pythia8'
config.Data.splitting = 'EventBased'
config.Data.unitsPerJob = 10000
NJOBS = 400 # This is not a configuration parameter, but an auxiliary variable that we use in the next line.
config.Data.totalUnits = config.Data.unitsPerJob * NJOBS
config.Data.outLFNDirBase = '/store/user/%s/' % (getUsername())
config.Data.publication = True
config.Data.outputDatasetTag = 'GT_stEFT_dim2_930X' + timestamp
config.section_('User')
config.User.voGroup = 'dcms'
config.section_('Site')
config.Site.storageSite = 'T1_DE_KIT_Disk'
crabCommand('submit', config=config)
def __init__(self, proxy=None, user=None):
if proxy is None:
proxy = os.getenv('X509_USER_PROXY')
if not proxy or not os.path.isfile(proxy):
raise CRABToolsException(
'X509_USER_PROXY is %r, get grid proxy first' % proxy)
self.proxy = proxy
if user is None:
user = getUsername()
if not user:
raise CRABToolsException(
'could not get username from sitedb, returned %r' % user)
self.user = user
def submit(config_name, tar_ball, name, dataset_name):
config = Configuration()
config.section_('General')
config.General.requestName = name
config.General.workArea = 'crab_projects'
config.General.transferOutputs = True
config.General.transferLogs = True
config.section_('JobType')
config.JobType.pluginName = 'PrivateMC'
config.JobType.psetName = config_name
config.JobType.inputFiles = [tar_ball]
config.section_('Data')
# config.Data.outputPrimaryDataset = 'ST_t-channel_4f_elmuDecays_13TeV-amcatnloFXFX-pythia8'
# config.Data.outputPrimaryDataset = 'TT_NoFullyHadronic_13TeV-amcatnloFXFX-pythia8'
config.Data.outputPrimaryDataset = dataset_name
config.Data.splitting = 'EventBased'
config.Data.unitsPerJob = 5000
NJOBS = 200 # This is not a configuration parameter, but an auxiliary variable that we use in the next line.
config.Data.totalUnits = config.Data.unitsPerJob * NJOBS
config.Data.outLFNDirBase = '/store/user/%s/' % (getUsername())
config.Data.publication = True
config.Data.outputDatasetTag = 'GT_stEFT_dim2_930X' + timestamp
config.JobType.maxMemoryMB = 9000
config.JobType.numCores = 8
config.section_('User')
config.User.voGroup = 'dcms'
config.section_('Site')
config.Site.storageSite = 'T1_DE_KIT_Disk'
crabCommand('submit', config=config)
config.General.transferOutputs = True
config.General.transferLogs = False
config.Data.publication = False
config.Site.storageSite = args.site
if len(args.vomsGroup) != 0:
config.User.voGroup = args.vomsGroup
if len(args.vomsRole) != 0:
config.User.voRole = args.vomsRole
if args.output[0] == '/':
config.Data.outLFNDirBase = args.output
else:
config.Data.outLFNDirBase = "/store/user/{}/{}".format(
getUsername(), args.output)
if len(args.blacklist) != 0:
config.Site.blacklist = re.split(',', args.blacklist)
if len(args.whitelist) != 0:
config.Site.whitelist = re.split(',', args.whitelist)
job_names = set(filter(lambda s: len(s) != 0, re.split(",", args.jobNames)))
from TauMLTools.Production.crab_tools import JobCollection
try:
job_collection = JobCollection(args.jobFile, job_names, args.lumiMask,
args.jobNameSuffix)
print(args.jobFile)
print(job_collection)
print("Splitting: {} with {} units per job".format(args.splitting,
from CRABClient.UserUtilities import config
from CRABClient.UserUtilities import getUsername
username = getUsername()
config = config()
config.section_('General')
config.General.workArea = 'crab_SPSTo4mu_psi2SJ-RECO'
config.General.requestName = 'crab_SPSTo4mu_psi2SJ-RECO'
config.General.transferOutputs = True
config.General.transferLogs = True
config.JobType.pluginName = 'Analysis'
config.JobType.psetName = 'BPH-RunIIAutumn18DRPremix-01798_DPS_cfg_2.py'
config.JobType.maxMemoryMB = 5000
config.JobType.allowUndistributedCMSSW = True
config.Data.inputDBS = 'phys03'
config.Data.inputDataset = '/SPSTo4mu_psi2SJ/xgao-RunIIAutumn18MiniAOD-102X_upgrade2018_realistic_v15-v1-DIGI_psi2SJ-98b2ebbf2a73ad4baba14fc5516a6604/USER'
config.Data.splitting = 'FileBased'
config.Data.unitsPerJob = 1
config.Data.outputDatasetTag = 'RunIIAutumn18MiniAOD-102X_upgrade2018_realistic_v15-v1-RECO'
config.Data.publication = True
#config.Data.outLFNDirBase = '/store/group/l1upgrades/%s/MC2020/Sample_Production/bbarTo4mu/bbarTo4mu-RECO/'%username
config.Data.outLFNDirBase = '/store/user/xgao/samples-20200813-DiJpsi/psi2SJ-RECO'
config.section_('User')
config.section_('Site')
#config.Site.storageSite = 'T3_US_FNALLPC'
#config.Site.whitelist = ['T2_US_*']
config.Site.storageSite = 'T2_BE_IIHE'
config.User.voGroup = 'becms'
def __call__(self):
username = getUsername(self.proxyfilename, logger=self.logger)
self.logger.info("Username is: %s", username)
return username
},
"run327327": {
"mTrigger": "ZeroBias",
"mRunRange": "327327",
"mDataset": "/HIForward/HIRun2018A-04Apr2019-v1/AOD"
},
"run327488": {
"mTrigger": "ZeroBias",
"mRunRange": "327488",
"mDataset": "/HIForward/HIRun2018A-04Apr2019-v1/AOD"
},
"run327554": {
"mTrigger": "ZeroBias",
"mRunRange": "327554",
"mDataset": "/HIForward/HIRun2018A-04Apr2019-v1/AOD"
},
# "run326942": {"mTrigger": "MinimumBias", "mRunRange": "326942", "mDataset": "/HIMinimumBias2/HIRun2018A-04Apr2019-v1/AOD"},
}
## Submit the muon PDs
for key, val in dataMap.items():
config.General.requestName = 'EvtInfoTree_' + val['mTrigger'] + '_' + key
config.Data.inputDataset = val['mDataset']
config.Data.outputDatasetTag = config.General.requestName
config.Data.runRange = val['mRunRange']
config.Data.outLFNDirBase = '/store/group/phys_heavyions/%s/L1Emulator/EvtInfoTree/%s' % (
getUsername(), config.General.requestName)
print("Submitting CRAB job for: " + key)
submit(config)
def createNewMyProxy2(self, timeleftthreshold=0, nokey=False):
"""
Handles the MyProxy creation. In this version the credential name will be simply
<username>_CRAB like e.g. belforte_CRAB where username is the CERN username
Let the following variables be
timeleftthreshold: the proxy in myproxy should be delegated for at least this time (14 days)
myproxytimeleft: current validity of your proxy in myproxy
usercertDaysLeft: the number of days left before your user certificate expire
myproxyDesiredValidity: delegate the proxy in myproxy for that time (30 days)
If we need to renew the proxy in myproxy because its atributes has changed or because it is valid for
less time than timeleftthreshold then we do it.
Before doing that, we check when the user certificate is expiring. If it's within the timeleftthreshold (myproxytimeleft < timeleftthreshold)
we delegate the proxy just for the time we need (checking first if we did not already do it since at some point
usercertDaysLeft ~= myproxytimeleft and we don't need to delegate it at every command even though myproxytimeleft < timeleftthreshold).
Note that a warning message is printed at every command it usercertDaysLeft < timeleftthreshold
:returns a tupla with info in the credential in myprosxy: (credentialName, myproxytimeleft)
credentialName : username to use in myproxy -l username
myproxytimeleft: validity of the credential in seconds
"""
defaultDelegation = self.defaultDelegation
defaultDelegation['myproxyAccount'] = None
from CRABClient.UserUtilities import getUsername
username = getUsername(proxyFile=self.proxyInfo['filename'],
logger=self.logger)
credentialName = username + '_CRAB'
defaultDelegation['userName'] = credentialName
myproxy = Proxy(defaultDelegation)
#userDNFromCert = myproxy.getSubjectFromCert(self.certLocation)
#if userDNFromCert:
# myproxy.userDN = userDNFromCert
myproxytimeleft = 0
self.logger.debug("Getting myproxy life time left for %s" %
self.defaultDelegation["myProxySvr"])
# return an integer that indicates the number of seconds to the expiration of the proxy in myproxy
# Also catch the exception in case WMCore encounters a problem with the proxy itself (one such case was #4532)
try:
myproxytimeleft = myproxy.getMyProxyTimeLeft(serverRenewer=True,
nokey=nokey)
except CredentialException as ex:
msg = "WMCore could not computer valid time for credential %s .\n Error detail: " % credentialName
msg += "%s" % str(ex._message)
msg += "\nTry to remove old myproxy credentials as per https://twiki.cern.ch/twiki/bin/view/CMSPublic/CRAB3FAQ#crab_command_fails_with_Impossib"
self.logger.error(msg)
raise ProxyCreationException("no valid credential for %s" %
credentialName)
except Exception as ex:
logging.exception(
"Problems calculating proxy lifetime, logging stack trace and raising ProxyCreationException"
)
# WMException may contain the _message attribute. Otherwise, take the exception as a string.
msg = ex._message if hasattr(ex, "_message") else str(ex) # pylint: disable=protected-access, no-member
raise ProxyCreationException(
"Problems calculating the time left until the expiration of the proxy."
+
" Please reset your environment or contact [email protected] if the problem persists.\n%s"
% msg)
self.logger.debug("Myproxy is valid: %i", myproxytimeleft)
trustRetrListChanged = myproxy.trustedRetrievers != self.defaultDelegation[
'serverDN'] #list on the REST and on myproxy are different
if myproxytimeleft < timeleftthreshold or self.proxyChanged or trustRetrListChanged:
# checking the enddate of the user certificate
usercertDaysLeft = myproxy.getUserCertEnddate()
if usercertDaysLeft == 0:
msg = "%sYOUR USER CERTIFICATE IS EXPIRED (OR WILL EXPIRE TODAY)." % colors.RED
msg += " YOU CANNOT USE THE CRAB3 CLIENT."
msg += " PLEASE REQUEST A NEW CERTIFICATE HERE https://gridca.cern.ch/gridca/"
msg += " AND SEE https://ca.cern.ch/ca/Help/?kbid=024010%s" % colors.NORMAL
raise ProxyCreationException(msg)
#if the certificate is going to expire print a warning. This is going to bre printed at every command if
#the myproxytimeleft is inferior to the timeleftthreshold
if usercertDaysLeft < self.myproxyDesiredValidity:
msg = "%sYour user certificate is going to expire in %s days." % (
colors.RED, usercertDaysLeft)
msg += " See: https://twiki.cern.ch/twiki/bin/view/CMSPublic/WorkBookStartingGrid#ObtainingCert %s" % colors.NORMAL
self.logger.info(msg)
#check if usercertDaysLeft ~= myproxytimeleft which means we already delegated the proxy for as long as we could
if abs(
usercertDaysLeft * 60 * 60 * 24 - myproxytimeleft
) < 60 * 60 * 24 and not trustRetrListChanged: #less than one day between usercertDaysLeft and myproxytimeleft
return (credentialName, myproxytimeleft)
#adjust the myproxy delegation time accordingly to the user cert validity
self.logger.info(
"%sDelegating your proxy for %s days instead of %s %s",
colors.RED, usercertDaysLeft, self.myproxyDesiredValidity,
colors.NORMAL)
myproxy.myproxyValidity = "%i:00" % (usercertDaysLeft * 24)
# creating the proxy
self.logger.debug("Delegating a myproxy for %s hours",
myproxy.myproxyValidity)
try:
myproxy.delegate(serverRenewer=True, nokey=nokey)
myproxytimeleft = myproxy.getMyProxyTimeLeft(
serverRenewer=True, nokey=nokey)
if myproxytimeleft <= 0:
raise ProxyCreationException("It seems your proxy has not been delegated to myproxy. Please check the logfile for the exact error "+\
"(it might simply you typed a wrong password)")
else:
self.logger.debug("My-proxy delegated.")
except Exception as ex:
msg = ex._message if hasattr(ex, '_message') else str(ex) # pylint: disable=protected-access, no-member
raise ProxyCreationException(
"Problems delegating My-proxy. %s" % msg)
return (credentialName, myproxytimeleft)
def __call__(self):
username = getUsername(self.proxyfilename, logger=self.logger)
if hasattr(self.options, 'userlfn') and self.options.userlfn != None:
self.lfnPrefix = self.options.userlfn
else:
## If the user didn't provide an LFN path where to check the write permission,
## assume he/she wants to check in /store/user/<username>
self.logger.info(
'Will check write permission in the default location /store/user/<username>'
)
self.lfnPrefix = '/store/user/' + username
## Check that the location where we want to check write permission
## is one where the user will be allowed to stageout.
self.logger.info("Validating LFN %s...", self.lfnPrefix)
# if an error message is needed later, prepare it now to keep code below tidy
errMsg = "Refusing to check write permission in %s, because this is not an allowed LFN for stageout." % (
self.lfnPrefix)
errMsg += "\nThe LFN must start with either"
errMsg += " '/store/user/<username>/' or '/store/group/<groupname>/'"
errMsg += " (or '/store/local/<something>/' if publication is off),"
errMsg += " where username is your username as registered in CMS"
errMsg += " (i.e. the username of your CERN primary account)."
errMsg += "\nLFN %s is not valid." % (self.lfnPrefix)
if not checkOutLFN(self.lfnPrefix, username):
self.logger.info(errMsg)
return {'status': 'FAILED'}
else:
self.logger.info("LFN %s is valid.", self.lfnPrefix)
cp_cmd = ""
if cmd_exist("gfal-copy") and cmd_exist(
"gfal-rm") and self.command in [None, "GFAL"]:
self.logger.info(
"Will use `gfal-copy`, `gfal-rm` commands for checking write permissions"
)
cp_cmd = "gfal-copy -p -v -t 180 "
delfile_cmd = "gfal-rm -v -t 180 "
deldir_cmd = "gfal-rm -r -v -t 180 "
if self.checksum:
cp_cmd += "-K %s " % self.checksum
elif cmd_exist("lcg-cp") and cmd_exist("lcg-del"):
self.logger.info(
"Will use `lcg-cp`, `lcg-del` commands for checking write permissions"
)
cp_cmd = "lcg-cp -v -b -D srmv2 --connect-timeout 180 "
delfile_cmd = "lcg-del --connect-timeout 180 -b -l -D srmv2 "
deldir_cmd = "lcg-del -d --connect-timeout 180 -b -l -D srmv2 "
if self.checksum:
cp_cmd += "--checksum-type %s " % self.checksum
else:
self.logger.info("Neither gfal nor lcg command was found")
return {'status': 'FAILED'}
self.logger.info('Will check write permission in %s on site %s',
self.lfnPrefix, self.options.sitename)
timestamp = str(time.strftime("%Y%m%d_%H%M%S"))
self.filename = 'crab3checkwrite_' + timestamp + '.tmp'
self.subdir = 'crab3checkwrite_' + timestamp
lfn = self.lfnPrefix + '/' + self.subdir + '/' + self.filename
site = self.options.sitename
try:
pfn = self.getPFN(site=site, lfn=lfn, username=username)
except Exception:
return {'status': 'FAILED'}
self.createFile()
self.logger.info("Will use PFN: %s", pfn)
dirpfn = pfn[:len(pfn) - len(self.filename)]
self.logger.info(
'\nAttempting to create (dummy) directory %s and copy (dummy) file %s to %s\n'
% (self.subdir, self.filename, self.lfnPrefix))
cpout, cperr, cpexitcode = self.cp(pfn, cp_cmd)
if cpexitcode == 0:
self.logger.info(
'\nSuccessfully created directory %s and copied file %s to %s'
% (self.subdir, self.filename, self.lfnPrefix))
self.logger.info('\nAttempting to delete file %s\n' % (pfn))
delexitcode = self.delete(pfn, delfile_cmd)
if delexitcode:
self.logger.info('\nFailed to delete file %s' % (pfn))
finalmsg = '%sError%s: CRAB3 is able to copy but unable to delete file in %s on site %s. Asynchronous Stage Out with CRAB3 will fail.' % (
colors.RED, colors.NORMAL, self.lfnPrefix,
self.options.sitename)
finalmsg += '\n You may want to contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
returndict = {'status': 'FAILED'}
else:
self.logger.info('\nSuccessfully deleted file %s' % (pfn))
self.logger.info('\nAttempting to delete directory %s\n' %
(dirpfn))
delexitcode = self.delete(dirpfn, deldir_cmd)
if delexitcode:
self.logger.info('\nFailed to delete directory %s' %
(dirpfn))
finalmsg = '%sError%s: CRAB3 is able to copy but unable to delete directory in %s on site %s. Asynchronous Stage Out with CRAB3 will fail.' % (
colors.RED, colors.NORMAL, self.lfnPrefix,
self.options.sitename)
finalmsg += '\n You may want to contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
returndict = {'status': 'FAILED'}
else:
self.logger.info('\nSuccessfully deleted directory %s' %
(dirpfn))
finalmsg = '%sSuccess%s: Able to write in %s on site %s' % (
colors.GREEN, colors.NORMAL, self.lfnPrefix,
self.options.sitename)
returndict = {'status': 'SUCCESS'}
else:
if 'Permission denied' in cperr or 'mkdir: cannot create directory' in cperr:
finalmsg = '%sError%s: Unable to write in %s on site %s' % (
colors.RED, colors.NORMAL, self.lfnPrefix,
self.options.sitename)
finalmsg += '\n You may want to contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
returndict = {'status': 'FAILED'}
elif 'timeout' in cpout or 'timeout' in cperr:
self.logger.info('Connection time out.')
finalmsg = '\nUnable to check write permission in %s on site %s' % (
self.lfnPrefix, self.options.sitename)
finalmsg += '\nPlease try again later or contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
returndict = {'status': 'FAILED'}
else:
finalmsg = 'Unable to check write permission in %s on site %s' % (
self.lfnPrefix, self.options.sitename)
finalmsg += '\nPlease try again later or contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
returndict = {'status': 'FAILED'}
self.removeFile()
self.logger.info('\nCheckwrite Result:')
self.logger.info(finalmsg)
if returndict['status'] == 'FAILED':
self.logger.info(
'%sNote%s: You cannot write to a site if you did not ask permission.'
% (colors.BOLD, colors.NORMAL))
if 'CH_CERN' in self.options.sitename:
dbgmsg = '%sAdditional diagnostic info for CERN EOS%s\n' % (
colors.RED, colors.NORMAL)
dbgcmd = "echo '== id ==>:';id"
dbgcmd += ";echo '== voms-proxy-info -all ==>:';voms-proxy-info -all"
dbgcmd += ";which uberftp > /dev/null 2>&1 && echo '== uberftp eoscmsftp.cern.ch pwd ==>:'"
dbgcmd += ";which uberftp > /dev/null 2>&1 && uberftp eoscmsftp.cern.ch pwd"
dbgcmd += ";which uberftp > /dev/null 2>&1 || echo 'WARNING uberftp command not found. To get additional diagnostic info'"
dbgcmd += ";which uberftp > /dev/null 2>&1 || echo ' log on lxplus, get a proxy and execute:'"
dbgcmd += ";which uberftp > /dev/null 2>&1 || echo ' uberftp eoscmsftp.cern.ch pwd'"
#self.logger.info('Executing command: %s' % cmd)
#self.logger.info('Please wait...')
output, _, _ = execute_command(command=dbgcmd)
dbgmsg += output
self.logger.info(dbgmsg)
return returndict
def main():
# Read options
options = getOptions()
doData = options.sampleType in ['all', 'data']
doMC = options.sampleType in ['all', 'mc']
eraDB = options.eraDB
subEra = options.subEra
customSuffix = options.customSuffix
numThreads = options.numThreads
storageSite = options.storageSite
crabCmd = options.crabCmd
crabCmdOpts = options.crabCmdOpts
workArea = options.workArea
configFile = options.configFile
# The submit command needs special treatment.
if crabCmd == 'submit':
#--------------------------------------------------------
# This is the base config:
#--------------------------------------------------------
from CRABClient.UserUtilities import config, getUsername
config = config()
config.General.workArea = workArea
config.General.transferOutputs = True
config.General.transferLogs = True
config.JobType.pluginName = 'Analysis'
config.JobType.psetName = configFile
config.JobType.numCores = numThreads
#config.JobType.maxMemoryMB = 4000
config.Data.splitting = 'Automatic'
config.Data.publication = False
#config.Data.ignoreLocality = True
if storageSite == 'FNAL':
# Requires write access to FNAL EOS space
config.Site.storageSite = 'T3_US_FNALLPC'
config.Data.outLFNDirBase = '/store/user/%s/TnP_ntuples/%s/' % (
getUsername(), eraDB.split('/')[-1].split('.')[0])
elif storageSite == 'CERN':
# Requires write access to Muon POG EOS space at CERN
config.Site.storageSite = 'T2_CH_CERN'
config.Data.outLFNDirBase = '/store/group/phys_muon/%s/TnP_ntuples/%s' % (
getUsername(), eraDB.split('/')[-1].split('.')[0])
elif storageSite == 'CERNBOX':
# CERNBOX write access from CRAB requires special permission from CERN IT
# See https://twiki.cern.ch/twiki/bin/view/CMSPublic/CRAB3FAQ#Can_I_send_CRAB_output_to_CERNBO
# and to ask permission: https://cern.service-now.com/service-portal?id=sc_cat_item&name=request-map-dn-to-gridmap&se=CERNBox-Service
config.Site.storageSite = 'T2_CH_CERNBOX'
config.Data.outLFNDirBase = '/store/user/%s/TnP_ntuples/%s/' % (
getUsername(), eraDB.split('/')[-1].split('.')[0])
#config.Site.ignoreGlobalBlacklist = True
#--------------------------------------------------------
with open(eraDB, 'r') as db_file:
db = json.load(db_file)
era = db['era']
resonance = db['resonance']
datasets = db['datasets']
samples = {}
try:
if subEra == 'all':
samples = datasets
else:
samples = dict({subEra: datasets[subEra]})
except:
print "Error!! Requested era+sub-era is likely not valid. Please check argument."
sys.exit()
for curr_subEra, configs in samples.items():
isData = 'Run' in curr_subEra
globalTag = configs['globalTag'] if 'globalTag' in configs else ''
input_dataset = configs['dataset']
if isData and not doData: continue
if not isData and not doMC: continue
if isData:
#config.Data.splitting = 'LumiBased'
#config.Data.unitsPerJob = 100
if '2018' in era:
config.Data.lumiMask = 'https://cms-service-dqm.web.cern.ch/cms-service-dqm/CAF/certification/Collisions18/13TeV/ReReco/Cert_314472-325175_13TeV_17SeptEarlyReReco2018ABC_PromptEraD_Collisions18_JSON.txt'
elif '2017' in era:
config.Data.lumiMask = 'https://cms-service-dqm.web.cern.ch/cms-service-dqm/CAF/certification/Collisions17/13TeV/ReReco/Cert_294927-306462_13TeV_EOY2017ReReco_Collisions17_JSON_v1.txt'
elif '2016' in era:
config.Data.lumiMask = 'https://cms-service-dqm.web.cern.ch/cms-service-dqm/CAF/certification/Collisions16/13TeV/ReReco/Final/Cert_271036-284044_13TeV_ReReco_07Aug2017_Collisions16_JSON.txt'
config.JobType.pyCfgParams = [
'resonance={}'.format(resonance), 'isFullAOD={}'.format(True),
'isMC={}'.format(not isData), 'globalTag={}'.format(globalTag),
'numThreads={}'.format(numThreads)
]
config.Data.inputDataset = input_dataset
config.JobType.allowUndistributedCMSSW = True
if customSuffix != '':
config.General.requestName = 'muonAnalyzer_' + resonance + '_' + era + '_' + curr_subEra + '_' + customSuffix
else:
config.General.requestName = 'muonAnalyzer_' + resonance + '_' + era + '_' + curr_subEra
#config.Data.outputDatasetTag = sample (default CRAB dataset tag is 'crab_' + requestName)
# If we need to pull input files from a list file instead of CRAB:
# config.Data.userInputFiles = open(basedir + sample + '.list').readlines()
# Submit.
def submit(config, options):
try:
print "Submitting for input dataset %s with options %s" % (
input_dataset, options.crabCmdOpts)
crabCommand(options.crabCmd,
config=config,
*options.crabCmdOpts.split())
except HTTPException as hte:
print "Submission for input dataset %s failed: %s" % (
input_dataset, hte.headers)
except ClientException as cle:
print "Submission for input dataset %s failed: %s" % (
input_dataset, cle)
# Need to submit using multiprocessing module because of CRAB issue with different configs
p = Process(target=submit, args=(
config,
options,
))
p.start()
p.join()
# All other commands can be simply executed.
elif workArea:
for dir in os.listdir(workArea):
projDir = os.path.join(workArea, dir)
if not os.path.isdir(projDir):
continue
# Execute the crab command.
msg = "Executing (the equivalent of): crab %s --dir %s %s" % (
crabCmd, projDir, crabCmdOpts)
print "-" * len(msg)
print msg
print "-" * len(msg)
try:
crabCommand(crabCmd, dir=projDir, *crabCmdOpts.split())
except HTTPException as hte:
print "Failed executing command %s for task %s: %s" % (
crabCmd, projDir, hte.headers)
except ClientException as cle:
print "Failed executing command %s for task %s: %s" % (
crabCmd, projDir, cle)
def uploadlogfile(logger, proxyfilename, logfilename = None, logpath = None, instance = 'prod', serverurl = None, username = None):
## WMCore dependencies. Moved here to minimize dependencies in the bootstrap script
from WMCore.Services.UserFileCache.UserFileCache import UserFileCache
doupload = True
if logfilename == None:
logfilename = str(time.strftime("%Y-%m-%d_%H%M%S"))+'_crab.log'
logger.info('Fetching user enviroment to log file')
try:
cmd = 'env'
logger.debug('Running env command')
pipe = subprocess.Popen(cmd, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = True)
stdout, dummyStderr = pipe.communicate()
logger.debug('\n\n\nUSER ENVIROMENT\n%s' % stdout)
except Exception as se:
logger.debug('Failed to get the user env\nException message: %s' % (se))
if logpath != None:
if not os.path.exists(logpath):
doupload = False
logger.debug('%sError%s: %s does not exist' %(colors.RED, colors.NORMAL, logpath))
else:
if os.path.exists(str(os.getcwd()) + '/crab.log'):
logpath = str(os.getcwd())+'/crab.log'
else:
logger.debug('%sError%s: Failed to find crab.log in current directory %s' % (colors.RED, colors.NORMAL, str(os.getcwd())))
if serverurl == None and instance in SERVICE_INSTANCES.keys():
serverurl = SERVICE_INSTANCES[instance]
elif not instance in SERVICE_INSTANCES.keys() and serverurl == None:
logger.debug('%sError%s: serverurl is None' % (colors.RED, colors.NORMAL))
doupload = False
if proxyfilename == None:
logger.debug('No proxy was given')
doupload = False
baseurl = getUrl(dbInstance=instance , resource = 'info')
if doupload:
cacheurl = server_info('backendurls', serverurl, proxyfilename, baseurl)
# Encode in ascii because old pycurl present in old CMSSW versions
# doesn't support unicode.
cacheurl = cacheurl['cacheSSL'].encode('ascii')
cacheurldict = {'endpoint': cacheurl, "pycurl": True}
ufc = UserFileCache(cacheurldict)
logger.debug("cacheURL: %s\nLog file name: %s" % (cacheurl, logfilename))
logger.info("Uploading log file...")
ufc.uploadLog(logpath, logfilename)
logger.info("%sSuccess%s: Log file uploaded successfully." % (colors.GREEN, colors.NORMAL))
logfileurl = cacheurl + '/logfile?name='+str(logfilename)
if not username:
from CRABClient.UserUtilities import getUsername
username = getUsername(proxyFile=proxyfilename, logger=logger)
logfileurl += '&username='******'Failed to upload the log file')
logfileurl = False
return logfileurl
config.JobType.psetName = args.cfg
config.JobType.maxMemoryMB = args.maxMemory
config.JobType.numCores = args.numCores
config.Data.inputDBS = args.inputDBS
config.Data.allowNonValidInputDataset = args.allowNonValid
config.General.transferOutputs = True
config.General.transferLogs = False
config.Data.publication = False
config.Site.storageSite = args.site
if args.output[0] == '/':
config.Data.outLFNDirBase = args.output
else:
config.Data.outLFNDirBase = "/store/user/{}/{}".format(getUsername(), args.output)
if len(args.blacklist) != 0:
config.Site.blacklist = re.split(',', args.blacklist)
if len(args.whitelist) != 0:
config.Site.whitelist = re.split(',', args.whitelist)
job_names = Set(filter(lambda s: len(s) != 0, re.split(",", args.jobNames)))
from TauMLTools.Production.crab_tools import JobCollection
try:
job_collection = JobCollection(args.jobFile, job_names, args.lumiMask, args.jobNameSuffix)
print args.jobFile
print job_collection
print "Splitting: {} with {} units per job".format(args.splitting, args.unitsPerJob)
job_collection.submit(config, args.splitting, args.unitsPerJob, args.dryrun)
from CRABClient.UserUtilities import config, getUsername config = config() config.General.requestName = 'DYJetsToLL_M-10to50_TuneCP5_13TeV-madgraphMLM-pythia8' ## EDIT config.General.workArea = 'crab_projects' config.General.transferLogs = False config.General.transferOutputs = True config.JobType.pluginName = 'Analysis' config.JobType.psetName = 'RunSKFlatMaker.py' config.JobType.sendExternalFolder = True config.JobType.pyCfgParams = [] ## EDIT #config.JobType.numCores = 8 #config.JobType.maxMemoryMB = 4000 config.JobType.allowUndistributedCMSSW = True config.Data.inputDataset = '/DYJetsToLL_M-10to50_TuneCP5_13TeV-madgraphMLM-pythia8/RunIIFall17MiniAOD-94X_mc2017_realistic_v10-v2/MINIAODSIM' ## EDIT config.Data.splitting = 'FileBased' ## EDIT config.Data.unitsPerJob = 1 config.Data.outLFNDirBase = '/store/user/%s/SKFlat/' % (getUsername()) config.Data.publication = False config.Data.outputDatasetTag = 'SKFlatMaker_2017_v1' config.Site.storageSite = 'T3_KR_KISTI'
def uploadlogfile(logger,
proxyfilename,
taskname=None,
logfilename=None,
logpath=None,
instance=None,
serverurl=None,
username=None):
## WMCore dependencies. Moved here to minimize dependencies in the bootstrap script
from WMCore.Services.UserFileCache.UserFileCache import UserFileCache
doupload = True
if logfilename == None:
logfilename = str(time.strftime("%Y-%m-%d_%H%M%S")) + '_crab.log'
logger.info('Fetching user enviroment to log file')
try:
logger.debug('Running env command')
stdout, _, _ = execute_command(command='env')
logger.debug('\n\n\nUSER ENVIROMENT\n%s' % stdout)
except Exception as se:
logger.debug('Failed to get the user env\nException message: %s' %
(se))
if logpath != None:
if not os.path.exists(logpath):
doupload = False
logger.debug('%sError%s: %s does not exist' %
(colors.RED, colors.NORMAL, logpath))
else:
if os.path.exists(str(os.getcwd()) + '/crab.log'):
logpath = str(os.getcwd()) + '/crab.log'
else:
logger.debug(
'%sError%s: Failed to find crab.log in current directory %s' %
(colors.RED, colors.NORMAL, str(os.getcwd())))
if proxyfilename == None:
logger.debug('No proxy was given')
doupload = False
if doupload:
# uploadLog is executed directly from crab main script, does not inherit from SubCommand
# so it needs its own REST server instantiation
restClass = CRABClient.Emulator.getEmulator('rest')
crabserver = restClass(hostname=serverurl,
localcert=proxyfilename,
localkey=proxyfilename,
retry=2,
logger=logger,
verbose=False,
version=__version__,
userAgent='CRABClient')
crabserver.setDbInstance(instance)
cacheurl = server_info(crabserver=crabserver,
subresource='backendurls')['cacheSSL']
logger.info("Uploading log file...")
if 'S3' in cacheurl.upper():
objecttype = 'clientlog'
uploadToS3(crabserver=crabserver,
filepath=logpath,
objecttype=objecttype,
taskname=taskname,
logger=logger)
logfileurl = getDownloadUrlFromS3(crabserver=crabserver,
objecttype=objecttype,
taskname=taskname,
logger=logger)
else:
cacheurldict = {'endpoint': cacheurl, "pycurl": True}
ufc = UserFileCache(cacheurldict)
logger.debug("cacheURL: %s\nLog file name: %s" %
(cacheurl, logfilename))
ufc.uploadLog(logpath, logfilename)
logfileurl = cacheurl + '/logfile?name=' + str(logfilename)
if not username:
from CRABClient.UserUtilities import getUsername
username = getUsername(proxyFile=proxyfilename, logger=logger)
logfileurl += '&username='******'Failed to upload the log file')
logfileurl = False
return logfileurl
