def creatorProcess(work, jobCacheDir):
"""
_creatorProcess_
Creator work areas and pickle job objects
"""
createWorkArea = CreateWorkArea()
try:
wmbsJobGroup = work.get('jobGroup')
workflow = work.get('workflow')
wmWorkload = work.get('wmWorkload')
wmTaskName = work.get('wmTaskName')
sandbox = work.get('sandbox')
owner = work.get('owner')
ownerDN = work.get('ownerDN', None)
ownerGroup = work.get('ownerGroup', '')
ownerRole = work.get('ownerRole', '')
scramArch = work.get('scramArch', None)
swVersion = work.get('swVersion', None)
agentNumber = work.get('agentNumber', 0)
numberOfCores = work.get('numberOfCores', 1)
multicoreEnabled = work.get('multicoreEnabled', False)
if ownerDN == None:
ownerDN = owner
jobNumber = work.get('jobNumber', 0)
except KeyError, ex:
msg = "Could not find critical key-value in work input.\n"
msg += str(ex)
logging.error(msg)
raise JobCreatorException(msg)
def __init__(self, **configDict):
"""
init jobCreator
"""
myThread = threading.currentThread()
self.transaction = myThread.transaction
#DAO factory for WMBS objects
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=logging,
dbinterface=myThread.dbi)
# WMCore splitter factory for splitting up jobs.
self.splitterFactory = SplitterFactory()
config = Configuration()
config.section_("JobStateMachine")
config.JobStateMachine.couchurl = configDict["couchURL"]
config.JobStateMachine.couch_retries = configDict["defaultRetries"]
config.JobStateMachine.couchDBName = configDict["couchDBName"]
self.config = config
#Variables
self.jobCacheDir = configDict['jobCacheDir']
self.defaultJobType = configDict['defaultJobType']
self.limit = configDict.get('fileLoadLimit', 500)
self.createWorkArea = CreateWorkArea()
self.changeState = ChangeState(self.config)
return
def creatorProcess(work, jobCacheDir):
"""
_creatorProcess_
Creator work areas and pickle job objects
"""
createWorkArea = CreateWorkArea()
try:
wmbsJobGroup = work.get('jobGroup')
workflow = work.get('workflow')
wmWorkload = work.get('wmWorkload')
work['ownerDN'] = work.get('owner') if work.get(
'ownerDN', None) is None else work.get('ownerDN')
except KeyError as ex:
msg = "Could not find critical key-value in work input.\n"
msg += str(ex)
logging.error(msg)
raise JobCreatorException(msg)
except Exception as ex:
msg = "Exception in opening work package. Error: %s" % str(ex)
logging.exception(msg)
raise JobCreatorException(msg)
try:
createWorkArea.processJobs(jobGroup=wmbsJobGroup,
startDir=jobCacheDir,
workflow=workflow,
wmWorkload=wmWorkload,
cache=False)
thisJobNumber = work.get('jobNumber', 0)
for job in wmbsJobGroup.jobs:
thisJobNumber += 1
saveJob(job, thisJobNumber, **work)
except Exception as ex:
msg = "Exception in processing wmbsJobGroup %i\n. Error: %s" % (
wmbsJobGroup.id, str(ex))
logging.exception(msg)
raise JobCreatorException(msg)
return wmbsJobGroup
def creatorProcess(work, jobCacheDir):
"""
_creatorProcess_
Creator work areas and pickle job objects
"""
createWorkArea = CreateWorkArea()
try:
wmbsJobGroup = work.get('jobGroup')
workflow = work.get('workflow')
wmWorkload = work.get('wmWorkload')
wmTaskName = work.get('wmTaskName')
sandbox = work.get('sandbox')
owner = work.get('owner')
ownerDN = work.get('ownerDN', None)
ownerGroup = work.get('ownerGroup', '')
ownerRole = work.get('ownerRole', '')
scramArch = work.get('scramArch', None)
swVersion = work.get('swVersion', None)
agentNumber = work.get('agentNumber', 0)
numberOfCores = work.get('numberOfCores', 1)
inputDataset = work.get('inputDataset', None)
inputDatasetLocations = work.get('inputDatasetLocations', None)
inputPileup = work.get('inputPileup', None)
allowOpportunistic = work.get('allowOpportunistic', False)
agentName = work.get('agentName', '')
if ownerDN is None:
ownerDN = owner
jobNumber = work.get('jobNumber', 0)
except KeyError as ex:
msg = "Could not find critical key-value in work input.\n"
msg += str(ex)
logging.error(msg)
raise JobCreatorException(msg)
except Exception as ex:
msg = "Exception in opening work package. Error: %s" % str(ex)
logging.exception(msg)
raise JobCreatorException(msg)
try:
createWorkArea.processJobs(jobGroup=wmbsJobGroup,
startDir=jobCacheDir,
workflow=workflow,
wmWorkload=wmWorkload,
cache=False)
for job in wmbsJobGroup.jobs:
jobNumber += 1
saveJob(job=job, workflow=workflow,
wmTask=wmTaskName,
jobNumber=jobNumber,
sandbox=sandbox,
owner=owner,
ownerDN=ownerDN,
ownerGroup=ownerGroup,
ownerRole=ownerRole,
scramArch=scramArch,
swVersion=swVersion,
agentNumber=agentNumber,
numberOfCores=numberOfCores,
inputDataset=inputDataset,
inputDatasetLocations=inputDatasetLocations,
inputPileup=inputPileup,
allowOpportunistic=allowOpportunistic,
agentName=agentName)
except Exception as ex:
msg = "Exception in processing wmbsJobGroup %i\n. Error: %s" % (wmbsJobGroup.id, str(ex))
logging.exception(msg)
raise JobCreatorException(msg)
return wmbsJobGroup
def creatorProcess(work, jobCacheDir):
"""
_creatorProcess_
Creator work areas and pickle job objects
"""
createWorkArea = CreateWorkArea()
try:
wmbsJobGroup = work.get('jobGroup')
workflow = work.get('workflow')
wmWorkload = work.get('wmWorkload')
wmTaskName = work.get('wmTaskName')
sandbox = work.get('sandbox')
owner = work.get('owner')
ownerDN = work.get('ownerDN', None)
ownerGroup = work.get('ownerGroup', '')
ownerRole = work.get('ownerRole', '')
scramArch = work.get('scramArch', None)
swVersion = work.get('swVersion', None)
agentNumber = work.get('agentNumber', 0)
numberOfCores = work.get('numberOfCores', 1)
if ownerDN == None:
ownerDN = owner
jobNumber = work.get('jobNumber', 0)
except KeyError as ex:
msg = "Could not find critical key-value in work input.\n"
msg += str(ex)
logging.error(msg)
raise JobCreatorException(msg)
except Exception as ex:
msg = "Exception in opening work package.\n"
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
raise JobCreatorException(msg)
try:
createWorkArea.processJobs(jobGroup=wmbsJobGroup,
startDir=jobCacheDir,
workflow=workflow,
wmWorkload=wmWorkload,
cache=False)
for job in wmbsJobGroup.jobs:
jobNumber += 1
saveJob(job=job,
workflow=workflow,
wmTask=wmTaskName,
jobNumber=jobNumber,
sandbox=sandbox,
owner=owner,
ownerDN=ownerDN,
ownerGroup=ownerGroup,
ownerRole=ownerRole,
scramArch=scramArch,
swVersion=swVersion,
agentNumber=agentNumber,
numberOfCores=numberOfCores)
except Exception as ex:
# Register as failure; move on
msg = "Exception in processing wmbsJobGroup %i\n" % wmbsJobGroup.id
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
raise JobCreatorException(msg)
return wmbsJobGroup
