class DBSBufferBlock(object):
"""
_DBSBufferBlock_
"""
def __init__(self, name, location, datasetpath):
"""
Just the necessary objects
Expects:
name: The blockname in full
location: The PNN of the site the block is at
"""
self.data = {
'dataset_conf_list': [], # List of dataset configurations
'file_conf_list': [], # List of files, the configuration for each
'files': [], # List of file objects
'block': {}, # Dict of block info
'processing_era': {}, # Dict of processing era info
'acquisition_era': {}, # Dict of acquisition era information
'primds': {}, # Dict of primary dataset info
'dataset': {}, # Dict of processed dataset info
'file_parent_list': [], # List of file parents
'dataset_parent_list':
[], # List of parent datasets (DBS requires this as list although it only allows one parent)
'close_settings': {}
} # Dict of info about block close settings
self.files = []
self.encoder = JSONRequests()
self.status = 'Open'
self.inBuff = False
self.startTime = time.time()
self.name = name
self.location = location
self.datasetpath = datasetpath
self.workflows = set()
self.data['block']['block_name'] = name
self.data['block']['origin_site_name'] = location
self.data['block']['open_for_writing'] = 1
self.data['block']['create_by'] = "WMAgent"
self.data['block']['creation_date'] = int(time.time())
self.data['block']['block_size'] = 0
self.data['block']['file_count'] = 0
self.data['block']['block_events'] = 0
self.data['close_settings'] = {}
self.data['close_settings']['block_close_max_wait_time'] = None
self.data['close_settings']['block_close_max_events'] = None
self.data['close_settings']['block_close_max_size'] = None
self.data['close_settings']['block_close_max_files'] = None
return
def encode(self):
"""
_encode_
Turn this into a JSON object for transmission
to DBS
"""
return self.encoder.encode(data=self.data)
def addFile(self, dbsFile, datasetType, primaryDatasetType):
"""
_addFile_
Add a DBSBufferFile object to our block
"""
if dbsFile['id'] in [x['id'] for x in self.files]:
msg = "Duplicate file inserted into DBSBufferBlock: %i\n" % (
dbsFile['id'])
msg += "Ignoring this file for now!\n"
logging.error(msg)
logging.debug("Block length: %i", len(self.files))
l = sorted([x['id'] for x in self.files])
logging.debug("First file: %s Last file: %s", l[0], l[-1])
return
for setting in self.data['close_settings']:
if self.data['close_settings'][setting] is None:
self.data['close_settings'][setting] = dbsFile[setting]
self.workflows.add(dbsFile['workflow'])
self.files.append(dbsFile)
self.data['block']['block_size'] += int(dbsFile['size'])
self.data['block']['file_count'] += 1
self.data['block']['block_events'] += int(dbsFile['events'])
# Assemble information for the file itself
fileDict = {}
fileDict['file_type'] = 'EDM'
fileDict['logical_file_name'] = dbsFile['lfn']
fileDict['file_size'] = dbsFile['size']
fileDict['event_count'] = dbsFile['events']
fileDict['last_modified_by'] = "WMAgent"
fileDict['last_modification_date'] = int(time.time())
fileDict['auto_cross_section'] = 0.0
# Do the checksums
for cktype in dbsFile['checksums']:
cksum = dbsFile['checksums'][cktype]
if cktype.lower() == 'cksum':
fileDict['check_sum'] = cksum
elif cktype.lower() == 'adler32':
fileDict['adler32'] = cksum
elif cktype.lower() == 'md5':
fileDict['md5'] = cksum
# Do the runs
lumiList = []
for run in dbsFile.getRuns():
for lumi in run.lumis:
dbsLumiDict = {'lumi_section_num': lumi, 'run_num': run.run}
if run.getEventsByLumi(lumi) is not None:
# if events is not None update event for dbs upload
dbsLumiDict['event_count'] = run.getEventsByLumi(lumi)
lumiList.append(dbsLumiDict)
fileDict['file_lumi_list'] = lumiList
# Append to the files list
self.data['files'].append(fileDict)
# If dataset_parent_list is defined don't add the file parentage.
# This means it is block from StepChain workflow and parentage of file will be resloved later
if not self.data['dataset_parent_list']:
# now add file to data
parentLFNs = dbsFile.getParentLFNs()
for lfn in parentLFNs:
self.addFileParent(child=dbsFile['lfn'], parent=lfn)
# Do the algo
algo = self.addConfiguration(release=dbsFile['appVer'],
psetHash=dbsFile['psetHash'],
appName=dbsFile['appName'],
outputLabel=dbsFile['appFam'],
globalTag=dbsFile['globalTag'])
# Now add the file with the algo
# Try to avoid messing with pointers here
fileAlgo = {}
fileAlgo.update(algo)
fileAlgo['lfn'] = dbsFile['lfn']
self.data['file_conf_list'].append(fileAlgo)
if dbsFile.get('acquisition_era', False):
self.setAcquisitionEra(dbsFile['acquisition_era'])
elif dbsFile.get('acquisitionEra', False):
self.setAcquisitionEra(dbsFile['acquisitionEra'])
if dbsFile.get('processingVer', False):
self.setProcessingVer(dbsFile['processingVer'])
elif dbsFile.get('processing_ver', False):
self.setProcessingVer(dbsFile['processing_ver'])
# Take care of the dataset
self.setDataset(datasetName=dbsFile['datasetPath'],
primaryType=primaryDatasetType,
datasetType=datasetType,
physicsGroup=dbsFile.get('physicsGroup', None),
prep_id=dbsFile.get('prep_id', None))
return
def addFileParent(self, child, parent):
"""
_addFileParent_
Add file parents to the data block
"""
info = {'parent_logical_file_name': parent, 'logical_file_name': child}
self.data['file_parent_list'].append(info)
return
def addBlockParent(self, parent):
"""
_addBlockParent_
Add the parents of the block
"""
self.data['block_parent_list'].append({'block_name': parent})
return
def addDatasetParent(self, parent):
"""
_addDatasetParent_
Add the parent datasets to the data block
"""
self.data['dataset_parent_list'].append(parent)
return
def setProcessingVer(self, procVer):
"""
_setProcessingVer_
Set the block's processing version.
"""
# compatibility statement for old style proc ver (still needed ?)
if procVer.count("-") == 1:
self.data["processing_era"]["processing_version"] = procVer.split(
"-v")[1]
else:
self.data["processing_era"]["processing_version"] = procVer
self.data["processing_era"]["create_by"] = "WMAgent"
self.data["processing_era"]["description"] = ""
return
def setAcquisitionEra(self, era, date=123456789):
"""
_setAcquisitionEra_
Set the acquisition era for the block
"""
self.data['acquisition_era']['acquisition_era_name'] = era
self.data['acquisition_era']['start_date'] = date
return
def setPhysicsGroup(self, group):
"""
_setPhysicsGroup_
Sets the name of the physics group to which the dataset is attached
"""
self.data['dataset']['physics_group_name'] = group
return
def getDatasetPath(self):
"""
_getDatasetPath_
Return the datasetpath
"""
return self.datasetpath
def getDataset(self):
"""
_getDataset_
Return the dataset (None if not set)
"""
return self.data['dataset'].get('dataset', None)
def setDataset(self,
datasetName,
primaryType,
datasetType,
physicsGroup=None,
prep_id=None,
overwrite=False):
"""
_setDataset_
Set all the information concerning a single dataset, including
the primary, processed and tier info
"""
if self.getDataset() != None and not overwrite:
# Do nothing, we already have a dataset
return
Lexicon.primaryDatasetType(primaryType)
if not datasetType in [
'VALID', 'PRODUCTION', 'INVALID', 'DEPRECATED', 'DELETED'
]:
msg = "Invalid processedDatasetType %s\n" % datasetType
logging.error(msg)
raise DBSBufferBlockException(msg)
try:
if datasetName[0] == '/':
_, primary, processed, tier = datasetName.split('/')
else:
primary, processed, tier = datasetName.split('/')
except Exception:
msg = "Invalid dataset name %s" % datasetName
logging.error(msg)
raise DBSBufferBlockException(msg)
# Do the primary dataset
self.data['primds']['primary_ds_name'] = primary
self.data['primds']['primary_ds_type'] = primaryType
self.data['primds']['create_by'] = "WMAgent"
self.data['primds']['creation_date'] = int(time.time())
# Do the processed
self.data['dataset']['physics_group_name'] = physicsGroup
self.data['dataset']['processed_ds_name'] = processed
self.data['dataset']['data_tier_name'] = tier
self.data['dataset']['dataset_access_type'] = datasetType
self.data['dataset']['dataset'] = datasetName
self.data['dataset']['prep_id'] = prep_id
# Add misc meta data.
self.data['dataset']['create_by'] = "WMAgent"
self.data['dataset']['last_modified_by'] = "WMAgent"
self.data['dataset']['creation_date'] = int(time.time())
self.data['dataset']['last_modification_date'] = int(time.time())
return
def addConfiguration(self,
release,
psetHash,
appName='cmsRun',
outputLabel='Merged',
globalTag='None'):
"""
_addConfiguration_
Add the algorithm config to the data block
"""
algo = {
'release_version': release,
'pset_hash': psetHash,
'app_name': appName,
'output_module_label': outputLabel,
'global_tag': globalTag
}
if not algo in self.data['dataset_conf_list']:
self.data['dataset_conf_list'].append(algo)
return algo
def getNFiles(self):
"""
_getNFiles_
Return the number of files in the block
"""
return len(self.files)
def getSize(self):
"""
_getSize_
Get size of block
"""
return self.data['block']['block_size']
def getNumEvents(self):
"""
_getNumEvents_
Get the number of events in the block
"""
return self.data['block']['block_events']
def getTime(self):
"""
_getTime_
Return the time the block has been running
"""
return time.time() - self.startTime
def getMaxBlockTime(self):
"""
_getMaxBlockTime_
Return the max time that the block should stay open
"""
return self.data['close_settings']['block_close_max_wait_time']
def getMaxBlockSize(self):
"""
_getMaxBlockSize_
Return the max size allowed for the block
"""
return self.data['close_settings']['block_close_max_size']
def getMaxBlockNumEvents(self):
"""
_getMaxBlockNumEvents_
Return the max number of events allowed for the block
"""
return self.data['close_settings']['block_close_max_events']
def getMaxBlockFiles(self):
"""
_getMaxBlockFiles_
Return the max number of files allowed for the block
"""
return self.data['close_settings']['block_close_max_files']
def getName(self):
"""
_getName_
Get Name
"""
return self.name
def getLocation(self):
"""
_getLocation_
Get location
"""
return self.location
def getStartTime(self):
"""
_getStartTime_
Get the time the block was opened at
"""
return self.startTime
def FillFromDBSBuffer(self, blockInfo):
"""
_FillFromDBSBuffer_
Take the info provided by LoadBlocks and
use it to create a block object
"""
# Blocks loaded out of the buffer should
# have both a creation time, and should
# be in the buffer (duh)
self.startTime = blockInfo.get('creation_date')
self.inBuff = True
if 'status' in blockInfo:
self.status = blockInfo['status']
if self.status == "Pending":
self.data['block']['open_for_writing'] = 0
del blockInfo['status']
for key in blockInfo:
self.data['block'][key] = blockInfo.get(key)
def convertToDBSBlock(self):
"""
convert to DBSBlock structure to upload to dbs
TODO: check file lumi event and validate event is not null
"""
block = {}
#TODO: instead of using key to remove need to change to keyToKeep
# Ask dbs team to publish the list (API)
keyToRemove = [
'insertedFiles', 'newFiles', 'file_count', 'block_size',
'origin_site_name', 'creation_date', 'open', 'Name',
'close_settings'
]
nestedKeyToRemove = [
'block.block_events', 'block.datasetpath', 'block.workflows'
]
dbsBufferToDBSBlockKey = {
'block_size': 'BlockSize',
'creation_date': 'CreationDate',
'file_count': 'NumberOfFiles',
'origin_site_name': 'location'
}
# clone the new DBSBlock dict after filtering out the data.
for key in self.data:
if key in keyToRemove:
continue
elif key in dbsBufferToDBSBlockKey:
block[dbsBufferToDBSBlockKey[key]] = copy.deepcopy(
self.data[key])
else:
block[key] = copy.deepcopy(self.data[key])
# delete nested key dictionary
for nestedKey in nestedKeyToRemove:
firstkey, subkey = nestedKey.split('.', 1)
if firstkey in block and subkey in block[firstkey]:
del block[firstkey][subkey]
return block
def setPendingAndCloseBlock(self):
"set the block status as Pending for upload as well as closed"
# Pending means ready to upload
self.status = "Pending"
# close block on DBS3 status
self.data['block']['open_for_writing'] = 0
class DBSBlock:
"""
DBSBlock
Class for holding all the necessary equipment for a DBSBlock
"""
def __init__(self, name, location, das):
"""
Just the necessary objects
Expects:
name: The blockname in full
location: The SE-name of the site the block is at
"""
self.data = {'dataset_conf_list': [], # List of dataset configurations
'file_conf_list': [], # List of files, with the configuration for each
'files': [], # List of file objects
'block': {}, # Dict of block info
'block_parent_list': [], # List of block parents
'processing_era': {}, # Dict of processing era info
'ds_parent_list': [], # List of parent datasets
'acquisition_era': {}, # Dict of acquisition era information
'primds': {}, # Dict of primary dataset info
'dataset': {}, # Dict of processed dataset info
'physics_group_name': {}, # Physics Group Name
'file_parent_list': []} # List of file parents
self.files = []
self.encoder = JSONRequests()
self.status = 'Open'
self.inBuff = False
self.startTime = time.time()
self.name = name
self.location = location
self.das = das
self.data['block']['block_name'] = name
self.data['block']['origin_site_name'] = location
self.data['block']['open_for_writing'] = 0 # If we're sending a block, it better be open
return
def encode(self):
"""
_encode_
Turn this into a JSON object for transmission
to DBS
"""
return self.encoder.encode(data = self.data)
def addFile(self, dbsFile):
"""
_addFile_
Add a DBSBufferFile object to our block
"""
if dbsFile['id'] in [x['id'] for x in self.files]:
msg = "Duplicate file inserted into DBSBlock: %i\n" % (dbsFile['id'])
msg += "Ignoring this file for now!\n"
logging.error(msg)
logging.debug("Block length: %i" % len(self.files))
l = [x['id'] for x in self.files]
l.sort()
logging.debug("First file: %s Last file: %s" % (l[0], l[-1]))
return
self.files.append(dbsFile)
# Assemble information for the file itself
fileDict = {}
fileDict['file_type'] = 'EDM'
fileDict['logical_file_name'] = dbsFile['lfn']
fileDict['file_size'] = dbsFile['size']
fileDict['event_count'] = dbsFile['events']
# Do the checksums
for cktype in dbsFile['checksums'].keys():
cksum = dbsFile['checksums'][cktype]
if cktype.lower() == 'cksum':
fileDict['check_sum'] = cksum
elif cktype.lower() == 'adler32':
fileDict['adler32'] = cksum
elif cktype.lower() == 'md5':
fileDict['md5'] = cksum
# Do the runs
lumiList = []
for run in dbsFile.getRuns():
for lumi in run.lumis:
lumiList.append({'lumi_section_num': lumi, 'run_num': run.run})
fileDict['file_lumi_list'] = lumiList
# Append to the files list
self.data['files'].append(fileDict)
# now add file to data
parentLFNs = dbsFile.getParentLFNs()
for lfn in parentLFNs:
self.addFileParent(child = dbsFile['lfn'], parent = lfn)
# Do the algo
algo = self.addConfiguration(release = dbsFile['appVer'],
psetHash = dbsFile['psetHash'],
appName = dbsFile['appName'],
outputLabel = dbsFile['appFam'],
globalTag = dbsFile['globalTag'])
# Now add the file with the algo
# Try to avoid messing with pointers here
fileAlgo = {}
fileAlgo.update(algo)
fileAlgo['lfn'] = dbsFile['lfn']
self.data['file_conf_list'].append(fileAlgo)
if dbsFile.get('acquisition_era', False):
self.setAcquisitionEra(dbsFile['acquisition_era'])
elif dbsFile.get('acquisitionEra', False):
self.setAcquisitionEra(dbsFile['acquisitionEra'])
if dbsFile.get('processingVer', False):
self.setProcessingVer(dbsFile['processingVer'])
elif dbsFile.get('processing_ver', False):
self.setProcessingVer(dbsFile['processing_ver'])
# Take care of the dataset
self.setDataset(datasetName = dbsFile['datasetPath'],
primaryType = dbsFile.get('primaryType', 'DATA'),
datasetType = dbsFile.get('datasetType', 'PRODUCTION'),
physicsGroup = dbsFile.get('physicsGroup', None))
return
def addFileParent(self, child, parent):
"""
_addFileParent_
Add file parents to the data block
"""
info = {'parent_logical_file_name': parent,
'logical_file_name': child}
self.data['file_parent_list'].append(info)
return
def addBlockParent(self, parent):
"""
_addBlockParent_
Add the parents of the block
"""
self.data['block_parent_list'].append({'block_name': parent})
return
def addDatasetParent(self, parent):
"""
_addDatasetParent_
Add the parent datasets to the data block
"""
self.data['ds_parent_list'].append({'parent_dataset': parent})
return
def setProcessingVer(self, era):
"""
_setProcessingEra_
Set the block's processing era
"""
self.data['processing_era']['processing_version'] = era
return
def setAcquisitionEra(self, era, date = 123456789):
"""
_setAcquisitionEra_
Set the acquisition era for the block
"""
self.data['acquisition_era']['acquisition_era_name'] = era
self.data['acquisition_era']['start_date'] = date
return
def setPhysicsGroup(self, group):
"""
_setPhysicsGroup_
Sets the name of the physics group to which the dataset is attached
"""
self.data['dataset']['physics_group_name'] = group
return
def hasDataset(self):
"""
_hasDataset_
Check and see if the dataset has been properly set
"""
return self.data['dataset'].get('dataset', False)
def setDataset(self, datasetName, primaryType,
datasetType, physicsGroup = None, overwrite = False, valid = 1):
"""
_setDataset_
Set all the information concerning a single dataset, including
the primary, processed and tier info
"""
if self.hasDataset() and not overwrite:
# Do nothing, we already have a dataset
return
if not primaryType in ['MC', 'DATA', 'TEST']:
msg = "Invalid primaryDatasetType %s\n" % primaryType
logging.error(msg)
raise DBSBlockException(msg)
if not datasetType in ['VALID', 'PRODUCTION', 'INVALID', 'DEPRECATED', 'DELETED']:
msg = "Invalid processedDatasetType %s\n" % datasetType
logging.error(msg)
raise DBSBlockException(msg)
try:
if datasetName[0] == '/':
junk, primary, processed, tier = datasetName.split('/')
else:
primary, processed, tier = datasetName.split('/')
except Exception, ex:
msg = "Invalid dataset name %s" % datasetName
logging.error(msg)
raise DBSBlockException(msg)
# Do the primary dataset
self.data['primds']['primary_ds_name'] = primary
self.data['primds']['primary_ds_type'] = primaryType
# Do the processed
self.data['dataset']['physics_group_name'] = physicsGroup
self.data['dataset']['processed_ds_name'] = processed
self.data['dataset']['data_tier_name'] = tier
self.data['dataset']['dataset_access_type'] = datasetType
self.data['dataset']['dataset'] = datasetName
return
class ProcessPool:
def __init__(self, slaveClassName, totalSlaves, componentDir,
config, namespace = 'WMComponent', inPort = '5555',
outPort = '5558'):
"""
__init__
Constructor for the process pool. The slave class name must be based
inside the WMComponent namespace. For examples, the JobAccountant would
pass in 'JobAccountant.AccountantWorker' to run the AccountantWorker
class. All log files will be stored in the component directory that is
passed in. Each slave will have its own log file.
Note that the config is only used to determine database connection
parameters. It is not passed to the slave class. The slaveInit
parameter will be serialized and passed to the slave class's
constructor.
"""
self.enqueueIndex = 0
self.dequeueIndex = 0
self.runningWork = 0
#Use the Services.Requests JSONizer, which handles __to_json__ calls
self.jsonHandler = JSONRequests()
# heartbeat should be registered at this point
if getattr(config.Agent, "useHeartbeat", True):
self.heartbeatAPI = HeartbeatAPI(getattr(config.Agent, "componentName", "ProcPoolSlave"))
self.slaveClassName = slaveClassName
self.componentDir = componentDir
self.config = config
# Grab the python version from the current version
# Assume naming convention pythonA.B, i.e., python2.4 for v2.4.X
majorVersion = sys.version_info[0]
minorVersion = sys.version_info[1]
if majorVersion and minorVersion:
self.versionString = "python%i.%i" % (majorVersion, minorVersion)
else:
self.versionString = "python2.6"
self.workers = []
self.nSlaves = totalSlaves
self.namespace = namespace
self.inPort = inPort
self.outPort = outPort
# Pickle the config
self.configPath = os.path.join(componentDir, '%s_config.pkl' % slaveClassName)
if os.path.exists(self.configPath):
# Then we note it and overwrite it
msg = "Something's in the way of the ProcessPool config: %s" % self.configPath
logging.error(msg)
f = open(self.configPath, 'w')
cPickle.dump(config, f)
f.close()
# Set up ZMQ
try:
context = zmq.Context()
self.sender = context.socket(zmq.PUSH)
self.sender.bind("tcp://*:%s" % inPort)
self.sink = context.socket(zmq.PULL)
self.sink.bind("tcp://*:%s" % outPort)
except zmq.ZMQError:
# Try this again in a moment to see
# if it's just being held by something pre-existing
import time
time.sleep(1)
logging.error("Blocked socket on startup: Attempting sleep to give it time to clear.")
try:
context = zmq.Context()
self.sender = context.socket(zmq.PUSH)
self.sender.bind("tcp://*:%s" % inPort)
self.sink = context.socket(zmq.PULL)
self.sink.bind("tcp://*:%s" % outPort)
except Exception as ex:
msg = "Error attempting to open TCP sockets\n"
msg += str(ex)
logging.error(msg)
import traceback
print traceback.format_exc()
raise ProcessPoolException(msg)
# Now actually create the slaves
self.createSlaves()
return
def createSlaves(self):
"""
_createSlaves_
Create the slaves by using the values from __init__()
Moving it into a separate function allows us to restart
all of them.
"""
totalSlaves = self.nSlaves
slaveClassName = self.slaveClassName
config = self.config
namespace = self.namespace
inPort = self.inPort
outPort = self.outPort
slaveArgs = [self.versionString, __file__, self.slaveClassName, inPort,
outPort, self.configPath, self.componentDir, self.namespace]
count = 0
while totalSlaves > 0:
#For each worker you want create a slave process
#That process calls this code (WMCore.ProcessPool) and opens
#A process pool that loads the designated class
slaveProcess = subprocess.Popen(slaveArgs, stdin = subprocess.PIPE,
stdout = subprocess.PIPE)
self.workers.append(slaveProcess)
totalSlaves -= 1
count += 1
return
def _subProcessName(self, slaveClassName, sequence):
""" subProcessName for heartbeat
could change to use process ID as a suffix
"""
return "%s_%s" % (slaveClassName, sequence + 1)
def __del__(self):
"""
__del__
Kill all the workers processes by sending them an invalid JSON object.
This will cause them to shut down.
"""
self.close()
return
def close(self):
"""
_close_
Close shuts down all the active systems by:
a) Sending STOP commands for all workers
b) Closing the pipes
c) Shutting down the workers themselves
"""
for i in range(self.nSlaves):
try:
encodedWork = self.jsonHandler.encode('STOP')
self.sender.send(encodedWork)
except Exception as ex:
# Might be already failed. Nothing you can
# really do about that.
logging.error("Failure killing running process: %s" % str(ex))
pass
try:
self.sender.close()
except:
# We can't really do anything if we fail
pass
try:
self.sink.close()
except:
# We can't do anything if we fail
pass
# Now close the workers by hand
for worker in self.workers:
try:
worker.join()
except Exception as ex:
try:
worker.terminate()
except Exception as ex2:
logging.error("Failure to join or terminate process")
logging.error(str(ex))
logging.error(str(ex2))
continue
self.workers = []
return
def enqueue(self, work, list = False):
"""
__enqeue__
Assign work to the workers processes. The work parameters must be a
list where each item in the list can be serialized into JSON.
If list is True, the entire list is sent as one piece of work
"""
if len(self.workers) < 1:
# Someone's shut down the system
msg = "Attempting to send work after system failure and shutdown!\n"
logging.error(msg)
raise ProcessPoolException(msg)
if not list:
for w in work:
encodedWork = self.jsonHandler.encode(w)
self.sender.send(encodedWork)
self.runningWork += 1
else:
encodedWork = self.jsonHandler.encode(work)
self.sender.send(encodedWork)
self.runningWork += 1
return
def dequeue(self, totalItems = 1):
"""
__dequeue__
Retrieve completed work from the slave workers. This method will block
until enough work has been completed.
"""
completedWork = []
if totalItems > self.runningWork:
msg = "Asked to dequeue more work then is running!\n"
msg += "Failing"
logging.error(msg)
raise ProcessPoolException(msg)
while totalItems > 0:
try:
output = self.sink.recv()
decode = self.jsonHandler.decode(output)
if type(decode) == type({}) and decode.get('type', None) == 'ERROR':
# Then we had some kind of error
msg = decode.get('msg', 'Unknown Error in ProcessPool')
logging.error("Received Error Message from ProcessPool Slave")
logging.error(msg)
self.close()
raise ProcessPoolException(msg)
completedWork.append(decode)
self.runningWork -= 1
totalItems -= 1
except Exception as ex:
msg = "Exception while getting slave outputin ProcessPool.\n"
msg += str(ex)
logging.error(msg)
break
return completedWork
def restart(self):
"""
_restart_
Delete everything and restart all pools
"""
self.close()
self.createSlaves()
return
try:
logging.error(input)
output = slaveClass(input)
except Exception as ex:
crashMessage = "Slave process crashed with exception: " + str(ex)
crashMessage += "\nStacktrace:\n"
stackTrace = traceback.format_tb(sys.exc_info()[2], None)
for stackFrame in stackTrace:
crashMessage += stackFrame
logging.error(crashMessage)
try:
output = {'type': 'ERROR', 'msg': crashMessage}
encodedOutput = jsonHandler.encode(output)
sender.send(encodedOutput)
logging.error("Sent error message and now breaking")
break
except Exception as ex:
logging.error("Failed to send error message")
logging.error(str(ex))
del jsonHandler
sys.exit(1)
if output != None:
if type(output) == list:
for item in output:
encodedOutput = jsonHandler.encode(item)
sender.send(encodedOutput)
else:
class DBSBufferBlock:
"""
_DBSBufferBlock_
"""
def __init__(self, name, location, datasetpath):
"""
Just the necessary objects
Expects:
name: The blockname in full
location: The PNN of the site the block is at
"""
self.data = {'dataset_conf_list': [], # List of dataset configurations
'file_conf_list': [], # List of files, the configuration for each
'files': [], # List of file objects
'block': {}, # Dict of block info
'processing_era': {}, # Dict of processing era info
'acquisition_era': {}, # Dict of acquisition era information
'primds': {}, # Dict of primary dataset info
'dataset': {}, # Dict of processed dataset info
'file_parent_list': [], # List of file parents
'close_settings': {}} # Dict of info about block close settings
self.files = []
self.encoder = JSONRequests()
self.status = 'Open'
self.inBuff = False
self.startTime = time.time()
self.name = name
self.location = location
self.datasetpath = datasetpath
self.workflows = set()
self.data['block']['block_name'] = name
self.data['block']['origin_site_name'] = location
self.data['block']['open_for_writing'] = 1
self.data['block']['create_by'] = "WMAgent"
self.data['block']['creation_date'] = int(time.time())
self.data['block']['block_size'] = 0
self.data['block']['file_count'] = 0
self.data['block']['block_events'] = 0
self.data['close_settings'] = {}
self.data['close_settings']['block_close_max_wait_time'] = None
self.data['close_settings']['block_close_max_events'] = None
self.data['close_settings']['block_close_max_size'] = None
self.data['close_settings']['block_close_max_files'] = None
return
def encode(self):
"""
_encode_
Turn this into a JSON object for transmission
to DBS
"""
return self.encoder.encode(data = self.data)
def addFile(self, dbsFile, datasetType, primaryDatasetType):
"""
_addFile_
Add a DBSBufferFile object to our block
"""
if dbsFile['id'] in [x['id'] for x in self.files]:
msg = "Duplicate file inserted into DBSBufferBlock: %i\n" % (dbsFile['id'])
msg += "Ignoring this file for now!\n"
logging.error(msg)
logging.debug("Block length: %i" % len(self.files))
l = sorted([x['id'] for x in self.files])
logging.debug("First file: %s Last file: %s" % (l[0], l[-1]))
return
for setting in self.data['close_settings']:
if self.data['close_settings'][setting] is None:
self.data['close_settings'][setting] = dbsFile[setting]
self.workflows.add(dbsFile['workflow'])
self.files.append(dbsFile)
self.data['block']['block_size'] += int(dbsFile['size'])
self.data['block']['file_count'] += 1
self.data['block']['block_events'] += int(dbsFile['events'])
# Assemble information for the file itself
fileDict = {}
fileDict['file_type'] = 'EDM'
fileDict['logical_file_name'] = dbsFile['lfn']
fileDict['file_size'] = dbsFile['size']
fileDict['event_count'] = dbsFile['events']
fileDict['last_modified_by'] = "WMAgent"
fileDict['last_modification_date'] = int(time.time())
fileDict['auto_cross_section'] = 0.0
# Do the checksums
for cktype in dbsFile['checksums'].keys():
cksum = dbsFile['checksums'][cktype]
if cktype.lower() == 'cksum':
fileDict['check_sum'] = cksum
elif cktype.lower() == 'adler32':
fileDict['adler32'] = cksum
elif cktype.lower() == 'md5':
fileDict['md5'] = cksum
# Do the runs
lumiList = []
for run in dbsFile.getRuns():
for lumi in run.lumis:
dbsLumiDict = {'lumi_section_num': lumi, 'run_num': run.run}
if run.getEventsByLumi(lumi) is not None:
# if events is not None update event for dbs upload
dbsLumiDict['event_count'] = run.getEventsByLumi(lumi)
lumiList.append(dbsLumiDict)
fileDict['file_lumi_list'] = lumiList
# Append to the files list
self.data['files'].append(fileDict)
# now add file to data
parentLFNs = dbsFile.getParentLFNs()
for lfn in parentLFNs:
self.addFileParent(child = dbsFile['lfn'], parent = lfn)
# Do the algo
algo = self.addConfiguration(release = dbsFile['appVer'],
psetHash = dbsFile['psetHash'],
appName = dbsFile['appName'],
outputLabel = dbsFile['appFam'],
globalTag = dbsFile['globalTag'])
# Now add the file with the algo
# Try to avoid messing with pointers here
fileAlgo = {}
fileAlgo.update(algo)
fileAlgo['lfn'] = dbsFile['lfn']
self.data['file_conf_list'].append(fileAlgo)
if dbsFile.get('acquisition_era', False):
self.setAcquisitionEra(dbsFile['acquisition_era'])
elif dbsFile.get('acquisitionEra', False):
self.setAcquisitionEra(dbsFile['acquisitionEra'])
if dbsFile.get('processingVer', False):
self.setProcessingVer(dbsFile['processingVer'])
elif dbsFile.get('processing_ver', False):
self.setProcessingVer(dbsFile['processing_ver'])
# Take care of the dataset
self.setDataset(datasetName = dbsFile['datasetPath'],
primaryType = primaryDatasetType,
datasetType = datasetType,
physicsGroup = dbsFile.get('physicsGroup', None),
prep_id = dbsFile.get('prep_id', None))
return
def addFileParent(self, child, parent):
"""
_addFileParent_
Add file parents to the data block
"""
info = {'parent_logical_file_name': parent,
'logical_file_name': child}
self.data['file_parent_list'].append(info)
return
def addBlockParent(self, parent):
"""
_addBlockParent_
Add the parents of the block
"""
self.data['block_parent_list'].append({'block_name': parent})
return
def addDatasetParent(self, parent):
"""
_addDatasetParent_
Add the parent datasets to the data block
"""
self.data['ds_parent_list'].append({'parent_dataset': parent})
return
def setProcessingVer(self, procVer):
"""
_setProcessingVer_
Set the block's processing version.
"""
# compatibility statement for old style proc ver (still needed ?)
if procVer.count("-") == 1:
self.data["processing_era"]["processing_version"] = procVer.split("-v")[1]
else:
self.data["processing_era"]["processing_version"] = procVer
self.data["processing_era"]["create_by"] = "WMAgent"
self.data["processing_era"]["description"] = ""
return
def setAcquisitionEra(self, era, date = 123456789):
"""
_setAcquisitionEra_
Set the acquisition era for the block
"""
self.data['acquisition_era']['acquisition_era_name'] = era
self.data['acquisition_era']['start_date'] = date
return
def setPhysicsGroup(self, group):
"""
_setPhysicsGroup_
Sets the name of the physics group to which the dataset is attached
"""
self.data['dataset']['physics_group_name'] = group
return
def getDatasetPath(self):
"""
_getDatasetPath_
Return the datasetpath
"""
return self.datasetpath
def getDataset(self):
"""
_getDataset_
Return the dataset (None if not set)
"""
return self.data['dataset'].get('dataset', None)
def setDataset(self, datasetName, primaryType,
datasetType, physicsGroup = None,
prep_id = None, overwrite = False):
"""
_setDataset_
Set all the information concerning a single dataset, including
the primary, processed and tier info
"""
if self.getDataset() != None and not overwrite:
# Do nothing, we already have a dataset
return
Lexicon.primaryDatasetType(primaryType)
if not datasetType in ['VALID', 'PRODUCTION', 'INVALID', 'DEPRECATED', 'DELETED']:
msg = "Invalid processedDatasetType %s\n" % datasetType
logging.error(msg)
raise DBSBufferBlockException(msg)
try:
if datasetName[0] == '/':
junk, primary, processed, tier = datasetName.split('/')
else:
primary, processed, tier = datasetName.split('/')
except Exception:
msg = "Invalid dataset name %s" % datasetName
logging.error(msg)
raise DBSBufferBlockException(msg)
# Do the primary dataset
self.data['primds']['primary_ds_name'] = primary
self.data['primds']['primary_ds_type'] = primaryType
self.data['primds']['create_by'] = "WMAgent"
self.data['primds']['creation_date'] = int(time.time())
# Do the processed
self.data['dataset']['physics_group_name'] = physicsGroup
self.data['dataset']['processed_ds_name'] = processed
self.data['dataset']['data_tier_name'] = tier
self.data['dataset']['dataset_access_type'] = datasetType
self.data['dataset']['dataset'] = datasetName
self.data['dataset']['prep_id'] = prep_id
# Add misc meta data.
self.data['dataset']['create_by'] = "WMAgent"
self.data['dataset']['last_modified_by'] = "WMAgent"
self.data['dataset']['creation_date'] = int(time.time())
self.data['dataset']['last_modification_date'] = int(time.time())
return
def addConfiguration(self, release, psetHash,
appName = 'cmsRun', outputLabel = 'Merged', globalTag = 'None'):
"""
_addConfiguration_
Add the algorithm config to the data block
"""
algo = {'release_version': release,
'pset_hash': psetHash,
'app_name': appName,
'output_module_label': outputLabel,
'global_tag': globalTag}
if not algo in self.data['dataset_conf_list']:
self.data['dataset_conf_list'].append(algo)
return algo
def getNFiles(self):
"""
_getNFiles_
Return the number of files in the block
"""
return len(self.files)
def getSize(self):
"""
_getSize_
Get size of block
"""
return self.data['block']['block_size']
def getNumEvents(self):
"""
_getNumEvents_
Get the number of events in the block
"""
return self.data['block']['block_events']
def getTime(self):
"""
_getTime_
Return the time the block has been running
"""
return time.time() - self.startTime
def getMaxBlockTime(self):
"""
_getMaxBlockTime_
Return the max time that the block should stay open
"""
return self.data['close_settings']['block_close_max_wait_time']
def getMaxBlockSize(self):
"""
_getMaxBlockSize_
Return the max size allowed for the block
"""
return self.data['close_settings']['block_close_max_size']
def getMaxBlockNumEvents(self):
"""
_getMaxBlockNumEvents_
Return the max number of events allowed for the block
"""
return self.data['close_settings']['block_close_max_events']
def getMaxBlockFiles(self):
"""
_getMaxBlockFiles_
Return the max number of files allowed for the block
"""
return self.data['close_settings']['block_close_max_files']
def getName(self):
"""
_getName_
Get Name
"""
return self.name
def getLocation(self):
"""
_getLocation_
Get location
"""
return self.location
def getStartTime(self):
"""
_getStartTime_
Get the time the block was opened at
"""
return self.startTime
def FillFromDBSBuffer(self, blockInfo):
"""
_FillFromDBSBuffer_
Take the info provided by LoadBlocks and
use it to create a block object
"""
# Blocks loaded out of the buffer should
# have both a creation time, and should
# be in the buffer (duh)
self.startTime = blockInfo.get('creation_date')
self.inBuff = True
if 'status' in blockInfo.keys():
self.status = blockInfo['status']
if self.status == "Pending":
self.data['block']['open_for_writing'] = 0
del blockInfo['status']
for key in blockInfo.keys():
self.data['block'][key] = blockInfo.get(key)
def convertToDBSBlock(self):
"""
convert to DBSBlock structure to upload to dbs
TODO: check file lumi event and validate event is not null
"""
block = {}
#TODO: instead of using key to remove need to change to keyToKeep
# Ask dbs team to publish the list (API)
keyToRemove = ['insertedFiles', 'newFiles', 'file_count', 'block_size',
'origin_site_name', 'creation_date', 'open',
'Name', 'close_settings']
nestedKeyToRemove = ['block.block_events', 'block.datasetpath', 'block.workflows']
dbsBufferToDBSBlockKey = {'block_size': 'BlockSize',
'creation_date': 'CreationDate',
'file_count': 'NumberOfFiles',
'origin_site_name': 'location'}
# clone the new DBSBlock dict after filtering out the data.
for key in self.data:
if key in keyToRemove:
continue
elif key in dbsBufferToDBSBlockKey.keys():
block[dbsBufferToDBSBlockKey[key]] = copy.deepcopy(self.data[key])
else:
block[key] = copy.deepcopy(self.data[key])
# delete nested key dictionary
for nestedKey in nestedKeyToRemove:
firstkey, subkey = nestedKey.split('.', 1)
if firstkey in block and subkey in block[firstkey]:
del block[firstkey][subkey]
return block
def setPendingAndCloseBlock(self):
"set the block status as Pending for upload as well as closed"
# Pending means ready to upload
self.status = "Pending"
# close block on DBS3 status
self.data['block']['open_for_writing'] = 0
class DBSBlock:
"""
DBSBlock
Class for holding all the necessary equipment for a DBSBlock
"""
def __init__(self, name, location, das, workflow):
"""
Just the necessary objects
Expects:
name: The blockname in full
location: The SE-name of the site the block is at
"""
self.data = {
'dataset_conf_list': [], # List of dataset configurations
'file_conf_list': [], # List of files, the configuration for each
'files': [], # List of file objects
'block': {}, # Dict of block info
'processing_era': {}, # Dict of processing era info
'acquisition_era': {}, # Dict of acquisition era information
'primds': {}, # Dict of primary dataset info
'dataset': {}, # Dict of processed dataset info
'file_parent_list': [], # List of file parents
'close_settings': {}
} # Dict of info about block close settings
self.files = []
self.encoder = JSONRequests()
self.status = 'Open'
self.inBuff = False
self.startTime = time.time()
self.name = name
self.location = location
self.das = das
self.workflow = workflow
self.data['block']['block_name'] = name
self.data['block']['origin_site_name'] = location
self.data['block']['open_for_writing'] = 1
self.data['block']['create_by'] = "WMAgent"
self.data['block']['creation_date'] = int(time.time())
self.data['block']['block_size'] = 0
self.data['block']['file_count'] = 0
self.data['block']['block_events'] = 0
self.data['close_settings'] = {}
self.data['close_settings']['block_close_max_wait_time'] = None
self.data['close_settings']['block_close_max_events'] = None
self.data['close_settings']['block_close_max_size'] = None
self.data['close_settings']['block_close_max_files'] = None
return
def encode(self):
"""
_encode_
Turn this into a JSON object for transmission
to DBS
"""
return self.encoder.encode(data=self.data)
def addFile(self, dbsFile, datasetType, primaryDatasetType):
"""
_addFile_
Add a DBSBufferFile object to our block
"""
if dbsFile['id'] in [x['id'] for x in self.files]:
msg = "Duplicate file inserted into DBSBlock: %i\n" % (
dbsFile['id'])
msg += "Ignoring this file for now!\n"
logging.error(msg)
logging.debug("Block length: %i" % len(self.files))
l = [x['id'] for x in self.files]
l.sort()
logging.debug("First file: %s Last file: %s" % (l[0], l[-1]))
return
for setting in self.data['close_settings']:
if self.data['close_settings'][setting] is None:
self.data['close_settings'][setting] = dbsFile[setting]
self.files.append(dbsFile)
self.data['block']['block_size'] += int(dbsFile['size'])
self.data['block']['file_count'] += 1
self.data['block']['block_events'] += int(dbsFile['events'])
# Assemble information for the file itself
fileDict = {}
fileDict['file_type'] = 'EDM'
fileDict['logical_file_name'] = dbsFile['lfn']
fileDict['file_size'] = dbsFile['size']
fileDict['event_count'] = dbsFile['events']
fileDict['adler32'] = "NOTSET"
fileDict['md5'] = "NOTSET"
fileDict['last_modified_by'] = "WMAgent"
fileDict['last_modification_date'] = int(time.time())
fileDict['auto_cross_section'] = 0.0
# Do the checksums
for cktype in dbsFile['checksums'].keys():
cksum = dbsFile['checksums'][cktype]
if cktype.lower() == 'cksum':
fileDict['check_sum'] = cksum
elif cktype.lower() == 'adler32':
fileDict['adler32'] = cksum
elif cktype.lower() == 'md5':
fileDict['md5'] = cksum
# Do the runs
lumiList = []
for run in dbsFile.getRuns():
for lumi in run.lumis:
lumiList.append({'lumi_section_num': lumi, 'run_num': run.run})
fileDict['file_lumi_list'] = lumiList
# Append to the files list
self.data['files'].append(fileDict)
# now add file to data
parentLFNs = dbsFile.getParentLFNs()
for lfn in parentLFNs:
self.addFileParent(child=dbsFile['lfn'], parent=lfn)
# Do the algo
algo = self.addConfiguration(release=dbsFile['appVer'],
psetHash=dbsFile['psetHash'],
appName=dbsFile['appName'],
outputLabel=dbsFile['appFam'],
globalTag=dbsFile['globalTag'])
# Now add the file with the algo
# Try to avoid messing with pointers here
fileAlgo = {}
fileAlgo.update(algo)
fileAlgo['lfn'] = dbsFile['lfn']
self.data['file_conf_list'].append(fileAlgo)
if dbsFile.get('acquisition_era', False):
self.setAcquisitionEra(dbsFile['acquisition_era'])
elif dbsFile.get('acquisitionEra', False):
self.setAcquisitionEra(dbsFile['acquisitionEra'])
if dbsFile.get('processingVer', False):
self.setProcessingVer(dbsFile['processingVer'])
elif dbsFile.get('processing_ver', False):
self.setProcessingVer(dbsFile['processing_ver'])
# Take care of the dataset
self.setDataset(datasetName=dbsFile['datasetPath'],
primaryType=primaryDatasetType,
datasetType=datasetType,
physicsGroup=dbsFile.get('physicsGroup', None),
prep_id=dbsFile.get('prep_id', None))
return
def addFileParent(self, child, parent):
"""
_addFileParent_
Add file parents to the data block
"""
info = {'parent_logical_file_name': parent, 'logical_file_name': child}
self.data['file_parent_list'].append(info)
return
def addBlockParent(self, parent):
"""
_addBlockParent_
Add the parents of the block
"""
self.data['block_parent_list'].append({'block_name': parent})
return
def addDatasetParent(self, parent):
"""
_addDatasetParent_
Add the parent datasets to the data block
"""
self.data['ds_parent_list'].append({'parent_dataset': parent})
return
def setProcessingVer(self, procVer):
"""
_setProcessingVer_
Set the block's processing version.
"""
if procVer.count("-") == 1:
(junk, self.data["processing_era"]["processing_version"]
) = procVer.split("-v")
else:
self.data["processing_era"]["processing_version"] = procVer
self.data["processing_era"]["create_by"] = "WMAgent"
self.data["processing_era"]["description"] = ""
return
def setAcquisitionEra(self, era, date=123456789):
"""
_setAcquisitionEra_
Set the acquisition era for the block
"""
self.data['acquisition_era']['acquisition_era_name'] = era
self.data['acquisition_era']['start_date'] = date
return
def setPhysicsGroup(self, group):
"""
_setPhysicsGroup_
Sets the name of the physics group to which the dataset is attached
"""
self.data['dataset']['physics_group_name'] = group
return
def hasDataset(self):
"""
_hasDataset_
Check and see if the dataset has been properly set
"""
return self.data['dataset'].get('dataset', False)
def setDataset(self,
datasetName,
primaryType,
datasetType,
physicsGroup=None,
prep_id=None,
overwrite=False,
valid=1):
"""
_setDataset_
Set all the information concerning a single dataset, including
the primary, processed and tier info
"""
if self.hasDataset() and not overwrite:
# Do nothing, we already have a dataset
return
Lexicon.primaryDatasetType(primaryType)
if not datasetType in [
'VALID', 'PRODUCTION', 'INVALID', 'DEPRECATED', 'DELETED'
]:
msg = "Invalid processedDatasetType %s\n" % datasetType
logging.error(msg)
raise DBSBlockException(msg)
try:
if datasetName[0] == '/':
junk, primary, processed, tier = datasetName.split('/')
else:
primary, processed, tier = datasetName.split('/')
except Exception, ex:
msg = "Invalid dataset name %s" % datasetName
logging.error(msg)
raise DBSBlockException(msg)
# Do the primary dataset
self.data['primds']['primary_ds_name'] = primary
self.data['primds']['primary_ds_type'] = primaryType
self.data['primds']['create_by'] = "WMAgent"
self.data['primds']['creation_date'] = int(time.time())
# Do the processed
self.data['dataset']['physics_group_name'] = physicsGroup
self.data['dataset']['processed_ds_name'] = processed
self.data['dataset']['data_tier_name'] = tier
self.data['dataset']['dataset_access_type'] = datasetType
self.data['dataset']['dataset'] = datasetName
self.data['dataset']['prep_id'] = prep_id
# Add misc meta data.
self.data['dataset']['create_by'] = "WMAgent"
self.data['dataset']['last_modified_by'] = "WMAgent"
self.data['dataset']['creation_date'] = int(time.time())
self.data['dataset']['last_modification_date'] = int(time.time())
return
class ProcessPool:
def __init__(self, slaveClassName, totalSlaves, componentDir,
config, slaveInit = None, namespace = None):
"""
__init__
Constructor for the process pool. The slave class name must be based
inside the WMComponent namespace. For examples, the JobAccountant would
pass in 'JobAccountant.AccountantWorker' to run the AccountantWorker
class. All log files will be stored in the component directory that is
passed in. Each slave will have its own log file.
Note that the config is only used to determine database connection
parameters. It is not passed to the slave class. The slaveInit
parameter will be serialized and passed to the slave class's
constructor.
"""
self.enqueueIndex = 0
self.dequeueIndex = 0
self.runningWork = 0
#Use the Services.Requests JSONizer, which handles __to_json__ calls
self.jsonHandler = JSONRequests()
# heartbeat should be registered at this point
if getattr(config.Agent, "useHeartbeat", True):
self.heartbeatAPI = HeartbeatAPI(getattr(config.Agent, "componentName", "ProcPoolSlave"))
self.slaveClassName = slaveClassName
self.componentDir = componentDir
self.config = config
# Grab the python version from the current version
# Assume naming convention pythonA.B, i.e., python2.4 for v2.4.X
majorVersion = sys.version_info[0]
minorVersion = sys.version_info[1]
if majorVersion and minorVersion:
self.versionString = "python%i.%i" % (majorVersion, minorVersion)
else:
self.versionString = "python2.4"
self.workers = []
self.nSlaves = totalSlaves
self.slaveInit = slaveInit
self.namespace = namespace
# Now actually create the slaves
self.createSlaves()
return
def createSlaves(self):
"""
_createSlaves_
Create the slaves by using the values from __init__()
Moving it into a separate function allows us to restart
all of them.
"""
totalSlaves = self.nSlaves
slaveClassName = self.slaveClassName
config = self.config
slaveInit = self.slaveInit
namespace = self.namespace
slaveArgs = [self.versionString, __file__, self.slaveClassName]
if hasattr(config.CoreDatabase, "socket"):
socket = config.CoreDatabase.socket
else:
socket = None
(connectDialect, junk) = config.CoreDatabase.connectUrl.split(":", 1)
if connectDialect.lower() == "mysql":
dialect = "MySQL"
elif connectDialect.lower() == "oracle":
dialect = "Oracle"
elif connectDialect.lower() == "sqlite":
dialect = "SQLite"
dbConfig = {"dialect": dialect,
"connectUrl": config.CoreDatabase.connectUrl,
"socket": socket,
"componentDir": self.componentDir}
if namespace:
# Then add a namespace to the config
dbConfig['namespace'] = namespace
encodedDBConfig = self.jsonHandler.encode(dbConfig)
if slaveInit == None:
encodedSlaveInit = None
else:
encodedSlaveInit = self.jsonHandler.encode(slaveInit)
count = 0
while totalSlaves > 0:
#For each worker you want create a slave process
#That process calls this code (WMCore.ProcessPool) and opens
#A process pool that loads the designated class
slaveProcess = subprocess.Popen(slaveArgs, stdin = subprocess.PIPE,
stdout = subprocess.PIPE)
slaveProcess.stdin.write("%s\n" % encodedDBConfig)
if encodedSlaveInit == None:
slaveProcess.stdin.write("\n")
else:
slaveProcess.stdin.write("%s\n" % encodedSlaveInit)
slaveProcess.stdin.flush()
self.workers.append(WorkerProcess(subproc = slaveProcess))
workerName = self._subProcessName(self.slaveClassName, count)
if getattr(self.config.Agent, "useHeartbeat", True):
self.heartbeatAPI.updateWorkerHeartbeat(workerName,
pid = slaveProcess.pid)
totalSlaves -= 1
count += 1
return
def _subProcessName(self, slaveClassName, sequence):
""" subProcessName for heartbeat
could change to use process ID as a suffix
"""
return "%s_%s" % (slaveClassName, sequence + 1)
def __del__(self):
"""
__del__
Kill all the workers processes by sending them an invalid JSON object.
This will cause them to shut down.
"""
for worker in self.workers:
try:
worker.delete()
except Exception, ex:
pass
self.workers = []
return
