def handleTaskChain(request): # Check if it's MC from scratch if 'RequestNumEvents' in request['Task1']: if request['Task1']['RequestNumEvents'] is not None: return request['Task1']['RequestNumEvents'] blockWhitelist = blockBlacklist = runWhitelist = runBlacklist = [] if 'InputDataset' in request['Task1']: inputDataSet=request['Task1']['InputDataset'] if 'BlockWhitelist' in request['Task1']: blockWhitelist=request['Task1']['BlockWhitelist'] if 'BlockBlacklist' in request['Task1']: blockBlacklist=request['Task1']['BlockBlacklist'] if 'RunWhitelist' in request['Task1']: runWhitelist=request['Task1']['RunWhitelist'] if 'RunBlacklist' in request['Task1']: runBlacklist=request['Task1']['RunBlacklist'] if blockWhitelist: return dbs3.getEventCountDataSetBlockList(inputDataSet,blockWhitelist) if blockBlacklist: return dbs3.getEventCountDataset(inputDataSet) - dbs3.getEventCountDataSetBlockList(inputDataSet,blockBlacklist) if runWhitelist: return dbs3.getEventCountDataSetRunList(inputDataSet, runWhitelist) else: return dbs3.getEventCountDataset(inputDataSet)
def getInputEventsTaskChain(request): """ Calculates input events for a taskchain based on the TaskChain properties and subtype """ #TODO filter by subtype #if it's MC from scratch, it has a set number of requested events if 'RequestNumEvents' in request['Task1']: if request['Task1']['RequestNumEvents'] is not None: return request['Task1']['RequestNumEvents'] #if it has an input dataset blockWhitelist = blockBlacklist = runWhitelist = runBlacklist = [] if 'InputDataset' in request['Task1']: inputDataSet = request['Task1']['InputDataset'] if 'BlockWhitelist' in request['Task1']: blockWhitelist = request['Task1']['BlockWhitelist'] if 'BlockBlacklist' in request['Task1']: blockBlacklist = request['Task1']['BlockBlacklist'] if 'RunWhitelist' in request['Task1']: runWhitelist = request['Task1']['RunWhitelist'] if 'RunBlacklist' in request['Task1']: runBlacklist = request['Task1']['RunBlacklist'] if blockWhitelist: return dbs3.getEventCountDataSetBlockList(inputDataSet, blockWhitelist) if blockBlacklist: return dbs3.getEventCountDataset( inputDataSet) - dbs3.getEventCountDataSetBlockList( inputDataSet, blockBlacklist) if runWhitelist: return dbs3.getEventCountDataSetRunList(inputDataSet, runWhitelist) else: return dbs3.getEventCountDataset(inputDataSet)
def getInputEventsTaskChain(request): """ Calculates input events for a taskchain based on the TaskChain properties and subtype """ #TODO filter by subtype #if it's MC from scratch, it has a set number of requested events if 'RequestNumEvents' in request['Task1']: if request['Task1']['RequestNumEvents'] is not None: return request['Task1']['RequestNumEvents'] #if it has an input dataset blockWhitelist = blockBlacklist = runWhitelist = runBlacklist = [] if 'InputDataset' in request['Task1']: inputDataSet=request['Task1']['InputDataset'] if 'BlockWhitelist' in request['Task1']: blockWhitelist=request['Task1']['BlockWhitelist'] if 'BlockBlacklist' in request['Task1']: blockBlacklist=request['Task1']['BlockBlacklist'] if 'RunWhitelist' in request['Task1']: runWhitelist=request['Task1']['RunWhitelist'] if 'RunBlacklist' in request['Task1']: runBlacklist=request['Task1']['RunBlacklist'] if blockWhitelist: return dbs3.getEventCountDataSetBlockList(inputDataSet,blockWhitelist) if blockBlacklist: return dbs3.getEventCountDataset(inputDataSet) - dbs3.getEventCountDataSetBlockList(inputDataSet,blockBlacklist) if runWhitelist: return dbs3.getEventCountDataSetRunList(inputDataSet, runWhitelist) else: return dbs3.getEventCountDataset(inputDataSet)
def getInputEvents(url, workflow): """ Gets the inputs events of a given workflow depending of the kind of workflow """ request = getWorkflowInfo(url,workflow) requestType=request['RequestType'] #if request is montecarlo or Step0, the numer of #input events is by the requsted events if requestType == 'MonteCarlo' or requestType == 'LHEStepZero': if 'RequestNumEvents' in request: if request['RequestNumEvents']>0: return request['RequestNumEvents'] if 'RequestSizeEvents' in request: return request['RequestSizeEvents'] else: return 0 if requestType == 'TaskChain': return handleTaskChain(request) #if request is not montecarlo, then we need to check the size #of input datasets #This loops fixes the white and blacklists in the workflow #information, for listitem in ["RunWhitelist", "RunBlacklist", "BlockWhitelist", "BlockBlacklist"]: if listitem in request: #if empty if request[listitem]=='[]' or request[listitem]=='': request[listitem]=[] #if there is not a list but some elements it creates a list if type(request[listitem]) is not list: request[listitem]= eval(request[listitem]) #if not, an empty list will do else: request[listitem]=[] inputDataSet=request['InputDataset'] #it the request is rereco, we valiate white/black lists if requestType=='ReReco': # if there is block whte list, count only the selected block if request['BlockWhitelist']: events = dbs3.getEventCountDataSetBlockList(inputDataSet,request['BlockWhitelist']) # if there is block black list, substract them from the total if request['BlockBlacklist']: events = (dbs3.getEventCountDataSet(inputDataSet) - dbs3.getEventCountDataSet(inputDataSet,request['BlockBlacklist'])) return events # same if a run whitelist if request['RunWhitelist']: events = dbs3.getEventCountDataSetRunList(inputDataSet, request['RunWhitelist']) return events # otherwize, the full lumi count else: events = dbs3.getEventCountDataset(inputDataSet) return events events = dbs3.getEventCountDataSet(inputDataSet) # if black list, subsctract them if request['BlockBlacklist']: events=events-dbs3.getEventCountDataSetBlockList(inputDataSet, request['BlockBlacklist']) # if white list, only the ones in the whitelist. if request['RunWhitelist']: events=dbs3.getEventCountDataSetRunList(inputDataSet, request['RunWhitelist']) # if white list of blocks if request['BlockWhitelist']: events=dbs3.getEventCountDataSetBlockList(inputDataSet, request['BlockWhitelist']) if 'FilterEfficiency' in request: return float(request['FilterEfficiency'])*events else: return events
def getInputEvents(url, workflow): """ Gets the inputs events of a given workflow depending of the kind of workflow TODO this can be replaced by getting the info from the workload cache """ request = getWorkflowInfo(url, workflow) requestType = request['RequestType'] #if request is montecarlo or Step0, the numer of #input events is by the requsted events if requestType == 'MonteCarlo' or requestType == 'LHEStepZero': if 'RequestNumEvents' in request: if request['RequestNumEvents'] > 0: return request['RequestNumEvents'] if 'RequestSizeEvents' in request: return request['RequestSizeEvents'] else: return 0 if requestType == 'TaskChain': return getInputEventsTaskChain(request) #if request is not montecarlo, then we need to check the size #of input datasets #This loops fixes the white and blacklists in the workflow #information, for listitem in [ "RunWhitelist", "RunBlacklist", "BlockWhitelist", "BlockBlacklist" ]: if listitem in request: #if empty if request[listitem] == '[]' or request[listitem] == '': request[listitem] = [] #if there is not a list but some elements it creates a list if type(request[listitem]) is not list: # if doesn't contain "[" is a single block if '[' not in request[listitem]: #wrap in a list request[listitem] = [request[listitem]] #else parse a list else: request[listitem] = eval(request[listitem]) #if not, an empty list will do else: request[listitem] = [] inputDataSet = request['InputDataset'] #it the request is rereco, we valiate white/black lists if requestType == 'ReReco': # if there is block whte list, count only the selected block if request['BlockWhitelist']: events = dbs3.getEventCountDataSetBlockList( inputDataSet, request['BlockWhitelist']) # if there is block black list, substract them from the total if request['BlockBlacklist']: events = (dbs3.getEventCountDataSet(inputDataSet) - dbs3.getEventCountDataSet(inputDataSet, request['BlockBlacklist'])) return events # same if a run whitelist if request['RunWhitelist']: events = dbs3.getEventCountDataSetRunList(inputDataSet, request['RunWhitelist']) return events # otherwize, the full lumi count else: events = dbs3.getEventCountDataset(inputDataSet) return events events = dbs3.getEventCountDataSet(inputDataSet) # if black list, subsctract them if request['BlockBlacklist']: events = events - dbs3.getEventCountDataSetBlockList( inputDataSet, request['BlockBlacklist']) # if white list, only the ones in the whitelist. if request['RunWhitelist']: events = dbs3.getEventCountDataSetRunList(inputDataSet, request['RunWhitelist']) # if white list of blocks if request['BlockWhitelist']: events = dbs3.getEventCountDataSetBlockList(inputDataSet, request['BlockWhitelist']) #TODO delete FilterEfficiency from here. TEST #if 'FilterEfficiency' in request: #return float(request['FilterEfficiency'])*events #else: return events