def main():
url='cmsweb.cern.ch'
parser = optparse.OptionParser()
parser.add_option('-f', '--filename', help='Filename',dest='filename')
parser.add_option('-w', '--workflow', help='Workflow',dest='userWorkflow')
parser.add_option('-t', '--team', help='Type of Requests',dest='team')
parser.add_option('-s', '--site', help='Force workflow to run at this site. For HLT/AI just put use HLT.',dest='site')
parser.add_option('-k', '--ignore-restrictions', help='Ignore site restrictions',action="store_true",dest='ignoresite')
parser.add_option('-u', '--new-priority', help='Change workflow priority to #',dest='newpriority')
parser.add_option('-c', '--custodial', help='Custodial site',dest='siteCust')
parser.add_option('-p', '--procstring', help='Process String',dest='inprocstring')
parser.add_option('-m', '--procversion', help='Process Version',dest='inprocversion')
parser.add_option('-e', '--execute', help='Actually assign workflows',action="store_true",dest='execute')
parser.add_option('-x', '--restrict', help='Only assign workflows for this site',dest='restrict')
parser.add_option('-z', '--threshold', help='Threshold for completeness of input dataset at site',dest='threshold')
parser.add_option('-r', '--rssmax', help='Max RSS',dest='maxRSS')
parser.add_option('-v', '--vsizemax', help='Max VMem',dest='maxVSize')
parser.add_option('-o', '--xrootd', help='Read input using xrootd',action="store_true",dest='xrootd')
parser.add_option('-i', '--ignore', help='Ignore any errors',action="store_true",dest='ignore')
(options,args) = parser.parse_args()
activity='reprocessing'
if not options.restrict:
restrict='None'
else:
restrict=options.restrict
maxRSS = 2800000
if not options.maxRSS:
maxRSS = 3000000
else:
maxRSS=options.maxRSS
maxRSSdefault = maxRSS
maxVSize = 4100000000
if not options.maxVSize:
maxVSize = 4100000000
else:
maxVSize=options.maxVSize
filename=options.filename
ignore = 0
if options.ignore:
ignore = 1
ignoresiterestrictions = 0
if options.ignoresite:
ignoresiterestrictions = 1
if not options.newpriority:
newpriority=0
else:
newpriority=options.newpriority
# Valid Tier-1 sites
sites = ['T1_DE_KIT', 'T1_FR_CCIN2P3', 'T1_IT_CNAF', 'T1_ES_PIC', 'T1_TW_ASGC', 'T1_UK_RAL', 'T1_US_FNAL', 'T1_RU_JINR', 'T2_CH_CERN', 'HLT']
# only assign workflows from these campaigns
valids = ['Fall11R1', 'Fall11R2', 'Fall11R4', 'Spring14dr', 'Fall13dr', 'Summer12DR53X', 'pAWinter13DR53X', 'Cosmic70DR', 'HiFall13DR53X', 'Phys14DR', 'Summer11LegDR','Fall14DR', 'Fall14DR73', 'TP2023SHCALDR', '2019GEMUpg14DR', 'HiWinter13DR53X', 'RunIWinter15DR', '2023SHCALUpg14DR']
# Tier-1s with no tape left, so use CERN instead
sitesNoTape = ['T1_RU_JINR']
if options.filename:
f = open(filename,'r')
elif options.userWorkflow:
f = [options.userWorkflow]
else:
f = getWorkflows(url)
workflowsNotAssignedInput = []
workflowsAssigned = {}
for workflow in f:
workflow = workflow.rstrip('\n')
if not options.xrootd:
useX = 0
else:
useX = 1
# Double check that the workflow really is in assignment-approved
currentStatus = getCurrentStatus(url, workflow)
if currentStatus != 'assignment-approved':
print 'NOTE: Due to workflow status (',currentStatus,') skipping',workflow
continue
# Only automatically assign workflows from specified campaigns
campaign = getCampaign(url, workflow)
if campaign not in valids and not options.userWorkflow and not options.filename:
print 'NOTE: Due to campaign skipping',workflow
continue
siteUse=options.site
if siteUse == 'T2_US':
siteUse = ['T2_US_Caltech', 'T2_US_Florida', 'T2_US_MIT', 'T2_US_Nebraska', 'T3_US_Omaha', 'T2_US_Purdue', 'T2_US_UCSD', 'T2_US_Vanderbilt', 'T2_US_Wisconsin']
if not options.siteCust:
print 'ERROR: A custodial site must be specified'
continue
siteCust = options.siteCust
# Check status of input dataset
inputDataset = reqMgrClient.getInputDataSet(url, workflow)
inputDatasetStatus = getDatasetStatus(inputDataset)
if inputDatasetStatus != 'VALID' and inputDatasetStatus != 'PRODUCTION':
print 'ERROR: Unable to assign',workflow,' because input dataset is not PRODUCTION or VALID, value is',inputDatasetStatus
continue
if not siteUse or siteUse == 'None':
# Find site to run workflow if no site specified
threshold = 98.0
if options.threshold:
threshold = options.threshold
[siteUse,completeness] = getSiteWithMostInput(inputDataset, threshold)
if siteUse == 'None' or ('T1_' not in siteUse and 'T2_CH_CERN' not in siteUse and campaign != 'Spring14miniaod'):
workflowsNotAssignedInput.append(workflow)
continue
if completeness < 100.0:
print 'Input dataset is < 100% complete (',completeness,') so enabling xrootd'
useX = 1
# Set the custodial location if necessary
if not options.site or options.site != 'T2_US':
if not options.siteCust:
siteCust = siteUse
else:
siteCust = options.siteCust
if options.site == 'HLT':
siteUse = ['T2_CH_CERN_HLT', 'T2_CH_CERN']
# Some sites have no free space on tape, so send the data to CERN
if siteUse in sitesNoTape:
siteCust = 'T0_CH_CERN'
# Don't specify a custodial site for miniaod
if campaign == 'Spring14miniaod':
siteCust = 'None'
# Check if input dataset subscribed to disk endpoint
siteSE = siteUse
if 'T1' in siteUse:
siteSE = siteSE + '_Disk'
[subscribedOurSite, subscribedOtherSite] = checkAcceptedSubscriptionRequest(url, inputDataset, siteSE)
if not subscribedOurSite and not options.xrootd and not ignore:
print 'ERROR: input dataset not subscribed/approved to required Disk endpoint and xrootd option not enabled (',subscribedOurSite,subscribedOtherSite,workflow,siteSE,')'
workflowsNotAssignedInput.append(workflow)
continue
if options.xrootd and not subscribedOtherSite and not ignore:
print 'ERROR: input dataset not subscribed/approved to any Disk endpoint (',subscribedOurSite,subscribedOtherSite,')'
continue
# Check if pileup dataset subscribed to disk endpoint
pileupDataset = getPileupDataset(url, workflow)
if pileupDataset != 'None':
[subscribedOurSite, subscribedOtherSite] = checkAcceptedSubscriptionRequest(url, pileupDataset, siteSE)
if not subscribedOurSite and not ignore:
print 'ERROR: pileup dataset (',pileupDataset,') not subscribed/approved to required Disk endpoint',siteSE,' for workflow',workflow
continue
# Decide which team to use if not already defined
# - currently we only use production for all workflows
if options.team:
team = options.team
else:
team = 'production'
# Get LFN base from input dataset
lfn = getLFNbase(url, inputDataset)
# Set maxRSS
maxRSS = maxRSSdefault
if ('HiFall11' in workflow or 'HiFall13DR53X' in workflow) and 'IN2P3' in siteUse:
maxRSS = 4000000
# Set max number of merge events
maxmergeevents = 50000
if 'Fall11R1' in workflow:
maxmergeevents = 6000
if 'DR61SLHCx' in workflow:
maxmergeevents = 5000
# Acquisition era
era = getCampaign(url, workflow)
# Correct situations where campaign name cannot be used as acquisition era
if era == '2019GEMUpg14DR':
era = 'GEM2019Upg14DR'
if era == '2023SHCALUpg14DR':
era = 'SHCAL2023Upg14DR'
if era == 'None':
print 'ERROR: unable to get campaign for workflow', workflow
continue
# Processing string
if options.inprocstring:
procstring = options.inprocstring
else:
procstring = getProcString(url, workflow)
# ProcessingVersion
if not options.inprocversion:
procversion = getDatasetVersion(url, workflow, era, procstring)
else:
procversion = options.inprocversion
# Handle run-dependent MC
if 'PU_RD' in procstring:
numEvents = getRequestNumEvents(url, workflow)
reqJobs = 500
if 'PU_RD2' in procstring:
reqJobs = 2000
eventsPerJob = int(numEvents/(reqJobs*1.4))
if eventsPerJob < 2000:
if options.execute:
print 'Changing splitting to',eventsPerJob,'events per job'
changeSplitting(url, workflow, eventsPerJob)
else:
print 'Would change splitting to',eventsPerJob,'events per job'
# Site checking
if siteUse not in sites and options.site != 'T2_US' and siteUse != ['T2_CH_CERN_T0', 'T2_CH_CERN_HLT', 'T2_CH_CERN'] and not ignoresiterestrictions and siteUse != ['T2_CH_CERN_HLT', 'T2_CH_CERN']:
if 'AODSIM' not in inputDataset:
print 'ERROR: invalid site',siteUse
continue
workflowsAssigned[workflow] = siteUse
if options.execute:
if restrict == 'None' or restrict == siteUse:
assignRequest(url, workflow, team, siteUse, era, procversion, procstring, activity, lfn, maxmergeevents, maxRSS, maxVSize, useX, siteCust)
if (newpriority !=0 ):
changePriorityWorkflow(url,workflow,newpriority)
print "Priority reset to %i" % newpriority
else:
print 'Skipping workflow ',workflow
else:
if restrict == 'None' or restrict == siteUse:
print 'Would assign ',workflow,' with ','Acquisition Era:',era,'ProcessingString:',procstring,'ProcessingVersion:',procversion,'lfn:',lfn,'Site(s):',siteUse,'Custodial Site:',siteCust,'team:',team
if (newpriority !=0 ):
print "Would reset priority to %i" % newpriority
else:
print 'Would skip workflow ',workflow
print ''
print 'SUMMARY'
print ''
# List assigned workflows
if len(workflowsAssigned) > 0:
if options.execute:
print 'Workflows assigned:'
else:
print 'Workflows which can be assigned:'
for workflow in workflowsAssigned:
print ' ',workflow,workflowsAssigned[workflow]
# List workflows not assigned because of input dataset
if len(workflowsNotAssignedInput) > 0:
print 'Workflows not assigned because input datasets are not complete on any site:'
for workflow in workflowsNotAssignedInput:
print ' ',workflow
sys.exit(0)
#encodedParams = urllib.urlencode(schema, True)
headers = {"Content-type": "application/x-www-form-urlencoded",
"Accept": "text/plain"}
conn = httplib.HTTPSConnection("cmsweb.cern.ch", cert_file = os.getenv('X509_USER_PROXY'), key_file = os.getenv('X509_USER_PROXY'))
conn.request("POST", "/reqmgr/create/makeSchema", encodedParams, headers)
response = conn.getresponse()
print response.status, response.reason
data = response.read()
print data
details=re.search("details\/(.*)\'",data)
return details.group(1)
if __name__ == "__main__":
if len(sys.argv) != 4:
print "Usage:"
print " ./resubmit WORKFLOW_NAME USER GROUP"
sys.exit(0)
user=sys.argv[2]
group=sys.argv[3]
#print "Going to attempt to resubmit %s..." % sys.argv[1]
schema = retrieveSchema(sys.argv[1], user, group)
#print schema
newWorkflow=submitWorkflow(schema)
approveRequest('cmsweb.cern.ch',newWorkflow)
print 'Cloned workflow:',newWorkflow
newPriority=getPriorityWorkflow('cmsweb.cern.ch',sys.argv[1])
if newPriority>2:
changePriorityWorkflow.changePriorityWorkflow('cmsweb.cern.ch', newWorkflow, newPriority)
sys.exit(0)
conn = httplib.HTTPSConnection("cmsweb.cern.ch",
cert_file=os.getenv('X509_USER_PROXY'),
key_file=os.getenv('X509_USER_PROXY'))
conn.request("POST", "/reqmgr/create/makeSchema", encodedParams, headers)
response = conn.getresponse()
print response.status, response.reason
data = response.read()
print data
details = re.search("details\/(.*)\'", data)
return details.group(1)
if __name__ == "__main__":
if len(sys.argv) != 4:
print "Usage:"
print " ./resubmit WORKFLOW_NAME USER GROUP"
sys.exit(0)
user = sys.argv[2]
group = sys.argv[3]
#print "Going to attempt to resubmit %s..." % sys.argv[1]
schema = retrieveSchema(sys.argv[1], user, group)
#print schema
newWorkflow = submitWorkflow(schema)
approveRequest('cmsweb.cern.ch', newWorkflow)
print 'Cloned workflow:', newWorkflow
newPriority = getPriorityWorkflow('cmsweb.cern.ch', sys.argv[1])
if newPriority > 2:
changePriorityWorkflow.changePriorityWorkflow('cmsweb.cern.ch',
newWorkflow, newPriority)
sys.exit(0)
conn = httplib.HTTPSConnection(
"cmsweb.cern.ch", cert_file=os.getenv("X509_USER_PROXY"), key_file=os.getenv("X509_USER_PROXY")
)
conn.request("POST", "/reqmgr/create/makeSchema", encodedParams, headers)
response = conn.getresponse()
print response.status, response.reason
data = response.read()
print data
details = re.search("details\/(.*)'", data)
return details.group(1)
if __name__ == "__main__":
if len(sys.argv) != 4:
print "Usage:"
print " ./resubmit WORKFLOW_NAME USER GROUP"
sys.exit(0)
user = sys.argv[2]
group = sys.argv[3]
# print "Going to attempt to resubmit %s..." % sys.argv[1]
schema = retrieveSchema(sys.argv[1], user, group)
# print schema
newWorkflow = submitWorkflow(schema)
approveRequest("cmsweb.cern.ch", newWorkflow)
print "Cloned workflow:", newWorkflow
newPriority = getPriorityWorkflow("cmsweb.cern.ch", sys.argv[1])
if newPriority > 2:
changePriorityWorkflow.changePriorityWorkflow("cmsweb.cern.ch", newWorkflow, newPriority)
sys.exit(0)
