def GetTargetStepFromDAS(TargetStep, DAS):
mcm = McM(dev=False)
input_request = mcm.get("requests", query="produce=" + DAS)
member_of_chain = input_request[0]['member_of_chain']
mychain = ''
for chain in member_of_chain:
if TargetStep in chain:
mychain = chain
chained_request = mcm.get('chained_requests', mychain)
req = ''
for member in chained_request['chain']:
if TargetStep in member:
req = member
return req
def clone_and_edit_jhchoi(request_prepid_to_clone, fragment, dataset_name,
notes, total_events, generators):
mcm = McM(dev=False)
modifications = {
'fragment': fragment,
'dataset_name': dataset_name,
'notes': notes,
'total_events': total_events,
"generators": generators
}
#request_prepid_to_clone = "SUS-RunIIWinter15wmLHE-00040"
request = mcm.get('requests', request_prepid_to_clone)
for key in modifications:
request[key] = modifications[key]
clone_answer = mcm.clone_request(request)
if clone_answer.get('results'):
print "@@"
new_prepid = clone_answer['prepid']
print('Clone PrepID: %s' % (clone_answer['prepid']))
new_request = mcm.get('requests', new_prepid)
print new_request['dataset_name']
mcm.approve('requests', new_prepid, None)
print "---"
f = open('success.txt', 'a')
f.write(new_prepid + "\n")
f.close()
else:
print('Something went wrong while cloning a request. %s' %
(dumps(clone_answer)))
f = open('fail.txt', 'a')
f.write(dataset_name + "\n")
f.close()
def createrequest(self, clonequeue):
self.needsupdate = True
return clonequeue.add(self, self.pwg, self.newcampaign)
if jobtype(): return "run locally to submit to McM"
mcm = McM()
clone_req = mcm.get('requests', self.originalprepid)
clone_req['member_of_campaign'] = self.campaign
answer = mcm.clone(self.originalprepid, clone_req)
if not (answer and answer.get("results")):
raise RuntimeError("Failed to create the request on McM\n{}\n{}".format(self, answer))
self.getprepid()
if self.prepid != answer["prepid"]:
raise RuntimeError("Wrong prepid?? {} {}".format(self.prepid, answer["prepid"]))
self.updaterequest()
return "cloned request "+self.originalprepid+" as "+self.prepid+" on McM"
# print(root_requests_from_ticket(ticket))
prepid = []
for rr in root_requests_from_ticket(ticket):
if 'GS' in rr or 'wmLHE' in rr or 'pLHE' in rr:
prepid.append(rr)
prepid = list(set(prepid)) #to avoid requests appearing x times if x chains have the same request
for x in range(0,len(prepid)):
print(prepid[x])
for num in range(0,len(prepid)):
query_str = 'prepid='+prepid[num]
res = mcm.get('requests', query=query_str)
if len(res) == 0 :
print "***********************************************************************************"
print "Something's wrong - can not get the request parameters"
print "***********************************************************************************"
my_path = '/tmp/'+os.environ['USER']+'/gridpacks/'
print ""
print "***********************************************************************************"
for r in res:
pi = r['prepid']
dn = r['dataset_name']
te = r['time_event']
totalevents = r['total_events']
cmssw = r['cmssw_release']
modifiedPrepid = prepidToClone
modifiedPrepid['interested_pwg'] = ['EGM']
modifiedPrepid['member_of_campaign'] = campaign
modifiedPrepid['ppd_tags'] = tag
print 'Will clone to ', campaign, 'with tag', tag
cloned_prepid = mcm.clone_request(modifiedPrepid)
return mcm.get('requests', cloned_prepid['prepid'], method='get')
for UL16PrepidName in UL16PrepidsToClone:
print '\n---> Cloning', UL16PrepidName
if not ('EGM-RunIISummer19UL16GEN'
or 'EGM-RunIISummer19UL16wmLHEGEN' in UL16PrepidName):
print 'This does not look like a UL16 prepid, skipping :', UL16PrepidName
continue
# Get prepid from McM, check it exists
UL16PrepidFromMcM = mcm.get('requests', UL16PrepidName, method='get')
if len(UL16PrepidFromMcM) == 0:
print 'This prepid does not exist in McM :', UL16PrepidFromMcM
print 'EXITING'
sys.exit()
# Will clone to the 2016 UL campaign
UL16APVPrepid = cloneToUL16PreVFP(prepidToClone=UL16PrepidFromMcM)
if not UL16APVPrepid:
print 'Dataset already existed in 2016 UL PreVFP campaign'
else:
print 'Cloned prepid to: %s' % UL16APVPrepid['prepid']
dry_run = False
# McM instance
mcm = McM(dev=True)#, cookie='/afs/cern.ch/work/s/snorberg/miniAODv3/cookie.txt')#/afs/cern.ch/user/j/jrumsevi/private/dev_cookie.txt')
# Set list of PWGs here. If no list is set, all PWGs from McM will be used
pwgs = ['SMP']
if not pwgs:
# Get list of PWGs
pwgs = mcm._McM__get('restapi/users/get_pwg')['results']
print('Got PWGs: %s' % (pwgs))
# Create a dict of dicts of dicts of integers
collector = defaultdict(lambda: defaultdict(lambda: defaultdict(int)))
for pwg in pwgs:
chained_requests = mcm.get('chained_requests', query='member_of_campaign=%s&pwg=%s' % (org_chained_campaign, pwg))
# print('PWG:%s\tChained requests:%s' % (pwg, len(chained_requests)))
for chained_request in chained_requests:
root_id = chained_request['chain'][0]
campaign = root_id.split('-')[1]
collector[pwg][campaign][root_id] += 1
# Collector now contains all root requests of chains that are member of org_chained_campaign campaign
# It's structured like this: PWG -> campaign name of root request -> request prepid -> count
# print('This is the list of %s requests that are deemed chainable:\n%s' % (pwgs, json.dumps(dict(collector), indent=4)))
all_tickets = []
chained_campaign = mcm.get('chained_campaigns', dest_chained_campaign, method='get')
for pwg in pwgs:
# Create a ticket for the correct chain
chained_campaign_prepid = chained_campaign['prepid']
def Run(idx_row):
output_msg = ''
KEY = '1bAdzoB9bg6nbyxCOBsTAWSSJy6FHNfmkrIDqY-Nmdi0'
sheet = '2016'
title_list = GetTitleFromSpreadsheet(KEY,
sheet,
i_column="A",
f_column="Z")
info_list = GetInfoFromSpreadsheet(idx_row,
KEY,
sheet,
i_column="A",
f_column="Z")
this_dic = {}
for i in range(0, len(title_list)):
title = title_list[i]
#print title
value = info_list[i]
#print value
this_dic[title] = value
FullFragment = MakeFullFragment(this_dic["gridpackPATH"],
this_dic["PSfragment"],
this_dic["card_ref"])
generator_parameters=Make_generator_parameters(this_dic["cross_section"],\
this_dic["filter_efficiency"],\
this_dic["filter_efficiency_error"],\
this_dic["match_efficiency"],\
this_dic["match_efficiency_error"],\
this_dic["negative_weights_fraction"])
print this_dic['dataset_name']
new_request = {
'pwg': 'HIG',
'member_of_campaign': this_dic['member_of_campaign'],
'dataset_name': this_dic['dataset_name'],
'generators': this_dic['generators'].split(','),
"fragment": FullFragment,
'generator_parameters': generator_parameters,
'total_events': int(this_dic['total_events']),
'mcdb_id': 0,
'size_event': [600],
'time_event': [60.]
}
if this_dic["PrepID"] != '':
#print "ALREADY"
mcm = McM(dev=False)
this_request = mcm.get("requests", this_dic["PrepID"])
for key in new_request:
this_request[key] = new_request[key]
update = mcm.update('requests', this_request)
return str(update)
# push it to McM
#mcm = McM(dev=False,cookie="dev-cookie.txt")
#mcm = McM(dev=False,cookie='prod-cookie.txt')
mcm = McM(dev=False)
put_answer = mcm.put('requests', new_request)
if put_answer.get('results'):
prepid = put_answer['prepid']
print('New PrepID: %s' % (prepid))
#f.write(prepid+'\n')
output_msg = str(prepid)
else:
print('Something went wrong while creating a request. %s' %
(dumps(put_answer)))
#f.write("fail,idx_row-->"+this_dic['dataset_name']+'\n')
output_msg = "fail" + str(idx_row) + "-->" + this_dic['dataset_name']
#print output_msg
return output_msg
import sys
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
sys.stdout.flush()
mcm = McM(dev=False)
# Example to edit a request parameter(-s) and save it back in McM
# request_prepid_to_update = 'HIG-Summer12-01257' # Doesn't exist
#requests = mcm.get('requests', query='priority=110000&status=submitted&prepid=*Autumn18DR*')
#requests = mcm.get('requests', query='status=submitted&prepid=*MTDTDR*GS*')
requests = mcm.get('requests',
query='status=submitted&tags=MTDTDRAutumn18PU200')
#requests = mcm.get('requests', query='tags=Summer16MiniAODv3T2')
#requests = mcm.get('requests', query='tags=Autumn18P1POGDR')
i = 0
for request in requests:
chained_request_id = request['member_of_chain'][0]
chained_requests = mcm.get('chained_requests',
query='prepid=%s' % (chained_request_id))
if (len(chained_requests[0]['chain']) < 3):
continue
if ('PU0' in chained_requests[0]['prepid']):
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
from json import dumps
mcm = McM(dev=False)
# Example to get ALL requesst which are member of a given campaign and are submitted
# It uses a generic search for specified columns: query='status=submitted'
# Queries can be combined: query='status=submitted&member_of_campaign=Summer12'
#campaign_requests = mcm.get('requests', query='dataset_name=NMSSMCascade_mH-*_mSUSY-*_TuneCUEP8M1_13TeV-madgraph-pythia8')
#for request in campaign_requests:
# print(request['prepid'])
# Example to retrieve single request dictionary
# More methods are here:
# https://cms-pdmv.cern.ch/mcm/restapi/requests/
single_request_prepid = 'HIG-RunIISummer15wmLHEGS-02135'
single_request = mcm.get('requests', single_request_prepid, method='get')
#print('Single request "%s":\n%s' % (single_request_prepid, single_request['generators']))#dumps(single_request, indent=4)))
for key in single_request:
print key
## Example how to get multiple requests using range
#requests_query = """
# B2G-Fall13-00001
# B2G-Fall13-00005 -> B2G-Fall13-00015
#"""
#range_of_requests = mcm.get_range_of_requests(requests_query)
#print('Found %s requests' % (len(range_of_requests)))
#for request in range_of_requests:
# print(request['prepid'])
# print(root_requests_from_ticket(ticket))
prepid = []
for rr in root_requests_from_ticket(ticket):
if 'GS' in rr or 'wmLHE' in rr or 'pLHE' in rr:
prepid.append(rr)
prepid = list(
set(prepid)
) #to avoid requests appearing x times if x chains have the same request
for x in range(0, len(prepid)):
print(prepid[x])
for num in range(0, len(prepid)):
query_str = 'prepid=' + prepid[num]
res = mcm.get('requests', query=query_str)
if len(res) == 0:
print "***********************************************************************************"
print "Something's wrong - can not get the request parameters"
print "***********************************************************************************"
my_path = '/tmp/' + os.environ['USER'] + '/gridpacks/'
print ""
print "***********************************************************************************"
for r in res:
pi = r['prepid']
dn = r['dataset_name']
te = r['time_event']
totalevents = r['total_events']
cmssw = r['cmssw_release']
mcm = McM(dev=False)
# Script clones a request to other campaign.
# Fefine list of modifications
# If member_of_campaign is different, it will clone to other campaign
# Get a request object which we want to clone
#chained_requests = mcm.get('chained_requests', 'B2G-chain_RunIISummer15wmLHEGS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3-00208')
dest_chained_campaign = 'chain_RunIISummer15wmLHEGS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3'
#dest_chained_campaign = 'chain_RunIIWinter15pLHE_flowLHE2Summer15GS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3'
#dest_chained_campaign = 'chain_RunIISummer15GS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3'
#dest_chained_campaign = 'chain_RunIIWinter15wmLHE_flowLHE2Summer15GS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3'
chained_requests = mcm.get('chained_requests',
query='member_of_campaign=%s&pwg=HIG' %
(dest_chained_campaign))
# Make predefined modifications
for chained_request in chained_requests:
if (len(chained_request['chain']) < 4):
continue
root_id = chained_request['chain'][0]
dr_id = chained_request['chain'][1]
miniAOD_id = chained_request['chain'][2]
nanoAOD_id = chained_request['chain'][3]
},
}
#write the prepids of the new cloned requests to a txt file
f = open('new_requests.txt', 'w')
for model in models:
if 'mother_request' in models[model].keys():
if models[model]['mother_request'] != '':
print('Cloning and modifying {0}'.format(
models[model]['mother_request']))
# Get a request object which we want to clone
request = mcm.get('requests', models[model]['mother_request'])
#if not request.get('results') :
# print ' Request', models[model]['mother_request'], 'not found'
# continue
#print request['generator_parameters'][0]['match_efficiency']
model_mother = ctau_mother = mLSP_mother = massLSP_mother = ''
datasetname = request['dataset_name'].split('_')
for datastring in datasetname:
if 'SMS-T' in datastring:
model_mother = datastring.replace('SMS-', '')
elif 'ctau' in datastring:
ctau_mother = datastring
elif 'mLSP' in datastring:
import sys
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
from json import dumps
from subprocess import Popen, PIPE
mcm = McM(dev=False)
dest_output = sys.argv[1]
requests = mcm.get('requests', query='produce=%s' % (dest_output))
for request in requests:
print request['prepid']
chained_requests = request['member_of_chain']
chained_request = mcm.get('chained_requests', chained_requests[0])
root_id = chained_request['chain'][0]
root_id_req = mcm.get('requests', root_id)
prepid = root_id_req['prepid']
print prepid
file_to_download = 'https://cms-pdmv.cern.ch/mcm/public/restapi/requests/get_test/'+str(prepid)
script = ("wget "+ str(file_to_download))
script += ("; mv "+ str(prepid) +" script.sh")
import sys
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
sys.stdout.flush()
mcm = McM(dev=False)
# Example to edit a request parameter(-s) and save it back in McM
# request_prepid_to_update = 'HIG-Summer12-01257' # Doesn't exist
#requests = mcm.get('requests', query='priority=110000&status=submitted&prepid=*Autumn18DR*')
#requests = mcm.get('requests', query='status=submitted&tags=Summer16MiniAODv3T2sub2')
#requests = mcm.get('requests', query='status=submitted&member_of_campaign=*MiniAODv3&prepid=HIG*')
requests = mcm.get('requests',
query='member_of_campaign=*MiniAODv3&prepid=HIG*')
#requests = mcm.get('requests', query='status=submitted&tags=Summer16MiniAODv3T4')
#requests = mcm.get('requests', query='tags=Summer16MiniAODv3T2')
#requests = mcm.get('requests', query='tags=Autumn18P1POGDR')
i = 0
for request in requests:
i += 1
print i
request_prepid_to_update = request['prepid']
request_prepid_to_update_req = mcm.get('requests',
request_prepid_to_update)
)
mcm = McM(dev=False)
output_file = open('TWIKI/{campaign}.txt'.format(campaign=args.campaign), 'w')
#print table header
#output_file.write('---+++ Current EGM samples for {campaign} campaign\n'.format(campaign = args.campaign))
output_file.write('---+++ {campaign}\n'.format(campaign=args.campaign))
output_file.write('\n')
output_file.write(
'| *Sample Name* | *status* | *Monitoring* | *Total Events* | *pileup* | *GT* | *CMSSW release* | *data tier* | *chain* |\n'
)
#find request in campaign and print each entry
foundRequests = mcm.get('requests',
query='pwg=EGM&member_of_campaign={campaign}'.format(
campaign=args.campaign),
method='get')
if foundRequests != None:
for foundRequest in foundRequests:
for chain in range(0, len(foundRequest['member_of_chain'])):
output_file.write(
'| {dataset_name} | {status} | {monitoring} | {tot_events} M | {pu_scheme} | {GT} | {cmssw_release} | {datatier} | {chain} |\n'
.format(
dataset_name=foundRequest['dataset_name'],
status=colorStatus(foundRequest['status']),
monitoring=tinyURL(
'https://dmytro.web.cern.ch/dmytro/cmsprodmon/workflows.php?prep_id=task_{0}'
.format(foundRequest['prepid']),
foundRequest['prepid']),
tot_events=float(foundRequest['total_events']) / 1000000.,
pu_scheme=find_puScheme(
das_cmd = "/cvmfs/cms.cern.ch/common/dasgoclient"
# if mcm is specified, retrieve dataset name from prepID:
if mcm:
if "/" in str(args.inputdataset):
print "not a McM prepID format, please check"
sys.exit(1)
# load McM
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
mcm = McM()
# retrieve request with given prepid
temp = sys.stdout
f = open('/dev/null', 'w')
sys.stdout = f
request = mcm.get('requests', str(args.inputdataset))
sys.stdout = temp
if debug: print 'request prepid',request['prepid']
# search dataset name as returned by mcm
dataset_used = str(request['output_dataset'][0])
primary_dataset_name = dataset_used.split('/')[1]
else:
# search dataset name as name + campaign + datatier
primary_dataset_name = args.inputdataset.split('/')[1]
command=das_cmd+" --limit=0 --query=\"dataset dataset=/"+primary_dataset_name+"/*"+args.campaign+"*/"+args.datatier+"\""
dataset_used = commands.getstatusoutput(command)[1].split("\n")
if debug: print 'command',command,'\n'
dataset_used = [x.strip() for x in dataset_used][0]
if skipexisting and os.path.isfile("xsec_"+primary_dataset_name+".log"):
print "xsec_"+primary_dataset_name+".log existing and NO skipexisting asked, skipping"
import sys
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
mcm = McM(dev=False)
# Example to edit a request parameter(-s) and save it back in McM
# request_prepid_to_update = 'HIG-Summer12-01257' # Doesn't exist
for i in range(0, 20):
request_prepid_to_update = 'EGM-RunIIFall18GS-000' + str(i)
field_to_update = 'tags'
# get a the dictionnary of a request
request = mcm.get('requests', request_prepid_to_update)
if 'prepid' not in request:
# In case the request doesn't exist, there is nothing to update
print('Request "%s" doesn\'t exist' % (request_prepid_to_update))
else:
print('Request\'s "%s" field "%s" BEFORE update: %s' %
(request_prepid_to_update, field_to_update,
request[field_to_update]))
# Modify what we want
# time_event is a list for each sequence step
request[field_to_update] = ["Fall18P1POGGS"]
# Push it back to McM
update_response = mcm.update('requests', request)
print('Update response: %s' % (update_response))
# NEED this to be sourced before
# export PYTHONPATH=/afs/cern.ch/cms/PPD/PdmV/tools/wmcontrol:${PYTHONPATH}
# export PATH=/afs/cern.ch/cms/PPD/PdmV/tools/wmcontrol:${PATH}
# source /afs/cern.ch/cms/PPD/PdmV/tools/wmclient/current/etc/wmclient.sh
import os
import sys
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
mcm = McM(dev=True)
requests = mcm.get('requests', query='tags=M17p1A')
print('Found %s requests' % (len(requests)))
for request in requests:
if len(request['reqmgr_name']) > 0:
# We change priority only if request has a registered workflow
# Remove echo command to acutally execute it
# Change priority to 90000
result = os.system("echo 'wmpriority.py %s %s'" %
(request['reqmgr_name'][-1]['name'], 90000))
if result != 0:
print('Change of priority failed for: %s. Exit code: %s' %
(request['prepid'], result))
else:
print('Workflow is not registered for %s' % (request['prepid']))
if len(member_of_chains) <= 0:
print('%s is member of none or one chain. Skipping' %
(request['prepid']))
continue
#print('Will leave %s' % (member_of_chains[-1]))
# Leave the last (with biggest number) chain
#member_of_chains = member_of_chains[:-1]
print('Will delete:')
for chain_prepid in member_of_chains:
print(' %s' % (chain_prepid))
for chain_prepid in member_of_chains:
print('Deleting %s' % (chain_prepid))
chain = mcm.get('chained_requests', chain_prepid)
chain['action_parameters']['flag'] = False
mcm.update('chained_requests', chain)
requests_in_chain = mcm.get('requests',
query='member_of_chain=%s' %
(chain_prepid))
requests_inverted = []
for request_in_chain in requests_in_chain:
requests_inverted.append(request_in_chain)
requests_inverted = list(reversed(requests_inverted))
for request_in_chain in requests_inverted:
'task_EXO-RunIISummer15wmLHEGS-06075', 'task_HIG-RunIIFall17wmLHEGS-01609',
'task_EXO-RunIISummer15wmLHEGS-06052', 'task_EXO-RunIIFall17wmLHEGS-00201',
'task_EXO-RunIIFall17GS-00231', 'task_EXO-RunIISummer15GS-11264',
'task_B2G-RunIISummer15wmLHEGS-01650'
]
print(
' Prepid Dataset name Completed events Total events Link to MCM'
)
for i in List:
name = i.replace('task_', '')
#split the strings
request = mcm.get('requests', query='prepid=' + str(name))
chained_requests = request[0]['member_of_chain']
for j in range(0, len(chained_requests)):
chained_request = mcm.get('chained_requests', chained_requests[j])
if (len(chained_request['chain']) > 2):
root_id = chained_request['chain'][2]
root_id_req = mcm.get('requests', root_id)
print(
str(root_id_req['prepid']) + ' ' +
str(root_id_req['dataset_name']) + ' ' +
str(root_id_req['completed_events']) + ' ' +
str(root_id_req['total_events']) +
#mcm = McM(dev=True,cookie='/afs/cern.ch/user/p/pgunnell/private/prod-cookie.txt')
mcm = McM(dev=False)
# Script clones a request to other campaign.
# Fefine list of modifications
# If member_of_campaign is different, it will clone to other campaign
# Get a request object which we want to clone
#chained_requests = mcm.get('chained_requests', 'B2G-chain_RunIISummer15wmLHEGS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3-00208')
dest_chained_campaign = 'RunIISummer16MiniAODv3'
#requests = mcm.get('requests', query='member_of_campaign=%s&status=new&tags=Summer16MiniAODv3T1' % (dest_chained_campaign))
requests = mcm.get(
'requests',
query='member_of_campaign=%s&status=new&tags=Summer16MiniAODv3T4' %
(dest_chained_campaign))
#requests = mcm.get('requests', query='prepid=B2G*&member_of_campaign=%s&approval=submit&status=approved&tags=Summer16MiniAODv3T3' % (dest_chained_campaign))
print len(requests)
i = 0
for request in requests:
#answer = mcm._McM__get('restapi/requests/approve/%s' % (request['prepid']))
#continue
i += 1
from json import dumps
#mcm = McM(dev=True,cookie='/afs/cern.ch/user/p/pgunnell/private/prod-cookie.txt')
mcm = McM(dev=False)
# Script clones a request to other campaign.
# Fefine list of modifications
# If member_of_campaign is different, it will clone to other campaign
# Get a request object which we want to clone
#chained_requests = mcm.get('chained_requests', 'B2G-chain_RunIISummer15wmLHEGS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3-00208')
dest_chained_campaign = 'RunIIFall17DRPremix'
requests = mcm.get('requests',
query='member_of_campaign=%s&status=approved' %
(dest_chained_campaign))
# Make predefined modifications
#print('Number of requests '+str(len(requests)))
print 'Dataset name PrepID root request Link to PrepID root request'
for request in requests:
chained_request = request['member_of_chain']
chained_request = mcm.get('chained_requests', chained_request[0])
root_id = chained_request['chain'][0]
root_id_req = mcm.get('requests', root_id)
import sys
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
from json import dumps
mcm = McM(dev=True)
# Script clones a request to other campaign.
# Fefine list of modifications
# If member_of_campaign is different, it will clone to other campaign
modifications = {'extension': 1,
'total_events': 101,
'member_of_campaign': 'Summer12'}
request_prepid_to_clone = "SUS-RunIIWinter15wmLHE-00040"
# Get a request object which we want to clone
request = mcm.get('requests', request_prepid_to_clone)
# Make predefined modifications
for key in modifications:
request[key] = modifications[key]
clone_answer = mcm.clone_request(request)
if clone_answer.get('results'):
print('Clone PrepID: %s' % (clone_answer['prepid']))
else:
print('Something went wrong while cloning a request. %s' % (dumps(clone_answer)))
"EXO-RunIIFall17GS-01245", "EXO-RunIIFall17GS-01246",
"EXO-RunIIFall17GS-01247", "EXO-RunIIFall17GS-01248",
"EXO-RunIIFall17GS-01249", "EXO-RunIIFall17GS-01250",
"EXO-RunIIFall17GS-01251", "EXO-RunIIFall17GS-01252",
"EXO-RunIIFall17GS-01253", "EXO-RunIIFall17GS-01254",
"EXO-RunIIFall17GS-01255", "EXO-RunIIFall17GS-01256",
"EXO-RunIIFall17GS-01257", "EXO-RunIIFall17GS-01258",
"EXO-RunIIFall17GS-01259", "EXO-RunIIFall17GS-01260",
"EXO-RunIIFall17GS-01261", "EXO-RunIIFall17GS-01262",
"EXO-RunIIFall17GS-01263", "EXO-RunIIFall17GS-01264",
"EXO-RunIIFall17GS-01266", "EXO-RunIIFall17GS-01267",
"EXO-RunIIFall17GS-01268", "EXO-RunIIFall17GS-01269"
]
for input_prepid in input_prepids:
request = mcm.get('requests', input_prepid)
# Make predefined modifications
for key in modifications:
request[key] = modifications[key]
frag17 = request["fragment"]
if "PSweights" in frag17:
continue
frag18 = frag17.replace("from Configuration.Generator.MCTunes2017.PythiaCP5Settings_cfi import *\n",
"from Configuration.Generator.MCTunes2017.PythiaCP5Settings_cfi import *\nfrom Configuration.Generator.PSweightsPythia.PythiaPSweightsSettings_cfi import *\n")\
.replace("pythia8CP5SettingsBlock,\n",
"pythia8CP5SettingsBlock,\n pythia8PSweightsSettingsBlock,\n")\
.replace("\'pythia8CP5Settings\',\n",
"\'pythia8CP5Settings\',\n \'pythia8PSweightsSettings\',\n")
request["fragment"] = frag18
import sys
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
sys.stdout.flush()
mcm = McM(dev=False)
# Example to edit a request parameter(-s) and save it back in McM
# request_prepid_to_update = 'HIG-Summer12-01257' # Doesn't exist
#requests = mcm.get('requests', query='priority=110000&status=submitted&prepid=*Autumn18DR*')
#requests = mcm.get('requests', query='tags=Autumn18POGP1DR&&priority=110000&status=submitted&prepid=MUO*')
requests = mcm.get(
'requests', query='prepid=*Autumn18DR*&&priority=85000&status=submitted')
#requests = mcm.get('requests', query='tags=Autumn18P1POGDR')
tot_events = 0
for request in requests:
chained_requests = request['member_of_chain']
chained_request = mcm.get('chained_requests', chained_requests[0])
tot_events += request['total_events']
root_id = chained_request['chain'][0]
root_id_req = mcm.get('requests', root_id)
chained_rootid = root_id_req['member_of_chain']
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
from json import dumps
mcm = McM(dev=False,
cookie='/afs/cern.ch/user/p/pgunnell/private/prod-cookie.txt')
# Script clones a request to other campaign.
# Fefine list of modifications
# If member_of_campaign is different, it will clone to other campaign
list_of_nonflowing = "list_of_forceflow"
# Get a request object which we want to clone
chained_requests = mcm.get('lists', list_of_nonflowing, method='get')
chained_requests = list(reversed(chained_requests['value']))
print chained_requests
# Make predefined modifications
print len(chained_requests)
for i in range(0, len(chained_requests)):
if (chained_requests[i].find('MiniAODv3') != -1):
print chained_requests[i]
chained_request = mcm.get('chained_requests', chained_requests[i])
ochain = 'chain_RunIIFall17GS_flowRunIIFall17DRPremixPU2017_flowRunIIFall17MiniAODv2_flowRunIIFall17NanoAOD'
dchain = 'chain_RunIIFall17GS_flowRunIIFall17DRPremixPU2017_flowRunIIFall17MiniAODv2_flowRunIIFall17NanoAODv4'
ticketfilename = dchain + '.json'
collector = defaultdict(lambda: defaultdict(lambda: defaultdict(int)))
#for cc in ccs:
print 50 * "-"
listprep = ['TOP']
for pwg in listprep:
#if pwg != "TOP": continue
## get all chains from that pwg in that chained campaign
#crs = mcm.getA('chained_requests', query='member_of_campaign=%s&pwg=%s'%(cc['prepid'],pwg))
crs = mcm.get('chained_requests',
query='member_of_campaign=%s&pwg=%s' % (ochain, pwg))
print "\t", pwg, ":\t", len(crs)
for cr in crs:
#print "\t\t",cr['prepid'], cr['chain']
root_id = cr['chain'][0]
#print "\t\t",root_id
#chainchecks = mcm.getA('chained_requests',query='contains=%s'%(root_id))
#print chainchecks
campaign = root_id.split('-')[1]
collector[pwg][campaign][root_id] += 1
print "This is the list of %s requests that are deemed chainable: " % (
listprep)
#pprint.pprint( dict(collector) )
#print collector
all_ticket = []
#pwgs = ['MUO','BTV','JME','TAU','EGM','TSG','SMP','HCA','B2G']
pwgs = ['SUS']
#pwgs = ['SUS','TOP','HIG']
#pwgs = ['BPH']
if not pwgs:
# Get list of PWGs
pwgs = mcm._McM__get('restapi/users/get_pwg')['results']
print('Got PWGs: %s' % (pwgs))
# Create a dict of dicts of dicts of integers
collector = defaultdict(lambda: defaultdict(lambda: defaultdict(int)))
for pwg in pwgs:
chained_requests = mcm.get('chained_requests',
query='member_of_campaign=%s&pwg=%s' %
(org_chained_campaign, pwg))
print('PWG:%s\tChained requests:%s' % (pwg, len(chained_requests)))
for chained_request in chained_requests:
root_id = chained_request['chain'][0]
campaign = root_id.split('-')[1]
existing_chains = mcm.get('chained_requests',
query='contains=%s&pwg=%s' % (root_id, pwg))
chain_present = False
for existing_chain in existing_chains:
if not devMode:
answer = None
while answer not in ["y", "n"]:
answer = raw_input("Not in dev mode, ok to continue [Y/N]? ").lower()
if not answer == 'y':
sys.exit()
os.system('source /afs/cern.ch/cms/PPD/PdmV/tools/McM/getCookie.sh')
os.system(
'cern-get-sso-cookie -u https://cms-pdmv.cern.ch/mcm/ -o ~/private/prod-cookie.txt --krb --reprocess'
)
mcm = McM(dev=devMode)
for req_prepid in prepidToSearch:
request = mcm.get('requests',
query='prepid={0}'.format(req_prepid),
method='get')
if len(request) == 1:
request = request[0]
if request['approval'] == 'none' or (
request['approval'] == 'validation'
and request['status'] == 'validation'):
print ''
print 'I will trigger validation of {0}'.format(request['prepid'])
print '-- request : ', request['dataset_name']
print '-- link : https://cms-pdmv.cern.ch/mcm/requests?prepid={0}'.format(
request['prepid'])
mcm.approve('requests', request['prepid'])
print ''
else:
print ''
#export X509_USER_PROXY=$(voms-proxy-info --path)
#export PYTHONPATH=/afs/cern.ch/cms/PPD/PdmV/tools/wmcontrol:${PYTHONPATH}
#export PATH=/afs/cern.ch/cms/PPD/PdmV/tools/wmcontrol:${PATH}
# source /afs/cern.ch/cms/PPD/PdmV/tools/wmclient/current/etc/wmclient.sh
import os
import sys
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
mcm = McM(dev=False)
#requests = mcm.get('requests', query='tags=Summer16MiniAODv3T2sub1&priority=90000')
#requests = mcm.get('requests', query='tags=Summer16MiniAODv3T2sub2&priority=90000')
#requests = mcm.get('requests', query='priority=85000&status=submitted&prepid=*Autumn18DR*')
requests = mcm.get('requests',
query='status=submitted&tags=PAGLHCP2019&priority=85000')
#requests = mcm.get('requests', query='prepid=EXO-RunIIFall17GS-009*&dataset_name=Mustar*')
#requests = mcm.get('requests', query='status=submitted&tags=Summer16MiniAODv3T3')
#requests = mcm.get('requests', query='status=submitted&prepid=HIG-PhaseIIMTDTDRAutumn18wmLHEGS-0000*')
#requests = mcm.get('requests', query='status=submitted&dataset_name=VBF_BulkGravToWW_narrow_M-*')
#requests = mcm.get('requests', query='status=submitted&prepid=SMP-*LowPU*GS*')
#requests = mcm.get('requests', query='prepid=BPH-RunIIFall18GS-0006*')
print('Found %s requests' % (len(requests)))
for request in requests:
if len(request['reqmgr_name']) > 0:
# We change priority only if request has a registered workflow
# Remove echo command to acutally execute it -> already removed
# Change priority to 200000
result = os.system("wmpriority.py %s %s" %
list0 = list('SUS-RunIIFall17GS-000' + str(i) for i in range(xxx, yyy + 1))
for req in list0:
requests_to_clone.append(req)
#write the prepids of the new cloned requests to a txt file
f = open('new_requests.txt', 'w')
# If member_of_campaign is different, it will clone to other campaign
campaign_modifications = {'member_of_campaign': 'RunIIFall18GS'}
for request_prepid_to_clone in requests_to_clone:
print('Cloning and modifying {0}'.format(request_prepid_to_clone))
# Get a request object which we want to clone
request = mcm.get('requests', request_prepid_to_clone)
# Make predefined modifications
for key in campaign_modifications:
request[key] = campaign_modifications[key]
#clone request and print error if it fails
clone_answer = mcm.clone_request(request)
if clone_answer.get('results'):
pass
else:
print('Something went wrong while cloning a request. %s' %
(dumps(clone_answer)))
#write new prepid to file
f.write(clone_answer['prepid'] + '\n')
print "This is the list of %s requests that are deemed chainable: " % (
listprep)
all_tickets = []
campaign_name = "RunIISummer16MiniAODv3"
# Get requests that are member of given campaign
requests_for_that_repeat = []
for pwg in listprep:
page = 0
requests = [{}]
while len(requests) > 0:
if (campaigntochain == 'wmLHEGS'):
requests = mcm.get(
'requests',
query=
'status=done&member_of_campaign=%s&pwg=%s&member_of_chain=*wmLHEGS_flowRunIISummer16DR80PremixPUMoriond17_*'
% (campaign_name, pwg),
page=page)
page += 1
elif (campaigntochain == 'GS'):
requests = mcm.get(
'requests',
query=
'status=done&member_of_campaign=%s&pwg=%s&member_of_chain=*chain_RunIISummer15GS_flowRunIISummer16DR80PremixPUMoriond17_*'
% (campaign_name, pwg),
page=page)
page += 1
elif (campaigntochain == 'pLHE'):
requests = mcm.get(
