def GetTargetStepFromDAS(TargetStep, DAS): mcm = McM(dev=False) input_request = mcm.get("requests", query="produce=" + DAS) member_of_chain = input_request[0]['member_of_chain'] mychain = '' for chain in member_of_chain: if TargetStep in chain: mychain = chain chained_request = mcm.get('chained_requests', mychain) req = '' for member in chained_request['chain']: if TargetStep in member: req = member return req
def clone_and_edit_jhchoi(request_prepid_to_clone, fragment, dataset_name, notes, total_events, generators): mcm = McM(dev=False) modifications = { 'fragment': fragment, 'dataset_name': dataset_name, 'notes': notes, 'total_events': total_events, "generators": generators } #request_prepid_to_clone = "SUS-RunIIWinter15wmLHE-00040" request = mcm.get('requests', request_prepid_to_clone) for key in modifications: request[key] = modifications[key] clone_answer = mcm.clone_request(request) if clone_answer.get('results'): print "@@" new_prepid = clone_answer['prepid'] print('Clone PrepID: %s' % (clone_answer['prepid'])) new_request = mcm.get('requests', new_prepid) print new_request['dataset_name'] mcm.approve('requests', new_prepid, None) print "---" f = open('success.txt', 'a') f.write(new_prepid + "\n") f.close() else: print('Something went wrong while cloning a request. %s' % (dumps(clone_answer))) f = open('fail.txt', 'a') f.write(dataset_name + "\n") f.close()
def createrequest(self, clonequeue): self.needsupdate = True return clonequeue.add(self, self.pwg, self.newcampaign) if jobtype(): return "run locally to submit to McM" mcm = McM() clone_req = mcm.get('requests', self.originalprepid) clone_req['member_of_campaign'] = self.campaign answer = mcm.clone(self.originalprepid, clone_req) if not (answer and answer.get("results")): raise RuntimeError("Failed to create the request on McM\n{}\n{}".format(self, answer)) self.getprepid() if self.prepid != answer["prepid"]: raise RuntimeError("Wrong prepid?? {} {}".format(self.prepid, answer["prepid"])) self.updaterequest() return "cloned request "+self.originalprepid+" as "+self.prepid+" on McM"
# print(root_requests_from_ticket(ticket)) prepid = [] for rr in root_requests_from_ticket(ticket): if 'GS' in rr or 'wmLHE' in rr or 'pLHE' in rr: prepid.append(rr) prepid = list(set(prepid)) #to avoid requests appearing x times if x chains have the same request for x in range(0,len(prepid)): print(prepid[x]) for num in range(0,len(prepid)): query_str = 'prepid='+prepid[num] res = mcm.get('requests', query=query_str) if len(res) == 0 : print "***********************************************************************************" print "Something's wrong - can not get the request parameters" print "***********************************************************************************" my_path = '/tmp/'+os.environ['USER']+'/gridpacks/' print "" print "***********************************************************************************" for r in res: pi = r['prepid'] dn = r['dataset_name'] te = r['time_event'] totalevents = r['total_events'] cmssw = r['cmssw_release']
modifiedPrepid = prepidToClone modifiedPrepid['interested_pwg'] = ['EGM'] modifiedPrepid['member_of_campaign'] = campaign modifiedPrepid['ppd_tags'] = tag print 'Will clone to ', campaign, 'with tag', tag cloned_prepid = mcm.clone_request(modifiedPrepid) return mcm.get('requests', cloned_prepid['prepid'], method='get') for UL16PrepidName in UL16PrepidsToClone: print '\n---> Cloning', UL16PrepidName if not ('EGM-RunIISummer19UL16GEN' or 'EGM-RunIISummer19UL16wmLHEGEN' in UL16PrepidName): print 'This does not look like a UL16 prepid, skipping :', UL16PrepidName continue # Get prepid from McM, check it exists UL16PrepidFromMcM = mcm.get('requests', UL16PrepidName, method='get') if len(UL16PrepidFromMcM) == 0: print 'This prepid does not exist in McM :', UL16PrepidFromMcM print 'EXITING' sys.exit() # Will clone to the 2016 UL campaign UL16APVPrepid = cloneToUL16PreVFP(prepidToClone=UL16PrepidFromMcM) if not UL16APVPrepid: print 'Dataset already existed in 2016 UL PreVFP campaign' else: print 'Cloned prepid to: %s' % UL16APVPrepid['prepid']
dry_run = False # McM instance mcm = McM(dev=True)#, cookie='/afs/cern.ch/work/s/snorberg/miniAODv3/cookie.txt')#/afs/cern.ch/user/j/jrumsevi/private/dev_cookie.txt') # Set list of PWGs here. If no list is set, all PWGs from McM will be used pwgs = ['SMP'] if not pwgs: # Get list of PWGs pwgs = mcm._McM__get('restapi/users/get_pwg')['results'] print('Got PWGs: %s' % (pwgs)) # Create a dict of dicts of dicts of integers collector = defaultdict(lambda: defaultdict(lambda: defaultdict(int))) for pwg in pwgs: chained_requests = mcm.get('chained_requests', query='member_of_campaign=%s&pwg=%s' % (org_chained_campaign, pwg)) # print('PWG:%s\tChained requests:%s' % (pwg, len(chained_requests))) for chained_request in chained_requests: root_id = chained_request['chain'][0] campaign = root_id.split('-')[1] collector[pwg][campaign][root_id] += 1 # Collector now contains all root requests of chains that are member of org_chained_campaign campaign # It's structured like this: PWG -> campaign name of root request -> request prepid -> count # print('This is the list of %s requests that are deemed chainable:\n%s' % (pwgs, json.dumps(dict(collector), indent=4))) all_tickets = [] chained_campaign = mcm.get('chained_campaigns', dest_chained_campaign, method='get') for pwg in pwgs: # Create a ticket for the correct chain chained_campaign_prepid = chained_campaign['prepid']
def Run(idx_row): output_msg = '' KEY = '1bAdzoB9bg6nbyxCOBsTAWSSJy6FHNfmkrIDqY-Nmdi0' sheet = '2016' title_list = GetTitleFromSpreadsheet(KEY, sheet, i_column="A", f_column="Z") info_list = GetInfoFromSpreadsheet(idx_row, KEY, sheet, i_column="A", f_column="Z") this_dic = {} for i in range(0, len(title_list)): title = title_list[i] #print title value = info_list[i] #print value this_dic[title] = value FullFragment = MakeFullFragment(this_dic["gridpackPATH"], this_dic["PSfragment"], this_dic["card_ref"]) generator_parameters=Make_generator_parameters(this_dic["cross_section"],\ this_dic["filter_efficiency"],\ this_dic["filter_efficiency_error"],\ this_dic["match_efficiency"],\ this_dic["match_efficiency_error"],\ this_dic["negative_weights_fraction"]) print this_dic['dataset_name'] new_request = { 'pwg': 'HIG', 'member_of_campaign': this_dic['member_of_campaign'], 'dataset_name': this_dic['dataset_name'], 'generators': this_dic['generators'].split(','), "fragment": FullFragment, 'generator_parameters': generator_parameters, 'total_events': int(this_dic['total_events']), 'mcdb_id': 0, 'size_event': [600], 'time_event': [60.] } if this_dic["PrepID"] != '': #print "ALREADY" mcm = McM(dev=False) this_request = mcm.get("requests", this_dic["PrepID"]) for key in new_request: this_request[key] = new_request[key] update = mcm.update('requests', this_request) return str(update) # push it to McM #mcm = McM(dev=False,cookie="dev-cookie.txt") #mcm = McM(dev=False,cookie='prod-cookie.txt') mcm = McM(dev=False) put_answer = mcm.put('requests', new_request) if put_answer.get('results'): prepid = put_answer['prepid'] print('New PrepID: %s' % (prepid)) #f.write(prepid+'\n') output_msg = str(prepid) else: print('Something went wrong while creating a request. %s' % (dumps(put_answer))) #f.write("fail,idx_row-->"+this_dic['dataset_name']+'\n') output_msg = "fail" + str(idx_row) + "-->" + this_dic['dataset_name'] #print output_msg return output_msg
import sys sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import McM sys.stdout.flush() mcm = McM(dev=False) # Example to edit a request parameter(-s) and save it back in McM # request_prepid_to_update = 'HIG-Summer12-01257' # Doesn't exist #requests = mcm.get('requests', query='priority=110000&status=submitted&prepid=*Autumn18DR*') #requests = mcm.get('requests', query='status=submitted&prepid=*MTDTDR*GS*') requests = mcm.get('requests', query='status=submitted&tags=MTDTDRAutumn18PU200') #requests = mcm.get('requests', query='tags=Summer16MiniAODv3T2') #requests = mcm.get('requests', query='tags=Autumn18P1POGDR') i = 0 for request in requests: chained_request_id = request['member_of_chain'][0] chained_requests = mcm.get('chained_requests', query='prepid=%s' % (chained_request_id)) if (len(chained_requests[0]['chain']) < 3): continue if ('PU0' in chained_requests[0]['prepid']):
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import McM from json import dumps mcm = McM(dev=False) # Example to get ALL requesst which are member of a given campaign and are submitted # It uses a generic search for specified columns: query='status=submitted' # Queries can be combined: query='status=submitted&member_of_campaign=Summer12' #campaign_requests = mcm.get('requests', query='dataset_name=NMSSMCascade_mH-*_mSUSY-*_TuneCUEP8M1_13TeV-madgraph-pythia8') #for request in campaign_requests: # print(request['prepid']) # Example to retrieve single request dictionary # More methods are here: # https://cms-pdmv.cern.ch/mcm/restapi/requests/ single_request_prepid = 'HIG-RunIISummer15wmLHEGS-02135' single_request = mcm.get('requests', single_request_prepid, method='get') #print('Single request "%s":\n%s' % (single_request_prepid, single_request['generators']))#dumps(single_request, indent=4))) for key in single_request: print key ## Example how to get multiple requests using range #requests_query = """ # B2G-Fall13-00001 # B2G-Fall13-00005 -> B2G-Fall13-00015 #""" #range_of_requests = mcm.get_range_of_requests(requests_query) #print('Found %s requests' % (len(range_of_requests))) #for request in range_of_requests: # print(request['prepid'])
# print(root_requests_from_ticket(ticket)) prepid = [] for rr in root_requests_from_ticket(ticket): if 'GS' in rr or 'wmLHE' in rr or 'pLHE' in rr: prepid.append(rr) prepid = list( set(prepid) ) #to avoid requests appearing x times if x chains have the same request for x in range(0, len(prepid)): print(prepid[x]) for num in range(0, len(prepid)): query_str = 'prepid=' + prepid[num] res = mcm.get('requests', query=query_str) if len(res) == 0: print "***********************************************************************************" print "Something's wrong - can not get the request parameters" print "***********************************************************************************" my_path = '/tmp/' + os.environ['USER'] + '/gridpacks/' print "" print "***********************************************************************************" for r in res: pi = r['prepid'] dn = r['dataset_name'] te = r['time_event'] totalevents = r['total_events'] cmssw = r['cmssw_release']
mcm = McM(dev=False) # Script clones a request to other campaign. # Fefine list of modifications # If member_of_campaign is different, it will clone to other campaign # Get a request object which we want to clone #chained_requests = mcm.get('chained_requests', 'B2G-chain_RunIISummer15wmLHEGS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3-00208') dest_chained_campaign = 'chain_RunIISummer15wmLHEGS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3' #dest_chained_campaign = 'chain_RunIIWinter15pLHE_flowLHE2Summer15GS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3' #dest_chained_campaign = 'chain_RunIISummer15GS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3' #dest_chained_campaign = 'chain_RunIIWinter15wmLHE_flowLHE2Summer15GS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3' chained_requests = mcm.get('chained_requests', query='member_of_campaign=%s&pwg=HIG' % (dest_chained_campaign)) # Make predefined modifications for chained_request in chained_requests: if (len(chained_request['chain']) < 4): continue root_id = chained_request['chain'][0] dr_id = chained_request['chain'][1] miniAOD_id = chained_request['chain'][2] nanoAOD_id = chained_request['chain'][3]
}, } #write the prepids of the new cloned requests to a txt file f = open('new_requests.txt', 'w') for model in models: if 'mother_request' in models[model].keys(): if models[model]['mother_request'] != '': print('Cloning and modifying {0}'.format( models[model]['mother_request'])) # Get a request object which we want to clone request = mcm.get('requests', models[model]['mother_request']) #if not request.get('results') : # print ' Request', models[model]['mother_request'], 'not found' # continue #print request['generator_parameters'][0]['match_efficiency'] model_mother = ctau_mother = mLSP_mother = massLSP_mother = '' datasetname = request['dataset_name'].split('_') for datastring in datasetname: if 'SMS-T' in datastring: model_mother = datastring.replace('SMS-', '') elif 'ctau' in datastring: ctau_mother = datastring elif 'mLSP' in datastring:
import sys sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import McM from json import dumps from subprocess import Popen, PIPE mcm = McM(dev=False) dest_output = sys.argv[1] requests = mcm.get('requests', query='produce=%s' % (dest_output)) for request in requests: print request['prepid'] chained_requests = request['member_of_chain'] chained_request = mcm.get('chained_requests', chained_requests[0]) root_id = chained_request['chain'][0] root_id_req = mcm.get('requests', root_id) prepid = root_id_req['prepid'] print prepid file_to_download = 'https://cms-pdmv.cern.ch/mcm/public/restapi/requests/get_test/'+str(prepid) script = ("wget "+ str(file_to_download)) script += ("; mv "+ str(prepid) +" script.sh")
import sys sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import McM sys.stdout.flush() mcm = McM(dev=False) # Example to edit a request parameter(-s) and save it back in McM # request_prepid_to_update = 'HIG-Summer12-01257' # Doesn't exist #requests = mcm.get('requests', query='priority=110000&status=submitted&prepid=*Autumn18DR*') #requests = mcm.get('requests', query='status=submitted&tags=Summer16MiniAODv3T2sub2') #requests = mcm.get('requests', query='status=submitted&member_of_campaign=*MiniAODv3&prepid=HIG*') requests = mcm.get('requests', query='member_of_campaign=*MiniAODv3&prepid=HIG*') #requests = mcm.get('requests', query='status=submitted&tags=Summer16MiniAODv3T4') #requests = mcm.get('requests', query='tags=Summer16MiniAODv3T2') #requests = mcm.get('requests', query='tags=Autumn18P1POGDR') i = 0 for request in requests: i += 1 print i request_prepid_to_update = request['prepid'] request_prepid_to_update_req = mcm.get('requests', request_prepid_to_update)
) mcm = McM(dev=False) output_file = open('TWIKI/{campaign}.txt'.format(campaign=args.campaign), 'w') #print table header #output_file.write('---+++ Current EGM samples for {campaign} campaign\n'.format(campaign = args.campaign)) output_file.write('---+++ {campaign}\n'.format(campaign=args.campaign)) output_file.write('\n') output_file.write( '| *Sample Name* | *status* | *Monitoring* | *Total Events* | *pileup* | *GT* | *CMSSW release* | *data tier* | *chain* |\n' ) #find request in campaign and print each entry foundRequests = mcm.get('requests', query='pwg=EGM&member_of_campaign={campaign}'.format( campaign=args.campaign), method='get') if foundRequests != None: for foundRequest in foundRequests: for chain in range(0, len(foundRequest['member_of_chain'])): output_file.write( '| {dataset_name} | {status} | {monitoring} | {tot_events} M | {pu_scheme} | {GT} | {cmssw_release} | {datatier} | {chain} |\n' .format( dataset_name=foundRequest['dataset_name'], status=colorStatus(foundRequest['status']), monitoring=tinyURL( 'https://dmytro.web.cern.ch/dmytro/cmsprodmon/workflows.php?prep_id=task_{0}' .format(foundRequest['prepid']), foundRequest['prepid']), tot_events=float(foundRequest['total_events']) / 1000000., pu_scheme=find_puScheme(
das_cmd = "/cvmfs/cms.cern.ch/common/dasgoclient" # if mcm is specified, retrieve dataset name from prepID: if mcm: if "/" in str(args.inputdataset): print "not a McM prepID format, please check" sys.exit(1) # load McM sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import McM mcm = McM() # retrieve request with given prepid temp = sys.stdout f = open('/dev/null', 'w') sys.stdout = f request = mcm.get('requests', str(args.inputdataset)) sys.stdout = temp if debug: print 'request prepid',request['prepid'] # search dataset name as returned by mcm dataset_used = str(request['output_dataset'][0]) primary_dataset_name = dataset_used.split('/')[1] else: # search dataset name as name + campaign + datatier primary_dataset_name = args.inputdataset.split('/')[1] command=das_cmd+" --limit=0 --query=\"dataset dataset=/"+primary_dataset_name+"/*"+args.campaign+"*/"+args.datatier+"\"" dataset_used = commands.getstatusoutput(command)[1].split("\n") if debug: print 'command',command,'\n' dataset_used = [x.strip() for x in dataset_used][0] if skipexisting and os.path.isfile("xsec_"+primary_dataset_name+".log"): print "xsec_"+primary_dataset_name+".log existing and NO skipexisting asked, skipping"
import sys sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import McM mcm = McM(dev=False) # Example to edit a request parameter(-s) and save it back in McM # request_prepid_to_update = 'HIG-Summer12-01257' # Doesn't exist for i in range(0, 20): request_prepid_to_update = 'EGM-RunIIFall18GS-000' + str(i) field_to_update = 'tags' # get a the dictionnary of a request request = mcm.get('requests', request_prepid_to_update) if 'prepid' not in request: # In case the request doesn't exist, there is nothing to update print('Request "%s" doesn\'t exist' % (request_prepid_to_update)) else: print('Request\'s "%s" field "%s" BEFORE update: %s' % (request_prepid_to_update, field_to_update, request[field_to_update])) # Modify what we want # time_event is a list for each sequence step request[field_to_update] = ["Fall18P1POGGS"] # Push it back to McM update_response = mcm.update('requests', request) print('Update response: %s' % (update_response))
# NEED this to be sourced before # export PYTHONPATH=/afs/cern.ch/cms/PPD/PdmV/tools/wmcontrol:${PYTHONPATH} # export PATH=/afs/cern.ch/cms/PPD/PdmV/tools/wmcontrol:${PATH} # source /afs/cern.ch/cms/PPD/PdmV/tools/wmclient/current/etc/wmclient.sh import os import sys sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import McM mcm = McM(dev=True) requests = mcm.get('requests', query='tags=M17p1A') print('Found %s requests' % (len(requests))) for request in requests: if len(request['reqmgr_name']) > 0: # We change priority only if request has a registered workflow # Remove echo command to acutally execute it # Change priority to 90000 result = os.system("echo 'wmpriority.py %s %s'" % (request['reqmgr_name'][-1]['name'], 90000)) if result != 0: print('Change of priority failed for: %s. Exit code: %s' % (request['prepid'], result)) else: print('Workflow is not registered for %s' % (request['prepid']))
if len(member_of_chains) <= 0: print('%s is member of none or one chain. Skipping' % (request['prepid'])) continue #print('Will leave %s' % (member_of_chains[-1])) # Leave the last (with biggest number) chain #member_of_chains = member_of_chains[:-1] print('Will delete:') for chain_prepid in member_of_chains: print(' %s' % (chain_prepid)) for chain_prepid in member_of_chains: print('Deleting %s' % (chain_prepid)) chain = mcm.get('chained_requests', chain_prepid) chain['action_parameters']['flag'] = False mcm.update('chained_requests', chain) requests_in_chain = mcm.get('requests', query='member_of_chain=%s' % (chain_prepid)) requests_inverted = [] for request_in_chain in requests_in_chain: requests_inverted.append(request_in_chain) requests_inverted = list(reversed(requests_inverted)) for request_in_chain in requests_inverted:
'task_EXO-RunIISummer15wmLHEGS-06075', 'task_HIG-RunIIFall17wmLHEGS-01609', 'task_EXO-RunIISummer15wmLHEGS-06052', 'task_EXO-RunIIFall17wmLHEGS-00201', 'task_EXO-RunIIFall17GS-00231', 'task_EXO-RunIISummer15GS-11264', 'task_B2G-RunIISummer15wmLHEGS-01650' ] print( ' Prepid Dataset name Completed events Total events Link to MCM' ) for i in List: name = i.replace('task_', '') #split the strings request = mcm.get('requests', query='prepid=' + str(name)) chained_requests = request[0]['member_of_chain'] for j in range(0, len(chained_requests)): chained_request = mcm.get('chained_requests', chained_requests[j]) if (len(chained_request['chain']) > 2): root_id = chained_request['chain'][2] root_id_req = mcm.get('requests', root_id) print( str(root_id_req['prepid']) + ' ' + str(root_id_req['dataset_name']) + ' ' + str(root_id_req['completed_events']) + ' ' + str(root_id_req['total_events']) +
#mcm = McM(dev=True,cookie='/afs/cern.ch/user/p/pgunnell/private/prod-cookie.txt') mcm = McM(dev=False) # Script clones a request to other campaign. # Fefine list of modifications # If member_of_campaign is different, it will clone to other campaign # Get a request object which we want to clone #chained_requests = mcm.get('chained_requests', 'B2G-chain_RunIISummer15wmLHEGS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3-00208') dest_chained_campaign = 'RunIISummer16MiniAODv3' #requests = mcm.get('requests', query='member_of_campaign=%s&status=new&tags=Summer16MiniAODv3T1' % (dest_chained_campaign)) requests = mcm.get( 'requests', query='member_of_campaign=%s&status=new&tags=Summer16MiniAODv3T4' % (dest_chained_campaign)) #requests = mcm.get('requests', query='prepid=B2G*&member_of_campaign=%s&approval=submit&status=approved&tags=Summer16MiniAODv3T3' % (dest_chained_campaign)) print len(requests) i = 0 for request in requests: #answer = mcm._McM__get('restapi/requests/approve/%s' % (request['prepid'])) #continue i += 1
from json import dumps #mcm = McM(dev=True,cookie='/afs/cern.ch/user/p/pgunnell/private/prod-cookie.txt') mcm = McM(dev=False) # Script clones a request to other campaign. # Fefine list of modifications # If member_of_campaign is different, it will clone to other campaign # Get a request object which we want to clone #chained_requests = mcm.get('chained_requests', 'B2G-chain_RunIISummer15wmLHEGS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3-00208') dest_chained_campaign = 'RunIIFall17DRPremix' requests = mcm.get('requests', query='member_of_campaign=%s&status=approved' % (dest_chained_campaign)) # Make predefined modifications #print('Number of requests '+str(len(requests))) print 'Dataset name PrepID root request Link to PrepID root request' for request in requests: chained_request = request['member_of_chain'] chained_request = mcm.get('chained_requests', chained_request[0]) root_id = chained_request['chain'][0] root_id_req = mcm.get('requests', root_id)
import sys sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import McM from json import dumps mcm = McM(dev=True) # Script clones a request to other campaign. # Fefine list of modifications # If member_of_campaign is different, it will clone to other campaign modifications = {'extension': 1, 'total_events': 101, 'member_of_campaign': 'Summer12'} request_prepid_to_clone = "SUS-RunIIWinter15wmLHE-00040" # Get a request object which we want to clone request = mcm.get('requests', request_prepid_to_clone) # Make predefined modifications for key in modifications: request[key] = modifications[key] clone_answer = mcm.clone_request(request) if clone_answer.get('results'): print('Clone PrepID: %s' % (clone_answer['prepid'])) else: print('Something went wrong while cloning a request. %s' % (dumps(clone_answer)))
"EXO-RunIIFall17GS-01245", "EXO-RunIIFall17GS-01246", "EXO-RunIIFall17GS-01247", "EXO-RunIIFall17GS-01248", "EXO-RunIIFall17GS-01249", "EXO-RunIIFall17GS-01250", "EXO-RunIIFall17GS-01251", "EXO-RunIIFall17GS-01252", "EXO-RunIIFall17GS-01253", "EXO-RunIIFall17GS-01254", "EXO-RunIIFall17GS-01255", "EXO-RunIIFall17GS-01256", "EXO-RunIIFall17GS-01257", "EXO-RunIIFall17GS-01258", "EXO-RunIIFall17GS-01259", "EXO-RunIIFall17GS-01260", "EXO-RunIIFall17GS-01261", "EXO-RunIIFall17GS-01262", "EXO-RunIIFall17GS-01263", "EXO-RunIIFall17GS-01264", "EXO-RunIIFall17GS-01266", "EXO-RunIIFall17GS-01267", "EXO-RunIIFall17GS-01268", "EXO-RunIIFall17GS-01269" ] for input_prepid in input_prepids: request = mcm.get('requests', input_prepid) # Make predefined modifications for key in modifications: request[key] = modifications[key] frag17 = request["fragment"] if "PSweights" in frag17: continue frag18 = frag17.replace("from Configuration.Generator.MCTunes2017.PythiaCP5Settings_cfi import *\n", "from Configuration.Generator.MCTunes2017.PythiaCP5Settings_cfi import *\nfrom Configuration.Generator.PSweightsPythia.PythiaPSweightsSettings_cfi import *\n")\ .replace("pythia8CP5SettingsBlock,\n", "pythia8CP5SettingsBlock,\n pythia8PSweightsSettingsBlock,\n")\ .replace("\'pythia8CP5Settings\',\n", "\'pythia8CP5Settings\',\n \'pythia8PSweightsSettings\',\n") request["fragment"] = frag18
import sys sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import McM sys.stdout.flush() mcm = McM(dev=False) # Example to edit a request parameter(-s) and save it back in McM # request_prepid_to_update = 'HIG-Summer12-01257' # Doesn't exist #requests = mcm.get('requests', query='priority=110000&status=submitted&prepid=*Autumn18DR*') #requests = mcm.get('requests', query='tags=Autumn18POGP1DR&&priority=110000&status=submitted&prepid=MUO*') requests = mcm.get( 'requests', query='prepid=*Autumn18DR*&&priority=85000&status=submitted') #requests = mcm.get('requests', query='tags=Autumn18P1POGDR') tot_events = 0 for request in requests: chained_requests = request['member_of_chain'] chained_request = mcm.get('chained_requests', chained_requests[0]) tot_events += request['total_events'] root_id = chained_request['chain'][0] root_id_req = mcm.get('requests', root_id) chained_rootid = root_id_req['member_of_chain']
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import McM from json import dumps mcm = McM(dev=False, cookie='/afs/cern.ch/user/p/pgunnell/private/prod-cookie.txt') # Script clones a request to other campaign. # Fefine list of modifications # If member_of_campaign is different, it will clone to other campaign list_of_nonflowing = "list_of_forceflow" # Get a request object which we want to clone chained_requests = mcm.get('lists', list_of_nonflowing, method='get') chained_requests = list(reversed(chained_requests['value'])) print chained_requests # Make predefined modifications print len(chained_requests) for i in range(0, len(chained_requests)): if (chained_requests[i].find('MiniAODv3') != -1): print chained_requests[i] chained_request = mcm.get('chained_requests', chained_requests[i])
ochain = 'chain_RunIIFall17GS_flowRunIIFall17DRPremixPU2017_flowRunIIFall17MiniAODv2_flowRunIIFall17NanoAOD' dchain = 'chain_RunIIFall17GS_flowRunIIFall17DRPremixPU2017_flowRunIIFall17MiniAODv2_flowRunIIFall17NanoAODv4' ticketfilename = dchain + '.json' collector = defaultdict(lambda: defaultdict(lambda: defaultdict(int))) #for cc in ccs: print 50 * "-" listprep = ['TOP'] for pwg in listprep: #if pwg != "TOP": continue ## get all chains from that pwg in that chained campaign #crs = mcm.getA('chained_requests', query='member_of_campaign=%s&pwg=%s'%(cc['prepid'],pwg)) crs = mcm.get('chained_requests', query='member_of_campaign=%s&pwg=%s' % (ochain, pwg)) print "\t", pwg, ":\t", len(crs) for cr in crs: #print "\t\t",cr['prepid'], cr['chain'] root_id = cr['chain'][0] #print "\t\t",root_id #chainchecks = mcm.getA('chained_requests',query='contains=%s'%(root_id)) #print chainchecks campaign = root_id.split('-')[1] collector[pwg][campaign][root_id] += 1 print "This is the list of %s requests that are deemed chainable: " % ( listprep) #pprint.pprint( dict(collector) ) #print collector all_ticket = []
#pwgs = ['MUO','BTV','JME','TAU','EGM','TSG','SMP','HCA','B2G'] pwgs = ['SUS'] #pwgs = ['SUS','TOP','HIG'] #pwgs = ['BPH'] if not pwgs: # Get list of PWGs pwgs = mcm._McM__get('restapi/users/get_pwg')['results'] print('Got PWGs: %s' % (pwgs)) # Create a dict of dicts of dicts of integers collector = defaultdict(lambda: defaultdict(lambda: defaultdict(int))) for pwg in pwgs: chained_requests = mcm.get('chained_requests', query='member_of_campaign=%s&pwg=%s' % (org_chained_campaign, pwg)) print('PWG:%s\tChained requests:%s' % (pwg, len(chained_requests))) for chained_request in chained_requests: root_id = chained_request['chain'][0] campaign = root_id.split('-')[1] existing_chains = mcm.get('chained_requests', query='contains=%s&pwg=%s' % (root_id, pwg)) chain_present = False for existing_chain in existing_chains:
if not devMode: answer = None while answer not in ["y", "n"]: answer = raw_input("Not in dev mode, ok to continue [Y/N]? ").lower() if not answer == 'y': sys.exit() os.system('source /afs/cern.ch/cms/PPD/PdmV/tools/McM/getCookie.sh') os.system( 'cern-get-sso-cookie -u https://cms-pdmv.cern.ch/mcm/ -o ~/private/prod-cookie.txt --krb --reprocess' ) mcm = McM(dev=devMode) for req_prepid in prepidToSearch: request = mcm.get('requests', query='prepid={0}'.format(req_prepid), method='get') if len(request) == 1: request = request[0] if request['approval'] == 'none' or ( request['approval'] == 'validation' and request['status'] == 'validation'): print '' print 'I will trigger validation of {0}'.format(request['prepid']) print '-- request : ', request['dataset_name'] print '-- link : https://cms-pdmv.cern.ch/mcm/requests?prepid={0}'.format( request['prepid']) mcm.approve('requests', request['prepid']) print '' else: print ''
#export X509_USER_PROXY=$(voms-proxy-info --path) #export PYTHONPATH=/afs/cern.ch/cms/PPD/PdmV/tools/wmcontrol:${PYTHONPATH} #export PATH=/afs/cern.ch/cms/PPD/PdmV/tools/wmcontrol:${PATH} # source /afs/cern.ch/cms/PPD/PdmV/tools/wmclient/current/etc/wmclient.sh import os import sys sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import McM mcm = McM(dev=False) #requests = mcm.get('requests', query='tags=Summer16MiniAODv3T2sub1&priority=90000') #requests = mcm.get('requests', query='tags=Summer16MiniAODv3T2sub2&priority=90000') #requests = mcm.get('requests', query='priority=85000&status=submitted&prepid=*Autumn18DR*') requests = mcm.get('requests', query='status=submitted&tags=PAGLHCP2019&priority=85000') #requests = mcm.get('requests', query='prepid=EXO-RunIIFall17GS-009*&dataset_name=Mustar*') #requests = mcm.get('requests', query='status=submitted&tags=Summer16MiniAODv3T3') #requests = mcm.get('requests', query='status=submitted&prepid=HIG-PhaseIIMTDTDRAutumn18wmLHEGS-0000*') #requests = mcm.get('requests', query='status=submitted&dataset_name=VBF_BulkGravToWW_narrow_M-*') #requests = mcm.get('requests', query='status=submitted&prepid=SMP-*LowPU*GS*') #requests = mcm.get('requests', query='prepid=BPH-RunIIFall18GS-0006*') print('Found %s requests' % (len(requests))) for request in requests: if len(request['reqmgr_name']) > 0: # We change priority only if request has a registered workflow # Remove echo command to acutally execute it -> already removed # Change priority to 200000 result = os.system("wmpriority.py %s %s" %
list0 = list('SUS-RunIIFall17GS-000' + str(i) for i in range(xxx, yyy + 1)) for req in list0: requests_to_clone.append(req) #write the prepids of the new cloned requests to a txt file f = open('new_requests.txt', 'w') # If member_of_campaign is different, it will clone to other campaign campaign_modifications = {'member_of_campaign': 'RunIIFall18GS'} for request_prepid_to_clone in requests_to_clone: print('Cloning and modifying {0}'.format(request_prepid_to_clone)) # Get a request object which we want to clone request = mcm.get('requests', request_prepid_to_clone) # Make predefined modifications for key in campaign_modifications: request[key] = campaign_modifications[key] #clone request and print error if it fails clone_answer = mcm.clone_request(request) if clone_answer.get('results'): pass else: print('Something went wrong while cloning a request. %s' % (dumps(clone_answer))) #write new prepid to file f.write(clone_answer['prepid'] + '\n')
print "This is the list of %s requests that are deemed chainable: " % ( listprep) all_tickets = [] campaign_name = "RunIISummer16MiniAODv3" # Get requests that are member of given campaign requests_for_that_repeat = [] for pwg in listprep: page = 0 requests = [{}] while len(requests) > 0: if (campaigntochain == 'wmLHEGS'): requests = mcm.get( 'requests', query= 'status=done&member_of_campaign=%s&pwg=%s&member_of_chain=*wmLHEGS_flowRunIISummer16DR80PremixPUMoriond17_*' % (campaign_name, pwg), page=page) page += 1 elif (campaigntochain == 'GS'): requests = mcm.get( 'requests', query= 'status=done&member_of_campaign=%s&pwg=%s&member_of_chain=*chain_RunIISummer15GS_flowRunIISummer16DR80PremixPUMoriond17_*' % (campaign_name, pwg), page=page) page += 1 elif (campaigntochain == 'pLHE'): requests = mcm.get(