def Run(prepid): mcm = McM(dev=False) #mcm.approve("requests", "EXO-RunIISummer15wmLHEGS-02343", 1) #0:new #1:validation #2:define #3:approved #4:submit approve = mcm.approve('requests', prepid, 1) print approve['results']
def createrequest(self, clonequeue): self.needsupdate = True return clonequeue.add(self, self.pwg, self.newcampaign) if jobtype(): return "run locally to submit to McM" mcm = McM() clone_req = mcm.get('requests', self.originalprepid) clone_req['member_of_campaign'] = self.campaign answer = mcm.clone(self.originalprepid, clone_req) if not (answer and answer.get("results")): raise RuntimeError("Failed to create the request on McM\n{}\n{}".format(self, answer)) self.getprepid() if self.prepid != answer["prepid"]: raise RuntimeError("Wrong prepid?? {} {}".format(self.prepid, answer["prepid"])) self.updaterequest() return "cloned request "+self.originalprepid+" as "+self.prepid+" on McM"
def __exit__(self, *err): for level, prepids in self.approvals.iteritems(): print print "approving", len(prepids), "requests to level", level for prepid in sorted(prepids): print " ", prepid print McM().approve("requests", ",".join(prepids), level)
def GetTargetStepFromDAS(TargetStep, DAS): mcm = McM(dev=False) input_request = mcm.get("requests", query="produce=" + DAS) member_of_chain = input_request[0]['member_of_chain'] mychain = '' for chain in member_of_chain: if TargetStep in chain: mychain = chain chained_request = mcm.get('chained_requests', mychain) req = '' for member in chained_request['chain']: if TargetStep in member: req = member return req
def originalfullinfo(self): result = McM().get("requests", query="prepid="+self.originalprepid) if not result: raise ValueError("mcm query for prepid="+self.originalprepid+" returned None!") if len(result) == 0: raise ValueError("mcm query for prepid="+self.originalprepid+" returned nothing!") if len(result) > 1: raise ValueError("mcm query for prepid="+self.originalprepid+" returned multiple results!") return result[0]
def getprepid(self): super(ClonedRequest, self).getprepid() if self.prepid: return if jobtype(): return query = "dataset_name={}&extension={}&prepid={}-{}-*".format(self.originalfullinfo["dataset_name"], self.extensionnumber, self.pwg, self.campaign) output = McM().get('requests', query=query) prepids = {_["prepid"] for _ in output} if not prepids: return None if len(prepids) != 1: raise RuntimeError("Multiple prepids for {} ({})".format(self, self.datasetname, query)) assert len(prepids) == 1, prepids self.prepid = prepids.pop()
def clone_and_edit_jhchoi(request_prepid_to_clone, fragment, dataset_name, notes, total_events, generators): mcm = McM(dev=False) modifications = { 'fragment': fragment, 'dataset_name': dataset_name, 'notes': notes, 'total_events': total_events, "generators": generators } #request_prepid_to_clone = "SUS-RunIIWinter15wmLHE-00040" request = mcm.get('requests', request_prepid_to_clone) for key in modifications: request[key] = modifications[key] clone_answer = mcm.clone_request(request) if clone_answer.get('results'): print "@@" new_prepid = clone_answer['prepid'] print('Clone PrepID: %s' % (clone_answer['prepid'])) new_request = mcm.get('requests', new_prepid) print new_request['dataset_name'] mcm.approve('requests', new_prepid, None) print "---" f = open('success.txt', 'a') f.write(new_prepid + "\n") f.close() else: print('Something went wrong while cloning a request. %s' % (dumps(clone_answer))) f = open('fail.txt', 'a') f.write(dataset_name + "\n") f.close()
sys.exit() # Double check if it's ok to run not in dev mode devMode = not args.notDev if not devMode: answer = None while answer not in ["y", "n"]: answer = raw_input("Not in dev mode, ok to continue [Y/N]? ").lower() if not answer == 'y': sys.exit() os.system('source /afs/cern.ch/cms/PPD/PdmV/tools/McM/getCookie.sh') os.system( 'cern-get-sso-cookie -u https://cms-pdmv.cern.ch/mcm/ -o ~/private/prod-cookie.txt --krb --reprocess' ) mcm = McM(dev=devMode) for req_prepid in prepidToSearch: request = mcm.get('requests', query='prepid={0}'.format(req_prepid), method='get') if len(request) == 1: request = request[0] if request['approval'] == 'none' or ( request['approval'] == 'validation' and request['status'] == 'validation'): print '' print 'I will trigger validation of {0}'.format(request['prepid']) print '-- request : ', request['dataset_name'] print '-- link : https://cms-pdmv.cern.ch/mcm/requests?prepid={0}'.format( request['prepid'])
import sys sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import McM from json import dumps mcm = McM(dev=False) #mcm = McM(dev=True) # Example to get ALL requesst which are member of a given campaign and are submitted # It uses a generic search for specified columns: query='status=submitted' # Queries can be combined: query='status=submitted&member_of_campaign=Summer12' #campaign_requests = mcm.get('requests', query='member_of_campaign=Summer12&status=submitted') #for request in campaign_requests: # print(request['prepid']) # Example to retrieve single request dictionary # More methods are here: # https://cms-pdmv.cern.ch/mcm/restapi/requests/ #single_request_prepid = 'TOP-Summer12-00368' #single_request = mcm.get('requests', single_request_prepid, method='get') #print('Single request "%s":\n%s' % (single_request_prepid, dumps(single_request, indent=4))) # Example how to get multiple requests using range requests_query = """ HIG-RunIIFall17wmLHEGS-01712 -> HIG-RunIIFall17wmLHEGS-01714 """ range_of_requests = mcm.get_range_of_requests(requests_query) print('Found %s requests' % (len(range_of_requests))) for request in range_of_requests:
list_of_root_request = [r for r in list_of_root_request if r not in set_of_root_request_prepid_without_aodsim] return set_of_root_request_prepid_without_aodsim N_REQUESTS_PER_TICKET = 100 PRIORITY_BLOCK = 2 TICKET_NOTE ='Summer16 miniAODv3/nanoAODv3 reminiAOD' org_chained_campaign = 'chain_RunIIWinter15wmLHE_flowLHE2Summer15GS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16PremixMiniAODv2_flowRunIISummer16NanoAOD' dest_chained_campaign = 'chain_RunIIWinter15wmLHE_flowLHE2Summer15GS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3' # If dry run is enabled, nothing will be uploaded to McM dry_run = False # McM instance mcm = McM(dev=True)#, cookie='/afs/cern.ch/work/s/snorberg/miniAODv3/cookie.txt')#/afs/cern.ch/user/j/jrumsevi/private/dev_cookie.txt') # Set list of PWGs here. If no list is set, all PWGs from McM will be used pwgs = ['SMP'] if not pwgs: # Get list of PWGs pwgs = mcm._McM__get('restapi/users/get_pwg')['results'] print('Got PWGs: %s' % (pwgs)) # Create a dict of dicts of dicts of integers collector = defaultdict(lambda: defaultdict(lambda: defaultdict(int))) for pwg in pwgs: chained_requests = mcm.get('chained_requests', query='member_of_campaign=%s&pwg=%s' % (org_chained_campaign, pwg)) # print('PWG:%s\tChained requests:%s' % (pwg, len(chained_requests))) for chained_request in chained_requests: root_id = chained_request['chain'][0]
def Run(idx_row): output_msg = '' KEY = '1bAdzoB9bg6nbyxCOBsTAWSSJy6FHNfmkrIDqY-Nmdi0' sheet = '2016' title_list = GetTitleFromSpreadsheet(KEY, sheet, i_column="A", f_column="Z") info_list = GetInfoFromSpreadsheet(idx_row, KEY, sheet, i_column="A", f_column="Z") this_dic = {} for i in range(0, len(title_list)): title = title_list[i] #print title value = info_list[i] #print value this_dic[title] = value FullFragment = MakeFullFragment(this_dic["gridpackPATH"], this_dic["PSfragment"], this_dic["card_ref"]) generator_parameters=Make_generator_parameters(this_dic["cross_section"],\ this_dic["filter_efficiency"],\ this_dic["filter_efficiency_error"],\ this_dic["match_efficiency"],\ this_dic["match_efficiency_error"],\ this_dic["negative_weights_fraction"]) print this_dic['dataset_name'] new_request = { 'pwg': 'HIG', 'member_of_campaign': this_dic['member_of_campaign'], 'dataset_name': this_dic['dataset_name'], 'generators': this_dic['generators'].split(','), "fragment": FullFragment, 'generator_parameters': generator_parameters, 'total_events': int(this_dic['total_events']), 'mcdb_id': 0, 'size_event': [600], 'time_event': [60.] } if this_dic["PrepID"] != '': #print "ALREADY" mcm = McM(dev=False) this_request = mcm.get("requests", this_dic["PrepID"]) for key in new_request: this_request[key] = new_request[key] update = mcm.update('requests', this_request) return str(update) # push it to McM #mcm = McM(dev=False,cookie="dev-cookie.txt") #mcm = McM(dev=False,cookie='prod-cookie.txt') mcm = McM(dev=False) put_answer = mcm.put('requests', new_request) if put_answer.get('results'): prepid = put_answer['prepid'] print('New PrepID: %s' % (prepid)) #f.write(prepid+'\n') output_msg = str(prepid) else: print('Something went wrong while creating a request. %s' % (dumps(put_answer))) #f.write("fail,idx_row-->"+this_dic['dataset_name']+'\n') output_msg = "fail" + str(idx_row) + "-->" + this_dic['dataset_name'] #print output_msg return output_msg
import sys sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import McM from json import dumps mcm = McM(dev=True) # Script clones a request to other campaign. # Fefine list of modifications # If member_of_campaign is different, it will clone to other campaign modifications = {'extension': 1, 'total_events': 101, 'member_of_campaign': 'Summer12'} request_prepid_to_clone = "SUS-RunIIWinter15wmLHE-00040" # Get a request object which we want to clone request = mcm.get('requests', request_prepid_to_clone) # Make predefined modifications for key in modifications: request[key] = modifications[key] clone_answer = mcm.clone_request(request) if clone_answer.get('results'): print('Clone PrepID: %s' % (clone_answer['prepid'])) else: print('Something went wrong while cloning a request. %s' % (dumps(clone_answer)))
import sys sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import McM sys.stdout.flush() mcm = McM(dev=False) # Example to edit a request parameter(-s) and save it back in McM # request_prepid_to_update = 'HIG-Summer12-01257' # Doesn't exist #requests = mcm.get('requests', query='priority=110000&status=submitted&prepid=*Autumn18DR*') #requests = mcm.get('requests', query='status=submitted&prepid=*MTDTDR*GS*') requests = mcm.get('requests', query='status=submitted&tags=MTDTDRAutumn18PU200') #requests = mcm.get('requests', query='tags=Summer16MiniAODv3T2') #requests = mcm.get('requests', query='tags=Autumn18P1POGDR') i = 0 for request in requests: chained_request_id = request['member_of_chain'][0] chained_requests = mcm.get('chained_requests', query='prepid=%s' % (chained_request_id)) if (len(chained_requests[0]['chain']) < 3): continue if ('PU0' in chained_requests[0]['prepid']):
# NEED this to be sourced before #voms-proxy-init -voms cms #export X509_USER_PROXY=$(voms-proxy-info --path) #export PYTHONPATH=/afs/cern.ch/cms/PPD/PdmV/tools/wmcontrol:${PYTHONPATH} #export PATH=/afs/cern.ch/cms/PPD/PdmV/tools/wmcontrol:${PATH} # source /afs/cern.ch/cms/PPD/PdmV/tools/wmclient/current/etc/wmclient.sh import os import sys sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import McM mcm = McM(dev=False) #requests = mcm.get('requests', query='tags=Summer16MiniAODv3T2sub1&priority=90000') #requests = mcm.get('requests', query='tags=Summer16MiniAODv3T2sub2&priority=90000') #requests = mcm.get('requests', query='priority=85000&status=submitted&prepid=*Autumn18DR*') requests = mcm.get('requests', query='status=submitted&tags=PAGLHCP2019&priority=85000') #requests = mcm.get('requests', query='prepid=EXO-RunIIFall17GS-009*&dataset_name=Mustar*') #requests = mcm.get('requests', query='status=submitted&tags=Summer16MiniAODv3T3') #requests = mcm.get('requests', query='status=submitted&prepid=HIG-PhaseIIMTDTDRAutumn18wmLHEGS-0000*') #requests = mcm.get('requests', query='status=submitted&dataset_name=VBF_BulkGravToWW_narrow_M-*') #requests = mcm.get('requests', query='status=submitted&prepid=SMP-*LowPU*GS*') #requests = mcm.get('requests', query='prepid=BPH-RunIIFall18GS-0006*') print('Found %s requests' % (len(requests))) for request in requests: if len(request['reqmgr_name']) > 0: # We change priority only if request has a registered workflow
#dest_chained_campaign = 'chain_RunIIWinter15wmLHE_flowLHE2Summer15GS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3' #org_chained_campaign = 'chain_RunIISummer15wmLHEGS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16PremixMiniAODv2_flowRunIISummer16NanoAOD' #dest_chained_campaign = 'chain_RunIISummer15wmLHEGS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3' #chain_RunIIWinter15wmLHE_flowLHE2Summer15GS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3' #org_chained_campaign = 'chain_RunIISummer15GS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16PremixMiniAODv2_flowRunIISummer16NanoAOD' #dest_chained_campaign = 'chain_RunIISummer15GS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3' #chain_RunIIWinter15wmLHE_flowLHE2Summer15GS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3' org_chained_campaign = 'chain_RunIIWinter15wmLHE_flowLHE2Summer15GS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16PremixMiniAODv2_flowRunIISummer16NanoAOD' dest_chained_campaign = 'chain_RunIIWinter15wmLHE_flowLHE2Summer15GS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3' # If dry run is enabled, nothing will be uploaded to McM dry_run = False #True # McM instance #mcm = McM(dev=True, cookie='/afs/cern.ch/work/s/snorberg/miniAODv3/cookie.txt')#/afs/cern.ch/user/j/jrumsevi/private/dev_cookie.txt') mcm = McM(dev=False) #/afs/cern.ch/user/j/jrumsevi/private/dev_cookie.txt') # Set list of PWGs here. If no list is set, all PWGs from McM will be used #pwgs = ['MUO','BTV','JME','TAU','EGM','TSG','SMP','HCA','B2G'] pwgs = ['SUS'] #pwgs = ['SUS','TOP','HIG'] #pwgs = ['BPH'] if not pwgs: # Get list of PWGs pwgs = mcm._McM__get('restapi/users/get_pwg')['results'] print('Got PWGs: %s' % (pwgs)) # Create a dict of dicts of dicts of integers collector = defaultdict(lambda: defaultdict(lambda: defaultdict(int))) for pwg in pwgs:
print ' number of events = ' + str(args.events) print ' use McM prepID = ' + str(mcm) print ' skipexisting = ' + str(skipexisting) print das_cmd = "/cvmfs/cms.cern.ch/common/dasgoclient" # if mcm is specified, retrieve dataset name from prepID: if mcm: if "/" in str(args.inputdataset): print "not a McM prepID format, please check" sys.exit(1) # load McM sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import McM mcm = McM() # retrieve request with given prepid temp = sys.stdout f = open('/dev/null', 'w') sys.stdout = f request = mcm.get('requests', str(args.inputdataset)) sys.stdout = temp if debug: print 'request prepid',request['prepid'] # search dataset name as returned by mcm dataset_used = str(request['output_dataset'][0]) primary_dataset_name = dataset_used.split('/')[1] else: # search dataset name as name + campaign + datatier primary_dataset_name = args.inputdataset.split('/')[1] command=das_cmd+" --limit=0 --query=\"dataset dataset=/"+primary_dataset_name+"/*"+args.campaign+"*/"+args.datatier+"\"" dataset_used = commands.getstatusoutput(command)[1].split("\n")
import sys sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import McM from json import dumps #mcm = McM(dev=True,cookie='/afs/cern.ch/user/p/pgunnell/private/prod-cookie.txt') mcm = McM(dev=False) # Script clones a request to other campaign. # Fefine list of modifications # If member_of_campaign is different, it will clone to other campaign # Get a request object which we want to clone #chained_requests = mcm.get('chained_requests', 'B2G-chain_RunIISummer15wmLHEGS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3-00208') dest_chained_campaign = 'RunIISummer16MiniAODv3' #requests = mcm.get('requests', query='member_of_campaign=%s&status=new&tags=Summer16MiniAODv3T1' % (dest_chained_campaign)) requests = mcm.get( 'requests', query='member_of_campaign=%s&status=new&tags=Summer16MiniAODv3T4' % (dest_chained_campaign)) #requests = mcm.get('requests', query='prepid=B2G*&member_of_campaign=%s&approval=submit&status=approved&tags=Summer16MiniAODv3T3' % (dest_chained_campaign)) print len(requests) i = 0 for request in requests: #answer = mcm._McM__get('restapi/requests/approve/%s' % (request['prepid']))
print 'This is DRYRUN! No tickets will be created' else: print 'WARNING!' * 10 print 'REAL QUERIES WILL BE MADE!!! Tickets will be created' print 'WARNING!' * 10 if use_dev_instance: cookie_file = 'dev-cookie.txt' #dev print 'Running on dev instance!' else: cookie_file = 'cookie.txt' #prod print 'WARNING!' * 10 print 'Running on prod instance!!!' print 'WARNING!' * 10 mcm = McM(dev=use_dev_instance, debug=True) #pwgs=mcm._McM__get('restapi/users/get_pwg')['results'] #pwgs=mcm.get('restapi/users/get_pwg')['results'] # submit only these groups N_REQUESTS_PER_TICKET = 30 PRIORITY_BLOCK = 1 TICKET_NOTE = "NanoAODv2 NanoAODv4 central migration" campaigntochain = 'GS' # 'GS' # 'wmLHEGS' #'pLHE' #wmLHE #[2] Choose one campaign types #For wmLHEGS if (campaigntochain == 'wmLHEGS'): ochain = 'chain_RunIISummer15wmLHEGS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3' dchain = 'chain_RunIISummer15wmLHEGS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv4'
parser.add_argument('--ticket', type=str, help="check mcm requests using ticket number", nargs=1) parser.add_argument('--bypass_status', help="don't check request status in mcm", action='store_false') parser.add_argument('--bypass_validation', help="proceed to next prepid even if there are errors", action='store_true') parser.add_argument('--apply_many_threads_patch', help="apply the many threads MG5_aMC@NLO LO patch if necessary", action='store_true') parser.add_argument('--dev', help="Run on DEV instance of McM", action='store_true') parser.add_argument('--debug', help="Print debugging information", action='store_true') args = parser.parse_args() if args.prepid is not None: print "---> "+str(len(args.prepid))+" requests will be checked:" prepid = args.prepid print " " # Use no-id as identification mode in order not to use a SSO cookie mcm = McM(id='no-id', dev=args.dev, debug=args.debug) def get_request(prepid): result = mcm._McM__get('public/restapi/requests/get/%s' % (prepid)) if not result: return {} result = result.get('results', {}) return result def get_range_of_requests(query): result = mcm._McM__put('public/restapi/requests/listwithfile', data={'contents': query}) if not result: return {}
import sys sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import McM mcm = McM(dev=False) # Example to edit a request parameter(-s) and save it back in McM # request_prepid_to_update = 'HIG-Summer12-01257' # Doesn't exist for i in range(0, 20): request_prepid_to_update = 'EGM-RunIIFall18GS-000' + str(i) field_to_update = 'tags' # get a the dictionnary of a request request = mcm.get('requests', request_prepid_to_update) if 'prepid' not in request: # In case the request doesn't exist, there is nothing to update print('Request "%s" doesn\'t exist' % (request_prepid_to_update)) else: print('Request\'s "%s" field "%s" BEFORE update: %s' % (request_prepid_to_update, field_to_update, request[field_to_update])) # Modify what we want # time_event is a list for each sequence step request[field_to_update] = ["Fall18P1POGGS"] # Push it back to McM update_response = mcm.update('requests', request) print('Update response: %s' % (update_response))
""" import sys import sqlite3 import logging #pylint: disable=wrong-import-position,import-error sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import McM #pylint: enable=wrong-import-position,import-error # Logger logging.basicConfig(format='[%(asctime)s][%(levelname)s] %(message)s', level=logging.INFO) logger = logging.getLogger() # McM instance mcm = McM(dev=False, cookie='cookie.txt') def main(): """ Clear mcm_users table and update with users from McM """ conn = sqlite3.connect('data.db') cursor = conn.cursor() # Create table if it does not exist cursor.execute('''CREATE TABLE IF NOT EXISTS mcm_users (username text PRIMARY KEY NOT NULL, role text NOT NULL)''') # Clear the table cursor.execute('DELETE FROM `mcm_users`') # Create action history table
parser.parse_args('--prepid 1'.split()) print "---> " + str(len(args.prepid)) + " requests will be checked:" prepid = args.prepid print " " os.system('source /afs/cern.ch/cms/PPD/PdmV/tools/McM/getCookie.sh') os.system( 'cern-get-sso-cookie -u https://cms-pdmv.cern.ch/mcm/ -o ~/private/prod-cookie.txt --krb --reprocess' ) sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import McM from json import dumps from itertools import groupby mcm = McM(dev=False) def root_requests_from_ticket(ticket_prepid, include_docs=False): """ Return list of all root (first ones in the chain) requests of a ticket. By default function returns list of prepids. If include_docs is set to True, function will return whole documents """ mccm = mcm.get('mccms', ticket_prepid) query = '' for root_request in mccm.get('requests', []): if isinstance(root_request, str) or isinstance(root_request, unicode): query += '%s\n' % (root_request) elif isinstance(root_request, list): # List always contains two elements - start and end of a range
import sys sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import McM sys.stdout.flush() mcm = McM(dev=False) # Example to edit a request parameter(-s) and save it back in McM # request_prepid_to_update = 'HIG-Summer12-01257' # Doesn't exist #requests = mcm.get('requests', query='priority=110000&status=submitted&prepid=*Autumn18DR*') #requests = mcm.get('requests', query='tags=Autumn18POGP1DR&&priority=110000&status=submitted&prepid=MUO*') requests = mcm.get( 'requests', query='prepid=*Autumn18DR*&&priority=85000&status=submitted') #requests = mcm.get('requests', query='tags=Autumn18P1POGDR') tot_events = 0 for request in requests: chained_requests = request['member_of_chain'] chained_request = mcm.get('chained_requests', chained_requests[0]) tot_events += request['total_events'] root_id = chained_request['chain'][0] root_id_req = mcm.get('requests', root_id) chained_rootid = root_id_req['member_of_chain']
import sys import os sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import McM from json import dumps import random import csv mcm = McM(dev=False) def MakeLHEpart(gridpackPATH): LHEpart='import FWCore.ParameterSet.Config as cms\nexternalLHEProducer = cms.EDProducer("ExternalLHEProducer",\n args = cms.vstring('+\ gridpackPATH\ +"),\n nEvents = cms.untracked.uint32(5000),\n numberOfParameters = cms.uint32(1),\n outputFile = cms.string('cmsgrid_final.lhe'),\n scriptName = cms.FileInPath('GeneratorInterface/LHEInterface/data/run_generic_tarball_cvmfs.sh')\n )\n" return LHEpart def MakeFullFragment(gridpackPATH, PSfragment, card_ref): LHEpart = MakeLHEpart(gridpackPATH) PSpart = PSfragment ref_list = card_ref.split('\n') ref_part = '' for ref in ref_list: ref_part += "#" + ref + '\n' FullFragment = LHEpart + '\n' + ref_part + '\n' + PSpart return FullFragment #print FullFragment
print 'This is DRYRUN!' else: print 'WARNING!' * 10 print 'REAL QUERIES WILL BE MADE!!!' print 'WARNING!' * 10 if is_dev_instance: cookie_file = 'dev-cookie.txt' #dev print 'Running on dev instance!' else: cookie_file = 'cookie.txt' #prod print 'WARNING!' * 10 print 'Running on prod instance!!!' print 'WARNING!' * 10 mcm = McM(dev=is_dev_instance, cookie=cookie_file, debug=True) #pwgs=mcm._McM__get('restapi/users/get_pwg')['results'] #pwgs=mcm.get('restapi/users/get_pwg')['results'] # submit only these groups ochain = '' dchain = '' N_REQUESTS_PER_TICKET = 30 PRIORITY_BLOCK = 1 TICKET_NOTE = "NanoAODv2 NanoAODv4 central migration" #[2] Choose one campaign types ochain = 'chain_RunIIFall17GS_flowRunIIFall17DRPremixPU2017_flowRunIIFall17MiniAODv2_flowRunIIFall17NanoAOD' dchain = 'chain_RunIIFall17GS_flowRunIIFall17DRPremixPU2017_flowRunIIFall17MiniAODv2_flowRunIIFall17NanoAODv4'
""" This module handles user synchronization between Samples page and McM """ import sys import sqlite3 import logging #pylint: disable=wrong-import-position,import-error sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import McM #pylint: enable=wrong-import-position,import-error # McM instance mcm = McM(dev=('--dev' in sys.argv), cookie='cookie.txt') # Logger logging.basicConfig(format='[%(asctime)s][%(levelname)s] %(message)s', level=logging.INFO) logger = logging.getLogger() def update_users(conn): """ Clear mcm_users table and update with users from McM """ # Create table if it does not exist conn.execute('''CREATE TABLE IF NOT EXISTS mcm_users (username text PRIMARY KEY NOT NULL, name text NOT NULL, role text NOT NULL)''') # Clear the table conn.execute('DELETE FROM `mcm_users`') conn.commit()
sys.exit() # Double check if it's ok to run not in dev mode devMode = not args.notDev if not devMode: answer = None while answer not in ["y", "n"]: answer = raw_input("Not in dev mode, ok to continue [Y/N]? ").lower() if not answer == 'y': sys.exit() os.system('source /afs/cern.ch/cms/PPD/PdmV/tools/McM/getCookie.sh') os.system( 'cern-get-sso-cookie -u https://cms-pdmv.cern.ch/mcm/ -o ~/private/prod-cookie.txt --krb --reprocess' ) mcm = McM(dev=devMode) def cloneToUL16PreVFP(prepidToClone): campaign = 'RunIISummer19UL16wmLHEGENAPV' if "RunIISummer19UL16wmLHEGEN" in prepidToClone[ 'prepid'] else 'RunIISummer19UL16GENAPV' tag = ['ULPAG16'] # Check dataset name doesn't already exist in this campaign otherRequests = mcm.get( 'requests', query='dataset_name={dataset}&member_of_campaign={campaign}'.format( dataset=prepidToClone['dataset_name'], campaign=campaign), method='get') if otherRequests != None:
""" Get all requests in given range. Take all chained requests that these requests are members of. Leave only those chained requests that have MiniAODv3 in their name. Skip (remove) chained request with biggest number (the last one). For all chained requests that are left in the list, set action_parameters.flag to False, delete requests in them if these requests appear only in that chained request, delete chained request itself. """ import sys sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import McM # McM instance mcm = McM(dev=False, cookie='/afs/cern.ch/user/p/pgunnell/private/prod-cookie.txt') requests_query = """ EXO-RunIIWinter15pLHE-03395 -> EXO-RunIIWinter15pLHE-03398 """ list_of_requests = mcm.get_range_of_requests(requests_query) # Iterate through all requests in the range for request in list_of_requests: print('\n\nProcessing %s' % (request['prepid'])) # Get chains that this request is member of. Leave only those that have MiniAODv3 in them # Sort prepids, so biggest number would be last member_of_chains = sorted(request.get('member_of_chain', [])) member_of_chains = [ chain for chain in member_of_chains if 'MiniAODv3' in chain ]
if args.prepid is not None: parser.parse_args('--prepid 1'.split()) print "---> "+str(len(args.prepid))+" requests will be checked:" prepid = args.prepid print " " os.system('source /afs/cern.ch/cms/PPD/PdmV/tools/McM/getCookie.sh') os.system('cern-get-sso-cookie -u https://cms-pdmv.cern.ch/mcm/ -o ~/private/prod-cookie.txt --krb --reprocess') sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import McM from json import dumps from itertools import groupby mcm = McM(dev=False) def root_requests_from_ticket(ticket_prepid, include_docs=False): """ Return list of all root (first ones in the chain) requests of a ticket. By default function returns list of prepids. If include_docs is set to True, function will return whole documents """ mccm = mcm.get('mccms',ticket_prepid) query = '' for root_request in mccm.get('requests',[]): if isinstance(root_request,str) or isinstance(root_request,unicode): query += '%s\n' % (root_request) elif isinstance(root_request,list): # List always contains two elements - start and end of a range query += '%s -> %s\n' % (root_request[0], root_request[1])
import sys import os import time sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import McM mcm = McM(dev=True) def GetRootRequstList(KEY, sheet, cell_range): ##KEY=> #print 'https://docs.google.com/spreadsheets/d/1BG4BaHfPBDlYOVi-BW63sQwdS5WQ2ggZv2HWkIk29oU/edit#gid=986135503' #KEY = 1BG4BaHfPBDlYOVi-BW63sQwdS5WQ2ggZv2HWkIk29oU #cell_ragne => "B3:B1000" download_url = '"https://docs.google.com/spreadsheets/d/' + KEY + '/gviz/tq?tqx=out:csv&sheet=' + sheet + '&range=' + cell_range + '"' filename = '_temp' + str(time.time()) + '.txt' #print 'wget -q -O '+filename+" "+download_url os.system('wget -q -O ' + filename + " " + download_url) f = open(filename, 'r') mylist = [] lines = f.readlines() for line in lines: if len(line.split('-')) == 3: line = line.replace('"', '').replace('\n', '') #if line.startswith('#'):continue mylist.append(line)
import sys sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import McM from json import dumps #set dev to False to run the final version of the script (i.e. send the modifications made to MCM) #while editing keep dev set to True mcm = McM(dev=False) #list of requests to clone requests_to_clone = [] list0 = list('SUS-RunIIFall17GS-000' + str(i) for i in range(xxx, yyy + 1)) for req in list0: requests_to_clone.append(req) #write the prepids of the new cloned requests to a txt file f = open('new_requests.txt', 'w') # If member_of_campaign is different, it will clone to other campaign campaign_modifications = {'member_of_campaign': 'RunIIFall18GS'} for request_prepid_to_clone in requests_to_clone: print('Cloning and modifying {0}'.format(request_prepid_to_clone)) # Get a request object which we want to clone request = mcm.get('requests', request_prepid_to_clone) # Make predefined modifications for key in campaign_modifications: request[key] = campaign_modifications[key]
import sys sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/') from rest import McM from json import dumps mcm = McM(dev=False) # Example to get ALL requesst which are member of a given campaign and are submitted # It uses a generic search for specified columns: query='status=submitted' # Queries can be combined: query='status=submitted&member_of_campaign=Summer12' #campaign_requests = mcm.get('requests', query='dataset_name=NMSSMCascade_mH-*_mSUSY-*_TuneCUEP8M1_13TeV-madgraph-pythia8') #for request in campaign_requests: # print(request['prepid']) # Example to retrieve single request dictionary # More methods are here: # https://cms-pdmv.cern.ch/mcm/restapi/requests/ single_request_prepid = 'HIG-RunIISummer15wmLHEGS-02135' single_request = mcm.get('requests', single_request_prepid, method='get') #print('Single request "%s":\n%s' % (single_request_prepid, single_request['generators']))#dumps(single_request, indent=4))) for key in single_request: print key ## Example how to get multiple requests using range #requests_query = """ # B2G-Fall13-00001 # B2G-Fall13-00005 -> B2G-Fall13-00015 #""" #range_of_requests = mcm.get_range_of_requests(requests_query) #print('Found %s requests' % (len(range_of_requests))) #for request in range_of_requests: # print(request['prepid'])