def Run(prepid):
mcm = McM(dev=False)
#mcm.approve("requests", "EXO-RunIISummer15wmLHEGS-02343", 1)
#0:new
#1:validation
#2:define
#3:approved
#4:submit
approve = mcm.approve('requests', prepid, 1)
print approve['results']
def createrequest(self, clonequeue):
self.needsupdate = True
return clonequeue.add(self, self.pwg, self.newcampaign)
if jobtype(): return "run locally to submit to McM"
mcm = McM()
clone_req = mcm.get('requests', self.originalprepid)
clone_req['member_of_campaign'] = self.campaign
answer = mcm.clone(self.originalprepid, clone_req)
if not (answer and answer.get("results")):
raise RuntimeError("Failed to create the request on McM\n{}\n{}".format(self, answer))
self.getprepid()
if self.prepid != answer["prepid"]:
raise RuntimeError("Wrong prepid?? {} {}".format(self.prepid, answer["prepid"]))
self.updaterequest()
return "cloned request "+self.originalprepid+" as "+self.prepid+" on McM"
def __exit__(self, *err):
for level, prepids in self.approvals.iteritems():
print
print "approving", len(prepids), "requests to level", level
for prepid in sorted(prepids):
print " ", prepid
print McM().approve("requests", ",".join(prepids), level)
def GetTargetStepFromDAS(TargetStep, DAS):
mcm = McM(dev=False)
input_request = mcm.get("requests", query="produce=" + DAS)
member_of_chain = input_request[0]['member_of_chain']
mychain = ''
for chain in member_of_chain:
if TargetStep in chain:
mychain = chain
chained_request = mcm.get('chained_requests', mychain)
req = ''
for member in chained_request['chain']:
if TargetStep in member:
req = member
return req
def originalfullinfo(self):
result = McM().get("requests", query="prepid="+self.originalprepid)
if not result:
raise ValueError("mcm query for prepid="+self.originalprepid+" returned None!")
if len(result) == 0:
raise ValueError("mcm query for prepid="+self.originalprepid+" returned nothing!")
if len(result) > 1:
raise ValueError("mcm query for prepid="+self.originalprepid+" returned multiple results!")
return result[0]
def getprepid(self):
super(ClonedRequest, self).getprepid()
if self.prepid: return
if jobtype(): return
query = "dataset_name={}&extension={}&prepid={}-{}-*".format(self.originalfullinfo["dataset_name"], self.extensionnumber, self.pwg, self.campaign)
output = McM().get('requests', query=query)
prepids = {_["prepid"] for _ in output}
if not prepids:
return None
if len(prepids) != 1:
raise RuntimeError("Multiple prepids for {} ({})".format(self, self.datasetname, query))
assert len(prepids) == 1, prepids
self.prepid = prepids.pop()
def clone_and_edit_jhchoi(request_prepid_to_clone, fragment, dataset_name,
notes, total_events, generators):
mcm = McM(dev=False)
modifications = {
'fragment': fragment,
'dataset_name': dataset_name,
'notes': notes,
'total_events': total_events,
"generators": generators
}
#request_prepid_to_clone = "SUS-RunIIWinter15wmLHE-00040"
request = mcm.get('requests', request_prepid_to_clone)
for key in modifications:
request[key] = modifications[key]
clone_answer = mcm.clone_request(request)
if clone_answer.get('results'):
print "@@"
new_prepid = clone_answer['prepid']
print('Clone PrepID: %s' % (clone_answer['prepid']))
new_request = mcm.get('requests', new_prepid)
print new_request['dataset_name']
mcm.approve('requests', new_prepid, None)
print "---"
f = open('success.txt', 'a')
f.write(new_prepid + "\n")
f.close()
else:
print('Something went wrong while cloning a request. %s' %
(dumps(clone_answer)))
f = open('fail.txt', 'a')
f.write(dataset_name + "\n")
f.close()
sys.exit()
# Double check if it's ok to run not in dev mode
devMode = not args.notDev
if not devMode:
answer = None
while answer not in ["y", "n"]:
answer = raw_input("Not in dev mode, ok to continue [Y/N]? ").lower()
if not answer == 'y':
sys.exit()
os.system('source /afs/cern.ch/cms/PPD/PdmV/tools/McM/getCookie.sh')
os.system(
'cern-get-sso-cookie -u https://cms-pdmv.cern.ch/mcm/ -o ~/private/prod-cookie.txt --krb --reprocess'
)
mcm = McM(dev=devMode)
for req_prepid in prepidToSearch:
request = mcm.get('requests',
query='prepid={0}'.format(req_prepid),
method='get')
if len(request) == 1:
request = request[0]
if request['approval'] == 'none' or (
request['approval'] == 'validation'
and request['status'] == 'validation'):
print ''
print 'I will trigger validation of {0}'.format(request['prepid'])
print '-- request : ', request['dataset_name']
print '-- link : https://cms-pdmv.cern.ch/mcm/requests?prepid={0}'.format(
request['prepid'])
import sys
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
from json import dumps
mcm = McM(dev=False)
#mcm = McM(dev=True)
# Example to get ALL requesst which are member of a given campaign and are submitted
# It uses a generic search for specified columns: query='status=submitted'
# Queries can be combined: query='status=submitted&member_of_campaign=Summer12'
#campaign_requests = mcm.get('requests', query='member_of_campaign=Summer12&status=submitted')
#for request in campaign_requests:
# print(request['prepid'])
# Example to retrieve single request dictionary
# More methods are here:
# https://cms-pdmv.cern.ch/mcm/restapi/requests/
#single_request_prepid = 'TOP-Summer12-00368'
#single_request = mcm.get('requests', single_request_prepid, method='get')
#print('Single request "%s":\n%s' % (single_request_prepid, dumps(single_request, indent=4)))
# Example how to get multiple requests using range
requests_query = """
HIG-RunIIFall17wmLHEGS-01712 -> HIG-RunIIFall17wmLHEGS-01714
"""
range_of_requests = mcm.get_range_of_requests(requests_query)
print('Found %s requests' % (len(range_of_requests)))
for request in range_of_requests:
list_of_root_request = [r for r in list_of_root_request if r not in set_of_root_request_prepid_without_aodsim]
return set_of_root_request_prepid_without_aodsim
N_REQUESTS_PER_TICKET = 100
PRIORITY_BLOCK = 2
TICKET_NOTE ='Summer16 miniAODv3/nanoAODv3 reminiAOD'
org_chained_campaign = 'chain_RunIIWinter15wmLHE_flowLHE2Summer15GS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16PremixMiniAODv2_flowRunIISummer16NanoAOD'
dest_chained_campaign = 'chain_RunIIWinter15wmLHE_flowLHE2Summer15GS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3'
# If dry run is enabled, nothing will be uploaded to McM
dry_run = False
# McM instance
mcm = McM(dev=True)#, cookie='/afs/cern.ch/work/s/snorberg/miniAODv3/cookie.txt')#/afs/cern.ch/user/j/jrumsevi/private/dev_cookie.txt')
# Set list of PWGs here. If no list is set, all PWGs from McM will be used
pwgs = ['SMP']
if not pwgs:
# Get list of PWGs
pwgs = mcm._McM__get('restapi/users/get_pwg')['results']
print('Got PWGs: %s' % (pwgs))
# Create a dict of dicts of dicts of integers
collector = defaultdict(lambda: defaultdict(lambda: defaultdict(int)))
for pwg in pwgs:
chained_requests = mcm.get('chained_requests', query='member_of_campaign=%s&pwg=%s' % (org_chained_campaign, pwg))
# print('PWG:%s\tChained requests:%s' % (pwg, len(chained_requests)))
for chained_request in chained_requests:
root_id = chained_request['chain'][0]
def Run(idx_row):
output_msg = ''
KEY = '1bAdzoB9bg6nbyxCOBsTAWSSJy6FHNfmkrIDqY-Nmdi0'
sheet = '2016'
title_list = GetTitleFromSpreadsheet(KEY,
sheet,
i_column="A",
f_column="Z")
info_list = GetInfoFromSpreadsheet(idx_row,
KEY,
sheet,
i_column="A",
f_column="Z")
this_dic = {}
for i in range(0, len(title_list)):
title = title_list[i]
#print title
value = info_list[i]
#print value
this_dic[title] = value
FullFragment = MakeFullFragment(this_dic["gridpackPATH"],
this_dic["PSfragment"],
this_dic["card_ref"])
generator_parameters=Make_generator_parameters(this_dic["cross_section"],\
this_dic["filter_efficiency"],\
this_dic["filter_efficiency_error"],\
this_dic["match_efficiency"],\
this_dic["match_efficiency_error"],\
this_dic["negative_weights_fraction"])
print this_dic['dataset_name']
new_request = {
'pwg': 'HIG',
'member_of_campaign': this_dic['member_of_campaign'],
'dataset_name': this_dic['dataset_name'],
'generators': this_dic['generators'].split(','),
"fragment": FullFragment,
'generator_parameters': generator_parameters,
'total_events': int(this_dic['total_events']),
'mcdb_id': 0,
'size_event': [600],
'time_event': [60.]
}
if this_dic["PrepID"] != '':
#print "ALREADY"
mcm = McM(dev=False)
this_request = mcm.get("requests", this_dic["PrepID"])
for key in new_request:
this_request[key] = new_request[key]
update = mcm.update('requests', this_request)
return str(update)
# push it to McM
#mcm = McM(dev=False,cookie="dev-cookie.txt")
#mcm = McM(dev=False,cookie='prod-cookie.txt')
mcm = McM(dev=False)
put_answer = mcm.put('requests', new_request)
if put_answer.get('results'):
prepid = put_answer['prepid']
print('New PrepID: %s' % (prepid))
#f.write(prepid+'\n')
output_msg = str(prepid)
else:
print('Something went wrong while creating a request. %s' %
(dumps(put_answer)))
#f.write("fail,idx_row-->"+this_dic['dataset_name']+'\n')
output_msg = "fail" + str(idx_row) + "-->" + this_dic['dataset_name']
#print output_msg
return output_msg
import sys
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
from json import dumps
mcm = McM(dev=True)
# Script clones a request to other campaign.
# Fefine list of modifications
# If member_of_campaign is different, it will clone to other campaign
modifications = {'extension': 1,
'total_events': 101,
'member_of_campaign': 'Summer12'}
request_prepid_to_clone = "SUS-RunIIWinter15wmLHE-00040"
# Get a request object which we want to clone
request = mcm.get('requests', request_prepid_to_clone)
# Make predefined modifications
for key in modifications:
request[key] = modifications[key]
clone_answer = mcm.clone_request(request)
if clone_answer.get('results'):
print('Clone PrepID: %s' % (clone_answer['prepid']))
else:
print('Something went wrong while cloning a request. %s' % (dumps(clone_answer)))
import sys
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
sys.stdout.flush()
mcm = McM(dev=False)
# Example to edit a request parameter(-s) and save it back in McM
# request_prepid_to_update = 'HIG-Summer12-01257' # Doesn't exist
#requests = mcm.get('requests', query='priority=110000&status=submitted&prepid=*Autumn18DR*')
#requests = mcm.get('requests', query='status=submitted&prepid=*MTDTDR*GS*')
requests = mcm.get('requests',
query='status=submitted&tags=MTDTDRAutumn18PU200')
#requests = mcm.get('requests', query='tags=Summer16MiniAODv3T2')
#requests = mcm.get('requests', query='tags=Autumn18P1POGDR')
i = 0
for request in requests:
chained_request_id = request['member_of_chain'][0]
chained_requests = mcm.get('chained_requests',
query='prepid=%s' % (chained_request_id))
if (len(chained_requests[0]['chain']) < 3):
continue
if ('PU0' in chained_requests[0]['prepid']):
# NEED this to be sourced before
#voms-proxy-init -voms cms
#export X509_USER_PROXY=$(voms-proxy-info --path)
#export PYTHONPATH=/afs/cern.ch/cms/PPD/PdmV/tools/wmcontrol:${PYTHONPATH}
#export PATH=/afs/cern.ch/cms/PPD/PdmV/tools/wmcontrol:${PATH}
# source /afs/cern.ch/cms/PPD/PdmV/tools/wmclient/current/etc/wmclient.sh
import os
import sys
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
mcm = McM(dev=False)
#requests = mcm.get('requests', query='tags=Summer16MiniAODv3T2sub1&priority=90000')
#requests = mcm.get('requests', query='tags=Summer16MiniAODv3T2sub2&priority=90000')
#requests = mcm.get('requests', query='priority=85000&status=submitted&prepid=*Autumn18DR*')
requests = mcm.get('requests',
query='status=submitted&tags=PAGLHCP2019&priority=85000')
#requests = mcm.get('requests', query='prepid=EXO-RunIIFall17GS-009*&dataset_name=Mustar*')
#requests = mcm.get('requests', query='status=submitted&tags=Summer16MiniAODv3T3')
#requests = mcm.get('requests', query='status=submitted&prepid=HIG-PhaseIIMTDTDRAutumn18wmLHEGS-0000*')
#requests = mcm.get('requests', query='status=submitted&dataset_name=VBF_BulkGravToWW_narrow_M-*')
#requests = mcm.get('requests', query='status=submitted&prepid=SMP-*LowPU*GS*')
#requests = mcm.get('requests', query='prepid=BPH-RunIIFall18GS-0006*')
print('Found %s requests' % (len(requests)))
for request in requests:
if len(request['reqmgr_name']) > 0:
# We change priority only if request has a registered workflow
#dest_chained_campaign = 'chain_RunIIWinter15wmLHE_flowLHE2Summer15GS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3'
#org_chained_campaign = 'chain_RunIISummer15wmLHEGS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16PremixMiniAODv2_flowRunIISummer16NanoAOD'
#dest_chained_campaign = 'chain_RunIISummer15wmLHEGS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3' #chain_RunIIWinter15wmLHE_flowLHE2Summer15GS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3'
#org_chained_campaign = 'chain_RunIISummer15GS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16PremixMiniAODv2_flowRunIISummer16NanoAOD'
#dest_chained_campaign = 'chain_RunIISummer15GS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3' #chain_RunIIWinter15wmLHE_flowLHE2Summer15GS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3'
org_chained_campaign = 'chain_RunIIWinter15wmLHE_flowLHE2Summer15GS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16PremixMiniAODv2_flowRunIISummer16NanoAOD'
dest_chained_campaign = 'chain_RunIIWinter15wmLHE_flowLHE2Summer15GS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3'
# If dry run is enabled, nothing will be uploaded to McM
dry_run = False #True
# McM instance
#mcm = McM(dev=True, cookie='/afs/cern.ch/work/s/snorberg/miniAODv3/cookie.txt')#/afs/cern.ch/user/j/jrumsevi/private/dev_cookie.txt')
mcm = McM(dev=False) #/afs/cern.ch/user/j/jrumsevi/private/dev_cookie.txt')
# Set list of PWGs here. If no list is set, all PWGs from McM will be used
#pwgs = ['MUO','BTV','JME','TAU','EGM','TSG','SMP','HCA','B2G']
pwgs = ['SUS']
#pwgs = ['SUS','TOP','HIG']
#pwgs = ['BPH']
if not pwgs:
# Get list of PWGs
pwgs = mcm._McM__get('restapi/users/get_pwg')['results']
print('Got PWGs: %s' % (pwgs))
# Create a dict of dicts of dicts of integers
collector = defaultdict(lambda: defaultdict(lambda: defaultdict(int)))
for pwg in pwgs:
print ' number of events = ' + str(args.events)
print ' use McM prepID = ' + str(mcm)
print ' skipexisting = ' + str(skipexisting)
print
das_cmd = "/cvmfs/cms.cern.ch/common/dasgoclient"
# if mcm is specified, retrieve dataset name from prepID:
if mcm:
if "/" in str(args.inputdataset):
print "not a McM prepID format, please check"
sys.exit(1)
# load McM
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
mcm = McM()
# retrieve request with given prepid
temp = sys.stdout
f = open('/dev/null', 'w')
sys.stdout = f
request = mcm.get('requests', str(args.inputdataset))
sys.stdout = temp
if debug: print 'request prepid',request['prepid']
# search dataset name as returned by mcm
dataset_used = str(request['output_dataset'][0])
primary_dataset_name = dataset_used.split('/')[1]
else:
# search dataset name as name + campaign + datatier
primary_dataset_name = args.inputdataset.split('/')[1]
command=das_cmd+" --limit=0 --query=\"dataset dataset=/"+primary_dataset_name+"/*"+args.campaign+"*/"+args.datatier+"\""
dataset_used = commands.getstatusoutput(command)[1].split("\n")
import sys
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
from json import dumps
#mcm = McM(dev=True,cookie='/afs/cern.ch/user/p/pgunnell/private/prod-cookie.txt')
mcm = McM(dev=False)
# Script clones a request to other campaign.
# Fefine list of modifications
# If member_of_campaign is different, it will clone to other campaign
# Get a request object which we want to clone
#chained_requests = mcm.get('chained_requests', 'B2G-chain_RunIISummer15wmLHEGS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3-00208')
dest_chained_campaign = 'RunIISummer16MiniAODv3'
#requests = mcm.get('requests', query='member_of_campaign=%s&status=new&tags=Summer16MiniAODv3T1' % (dest_chained_campaign))
requests = mcm.get(
'requests',
query='member_of_campaign=%s&status=new&tags=Summer16MiniAODv3T4' %
(dest_chained_campaign))
#requests = mcm.get('requests', query='prepid=B2G*&member_of_campaign=%s&approval=submit&status=approved&tags=Summer16MiniAODv3T3' % (dest_chained_campaign))
print len(requests)
i = 0
for request in requests:
#answer = mcm._McM__get('restapi/requests/approve/%s' % (request['prepid']))
print 'This is DRYRUN! No tickets will be created'
else:
print 'WARNING!' * 10
print 'REAL QUERIES WILL BE MADE!!! Tickets will be created'
print 'WARNING!' * 10
if use_dev_instance:
cookie_file = 'dev-cookie.txt' #dev
print 'Running on dev instance!'
else:
cookie_file = 'cookie.txt' #prod
print 'WARNING!' * 10
print 'Running on prod instance!!!'
print 'WARNING!' * 10
mcm = McM(dev=use_dev_instance, debug=True)
#pwgs=mcm._McM__get('restapi/users/get_pwg')['results']
#pwgs=mcm.get('restapi/users/get_pwg')['results']
# submit only these groups
N_REQUESTS_PER_TICKET = 30
PRIORITY_BLOCK = 1
TICKET_NOTE = "NanoAODv2 NanoAODv4 central migration"
campaigntochain = 'GS' # 'GS' # 'wmLHEGS' #'pLHE' #wmLHE
#[2] Choose one campaign types
#For wmLHEGS
if (campaigntochain == 'wmLHEGS'):
ochain = 'chain_RunIISummer15wmLHEGS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3'
dchain = 'chain_RunIISummer15wmLHEGS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv4'
parser.add_argument('--ticket', type=str, help="check mcm requests using ticket number", nargs=1)
parser.add_argument('--bypass_status', help="don't check request status in mcm", action='store_false')
parser.add_argument('--bypass_validation', help="proceed to next prepid even if there are errors", action='store_true')
parser.add_argument('--apply_many_threads_patch', help="apply the many threads MG5_aMC@NLO LO patch if necessary", action='store_true')
parser.add_argument('--dev', help="Run on DEV instance of McM", action='store_true')
parser.add_argument('--debug', help="Print debugging information", action='store_true')
args = parser.parse_args()
if args.prepid is not None:
print "---> "+str(len(args.prepid))+" requests will be checked:"
prepid = args.prepid
print " "
# Use no-id as identification mode in order not to use a SSO cookie
mcm = McM(id='no-id', dev=args.dev, debug=args.debug)
def get_request(prepid):
result = mcm._McM__get('public/restapi/requests/get/%s' % (prepid))
if not result:
return {}
result = result.get('results', {})
return result
def get_range_of_requests(query):
result = mcm._McM__put('public/restapi/requests/listwithfile', data={'contents': query})
if not result:
return {}
import sys
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
mcm = McM(dev=False)
# Example to edit a request parameter(-s) and save it back in McM
# request_prepid_to_update = 'HIG-Summer12-01257' # Doesn't exist
for i in range(0, 20):
request_prepid_to_update = 'EGM-RunIIFall18GS-000' + str(i)
field_to_update = 'tags'
# get a the dictionnary of a request
request = mcm.get('requests', request_prepid_to_update)
if 'prepid' not in request:
# In case the request doesn't exist, there is nothing to update
print('Request "%s" doesn\'t exist' % (request_prepid_to_update))
else:
print('Request\'s "%s" field "%s" BEFORE update: %s' %
(request_prepid_to_update, field_to_update,
request[field_to_update]))
# Modify what we want
# time_event is a list for each sequence step
request[field_to_update] = ["Fall18P1POGGS"]
# Push it back to McM
update_response = mcm.update('requests', request)
print('Update response: %s' % (update_response))
"""
import sys
import sqlite3
import logging
#pylint: disable=wrong-import-position,import-error
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
#pylint: enable=wrong-import-position,import-error
# Logger
logging.basicConfig(format='[%(asctime)s][%(levelname)s] %(message)s', level=logging.INFO)
logger = logging.getLogger()
# McM instance
mcm = McM(dev=False, cookie='cookie.txt')
def main():
"""
Clear mcm_users table and update with users from McM
"""
conn = sqlite3.connect('data.db')
cursor = conn.cursor()
# Create table if it does not exist
cursor.execute('''CREATE TABLE IF NOT EXISTS mcm_users
(username text PRIMARY KEY NOT NULL,
role text NOT NULL)''')
# Clear the table
cursor.execute('DELETE FROM `mcm_users`')
# Create action history table
parser.parse_args('--prepid 1'.split())
print "---> " + str(len(args.prepid)) + " requests will be checked:"
prepid = args.prepid
print " "
os.system('source /afs/cern.ch/cms/PPD/PdmV/tools/McM/getCookie.sh')
os.system(
'cern-get-sso-cookie -u https://cms-pdmv.cern.ch/mcm/ -o ~/private/prod-cookie.txt --krb --reprocess'
)
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
from json import dumps
from itertools import groupby
mcm = McM(dev=False)
def root_requests_from_ticket(ticket_prepid, include_docs=False):
"""
Return list of all root (first ones in the chain) requests of a ticket.
By default function returns list of prepids.
If include_docs is set to True, function will return whole documents
"""
mccm = mcm.get('mccms', ticket_prepid)
query = ''
for root_request in mccm.get('requests', []):
if isinstance(root_request, str) or isinstance(root_request, unicode):
query += '%s\n' % (root_request)
elif isinstance(root_request, list):
# List always contains two elements - start and end of a range
import sys
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
sys.stdout.flush()
mcm = McM(dev=False)
# Example to edit a request parameter(-s) and save it back in McM
# request_prepid_to_update = 'HIG-Summer12-01257' # Doesn't exist
#requests = mcm.get('requests', query='priority=110000&status=submitted&prepid=*Autumn18DR*')
#requests = mcm.get('requests', query='tags=Autumn18POGP1DR&&priority=110000&status=submitted&prepid=MUO*')
requests = mcm.get(
'requests', query='prepid=*Autumn18DR*&&priority=85000&status=submitted')
#requests = mcm.get('requests', query='tags=Autumn18P1POGDR')
tot_events = 0
for request in requests:
chained_requests = request['member_of_chain']
chained_request = mcm.get('chained_requests', chained_requests[0])
tot_events += request['total_events']
root_id = chained_request['chain'][0]
root_id_req = mcm.get('requests', root_id)
chained_rootid = root_id_req['member_of_chain']
import sys
import os
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
from json import dumps
import random
import csv
mcm = McM(dev=False)
def MakeLHEpart(gridpackPATH):
LHEpart='import FWCore.ParameterSet.Config as cms\nexternalLHEProducer = cms.EDProducer("ExternalLHEProducer",\n args = cms.vstring('+\
gridpackPATH\
+"),\n nEvents = cms.untracked.uint32(5000),\n numberOfParameters = cms.uint32(1),\n outputFile = cms.string('cmsgrid_final.lhe'),\n scriptName = cms.FileInPath('GeneratorInterface/LHEInterface/data/run_generic_tarball_cvmfs.sh')\n )\n"
return LHEpart
def MakeFullFragment(gridpackPATH, PSfragment, card_ref):
LHEpart = MakeLHEpart(gridpackPATH)
PSpart = PSfragment
ref_list = card_ref.split('\n')
ref_part = ''
for ref in ref_list:
ref_part += "#" + ref + '\n'
FullFragment = LHEpart + '\n' + ref_part + '\n' + PSpart
return FullFragment
#print FullFragment
print 'This is DRYRUN!'
else:
print 'WARNING!' * 10
print 'REAL QUERIES WILL BE MADE!!!'
print 'WARNING!' * 10
if is_dev_instance:
cookie_file = 'dev-cookie.txt' #dev
print 'Running on dev instance!'
else:
cookie_file = 'cookie.txt' #prod
print 'WARNING!' * 10
print 'Running on prod instance!!!'
print 'WARNING!' * 10
mcm = McM(dev=is_dev_instance, cookie=cookie_file, debug=True)
#pwgs=mcm._McM__get('restapi/users/get_pwg')['results']
#pwgs=mcm.get('restapi/users/get_pwg')['results']
# submit only these groups
ochain = ''
dchain = ''
N_REQUESTS_PER_TICKET = 30
PRIORITY_BLOCK = 1
TICKET_NOTE = "NanoAODv2 NanoAODv4 central migration"
#[2] Choose one campaign types
ochain = 'chain_RunIIFall17GS_flowRunIIFall17DRPremixPU2017_flowRunIIFall17MiniAODv2_flowRunIIFall17NanoAOD'
dchain = 'chain_RunIIFall17GS_flowRunIIFall17DRPremixPU2017_flowRunIIFall17MiniAODv2_flowRunIIFall17NanoAODv4'
"""
This module handles user synchronization between Samples page and McM
"""
import sys
import sqlite3
import logging
#pylint: disable=wrong-import-position,import-error
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
#pylint: enable=wrong-import-position,import-error
# McM instance
mcm = McM(dev=('--dev' in sys.argv), cookie='cookie.txt')
# Logger
logging.basicConfig(format='[%(asctime)s][%(levelname)s] %(message)s',
level=logging.INFO)
logger = logging.getLogger()
def update_users(conn):
"""
Clear mcm_users table and update with users from McM
"""
# Create table if it does not exist
conn.execute('''CREATE TABLE IF NOT EXISTS mcm_users
(username text PRIMARY KEY NOT NULL,
name text NOT NULL,
role text NOT NULL)''')
# Clear the table
conn.execute('DELETE FROM `mcm_users`')
conn.commit()
sys.exit()
# Double check if it's ok to run not in dev mode
devMode = not args.notDev
if not devMode:
answer = None
while answer not in ["y", "n"]:
answer = raw_input("Not in dev mode, ok to continue [Y/N]? ").lower()
if not answer == 'y':
sys.exit()
os.system('source /afs/cern.ch/cms/PPD/PdmV/tools/McM/getCookie.sh')
os.system(
'cern-get-sso-cookie -u https://cms-pdmv.cern.ch/mcm/ -o ~/private/prod-cookie.txt --krb --reprocess'
)
mcm = McM(dev=devMode)
def cloneToUL16PreVFP(prepidToClone):
campaign = 'RunIISummer19UL16wmLHEGENAPV' if "RunIISummer19UL16wmLHEGEN" in prepidToClone[
'prepid'] else 'RunIISummer19UL16GENAPV'
tag = ['ULPAG16']
# Check dataset name doesn't already exist in this campaign
otherRequests = mcm.get(
'requests',
query='dataset_name={dataset}&member_of_campaign={campaign}'.format(
dataset=prepidToClone['dataset_name'], campaign=campaign),
method='get')
if otherRequests != None:
"""
Get all requests in given range.
Take all chained requests that these requests are members of.
Leave only those chained requests that have MiniAODv3 in their name.
Skip (remove) chained request with biggest number (the last one).
For all chained requests that are left in the list, set action_parameters.flag to False,
delete requests in them if these requests appear only in that chained request,
delete chained request itself.
"""
import sys
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
# McM instance
mcm = McM(dev=False,
cookie='/afs/cern.ch/user/p/pgunnell/private/prod-cookie.txt')
requests_query = """
EXO-RunIIWinter15pLHE-03395 -> EXO-RunIIWinter15pLHE-03398
"""
list_of_requests = mcm.get_range_of_requests(requests_query)
# Iterate through all requests in the range
for request in list_of_requests:
print('\n\nProcessing %s' % (request['prepid']))
# Get chains that this request is member of. Leave only those that have MiniAODv3 in them
# Sort prepids, so biggest number would be last
member_of_chains = sorted(request.get('member_of_chain', []))
member_of_chains = [
chain for chain in member_of_chains if 'MiniAODv3' in chain
]
if args.prepid is not None:
parser.parse_args('--prepid 1'.split())
print "---> "+str(len(args.prepid))+" requests will be checked:"
prepid = args.prepid
print " "
os.system('source /afs/cern.ch/cms/PPD/PdmV/tools/McM/getCookie.sh')
os.system('cern-get-sso-cookie -u https://cms-pdmv.cern.ch/mcm/ -o ~/private/prod-cookie.txt --krb --reprocess')
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
from json import dumps
from itertools import groupby
mcm = McM(dev=False)
def root_requests_from_ticket(ticket_prepid, include_docs=False):
"""
Return list of all root (first ones in the chain) requests of a ticket.
By default function returns list of prepids.
If include_docs is set to True, function will return whole documents
"""
mccm = mcm.get('mccms',ticket_prepid)
query = ''
for root_request in mccm.get('requests',[]):
if isinstance(root_request,str) or isinstance(root_request,unicode):
query += '%s\n' % (root_request)
elif isinstance(root_request,list):
# List always contains two elements - start and end of a range
query += '%s -> %s\n' % (root_request[0], root_request[1])
import sys
import os
import time
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
mcm = McM(dev=True)
def GetRootRequstList(KEY, sheet, cell_range):
##KEY=>
#print 'https://docs.google.com/spreadsheets/d/1BG4BaHfPBDlYOVi-BW63sQwdS5WQ2ggZv2HWkIk29oU/edit#gid=986135503'
#KEY = 1BG4BaHfPBDlYOVi-BW63sQwdS5WQ2ggZv2HWkIk29oU
#cell_ragne => "B3:B1000"
download_url = '"https://docs.google.com/spreadsheets/d/' + KEY + '/gviz/tq?tqx=out:csv&sheet=' + sheet + '&range=' + cell_range + '"'
filename = '_temp' + str(time.time()) + '.txt'
#print 'wget -q -O '+filename+" "+download_url
os.system('wget -q -O ' + filename + " " + download_url)
f = open(filename, 'r')
mylist = []
lines = f.readlines()
for line in lines:
if len(line.split('-')) == 3:
line = line.replace('"', '').replace('\n', '')
#if line.startswith('#'):continue
mylist.append(line)
import sys
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
from json import dumps
#set dev to False to run the final version of the script (i.e. send the modifications made to MCM)
#while editing keep dev set to True
mcm = McM(dev=False)
#list of requests to clone
requests_to_clone = []
list0 = list('SUS-RunIIFall17GS-000' + str(i) for i in range(xxx, yyy + 1))
for req in list0:
requests_to_clone.append(req)
#write the prepids of the new cloned requests to a txt file
f = open('new_requests.txt', 'w')
# If member_of_campaign is different, it will clone to other campaign
campaign_modifications = {'member_of_campaign': 'RunIIFall18GS'}
for request_prepid_to_clone in requests_to_clone:
print('Cloning and modifying {0}'.format(request_prepid_to_clone))
# Get a request object which we want to clone
request = mcm.get('requests', request_prepid_to_clone)
# Make predefined modifications
for key in campaign_modifications:
request[key] = campaign_modifications[key]
import sys
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
from json import dumps
mcm = McM(dev=False)
# Example to get ALL requesst which are member of a given campaign and are submitted
# It uses a generic search for specified columns: query='status=submitted'
# Queries can be combined: query='status=submitted&member_of_campaign=Summer12'
#campaign_requests = mcm.get('requests', query='dataset_name=NMSSMCascade_mH-*_mSUSY-*_TuneCUEP8M1_13TeV-madgraph-pythia8')
#for request in campaign_requests:
# print(request['prepid'])
# Example to retrieve single request dictionary
# More methods are here:
# https://cms-pdmv.cern.ch/mcm/restapi/requests/
single_request_prepid = 'HIG-RunIISummer15wmLHEGS-02135'
single_request = mcm.get('requests', single_request_prepid, method='get')
#print('Single request "%s":\n%s' % (single_request_prepid, single_request['generators']))#dumps(single_request, indent=4)))
for key in single_request:
print key
## Example how to get multiple requests using range
#requests_query = """
# B2G-Fall13-00001
# B2G-Fall13-00005 -> B2G-Fall13-00015
#"""
#range_of_requests = mcm.get_range_of_requests(requests_query)
#print('Found %s requests' % (len(range_of_requests)))
#for request in range_of_requests:
# print(request['prepid'])
