def getCampaign(name):
import sys
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
#mcm = restful(dev=False, cookie='dev-cookie.txt')
mcm = McM(dev=False, cookie='cookies/cookie.txt')
# example to search ALL requesst which are member of a campaign
# it uses a generic search for specified columns: query='status=submitted'
# queries can be combined: query='status=submitted&member_of_campaign=Summer12'
lines = []
with open("lists/" + name + ".txt") as f:
lines = [line.rstrip('\n') for line in f]
dataSets = []
for d in lines:
ds = d.strip()
if len(ds) < 1: continue
if ds[0] == "%": continue
if ds[0] == "#":
dataSets.append((ds[1:], []))
else:
dataSets[-1][1].append(ds)
#print "RADEK"
#print dataSets
#import sys
#sys.exit()
#lines = set(lines) #To have it unique
allRequests = []
for item in dataSets:
for r in item[1]:
req = mcm.get('requests',
query='dataset_name=' + r + '&prepid=*-RunII' +
name + '*')
if req != None:
allRequests += req
chains = getChains(allRequests)
for dn in chains:
chAll = chains[dn]
for ch in chAll:
req = mcm.get('requests', query='member_of_chain=' + ch)
for r in req:
#print ch, r['prepid']
n = r['prepid']
if 'GS-' in n:
pass
elif 'DR' in n:
pass
elif 'ReReco' in n:
pass
elif 'MiniAOD' in n:
pass
elif 'NanoAOD' in n:
pass
else:
#print len(req), ch
print n
if r not in allRequests:
allRequests.append(r)
#print 'Adding ',r['dataset_name'], r['prepid']
else:
pass
#print 'Existing ',r['dataset_name'], r['prepid']
#test
# for r in allRequests:
# if len(r['reqmgr_name']) > 1:
# print r['reqmgr_name']
# import sys
# sys.exit()
#allRequests = allRequests1 + allRequests2 + allRequests3 + allRequests4
return (allRequests, dataSets)
def Run(idx_row, KEY, sheet):
output_msg = ''
#KEY='1bAdzoB9bg6nbyxCOBsTAWSSJy6FHNfmkrIDqY-Nmdi0'
#sheet='2016'
title_list = GetTitleFromSpreadsheet(KEY,
sheet,
i_column="A",
f_column="Z")
info_list = GetInfoFromSpreadsheet(idx_row,
KEY,
sheet,
i_column="A",
f_column="Z")
this_dic = {}
for i in range(0, len(title_list)):
title = title_list[i]
#print title
value = info_list[i]
#print value
this_dic[title] = value
FullFragment = MakeFullFragment(this_dic["gridpackPATH"],
this_dic["PSfragment"],
this_dic["card_ref"])
generator_parameters=Make_generator_parameters(this_dic["cross_section"],\
this_dic["filter_efficiency"],\
this_dic["filter_efficiency_error"],\
this_dic["match_efficiency"],\
this_dic["match_efficiency_error"],\
this_dic["negative_weights_fraction"])
print this_dic['dataset_name']
new_request = {
'pwg': 'HIG',
'member_of_campaign': this_dic['member_of_campaign'],
'dataset_name': this_dic['dataset_name'],
'generators': this_dic['generators'].split(','),
"fragment": FullFragment,
'generator_parameters': generator_parameters,
'total_events': int(this_dic['total_events']),
'mcdb_id': 0,
'size_event': [600],
'time_event': [60.]
}
if this_dic["PrepID"] != '':
#print "ALREADY"
mcm = McM(dev=False)
this_request = mcm.get("requests", this_dic["PrepID"])
for key in new_request:
this_request[key] = new_request[key]
update = mcm.update('requests', this_request)
return str(update)
# push it to McM
#mcm = McM(dev=False,cookie="dev-cookie.txt")
#mcm = McM(dev=False,cookie='prod-cookie.txt')
mcm = McM(dev=False)
put_answer = mcm.put('requests', new_request)
if put_answer.get('results'):
prepid = put_answer['prepid']
print('New PrepID: %s' % (prepid))
#f.write(prepid+'\n')
output_msg = str(prepid)
else:
print('Something went wrong while creating a request. %s' %
(dumps(put_answer)))
#f.write("fail,idx_row-->"+this_dic['dataset_name']+'\n')
output_msg = "fail" + str(idx_row) + "-->" + this_dic['dataset_name']
#print output_msg
return output_msg
import sys
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
from json import dumps
#mcm = McM(dev=True,cookie='/afs/cern.ch/user/p/pgunnell/private/prod-cookie.txt')
mcm = McM(dev=False)
# Script clones a request to other campaign.
# Fefine list of modifications
# If member_of_campaign is different, it will clone to other campaign
# Get a request object which we want to clone
#chained_requests = mcm.get('chained_requests', 'B2G-chain_RunIISummer15wmLHEGS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3-00208')
#dest_chained_campaign = 'RunIIFall17MiniAODv2'
#requests = mcm.get('requests', query='member_of_campaign=%s&status=approved&prepid=*' % (dest_chained_campaign))
#requests = mcm.get('requests', query='prepid=SUS-RunIIFall17NanoAOD-00082')
#requests = mcm.get('requests', query='prepid=HIG-RunIIFall17MiniAODv2-01676')
#requests = mcm.get('requests', query='member_of_campaign=%s&status=new&prepid=*' % (dest_chained_campaign))
dest_chained_campaign = 'TOP-RunIISummer16MiniAODv3-00171,TOP-RunIISummer16MiniAODv3-00175'
#requests = mcm.get('requests', query='range=%s' % (dest_chained_campaign))
requests = mcm.get('requests', query='prepid=TOP-RunIISummer16MiniAODv3-00172')
# Make predefined modifications
#print('Number of requests '+str(len(requests)))
print 'Dataset name PrepID root request Link to PrepID root request'
"""
TODO: Document
"""
import sys
import sqlite3
import logging
#pylint: disable=wrong-import-position,import-error
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
#pylint: enable=wrong-import-position,import-error
# McM instance
mcm = McM(dev=False, cookie='cookie.txt')
# Logger
logging.basicConfig(format='[%(asctime)s][%(levelname)s] %(message)s', level=logging.INFO)
logger = logging.getLogger()
def main():
"""
TODO: Document
"""
conn = sqlite3.connect('data.db')
cursor = conn.cursor()
# Create table if it does not exist
cursor.execute('''CREATE TABLE IF NOT EXISTS run3_samples
(prepid text PRIMARY KEY NOT NULL,
dataset text NOT NULL,
total_events integer NOT NULL,
import os, sys, time, string, re
os.system('source /afs/cern.ch/cms/PPD/PdmV/tools/McM/getCookie.sh')
os.system(
'cern-get-sso-cookie -u https://cms-pdmv.cern.ch/mcm/ -o ~/private/prod-cookie.txt --krb --reprocess'
)
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
from json import dumps
from itertools import groupby
mcm = McM(dev=True)
query_str = 'prepid=' + str(sys.argv[1])
res = mcm.get('requests', query=query_str)
if len(res) == 0:
print "***********************************************************************************"
print "Something's wrong - can not get the request parameters"
print "***********************************************************************************"
exit()
my_path = '/tmp/' + os.environ['USER'] + '/gridpacks/'
print ""
print "***********************************************************************************"
for r in res:
pi = r['prepid']
dn = r['dataset_name']
te = r['time_event']
totalevents = r['total_events']
cmssw = r['cmssw_release']
value = args.value
# Double check if it's ok to run not in dev mode
devMode = not args.notDev
if not devMode:
answer = None
while answer not in ["y", "n"]:
answer = raw_input("Not in dev mode, ok to continue [Y/N]? ").lower()
if not answer == 'y':
sys.exit()
os.system('source /afs/cern.ch/cms/PPD/PdmV/tools/McM/getCookie.sh')
os.system(
'cern-get-sso-cookie -u https://cms-pdmv.cern.ch/mcm/ -o ~/private/prod-cookie.txt --krb --reprocess'
)
mcm = McM(dev=devMode)
for req_prepid in prepidToSearch:
request = mcm.get('requests',
query='prepid={0}'.format(req_prepid),
method='get')
if len(request) == 1:
request = request[0]
if request['approval'] == 'none':
print ''
print 'I will modify {0}'.format(request['prepid'])
print 'Request {0} field {1} BEFORE update: {2}'.format(
request['prepid'], field_to_update, request[field_to_update])
# Modify what we want
# time_event is a list for each sequence step
def GetPileUpDatasetName(prepid):
mcm=McM(dev=False)
request=mcm.get('requests',prepid)
return request['pileup_dataset_name']
return set_of_root_request_prepid_without_aodsim
N_REQUESTS_PER_TICKET = 20
PRIORITY_BLOCK = 3
TICKET_NOTE = 'Resubmission of Fall17 not chained requests'
#dest_chained_campaign = 'chain_RunIIFall17wmLHEGS_flowRunIIFall17DRPremixPU2017_flowRunIIFall17MiniAODv2_flowRunIIFall17NanoAOD'
dest_chained_campaign = 'chain_RunIIFall17GS_flowRunIIFall17DRPremixPU2017_flowRunIIFall17MiniAODv2_flowRunIIFall17NanoAOD'
# If dry run is enabled, nothing will be uploaded to McM
dry_run = False
# McM instance
mcm = McM(dev=False) #/afs/cern.ch/user/j/jrumsevi/private/dev_cookie.txt')
requests = []
requests_for_that_repeat = []
i = 0
all_tickets = []
prepids = [
'HIG-RunIIFall17DRPremix-00040',
'HIG-RunIIFall17DRPremix-00041',
'HIG-RunIIFall17DRPremix-00044',
'HIG-RunIIFall17DRPremix-01319',
'HIG-RunIIFall17DRPremix-01318',
'HIG-RunIIFall17DRPremix-01315',
'HIG-RunIIFall17DRPremix-01314',
'HIG-RunIIFall17DRPremix-01316',
print ' number of events = ' + str(args.events)
print ' use McM prepID = ' + str(mcm)
print ' skipexisting = ' + str(skipexisting)
print
das_cmd = "/cvmfs/cms.cern.ch/common/dasgoclient"
# if mcm is specified, retrieve dataset name from prepID:
if mcm:
if "/" in str(args.inputdataset):
print "not a McM prepID format, please check"
sys.exit(1)
# load McM
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
mcm = McM()
# retrieve request with given prepid
temp = sys.stdout
f = open('/dev/null', 'w')
sys.stdout = f
request = mcm.get('requests', str(args.inputdataset))
sys.stdout = temp
if debug: print 'request prepid', request['prepid']
# search dataset name as returned by mcm
dataset_used = str(request['output_dataset'][0])
primary_dataset_name = dataset_used.split('/')[1]
else:
# search dataset name as name + campaign + datatier
primary_dataset_name = args.inputdataset.split('/')[1]
command = das_cmd + " --limit=0 --query=\"dataset dataset=/" + primary_dataset_name + "/*" + args.campaign + "*/" + args.datatier + "\""
dataset_used = commands.getstatusoutput(command)[1].split("\n")
print 'This is DRYRUN!'
else:
print 'WARNING!' * 10
print 'REAL QUERIES WILL BE MADE!!!'
print 'WARNING!' * 10
if is_dev_instance:
cookie_file = 'dev-cookie.txt' #dev
print 'Running on dev instance!'
else:
cookie_file = 'cookie.txt' #prod
print 'WARNING!' * 10
print 'Running on prod instance!!!'
print 'WARNING!' * 10
mcm = McM(dev=is_dev_instance, debug=True)
#pwgs=mcm._McM__get('restapi/users/get_pwg')['results']
#pwgs=mcm.get('restapi/users/get_pwg')['results']
# submit only these groups
N_REQUESTS_PER_TICKET = 30
PRIORITY_BLOCK = 1
TICKET_NOTE = "NanoAODv2 NanoAODv4 central migration"
#[2] Choose one campaign types
ochain = 'chain_RunIIFall17GS_flowRunIIFall17DRPremixPU2017_flowRunIIFall17MiniAODv2_flowRunIIFall17NanoAOD'
dchain = 'chain_RunIIFall17GS_flowRunIIFall17DRPremixPU2017_flowRunIIFall17MiniAODv2_flowRunIIFall17NanoAODv4'
ticketfilename = dchain + '.json'
print 'This is DRYRUN! No tickets will be created'
else:
print 'WARNING!' * 10
print 'REAL QUERIES WILL BE MADE!!! Tickets will be created'
print 'WARNING!' * 10
if use_dev_instance:
cookie_file = 'dev-cookie.txt' #dev
print 'Running on dev instance!'
else:
cookie_file = 'cookie.txt' #prod
print 'WARNING!' * 10
print 'Running on prod instance!!!'
print 'WARNING!' * 10
mcm = McM(dev=use_dev_instance, debug=False)
#pwgs=mcm._McM__get('restapi/users/get_pwg')['results']
#pwgs=mcm.get('restapi/users/get_pwg')['results']
# submit only these groups
N_REQUESTS_PER_TICKET = 30
PRIORITY_BLOCK = 1
TICKET_NOTE = "NanoAODv2 NanoAODv4 central migration"
#[2] Choose one campaign types
#ochain='chain_RunIIWinter15wmLHE_flowLHE2Summer15GS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16PremixMiniAODv2_flowRunIISummer16NanoAOD'
ochain = 'chain_RunIISummer15wmLHEGS_flowRunIISummer16DR80PUMoriond17_flowRunIISummer16MiniAODv2_flowRunIISummer16NanoAOD'
#dchain = 'chain_RunIIWinter15wmLHE_flowLHE2Summer15GS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3'
dchain = 'chain_RunIISummer15wmLHEGS_flowRunIISummer16DR80PUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3'
"""
Get all requests in given range.
Take all chained requests that these requests are members of.
Leave only those chained requests that have MiniAODv3 in their name.
Skip (remove) chained request with biggest number (the last one).
For all chained requests that are left in the list, set action_parameters.flag to False,
delete requests in them if these requests appear only in that chained request,
delete chained request itself.
"""
import sys
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
# McM instance
mcm = McM(dev=True)#, cookie='/afs/cern.ch/user/j/jrumsevi/private/dev_cookie.txt')
requests_query = """
B2G-RunIIWinter15wmLHE-00035 -> B2G-RunIIWinter15wmLHE-00035
"""
list_of_requests = mcm.get_range_of_requests(requests_query)
# Iterate through all requests in the range
for request in list_of_requests:
print('\n\nProcessing %s' % (request['prepid']))
# Get chains that this request is member of. Leave only those that have MiniAODv3 in them
# Sort prepids, so biggest number would be last
member_of_chains = sorted(request.get('member_of_chain', []))
member_of_chains = [chain for chain in member_of_chains if 'MiniAODv3' in chain]
print member_of_chains
if len(member_of_chains) <= 1:
action='store_true')
parser.add_argument('--dev',
help="Run on DEV instance of McM",
action='store_true')
parser.add_argument('--debug',
help="Print debugging information",
action='store_true')
args = parser.parse_args()
if args.prepid is not None:
print "---> " + str(len(args.prepid)) + " requests will be checked:"
prepid = args.prepid
print " "
# Use no-id as identification mode in order not to use a SSO cookie
mcm = McM(id='no-id', dev=args.dev, debug=args.debug)
def get_request(prepid):
result = mcm._McM__get('public/restapi/requests/get/%s' % (prepid))
if not result:
return {}
result = result.get('results', {})
return result
def get_range_of_requests(query):
result = mcm._McM__put('public/restapi/requests/listwithfile',
data={'contents': query})
if not result:
import sys
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
sys.stdout.flush()
mcm = McM(dev=False,
cookie='/afs/cern.ch/user/p/pgunnell/private/prod-cookie.txt')
# Example to edit a request parameter(-s) and save it back in McM
# request_prepid_to_update = 'HIG-Summer12-01257' # Doesn't exist
for i in range(0, 28):
print str(0) + ','
for i in range(28, 63):
print str(0.029411) + ','
#pwgPOG = ['MUO','EGM','TAU','BTV','TSG','HCA','EXO']
pwgPOG = ['JME']
#pwgPAG = ['HIG','SUS','SMP','EXO','BPH','TOP']
campaigns = ['GS', 'DR', 'DRPremix']
for i in range(1, 50):
#request_prepid_to_update = 'BTV-RunIIFall18DRPremix-000'+str(i)
for name in pwgPOG:
if i < 10:
request_prepid_to_update = name + '-RunIIAutumn18DR-0000' + str(i)