-
Notifications
You must be signed in to change notification settings - Fork 0
/
ticket-miniaodv2-nanoaod.py
148 lines (129 loc) · 5.55 KB
/
ticket-miniaodv2-nanoaod.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
import sys
import time
from collections import defaultdict
import pprint
import copy
import json
today = time.mktime( time.localtime() )
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import restful
def chunks(l, n):
"""Yield successive n-sized chunks from l."""
for i in range(0, len(l), n):
yield l[i:i + n]
def dr_output_not_saved(list_of_root_request,ochain):
"""Verify that all requests have AODSIM output"""
set_of_root_request_prepid_without_aodsim = set()
for root_request_prepid in list_of_root_request:
crs = mcm.getA('chained_requests', query='contains=%s&prepid=%s'%(root_request_prepid,ochain))
for cr in crs:
list_of_dr_request_prepid = [dr for dr in cr['chain'] if 'DR' in dr]
if len(list_of_dr_request_prepid)==1:
dr_request = mcm.getA('requests', query='prepid=%s'%(dr_request_prepid[0]))[0]
# last step of the request must be saved
if dr_request['keep_output'][-1]: continue
else: set_of_root_request_prepid_without_aodsim.add(root_request_prepid)
else: set_of_root_request_prepid_without_aodsim.add(root_request_prepid)
#remove not chainable requests from the original list
list_of_root_request = [r for r in list_of_root_request if r not in set_of_root_request_prepid_without_aodsim]
return set_of_root_request_prepid_without_aodsim
# Global settings defining access to McM
cookie_file = ''
is_dev_instance = True
#[1] Use False first to check
#Remember to use False first, to make sure that all tickets are creatable.
is_dry_run = True
if is_dry_run:
print 'This is DRYRUN!'
else:
print 'WARNING!'*10
print 'REAL QUERIES WILL BE MADE!!!'
print 'WARNING!'*10
if is_dev_instance:
cookie_file = 'dev-cookie.txt' #dev
print 'Running on dev instance!'
else:
cookie_file = 'cookie.txt' #prod
print 'WARNING!'*10
print 'Running on prod instance!!!'
print 'WARNING!'*10
mcm=restful(dev=is_dev_instance, cookie=cookie_file, debug=True)
pwgs=mcm.get('restapi/users/get_pwg')['results']
# submit only these groups
#pwgs=['B2G','BPH','BTV','EXO']
print pwgs
ochain = ''
dchain = ''
N_REQUESTS_PER_TICKET = 30
PRIORITY_BLOCK = 1
TICKET_NOTE = "Fall17MiniAOD to Fall17MiniAODv2+NanoAOD central migration"
#[2] Choose one campaign types
ochain = 'chain_RunIIFall17wmLHEGS_flowRunIIFall17DRPremixPU2017_flowRunIIFall17MiniAOD'
dchain = 'chain_RunIIFall17wmLHEGS_flowRunIIFall17DRPremixPU2017_flowRunIIFall17MiniAODv2_flowRunIIFall17NanoAOD'
ticketfilename = dchain+'.json'
collector=defaultdict(lambda : defaultdict( lambda : defaultdict( int )))
#for cc in ccs:
print 50*"-"
for pwg in pwgs:
#if pwg != "TOP": continue
## get all chains from that pwg in that chained campaign
#crs = mcm.getA('chained_requests', query='member_of_campaign=%s&pwg=%s'%(cc['prepid'],pwg))
crs = mcm.getA('chained_requests', query='member_of_campaign=%s&pwg=%s'%(ochain,pwg))
print "\t",pwg,":\t",len(crs)
for cr in crs:
#print "\t\t",cr['prepid'], cr['chain']
root_id = cr['chain'][0]
#print "\t\t",root_id
#chainchecks = mcm.getA('chained_requests',query='contains=%s'%(root_id))
#print chainchecks
campaign = root_id.split('-')[1]
collector[pwg][campaign][root_id]+=1
print "This is the list of %s requests that are deemed chainable: "%(pwgs)
#pprint.pprint( dict(collector) )
#print collector
all_ticket=[]
#ccs = mcm.getA('chained_campaigns', query='contains=....')
ccs = mcm.getA('chained_campaigns', query='prepid=%s'%(dchain))
for pwg in pwgs:
## create a ticket for the correct chain
for cc in ccs:
chain_prepid = cc['prepid']
root_campaign = cc['campaigns'][0][0]
# create tickets with different repetition numbers for root requests
for repeat in range(10):
# list of root requests in cc['prepid'] chained campaign with 'repeat' repetition number
requests_for_that_repeat = map(lambda i : i[0], filter(lambda i : i[1]==repeat, collector[pwg][root_campaign].items()))
if not requests_for_that_repeat: continue
#print requests_for_that_repeat
requests_for_that_repeat.sort()
set_of_invalid_requests = dr_output_not_saved(requests_for_that_repeat,ochain)
if len(set_of_invalid_requests) > 0: print 'NOT CHAINED: ', set_of_invalid_requests
## create a ticket with that content
for chunk in chunks(requests_for_that_repeat,N_REQUESTS_PER_TICKET):
mccm_ticket = { 'prepid' : pwg, ## this is how one passes it in the first place
'pwg' : pwg,
'requests' : chunk,
'notes' : TICKET_NOTE,
'chains' : [ chain_prepid ],
'repetitions' : repeat,
'block' : PRIORITY_BLOCK
}
all_ticket.append( copy.deepcopy( mccm_ticket ) )
#print mccm_ticket
## you'll be able to re-read all tickets from the created json
open(ticketfilename,'w').write(json.dumps( all_ticket ))
#all_ticket = json.loads(open('all_tickets_pythia_nooutput.json').read())
for ticket in all_ticket:
### flip the switch
if not is_dry_run:
res = mcm.putA('mccms', ticket )
print "Create ticket"
print res
ticket_prepid=res.get('prepid',None)
print "Ticket prepid: ", ticket_prepid
if ticket_prepid: mcm.get('restapi/mccms/generate/%s/reserve'%(ticket_prepid))
else:
print "Error: no ticket"
print ticket
time.sleep(60)
pass