root_gs_req = mcm.get('requests', root_gs)
root_gsdr = chained_request['chain'][2]
root_gsdr_req = mcm.get('requests', root_gsdr)
if (root_id_req['status'] == 'done' and root_gs_req['status'] == 'done'):
print str(request['prepid']) + ' ' + str(request['dataset_name'])
#answer = mcm._McM__get('restapi/chained_requests/rewind/%s' % (chained_request['prepid']))
#print answer
#answer = mcm._McM__get('restapi/chained_requests/rewind/%s' % (chained_request['prepid']))
#print answer
answer = mcm._McM__get('restapi/chained_requests/flow/%s' %
(chained_request['prepid']))
print answer
#answer = mcm._McM__get('restapi/requests/approve/%s' % (request['prepid']))
#print answer
root_dr = chained_request['chain'][2]
root_dr_req = mcm.get('requests', root_dr)
root_dr1 = chained_request['chain'][3]
root_dr1_req = mcm.get('requests', root_dr1)
answer = mcm._McM__get('restapi/chained_requests/inject/%s' %
(chained_request['prepid']))
if (len(chained_request['chain']) > 4):
PRIORITY_BLOCK = 2
TICKET_NOTE ='Summer16 miniAODv3/nanoAODv3 reminiAOD'
org_chained_campaign = 'chain_RunIIWinter15wmLHE_flowLHE2Summer15GS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16PremixMiniAODv2_flowRunIISummer16NanoAOD'
dest_chained_campaign = 'chain_RunIIWinter15wmLHE_flowLHE2Summer15GS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3'
# If dry run is enabled, nothing will be uploaded to McM
dry_run = False
# McM instance
mcm = McM(dev=True)#, cookie='/afs/cern.ch/work/s/snorberg/miniAODv3/cookie.txt')#/afs/cern.ch/user/j/jrumsevi/private/dev_cookie.txt')
# Set list of PWGs here. If no list is set, all PWGs from McM will be used
pwgs = ['SMP']
if not pwgs:
# Get list of PWGs
pwgs = mcm._McM__get('restapi/users/get_pwg')['results']
print('Got PWGs: %s' % (pwgs))
# Create a dict of dicts of dicts of integers
collector = defaultdict(lambda: defaultdict(lambda: defaultdict(int)))
for pwg in pwgs:
chained_requests = mcm.get('chained_requests', query='member_of_campaign=%s&pwg=%s' % (org_chained_campaign, pwg))
# print('PWG:%s\tChained requests:%s' % (pwg, len(chained_requests)))
for chained_request in chained_requests:
root_id = chained_request['chain'][0]
campaign = root_id.split('-')[1]
collector[pwg][campaign][root_id] += 1
# Collector now contains all root requests of chains that are member of org_chained_campaign campaign
# It's structured like this: PWG -> campaign name of root request -> request prepid -> count
for chunk in chunks(requests_for_that_repeat, N_REQUESTS_PER_TICKET):
mccm_ticket = {
'prepid': 'PPD',
'pwg': 'PPD',
'requests': chunk,
'notes': TICKET_NOTE,
'chains': [str(dchain)],
'block': PRIORITY_BLOCK
}
all_tickets.append(mccm_ticket)
for ticket in all_tickets:
print('Create ticket:\n%s' % (json.dumps(ticket, indent=4)))
if not is_dry_run:
res = mcm.put('mccms', ticket)
ticket_prepid = res.get('prepid', None)
print('Ticket prepid: %s' % (ticket_prepid))
print('Generating and reserving requests...')
res = mcm._McM__get('restapi/mccms/generate/%s/reserve/' %
(ticket_prepid))
#if res is not None:
# for ch_req in res.get('message'):
# print('Flow %s' % (ch_req['prepid']))
# flow_res = mcm.get('chained_requests', ch_req['prepid'], 'flow')
# print(flow_res)
# time.sleep(0.1)
#else:
# print('Error generating and reserving for %s' % (ticket_prepid))
time.sleep(0.2)
print("Which requests are in the chain? " +
str(request_in_chain['prepid']))
if ((request_in_chain['prepid'].find("NanoAODv3"))
and (request_in_chain['status'] == "submitted")):
print(' Found ' + str(chain_prepid))
chain['action_parameters']['flag'] = True
continue
if len(request_in_chain.get('member_of_chain', [])) > 1:
print(
' Not deleting %s because it is a member of %d chains' %
(request_in_chain['prepid'],
len(request_in_chain.get('member_of_chain', []))))
else:
print('Going to operate in ' + str(request_in_chain['prepid']))
if (request_in_chain['status'] == "new"):
mcm._McM__get('restapi/chained_requests/rewind/%s' %
(chain_prepid))
#mcm.get('requests', request_in_chain['prepid'], method='none')
request_delete_result = mcm.delete(
'requests', request_in_chain['prepid'])
print(' Deleted %s. %s' %
(request_in_chain['prepid'], request_delete_result))
#chain_delete_result = mcm.delete('chained_requests', chain_prepid)
chain_delete_result = mcm.delete('chained_requests', chain_prepid)
print(' Deleted %s. %s' % (chain_prepid, chain_delete_result))
from json import dumps
mcm = McM(dev=False,
cookie='/afs/cern.ch/user/p/pgunnell/private/prod-cookie.txt')
# Script clones a request to other campaign.
# Fefine list of modifications
# If member_of_campaign is different, it will clone to other campaign
list_of_nonflowing = "list_of_forceflow"
# Get a request object which we want to clone
chained_requests = mcm.get('lists', list_of_nonflowing, method='get')
chained_requests = list(reversed(chained_requests['value']))
print chained_requests
# Make predefined modifications
print len(chained_requests)
for i in range(0, len(chained_requests)):
if (chained_requests[i].find('MiniAODv3') != -1):
print chained_requests[i]
chained_request = mcm.get('chained_requests', chained_requests[i])
mcm._McM__get('restapi/chained_requests/flow/%s/force' %
(chained_request['prepid']))
# If dry run is enabled, nothing will be uploaded to McM
dry_run = False #True
# McM instance
#mcm = McM(dev=True, cookie='/afs/cern.ch/work/s/snorberg/miniAODv3/cookie.txt')#/afs/cern.ch/user/j/jrumsevi/private/dev_cookie.txt')
mcm = McM(dev=False) #/afs/cern.ch/user/j/jrumsevi/private/dev_cookie.txt')
# Set list of PWGs here. If no list is set, all PWGs from McM will be used
#pwgs = ['MUO','BTV','JME','TAU','EGM','TSG','SMP','HCA','B2G']
pwgs = ['SUS']
#pwgs = ['SUS','TOP','HIG']
#pwgs = ['BPH']
if not pwgs:
# Get list of PWGs
pwgs = mcm._McM__get('restapi/users/get_pwg')['results']
print('Got PWGs: %s' % (pwgs))
# Create a dict of dicts of dicts of integers
collector = defaultdict(lambda: defaultdict(lambda: defaultdict(int)))
for pwg in pwgs:
chained_requests = mcm.get('chained_requests',
query='member_of_campaign=%s&pwg=%s' %
(org_chained_campaign, pwg))
print('PWG:%s\tChained requests:%s' % (pwg, len(chained_requests)))
for chained_request in chained_requests:
root_id = chained_request['chain'][0]
import sys
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
from json import dumps
mcm = McM(dev=False)
# Script clones a request to other campaign.
# Fefine list of modifications
# If member_of_campaign is different, it will clone to other campaign
# Get a request object which we want to clone
requests = mcm.get('requests', query='status=approved&approval=submit')
print len(requests)
for request in requests:
chained_requests = request['member_of_chain']
for chained_request_loop in chained_requests:
answer = mcm._McM__get('restapi/chained_requests/inject/%s' % (chained_request_loop))
print answer
if (root_id_req['keep_output'][0] != True
and root_id_req['status'] == 'done'):
print(
str(root_id_req['dataset_name']) + ' ' +
str(root_id_req['prepid']) +
' https://cms-pdmv.cern.ch/mcm/requests?prepid=' +
str(root_id_req['prepid']))
#if(chained_request)
chained_request['action_parameters']['flag'] = False
mcm.update('chained_requests', chained_request)
if ('MiniAODv2' in chained_request['prepid']):
mcm._McM__get('restapi/chained_requests/rewind/%s' %
(chained_request['prepid']))
mcm._McM__get('restapi/requests/reset/%s' %
(root_nano_req['prepid']))
request_delete_result = mcm.delete('requests',
root_nano_req['prepid'])
mcm._McM__get('restapi/chained_requests/rewind/%s' %
(chained_request['prepid']))
mcm._McM__get('restapi/requests/reset/%s' %
(root_mini_req['prepid']))
request_delete_result = mcm.delete('requests',
root_mini_req['prepid'])
mcm._McM__get('restapi/chained_requests/rewind/%s' %
(chained_request['prepid']))
mcm._McM__get('restapi/requests/reset/%s' %
if (len(chained_request['chain']) > 3):
nano_id = chained_request['chain'][3]
root_nano_req = mcm.get('requests', nano_id)
#if(root_id_req['keep_output'][0] != True and root_id_req['status']=='done' and root_dr_req['status']=='new')
#if(root_id_req['keep_output'][0] != True and root_id_req['status']=='done' and root_dr_req['status']=='done'):
if (root_id_req['status'] == 'done' and root_dr_req['status'] == 'done'
and 'AOD' in root_mini_req['input_dataset']):
print(
str(root_id_req['dataset_name']) + ' ' +
str(root_id_req['prepid']) +
' https://cms-pdmv.cern.ch/mcm/requests?prepid=' +
str(root_id_req['prepid']))
answer = mcm._McM__get('restapi/requests/approve/%s' %
(root_mini_req['prepid']))
answer = mcm._McM__get('restapi/requests/approve/%s' %
(root_nano_req['prepid']))
#mcm._McM__get('restapi/chained_requests/flow/%s/force' % (chained_request['prepid']))
continue
chained_request['action_parameters']['flag'] = False
answer = mcm.update('chained_requests', chained_request)
print answer
if (len(chained_request['chain']) > 3):
mcm._McM__get('restapi/chained_requests/rewind/%s' %
(chained_request['prepid']))
#mcm._McM__get('restapi/requests/reset/%s' % (root_nano_req['prepid']))
answer = mcm.delete('requests', root_nano_req['prepid'])
chained_requests = request['member_of_chain']
chained_request = mcm.get('chained_requests', chained_requests[0])
root_id = chained_request['chain'][0]
root_id_req = mcm.get('requests', root_id)
root_gs = chained_request['chain'][1]
root_gs_req = mcm.get('requests', root_gs)
if(root_id_req['status']=='done' and root_gs_req['status']=='done'):
print str(request['prepid'])+' '+str(request['dataset_name'])
answer = mcm._McM__get('restapi/requests/reset/%s' % (request['prepid']))
print answer
root_dr = chained_request['chain'][2]
root_dr_req = mcm.get('requests', root_dr)
root_dr1 = chained_request['chain'][3]
root_dr1_req = mcm.get('requests', root_dr1)
if('MiniAODv3' in root_dr_req['prepid']):
answer = mcm._McM__get('restapi/requests/reset/%s' % (root_dr_req['prepid']))
print answer
if('MiniAODv3' in root_dr1_req['prepid']):
answer = mcm._McM__get('restapi/requests/reset/%s' % (root_dr1_req['prepid']))
print answer
# Fefine list of modifications
# If member_of_campaign is different, it will clone to other campaign
# Get a request object which we want to clone
#chained_requests = mcm.get('chained_requests', 'B2G-chain_RunIISummer15wmLHEGS_flowRunIISummer16DR80PremixPUMoriond17_flowRunIISummer16MiniAODv3_flowRunIISummer16NanoAODv3-00208')
dest_chained_campaign = 'RunIIFall17DRPremix'
requests = mcm.get('requests',
query='member_of_campaign=%s&status=approved' %
(dest_chained_campaign))
# Make predefined modifications
print('Number of requests ' + str(len(requests)))
for request in requests:
chained_request = request['member_of_chain']
chained_request = mcm.get('chained_requests', chained_request[0])
root_id = chained_request['chain'][0]
root_id_req = mcm.get('requests', root_id)
print root_id_req['keep_output']
if (root_id_req['keep_output']):
print chained_request['prepid']
mcm._McM__get('restapi/chained_requests/inject/%s' %
(chained_request['prepid']))
import sys
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM
from json import dumps
mcm = McM(dev=False)
# Script clones a request to other campaign.
# Fefine list of modifications
# If member_of_campaign is different, it will clone to other campaign
list_of_nonflowing = "list_of_nonflowing_chains"
# Get a request object which we want to clone
chained_requests = mcm.get('lists', list_of_nonflowing, method='get')
chained_requests = chained_requests['value']
# Make predefined modifications
print len(chained_requests)
for i in range(0, len(chained_requests)):
if (chained_requests[i]['chain'].find('MiniAODv3') != -1):
print chained_requests[i]['chain']
mcm._McM__get('restapi/chained_requests/flow/%s/force' %
(chained_requests[i]['chain']))
Total = 0
request_n=0
for i in range(0,400):
requests = mcm.get('requests', query='prepid=%s' % (numberEvents[i][1]))
for request in requests:
chained_request_id = request['member_of_chain'][0]
chained_requests = mcm.get('chained_requests', query='prepid=%s' % (chained_request_id))
root_request = chained_requests[0]['chain'][0]
dr_request = chained_requests[0]['chain'][1]
request_to_check = mcm.get('requests', root_request)
dr_request_to_check = mcm.get('requests', dr_request)
if(dr_request_to_check['status'] == 'done' and request_to_check['status'] == 'done'):
request_n+=1
print numberEvents[i][1]
Total += numberEvents[i][0]
for ch_request in chained_requests:
print(mcm._McM__get('restapi/chained_requests/rewind/%s' % (ch_request['prepid'])))
print(mcm._McM__get('restapi/chained_requests/rewind/%s' % (ch_request['prepid'])))
print(mcm.forceflow(ch_request['prepid']))
print("Total is "+str(Total))
