def get_cmsDriver(self, data):
try:
camp_mcm = campaign(json_input=data)
except campaign.IllegalAttributeName as ex:
return {"results":''}
return {"results":camp_mcm.build_cmsDrivers()}
def create_campaign(self, data):
db = database('campaigns')
try:
camp_mcm = campaign(json_input=threaded_loads(data))
except campaign.IllegalAttributeName as ex:
return {"results":False}
#id = RequestPrepId().generate_prepid(self.json['pwg'], self.json['member_of_campaign'])
#self.json['prepid'] = loads(id)['prepid']
if not camp_mcm.get_attribute('prepid'):
self.logger.error('Invalid prepid: Prepid returned None')
return {"results":False}
if '_' in camp_mcm.get_attribute('prepid'):
self.logger.error('Invalid campaign name %s'%(camp_mcm.get_attribute('prepid')))
return {"results":False}
camp_mcm.set_attribute('_id', camp_mcm.get_attribute('prepid'))
camp_mcm.update_history({'action':'created'})
## this is to create, not to update
if db.document_exists( camp_mcm.get_attribute('prepid') ):
return {"results":False}
# save to db
if not db.save(camp_mcm.json()):
self.logger.error('Could not save object to database')
return {"results":False}
# create dedicated chained campaign
self.create_chained_campaign(camp_mcm.get_attribute('_id'), db)
return {"results":True}
def next_prepid(self, pwg, camp):
if not pwg or not camp:
return None
with locker.lock("{0}-{1}".format(pwg, camp)):
db = database(self.db_name)
query_results = db.raw_query('serial_number', {
'group': True,
'key': [camp, pwg]
})
sn = 1
if query_results:
sn = query_results[0]['value'] + 1
pid = '%s-%s-%05d' % (pwg, camp, sn)
if sn == 1:
self.logger.info('Beginning new prepid family: %s-%s' %
(pwg, camp))
db_camp = database('campaigns', cache_enabled=True)
req_camp = campaign(db_camp.get(camp))
new_request = request(
req_camp.add_request({
'_id': pid,
'prepid': pid,
'pwg': pwg,
'member_of_campaign': camp
}))
new_request.update_history({'action': 'created'})
db.save(new_request.json())
self.logger.info('New prepid : %s ' % pid)
return pid
def post(self):
"""
Change campaign hold status. Content type 'application/json' must be set.
Request is a JSON with 'prepid' and 'on_hold' (0/1) attributes.
"""
data = flask.request.data
try:
data_json = loads(data)
campaign_name = data_json['prepid']
campaign_hold = int(data_json['on_hold'])
except:
return self.form_response({'error': 'Request must have \'prepid\' and \'on_hold\' attributes'}, code=400)
if campaign_hold != 0 and campaign_hold != 1:
return self.form_response({'error': 'campaign_hold must be (0/1) or (true/false)'}, code=400)
db = database('campaigns')
if not db.document_exists(campaign_name):
return self.form_response({'error': 'Campaign %s is not found' % (campaign_name)}, code=404)
camp = campaign(json_input=db.get(campaign_name))
camp.set_attribute('on_hold', campaign_hold)
result = db.update(camp.json())
if result:
return self.form_response({'result': 'success'}, code=200)
else:
return self.form_response({'result': 'failure'}, code=500)
def are_campaigns_correct(self, next_c, allowed_c, cdb):
if next_c:
if not cdb.document_exists(next_c):
return {
"results":
False,
"message":
'{0} is not a valid campaign for next'.format(next_c)
}
next_mcm_c = campaign(cdb.get(next_c))
if not self.is_energy_consistent(next_mcm_c, allowed_c, cdb):
return {
"results":
False,
"message":
'Next campaign {0} and allowed campaigns have inconsistent energies'
.format(next_c)
}
##consistency check
if next_c in allowed_c:
return {
"results": False,
"message":
"Cannot have next campaign in the allowed campaign"
}
return True
def update_campaign(self, data):
if '_rev' not in data:
return {
"results": False,
'message': 'There is no previous revision provided'
}
try:
camp_mcm = campaign(json_input=data)
except campaign.IllegalAttributeName:
return {"results": False}
if not camp_mcm.get_attribute('prepid') and not camp_mcm.get_attribute(
'_id'):
raise ValueError('Prepid returned was None')
# cast schema evolution of sequences
sequences = camp_mcm.get_attribute('sequences')
for steps in sequences:
for label in steps:
steps[label] = sequence(steps[label]).json()
camp_mcm.set_attribute('sequences', sequences)
# create dedicated chained campaign
self.create_chained_campaign(camp_mcm.get_attribute('_id'),
camp_mcm.get_attribute('root'))
camp_mcm.update_history({'action': 'update'})
return self.save_campaign(camp_mcm)
def create_campaign(self, data):
db = database('campaigns')
try:
camp_mcm = campaign(json_input=loads(data))
except campaign.IllegalAttributeName as ex:
return {"results": False}
if not camp_mcm.get_attribute('prepid'):
self.logger.error('Invalid prepid: Prepid returned None')
return {"results": False}
if '_' in camp_mcm.get_attribute('prepid'):
self.logger.error('Invalid campaign name %s' %
(camp_mcm.get_attribute('prepid')))
return {"results": False}
camp_mcm.set_attribute('_id', camp_mcm.get_attribute('prepid'))
camp_mcm.update_history({'action': 'created'})
# this is to create, not to update
if db.document_exists(camp_mcm.get_attribute('prepid')):
return {"results": False}
# save to db
if not db.save(camp_mcm.json()):
self.logger.error('Could not save object to database')
return {"results": False}
# create dedicated chained campaign
self.create_chained_campaign(camp_mcm.get_attribute('_id'), db)
return {"results": True}
def get_cmsDriver(self, data):
try:
camp_mcm = campaign(json_input=data)
except campaign.IllegalAttributeName:
return {"results": ''}
return {"results": camp_mcm.build_cmsDrivers()}
def toggle_campaign(self, rid):
db = database('campaigns')
if not db.document_exists(rid):
return {"results": 'Error: The given campaign id does not exist.'}
camp = campaign(json_input=db.get(rid))
camp.toggle_approval()
return {"results": db.update(camp.json())}
def toggle_campaign(self, rid):
db = database('campaigns')
if not db.document_exists(rid):
return {"results":'Error: The given campaign id does not exist.'}
camp = campaign(json_input=db.get(rid))
camp.toggle_approval()
return {"results":db.update(camp.json())}
def is_energy_consistent(self, next_c, allowed_c, cdb):
"""
Checks if the energy of campaigns is consistent (it cannot differ)
"""
next_energy = next_c.get_attribute('energy')
for camp in allowed_c:
mcm_c = campaign(cdb.get(camp))
if mcm_c.get_attribute('energy') != next_energy:
return False
return True
def toggle_campaign(self, rid, index):
db = database('campaigns')
if not db.document_exists(rid):
return {"prepid": rid, "results":'Error: The given campaign id does not exist.'}
camp = campaign(json_input=db.get(rid))
camp.approve(int(index))
if int(index)==0:
camp.set_status(0)
res=db.update(camp.json())
return {"prepid": rid, "results": res, "approval" : camp.get_attribute('approval')}
def is_energy_consistent(self, next_c, allowed_c, cdb):
"""
Checks if the energy of campaigns is consistent (it cannot differ)
"""
next_energy = next_c.get_attribute('energy')
for camp in allowed_c:
mcm_c = campaign(cdb.get(camp))
if mcm_c.get_attribute('energy') != next_energy:
return False
return True
def are_campaigns_correct(self, next_c, allowed_c, cdb):
if next_c:
if not cdb.document_exists(next_c):
return {"results": False, "message": '{0} is not a valid campaign for next'.format(next_c)}
next_mcm_c = campaign(cdb.get(next_c))
if not self.is_energy_consistent(next_mcm_c, allowed_c, cdb):
return {"results": False,
"message": 'Next campaign {0} and allowed campaigns have inconsistent energies'.format(next_c)}
##consistency check
if next_c in allowed_c:
return {"results": False, "message": "Cannot have next campaign in the allowed campaign"}
return True
def toggle_campaign(self, rid):
db = database('campaigns')
if not db.document_exists(rid):
return {"results": 'Error: The given campaign id does not exist.'}
camp = campaign(json_input=db.get(rid))
try:
camp.toggle_status()
saved = db.update(camp.json())
if saved:
return {"results": True}
else:
return {"results": False, "message": "Could not save request"}
except Exception as ex:
return {"results": False, "message": str(ex)}
def toggle_campaign(self, rid):
db = database('campaigns')
if not db.document_exists(rid):
return {"results":'Error: The given campaign id does not exist.'}
camp = campaign(json_input=db.get(rid))
try:
camp.toggle_status()
saved = db.update(camp.json())
if saved:
return {"results":True}
else:
return {"results":False, "message":"Could not save request"}
except Exception as ex:
return {"results":False, "message": str(ex) }
def toggle_campaign(self, rid, index):
db = database('campaigns')
if not db.document_exists(rid):
return {
"prepid": rid,
"results": 'Error: The given campaign id does not exist.'
}
camp = campaign(json_input=db.get(rid))
camp.approve(int(index))
if int(index) == 0:
camp.set_status(0)
res = db.update(camp.json())
return {
"prepid": rid,
"results": res,
"approval": camp.get_attribute('approval')
}
def next_prepid(self, pwg, camp):
if not pwg or not camp:
return None
with locker.lock("{0}-{1}".format(pwg, camp)):
db = database(self.db_name)
query_results = db.raw_query('serial_number', {'group':True, 'key':[camp, pwg]})
sn = 1
if query_results:
sn = query_results[0]['value']+1
pid='%s-%s-%05d'%( pwg, camp , sn)
if sn==1:
self.logger.log('Beginning new prepid family: %s-%s' %( pwg, camp))
db_camp = database('campaigns', cache=True)
req_camp = campaign(db_camp.get(camp))
new_request = request(req_camp.add_request({'_id':pid, 'prepid':pid, 'pwg':pwg, 'member_of_campaign':camp}))
new_request.update_history({'action':'created'})
db.save(new_request.json())
self.logger.log('New prepid : %s '%pid)
return pid
def update_campaign(self, data):
if '_rev' not in data:
return {"results": False, 'message': 'There is no previous revision provided'}
try:
camp_mcm = campaign(json_input=data)
except campaign.IllegalAttributeName:
return {"results": False}
if not camp_mcm.get_attribute('prepid') and not camp_mcm.get_attribute('_id'):
raise ValueError('Prepid returned was None')
# cast schema evolution of sequences
sequences = camp_mcm.get_attribute('sequences')
for steps in sequences:
for label in steps:
steps[label] = sequence(steps[label]).json()
camp_mcm.set_attribute('sequences', sequences)
# create dedicated chained campaign
self.create_chained_campaign(camp_mcm.get_attribute('_id'), camp_mcm.get_attribute('root'))
camp_mcm.update_history({'action': 'update'})
return self.save_campaign(camp_mcm)
def update_campaigns(self, next_c, allowed, cdb):
# check to see if next_c is legal
if not cdb.document_exists(next_c):
raise ValueError('Campaign ' + str(next_c) + ' does not exist.')
if not next_c:
return
n = cdb.get(next_c)
if n['root'] == 0:
raise ValueError('Campaign ' + str(next_c) + ' is a root campaign.')
# iterate through all allowed campaigns and update the next_c field
for c in allowed:
camp = campaign(json_input=cdb.get(c))
try:
# append campaign
camp.add_next(next_c)
except campaign.CampaignExistsException:
pass
# save to database
cdb.update(camp.json())
def flow_to_next_step(self,
input_dataset='',
block_black_list=None,
block_white_list=None,
check_stats=True,
reserve=False):
if not block_white_list: block_white_list = []
if not block_black_list: block_black_list = []
self.logger.log('Flowing chained_request %s to next step...' %
(self.get_attribute('_id')))
if not self.get_attribute('chain'):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'chained_request %s has got no root' %
(self.get_attribute('_id')))
# check on the approval of the chained request before all
## let it be flowing regardless
#if self.get_attribute('approval') == 'none':
# raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
# 'The approval of the chained request is none, and therefore flow cannot happen')
#this operation requires to access all sorts of objects
rdb = database('requests')
cdb = database('campaigns')
ccdb = database('chained_campaigns')
crdb = database('chained_requests')
fdb = database('flows')
adb = database('actions')
l_type = locator()
current_step = len(self.get_attribute(
'chain')) - 1 if reserve else self.get_attribute('step')
current_id = self.get_attribute('chain')[current_step]
next_step = current_step + 1
if not rdb.document_exists(current_id):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'the request %s does not exist' % current_id)
current_request = request(rdb.get(current_id))
current_campaign = campaign(
cdb.get(current_request.get_attribute('member_of_campaign')))
if not ccdb.document_exists(self.get_attribute('member_of_campaign')):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'the chain request %s is member of %s that does not exist' %
(self.get_attribute('_id'),
self.get_attribute('member_of_campaign')))
mcm_cc = ccdb.get(self.get_attribute('member_of_campaign'))
if next_step >= len(mcm_cc['campaigns']):
if reserve: return False
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'chained_campaign %s does not allow any further flowing.' %
(self.get_attribute('member_of_campaign')))
if not reserve:
## is the current request in the proper approval
allowed_request_approvals = ['submit']
if current_request.get_attribute(
'approval') not in allowed_request_approvals:
raise self.NotApprovedException(
current_request.get_attribute('prepid'),
current_request.get_attribute('approval'),
allowed_request_approvals)
## is the current request in the proper status
allowed_request_statuses = ['submitted', 'done']
if current_request.get_attribute(
'status') not in allowed_request_statuses:
raise self.NotInProperStateException(
current_request.get_attribute('prepid'),
current_request.get_attribute('status'),
allowed_request_statuses)
original_action_id = self.get_attribute('chain')[0]
original_action_item = self.retrieve_original_action_item(
adb, original_action_id)
## what is the campaign to go to next and with which flow
(next_campaign_id, flow_name) = mcm_cc['campaigns'][next_step]
if not fdb.document_exists(flow_name):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'The flow %s does not exist' % flow_name)
mcm_f = flow(fdb.get(flow_name))
if not 'sequences' in mcm_f.get_attribute('request_parameters'):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'The flow %s does not contain sequences information.' %
(flow_name))
if not cdb.document_exists(next_campaign_id):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'The next campaign %s does not exist' % next_campaign_id)
next_campaign = campaign(cdb.get(next_campaign_id))
if len(next_campaign.get_attribute('sequences')) != len(
mcm_f.get_attribute('request_parameters')['sequences']):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'the sequences changes in flow %s are not consistent with the next campaign %s'
% (flow_name, next_campaign_id))
if next_campaign.get_attribute(
'energy') != current_campaign.get_attribute('energy'):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'cannot flow any further. Request {0} has inconsistent energy.'
.format(next_campaign.get_attribute("prepid")))
if not next_campaign.is_release_greater_or_equal_to(
current_campaign.get_attribute('cmssw_release')):
raise self.ChainedRequestCannotFlowException(
self.get_attribute("_id"),
'cannot flow any further. Request {0} has lower release version.'
.format(next_campaign.get_attribute("prepid")))
if next_campaign.get_attribute('type') == 'MCReproc' and (
not 'time_event' in mcm_f.get_attribute('request_parameters')):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'the flow is getting into a MCReproc campaign but not time per event is specified'
)
if next_campaign.get_attribute('type') == 'MCReproc' and (
not 'size_event' in mcm_f.get_attribute('request_parameters')):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'the flow is getting into a MCReproc campaign but not size per event is specified'
)
## check that it is allowed to flow
allowed_flow_approvals = ['flow', 'submit']
###### cascade of checks
"""
if not reserve and not mcm_f.get_attribute('approval') in allowed_flow_approvals:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The flow (%s) is not in proper approval state (%s)'%(
mcm_f.get_attribute('prepid'),
mcm_f.get_attribute('approval')))
if not reserve and not self.get_attribute('approval') in allowed_flow_approvals:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The chained request (%s) is not in the proper approval state (%s)'% (
self.get_attribute('_id'),
self.get_attribute('approval')))
"""
if not reserve and not mcm_f.get_attribute(
'approval') in allowed_flow_approvals:
if not self.get_attribute('approval') in allowed_flow_approvals:
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'Neither the flow (%s) nor the chained request (%s) approvals allow for flowing'
% (mcm_f.get_attribute('approval'),
self.get_attribute('approval')))
if next_campaign.get_attribute('status') == 'stopped':
raise self.CampaignStoppedException(next_campaign_id)
#what is going to be the required number of events for the next request
#update the stats to its best
if not reserve:
current_request.get_stats()
next_total_events = current_request.get_attribute(
'completed_events')
## get the original expected events and allow a margin of 5% less statistics
statistics_fraction = settings().get_value('statistics_fraction')
current_eff_error = 1. - current_request.get_efficiency_error()
statistics_fraction = min(statistics_fraction, current_eff_error)
completed_events_to_pass = int(
current_request.get_attribute('total_events') *
statistics_fraction)
notify_on_fail = True ## to be tuned according to the specific cases
if current_request.get_attribute('completed_events') <= 0:
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'The number of events completed is negative or null')
else:
allowed_request_statuses = ['done']
## determine if this is a root -> non-root transition to potentially apply staged number
at_a_transition = (current_campaign.get_attribute('root') != 1
and next_campaign.get_attribute('root')
== 1)
if ('staged' in original_action_item or 'threshold'
in original_action_item) and at_a_transition:
allowed_request_statuses.append('submitted')
##check status
if not current_request.get_attribute(
'status') in allowed_request_statuses:
raise self.NotInProperStateException(
current_request.get_attribute('prepid'),
current_request.get_attribute('status'),
allowed_request_statuses)
##special check at transition that the statistics is good enough
if at_a_transition:
# at a root -> non-root transition only does the staged/threshold functions !
if 'staged' in original_action_item:
next_total_events = int(original_action_item['staged'])
completed_events_to_pass = next_total_events
if 'threshold' in original_action_item:
next_total_events = int(
current_request.get_attribute('total_events') *
float(original_action_item['threshold'] / 100.))
completed_events_to_pass = next_total_events
if check_stats and (
current_request.get_attribute('completed_events') <
completed_events_to_pass):
if notify_on_fail:
current_request.notify(
'Flowing %s with not enough statistics' %
(current_request.get_attribute('prepid')),
'For the request %s, the completed statistics %s is not enough to fullfill the requirement to the next level : need at least %s in chain %s \n\n Please report to the operation HN or at the next MccM what action should be taken.\n\n %srequests?prepid=%s\n%schained_requests?contains=%s\n%schained_requests?prepid=%s '
% (current_request.get_attribute('prepid'),
current_request.get_attribute('completed_events'),
completed_events_to_pass,
self.get_attribute('prepid'), l_type.baseurl(),
current_request.get_attribute('prepid'),
l_type.baseurl(),
current_request.get_attribute('prepid'),
l_type.baseurl(), self.get_attribute('prepid')),
accumulate=True)
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'The number of events completed (%s) is not enough for the requirement (%s)'
% (current_request.get_attribute('completed_events'),
completed_events_to_pass))
## select what is to happened : [create, patch, use]
next_id = None
approach = None
next_request = None
if next_step != len(self.get_attribute('chain')):
#not at the end
next_id = self.get_attribute('chain')[next_step]
if not rdb.document_exists(next_id):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'The next request (%s) according to the step (%s) does not exist'
% (next_id, next_step))
next_request = request(rdb.get(next_id))
if next_request.get_attribute('status') == 'new':
#most likely a rewind + resub
approach = 'patch'
else:
##this is always the case in chains reserved from existing things: so use the next request
approach = 'use'
#raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
#'This should never happen. (%s) is next according to step (%s), but is not in new status (%s)' % (
#next_id, next_step, next_request.get_attribute('status')))
else:
## look in *other* chained campaigns whether you can suck in an existing request
## look up all chained requests that start from the same root request
## remove <pwg>-chain_ and the -serial number, replacing _ with a .
toMatch = '.'.join(
self.get_attribute('prepid').split('_')[1:][0:next_step +
1]).split('-')[0]
## make sure they get ordered by prepid
related_crs = sorted(crdb.queries(
['root_request==%s' % original_action_id]),
key=lambda cr: cr['prepid'])
vetoed_last = []
for existing_cr in related_crs:
## exclude itself
if existing_cr['prepid'] == self.get_attribute('prepid'):
continue
## prevent from using a request from within the same exact chained_campaigns
if existing_cr['member_of_campaign'] == self.get_attribute(
'member_of_campaign'):
mcm_cr = chained_request(crdb.get(existing_cr['prepid']))
if len(mcm_cr.get_attribute('chain')) > next_step:
## one existing request in the very same chained campaign has already something used, make sure it is not going to be used
vetoed_last.append(
mcm_cr.get_attribute('chain')[next_step])
continue
else:
continue
for existing_cr in related_crs:
## exclude itself
if existing_cr['prepid'] == self.get_attribute('prepid'):
continue
## prevent from using a request from within the same exact chained_campaigns
if existing_cr['member_of_campaign'] == self.get_attribute(
'member_of_campaign'):
continue
truncated = '.'.join(
existing_cr['prepid'].split('_')[1:][0:next_step +
1]).split('-')[0]
self.logger.error('to match : %s , this one %s' %
(toMatch, truncated))
if truncated == toMatch:
#we found a chained request that starts with all same steps
mcm_cr = chained_request(crdb.get(existing_cr['prepid']))
if len(mcm_cr.get_attribute('chain')) <= next_step:
#found one, but it has not enough content either
continue
if mcm_cr.get_attribute('chain')[next_step] in vetoed_last:
continue
next_id = mcm_cr.get_attribute('chain')[next_step]
break
if next_id:
approach = 'use'
else:
approach = 'create'
if approach == 'create':
from rest_api.RequestPrepId import RequestPrepId
next_id = RequestPrepId().next_prepid(
current_request.get_attribute('pwg'), next_campaign_id)
next_request = request(rdb.get(next_id))
request.transfer(current_request, next_request)
self.request_join(next_request)
elif approach == 'use':
## there exists a request in another chained campaign that can be re-used here.
# take this one. advance and go on
next_request = request(rdb.get(next_id))
if not reserve:
self.set_attribute('step', next_step)
self.set_attribute('last_status',
next_request.get_attribute('status'))
self.update_history({'action': 'flow', 'step': str(next_id)})
self.set_attribute('status', 'processing')
if not self.get_attribute("prepid") in next_request.get_attribute(
"member_of_chain"):
## register the chain to the next request
self.request_join(next_request)
saved = rdb.update(next_request.json())
if not saved:
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'Unable to save %s with updated member_of_chains' %
next_id)
return True
elif approach == 'patch':
## there exists already a request in the chain (step!=last) and it is usable for the next stage
next_request = request(next_campaign.add_request(rdb.get(next_id)))
## propagate again some of the fields of the previous request.
request.transfer(current_request, next_request)
else:
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'Unrecognized approach %s' % approach)
#current_request -> next_request
#current_campaign -> next_campaign
##determine whether we have an input dataset for the next request
if len(current_request.get_attribute('reqmgr_name')):
last_wma = current_request.get_attribute('reqmgr_name')[-1]
if 'content' in last_wma and 'pdmv_dataset_name' in last_wma[
'content']:
input_dataset = last_wma['content']['pdmv_dataset_name']
else:
statsDB = database('stats',
url='http://cms-pdmv-stats.cern.ch:5984/')
if statsDB.document_exists(last_wma['name']):
latestStatus = statsDB.get(last_wma['name'])
input_dataset = latestStatus['pdmv_dataset_name']
if input_dataset:
next_request.set_attribute('input_dataset', input_dataset)
## set blocks restriction if any
if block_black_list:
next_request.set_attribute('block_black_list', block_black_list)
if block_white_list:
next_request.set_attribute('block_white_list', block_white_list)
## register the flow to the request
next_request.set_attribute('flown_with', flow_name)
##assemble the campaign+flow => request
request.put_together(next_campaign, mcm_f, next_request)
if not reserve:
#already taking stage and threshold into account
next_request.set_attribute('total_events', next_total_events)
next_request.update_history({
'action': 'flow',
'step': self.get_attribute('prepid')
})
request_saved = rdb.save(next_request.json())
if not request_saved:
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'Could not save the new request %s' %
(next_request.get_attribute('prepid')))
## inspect priority
self.set_priority(original_action_item['block_number'])
if not reserve:
# sync last status
self.set_attribute('last_status',
next_request.get_attribute('status'))
# we can only be processing at this point
self.set_attribute('status', 'processing')
# set to next step
self.set_attribute('step', next_step)
self.update_history({
'action': 'flow',
'step': next_request.get_attribute('prepid')
})
if not reserve:
notification_subject = 'Flow for request %s in %s' % (
current_request.get_attribute('prepid'), next_campaign_id)
notification_text = 'The request %s has been flown within:\n \t %s \n into campaign:\n \t %s \n using:\n \t %s \n creating the new request:\n \t %s \n as part of:\n \t %s \n and from the produced dataset:\n %s \n\n%srequests?prepid=%s \n%srequests?prepid=%s \n' % (
current_request.get_attribute('prepid'),
self.get_attribute('member_of_campaign'), next_campaign_id,
flow_name, next_request.get_attribute('prepid'),
self.get_attribute('prepid'),
next_request.get_attribute('input_dataset'), l_type.baseurl(),
current_request.get_attribute('prepid'), l_type.baseurl(),
next_request.get_attribute('prepid'))
current_request.notify(notification_subject,
notification_text,
accumulate=True)
else:
notification_subject = 'Reservation of request {0}'.format(
next_request.get_attribute('prepid'))
notification_text = 'The request {0} of campaign \n\t{2}\nhas been reserved as part of \n\t{1}\nas the next step for {4}\n\n{3}requests?prepid={4}\n{5}requests?prepid={6}\n'.format(
next_request.get_attribute('prepid'),
self.get_attribute('prepid'),
next_campaign_id,
l_type.baseurl(),
current_request.get_attribute('prepid'),
l_type.baseurl(),
next_request.get_attribute('prepid'),
)
next_request.notify(notification_subject,
notification_text,
accumulate=True)
return True
def flow_to_next_step(self, input_dataset='', block_black_list=None, block_white_list=None, check_stats=True, reserve=False):
if not block_white_list: block_white_list = []
if not block_black_list: block_black_list = []
self.logger.log('Flowing chained_request %s to next step...' % (self.get_attribute('_id')))
if not self.get_attribute('chain'):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'chained_request %s has got no root' % (
self.get_attribute('_id')))
# check on the approval of the chained request before all
## let it be flowing regardless
#if self.get_attribute('approval') == 'none':
# raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
# 'The approval of the chained request is none, and therefore flow cannot happen')
#this operation requires to access all sorts of objects
rdb = database('requests')
cdb = database('campaigns')
ccdb = database('chained_campaigns')
crdb = database('chained_requests')
fdb = database('flows')
adb = database('actions')
l_type=locator()
current_step = len(self.get_attribute('chain'))-1 if reserve else self.get_attribute('step')
current_id = self.get_attribute('chain')[current_step]
next_step = current_step + 1
if not rdb.document_exists(current_id):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'the request %s does not exist' % current_id)
current_request = request(rdb.get(current_id))
current_campaign = campaign(cdb.get(current_request.get_attribute('member_of_campaign')))
if not ccdb.document_exists(self.get_attribute('member_of_campaign')):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'the chain request %s is member of %s that does not exist' % (
self.get_attribute('_id'),
self.get_attribute('member_of_campaign')))
mcm_cc = ccdb.get(self.get_attribute('member_of_campaign'))
if next_step >= len(mcm_cc['campaigns']):
if reserve: return False
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'chained_campaign %s does not allow any further flowing.' % (
self.get_attribute('member_of_campaign')))
if not reserve:
## is the current request in the proper approval
allowed_request_approvals = ['submit']
if current_request.get_attribute('approval') not in allowed_request_approvals:
raise self.NotApprovedException(current_request.get_attribute('prepid'),
current_request.get_attribute('approval'), allowed_request_approvals)
## is the current request in the proper status
allowed_request_statuses = ['submitted', 'done']
if current_request.get_attribute('status') not in allowed_request_statuses:
raise self.NotInProperStateException(current_request.get_attribute('prepid'),
current_request.get_attribute('status'),
allowed_request_statuses)
original_action_id = self.get_attribute('chain')[0]
original_action_item = self.retrieve_original_action_item(adb, original_action_id)
## what is the campaign to go to next and with which flow
(next_campaign_id, flow_name) = mcm_cc['campaigns'][next_step]
if not fdb.document_exists(flow_name):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The flow %s does not exist' % flow_name )
mcm_f = flow(fdb.get(flow_name))
if not 'sequences' in mcm_f.get_attribute('request_parameters'):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The flow %s does not contain sequences information.' % (
flow_name))
if not cdb.document_exists(next_campaign_id):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The next campaign %s does not exist' % next_campaign_id)
next_campaign = campaign(cdb.get(next_campaign_id))
if len(next_campaign.get_attribute('sequences')) != len(mcm_f.get_attribute('request_parameters')['sequences']):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'the sequences changes in flow %s are not consistent with the next campaign %s' % (
flow_name, next_campaign_id))
if next_campaign.get_attribute('energy') != current_campaign.get_attribute('energy'):
raise self.EnergyInconsistentException(next_campaign.get_attribute('prepid'))
if next_campaign.get_attribute('type') == 'MCReproc' and (not 'time_event' in mcm_f.get_attribute('request_parameters')):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'the flow is getting into a MCReproc campaign but not time per event is specified')
if next_campaign.get_attribute('type') == 'MCReproc' and (not 'size_event' in mcm_f.get_attribute('request_parameters')):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'the flow is getting into a MCReproc campaign but not size per event is specified')
## check that it is allowed to flow
allowed_flow_approvals = ['flow', 'submit']
###### cascade of checks
"""
if not reserve and not mcm_f.get_attribute('approval') in allowed_flow_approvals:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The flow (%s) is not in proper approval state (%s)'%(
mcm_f.get_attribute('prepid'),
mcm_f.get_attribute('approval')))
if not reserve and not self.get_attribute('approval') in allowed_flow_approvals:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The chained request (%s) is not in the proper approval state (%s)'% (
self.get_attribute('_id'),
self.get_attribute('approval')))
"""
if not reserve and not mcm_f.get_attribute('approval') in allowed_flow_approvals:
if not self.get_attribute('approval') in allowed_flow_approvals:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'Neither the flow (%s) nor the chained request (%s) approvals allow for flowing' % (
mcm_f.get_attribute('approval'),
self.get_attribute('approval')))
if next_campaign.get_attribute('status') == 'stopped':
raise self.CampaignStoppedException(next_campaign_id)
#what is going to be the required number of events for the next request
#update the stats to its best
if not reserve:
current_request.get_stats()
next_total_events=current_request.get_attribute('completed_events')
notify_on_fail=True ## to be tuned according to the specific cases
if current_request.get_attribute('completed_events') <= 0:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The number of events completed is negative or null')
else:
allowed_request_statuses = ['done']
## determine if this is a root -> non-root transition to potentially apply staged number
at_a_transition=(current_campaign.get_attribute('root') != 1 and next_campaign.get_attribute('root') == 1)
if ('staged' in original_action_item or 'threshold' in original_action_item) and at_a_transition:
allowed_request_statuses.append('submitted')
##check status
if not current_request.get_attribute('status') in allowed_request_statuses:
raise self.NotInProperStateException(current_request.get_attribute('prepid'),
current_request.get_attribute('status'),
allowed_request_statuses)
##special check at transition that the statistics is good enough
if at_a_transition:
# at a root -> non-root transition only does the staged/threshold functions !
if 'staged' in original_action_item:
next_total_events = int(original_action_item['staged'])
if 'threshold' in original_action_item:
next_total_events = int(current_request.get_attribute('total_events') * float(original_action_item['threshold'] / 100.))
completed_events_to_pass = next_total_events
else:
## get the original expected events and allow a margin of 5% less statistics
completed_events_to_pass = int(current_request.get_attribute('total_events') * 0.95)
if check_stats and (current_request.get_attribute('completed_events') < completed_events_to_pass):
if notify_on_fail:
current_request.notify('Flowing for %s: not enough statistics'%( current_request.get_attribute('prepid')),
'For this request, the completed statistics %s is not enough to fullfill the requirement to the next level : need at least %s \n\n %srequests?prepid=%s'%( current_request.get_attribute('completed_events'),
completed_events_to_pass,
l_type.baseurl(),
current_request.get_attribute('prepid')))
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The number of events completed (%s) is not enough for the requirement (%s)'%(current_request.get_attribute('completed_events'), completed_events_to_pass))
## select what is to happened : [create, patch, use]
next_id = None
approach = None
next_request = None
if next_step != len(self.get_attribute('chain')):
#not at the end
next_id = self.get_attribute('chain')[next_step]
if not rdb.document_exists(next_id):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The next request (%s) according to the step (%s) does not exist' % (
next_id, next_step))
next_request = request(rdb.get(next_id))
if next_request.get_attribute('status') == 'new':
#most likely a rewind + resub
approach = 'patch'
else:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'This should never happen. (%s) is next according to step (%s), but is not in new status (%s)' % (
next_id, next_step, next_request.get_attribute('status')))
else:
## look in *other* chained campaigns whether you can suck in an existing request
## look up all chained requests that start from the same root request
## remove <pwg>-chain_ and the -serial number, replacing _ with a .
toMatch = '.'.join(self.get_attribute('prepid').split('_')[1:][0:next_step + 1]).split('-')[0]
## make sure they get ordered by prepid
related_crs = sorted(crdb.queries(['root_request==%s' % original_action_id]), key=lambda cr : cr['prepid'])
vetoed_last=[]
for existing_cr in related_crs:
## exclude itself
if existing_cr['prepid']==self.get_attribute('prepid'):
continue
## prevent from using a request from within the same exact chained_campaigns
if existing_cr['member_of_campaign'] == self.get_attribute('member_of_campaign'):
mcm_cr = chained_request(crdb.get(existing_cr['prepid']))
if len(mcm_cr.get_attribute('chain')) > next_step:
## one existing request in the very same chained campaign has already something used, make sure it is not going to be used
vetoed_last.append( mcm_cr.get_attribute('chain')[next_step])
continue
else:
continue
for existing_cr in related_crs:
## exclude itself
if existing_cr['prepid']==self.get_attribute('prepid'):
continue
## prevent from using a request from within the same exact chained_campaigns
if existing_cr['member_of_campaign'] == self.get_attribute('member_of_campaign'):
continue
truncated = '.'.join(existing_cr['prepid'].split('_')[1:][0:next_step + 1]).split('-')[0]
self.logger.error('to match : %s , this one %s' % ( toMatch, truncated ))
if truncated == toMatch:
#we found a chained request that starts with all same steps
mcm_cr = chained_request(crdb.get(existing_cr['prepid']))
if len(mcm_cr.get_attribute('chain')) <= next_step:
#found one, but it has not enough content either
continue
if mcm_cr.get_attribute('chain')[next_step] in vetoed_last:
continue
next_id = mcm_cr.get_attribute('chain')[next_step]
break
if next_id:
approach = 'use'
else:
approach = 'create'
if approach == 'create':
from rest_api.RequestPrepId import RequestPrepId
next_id = RequestPrepId().next_prepid(current_request.get_attribute('pwg'), next_campaign_id)
next_request = request(rdb.get(next_id))
request.transfer( current_request, next_request)
self.request_join(next_request)
elif approach == 'use':
## there exists a request in another chained campaign that can be re-used here.
# take this one. advance and go on
next_request = request(rdb.get(next_id))
if not reserve:
self.set_attribute('step', next_step)
self.set_attribute('last_status', next_request.get_attribute('status'))
self.update_history({'action': 'flow', 'step': str(next_step)})
self.set_attribute('status', 'processing')
if not self.get_attribute("prepid") in next_request.get_attribute("member_of_chain"):
## register the chain to the next request
self.request_join(next_request)
saved = rdb.update(next_request.json())
if not saved:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'Unable to save %s with updated member_of_chains' % next_id)
return True
elif approach == 'patch':
## there exists already a request in the chain (step!=last) and it is usable for the next stage
next_request = request( next_campaign.add_request( rdb.get(next_id)))
### shouldn't this be added ?
#transfer( current_request, next_request)
else:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'Unrecognized approach %s' % approach )
#current_request -> next_request
#current_campaign -> next_campaign
##determine whether we have an input dataset for the next request
if len(current_request.get_attribute('reqmgr_name')):
last_wma = current_request.get_attribute('reqmgr_name')[-1]
if 'content' in last_wma and 'pdmv_dataset_name' in last_wma['content']:
input_dataset = last_wma['content']['pdmv_dataset_name']
else:
statsDB = database('stats', url='http://cms-pdmv-stats.cern.ch:5984/')
if statsDB.document_exists(last_wma['name']):
latestStatus = statsDB.get(last_wma['name'])
input_dataset = latestStatus['pdmv_dataset_name']
if input_dataset:
next_request.set_attribute('input_filename', input_dataset)
## set blocks restriction if any
if block_black_list:
next_request.set_attribute('block_black_list', block_black_list)
if block_white_list:
next_request.set_attribute('block_white_list', block_white_list)
## register the flow to the request
next_request.set_attribute('flown_with', flow_name)
##assemble the campaign+flow => request
request.put_together(next_campaign, mcm_f, next_request)
if not reserve:
#already taking stage and threshold into account
next_request.set_attribute('total_events', next_total_events)
next_request.update_history({'action': 'flow', 'step': self.get_attribute('prepid')})
request_saved = rdb.save(next_request.json())
if not request_saved:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'Could not save the new request %s' % (
next_request.get_attribute('prepid')))
## inspect priority
self.set_priority(original_action_item['block_number'])
if not reserve:
# sync last status
self.set_attribute('last_status', next_request.get_attribute('status'))
# we can only be processing at this point
self.set_attribute('status', 'processing')
# set to next step
self.set_attribute('step', next_step)
if not reserve:
notification_subject = 'Flow for request %s in %s' % (current_request.get_attribute('prepid'), next_campaign_id)
notification_text = 'The request %s has been flown within:\n \t %s \n into campaign:\n \t %s \n using:\n \t %s \n creating the new request:\n \t %s \n as part of:\n \t %s \n and from the produced dataset:\n %s \n\n%srequests?prepid=%s \n%srequests?prepid=%s \n' % (
current_request.get_attribute('prepid'),
self.get_attribute('member_of_campaign'),
next_campaign_id,
flow_name,
next_request.get_attribute('prepid'),
self.get_attribute('prepid'),
next_request.get_attribute('input_filename'),
l_type.baseurl(),
current_request.get_attribute('prepid'),
l_type.baseurl(),
next_request.get_attribute('prepid')
)
current_request.notify(notification_subject, notification_text)
else:
notification_subject = 'Reservation of request {0}'.format(next_request.get_attribute('prepid'))
notification_text = 'The request {0} of campaign \n\t{2}\nhas been reserved as part of \n\t{1}\nas the next step for {4}\n\n{3}requests?prepid={4}\n{5}requests?prepid={6}\n'.format(
next_request.get_attribute('prepid'),
self.get_attribute('prepid'),
next_campaign_id,
l_type.baseurl(), current_request.get_attribute('prepid'),
l_type.baseurl(), next_request.get_attribute('prepid'),
)
next_request.notify(notification_subject, notification_text)
return True
