def toggle_last_request(self):
## let it toggle the last request to a given approval only if the chained request allows it
if self.get_attribute('approval') == 'none':
return
ccdb = database('chained_campaigns')
mcm_cc = ccdb.get(self.get_attribute('member_of_campaign'))
(next_campaign_id, flow_name) = mcm_cc['campaigns'][self.get_attribute('step')]
fdb = database('flows')
mcm_f = flow(fdb.get(flow_name))
# check whether we have to do something even more subtle with the request
if mcm_f.get_attribute('approval') == 'submit' or self.get_attribute('approval') == 'submit':
rdb = database('requests')
next_request = request(rdb.get(self.get_attribute('chain')[self.get_attribute('step')]))
current_r_approval = next_request.get_attribute('approval')
time_out = 0
#self.logger.error('Trying to move %s from %s to submit'% (next_request.get_attribute('prepid'), current_r_approval))
while current_r_approval != 'submit' and time_out <= 10:
time_out += 1
#get it back from db to avoid _red issues
next_request = request(rdb.get(next_request.get_attribute('prepid')))
with locker.lock('{0}-wait-for-approval'.format( next_request.get_attribute('prepid') )):
next_request.approve()
request_saved = rdb.save(next_request.json())
if not request_saved:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'Could not save the new request %s while trying to move to submit approval' % (
next_request.get_attribute('prepid')))
current_r_approval = next_request.get_attribute('approval')
pass
return True
def internal_run(self):
from tools.installer import installer
from tools.batch_control import batch_control
location = installer( self.crid, care_on_existing=False, clean_on_exit=True)
try:
crdb = database('chained_requests')
rdb = database('requests')
mcm_cr = chained_request(crdb.get(self.crid))
mcm_rs = []
for rid in mcm_cr.get_attribute('chain'):
mcm_rs.append( request( rdb.get( rid ) ))
test_script = location.location() + 'validation_run_test.sh'
with open(test_script, 'w') as there:
there.write(mcm_cr.get_setup(directory=location.location(), run=True, validation=True))
batch_test = batch_control( self.crid, test_script )
try:
success = batch_test.test()
except:
self.reset_all( traceback.format_exc() )
return
if not success:
self.reset_all( '\t .out \n%s\n\t .err \n%s\n ' % ( batch_test.log_out, batch_test.log_err) )
return
last_fail=mcm_rs[0]
trace=""
for mcm_r in mcm_rs:
### if not mcm_r.is_root: continue ##disable for dr request
(success,trace) = mcm_r.pickup_all_performance(location.location())
if not success:
last_fail = mcm_r
break
self.logger.error('I came all the way to here and %s (request %s)' % ( success, self.crid ))
if success:
for mcm_r in mcm_rs:
if mcm_r.is_root:
mcm_current = request( rdb.get(mcm_r.get_attribute('prepid')))
if mcm_current.json()['_rev'] == mcm_r.json()['_rev']:
mcm_r.set_status(with_notification=True)
if not mcm_r.reload():
self.reset_all( 'The request %s could not be saved after the runtest procedure' % (mcm_r.get_attribute('prepid')))
return
else:
self.reset_all( 'The request %s has changed during the run test procedure'%(mcm_r.get_attribute('prepid')), notify_one = mcm_r.get_attribute('prepid'))
return
else:
self.reset_all( trace , notify_one = last_fail.get_attribute('prepid') )
return
except:
mess = 'We have been taken out of run_safe of runtest_genvalid for %s because \n %s \n During an un-excepted exception. Please contact support.' % (
self.crid, traceback.format_exc())
self.logger.error(mess)
finally:
location.close()
def GET(self, *args):
"""
Perform test for chained requests
"""
if not len(args):
return dumps({"results" : False, "message" : "no argument provided"})
from tools.handlers import RunChainValid, validation_pool
## now in the core of the api
runtest = RunChainValid(crid=args[0], lock=locker.lock(args[0]))
crdb = database('chained_requests')
rdb = database('requests')
mcm_cr = chained_request(crdb.get(args[0]))
mcm_rs = []
for rid in mcm_cr.get_attribute('chain')[mcm_cr.get_attribute('step'):]:
mcm_r = request( rdb.get( rid ) )
if mcm_r.get_attribute('status') in ['approved','submitted','done']:
return dumps({"results" : False, "prepid" : args[0],
"message" : "request %s is in status %s" % (
rid, mcm_r.get_attribute('status'))})
for rid in mcm_cr.get_attribute('chain')[mcm_cr.get_attribute('step'):]:
mcm_r = request(rdb.get(rid))
next = 'validation'
if not mcm_r.is_root: next = 'approve'
try:
if mcm_r.get_attribute('approval') == 'none':
## no need to try and move it along if already further than that
getattr(mcm_r,'ok_to_move_to_approval_%s' % next)(for_chain=True)
mcm_r.update_history({'action' : 'approve', 'step' : next})
mcm_r.set_attribute('approval', next)
mcm_r.reload()
else:
pass
## fail this for the moment. there is no way to handle this yet
#text="It is not supported for the moment to test a chain of requests which are partially not new. Please contact an administrator"
#runtest.reset_all( text , notify_one = rid )
#return dumps({"results" : False, "message" : text, "prepid" : args[0]})
text = 'Within chain %s \n'% mcm_cr.get_attribute('prepid')
text += mcm_r.textified()
mcm_r.notify('Approval %s in chain %s for request %s' % (next,
mcm_cr.get_attribute('prepid'), mcm_r.get_attribute('prepid')),
text, accumulate=True)
except Exception as e:
runtest.reset_all(str(e), notify_one=rid)
return dumps({"results" : False, "message" : str(e),"prepid" : args[0]})
validation_pool.add_task(runtest.internal_run)
#runtest.start()
return dumps({"results" : True, "message" : "run test started","prepid" : args[0]})
def rewind_one(self, crid):
crdb = database('chained_requests')
rdb = database('requests')
if not crdb.document_exists( crid ):
return {"results":False, "message":"does not exist","prepid" : crid}
mcm_cr = chained_request( crdb.get( crid) )
current_step = mcm_cr.get_attribute('step')
if current_step==0:
## or should it be possible to cancel the initial requests of a chained request
return {"results":False, "message":"already at the root","prepid" : crid}
## supposedly all the other requests were already reset!
for next in mcm_cr.get_attribute('chain')[current_step+1:]:
## what if that next one is not in the db
if not rdb.document_exists( next):
self.logger.error('%s is part of %s but does not exist'%( next, crid))
continue
mcm_r = request(rdb.get( next ))
if mcm_r.get_attribute('status')!='new':
# this cannot be right!
self.logger.error('%s is after the current request and is not new: %s' % (
next, mcm_r.get_attribute('status')))
return {"results":False, "message":"%s is not new" % (next), "prepid" : crid}
##get the one to be reset
current_id=mcm_cr.get_attribute('chain')[current_step]
mcm_r = request( rdb.get( current_id ))
mcm_r.reset()
saved = rdb.update( mcm_r.json() )
if not saved:
{"results":False, "message":"could not save the last request of the chain","prepid" : crid}
## the current chained request has very likely been updated :
## reload it as you have not changed anything to it yet
mcm_cr = chained_request( crdb.get( crid) )
mcm_cr.set_attribute('step',current_step -1 )
# set status, last status
mcm_cr.set_last_status()
mcm_cr.set_attribute('status','processing')
saved = crdb.update( mcm_cr.json())
if saved:
return {"results":True,"prepid" : crid}
else:
return {"results" : False,
"message" : "could not save chained requests. the DB is going to be inconsistent !",
"prepid" : crid}
def multiple_inspect(self, cid, in_statuses=['submitted','approved']):
clist = list(set(cid.rsplit(',')))
res = []
rdb = database('requests')
for c in clist:
## this query needs to be modified if we want to
## also inspect the request for submit !
rlist = []
for in_status in in_statuses:
rlist.extend(rdb.queries(["member_of_campaign==%s" % (c),
"status==%s" % (in_status)]))
for r in rlist:
mcm_r = request(r)
if mcm_r:
res.append(mcm_r.inspect())
else:
res.append({"prepid": r, "results":False,
'message' : '%s does not exist' % (r)})
if len(res) > 1:
return res
elif len(res):
return res[0]
else:
return []
def get(self, batch_ids):
"""
Reset all requests in a batch (or list of) and set the status to reset
"""
res = []
bdb = database('batches')
rdb = database('requests')
for bid in batch_ids.split(','):
mcm_b = bdb.get(bid)
for r in mcm_b['requests']:
if 'pdmv_prep_id' not in r['content']:
continue
rid = r['content']['pdmv_prep_id']
if not rdb.document_exists(rid):
continue
mcm_r = request(rdb.get(rid))
try:
mcm_r.reset()
rdb.update(mcm_r.json())
except Exception:
continue
batch_to_update = batch(mcm_b)
batch_to_update.set_attribute('status', 'reset')
batch_to_update.update_history({'action': 'set status', 'step': 'reset'})
bdb.update(batch_to_update.json())
res.append({'prepid': bid, 'results': True})
return res
def get(self, batch_ids):
"""
Reset all requests in a batch (or list of) and set the status to reset
"""
res = []
bdb = database('batches')
rdb = database('requests')
for bid in batch_ids.split(','):
mcm_b = bdb.get(bid)
for r in mcm_b['requests']:
if 'pdmv_prep_id' not in r['content']:
continue
rid = r['content']['pdmv_prep_id']
if not rdb.document_exists(rid):
continue
mcm_r = request(rdb.get(rid))
try:
mcm_r.reset()
rdb.update(mcm_r.json())
except Exception:
continue
batch_to_update = batch(mcm_b)
batch_to_update.set_attribute('status', 'reset')
batch_to_update.update_history({
'action': 'set status',
'step': 'reset'
})
bdb.update(batch_to_update.json())
res.append({'prepid': bid, 'results': True})
return res
def multiple_inspect(self, cid, in_statuses=['submitted', 'approved']):
clist = list(set(cid.rsplit(',')))
res = []
rdb = database('requests')
for c in clist:
## this query needs to be modified if we want to also inspect the request for submit !
rlist = []
for in_status in in_statuses:
rlist.extend(
rdb.queries([
"member_of_campaign==%s" % (c),
"status==%s" % (in_status)
]))
for r in rlist:
mcm_r = request(r)
if mcm_r:
res.append(mcm_r.inspect())
else:
res.append({
"prepid": r,
"results": False,
'message': '%s does not exist' % (r)
})
if len(res) > 1:
return res
elif len(res):
return res[0]
else:
return []
def find_chains(self):
# validate request
if not self.get_attribute('prepid'):
raise self.PrepIdNotDefinedException()
# initialize db connections
try:
reqdb = database('requests')
campaigndb = database('campaigns')
except database.DatabaseAccessError as ex:
return False
# validate prepid
if not reqdb.document_exists(self.get_attribute('prepid')):
raise self.PrepIdDoesNotExistException(
self.get_attribute('prepid'))
# get campaign id
req = request(json_input=reqdb.get(self.get_attribute('prepid')))
# check if campaign exists
campid = req.get_attribute('member_of_campaign')
if not campid:
self.logger.error('action %s has not a campaign defined' %
(self.get_attribute('prepid')))
raise ValueError('Error: Campaign was not set for',
self.get_attribute('prepid'))
if not campaigndb.document_exists(campid):
raise self.PrepIdDoesNotExistException(campid)
# get all chains
return self.__retrieve_chains(self.get_attribute('prepid'), campid)
def find_chains(self):
# validate request
if not self.get_attribute('prepid'):
raise self.PrepIdNotDefinedException()
# initialize db connections
try:
reqdb = database('requests')
campaigndb = database('campaigns')
except database.DatabaseAccessError as ex:
return False
# validate prepid
if not reqdb.document_exists(self.get_attribute('prepid')):
raise self.PrepIdDoesNotExistException(self.get_attribute('prepid'))
# get campaign id
req = request(json_input=reqdb.get(self.get_attribute('prepid')))
# check if campaign exists
campid = req.get_attribute('member_of_campaign')
if not campid:
self.logger.error('action %s has not a campaign defined' % (self.get_attribute('prepid')))
raise ValueError('Error: Campaign was not set for', self.get_attribute('prepid'))
if not campaigndb.document_exists(campid):
raise self.PrepIdDoesNotExistException(campid)
# get all chains
return self.__retrieve_chains(self.get_attribute('prepid'), campid)
def GET(self, *args):
"""
Reset all requests in a batch (or list of) and set the status to reset
"""
res=[]
bdb = database('batches')
rdb = database('requests')
bids = args[0]
for bid in bids.split(','):
mcm_b = bdb.get(bid)
for r in mcm_b['requests']:
if not 'pdmv_prep_id' in r['content']:
continue
rid = r['content']['pdmv_prep_id']
if not rdb.document_exists( rid ):
continue
mcm_r = request( rdb.get( rid ) )
try:
mcm_r.reset()
rdb.update( mcm_r.json() )
except Exception as ex:
continue
mcm_b['status'] = 'reset'
bdb.update( mcm_b )
res.append({'prepid':bid, 'results': True})
return dumps(res)
def reset_all(self, message, what = 'Chained validation run test', notify_one=None):
crdb = database('chained_requests')
rdb = database('requests')
mcm_cr = chained_request(crdb.get(self.crid))
if self.scratch:
chain = mcm_cr.get_attribute('chain')
else:
chain = mcm_cr.get_attribute('chain')[mcm_cr.get_attribute('step'):]
for rid in chain:
mcm_r = request(rdb.get(rid ))
s_label = 'chainvalid-%s' % rid
semaphore_events.decrement(s_label)
if not semaphore_events.is_set(s_label):
##someone else is still validating that chain, so no reset !
mcm_r.notify('%s failed for request %s' % (what,
mcm_r.get_attribute('prepid')), message)
continue
## do not reset anything that does not look ok already
# this might leave things half-way inconsistent in terms of status
if mcm_r.get_attribute('status') != 'new':
mcm_r.notify('%s failed for request %s' % (what,
mcm_r.get_attribute('prepid')), message)
continue
notify = True
if notify_one and notify_one != rid:
notify = False
mcm_r.test_failure( message,
what = what, rewind=True, with_notification=notify)
def GET(self, *args):
"""
Does a soft reset to all relevant request in the chain
"""
if not len(args):
return dumps({"results" : False, "message" : "no argument provided"})
arg0 = args[0]
crdb = database('chained_requests')
rdb = database('requests')
mcm_cr = chained_request(crdb.get(arg0))
for rid in reversed(mcm_cr.get_attribute('chain')[:mcm_cr.get_attribute('step')+1]):
## from the current one to the first one REVERSED
mcm_r = request(rdb.get(rid))
try:
mcm_r.reset(hard=False)
except Exception as e:
return dumps({'prepid' : arg0, 'results' : False, 'message' : str(e)})
mcm_r.reload()
mcm_cr = chained_request(crdb.get(arg0))
mcm_cr.set_attribute('step', max(0, mcm_cr.get_attribute('chain').index(rid)-1))
mcm_cr.reload()
return dumps({'prepid' : arg0, 'results':True})
def next_prepid(self, pwg, camp):
if not pwg or not camp:
return None
with locker.lock("{0}-{1}".format(pwg, camp)):
db = database(self.db_name)
query_results = db.raw_query('serial_number', {
'group': True,
'key': [camp, pwg]
})
sn = 1
if query_results:
sn = query_results[0]['value'] + 1
pid = '%s-%s-%05d' % (pwg, camp, sn)
if sn == 1:
self.logger.info('Beginning new prepid family: %s-%s' %
(pwg, camp))
db_camp = database('campaigns', cache_enabled=True)
req_camp = campaign(db_camp.get(camp))
new_request = request(
req_camp.add_request({
'_id': pid,
'prepid': pid,
'pwg': pwg,
'member_of_campaign': camp
}))
new_request.update_history({'action': 'created'})
db.save(new_request.json())
self.logger.info('New prepid : %s ' % pid)
return pid
def GET(self, *args):
"""
Reset all requests in a batch (or list of) and set the status to reset
"""
res = []
bdb = database('batches')
rdb = database('requests')
bids = args[0]
for bid in bids.split(','):
mcm_b = bdb.get(bid)
for r in mcm_b['requests']:
if not 'pdmv_prep_id' in r['content']:
continue
rid = r['content']['pdmv_prep_id']
if not rdb.document_exists(rid):
continue
mcm_r = request(rdb.get(rid))
try:
mcm_r.reset()
rdb.update(mcm_r.json())
except Exception as ex:
continue
mcm_b['status'] = 'reset'
bdb.update(mcm_b)
res.append({'prepid': bid, 'results': True})
return dumps(res)
def get(self, chained_request_id):
"""
Does a soft reset to all relevant request in the chain
"""
crdb = database('chained_requests')
rdb = database('requests')
mcm_cr = chained_request(crdb.get(chained_request_id))
for rid in reversed(
mcm_cr.get_attribute('chain')[:mcm_cr.get_attribute('step') +
1]):
# from the current one to the first one REVERSED
mcm_r = request(rdb.get(rid))
try:
mcm_r.reset(hard=False)
except Exception as e:
return {
'prepid': chained_request_id,
'results': False,
'message': str(e)
}
mcm_r.reload()
mcm_cr = chained_request(crdb.get(chained_request_id))
mcm_cr.set_attribute(
'step', max(0,
mcm_cr.get_attribute('chain').index(rid) - 1))
mcm_cr.reload()
return {'prepid': chained_request_id, 'results': True}
def submit_request(self, prepid, run_test_path):
mcm_request = request(self.request_db.get(prepid))
# check if the request should be validated as part of a chain
for chain_prepid in mcm_request.get_attribute('member_of_chain'):
mcm_chain = chained_request(self.chained_request_db.get(chain_prepid))
if mcm_chain.get_attribute('validate') and prepid in mcm_chain.get_attribute('chain')[mcm_chain.get_attribute('step'):]:
return {}
aux_validation = mcm_request.get_attribute(self.DOC_VALIDATION)
to_write = mcm_request.get_setup_file(run=True, do_valid=True, for_validation=True, gen_script=True)
file_name = self.TEST_FILE_NAME % prepid
if not self.create_test_file(to_write, run_test_path, file_name):
mcm_request.set_attribute(self.DOC_VALIDATION, aux_validation)
mcm_request.test_failure(
message='There was a problem while creating the file for prepid: %s' % (mcm_request.get_attribute('prepid')),
what='Validation run test',
rewind=True)
return {}
timeout = mcm_request.get_timeout()
memory = mcm_request.get_attribute("memory")
threads = mcm_request.get_core_num()
self.create_htcondor_config_file(run_test_path, prepid, timeout, memory, threads, [prepid], mcm_request)
job_info = self.execute_command_submission(prepid, run_test_path)
if 'error' in job_info:
mcm_request.test_failure(message=job_info['error'], what='Validation run test', rewind=True)
return {}
job_info[self.DOC_REV] = mcm_request.json()[self.DOC_REV]
job_info[self.DOC_VALIDATION] = mcm_request.get_attribute(self.DOC_VALIDATION) # this field change when calling request.get_setup_file, to be propagated in case of success
return job_info
def internal_run(self):
if not self.lock.acquire(blocking=False):
return False
try:
req = request(self.request_db.get(self.prepid))
ret = req.prepare_and_upload_config()
return True if ret else False
finally:
self.lock.release()
def submit_chain(self, prepid, run_test_path):
mcm_chained_request = chained_request(self.chained_request_db.get(prepid))
except_requests = []
reset = False
# If a request of a chain was singly submmited to validation and then somebody reseted it, we will find it here
for request_prepid in mcm_chained_request.get_attribute('chain')[mcm_chained_request.get_attribute('step'):]:
if request_prepid in self.submmited_prepids_set:
except_requests.append(request_prepid)
reset = True
if reset:
message = "Requests %s of the chain %s are already in validation" % (except_requests, prepid)
self.logger.error(message)
mcm_chained_request.reset_requests(message, except_requests=except_requests)
return {}
to_write = mcm_chained_request.get_setup(run=True, validation=True, for_validation=True)
file_name = self.TEST_FILE_NAME % prepid
if not self.create_test_file(to_write, run_test_path, file_name):
mcm_chained_request.reset_requests('There was a problem while creating the file for prepid: %s' % (mcm_chained_request.get_attribute('prepid')))
return {}
requests_in_chain = {}
first_request_in_chain = None
for request_prepid in mcm_chained_request.get_attribute('chain')[mcm_chained_request.get_attribute('step'):]:
mcm_request = request(self.request_db.get(request_prepid))
if not mcm_request.is_root and 'validation' not in mcm_request._json_base__status: # only root or possible root requests
break
status = mcm_request.get_attribute('status')
approval = mcm_request.get_attribute('approval')
if status != 'new' or approval != 'validation':
message = "The request %s of chain %s is in status: %s approval: %s" % (request_prepid, prepid, status, approval)
self.logger.error(message)
mcm_chained_request.reset_requests(message)
return {}
requests_in_chain[request_prepid] = mcm_request.json()[self.DOC_REV]
if not first_request_in_chain:
first_request_in_chain = mcm_request
if not len(requests_in_chain):
message = 'No requests to be validated in chain: %s' % prepid
self.logger.info(message)
mcm_chained_request.reset_requests(message)
return {}
timeout, memory, threads = mcm_chained_request.get_timeout_memory_threads()
self.create_htcondor_config_file(run_test_path, prepid, timeout, memory, threads, list(requests_in_chain.iterkeys()), first_request_in_chain)
job_info = self.execute_command_submission(prepid, run_test_path)
if 'error' in job_info:
mcm_chained_request.reset_requests(job_info['error'])
return {}
job_info[self.CHAIN_REQUESTS] = requests_in_chain
return job_info
def announce_with_text(self, bid, message):
bdb = database('batches')
if not semaphore_events.is_set(bid):
return {"results": False, "message":
"Batch {0} has on-going submissions.".format(bid), "prepid": bid}
b = batch(bdb.get(bid))
workflows = ''
for dictionary in b.get_attribute('requests'):
workflows += dictionary['name'] + ','
workflows = workflows[:-1]
r = ''
result = {}
if workflows != '':
approver = RequestApprover(bid, workflows)
result = approver.internal_run()
if (result['results']):
r = b.announce(message)
else:
r = b.announce(message)
if r:
map_wf_to_prepid = {}
for dictionary in b.get_attribute('requests'):
wf = dictionary.get('name')
prepid = dictionary.get('content', {}).get('pdmv_prep_id')
if not wf or not prepid:
continue
if wf not in map_wf_to_prepid:
map_wf_to_prepid[wf] = []
map_wf_to_prepid[wf].append(prepid)
rdb = database('requests')
priority_coeff = settings.get_value('nanoaod_priority_increase_coefficient')
for wf, requests in map_wf_to_prepid.iteritems():
if len(requests) == 1 and 'nanoaod' in requests[0].lower():
for r_prepid in requests:
req = request(rdb.get(r_prepid))
current_priority = req.get_attribute('priority')
new_priority = int(current_priority + priority_coeff * 1000)
req.change_priority(new_priority)
return {
"results": bdb.update(b.json()),
"message": r,
"prepid": bid
}
else:
return {
"results": False,
"prepid": bid,
"message": result['message'] if 'message' in result and not r else r
}
def set_priority(self, level):
rdb = database('requests')
okay = True
for r in self.get_attribute('chain'):
req = request(rdb.get(r))
##only those that can still be changed
#set to the maximum priority
if not req.change_priority(priority().priority(level)):
self.logger.log('Could not save updated priority for %s' % r)
okay = False
return okay
def set_priority(self, level):
rdb = database('requests')
okay = True
for r in self.get_attribute('chain'):
req = request(rdb.get(r))
##only those that can still be changed
#set to the maximum priority
if not req.change_priority(priority().priority(level)):
self.logger.log('Could not save updated priority for %s' % r)
okay = False
return okay
def get_setup(self, directory='', events=None, run=False, validation=False):
req_ids = self.get_attribute('chain')
rdb = database('requests')
setup_file = ''
for (index,req_id) in enumerate(req_ids):
req = request(rdb.get(req_id))
ev = events
if not ev and index!=0 and not req.is_root:
ev = -1
setup_file += req.get_setup_file(directory=directory, events=ev, run=run, do_valid=validation)
return setup_file
def toggle_last_request(self):
## let it toggle the last request to a given approval only if the chained request allows it
if self.get_attribute('approval') == 'none':
return
ccdb = database('chained_campaigns')
mcm_cc = ccdb.get(self.get_attribute('member_of_campaign'))
(next_campaign_id,
flow_name) = mcm_cc['campaigns'][self.get_attribute('step')]
fdb = database('flows')
mcm_f = flow(fdb.get(flow_name))
# check whether we have to do something even more subtle with the request
if mcm_f.get_attribute('approval') == 'submit' or self.get_attribute(
'approval') == 'submit':
rdb = database('requests')
next_request = request(
rdb.get(
self.get_attribute('chain')[self.get_attribute('step')]))
current_r_approval = next_request.get_attribute('approval')
time_out = 0
#self.logger.error('Trying to move %s from %s to submit'% (next_request.get_attribute('prepid'), current_r_approval))
while current_r_approval != 'submit' and time_out <= 10:
time_out += 1
#get it back from db to avoid _red issues
next_request = request(
rdb.get(next_request.get_attribute('prepid')))
with locker.lock('{0}-wait-for-approval'.format(
next_request.get_attribute('prepid'))):
next_request.approve()
request_saved = rdb.save(next_request.json())
if not request_saved:
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'Could not save the new request %s while trying to move to submit approval'
% (next_request.get_attribute('prepid')))
current_r_approval = next_request.get_attribute('approval')
pass
return True
def multiple_inspect(self, cid, in_statuses=['submitted', 'approved']):
clist = list(set(cid.rsplit(',')))
res = []
rdb = database('requests')
index = 0
self.logger.error("Chain inspect begin. Number of chains to be inspected: %s" % (len(clist)))
try:
while len(clist) > index:
yield dumps({"current cr element": "%s/%s" % (index, len(clist))}, indent=2)
query = rdb.construct_lucene_complex_query([
('member_of_campaign', {'value': clist[index: index + 1]}),
('status', {'value': in_statuses})
])
##do another loop over the requests themselves
req_page = 0
request_res = rdb.full_text_search('search', query, page=req_page)
while len(request_res) > 0:
self.logger.info("inspecting single requests. page: %s" % (req_page))
for r in request_res:
self.logger.info("running inspect on request: %s" % (r['prepid']))
mcm_r = request(r)
if mcm_r:
#making it as a stream
yield dumps(mcm_r.inspect(), indent=4)
else:
#making it as a stream
yield dumps({"prepid": r, "results": False,
'message': '%s does not exist' % (r)}, indent=4)
req_page += 1
request_res = rdb.full_text_search('search', query, page=req_page)
time.sleep(0.5)
index += 1
time.sleep(1)
except Exception as e:
subject = "Exception while inspecting request "
message = "Request: %s \n %s traceback: \n %s" % (mcm_r.get_attribute('prepid'), str(e), traceback.format_exc())
self.logger.error(subject + message)
notification(
subject,
message,
[],
group=notification.REQUEST_OPERATIONS,
action_objects=[mcm_r.get_attribute('prepid')],
object_type='requests',
base_object=mcm_r)
mcm_r.notify(subject, message, accumulate=True)
self.logger.info("Campaign inspection finished!")
def process_finished_request_failed(self, prepid, job_out, error_out, was_exited=True, job_error_out='', out_path='', log_out=''):
mcm_request = request(self.request_db.get(prepid))
# need to provide all the information back
if not was_exited:
no_success_message = "File %s does not look properly formatted or does not exist. \n %s \n %s \n Error out: \n%s \n Log out: \n%s" % (
out_path, job_out, job_error_out, error_out, log_out)
else:
no_success_message = '\t Job out: \n%s\n\t Error out: \n%s\n Log out: \n%s ' % (job_out, error_out, log_out)
mcm_request.test_failure(
message=no_success_message,
what='Validation run test',
rewind=True)
def reset_all(self, message, what = 'Chained validation run test', notify_one=None):
crdb = database('chained_requests')
rdb = database('requests')
mcm_cr = chained_request(crdb.get(self.crid))
for rid in mcm_cr.get_attribute('chain'):
mcm_r = request( rdb.get( rid ) )
notify = True
if notify_one and notify_one != rid:
notify = False
mcm_r.test_failure( message,
what = what,
rewind=True,
with_notification=notify)
def delete_request(self, crid):
crdb = database('chained_requests')
rdb = database('requests')
adb = database('actions')
mcm_cr = chained_request(crdb.get(crid))
mcm_a = None
## get all objects
mcm_r_s=[]
for (i,rid) in enumerate(mcm_cr.get_attribute('chain')):
mcm_r = request(rdb.get(rid))
#this is not a valid check as it is allowed to remove a chain around already running requests
# if mcm_r.get_attribute('status') != 'new':
# return {"results":False,"message" : "the request %s part of the chain %s for action %s is not in new status"%( mcm_r.get_attribute('prepid'),
# crid,
# mcm_a.get_attribute('prepid'))}
in_chains = mcm_r.get_attribute('member_of_chain')
in_chains.remove( crid )
mcm_r.set_attribute('member_of_chain', in_chains)
if i==0:
# the root is the action id
mcm_a = action(adb.get(rid))
if len(in_chains)==0 and mcm_r.get_attribute('status')!='new':
return {"results":False, "message" : "the request %s, not in status new, at the root of the chain will not be chained anymore"% rid}
else:
if len(in_chains)==0:
return {"results":False,"message" : "the request %s, not at the root of the chain will not be chained anymore"% rid}
mcm_r.update_history({'action':'leave','step':crid})
mcm_r_s.append( mcm_r )
## check if possible to get rid of it !
# action for the chain is disabled
chains = mcm_a.get_chains( mcm_cr.get_attribute('member_of_campaign'))
if chains[crid]['flag']:
return {"results":False,"message" : "the action %s for %s is not disabled"%(mcm_a.get_attribute('prepid'), crid)}
#take it out
mcm_a.remove_chain( mcm_cr.get_attribute('member_of_campaign'), mcm_cr.get_attribute('prepid') )
if not adb.update( mcm_a.json()):
return {"results":False,"message" : "Could not save action "+ mcm_a.get_attribute('prepid')}
## then save all changes
for mcm_r in mcm_r_s:
if not rdb.update( mcm_r.json()):
return {"results":False,"message" : "Could not save request "+ mcm_r.get_attribute('prepid')}
else:
mcm_r.notify("Request {0} left chain".format( mcm_r.get_attribute('prepid')),
"Request {0} has successfuly left chain {1}".format(
mcm_r.get_attribute('prepid'), crid))
return {"results": crdb.delete(crid)}
def report_error(self, prepid, message):
self.logger.error(message)
try:
if 'chain' in prepid:
mcm_chained_request = chained_request(self.chained_request_db.get(prepid))
mcm_chained_request.reset_requests(message)
else:
mcm_request = request(self.request_db.get(prepid))
mcm_request.test_failure(
message=message,
what='Validation run test',
rewind=True)
except Exception as e:
self.logger.error("Exception while reporting an error for %s message: %s \ntraceback: %s" % (prepid, str(e), traceback.format_exc()))
def check_request(self):
if not self.request_db.document_exists(self.prepid):
self.logger.inject("The request {0} does not exist".format(self.prepid), level='error', handler=self.prepid)
return False, None
req = request(self.request_db.get(self.prepid))
if self.check_approval and req.get_attribute('approval') != 'submit':
self.injection_error("The request is in approval {0}, while submit is required".format(req.get_attribute('approval')), req)
return False, None
if req.get_attribute('status') != 'approved':
self.injection_error("The request is in status {0}, while approved is required".format(req.get_attribute('status')), req)
return False, None
return True, req
def generate_chained_requests(self, mccm_ticket, request_prepid, mcm_chained_campaign, reserve=False, with_notify=True, special=False):
try:
mcm_chained_campaign.reload(save_current=False)
generated_chained_request = chained_request(mcm_chained_campaign.generate_request(request_prepid))
except Exception as e:
message = "Unable to generate chained request for ticket %s request %s, message: " % (mccm_ticket.get_attribute('prepid'), request_prepid, str(e))
self.logger.error(message)
return {
"results": False,
"message": message}
requests_db = database('requests')
self.overwrite_action_parameters_from_ticket(generated_chained_request, mccm_ticket)
mcm_request = request(json_input=requests_db.get(request_prepid))
generated_chained_request.set_attribute('last_status', mcm_request.get_attribute('status'))
if generated_chained_request.get_attribute('last_status') in ['submitted', 'done']:
generated_chained_request.set_attribute('status', 'processing')
if special:
generated_chained_request.set_attribute('approval', 'none')
new_chain_prepid = generated_chained_request.get_attribute('prepid')
if not generated_chained_request.reload():
return {
'results': False,
'message': 'Unable to save chained request %s' % new_chain_prepid}
# update the history of chained campaign
mcm_chained_campaign.save()
# let the root request know that it is part of a chained request
chains = mcm_request.get_attribute('member_of_chain')
chains.append(new_chain_prepid)
chains.sort()
mcm_request.set_attribute('member_of_chain', list(set(chains)))
mcm_request.update_history({'action': 'join chain', 'step': new_chain_prepid})
mcm_request.save()
# do the reservation of the whole chain ?
generated_requests = []
if reserve:
results_dict = generated_chained_request.reserve(limit=reserve, save_requests=False)
if results_dict['results'] and 'generated_requests' in results_dict:
generated_requests = results_dict['generated_requests']
results_dict.pop('generated_requests')
else:
return {
"results": False,
"prepid": new_chain_prepid,
"message": results_dict['message']}
return {
"results":True,
"prepid": new_chain_prepid,
'generated_requests': generated_requests}
def process_finished_request_success(self, prepid, doc_info, job_out,
error_out, log_out):
mcm_request = request(self.request_db.get(prepid))
doc_revision = doc_info[self.DOC_REV]
doc_validation = doc_info[self.DOC_VALIDATION]
if doc_revision != mcm_request.json()[self.DOC_REV]:
message = 'The request %s has changed during the run test procedure, preventing from being saved' % (
prepid)
self.logger.error(message)
mcm_request.test_failure(message=message,
what='Validation run test',
rewind=True)
return
path = self.test_directory_path + prepid + '/'
error = ''
is_success = False
mcm_request.inc_validations_counter()
try:
(is_success, error) = mcm_request.pickup_all_performance(path)
error = 'Error:\n%s\n Error out:\n%s\n' % (error, error_out)
except request.WrongTimeEvent as ex:
self.logger.error('Exception: %s' % (ex))
retry_validation = self.process_request_wrong_time_event(
mcm_request)
if retry_validation:
return
if not is_success:
self.process_finished_request_failed(prepid,
job_out,
error,
log_out=log_out)
return
mcm_request.set_status(with_notification=True)
aux_validation = mcm_request.get_attribute(self.DOC_VALIDATION)
mcm_request.set_attribute(self.DOC_VALIDATION, doc_validation)
saved = self.request_db.update(mcm_request.json())
if not saved:
mcm_request.set_attribute(self.DOC_VALIDATION, aux_validation)
mcm_request.test_failure(
message=
'The request could not be saved after the run test procedure',
what='Validation run test',
rewind=True)
return
self.logger.info('Validation job for prepid %s SUCCESSFUL!!!' % prepid)
def check_request(self):
if not self.request_db.document_exists(self.prepid):
self.logger.inject("The request {0} does not exist".format(self.prepid), level='error', handler=self.prepid)
return False, None
req = request(self.request_db.get(self.prepid))
if self.check_approval and req.get_attribute('approval') != 'submit':
self.injection_error(
"The request is in approval {0}, while submit is required".format(req.get_attribute('approval')), req)
return False, None
if req.get_attribute('status') != 'approved':
self.injection_error(
"The request is in status {0}, while approved is required".format(req.get_attribute('status')), req)
return False, None
return True, req
def report_error(self, prepid, message):
self.logger.error(message)
try:
if 'chain' in prepid:
mcm_chained_request = chained_request(
self.chained_request_db.get(prepid))
mcm_chained_request.reset_requests(message)
else:
mcm_request = request(self.request_db.get(prepid))
mcm_request.test_failure(message=message,
what='Validation run test',
rewind=True)
except Exception as e:
self.logger.error(
"Exception while reporting an error for %s message: %s \ntraceback: %s"
% (prepid, str(e), traceback.format_exc()))
def process_finished_request_success(self, prepid, doc_info, job_out, error_out, log_out):
mcm_request = request(self.request_db.get(prepid))
doc_revision = doc_info[self.DOC_REV]
doc_validation = doc_info[self.DOC_VALIDATION]
if doc_revision != mcm_request.json()[self.DOC_REV]:
message = 'The request %s has changed during the run test procedure, preventing from being saved' % (prepid)
self.logger.error(message)
mcm_request.test_failure(
message=message,
what='Validation run test',
rewind=True)
return
path = self.test_directory_path + prepid + '/'
error = ''
is_success = False
mcm_request.inc_validations_counter()
try:
(is_success, error) = mcm_request.pickup_all_performance(path)
error = 'Error:\n%s\n Error out:\n%s\n' % (error, error_out)
except request.WrongTimeEvent as ex:
self.logger.error('Exception: %s' % (ex))
retry_validation = self.process_request_wrong_time_event(mcm_request)
if retry_validation:
return
if not is_success:
self.process_finished_request_failed(prepid, job_out, error, log_out=log_out)
return
mcm_request.set_status(with_notification=True)
aux_validation = mcm_request.get_attribute(self.DOC_VALIDATION)
mcm_request.set_attribute(self.DOC_VALIDATION, doc_validation)
saved = self.request_db.update(mcm_request.json())
if not saved:
mcm_request.set_attribute(self.DOC_VALIDATION, aux_validation)
mcm_request.test_failure(
message='The request could not be saved after the run test procedure',
what='Validation run test',
rewind=True)
return
mcm_request.test_success(
what='Validation',
message='Validation was successful for %s. Request:\n%s' % (mcm_request.get_attribute('prepid'), mcm_request.textified()))
self.logger.info('Validation job for prepid %s SUCCESSFUL!!!' % prepid)
def internal_run(self):
if not self.lock.acquire(blocking=False):
self.logger.error(
"Could not acquire lock for ConfigMakerAndUploader. prepid %s"
% (self.prepid))
return False
try:
self.logger.info(
"Acquired lock for ConfigMakerAndUploader. prepid %s" %
(self.prepid))
req = request(self.request_db.get(self.prepid))
ret = req.prepare_and_upload_config()
return True if ret else False
finally:
self.logger.info(
"Releasing a lock for ConfigMakerAndUploader. prepid %s" %
(self.prepid))
self.lock.release()
def internal_run(self):
if not self.lock.acquire(blocking=False):
self.logger.error("Could not acquire lock for ConfigMakerAndUploader. prepid %s" % (
self.prepid))
return False
try:
self.logger.error("Acquired lock for ConfigMakerAndUploader. prepid %s" % (
self.prepid))
req = request(self.request_db.get(self.prepid))
ret = req.prepare_and_upload_config()
return True if ret else False
finally:
self.logger.error("Releasing a lock for ConfigMakerAndUploader. prepid %s" % (
self.prepid))
self.lock.release()
def get_setup(self,
directory='',
events=None,
run=False,
validation=False):
req_ids = self.get_attribute('chain')
rdb = database('requests')
setup_file = ''
for (index, req_id) in enumerate(req_ids):
req = request(rdb.get(req_id))
ev = events
if not ev and index != 0 and not req.is_root:
ev = -1
setup_file += req.get_setup_file(directory=directory,
events=ev,
run=run,
do_valid=validation)
return setup_file
def next_prepid(self, pwg, camp):
if not pwg or not camp:
return None
with locker.lock("{0}-{1}".format(pwg, camp)):
db = database(self.db_name)
query_results = db.raw_query('serial_number', {'group':True, 'key':[camp, pwg]})
sn = 1
if query_results:
sn = query_results[0]['value']+1
pid='%s-%s-%05d'%( pwg, camp , sn)
if sn==1:
self.logger.log('Beginning new prepid family: %s-%s' %( pwg, camp))
db_camp = database('campaigns', cache=True)
req_camp = campaign(db_camp.get(camp))
new_request = request(req_camp.add_request({'_id':pid, 'prepid':pid, 'pwg':pwg, 'member_of_campaign':camp}))
new_request.update_history({'action':'created'})
db.save(new_request.json())
self.logger.log('New prepid : %s '%pid)
return pid
def process_finished_request_failed(self,
prepid,
job_out,
error_out,
was_exited=True,
job_error_out='',
out_path='',
log_out=''):
mcm_request = request(self.request_db.get(prepid))
# need to provide all the information back
if not was_exited:
no_success_message = "File %s does not look properly formatted or does not exist. \n %s \n %s \n Error out: \n%s \n Log out: \n%s" % (
out_path, job_out, job_error_out, error_out, log_out)
else:
no_success_message = '\t Job out: \n%s\n\t Error out: \n%s\n Log out: \n%s ' % (
job_out, error_out, log_out)
mcm_request.test_failure(message=no_success_message,
what='Validation run test',
rewind=True)
def inspect_priority(self, forChains=None):
##JR: until put in the proper place
chains = self.get_attribute('chains')
crdb = database('chained_requests')
okay = True
for inCC in chains:
### this is the old convention
#if 'flag' in chains[inCC] and chains[inCC]['flag']:
# if 'chains' in chains[inCC]:
# for acr in chains[inCC]['chains']:
# if forChains and not acr in forChains: continue
# cr=chained_request(crdb.get(acr))
# cc=cr.get_attribute('member_of_campaign')
# #if 'block_number' in chains[cc] and chains[cc]['block_number']:
# if chains[cc]['block_number']:
# cr.set_priority(chains[cc]['block_number'])
# self.logger.log('Set priority block %s to %s'%(chains[cc]['block_number'],cr.get_attribute('prepid')))
# else:
# self.logger.error('Could not set block %s to %s'%(chains[cc]['block_number'],cr.get_attribute('prepid')))
## new convention
if 'chains' in chains[inCC] and type(
chains[inCC]['chains']) == dict:
for acr in chains[inCC]['chains']:
if forChains and not acr in forChains: continue
bn = chains[inCC]['chains'][acr]['block_number']
cr = chained_request(crdb.get(acr))
if bn:
self.logger.log('Set priority block %s to %s' %
(bn, acr))
if not cr.set_priority(bn):
okay = False
else:
self.logger.error('Could not set block %s to %s' %
(bn, acr))
rd = database('requests')
if rd.document_exists(self.get_attribute('prepid')):
r = request(rd.get(self.get_attribute('prepid')))
self.set_attribute('dataset_name', r.get_attribute('dataset_name'))
return okay
def add_to_chain(self, data):
rdb = database('requests')
db = database('chained_requests')
self.logger.log('Adding a new request to chained_request')
try:
from json_layer.request import request
except ImportError as ex:
self.logger.error('Could not import request object class.', level='critical')
return {"results":False}
try:
req = request(json_input=loads(data))
except request.IllegalAttributeName as ex:
return {"results":str(ex)}
if not req.get_attribute("member_of_chain"):
self.logger.error('Attribute "member_of_chain" attribute was None')
return {"results":'Error: "member_of_chain" attribute was None.'}
if not req.get_attribute("member_of_campaign"):
self.logger.error('Attribute "member_of_campaign" attribute was None.')
return {"results":'Error: "member_of_campaign" attribute was None.'}
try:
creq = chained_request(json_input=db.get(req.get_attribute('member_of_chain')))
except chained_request.IllegalAttributeName as ex:
return {"results":str(ex)}
try:
new_req = creq.add_request(req.json())
except chained_request.CampaignAlreadyInChainException as ex:
return {"results":str(ex)}
if not new_req:
self.logger.error('Could not save newly created request to database')
return {"results":False}
# finalize and make persistent
db.update(creq.json())
rdb.save(new_req)
return {"results":True}
def submit_request(self, prepid, run_test_path):
mcm_request = request(self.request_db.get(prepid))
# check if the request should be validated as part of a chain
for chain_prepid in mcm_request.get_attribute('member_of_chain'):
mcm_chain = chained_request(
self.chained_request_db.get(chain_prepid))
if mcm_chain.get_attribute(
'validate') and prepid in mcm_chain.get_attribute(
'chain')[mcm_chain.get_attribute('step'):]:
return {}
aux_validation = mcm_request.get_attribute(self.DOC_VALIDATION)
to_write = mcm_request.get_setup_file(run=True,
do_valid=True,
for_validation=True)
file_name = self.TEST_FILE_NAME % prepid
if not self.create_test_file(to_write, run_test_path, file_name):
mcm_request.set_attribute(self.DOC_VALIDATION, aux_validation)
mcm_request.test_failure(
message=
'There was a problem while creating the file for prepid: %s' %
(mcm_request.get_attribute('prepid')),
what='Validation run test',
rewind=True)
return {}
timeout = mcm_request.get_timeout()
memory = mcm_request.get_attribute("memory")
threads = mcm_request.get_core_num()
self.create_htcondor_config_file(run_test_path, prepid, timeout,
memory, threads, [prepid])
job_info = self.execute_command_submission(prepid, run_test_path)
if 'error' in job_info:
mcm_request.test_failure(message=job_info['error'],
what='Validation run test',
rewind=True)
return {}
job_info[self.DOC_REV] = mcm_request.json()[self.DOC_REV]
job_info[self.DOC_VALIDATION] = mcm_request.get_attribute(
self.DOC_VALIDATION
) # this field change when calling request.get_setup_file, to be propagated in case of success
return job_info
def delete_request(self, crid):
crdb = database('chained_requests')
rdb = database('requests')
mcm_cr = chained_request(crdb.get(crid))
# get all objects
mcm_r_s = []
for (i, rid) in enumerate(mcm_cr.get_attribute('chain')):
mcm_r = request(rdb.get(rid))
# this is not a valid check as it is allowed to remove a chain around already running requests
# if mcm_r.get_attribute('status') != 'new':
# return {"results":False,"message" : "the request %s part of the chain %s for action %s is not in new status"%( mcm_r.get_attribute('prepid'),
# crid,
# mcm_a.get_attribute('prepid'))}
in_chains = mcm_r.get_attribute('member_of_chain')
in_chains.remove(crid)
self.logger.debug("Removing ChainAction member_of_chain: %s to request: %s" % (
mcm_cr.get_attribute("prepid"), mcm_r.get_attribute('prepid')))
mcm_r.set_attribute('member_of_chain', in_chains)
if i == 0:
if len(in_chains) == 0 and mcm_r.get_attribute('status') != 'new':
return {"results": False, "message": "the request %s, not in status new, at the root of the chain will not be chained anymore" % rid}
else:
if len(in_chains) == 0:
return {"results": False, "message": "the request %s, not at the root of the chain will not be chained anymore" % rid}
mcm_r.update_history({'action': 'leave', 'step': crid})
mcm_r_s.append(mcm_r)
if mcm_cr.get_attribute('action_parameters')['flag']:
return {
"results": False,
"message": "the action for %s is not disabled" % (crid)
}
# then save all changes
for mcm_r in mcm_r_s:
if not rdb.update(mcm_r.json()):
return {"results": False, "message": "Could not save request " + mcm_r.get_attribute('prepid')}
return {"results": crdb.delete(crid)}
def get(self, chained_request_id):
"""
Does a soft reset to all relevant request in the chain
"""
crdb = database('chained_requests')
rdb = database('requests')
mcm_cr = chained_request(crdb.get(chained_request_id))
for rid in reversed(mcm_cr.get_attribute('chain')[:mcm_cr.get_attribute('step') + 1]):
# from the current one to the first one REVERSED
mcm_r = request(rdb.get(rid))
try:
mcm_r.reset(hard=False)
except Exception as e:
return {'prepid': chained_request_id, 'results': False, 'message': str(e)}
mcm_r.reload()
mcm_cr = chained_request(crdb.get(chained_request_id))
mcm_cr.set_attribute('step', max(0, mcm_cr.get_attribute('chain').index(rid) - 1))
mcm_cr.reload()
return {'prepid': chained_request_id, 'results': True}
def inspect_priority(self, forChains=None):
##JR: until put in the proper place
chains=self.get_attribute('chains')
crdb = database('chained_requests')
okay = True
for inCC in chains:
### this is the old convention
#if 'flag' in chains[inCC] and chains[inCC]['flag']:
# if 'chains' in chains[inCC]:
# for acr in chains[inCC]['chains']:
# if forChains and not acr in forChains: continue
# cr=chained_request(crdb.get(acr))
# cc=cr.get_attribute('member_of_campaign')
# #if 'block_number' in chains[cc] and chains[cc]['block_number']:
# if chains[cc]['block_number']:
# cr.set_priority(chains[cc]['block_number'])
# self.logger.log('Set priority block %s to %s'%(chains[cc]['block_number'],cr.get_attribute('prepid')))
# else:
# self.logger.error('Could not set block %s to %s'%(chains[cc]['block_number'],cr.get_attribute('prepid')))
## new convention
if 'chains' in chains[inCC] and type(chains[inCC]['chains'])==dict:
for acr in chains[inCC]['chains']:
if forChains and not acr in forChains: continue
bn=chains[inCC]['chains'][acr]['block_number']
cr=chained_request(crdb.get(acr))
if bn:
self.logger.log('Set priority block %s to %s'%( bn , acr))
if not cr.set_priority( bn ):
okay = False
else:
self.logger.error('Could not set block %s to %s'%( bn , acr))
rd = database('requests')
if rd.document_exists(self.get_attribute('prepid')):
r= request(rd.get(self.get_attribute('prepid')))
self.set_attribute('dataset_name',r.get_attribute('dataset_name'))
return okay
def internal_run(self):
if not self.lock.acquire(blocking=False):
self.logger.error(
"Could not acquire lock for ChainRequestInjector. prepid %s" %
(self.prepid))
return False
try:
crdb = database('chained_requests')
rdb = database('requests')
batch_name = None
if not crdb.document_exists(self.prepid):
# it's a request actually, pick up all chains containing it
mcm_r = rdb.get(self.prepid)
# mcm_crs = crdb.query(query="root_request==%s"% self.prepid) ## not only when its the root of
mcm_crs = crdb.query(query="contains==%s" % self.prepid)
task_name = 'task_' + self.prepid
batch_type = 'Task_' + mcm_r['member_of_campaign']
else:
mcm_crs = [crdb.get(self.prepid)]
current_step_prepid = mcm_crs[0]['chain'][mcm_crs[0]['step']]
mcm_request = rdb.get(current_step_prepid)
task_name = 'task_' + current_step_prepid
batch_type = 'Task_' + mcm_request['member_of_campaign']
if len(mcm_crs) == 0:
return False
mcm_rs = []
# upload all config files to config cache, with "configuration economy" already implemented
for cr in mcm_crs:
mcm_cr = chained_request(cr)
chain = mcm_cr.get_attribute(
'chain')[mcm_cr.get_attribute('step'):]
for request_prepid in chain:
mcm_rs.append(request(rdb.get(request_prepid)))
if self.check_approval and mcm_rs[-1].get_attribute(
'approval') != 'submit':
message = 'requests %s is in "%s"/"%s" status/approval, requires "approved"/"submit"' % (
request_prepid, mcm_rs[-1].get_attribute('status'),
mcm_rs[-1].get_attribute('approval'))
self.logger.error(message)
subject = '%s injection failed' % mcm_cr.get_attribute(
'prepid')
notification(
subject,
message, [],
group=notification.CHAINED_REQUESTS,
action_objects=[mcm_cr.get_attribute('prepid')],
object_type='chained_requests',
base_object=mcm_cr)
mcm_cr.notify(subject, message)
return False
if mcm_rs[-1].get_attribute('status') != 'approved':
# change the return format to percolate the error message
message = 'requests %s in in "%s"/"%s" status/approval, requires "approved"/"submit"' % (
request_prepid, mcm_rs[-1].get_attribute('status'),
mcm_rs[-1].get_attribute('approval'))
self.logger.error(message)
subject = '%s injection failed' % mcm_cr.get_attribute(
'prepid')
notification(
subject,
message, [],
group=notification.CHAINED_REQUESTS,
action_objects=[mcm_cr.get_attribute('prepid')],
object_type='chained_requests',
base_object=mcm_cr)
mcm_cr.notify(subject, message)
return False
uploader = ConfigMakerAndUploader(
prepid=request_prepid,
lock=locker.lock(request_prepid))
if not uploader.internal_run():
message = 'Problem with uploading the configuration for request %s' % (
request_prepid)
notification(
'Configuration upload failed',
message, [],
group=notification.CHAINED_REQUESTS,
action_objects=[mcm_cr.get_attribute('prepid')],
object_type='chained_requests',
base_object=mcm_cr)
mcm_cr.notify('Configuration upload failed', message)
self.logger.error(message)
return False
mcm_r = mcm_rs[-1]
batch_name = BatchPrepId().next_batch_id(batch_type,
create_batch=True)
semaphore_events.increment(batch_name)
self.logger.error('found batch %s' % batch_name)
with ssh_executor(server='vocms081.cern.ch') as ssh:
cmd = self.make_command(mcm_r)
self.logger.error('prepared command %s' % cmd)
# modify here to have the command to be executed
_, stdout, stderr = ssh.execute(cmd)
output = stdout.read()
error = stderr.read()
self.logger.info(output)
self.logger.info(error)
injected_requests = [
l.split()[-1] for l in output.split('\n')
if l.startswith('Injected workflow:')
]
if not injected_requests:
self.injection_error(
'Injection has succeeded but no request manager names were registered. Check with administrators. \nOutput: \n%s\n\nError: \n%s'
% (output, error), mcm_rs)
return False
# what gets printed into the batch object
added_requests = []
once = set()
for mcm_r in mcm_rs:
if mcm_r.get_attribute('prepid') in once:
continue
once.add(mcm_r.get_attribute('prepid'))
added = [{
'name': app_req,
'content': {
'pdmv_prep_id': mcm_r.get_attribute('prepid')
}
} for app_req in injected_requests]
added_requests.extend(added)
# edit the batch object
with locker.lock(batch_name):
bdb = database('batches')
bat = batch(bdb.get(batch_name))
bat.add_requests(added_requests)
bat.update_history({
'action': 'updated',
'step': task_name
})
bat.reload()
# reload the content of all requests as they might have changed already
added = [{
'name': app_req,
'content': {
'pdmv_prep_id': task_name
}
} for app_req in injected_requests]
seen = set()
for cr in mcm_crs:
mcm_cr = chained_request(cr)
chain = mcm_cr.get_attribute(
'chain')[mcm_cr.get_attribute('step'):]
message = ""
for rn in chain:
if rn in seen:
continue # don't do it twice
seen.add(rn)
mcm_r = request(rdb.get(rn))
message += mcm_r.textified()
message += "\n\n"
mcm_r.set_attribute('reqmgr_name', added)
mcm_r.update_history({
'action': 'inject',
'step': batch_name
})
if not self.check_approval:
mcm_r.set_attribute('approval', 'submit')
# set the status to submitted
mcm_r.set_status(
step=mcm_r._json_base__status.index('submitted'),
with_notification=False)
mcm_r.reload()
mcm_cr.set_attribute('last_status',
mcm_r.get_attribute('status'))
# re-get the object
mcm_cr = chained_request(crdb.get(cr['prepid']))
# take care of changes to the chain
mcm_cr.update_history({
'action': 'inject',
'step': batch_name
})
mcm_cr.set_attribute(
'step',
len(mcm_cr.get_attribute('chain')) - 1)
mcm_cr.set_attribute('status', 'processing')
subject = 'Injection succeeded for %s' % mcm_cr.get_attribute(
'prepid')
notification(
subject,
message, [],
group=notification.CHAINED_REQUESTS,
action_objects=[mcm_cr.get_attribute('prepid')],
object_type='chained_requests',
base_object=mcm_cr)
mcm_cr.notify(subject, message)
mcm_cr.reload()
return True
except Exception:
self.injection_error(
"Error with injecting chains for %s :\n %s" %
(self.prepid, traceback.format_exc()), [])
finally: # we decrement batch id and release lock on prepid+lower semaphore
if batch_name: # ditry thing for now. Because batch name can be None for certain use-cases in code above
semaphore_events.decrement(batch_name)
self.lock.release()
self.queue_lock.release()
def submit_chain(self, prepid, run_test_path):
mcm_chained_request = chained_request(
self.chained_request_db.get(prepid))
except_requests = []
reset = False
# If a request of a chain was singly submmited to validation and then somebody reseted it, we will find it here
for request_prepid in mcm_chained_request.get_attribute(
'chain')[mcm_chained_request.get_attribute('step'):]:
if request_prepid in self.submmited_prepids_set:
except_requests.append(request_prepid)
reset = True
if reset:
message = "Requests %s of the chain %s are already in validation" % (
except_requests, prepid)
self.logger.error(message)
mcm_chained_request.reset_requests(message,
except_requests=except_requests)
return {}
to_write = mcm_chained_request.get_setup(run=True,
validation=True,
for_validation=True)
file_name = self.TEST_FILE_NAME % prepid
if not self.create_test_file(to_write, run_test_path, file_name):
mcm_chained_request.reset_requests(
'There was a problem while creating the file for prepid: %s' %
(mcm_chained_request.get_attribute('prepid')))
return {}
requests_in_chain = {}
for request_prepid in mcm_chained_request.get_attribute(
'chain')[mcm_chained_request.get_attribute('step'):]:
mcm_request = request(self.request_db.get(request_prepid))
if not mcm_request.is_root and 'validation' not in mcm_request._json_base__status: # only root or possible root requests
break
status = mcm_request.get_attribute('status')
approval = mcm_request.get_attribute('approval')
if status != 'new' or approval != 'validation':
message = "The request %s of chain %s is in status: %s approval: %s" % (
request_prepid, prepid, status, approval)
self.logger.error(message)
mcm_chained_request.reset_requests(message)
return {}
requests_in_chain[request_prepid] = mcm_request.json()[
self.DOC_REV]
if not len(requests_in_chain):
message = 'No requests to be validated in chain: %s' % prepid
self.logger.info(message)
mcm_chained_request.reset_requests(message)
return {}
timeout, memory, threads = mcm_chained_request.get_timeout_memory_threads(
)
self.create_htcondor_config_file(run_test_path, prepid, timeout,
memory, threads,
list(requests_in_chain.iterkeys()))
job_info = self.execute_command_submission(prepid, run_test_path)
if 'error' in job_info:
mcm_chained_request.reset_requests(job_info['error'])
return {}
job_info[self.CHAIN_REQUESTS] = requests_in_chain
return job_info
def process_finished_chain_success(self, prepid, doc_info, job_out,
error_out, log_out):
mcm_chained_request = chained_request(
self.chained_request_db.get(prepid))
requests_in_chain = []
success = True
failed_request_prepid = None
for request_prepid, doc_rev in doc_info[
self.CHAIN_REQUESTS].iteritems():
mcm_request = request(self.request_db.get(request_prepid))
# Increase counters for all requests, but save error message only for the first failed request
if success and doc_rev != mcm_request.json()[self.DOC_REV]:
message = 'The request %s in the chain %s has changed during the run test procedure, preventing from being saved' % (
request_prepid, prepid)
success = False
failed_request_prepid = request_prepid
mcm_request.inc_validations_counter()
mcm_request.reload()
if not success:
mcm_chained_request.reset_requests(
message, notify_one=failed_request_prepid)
return
for request_prepid, doc_rev in doc_info[
self.CHAIN_REQUESTS].iteritems():
self.logger.info('Processing request %s in chain %s' %
(request_prepid, prepid))
mcm_request = request(self.request_db.get(request_prepid))
success = True
path = self.test_directory_path + prepid + '/'
try:
success, error = mcm_request.pickup_all_performance(path)
error = 'Error:\n%s\n Error out:\n%s\n' % (error, error_out)
except request.WrongTimeEvent as ex:
self.logger.error('Exception: %s' % (ex))
error = 'Exception:\n%s\n' % (ex)
success = False
retry_validation = self.process_request_wrong_time_event(
mcm_request, prepid)
if retry_validation:
return
if not success:
message = 'Error while picking up all the performance for request %s of chain %s: \n Error:\n%s\n Job out:\n%s\n Error out: \n%s\n Log out: \n%s\n' % (
request_prepid, prepid, error, job_out, error_out, log_out)
mcm_chained_request.reset_requests(message,
notify_one=request_prepid)
return
requests_in_chain.append(mcm_request)
for mcm_request in requests_in_chain:
mcm_request.set_status(with_notification=True)
if not self.request_db.update(mcm_request.json()):
request_prepid = mcm_request.get_attribute('prepid')
message = "The request %s of chain %s could not be saved after the runtest procedure" % (
request_prepid, prepid)
self.logger.error(message)
# reset it and keep saving requests
mcm_request.test_failure(message=message,
what='Chain validation run test',
rewind=True)
mcm_chained_request.reload(
save_current=False
) # setting new requests status change the chain object
mcm_chained_request.set_attribute('validate', 0)
if not self.chained_request_db.update(mcm_chained_request.json()):
message = 'Problem saving changes in chain %s, set validate = False ASAP!' % prepid
self.logger.error(message)
notification(
'Chained validation run test',
message, [],
group=notification.CHAINED_REQUESTS,
action_objects=[mcm_chained_request.get_attribute('prepid')],
object_type='chained_requests',
base_object=mcm_chained_request)
mcm_chained_request.notify('Chained validation run test', message)
return
self.logger.info('Validation job for prepid %s SUCCESSFUL!!!' % prepid)
def generate_request(self,
aid,
reserve=False,
with_notify=True,
special=False):
adb = database('actions')
ccdb = database('chained_campaigns')
crdb = database('chained_requests')
rdb = database('requests')
if not adb.document_exists(aid):
return {'results': False, 'message': '%s does not exist' % (aid)}
self.logger.log(
'Generating all selected chained_requests for action %s' % (aid))
mcm_a = action(adb.get(aid))
chains = mcm_a.get_attribute('chains')
hasChainsChanged = False
new_chains = []
for cc in chains:
if 'flag' in chains[cc] and chains[cc]['flag']:
## in the new convention, that means that something needs to be created
mcm_cc = chained_campaign(ccdb.get(cc))
new_cr = mcm_cc.generate_request(aid)
if not 'chains' in chains[cc]:
chains[cc]['chains'] = {}
chains[cc]['chains'][new_cr['prepid']] = {}
to_transfer = ['flag', 'threshold', 'staged', 'block_number']
for item in to_transfer:
if item in chains[cc]:
chains[cc]['chains'][
new_cr['prepid']][item] = chains[cc][item]
chains[cc].pop(item)
hasChainsChanged = True
## get the root request
req = request(json_input=rdb.get(aid))
new_cr['last_status'] = req.get_attribute('status')
if new_cr['last_status'] in ['submitted', 'done']:
new_cr['status'] = 'processing'
if special:
new_cr['approval'] = 'none'
if not crdb.update(new_cr):
return {
'results': False,
'message': 'could not save %s' % (new_cr['prepid'])
}
## update the cc history
ccdb.update(mcm_cc.json())
new_chains.append(new_cr['prepid'])
# then let the root request know that it is part of a chained request
inchains = req.get_attribute('member_of_chain')
inchains.append(new_cr['prepid'])
inchains.sort()
req.set_attribute('member_of_chain', list(set(inchains)))
req.update_history({
'action': 'join chain',
'step': new_cr['prepid']
})
if with_notify:
req.notify(
"Request {0} joined chain".format(
req.get_attribute('prepid')),
"Request {0} has successfully joined chain {1}".format(
req.get_attribute('prepid'), new_cr['prepid']),
Nchild=0,
accumulate=True)
mcm_a.update_history({
'action': 'add',
'step': new_cr['prepid']
})
rdb.update(req.json())
if hasChainsChanged:
#the chains parameter might have changed
mcm_a.set_attribute('chains', chains)
adb.update(mcm_a.json())
#and set priorities properly to all requests concerned
priority_set = mcm_a.inspect_priority(forChains=new_chains)
## do the reservation of the whole chain ?
res = []
if reserve:
for cr in new_chains:
mcm_cr = chained_request(crdb.get(cr))
res.append(mcm_cr.reserve())
crdb.update(mcm_cr.json())
if priority_set:
return {"results": True, "prepid": mcm_a.get_attribute('prepid')}
else:
return {
"results": False,
"prepid": mcm_a.get_attribute('prepid'),
"message": "Priorities not set properly"
}
def flow_to_next_step(self,
input_dataset='',
block_black_list=None,
block_white_list=None,
check_stats=True,
reserve=False):
if not block_white_list: block_white_list = []
if not block_black_list: block_black_list = []
self.logger.log('Flowing chained_request %s to next step...' %
(self.get_attribute('_id')))
if not self.get_attribute('chain'):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'chained_request %s has got no root' %
(self.get_attribute('_id')))
# check on the approval of the chained request before all
## let it be flowing regardless
#if self.get_attribute('approval') == 'none':
# raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
# 'The approval of the chained request is none, and therefore flow cannot happen')
#this operation requires to access all sorts of objects
rdb = database('requests')
cdb = database('campaigns')
ccdb = database('chained_campaigns')
crdb = database('chained_requests')
fdb = database('flows')
adb = database('actions')
l_type = locator()
current_step = len(self.get_attribute(
'chain')) - 1 if reserve else self.get_attribute('step')
current_id = self.get_attribute('chain')[current_step]
next_step = current_step + 1
if not rdb.document_exists(current_id):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'the request %s does not exist' % current_id)
current_request = request(rdb.get(current_id))
current_campaign = campaign(
cdb.get(current_request.get_attribute('member_of_campaign')))
if not ccdb.document_exists(self.get_attribute('member_of_campaign')):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'the chain request %s is member of %s that does not exist' %
(self.get_attribute('_id'),
self.get_attribute('member_of_campaign')))
mcm_cc = ccdb.get(self.get_attribute('member_of_campaign'))
if next_step >= len(mcm_cc['campaigns']):
if reserve: return False
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'chained_campaign %s does not allow any further flowing.' %
(self.get_attribute('member_of_campaign')))
if not reserve:
## is the current request in the proper approval
allowed_request_approvals = ['submit']
if current_request.get_attribute(
'approval') not in allowed_request_approvals:
raise self.NotApprovedException(
current_request.get_attribute('prepid'),
current_request.get_attribute('approval'),
allowed_request_approvals)
## is the current request in the proper status
allowed_request_statuses = ['submitted', 'done']
if current_request.get_attribute(
'status') not in allowed_request_statuses:
raise self.NotInProperStateException(
current_request.get_attribute('prepid'),
current_request.get_attribute('status'),
allowed_request_statuses)
original_action_id = self.get_attribute('chain')[0]
original_action_item = self.retrieve_original_action_item(
adb, original_action_id)
## what is the campaign to go to next and with which flow
(next_campaign_id, flow_name) = mcm_cc['campaigns'][next_step]
if not fdb.document_exists(flow_name):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'The flow %s does not exist' % flow_name)
mcm_f = flow(fdb.get(flow_name))
if not 'sequences' in mcm_f.get_attribute('request_parameters'):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'The flow %s does not contain sequences information.' %
(flow_name))
if not cdb.document_exists(next_campaign_id):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'The next campaign %s does not exist' % next_campaign_id)
next_campaign = campaign(cdb.get(next_campaign_id))
if len(next_campaign.get_attribute('sequences')) != len(
mcm_f.get_attribute('request_parameters')['sequences']):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'the sequences changes in flow %s are not consistent with the next campaign %s'
% (flow_name, next_campaign_id))
if next_campaign.get_attribute(
'energy') != current_campaign.get_attribute('energy'):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'cannot flow any further. Request {0} has inconsistent energy.'
.format(next_campaign.get_attribute("prepid")))
if not next_campaign.is_release_greater_or_equal_to(
current_campaign.get_attribute('cmssw_release')):
raise self.ChainedRequestCannotFlowException(
self.get_attribute("_id"),
'cannot flow any further. Request {0} has lower release version.'
.format(next_campaign.get_attribute("prepid")))
if next_campaign.get_attribute('type') == 'MCReproc' and (
not 'time_event' in mcm_f.get_attribute('request_parameters')):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'the flow is getting into a MCReproc campaign but not time per event is specified'
)
if next_campaign.get_attribute('type') == 'MCReproc' and (
not 'size_event' in mcm_f.get_attribute('request_parameters')):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'the flow is getting into a MCReproc campaign but not size per event is specified'
)
## check that it is allowed to flow
allowed_flow_approvals = ['flow', 'submit']
###### cascade of checks
"""
if not reserve and not mcm_f.get_attribute('approval') in allowed_flow_approvals:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The flow (%s) is not in proper approval state (%s)'%(
mcm_f.get_attribute('prepid'),
mcm_f.get_attribute('approval')))
if not reserve and not self.get_attribute('approval') in allowed_flow_approvals:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The chained request (%s) is not in the proper approval state (%s)'% (
self.get_attribute('_id'),
self.get_attribute('approval')))
"""
if not reserve and not mcm_f.get_attribute(
'approval') in allowed_flow_approvals:
if not self.get_attribute('approval') in allowed_flow_approvals:
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'Neither the flow (%s) nor the chained request (%s) approvals allow for flowing'
% (mcm_f.get_attribute('approval'),
self.get_attribute('approval')))
if next_campaign.get_attribute('status') == 'stopped':
raise self.CampaignStoppedException(next_campaign_id)
#what is going to be the required number of events for the next request
#update the stats to its best
if not reserve:
current_request.get_stats()
next_total_events = current_request.get_attribute(
'completed_events')
## get the original expected events and allow a margin of 5% less statistics
statistics_fraction = settings().get_value('statistics_fraction')
current_eff_error = 1. - current_request.get_efficiency_error()
statistics_fraction = min(statistics_fraction, current_eff_error)
completed_events_to_pass = int(
current_request.get_attribute('total_events') *
statistics_fraction)
notify_on_fail = True ## to be tuned according to the specific cases
if current_request.get_attribute('completed_events') <= 0:
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'The number of events completed is negative or null')
else:
allowed_request_statuses = ['done']
## determine if this is a root -> non-root transition to potentially apply staged number
at_a_transition = (current_campaign.get_attribute('root') != 1
and next_campaign.get_attribute('root')
== 1)
if ('staged' in original_action_item or 'threshold'
in original_action_item) and at_a_transition:
allowed_request_statuses.append('submitted')
##check status
if not current_request.get_attribute(
'status') in allowed_request_statuses:
raise self.NotInProperStateException(
current_request.get_attribute('prepid'),
current_request.get_attribute('status'),
allowed_request_statuses)
##special check at transition that the statistics is good enough
if at_a_transition:
# at a root -> non-root transition only does the staged/threshold functions !
if 'staged' in original_action_item:
next_total_events = int(original_action_item['staged'])
completed_events_to_pass = next_total_events
if 'threshold' in original_action_item:
next_total_events = int(
current_request.get_attribute('total_events') *
float(original_action_item['threshold'] / 100.))
completed_events_to_pass = next_total_events
if check_stats and (
current_request.get_attribute('completed_events') <
completed_events_to_pass):
if notify_on_fail:
current_request.notify(
'Flowing %s with not enough statistics' %
(current_request.get_attribute('prepid')),
'For the request %s, the completed statistics %s is not enough to fullfill the requirement to the next level : need at least %s in chain %s \n\n Please report to the operation HN or at the next MccM what action should be taken.\n\n %srequests?prepid=%s\n%schained_requests?contains=%s\n%schained_requests?prepid=%s '
% (current_request.get_attribute('prepid'),
current_request.get_attribute('completed_events'),
completed_events_to_pass,
self.get_attribute('prepid'), l_type.baseurl(),
current_request.get_attribute('prepid'),
l_type.baseurl(),
current_request.get_attribute('prepid'),
l_type.baseurl(), self.get_attribute('prepid')),
accumulate=True)
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'The number of events completed (%s) is not enough for the requirement (%s)'
% (current_request.get_attribute('completed_events'),
completed_events_to_pass))
## select what is to happened : [create, patch, use]
next_id = None
approach = None
next_request = None
if next_step != len(self.get_attribute('chain')):
#not at the end
next_id = self.get_attribute('chain')[next_step]
if not rdb.document_exists(next_id):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'The next request (%s) according to the step (%s) does not exist'
% (next_id, next_step))
next_request = request(rdb.get(next_id))
if next_request.get_attribute('status') == 'new':
#most likely a rewind + resub
approach = 'patch'
else:
##this is always the case in chains reserved from existing things: so use the next request
approach = 'use'
#raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
#'This should never happen. (%s) is next according to step (%s), but is not in new status (%s)' % (
#next_id, next_step, next_request.get_attribute('status')))
else:
## look in *other* chained campaigns whether you can suck in an existing request
## look up all chained requests that start from the same root request
## remove <pwg>-chain_ and the -serial number, replacing _ with a .
toMatch = '.'.join(
self.get_attribute('prepid').split('_')[1:][0:next_step +
1]).split('-')[0]
## make sure they get ordered by prepid
related_crs = sorted(crdb.queries(
['root_request==%s' % original_action_id]),
key=lambda cr: cr['prepid'])
vetoed_last = []
for existing_cr in related_crs:
## exclude itself
if existing_cr['prepid'] == self.get_attribute('prepid'):
continue
## prevent from using a request from within the same exact chained_campaigns
if existing_cr['member_of_campaign'] == self.get_attribute(
'member_of_campaign'):
mcm_cr = chained_request(crdb.get(existing_cr['prepid']))
if len(mcm_cr.get_attribute('chain')) > next_step:
## one existing request in the very same chained campaign has already something used, make sure it is not going to be used
vetoed_last.append(
mcm_cr.get_attribute('chain')[next_step])
continue
else:
continue
for existing_cr in related_crs:
## exclude itself
if existing_cr['prepid'] == self.get_attribute('prepid'):
continue
## prevent from using a request from within the same exact chained_campaigns
if existing_cr['member_of_campaign'] == self.get_attribute(
'member_of_campaign'):
continue
truncated = '.'.join(
existing_cr['prepid'].split('_')[1:][0:next_step +
1]).split('-')[0]
self.logger.error('to match : %s , this one %s' %
(toMatch, truncated))
if truncated == toMatch:
#we found a chained request that starts with all same steps
mcm_cr = chained_request(crdb.get(existing_cr['prepid']))
if len(mcm_cr.get_attribute('chain')) <= next_step:
#found one, but it has not enough content either
continue
if mcm_cr.get_attribute('chain')[next_step] in vetoed_last:
continue
next_id = mcm_cr.get_attribute('chain')[next_step]
break
if next_id:
approach = 'use'
else:
approach = 'create'
if approach == 'create':
from rest_api.RequestPrepId import RequestPrepId
next_id = RequestPrepId().next_prepid(
current_request.get_attribute('pwg'), next_campaign_id)
next_request = request(rdb.get(next_id))
request.transfer(current_request, next_request)
self.request_join(next_request)
elif approach == 'use':
## there exists a request in another chained campaign that can be re-used here.
# take this one. advance and go on
next_request = request(rdb.get(next_id))
if not reserve:
self.set_attribute('step', next_step)
self.set_attribute('last_status',
next_request.get_attribute('status'))
self.update_history({'action': 'flow', 'step': str(next_id)})
self.set_attribute('status', 'processing')
if not self.get_attribute("prepid") in next_request.get_attribute(
"member_of_chain"):
## register the chain to the next request
self.request_join(next_request)
saved = rdb.update(next_request.json())
if not saved:
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'Unable to save %s with updated member_of_chains' %
next_id)
return True
elif approach == 'patch':
## there exists already a request in the chain (step!=last) and it is usable for the next stage
next_request = request(next_campaign.add_request(rdb.get(next_id)))
## propagate again some of the fields of the previous request.
request.transfer(current_request, next_request)
else:
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'Unrecognized approach %s' % approach)
#current_request -> next_request
#current_campaign -> next_campaign
##determine whether we have an input dataset for the next request
if len(current_request.get_attribute('reqmgr_name')):
last_wma = current_request.get_attribute('reqmgr_name')[-1]
if 'content' in last_wma and 'pdmv_dataset_name' in last_wma[
'content']:
input_dataset = last_wma['content']['pdmv_dataset_name']
else:
statsDB = database('stats',
url='http://cms-pdmv-stats.cern.ch:5984/')
if statsDB.document_exists(last_wma['name']):
latestStatus = statsDB.get(last_wma['name'])
input_dataset = latestStatus['pdmv_dataset_name']
if input_dataset:
next_request.set_attribute('input_dataset', input_dataset)
## set blocks restriction if any
if block_black_list:
next_request.set_attribute('block_black_list', block_black_list)
if block_white_list:
next_request.set_attribute('block_white_list', block_white_list)
## register the flow to the request
next_request.set_attribute('flown_with', flow_name)
##assemble the campaign+flow => request
request.put_together(next_campaign, mcm_f, next_request)
if not reserve:
#already taking stage and threshold into account
next_request.set_attribute('total_events', next_total_events)
next_request.update_history({
'action': 'flow',
'step': self.get_attribute('prepid')
})
request_saved = rdb.save(next_request.json())
if not request_saved:
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'Could not save the new request %s' %
(next_request.get_attribute('prepid')))
## inspect priority
self.set_priority(original_action_item['block_number'])
if not reserve:
# sync last status
self.set_attribute('last_status',
next_request.get_attribute('status'))
# we can only be processing at this point
self.set_attribute('status', 'processing')
# set to next step
self.set_attribute('step', next_step)
self.update_history({
'action': 'flow',
'step': next_request.get_attribute('prepid')
})
if not reserve:
notification_subject = 'Flow for request %s in %s' % (
current_request.get_attribute('prepid'), next_campaign_id)
notification_text = 'The request %s has been flown within:\n \t %s \n into campaign:\n \t %s \n using:\n \t %s \n creating the new request:\n \t %s \n as part of:\n \t %s \n and from the produced dataset:\n %s \n\n%srequests?prepid=%s \n%srequests?prepid=%s \n' % (
current_request.get_attribute('prepid'),
self.get_attribute('member_of_campaign'), next_campaign_id,
flow_name, next_request.get_attribute('prepid'),
self.get_attribute('prepid'),
next_request.get_attribute('input_dataset'), l_type.baseurl(),
current_request.get_attribute('prepid'), l_type.baseurl(),
next_request.get_attribute('prepid'))
current_request.notify(notification_subject,
notification_text,
accumulate=True)
else:
notification_subject = 'Reservation of request {0}'.format(
next_request.get_attribute('prepid'))
notification_text = 'The request {0} of campaign \n\t{2}\nhas been reserved as part of \n\t{1}\nas the next step for {4}\n\n{3}requests?prepid={4}\n{5}requests?prepid={6}\n'.format(
next_request.get_attribute('prepid'),
self.get_attribute('prepid'),
next_campaign_id,
l_type.baseurl(),
current_request.get_attribute('prepid'),
l_type.baseurl(),
next_request.get_attribute('prepid'),
)
next_request.notify(notification_subject,
notification_text,
accumulate=True)
return True
def get_actors(self, N=-1, what='author_username', Nchild=-1):
rdb = database('requests')
last_r = request(rdb.get(self.get_attribute('chain')[-1]))
return last_r.get_actors(N, what, Nchild)
def multiple_inspect(self, cid, in_statuses=['submitted', 'approved']):
clist = list(set(cid.rsplit(',')))
res = []
rdb = database('requests')
index = 0
self.logger.error(
"Chain inspect begin. Number of chains to be inspected: %s" %
(len(clist)))
try:
while len(clist) > index:
yield dumps(
{"current cr element": "%s/%s" % (index, len(clist))},
indent=2)
query = rdb.construct_lucene_complex_query([
('member_of_campaign', {
'value': clist[index:index + 1]
}), ('status', {
'value': in_statuses
})
])
##do another loop over the requests themselves
req_page = 0
request_res = rdb.full_text_search('search',
query,
page=req_page)
while len(request_res) > 0:
self.logger.info("inspecting single requests. page: %s" %
(req_page))
for r in request_res:
self.logger.info("running inspect on request: %s" %
(r['prepid']))
mcm_r = request(r)
if mcm_r:
#making it as a stream
yield dumps(mcm_r.inspect(), indent=4)
else:
#making it as a stream
yield dumps(
{
"prepid": r,
"results": False,
'message': '%s does not exist' % (r)
},
indent=4)
req_page += 1
request_res = rdb.full_text_search('search',
query,
page=req_page)
time.sleep(0.5)
index += 1
time.sleep(1)
except Exception as e:
subject = "Exception while inspecting request "
message = "Request: %s \n %s traceback: \n %s" % (
mcm_r.get_attribute('prepid'), str(e), traceback.format_exc())
self.logger.error(subject + message)
notification(subject,
message, [],
group=notification.REQUEST_OPERATIONS,
action_objects=[mcm_r.get_attribute('prepid')],
object_type='requests',
base_object=mcm_r)
mcm_r.notify(subject, message, accumulate=True)
self.logger.info("Campaign inspection finished!")
def get(self, chained_request_id):
"""
Provide the taskchain dictionnary for uploading to request manager
"""
kwargs = self.parser.parse_args()
crdb = database('chained_requests')
rdb = database('requests')
settingsDB = database('settings')
__DT_prio = settingsDB.get('datatier_input')["value"]
def tranform_to_step_chain(wma_dict, total_time_evt, total_size_evt):
# replace Task -> Step in inside dictionaries
for task_num in range(wma_dict["TaskChain"]):
for elem in wma_dict["Task%s" % (task_num + 1)]:
if "Task" in elem:
wma_dict["Task%s" % (task_num + 1)][elem.replace(
"Task",
"Step")] = wma_dict["Task%s" %
(task_num + 1)].pop(elem)
# we later add the global fields
del (wma_dict["Task%s" % (task_num + 1)]["TimePerEvent"])
del (wma_dict["Task%s" % (task_num + 1)]["SizePerEvent"])
# we do same replacement on top level
for el in wma_dict:
if wma_dict[el].__class__ == str and "task" in wma_dict[el]:
wma_dict[el] = wma_dict[el].replace("task", "step")
if "Task" in el:
wma_dict[el.replace("Task", "Step")] = wma_dict.pop(el)
wma_dict["RequestType"] = "StepChain"
# as of 2017-05 StepChain needs these as sum of internal Tasks
wma_dict["TimePerEvent"] = total_time_evt
wma_dict["SizePerEvent"] = total_size_evt
return wma_dict
if not crdb.document_exists(chained_request_id):
# it's a request actually, pick up all chains containing it
mcm_r = rdb.get(chained_request_id)
# mcm_crs = crdb.query(query="root_request==%s"% chained_request_id) ## not only when its the root of
mcm_crs = crdb.query(query="contains==%s" % chained_request_id)
task_name = 'task_' + chained_request_id
else:
mcm_crs = [crdb.get(chained_request_id)]
# here name should be task_chain's[curr_step] request_prepid
# so it would be task_prepid-of-current-request same as in top
__req_id = mcm_crs[0]['chain'][mcm_crs[0]['step']]
task_name = 'task_' + __req_id
if len(mcm_crs) == 0:
return {}
tasktree = {}
ignore_status = False
__total_time_evt = 0
__total_size_evt = 0
if kwargs['scratch'] is not None:
ignore_status = True
veto_point = None
if kwargs['upto'] is not None:
veto_point = kwargs['upto']
__chains_type = []
for mcm_cr in mcm_crs:
__chains_type.append(mcm_cr["chain_type"])
starting_point = mcm_cr['step']
if ignore_status:
starting_point = 0
for (ir, r) in enumerate(mcm_cr['chain']):
if (ir < starting_point):
continue # ad no task for things before what is already done
if veto_point and (ir > veto_point):
continue
mcm_r = request(rdb.get(r))
if mcm_r.get_attribute(
'status') == 'done' and not ignore_status:
continue
if r not in tasktree:
tasktree[r] = {'next': [], 'dict': [], 'rank': ir}
base = ir == 0 and mcm_r.get_wmagent_type() in [
'MonteCarlo', 'LHEStepZero'
]
depend = (
ir > starting_point
) # all the ones later than the starting point depend on a previous task
if ir < (len(mcm_cr['chain']) - 1):
tasktree[r]['next'].append(mcm_cr['chain'][ir + 1])
tasktree[r]['dict'] = mcm_r.request_to_tasks(base, depend)
# if request is added to tasktree, we save global sums for StepChains
__total_time_evt += mcm_r.get_sum_time_events()
__total_size_evt += sum(mcm_r.get_attribute("size_event"))
for (r, item) in tasktree.items():
# here we should generate unique list of steps+output tiers
# as we iterate over requests in tasktree
__uniq_tiers = []
for el in item['dict']:
# map of tiers and taskID in order of steps
__uniq_tiers.append((el['TaskName'], el['_output_tiers_']))
item['unique_tiers_'] = __uniq_tiers
for n in item['next']:
# here we should take input from datatier selection;
# have a map of tiers -> taskNames and select appropriate one
__input_tier = tasktree[n]['dict'][0]['_first_step_']
tModule = tName = ""
if __input_tier in __DT_prio:
# in case there is a possible DataTier in global_dict
tModule, tName = request.do_datatier_selection(
__DT_prio[__input_tier], __uniq_tiers)
if tModule != "" and tName != "":
tasktree[n]['dict'][0].update({
"InputFromOutputModule": tModule,
"InputTask": tName
})
else:
# default & fallback solution
tasktree[n]['dict'][0].update({
"InputFromOutputModule":
item['dict'][-1]['output_'],
"InputTask":
item['dict'][-1]['TaskName']
})
wma = {
"RequestType": "TaskChain",
"Group": "ppd",
"Requestor": "pdmvserv",
"TaskChain": 0,
"ProcessingVersion": 1,
"RequestPriority": 0,
"SubRequestType": "MC",
# we default to 1 in multicore global
"Multicore": 1
}
task = 1
for (r, item) in sorted(tasktree.items(), key=lambda d: d[1]['rank']):
for d in item['dict']:
if d['priority_'] > wma['RequestPriority']:
wma['RequestPriority'] = d['priority_']
if d['request_type_'] in ['ReDigi']:
wma['SubRequestType'] = 'ReDigi'
for k in d.keys():
if k.endswith('_'):
d.pop(k)
wma['Task%d' % task] = d
task += 1
wma['TaskChain'] = task - 1
if wma['TaskChain'] == 0:
return dumps({})
for item in [
'CMSSWVersion', 'ScramArch', 'TimePerEvent', 'SizePerEvent',
'GlobalTag', 'Memory'
]:
wma[item] = wma['Task%d' % wma['TaskChain']][item]
# since 2016-11, processingString and AcquisitionEra is mandatory in global params
wma['AcquisitionEra'] = wma['Task1']['AcquisitionEra']
wma['ProcessingString'] = wma['Task1']['ProcessingString']
wma['Campaign'] = wma['Task1']['Campaign']
wma['PrepID'] = task_name
wma['RequestString'] = wma['PrepID']
if __chains_type.count("StepChain") == len(__chains_type):
return dumps(tranform_to_step_chain(wma, __total_time_evt,
__total_size_evt),
indent=4)
else:
return dumps(wma, indent=4)
def internal_run(self):
location = installer(self.rid, care_on_existing=False, clean_on_exit=True)
try:
test_script = location.location() + 'validation_run_test.sh'
timeout=None
with open(test_script, 'w') as there:
## one has to wait just a bit, so that the approval change operates, and the get retrieves the latest greatest _rev number
#self.logger.error('Revision %s'%( self.db.get(self.rid)['_rev']))
time.sleep(10)
mcm_r = request(self.db.get(self.rid))
#self.logger.error('Revision %s'%( self.db.get(self.rid)['_rev']))
## the following does change something on the request object, to be propagated in case of success
there.write(mcm_r.get_setup_file(location.location(), run=True, do_valid=True))
timeout = mcm_r.get_timeout()
batch_test = batch_control(self.rid, test_script, timeout=timeout)
try:
success = batch_test.test()
except:
batch_test.log_err = traceback.format_exc()
success = False
if success:
self.logger.log("batch_test result is %s" % success)
(success,batch_test.log_err) = mcm_r.pickup_all_performance(location.location())
self.logger.error('I came all the way to here and %s (request %s)' % ( success, self.rid ))
if not success:
mcm_r = request(self.db.get(self.rid))
## need to provide all the information back
if settings().get_value('check_term_runlimit') and "TERM_RUNLIMIT" in batch_test.log_out:
no_success_message = "LSF job was terminated after reaching run time limit.\n\n"
no_success_message += "Average CPU time per event specified for request was {0} seconds. \n\n".format(
mcm_r.get_attribute("time_event"))
additiona_message = "Time report not found in LSF job."
split_log = batch_test.log_err.split('\n')
for l_id, line in izip(reversed(xrange(len(split_log))), reversed(split_log)):
if "TimeReport>" in line:
additiona_message = "\n".join(split_log[l_id:l_id + 12])
no_success_message += additiona_message
else:
no_success_message = '\t .out \n%s\n\t .err \n%s\n ' % ( batch_test.log_out, batch_test.log_err)
#self.logger.error('Revision %s'%( self.db.get(self.rid)['_rev']))
# reset the content of the request
mcm_r.test_failure(message=no_success_message, what='Validation run test', rewind=True)
#self.logger.error('Revision %s'%( self.db.get(self.rid)['_rev']))
else:
#self.logger.error('Revision %s'%( self.db.get(self.rid)['_rev']))
## change the status with notification
mcm_current = request(self.db.get(self.rid))
if mcm_current.json()['_rev'] == mcm_r.json()['_rev']:
## it's fine to push it through
mcm_r.set_status(with_notification=True)
saved = self.db.update(mcm_r.json())
if not saved:
mcm_current.test_failure(message='The request could not be saved after the run test procedure',
what='Validation run test', rewind=True)
else:
mcm_current.test_failure(
message='The request has changed during the run test procedure, preventing from being saved',
what='Validation run test', rewind=True)
#self.logger.error('Revision %s'%( self.db.get(self.rid)['_rev']))
except:
mess = 'We have been taken out of run_safe of runtest_genvalid for %s because \n %s \n During an un-excepted exception. Please contact support.' % (
self.rid, traceback.format_exc())
self.logger.error(mess)
mcm_r = request(self.db.get(self.rid))
mcm_r.test_failure(message=mess, what='Validation run test', rewind=True)
finally:
location.close()
def get(self, chained_request_id):
"""
Perform test for chained requests
"""
crdb = database('chained_requests')
rdb = database('requests')
settingsDB = database('settings')
mcm_cr = chained_request(crdb.get(chained_request_id))
if settingsDB.get('validation_stop')['value']:
return {
"results":
False,
'message':
('validation jobs are halted to allow forthcoming mcm '
'restart - try again later'),
"prepid":
chained_request_id
}
requires_validation = False
for rid in mcm_cr.get_attribute(
'chain')[mcm_cr.get_attribute('step'):]:
mcm_r = request(rdb.get(rid))
if not mcm_r.is_root and 'validation' not in mcm_r._json_base__status: # We dont care about non root request because they are not being used on chain run test
break
requires_validation = True
if mcm_r.get_attribute('status') != 'new' or mcm_r.get_attribute(
'approval') != 'none':
return {
"results":
False,
"prepid":
chained_request_id,
"message":
"request %s is in status %s, approval: %s" %
(rid, mcm_r.get_attribute('status'),
mcm_r.get_attribute('approval'))
}
try:
mcm_r.ok_to_move_to_approval_validation(for_chain=True)
mcm_r.update_history({
'action': 'approve',
'step': 'validation'
})
mcm_r.set_attribute('approval', 'validation')
mcm_r.reset_validations_counter()
mcm_r.reload()
text = 'Within chain %s \n' % mcm_cr.get_attribute('prepid')
text += mcm_r.textified()
subject = 'Approval %s in chain %s for request %s' % (
'validation', mcm_cr.get_attribute('prepid'),
mcm_r.get_attribute('prepid'))
notification(subject,
text, [],
group=notification.REQUEST_APPROVALS,
action_objects=[mcm_r.get_attribute('prepid')],
object_type='requests',
base_object=mcm_r)
mcm_r.notify(subject, text, accumulate=True)
except Exception as e:
mcm_cr.reset_requests(str(e), notify_one=rid)
return {
"results": False,
"message": str(e),
"prepid": chained_request_id
}
if not requires_validation:
return {
"results": True,
"message": "No validation required",
"prepid": chained_request_id
}
mcm_cr.set_attribute('validate', 1)
if not crdb.update(mcm_cr.json()):
return {
"results": False,
"message": "Failed while trying to update the document in DB",
"prepid": chained_request_id
}
return {
"results": True,
"message": "run test will start soon",
"prepid": chained_request_id
}
def rewind_one(self, crid):
crdb = database('chained_requests')
rdb = database('requests')
if not crdb.document_exists(crid):
return {
"results": False,
"message": "does not exist",
"prepid": crid
}
mcm_cr = chained_request(crdb.get(crid))
current_step = mcm_cr.get_attribute('step')
if current_step == 0:
# or should it be possible to cancel the initial requests of a chained request
return {
"results": False,
"message": "already at the root",
"prepid": crid
}
# supposedly all the other requests were already reset!
for next in mcm_cr.get_attribute('chain')[current_step + 1:]:
# what if that next one is not in the db
if not rdb.document_exists(next):
self.logger.error('%s is part of %s but does not exist' %
(next, crid))
continue
mcm_r = request(rdb.get(next))
if mcm_r.get_attribute('status') != 'new':
# this cannot be right!
self.logger.error(
'%s is after the current request and is not new: %s' %
(next, mcm_r.get_attribute('status')))
return {
"results": False,
"message": "%s is not new" % (next),
"prepid": crid
}
# get the one to be reset
current_id = mcm_cr.get_attribute('chain')[current_step]
mcm_r = request(rdb.get(current_id))
mcm_r.reset()
saved = rdb.update(mcm_r.json())
if not saved:
{
"results": False,
"message": "could not save the last request of the chain",
"prepid": crid
}
# the current chained request has very likely been updated :
# reload it as you have not changed anything to it yet
mcm_cr = chained_request(crdb.get(crid))
mcm_cr.set_attribute('step', current_step - 1)
# set status, last status
mcm_cr.set_last_status()
mcm_cr.set_attribute('status', 'processing')
saved = crdb.update(mcm_cr.json())
if saved:
return {"results": True, "prepid": crid}
else:
return {
"results": False,
"message":
"could not save chained requests. the DB is going to be inconsistent !",
"prepid": crid
}
def delete_request(self, crid):
crdb = database('chained_requests')
rdb = database('requests')
mcm_cr = chained_request(crdb.get(crid))
# get all objects
mcm_r_s = []
for (i, rid) in enumerate(mcm_cr.get_attribute('chain')):
mcm_r = request(rdb.get(rid))
# this is not a valid check as it is allowed to remove a chain around already running requests
# if mcm_r.get_attribute('status') != 'new':
# return {"results":False,"message" : "the request %s part of the chain %s for action %s is not in new status"%( mcm_r.get_attribute('prepid'),
# crid,
# mcm_a.get_attribute('prepid'))}
in_chains = mcm_r.get_attribute('member_of_chain')
in_chains.remove(crid)
self.logger.debug(
"Removing ChainAction member_of_chain: %s to request: %s" %
(mcm_cr.get_attribute("prepid"),
mcm_r.get_attribute('prepid')))
mcm_r.set_attribute('member_of_chain', in_chains)
if i == 0:
if len(in_chains
) == 0 and mcm_r.get_attribute('status') != 'new':
return {
"results":
False,
"message":
"the request %s, not in status new, at the root of the chain will not be chained anymore"
% rid
}
else:
if len(in_chains) == 0:
return {
"results":
False,
"message":
"the request %s, not at the root of the chain will not be chained anymore"
% rid
}
mcm_r.update_history({'action': 'leave', 'step': crid})
mcm_r_s.append(mcm_r)
if mcm_cr.get_attribute('action_parameters')['flag']:
return {
"results": False,
"message": "the action for %s is not disabled" % (crid)
}
# then save all changes
for mcm_r in mcm_r_s:
if not rdb.update(mcm_r.json()):
return {
"results":
False,
"message":
"Could not save request " + mcm_r.get_attribute('prepid')
}
else:
subject = "Request {0} left chain".format(
mcm_r.get_attribute('prepid'))
message = "Request {0} has successfuly left chain {1}".format(
mcm_r.get_attribute('prepid'), crid)
notification(subject,
message, [],
group=notification.REQUEST_OPERATIONS,
action_objects=[mcm_r.get_attribute('prepid')],
object_type='requests',
base_object=mcm_r)
mcm_r.notify(subject, message)
return {"results": crdb.delete(crid)}
def generate_chained_requests(self, mccm_ticket, request_prepid, mcm_chained_campaign, reserve=False, with_notify=True, special=False):
try:
mcm_chained_campaign.reload(save_current=False)
generated_chained_request = chained_request(mcm_chained_campaign.generate_request(request_prepid))
except Exception as e:
message = "Unable to generate chained request for ticket %s request %s, message: " % (mccm_ticket.get_attribute('prepid'), request_prepid, str(e))
self.logger.error(message)
return {
"results": False,
"message": message}
requests_db = database('requests')
self.overwrite_action_parameters_from_ticket(generated_chained_request, mccm_ticket)
mcm_request = request(json_input=requests_db.get(request_prepid))
generated_chained_request.set_attribute('last_status', mcm_request.get_attribute('status'))
if generated_chained_request.get_attribute('last_status') in ['submitted', 'done']:
generated_chained_request.set_attribute('status', 'processing')
if special:
generated_chained_request.set_attribute('approval', 'none')
new_chain_prepid = generated_chained_request.get_attribute('prepid')
if not generated_chained_request.reload():
return {
'results': False,
'message': 'Unable to save chained request %s' % new_chain_prepid}
# update the history of chained campaign
mcm_chained_campaign.save()
# let the root request know that it is part of a chained request
chains = mcm_request.get_attribute('member_of_chain')
chains.append(new_chain_prepid)
chains.sort()
mcm_request.set_attribute('member_of_chain', list(set(chains)))
mcm_request.update_history({'action': 'join chain', 'step': new_chain_prepid})
if with_notify:
subject = "Request %s joined chain" % mcm_request.get_attribute('prepid')
message = "Request %s has successfully joined chain %s" % (mcm_request.get_attribute('prepid'), new_chain_prepid)
notification(
subject,
message,
[],
group=notification.REQUEST_OPERATIONS,
action_objects=[mcm_request.get_attribute('prepid')],
object_type='requests',
base_object=mcm_request)
mcm_request.notify(
subject,
message,
Nchild=0,
accumulate=True)
mcm_request.save()
# do the reservation of the whole chain ?
generated_requests = []
if reserve:
results_dict = generated_chained_request.reserve(limit=reserve, save_requests=False)
if results_dict['results'] and 'generated_requests' in results_dict:
generated_requests = results_dict['generated_requests']
results_dict.pop('generated_requests')
else:
return {
"results": False,
"prepid": new_chain_prepid,
"message": results_dict['message']}
return {
"results":True,
"prepid": new_chain_prepid,
'generated_requests': generated_requests}
