def import_request(self, data):
db = database(self.db_name)
json_input=threaded_loads(data)
if 'pwg' not in json_input or 'member_of_campaign' not in json_input:
self.logger.error('Now pwg or member of campaign attribute for new chained request')
return {"results":False}
if 'prepid' in json_input:
req = chained_request(json_input)
cr_id = req.get_attribute('prepid')
else:
cr_id = ChainedRequestPrepId().next_id(json_input['pwg'], json_input['member_of_campaign'])
if not cr_id:
return {"results":False}
req = chained_request(db.get(cr_id))
for key in json_input:
if key not in ['prepid', '_id', '_rev', 'history']:
req.set_attribute(key, json_input[key])
if not req.get_attribute('prepid'):
self.logger.error('prepid returned was None')
raise ValueError('Prepid returned was None')
self.logger.log('Created new chained_request %s' % cr_id)
# update history with the submission details
req.update_history({'action': 'created'})
return self.save_request(db, req)
def get(self, chained_request_id):
"""
Does a soft reset to all relevant request in the chain
"""
crdb = database('chained_requests')
rdb = database('requests')
mcm_cr = chained_request(crdb.get(chained_request_id))
for rid in reversed(
mcm_cr.get_attribute('chain')[:mcm_cr.get_attribute('step') +
1]):
# from the current one to the first one REVERSED
mcm_r = request(rdb.get(rid))
try:
mcm_r.reset(hard=False)
except Exception as e:
return {
'prepid': chained_request_id,
'results': False,
'message': str(e)
}
mcm_r.reload()
mcm_cr = chained_request(crdb.get(chained_request_id))
mcm_cr.set_attribute(
'step', max(0,
mcm_cr.get_attribute('chain').index(rid) - 1))
mcm_cr.reload()
return {'prepid': chained_request_id, 'results': True}
def import_request(self, data):
db = database(self.db_name)
json_input = loads(data)
if 'pwg' not in json_input or 'member_of_campaign' not in json_input:
self.logger.error(
'Now pwg or member of campaign attribute for new chained request'
)
return {"results": False}
if 'prepid' in json_input:
req = chained_request(json_input)
cr_id = req.get_attribute('prepid')
else:
cr_id = ChainedRequestPrepId().next_id(
json_input['pwg'], json_input['member_of_campaign'])
if not cr_id:
return {"results": False}
req = chained_request(db.get(cr_id))
for key in json_input:
if key not in ['prepid', '_id', '_rev', 'history']:
req.set_attribute(key, json_input[key])
if not req.get_attribute('prepid'):
self.logger.error('prepid returned was None')
raise ValueError('Prepid returned was None')
self.logger.info('Created new chained_request %s' % cr_id)
# update history with the submission details
req.update_history({'action': 'created'})
return self.save_request(db, req)
def GET(self, *args):
"""
Does a soft reset to all relevant request in the chain
"""
if not len(args):
return dumps({"results" : False, "message" : "no argument provided"})
arg0 = args[0]
crdb = database('chained_requests')
rdb = database('requests')
mcm_cr = chained_request(crdb.get(arg0))
for rid in reversed(mcm_cr.get_attribute('chain')[:mcm_cr.get_attribute('step')+1]):
## from the current one to the first one REVERSED
mcm_r = request(rdb.get(rid))
try:
mcm_r.reset(hard=False)
except Exception as e:
return dumps({'prepid' : arg0, 'results' : False, 'message' : str(e)})
mcm_r.reload()
mcm_cr = chained_request(crdb.get(arg0))
mcm_cr.set_attribute('step', max(0, mcm_cr.get_attribute('chain').index(rid)-1))
mcm_cr.reload()
return dumps({'prepid' : arg0, 'results':True})
def rewind_one(self, crid):
crdb = database('chained_requests')
rdb = database('requests')
if not crdb.document_exists( crid ):
return {"results":False, "message":"does not exist","prepid" : crid}
mcm_cr = chained_request( crdb.get( crid) )
current_step = mcm_cr.get_attribute('step')
if current_step==0:
## or should it be possible to cancel the initial requests of a chained request
return {"results":False, "message":"already at the root","prepid" : crid}
## supposedly all the other requests were already reset!
for next in mcm_cr.get_attribute('chain')[current_step+1:]:
## what if that next one is not in the db
if not rdb.document_exists( next):
self.logger.error('%s is part of %s but does not exist'%( next, crid))
continue
mcm_r = request(rdb.get( next ))
if mcm_r.get_attribute('status')!='new':
# this cannot be right!
self.logger.error('%s is after the current request and is not new: %s' % (
next, mcm_r.get_attribute('status')))
return {"results":False, "message":"%s is not new" % (next), "prepid" : crid}
##get the one to be reset
current_id=mcm_cr.get_attribute('chain')[current_step]
mcm_r = request( rdb.get( current_id ))
mcm_r.reset()
saved = rdb.update( mcm_r.json() )
if not saved:
{"results":False, "message":"could not save the last request of the chain","prepid" : crid}
## the current chained request has very likely been updated :
## reload it as you have not changed anything to it yet
mcm_cr = chained_request( crdb.get( crid) )
mcm_cr.set_attribute('step',current_step -1 )
# set status, last status
mcm_cr.set_last_status()
mcm_cr.set_attribute('status','processing')
saved = crdb.update( mcm_cr.json())
if saved:
return {"results":True,"prepid" : crid}
else:
return {"results" : False,
"message" : "could not save chained requests. the DB is going to be inconsistent !",
"prepid" : crid}
def get(self, chained_request_id):
kwargs = self.parser.parse_args()
crdb = database('chained_requests')
if not crdb.document_exists(chained_request_id):
return {
"results":
False,
"message":
"Chained request with prepid {0} does not exist".format(
chained_request_id)
}
cr = chained_request(crdb.get(chained_request_id))
events = None
run = False
valid = False
directory = ''
__scratch = kwargs["scratch"].lower() == 'true'
if self.opt == 'test' or self.opt == 'valid':
run = True
if self.opt == 'valid':
valid = True
return cr.get_setup(directory=kwargs['directory'],
run=run,
events=kwargs['events'],
validation=valid,
scratch=__scratch)
def submit_request(self, prepid, run_test_path):
mcm_request = request(self.request_db.get(prepid))
# check if the request should be validated as part of a chain
for chain_prepid in mcm_request.get_attribute('member_of_chain'):
mcm_chain = chained_request(self.chained_request_db.get(chain_prepid))
if mcm_chain.get_attribute('validate') and prepid in mcm_chain.get_attribute('chain')[mcm_chain.get_attribute('step'):]:
return {}
aux_validation = mcm_request.get_attribute(self.DOC_VALIDATION)
to_write = mcm_request.get_setup_file(run=True, do_valid=True, for_validation=True, gen_script=True)
file_name = self.TEST_FILE_NAME % prepid
if not self.create_test_file(to_write, run_test_path, file_name):
mcm_request.set_attribute(self.DOC_VALIDATION, aux_validation)
mcm_request.test_failure(
message='There was a problem while creating the file for prepid: %s' % (mcm_request.get_attribute('prepid')),
what='Validation run test',
rewind=True)
return {}
timeout = mcm_request.get_timeout()
memory = mcm_request.get_attribute("memory")
threads = mcm_request.get_core_num()
self.create_htcondor_config_file(run_test_path, prepid, timeout, memory, threads, [prepid], mcm_request)
job_info = self.execute_command_submission(prepid, run_test_path)
if 'error' in job_info:
mcm_request.test_failure(message=job_info['error'], what='Validation run test', rewind=True)
return {}
job_info[self.DOC_REV] = mcm_request.json()[self.DOC_REV]
job_info[self.DOC_VALIDATION] = mcm_request.get_attribute(self.DOC_VALIDATION) # this field change when calling request.get_setup_file, to be propagated in case of success
return job_info
def flow2(self, data):
try:
vdata = loads(data)
except ValueError as ex:
self.logger.error('Could not start flowing to next step. Reason: %s' % (ex))
return {"results":str(ex)}
db = database('chained_requests')
try:
creq = chained_request(json_input=db.get(vdata['prepid']))
except Exception as ex:
self.logger.error('Could not initialize chained_request object. Reason: %s' % (ex))
return {"results":str(ex)}
self.logger.log('Attempting to flow to next step for chained_request %s' % (creq.get_attribute('_id')))
# if the chained_request can flow, do it
inputds = ''
inblack = []
inwhite = []
if 'input_filename' in vdata:
inputds = vdata['input_filename']
if 'block_black_list' in vdata:
inblack = vdata['block_black_list']
if 'block_white_list' in vdata:
inwhite = vdata['block_white_list']
if 'force' in vdata:
check_stats = vdata['force']!='force'
if 'reserve' in vdata and vdata["reserve"]:
return creq.reserve()
return creq.flow_trial( inputds, inblack, inwhite, check_stats)
def force_status_done(self, prepid):
if not self.crdb.document_exists(prepid):
return dumps(
{
"results":
False,
"message":
"Chained request with prepid {0} does not exist".format(
prepid)
},
indent=4)
cr = chained_request(self.crdb.get(prepid))
if not (cr.get_attribute("status") in ["done", "force_done"]):
cr.set_status(to_status="force_done")
self.logger.debug(
"forcing chain_req status to done. cr status:%s" %
(cr.get_attribute("status")))
ret = self.crdb.save(cr.json())
return dumps({
'prepid': prepid,
'message': ret,
'results': True
},
indent=4)
else:
ret = "Chained request already in status done"
return dumps({
'prepid': prepid,
'message': ret,
'results': False
},
indent=4)
def multiple_inspect(self, ccids):
crdb = database('chained_requests')
res = []
for ccid in ccids.split(','):
crlist = crdb.queries(["member_of_campaign==%s"% ccid,
"last_status==done",
"status==processing"])
self.logger.log('crlist %s in chained_camp %s ' % (crlist, ccid))
for cr in crlist:
mcm_cr = chained_request( cr )
if mcm_cr:
__inspect_ret = mcm_cr.inspect()
else:
__inspect_ret = {"prepid":cr, "results":False,
'message' : '%s does not exist' % cr}
self.logger.log("Inspection for: %s returned: %s" %(cr, __inspect_ret))
res.append(__inspect_ret)
if len(res)>1:
return res
elif len(res):
self.logger.log("Inspection finished. Returned: %s" %(res[0]))
return res[0]
else:
self.logger.log("Inspection finished. Returned: %s" %([]))
return []
def multiple_inspect(self, ccids):
crdb = database('chained_requests')
res = []
for ccid in ccids.split(','):
crlist = crdb.queries([
"member_of_campaign==%s" % ccid, "last_status==done",
"status==processing"
])
self.logger.log('crlist %s in chained_camp %s ' % (crlist, ccid))
for cr in crlist:
mcm_cr = chained_request(cr)
if mcm_cr:
res.append(mcm_cr.inspect())
else:
res.append({
"prepid": cr,
"results": False,
'message': '%s does not exist' % cr
})
if len(res) > 1:
return res
elif len(res):
return res[0]
else:
return []
def flow2(self, data):
db = database('chained_requests')
chain_id = data['prepid']
try:
creq = chained_request(json_input=db.get(chain_id))
except Exception as ex:
self.logger.error(
'Could not initialize chained_request object. Reason: %s' %
(ex))
return {"results": str(ex)}
self.logger.info(
'Attempting to flow to next step for chained_request %s' %
(creq.get_attribute('_id')))
# if the chained_request can flow, do it
inputds = ''
inblack = []
inwhite = []
if 'input_dataset' in data:
inputds = data['input_dataset']
if 'block_black_list' in data:
inblack = data['block_black_list']
if 'block_white_list' in data:
inwhite = data['block_white_list']
if 'force' in data:
check_stats = data['force'] != 'force'
if 'reserve' in data and data["reserve"]:
reserve = data["reserve"]
return creq.reserve(limit=reserve)
return creq.flow_trial(inputds, inblack, inwhite, check_stats)
def flow2(self, data):
db = database('chained_requests')
chain_id = data['prepid']
try:
creq = chained_request(json_input=db.get(chain_id))
except Exception as ex:
self.logger.error('Could not initialize chained_request object. Reason: %s' % (ex))
return {"results": str(ex)}
self.logger.info('Attempting to flow to next step for chained_request %s' % (
creq.get_attribute('_id')))
# if the chained_request can flow, do it
inputds = ''
inblack = []
inwhite = []
if 'input_dataset' in data:
inputds = data['input_dataset']
if 'block_black_list' in data:
inblack = data['block_black_list']
if 'block_white_list' in data:
inwhite = data['block_white_list']
if 'force' in data:
check_stats = data['force'] != 'force'
if 'reserve' in data and data["reserve"]:
reserve = data["reserve"]
return creq.reserve(limit=reserve)
return creq.flow_trial(inputds, inblack, inwhite, check_stats)
def next_id(self, pwg, campaign):
ccamp_db = database(self.ccamp_db_name)
creq_db = database(self.creq_db_name)
if not pwg:
self.logger.error('Physics working group provided is None.')
return None
if not campaign:
self.logger.error('Campaign id provided is None.')
return None
with locker.lock("{0}-{1}".format(pwg, campaign)):
if not ccamp_db.document_exists(campaign):
self.logger.error('Campaign id {0} does not exist.'.format(campaign))
return None
if (campaign, pwg) in self.serial_number_cache:
sn = self.serial_number_cache[(campaign, pwg)] + 1
else:
sn=1
serial_number_lookup = creq_db.raw_query('serial_number', {'group':True, 'key':[campaign, pwg]})
if serial_number_lookup:
sn = serial_number_lookup[0]['value']+1
## construct the new id
new_prepid = pwg + '-' + campaign + '-' + str(sn).zfill(5)
if sn==1:
self.logger.log('Beginning new prepid family: %s' % (new_prepid))
new_request = chained_request({'_id':new_prepid, 'prepid':new_prepid, 'pwg':pwg, 'member_of_campaign':campaign})
new_request.update_history({'action':'created'})
creq_db.save(new_request.json())
self.serial_number_cache[(campaign, pwg)] = sn
self.logger.log('New chain id: %s' % new_prepid, level='debug')
return new_prepid
def reset_all(self, message, what = 'Chained validation run test', notify_one=None):
crdb = database('chained_requests')
rdb = database('requests')
mcm_cr = chained_request(crdb.get(self.crid))
if self.scratch:
chain = mcm_cr.get_attribute('chain')
else:
chain = mcm_cr.get_attribute('chain')[mcm_cr.get_attribute('step'):]
for rid in chain:
mcm_r = request(rdb.get(rid ))
s_label = 'chainvalid-%s' % rid
semaphore_events.decrement(s_label)
if not semaphore_events.is_set(s_label):
##someone else is still validating that chain, so no reset !
mcm_r.notify('%s failed for request %s' % (what,
mcm_r.get_attribute('prepid')), message)
continue
## do not reset anything that does not look ok already
# this might leave things half-way inconsistent in terms of status
if mcm_r.get_attribute('status') != 'new':
mcm_r.notify('%s failed for request %s' % (what,
mcm_r.get_attribute('prepid')), message)
continue
notify = True
if notify_one and notify_one != rid:
notify = False
mcm_r.test_failure( message,
what = what, rewind=True, with_notification=notify)
def force_status(self, prepid):
if not self.crdb.document_exists(prepid):
return dumps({
"results":
False,
"message":
"Chained request with prepid {0} does not exist".format(prepid)
})
cr = chained_request(self.crdb.get(prepid))
if cr.get_attribute("status") == "force_done":
cr.set_status(to_status="processing")
self.logger.debug(
"Moving chain_req back to satus 'processing'. cr status:%s" %
(cr.get_attribute("status")))
ret = self.crdb.save(cr.json())
return dumps({
'prepid': prepid,
'message': ret,
'results': True
},
indent=4)
else:
ret = "Chained request not in status force_done"
return dumps({
'prepid': prepid,
'message': ret,
'results': False
},
indent=4)
def GET(self, *args, **kwargs):
if not len(args):
return dumps({"results": False, "message": "Chained request prepid not given"})
crdb = database('chained_requests')
if not crdb.document_exists(args[0]):
return dumps({"results": False,
"message": "Chained request with prepid {0} does not exist".format(args[0])})
cr = chained_request(crdb.get(args[0]))
events = None
run = False
valid = False
directory = ''
__scratch = False
if 'scratch' in kwargs:
__scratch = kwargs["scratch"]
if self.opt == 'test' or self.opt == 'valid':
run = True
if self.opt == 'valid':
valid = True
if 'events' in kwargs:
events = int(kwargs['events'])
if 'directory' in kwargs:
directory = kwargs['directory']
cherrypy.response.headers['Content-Type'] = 'text/plain'
return cr.get_setup(directory=directory, run=run, events=events,
validation=valid, scratch=__scratch)
def get(self, chained_request_ids):
"""
Add selected prepid's to global force complete list for later action
"""
if ',' in chained_request_ids:
rlist = chained_request_ids.rsplit(',')
else:
rlist = [chained_request_ids]
res = []
__updated = False
forceflow_list = self.ldb.get("list_of_forceflow")
# TO-DO check if prepid exists!
# TO-DO check the status of chain_req!
for el in rlist:
if el not in forceflow_list["value"]:
forceflow_list["value"].append(el)
chain_req = chained_request(self.cdb.get(el))
chain_req.update_history({'action': 'add_to_forceflow'})
self.cdb.save(chain_req.json())
res.append({"prepid": el, 'results': True, 'message': 'OK'})
__updated = True
else:
res.append({"prepid": el, 'results': False, 'message': 'Chained request already in forceflow list'})
# TO-DO check the update return value
if __updated:
self.ldb.update(forceflow_list)
return res
def internal_run(self):
from tools.installer import installer
from tools.batch_control import batch_control
location = installer( self.crid, care_on_existing=False, clean_on_exit=True)
try:
crdb = database('chained_requests')
rdb = database('requests')
mcm_cr = chained_request(crdb.get(self.crid))
mcm_rs = []
for rid in mcm_cr.get_attribute('chain'):
mcm_rs.append( request( rdb.get( rid ) ))
test_script = location.location() + 'validation_run_test.sh'
with open(test_script, 'w') as there:
there.write(mcm_cr.get_setup(directory=location.location(), run=True, validation=True))
batch_test = batch_control( self.crid, test_script )
try:
success = batch_test.test()
except:
self.reset_all( traceback.format_exc() )
return
if not success:
self.reset_all( '\t .out \n%s\n\t .err \n%s\n ' % ( batch_test.log_out, batch_test.log_err) )
return
last_fail=mcm_rs[0]
trace=""
for mcm_r in mcm_rs:
### if not mcm_r.is_root: continue ##disable for dr request
(success,trace) = mcm_r.pickup_all_performance(location.location())
if not success:
last_fail = mcm_r
break
self.logger.error('I came all the way to here and %s (request %s)' % ( success, self.crid ))
if success:
for mcm_r in mcm_rs:
if mcm_r.is_root:
mcm_current = request( rdb.get(mcm_r.get_attribute('prepid')))
if mcm_current.json()['_rev'] == mcm_r.json()['_rev']:
mcm_r.set_status(with_notification=True)
if not mcm_r.reload():
self.reset_all( 'The request %s could not be saved after the runtest procedure' % (mcm_r.get_attribute('prepid')))
return
else:
self.reset_all( 'The request %s has changed during the run test procedure'%(mcm_r.get_attribute('prepid')), notify_one = mcm_r.get_attribute('prepid'))
return
else:
self.reset_all( trace , notify_one = last_fail.get_attribute('prepid') )
return
except:
mess = 'We have been taken out of run_safe of runtest_genvalid for %s because \n %s \n During an un-excepted exception. Please contact support.' % (
self.crid, traceback.format_exc())
self.logger.error(mess)
finally:
location.close()
def submit_chain(self, prepid, run_test_path):
mcm_chained_request = chained_request(self.chained_request_db.get(prepid))
except_requests = []
reset = False
# If a request of a chain was singly submmited to validation and then somebody reseted it, we will find it here
for request_prepid in mcm_chained_request.get_attribute('chain')[mcm_chained_request.get_attribute('step'):]:
if request_prepid in self.submmited_prepids_set:
except_requests.append(request_prepid)
reset = True
if reset:
message = "Requests %s of the chain %s are already in validation" % (except_requests, prepid)
self.logger.error(message)
mcm_chained_request.reset_requests(message, except_requests=except_requests)
return {}
to_write = mcm_chained_request.get_setup(run=True, validation=True, for_validation=True)
file_name = self.TEST_FILE_NAME % prepid
if not self.create_test_file(to_write, run_test_path, file_name):
mcm_chained_request.reset_requests('There was a problem while creating the file for prepid: %s' % (mcm_chained_request.get_attribute('prepid')))
return {}
requests_in_chain = {}
first_request_in_chain = None
for request_prepid in mcm_chained_request.get_attribute('chain')[mcm_chained_request.get_attribute('step'):]:
mcm_request = request(self.request_db.get(request_prepid))
if not mcm_request.is_root and 'validation' not in mcm_request._json_base__status: # only root or possible root requests
break
status = mcm_request.get_attribute('status')
approval = mcm_request.get_attribute('approval')
if status != 'new' or approval != 'validation':
message = "The request %s of chain %s is in status: %s approval: %s" % (request_prepid, prepid, status, approval)
self.logger.error(message)
mcm_chained_request.reset_requests(message)
return {}
requests_in_chain[request_prepid] = mcm_request.json()[self.DOC_REV]
if not first_request_in_chain:
first_request_in_chain = mcm_request
if not len(requests_in_chain):
message = 'No requests to be validated in chain: %s' % prepid
self.logger.info(message)
mcm_chained_request.reset_requests(message)
return {}
timeout, memory, threads = mcm_chained_request.get_timeout_memory_threads()
self.create_htcondor_config_file(run_test_path, prepid, timeout, memory, threads, list(requests_in_chain.iterkeys()), first_request_in_chain)
job_info = self.execute_command_submission(prepid, run_test_path)
if 'error' in job_info:
mcm_chained_request.reset_requests(job_info['error'])
return {}
job_info[self.CHAIN_REQUESTS] = requests_in_chain
return job_info
def multiple_inspect(self, ccids):
settings.set_value('inspect_chained_campaigns_running', True)
crdb = database('chained_requests')
try:
ccids = ccids.split(',')
index = 0
while len(ccids) > index:
query = crdb.construct_lucene_complex_query([
('member_of_campaign', {
'value': ccids[index:index + 20]
}), ('last_status', {
'value': 'done'
}), ('status', {
'value': 'processing'
})
])
crlist = crdb.full_text_search('search', query, page=-1)
# we yield len of cr_list so we would know how much data later on we processed
yield dumps(
{
'prepids': ccids[index:index + 20],
'cr_len': len(crlist)
},
indent=2)
index += 20
for cr in crlist:
time.sleep(0.5)
mcm_cr = chained_request(cr)
if mcm_cr:
__inspect_ret = mcm_cr.inspect()
else:
__inspect_ret = {
"prepid": cr,
"results": False,
'message': '%s does not exist' % cr['prepid']
}
self.logger.info("Inspection for: %s returned: %s" %
(cr['prepid'], __inspect_ret))
yield dumps(__inspect_ret, indent=8)
# force slowing-down of inspect to not abuse the DB
time.sleep(1)
self.logger.info("ChainedCampaigns inspection finished")
except Exception as ex:
self.logger.error(
"ChainedCampaigns inspection crashed. reason: %s" % str(ex))
yield dumps({
'message': 'crlist crashed: %s' % (str(ex)),
'last_used_query': query
})
finally:
settings.set_value('inspect_chained_campaigns_running', False)
def GET(self, *args):
"""
Perform test for chained requests
"""
if not len(args):
return dumps({"results" : False, "message" : "no argument provided"})
from tools.handlers import RunChainValid, validation_pool
## now in the core of the api
runtest = RunChainValid(crid=args[0], lock=locker.lock(args[0]))
crdb = database('chained_requests')
rdb = database('requests')
mcm_cr = chained_request(crdb.get(args[0]))
mcm_rs = []
for rid in mcm_cr.get_attribute('chain')[mcm_cr.get_attribute('step'):]:
mcm_r = request( rdb.get( rid ) )
if mcm_r.get_attribute('status') in ['approved','submitted','done']:
return dumps({"results" : False, "prepid" : args[0],
"message" : "request %s is in status %s" % (
rid, mcm_r.get_attribute('status'))})
for rid in mcm_cr.get_attribute('chain')[mcm_cr.get_attribute('step'):]:
mcm_r = request(rdb.get(rid))
next = 'validation'
if not mcm_r.is_root: next = 'approve'
try:
if mcm_r.get_attribute('approval') == 'none':
## no need to try and move it along if already further than that
getattr(mcm_r,'ok_to_move_to_approval_%s' % next)(for_chain=True)
mcm_r.update_history({'action' : 'approve', 'step' : next})
mcm_r.set_attribute('approval', next)
mcm_r.reload()
else:
pass
## fail this for the moment. there is no way to handle this yet
#text="It is not supported for the moment to test a chain of requests which are partially not new. Please contact an administrator"
#runtest.reset_all( text , notify_one = rid )
#return dumps({"results" : False, "message" : text, "prepid" : args[0]})
text = 'Within chain %s \n'% mcm_cr.get_attribute('prepid')
text += mcm_r.textified()
mcm_r.notify('Approval %s in chain %s for request %s' % (next,
mcm_cr.get_attribute('prepid'), mcm_r.get_attribute('prepid')),
text, accumulate=True)
except Exception as e:
runtest.reset_all(str(e), notify_one=rid)
return dumps({"results" : False, "message" : str(e),"prepid" : args[0]})
validation_pool.add_task(runtest.internal_run)
#runtest.start()
return dumps({"results" : True, "message" : "run test started","prepid" : args[0]})
def process_finished_chain_failed(self, prepid, job_out, job_error_out,
error_out, was_exited, out_path,
log_out):
mcm_chained_request = chained_request(
self.chained_request_db.get(prepid))
if not was_exited:
message = "File %s does not look properly formatted or does not exist. \n %s \n %s \n Error out: \n%s \n Log out:\n %s" % (
out_path, job_out, job_error_out, error_out, log_out)
else:
message = "Job validation failed for chain %s \nJob out: \n%s \n Error out: \n%s \n Log out: \n%s" % (
prepid, job_out, error_out, log_out)
mcm_chained_request.reset_requests(message)
def process_finished_chain_failed(self, prepid, job_out, job_error_out, error_out, was_exited, out_path, log_out):
mcm_chained_request = chained_request(self.chained_request_db.get(prepid))
if not was_exited:
message = "File %s does not look properly formatted or does not exist. \n %s \n %s \n Error out: \n%s \n Log out:\n %s" % (
out_path,
job_out,
job_error_out,
error_out,
log_out)
else:
message = "Job validation failed for chain %s \nJob out: \n%s \n Error out: \n%s \n Log out: \n%s" % (prepid, job_out, error_out, log_out)
mcm_chained_request.reset_requests(message)
def force_status_done(self, prepid):
if not self.crdb.document_exists(prepid):
return dumps({"results": False, "message": "Chained request with prepid {0} does not exist".format(prepid)}, indent=4)
cr = chained_request(self.crdb.get(prepid))
if not (cr.get_attribute("status") in ["done", "force_done"]):
cr.set_status(to_status="force_done")
cr.remove_from_nonflowing_list()
self.logger.debug("forcing chain_req status to done. cr status:%s" % (cr.get_attribute("status")))
ret = self.crdb.save(cr.json())
return {'prepid': prepid, 'message': ret, 'results': True}
else:
ret = "Chained request already in status done"
return {'prepid': prepid, 'message': ret, 'results': False}
def GET(self, *args, **kwargs):
if not len(args):
return dumps({"results": False, "message": "Chained request prepid not given"})
crdb = database('chained_requests')
if not crdb.document_exists(args[0]):
return dumps({"results": False,
"message": "Chained request with prepid {0} does not exist".format(args[0])})
cr = chained_request(crdb.get(args[0]))
cherrypy.response.headers['Content-Type'] = 'text/plain'
return cr.test_output_ds()
def get(self, chained_request_id):
"""
Does a soft reset to all relevant request in the chain
"""
crdb = database('chained_requests')
rdb = database('requests')
mcm_cr = chained_request(crdb.get(chained_request_id))
for rid in reversed(mcm_cr.get_attribute('chain')[:mcm_cr.get_attribute('step') + 1]):
# from the current one to the first one REVERSED
mcm_r = request(rdb.get(rid))
try:
mcm_r.reset(hard=False)
except Exception as e:
return {'prepid': chained_request_id, 'results': False, 'message': str(e)}
mcm_r.reload()
mcm_cr = chained_request(crdb.get(chained_request_id))
mcm_cr.set_attribute('step', max(0, mcm_cr.get_attribute('chain').index(rid) - 1))
mcm_cr.reload()
return {'prepid': chained_request_id, 'results': True}
def reset_all(self, message, what = 'Chained validation run test', notify_one=None):
crdb = database('chained_requests')
rdb = database('requests')
mcm_cr = chained_request(crdb.get(self.crid))
for rid in mcm_cr.get_attribute('chain'):
mcm_r = request( rdb.get( rid ) )
notify = True
if notify_one and notify_one != rid:
notify = False
mcm_r.test_failure( message,
what = what,
rewind=True,
with_notification=notify)
def delete_request(self, crid):
crdb = database('chained_requests')
rdb = database('requests')
adb = database('actions')
mcm_cr = chained_request(crdb.get(crid))
mcm_a = None
## get all objects
mcm_r_s=[]
for (i,rid) in enumerate(mcm_cr.get_attribute('chain')):
mcm_r = request(rdb.get(rid))
#this is not a valid check as it is allowed to remove a chain around already running requests
# if mcm_r.get_attribute('status') != 'new':
# return {"results":False,"message" : "the request %s part of the chain %s for action %s is not in new status"%( mcm_r.get_attribute('prepid'),
# crid,
# mcm_a.get_attribute('prepid'))}
in_chains = mcm_r.get_attribute('member_of_chain')
in_chains.remove( crid )
mcm_r.set_attribute('member_of_chain', in_chains)
if i==0:
# the root is the action id
mcm_a = action(adb.get(rid))
if len(in_chains)==0 and mcm_r.get_attribute('status')!='new':
return {"results":False, "message" : "the request %s, not in status new, at the root of the chain will not be chained anymore"% rid}
else:
if len(in_chains)==0:
return {"results":False,"message" : "the request %s, not at the root of the chain will not be chained anymore"% rid}
mcm_r.update_history({'action':'leave','step':crid})
mcm_r_s.append( mcm_r )
## check if possible to get rid of it !
# action for the chain is disabled
chains = mcm_a.get_chains( mcm_cr.get_attribute('member_of_campaign'))
if chains[crid]['flag']:
return {"results":False,"message" : "the action %s for %s is not disabled"%(mcm_a.get_attribute('prepid'), crid)}
#take it out
mcm_a.remove_chain( mcm_cr.get_attribute('member_of_campaign'), mcm_cr.get_attribute('prepid') )
if not adb.update( mcm_a.json()):
return {"results":False,"message" : "Could not save action "+ mcm_a.get_attribute('prepid')}
## then save all changes
for mcm_r in mcm_r_s:
if not rdb.update( mcm_r.json()):
return {"results":False,"message" : "Could not save request "+ mcm_r.get_attribute('prepid')}
else:
mcm_r.notify("Request {0} left chain".format( mcm_r.get_attribute('prepid')),
"Request {0} has successfuly left chain {1}".format(
mcm_r.get_attribute('prepid'), crid))
return {"results": crdb.delete(crid)}
def report_error(self, prepid, message):
self.logger.error(message)
try:
if 'chain' in prepid:
mcm_chained_request = chained_request(self.chained_request_db.get(prepid))
mcm_chained_request.reset_requests(message)
else:
mcm_request = request(self.request_db.get(prepid))
mcm_request.test_failure(
message=message,
what='Validation run test',
rewind=True)
except Exception as e:
self.logger.error("Exception while reporting an error for %s message: %s \ntraceback: %s" % (prepid, str(e), traceback.format_exc()))
def force_status(self, prepid):
if not self.crdb.document_exists(prepid):
return dumps({"results": False,
"message": "Chained request with prepid {0} does not exist".format(prepid)})
cr = chained_request(self.crdb.get(prepid))
if cr.get_attribute("status") == "force_done":
cr.set_status(to_status="processing")
self.logger.debug("Moving chain_req back to satus 'processing'. cr status:%s" % (
cr.get_attribute("status")))
ret = self.crdb.save(cr.json())
return {'prepid': prepid, 'message': ret, 'results': True}
else:
ret = "Chained request already in status done"
return {'prepid': prepid, 'message': ret, 'results': False}
def multiple_inspect(self, crid):
crlist=crid.rsplit(',')
res = []
crdb = database('chained_requests')
for cr in crlist:
if crdb.document_exists( cr):
mcm_cr = chained_request( crdb.get( cr) )
res.append( mcm_cr.inspect() )
else:
res.append( {"prepid": cr, "results":False, 'message' : '%s does not exist'% cr})
if len(res)>1:
return res
else:
return res[0]
def approve(self, rid, val=-1):
db = database('chained_requests')
if not db.document_exists(rid):
return {"prepid": rid, "results":'Error: The given chained_request id does not exist.'}
creq = chained_request(json_input=db.get(rid))
try:
creq.approve(val)
except Exception as ex:
return {"prepid": rid, "results":False, 'message' : str(ex)}
saved = db.update(creq.json())
if saved:
return {"prepid":rid, "results":True}
else:
return {"prepid":rid, "results":False , 'message': 'unable to save the updated chained request'}
def flow(self, chainid, check_stats=True, reserve = False):
try:
db = database('chained_requests')
creq = chained_request(json_input=db.get(chainid))
except Exception as ex:
self.logger.error('Could not initialize chained_request object. Reason: %s' % (ex))
return {"results":str(ex)}
# if the chained_request can flow, do it
if reserve:
self.logger.log('Attempting to reserve to next step for chained_request %s' % (creq.get_attribute('_id')))
return creq.reserve()
self.logger.log('Attempting to flow to next step for chained_request %s' % (creq.get_attribute('_id')))
return creq.flow_trial(check_stats=check_stats)
def generate_chained_requests(self, mccm_ticket, request_prepid, mcm_chained_campaign, reserve=False, with_notify=True, special=False):
try:
mcm_chained_campaign.reload(save_current=False)
generated_chained_request = chained_request(mcm_chained_campaign.generate_request(request_prepid))
except Exception as e:
message = "Unable to generate chained request for ticket %s request %s, message: " % (mccm_ticket.get_attribute('prepid'), request_prepid, str(e))
self.logger.error(message)
return {
"results": False,
"message": message}
requests_db = database('requests')
self.overwrite_action_parameters_from_ticket(generated_chained_request, mccm_ticket)
mcm_request = request(json_input=requests_db.get(request_prepid))
generated_chained_request.set_attribute('last_status', mcm_request.get_attribute('status'))
if generated_chained_request.get_attribute('last_status') in ['submitted', 'done']:
generated_chained_request.set_attribute('status', 'processing')
if special:
generated_chained_request.set_attribute('approval', 'none')
new_chain_prepid = generated_chained_request.get_attribute('prepid')
if not generated_chained_request.reload():
return {
'results': False,
'message': 'Unable to save chained request %s' % new_chain_prepid}
# update the history of chained campaign
mcm_chained_campaign.save()
# let the root request know that it is part of a chained request
chains = mcm_request.get_attribute('member_of_chain')
chains.append(new_chain_prepid)
chains.sort()
mcm_request.set_attribute('member_of_chain', list(set(chains)))
mcm_request.update_history({'action': 'join chain', 'step': new_chain_prepid})
mcm_request.save()
# do the reservation of the whole chain ?
generated_requests = []
if reserve:
results_dict = generated_chained_request.reserve(limit=reserve, save_requests=False)
if results_dict['results'] and 'generated_requests' in results_dict:
generated_requests = results_dict['generated_requests']
results_dict.pop('generated_requests')
else:
return {
"results": False,
"prepid": new_chain_prepid,
"message": results_dict['message']}
return {
"results":True,
"prepid": new_chain_prepid,
'generated_requests': generated_requests}
def report_error(self, prepid, message):
self.logger.error(message)
try:
if 'chain' in prepid:
mcm_chained_request = chained_request(
self.chained_request_db.get(prepid))
mcm_chained_request.reset_requests(message)
else:
mcm_request = request(self.request_db.get(prepid))
mcm_request.test_failure(message=message,
what='Validation run test',
rewind=True)
except Exception as e:
self.logger.error(
"Exception while reporting an error for %s message: %s \ntraceback: %s"
% (prepid, str(e), traceback.format_exc()))
def update_request(self, data):
try:
req = chained_request(json_input=loads(data))
except chained_request.IllegalAttributeName as ex:
return {"results":False}
if not req.get_attribute('prepid') and not req.get_attribute('_id'):
self.logger.error('prepid returned was None')
raise ValueError('Prepid returned was None')
#req.set_attribute('_id', req.get_attribute('prepid')
self.logger.log('Updating chained_request %s' % (req.get_attribute('_id')))
# update history
req.update_history({'action': 'update'})
return self.save_request(req)
def update_request(self, data):
try:
req = chained_request(json_input=loads(data))
except chained_request.IllegalAttributeName:
return {"results": False}
if not req.get_attribute('prepid') and not req.get_attribute('_id'):
self.logger.error('prepid returned was None')
raise ValueError('Prepid returned was None')
# req.set_attribute('_id', req.get_attribute('prepid')
self.logger.info('Updating chained_request %s' % (req.get_attribute('_id')))
self.logger.info('wtf %s' % (str(req.get_attribute('approval'))))
# update history
req.update_history({'action': 'update'})
new_priority = req.get_attribute('action_parameters')['block_number']
req.set_priority(new_priority)
return self.save_request(req)
def get(self, chained_request_id):
kwargs = self.parser.parse_args()
crdb = database('chained_requests')
if not crdb.document_exists(chained_request_id):
return {"results": False,
"message": "Chained request with prepid {0} does not exist".format(chained_request_id)}
cr = chained_request(crdb.get(chained_request_id))
events = None
run = False
valid = False
directory = ''
__scratch = kwargs["scratch"].lower() == 'true'
if self.opt == 'test' or self.opt == 'valid':
run = True
if self.opt == 'valid':
valid = True
return cr.get_setup(directory=kwargs['directory'], run=run, events=kwargs['events'],
validation=valid, scratch=__scratch)
def next_id(self, pwg, campaign):
ccamp_db = database(self.ccamp_db_name)
creq_db = database(self.creq_db_name)
if not pwg:
self.logger.error('Physics working group provided is None.')
return None
if not campaign:
self.logger.error('Campaign id provided is None.')
return None
with locker.lock("{0}-{1}".format(pwg, campaign)):
if not ccamp_db.document_exists(campaign):
self.logger.error(
'Campaign id {0} does not exist.'.format(campaign))
return None
if (campaign, pwg) in self.serial_number_cache:
sn = self.serial_number_cache[(campaign, pwg)] + 1
else:
sn = 1
serial_number_lookup = creq_db.raw_query(
'serial_number', {
'group': True,
'key': [campaign, pwg]
})
if serial_number_lookup:
sn = serial_number_lookup[0]['value'] + 1
# construct the new id
new_prepid = pwg + '-' + campaign + '-' + str(sn).zfill(5)
if sn == 1:
self.logger.info('Beginning new prepid family: %s' %
(new_prepid))
new_request = chained_request({
'_id': new_prepid,
'prepid': new_prepid,
'pwg': pwg,
'member_of_campaign': campaign
})
new_request.update_history({'action': 'created'})
creq_db.save(new_request.json())
self.serial_number_cache[(campaign, pwg)] = sn
self.logger.info('New chain id: %s' % new_prepid)
return new_prepid
def update_request(self, data):
try:
req = chained_request(json_input=loads(data))
except chained_request.IllegalAttributeName:
return {"results": False}
if not req.get_attribute('prepid') and not req.get_attribute('_id'):
self.logger.error('prepid returned was None')
raise ValueError('Prepid returned was None')
# req.set_attribute('_id', req.get_attribute('prepid')
self.logger.info('Updating chained_request %s' %
(req.get_attribute('_id')))
self.logger.info('wtf %s' % (str(req.get_attribute('approval'))))
# update history
req.update_history({'action': 'update'})
new_priority = req.get_attribute('action_parameters')['block_number']
req.set_priority(new_priority)
return self.save_request(req)
def post(self):
fails = []
for chain in loads(flask.request.data):
chain_prepid = chain['prepid']
mcm_chained_request = chained_request(self.chained_requests_db.get(chain_prepid))
action_parameters = chain['action_parameters']
if not mcm_chained_request.set_priority(action_parameters['block_number']):
message = 'Unable to set new priority in request %s' % chain_prepid
fails.append(message)
self.logger.error(message)
else:
mcm_chained_request.set_attribute('action_parameters', action_parameters)
if not mcm_chained_request.save():
message = 'Unable to save chained request %s' % chain_prepid
fails.append(message)
self.logger.error(message)
return {
'results': True if len(fails) == 0 else False,
'message': fails}
def multiple_inspect(self, crid):
crlist = crid.rsplit(',')
res = []
crdb = database('chained_requests')
for cr in crlist:
if crdb.document_exists(cr):
mcm_cr = chained_request(crdb.get(cr))
res.append(mcm_cr.inspect())
else:
res.append({
"prepid": cr,
"results": False,
'message': '%s does not exist' % cr
})
if len(res) > 1:
return res
else:
return res[0]
def inspect_priority(self, forChains=None):
##JR: until put in the proper place
chains = self.get_attribute('chains')
crdb = database('chained_requests')
okay = True
for inCC in chains:
### this is the old convention
#if 'flag' in chains[inCC] and chains[inCC]['flag']:
# if 'chains' in chains[inCC]:
# for acr in chains[inCC]['chains']:
# if forChains and not acr in forChains: continue
# cr=chained_request(crdb.get(acr))
# cc=cr.get_attribute('member_of_campaign')
# #if 'block_number' in chains[cc] and chains[cc]['block_number']:
# if chains[cc]['block_number']:
# cr.set_priority(chains[cc]['block_number'])
# self.logger.log('Set priority block %s to %s'%(chains[cc]['block_number'],cr.get_attribute('prepid')))
# else:
# self.logger.error('Could not set block %s to %s'%(chains[cc]['block_number'],cr.get_attribute('prepid')))
## new convention
if 'chains' in chains[inCC] and type(
chains[inCC]['chains']) == dict:
for acr in chains[inCC]['chains']:
if forChains and not acr in forChains: continue
bn = chains[inCC]['chains'][acr]['block_number']
cr = chained_request(crdb.get(acr))
if bn:
self.logger.log('Set priority block %s to %s' %
(bn, acr))
if not cr.set_priority(bn):
okay = False
else:
self.logger.error('Could not set block %s to %s' %
(bn, acr))
rd = database('requests')
if rd.document_exists(self.get_attribute('prepid')):
r = request(rd.get(self.get_attribute('prepid')))
self.set_attribute('dataset_name', r.get_attribute('dataset_name'))
return okay
def submit_request(self, prepid, run_test_path):
mcm_request = request(self.request_db.get(prepid))
# check if the request should be validated as part of a chain
for chain_prepid in mcm_request.get_attribute('member_of_chain'):
mcm_chain = chained_request(
self.chained_request_db.get(chain_prepid))
if mcm_chain.get_attribute(
'validate') and prepid in mcm_chain.get_attribute(
'chain')[mcm_chain.get_attribute('step'):]:
return {}
aux_validation = mcm_request.get_attribute(self.DOC_VALIDATION)
to_write = mcm_request.get_setup_file(run=True,
do_valid=True,
for_validation=True)
file_name = self.TEST_FILE_NAME % prepid
if not self.create_test_file(to_write, run_test_path, file_name):
mcm_request.set_attribute(self.DOC_VALIDATION, aux_validation)
mcm_request.test_failure(
message=
'There was a problem while creating the file for prepid: %s' %
(mcm_request.get_attribute('prepid')),
what='Validation run test',
rewind=True)
return {}
timeout = mcm_request.get_timeout()
memory = mcm_request.get_attribute("memory")
threads = mcm_request.get_core_num()
self.create_htcondor_config_file(run_test_path, prepid, timeout,
memory, threads, [prepid])
job_info = self.execute_command_submission(prepid, run_test_path)
if 'error' in job_info:
mcm_request.test_failure(message=job_info['error'],
what='Validation run test',
rewind=True)
return {}
job_info[self.DOC_REV] = mcm_request.json()[self.DOC_REV]
job_info[self.DOC_VALIDATION] = mcm_request.get_attribute(
self.DOC_VALIDATION
) # this field change when calling request.get_setup_file, to be propagated in case of success
return job_info
def flow(self, chainid, check_stats=True, reserve=False):
try:
db = database('chained_requests')
creq = chained_request(json_input=db.get(chainid))
except Exception as ex:
self.logger.error(
'Could not initialize chained_request object. Reason: %s' %
(ex))
return {"results": str(ex)}
# TO-DO check if chained_request is in settings forceflow_list and remove it!
# if the chained_request can flow, do it
if reserve:
self.logger.info(
'Attempting to reserve to next step for chained_request %s' %
(creq.get_attribute('_id')))
return creq.reserve(limit=reserve, save_requests=False)
self.logger.info(
'Attempting to flow to next step for chained_request %s' %
(creq.get_attribute('_id')))
return creq.flow_trial(check_stats=check_stats)
def post(self):
fails = []
for chain in loads(flask.request.data):
chain_prepid = chain['prepid']
mcm_chained_request = chained_request(
self.chained_requests_db.get(chain_prepid))
action_parameters = chain['action_parameters']
if not mcm_chained_request.set_priority(
action_parameters['block_number']):
message = 'Unable to set new priority in request %s' % chain_prepid
fails.append(message)
self.logger.error(message)
else:
mcm_chained_request.set_attribute('action_parameters',
action_parameters)
if not mcm_chained_request.save():
message = 'Unable to save chained request %s' % chain_prepid
fails.append(message)
self.logger.error(message)
return {
'results': True if len(fails) == 0 else False,
'message': fails
}
def approve(self, rid, val=-1):
db = database('chained_requests')
if not db.document_exists(rid):
return {
"prepid": rid,
"results":
'Error: The given chained_request id does not exist.'
}
creq = chained_request(json_input=db.get(rid))
try:
creq.approve(val)
except Exception as ex:
return {"prepid": rid, "results": False, 'message': str(ex)}
saved = db.update(creq.json())
if saved:
return {"prepid": rid, "results": True}
else:
return {
"prepid": rid,
"results": False,
'message': 'unable to save the updated chained request'
}
def get(self, chained_request_ids):
"""
Add selected prepid's to global force complete list for later action
"""
if ',' in chained_request_ids:
rlist = chained_request_ids.rsplit(',')
else:
rlist = [chained_request_ids]
res = []
__updated = False
forceflow_list = self.sdb.get("list_of_forceflow")
# TO-DO check if prepid exists!
# TO-DO check the status of chain_req!
for el in rlist:
if el not in forceflow_list["value"]:
forceflow_list["value"].append(el)
chain_req = chained_request(self.cdb.get(el))
chain_req.update_history({'action': 'add_to_forceflow'})
self.cdb.save(chain_req.json())
res.append({"prepid": el, 'results': True, 'message': 'OK'})
__updated = True
else:
res.append({
"prepid":
el,
'results':
False,
'message':
'Chained request already in forceflow list'
})
# TO-DO check the update return value
if __updated:
self.sdb.update(forceflow_list)
return res
def get(self, chained_request_id):
"""
Perform test for chained requests
"""
crdb = database('chained_requests')
rdb = database('requests')
settingsDB = database('settings')
mcm_cr = chained_request(crdb.get(chained_request_id))
if settingsDB.get('validation_stop')['value']:
return {
"results":
False,
'message':
('validation jobs are halted to allow forthcoming mcm '
'restart - try again later'),
"prepid":
chained_request_id
}
requires_validation = False
for rid in mcm_cr.get_attribute(
'chain')[mcm_cr.get_attribute('step'):]:
mcm_r = request(rdb.get(rid))
if not mcm_r.is_root and 'validation' not in mcm_r._json_base__status: # We dont care about non root request because they are not being used on chain run test
break
requires_validation = True
if mcm_r.get_attribute('status') != 'new' or mcm_r.get_attribute(
'approval') != 'none':
return {
"results":
False,
"prepid":
chained_request_id,
"message":
"request %s is in status %s, approval: %s" %
(rid, mcm_r.get_attribute('status'),
mcm_r.get_attribute('approval'))
}
try:
mcm_r.ok_to_move_to_approval_validation(for_chain=True)
mcm_r.update_history({
'action': 'approve',
'step': 'validation'
})
mcm_r.set_attribute('approval', 'validation')
mcm_r.reset_validations_counter()
mcm_r.reload()
text = 'Within chain %s \n' % mcm_cr.get_attribute('prepid')
text += mcm_r.textified()
subject = 'Approval %s in chain %s for request %s' % (
'validation', mcm_cr.get_attribute('prepid'),
mcm_r.get_attribute('prepid'))
notification(subject,
text, [],
group=notification.REQUEST_APPROVALS,
action_objects=[mcm_r.get_attribute('prepid')],
object_type='requests',
base_object=mcm_r)
mcm_r.notify(subject, text, accumulate=True)
except Exception as e:
mcm_cr.reset_requests(str(e), notify_one=rid)
return {
"results": False,
"message": str(e),
"prepid": chained_request_id
}
if not requires_validation:
return {
"results": True,
"message": "No validation required",
"prepid": chained_request_id
}
mcm_cr.set_attribute('validate', 1)
if not crdb.update(mcm_cr.json()):
return {
"results": False,
"message": "Failed while trying to update the document in DB",
"prepid": chained_request_id
}
return {
"results": True,
"message": "run test will start soon",
"prepid": chained_request_id
}
def internal_run(self):
if not self.lock.acquire(blocking=False):
self.logger.error(
"Could not acquire lock for ChainRequestInjector. prepid %s" %
(self.prepid))
return False
try:
crdb = database('chained_requests')
rdb = database('requests')
batch_name = None
if not crdb.document_exists(self.prepid):
# it's a request actually, pick up all chains containing it
mcm_r = rdb.get(self.prepid)
# mcm_crs = crdb.query(query="root_request==%s"% self.prepid) ## not only when its the root of
mcm_crs = crdb.query(query="contains==%s" % self.prepid)
task_name = 'task_' + self.prepid
batch_type = 'Task_' + mcm_r['member_of_campaign']
else:
mcm_crs = [crdb.get(self.prepid)]
current_step_prepid = mcm_crs[0]['chain'][mcm_crs[0]['step']]
mcm_request = rdb.get(current_step_prepid)
task_name = 'task_' + current_step_prepid
batch_type = 'Task_' + mcm_request['member_of_campaign']
if len(mcm_crs) == 0:
return False
mcm_rs = []
# upload all config files to config cache, with "configuration economy" already implemented
for cr in mcm_crs:
mcm_cr = chained_request(cr)
chain = mcm_cr.get_attribute(
'chain')[mcm_cr.get_attribute('step'):]
for request_prepid in chain:
mcm_rs.append(request(rdb.get(request_prepid)))
if self.check_approval and mcm_rs[-1].get_attribute(
'approval') != 'submit':
message = 'requests %s is in "%s"/"%s" status/approval, requires "approved"/"submit"' % (
request_prepid, mcm_rs[-1].get_attribute('status'),
mcm_rs[-1].get_attribute('approval'))
self.logger.error(message)
subject = '%s injection failed' % mcm_cr.get_attribute(
'prepid')
notification(
subject,
message, [],
group=notification.CHAINED_REQUESTS,
action_objects=[mcm_cr.get_attribute('prepid')],
object_type='chained_requests',
base_object=mcm_cr)
mcm_cr.notify(subject, message)
return False
if mcm_rs[-1].get_attribute('status') != 'approved':
# change the return format to percolate the error message
message = 'requests %s in in "%s"/"%s" status/approval, requires "approved"/"submit"' % (
request_prepid, mcm_rs[-1].get_attribute('status'),
mcm_rs[-1].get_attribute('approval'))
self.logger.error(message)
subject = '%s injection failed' % mcm_cr.get_attribute(
'prepid')
notification(
subject,
message, [],
group=notification.CHAINED_REQUESTS,
action_objects=[mcm_cr.get_attribute('prepid')],
object_type='chained_requests',
base_object=mcm_cr)
mcm_cr.notify(subject, message)
return False
uploader = ConfigMakerAndUploader(
prepid=request_prepid,
lock=locker.lock(request_prepid))
if not uploader.internal_run():
message = 'Problem with uploading the configuration for request %s' % (
request_prepid)
notification(
'Configuration upload failed',
message, [],
group=notification.CHAINED_REQUESTS,
action_objects=[mcm_cr.get_attribute('prepid')],
object_type='chained_requests',
base_object=mcm_cr)
mcm_cr.notify('Configuration upload failed', message)
self.logger.error(message)
return False
mcm_r = mcm_rs[-1]
batch_name = BatchPrepId().next_batch_id(batch_type,
create_batch=True)
semaphore_events.increment(batch_name)
self.logger.error('found batch %s' % batch_name)
with ssh_executor(server='vocms081.cern.ch') as ssh:
cmd = self.make_command(mcm_r)
self.logger.error('prepared command %s' % cmd)
# modify here to have the command to be executed
_, stdout, stderr = ssh.execute(cmd)
output = stdout.read()
error = stderr.read()
self.logger.info(output)
self.logger.info(error)
injected_requests = [
l.split()[-1] for l in output.split('\n')
if l.startswith('Injected workflow:')
]
if not injected_requests:
self.injection_error(
'Injection has succeeded but no request manager names were registered. Check with administrators. \nOutput: \n%s\n\nError: \n%s'
% (output, error), mcm_rs)
return False
# what gets printed into the batch object
added_requests = []
once = set()
for mcm_r in mcm_rs:
if mcm_r.get_attribute('prepid') in once:
continue
once.add(mcm_r.get_attribute('prepid'))
added = [{
'name': app_req,
'content': {
'pdmv_prep_id': mcm_r.get_attribute('prepid')
}
} for app_req in injected_requests]
added_requests.extend(added)
# edit the batch object
with locker.lock(batch_name):
bdb = database('batches')
bat = batch(bdb.get(batch_name))
bat.add_requests(added_requests)
bat.update_history({
'action': 'updated',
'step': task_name
})
bat.reload()
# reload the content of all requests as they might have changed already
added = [{
'name': app_req,
'content': {
'pdmv_prep_id': task_name
}
} for app_req in injected_requests]
seen = set()
for cr in mcm_crs:
mcm_cr = chained_request(cr)
chain = mcm_cr.get_attribute(
'chain')[mcm_cr.get_attribute('step'):]
message = ""
for rn in chain:
if rn in seen:
continue # don't do it twice
seen.add(rn)
mcm_r = request(rdb.get(rn))
message += mcm_r.textified()
message += "\n\n"
mcm_r.set_attribute('reqmgr_name', added)
mcm_r.update_history({
'action': 'inject',
'step': batch_name
})
if not self.check_approval:
mcm_r.set_attribute('approval', 'submit')
# set the status to submitted
mcm_r.set_status(
step=mcm_r._json_base__status.index('submitted'),
with_notification=False)
mcm_r.reload()
mcm_cr.set_attribute('last_status',
mcm_r.get_attribute('status'))
# re-get the object
mcm_cr = chained_request(crdb.get(cr['prepid']))
# take care of changes to the chain
mcm_cr.update_history({
'action': 'inject',
'step': batch_name
})
mcm_cr.set_attribute(
'step',
len(mcm_cr.get_attribute('chain')) - 1)
mcm_cr.set_attribute('status', 'processing')
subject = 'Injection succeeded for %s' % mcm_cr.get_attribute(
'prepid')
notification(
subject,
message, [],
group=notification.CHAINED_REQUESTS,
action_objects=[mcm_cr.get_attribute('prepid')],
object_type='chained_requests',
base_object=mcm_cr)
mcm_cr.notify(subject, message)
mcm_cr.reload()
return True
except Exception:
self.injection_error(
"Error with injecting chains for %s :\n %s" %
(self.prepid, traceback.format_exc()), [])
finally: # we decrement batch id and release lock on prepid+lower semaphore
if batch_name: # ditry thing for now. Because batch name can be None for certain use-cases in code above
semaphore_events.decrement(batch_name)
self.lock.release()
self.queue_lock.release()
def delete_request(self, crid):
crdb = database('chained_requests')
rdb = database('requests')
mcm_cr = chained_request(crdb.get(crid))
# get all objects
mcm_r_s = []
for (i, rid) in enumerate(mcm_cr.get_attribute('chain')):
mcm_r = request(rdb.get(rid))
# this is not a valid check as it is allowed to remove a chain around already running requests
# if mcm_r.get_attribute('status') != 'new':
# return {"results":False,"message" : "the request %s part of the chain %s for action %s is not in new status"%( mcm_r.get_attribute('prepid'),
# crid,
# mcm_a.get_attribute('prepid'))}
in_chains = mcm_r.get_attribute('member_of_chain')
in_chains.remove(crid)
self.logger.debug(
"Removing ChainAction member_of_chain: %s to request: %s" %
(mcm_cr.get_attribute("prepid"),
mcm_r.get_attribute('prepid')))
mcm_r.set_attribute('member_of_chain', in_chains)
if i == 0:
if len(in_chains
) == 0 and mcm_r.get_attribute('status') != 'new':
return {
"results":
False,
"message":
"the request %s, not in status new, at the root of the chain will not be chained anymore"
% rid
}
else:
if len(in_chains) == 0:
return {
"results":
False,
"message":
"the request %s, not at the root of the chain will not be chained anymore"
% rid
}
mcm_r.update_history({'action': 'leave', 'step': crid})
mcm_r_s.append(mcm_r)
if mcm_cr.get_attribute('action_parameters')['flag']:
return {
"results": False,
"message": "the action for %s is not disabled" % (crid)
}
# then save all changes
for mcm_r in mcm_r_s:
if not rdb.update(mcm_r.json()):
return {
"results":
False,
"message":
"Could not save request " + mcm_r.get_attribute('prepid')
}
else:
subject = "Request {0} left chain".format(
mcm_r.get_attribute('prepid'))
message = "Request {0} has successfuly left chain {1}".format(
mcm_r.get_attribute('prepid'), crid)
notification(subject,
message, [],
group=notification.REQUEST_OPERATIONS,
action_objects=[mcm_r.get_attribute('prepid')],
object_type='requests',
base_object=mcm_r)
mcm_r.notify(subject, message)
return {"results": crdb.delete(crid)}
def generate_request(self,
aid,
reserve=False,
with_notify=True,
special=False):
adb = database('actions')
ccdb = database('chained_campaigns')
crdb = database('chained_requests')
rdb = database('requests')
if not adb.document_exists(aid):
return {'results': False, 'message': '%s does not exist' % (aid)}
self.logger.log(
'Generating all selected chained_requests for action %s' % (aid))
mcm_a = action(adb.get(aid))
chains = mcm_a.get_attribute('chains')
hasChainsChanged = False
new_chains = []
for cc in chains:
if 'flag' in chains[cc] and chains[cc]['flag']:
## in the new convention, that means that something needs to be created
mcm_cc = chained_campaign(ccdb.get(cc))
new_cr = mcm_cc.generate_request(aid)
if not 'chains' in chains[cc]:
chains[cc]['chains'] = {}
chains[cc]['chains'][new_cr['prepid']] = {}
to_transfer = ['flag', 'threshold', 'staged', 'block_number']
for item in to_transfer:
if item in chains[cc]:
chains[cc]['chains'][
new_cr['prepid']][item] = chains[cc][item]
chains[cc].pop(item)
hasChainsChanged = True
## get the root request
req = request(json_input=rdb.get(aid))
new_cr['last_status'] = req.get_attribute('status')
if new_cr['last_status'] in ['submitted', 'done']:
new_cr['status'] = 'processing'
if special:
new_cr['approval'] = 'none'
if not crdb.update(new_cr):
return {
'results': False,
'message': 'could not save %s' % (new_cr['prepid'])
}
## update the cc history
ccdb.update(mcm_cc.json())
new_chains.append(new_cr['prepid'])
# then let the root request know that it is part of a chained request
inchains = req.get_attribute('member_of_chain')
inchains.append(new_cr['prepid'])
inchains.sort()
req.set_attribute('member_of_chain', list(set(inchains)))
req.update_history({
'action': 'join chain',
'step': new_cr['prepid']
})
if with_notify:
req.notify(
"Request {0} joined chain".format(
req.get_attribute('prepid')),
"Request {0} has successfully joined chain {1}".format(
req.get_attribute('prepid'), new_cr['prepid']),
Nchild=0,
accumulate=True)
mcm_a.update_history({
'action': 'add',
'step': new_cr['prepid']
})
rdb.update(req.json())
if hasChainsChanged:
#the chains parameter might have changed
mcm_a.set_attribute('chains', chains)
adb.update(mcm_a.json())
#and set priorities properly to all requests concerned
priority_set = mcm_a.inspect_priority(forChains=new_chains)
## do the reservation of the whole chain ?
res = []
if reserve:
for cr in new_chains:
mcm_cr = chained_request(crdb.get(cr))
res.append(mcm_cr.reserve())
crdb.update(mcm_cr.json())
if priority_set:
return {"results": True, "prepid": mcm_a.get_attribute('prepid')}
else:
return {
"results": False,
"prepid": mcm_a.get_attribute('prepid'),
"message": "Priorities not set properly"
}
def generate_chained_requests(self, mccm_ticket, request_prepid, mcm_chained_campaign, reserve=False, with_notify=True, special=False):
try:
mcm_chained_campaign.reload(save_current=False)
generated_chained_request = chained_request(mcm_chained_campaign.generate_request(request_prepid))
except Exception as e:
message = "Unable to generate chained request for ticket %s request %s, message: " % (mccm_ticket.get_attribute('prepid'), request_prepid, str(e))
self.logger.error(message)
return {
"results": False,
"message": message}
requests_db = database('requests')
self.overwrite_action_parameters_from_ticket(generated_chained_request, mccm_ticket)
mcm_request = request(json_input=requests_db.get(request_prepid))
generated_chained_request.set_attribute('last_status', mcm_request.get_attribute('status'))
if generated_chained_request.get_attribute('last_status') in ['submitted', 'done']:
generated_chained_request.set_attribute('status', 'processing')
if special:
generated_chained_request.set_attribute('approval', 'none')
new_chain_prepid = generated_chained_request.get_attribute('prepid')
if not generated_chained_request.reload():
return {
'results': False,
'message': 'Unable to save chained request %s' % new_chain_prepid}
# update the history of chained campaign
mcm_chained_campaign.save()
# let the root request know that it is part of a chained request
chains = mcm_request.get_attribute('member_of_chain')
chains.append(new_chain_prepid)
chains.sort()
mcm_request.set_attribute('member_of_chain', list(set(chains)))
mcm_request.update_history({'action': 'join chain', 'step': new_chain_prepid})
if with_notify:
subject = "Request %s joined chain" % mcm_request.get_attribute('prepid')
message = "Request %s has successfully joined chain %s" % (mcm_request.get_attribute('prepid'), new_chain_prepid)
notification(
subject,
message,
[],
group=notification.REQUEST_OPERATIONS,
action_objects=[mcm_request.get_attribute('prepid')],
object_type='requests',
base_object=mcm_request)
mcm_request.notify(
subject,
message,
Nchild=0,
accumulate=True)
mcm_request.save()
# do the reservation of the whole chain ?
generated_requests = []
if reserve:
results_dict = generated_chained_request.reserve(limit=reserve, save_requests=False)
if results_dict['results'] and 'generated_requests' in results_dict:
generated_requests = results_dict['generated_requests']
results_dict.pop('generated_requests')
else:
return {
"results": False,
"prepid": new_chain_prepid,
"message": results_dict['message']}
return {
"results":True,
"prepid": new_chain_prepid,
'generated_requests': generated_requests}
def process_finished_chain_success(self, prepid, doc_info, job_out,
error_out, log_out):
mcm_chained_request = chained_request(
self.chained_request_db.get(prepid))
requests_in_chain = []
success = True
failed_request_prepid = None
for request_prepid, doc_rev in doc_info[
self.CHAIN_REQUESTS].iteritems():
mcm_request = request(self.request_db.get(request_prepid))
# Increase counters for all requests, but save error message only for the first failed request
if success and doc_rev != mcm_request.json()[self.DOC_REV]:
message = 'The request %s in the chain %s has changed during the run test procedure, preventing from being saved' % (
request_prepid, prepid)
success = False
failed_request_prepid = request_prepid
mcm_request.inc_validations_counter()
mcm_request.reload()
if not success:
mcm_chained_request.reset_requests(
message, notify_one=failed_request_prepid)
return
for request_prepid, doc_rev in doc_info[
self.CHAIN_REQUESTS].iteritems():
self.logger.info('Processing request %s in chain %s' %
(request_prepid, prepid))
mcm_request = request(self.request_db.get(request_prepid))
success = True
path = self.test_directory_path + prepid + '/'
try:
success, error = mcm_request.pickup_all_performance(path)
error = 'Error:\n%s\n Error out:\n%s\n' % (error, error_out)
except request.WrongTimeEvent as ex:
self.logger.error('Exception: %s' % (ex))
error = 'Exception:\n%s\n' % (ex)
success = False
retry_validation = self.process_request_wrong_time_event(
mcm_request, prepid)
if retry_validation:
return
if not success:
message = 'Error while picking up all the performance for request %s of chain %s: \n Error:\n%s\n Job out:\n%s\n Error out: \n%s\n Log out: \n%s\n' % (
request_prepid, prepid, error, job_out, error_out, log_out)
mcm_chained_request.reset_requests(message,
notify_one=request_prepid)
return
requests_in_chain.append(mcm_request)
for mcm_request in requests_in_chain:
mcm_request.set_status(with_notification=True)
if not self.request_db.update(mcm_request.json()):
request_prepid = mcm_request.get_attribute('prepid')
message = "The request %s of chain %s could not be saved after the runtest procedure" % (
request_prepid, prepid)
self.logger.error(message)
# reset it and keep saving requests
mcm_request.test_failure(message=message,
what='Chain validation run test',
rewind=True)
mcm_chained_request.reload(
save_current=False
) # setting new requests status change the chain object
mcm_chained_request.set_attribute('validate', 0)
if not self.chained_request_db.update(mcm_chained_request.json()):
message = 'Problem saving changes in chain %s, set validate = False ASAP!' % prepid
self.logger.error(message)
notification(
'Chained validation run test',
message, [],
group=notification.CHAINED_REQUESTS,
action_objects=[mcm_chained_request.get_attribute('prepid')],
object_type='chained_requests',
base_object=mcm_chained_request)
mcm_chained_request.notify('Chained validation run test', message)
return
self.logger.info('Validation job for prepid %s SUCCESSFUL!!!' % prepid)
def submit_chain(self, prepid, run_test_path):
mcm_chained_request = chained_request(
self.chained_request_db.get(prepid))
except_requests = []
reset = False
# If a request of a chain was singly submmited to validation and then somebody reseted it, we will find it here
for request_prepid in mcm_chained_request.get_attribute(
'chain')[mcm_chained_request.get_attribute('step'):]:
if request_prepid in self.submmited_prepids_set:
except_requests.append(request_prepid)
reset = True
if reset:
message = "Requests %s of the chain %s are already in validation" % (
except_requests, prepid)
self.logger.error(message)
mcm_chained_request.reset_requests(message,
except_requests=except_requests)
return {}
to_write = mcm_chained_request.get_setup(run=True,
validation=True,
for_validation=True)
file_name = self.TEST_FILE_NAME % prepid
if not self.create_test_file(to_write, run_test_path, file_name):
mcm_chained_request.reset_requests(
'There was a problem while creating the file for prepid: %s' %
(mcm_chained_request.get_attribute('prepid')))
return {}
requests_in_chain = {}
for request_prepid in mcm_chained_request.get_attribute(
'chain')[mcm_chained_request.get_attribute('step'):]:
mcm_request = request(self.request_db.get(request_prepid))
if not mcm_request.is_root and 'validation' not in mcm_request._json_base__status: # only root or possible root requests
break
status = mcm_request.get_attribute('status')
approval = mcm_request.get_attribute('approval')
if status != 'new' or approval != 'validation':
message = "The request %s of chain %s is in status: %s approval: %s" % (
request_prepid, prepid, status, approval)
self.logger.error(message)
mcm_chained_request.reset_requests(message)
return {}
requests_in_chain[request_prepid] = mcm_request.json()[
self.DOC_REV]
if not len(requests_in_chain):
message = 'No requests to be validated in chain: %s' % prepid
self.logger.info(message)
mcm_chained_request.reset_requests(message)
return {}
timeout, memory, threads = mcm_chained_request.get_timeout_memory_threads(
)
self.create_htcondor_config_file(run_test_path, prepid, timeout,
memory, threads,
list(requests_in_chain.iterkeys()))
job_info = self.execute_command_submission(prepid, run_test_path)
if 'error' in job_info:
mcm_chained_request.reset_requests(job_info['error'])
return {}
job_info[self.CHAIN_REQUESTS] = requests_in_chain
return job_info
def rewind_one(self, crid):
crdb = database('chained_requests')
rdb = database('requests')
if not crdb.document_exists(crid):
return {
"results": False,
"message": "does not exist",
"prepid": crid
}
mcm_cr = chained_request(crdb.get(crid))
current_step = mcm_cr.get_attribute('step')
if current_step == 0:
# or should it be possible to cancel the initial requests of a chained request
return {
"results": False,
"message": "already at the root",
"prepid": crid
}
# supposedly all the other requests were already reset!
for next in mcm_cr.get_attribute('chain')[current_step + 1:]:
# what if that next one is not in the db
if not rdb.document_exists(next):
self.logger.error('%s is part of %s but does not exist' %
(next, crid))
continue
mcm_r = request(rdb.get(next))
if mcm_r.get_attribute('status') != 'new':
# this cannot be right!
self.logger.error(
'%s is after the current request and is not new: %s' %
(next, mcm_r.get_attribute('status')))
return {
"results": False,
"message": "%s is not new" % (next),
"prepid": crid
}
# get the one to be reset
current_id = mcm_cr.get_attribute('chain')[current_step]
mcm_r = request(rdb.get(current_id))
mcm_r.reset()
saved = rdb.update(mcm_r.json())
if not saved:
{
"results": False,
"message": "could not save the last request of the chain",
"prepid": crid
}
# the current chained request has very likely been updated :
# reload it as you have not changed anything to it yet
mcm_cr = chained_request(crdb.get(crid))
mcm_cr.set_attribute('step', current_step - 1)
# set status, last status
mcm_cr.set_last_status()
mcm_cr.set_attribute('status', 'processing')
saved = crdb.update(mcm_cr.json())
if saved:
return {"results": True, "prepid": crid}
else:
return {
"results": False,
"message":
"could not save chained requests. the DB is going to be inconsistent !",
"prepid": crid
}
