def toggle_last_request(self): ## let it toggle the last request to a given approval only if the chained request allows it if self.get_attribute('approval') == 'none': return ccdb = database('chained_campaigns') mcm_cc = ccdb.get(self.get_attribute('member_of_campaign')) (next_campaign_id, flow_name) = mcm_cc['campaigns'][self.get_attribute('step')] fdb = database('flows') mcm_f = flow(fdb.get(flow_name)) # check whether we have to do something even more subtle with the request if mcm_f.get_attribute('approval') == 'submit' or self.get_attribute('approval') == 'submit': rdb = database('requests') next_request = request(rdb.get(self.get_attribute('chain')[self.get_attribute('step')])) current_r_approval = next_request.get_attribute('approval') time_out = 0 #self.logger.error('Trying to move %s from %s to submit'% (next_request.get_attribute('prepid'), current_r_approval)) while current_r_approval != 'submit' and time_out <= 10: time_out += 1 #get it back from db to avoid _red issues next_request = request(rdb.get(next_request.get_attribute('prepid'))) with locker.lock('{0}-wait-for-approval'.format( next_request.get_attribute('prepid') )): next_request.approve() request_saved = rdb.save(next_request.json()) if not request_saved: raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'Could not save the new request %s while trying to move to submit approval' % ( next_request.get_attribute('prepid'))) current_r_approval = next_request.get_attribute('approval') pass return True
def internal_run(self): from tools.installer import installer from tools.batch_control import batch_control location = installer( self.crid, care_on_existing=False, clean_on_exit=True) try: crdb = database('chained_requests') rdb = database('requests') mcm_cr = chained_request(crdb.get(self.crid)) mcm_rs = [] for rid in mcm_cr.get_attribute('chain'): mcm_rs.append( request( rdb.get( rid ) )) test_script = location.location() + 'validation_run_test.sh' with open(test_script, 'w') as there: there.write(mcm_cr.get_setup(directory=location.location(), run=True, validation=True)) batch_test = batch_control( self.crid, test_script ) try: success = batch_test.test() except: self.reset_all( traceback.format_exc() ) return if not success: self.reset_all( '\t .out \n%s\n\t .err \n%s\n ' % ( batch_test.log_out, batch_test.log_err) ) return last_fail=mcm_rs[0] trace="" for mcm_r in mcm_rs: ### if not mcm_r.is_root: continue ##disable for dr request (success,trace) = mcm_r.pickup_all_performance(location.location()) if not success: last_fail = mcm_r break self.logger.error('I came all the way to here and %s (request %s)' % ( success, self.crid )) if success: for mcm_r in mcm_rs: if mcm_r.is_root: mcm_current = request( rdb.get(mcm_r.get_attribute('prepid'))) if mcm_current.json()['_rev'] == mcm_r.json()['_rev']: mcm_r.set_status(with_notification=True) if not mcm_r.reload(): self.reset_all( 'The request %s could not be saved after the runtest procedure' % (mcm_r.get_attribute('prepid'))) return else: self.reset_all( 'The request %s has changed during the run test procedure'%(mcm_r.get_attribute('prepid')), notify_one = mcm_r.get_attribute('prepid')) return else: self.reset_all( trace , notify_one = last_fail.get_attribute('prepid') ) return except: mess = 'We have been taken out of run_safe of runtest_genvalid for %s because \n %s \n During an un-excepted exception. Please contact support.' % ( self.crid, traceback.format_exc()) self.logger.error(mess) finally: location.close()
def GET(self, *args): """ Perform test for chained requests """ if not len(args): return dumps({"results" : False, "message" : "no argument provided"}) from tools.handlers import RunChainValid, validation_pool ## now in the core of the api runtest = RunChainValid(crid=args[0], lock=locker.lock(args[0])) crdb = database('chained_requests') rdb = database('requests') mcm_cr = chained_request(crdb.get(args[0])) mcm_rs = [] for rid in mcm_cr.get_attribute('chain')[mcm_cr.get_attribute('step'):]: mcm_r = request( rdb.get( rid ) ) if mcm_r.get_attribute('status') in ['approved','submitted','done']: return dumps({"results" : False, "prepid" : args[0], "message" : "request %s is in status %s" % ( rid, mcm_r.get_attribute('status'))}) for rid in mcm_cr.get_attribute('chain')[mcm_cr.get_attribute('step'):]: mcm_r = request(rdb.get(rid)) next = 'validation' if not mcm_r.is_root: next = 'approve' try: if mcm_r.get_attribute('approval') == 'none': ## no need to try and move it along if already further than that getattr(mcm_r,'ok_to_move_to_approval_%s' % next)(for_chain=True) mcm_r.update_history({'action' : 'approve', 'step' : next}) mcm_r.set_attribute('approval', next) mcm_r.reload() else: pass ## fail this for the moment. there is no way to handle this yet #text="It is not supported for the moment to test a chain of requests which are partially not new. Please contact an administrator" #runtest.reset_all( text , notify_one = rid ) #return dumps({"results" : False, "message" : text, "prepid" : args[0]}) text = 'Within chain %s \n'% mcm_cr.get_attribute('prepid') text += mcm_r.textified() mcm_r.notify('Approval %s in chain %s for request %s' % (next, mcm_cr.get_attribute('prepid'), mcm_r.get_attribute('prepid')), text, accumulate=True) except Exception as e: runtest.reset_all(str(e), notify_one=rid) return dumps({"results" : False, "message" : str(e),"prepid" : args[0]}) validation_pool.add_task(runtest.internal_run) #runtest.start() return dumps({"results" : True, "message" : "run test started","prepid" : args[0]})
def rewind_one(self, crid): crdb = database('chained_requests') rdb = database('requests') if not crdb.document_exists( crid ): return {"results":False, "message":"does not exist","prepid" : crid} mcm_cr = chained_request( crdb.get( crid) ) current_step = mcm_cr.get_attribute('step') if current_step==0: ## or should it be possible to cancel the initial requests of a chained request return {"results":False, "message":"already at the root","prepid" : crid} ## supposedly all the other requests were already reset! for next in mcm_cr.get_attribute('chain')[current_step+1:]: ## what if that next one is not in the db if not rdb.document_exists( next): self.logger.error('%s is part of %s but does not exist'%( next, crid)) continue mcm_r = request(rdb.get( next )) if mcm_r.get_attribute('status')!='new': # this cannot be right! self.logger.error('%s is after the current request and is not new: %s' % ( next, mcm_r.get_attribute('status'))) return {"results":False, "message":"%s is not new" % (next), "prepid" : crid} ##get the one to be reset current_id=mcm_cr.get_attribute('chain')[current_step] mcm_r = request( rdb.get( current_id )) mcm_r.reset() saved = rdb.update( mcm_r.json() ) if not saved: {"results":False, "message":"could not save the last request of the chain","prepid" : crid} ## the current chained request has very likely been updated : ## reload it as you have not changed anything to it yet mcm_cr = chained_request( crdb.get( crid) ) mcm_cr.set_attribute('step',current_step -1 ) # set status, last status mcm_cr.set_last_status() mcm_cr.set_attribute('status','processing') saved = crdb.update( mcm_cr.json()) if saved: return {"results":True,"prepid" : crid} else: return {"results" : False, "message" : "could not save chained requests. the DB is going to be inconsistent !", "prepid" : crid}
def multiple_inspect(self, cid, in_statuses=['submitted','approved']): clist = list(set(cid.rsplit(','))) res = [] rdb = database('requests') for c in clist: ## this query needs to be modified if we want to ## also inspect the request for submit ! rlist = [] for in_status in in_statuses: rlist.extend(rdb.queries(["member_of_campaign==%s" % (c), "status==%s" % (in_status)])) for r in rlist: mcm_r = request(r) if mcm_r: res.append(mcm_r.inspect()) else: res.append({"prepid": r, "results":False, 'message' : '%s does not exist' % (r)}) if len(res) > 1: return res elif len(res): return res[0] else: return []
def get(self, batch_ids): """ Reset all requests in a batch (or list of) and set the status to reset """ res = [] bdb = database('batches') rdb = database('requests') for bid in batch_ids.split(','): mcm_b = bdb.get(bid) for r in mcm_b['requests']: if 'pdmv_prep_id' not in r['content']: continue rid = r['content']['pdmv_prep_id'] if not rdb.document_exists(rid): continue mcm_r = request(rdb.get(rid)) try: mcm_r.reset() rdb.update(mcm_r.json()) except Exception: continue batch_to_update = batch(mcm_b) batch_to_update.set_attribute('status', 'reset') batch_to_update.update_history({'action': 'set status', 'step': 'reset'}) bdb.update(batch_to_update.json()) res.append({'prepid': bid, 'results': True}) return res
def get(self, batch_ids): """ Reset all requests in a batch (or list of) and set the status to reset """ res = [] bdb = database('batches') rdb = database('requests') for bid in batch_ids.split(','): mcm_b = bdb.get(bid) for r in mcm_b['requests']: if 'pdmv_prep_id' not in r['content']: continue rid = r['content']['pdmv_prep_id'] if not rdb.document_exists(rid): continue mcm_r = request(rdb.get(rid)) try: mcm_r.reset() rdb.update(mcm_r.json()) except Exception: continue batch_to_update = batch(mcm_b) batch_to_update.set_attribute('status', 'reset') batch_to_update.update_history({ 'action': 'set status', 'step': 'reset' }) bdb.update(batch_to_update.json()) res.append({'prepid': bid, 'results': True}) return res
def multiple_inspect(self, cid, in_statuses=['submitted', 'approved']): clist = list(set(cid.rsplit(','))) res = [] rdb = database('requests') for c in clist: ## this query needs to be modified if we want to also inspect the request for submit ! rlist = [] for in_status in in_statuses: rlist.extend( rdb.queries([ "member_of_campaign==%s" % (c), "status==%s" % (in_status) ])) for r in rlist: mcm_r = request(r) if mcm_r: res.append(mcm_r.inspect()) else: res.append({ "prepid": r, "results": False, 'message': '%s does not exist' % (r) }) if len(res) > 1: return res elif len(res): return res[0] else: return []
def find_chains(self): # validate request if not self.get_attribute('prepid'): raise self.PrepIdNotDefinedException() # initialize db connections try: reqdb = database('requests') campaigndb = database('campaigns') except database.DatabaseAccessError as ex: return False # validate prepid if not reqdb.document_exists(self.get_attribute('prepid')): raise self.PrepIdDoesNotExistException( self.get_attribute('prepid')) # get campaign id req = request(json_input=reqdb.get(self.get_attribute('prepid'))) # check if campaign exists campid = req.get_attribute('member_of_campaign') if not campid: self.logger.error('action %s has not a campaign defined' % (self.get_attribute('prepid'))) raise ValueError('Error: Campaign was not set for', self.get_attribute('prepid')) if not campaigndb.document_exists(campid): raise self.PrepIdDoesNotExistException(campid) # get all chains return self.__retrieve_chains(self.get_attribute('prepid'), campid)
def find_chains(self): # validate request if not self.get_attribute('prepid'): raise self.PrepIdNotDefinedException() # initialize db connections try: reqdb = database('requests') campaigndb = database('campaigns') except database.DatabaseAccessError as ex: return False # validate prepid if not reqdb.document_exists(self.get_attribute('prepid')): raise self.PrepIdDoesNotExistException(self.get_attribute('prepid')) # get campaign id req = request(json_input=reqdb.get(self.get_attribute('prepid'))) # check if campaign exists campid = req.get_attribute('member_of_campaign') if not campid: self.logger.error('action %s has not a campaign defined' % (self.get_attribute('prepid'))) raise ValueError('Error: Campaign was not set for', self.get_attribute('prepid')) if not campaigndb.document_exists(campid): raise self.PrepIdDoesNotExistException(campid) # get all chains return self.__retrieve_chains(self.get_attribute('prepid'), campid)
def GET(self, *args): """ Reset all requests in a batch (or list of) and set the status to reset """ res=[] bdb = database('batches') rdb = database('requests') bids = args[0] for bid in bids.split(','): mcm_b = bdb.get(bid) for r in mcm_b['requests']: if not 'pdmv_prep_id' in r['content']: continue rid = r['content']['pdmv_prep_id'] if not rdb.document_exists( rid ): continue mcm_r = request( rdb.get( rid ) ) try: mcm_r.reset() rdb.update( mcm_r.json() ) except Exception as ex: continue mcm_b['status'] = 'reset' bdb.update( mcm_b ) res.append({'prepid':bid, 'results': True}) return dumps(res)
def reset_all(self, message, what = 'Chained validation run test', notify_one=None): crdb = database('chained_requests') rdb = database('requests') mcm_cr = chained_request(crdb.get(self.crid)) if self.scratch: chain = mcm_cr.get_attribute('chain') else: chain = mcm_cr.get_attribute('chain')[mcm_cr.get_attribute('step'):] for rid in chain: mcm_r = request(rdb.get(rid )) s_label = 'chainvalid-%s' % rid semaphore_events.decrement(s_label) if not semaphore_events.is_set(s_label): ##someone else is still validating that chain, so no reset ! mcm_r.notify('%s failed for request %s' % (what, mcm_r.get_attribute('prepid')), message) continue ## do not reset anything that does not look ok already # this might leave things half-way inconsistent in terms of status if mcm_r.get_attribute('status') != 'new': mcm_r.notify('%s failed for request %s' % (what, mcm_r.get_attribute('prepid')), message) continue notify = True if notify_one and notify_one != rid: notify = False mcm_r.test_failure( message, what = what, rewind=True, with_notification=notify)
def GET(self, *args): """ Does a soft reset to all relevant request in the chain """ if not len(args): return dumps({"results" : False, "message" : "no argument provided"}) arg0 = args[0] crdb = database('chained_requests') rdb = database('requests') mcm_cr = chained_request(crdb.get(arg0)) for rid in reversed(mcm_cr.get_attribute('chain')[:mcm_cr.get_attribute('step')+1]): ## from the current one to the first one REVERSED mcm_r = request(rdb.get(rid)) try: mcm_r.reset(hard=False) except Exception as e: return dumps({'prepid' : arg0, 'results' : False, 'message' : str(e)}) mcm_r.reload() mcm_cr = chained_request(crdb.get(arg0)) mcm_cr.set_attribute('step', max(0, mcm_cr.get_attribute('chain').index(rid)-1)) mcm_cr.reload() return dumps({'prepid' : arg0, 'results':True})
def next_prepid(self, pwg, camp): if not pwg or not camp: return None with locker.lock("{0}-{1}".format(pwg, camp)): db = database(self.db_name) query_results = db.raw_query('serial_number', { 'group': True, 'key': [camp, pwg] }) sn = 1 if query_results: sn = query_results[0]['value'] + 1 pid = '%s-%s-%05d' % (pwg, camp, sn) if sn == 1: self.logger.info('Beginning new prepid family: %s-%s' % (pwg, camp)) db_camp = database('campaigns', cache_enabled=True) req_camp = campaign(db_camp.get(camp)) new_request = request( req_camp.add_request({ '_id': pid, 'prepid': pid, 'pwg': pwg, 'member_of_campaign': camp })) new_request.update_history({'action': 'created'}) db.save(new_request.json()) self.logger.info('New prepid : %s ' % pid) return pid
def GET(self, *args): """ Reset all requests in a batch (or list of) and set the status to reset """ res = [] bdb = database('batches') rdb = database('requests') bids = args[0] for bid in bids.split(','): mcm_b = bdb.get(bid) for r in mcm_b['requests']: if not 'pdmv_prep_id' in r['content']: continue rid = r['content']['pdmv_prep_id'] if not rdb.document_exists(rid): continue mcm_r = request(rdb.get(rid)) try: mcm_r.reset() rdb.update(mcm_r.json()) except Exception as ex: continue mcm_b['status'] = 'reset' bdb.update(mcm_b) res.append({'prepid': bid, 'results': True}) return dumps(res)
def get(self, chained_request_id): """ Does a soft reset to all relevant request in the chain """ crdb = database('chained_requests') rdb = database('requests') mcm_cr = chained_request(crdb.get(chained_request_id)) for rid in reversed( mcm_cr.get_attribute('chain')[:mcm_cr.get_attribute('step') + 1]): # from the current one to the first one REVERSED mcm_r = request(rdb.get(rid)) try: mcm_r.reset(hard=False) except Exception as e: return { 'prepid': chained_request_id, 'results': False, 'message': str(e) } mcm_r.reload() mcm_cr = chained_request(crdb.get(chained_request_id)) mcm_cr.set_attribute( 'step', max(0, mcm_cr.get_attribute('chain').index(rid) - 1)) mcm_cr.reload() return {'prepid': chained_request_id, 'results': True}
def submit_request(self, prepid, run_test_path): mcm_request = request(self.request_db.get(prepid)) # check if the request should be validated as part of a chain for chain_prepid in mcm_request.get_attribute('member_of_chain'): mcm_chain = chained_request(self.chained_request_db.get(chain_prepid)) if mcm_chain.get_attribute('validate') and prepid in mcm_chain.get_attribute('chain')[mcm_chain.get_attribute('step'):]: return {} aux_validation = mcm_request.get_attribute(self.DOC_VALIDATION) to_write = mcm_request.get_setup_file(run=True, do_valid=True, for_validation=True, gen_script=True) file_name = self.TEST_FILE_NAME % prepid if not self.create_test_file(to_write, run_test_path, file_name): mcm_request.set_attribute(self.DOC_VALIDATION, aux_validation) mcm_request.test_failure( message='There was a problem while creating the file for prepid: %s' % (mcm_request.get_attribute('prepid')), what='Validation run test', rewind=True) return {} timeout = mcm_request.get_timeout() memory = mcm_request.get_attribute("memory") threads = mcm_request.get_core_num() self.create_htcondor_config_file(run_test_path, prepid, timeout, memory, threads, [prepid], mcm_request) job_info = self.execute_command_submission(prepid, run_test_path) if 'error' in job_info: mcm_request.test_failure(message=job_info['error'], what='Validation run test', rewind=True) return {} job_info[self.DOC_REV] = mcm_request.json()[self.DOC_REV] job_info[self.DOC_VALIDATION] = mcm_request.get_attribute(self.DOC_VALIDATION) # this field change when calling request.get_setup_file, to be propagated in case of success return job_info
def internal_run(self): if not self.lock.acquire(blocking=False): return False try: req = request(self.request_db.get(self.prepid)) ret = req.prepare_and_upload_config() return True if ret else False finally: self.lock.release()
def submit_chain(self, prepid, run_test_path): mcm_chained_request = chained_request(self.chained_request_db.get(prepid)) except_requests = [] reset = False # If a request of a chain was singly submmited to validation and then somebody reseted it, we will find it here for request_prepid in mcm_chained_request.get_attribute('chain')[mcm_chained_request.get_attribute('step'):]: if request_prepid in self.submmited_prepids_set: except_requests.append(request_prepid) reset = True if reset: message = "Requests %s of the chain %s are already in validation" % (except_requests, prepid) self.logger.error(message) mcm_chained_request.reset_requests(message, except_requests=except_requests) return {} to_write = mcm_chained_request.get_setup(run=True, validation=True, for_validation=True) file_name = self.TEST_FILE_NAME % prepid if not self.create_test_file(to_write, run_test_path, file_name): mcm_chained_request.reset_requests('There was a problem while creating the file for prepid: %s' % (mcm_chained_request.get_attribute('prepid'))) return {} requests_in_chain = {} first_request_in_chain = None for request_prepid in mcm_chained_request.get_attribute('chain')[mcm_chained_request.get_attribute('step'):]: mcm_request = request(self.request_db.get(request_prepid)) if not mcm_request.is_root and 'validation' not in mcm_request._json_base__status: # only root or possible root requests break status = mcm_request.get_attribute('status') approval = mcm_request.get_attribute('approval') if status != 'new' or approval != 'validation': message = "The request %s of chain %s is in status: %s approval: %s" % (request_prepid, prepid, status, approval) self.logger.error(message) mcm_chained_request.reset_requests(message) return {} requests_in_chain[request_prepid] = mcm_request.json()[self.DOC_REV] if not first_request_in_chain: first_request_in_chain = mcm_request if not len(requests_in_chain): message = 'No requests to be validated in chain: %s' % prepid self.logger.info(message) mcm_chained_request.reset_requests(message) return {} timeout, memory, threads = mcm_chained_request.get_timeout_memory_threads() self.create_htcondor_config_file(run_test_path, prepid, timeout, memory, threads, list(requests_in_chain.iterkeys()), first_request_in_chain) job_info = self.execute_command_submission(prepid, run_test_path) if 'error' in job_info: mcm_chained_request.reset_requests(job_info['error']) return {} job_info[self.CHAIN_REQUESTS] = requests_in_chain return job_info
def announce_with_text(self, bid, message): bdb = database('batches') if not semaphore_events.is_set(bid): return {"results": False, "message": "Batch {0} has on-going submissions.".format(bid), "prepid": bid} b = batch(bdb.get(bid)) workflows = '' for dictionary in b.get_attribute('requests'): workflows += dictionary['name'] + ',' workflows = workflows[:-1] r = '' result = {} if workflows != '': approver = RequestApprover(bid, workflows) result = approver.internal_run() if (result['results']): r = b.announce(message) else: r = b.announce(message) if r: map_wf_to_prepid = {} for dictionary in b.get_attribute('requests'): wf = dictionary.get('name') prepid = dictionary.get('content', {}).get('pdmv_prep_id') if not wf or not prepid: continue if wf not in map_wf_to_prepid: map_wf_to_prepid[wf] = [] map_wf_to_prepid[wf].append(prepid) rdb = database('requests') priority_coeff = settings.get_value('nanoaod_priority_increase_coefficient') for wf, requests in map_wf_to_prepid.iteritems(): if len(requests) == 1 and 'nanoaod' in requests[0].lower(): for r_prepid in requests: req = request(rdb.get(r_prepid)) current_priority = req.get_attribute('priority') new_priority = int(current_priority + priority_coeff * 1000) req.change_priority(new_priority) return { "results": bdb.update(b.json()), "message": r, "prepid": bid } else: return { "results": False, "prepid": bid, "message": result['message'] if 'message' in result and not r else r }
def set_priority(self, level): rdb = database('requests') okay = True for r in self.get_attribute('chain'): req = request(rdb.get(r)) ##only those that can still be changed #set to the maximum priority if not req.change_priority(priority().priority(level)): self.logger.log('Could not save updated priority for %s' % r) okay = False return okay
def set_priority(self, level): rdb = database('requests') okay = True for r in self.get_attribute('chain'): req = request(rdb.get(r)) ##only those that can still be changed #set to the maximum priority if not req.change_priority(priority().priority(level)): self.logger.log('Could not save updated priority for %s' % r) okay = False return okay
def get_setup(self, directory='', events=None, run=False, validation=False): req_ids = self.get_attribute('chain') rdb = database('requests') setup_file = '' for (index,req_id) in enumerate(req_ids): req = request(rdb.get(req_id)) ev = events if not ev and index!=0 and not req.is_root: ev = -1 setup_file += req.get_setup_file(directory=directory, events=ev, run=run, do_valid=validation) return setup_file
def toggle_last_request(self): ## let it toggle the last request to a given approval only if the chained request allows it if self.get_attribute('approval') == 'none': return ccdb = database('chained_campaigns') mcm_cc = ccdb.get(self.get_attribute('member_of_campaign')) (next_campaign_id, flow_name) = mcm_cc['campaigns'][self.get_attribute('step')] fdb = database('flows') mcm_f = flow(fdb.get(flow_name)) # check whether we have to do something even more subtle with the request if mcm_f.get_attribute('approval') == 'submit' or self.get_attribute( 'approval') == 'submit': rdb = database('requests') next_request = request( rdb.get( self.get_attribute('chain')[self.get_attribute('step')])) current_r_approval = next_request.get_attribute('approval') time_out = 0 #self.logger.error('Trying to move %s from %s to submit'% (next_request.get_attribute('prepid'), current_r_approval)) while current_r_approval != 'submit' and time_out <= 10: time_out += 1 #get it back from db to avoid _red issues next_request = request( rdb.get(next_request.get_attribute('prepid'))) with locker.lock('{0}-wait-for-approval'.format( next_request.get_attribute('prepid'))): next_request.approve() request_saved = rdb.save(next_request.json()) if not request_saved: raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'Could not save the new request %s while trying to move to submit approval' % (next_request.get_attribute('prepid'))) current_r_approval = next_request.get_attribute('approval') pass return True
def multiple_inspect(self, cid, in_statuses=['submitted', 'approved']): clist = list(set(cid.rsplit(','))) res = [] rdb = database('requests') index = 0 self.logger.error("Chain inspect begin. Number of chains to be inspected: %s" % (len(clist))) try: while len(clist) > index: yield dumps({"current cr element": "%s/%s" % (index, len(clist))}, indent=2) query = rdb.construct_lucene_complex_query([ ('member_of_campaign', {'value': clist[index: index + 1]}), ('status', {'value': in_statuses}) ]) ##do another loop over the requests themselves req_page = 0 request_res = rdb.full_text_search('search', query, page=req_page) while len(request_res) > 0: self.logger.info("inspecting single requests. page: %s" % (req_page)) for r in request_res: self.logger.info("running inspect on request: %s" % (r['prepid'])) mcm_r = request(r) if mcm_r: #making it as a stream yield dumps(mcm_r.inspect(), indent=4) else: #making it as a stream yield dumps({"prepid": r, "results": False, 'message': '%s does not exist' % (r)}, indent=4) req_page += 1 request_res = rdb.full_text_search('search', query, page=req_page) time.sleep(0.5) index += 1 time.sleep(1) except Exception as e: subject = "Exception while inspecting request " message = "Request: %s \n %s traceback: \n %s" % (mcm_r.get_attribute('prepid'), str(e), traceback.format_exc()) self.logger.error(subject + message) notification( subject, message, [], group=notification.REQUEST_OPERATIONS, action_objects=[mcm_r.get_attribute('prepid')], object_type='requests', base_object=mcm_r) mcm_r.notify(subject, message, accumulate=True) self.logger.info("Campaign inspection finished!")
def process_finished_request_failed(self, prepid, job_out, error_out, was_exited=True, job_error_out='', out_path='', log_out=''): mcm_request = request(self.request_db.get(prepid)) # need to provide all the information back if not was_exited: no_success_message = "File %s does not look properly formatted or does not exist. \n %s \n %s \n Error out: \n%s \n Log out: \n%s" % ( out_path, job_out, job_error_out, error_out, log_out) else: no_success_message = '\t Job out: \n%s\n\t Error out: \n%s\n Log out: \n%s ' % (job_out, error_out, log_out) mcm_request.test_failure( message=no_success_message, what='Validation run test', rewind=True)
def reset_all(self, message, what = 'Chained validation run test', notify_one=None): crdb = database('chained_requests') rdb = database('requests') mcm_cr = chained_request(crdb.get(self.crid)) for rid in mcm_cr.get_attribute('chain'): mcm_r = request( rdb.get( rid ) ) notify = True if notify_one and notify_one != rid: notify = False mcm_r.test_failure( message, what = what, rewind=True, with_notification=notify)
def delete_request(self, crid): crdb = database('chained_requests') rdb = database('requests') adb = database('actions') mcm_cr = chained_request(crdb.get(crid)) mcm_a = None ## get all objects mcm_r_s=[] for (i,rid) in enumerate(mcm_cr.get_attribute('chain')): mcm_r = request(rdb.get(rid)) #this is not a valid check as it is allowed to remove a chain around already running requests # if mcm_r.get_attribute('status') != 'new': # return {"results":False,"message" : "the request %s part of the chain %s for action %s is not in new status"%( mcm_r.get_attribute('prepid'), # crid, # mcm_a.get_attribute('prepid'))} in_chains = mcm_r.get_attribute('member_of_chain') in_chains.remove( crid ) mcm_r.set_attribute('member_of_chain', in_chains) if i==0: # the root is the action id mcm_a = action(adb.get(rid)) if len(in_chains)==0 and mcm_r.get_attribute('status')!='new': return {"results":False, "message" : "the request %s, not in status new, at the root of the chain will not be chained anymore"% rid} else: if len(in_chains)==0: return {"results":False,"message" : "the request %s, not at the root of the chain will not be chained anymore"% rid} mcm_r.update_history({'action':'leave','step':crid}) mcm_r_s.append( mcm_r ) ## check if possible to get rid of it ! # action for the chain is disabled chains = mcm_a.get_chains( mcm_cr.get_attribute('member_of_campaign')) if chains[crid]['flag']: return {"results":False,"message" : "the action %s for %s is not disabled"%(mcm_a.get_attribute('prepid'), crid)} #take it out mcm_a.remove_chain( mcm_cr.get_attribute('member_of_campaign'), mcm_cr.get_attribute('prepid') ) if not adb.update( mcm_a.json()): return {"results":False,"message" : "Could not save action "+ mcm_a.get_attribute('prepid')} ## then save all changes for mcm_r in mcm_r_s: if not rdb.update( mcm_r.json()): return {"results":False,"message" : "Could not save request "+ mcm_r.get_attribute('prepid')} else: mcm_r.notify("Request {0} left chain".format( mcm_r.get_attribute('prepid')), "Request {0} has successfuly left chain {1}".format( mcm_r.get_attribute('prepid'), crid)) return {"results": crdb.delete(crid)}
def report_error(self, prepid, message): self.logger.error(message) try: if 'chain' in prepid: mcm_chained_request = chained_request(self.chained_request_db.get(prepid)) mcm_chained_request.reset_requests(message) else: mcm_request = request(self.request_db.get(prepid)) mcm_request.test_failure( message=message, what='Validation run test', rewind=True) except Exception as e: self.logger.error("Exception while reporting an error for %s message: %s \ntraceback: %s" % (prepid, str(e), traceback.format_exc()))
def check_request(self): if not self.request_db.document_exists(self.prepid): self.logger.inject("The request {0} does not exist".format(self.prepid), level='error', handler=self.prepid) return False, None req = request(self.request_db.get(self.prepid)) if self.check_approval and req.get_attribute('approval') != 'submit': self.injection_error("The request is in approval {0}, while submit is required".format(req.get_attribute('approval')), req) return False, None if req.get_attribute('status') != 'approved': self.injection_error("The request is in status {0}, while approved is required".format(req.get_attribute('status')), req) return False, None return True, req
def generate_chained_requests(self, mccm_ticket, request_prepid, mcm_chained_campaign, reserve=False, with_notify=True, special=False): try: mcm_chained_campaign.reload(save_current=False) generated_chained_request = chained_request(mcm_chained_campaign.generate_request(request_prepid)) except Exception as e: message = "Unable to generate chained request for ticket %s request %s, message: " % (mccm_ticket.get_attribute('prepid'), request_prepid, str(e)) self.logger.error(message) return { "results": False, "message": message} requests_db = database('requests') self.overwrite_action_parameters_from_ticket(generated_chained_request, mccm_ticket) mcm_request = request(json_input=requests_db.get(request_prepid)) generated_chained_request.set_attribute('last_status', mcm_request.get_attribute('status')) if generated_chained_request.get_attribute('last_status') in ['submitted', 'done']: generated_chained_request.set_attribute('status', 'processing') if special: generated_chained_request.set_attribute('approval', 'none') new_chain_prepid = generated_chained_request.get_attribute('prepid') if not generated_chained_request.reload(): return { 'results': False, 'message': 'Unable to save chained request %s' % new_chain_prepid} # update the history of chained campaign mcm_chained_campaign.save() # let the root request know that it is part of a chained request chains = mcm_request.get_attribute('member_of_chain') chains.append(new_chain_prepid) chains.sort() mcm_request.set_attribute('member_of_chain', list(set(chains))) mcm_request.update_history({'action': 'join chain', 'step': new_chain_prepid}) mcm_request.save() # do the reservation of the whole chain ? generated_requests = [] if reserve: results_dict = generated_chained_request.reserve(limit=reserve, save_requests=False) if results_dict['results'] and 'generated_requests' in results_dict: generated_requests = results_dict['generated_requests'] results_dict.pop('generated_requests') else: return { "results": False, "prepid": new_chain_prepid, "message": results_dict['message']} return { "results":True, "prepid": new_chain_prepid, 'generated_requests': generated_requests}
def process_finished_request_success(self, prepid, doc_info, job_out, error_out, log_out): mcm_request = request(self.request_db.get(prepid)) doc_revision = doc_info[self.DOC_REV] doc_validation = doc_info[self.DOC_VALIDATION] if doc_revision != mcm_request.json()[self.DOC_REV]: message = 'The request %s has changed during the run test procedure, preventing from being saved' % ( prepid) self.logger.error(message) mcm_request.test_failure(message=message, what='Validation run test', rewind=True) return path = self.test_directory_path + prepid + '/' error = '' is_success = False mcm_request.inc_validations_counter() try: (is_success, error) = mcm_request.pickup_all_performance(path) error = 'Error:\n%s\n Error out:\n%s\n' % (error, error_out) except request.WrongTimeEvent as ex: self.logger.error('Exception: %s' % (ex)) retry_validation = self.process_request_wrong_time_event( mcm_request) if retry_validation: return if not is_success: self.process_finished_request_failed(prepid, job_out, error, log_out=log_out) return mcm_request.set_status(with_notification=True) aux_validation = mcm_request.get_attribute(self.DOC_VALIDATION) mcm_request.set_attribute(self.DOC_VALIDATION, doc_validation) saved = self.request_db.update(mcm_request.json()) if not saved: mcm_request.set_attribute(self.DOC_VALIDATION, aux_validation) mcm_request.test_failure( message= 'The request could not be saved after the run test procedure', what='Validation run test', rewind=True) return self.logger.info('Validation job for prepid %s SUCCESSFUL!!!' % prepid)
def check_request(self): if not self.request_db.document_exists(self.prepid): self.logger.inject("The request {0} does not exist".format(self.prepid), level='error', handler=self.prepid) return False, None req = request(self.request_db.get(self.prepid)) if self.check_approval and req.get_attribute('approval') != 'submit': self.injection_error( "The request is in approval {0}, while submit is required".format(req.get_attribute('approval')), req) return False, None if req.get_attribute('status') != 'approved': self.injection_error( "The request is in status {0}, while approved is required".format(req.get_attribute('status')), req) return False, None return True, req
def report_error(self, prepid, message): self.logger.error(message) try: if 'chain' in prepid: mcm_chained_request = chained_request( self.chained_request_db.get(prepid)) mcm_chained_request.reset_requests(message) else: mcm_request = request(self.request_db.get(prepid)) mcm_request.test_failure(message=message, what='Validation run test', rewind=True) except Exception as e: self.logger.error( "Exception while reporting an error for %s message: %s \ntraceback: %s" % (prepid, str(e), traceback.format_exc()))
def process_finished_request_success(self, prepid, doc_info, job_out, error_out, log_out): mcm_request = request(self.request_db.get(prepid)) doc_revision = doc_info[self.DOC_REV] doc_validation = doc_info[self.DOC_VALIDATION] if doc_revision != mcm_request.json()[self.DOC_REV]: message = 'The request %s has changed during the run test procedure, preventing from being saved' % (prepid) self.logger.error(message) mcm_request.test_failure( message=message, what='Validation run test', rewind=True) return path = self.test_directory_path + prepid + '/' error = '' is_success = False mcm_request.inc_validations_counter() try: (is_success, error) = mcm_request.pickup_all_performance(path) error = 'Error:\n%s\n Error out:\n%s\n' % (error, error_out) except request.WrongTimeEvent as ex: self.logger.error('Exception: %s' % (ex)) retry_validation = self.process_request_wrong_time_event(mcm_request) if retry_validation: return if not is_success: self.process_finished_request_failed(prepid, job_out, error, log_out=log_out) return mcm_request.set_status(with_notification=True) aux_validation = mcm_request.get_attribute(self.DOC_VALIDATION) mcm_request.set_attribute(self.DOC_VALIDATION, doc_validation) saved = self.request_db.update(mcm_request.json()) if not saved: mcm_request.set_attribute(self.DOC_VALIDATION, aux_validation) mcm_request.test_failure( message='The request could not be saved after the run test procedure', what='Validation run test', rewind=True) return mcm_request.test_success( what='Validation', message='Validation was successful for %s. Request:\n%s' % (mcm_request.get_attribute('prepid'), mcm_request.textified())) self.logger.info('Validation job for prepid %s SUCCESSFUL!!!' % prepid)
def internal_run(self): if not self.lock.acquire(blocking=False): self.logger.error( "Could not acquire lock for ConfigMakerAndUploader. prepid %s" % (self.prepid)) return False try: self.logger.info( "Acquired lock for ConfigMakerAndUploader. prepid %s" % (self.prepid)) req = request(self.request_db.get(self.prepid)) ret = req.prepare_and_upload_config() return True if ret else False finally: self.logger.info( "Releasing a lock for ConfigMakerAndUploader. prepid %s" % (self.prepid)) self.lock.release()
def internal_run(self): if not self.lock.acquire(blocking=False): self.logger.error("Could not acquire lock for ConfigMakerAndUploader. prepid %s" % ( self.prepid)) return False try: self.logger.error("Acquired lock for ConfigMakerAndUploader. prepid %s" % ( self.prepid)) req = request(self.request_db.get(self.prepid)) ret = req.prepare_and_upload_config() return True if ret else False finally: self.logger.error("Releasing a lock for ConfigMakerAndUploader. prepid %s" % ( self.prepid)) self.lock.release()
def get_setup(self, directory='', events=None, run=False, validation=False): req_ids = self.get_attribute('chain') rdb = database('requests') setup_file = '' for (index, req_id) in enumerate(req_ids): req = request(rdb.get(req_id)) ev = events if not ev and index != 0 and not req.is_root: ev = -1 setup_file += req.get_setup_file(directory=directory, events=ev, run=run, do_valid=validation) return setup_file
def next_prepid(self, pwg, camp): if not pwg or not camp: return None with locker.lock("{0}-{1}".format(pwg, camp)): db = database(self.db_name) query_results = db.raw_query('serial_number', {'group':True, 'key':[camp, pwg]}) sn = 1 if query_results: sn = query_results[0]['value']+1 pid='%s-%s-%05d'%( pwg, camp , sn) if sn==1: self.logger.log('Beginning new prepid family: %s-%s' %( pwg, camp)) db_camp = database('campaigns', cache=True) req_camp = campaign(db_camp.get(camp)) new_request = request(req_camp.add_request({'_id':pid, 'prepid':pid, 'pwg':pwg, 'member_of_campaign':camp})) new_request.update_history({'action':'created'}) db.save(new_request.json()) self.logger.log('New prepid : %s '%pid) return pid
def process_finished_request_failed(self, prepid, job_out, error_out, was_exited=True, job_error_out='', out_path='', log_out=''): mcm_request = request(self.request_db.get(prepid)) # need to provide all the information back if not was_exited: no_success_message = "File %s does not look properly formatted or does not exist. \n %s \n %s \n Error out: \n%s \n Log out: \n%s" % ( out_path, job_out, job_error_out, error_out, log_out) else: no_success_message = '\t Job out: \n%s\n\t Error out: \n%s\n Log out: \n%s ' % ( job_out, error_out, log_out) mcm_request.test_failure(message=no_success_message, what='Validation run test', rewind=True)
def inspect_priority(self, forChains=None): ##JR: until put in the proper place chains = self.get_attribute('chains') crdb = database('chained_requests') okay = True for inCC in chains: ### this is the old convention #if 'flag' in chains[inCC] and chains[inCC]['flag']: # if 'chains' in chains[inCC]: # for acr in chains[inCC]['chains']: # if forChains and not acr in forChains: continue # cr=chained_request(crdb.get(acr)) # cc=cr.get_attribute('member_of_campaign') # #if 'block_number' in chains[cc] and chains[cc]['block_number']: # if chains[cc]['block_number']: # cr.set_priority(chains[cc]['block_number']) # self.logger.log('Set priority block %s to %s'%(chains[cc]['block_number'],cr.get_attribute('prepid'))) # else: # self.logger.error('Could not set block %s to %s'%(chains[cc]['block_number'],cr.get_attribute('prepid'))) ## new convention if 'chains' in chains[inCC] and type( chains[inCC]['chains']) == dict: for acr in chains[inCC]['chains']: if forChains and not acr in forChains: continue bn = chains[inCC]['chains'][acr]['block_number'] cr = chained_request(crdb.get(acr)) if bn: self.logger.log('Set priority block %s to %s' % (bn, acr)) if not cr.set_priority(bn): okay = False else: self.logger.error('Could not set block %s to %s' % (bn, acr)) rd = database('requests') if rd.document_exists(self.get_attribute('prepid')): r = request(rd.get(self.get_attribute('prepid'))) self.set_attribute('dataset_name', r.get_attribute('dataset_name')) return okay
def add_to_chain(self, data): rdb = database('requests') db = database('chained_requests') self.logger.log('Adding a new request to chained_request') try: from json_layer.request import request except ImportError as ex: self.logger.error('Could not import request object class.', level='critical') return {"results":False} try: req = request(json_input=loads(data)) except request.IllegalAttributeName as ex: return {"results":str(ex)} if not req.get_attribute("member_of_chain"): self.logger.error('Attribute "member_of_chain" attribute was None') return {"results":'Error: "member_of_chain" attribute was None.'} if not req.get_attribute("member_of_campaign"): self.logger.error('Attribute "member_of_campaign" attribute was None.') return {"results":'Error: "member_of_campaign" attribute was None.'} try: creq = chained_request(json_input=db.get(req.get_attribute('member_of_chain'))) except chained_request.IllegalAttributeName as ex: return {"results":str(ex)} try: new_req = creq.add_request(req.json()) except chained_request.CampaignAlreadyInChainException as ex: return {"results":str(ex)} if not new_req: self.logger.error('Could not save newly created request to database') return {"results":False} # finalize and make persistent db.update(creq.json()) rdb.save(new_req) return {"results":True}
def submit_request(self, prepid, run_test_path): mcm_request = request(self.request_db.get(prepid)) # check if the request should be validated as part of a chain for chain_prepid in mcm_request.get_attribute('member_of_chain'): mcm_chain = chained_request( self.chained_request_db.get(chain_prepid)) if mcm_chain.get_attribute( 'validate') and prepid in mcm_chain.get_attribute( 'chain')[mcm_chain.get_attribute('step'):]: return {} aux_validation = mcm_request.get_attribute(self.DOC_VALIDATION) to_write = mcm_request.get_setup_file(run=True, do_valid=True, for_validation=True) file_name = self.TEST_FILE_NAME % prepid if not self.create_test_file(to_write, run_test_path, file_name): mcm_request.set_attribute(self.DOC_VALIDATION, aux_validation) mcm_request.test_failure( message= 'There was a problem while creating the file for prepid: %s' % (mcm_request.get_attribute('prepid')), what='Validation run test', rewind=True) return {} timeout = mcm_request.get_timeout() memory = mcm_request.get_attribute("memory") threads = mcm_request.get_core_num() self.create_htcondor_config_file(run_test_path, prepid, timeout, memory, threads, [prepid]) job_info = self.execute_command_submission(prepid, run_test_path) if 'error' in job_info: mcm_request.test_failure(message=job_info['error'], what='Validation run test', rewind=True) return {} job_info[self.DOC_REV] = mcm_request.json()[self.DOC_REV] job_info[self.DOC_VALIDATION] = mcm_request.get_attribute( self.DOC_VALIDATION ) # this field change when calling request.get_setup_file, to be propagated in case of success return job_info
def delete_request(self, crid): crdb = database('chained_requests') rdb = database('requests') mcm_cr = chained_request(crdb.get(crid)) # get all objects mcm_r_s = [] for (i, rid) in enumerate(mcm_cr.get_attribute('chain')): mcm_r = request(rdb.get(rid)) # this is not a valid check as it is allowed to remove a chain around already running requests # if mcm_r.get_attribute('status') != 'new': # return {"results":False,"message" : "the request %s part of the chain %s for action %s is not in new status"%( mcm_r.get_attribute('prepid'), # crid, # mcm_a.get_attribute('prepid'))} in_chains = mcm_r.get_attribute('member_of_chain') in_chains.remove(crid) self.logger.debug("Removing ChainAction member_of_chain: %s to request: %s" % ( mcm_cr.get_attribute("prepid"), mcm_r.get_attribute('prepid'))) mcm_r.set_attribute('member_of_chain', in_chains) if i == 0: if len(in_chains) == 0 and mcm_r.get_attribute('status') != 'new': return {"results": False, "message": "the request %s, not in status new, at the root of the chain will not be chained anymore" % rid} else: if len(in_chains) == 0: return {"results": False, "message": "the request %s, not at the root of the chain will not be chained anymore" % rid} mcm_r.update_history({'action': 'leave', 'step': crid}) mcm_r_s.append(mcm_r) if mcm_cr.get_attribute('action_parameters')['flag']: return { "results": False, "message": "the action for %s is not disabled" % (crid) } # then save all changes for mcm_r in mcm_r_s: if not rdb.update(mcm_r.json()): return {"results": False, "message": "Could not save request " + mcm_r.get_attribute('prepid')} return {"results": crdb.delete(crid)}
def get(self, chained_request_id): """ Does a soft reset to all relevant request in the chain """ crdb = database('chained_requests') rdb = database('requests') mcm_cr = chained_request(crdb.get(chained_request_id)) for rid in reversed(mcm_cr.get_attribute('chain')[:mcm_cr.get_attribute('step') + 1]): # from the current one to the first one REVERSED mcm_r = request(rdb.get(rid)) try: mcm_r.reset(hard=False) except Exception as e: return {'prepid': chained_request_id, 'results': False, 'message': str(e)} mcm_r.reload() mcm_cr = chained_request(crdb.get(chained_request_id)) mcm_cr.set_attribute('step', max(0, mcm_cr.get_attribute('chain').index(rid) - 1)) mcm_cr.reload() return {'prepid': chained_request_id, 'results': True}
def inspect_priority(self, forChains=None): ##JR: until put in the proper place chains=self.get_attribute('chains') crdb = database('chained_requests') okay = True for inCC in chains: ### this is the old convention #if 'flag' in chains[inCC] and chains[inCC]['flag']: # if 'chains' in chains[inCC]: # for acr in chains[inCC]['chains']: # if forChains and not acr in forChains: continue # cr=chained_request(crdb.get(acr)) # cc=cr.get_attribute('member_of_campaign') # #if 'block_number' in chains[cc] and chains[cc]['block_number']: # if chains[cc]['block_number']: # cr.set_priority(chains[cc]['block_number']) # self.logger.log('Set priority block %s to %s'%(chains[cc]['block_number'],cr.get_attribute('prepid'))) # else: # self.logger.error('Could not set block %s to %s'%(chains[cc]['block_number'],cr.get_attribute('prepid'))) ## new convention if 'chains' in chains[inCC] and type(chains[inCC]['chains'])==dict: for acr in chains[inCC]['chains']: if forChains and not acr in forChains: continue bn=chains[inCC]['chains'][acr]['block_number'] cr=chained_request(crdb.get(acr)) if bn: self.logger.log('Set priority block %s to %s'%( bn , acr)) if not cr.set_priority( bn ): okay = False else: self.logger.error('Could not set block %s to %s'%( bn , acr)) rd = database('requests') if rd.document_exists(self.get_attribute('prepid')): r= request(rd.get(self.get_attribute('prepid'))) self.set_attribute('dataset_name',r.get_attribute('dataset_name')) return okay
def internal_run(self): if not self.lock.acquire(blocking=False): self.logger.error( "Could not acquire lock for ChainRequestInjector. prepid %s" % (self.prepid)) return False try: crdb = database('chained_requests') rdb = database('requests') batch_name = None if not crdb.document_exists(self.prepid): # it's a request actually, pick up all chains containing it mcm_r = rdb.get(self.prepid) # mcm_crs = crdb.query(query="root_request==%s"% self.prepid) ## not only when its the root of mcm_crs = crdb.query(query="contains==%s" % self.prepid) task_name = 'task_' + self.prepid batch_type = 'Task_' + mcm_r['member_of_campaign'] else: mcm_crs = [crdb.get(self.prepid)] current_step_prepid = mcm_crs[0]['chain'][mcm_crs[0]['step']] mcm_request = rdb.get(current_step_prepid) task_name = 'task_' + current_step_prepid batch_type = 'Task_' + mcm_request['member_of_campaign'] if len(mcm_crs) == 0: return False mcm_rs = [] # upload all config files to config cache, with "configuration economy" already implemented for cr in mcm_crs: mcm_cr = chained_request(cr) chain = mcm_cr.get_attribute( 'chain')[mcm_cr.get_attribute('step'):] for request_prepid in chain: mcm_rs.append(request(rdb.get(request_prepid))) if self.check_approval and mcm_rs[-1].get_attribute( 'approval') != 'submit': message = 'requests %s is in "%s"/"%s" status/approval, requires "approved"/"submit"' % ( request_prepid, mcm_rs[-1].get_attribute('status'), mcm_rs[-1].get_attribute('approval')) self.logger.error(message) subject = '%s injection failed' % mcm_cr.get_attribute( 'prepid') notification( subject, message, [], group=notification.CHAINED_REQUESTS, action_objects=[mcm_cr.get_attribute('prepid')], object_type='chained_requests', base_object=mcm_cr) mcm_cr.notify(subject, message) return False if mcm_rs[-1].get_attribute('status') != 'approved': # change the return format to percolate the error message message = 'requests %s in in "%s"/"%s" status/approval, requires "approved"/"submit"' % ( request_prepid, mcm_rs[-1].get_attribute('status'), mcm_rs[-1].get_attribute('approval')) self.logger.error(message) subject = '%s injection failed' % mcm_cr.get_attribute( 'prepid') notification( subject, message, [], group=notification.CHAINED_REQUESTS, action_objects=[mcm_cr.get_attribute('prepid')], object_type='chained_requests', base_object=mcm_cr) mcm_cr.notify(subject, message) return False uploader = ConfigMakerAndUploader( prepid=request_prepid, lock=locker.lock(request_prepid)) if not uploader.internal_run(): message = 'Problem with uploading the configuration for request %s' % ( request_prepid) notification( 'Configuration upload failed', message, [], group=notification.CHAINED_REQUESTS, action_objects=[mcm_cr.get_attribute('prepid')], object_type='chained_requests', base_object=mcm_cr) mcm_cr.notify('Configuration upload failed', message) self.logger.error(message) return False mcm_r = mcm_rs[-1] batch_name = BatchPrepId().next_batch_id(batch_type, create_batch=True) semaphore_events.increment(batch_name) self.logger.error('found batch %s' % batch_name) with ssh_executor(server='vocms081.cern.ch') as ssh: cmd = self.make_command(mcm_r) self.logger.error('prepared command %s' % cmd) # modify here to have the command to be executed _, stdout, stderr = ssh.execute(cmd) output = stdout.read() error = stderr.read() self.logger.info(output) self.logger.info(error) injected_requests = [ l.split()[-1] for l in output.split('\n') if l.startswith('Injected workflow:') ] if not injected_requests: self.injection_error( 'Injection has succeeded but no request manager names were registered. Check with administrators. \nOutput: \n%s\n\nError: \n%s' % (output, error), mcm_rs) return False # what gets printed into the batch object added_requests = [] once = set() for mcm_r in mcm_rs: if mcm_r.get_attribute('prepid') in once: continue once.add(mcm_r.get_attribute('prepid')) added = [{ 'name': app_req, 'content': { 'pdmv_prep_id': mcm_r.get_attribute('prepid') } } for app_req in injected_requests] added_requests.extend(added) # edit the batch object with locker.lock(batch_name): bdb = database('batches') bat = batch(bdb.get(batch_name)) bat.add_requests(added_requests) bat.update_history({ 'action': 'updated', 'step': task_name }) bat.reload() # reload the content of all requests as they might have changed already added = [{ 'name': app_req, 'content': { 'pdmv_prep_id': task_name } } for app_req in injected_requests] seen = set() for cr in mcm_crs: mcm_cr = chained_request(cr) chain = mcm_cr.get_attribute( 'chain')[mcm_cr.get_attribute('step'):] message = "" for rn in chain: if rn in seen: continue # don't do it twice seen.add(rn) mcm_r = request(rdb.get(rn)) message += mcm_r.textified() message += "\n\n" mcm_r.set_attribute('reqmgr_name', added) mcm_r.update_history({ 'action': 'inject', 'step': batch_name }) if not self.check_approval: mcm_r.set_attribute('approval', 'submit') # set the status to submitted mcm_r.set_status( step=mcm_r._json_base__status.index('submitted'), with_notification=False) mcm_r.reload() mcm_cr.set_attribute('last_status', mcm_r.get_attribute('status')) # re-get the object mcm_cr = chained_request(crdb.get(cr['prepid'])) # take care of changes to the chain mcm_cr.update_history({ 'action': 'inject', 'step': batch_name }) mcm_cr.set_attribute( 'step', len(mcm_cr.get_attribute('chain')) - 1) mcm_cr.set_attribute('status', 'processing') subject = 'Injection succeeded for %s' % mcm_cr.get_attribute( 'prepid') notification( subject, message, [], group=notification.CHAINED_REQUESTS, action_objects=[mcm_cr.get_attribute('prepid')], object_type='chained_requests', base_object=mcm_cr) mcm_cr.notify(subject, message) mcm_cr.reload() return True except Exception: self.injection_error( "Error with injecting chains for %s :\n %s" % (self.prepid, traceback.format_exc()), []) finally: # we decrement batch id and release lock on prepid+lower semaphore if batch_name: # ditry thing for now. Because batch name can be None for certain use-cases in code above semaphore_events.decrement(batch_name) self.lock.release() self.queue_lock.release()
def submit_chain(self, prepid, run_test_path): mcm_chained_request = chained_request( self.chained_request_db.get(prepid)) except_requests = [] reset = False # If a request of a chain was singly submmited to validation and then somebody reseted it, we will find it here for request_prepid in mcm_chained_request.get_attribute( 'chain')[mcm_chained_request.get_attribute('step'):]: if request_prepid in self.submmited_prepids_set: except_requests.append(request_prepid) reset = True if reset: message = "Requests %s of the chain %s are already in validation" % ( except_requests, prepid) self.logger.error(message) mcm_chained_request.reset_requests(message, except_requests=except_requests) return {} to_write = mcm_chained_request.get_setup(run=True, validation=True, for_validation=True) file_name = self.TEST_FILE_NAME % prepid if not self.create_test_file(to_write, run_test_path, file_name): mcm_chained_request.reset_requests( 'There was a problem while creating the file for prepid: %s' % (mcm_chained_request.get_attribute('prepid'))) return {} requests_in_chain = {} for request_prepid in mcm_chained_request.get_attribute( 'chain')[mcm_chained_request.get_attribute('step'):]: mcm_request = request(self.request_db.get(request_prepid)) if not mcm_request.is_root and 'validation' not in mcm_request._json_base__status: # only root or possible root requests break status = mcm_request.get_attribute('status') approval = mcm_request.get_attribute('approval') if status != 'new' or approval != 'validation': message = "The request %s of chain %s is in status: %s approval: %s" % ( request_prepid, prepid, status, approval) self.logger.error(message) mcm_chained_request.reset_requests(message) return {} requests_in_chain[request_prepid] = mcm_request.json()[ self.DOC_REV] if not len(requests_in_chain): message = 'No requests to be validated in chain: %s' % prepid self.logger.info(message) mcm_chained_request.reset_requests(message) return {} timeout, memory, threads = mcm_chained_request.get_timeout_memory_threads( ) self.create_htcondor_config_file(run_test_path, prepid, timeout, memory, threads, list(requests_in_chain.iterkeys())) job_info = self.execute_command_submission(prepid, run_test_path) if 'error' in job_info: mcm_chained_request.reset_requests(job_info['error']) return {} job_info[self.CHAIN_REQUESTS] = requests_in_chain return job_info
def process_finished_chain_success(self, prepid, doc_info, job_out, error_out, log_out): mcm_chained_request = chained_request( self.chained_request_db.get(prepid)) requests_in_chain = [] success = True failed_request_prepid = None for request_prepid, doc_rev in doc_info[ self.CHAIN_REQUESTS].iteritems(): mcm_request = request(self.request_db.get(request_prepid)) # Increase counters for all requests, but save error message only for the first failed request if success and doc_rev != mcm_request.json()[self.DOC_REV]: message = 'The request %s in the chain %s has changed during the run test procedure, preventing from being saved' % ( request_prepid, prepid) success = False failed_request_prepid = request_prepid mcm_request.inc_validations_counter() mcm_request.reload() if not success: mcm_chained_request.reset_requests( message, notify_one=failed_request_prepid) return for request_prepid, doc_rev in doc_info[ self.CHAIN_REQUESTS].iteritems(): self.logger.info('Processing request %s in chain %s' % (request_prepid, prepid)) mcm_request = request(self.request_db.get(request_prepid)) success = True path = self.test_directory_path + prepid + '/' try: success, error = mcm_request.pickup_all_performance(path) error = 'Error:\n%s\n Error out:\n%s\n' % (error, error_out) except request.WrongTimeEvent as ex: self.logger.error('Exception: %s' % (ex)) error = 'Exception:\n%s\n' % (ex) success = False retry_validation = self.process_request_wrong_time_event( mcm_request, prepid) if retry_validation: return if not success: message = 'Error while picking up all the performance for request %s of chain %s: \n Error:\n%s\n Job out:\n%s\n Error out: \n%s\n Log out: \n%s\n' % ( request_prepid, prepid, error, job_out, error_out, log_out) mcm_chained_request.reset_requests(message, notify_one=request_prepid) return requests_in_chain.append(mcm_request) for mcm_request in requests_in_chain: mcm_request.set_status(with_notification=True) if not self.request_db.update(mcm_request.json()): request_prepid = mcm_request.get_attribute('prepid') message = "The request %s of chain %s could not be saved after the runtest procedure" % ( request_prepid, prepid) self.logger.error(message) # reset it and keep saving requests mcm_request.test_failure(message=message, what='Chain validation run test', rewind=True) mcm_chained_request.reload( save_current=False ) # setting new requests status change the chain object mcm_chained_request.set_attribute('validate', 0) if not self.chained_request_db.update(mcm_chained_request.json()): message = 'Problem saving changes in chain %s, set validate = False ASAP!' % prepid self.logger.error(message) notification( 'Chained validation run test', message, [], group=notification.CHAINED_REQUESTS, action_objects=[mcm_chained_request.get_attribute('prepid')], object_type='chained_requests', base_object=mcm_chained_request) mcm_chained_request.notify('Chained validation run test', message) return self.logger.info('Validation job for prepid %s SUCCESSFUL!!!' % prepid)
def generate_request(self, aid, reserve=False, with_notify=True, special=False): adb = database('actions') ccdb = database('chained_campaigns') crdb = database('chained_requests') rdb = database('requests') if not adb.document_exists(aid): return {'results': False, 'message': '%s does not exist' % (aid)} self.logger.log( 'Generating all selected chained_requests for action %s' % (aid)) mcm_a = action(adb.get(aid)) chains = mcm_a.get_attribute('chains') hasChainsChanged = False new_chains = [] for cc in chains: if 'flag' in chains[cc] and chains[cc]['flag']: ## in the new convention, that means that something needs to be created mcm_cc = chained_campaign(ccdb.get(cc)) new_cr = mcm_cc.generate_request(aid) if not 'chains' in chains[cc]: chains[cc]['chains'] = {} chains[cc]['chains'][new_cr['prepid']] = {} to_transfer = ['flag', 'threshold', 'staged', 'block_number'] for item in to_transfer: if item in chains[cc]: chains[cc]['chains'][ new_cr['prepid']][item] = chains[cc][item] chains[cc].pop(item) hasChainsChanged = True ## get the root request req = request(json_input=rdb.get(aid)) new_cr['last_status'] = req.get_attribute('status') if new_cr['last_status'] in ['submitted', 'done']: new_cr['status'] = 'processing' if special: new_cr['approval'] = 'none' if not crdb.update(new_cr): return { 'results': False, 'message': 'could not save %s' % (new_cr['prepid']) } ## update the cc history ccdb.update(mcm_cc.json()) new_chains.append(new_cr['prepid']) # then let the root request know that it is part of a chained request inchains = req.get_attribute('member_of_chain') inchains.append(new_cr['prepid']) inchains.sort() req.set_attribute('member_of_chain', list(set(inchains))) req.update_history({ 'action': 'join chain', 'step': new_cr['prepid'] }) if with_notify: req.notify( "Request {0} joined chain".format( req.get_attribute('prepid')), "Request {0} has successfully joined chain {1}".format( req.get_attribute('prepid'), new_cr['prepid']), Nchild=0, accumulate=True) mcm_a.update_history({ 'action': 'add', 'step': new_cr['prepid'] }) rdb.update(req.json()) if hasChainsChanged: #the chains parameter might have changed mcm_a.set_attribute('chains', chains) adb.update(mcm_a.json()) #and set priorities properly to all requests concerned priority_set = mcm_a.inspect_priority(forChains=new_chains) ## do the reservation of the whole chain ? res = [] if reserve: for cr in new_chains: mcm_cr = chained_request(crdb.get(cr)) res.append(mcm_cr.reserve()) crdb.update(mcm_cr.json()) if priority_set: return {"results": True, "prepid": mcm_a.get_attribute('prepid')} else: return { "results": False, "prepid": mcm_a.get_attribute('prepid'), "message": "Priorities not set properly" }
def flow_to_next_step(self, input_dataset='', block_black_list=None, block_white_list=None, check_stats=True, reserve=False): if not block_white_list: block_white_list = [] if not block_black_list: block_black_list = [] self.logger.log('Flowing chained_request %s to next step...' % (self.get_attribute('_id'))) if not self.get_attribute('chain'): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'chained_request %s has got no root' % (self.get_attribute('_id'))) # check on the approval of the chained request before all ## let it be flowing regardless #if self.get_attribute('approval') == 'none': # raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), # 'The approval of the chained request is none, and therefore flow cannot happen') #this operation requires to access all sorts of objects rdb = database('requests') cdb = database('campaigns') ccdb = database('chained_campaigns') crdb = database('chained_requests') fdb = database('flows') adb = database('actions') l_type = locator() current_step = len(self.get_attribute( 'chain')) - 1 if reserve else self.get_attribute('step') current_id = self.get_attribute('chain')[current_step] next_step = current_step + 1 if not rdb.document_exists(current_id): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'the request %s does not exist' % current_id) current_request = request(rdb.get(current_id)) current_campaign = campaign( cdb.get(current_request.get_attribute('member_of_campaign'))) if not ccdb.document_exists(self.get_attribute('member_of_campaign')): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'the chain request %s is member of %s that does not exist' % (self.get_attribute('_id'), self.get_attribute('member_of_campaign'))) mcm_cc = ccdb.get(self.get_attribute('member_of_campaign')) if next_step >= len(mcm_cc['campaigns']): if reserve: return False raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'chained_campaign %s does not allow any further flowing.' % (self.get_attribute('member_of_campaign'))) if not reserve: ## is the current request in the proper approval allowed_request_approvals = ['submit'] if current_request.get_attribute( 'approval') not in allowed_request_approvals: raise self.NotApprovedException( current_request.get_attribute('prepid'), current_request.get_attribute('approval'), allowed_request_approvals) ## is the current request in the proper status allowed_request_statuses = ['submitted', 'done'] if current_request.get_attribute( 'status') not in allowed_request_statuses: raise self.NotInProperStateException( current_request.get_attribute('prepid'), current_request.get_attribute('status'), allowed_request_statuses) original_action_id = self.get_attribute('chain')[0] original_action_item = self.retrieve_original_action_item( adb, original_action_id) ## what is the campaign to go to next and with which flow (next_campaign_id, flow_name) = mcm_cc['campaigns'][next_step] if not fdb.document_exists(flow_name): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'The flow %s does not exist' % flow_name) mcm_f = flow(fdb.get(flow_name)) if not 'sequences' in mcm_f.get_attribute('request_parameters'): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'The flow %s does not contain sequences information.' % (flow_name)) if not cdb.document_exists(next_campaign_id): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'The next campaign %s does not exist' % next_campaign_id) next_campaign = campaign(cdb.get(next_campaign_id)) if len(next_campaign.get_attribute('sequences')) != len( mcm_f.get_attribute('request_parameters')['sequences']): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'the sequences changes in flow %s are not consistent with the next campaign %s' % (flow_name, next_campaign_id)) if next_campaign.get_attribute( 'energy') != current_campaign.get_attribute('energy'): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'cannot flow any further. Request {0} has inconsistent energy.' .format(next_campaign.get_attribute("prepid"))) if not next_campaign.is_release_greater_or_equal_to( current_campaign.get_attribute('cmssw_release')): raise self.ChainedRequestCannotFlowException( self.get_attribute("_id"), 'cannot flow any further. Request {0} has lower release version.' .format(next_campaign.get_attribute("prepid"))) if next_campaign.get_attribute('type') == 'MCReproc' and ( not 'time_event' in mcm_f.get_attribute('request_parameters')): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'the flow is getting into a MCReproc campaign but not time per event is specified' ) if next_campaign.get_attribute('type') == 'MCReproc' and ( not 'size_event' in mcm_f.get_attribute('request_parameters')): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'the flow is getting into a MCReproc campaign but not size per event is specified' ) ## check that it is allowed to flow allowed_flow_approvals = ['flow', 'submit'] ###### cascade of checks """ if not reserve and not mcm_f.get_attribute('approval') in allowed_flow_approvals: raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'The flow (%s) is not in proper approval state (%s)'%( mcm_f.get_attribute('prepid'), mcm_f.get_attribute('approval'))) if not reserve and not self.get_attribute('approval') in allowed_flow_approvals: raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'The chained request (%s) is not in the proper approval state (%s)'% ( self.get_attribute('_id'), self.get_attribute('approval'))) """ if not reserve and not mcm_f.get_attribute( 'approval') in allowed_flow_approvals: if not self.get_attribute('approval') in allowed_flow_approvals: raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'Neither the flow (%s) nor the chained request (%s) approvals allow for flowing' % (mcm_f.get_attribute('approval'), self.get_attribute('approval'))) if next_campaign.get_attribute('status') == 'stopped': raise self.CampaignStoppedException(next_campaign_id) #what is going to be the required number of events for the next request #update the stats to its best if not reserve: current_request.get_stats() next_total_events = current_request.get_attribute( 'completed_events') ## get the original expected events and allow a margin of 5% less statistics statistics_fraction = settings().get_value('statistics_fraction') current_eff_error = 1. - current_request.get_efficiency_error() statistics_fraction = min(statistics_fraction, current_eff_error) completed_events_to_pass = int( current_request.get_attribute('total_events') * statistics_fraction) notify_on_fail = True ## to be tuned according to the specific cases if current_request.get_attribute('completed_events') <= 0: raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'The number of events completed is negative or null') else: allowed_request_statuses = ['done'] ## determine if this is a root -> non-root transition to potentially apply staged number at_a_transition = (current_campaign.get_attribute('root') != 1 and next_campaign.get_attribute('root') == 1) if ('staged' in original_action_item or 'threshold' in original_action_item) and at_a_transition: allowed_request_statuses.append('submitted') ##check status if not current_request.get_attribute( 'status') in allowed_request_statuses: raise self.NotInProperStateException( current_request.get_attribute('prepid'), current_request.get_attribute('status'), allowed_request_statuses) ##special check at transition that the statistics is good enough if at_a_transition: # at a root -> non-root transition only does the staged/threshold functions ! if 'staged' in original_action_item: next_total_events = int(original_action_item['staged']) completed_events_to_pass = next_total_events if 'threshold' in original_action_item: next_total_events = int( current_request.get_attribute('total_events') * float(original_action_item['threshold'] / 100.)) completed_events_to_pass = next_total_events if check_stats and ( current_request.get_attribute('completed_events') < completed_events_to_pass): if notify_on_fail: current_request.notify( 'Flowing %s with not enough statistics' % (current_request.get_attribute('prepid')), 'For the request %s, the completed statistics %s is not enough to fullfill the requirement to the next level : need at least %s in chain %s \n\n Please report to the operation HN or at the next MccM what action should be taken.\n\n %srequests?prepid=%s\n%schained_requests?contains=%s\n%schained_requests?prepid=%s ' % (current_request.get_attribute('prepid'), current_request.get_attribute('completed_events'), completed_events_to_pass, self.get_attribute('prepid'), l_type.baseurl(), current_request.get_attribute('prepid'), l_type.baseurl(), current_request.get_attribute('prepid'), l_type.baseurl(), self.get_attribute('prepid')), accumulate=True) raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'The number of events completed (%s) is not enough for the requirement (%s)' % (current_request.get_attribute('completed_events'), completed_events_to_pass)) ## select what is to happened : [create, patch, use] next_id = None approach = None next_request = None if next_step != len(self.get_attribute('chain')): #not at the end next_id = self.get_attribute('chain')[next_step] if not rdb.document_exists(next_id): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'The next request (%s) according to the step (%s) does not exist' % (next_id, next_step)) next_request = request(rdb.get(next_id)) if next_request.get_attribute('status') == 'new': #most likely a rewind + resub approach = 'patch' else: ##this is always the case in chains reserved from existing things: so use the next request approach = 'use' #raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), #'This should never happen. (%s) is next according to step (%s), but is not in new status (%s)' % ( #next_id, next_step, next_request.get_attribute('status'))) else: ## look in *other* chained campaigns whether you can suck in an existing request ## look up all chained requests that start from the same root request ## remove <pwg>-chain_ and the -serial number, replacing _ with a . toMatch = '.'.join( self.get_attribute('prepid').split('_')[1:][0:next_step + 1]).split('-')[0] ## make sure they get ordered by prepid related_crs = sorted(crdb.queries( ['root_request==%s' % original_action_id]), key=lambda cr: cr['prepid']) vetoed_last = [] for existing_cr in related_crs: ## exclude itself if existing_cr['prepid'] == self.get_attribute('prepid'): continue ## prevent from using a request from within the same exact chained_campaigns if existing_cr['member_of_campaign'] == self.get_attribute( 'member_of_campaign'): mcm_cr = chained_request(crdb.get(existing_cr['prepid'])) if len(mcm_cr.get_attribute('chain')) > next_step: ## one existing request in the very same chained campaign has already something used, make sure it is not going to be used vetoed_last.append( mcm_cr.get_attribute('chain')[next_step]) continue else: continue for existing_cr in related_crs: ## exclude itself if existing_cr['prepid'] == self.get_attribute('prepid'): continue ## prevent from using a request from within the same exact chained_campaigns if existing_cr['member_of_campaign'] == self.get_attribute( 'member_of_campaign'): continue truncated = '.'.join( existing_cr['prepid'].split('_')[1:][0:next_step + 1]).split('-')[0] self.logger.error('to match : %s , this one %s' % (toMatch, truncated)) if truncated == toMatch: #we found a chained request that starts with all same steps mcm_cr = chained_request(crdb.get(existing_cr['prepid'])) if len(mcm_cr.get_attribute('chain')) <= next_step: #found one, but it has not enough content either continue if mcm_cr.get_attribute('chain')[next_step] in vetoed_last: continue next_id = mcm_cr.get_attribute('chain')[next_step] break if next_id: approach = 'use' else: approach = 'create' if approach == 'create': from rest_api.RequestPrepId import RequestPrepId next_id = RequestPrepId().next_prepid( current_request.get_attribute('pwg'), next_campaign_id) next_request = request(rdb.get(next_id)) request.transfer(current_request, next_request) self.request_join(next_request) elif approach == 'use': ## there exists a request in another chained campaign that can be re-used here. # take this one. advance and go on next_request = request(rdb.get(next_id)) if not reserve: self.set_attribute('step', next_step) self.set_attribute('last_status', next_request.get_attribute('status')) self.update_history({'action': 'flow', 'step': str(next_id)}) self.set_attribute('status', 'processing') if not self.get_attribute("prepid") in next_request.get_attribute( "member_of_chain"): ## register the chain to the next request self.request_join(next_request) saved = rdb.update(next_request.json()) if not saved: raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'Unable to save %s with updated member_of_chains' % next_id) return True elif approach == 'patch': ## there exists already a request in the chain (step!=last) and it is usable for the next stage next_request = request(next_campaign.add_request(rdb.get(next_id))) ## propagate again some of the fields of the previous request. request.transfer(current_request, next_request) else: raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'Unrecognized approach %s' % approach) #current_request -> next_request #current_campaign -> next_campaign ##determine whether we have an input dataset for the next request if len(current_request.get_attribute('reqmgr_name')): last_wma = current_request.get_attribute('reqmgr_name')[-1] if 'content' in last_wma and 'pdmv_dataset_name' in last_wma[ 'content']: input_dataset = last_wma['content']['pdmv_dataset_name'] else: statsDB = database('stats', url='http://cms-pdmv-stats.cern.ch:5984/') if statsDB.document_exists(last_wma['name']): latestStatus = statsDB.get(last_wma['name']) input_dataset = latestStatus['pdmv_dataset_name'] if input_dataset: next_request.set_attribute('input_dataset', input_dataset) ## set blocks restriction if any if block_black_list: next_request.set_attribute('block_black_list', block_black_list) if block_white_list: next_request.set_attribute('block_white_list', block_white_list) ## register the flow to the request next_request.set_attribute('flown_with', flow_name) ##assemble the campaign+flow => request request.put_together(next_campaign, mcm_f, next_request) if not reserve: #already taking stage and threshold into account next_request.set_attribute('total_events', next_total_events) next_request.update_history({ 'action': 'flow', 'step': self.get_attribute('prepid') }) request_saved = rdb.save(next_request.json()) if not request_saved: raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'Could not save the new request %s' % (next_request.get_attribute('prepid'))) ## inspect priority self.set_priority(original_action_item['block_number']) if not reserve: # sync last status self.set_attribute('last_status', next_request.get_attribute('status')) # we can only be processing at this point self.set_attribute('status', 'processing') # set to next step self.set_attribute('step', next_step) self.update_history({ 'action': 'flow', 'step': next_request.get_attribute('prepid') }) if not reserve: notification_subject = 'Flow for request %s in %s' % ( current_request.get_attribute('prepid'), next_campaign_id) notification_text = 'The request %s has been flown within:\n \t %s \n into campaign:\n \t %s \n using:\n \t %s \n creating the new request:\n \t %s \n as part of:\n \t %s \n and from the produced dataset:\n %s \n\n%srequests?prepid=%s \n%srequests?prepid=%s \n' % ( current_request.get_attribute('prepid'), self.get_attribute('member_of_campaign'), next_campaign_id, flow_name, next_request.get_attribute('prepid'), self.get_attribute('prepid'), next_request.get_attribute('input_dataset'), l_type.baseurl(), current_request.get_attribute('prepid'), l_type.baseurl(), next_request.get_attribute('prepid')) current_request.notify(notification_subject, notification_text, accumulate=True) else: notification_subject = 'Reservation of request {0}'.format( next_request.get_attribute('prepid')) notification_text = 'The request {0} of campaign \n\t{2}\nhas been reserved as part of \n\t{1}\nas the next step for {4}\n\n{3}requests?prepid={4}\n{5}requests?prepid={6}\n'.format( next_request.get_attribute('prepid'), self.get_attribute('prepid'), next_campaign_id, l_type.baseurl(), current_request.get_attribute('prepid'), l_type.baseurl(), next_request.get_attribute('prepid'), ) next_request.notify(notification_subject, notification_text, accumulate=True) return True
def get_actors(self, N=-1, what='author_username', Nchild=-1): rdb = database('requests') last_r = request(rdb.get(self.get_attribute('chain')[-1])) return last_r.get_actors(N, what, Nchild)
def multiple_inspect(self, cid, in_statuses=['submitted', 'approved']): clist = list(set(cid.rsplit(','))) res = [] rdb = database('requests') index = 0 self.logger.error( "Chain inspect begin. Number of chains to be inspected: %s" % (len(clist))) try: while len(clist) > index: yield dumps( {"current cr element": "%s/%s" % (index, len(clist))}, indent=2) query = rdb.construct_lucene_complex_query([ ('member_of_campaign', { 'value': clist[index:index + 1] }), ('status', { 'value': in_statuses }) ]) ##do another loop over the requests themselves req_page = 0 request_res = rdb.full_text_search('search', query, page=req_page) while len(request_res) > 0: self.logger.info("inspecting single requests. page: %s" % (req_page)) for r in request_res: self.logger.info("running inspect on request: %s" % (r['prepid'])) mcm_r = request(r) if mcm_r: #making it as a stream yield dumps(mcm_r.inspect(), indent=4) else: #making it as a stream yield dumps( { "prepid": r, "results": False, 'message': '%s does not exist' % (r) }, indent=4) req_page += 1 request_res = rdb.full_text_search('search', query, page=req_page) time.sleep(0.5) index += 1 time.sleep(1) except Exception as e: subject = "Exception while inspecting request " message = "Request: %s \n %s traceback: \n %s" % ( mcm_r.get_attribute('prepid'), str(e), traceback.format_exc()) self.logger.error(subject + message) notification(subject, message, [], group=notification.REQUEST_OPERATIONS, action_objects=[mcm_r.get_attribute('prepid')], object_type='requests', base_object=mcm_r) mcm_r.notify(subject, message, accumulate=True) self.logger.info("Campaign inspection finished!")
def get(self, chained_request_id): """ Provide the taskchain dictionnary for uploading to request manager """ kwargs = self.parser.parse_args() crdb = database('chained_requests') rdb = database('requests') settingsDB = database('settings') __DT_prio = settingsDB.get('datatier_input')["value"] def tranform_to_step_chain(wma_dict, total_time_evt, total_size_evt): # replace Task -> Step in inside dictionaries for task_num in range(wma_dict["TaskChain"]): for elem in wma_dict["Task%s" % (task_num + 1)]: if "Task" in elem: wma_dict["Task%s" % (task_num + 1)][elem.replace( "Task", "Step")] = wma_dict["Task%s" % (task_num + 1)].pop(elem) # we later add the global fields del (wma_dict["Task%s" % (task_num + 1)]["TimePerEvent"]) del (wma_dict["Task%s" % (task_num + 1)]["SizePerEvent"]) # we do same replacement on top level for el in wma_dict: if wma_dict[el].__class__ == str and "task" in wma_dict[el]: wma_dict[el] = wma_dict[el].replace("task", "step") if "Task" in el: wma_dict[el.replace("Task", "Step")] = wma_dict.pop(el) wma_dict["RequestType"] = "StepChain" # as of 2017-05 StepChain needs these as sum of internal Tasks wma_dict["TimePerEvent"] = total_time_evt wma_dict["SizePerEvent"] = total_size_evt return wma_dict if not crdb.document_exists(chained_request_id): # it's a request actually, pick up all chains containing it mcm_r = rdb.get(chained_request_id) # mcm_crs = crdb.query(query="root_request==%s"% chained_request_id) ## not only when its the root of mcm_crs = crdb.query(query="contains==%s" % chained_request_id) task_name = 'task_' + chained_request_id else: mcm_crs = [crdb.get(chained_request_id)] # here name should be task_chain's[curr_step] request_prepid # so it would be task_prepid-of-current-request same as in top __req_id = mcm_crs[0]['chain'][mcm_crs[0]['step']] task_name = 'task_' + __req_id if len(mcm_crs) == 0: return {} tasktree = {} ignore_status = False __total_time_evt = 0 __total_size_evt = 0 if kwargs['scratch'] is not None: ignore_status = True veto_point = None if kwargs['upto'] is not None: veto_point = kwargs['upto'] __chains_type = [] for mcm_cr in mcm_crs: __chains_type.append(mcm_cr["chain_type"]) starting_point = mcm_cr['step'] if ignore_status: starting_point = 0 for (ir, r) in enumerate(mcm_cr['chain']): if (ir < starting_point): continue # ad no task for things before what is already done if veto_point and (ir > veto_point): continue mcm_r = request(rdb.get(r)) if mcm_r.get_attribute( 'status') == 'done' and not ignore_status: continue if r not in tasktree: tasktree[r] = {'next': [], 'dict': [], 'rank': ir} base = ir == 0 and mcm_r.get_wmagent_type() in [ 'MonteCarlo', 'LHEStepZero' ] depend = ( ir > starting_point ) # all the ones later than the starting point depend on a previous task if ir < (len(mcm_cr['chain']) - 1): tasktree[r]['next'].append(mcm_cr['chain'][ir + 1]) tasktree[r]['dict'] = mcm_r.request_to_tasks(base, depend) # if request is added to tasktree, we save global sums for StepChains __total_time_evt += mcm_r.get_sum_time_events() __total_size_evt += sum(mcm_r.get_attribute("size_event")) for (r, item) in tasktree.items(): # here we should generate unique list of steps+output tiers # as we iterate over requests in tasktree __uniq_tiers = [] for el in item['dict']: # map of tiers and taskID in order of steps __uniq_tiers.append((el['TaskName'], el['_output_tiers_'])) item['unique_tiers_'] = __uniq_tiers for n in item['next']: # here we should take input from datatier selection; # have a map of tiers -> taskNames and select appropriate one __input_tier = tasktree[n]['dict'][0]['_first_step_'] tModule = tName = "" if __input_tier in __DT_prio: # in case there is a possible DataTier in global_dict tModule, tName = request.do_datatier_selection( __DT_prio[__input_tier], __uniq_tiers) if tModule != "" and tName != "": tasktree[n]['dict'][0].update({ "InputFromOutputModule": tModule, "InputTask": tName }) else: # default & fallback solution tasktree[n]['dict'][0].update({ "InputFromOutputModule": item['dict'][-1]['output_'], "InputTask": item['dict'][-1]['TaskName'] }) wma = { "RequestType": "TaskChain", "Group": "ppd", "Requestor": "pdmvserv", "TaskChain": 0, "ProcessingVersion": 1, "RequestPriority": 0, "SubRequestType": "MC", # we default to 1 in multicore global "Multicore": 1 } task = 1 for (r, item) in sorted(tasktree.items(), key=lambda d: d[1]['rank']): for d in item['dict']: if d['priority_'] > wma['RequestPriority']: wma['RequestPriority'] = d['priority_'] if d['request_type_'] in ['ReDigi']: wma['SubRequestType'] = 'ReDigi' for k in d.keys(): if k.endswith('_'): d.pop(k) wma['Task%d' % task] = d task += 1 wma['TaskChain'] = task - 1 if wma['TaskChain'] == 0: return dumps({}) for item in [ 'CMSSWVersion', 'ScramArch', 'TimePerEvent', 'SizePerEvent', 'GlobalTag', 'Memory' ]: wma[item] = wma['Task%d' % wma['TaskChain']][item] # since 2016-11, processingString and AcquisitionEra is mandatory in global params wma['AcquisitionEra'] = wma['Task1']['AcquisitionEra'] wma['ProcessingString'] = wma['Task1']['ProcessingString'] wma['Campaign'] = wma['Task1']['Campaign'] wma['PrepID'] = task_name wma['RequestString'] = wma['PrepID'] if __chains_type.count("StepChain") == len(__chains_type): return dumps(tranform_to_step_chain(wma, __total_time_evt, __total_size_evt), indent=4) else: return dumps(wma, indent=4)
def internal_run(self): location = installer(self.rid, care_on_existing=False, clean_on_exit=True) try: test_script = location.location() + 'validation_run_test.sh' timeout=None with open(test_script, 'w') as there: ## one has to wait just a bit, so that the approval change operates, and the get retrieves the latest greatest _rev number #self.logger.error('Revision %s'%( self.db.get(self.rid)['_rev'])) time.sleep(10) mcm_r = request(self.db.get(self.rid)) #self.logger.error('Revision %s'%( self.db.get(self.rid)['_rev'])) ## the following does change something on the request object, to be propagated in case of success there.write(mcm_r.get_setup_file(location.location(), run=True, do_valid=True)) timeout = mcm_r.get_timeout() batch_test = batch_control(self.rid, test_script, timeout=timeout) try: success = batch_test.test() except: batch_test.log_err = traceback.format_exc() success = False if success: self.logger.log("batch_test result is %s" % success) (success,batch_test.log_err) = mcm_r.pickup_all_performance(location.location()) self.logger.error('I came all the way to here and %s (request %s)' % ( success, self.rid )) if not success: mcm_r = request(self.db.get(self.rid)) ## need to provide all the information back if settings().get_value('check_term_runlimit') and "TERM_RUNLIMIT" in batch_test.log_out: no_success_message = "LSF job was terminated after reaching run time limit.\n\n" no_success_message += "Average CPU time per event specified for request was {0} seconds. \n\n".format( mcm_r.get_attribute("time_event")) additiona_message = "Time report not found in LSF job." split_log = batch_test.log_err.split('\n') for l_id, line in izip(reversed(xrange(len(split_log))), reversed(split_log)): if "TimeReport>" in line: additiona_message = "\n".join(split_log[l_id:l_id + 12]) no_success_message += additiona_message else: no_success_message = '\t .out \n%s\n\t .err \n%s\n ' % ( batch_test.log_out, batch_test.log_err) #self.logger.error('Revision %s'%( self.db.get(self.rid)['_rev'])) # reset the content of the request mcm_r.test_failure(message=no_success_message, what='Validation run test', rewind=True) #self.logger.error('Revision %s'%( self.db.get(self.rid)['_rev'])) else: #self.logger.error('Revision %s'%( self.db.get(self.rid)['_rev'])) ## change the status with notification mcm_current = request(self.db.get(self.rid)) if mcm_current.json()['_rev'] == mcm_r.json()['_rev']: ## it's fine to push it through mcm_r.set_status(with_notification=True) saved = self.db.update(mcm_r.json()) if not saved: mcm_current.test_failure(message='The request could not be saved after the run test procedure', what='Validation run test', rewind=True) else: mcm_current.test_failure( message='The request has changed during the run test procedure, preventing from being saved', what='Validation run test', rewind=True) #self.logger.error('Revision %s'%( self.db.get(self.rid)['_rev'])) except: mess = 'We have been taken out of run_safe of runtest_genvalid for %s because \n %s \n During an un-excepted exception. Please contact support.' % ( self.rid, traceback.format_exc()) self.logger.error(mess) mcm_r = request(self.db.get(self.rid)) mcm_r.test_failure(message=mess, what='Validation run test', rewind=True) finally: location.close()
def get(self, chained_request_id): """ Perform test for chained requests """ crdb = database('chained_requests') rdb = database('requests') settingsDB = database('settings') mcm_cr = chained_request(crdb.get(chained_request_id)) if settingsDB.get('validation_stop')['value']: return { "results": False, 'message': ('validation jobs are halted to allow forthcoming mcm ' 'restart - try again later'), "prepid": chained_request_id } requires_validation = False for rid in mcm_cr.get_attribute( 'chain')[mcm_cr.get_attribute('step'):]: mcm_r = request(rdb.get(rid)) if not mcm_r.is_root and 'validation' not in mcm_r._json_base__status: # We dont care about non root request because they are not being used on chain run test break requires_validation = True if mcm_r.get_attribute('status') != 'new' or mcm_r.get_attribute( 'approval') != 'none': return { "results": False, "prepid": chained_request_id, "message": "request %s is in status %s, approval: %s" % (rid, mcm_r.get_attribute('status'), mcm_r.get_attribute('approval')) } try: mcm_r.ok_to_move_to_approval_validation(for_chain=True) mcm_r.update_history({ 'action': 'approve', 'step': 'validation' }) mcm_r.set_attribute('approval', 'validation') mcm_r.reset_validations_counter() mcm_r.reload() text = 'Within chain %s \n' % mcm_cr.get_attribute('prepid') text += mcm_r.textified() subject = 'Approval %s in chain %s for request %s' % ( 'validation', mcm_cr.get_attribute('prepid'), mcm_r.get_attribute('prepid')) notification(subject, text, [], group=notification.REQUEST_APPROVALS, action_objects=[mcm_r.get_attribute('prepid')], object_type='requests', base_object=mcm_r) mcm_r.notify(subject, text, accumulate=True) except Exception as e: mcm_cr.reset_requests(str(e), notify_one=rid) return { "results": False, "message": str(e), "prepid": chained_request_id } if not requires_validation: return { "results": True, "message": "No validation required", "prepid": chained_request_id } mcm_cr.set_attribute('validate', 1) if not crdb.update(mcm_cr.json()): return { "results": False, "message": "Failed while trying to update the document in DB", "prepid": chained_request_id } return { "results": True, "message": "run test will start soon", "prepid": chained_request_id }
def rewind_one(self, crid): crdb = database('chained_requests') rdb = database('requests') if not crdb.document_exists(crid): return { "results": False, "message": "does not exist", "prepid": crid } mcm_cr = chained_request(crdb.get(crid)) current_step = mcm_cr.get_attribute('step') if current_step == 0: # or should it be possible to cancel the initial requests of a chained request return { "results": False, "message": "already at the root", "prepid": crid } # supposedly all the other requests were already reset! for next in mcm_cr.get_attribute('chain')[current_step + 1:]: # what if that next one is not in the db if not rdb.document_exists(next): self.logger.error('%s is part of %s but does not exist' % (next, crid)) continue mcm_r = request(rdb.get(next)) if mcm_r.get_attribute('status') != 'new': # this cannot be right! self.logger.error( '%s is after the current request and is not new: %s' % (next, mcm_r.get_attribute('status'))) return { "results": False, "message": "%s is not new" % (next), "prepid": crid } # get the one to be reset current_id = mcm_cr.get_attribute('chain')[current_step] mcm_r = request(rdb.get(current_id)) mcm_r.reset() saved = rdb.update(mcm_r.json()) if not saved: { "results": False, "message": "could not save the last request of the chain", "prepid": crid } # the current chained request has very likely been updated : # reload it as you have not changed anything to it yet mcm_cr = chained_request(crdb.get(crid)) mcm_cr.set_attribute('step', current_step - 1) # set status, last status mcm_cr.set_last_status() mcm_cr.set_attribute('status', 'processing') saved = crdb.update(mcm_cr.json()) if saved: return {"results": True, "prepid": crid} else: return { "results": False, "message": "could not save chained requests. the DB is going to be inconsistent !", "prepid": crid }
def delete_request(self, crid): crdb = database('chained_requests') rdb = database('requests') mcm_cr = chained_request(crdb.get(crid)) # get all objects mcm_r_s = [] for (i, rid) in enumerate(mcm_cr.get_attribute('chain')): mcm_r = request(rdb.get(rid)) # this is not a valid check as it is allowed to remove a chain around already running requests # if mcm_r.get_attribute('status') != 'new': # return {"results":False,"message" : "the request %s part of the chain %s for action %s is not in new status"%( mcm_r.get_attribute('prepid'), # crid, # mcm_a.get_attribute('prepid'))} in_chains = mcm_r.get_attribute('member_of_chain') in_chains.remove(crid) self.logger.debug( "Removing ChainAction member_of_chain: %s to request: %s" % (mcm_cr.get_attribute("prepid"), mcm_r.get_attribute('prepid'))) mcm_r.set_attribute('member_of_chain', in_chains) if i == 0: if len(in_chains ) == 0 and mcm_r.get_attribute('status') != 'new': return { "results": False, "message": "the request %s, not in status new, at the root of the chain will not be chained anymore" % rid } else: if len(in_chains) == 0: return { "results": False, "message": "the request %s, not at the root of the chain will not be chained anymore" % rid } mcm_r.update_history({'action': 'leave', 'step': crid}) mcm_r_s.append(mcm_r) if mcm_cr.get_attribute('action_parameters')['flag']: return { "results": False, "message": "the action for %s is not disabled" % (crid) } # then save all changes for mcm_r in mcm_r_s: if not rdb.update(mcm_r.json()): return { "results": False, "message": "Could not save request " + mcm_r.get_attribute('prepid') } else: subject = "Request {0} left chain".format( mcm_r.get_attribute('prepid')) message = "Request {0} has successfuly left chain {1}".format( mcm_r.get_attribute('prepid'), crid) notification(subject, message, [], group=notification.REQUEST_OPERATIONS, action_objects=[mcm_r.get_attribute('prepid')], object_type='requests', base_object=mcm_r) mcm_r.notify(subject, message) return {"results": crdb.delete(crid)}
def generate_chained_requests(self, mccm_ticket, request_prepid, mcm_chained_campaign, reserve=False, with_notify=True, special=False): try: mcm_chained_campaign.reload(save_current=False) generated_chained_request = chained_request(mcm_chained_campaign.generate_request(request_prepid)) except Exception as e: message = "Unable to generate chained request for ticket %s request %s, message: " % (mccm_ticket.get_attribute('prepid'), request_prepid, str(e)) self.logger.error(message) return { "results": False, "message": message} requests_db = database('requests') self.overwrite_action_parameters_from_ticket(generated_chained_request, mccm_ticket) mcm_request = request(json_input=requests_db.get(request_prepid)) generated_chained_request.set_attribute('last_status', mcm_request.get_attribute('status')) if generated_chained_request.get_attribute('last_status') in ['submitted', 'done']: generated_chained_request.set_attribute('status', 'processing') if special: generated_chained_request.set_attribute('approval', 'none') new_chain_prepid = generated_chained_request.get_attribute('prepid') if not generated_chained_request.reload(): return { 'results': False, 'message': 'Unable to save chained request %s' % new_chain_prepid} # update the history of chained campaign mcm_chained_campaign.save() # let the root request know that it is part of a chained request chains = mcm_request.get_attribute('member_of_chain') chains.append(new_chain_prepid) chains.sort() mcm_request.set_attribute('member_of_chain', list(set(chains))) mcm_request.update_history({'action': 'join chain', 'step': new_chain_prepid}) if with_notify: subject = "Request %s joined chain" % mcm_request.get_attribute('prepid') message = "Request %s has successfully joined chain %s" % (mcm_request.get_attribute('prepid'), new_chain_prepid) notification( subject, message, [], group=notification.REQUEST_OPERATIONS, action_objects=[mcm_request.get_attribute('prepid')], object_type='requests', base_object=mcm_request) mcm_request.notify( subject, message, Nchild=0, accumulate=True) mcm_request.save() # do the reservation of the whole chain ? generated_requests = [] if reserve: results_dict = generated_chained_request.reserve(limit=reserve, save_requests=False) if results_dict['results'] and 'generated_requests' in results_dict: generated_requests = results_dict['generated_requests'] results_dict.pop('generated_requests') else: return { "results": False, "prepid": new_chain_prepid, "message": results_dict['message']} return { "results":True, "prepid": new_chain_prepid, 'generated_requests': generated_requests}