def import_request(self, data): db = database(self.db_name) json_input=threaded_loads(data) if 'pwg' not in json_input or 'member_of_campaign' not in json_input: self.logger.error('Now pwg or member of campaign attribute for new chained request') return {"results":False} if 'prepid' in json_input: req = chained_request(json_input) cr_id = req.get_attribute('prepid') else: cr_id = ChainedRequestPrepId().next_id(json_input['pwg'], json_input['member_of_campaign']) if not cr_id: return {"results":False} req = chained_request(db.get(cr_id)) for key in json_input: if key not in ['prepid', '_id', '_rev', 'history']: req.set_attribute(key, json_input[key]) if not req.get_attribute('prepid'): self.logger.error('prepid returned was None') raise ValueError('Prepid returned was None') self.logger.log('Created new chained_request %s' % cr_id) # update history with the submission details req.update_history({'action': 'created'}) return self.save_request(db, req)
def get(self, chained_request_id): """ Does a soft reset to all relevant request in the chain """ crdb = database('chained_requests') rdb = database('requests') mcm_cr = chained_request(crdb.get(chained_request_id)) for rid in reversed( mcm_cr.get_attribute('chain')[:mcm_cr.get_attribute('step') + 1]): # from the current one to the first one REVERSED mcm_r = request(rdb.get(rid)) try: mcm_r.reset(hard=False) except Exception as e: return { 'prepid': chained_request_id, 'results': False, 'message': str(e) } mcm_r.reload() mcm_cr = chained_request(crdb.get(chained_request_id)) mcm_cr.set_attribute( 'step', max(0, mcm_cr.get_attribute('chain').index(rid) - 1)) mcm_cr.reload() return {'prepid': chained_request_id, 'results': True}
def import_request(self, data): db = database(self.db_name) json_input = loads(data) if 'pwg' not in json_input or 'member_of_campaign' not in json_input: self.logger.error( 'Now pwg or member of campaign attribute for new chained request' ) return {"results": False} if 'prepid' in json_input: req = chained_request(json_input) cr_id = req.get_attribute('prepid') else: cr_id = ChainedRequestPrepId().next_id( json_input['pwg'], json_input['member_of_campaign']) if not cr_id: return {"results": False} req = chained_request(db.get(cr_id)) for key in json_input: if key not in ['prepid', '_id', '_rev', 'history']: req.set_attribute(key, json_input[key]) if not req.get_attribute('prepid'): self.logger.error('prepid returned was None') raise ValueError('Prepid returned was None') self.logger.info('Created new chained_request %s' % cr_id) # update history with the submission details req.update_history({'action': 'created'}) return self.save_request(db, req)
def GET(self, *args): """ Does a soft reset to all relevant request in the chain """ if not len(args): return dumps({"results" : False, "message" : "no argument provided"}) arg0 = args[0] crdb = database('chained_requests') rdb = database('requests') mcm_cr = chained_request(crdb.get(arg0)) for rid in reversed(mcm_cr.get_attribute('chain')[:mcm_cr.get_attribute('step')+1]): ## from the current one to the first one REVERSED mcm_r = request(rdb.get(rid)) try: mcm_r.reset(hard=False) except Exception as e: return dumps({'prepid' : arg0, 'results' : False, 'message' : str(e)}) mcm_r.reload() mcm_cr = chained_request(crdb.get(arg0)) mcm_cr.set_attribute('step', max(0, mcm_cr.get_attribute('chain').index(rid)-1)) mcm_cr.reload() return dumps({'prepid' : arg0, 'results':True})
def rewind_one(self, crid): crdb = database('chained_requests') rdb = database('requests') if not crdb.document_exists( crid ): return {"results":False, "message":"does not exist","prepid" : crid} mcm_cr = chained_request( crdb.get( crid) ) current_step = mcm_cr.get_attribute('step') if current_step==0: ## or should it be possible to cancel the initial requests of a chained request return {"results":False, "message":"already at the root","prepid" : crid} ## supposedly all the other requests were already reset! for next in mcm_cr.get_attribute('chain')[current_step+1:]: ## what if that next one is not in the db if not rdb.document_exists( next): self.logger.error('%s is part of %s but does not exist'%( next, crid)) continue mcm_r = request(rdb.get( next )) if mcm_r.get_attribute('status')!='new': # this cannot be right! self.logger.error('%s is after the current request and is not new: %s' % ( next, mcm_r.get_attribute('status'))) return {"results":False, "message":"%s is not new" % (next), "prepid" : crid} ##get the one to be reset current_id=mcm_cr.get_attribute('chain')[current_step] mcm_r = request( rdb.get( current_id )) mcm_r.reset() saved = rdb.update( mcm_r.json() ) if not saved: {"results":False, "message":"could not save the last request of the chain","prepid" : crid} ## the current chained request has very likely been updated : ## reload it as you have not changed anything to it yet mcm_cr = chained_request( crdb.get( crid) ) mcm_cr.set_attribute('step',current_step -1 ) # set status, last status mcm_cr.set_last_status() mcm_cr.set_attribute('status','processing') saved = crdb.update( mcm_cr.json()) if saved: return {"results":True,"prepid" : crid} else: return {"results" : False, "message" : "could not save chained requests. the DB is going to be inconsistent !", "prepid" : crid}
def get(self, chained_request_id): kwargs = self.parser.parse_args() crdb = database('chained_requests') if not crdb.document_exists(chained_request_id): return { "results": False, "message": "Chained request with prepid {0} does not exist".format( chained_request_id) } cr = chained_request(crdb.get(chained_request_id)) events = None run = False valid = False directory = '' __scratch = kwargs["scratch"].lower() == 'true' if self.opt == 'test' or self.opt == 'valid': run = True if self.opt == 'valid': valid = True return cr.get_setup(directory=kwargs['directory'], run=run, events=kwargs['events'], validation=valid, scratch=__scratch)
def submit_request(self, prepid, run_test_path): mcm_request = request(self.request_db.get(prepid)) # check if the request should be validated as part of a chain for chain_prepid in mcm_request.get_attribute('member_of_chain'): mcm_chain = chained_request(self.chained_request_db.get(chain_prepid)) if mcm_chain.get_attribute('validate') and prepid in mcm_chain.get_attribute('chain')[mcm_chain.get_attribute('step'):]: return {} aux_validation = mcm_request.get_attribute(self.DOC_VALIDATION) to_write = mcm_request.get_setup_file(run=True, do_valid=True, for_validation=True, gen_script=True) file_name = self.TEST_FILE_NAME % prepid if not self.create_test_file(to_write, run_test_path, file_name): mcm_request.set_attribute(self.DOC_VALIDATION, aux_validation) mcm_request.test_failure( message='There was a problem while creating the file for prepid: %s' % (mcm_request.get_attribute('prepid')), what='Validation run test', rewind=True) return {} timeout = mcm_request.get_timeout() memory = mcm_request.get_attribute("memory") threads = mcm_request.get_core_num() self.create_htcondor_config_file(run_test_path, prepid, timeout, memory, threads, [prepid], mcm_request) job_info = self.execute_command_submission(prepid, run_test_path) if 'error' in job_info: mcm_request.test_failure(message=job_info['error'], what='Validation run test', rewind=True) return {} job_info[self.DOC_REV] = mcm_request.json()[self.DOC_REV] job_info[self.DOC_VALIDATION] = mcm_request.get_attribute(self.DOC_VALIDATION) # this field change when calling request.get_setup_file, to be propagated in case of success return job_info
def flow2(self, data): try: vdata = loads(data) except ValueError as ex: self.logger.error('Could not start flowing to next step. Reason: %s' % (ex)) return {"results":str(ex)} db = database('chained_requests') try: creq = chained_request(json_input=db.get(vdata['prepid'])) except Exception as ex: self.logger.error('Could not initialize chained_request object. Reason: %s' % (ex)) return {"results":str(ex)} self.logger.log('Attempting to flow to next step for chained_request %s' % (creq.get_attribute('_id'))) # if the chained_request can flow, do it inputds = '' inblack = [] inwhite = [] if 'input_filename' in vdata: inputds = vdata['input_filename'] if 'block_black_list' in vdata: inblack = vdata['block_black_list'] if 'block_white_list' in vdata: inwhite = vdata['block_white_list'] if 'force' in vdata: check_stats = vdata['force']!='force' if 'reserve' in vdata and vdata["reserve"]: return creq.reserve() return creq.flow_trial( inputds, inblack, inwhite, check_stats)
def force_status_done(self, prepid): if not self.crdb.document_exists(prepid): return dumps( { "results": False, "message": "Chained request with prepid {0} does not exist".format( prepid) }, indent=4) cr = chained_request(self.crdb.get(prepid)) if not (cr.get_attribute("status") in ["done", "force_done"]): cr.set_status(to_status="force_done") self.logger.debug( "forcing chain_req status to done. cr status:%s" % (cr.get_attribute("status"))) ret = self.crdb.save(cr.json()) return dumps({ 'prepid': prepid, 'message': ret, 'results': True }, indent=4) else: ret = "Chained request already in status done" return dumps({ 'prepid': prepid, 'message': ret, 'results': False }, indent=4)
def multiple_inspect(self, ccids): crdb = database('chained_requests') res = [] for ccid in ccids.split(','): crlist = crdb.queries(["member_of_campaign==%s"% ccid, "last_status==done", "status==processing"]) self.logger.log('crlist %s in chained_camp %s ' % (crlist, ccid)) for cr in crlist: mcm_cr = chained_request( cr ) if mcm_cr: __inspect_ret = mcm_cr.inspect() else: __inspect_ret = {"prepid":cr, "results":False, 'message' : '%s does not exist' % cr} self.logger.log("Inspection for: %s returned: %s" %(cr, __inspect_ret)) res.append(__inspect_ret) if len(res)>1: return res elif len(res): self.logger.log("Inspection finished. Returned: %s" %(res[0])) return res[0] else: self.logger.log("Inspection finished. Returned: %s" %([])) return []
def multiple_inspect(self, ccids): crdb = database('chained_requests') res = [] for ccid in ccids.split(','): crlist = crdb.queries([ "member_of_campaign==%s" % ccid, "last_status==done", "status==processing" ]) self.logger.log('crlist %s in chained_camp %s ' % (crlist, ccid)) for cr in crlist: mcm_cr = chained_request(cr) if mcm_cr: res.append(mcm_cr.inspect()) else: res.append({ "prepid": cr, "results": False, 'message': '%s does not exist' % cr }) if len(res) > 1: return res elif len(res): return res[0] else: return []
def flow2(self, data): db = database('chained_requests') chain_id = data['prepid'] try: creq = chained_request(json_input=db.get(chain_id)) except Exception as ex: self.logger.error( 'Could not initialize chained_request object. Reason: %s' % (ex)) return {"results": str(ex)} self.logger.info( 'Attempting to flow to next step for chained_request %s' % (creq.get_attribute('_id'))) # if the chained_request can flow, do it inputds = '' inblack = [] inwhite = [] if 'input_dataset' in data: inputds = data['input_dataset'] if 'block_black_list' in data: inblack = data['block_black_list'] if 'block_white_list' in data: inwhite = data['block_white_list'] if 'force' in data: check_stats = data['force'] != 'force' if 'reserve' in data and data["reserve"]: reserve = data["reserve"] return creq.reserve(limit=reserve) return creq.flow_trial(inputds, inblack, inwhite, check_stats)
def flow2(self, data): db = database('chained_requests') chain_id = data['prepid'] try: creq = chained_request(json_input=db.get(chain_id)) except Exception as ex: self.logger.error('Could not initialize chained_request object. Reason: %s' % (ex)) return {"results": str(ex)} self.logger.info('Attempting to flow to next step for chained_request %s' % ( creq.get_attribute('_id'))) # if the chained_request can flow, do it inputds = '' inblack = [] inwhite = [] if 'input_dataset' in data: inputds = data['input_dataset'] if 'block_black_list' in data: inblack = data['block_black_list'] if 'block_white_list' in data: inwhite = data['block_white_list'] if 'force' in data: check_stats = data['force'] != 'force' if 'reserve' in data and data["reserve"]: reserve = data["reserve"] return creq.reserve(limit=reserve) return creq.flow_trial(inputds, inblack, inwhite, check_stats)
def next_id(self, pwg, campaign): ccamp_db = database(self.ccamp_db_name) creq_db = database(self.creq_db_name) if not pwg: self.logger.error('Physics working group provided is None.') return None if not campaign: self.logger.error('Campaign id provided is None.') return None with locker.lock("{0}-{1}".format(pwg, campaign)): if not ccamp_db.document_exists(campaign): self.logger.error('Campaign id {0} does not exist.'.format(campaign)) return None if (campaign, pwg) in self.serial_number_cache: sn = self.serial_number_cache[(campaign, pwg)] + 1 else: sn=1 serial_number_lookup = creq_db.raw_query('serial_number', {'group':True, 'key':[campaign, pwg]}) if serial_number_lookup: sn = serial_number_lookup[0]['value']+1 ## construct the new id new_prepid = pwg + '-' + campaign + '-' + str(sn).zfill(5) if sn==1: self.logger.log('Beginning new prepid family: %s' % (new_prepid)) new_request = chained_request({'_id':new_prepid, 'prepid':new_prepid, 'pwg':pwg, 'member_of_campaign':campaign}) new_request.update_history({'action':'created'}) creq_db.save(new_request.json()) self.serial_number_cache[(campaign, pwg)] = sn self.logger.log('New chain id: %s' % new_prepid, level='debug') return new_prepid
def reset_all(self, message, what = 'Chained validation run test', notify_one=None): crdb = database('chained_requests') rdb = database('requests') mcm_cr = chained_request(crdb.get(self.crid)) if self.scratch: chain = mcm_cr.get_attribute('chain') else: chain = mcm_cr.get_attribute('chain')[mcm_cr.get_attribute('step'):] for rid in chain: mcm_r = request(rdb.get(rid )) s_label = 'chainvalid-%s' % rid semaphore_events.decrement(s_label) if not semaphore_events.is_set(s_label): ##someone else is still validating that chain, so no reset ! mcm_r.notify('%s failed for request %s' % (what, mcm_r.get_attribute('prepid')), message) continue ## do not reset anything that does not look ok already # this might leave things half-way inconsistent in terms of status if mcm_r.get_attribute('status') != 'new': mcm_r.notify('%s failed for request %s' % (what, mcm_r.get_attribute('prepid')), message) continue notify = True if notify_one and notify_one != rid: notify = False mcm_r.test_failure( message, what = what, rewind=True, with_notification=notify)
def force_status(self, prepid): if not self.crdb.document_exists(prepid): return dumps({ "results": False, "message": "Chained request with prepid {0} does not exist".format(prepid) }) cr = chained_request(self.crdb.get(prepid)) if cr.get_attribute("status") == "force_done": cr.set_status(to_status="processing") self.logger.debug( "Moving chain_req back to satus 'processing'. cr status:%s" % (cr.get_attribute("status"))) ret = self.crdb.save(cr.json()) return dumps({ 'prepid': prepid, 'message': ret, 'results': True }, indent=4) else: ret = "Chained request not in status force_done" return dumps({ 'prepid': prepid, 'message': ret, 'results': False }, indent=4)
def GET(self, *args, **kwargs): if not len(args): return dumps({"results": False, "message": "Chained request prepid not given"}) crdb = database('chained_requests') if not crdb.document_exists(args[0]): return dumps({"results": False, "message": "Chained request with prepid {0} does not exist".format(args[0])}) cr = chained_request(crdb.get(args[0])) events = None run = False valid = False directory = '' __scratch = False if 'scratch' in kwargs: __scratch = kwargs["scratch"] if self.opt == 'test' or self.opt == 'valid': run = True if self.opt == 'valid': valid = True if 'events' in kwargs: events = int(kwargs['events']) if 'directory' in kwargs: directory = kwargs['directory'] cherrypy.response.headers['Content-Type'] = 'text/plain' return cr.get_setup(directory=directory, run=run, events=events, validation=valid, scratch=__scratch)
def get(self, chained_request_ids): """ Add selected prepid's to global force complete list for later action """ if ',' in chained_request_ids: rlist = chained_request_ids.rsplit(',') else: rlist = [chained_request_ids] res = [] __updated = False forceflow_list = self.ldb.get("list_of_forceflow") # TO-DO check if prepid exists! # TO-DO check the status of chain_req! for el in rlist: if el not in forceflow_list["value"]: forceflow_list["value"].append(el) chain_req = chained_request(self.cdb.get(el)) chain_req.update_history({'action': 'add_to_forceflow'}) self.cdb.save(chain_req.json()) res.append({"prepid": el, 'results': True, 'message': 'OK'}) __updated = True else: res.append({"prepid": el, 'results': False, 'message': 'Chained request already in forceflow list'}) # TO-DO check the update return value if __updated: self.ldb.update(forceflow_list) return res
def internal_run(self): from tools.installer import installer from tools.batch_control import batch_control location = installer( self.crid, care_on_existing=False, clean_on_exit=True) try: crdb = database('chained_requests') rdb = database('requests') mcm_cr = chained_request(crdb.get(self.crid)) mcm_rs = [] for rid in mcm_cr.get_attribute('chain'): mcm_rs.append( request( rdb.get( rid ) )) test_script = location.location() + 'validation_run_test.sh' with open(test_script, 'w') as there: there.write(mcm_cr.get_setup(directory=location.location(), run=True, validation=True)) batch_test = batch_control( self.crid, test_script ) try: success = batch_test.test() except: self.reset_all( traceback.format_exc() ) return if not success: self.reset_all( '\t .out \n%s\n\t .err \n%s\n ' % ( batch_test.log_out, batch_test.log_err) ) return last_fail=mcm_rs[0] trace="" for mcm_r in mcm_rs: ### if not mcm_r.is_root: continue ##disable for dr request (success,trace) = mcm_r.pickup_all_performance(location.location()) if not success: last_fail = mcm_r break self.logger.error('I came all the way to here and %s (request %s)' % ( success, self.crid )) if success: for mcm_r in mcm_rs: if mcm_r.is_root: mcm_current = request( rdb.get(mcm_r.get_attribute('prepid'))) if mcm_current.json()['_rev'] == mcm_r.json()['_rev']: mcm_r.set_status(with_notification=True) if not mcm_r.reload(): self.reset_all( 'The request %s could not be saved after the runtest procedure' % (mcm_r.get_attribute('prepid'))) return else: self.reset_all( 'The request %s has changed during the run test procedure'%(mcm_r.get_attribute('prepid')), notify_one = mcm_r.get_attribute('prepid')) return else: self.reset_all( trace , notify_one = last_fail.get_attribute('prepid') ) return except: mess = 'We have been taken out of run_safe of runtest_genvalid for %s because \n %s \n During an un-excepted exception. Please contact support.' % ( self.crid, traceback.format_exc()) self.logger.error(mess) finally: location.close()
def submit_chain(self, prepid, run_test_path): mcm_chained_request = chained_request(self.chained_request_db.get(prepid)) except_requests = [] reset = False # If a request of a chain was singly submmited to validation and then somebody reseted it, we will find it here for request_prepid in mcm_chained_request.get_attribute('chain')[mcm_chained_request.get_attribute('step'):]: if request_prepid in self.submmited_prepids_set: except_requests.append(request_prepid) reset = True if reset: message = "Requests %s of the chain %s are already in validation" % (except_requests, prepid) self.logger.error(message) mcm_chained_request.reset_requests(message, except_requests=except_requests) return {} to_write = mcm_chained_request.get_setup(run=True, validation=True, for_validation=True) file_name = self.TEST_FILE_NAME % prepid if not self.create_test_file(to_write, run_test_path, file_name): mcm_chained_request.reset_requests('There was a problem while creating the file for prepid: %s' % (mcm_chained_request.get_attribute('prepid'))) return {} requests_in_chain = {} first_request_in_chain = None for request_prepid in mcm_chained_request.get_attribute('chain')[mcm_chained_request.get_attribute('step'):]: mcm_request = request(self.request_db.get(request_prepid)) if not mcm_request.is_root and 'validation' not in mcm_request._json_base__status: # only root or possible root requests break status = mcm_request.get_attribute('status') approval = mcm_request.get_attribute('approval') if status != 'new' or approval != 'validation': message = "The request %s of chain %s is in status: %s approval: %s" % (request_prepid, prepid, status, approval) self.logger.error(message) mcm_chained_request.reset_requests(message) return {} requests_in_chain[request_prepid] = mcm_request.json()[self.DOC_REV] if not first_request_in_chain: first_request_in_chain = mcm_request if not len(requests_in_chain): message = 'No requests to be validated in chain: %s' % prepid self.logger.info(message) mcm_chained_request.reset_requests(message) return {} timeout, memory, threads = mcm_chained_request.get_timeout_memory_threads() self.create_htcondor_config_file(run_test_path, prepid, timeout, memory, threads, list(requests_in_chain.iterkeys()), first_request_in_chain) job_info = self.execute_command_submission(prepid, run_test_path) if 'error' in job_info: mcm_chained_request.reset_requests(job_info['error']) return {} job_info[self.CHAIN_REQUESTS] = requests_in_chain return job_info
def multiple_inspect(self, ccids): settings.set_value('inspect_chained_campaigns_running', True) crdb = database('chained_requests') try: ccids = ccids.split(',') index = 0 while len(ccids) > index: query = crdb.construct_lucene_complex_query([ ('member_of_campaign', { 'value': ccids[index:index + 20] }), ('last_status', { 'value': 'done' }), ('status', { 'value': 'processing' }) ]) crlist = crdb.full_text_search('search', query, page=-1) # we yield len of cr_list so we would know how much data later on we processed yield dumps( { 'prepids': ccids[index:index + 20], 'cr_len': len(crlist) }, indent=2) index += 20 for cr in crlist: time.sleep(0.5) mcm_cr = chained_request(cr) if mcm_cr: __inspect_ret = mcm_cr.inspect() else: __inspect_ret = { "prepid": cr, "results": False, 'message': '%s does not exist' % cr['prepid'] } self.logger.info("Inspection for: %s returned: %s" % (cr['prepid'], __inspect_ret)) yield dumps(__inspect_ret, indent=8) # force slowing-down of inspect to not abuse the DB time.sleep(1) self.logger.info("ChainedCampaigns inspection finished") except Exception as ex: self.logger.error( "ChainedCampaigns inspection crashed. reason: %s" % str(ex)) yield dumps({ 'message': 'crlist crashed: %s' % (str(ex)), 'last_used_query': query }) finally: settings.set_value('inspect_chained_campaigns_running', False)
def GET(self, *args): """ Perform test for chained requests """ if not len(args): return dumps({"results" : False, "message" : "no argument provided"}) from tools.handlers import RunChainValid, validation_pool ## now in the core of the api runtest = RunChainValid(crid=args[0], lock=locker.lock(args[0])) crdb = database('chained_requests') rdb = database('requests') mcm_cr = chained_request(crdb.get(args[0])) mcm_rs = [] for rid in mcm_cr.get_attribute('chain')[mcm_cr.get_attribute('step'):]: mcm_r = request( rdb.get( rid ) ) if mcm_r.get_attribute('status') in ['approved','submitted','done']: return dumps({"results" : False, "prepid" : args[0], "message" : "request %s is in status %s" % ( rid, mcm_r.get_attribute('status'))}) for rid in mcm_cr.get_attribute('chain')[mcm_cr.get_attribute('step'):]: mcm_r = request(rdb.get(rid)) next = 'validation' if not mcm_r.is_root: next = 'approve' try: if mcm_r.get_attribute('approval') == 'none': ## no need to try and move it along if already further than that getattr(mcm_r,'ok_to_move_to_approval_%s' % next)(for_chain=True) mcm_r.update_history({'action' : 'approve', 'step' : next}) mcm_r.set_attribute('approval', next) mcm_r.reload() else: pass ## fail this for the moment. there is no way to handle this yet #text="It is not supported for the moment to test a chain of requests which are partially not new. Please contact an administrator" #runtest.reset_all( text , notify_one = rid ) #return dumps({"results" : False, "message" : text, "prepid" : args[0]}) text = 'Within chain %s \n'% mcm_cr.get_attribute('prepid') text += mcm_r.textified() mcm_r.notify('Approval %s in chain %s for request %s' % (next, mcm_cr.get_attribute('prepid'), mcm_r.get_attribute('prepid')), text, accumulate=True) except Exception as e: runtest.reset_all(str(e), notify_one=rid) return dumps({"results" : False, "message" : str(e),"prepid" : args[0]}) validation_pool.add_task(runtest.internal_run) #runtest.start() return dumps({"results" : True, "message" : "run test started","prepid" : args[0]})
def process_finished_chain_failed(self, prepid, job_out, job_error_out, error_out, was_exited, out_path, log_out): mcm_chained_request = chained_request( self.chained_request_db.get(prepid)) if not was_exited: message = "File %s does not look properly formatted or does not exist. \n %s \n %s \n Error out: \n%s \n Log out:\n %s" % ( out_path, job_out, job_error_out, error_out, log_out) else: message = "Job validation failed for chain %s \nJob out: \n%s \n Error out: \n%s \n Log out: \n%s" % ( prepid, job_out, error_out, log_out) mcm_chained_request.reset_requests(message)
def process_finished_chain_failed(self, prepid, job_out, job_error_out, error_out, was_exited, out_path, log_out): mcm_chained_request = chained_request(self.chained_request_db.get(prepid)) if not was_exited: message = "File %s does not look properly formatted or does not exist. \n %s \n %s \n Error out: \n%s \n Log out:\n %s" % ( out_path, job_out, job_error_out, error_out, log_out) else: message = "Job validation failed for chain %s \nJob out: \n%s \n Error out: \n%s \n Log out: \n%s" % (prepid, job_out, error_out, log_out) mcm_chained_request.reset_requests(message)
def force_status_done(self, prepid): if not self.crdb.document_exists(prepid): return dumps({"results": False, "message": "Chained request with prepid {0} does not exist".format(prepid)}, indent=4) cr = chained_request(self.crdb.get(prepid)) if not (cr.get_attribute("status") in ["done", "force_done"]): cr.set_status(to_status="force_done") cr.remove_from_nonflowing_list() self.logger.debug("forcing chain_req status to done. cr status:%s" % (cr.get_attribute("status"))) ret = self.crdb.save(cr.json()) return {'prepid': prepid, 'message': ret, 'results': True} else: ret = "Chained request already in status done" return {'prepid': prepid, 'message': ret, 'results': False}
def GET(self, *args, **kwargs): if not len(args): return dumps({"results": False, "message": "Chained request prepid not given"}) crdb = database('chained_requests') if not crdb.document_exists(args[0]): return dumps({"results": False, "message": "Chained request with prepid {0} does not exist".format(args[0])}) cr = chained_request(crdb.get(args[0])) cherrypy.response.headers['Content-Type'] = 'text/plain' return cr.test_output_ds()
def get(self, chained_request_id): """ Does a soft reset to all relevant request in the chain """ crdb = database('chained_requests') rdb = database('requests') mcm_cr = chained_request(crdb.get(chained_request_id)) for rid in reversed(mcm_cr.get_attribute('chain')[:mcm_cr.get_attribute('step') + 1]): # from the current one to the first one REVERSED mcm_r = request(rdb.get(rid)) try: mcm_r.reset(hard=False) except Exception as e: return {'prepid': chained_request_id, 'results': False, 'message': str(e)} mcm_r.reload() mcm_cr = chained_request(crdb.get(chained_request_id)) mcm_cr.set_attribute('step', max(0, mcm_cr.get_attribute('chain').index(rid) - 1)) mcm_cr.reload() return {'prepid': chained_request_id, 'results': True}
def reset_all(self, message, what = 'Chained validation run test', notify_one=None): crdb = database('chained_requests') rdb = database('requests') mcm_cr = chained_request(crdb.get(self.crid)) for rid in mcm_cr.get_attribute('chain'): mcm_r = request( rdb.get( rid ) ) notify = True if notify_one and notify_one != rid: notify = False mcm_r.test_failure( message, what = what, rewind=True, with_notification=notify)
def delete_request(self, crid): crdb = database('chained_requests') rdb = database('requests') adb = database('actions') mcm_cr = chained_request(crdb.get(crid)) mcm_a = None ## get all objects mcm_r_s=[] for (i,rid) in enumerate(mcm_cr.get_attribute('chain')): mcm_r = request(rdb.get(rid)) #this is not a valid check as it is allowed to remove a chain around already running requests # if mcm_r.get_attribute('status') != 'new': # return {"results":False,"message" : "the request %s part of the chain %s for action %s is not in new status"%( mcm_r.get_attribute('prepid'), # crid, # mcm_a.get_attribute('prepid'))} in_chains = mcm_r.get_attribute('member_of_chain') in_chains.remove( crid ) mcm_r.set_attribute('member_of_chain', in_chains) if i==0: # the root is the action id mcm_a = action(adb.get(rid)) if len(in_chains)==0 and mcm_r.get_attribute('status')!='new': return {"results":False, "message" : "the request %s, not in status new, at the root of the chain will not be chained anymore"% rid} else: if len(in_chains)==0: return {"results":False,"message" : "the request %s, not at the root of the chain will not be chained anymore"% rid} mcm_r.update_history({'action':'leave','step':crid}) mcm_r_s.append( mcm_r ) ## check if possible to get rid of it ! # action for the chain is disabled chains = mcm_a.get_chains( mcm_cr.get_attribute('member_of_campaign')) if chains[crid]['flag']: return {"results":False,"message" : "the action %s for %s is not disabled"%(mcm_a.get_attribute('prepid'), crid)} #take it out mcm_a.remove_chain( mcm_cr.get_attribute('member_of_campaign'), mcm_cr.get_attribute('prepid') ) if not adb.update( mcm_a.json()): return {"results":False,"message" : "Could not save action "+ mcm_a.get_attribute('prepid')} ## then save all changes for mcm_r in mcm_r_s: if not rdb.update( mcm_r.json()): return {"results":False,"message" : "Could not save request "+ mcm_r.get_attribute('prepid')} else: mcm_r.notify("Request {0} left chain".format( mcm_r.get_attribute('prepid')), "Request {0} has successfuly left chain {1}".format( mcm_r.get_attribute('prepid'), crid)) return {"results": crdb.delete(crid)}
def report_error(self, prepid, message): self.logger.error(message) try: if 'chain' in prepid: mcm_chained_request = chained_request(self.chained_request_db.get(prepid)) mcm_chained_request.reset_requests(message) else: mcm_request = request(self.request_db.get(prepid)) mcm_request.test_failure( message=message, what='Validation run test', rewind=True) except Exception as e: self.logger.error("Exception while reporting an error for %s message: %s \ntraceback: %s" % (prepid, str(e), traceback.format_exc()))
def force_status(self, prepid): if not self.crdb.document_exists(prepid): return dumps({"results": False, "message": "Chained request with prepid {0} does not exist".format(prepid)}) cr = chained_request(self.crdb.get(prepid)) if cr.get_attribute("status") == "force_done": cr.set_status(to_status="processing") self.logger.debug("Moving chain_req back to satus 'processing'. cr status:%s" % ( cr.get_attribute("status"))) ret = self.crdb.save(cr.json()) return {'prepid': prepid, 'message': ret, 'results': True} else: ret = "Chained request already in status done" return {'prepid': prepid, 'message': ret, 'results': False}
def multiple_inspect(self, crid): crlist=crid.rsplit(',') res = [] crdb = database('chained_requests') for cr in crlist: if crdb.document_exists( cr): mcm_cr = chained_request( crdb.get( cr) ) res.append( mcm_cr.inspect() ) else: res.append( {"prepid": cr, "results":False, 'message' : '%s does not exist'% cr}) if len(res)>1: return res else: return res[0]
def approve(self, rid, val=-1): db = database('chained_requests') if not db.document_exists(rid): return {"prepid": rid, "results":'Error: The given chained_request id does not exist.'} creq = chained_request(json_input=db.get(rid)) try: creq.approve(val) except Exception as ex: return {"prepid": rid, "results":False, 'message' : str(ex)} saved = db.update(creq.json()) if saved: return {"prepid":rid, "results":True} else: return {"prepid":rid, "results":False , 'message': 'unable to save the updated chained request'}
def flow(self, chainid, check_stats=True, reserve = False): try: db = database('chained_requests') creq = chained_request(json_input=db.get(chainid)) except Exception as ex: self.logger.error('Could not initialize chained_request object. Reason: %s' % (ex)) return {"results":str(ex)} # if the chained_request can flow, do it if reserve: self.logger.log('Attempting to reserve to next step for chained_request %s' % (creq.get_attribute('_id'))) return creq.reserve() self.logger.log('Attempting to flow to next step for chained_request %s' % (creq.get_attribute('_id'))) return creq.flow_trial(check_stats=check_stats)
def generate_chained_requests(self, mccm_ticket, request_prepid, mcm_chained_campaign, reserve=False, with_notify=True, special=False): try: mcm_chained_campaign.reload(save_current=False) generated_chained_request = chained_request(mcm_chained_campaign.generate_request(request_prepid)) except Exception as e: message = "Unable to generate chained request for ticket %s request %s, message: " % (mccm_ticket.get_attribute('prepid'), request_prepid, str(e)) self.logger.error(message) return { "results": False, "message": message} requests_db = database('requests') self.overwrite_action_parameters_from_ticket(generated_chained_request, mccm_ticket) mcm_request = request(json_input=requests_db.get(request_prepid)) generated_chained_request.set_attribute('last_status', mcm_request.get_attribute('status')) if generated_chained_request.get_attribute('last_status') in ['submitted', 'done']: generated_chained_request.set_attribute('status', 'processing') if special: generated_chained_request.set_attribute('approval', 'none') new_chain_prepid = generated_chained_request.get_attribute('prepid') if not generated_chained_request.reload(): return { 'results': False, 'message': 'Unable to save chained request %s' % new_chain_prepid} # update the history of chained campaign mcm_chained_campaign.save() # let the root request know that it is part of a chained request chains = mcm_request.get_attribute('member_of_chain') chains.append(new_chain_prepid) chains.sort() mcm_request.set_attribute('member_of_chain', list(set(chains))) mcm_request.update_history({'action': 'join chain', 'step': new_chain_prepid}) mcm_request.save() # do the reservation of the whole chain ? generated_requests = [] if reserve: results_dict = generated_chained_request.reserve(limit=reserve, save_requests=False) if results_dict['results'] and 'generated_requests' in results_dict: generated_requests = results_dict['generated_requests'] results_dict.pop('generated_requests') else: return { "results": False, "prepid": new_chain_prepid, "message": results_dict['message']} return { "results":True, "prepid": new_chain_prepid, 'generated_requests': generated_requests}
def report_error(self, prepid, message): self.logger.error(message) try: if 'chain' in prepid: mcm_chained_request = chained_request( self.chained_request_db.get(prepid)) mcm_chained_request.reset_requests(message) else: mcm_request = request(self.request_db.get(prepid)) mcm_request.test_failure(message=message, what='Validation run test', rewind=True) except Exception as e: self.logger.error( "Exception while reporting an error for %s message: %s \ntraceback: %s" % (prepid, str(e), traceback.format_exc()))
def update_request(self, data): try: req = chained_request(json_input=loads(data)) except chained_request.IllegalAttributeName as ex: return {"results":False} if not req.get_attribute('prepid') and not req.get_attribute('_id'): self.logger.error('prepid returned was None') raise ValueError('Prepid returned was None') #req.set_attribute('_id', req.get_attribute('prepid') self.logger.log('Updating chained_request %s' % (req.get_attribute('_id'))) # update history req.update_history({'action': 'update'}) return self.save_request(req)
def update_request(self, data): try: req = chained_request(json_input=loads(data)) except chained_request.IllegalAttributeName: return {"results": False} if not req.get_attribute('prepid') and not req.get_attribute('_id'): self.logger.error('prepid returned was None') raise ValueError('Prepid returned was None') # req.set_attribute('_id', req.get_attribute('prepid') self.logger.info('Updating chained_request %s' % (req.get_attribute('_id'))) self.logger.info('wtf %s' % (str(req.get_attribute('approval')))) # update history req.update_history({'action': 'update'}) new_priority = req.get_attribute('action_parameters')['block_number'] req.set_priority(new_priority) return self.save_request(req)
def get(self, chained_request_id): kwargs = self.parser.parse_args() crdb = database('chained_requests') if not crdb.document_exists(chained_request_id): return {"results": False, "message": "Chained request with prepid {0} does not exist".format(chained_request_id)} cr = chained_request(crdb.get(chained_request_id)) events = None run = False valid = False directory = '' __scratch = kwargs["scratch"].lower() == 'true' if self.opt == 'test' or self.opt == 'valid': run = True if self.opt == 'valid': valid = True return cr.get_setup(directory=kwargs['directory'], run=run, events=kwargs['events'], validation=valid, scratch=__scratch)
def next_id(self, pwg, campaign): ccamp_db = database(self.ccamp_db_name) creq_db = database(self.creq_db_name) if not pwg: self.logger.error('Physics working group provided is None.') return None if not campaign: self.logger.error('Campaign id provided is None.') return None with locker.lock("{0}-{1}".format(pwg, campaign)): if not ccamp_db.document_exists(campaign): self.logger.error( 'Campaign id {0} does not exist.'.format(campaign)) return None if (campaign, pwg) in self.serial_number_cache: sn = self.serial_number_cache[(campaign, pwg)] + 1 else: sn = 1 serial_number_lookup = creq_db.raw_query( 'serial_number', { 'group': True, 'key': [campaign, pwg] }) if serial_number_lookup: sn = serial_number_lookup[0]['value'] + 1 # construct the new id new_prepid = pwg + '-' + campaign + '-' + str(sn).zfill(5) if sn == 1: self.logger.info('Beginning new prepid family: %s' % (new_prepid)) new_request = chained_request({ '_id': new_prepid, 'prepid': new_prepid, 'pwg': pwg, 'member_of_campaign': campaign }) new_request.update_history({'action': 'created'}) creq_db.save(new_request.json()) self.serial_number_cache[(campaign, pwg)] = sn self.logger.info('New chain id: %s' % new_prepid) return new_prepid
def post(self): fails = [] for chain in loads(flask.request.data): chain_prepid = chain['prepid'] mcm_chained_request = chained_request(self.chained_requests_db.get(chain_prepid)) action_parameters = chain['action_parameters'] if not mcm_chained_request.set_priority(action_parameters['block_number']): message = 'Unable to set new priority in request %s' % chain_prepid fails.append(message) self.logger.error(message) else: mcm_chained_request.set_attribute('action_parameters', action_parameters) if not mcm_chained_request.save(): message = 'Unable to save chained request %s' % chain_prepid fails.append(message) self.logger.error(message) return { 'results': True if len(fails) == 0 else False, 'message': fails}
def multiple_inspect(self, crid): crlist = crid.rsplit(',') res = [] crdb = database('chained_requests') for cr in crlist: if crdb.document_exists(cr): mcm_cr = chained_request(crdb.get(cr)) res.append(mcm_cr.inspect()) else: res.append({ "prepid": cr, "results": False, 'message': '%s does not exist' % cr }) if len(res) > 1: return res else: return res[0]
def inspect_priority(self, forChains=None): ##JR: until put in the proper place chains = self.get_attribute('chains') crdb = database('chained_requests') okay = True for inCC in chains: ### this is the old convention #if 'flag' in chains[inCC] and chains[inCC]['flag']: # if 'chains' in chains[inCC]: # for acr in chains[inCC]['chains']: # if forChains and not acr in forChains: continue # cr=chained_request(crdb.get(acr)) # cc=cr.get_attribute('member_of_campaign') # #if 'block_number' in chains[cc] and chains[cc]['block_number']: # if chains[cc]['block_number']: # cr.set_priority(chains[cc]['block_number']) # self.logger.log('Set priority block %s to %s'%(chains[cc]['block_number'],cr.get_attribute('prepid'))) # else: # self.logger.error('Could not set block %s to %s'%(chains[cc]['block_number'],cr.get_attribute('prepid'))) ## new convention if 'chains' in chains[inCC] and type( chains[inCC]['chains']) == dict: for acr in chains[inCC]['chains']: if forChains and not acr in forChains: continue bn = chains[inCC]['chains'][acr]['block_number'] cr = chained_request(crdb.get(acr)) if bn: self.logger.log('Set priority block %s to %s' % (bn, acr)) if not cr.set_priority(bn): okay = False else: self.logger.error('Could not set block %s to %s' % (bn, acr)) rd = database('requests') if rd.document_exists(self.get_attribute('prepid')): r = request(rd.get(self.get_attribute('prepid'))) self.set_attribute('dataset_name', r.get_attribute('dataset_name')) return okay
def submit_request(self, prepid, run_test_path): mcm_request = request(self.request_db.get(prepid)) # check if the request should be validated as part of a chain for chain_prepid in mcm_request.get_attribute('member_of_chain'): mcm_chain = chained_request( self.chained_request_db.get(chain_prepid)) if mcm_chain.get_attribute( 'validate') and prepid in mcm_chain.get_attribute( 'chain')[mcm_chain.get_attribute('step'):]: return {} aux_validation = mcm_request.get_attribute(self.DOC_VALIDATION) to_write = mcm_request.get_setup_file(run=True, do_valid=True, for_validation=True) file_name = self.TEST_FILE_NAME % prepid if not self.create_test_file(to_write, run_test_path, file_name): mcm_request.set_attribute(self.DOC_VALIDATION, aux_validation) mcm_request.test_failure( message= 'There was a problem while creating the file for prepid: %s' % (mcm_request.get_attribute('prepid')), what='Validation run test', rewind=True) return {} timeout = mcm_request.get_timeout() memory = mcm_request.get_attribute("memory") threads = mcm_request.get_core_num() self.create_htcondor_config_file(run_test_path, prepid, timeout, memory, threads, [prepid]) job_info = self.execute_command_submission(prepid, run_test_path) if 'error' in job_info: mcm_request.test_failure(message=job_info['error'], what='Validation run test', rewind=True) return {} job_info[self.DOC_REV] = mcm_request.json()[self.DOC_REV] job_info[self.DOC_VALIDATION] = mcm_request.get_attribute( self.DOC_VALIDATION ) # this field change when calling request.get_setup_file, to be propagated in case of success return job_info
def flow(self, chainid, check_stats=True, reserve=False): try: db = database('chained_requests') creq = chained_request(json_input=db.get(chainid)) except Exception as ex: self.logger.error( 'Could not initialize chained_request object. Reason: %s' % (ex)) return {"results": str(ex)} # TO-DO check if chained_request is in settings forceflow_list and remove it! # if the chained_request can flow, do it if reserve: self.logger.info( 'Attempting to reserve to next step for chained_request %s' % (creq.get_attribute('_id'))) return creq.reserve(limit=reserve, save_requests=False) self.logger.info( 'Attempting to flow to next step for chained_request %s' % (creq.get_attribute('_id'))) return creq.flow_trial(check_stats=check_stats)
def post(self): fails = [] for chain in loads(flask.request.data): chain_prepid = chain['prepid'] mcm_chained_request = chained_request( self.chained_requests_db.get(chain_prepid)) action_parameters = chain['action_parameters'] if not mcm_chained_request.set_priority( action_parameters['block_number']): message = 'Unable to set new priority in request %s' % chain_prepid fails.append(message) self.logger.error(message) else: mcm_chained_request.set_attribute('action_parameters', action_parameters) if not mcm_chained_request.save(): message = 'Unable to save chained request %s' % chain_prepid fails.append(message) self.logger.error(message) return { 'results': True if len(fails) == 0 else False, 'message': fails }
def approve(self, rid, val=-1): db = database('chained_requests') if not db.document_exists(rid): return { "prepid": rid, "results": 'Error: The given chained_request id does not exist.' } creq = chained_request(json_input=db.get(rid)) try: creq.approve(val) except Exception as ex: return {"prepid": rid, "results": False, 'message': str(ex)} saved = db.update(creq.json()) if saved: return {"prepid": rid, "results": True} else: return { "prepid": rid, "results": False, 'message': 'unable to save the updated chained request' }
def get(self, chained_request_ids): """ Add selected prepid's to global force complete list for later action """ if ',' in chained_request_ids: rlist = chained_request_ids.rsplit(',') else: rlist = [chained_request_ids] res = [] __updated = False forceflow_list = self.sdb.get("list_of_forceflow") # TO-DO check if prepid exists! # TO-DO check the status of chain_req! for el in rlist: if el not in forceflow_list["value"]: forceflow_list["value"].append(el) chain_req = chained_request(self.cdb.get(el)) chain_req.update_history({'action': 'add_to_forceflow'}) self.cdb.save(chain_req.json()) res.append({"prepid": el, 'results': True, 'message': 'OK'}) __updated = True else: res.append({ "prepid": el, 'results': False, 'message': 'Chained request already in forceflow list' }) # TO-DO check the update return value if __updated: self.sdb.update(forceflow_list) return res
def get(self, chained_request_id): """ Perform test for chained requests """ crdb = database('chained_requests') rdb = database('requests') settingsDB = database('settings') mcm_cr = chained_request(crdb.get(chained_request_id)) if settingsDB.get('validation_stop')['value']: return { "results": False, 'message': ('validation jobs are halted to allow forthcoming mcm ' 'restart - try again later'), "prepid": chained_request_id } requires_validation = False for rid in mcm_cr.get_attribute( 'chain')[mcm_cr.get_attribute('step'):]: mcm_r = request(rdb.get(rid)) if not mcm_r.is_root and 'validation' not in mcm_r._json_base__status: # We dont care about non root request because they are not being used on chain run test break requires_validation = True if mcm_r.get_attribute('status') != 'new' or mcm_r.get_attribute( 'approval') != 'none': return { "results": False, "prepid": chained_request_id, "message": "request %s is in status %s, approval: %s" % (rid, mcm_r.get_attribute('status'), mcm_r.get_attribute('approval')) } try: mcm_r.ok_to_move_to_approval_validation(for_chain=True) mcm_r.update_history({ 'action': 'approve', 'step': 'validation' }) mcm_r.set_attribute('approval', 'validation') mcm_r.reset_validations_counter() mcm_r.reload() text = 'Within chain %s \n' % mcm_cr.get_attribute('prepid') text += mcm_r.textified() subject = 'Approval %s in chain %s for request %s' % ( 'validation', mcm_cr.get_attribute('prepid'), mcm_r.get_attribute('prepid')) notification(subject, text, [], group=notification.REQUEST_APPROVALS, action_objects=[mcm_r.get_attribute('prepid')], object_type='requests', base_object=mcm_r) mcm_r.notify(subject, text, accumulate=True) except Exception as e: mcm_cr.reset_requests(str(e), notify_one=rid) return { "results": False, "message": str(e), "prepid": chained_request_id } if not requires_validation: return { "results": True, "message": "No validation required", "prepid": chained_request_id } mcm_cr.set_attribute('validate', 1) if not crdb.update(mcm_cr.json()): return { "results": False, "message": "Failed while trying to update the document in DB", "prepid": chained_request_id } return { "results": True, "message": "run test will start soon", "prepid": chained_request_id }
def internal_run(self): if not self.lock.acquire(blocking=False): self.logger.error( "Could not acquire lock for ChainRequestInjector. prepid %s" % (self.prepid)) return False try: crdb = database('chained_requests') rdb = database('requests') batch_name = None if not crdb.document_exists(self.prepid): # it's a request actually, pick up all chains containing it mcm_r = rdb.get(self.prepid) # mcm_crs = crdb.query(query="root_request==%s"% self.prepid) ## not only when its the root of mcm_crs = crdb.query(query="contains==%s" % self.prepid) task_name = 'task_' + self.prepid batch_type = 'Task_' + mcm_r['member_of_campaign'] else: mcm_crs = [crdb.get(self.prepid)] current_step_prepid = mcm_crs[0]['chain'][mcm_crs[0]['step']] mcm_request = rdb.get(current_step_prepid) task_name = 'task_' + current_step_prepid batch_type = 'Task_' + mcm_request['member_of_campaign'] if len(mcm_crs) == 0: return False mcm_rs = [] # upload all config files to config cache, with "configuration economy" already implemented for cr in mcm_crs: mcm_cr = chained_request(cr) chain = mcm_cr.get_attribute( 'chain')[mcm_cr.get_attribute('step'):] for request_prepid in chain: mcm_rs.append(request(rdb.get(request_prepid))) if self.check_approval and mcm_rs[-1].get_attribute( 'approval') != 'submit': message = 'requests %s is in "%s"/"%s" status/approval, requires "approved"/"submit"' % ( request_prepid, mcm_rs[-1].get_attribute('status'), mcm_rs[-1].get_attribute('approval')) self.logger.error(message) subject = '%s injection failed' % mcm_cr.get_attribute( 'prepid') notification( subject, message, [], group=notification.CHAINED_REQUESTS, action_objects=[mcm_cr.get_attribute('prepid')], object_type='chained_requests', base_object=mcm_cr) mcm_cr.notify(subject, message) return False if mcm_rs[-1].get_attribute('status') != 'approved': # change the return format to percolate the error message message = 'requests %s in in "%s"/"%s" status/approval, requires "approved"/"submit"' % ( request_prepid, mcm_rs[-1].get_attribute('status'), mcm_rs[-1].get_attribute('approval')) self.logger.error(message) subject = '%s injection failed' % mcm_cr.get_attribute( 'prepid') notification( subject, message, [], group=notification.CHAINED_REQUESTS, action_objects=[mcm_cr.get_attribute('prepid')], object_type='chained_requests', base_object=mcm_cr) mcm_cr.notify(subject, message) return False uploader = ConfigMakerAndUploader( prepid=request_prepid, lock=locker.lock(request_prepid)) if not uploader.internal_run(): message = 'Problem with uploading the configuration for request %s' % ( request_prepid) notification( 'Configuration upload failed', message, [], group=notification.CHAINED_REQUESTS, action_objects=[mcm_cr.get_attribute('prepid')], object_type='chained_requests', base_object=mcm_cr) mcm_cr.notify('Configuration upload failed', message) self.logger.error(message) return False mcm_r = mcm_rs[-1] batch_name = BatchPrepId().next_batch_id(batch_type, create_batch=True) semaphore_events.increment(batch_name) self.logger.error('found batch %s' % batch_name) with ssh_executor(server='vocms081.cern.ch') as ssh: cmd = self.make_command(mcm_r) self.logger.error('prepared command %s' % cmd) # modify here to have the command to be executed _, stdout, stderr = ssh.execute(cmd) output = stdout.read() error = stderr.read() self.logger.info(output) self.logger.info(error) injected_requests = [ l.split()[-1] for l in output.split('\n') if l.startswith('Injected workflow:') ] if not injected_requests: self.injection_error( 'Injection has succeeded but no request manager names were registered. Check with administrators. \nOutput: \n%s\n\nError: \n%s' % (output, error), mcm_rs) return False # what gets printed into the batch object added_requests = [] once = set() for mcm_r in mcm_rs: if mcm_r.get_attribute('prepid') in once: continue once.add(mcm_r.get_attribute('prepid')) added = [{ 'name': app_req, 'content': { 'pdmv_prep_id': mcm_r.get_attribute('prepid') } } for app_req in injected_requests] added_requests.extend(added) # edit the batch object with locker.lock(batch_name): bdb = database('batches') bat = batch(bdb.get(batch_name)) bat.add_requests(added_requests) bat.update_history({ 'action': 'updated', 'step': task_name }) bat.reload() # reload the content of all requests as they might have changed already added = [{ 'name': app_req, 'content': { 'pdmv_prep_id': task_name } } for app_req in injected_requests] seen = set() for cr in mcm_crs: mcm_cr = chained_request(cr) chain = mcm_cr.get_attribute( 'chain')[mcm_cr.get_attribute('step'):] message = "" for rn in chain: if rn in seen: continue # don't do it twice seen.add(rn) mcm_r = request(rdb.get(rn)) message += mcm_r.textified() message += "\n\n" mcm_r.set_attribute('reqmgr_name', added) mcm_r.update_history({ 'action': 'inject', 'step': batch_name }) if not self.check_approval: mcm_r.set_attribute('approval', 'submit') # set the status to submitted mcm_r.set_status( step=mcm_r._json_base__status.index('submitted'), with_notification=False) mcm_r.reload() mcm_cr.set_attribute('last_status', mcm_r.get_attribute('status')) # re-get the object mcm_cr = chained_request(crdb.get(cr['prepid'])) # take care of changes to the chain mcm_cr.update_history({ 'action': 'inject', 'step': batch_name }) mcm_cr.set_attribute( 'step', len(mcm_cr.get_attribute('chain')) - 1) mcm_cr.set_attribute('status', 'processing') subject = 'Injection succeeded for %s' % mcm_cr.get_attribute( 'prepid') notification( subject, message, [], group=notification.CHAINED_REQUESTS, action_objects=[mcm_cr.get_attribute('prepid')], object_type='chained_requests', base_object=mcm_cr) mcm_cr.notify(subject, message) mcm_cr.reload() return True except Exception: self.injection_error( "Error with injecting chains for %s :\n %s" % (self.prepid, traceback.format_exc()), []) finally: # we decrement batch id and release lock on prepid+lower semaphore if batch_name: # ditry thing for now. Because batch name can be None for certain use-cases in code above semaphore_events.decrement(batch_name) self.lock.release() self.queue_lock.release()
def delete_request(self, crid): crdb = database('chained_requests') rdb = database('requests') mcm_cr = chained_request(crdb.get(crid)) # get all objects mcm_r_s = [] for (i, rid) in enumerate(mcm_cr.get_attribute('chain')): mcm_r = request(rdb.get(rid)) # this is not a valid check as it is allowed to remove a chain around already running requests # if mcm_r.get_attribute('status') != 'new': # return {"results":False,"message" : "the request %s part of the chain %s for action %s is not in new status"%( mcm_r.get_attribute('prepid'), # crid, # mcm_a.get_attribute('prepid'))} in_chains = mcm_r.get_attribute('member_of_chain') in_chains.remove(crid) self.logger.debug( "Removing ChainAction member_of_chain: %s to request: %s" % (mcm_cr.get_attribute("prepid"), mcm_r.get_attribute('prepid'))) mcm_r.set_attribute('member_of_chain', in_chains) if i == 0: if len(in_chains ) == 0 and mcm_r.get_attribute('status') != 'new': return { "results": False, "message": "the request %s, not in status new, at the root of the chain will not be chained anymore" % rid } else: if len(in_chains) == 0: return { "results": False, "message": "the request %s, not at the root of the chain will not be chained anymore" % rid } mcm_r.update_history({'action': 'leave', 'step': crid}) mcm_r_s.append(mcm_r) if mcm_cr.get_attribute('action_parameters')['flag']: return { "results": False, "message": "the action for %s is not disabled" % (crid) } # then save all changes for mcm_r in mcm_r_s: if not rdb.update(mcm_r.json()): return { "results": False, "message": "Could not save request " + mcm_r.get_attribute('prepid') } else: subject = "Request {0} left chain".format( mcm_r.get_attribute('prepid')) message = "Request {0} has successfuly left chain {1}".format( mcm_r.get_attribute('prepid'), crid) notification(subject, message, [], group=notification.REQUEST_OPERATIONS, action_objects=[mcm_r.get_attribute('prepid')], object_type='requests', base_object=mcm_r) mcm_r.notify(subject, message) return {"results": crdb.delete(crid)}
def generate_request(self, aid, reserve=False, with_notify=True, special=False): adb = database('actions') ccdb = database('chained_campaigns') crdb = database('chained_requests') rdb = database('requests') if not adb.document_exists(aid): return {'results': False, 'message': '%s does not exist' % (aid)} self.logger.log( 'Generating all selected chained_requests for action %s' % (aid)) mcm_a = action(adb.get(aid)) chains = mcm_a.get_attribute('chains') hasChainsChanged = False new_chains = [] for cc in chains: if 'flag' in chains[cc] and chains[cc]['flag']: ## in the new convention, that means that something needs to be created mcm_cc = chained_campaign(ccdb.get(cc)) new_cr = mcm_cc.generate_request(aid) if not 'chains' in chains[cc]: chains[cc]['chains'] = {} chains[cc]['chains'][new_cr['prepid']] = {} to_transfer = ['flag', 'threshold', 'staged', 'block_number'] for item in to_transfer: if item in chains[cc]: chains[cc]['chains'][ new_cr['prepid']][item] = chains[cc][item] chains[cc].pop(item) hasChainsChanged = True ## get the root request req = request(json_input=rdb.get(aid)) new_cr['last_status'] = req.get_attribute('status') if new_cr['last_status'] in ['submitted', 'done']: new_cr['status'] = 'processing' if special: new_cr['approval'] = 'none' if not crdb.update(new_cr): return { 'results': False, 'message': 'could not save %s' % (new_cr['prepid']) } ## update the cc history ccdb.update(mcm_cc.json()) new_chains.append(new_cr['prepid']) # then let the root request know that it is part of a chained request inchains = req.get_attribute('member_of_chain') inchains.append(new_cr['prepid']) inchains.sort() req.set_attribute('member_of_chain', list(set(inchains))) req.update_history({ 'action': 'join chain', 'step': new_cr['prepid'] }) if with_notify: req.notify( "Request {0} joined chain".format( req.get_attribute('prepid')), "Request {0} has successfully joined chain {1}".format( req.get_attribute('prepid'), new_cr['prepid']), Nchild=0, accumulate=True) mcm_a.update_history({ 'action': 'add', 'step': new_cr['prepid'] }) rdb.update(req.json()) if hasChainsChanged: #the chains parameter might have changed mcm_a.set_attribute('chains', chains) adb.update(mcm_a.json()) #and set priorities properly to all requests concerned priority_set = mcm_a.inspect_priority(forChains=new_chains) ## do the reservation of the whole chain ? res = [] if reserve: for cr in new_chains: mcm_cr = chained_request(crdb.get(cr)) res.append(mcm_cr.reserve()) crdb.update(mcm_cr.json()) if priority_set: return {"results": True, "prepid": mcm_a.get_attribute('prepid')} else: return { "results": False, "prepid": mcm_a.get_attribute('prepid'), "message": "Priorities not set properly" }
def generate_chained_requests(self, mccm_ticket, request_prepid, mcm_chained_campaign, reserve=False, with_notify=True, special=False): try: mcm_chained_campaign.reload(save_current=False) generated_chained_request = chained_request(mcm_chained_campaign.generate_request(request_prepid)) except Exception as e: message = "Unable to generate chained request for ticket %s request %s, message: " % (mccm_ticket.get_attribute('prepid'), request_prepid, str(e)) self.logger.error(message) return { "results": False, "message": message} requests_db = database('requests') self.overwrite_action_parameters_from_ticket(generated_chained_request, mccm_ticket) mcm_request = request(json_input=requests_db.get(request_prepid)) generated_chained_request.set_attribute('last_status', mcm_request.get_attribute('status')) if generated_chained_request.get_attribute('last_status') in ['submitted', 'done']: generated_chained_request.set_attribute('status', 'processing') if special: generated_chained_request.set_attribute('approval', 'none') new_chain_prepid = generated_chained_request.get_attribute('prepid') if not generated_chained_request.reload(): return { 'results': False, 'message': 'Unable to save chained request %s' % new_chain_prepid} # update the history of chained campaign mcm_chained_campaign.save() # let the root request know that it is part of a chained request chains = mcm_request.get_attribute('member_of_chain') chains.append(new_chain_prepid) chains.sort() mcm_request.set_attribute('member_of_chain', list(set(chains))) mcm_request.update_history({'action': 'join chain', 'step': new_chain_prepid}) if with_notify: subject = "Request %s joined chain" % mcm_request.get_attribute('prepid') message = "Request %s has successfully joined chain %s" % (mcm_request.get_attribute('prepid'), new_chain_prepid) notification( subject, message, [], group=notification.REQUEST_OPERATIONS, action_objects=[mcm_request.get_attribute('prepid')], object_type='requests', base_object=mcm_request) mcm_request.notify( subject, message, Nchild=0, accumulate=True) mcm_request.save() # do the reservation of the whole chain ? generated_requests = [] if reserve: results_dict = generated_chained_request.reserve(limit=reserve, save_requests=False) if results_dict['results'] and 'generated_requests' in results_dict: generated_requests = results_dict['generated_requests'] results_dict.pop('generated_requests') else: return { "results": False, "prepid": new_chain_prepid, "message": results_dict['message']} return { "results":True, "prepid": new_chain_prepid, 'generated_requests': generated_requests}
def process_finished_chain_success(self, prepid, doc_info, job_out, error_out, log_out): mcm_chained_request = chained_request( self.chained_request_db.get(prepid)) requests_in_chain = [] success = True failed_request_prepid = None for request_prepid, doc_rev in doc_info[ self.CHAIN_REQUESTS].iteritems(): mcm_request = request(self.request_db.get(request_prepid)) # Increase counters for all requests, but save error message only for the first failed request if success and doc_rev != mcm_request.json()[self.DOC_REV]: message = 'The request %s in the chain %s has changed during the run test procedure, preventing from being saved' % ( request_prepid, prepid) success = False failed_request_prepid = request_prepid mcm_request.inc_validations_counter() mcm_request.reload() if not success: mcm_chained_request.reset_requests( message, notify_one=failed_request_prepid) return for request_prepid, doc_rev in doc_info[ self.CHAIN_REQUESTS].iteritems(): self.logger.info('Processing request %s in chain %s' % (request_prepid, prepid)) mcm_request = request(self.request_db.get(request_prepid)) success = True path = self.test_directory_path + prepid + '/' try: success, error = mcm_request.pickup_all_performance(path) error = 'Error:\n%s\n Error out:\n%s\n' % (error, error_out) except request.WrongTimeEvent as ex: self.logger.error('Exception: %s' % (ex)) error = 'Exception:\n%s\n' % (ex) success = False retry_validation = self.process_request_wrong_time_event( mcm_request, prepid) if retry_validation: return if not success: message = 'Error while picking up all the performance for request %s of chain %s: \n Error:\n%s\n Job out:\n%s\n Error out: \n%s\n Log out: \n%s\n' % ( request_prepid, prepid, error, job_out, error_out, log_out) mcm_chained_request.reset_requests(message, notify_one=request_prepid) return requests_in_chain.append(mcm_request) for mcm_request in requests_in_chain: mcm_request.set_status(with_notification=True) if not self.request_db.update(mcm_request.json()): request_prepid = mcm_request.get_attribute('prepid') message = "The request %s of chain %s could not be saved after the runtest procedure" % ( request_prepid, prepid) self.logger.error(message) # reset it and keep saving requests mcm_request.test_failure(message=message, what='Chain validation run test', rewind=True) mcm_chained_request.reload( save_current=False ) # setting new requests status change the chain object mcm_chained_request.set_attribute('validate', 0) if not self.chained_request_db.update(mcm_chained_request.json()): message = 'Problem saving changes in chain %s, set validate = False ASAP!' % prepid self.logger.error(message) notification( 'Chained validation run test', message, [], group=notification.CHAINED_REQUESTS, action_objects=[mcm_chained_request.get_attribute('prepid')], object_type='chained_requests', base_object=mcm_chained_request) mcm_chained_request.notify('Chained validation run test', message) return self.logger.info('Validation job for prepid %s SUCCESSFUL!!!' % prepid)
def submit_chain(self, prepid, run_test_path): mcm_chained_request = chained_request( self.chained_request_db.get(prepid)) except_requests = [] reset = False # If a request of a chain was singly submmited to validation and then somebody reseted it, we will find it here for request_prepid in mcm_chained_request.get_attribute( 'chain')[mcm_chained_request.get_attribute('step'):]: if request_prepid in self.submmited_prepids_set: except_requests.append(request_prepid) reset = True if reset: message = "Requests %s of the chain %s are already in validation" % ( except_requests, prepid) self.logger.error(message) mcm_chained_request.reset_requests(message, except_requests=except_requests) return {} to_write = mcm_chained_request.get_setup(run=True, validation=True, for_validation=True) file_name = self.TEST_FILE_NAME % prepid if not self.create_test_file(to_write, run_test_path, file_name): mcm_chained_request.reset_requests( 'There was a problem while creating the file for prepid: %s' % (mcm_chained_request.get_attribute('prepid'))) return {} requests_in_chain = {} for request_prepid in mcm_chained_request.get_attribute( 'chain')[mcm_chained_request.get_attribute('step'):]: mcm_request = request(self.request_db.get(request_prepid)) if not mcm_request.is_root and 'validation' not in mcm_request._json_base__status: # only root or possible root requests break status = mcm_request.get_attribute('status') approval = mcm_request.get_attribute('approval') if status != 'new' or approval != 'validation': message = "The request %s of chain %s is in status: %s approval: %s" % ( request_prepid, prepid, status, approval) self.logger.error(message) mcm_chained_request.reset_requests(message) return {} requests_in_chain[request_prepid] = mcm_request.json()[ self.DOC_REV] if not len(requests_in_chain): message = 'No requests to be validated in chain: %s' % prepid self.logger.info(message) mcm_chained_request.reset_requests(message) return {} timeout, memory, threads = mcm_chained_request.get_timeout_memory_threads( ) self.create_htcondor_config_file(run_test_path, prepid, timeout, memory, threads, list(requests_in_chain.iterkeys())) job_info = self.execute_command_submission(prepid, run_test_path) if 'error' in job_info: mcm_chained_request.reset_requests(job_info['error']) return {} job_info[self.CHAIN_REQUESTS] = requests_in_chain return job_info
def rewind_one(self, crid): crdb = database('chained_requests') rdb = database('requests') if not crdb.document_exists(crid): return { "results": False, "message": "does not exist", "prepid": crid } mcm_cr = chained_request(crdb.get(crid)) current_step = mcm_cr.get_attribute('step') if current_step == 0: # or should it be possible to cancel the initial requests of a chained request return { "results": False, "message": "already at the root", "prepid": crid } # supposedly all the other requests were already reset! for next in mcm_cr.get_attribute('chain')[current_step + 1:]: # what if that next one is not in the db if not rdb.document_exists(next): self.logger.error('%s is part of %s but does not exist' % (next, crid)) continue mcm_r = request(rdb.get(next)) if mcm_r.get_attribute('status') != 'new': # this cannot be right! self.logger.error( '%s is after the current request and is not new: %s' % (next, mcm_r.get_attribute('status'))) return { "results": False, "message": "%s is not new" % (next), "prepid": crid } # get the one to be reset current_id = mcm_cr.get_attribute('chain')[current_step] mcm_r = request(rdb.get(current_id)) mcm_r.reset() saved = rdb.update(mcm_r.json()) if not saved: { "results": False, "message": "could not save the last request of the chain", "prepid": crid } # the current chained request has very likely been updated : # reload it as you have not changed anything to it yet mcm_cr = chained_request(crdb.get(crid)) mcm_cr.set_attribute('step', current_step - 1) # set status, last status mcm_cr.set_last_status() mcm_cr.set_attribute('status', 'processing') saved = crdb.update(mcm_cr.json()) if saved: return {"results": True, "prepid": crid} else: return { "results": False, "message": "could not save chained requests. the DB is going to be inconsistent !", "prepid": crid }