def get_requests(self, mcm_r):
ssh = ssh_executor(location.location(), mcm_r.get_attribute('prepid'))
stdin, stdout, stderr = ssh.execute('bash %s' % test_script)
fullOutPutText = stdout.read()
error = stderr.read()
Exceptions = []
for line in error.split('\n'):
if '[wmcontrol exception]' in line:
Exceptions.append(line)
if len(Exceptions):
self.logger.error('Executed \n %s' % fullOutPutText)
self.logger.error('Errors returned: %s' % error)
return False
requestNames = []
for line in fullOutPutText.split('\n'):
if line.startswith('Injected workflow:'):
requestNames.append(line.split()[2])
if not len(requestNames):
self.logger.error('There were no request manager name recorded \n %s' % fullOutPutText)
return False
self.logger.log('Injection output: %s' % fullOutPutText)
return True
def get_requests(self, mcm_r):
ssh = ssh_executor(location.location(), mcm_r.get_attribute('prepid'))
stdin, stdout, stderr = ssh.execute('bash %s' % test_script)
fullOutPutText = stdout.read()
error = stderr.read()
Exceptions = []
for line in error.split('\n'):
if '[wmcontrol exception]' in line:
Exceptions.append(line)
if len(Exceptions):
self.logger.error('Executed \n %s' % fullOutPutText)
self.logger.error('Errors returned: %s' % error)
return False
requestNames = []
for line in fullOutPutText.split('\n'):
if line.startswith('Injected workflow:'):
requestNames.append(line.split()[2])
if not len(requestNames):
self.logger.error(
'There were no request manager name recorded \n %s' %
fullOutPutText)
return False
self.logger.log('Injection output: %s' % fullOutPutText)
return True
def internal_run(self):
command = self.make_command()
executor = ssh_executor(server='vocms081.cern.ch')
try:
self.logger.info("Command being used for approve requests: " +
command)
trails = 1
while trails < 3:
self.logger.info("Wmapprove trail number: %s" % trails)
_, stdout, stderr = executor.execute(command)
if not stdout and not stderr:
self.logger.error(
'ssh error for request approvals, batch id: ' +
self.batch_id)
return
output = stdout.read()
error = stderr.read()
self.logger.info('Wmapprove output: %s' % output)
if not error and 'Something went wrong' not in output:
break
time.sleep(3)
trails += 1
if error or 'Something went wrong' in output:
message = 'Error in wmapprove: %s' % (
output if 'Something went wrong' in output else error)
self.logger.error(message)
self.send_email_failure(output, error)
return {'results': False, 'message': message}
except Exception as e:
message = 'Error while approving requests, batch id: %s, message: %s' % (
self.batch_id, str(e))
self.logger.error(message)
self.send_email_failure('', message)
return {'results': False, 'message': message}
return {'results': True}
def __init__(self, inspect=False, force=True):
self.ssh = ssh_executor(server='cmsdev04.cern.ch')
self.inspect = ""
if inspect:
self.inspect = '--inspect'
self.force = ""
if force:
self.force = '--force'
def __init__(self, inspect=False, force=True):
self.ssh = ssh_executor(server='cmsdev13.cern.ch')
self.inspect = ""
if inspect:
self.inspect = '--inspect'
self.force = ""
if force:
self.force = '--force'
def GET(self, *args):
"""
Renew certificate on pdmvserv-test.cern.ch
"""
ssh_exec = ssh_executor(server='pdmvserv-test.cern.ch')
try:
self.logger.log("Renewing certificate")
stdin, stdout, stderr = ssh_exec.execute(self.create_command())
self.logger.log("Certificate renewed:\n{0}".format(stdout.read()))
finally:
ssh_exec.close_executor()
def GET(self, *args):
"""
Renew certificate on pdmvserv-test.cern.ch
"""
ssh_exec = ssh_executor(server='pdmvserv-test.cern.ch')
try:
self.logger.log("Renewing certificate")
stdin, stdout, stderr = ssh_exec.execute(self.create_command())
self.logger.log("Certificate renewed:\n{0}".format(stdout.read()))
finally:
ssh_exec.close_executor()
def GET(self, *args):
"""
Renew certificate on pdmvserv-test.cern.ch
"""
machines = ["pdmvserv-test.cern.ch", "cms-pdmv-op.cern.ch"]
for elem in machines:
ssh_exec = ssh_executor(server=elem)
try:
self.logger.log("Renewing certificate for: %s" % (elem))
stdin, stdout, stderr = ssh_exec.execute(self.create_command(elem))
self.logger.log("Certificate renewed:\n{0}".format(stdout.read()))
finally:
ssh_exec.close_executor()
def GET(self, *args):
"""
this is test of connection through ssh to /node/iterations
"""
sta = 0
sto = 0
spend = []
server = args[0]
N = int(args[1])
success = 0
fail = 0
for i in range(N):
sta = time.time()
connect = ssh_executor(server=server)
try:
_, stdout, stderr = connect.execute('ls $HOME')
out = stdout.read()
err = stderr.read()
if ('private' in out and 'public' in out and 'PdmV' in out):
success += 1
self.logger.error(
"test: %s SUCCESS \n out:\n%s \n err:\n %s" %
(i, out, err))
else:
tail += 1
self.logger.error(
"test: %s failed \n out:\n%s \n err:\n %s" %
(i, out, err))
except:
fail += 1
self.logger.error("test: %s failed %s" %
(i, traceback.format_exc()))
sto = time.time()
spend.append(sto - sta)
mean = sum(spend) / len(spend)
rms = sqrt(
sum(map(lambda v: (v - mean) * (v - mean), spend)) / len(spend))
return dumps({
"server": server,
"trials": N,
"time": spend,
"mean": mean,
"rms": rms,
"success": success,
"fail": fail,
"max": max(spend),
"min": min(spend),
"total": sum(spend)
})
def get(self):
"""
Renew certificates on our request upload/injection machines
"""
# machines = ["cms-pdmv-op.cern.ch"]
machines = ["vocms081.cern.ch"]
for elem in machines:
ssh_exec = ssh_executor(server=elem)
try:
self.logger.info("Renewing certificate for: %s" % (elem))
stdin, stdout, stderr = ssh_exec.execute(self.create_command(elem))
self.logger.info("Certificate renewed:\n{0}".format(stdout.read()))
finally:
ssh_exec.close_executor()
def get(self, options=''):
"""
Get bjobs information regarding the condor clusters
"""
ssh_exec = ssh_executor()
try:
stdin, stdout, stderr = ssh_exec.execute(self.create_command(options))
out = stdout.read()
err = stderr.read()
if err:
return {"results": err}
return {"results": out}
finally:
ssh_exec.close_executor()
def get(self, options=''):
"""
Get bjobs information regarding the condor clusters
"""
ssh_exec = ssh_executor()
try:
stdin, stdout, stderr = ssh_exec.execute(self.create_command(options))
out = stdout.read()
err = stderr.read()
if err:
return {"results": err}
return {"results": out}
finally:
ssh_exec.close_executor()
def get(self):
"""
Renew certificates on our request upload/injection machines
"""
# machines = ["cms-pdmv-op.cern.ch"]
machines = ["vocms081.cern.ch"]
for elem in machines:
ssh_exec = ssh_executor(server=elem)
try:
self.logger.info("Renewing certificate for: %s" % (elem))
stdin, stdout, stderr = ssh_exec.execute(
self.create_command(elem))
self.logger.info("Certificate renewed:\n{0}".format(
stdout.read()))
finally:
ssh_exec.close_executor()
def GET(self, *args):
"""
Get bjobs information regarding the batch jobs
"""
ssh_exec = ssh_executor()
try:
stdin, stdout, stderr = ssh_exec.execute(self.create_command(args))
out = stdout.read()
err = stderr.read()
if err:
if "No job found in job group" in err: # so the shown string is consistent with production
return dumps({"results": 'No unfinished job found'})
return dumps({"results": err})
return dumps({"results": out})
finally:
ssh_exec.close_executor()
def GET(self, *args):
"""
Get bjobs information regarding the batch jobs
"""
ssh_exec = ssh_executor()
try:
stdin, stdout, stderr = ssh_exec.execute(self.create_command(args))
out = stdout.read()
err = stderr.read()
if err:
if "No job found in job group" in err: # so the shown string is consistent with production
return dumps({"results": 'No unfinished job found'})
return dumps({"results": err})
return dumps({"results": out})
finally:
ssh_exec.close_executor()
def __init__(self):
self.setup_directories()
self.setup_logger()
self.get_submmited_prepids()
self.batch_retry_timeout = settings.get_value('batch_retry_timeout')
self.check_term_runlimit = settings.get_value('check_term_runlimit')
try:
self.ssh_exec = ssh_executor()
except Exception as e:
self.ssh_exec = None
self.logger.error(str(e) + 'traceback %s ' % traceback.format_exc())
return
if locator().isDev():
self.group = '/dev'
else:
self.group = '/prod'
def __init__(self):
self.setup_directories()
self.setup_logger()
self.get_submmited_prepids()
self.batch_retry_timeout = settings.get_value('batch_retry_timeout')
self.check_term_runlimit = settings.get_value('check_term_runlimit')
try:
self.ssh_exec = ssh_executor()
except Exception as e:
self.ssh_exec = None
self.logger.error(
str(e) + 'traceback %s ' % traceback.format_exc())
return
if locator().isDev():
self.group = '/dev'
else:
self.group = '/prod'
def __init__(self, test_id, test_script):
self.script_for_test = test_script
self.test_id = test_id
self.test_err = os.path.abspath( self.script_for_test + '.err')
self.test_out = os.path.abspath( self.script_for_test + '.out')
locat = locator()
if locat.isDev():
self.group = '/dev'
else:
self.group = '/prod'
self.directory_for_test = os.path.dirname(self.script_for_test)
self.ssh_exec = ssh_executor(self.directory_for_test, self.test_id)
self.log_out = 'Not available'
self.log_err = 'Not available'
self.timeout = settings().get_value('batch_timeout')
def GET(self, *args):
"""
this is test of connection through ssh to /node/iterations
"""
sta=0
sto=0
spend=[]
server=args[0]
N=int(args[1])
success=0
fail=0
for i in range(N):
sta=time.time()
connect = ssh_executor(server=server)
try:
_, stdout, stderr = connect.execute('ls $HOME')
out = stdout.read()
err = stderr.read()
if ('private' in out and 'public' in out and 'PdmV' in out):
success+=1
self.logger.error("test: %s SUCCESS \n out:\n%s \n err:\n %s"%( i, out,err))
else:
tail+=1
self.logger.error("test: %s failed \n out:\n%s \n err:\n %s"%( i, out,err))
except:
fail+=1
self.logger.error("test: %s failed %s"%(i, traceback.format_exc()))
sto=time.time()
spend.append( sto - sta )
mean= sum(spend) / len(spend)
rms= sqrt(sum( map( lambda v : (v-mean)*(v-mean), spend)) / len(spend))
return dumps({"server" : server,
"trials" : N,
"time": spend,
"mean": mean,
"rms": rms,
"success" : success,
"fail" : fail,
"max" : max(spend),
"min": min(spend),
"total" : sum(spend)
})
def __init__(self, test_id, test_script, timeout=None):
self.script_for_test = test_script
self.test_id = test_id
self.test_err = os.path.abspath( self.script_for_test + '.err')
self.test_out = os.path.abspath( self.script_for_test + '.out')
locat = locator()
if locat.isDev():
self.group = '/dev'
else:
self.group = '/prod'
self.directory_for_test = os.path.dirname(self.script_for_test)
self.ssh_exec = ssh_executor(self.directory_for_test, self.test_id)
self.log_out = 'Not available'
self.log_err = 'Not available'
self.timeout = int(settings().get_value('batch_timeout'))
self.queue = '8nh'
if timeout:
self.timeout = int(timeout / 60.)
if (self.timeout / 3600. ) > 8.:
self.queue = '1nd' ## fall back to the one day queue at worse
def __init__(self, **kwargs):
Handler.__init__(self, **kwargs)
self.prepid = kwargs["prepid"]
self.check_approval = kwargs["check_approval"] if "check_approval" in kwargs else True
self.request_db = database('requests')
self.ssh_executor = ssh_executor(server='pdmvserv-test.cern.ch')
def internal_run(self):
if not self.lock.acquire(blocking=False):
self.logger.error("Could not acquire lock for ChainRequestInjector. prepid %s" % (
self.prepid))
return False
try:
crdb = database('chained_requests')
rdb = database('requests')
batch_name = None
if not crdb.document_exists( self.prepid ):
## it's a request actually, pick up all chains containing it
mcm_r = rdb.get( self.prepid )
#mcm_crs = crdb.query(query="root_request==%s"% self.prepid) ## not only when its the root of
mcm_crs = crdb.query(query="contains==%s" % self.prepid)
task_name = 'task_' + self.prepid
batch_type = 'Task_' + mcm_r['member_of_campaign']
else:
mcm_crs = [crdb.get( self.prepid )]
task_name = self.prepid
batch_type = mcm_crs[-1]['member_of_campaign']
if len(mcm_crs)==0:
return False
mcm_rs=[]
## upload all config files to config cache, with "configuration economy" already implemented
for cr in mcm_crs:
mcm_cr = chained_request(cr)
chain = mcm_cr.get_attribute('chain')[mcm_cr.get_attribute('step'):]
for rn in chain:
mcm_rs.append(request(rdb.get(rn)))
if self.check_approval and mcm_rs[-1].get_attribute('approval') != 'submit':
self.logger.error('requests %s is in "%s"/"%s" status/approval, requires "approved"/"submit"'%(
rn, mcm_rs[-1].get_attribute('status'),
mcm_rs[-1].get_attribute('approval')))
return False
if mcm_rs[-1].get_attribute('status') != 'approved':
## change the return format to percolate the error message
self.logger.error('requests %s in in "%s"/"%s" status/approval, requires "approved"/"submit"'%(
rn, mcm_rs[-1].get_attribute('status'),
mcm_rs[-1].get_attribute('approval')))
return False
uploader = ConfigMakerAndUploader(prepid=rn, lock=locker.lock(rn))
uploader.internal_run() ## we run in same thread that we locked in start
mcm_r = mcm_rs[-1]
batch_name = BatchPrepId().next_batch_id(batch_type, create_batch=True)
semaphore_events.increment(batch_name)
self.logger.error('found batch %s'% batch_name)
with ssh_executor(server = 'cms-pdmv-op.cern.ch') as ssh:
cmd = self.make_command(mcm_r)
self.logger.error('prepared command %s' % cmd)
## modify here to have the command to be executed
_, stdout, stderr = ssh.execute(cmd)
output = stdout.read()
error = stderr.read()
self.logger.log(output)
self.logger.log(error)
injected_requests = [l.split()[-1] for l in output.split('\n') if
l.startswith('Injected workflow:')]
approved_requests = [l.split()[-1] for l in output.split('\n') if
l.startswith('Approved workflow:')]
if not injected_requests:
self.injection_error('Injection has succeeded but no request manager names were registered. Check with administrators. \nOutput: \n%s\n\nError: \n%s'%(
output, error), mcm_rs)
return False
if injected_requests and not approved_requests:
self.injection_error("Request %s was injected but could not be approved" % (
injected_requests), mcm_rs)
## yet again...
objects_to_invalidate = [
{"_id": inv_req, "object": inv_req, "type": "request",
"status": "new", "prepid": self.prepid}
for inv_req in injected_requests if inv_req not in approved_requests]
if objects_to_invalidate:
self.logger.error("Some requests %s need to be invalidated" % (
objects_to_invalidate))
invalidation = database('invalidation')
saved = invalidation.save_all(objects_to_invalidate)
if not saved:
self.logger.error('Could not save the invalidations {0}'.format(
objects_to_invalidate))
return False
# what gets printed into the batch object
added_requests = []
once=set()
for mcm_r in mcm_rs:
if mcm_r.get_attribute('prepid') in once: continue
once.add(mcm_r.get_attribute('prepid'))
added = [{'name': app_req,
'content': {'pdmv_prep_id': mcm_r.get_attribute('prepid')}}
for app_req in approved_requests]
added_requests.extend(added)
##edit the batch object
with locker.lock(batch_name):
bdb = database('batches')
bat = batch(bdb.get(batch_name))
bat.add_requests(added_requests)
bat.update_history({'action': 'updated', 'step': task_name })
bat.reload()
## reload the content of all requests as they might have changed already
added = [{'name': app_req, 'content': {'pdmv_prep_id': task_name }}
for app_req in approved_requests]
seen = set()
for cr in mcm_crs:
mcm_cr = chained_request(cr)
chain = mcm_cr.get_attribute('chain')[mcm_cr.get_attribute('step'):]
message = ""
for rn in chain:
if rn in seen: continue # don't do it twice
seen.add(rn)
mcm_r = request(rdb.get(rn))
message += mcm_r.textified()
message += "\n\n"
mcm_r.set_attribute('reqmgr_name', added)
mcm_r.update_history({'action': 'inject','step' : batch_name})
if not self.check_approval:
mcm_r.set_attribute('approval', 'submit')
mcm_r.set_status(with_notification=False)
mcm_r.reload()
mcm_cr.set_attribute('last_status', mcm_r.get_attribute('status'))
## re-get the object
mcm_cr = chained_request(crdb.get(cr['prepid']))
#take care of changes to the chain
mcm_cr.update_history({'action' : 'inject','step': batch_name})
mcm_cr.set_attribute('step', len(mcm_cr.get_attribute('chain'))-1)
mcm_cr.set_attribute('status','processing')
mcm_cr.notify('Injection succeeded for %s' % mcm_cr.get_attribute('prepid'),
message)
mcm_cr.reload()
return True
except Exception as e:
self.injection_error("Error with injecting chains for %s :\n %s" % (
self.prepid, traceback.format_exc()),[])
finally: ##we decrement batch id and release lock on prepid+lower semaphore
if batch_name: ##ditry thing for now. Because batch name can be None for certain use-cases in code above
semaphore_events.decrement(batch_name)
self.lock.release()
self.queue_lock.release()
def internal_run(self):
try:
if not self.lock.acquire(blocking=False):
self.injection_error('Couldnt acquire lock', None)
return False
try:
okay, req = self.check_request()
if not okay:
return False
batch_name = BatchPrepId().next_batch_id(
req.get_attribute("member_of_campaign"), create_batch=True)
semaphore_events.increment(
batch_name
) # so it's not possible to announce while still injecting
executor = ssh_executor(server='vocms081.cern.ch')
try:
cmd = req.prepare_submit_command()
self.inject_logger.info(
"Command being used for injecting request {0}: {1}".
format(self.prepid, cmd))
_, stdout, stderr = executor.execute(cmd)
if not stdout and not stderr:
self.injection_error(
'ssh error for request {0} injection'.format(
self.prepid), req)
return False
output = stdout.read()
error = stderr.read()
self.injection_error(output, None)
self.injection_error(error, None)
if error and not output: # money on the table that it will break as well?
self.injection_error(
'Error in wmcontrol: {0}'.format(error), req)
return False
injected_requests = [
l.split()[-1] for l in output.split('\n')
if l.startswith('Injected workflow:')
]
if not injected_requests:
self.injection_error(
'Injection has succeeded but no request manager names were registered. Check with administrators. \nOutput: \n%s\n\nError: \n%s'
% (output, error), req)
return False
# another great structure
added_requests = [{
'name': app_req,
'content': {
'pdmv_prep_id': self.prepid
}
} for app_req in injected_requests]
requests = req.get_attribute('reqmgr_name')
requests.extend(added_requests)
req.set_attribute('reqmgr_name', requests)
# inject to batch
with locker.lock(batch_name):
bdb = database('batches')
bat = batch(bdb.get(batch_name))
bat.add_requests(added_requests)
bat.update_history({
'action': 'updated',
'step': self.prepid
})
saved = bdb.update(bat.json())
if not saved:
self.injection_error(
'There was a problem with registering request in the batch {0}'
.format(batch_name), req)
return False
# and in the end update request in database
req.update_history({
'action': 'inject',
'step': batch_name
})
req.set_status(
step=req._json_base__status.index('submitted'),
with_notification=True)
saved = self.request_db.update(req.json())
if not saved:
self.injection_error(
'Could not update request {0} in database'.format(
self.prepid), req)
return False
for added_req in added_requests:
self.inject_logger.info(
'Request {0} sent to {1}'.format(
added_req['name'], batch_name))
return True
finally: # lover batch semahore, created on submission time
semaphore_events.decrement(batch_name)
finally: # finally release Sumbitter lock
self.lock.release()
try:
executor.close_executor()
except UnboundLocalError:
pass
except Exception:
self.injection_error(
'Error with injecting the {0} request:\n{1}'.format(
self.prepid, traceback.format_exc()), None)
def internal_run(self):
if not self.lock.acquire(blocking=False):
self.logger.error(
"Could not acquire lock for ChainRequestInjector. prepid %s" %
(self.prepid))
return False
try:
crdb = database('chained_requests')
rdb = database('requests')
batch_name = None
if not crdb.document_exists(self.prepid):
# it's a request actually, pick up all chains containing it
mcm_r = rdb.get(self.prepid)
# mcm_crs = crdb.query(query="root_request==%s"% self.prepid) ## not only when its the root of
mcm_crs = crdb.query(query="contains==%s" % self.prepid)
task_name = 'task_' + self.prepid
batch_type = 'Task_' + mcm_r['member_of_campaign']
else:
mcm_crs = [crdb.get(self.prepid)]
current_step_prepid = mcm_crs[0]['chain'][mcm_crs[0]['step']]
mcm_request = rdb.get(current_step_prepid)
task_name = 'task_' + current_step_prepid
batch_type = 'Task_' + mcm_request['member_of_campaign']
if len(mcm_crs) == 0:
return False
mcm_rs = []
# upload all config files to config cache, with "configuration economy" already implemented
for cr in mcm_crs:
mcm_cr = chained_request(cr)
chain = mcm_cr.get_attribute(
'chain')[mcm_cr.get_attribute('step'):]
for request_prepid in chain:
mcm_rs.append(request(rdb.get(request_prepid)))
if self.check_approval and mcm_rs[-1].get_attribute(
'approval') != 'submit':
message = 'requests %s is in "%s"/"%s" status/approval, requires "approved"/"submit"' % (
request_prepid, mcm_rs[-1].get_attribute('status'),
mcm_rs[-1].get_attribute('approval'))
self.logger.error(message)
subject = '%s injection failed' % mcm_cr.get_attribute(
'prepid')
notification(
subject,
message, [],
group=notification.CHAINED_REQUESTS,
action_objects=[mcm_cr.get_attribute('prepid')],
object_type='chained_requests',
base_object=mcm_cr)
mcm_cr.notify(subject, message)
return False
if mcm_rs[-1].get_attribute('status') != 'approved':
# change the return format to percolate the error message
message = 'requests %s in in "%s"/"%s" status/approval, requires "approved"/"submit"' % (
request_prepid, mcm_rs[-1].get_attribute('status'),
mcm_rs[-1].get_attribute('approval'))
self.logger.error(message)
subject = '%s injection failed' % mcm_cr.get_attribute(
'prepid')
notification(
subject,
message, [],
group=notification.CHAINED_REQUESTS,
action_objects=[mcm_cr.get_attribute('prepid')],
object_type='chained_requests',
base_object=mcm_cr)
mcm_cr.notify(subject, message)
return False
uploader = ConfigMakerAndUploader(
prepid=request_prepid,
lock=locker.lock(request_prepid))
if not uploader.internal_run():
message = 'Problem with uploading the configuration for request %s' % (
request_prepid)
notification(
'Configuration upload failed',
message, [],
group=notification.CHAINED_REQUESTS,
action_objects=[mcm_cr.get_attribute('prepid')],
object_type='chained_requests',
base_object=mcm_cr)
mcm_cr.notify('Configuration upload failed', message)
self.logger.error(message)
return False
mcm_r = mcm_rs[-1]
batch_name = BatchPrepId().next_batch_id(batch_type,
create_batch=True)
semaphore_events.increment(batch_name)
self.logger.error('found batch %s' % batch_name)
with ssh_executor(server='vocms081.cern.ch') as ssh:
cmd = self.make_command(mcm_r)
self.logger.error('prepared command %s' % cmd)
# modify here to have the command to be executed
_, stdout, stderr = ssh.execute(cmd)
output = stdout.read()
error = stderr.read()
self.logger.info(output)
self.logger.info(error)
injected_requests = [
l.split()[-1] for l in output.split('\n')
if l.startswith('Injected workflow:')
]
if not injected_requests:
self.injection_error(
'Injection has succeeded but no request manager names were registered. Check with administrators. \nOutput: \n%s\n\nError: \n%s'
% (output, error), mcm_rs)
return False
# what gets printed into the batch object
added_requests = []
once = set()
for mcm_r in mcm_rs:
if mcm_r.get_attribute('prepid') in once:
continue
once.add(mcm_r.get_attribute('prepid'))
added = [{
'name': app_req,
'content': {
'pdmv_prep_id': mcm_r.get_attribute('prepid')
}
} for app_req in injected_requests]
added_requests.extend(added)
# edit the batch object
with locker.lock(batch_name):
bdb = database('batches')
bat = batch(bdb.get(batch_name))
bat.add_requests(added_requests)
bat.update_history({
'action': 'updated',
'step': task_name
})
bat.reload()
# reload the content of all requests as they might have changed already
added = [{
'name': app_req,
'content': {
'pdmv_prep_id': task_name
}
} for app_req in injected_requests]
seen = set()
for cr in mcm_crs:
mcm_cr = chained_request(cr)
chain = mcm_cr.get_attribute(
'chain')[mcm_cr.get_attribute('step'):]
message = ""
for rn in chain:
if rn in seen:
continue # don't do it twice
seen.add(rn)
mcm_r = request(rdb.get(rn))
message += mcm_r.textified()
message += "\n\n"
mcm_r.set_attribute('reqmgr_name', added)
mcm_r.update_history({
'action': 'inject',
'step': batch_name
})
if not self.check_approval:
mcm_r.set_attribute('approval', 'submit')
# set the status to submitted
mcm_r.set_status(
step=mcm_r._json_base__status.index('submitted'),
with_notification=False)
mcm_r.reload()
mcm_cr.set_attribute('last_status',
mcm_r.get_attribute('status'))
# re-get the object
mcm_cr = chained_request(crdb.get(cr['prepid']))
# take care of changes to the chain
mcm_cr.update_history({
'action': 'inject',
'step': batch_name
})
mcm_cr.set_attribute(
'step',
len(mcm_cr.get_attribute('chain')) - 1)
mcm_cr.set_attribute('status', 'processing')
subject = 'Injection succeeded for %s' % mcm_cr.get_attribute(
'prepid')
notification(
subject,
message, [],
group=notification.CHAINED_REQUESTS,
action_objects=[mcm_cr.get_attribute('prepid')],
object_type='chained_requests',
base_object=mcm_cr)
mcm_cr.notify(subject, message)
mcm_cr.reload()
return True
except Exception:
self.injection_error(
"Error with injecting chains for %s :\n %s" %
(self.prepid, traceback.format_exc()), [])
finally: # we decrement batch id and release lock on prepid+lower semaphore
if batch_name: # ditry thing for now. Because batch name can be None for certain use-cases in code above
semaphore_events.decrement(batch_name)
self.lock.release()
self.queue_lock.release()
def internal_run(self):
try:
if not self.lock.acquire(blocking=False):
return False
try:
okay, req = self.check_request()
if not okay: return False
batch_name = BatchPrepId().next_id(req.json())
semaphore_events.increment(batch_name) # so it's not possible to announce while still injecting
executor = ssh_executor(server='pdmvserv-test.cern.ch')
try:
cmd = req.prepare_submit_command(batch_name)
self.logger.inject("Command being used for injecting request {0}: {1}".format(self.prepid, cmd),
handler=self.prepid)
_, stdout, stderr = executor.execute(cmd)
if not stdout and not stderr:
self.injection_error('ssh error for request {0} injection'.format(self.prepid), req)
return False
output = stdout.read()
error = stderr.read()
if error and not output: # money on the table that it will break as well?
self.injection_error('Error in wmcontrol: {0}'.format(error), req)
return False
injected_requests = [l.split()[-1] for l in output.split('\n') if
l.startswith('Injected workflow:')]
approved_requests = [l.split()[-1] for l in output.split('\n') if
l.startswith('Approved workflow:')]
if not approved_requests:
self.injection_error(
'Injection has succeeded but no request manager names were registered. Check with administrators. \nOutput: \n{0}\n\nError: \n{1}'.format(
output, error), req)
return False
objects_to_invalidate = [
{"_id": inv_req, "object": inv_req, "type": "request", "status": "new", "prepid": self.prepid}
for inv_req in injected_requests if inv_req not in approved_requests]
if objects_to_invalidate:
self.logger.inject(
"Some of the workflows had to be invalidated: {0}".format(objects_to_invalidate),
handler=self.prepid)
invalidation = database('invalidation')
saved = invalidation.save_all(objects_to_invalidate)
if not saved:
self.injection_error('Could not save the invalidations {0}'.format(objects_to_invalidate),
req)
added_requests = [{'name': app_req, 'content': {'pdmv_prep_id': self.prepid}} for app_req in
approved_requests]
requests = req.get_attribute('reqmgr_name')
requests.extend(added_requests)
req.set_attribute('reqmgr_name', requests)
#inject to batch
with locker.lock(batch_name):
bdb = database('batches')
bat = batch(bdb.get(batch_name))
bat.add_requests(added_requests)
bat.update_history({'action': 'updated', 'step': self.prepid})
saved = bdb.update(bat.json())
if not saved:
self.injection_error(
'There was a problem with registering request in the batch {0}'.format(batch_name), req)
return False
#and in the end update request in database
req.update_history({'action': 'inject', 'step' : batch_name})
req.set_status(with_notification=True)
saved = self.request_db.update(req.json())
if not saved:
self.injection_error('Could not update request {0} in database'.format(self.prepid), req)
return False
for added_req in added_requests:
self.logger.inject('Request {0} sent to {1}'.format(added_req['name'], batch_name),
handler=self.prepid)
return True
finally:
semaphore_events.decrement(batch_name)
finally:
self.lock.release()
executor.close_executor()
except Exception as e:
self.injection_error(
'Error with injecting the {0} request:\n{1}'.format(self.prepid, traceback.format_exc()), req)
def __init__(self, **kwargs):
Handler.__init__(self, **kwargs)
self.prepid = kwargs["prepid"]
self.request_db = database("requests")
self.config_db = database("configs")
self.ssh_executor = ssh_executor(server='pdmvserv-test.cern.ch')
def run_flask():
print('Will do dummy ssh connection in order to initialize ssh_executor. Wait!')
ssh_executor().execute('echo pippo')
print('Finished ssh, McM will start shortly...')
app.run(host='0.0.0.0', port=443, threaded=True, ssl_context=('/etc/pki/tls/certs/localhost.crt', '/etc/pki/tls/private/localhost.key'))
def GET(self, *args):
"""
Provides the injection command and does the injection.
"""
crn= args[0]
crdb = database('chained_requests')
mcm_cr = chained_request(crdb.get(crn))
rdb = database('requests')
mcm_rs=[]
## upload all config files to config cache, with "configuration economy" already implemented
from tools.locker import locker
from tools.handlers import ConfigMakerAndUploader
for rn in mcm_cr.get_attribute('chain'):
mcm_rs.append( request( rdb.get( rn )))
if self.mode=='inject' and mcm_rs[-1].get_attribute('status') != 'approved':
return dumps({"results" : False, "message" : 'requests %s in in "%s" status, requires "approved"'%( rn, mcm_rs[-1].get_attribute('status'))})
uploader = ConfigMakerAndUploader(prepid=rn, lock = locker.lock(rn))
uploader.run()
mcm_r = mcm_rs[-1]
from rest_api.BatchPrepId import BatchPrepId
batch_name = BatchPrepId().next_batch_id( mcm_cr.get_attribute('member_of_campaign') , create_batch=self.mode=='inject')
from tools.locker import semaphore_events, locker
semaphore_events.increment(batch_name)
from tools.ssh_executor import ssh_executor
from tools.locator import locator
l_type = locator()
with ssh_executor(server = 'pdmvserv-test.cern.ch') as ssh:
cmd='cd /afs/cern.ch/cms/PPD/PdmV/work/McM/dev-submit/\n'
cmd+=mcm_r.make_release()
cmd+='export X509_USER_PROXY=/afs/cern.ch/user/p/pdmvserv/private/$HOST/voms_proxy.cert\n'
cmd+='export PATH=/afs/cern.ch/cms/PPD/PdmV/tools/wmcontrol:${PATH}\n'
## until we get into production
there='--wmtest --wmtesturl cmsweb-testbed.cern.ch'
cmd+='wmcontrol.py --url-dict %s/public/restapi/chained_requests/get_dict/%s %s \n'%(l_type.baseurl(), crn, there)
if self.mode == 'show':
cherrypy.response.headers['Content-Type'] = 'text/plain'
return cmd
else:
_, stdout, stderr = ssh.execute(cmd)
cherrypy.response.headers['Content-Type'] = 'text/plain'
output = stdout.read()
error = stderr.read()
self.logger.log(output)
self.logger.log(error)
injected_requests = [l.split()[-1] for l in output.split('\n') if
l.startswith('Injected workflow:')]
approved_requests = [l.split()[-1] for l in output.split('\n') if
l.startswith('Approved workflow:')]
if injected_requests and not approved_requests:
return dumps({"results" : False, "message" : "Request %s was injected but could not be approved" % ( injected_requests )})
objects_to_invalidate = [
{"_id": inv_req, "object": inv_req, "type": "request", "status": "new", "prepid": self.prepid}
for inv_req in injected_requests if inv_req not in approved_requests]
if objects_to_invalidate:
return dumps({"results" : False, "message" : "Some requests %s need to be invalidated"})
added_requests = []
for mcm_r in mcm_rs:
added = [{'name': app_req, 'content': {'pdmv_prep_id': mcm_r.get_attribute('prepid')}} for app_req in approved_requests]
added_requests.extend( added )
with locker.lock(batch_name):
bdb = database('batches')
bat = batch(bdb.get(batch_name))
bat.add_requests(added_requests)
bat.update_history({'action': 'updated', 'step': crn})
bat.reload()
for mcm_r in mcm_rs:
mcm_r.set_attribute('reqmgr_name', added_requests)
for mcm_r in mcm_rs:
added = [{'name': app_req, 'content': {'pdmv_prep_id': mcm_r.get_attribute('prepid')}} for app_req in approved_requests]
mcm_r.set_attribute('reqmgr_name', added )
mcm_r.update_history({'action': 'inject','step' : batch_name})
mcm_r.set_attribute('approval', 'submit')
mcm_r.set_status(with_notification=False) ## maybe change to false
mcm_r.reload()
mcm_cr.update_history({'action' : 'inject','step': batch_name})
mcm_cr.set_attribute('step', len(mcm_rs)-1)
mcm_cr.set_attribute('status','processing')
mcm_cr.set_attribute('last_status', mcm_rs[-1].get_attribute('status'))
message=""
for mcm_r in mcm_rs:
message+=mcm_r.textified()
message+="\n\n"
mcm_cr.notify('Injection succeeded for %s'% crn,
message)
mcm_cr.reload()
return dumps({"results" : True, "message" : "request send to batch %s"% batch_name})
