def isRunning(self, db):
if not self.node_ip:
self.log.error('Checking job %s is running but no node IP address'%self)
return False
# pylint: disable=E
updated = time.mktime(self.updated.timetuple())
# pylint: enable=E
if (time.time() - updated < 300):
# Allow 5 minutes for the gate PID to exist
return True
# Absolute maximum running time of 2 hours. Note that if by happy chance the tests have finished
# this result will be over-written by retrieveResults
if (time.time() - updated > Configuration().get_int('MAX_RUNNING_TIME')):
self.log.error('Timed out job %s (Running for %d seconds)'%(self, time.time()-updated))
self.update(db, result='Aborted: Timed out')
return False
try:
success = utils.execute_command('ssh -q -o BatchMode=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %s %s@%s ps -p `cat /home/jenkins/run_tests.pid`'%(
Configuration().NODE_KEY, Configuration().NODE_USERNAME, self.node_ip), silent=True)
self.log.info('Gate-is-running on job %s (%s) returned: %s'%(
self, self.node_ip, success))
return success
except Exception, e:
self.update(db, result='Aborted: Exception checking for pid')
self.log.exception(e)
return False
def isRunning(self, db):
if not self.node_ip:
self.log.error('Checking job %s is running but no node IP address'%self)
return False
# pylint: disable=E
updated = time.mktime(self.updated.timetuple())
# pylint: enable=E
if (time.time() - updated < 300):
# Allow 5 minutes for the gate PID to exist
return True
# Absolute maximum running time of 2 hours. Note that if by happy chance the tests have finished
# this result will be over-written by retrieveResults
if (time.time() - updated > Configuration().get_int('MAX_RUNNING_TIME')):
self.log.error('Timed out job %s (Running for %d seconds)'%(self, time.time()-updated))
self.update(db, result='Aborted: Timed out')
return False
try:
success = utils.execute_command('ssh -q -o BatchMode=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %s %s@%s ps -p `cat /home/jenkins/run_tests.pid`'%(
Configuration().NODE_KEY, Configuration().NODE_USERNAME, self.node_ip), silent=True)
self.log.info('Gate-is-running on job %s (%s) returned: %s'%(
self, self.node_ip, success))
return success
except Exception, e:
self.update(db, result='Aborted: Exception checking for pid')
self.log.exception(e)
return False
def runJob(self, db, nodepool):
if self.node_id:
nodepool.deleteNode(self.node_id)
self.update(db, node_id=0)
node_id, node_ip = nodepool.getNode()
if not node_id:
return
self.log.info("Running job for %s on %s/%s"%(self, node_id, node_ip))
if not utils.testSSH(node_ip, Configuration().NODE_USERNAME, Configuration().NODE_KEY):
self.log.error('Failed to get SSH object for node %s/%s. Deleting node.'%(node_id, node_ip))
nodepool.deleteNode(node_id)
self.update(db, node_id=0)
return
self.update(db, node_id=node_id, node_ip=node_ip, result='')
cmd = 'echo %s >> run_tests_env' % ' '.join(instructions.check_out_testrunner())
utils.execute_command('ssh -q -o BatchMode=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %s %s@%s %s'%(
Configuration().NODE_KEY, Configuration().NODE_USERNAME, node_ip, cmd))
cmd = 'echo "%s %s" >> run_tests_env' % (
' '.join(environment.get_environment(self.change_ref)),
' '.join(instructions.execute_test_runner()))
utils.execute_command('ssh -q -o BatchMode=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %s %s@%s %s'%(
Configuration().NODE_KEY, Configuration().NODE_USERNAME, node_ip, cmd))
# For some reason invoking this immediately fails...
time.sleep(5)
utils.execute_command('ssh$-q$-o$BatchMode=yes$-o$UserKnownHostsFile=/dev/null$-o$StrictHostKeyChecking=no$-i$%s$%s@%s$nohup bash /home/jenkins/run_tests_env < /dev/null > run_tests.log 2>&1 &'%(
Configuration().NODE_KEY, Configuration().NODE_USERNAME, node_ip), '$')
self.update(db, state=constants.RUNNING)
def retrieveResults(self, dest_path):
if not self.node_ip:
self.log.error('Attempting to retrieve results for %s but no node IP address'%self)
return constants.NO_IP
try:
code, stdout, stderr = utils.execute_command(
'ssh -q -o BatchMode=yes -o UserKnownHostsFile=/dev/null'
' -o StrictHostKeyChecking=no -i %s %s@%s cat result.txt' % (
Configuration().NODE_KEY,
Configuration().NODE_USERNAME,
self.node_ip),
silent=True,
return_streams=True
)
self.log.info('Result: %s (Err: %s)'%(stdout, stderr))
self.log.info('Downloading logs for %s'%self)
utils.copy_logs(
[
'/home/jenkins/workspace/testing/logs/*',
'/home/jenkins/run_test*',
'/etc/nova/*',
'/etc/swift/*',
'/etc/cinder/*',
'/etc/keystone/*',
],
dest_path,
self.node_ip,
Configuration().NODE_USERNAME,
Configuration().NODE_KEY,
upload=False
)
self.log.info('Downloading dom0 logs for %s'%self)
utils.copy_dom0_logs(
self.node_ip,
Configuration().NODE_USERNAME,
Configuration().NODE_KEY,
dest_path
)
if code != 0:
# This node is broken somehow... Mark it as aborted
if self.result and self.result.startswith('Aborted: '):
return self.result
return constants.NORESULT
return stdout.splitlines()[0]
except Exception, e:
self.log.exception(e)
return constants.COPYFAIL
def retrieveResults(self, dest_path):
if not self.node_ip:
self.log.error('Attempting to retrieve results for %s but no node IP address'%self)
return constants.NO_IP
try:
code, stdout, stderr = utils.execute_command(
'ssh -q -o BatchMode=yes -o UserKnownHostsFile=/dev/null'
' -o StrictHostKeyChecking=no -i %s %s@%s cat result.txt' % (
Configuration().NODE_KEY,
Configuration().NODE_USERNAME,
self.node_ip),
silent=True,
return_streams=True
)
self.log.info('Result: %s (Err: %s)'%(stdout, stderr))
self.log.info('Downloading logs for %s'%self)
utils.copy_logs(
[
'/home/jenkins/workspace/testing/logs/*',
'/home/jenkins/run_test*',
'/etc/nova/*',
'/etc/swift/*',
'/etc/cinder/*',
'/etc/keystone/*',
],
dest_path,
self.node_ip,
Configuration().NODE_USERNAME,
Configuration().NODE_KEY,
upload=False
)
self.log.info('Downloading dom0 logs for %s'%self)
utils.copy_dom0_logs(
self.node_ip,
Configuration().NODE_USERNAME,
Configuration().NODE_KEY,
dest_path
)
if code != 0:
# This node is broken somehow... Mark it as aborted
if self.result and self.result.startswith('Aborted: '):
return self.result
return constants.NORESULT
return stdout.splitlines()[0]
except Exception, e:
self.log.exception(e)
return constants.COPYFAIL
def runJob(self, db, nodepool):
if self.node_id:
nodepool.deleteNode(self.node_id)
self.update(db, node_id=0)
node_id, node_ip = nodepool.getNode()
if not node_id:
return
self.log.info("Running job for %s on %s/%s"%(self, node_id, node_ip))
if not utils.testSSH(node_ip, Configuration().NODE_USERNAME, Configuration().NODE_KEY):
self.log.error('Failed to get SSH object for node %s/%s. Deleting node.'%(node_id, node_ip))
nodepool.deleteNode(node_id)
self.update(db, node_id=0)
return
self.update(db, node_id=node_id, node_ip=node_ip, result='')
instruction_list = ["#!/bin/bash"]
for instruction in instructions.update_devstackgate('origin/master'):
instruction_list.append(" ".join(instruction))
instruction_list.append(" ".join(instructions.check_out_testrunner()))
if self.project_name == 'openstack/xenapi-os-testing':
for instruction in instructions.update_testrunner(self.change_ref):
instruction_list.append(" ".join(instruction))
instruction_list.append("%s %s"%(" ".join(environment.get_environment(self.project_name, self.change_ref, self.branch)),
" ".join(instructions.execute_test_runner())))
for instruction in instruction_list:
cmd = 'echo "%s" >> run_tests_env' % instruction
utils.execute_command('ssh -q -o BatchMode=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %s %s@%s %s'%(
Configuration().NODE_KEY, Configuration().NODE_USERNAME, node_ip, cmd))
cmd = "chmod +x run_tests_env"
utils.execute_command('ssh -q -o BatchMode=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %s %s@%s %s'%(
Configuration().NODE_KEY, Configuration().NODE_USERNAME, node_ip, cmd))
# For some reason invoking this immediately fails...
time.sleep(5)
utils.execute_command('ssh$-q$-o$BatchMode=yes$-o$UserKnownHostsFile=/dev/null$-o$StrictHostKeyChecking=no$-i$%s$%s@%s$'\
'nohup bash -c "source /opt/git/openstack-infra/devstack-gate/functions.sh; tsfilter /home/jenkins/run_tests_env" < /dev/null > run_tests.log 2>&1 &'%(
Configuration().NODE_KEY, Configuration().NODE_USERNAME, node_ip), '$')
self.update(db, state=constants.RUNNING)
def runJob(self, db, nodepool):
if self.node_id:
nodepool.deleteNode(self.node_id)
self.update(db, node_id=0)
node_id, node_ip = nodepool.getNode()
if not node_id:
return
self.log.info("Running job for %s on %s/%s"%(self, node_id, node_ip))
if not utils.testSSH(node_ip, Configuration().NODE_USERNAME, Configuration().NODE_KEY):
self.log.error('Failed to get SSH object for node %s/%s. Deleting node.'%(node_id, node_ip))
nodepool.deleteNode(node_id)
self.update(db, node_id=0)
return
self.update(db, node_id=node_id, node_ip=node_ip, result='')
instruction_list = ["#!/bin/bash"]
for instruction in instructions.update_devstackgate('origin/master'):
instruction_list.append(" ".join(instruction))
instruction_list.append(" ".join(instructions.check_out_testrunner()))
if self.project_name == 'stackforge/xenapi-os-testing':
for instruction in instructions.update_testrunner(self.change_ref):
instruction_list.append(" ".join(instruction))
instruction_list.append("%s %s"%(" ".join(environment.get_environment(self.project_name, self.change_ref)),
" ".join(instructions.execute_test_runner())))
for instruction in instruction_list:
cmd = 'echo "%s" >> run_tests_env' % instruction
utils.execute_command('ssh -q -o BatchMode=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %s %s@%s %s'%(
Configuration().NODE_KEY, Configuration().NODE_USERNAME, node_ip, cmd))
cmd = "chmod +x run_tests_env"
utils.execute_command('ssh -q -o BatchMode=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %s %s@%s %s'%(
Configuration().NODE_KEY, Configuration().NODE_USERNAME, node_ip, cmd))
# For some reason invoking this immediately fails...
time.sleep(5)
utils.execute_command('ssh$-q$-o$BatchMode=yes$-o$UserKnownHostsFile=/dev/null$-o$StrictHostKeyChecking=no$-i$%s$%s@%s$'\
'nohup bash -c "source /opt/git/openstack-infra/devstack-gate/functions.sh; tsfilter /home/jenkins/run_tests_env" < /dev/null > run_tests.log 2>&1 &'%(
Configuration().NODE_KEY, Configuration().NODE_USERNAME, node_ip), '$')
self.update(db, state=constants.RUNNING)