예제 #1
0
    def isRunning(self, db):
        if not self.node_ip:
            self.log.error('Checking job %s is running but no node IP address'%self)
            return False

        # pylint: disable=E
        updated = time.mktime(self.updated.timetuple())
        # pylint: enable=E

        if (time.time() - updated < 300):
            # Allow 5 minutes for the gate PID to exist
            return True

        # Absolute maximum running time of 2 hours.  Note that if by happy chance the tests have finished
        # this result will be over-written by retrieveResults
        if (time.time() - updated > Configuration().get_int('MAX_RUNNING_TIME')):
            self.log.error('Timed out job %s (Running for %d seconds)'%(self, time.time()-updated))
            self.update(db, result='Aborted: Timed out')
            return False

        try:
            success = utils.execute_command('ssh -q -o BatchMode=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %s %s@%s ps -p `cat /home/jenkins/run_tests.pid`'%(
                    Configuration().NODE_KEY, Configuration().NODE_USERNAME, self.node_ip), silent=True)
            self.log.info('Gate-is-running on job %s (%s) returned: %s'%(
                          self, self.node_ip, success))
            return success
        except Exception, e:
            self.update(db, result='Aborted: Exception checking for pid')
            self.log.exception(e)
            return False
예제 #2
0
    def isRunning(self, db):
        if not self.node_ip:
            self.log.error('Checking job %s is running but no node IP address'%self)
            return False

        # pylint: disable=E
        updated = time.mktime(self.updated.timetuple())
        # pylint: enable=E

        if (time.time() - updated < 300):
            # Allow 5 minutes for the gate PID to exist
            return True

        # Absolute maximum running time of 2 hours.  Note that if by happy chance the tests have finished
        # this result will be over-written by retrieveResults
        if (time.time() - updated > Configuration().get_int('MAX_RUNNING_TIME')):
            self.log.error('Timed out job %s (Running for %d seconds)'%(self, time.time()-updated))
            self.update(db, result='Aborted: Timed out')
            return False

        try:
            success = utils.execute_command('ssh -q -o BatchMode=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %s %s@%s ps -p `cat /home/jenkins/run_tests.pid`'%(
                    Configuration().NODE_KEY, Configuration().NODE_USERNAME, self.node_ip), silent=True)
            self.log.info('Gate-is-running on job %s (%s) returned: %s'%(
                          self, self.node_ip, success))
            return success
        except Exception, e:
            self.update(db, result='Aborted: Exception checking for pid')
            self.log.exception(e)
            return False
예제 #3
0
    def runJob(self, db, nodepool):
        if self.node_id:
            nodepool.deleteNode(self.node_id)
            self.update(db, node_id=0)

        node_id, node_ip = nodepool.getNode()

        if not node_id:
            return
        self.log.info("Running job for %s on %s/%s"%(self, node_id, node_ip))

        if not utils.testSSH(node_ip, Configuration().NODE_USERNAME, Configuration().NODE_KEY):
            self.log.error('Failed to get SSH object for node %s/%s.  Deleting node.'%(node_id, node_ip))
            nodepool.deleteNode(node_id)
            self.update(db, node_id=0)
            return

        self.update(db, node_id=node_id, node_ip=node_ip, result='')

        cmd = 'echo %s >> run_tests_env' % ' '.join(instructions.check_out_testrunner())
        utils.execute_command('ssh -q -o BatchMode=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %s %s@%s %s'%(
                Configuration().NODE_KEY, Configuration().NODE_USERNAME, node_ip, cmd))
        cmd = 'echo "%s %s" >> run_tests_env' % (
            ' '.join(environment.get_environment(self.change_ref)),
            ' '.join(instructions.execute_test_runner()))
        utils.execute_command('ssh -q -o BatchMode=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %s %s@%s %s'%(
                Configuration().NODE_KEY, Configuration().NODE_USERNAME, node_ip, cmd))
        # For some reason invoking this immediately fails...
        time.sleep(5)
        utils.execute_command('ssh$-q$-o$BatchMode=yes$-o$UserKnownHostsFile=/dev/null$-o$StrictHostKeyChecking=no$-i$%s$%s@%s$nohup bash /home/jenkins/run_tests_env < /dev/null > run_tests.log 2>&1 &'%(
                Configuration().NODE_KEY, Configuration().NODE_USERNAME, node_ip), '$')
        self.update(db, state=constants.RUNNING)
예제 #4
0
    def retrieveResults(self, dest_path):
        if not self.node_ip:
            self.log.error('Attempting to retrieve results for %s but no node IP address'%self)
            return constants.NO_IP
        try:
            code, stdout, stderr = utils.execute_command(
                'ssh -q -o BatchMode=yes -o UserKnownHostsFile=/dev/null'
                ' -o StrictHostKeyChecking=no -i %s %s@%s cat result.txt' % (
                    Configuration().NODE_KEY,
                    Configuration().NODE_USERNAME,
                    self.node_ip),
                silent=True,
                return_streams=True
            )
            self.log.info('Result: %s (Err: %s)'%(stdout, stderr))
            self.log.info('Downloading logs for %s'%self)
            utils.copy_logs(
                [
                    '/home/jenkins/workspace/testing/logs/*',
                    '/home/jenkins/run_test*',
                    '/etc/nova/*',
                    '/etc/swift/*',
                    '/etc/cinder/*',
                    '/etc/keystone/*',
                ],
                dest_path,
                self.node_ip,
                Configuration().NODE_USERNAME,
                Configuration().NODE_KEY,
                upload=False
            )

            self.log.info('Downloading dom0 logs for %s'%self)
            utils.copy_dom0_logs(
                self.node_ip,
                Configuration().NODE_USERNAME,
                Configuration().NODE_KEY,
                dest_path
            )

            if code != 0:
                # This node is broken somehow... Mark it as aborted
                if self.result and self.result.startswith('Aborted: '):
                    return self.result
                return constants.NORESULT

            return stdout.splitlines()[0]
        except Exception, e:
            self.log.exception(e)
            return constants.COPYFAIL
예제 #5
0
    def retrieveResults(self, dest_path):
        if not self.node_ip:
            self.log.error('Attempting to retrieve results for %s but no node IP address'%self)
            return constants.NO_IP
        try:
            code, stdout, stderr = utils.execute_command(
                'ssh -q -o BatchMode=yes -o UserKnownHostsFile=/dev/null'
                ' -o StrictHostKeyChecking=no -i %s %s@%s cat result.txt' % (
                    Configuration().NODE_KEY,
                    Configuration().NODE_USERNAME,
                    self.node_ip),
                silent=True,
                return_streams=True
            )
            self.log.info('Result: %s (Err: %s)'%(stdout, stderr))
            self.log.info('Downloading logs for %s'%self)
            utils.copy_logs(
                [
                    '/home/jenkins/workspace/testing/logs/*',
                    '/home/jenkins/run_test*',
                    '/etc/nova/*',
                    '/etc/swift/*',
                    '/etc/cinder/*',
                    '/etc/keystone/*',
                ],
                dest_path,
                self.node_ip,
                Configuration().NODE_USERNAME,
                Configuration().NODE_KEY,
                upload=False
            )

            self.log.info('Downloading dom0 logs for %s'%self)
            utils.copy_dom0_logs(
                self.node_ip,
                Configuration().NODE_USERNAME,
                Configuration().NODE_KEY,
                dest_path
            )

            if code != 0:
                # This node is broken somehow... Mark it as aborted
                if self.result and self.result.startswith('Aborted: '):
                    return self.result
                return constants.NORESULT

            return stdout.splitlines()[0]
        except Exception, e:
            self.log.exception(e)
            return constants.COPYFAIL
예제 #6
0
    def runJob(self, db, nodepool):
        if self.node_id:
            nodepool.deleteNode(self.node_id)
            self.update(db, node_id=0)

        node_id, node_ip = nodepool.getNode()

        if not node_id:
            return
        self.log.info("Running job for %s on %s/%s"%(self, node_id, node_ip))

        if not utils.testSSH(node_ip, Configuration().NODE_USERNAME, Configuration().NODE_KEY):
            self.log.error('Failed to get SSH object for node %s/%s.  Deleting node.'%(node_id, node_ip))
            nodepool.deleteNode(node_id)
            self.update(db, node_id=0)
            return

        self.update(db, node_id=node_id, node_ip=node_ip, result='')

        instruction_list = ["#!/bin/bash"]
        for instruction in instructions.update_devstackgate('origin/master'):
            instruction_list.append(" ".join(instruction))
        instruction_list.append(" ".join(instructions.check_out_testrunner()))
        if self.project_name == 'openstack/xenapi-os-testing':
            for instruction in instructions.update_testrunner(self.change_ref):
                instruction_list.append(" ".join(instruction))
        instruction_list.append("%s %s"%(" ".join(environment.get_environment(self.project_name, self.change_ref, self.branch)),
                                         " ".join(instructions.execute_test_runner())))

        for instruction in instruction_list:
            cmd = 'echo "%s" >> run_tests_env' % instruction
            utils.execute_command('ssh -q -o BatchMode=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %s %s@%s %s'%(
                Configuration().NODE_KEY, Configuration().NODE_USERNAME, node_ip, cmd))
        cmd = "chmod +x run_tests_env"
        utils.execute_command('ssh -q -o BatchMode=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %s %s@%s %s'%(
            Configuration().NODE_KEY, Configuration().NODE_USERNAME, node_ip, cmd))
        # For some reason invoking this immediately fails...
        time.sleep(5)
        utils.execute_command('ssh$-q$-o$BatchMode=yes$-o$UserKnownHostsFile=/dev/null$-o$StrictHostKeyChecking=no$-i$%s$%s@%s$'\
                              'nohup bash -c "source /opt/git/openstack-infra/devstack-gate/functions.sh; tsfilter /home/jenkins/run_tests_env" < /dev/null > run_tests.log 2>&1 &'%(
                Configuration().NODE_KEY, Configuration().NODE_USERNAME, node_ip), '$')
        self.update(db, state=constants.RUNNING)
예제 #7
0
    def runJob(self, db, nodepool):
        if self.node_id:
            nodepool.deleteNode(self.node_id)
            self.update(db, node_id=0)

        node_id, node_ip = nodepool.getNode()

        if not node_id:
            return
        self.log.info("Running job for %s on %s/%s"%(self, node_id, node_ip))

        if not utils.testSSH(node_ip, Configuration().NODE_USERNAME, Configuration().NODE_KEY):
            self.log.error('Failed to get SSH object for node %s/%s.  Deleting node.'%(node_id, node_ip))
            nodepool.deleteNode(node_id)
            self.update(db, node_id=0)
            return

        self.update(db, node_id=node_id, node_ip=node_ip, result='')

        instruction_list = ["#!/bin/bash"]
        for instruction in instructions.update_devstackgate('origin/master'):
            instruction_list.append(" ".join(instruction))
        instruction_list.append(" ".join(instructions.check_out_testrunner()))
        if self.project_name == 'stackforge/xenapi-os-testing':
            for instruction in instructions.update_testrunner(self.change_ref):
                instruction_list.append(" ".join(instruction))
        instruction_list.append("%s %s"%(" ".join(environment.get_environment(self.project_name, self.change_ref)),
                                         " ".join(instructions.execute_test_runner())))

        for instruction in instruction_list:
            cmd = 'echo "%s" >> run_tests_env' % instruction
            utils.execute_command('ssh -q -o BatchMode=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %s %s@%s %s'%(
                Configuration().NODE_KEY, Configuration().NODE_USERNAME, node_ip, cmd))
        cmd = "chmod +x run_tests_env"
        utils.execute_command('ssh -q -o BatchMode=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %s %s@%s %s'%(
            Configuration().NODE_KEY, Configuration().NODE_USERNAME, node_ip, cmd))
        # For some reason invoking this immediately fails...
        time.sleep(5)
        utils.execute_command('ssh$-q$-o$BatchMode=yes$-o$UserKnownHostsFile=/dev/null$-o$StrictHostKeyChecking=no$-i$%s$%s@%s$'\
                              'nohup bash -c "source /opt/git/openstack-infra/devstack-gate/functions.sh; tsfilter /home/jenkins/run_tests_env" < /dev/null > run_tests.log 2>&1 &'%(
                Configuration().NODE_KEY, Configuration().NODE_USERNAME, node_ip), '$')
        self.update(db, state=constants.RUNNING)