Пример #1
0
 def test_does_not_find_labels(self, m_requests):
     mock_response = MagicMock()
     job_config = '<?xml version="1.0" encoding="UTF-8"?><project></project>'
     mock_response.text = job_config
     m_requests.get.return_value = mock_response
     result = util.match_node_from_job_config("https://jenkins.ceph.com/job/ceph-pull-requests")
     assert not result
Пример #2
0
 def test_finds_labels(self, m_requests):
     mock_response = MagicMock()
     job_config = '<?xml version="1.0" encoding="UTF-8"?><project><assignedNode>amd64 &amp;&amp; debian</assignedNode></project>'
     mock_response.text = job_config
     m_requests.get.return_value = mock_response
     result = util.match_node_from_job_config("https://jenkins.ceph.com/job/ceph-pull-requests")
     assert result == "wheezy"
Пример #3
0
 def test_failed_to_fetch_job_config(self, m_requests):
     mock_response = MagicMock()
     mock_response.raise_for_status.side_effect = requests.exceptions.RequestException(
     )
     m_requests.get.return_value = mock_response
     result = util.match_node_from_job_config(
         "https://jenkins.ceph.com/job/ceph-pull-requests")
     assert not result
Пример #4
0
 def test_does_not_find_labels(self, m_requests):
     mock_response = MagicMock()
     job_config = '<?xml version="1.0" encoding="UTF-8"?><project></project>'
     mock_response.text = job_config
     m_requests.get.return_value = mock_response
     result = util.match_node_from_job_config(
         "https://jenkins.ceph.com/job/ceph-pull-requests")
     assert not result
Пример #5
0
 def test_finds_labels(self, m_requests):
     mock_response = MagicMock()
     job_config = '<?xml version="1.0" encoding="UTF-8"?><project><assignedNode>amd64 &amp;&amp; debian</assignedNode></project>'
     mock_response.text = job_config
     m_requests.get.return_value = mock_response
     result = util.match_node_from_job_config(
         "https://jenkins.ceph.com/job/ceph-pull-requests")
     assert result == "wheezy"
Пример #6
0
def check_queue():
    """
    Specifically checks for the status of the Jenkins queue. The docs are
    sparse here, but
    ``jenkins/core/src/main/resources/hudson/model/queue/CauseOfBlockage`` has
    the specific reasons this check needs:

    * *BecauseLabelIsBusy* Waiting for next available executor on {0}

    * *BecauseLabelIsOffline* All nodes of label \u2018{0}\u2019 are offline

    * *BecauseNodeIsBusy* Waiting for next available executor on {0}

    * *BecauseNodeLabelIsOffline* There are no nodes with the label \u2018{0}\u2019

    * *BecauseNodeIsOffline* {0} is offline

    The distinction is the need for a label or a node. In the case of a node,
    it will get matched directly to the nodes in the configuration, in case of a label
    it will go through the configured nodes and pick the first matched to its labels.

    Label needed example
    --------------------
    Jenkins queue reports that 'All nodes of label x86_64 are offline'. The
    configuration has::

        nodes: {
            'centos6': {
                ...
                'labels': ['x86_64', 'centos', 'centos6']
            }
        }

    Since 'x86_64' exists in the 'centos6' configured node, it will be sent off
    to create that one.

    Node needed example
    -------------------
    Jenkins reports that 'wheezy is offline'. The configuration has a few
    labels configured::

        nodes: {
            'wheezy': {
                ...
            }
            'centos6': {
                ...
            }
        }

    Since the key 'wheezy' matches the node required by the build system to
    continue it goes off to create it.
    """
    jenkins_url = pecan.conf.jenkins['url']
    jenkins_user = pecan.conf.jenkins['user']
    jenkins_token = pecan.conf.jenkins['token']
    conn = jenkins.Jenkins(jenkins_url, jenkins_user, jenkins_token)
    result = conn.get_queue_info()
    needed_nodes = {}

    if result:
        for task in result:
            if task['why'] is None:
                # this may happen when multiple tasks are getting pilled up (for a PR for example)
                # and there is no 'why' yet. So the API has a `None` for it which would break logic
                # to infer what is needed to get it unstuck
                continue
            if util.is_stuck(task['why']):
                logger.info('found stuck task with name: %s' %
                            task['task']['name'])
                logger.info('reason was: %s' % task['why'])
                node_name = util.match_node(task['why'])
                if not node_name:
                    # this usually happens when jenkins is waiting on an executor
                    # on a static slave whose labels are not a subset of a
                    # node in the mita config
                    logger.warning('unable to match a suitable node')
                    logger.warning('will infer from job labels')
                    job_url = task['task']['url']
                    job_name = util.job_from_url(job_url)
                    try:
                        conn.get_job_info(job_name)
                        logger.info("Job info found for: %s", job_name)
                        node_name = util.match_node_from_job_config(job_url)
                    except jenkins.JenkinsException:
                        logger.warning('No job info found for: %s', job_name)
                        logger.warning('Will assume the job is a matrix job')
                        node_name = util.match_node_from_matrix_job_name(
                            job_name)
                    if not node_name:
                        logger.warning(
                            'completely unable to match a node to provide')
                        continue
                logger.info('inferred node as: %s' % str(node_name))
                if node_name:
                    logger.info('matched a node name to config: %s' %
                                node_name)
                    # TODO: this should talk to the pecan app over HTTP using
                    # the `app.conf.pecan_app` configuration entry, and then follow this logic:
                    # * got asked to create a new node -> check for an entry in the DB for a node that
                    # matches the characteristics of it.
                    #  * if there is one already:
                    #    - check that if it has been running for more than N (configurable) minutes (defaults to 8):
                    #      * if it has, it means that it is probable busy already, so:
                    #        - create a new node in the cloud backend matching the characteristics needed
                    #      * if it hasn't, it means that it is still getting provisioned so:
                    #        - skip - do a log warning
                    #  * if there is more than one, and it has been more than
                    #    N (8) minutes since they got launched it is possible
                    #    that they are configured *incorrectly* and we should not
                    #    keep launching more, so log the warning and skip.
                    #    - now ask Jenkins about machines that have been idle
                    #      for N (configurable) minutes, and see if matches
                    #      a record in the DB for the characteristics that we are
                    #      being asked to create.
                    #      * if found/matched:
                    #        - log the warnings again, something is not working right.
                    if needed_nodes.get(node_name):
                        needed_nodes[node_name] += 1
                    else:
                        needed_nodes[node_name] = 1
                else:
                    logger.warning(
                        'could not match a node name to config for labels')
            else:
                logger.info('no tasks where fund in "stuck" state')
    elif result == []:
        logger.info('the Jenkins queue is empty, nothing to do')
    else:
        logger.warning('attempted to get queue info but got: %s' % result)
    # At this point we might have a bag of nodes that we need to create, go over that
    # mapping and ask as many as Jenkins needs:
    node_endpoint = get_mita_api('nodes')
    for node_name, count in needed_nodes.items():
        configured_node = pecan.conf.nodes[node_name]
        configured_node['name'] = node_name
        configured_node['count'] = count
        requests.post(node_endpoint, data=json.dumps(configured_node))
Пример #7
0
Файл: async.py Проект: ceph/mita
def check_queue():
    """
    Specifically checks for the status of the Jenkins queue. The docs are
    sparse here, but
    ``jenkins/core/src/main/resources/hudson/model/queue/CauseOfBlockage`` has
    the specific reasons this check needs:

    * *BecauseLabelIsBusy* Waiting for next available executor on {0}

    * *BecauseLabelIsOffline* All nodes of label \u2018{0}\u2019 are offline

    * *BecauseNodeIsBusy* Waiting for next available executor on {0}

    * *BecauseNodeLabelIsOffline* There are no nodes with the label \u2018{0}\u2019

    * *BecauseNodeIsOffline* {0} is offline

    The distinction is the need for a label or a node. In the case of a node,
    it will get matched directly to the nodes in the configuration, in case of a label
    it will go through the configured nodes and pick the first matched to its labels.

    Label needed example
    --------------------
    Jenkins queue reports that 'All nodes of label x86_64 are offline'. The
    configuration has::

        nodes: {
            'centos6': {
                ...
                'labels': ['x86_64', 'centos', 'centos6']
            }
        }

    Since 'x86_64' exists in the 'centos6' configured node, it will be sent off
    to create that one.

    Node needed example
    -------------------
    Jenkins reports that 'wheezy is offline'. The configuration has a few
    labels configured::

        nodes: {
            'wheezy': {
                ...
            }
            'centos6': {
                ...
            }
        }

    Since the key 'wheezy' matches the node required by the build system to
    continue it goes off to create it.
    """
    jenkins_url = pecan.conf.jenkins['url']
    jenkins_user = pecan.conf.jenkins['user']
    jenkins_token = pecan.conf.jenkins['token']
    conn = jenkins.Jenkins(jenkins_url, jenkins_user, jenkins_token)
    result = conn.get_queue_info()
    needed_nodes = {}

    if result:
        for task in result:
            if task['why'] is None:
                # this may happen when multiple tasks are getting pilled up (for a PR for example)
                # and there is no 'why' yet. So the API has a `None` for it which would break logic
                # to infer what is needed to get it unstuck
                continue
            if util.is_stuck(task['why']):
                logger.info('found stuck task with name: %s' % task['task']['name'])
                logger.info('reason was: %s' % task['why'])
                node_name = util.match_node(task['why'])
                if not node_name:
                    # this usually happens when jenkins is waiting on an executor
                    # on a static slave whose labels are not a subset of a
                    # node in the mita config
                    logger.warning('unable to match a suitable node')
                    logger.warning('will infer from job labels')
                    job_url = task['task']['url']
                    job_name = util.job_from_url(job_url)
                    try:
                        conn.get_job_info(job_name)
                        logger.info("Job info found for: %s", job_name)
                        node_name = util.match_node_from_job_config(job_url)
                    except jenkins.NotFoundException:
                        logger.warning('No job info found for: %s', job_name)
                        logger.warning('Will assume the job is a matrix job')
                        node_name = util.match_node_from_matrix_job_name(job_name)
                    if not node_name:
                        logger.warning('completely unable to match a node to provide')
                        continue
                logger.info('inferred node as: %s' % str(node_name))
                if node_name:
                    logger.info('matched a node name to config: %s' % node_name)
                    # TODO: this should talk to the pecan app over HTTP using
                    # the `app.conf.pecan_app` configuration entry, and then follow this logic:
                    # * got asked to create a new node -> check for an entry in the DB for a node that
                    # matches the characteristics of it.
                    #  * if there is one already:
                    #    - check that if it has been running for more than N (configurable) minutes (defaults to 8):
                    #      * if it has, it means that it is probable busy already, so:
                    #        - create a new node in the cloud backend matching the characteristics needed
                    #      * if it hasn't, it means that it is still getting provisioned so:
                    #        - skip - do a log warning
                    #  * if there is more than one, and it has been more than
                    #    N (8) minutes since they got launched it is possible
                    #    that they are configured *incorrectly* and we should not
                    #    keep launching more, so log the warning and skip.
                    #    - now ask Jenkins about machines that have been idle
                    #      for N (configurable) minutes, and see if matches
                    #      a record in the DB for the characteristics that we are
                    #      being asked to create.
                    #      * if found/matched:
                    #        - log the warnings again, something is not working right.
                    if needed_nodes.get(node_name):
                        needed_nodes[node_name] += 1
                    else:
                        needed_nodes[node_name] = 1
                else:
                    logger.warning('could not match a node name to config for labels')
            else:
                logger.info('no tasks where fund in "stuck" state')
    elif result == []:
        logger.info('the Jenkins queue is empty, nothing to do')
    else:
        logger.warning('attempted to get queue info but got: %s' % result)
    # At this point we might have a bag of nodes that we need to create, go over that
    # mapping and ask as many as Jenkins needs:
    node_endpoint = get_mita_api('nodes')
    for node_name, count in needed_nodes.items():
        configured_node = pecan.conf.nodes[node_name]
        configured_node['name'] = node_name
        configured_node['count'] = count
        requests.post(node_endpoint, data=json.dumps(configured_node))
Пример #8
0
 def test_failed_to_fetch_job_config(self, m_requests):
     mock_response = MagicMock()
     mock_response.raise_for_status.side_effect = requests.exceptions.RequestException()
     m_requests.get.return_value = mock_response
     result = util.match_node_from_job_config("https://jenkins.ceph.com/job/ceph-pull-requests")
     assert not result