Exemplo n.º 1
0
    def test_countRealRunning_match(self):
        cq_run_dict = glideinFrontendLib.getRunningCondorQ(self.condorq_dict)
        glideinFrontendLib.appendRealRunning(cq_run_dict, self.status_dict)

        match_obj = compile('True', "<string>", "eval")
        actual = glideinFrontendLib.countRealRunning(
            match_obj, cq_run_dict, self.glidein_dict, {})
        expected = (
            {self.glidein_dict_k1: 1,
             self.glidein_dict_k2: 4,
             self.glidein_dict_k3: 0},
            {self.glidein_dict_k1: 1, self.glidein_dict_k2: 3, self.glidein_dict_k3: 0})
        self.assertEqual(expected, actual)

        match_obj = compile('False', "<string>", "eval")
        actual = glideinFrontendLib.countRealRunning(
            match_obj, cq_run_dict, self.glidein_dict, {})
        expected = (
            {self.glidein_dict_k1: 0,
             self.glidein_dict_k2: 0,
             self.glidein_dict_k3: 0},
            {self.glidein_dict_k1: 0, self.glidein_dict_k2: 0, self.glidein_dict_k3: 0})
        self.assertEqual(expected, actual)

        match_expr = 'glidein["attrs"].get("GLIDEIN_Site") in job.get("DESIRED_Sites", [])'
        match_obj = compile(match_expr, "<string>", "eval")
        actual = glideinFrontendLib.countRealRunning(
            match_obj, cq_run_dict, self.glidein_dict, {})
        expected = (
            {self.glidein_dict_k1: 1,
             self.glidein_dict_k2: 1,
             self.glidein_dict_k3: 0},
            {self.glidein_dict_k1: 1, self.glidein_dict_k2: 1, self.glidein_dict_k3: 0})
        self.assertEqual(expected, actual)
Exemplo n.º 2
0
    def test_countRealRunning_match(self):
        cq_run_dict = glideinFrontendLib.getRunningCondorQ(self.condorq_dict)
        glideinFrontendLib.appendRealRunning(cq_run_dict, self.status_dict)

        match_obj = compile('True', "<string>", "eval")
        actual = glideinFrontendLib.countRealRunning(match_obj, cq_run_dict, self.glidein_dict, {})
        expected = (
            {self.glidein_dict_k1: 1, self.glidein_dict_k2: 4, self.glidein_dict_k3: 0},
            {self.glidein_dict_k1: 1, self.glidein_dict_k2: 3, self.glidein_dict_k3: 0})
        self.assertEqual(expected, actual)

        match_obj = compile('False', "<string>", "eval")
        actual = glideinFrontendLib.countRealRunning(match_obj, cq_run_dict, self.glidein_dict, {})
        expected = (
            {self.glidein_dict_k1: 0, self.glidein_dict_k2: 0, self.glidein_dict_k3: 0},
            {self.glidein_dict_k1: 0, self.glidein_dict_k2: 0, self.glidein_dict_k3: 0})
        self.assertEqual(expected, actual)

        match_expr = 'glidein["attrs"].get("GLIDEIN_Site") in job.get("DESIRED_Sites", [])'
        match_obj = compile(match_expr, "<string>", "eval")
        actual = glideinFrontendLib.countRealRunning(match_obj, cq_run_dict, self.glidein_dict, {})
        expected = (
            {self.glidein_dict_k1: 1, self.glidein_dict_k2: 1, self.glidein_dict_k3: 0},
            {self.glidein_dict_k1: 1, self.glidein_dict_k2: 1, self.glidein_dict_k3: 0})
        self.assertEqual(expected, actual)
Exemplo n.º 3
0
    def test_getRunningCondorQ(self):
        condor_ids = \
            glideinFrontendLib.getRunningCondorQ(self.condorq_dict)['sched1'].fetchStored().keys()

        self.assertItemsEqual(condor_ids, [(12345, 3), (12345, 4), (12345, 5),
                                           (12345, 10), (12345, 11),
                                           (12345, 12)])
Exemplo n.º 4
0
    def test_getRunningCondorQ(self):
        condor_ids = \
            glideinFrontendLib.getRunningCondorQ(
                self.condorq_dict)['sched1'].fetchStored().keys()

        self.assertItemsEqual(
            condor_ids, [
                (12345, 3), (12345, 4), (12345, 5), (12345, 10), (12345, 11), (12345, 12)])
Exemplo n.º 5
0
    def test_appendRealRunning(self):
        cq_run_dict = glideinFrontendLib.getRunningCondorQ(self.condorq_dict)
        glideinFrontendLib.appendRealRunning(cq_run_dict, self.status_dict)
        expected = ['Site_Name%s@v3_0@[email protected]' % (x) for x in [1,2,2,2,2]]
        expected.append('UNKNOWN')

        self.assertItemsEqual(
            [x['RunningOn'] for x in cq_run_dict['sched1'].fetchStored().values()],
            expected)
Exemplo n.º 6
0
    def test_countRealRunning_otherException(self):
        cq_run_dict = glideinFrontendLib.getRunningCondorQ(self.condorq_dict)
        glideinFrontendLib.appendRealRunning(cq_run_dict, self.status_dict)
        with mock.patch.object(glideinwms.frontend.glideinFrontendLib.logSupport.log, 'debug') as m_debug:
            match_obj = compile('3/0', "<string>", "eval")
            actual = glideinFrontendLib.countRealRunning(match_obj, cq_run_dict, self.glidein_dict, {})
            log_msg = m_debug.call_args[0]

            self.assertTrue('exceptions in countMatch subprocess. Most recent traceback') in log_msg
            self.assertTrue('ZeroDivisionError: integer division or modulo by zero') in log_msg
Exemplo n.º 7
0
    def test_countRealRunning_missingKey(self):
        cq_run_dict = glideinFrontendLib.getRunningCondorQ(self.condorq_dict)
        glideinFrontendLib.appendRealRunning(cq_run_dict, self.status_dict)

        with mock.patch.object(glideinwms.frontend.glideinFrontendLib.logSupport.log, 'debug') as m_debug:
            match_obj = compile('glidein["attrs"]["FOO"] == 3', "<string>", "eval")
            actual = glideinFrontendLib.countRealRunning(match_obj, cq_run_dict, self.glidein_dict, {})
            m_debug.assert_any_call(
                "Failed to evaluate resource match in countRealRunning. Possibly match_expr has "
                "errors and trying to reference job or site attribute(s) ''FOO'' in an inappropriate way.")
Exemplo n.º 8
0
    def test_appendRealRunning(self):
        cq_run_dict = glideinFrontendLib.getRunningCondorQ(self.condorq_dict)
        glideinFrontendLib.appendRealRunning(cq_run_dict, self.status_dict)
        expected = [
            'Site_Name%s@v3_0@[email protected]' %
            (x) for x in [
                1, 2, 2, 2, 2]]
        expected.append('UNKNOWN')

        self.assertItemsEqual(
            [x['RunningOn']
             for x in cq_run_dict['sched1'].fetchStored().values()],
            expected)
Exemplo n.º 9
0
    def test_countRealRunning_otherException(self):
        cq_run_dict = glideinFrontendLib.getRunningCondorQ(self.condorq_dict)
        glideinFrontendLib.appendRealRunning(cq_run_dict, self.status_dict)
        with mock.patch.object(glideinwms.frontend.glideinFrontendLib.logSupport.log, 'debug') as m_debug:
            match_obj = compile('3/0', "<string>", "eval")
            actual = glideinFrontendLib.countRealRunning(
                match_obj, cq_run_dict, self.glidein_dict, {})
            log_msg = m_debug.call_args[0]

            self.assertTrue(
                'Running glidein ids at ' in str(log_msg), log_msg)
            self.assertTrue(
                'total glideins: 0, total jobs 0, cluster matches: 0' in str(log_msg), log_msg)
Exemplo n.º 10
0
    def test_countRealRunning_otherException(self):
        cq_run_dict = glideinFrontendLib.getRunningCondorQ(self.condorq_dict)
        glideinFrontendLib.appendRealRunning(cq_run_dict, self.status_dict)
        with mock.patch.object(glideinwms.frontend.glideinFrontendLib.logSupport.log, 'debug') as m_debug:
            match_obj = compile('3/0', "<string>", "eval")
            actual = glideinFrontendLib.countRealRunning(
                match_obj, cq_run_dict, self.glidein_dict, {})
            log_msg = m_debug.call_args[0]

            self.assertTrue(
                'Running glidein ids at ' in str(log_msg), log_msg)
            self.assertTrue(
                'total glideins: 0, total jobs 0, cluster matches: 0' in str(log_msg), log_msg)
Exemplo n.º 11
0
    def test_countRealRunning_missingKey(self):
        cq_run_dict = glideinFrontendLib.getRunningCondorQ(self.condorq_dict)
        glideinFrontendLib.appendRealRunning(cq_run_dict, self.status_dict)

        with mock.patch.object(glideinwms.frontend.glideinFrontendLib.logSupport.log, 'debug') as m_debug:
            match_obj = compile(
                'glidein["attrs"]["FOO"] == 3',
                "<string>",
                "eval")
            actual = glideinFrontendLib.countRealRunning(
                match_obj, cq_run_dict, self.glidein_dict, {})
            m_debug.assert_any_call(
                "Failed to evaluate resource match in countRealRunning. Possibly match_expr has "
                "errors and trying to reference job or site attribute(s) ''FOO'' in an inappropriate way.")
Exemplo n.º 12
0
    def test_countRealRunning_otherException(self):
        cq_run_dict = glideinFrontendLib.getRunningCondorQ(self.condorq_dict)
        glideinFrontendLib.appendRealRunning(cq_run_dict, self.status_dict)
        with mock.patch.object(
                glideinwms.frontend.glideinFrontendLib.logSupport.log,
                'debug') as m_debug:
            match_obj = compile('3/0', "<string>", "eval")
            actual = glideinFrontendLib.countRealRunning(
                match_obj, cq_run_dict, self.glidein_dict, {})
            log_msg = m_debug.call_args[0]

            self.assertTrue(
                'exceptions in countMatch subprocess. Most recent traceback'
            ) in log_msg
            self.assertTrue(
                'ZeroDivisionError: integer division or modulo by zero'
            ) in log_msg
    except RuntimeError, e:
        # expect all errors logged already
        logSupport.log.info("Missing schedd, factory entry, and/or current glidein state information. " \
                            "Unable to calculate required glideins, terminating loop.")
        return
    logSupport.log.info("All children terminated")

    glidein_dict=pipe_out['entries']
    condorq_dict=pipe_out['jobs']
    status_dict=pipe_out['startds']
    
    condorq_dict_proxy=glideinFrontendLib.getIdleProxyCondorQ(condorq_dict)
    condorq_dict_voms=glideinFrontendLib.getIdleVomsCondorQ(condorq_dict)
    condorq_dict_idle = glideinFrontendLib.getIdleCondorQ(condorq_dict)
    condorq_dict_old_idle = glideinFrontendLib.getOldCondorQ(condorq_dict_idle, 600)
    condorq_dict_running = glideinFrontendLib.getRunningCondorQ(condorq_dict)

    condorq_dict_types = {'Idle':{'dict':condorq_dict_idle, 'abs':glideinFrontendLib.countCondorQ(condorq_dict_idle)},
                          'OldIdle':{'dict':condorq_dict_old_idle, 'abs':glideinFrontendLib.countCondorQ(condorq_dict_old_idle)},
                          'VomsIdle':{'dict':condorq_dict_voms, 'abs':glideinFrontendLib.countCondorQ(condorq_dict_voms)},
                        'ProxyIdle':{'dict':condorq_dict_proxy,'abs':glideinFrontendLib.countCondorQ(condorq_dict_proxy)},
                          'Running':{'dict':condorq_dict_running, 'abs':glideinFrontendLib.countCondorQ(condorq_dict_running)}}
    condorq_dict_abs = glideinFrontendLib.countCondorQ(condorq_dict);


    stats['group'].logJobs({'Total':condorq_dict_abs,
                            'Idle':condorq_dict_types['Idle']['abs'],
                            'OldIdle':condorq_dict_types['OldIdle']['abs'],
                            'Running':condorq_dict_types['Running']['abs']})

    logSupport.log.info("Jobs found total %i idle %i (old %i, grid %i, voms %i) running %i" % (condorq_dict_abs,