Python is_job_flow_done Examples

Example #1

File: test_terminate_idle_job_flows.py Project: yukatherin/mrjob

 def time_job_flow_idle(self, jf):
     if (is_job_flow_bootstrapping(jf) or
         is_job_flow_running(jf) or
         is_job_flow_done(jf)):
         return timedelta(0)
     else:
         return self.now - time_last_active(jf)

Example #2

File: test_terminate_idle_job_flows.py Project: slojo404/mrjob

    def test_done(self):
        jf = self.mock_emr_job_flows['j-DONE']

        self.assertEqual(is_job_flow_done(jf), True)
        self.assertEqual(is_job_flow_running(jf), False)
        self.assertEqual(is_job_flow_non_streaming(jf), False)
        self.assertEqual(time_job_flow_idle(jf, self.now), timedelta(0))

Example #3

File: test_terminate_idle_job_flows.py Project: bchess/mrjob

    def test_hadoop_debugging_job_flow(self):
        jf = self.mock_emr_job_flows['j-HADOOP_DEBUGGING']

        self.assertEqual(is_job_flow_done(jf), False)
        self.assertEqual(is_job_flow_running(jf), False)
        self.assertEqual(is_job_flow_non_streaming(jf), False)
        self.assertEqual(time_job_flow_idle(jf, self.now), timedelta(hours=2))

Example #4

    def test_idle_and_failed(self):
        jf = self.mock_emr_job_flows['j-IDLE_AND_FAILED']

        assert_equal(is_job_flow_done(jf), False)
        assert_equal(is_job_flow_running(jf), False)
        assert_equal(is_job_flow_non_streaming(jf), False)
        assert_equal(time_job_flow_idle(jf, self.now), timedelta(hours=3))

Example #5

File: terminate_idle_job_flows_test.py Project: gimlids/LTPM

    def test_done(self):
        jf = self.mock_emr_job_flows['j-DONE']

        assert_equal(is_job_flow_done(jf), True)
        assert_equal(is_job_flow_running(jf), False)
        assert_equal(is_job_flow_non_streaming(jf), False)
        assert_equal(time_job_flow_idle(jf, self.now), timedelta(0))

Example #6

File: test_terminate_idle_job_flows.py Project: bchess/mrjob

    def test_done_and_idle(self):
        jf = self.mock_emr_job_flows['j-DONE_AND_IDLE']

        self.assertEqual(is_job_flow_done(jf), False)
        self.assertEqual(is_job_flow_running(jf), False)
        self.assertEqual(is_job_flow_non_streaming(jf), False)
        self.assertEqual(time_job_flow_idle(jf, self.now), timedelta(hours=2))

Example #7

    def test_done_and_idle(self):
        jf = self.mock_emr_job_flows['j-DONE_AND_IDLE']

        assert_equal(is_job_flow_done(jf), False)
        assert_equal(is_job_flow_running(jf), False)
        assert_equal(is_job_flow_non_streaming(jf), False)
        assert_equal(time_job_flow_idle(jf, self.now), timedelta(hours=2))

Example #8

File: test_terminate_idle_job_flows.py Project: bchess/mrjob

    def test_hive_job_flow(self):
        jf = self.mock_emr_job_flows['j-HIVE']

        self.assertEqual(is_job_flow_done(jf), False)
        self.assertEqual(is_job_flow_running(jf), False)
        self.assertEqual(is_job_flow_non_streaming(jf), True)
        self.assertEqual(time_job_flow_idle(jf, self.now), timedelta(hours=4))

Example #9

    def test_hive_job_flow(self):
        jf = self.mock_emr_job_flows['j-HIVE']

        assert_equal(is_job_flow_done(jf), False)
        assert_equal(is_job_flow_running(jf), False)
        assert_equal(is_job_flow_non_streaming(jf), True)
        assert_equal(time_job_flow_idle(jf, self.now), timedelta(hours=4))

Example #10

File: test_terminate_idle_job_flows.py Project: bchess/mrjob

    def test_empty(self):
        jf = self.mock_emr_job_flows['j-EMPTY']

        self.assertEqual(is_job_flow_done(jf), False)
        self.assertEqual(is_job_flow_running(jf), False)
        self.assertEqual(is_job_flow_non_streaming(jf), False)
        self.assertEqual(time_job_flow_idle(jf, self.now), timedelta(hours=10))

Example #11

    def test_hadoop_debugging_job_flow(self):
        jf = self.mock_emr_job_flows['j-HADOOP_DEBUGGING']

        assert_equal(is_job_flow_done(jf), False)
        assert_equal(is_job_flow_running(jf), False)
        assert_equal(is_job_flow_non_streaming(jf), False)
        assert_equal(time_job_flow_idle(jf, self.now), timedelta(hours=2))

Example #12

    def test_empty(self):
        jf = self.mock_emr_job_flows['j-EMPTY']

        assert_equal(is_job_flow_done(jf), False)
        assert_equal(is_job_flow_running(jf), False)
        assert_equal(is_job_flow_non_streaming(jf), False)
        assert_equal(time_job_flow_idle(jf, self.now), timedelta(hours=10))

Example #13

File: test_terminate_idle_job_flows.py Project: bchess/mrjob

    def test_idle_and_failed(self):
        jf = self.mock_emr_job_flows['j-IDLE_AND_FAILED']

        self.assertEqual(is_job_flow_done(jf), False)
        self.assertEqual(is_job_flow_running(jf), False)
        self.assertEqual(is_job_flow_non_streaming(jf), False)
        self.assertEqual(time_job_flow_idle(jf, self.now), timedelta(hours=3))

Example #14

File: test_terminate_idle_job_flows.py Project: kevinburke/mrjob

 def test_currently_running(self):
     jf = self.mock_emr_job_flows['j-CURRENTLY_RUNNING']
     self.assertEqual(is_job_flow_done(jf), False)
     self.assertEqual(is_job_flow_running(jf), True)
     self.assertEqual(is_job_flow_non_streaming(jf), False)
     self.assertEqual(time_job_flow_idle(jf, self.now), timedelta(0))
     self.assertEqual(time_to_end_of_hour_for_job_flow(jf, self.now),
                      timedelta(minutes=45))
     self.assertEqual(job_flow_pool_name(jf), None)

Example #15

File: test_terminate_idle_job_flows.py Project: ddehghan/mrjob

 def test_currently_running(self):
     jf = self.mock_emr_job_flows['j-CURRENTLY_RUNNING']
     self.assertEqual(is_job_flow_done(jf), False)
     self.assertEqual(is_job_flow_running(jf), True)
     self.assertEqual(is_job_flow_non_streaming(jf), False)
     self.assertEqual(time_job_flow_idle(jf, self.now), timedelta(0))
     self.assertEqual(time_to_end_of_hour_for_job_flow(jf, self.now),
                      timedelta(minutes=45))
     self.assertEqual(job_flow_pool_name(jf), None)

Example #16

File: test_terminate_idle_job_flows.py Project: dataartisan/mrjob

    def test_hive_job_flow(self):
        jf = self.mock_emr_job_flows['j-HIVE']

        self.assertEqual(is_job_flow_done(jf), False)
        self.assertEqual(is_job_flow_running(jf), False)
        self.assertEqual(is_job_flow_non_streaming(jf), True)
        self.assertEqual(time_job_flow_idle(jf, self.now), timedelta(hours=4))
        self.assertEqual(est_time_to_hour(jf, self.now),
                         timedelta(hours=1))
        self.assertEqual(pool_hash_and_name(jf), (None, None))

Example #17

File: test_terminate_idle_job_flows.py Project: kevinburke/mrjob

    def test_idle_and_failed(self):
        jf = self.mock_emr_job_flows['j-IDLE_AND_FAILED']

        self.assertEqual(is_job_flow_done(jf), False)
        self.assertEqual(is_job_flow_running(jf), False)
        self.assertEqual(is_job_flow_non_streaming(jf), False)
        self.assertEqual(time_job_flow_idle(jf, self.now), timedelta(hours=3))
        self.assertEqual(time_to_end_of_hour_for_job_flow(jf, self.now),
                         timedelta(hours=1))
        self.assertEqual(job_flow_pool_name(jf), None)

Example #18

File: test_terminate_idle_job_flows.py Project: ddehghan/mrjob

    def test_hive_job_flow(self):
        jf = self.mock_emr_job_flows['j-HIVE']

        self.assertEqual(is_job_flow_done(jf), False)
        self.assertEqual(is_job_flow_running(jf), False)
        self.assertEqual(is_job_flow_non_streaming(jf), True)
        self.assertEqual(time_job_flow_idle(jf, self.now), timedelta(hours=4))
        self.assertEqual(time_to_end_of_hour_for_job_flow(jf, self.now),
                         timedelta(hours=1))
        self.assertEqual(job_flow_pool_name(jf), None)

Example #19

File: test_terminate_idle_job_flows.py Project: kevinburke/mrjob

    def test_hive_job_flow(self):
        jf = self.mock_emr_job_flows['j-HIVE']

        self.assertEqual(is_job_flow_done(jf), False)
        self.assertEqual(is_job_flow_running(jf), False)
        self.assertEqual(is_job_flow_non_streaming(jf), True)
        self.assertEqual(time_job_flow_idle(jf, self.now), timedelta(hours=4))
        self.assertEqual(time_to_end_of_hour_for_job_flow(jf, self.now),
                         timedelta(hours=1))
        self.assertEqual(job_flow_pool_name(jf), None)

Example #20

File: test_terminate_idle_job_flows.py Project: kevinburke/mrjob

    def test_hadoop_debugging_job_flow(self):
        jf = self.mock_emr_job_flows['j-HADOOP_DEBUGGING']

        self.assertEqual(is_job_flow_done(jf), False)
        self.assertEqual(is_job_flow_running(jf), False)
        self.assertEqual(is_job_flow_non_streaming(jf), False)
        self.assertEqual(time_job_flow_idle(jf, self.now), timedelta(hours=2))
        self.assertEqual(time_to_end_of_hour_for_job_flow(jf, self.now),
                         timedelta(hours=1))
        self.assertEqual(job_flow_pool_name(jf), None)

Example #21

File: test_terminate_idle_job_flows.py Project: ddehghan/mrjob

    def test_pooled(self):
        jf = self.mock_emr_job_flows['j-POOLED']

        self.assertEqual(is_job_flow_done(jf), False)
        self.assertEqual(is_job_flow_running(jf), False)
        self.assertEqual(is_job_flow_non_streaming(jf), False)
        self.assertEqual(time_job_flow_idle(jf, self.now),
                         timedelta(minutes=55))
        self.assertEqual(time_to_end_of_hour_for_job_flow(jf, self.now),
                         timedelta(minutes=5))
        self.assertEqual(job_flow_pool_name(jf), 'reflecting')

Example #22

File: test_terminate_idle_job_flows.py Project: kevinburke/mrjob

    def test_pooled(self):
        jf = self.mock_emr_job_flows['j-POOLED']

        self.assertEqual(is_job_flow_done(jf), False)
        self.assertEqual(is_job_flow_running(jf), False)
        self.assertEqual(is_job_flow_non_streaming(jf), False)
        self.assertEqual(time_job_flow_idle(jf, self.now),
                         timedelta(minutes=55))
        self.assertEqual(time_to_end_of_hour_for_job_flow(jf, self.now),
                         timedelta(minutes=5))
        self.assertEqual(job_flow_pool_name(jf), 'reflecting')

Example #23

File: test_terminate_idle_job_flows.py Project: dataartisan/mrjob

    def test_pooled(self):
        jf = self.mock_emr_job_flows['j-POOLED']

        self.assertEqual(is_job_flow_done(jf), False)
        self.assertEqual(is_job_flow_running(jf), False)
        self.assertEqual(is_job_flow_non_streaming(jf), False)
        self.assertEqual(time_job_flow_idle(jf, self.now),
                         timedelta(minutes=55))
        self.assertEqual(est_time_to_hour(jf, self.now),
                         timedelta(minutes=5))
        self.assertEqual(pool_hash_and_name(jf),
                         ('0123456789abcdef0123456789abcdef', 'reflecting'))

Example #24

File: test_terminate_idle_job_flows.py Project: yukatherin/mrjob

 def assertJobFlowIs(
     self, jf,
     bootstrapping=False,
     done=False,
     from_end_of_hour=timedelta(hours=1),
     has_pending_steps=False,
     idle_for=timedelta(0),
     pool_hash=None,
     pool_name=None,
     running=False,
     streaming=True,
 ):
     self.assertEqual(bootstrapping, is_job_flow_bootstrapping(jf))
     self.assertEqual(done, is_job_flow_done(jf))
     self.assertEqual(from_end_of_hour, est_time_to_hour(jf, self.now))
     self.assertEqual(has_pending_steps, job_flow_has_pending_steps(jf))
     self.assertEqual(idle_for, self.time_job_flow_idle(jf))
     self.assertEqual((pool_hash, pool_name), pool_hash_and_name(jf))
     self.assertEqual(running, is_job_flow_running(jf))
     self.assertEqual(streaming, is_job_flow_streaming(jf))

Example #25

File: test_terminate_idle_job_flows.py Project: bchess/mrjob

 def test_currently_running(self):
     jf = self.mock_emr_job_flows['j-CURRENTLY_RUNNING']
     self.assertEqual(is_job_flow_done(jf), False)
     self.assertEqual(is_job_flow_running(jf), True)
     self.assertEqual(is_job_flow_non_streaming(jf), False)
     self.assertEqual(time_job_flow_idle(jf, self.now), timedelta(0))

Example #26

 def test_currently_running(self):
     jf = self.mock_emr_job_flows['j-CURRENTLY_RUNNING']
     assert_equal(is_job_flow_done(jf), False)
     assert_equal(is_job_flow_running(jf), True)
     assert_equal(is_job_flow_non_streaming(jf), False)
     assert_equal(time_job_flow_idle(jf, self.now), timedelta(0))