def test_cluster_with_bad_lock(self):
# spaces in job key result in invalid lock
_attempt_to_lock_cluster(self.emr_client, self.cluster_id,
'blah blah blah')
acquired = _attempt_to_lock_cluster(self.emr_client, self.cluster_id,
self.our_key)
self.assertTrue(acquired)
def test_remove_existing_tags(self):
_attempt_to_lock_cluster(self.emr_client, self.cluster_id,
self.our_key)
self.assertIsNotNone(self._get_cluster_lock())
unlocked = _attempt_to_unlock_cluster(self.emr_client, self.cluster_id)
self.assertTrue(unlocked)
self.assertIsNone(self._get_cluster_lock())
def test_cluster_with_expired_lock(self):
they_acquired = _attempt_to_lock_cluster(self.emr_client,
self.cluster_id,
self.their_key)
self.assertTrue(they_acquired)
# 60 seconds later...
self.mock_time.side_effect = lambda: self.time() + 60.0
we_acquired = _attempt_to_lock_cluster(self.emr_client,
self.cluster_id, self.our_key)
self.assertTrue(we_acquired)
def test_cluster_with_current_lock(self):
they_acquired = _attempt_to_lock_cluster(self.emr_client,
self.cluster_id,
self.their_key)
self.assertTrue(they_acquired)
we_acquired = _attempt_to_lock_cluster(self.emr_client,
self.cluster_id, self.our_key)
self.assertFalse(we_acquired)
self.mock_sleep.assert_called_with(10.0)
def test_terminating_cluster(self):
self.emr_client.terminate_job_flows(JobFlowIds=[self.cluster_id])
acquired = _attempt_to_lock_cluster(self.emr_client, self.cluster_id,
self.our_key)
self.assertFalse(acquired)
self.assertFalse(self.mock_sleep.called)
def test_running_cluster(self):
self.mock_emr_clusters[self.cluster_id]['Status']['State'] = 'RUNNING'
acquired = _attempt_to_lock_cluster(self.emr_client, self.cluster_id,
self.our_key)
self.assertFalse(acquired)
self.assertFalse(self.mock_sleep.called)
def test_okay_if_cluster_terminated(self):
_attempt_to_lock_cluster(self.emr_client, self.cluster_id,
self.our_key)
self.assertIsNotNone(self._get_cluster_lock())
self.emr_client.terminate_job_flows(JobFlowIds=[self.cluster_id])
cluster = self.emr_client.describe_cluster(
ClusterId=self.cluster_id)['Cluster']
self.assertEqual(cluster['Status']['State'], 'TERMINATED')
unlocked = _attempt_to_unlock_cluster(self.emr_client, self.cluster_id)
# failed, but didn't crash
self.assertFalse(unlocked)
self.assertIsNotNone(self._get_cluster_lock())
def test_cluster_terminated_after_locking(self):
def _terminate_cluster(_):
self.emr_client.terminate_job_flows(JobFlowIds=[self.cluster_id])
self.mock_sleep.side_effect = _terminate_cluster
acquired = _attempt_to_lock_cluster(self.emr_client, self.cluster_id,
self.our_key)
self.assertFalse(acquired)
def test_too_slow_to_check_own_lock(self):
start = self.time()
self.mock_time.side_effect = [start, start + 3.0, start + 21.0]
acquired = _attempt_to_lock_cluster(self.emr_client, self.cluster_id,
self.our_key)
self.assertFalse(acquired)
self.assertTrue(self.mock_sleep.called)
def test_too_slow_to_read_tags(self):
start = self.time()
self.mock_time.side_effect = [start, start + 6.0]
acquired = _attempt_to_lock_cluster(self.emr_client, self.cluster_id,
self.our_key)
self.assertFalse(acquired)
self.assertFalse(self.mock_sleep.called)
def test_lock_deleted(self):
def _overwrite_lock(_):
self.emr_client.remove_tags(
ResourceId=self.cluster_id,
TagKeys=[_POOL_LOCK_KEY],
)
self.mock_sleep.side_effect = _overwrite_lock
acquired = _attempt_to_lock_cluster(self.emr_client, self.cluster_id,
self.our_key)
self.assertFalse(acquired)
def test_lock_overwritten_with_invalid_key(self):
def _overwrite_lock(_):
self.emr_client.add_tags(
ResourceId=self.cluster_id,
Tags=[dict(Key=_POOL_LOCK_KEY, Value='garbage')],
)
self.mock_sleep.side_effect = _overwrite_lock
acquired = _attempt_to_lock_cluster(self.emr_client, self.cluster_id,
self.our_key)
self.assertFalse(acquired)
def test_running_concurrent_cluster(self):
self.mock_emr_clusters[self.cluster_id]['Status']['State'] = 'RUNNING'
self.mock_emr_clusters[self.cluster_id]['StepConcurrencyLevel'] = 2
self.mock_emr_clusters[self.cluster_id]['_Steps'] = [
dict(Id='s-ONE', Status=dict(State='RUNNING'))
]
acquired = _attempt_to_lock_cluster(self.emr_client, self.cluster_id,
self.our_key)
self.assertTrue(acquired)
self.assertTrue(self.mock_sleep.called)
def test_concurrent_cluster_with_old_steps(self):
self.mock_emr_clusters[self.cluster_id]['Status']['State'] = 'RUNNING'
self.mock_emr_clusters[self.cluster_id]['StepConcurrencyLevel'] = 2
self.mock_emr_clusters[self.cluster_id]['_Steps'] = [
dict(Id='s-ONE', Status=dict(State='COMPLETE')),
dict(Id='s-TWO', Status=dict(State='CANCELLED')),
]
acquired = _attempt_to_lock_cluster(self.emr_client, self.cluster_id,
self.our_key)
self.assertTrue(acquired)
self.assertTrue(self.mock_sleep.called)
def test_concurrent_cluster_that_is_full(self):
self.mock_emr_clusters[self.cluster_id]['Status']['State'] = 'RUNNING'
self.mock_emr_clusters[self.cluster_id]['StepConcurrencyLevel'] = 2
self.mock_emr_clusters[self.cluster_id]['_Steps'] = [
dict(Id='s-ONE', Status=dict(State='RUNNING')),
dict(Id='s-TWO', Status=dict(State='PENDING')),
]
acquired = _attempt_to_lock_cluster(self.emr_client, self.cluster_id,
self.our_key)
self.assertFalse(acquired)
# only check number of steps once, after attempting to lock cluster
self.assertTrue(self.mock_sleep.called)
def test_cluster_with_no_lock(self):
acquired = _attempt_to_lock_cluster(self.emr_client, self.cluster_id,
self.our_key)
self.assertTrue(acquired)
self.mock_sleep.assert_called_once_with(10.0)
