def setUp(self):
self.prefix = 'this/dummy/prefix/'
self.client = consul.Consul()
self.agent = ConsulAgent()
self.prefixed_agent = ConsulAgent(prefix=self.prefix)
if self.client.kv.get('', recurse=True)[1]:
self.skipTest('Consul contains unknown values!')
def test_init(self):
"""
Tests ConsulAgent's init method and the data it's expected to receive and set.
"""
agent = ConsulAgent()
self.assertEqual(agent.prefix, '')
self.assertIsInstance(agent._client, consul.Consul)
# With custom parameters
prefix = 'custom_prefix'
agent = ConsulAgent(prefix=prefix)
self.assertEqual(agent.prefix, prefix)
self.assertIsInstance(agent._client, consul.Consul)
def _write_metadata_to_consul(self, configurations):
"""
Reflect passed configurations to Consul. Values on consul
will be updated only if they changed in this version of the
model using agent's `cas` parameter (Check-And-Set).
If we successfully updated at least one field in Consul
then the configurations' version number is gonna be incremented.
:note: This still doesn't apply removed-configurations case.
:param configurations: A dict object contains the configurations
to be written on Consul.
:return: A pair (version, changed) with the current version number and
a bool to indicate whether the information was updated.
"""
agent = ConsulAgent(prefix=self.consul_prefix)
version_updated = False
version_number = agent.get('version') or 0
for key, value in configurations.items():
index, stored_value = agent.get(key, index=True)
cas = index if stored_value is not None else 0
agent.put(key, value, cas=cas)
if not version_updated and value != stored_value:
version_updated = True
version_number += 1
agent.put('version', version_number)
return version_number, version_updated
def test_get_with_prefix(self):
"""
Tests getting a prefixed key of different data types from Consul's KEy-Value store.
"""
prefix = 'some-dummy/prefix/'
agent = ConsulAgent(prefix=prefix)
# Test string values
key = 'string_key'
stored_value = 'String Value'
self.client.kv.put(prefix + key, stored_value)
fetched_value = agent.get(key)
self.assertIsInstance(fetched_value, str)
self.assertEqual(fetched_value, stored_value)
# Test integer values
key = 'int_key'
stored_value = 23 # pylint: disable=redefined-variable-type
self.client.kv.put(prefix + key, str(stored_value))
fetched_value = agent.get(key)
self.assertIsInstance(fetched_value, int)
self.assertEqual(fetched_value, stored_value)
# Test float values
key = 'float_key'
stored_value = 23.23
self.client.kv.put(prefix + key, str(stored_value))
fetched_value = agent.get(key)
self.assertIsInstance(fetched_value, float)
self.assertEqual(fetched_value, stored_value)
# Test list values
key = 'list_key'
stored_value = [{'nice': 'good'}, {'awesome': 'things'}]
self.client.kv.put(prefix + key, json.dumps(stored_value))
fetched_value = agent.get(key)
self.assertIsInstance(fetched_value, list)
self.assertEqual(fetched_value, stored_value)
# Test dict values
key = 'dict_key'
stored_value = {'nice': 'good', 'awesome': 'things'}
self.client.kv.put(prefix + key, json.dumps(stored_value))
fetched_value = agent.get(key)
self.assertIsInstance(fetched_value, dict)
self.assertEqual(fetched_value, stored_value)
# Test other (boolean) objects
key = 'random_key'
stored_value = True
self.client.kv.put(prefix + key, str(stored_value))
fetched_value = agent.get(key)
self.assertIsInstance(fetched_value, str)
self.assertEqual(fetched_value, str(stored_value))
def test_put_with_prefix(self):
"""
Will test the put functionality on Consul with different data types after prefixing the keys.
"""
prefix = 'some/testing-prefix'
agent = ConsulAgent(prefix=prefix)
# Put string values
key = 'key'
value = 'value'
agent.put(key, value)
_, data = self.client.kv.get(prefix + key)
fetched_value = data['Value'].decode()
self.assertEqual(fetched_value, value)
# Put int values
key = 'key'
value = 1 # pylint: disable=redefined-variable-type
agent.put(key, value)
_, data = self.client.kv.get(prefix + key)
fetched_value = data['Value'].decode()
self.assertEqual(fetched_value, str(value))
# Put float values
key = 'key'
value = 1.1
agent.put(key, value)
_, data = self.client.kv.get(prefix + key)
fetched_value = data['Value'].decode()
self.assertEqual(fetched_value, str(value))
# Put list values
key = 'key'
value = [1, 2, 3, 5]
agent.put(key, value)
_, data = self.client.kv.get(prefix + key)
fetched_value = data['Value'].decode()
self.assertEqual(fetched_value, json.dumps(value))
# Put dict values
key = 'key'
value = {'key': 'value', 'another_key': 12}
agent.put(key, value)
_, data = self.client.kv.get(prefix + key)
fetched_value = data['Value'].decode()
self.assertEqual(fetched_value, json.dumps(value))
# Put other values
key = 'key'
value = False
agent.put(key, value)
_, data = self.client.kv.get(prefix + key)
fetched_value = data['Value'].decode()
self.assertEqual(fetched_value, json.dumps(value))
def test_get_no_prefix(self):
"""
Tests getting bare keys of different data types from Consul's Key-Value store.
"""
agent = ConsulAgent()
# Test string values
key = 'string_key'
stored_value = 'String Value'
self.client.kv.put(key, stored_value)
fetched_value = agent.get(key)
self.assertIsInstance(fetched_value, str)
self.assertEqual(fetched_value, stored_value)
# Test integer values
key = 'int_key'
stored_value = 23
self.client.kv.put(key, str(stored_value))
fetched_value = agent.get(key)
self.assertIsInstance(fetched_value, int)
self.assertEqual(fetched_value, stored_value)
# Test float values
key = 'float_key'
stored_value = 23.23
self.client.kv.put(key, str(stored_value))
fetched_value = agent.get(key)
self.assertIsInstance(fetched_value, float)
self.assertEqual(fetched_value, stored_value)
# Test list values
key = 'list_key'
stored_value = [{'nice': 'good'}, {'awesome': 'things'}]
self.client.kv.put(key, json.dumps(stored_value))
fetched_value = agent.get(key)
self.assertIsInstance(fetched_value, list)
self.assertEqual(fetched_value, stored_value)
# Test dict values
key = 'dict_key'
stored_value = {'nice': 'good', 'awesome': 'things'}
self.client.kv.put(key, json.dumps(stored_value))
fetched_value = agent.get(key)
self.assertIsInstance(fetched_value, dict)
self.assertEqual(fetched_value, stored_value)
# Test other (boolean) objects
key = 'random_key'
stored_value = True
self.client.kv.put(key, str(stored_value))
fetched_value = agent.get(key)
self.assertIsInstance(fetched_value, str)
self.assertEqual(fetched_value, str(stored_value))
def test_put_no_prefix(self):
"""
Will test the put functionality on Consul with different data types with no prefix on keys.
"""
agent = ConsulAgent()
# Put string values
key = 'key'
value = 'value'
agent.put(key, value)
_, data = self.client.kv.get(key)
fetched_value = data['Value'].decode()
self.assertEqual(fetched_value, value)
# Put int values
key = 'key'
value = 1
agent.put(key, value)
_, data = self.client.kv.get(key)
fetched_value = data['Value'].decode()
self.assertEqual(fetched_value, str(value))
# Put float values
key = 'key'
value = 1.1
agent.put(key, value)
_, data = self.client.kv.get(key)
fetched_value = data['Value'].decode()
self.assertEqual(fetched_value, str(value))
# Put list values
key = 'key'
value = [1, 2, 3, 5]
agent.put(key, value)
_, data = self.client.kv.get(key)
fetched_value = data['Value'].decode()
self.assertEqual(fetched_value, json.dumps(value))
# Put dict values
key = 'key'
value = {'key': 'value', 'another_key': 12}
agent.put(key, value)
_, data = self.client.kv.get(key)
fetched_value = data['Value'].decode()
self.assertEqual(fetched_value, json.dumps(value))
# Put other values
key = 'key'
value = False
agent.put(key, value)
_, data = self.client.kv.get(key)
fetched_value = data['Value'].decode()
self.assertEqual(fetched_value, json.dumps(value))
def test_txn_put_retries(self, mock_get_put_data, mock_txn_put):
"""
Tests that the txn_put method retries on error
"""
mock_get_put_data.return_value = (1, True)
mock_txn_put.side_effect = [consul.base.ClientError(), {}]
agent = ConsulAgent()
agent.txn_put({})
def _get_unused_id():
"""
Find an unused ID for Consul
"""
while True:
i = randint(2**30, 2**31)
if not ConsulAgent(prefix=settings.CONSUL_PREFIX.format(
ocim=settings.OCIM_ID, instance=i)).get(''):
return i
def test_txn_put_error_after_all_retries(self, mock_get_put_data,
mock_txn_put):
"""
Tests that the txn_put method raises the exception on error after exhausting all the retries
"""
mock_get_put_data.return_value = (1, True)
mock_txn_put.side_effect = consul.base.ClientError
with self.assertRaises(consul.base.ClientError):
agent = ConsulAgent()
agent.txn_put({})
def purge_consul_metadata(self):
"""
This method is responsible for purging all instances' metadata from
Consul if they exist.
"""
if not settings.CONSUL_ENABLED:
return
self.logger.info('Purging consul metadata with prefix: %s.', self.consul_prefix)
agent = ConsulAgent(prefix=self.consul_prefix)
agent.purge()
def clean_consul_metadata(self):
"""
This method will iterate over all archived instances and clean their
metadata from Consul.
"""
instances_ids = self.get_archived_instances()
agent = ConsulAgent()
self.stdout.write('Cleaning metadata for {} archived instances...'.format(len(instances_ids)))
for instances_id in instances_ids:
prefix = settings.CONSUL_PREFIX.format(ocim=settings.OCIM_ID, instance=instances_id)
agent.delete(prefix, recurse=True)
self.stdout.write(self.style.SUCCESS('Successfully cleaned archived instances\' metadata'))
def _write_metadata_to_consul(self, configurations):
"""
Reflect passed configurations to Consul.
If we successfully updated at least one field in Consul
then the configurations' version number is incremented.
:note: This still doesn't apply removed-configurations case.
:param configurations: A dict object contains the configurations
to be written on Consul.
:return: A pair (version, changed) with the current version number and
a bool to indicate whether the information was updated.
"""
return ConsulAgent(prefix=self.consul_prefix).txn_put(configurations)
def test_delete_no_prefix(self):
"""
Will test whether a key is gonna be deleted or not from the Key-Value store.
"""
agent = ConsulAgent()
self.client.kv.put('key', 'value')
self.client.kv.put('another_key', 'another value')
self.client.kv.put('dummy_key', '1')
_, values = self.client.kv.get('', recurse=True)
self.assertEqual(len(values), 3)
agent.delete('key')
_, values = self.client.kv.get('', recurse=True)
self.assertEqual(len(values), 2)
def test_purge_no_prefix(self):
"""
Purging with no prefix will remove all of the keys from Consul's Key-Value store
"""
agent = ConsulAgent()
self.client.kv.put('key', 'value')
self.client.kv.put('another_key', 'another value')
self.client.kv.put('dummy_key', '1')
_, values = self.client.kv.get('', recurse=True)
self.assertEqual(len(values), 3)
agent.purge()
_, values = self.client.kv.get('', recurse=True)
self.assertIsNone(values)
def test_purge_with_prefix(self):
"""
Purging with prefix should only remove the prefixed keys with the given prefix.
All other values must not be touched.
"""
prefix = 'nice-prefix'
agent = ConsulAgent(prefix=prefix)
self.client.kv.put(prefix + 'key', 'value')
self.client.kv.put(prefix + 'another_key', 'another value')
self.client.kv.put('dummy_key', '1')
_, values = self.client.kv.get('', recurse=True)
self.assertEqual(len(values), 3)
agent.purge()
_, values = self.client.kv.get('', recurse=True)
self.assertEqual(len(values), 1)
def test_delete_with_prefix(self):
"""
Delete with prefix will delete the given key from a prefixed agent.
"""
prefix = 'nice-prefix'
agent = ConsulAgent(prefix=prefix)
self.client.kv.put(prefix + 'key', 'value')
self.client.kv.put(prefix + 'another_key', 'another value')
self.client.kv.put('dummy_key', '1')
_, values = self.client.kv.get('', recurse=True)
self.assertEqual(len(values), 3)
agent.delete('key')
_, values = self.client.kv.get('', recurse=True)
self.assertEqual(len(values), 2)
agent.delete('dummy_key')
_, values = self.client.kv.get('', recurse=True)
self.assertEqual(len(values), 2)