def delete_record(self, relation_name, record):
"""Delete the given record.
Args:
relation_name (unicode): Name of the relation/table
record (object) : Relation/Table record
Raises:
RuntimeError : Fail to store data in Consul
TypeError : If relation_name is not a 'string' type and/or
record is None
"""
if not isinstance(relation_name, six.string_types) or (record is None):
raise TypeError
# Find the primary index of the relation
pi = self._get_relation_index(relation_name, 'primary')
# Prepare the key in the required format
key = consul_key_join(relation_name, pi, getattr(record, pi))
session_id = self.create_session()
try:
# Delete the KV pair in Consul
self.connection.kv.delete(key)
index, data = self.connection.kv.get(key)
if data is not None:
LOG.error("Unable to delete record in Consul")
raise RuntimeError
# Also delete the secondary index(es) records from Consul
si = self._get_relation_index(relation_name, 'secondary')
if si is None:
return
for index in si:
key = consul_key_join(relation_name, index,
getattr(record, index), pi,
getattr(record, pi))
self.connection.kv.delete(key)
consul_index, data = self.connection.kv.get(key)
if data is not None:
LOG.error("Unable to delete record in Consul")
raise RuntimeError
finally:
self.destroy_session(session_id)
def get_key_prefix(self, key_prefix, keys=False):
"""Fetch the records with the Key prefix in Consul
Args:
key_prefix (str): Key Prefix
keys (bool): If True, only get records' key from consul
Returns:
list: List of records
Raises:
TypeError: If Key is not a 'string' type
RuntimeError: Fail to get data from Consul
"""
if not isinstance(key_prefix, six.string_types):
raise TypeError
consul_index, data = self.connection.kv.get(
consul_key_join(key_prefix), recurse=True, keys=keys)
if data is None:
return []
if not keys:
# Prepare a list of all the Consul record 'Value' field
records = [record['Value'] for record in data]
else:
return data
return records
def get_records(self, relation_name):
"""Retrieve list of all records of a relation/table.
Args:
relation_name (unicode): Name of the relation/table
Returns:
list: All records in the relation.
Raises:
TypeError : If passed argument is not of 'string' type
"""
if not isinstance(relation_name, six.string_types):
raise TypeError
# Find the primary index of the relation
field = self._get_relation_index(relation_name, 'primary')
# Prepare the prefix of the Consul key in the required format
# e.g. ikepolicies/UUID/
key = consul_key_join(relation_name, field) + CONSUL_SEP
# Retrieve the list of all records from Consul
# Note : 'recurse=True' option fetches all the record with the
# given key prefix
consul_index, data = self.connection.kv.get(key, recurse=True)
if data is None:
return []
# Prepare a list of all the Consul record 'Value' field
records = [str_to_dict(record['Value']) for record in data]
return records
def get_record(self, relation_name, primary_index_value):
"""Retrieve a record from Consul with the required primary
index value
Args:
relation_name (unicode): Name of the relation/table
primary_index_value (unicode) : Primary index(key) value
Returns:
Empty List ([]) OR A record with matching primary index
value
Raises:
TypeError : If passed arguments are not of 'string' type
"""
if (not isinstance(relation_name, six.string_types)
or not isinstance(primary_index_value, six.string_types)):
raise TypeError
# Find the Primary Index of the relation
field = self._get_relation_index(relation_name, 'primary')
# Prepare the key in the required format
# e.g. ikepolicies/UUID/fc5221be-b9d0-11e5-8338-005056b46cff
key = consul_key_join(relation_name, field, primary_index_value)
# Fetch the record with the prepared key
consul_index, data = self.connection.kv.get(key)
if data is not None:
return str_to_dict(data['Value'])
else:
return []
def _prepare_secondary_indices(self, relation_name, record):
"""Store each secondary index of the relation/table along with
primary index value for the record. The list of secondary
index is present in the consul_config.
Args:
relation_name (unicode): Name of the relation/table
record (Any relation record object) : Relation/Table record
Raises:
RuntimeError : Fail to store data in Consul
"""
# Find the primary index of the relation
pi = self._get_relation_index(relation_name, 'primary')
# Prepare the Consul value
value = consul_key_join(relation_name, pi, getattr(record, pi))
# Find the list of Secondary Index
si_list = self._get_relation_index(relation_name, 'secondary')
if si_list is None:
return
for si in si_list:
# Adding primary index value to the key helps in storing
# multiple values for same index.
# For update(PUT/PATCH) operation of secondary index(es),
# first delete the existing secondary index Consul record
key_prefix = consul_key_join(relation_name, si) + CONSUL_SEP
key_suffix = CONSUL_SEP.join((pi, getattr(record, pi)))
consul_index, data = self.connection.kv.get(key_prefix,
keys=True,
recurse=True)
if data is not None:
regex = key_prefix + CONSUL_SEP.join(('*', key_suffix))
match_key = fnmatch.filter(data, regex)
if match_key:
self.connection.kv.delete(match_key[0])
# Store secondary index in Consul
key = key_prefix + CONSUL_SEP.join(
(getattr(record, si), key_suffix))
rvalue = self.connection.kv.put(key, value)
if rvalue is None:
LOG.error("Unable to store secondary index in Consul")
raise RuntimeError
def create_session(self):
"""Create a Consul session for critical section operations
Returns:
session_id (int): id of the Consul session
"""
lock_key = consul_key_join('CONSUL_LOCK')
lock_delay = 0 # in seconds
session_id = self.connection.session.create(lock_delay=lock_delay)
while not self.connection.kv.put(lock_key, '', acquire=session_id):
pass
return session_id
def get_records_by_secondary_index(self, relation_name, secondary_index,
field_value):
"""Retrieve a list of record for a secondary index from a
relation
Args:
relation_name (unicode): Name of the relation/table
secondary_index (unicode) : Required secondary index
field_value (unicode) : Secondary index value
Returns:
Empty list ([]) OR list of records with the given secondary
index value in the relation.
Raises:
TypeError : If passed arguments are not of 'string' type
"""
if (not isinstance(relation_name, six.string_types)
or not isinstance(secondary_index, six.string_types)
or not isinstance(field_value, six.string_types)):
raise TypeError
# Prepare the key prefix with secondary index in the required
# format
# e.g. ikepolicies/name/ike_1
key = consul_key_join(relation_name, secondary_index, field_value)
# Find the primary index value for the given secondary index
# value
consul_index, data = self.connection.kv.get(key, recurse=True)
if data is None:
return []
primary_index_records = (record['Value'] for record in data
if record['Key'].startswith(key + CONSUL_SEP))
if primary_index_records is None:
return []
records = []
# Fetch the record with the primary index value
for primary_index in primary_index_records:
consul_index, data = self.connection.kv.get(primary_index)
# Prepare a list of all the Consul records' 'Value' field
records.append(str_to_dict(data['Value']))
return records
def _store_record_in_consul(self, relation_name, record):
"""Store record in the Consul with primary index(pi) as the
'key' and the record in JSON format as the 'value'.
Args:
relation_name (unicode): Name of the relation/table
record (Any relation record object) : Relation/Table
record
Raises:
RuntimeError : Fail to store data in Consul
"""
session_id = self.create_session()
try:
# Find the primary index of the relation
pi = self._get_relation_index(relation_name, 'primary')
# Prepare the key in the required format
# e.g. ikepolicies/UUID/fc5221be-b9d0-11e5-8338-005056b46cff
key = consul_key_join(relation_name, pi, getattr(record, pi))
# Convert the object into JSON(dict)
value = json.dumps(record.__dict__, cls=CustomEncoder)
# Store secondary indexes in Consul
self._prepare_secondary_indices(relation_name, record)
# Store KV pair in Consul
rvalue = self.connection.kv.put(key, value)
finally:
self.destroy_session(session_id)
if rvalue is None:
LOG.error("Unable to store record in Consul")
raise RuntimeError