def delete_record(self, relation_name, record): """Delete the given record. Args: relation_name (unicode): Name of the relation/table record (object) : Relation/Table record Raises: RuntimeError : Fail to store data in Consul TypeError : If relation_name is not a 'string' type and/or record is None """ if not isinstance(relation_name, six.string_types) or (record is None): raise TypeError # Find the primary index of the relation pi = self._get_relation_index(relation_name, 'primary') # Prepare the key in the required format key = consul_key_join(relation_name, pi, getattr(record, pi)) session_id = self.create_session() try: # Delete the KV pair in Consul self.connection.kv.delete(key) index, data = self.connection.kv.get(key) if data is not None: LOG.error("Unable to delete record in Consul") raise RuntimeError # Also delete the secondary index(es) records from Consul si = self._get_relation_index(relation_name, 'secondary') if si is None: return for index in si: key = consul_key_join(relation_name, index, getattr(record, index), pi, getattr(record, pi)) self.connection.kv.delete(key) consul_index, data = self.connection.kv.get(key) if data is not None: LOG.error("Unable to delete record in Consul") raise RuntimeError finally: self.destroy_session(session_id)
def get_key_prefix(self, key_prefix, keys=False): """Fetch the records with the Key prefix in Consul Args: key_prefix (str): Key Prefix keys (bool): If True, only get records' key from consul Returns: list: List of records Raises: TypeError: If Key is not a 'string' type RuntimeError: Fail to get data from Consul """ if not isinstance(key_prefix, six.string_types): raise TypeError consul_index, data = self.connection.kv.get( consul_key_join(key_prefix), recurse=True, keys=keys) if data is None: return [] if not keys: # Prepare a list of all the Consul record 'Value' field records = [record['Value'] for record in data] else: return data return records
def get_records(self, relation_name): """Retrieve list of all records of a relation/table. Args: relation_name (unicode): Name of the relation/table Returns: list: All records in the relation. Raises: TypeError : If passed argument is not of 'string' type """ if not isinstance(relation_name, six.string_types): raise TypeError # Find the primary index of the relation field = self._get_relation_index(relation_name, 'primary') # Prepare the prefix of the Consul key in the required format # e.g. ikepolicies/UUID/ key = consul_key_join(relation_name, field) + CONSUL_SEP # Retrieve the list of all records from Consul # Note : 'recurse=True' option fetches all the record with the # given key prefix consul_index, data = self.connection.kv.get(key, recurse=True) if data is None: return [] # Prepare a list of all the Consul record 'Value' field records = [str_to_dict(record['Value']) for record in data] return records
def get_record(self, relation_name, primary_index_value): """Retrieve a record from Consul with the required primary index value Args: relation_name (unicode): Name of the relation/table primary_index_value (unicode) : Primary index(key) value Returns: Empty List ([]) OR A record with matching primary index value Raises: TypeError : If passed arguments are not of 'string' type """ if (not isinstance(relation_name, six.string_types) or not isinstance(primary_index_value, six.string_types)): raise TypeError # Find the Primary Index of the relation field = self._get_relation_index(relation_name, 'primary') # Prepare the key in the required format # e.g. ikepolicies/UUID/fc5221be-b9d0-11e5-8338-005056b46cff key = consul_key_join(relation_name, field, primary_index_value) # Fetch the record with the prepared key consul_index, data = self.connection.kv.get(key) if data is not None: return str_to_dict(data['Value']) else: return []
def _prepare_secondary_indices(self, relation_name, record): """Store each secondary index of the relation/table along with primary index value for the record. The list of secondary index is present in the consul_config. Args: relation_name (unicode): Name of the relation/table record (Any relation record object) : Relation/Table record Raises: RuntimeError : Fail to store data in Consul """ # Find the primary index of the relation pi = self._get_relation_index(relation_name, 'primary') # Prepare the Consul value value = consul_key_join(relation_name, pi, getattr(record, pi)) # Find the list of Secondary Index si_list = self._get_relation_index(relation_name, 'secondary') if si_list is None: return for si in si_list: # Adding primary index value to the key helps in storing # multiple values for same index. # For update(PUT/PATCH) operation of secondary index(es), # first delete the existing secondary index Consul record key_prefix = consul_key_join(relation_name, si) + CONSUL_SEP key_suffix = CONSUL_SEP.join((pi, getattr(record, pi))) consul_index, data = self.connection.kv.get(key_prefix, keys=True, recurse=True) if data is not None: regex = key_prefix + CONSUL_SEP.join(('*', key_suffix)) match_key = fnmatch.filter(data, regex) if match_key: self.connection.kv.delete(match_key[0]) # Store secondary index in Consul key = key_prefix + CONSUL_SEP.join( (getattr(record, si), key_suffix)) rvalue = self.connection.kv.put(key, value) if rvalue is None: LOG.error("Unable to store secondary index in Consul") raise RuntimeError
def create_session(self): """Create a Consul session for critical section operations Returns: session_id (int): id of the Consul session """ lock_key = consul_key_join('CONSUL_LOCK') lock_delay = 0 # in seconds session_id = self.connection.session.create(lock_delay=lock_delay) while not self.connection.kv.put(lock_key, '', acquire=session_id): pass return session_id
def get_records_by_secondary_index(self, relation_name, secondary_index, field_value): """Retrieve a list of record for a secondary index from a relation Args: relation_name (unicode): Name of the relation/table secondary_index (unicode) : Required secondary index field_value (unicode) : Secondary index value Returns: Empty list ([]) OR list of records with the given secondary index value in the relation. Raises: TypeError : If passed arguments are not of 'string' type """ if (not isinstance(relation_name, six.string_types) or not isinstance(secondary_index, six.string_types) or not isinstance(field_value, six.string_types)): raise TypeError # Prepare the key prefix with secondary index in the required # format # e.g. ikepolicies/name/ike_1 key = consul_key_join(relation_name, secondary_index, field_value) # Find the primary index value for the given secondary index # value consul_index, data = self.connection.kv.get(key, recurse=True) if data is None: return [] primary_index_records = (record['Value'] for record in data if record['Key'].startswith(key + CONSUL_SEP)) if primary_index_records is None: return [] records = [] # Fetch the record with the primary index value for primary_index in primary_index_records: consul_index, data = self.connection.kv.get(primary_index) # Prepare a list of all the Consul records' 'Value' field records.append(str_to_dict(data['Value'])) return records
def _store_record_in_consul(self, relation_name, record): """Store record in the Consul with primary index(pi) as the 'key' and the record in JSON format as the 'value'. Args: relation_name (unicode): Name of the relation/table record (Any relation record object) : Relation/Table record Raises: RuntimeError : Fail to store data in Consul """ session_id = self.create_session() try: # Find the primary index of the relation pi = self._get_relation_index(relation_name, 'primary') # Prepare the key in the required format # e.g. ikepolicies/UUID/fc5221be-b9d0-11e5-8338-005056b46cff key = consul_key_join(relation_name, pi, getattr(record, pi)) # Convert the object into JSON(dict) value = json.dumps(record.__dict__, cls=CustomEncoder) # Store secondary indexes in Consul self._prepare_secondary_indices(relation_name, record) # Store KV pair in Consul rvalue = self.connection.kv.put(key, value) finally: self.destroy_session(session_id) if rvalue is None: LOG.error("Unable to store record in Consul") raise RuntimeError