def _prepare_m2m_references(self, relation_name, record): """Store Many to Many references""" m2m_ref_relations = self._get_reference_info(relation_name, 'many_to_many') if not m2m_ref_relations: return False primary_resource_pi = self._get_relation_index(m2m_ref_relations[0], 'primary') primary_resource_pi_value = getattr(record, '_'.join((m2m_ref_relations[0], primary_resource_pi))) secondary_resource_pi = self._get_relation_index(m2m_ref_relations[1], 'primary') secondary_resource_pi_value = getattr(record, '_'.join((m2m_ref_relations[1], secondary_resource_pi))) key = consul_key_join(relation_name, primary_resource_pi, primary_resource_pi_value, secondary_resource_pi, secondary_resource_pi_value) value = consul_key_join(relation_name.rsplit(CONSUL_SEP, 1)[0], m2m_ref_relations[1], secondary_resource_pi, secondary_resource_pi_value) # Store KV pair in Consul rvalue = self.connection.kv.put(key, value) if rvalue is None: LOG.error("Unable to store record in Consul") raise RuntimeError key = consul_key_join(relation_name, secondary_resource_pi, secondary_resource_pi_value, primary_resource_pi, primary_resource_pi_value) value = consul_key_join(relation_name.rsplit(CONSUL_SEP, 1)[0], m2m_ref_relations[0], primary_resource_pi, primary_resource_pi_value) # Store KV pair in Consul rvalue = self.connection.kv.put(key, value) if rvalue is None: LOG.error("Unable to store record in Consul") raise RuntimeError return True
def delete_record(self, relation_name, record): """Delete the given record. Args: relation_name (unicode): Name of the relation/table record (object): Relation/Table record Raises: RuntimeError: Fail to store data in Consul TypeError: If relation_name is not a 'string' type and/or record is None """ if not isinstance(relation_name, six.string_types) or (record is None): raise TypeError # Find the primary index of the relation pi = self._get_relation_index(relation_name, 'primary') # Prepare the key in the required format key = consul_key_join(relation_name, pi, getattr(record, pi)) session_id = self.create_session() try: if self._delete_m2m_references(relation_name, record): return # Delete the KV pair in Consul self.connection.kv.delete(key) index, data = self.connection.kv.get(key) if data is not None: LOG.error("Unable to delete record in Consul") raise RuntimeError # Also delete the secondary index(es) records from Consul si = self._get_relation_index(relation_name, 'secondary') if si is None: return for index in si: key = consul_key_join(relation_name, index, getattr(record, index), pi, getattr(record, pi)) self.connection.kv.delete(key) consul_index, data = self.connection.kv.get(key) if data is not None: LOG.error("Unable to delete record in Consul") raise RuntimeError finally: self.destroy_session(session_id)
def get_key_prefix(self, key_prefix, keys=False): """Fetch the records with the Key prefix in Consul Args: key_prefix (str): Key Prefix keys (bool): If True, only get records' key from consul Returns: list: List of records Raises: TypeError: If Key is not a 'string' type RuntimeError: Fail to get data from Consul """ if not isinstance(key_prefix, six.string_types): raise TypeError consul_index, data = self.connection.kv.get(consul_key_join(key_prefix), recurse=True, keys=keys) if data is None: return [] if not keys: # Prepare a list of all the Consul record 'Value' field records = [record['Value'] for record in data] else: return data return records
def get_records(self, relation_name): """Retrieve list of all records of a relation/table. Args: relation_name (unicode): Name of the relation/table Returns: list: All records in the relation. Raises: TypeError: If passed argument is not of 'string' type """ if not isinstance(relation_name, six.string_types): raise TypeError # Find the primary index of the relation field = self._get_relation_index(relation_name, 'primary') # Prepare the prefix of the Consul key in the required format # e.g. ikepolicies/UUID/ key = consul_key_join(relation_name, field) + CONSUL_SEP # Retrieve the list of all records from Consul # Note : 'recurse=True' option fetches all the record with the # given key prefix consul_index, data = self.connection.kv.get(key, recurse=True) if data is None: return [] # Prepare a list of all the Consul record 'Value' field records = [self._prepare_record(relation_name, record['Value']) for record in data] return records
def get_record(self, relation_name, primary_index_value): """Retrieve a record from Consul with the required primary index value Args: relation_name (unicode): Name of the relation/table primary_index_value (unicode): Primary index(key) value Returns: Empty List ([]) OR A record with matching primary index value Raises: TypeError: If passed arguments are not of 'string' type """ if (not isinstance(relation_name, six.string_types) or not isinstance(primary_index_value, six.string_types)): raise TypeError # Find the Primary Index of the relation field = self._get_relation_index(relation_name, 'primary') # Prepare the key in the required format # e.g. ikepolicies/UUID/fc5221be-b9d0-11e5-8338-005056b46cff key = consul_key_join(relation_name, field, primary_index_value) # Fetch the record with the prepared key consul_index, data = self.connection.kv.get(key) if data is not None: return self._prepare_record(relation_name, data['Value']) else: return []
def _prepare_secondary_indices(self, relation_name, record): """Store each secondary index of the relation/table along with primary index value for the record. The list of secondary index is present in the consul_config. Args: relation_name (unicode): Name of the relation/table record (Any relation record object): Relation/Table record Raises: RuntimeError: Fail to store data in Consul """ # Find the primary index of the relation pi = self._get_relation_index(relation_name, 'primary') # Prepare the Consul value value = consul_key_join(relation_name, pi, getattr(record, pi)) # Find the list of Secondary Index si_list = self._get_relation_index(relation_name, 'secondary') if si_list is None: return for si in si_list: # Adding primary index value to the key helps in storing # multiple values for same index. # For update(PUT/PATCH) operation of secondary index(es), # first delete the existing secondary index Consul record key_prefix = consul_key_join(relation_name, si) + CONSUL_SEP key_suffix = CONSUL_SEP.join((pi, getattr(record, pi))) consul_index, data = self.connection.kv.get(key_prefix, keys=True, recurse=True) if data is not None: regex = key_prefix + CONSUL_SEP.join(('*', key_suffix)) match_key = fnmatch.filter(data, regex) if match_key: self.connection.kv.delete(match_key[0]) # Store secondary index in Consul key = key_prefix + CONSUL_SEP.join((getattr(record, si), key_suffix)) rvalue = self.connection.kv.put(key, value) if rvalue is None: LOG.error("Unable to store secondary index in Consul") raise RuntimeError
def _delete_m2m_references(self, relation_name, record): m2m_ref_relations = self._get_reference_info(relation_name, 'many_to_many') if not m2m_ref_relations: return False primary_resource_pi = self._get_relation_index(m2m_ref_relations[0], 'primary') primary_resource_pi_value = getattr(record, '_'.join((m2m_ref_relations[0], primary_resource_pi))) secondary_resource_pi = self._get_relation_index(m2m_ref_relations[1], 'primary') secondary_resource_pi_value = getattr(record, '_'.join((m2m_ref_relations[1], secondary_resource_pi))) key = consul_key_join(relation_name, primary_resource_pi, primary_resource_pi_value, secondary_resource_pi, secondary_resource_pi_value) self.connection.kv.delete(key) index, data = self.connection.kv.get(key) if data is not None: LOG.error("Unable to delete record in Consul") raise RuntimeError key = consul_key_join(relation_name, secondary_resource_pi, secondary_resource_pi_value, primary_resource_pi, primary_resource_pi_value) self.connection.kv.delete(key) index, data = self.connection.kv.get(key) if data is not None: LOG.error("Unable to delete record in Consul") raise RuntimeError return True
def create_session(self): """Create a Consul session for critical section operations Returns: session_id (int): id of the Consul session """ lock_key = consul_key_join('CONSUL_LOCK') lock_delay = 0 # in seconds session_id = self.connection.session.create(lock_delay=lock_delay) while not self.connection.kv.put(lock_key, '', acquire=session_id): pass return session_id
def get_records_by_secondary_index(self, relation_name, secondary_index, field_value): """Retrieve a list of record for a secondary index from a relation Args: relation_name (unicode): Name of the relation/table secondary_index (unicode): Required secondary index field_value (unicode): Secondary index value Returns: Empty list ([]) OR list of records with the given secondary index value in the relation. Raises: TypeError: If passed arguments are not of 'string' type """ if (not isinstance(relation_name, six.string_types) or not isinstance(secondary_index, six.string_types) or not isinstance(field_value, six.string_types)): raise TypeError # Prepare the key prefix with secondary index in the required # format # e.g. ikepolicies/name/ike_1 key = consul_key_join(relation_name, secondary_index, field_value) # Find the primary index value for the given secondary index # value consul_index, data = self.connection.kv.get(key, recurse=True) if data is None: return [] primary_index_records = (record['Value'] for record in data if record['Key'].startswith(key + CONSUL_SEP)) if primary_index_records is None: return [] records = [] # Fetch the record with the primary index value for primary_index in primary_index_records: consul_index, data = self.connection.kv.get(primary_index) # Prepare a list of all the Consul records' 'Value' field records.append(self._prepare_record(relation_name, data['Value'])) return records
def _store_record_in_consul(self, relation_name, record): """Store record in the Consul with primary index(pi) as the 'key' and the record in JSON format as the 'value'. Args: relation_name (unicode): Name of the relation/table record (Any relation record object): Relation/Table record Raises: RuntimeError: Fail to store data in Consul """ session_id = self.create_session() try: if self._prepare_m2m_references(relation_name, record): return # Find the primary index of the relation pi = self._get_relation_index(relation_name, 'primary') # Prepare the key in the required format # e.g. ikepolicies/UUID/fc5221be-b9d0-11e5-8338-005056b46cff key = consul_key_join(relation_name, pi, getattr(record, pi)) # Convert the object into JSON(dict) value = json.dumps(record.__dict__, cls=CustomEncoder) # Store secondary indexes in Consul self._prepare_secondary_indices(relation_name, record) # Store KV pair in Consul rvalue = self.connection.kv.put(key, value) finally: self.destroy_session(session_id) if rvalue is None: LOG.error("Unable to store record in Consul") raise RuntimeError
def _get_m2m_relation_records(self, relation_name, reference_key_prefix): """Fetch Many to Many records for a relation Args: relation_name (unicode): Name of the relation/table reference_key_prefix (unicode): Returns: dict: Many to Many records of the relation """ reference_relations = self._get_reference_info(relation_name, 'many') m2m_relation_records = {} for reference in reference_relations: m2m_relations = self._get_reference_info(reference, 'many_to_many') assert len(m2m_relations) == 2 primary_relation = self._get_relation_name(relation_name) assert primary_relation in m2m_relations if primary_relation == m2m_relations[0]: secondary_relation = m2m_relations[1] else: secondary_relation = m2m_relations[0] secondary_resource_pi = self._get_relation_index(secondary_relation, 'primary') m2m_relation_name = relation_name relation_name_parts = relation_name.split(CONSUL_SEP) if len(relation_name_parts) > 1: relation_name_parts[-1] = reference m2m_relation_name = CONSUL_SEP.join(relation_name_parts) m2m_key_prefix = consul_key_join(m2m_relation_name, reference_key_prefix, secondary_resource_pi) + CONSUL_SEP consul_index, data = self.connection.kv.get(m2m_key_prefix, recurse=True) if data is None: continue # Prepare a list of all the Consul record 'Value' field records = [record['Value'] for record in data] reference_records = [] for record in records: consul_index, data = self.connection.kv.get(record) if data is None: LOG.error("Unable to get Value for the Key in Consul") continue reference_records.append(str_to_dict(data['Value'])) m2m_relation_records[secondary_relation] = reference_records return m2m_relation_records