def get_data_sources_by_data_target_id(data_target_id):
"""Get all the data sources that associate to the given data target.
Args:
data_target_id (int): Id of the data target.
Returns:
List[:class:schematizer.models.consumer_group_data_source
.ConsumerGroupDataSource]: List of data sources associated to the
given data target.
Raises:
:class:schematizer.models.exceptions.EntityNotFoundError: if specified
data target id is not found.
"""
data_srcs = session.query(models.ConsumerGroupDataSource).join(
models.ConsumerGroup
).filter(
models.ConsumerGroup.data_target_id == data_target_id,
models.ConsumerGroup.id == (models.ConsumerGroupDataSource
.consumer_group_id)
).all()
if not data_srcs:
verify_entity_exists(session, models.DataTarget, data_target_id)
return data_srcs
def create_consumer_group(group_name, data_target_id):
"""Add a new consumer group that associates to given data target.
Args:
group_name (string): consumer group name. It should be unique among all
the consumer groups.
data_target_id (id): id of the data target which this consumer group
associates to
Returns:
:class:models.consumer_group.ConsumerGroup: the created consumer group.
Raises:
ValueError: if group name is empty or already exists.
:class:schematizer.models.exceptions.EntityNotFoundError: if specified
data target id is not found.
"""
verify_truthy_value(group_name, "consumer group name")
verify_entity_exists(session, models.DataTarget, data_target_id)
try:
return models.ConsumerGroup.create(
session,
group_name=group_name,
data_target_id=data_target_id
)
except (IntegrityError, exc.IntegrityError):
# TODO [clin|DATAPIPE-1471] see if there is a way to only handle one
# exception or the other.
raise ValueError(
"Consumer group {} already exists.".format(group_name)
)
def register_consumer_group_data_source(
consumer_group_id,
data_source_type,
data_source_id
):
"""Assign the specified data source to the given consumer group.
Args:
consumer_group_id (int): :class:models.consumer_group.ConsumerGroup
object Id.
data_src_type (
:class:models.consumer_group_data_source.DataSourceTypeEnum) data
source type
data_src_id (int): Id of the specified data source type
Returns:
:class:models.consumer_group_data_source.ConsumerGroupDataSource:
the newly created object.
Raises:
ValueError: if given data source type is invalid
:class:schematizer.models.exceptions.EntityNotFoundError: if specified
data source id is not found or consumer group id is not found.
"""
verify_entity_exists(session, models.ConsumerGroup, consumer_group_id)
_verify_data_source_exists(data_source_type, data_source_id)
return models.ConsumerGroupDataSource.create(
session,
consumer_group_id=consumer_group_id,
data_source_type=data_source_type,
data_source_id=data_source_id
)
def _verify_data_source_exists(data_src_type, data_src_id):
"""Check if the data source object of specified data source type and id
exists.
Args:
data_src_type (
:class:models.consumer_group_data_source.DataSourceTypeEnum) data
source type
data_src_id (int): Id of the specified data source type
Returns:
bool: returns True if the object exists.
Raises:
ValueError: if given data source type is invalid
:class:schematizer.models.exceptions.EntityNotFoundError: if specified
data source id is not found.
"""
data_src_type_to_cls_map = {
models.DataSourceTypeEnum.NAMESPACE: models.Namespace,
models.DataSourceTypeEnum.SOURCE: models.Source
}
data_src_cls = data_src_type_to_cls_map.get(data_src_type)
if not data_src_cls:
raise ValueError(
"Invalid data source type {}. It should be one of {}."
.format(data_src_type, models.DataSourceTypeEnum.__name__)
)
verify_entity_exists(session, data_src_cls, data_src_id)
def delete_meta_attribute_mapping_for_entity(entity_model, entity_id,
meta_attr_schema_id):
verify_entity_exists(session, entity_model, entity_id)
verify_entity_exists(session, AvroSchema, meta_attr_schema_id)
mapping_to_delete = MetaAttributeMappingStore.get_by_mapping(
entity_type=entity_model.__name__,
entity_id=entity_id,
meta_attr_schema_id=meta_attr_schema_id)
session.query(MetaAttributeMappingStore).filter(
MetaAttributeMappingStore.entity_type == entity_model.__name__,
MetaAttributeMappingStore.entity_id == entity_id,
MetaAttributeMappingStore.meta_attr_schema_id ==
meta_attr_schema_id).delete()
return mapping_to_delete
def register_meta_attribute_for_entity(entity_model, entity_id,
meta_attr_schema_id):
verify_entity_exists(session, entity_model, entity_id)
verify_entity_exists(session, AvroSchema, meta_attr_schema_id)
try:
with session.begin_nested():
new_mapping = MetaAttributeMappingStore(
entity_type=entity_model.__name__,
entity_id=entity_id,
meta_attr_schema_id=meta_attr_schema_id)
session.add(new_mapping)
except (IntegrityError, exc.IntegrityError):
# Ignore this error due to trying to create a duplicate mapping
new_mapping = MetaAttributeMappingStore.get_by_mapping(
entity_type=entity_model.__name__,
entity_id=entity_id,
meta_attr_schema_id=meta_attr_schema_id)
return new_mapping
def get_consumer_groups_by_data_target_id(data_target_id):
"""Get the list of consumer groups that associate to the given data target.
Args:
data_target_id (int): Id of the data target
Returns:
List[:class: schematizer.models.consumer_group.ConsumerGroup]: List of
consumer group objects.
Raises:
:class:schematizer.models.exceptions.EntityNotFoundError: if specified
data target id is not found.
"""
groups = session.query(models.ConsumerGroup).filter(
models.ConsumerGroup.data_target_id == data_target_id
).all()
if not groups:
verify_entity_exists(session, models.DataTarget, data_target_id)
return groups
def delete_meta_attribute_mapping_for_entity(
entity_model,
entity_id,
meta_attr_schema_id
):
verify_entity_exists(session, entity_model, entity_id)
verify_entity_exists(session, AvroSchema, meta_attr_schema_id)
mapping_to_delete = MetaAttributeMappingStore.get_by_mapping(
entity_type=entity_model.__name__,
entity_id=entity_id,
meta_attr_schema_id=meta_attr_schema_id
)
session.query(
MetaAttributeMappingStore
).filter(
MetaAttributeMappingStore.entity_type == entity_model.__name__,
MetaAttributeMappingStore.entity_id == entity_id,
MetaAttributeMappingStore.meta_attr_schema_id == meta_attr_schema_id
).delete()
return mapping_to_delete
def register_meta_attribute_for_entity(
entity_model,
entity_id,
meta_attr_schema_id
):
verify_entity_exists(session, entity_model, entity_id)
verify_entity_exists(session, AvroSchema, meta_attr_schema_id)
try:
with session.begin_nested():
new_mapping = MetaAttributeMappingStore(
entity_type=entity_model.__name__,
entity_id=entity_id,
meta_attr_schema_id=meta_attr_schema_id
)
session.add(new_mapping)
except (IntegrityError, exc.IntegrityError):
# Ignore this error due to trying to create a duplicate mapping
new_mapping = MetaAttributeMappingStore.get_by_mapping(
entity_type=entity_model.__name__,
entity_id=entity_id,
meta_attr_schema_id=meta_attr_schema_id
)
return new_mapping