def _lock_topic_and_schemas(topic):
if not topic:
return
session.query(
models.Topic).filter(models.Topic.id == topic.id).with_for_update()
session.query(models.AvroSchema).filter(
models.AvroSchema.topic_id == topic.id).with_for_update()
def test_filter_by_created_timestamp(
self,
dw_data_target,
biz_topic,
foo_topic,
dw_consumer_group_namespace_data_src,
dw_consumer_group_source_data_src
):
# set the creation timestamp of foo_topic 10 seconds behind biz_topic
new_created_at = biz_topic.created_at + datetime.timedelta(seconds=10)
session.query(models.Topic).filter(
models.Topic.id == foo_topic.id
).update(
{models.Topic.created_at: new_created_at}
)
actual = reg_repo.get_topics_by_data_target_id(
dw_data_target.id,
created_after=new_created_at + datetime.timedelta(seconds=-1)
)
asserts.assert_equal_entity_list(
actual_list=actual,
expected_list=[foo_topic],
assert_func=asserts.assert_equal_topic
)
def __call__(self, *args, **kwargs):
if can_watch_config:
self.get_watcher().reload_if_changed()
with session.connect_begin(ro=True):
session.query(models.Namespace).order_by(
models.Namespace.id).limit(1).all()
def __call__(self, *args, **kwargs):
if can_watch_config:
self.get_watcher().reload_if_changed()
with session.connect_begin(ro=True):
session.query(models.Namespace).order_by(
models.Namespace.id
).limit(1).all()
def _update_schema_status(schema_id, status):
session.query(
models.AvroSchema
).filter(
models.AvroSchema.id == schema_id
).update(
{'status': status}
)
session.flush()
def _lock_topic_and_schemas(topic):
if not topic:
return
session.query(
models.Topic
).filter(
models.Topic.id == topic.id
).with_for_update()
session.query(
models.AvroSchema
).filter(
models.AvroSchema.topic_id == topic.id
).with_for_update()
def test_delete_mapping(self, meta_attr_schema):
self._setup_meta_attribute_mapping(meta_attr_schema, self.entity.id)
actual = meta_attr_logic.delete_meta_attribute_mapping_for_entity(
self.entity_model, self.entity.id, meta_attr_schema.id)
expected = meta_attr_model(entity_type=self.entity_model.__name__,
entity_id=self.entity.id,
meta_attr_schema_id=meta_attr_schema.id)
self.assert_equal_meta_attr_partial(expected, actual)
with pytest.raises(orm_exc.NoResultFound):
session.query(meta_attr_model).filter(
meta_attr_model.entity_type == self.entity_model.__name__,
meta_attr_model.entity_id == self.entity.id,
meta_attr_model.meta_attr_schema_id ==
meta_attr_schema.id).one()
def get_or_create_namespace(namespace_name):
namespace = session.query(
models.Namespace
).filter(
models.Namespace.name == namespace_name
).first()
return namespace or create_namespace(namespace_name)
def get_by_id(cls, obj_id):
result = session.query(cls).filter(cls.id == obj_id).one_or_none()
if result is None:
raise EntityNotFoundError(
entity_desc='{} id {}'.format(cls.__name__, obj_id)
)
return result
def get_by_name(cls, name):
try:
return session.query(Namespace).filter(cls.name == name).one()
except orm_exc.NoResultFound:
raise EntityNotFoundError(
entity_desc='{} name `{}`'.format(cls.__name__, name)
)
def get_data_sources_by_data_target_id(data_target_id):
"""Get all the data sources that associate to the given data target.
Args:
data_target_id (int): Id of the data target.
Returns:
List[:class:schematizer.models.consumer_group_data_source
.ConsumerGroupDataSource]: List of data sources associated to the
given data target.
Raises:
:class:schematizer.models.exceptions.EntityNotFoundError: if specified
data target id is not found.
"""
data_srcs = session.query(models.ConsumerGroupDataSource).join(
models.ConsumerGroup
).filter(
models.ConsumerGroup.data_target_id == data_target_id,
models.ConsumerGroup.id == (models.ConsumerGroupDataSource
.consumer_group_id)
).all()
if not data_srcs:
verify_entity_exists(session, models.DataTarget, data_target_id)
return data_srcs
def get_notes_by_schemas_and_elements(schemas, elements):
if not (schemas or elements):
return []
# Can't use a join since sqlalchemy doesn't like
# the composite foreign key of Note
schema_ids = [schema.id for schema in schemas]
element_ids = [element.id for element in elements]
if schema_ids and element_ids:
note_filter = or_(
and_(models.Note.reference_type == models.ReferenceTypeEnum.SCHEMA,
models.Note.reference_id.in_(schema_ids)),
and_(
models.Note.reference_type ==
models.ReferenceTypeEnum.SCHEMA_ELEMENT,
models.Note.reference_id.in_(element_ids)))
elif schema_ids:
note_filter = and_(
models.Note.reference_type == models.ReferenceTypeEnum.SCHEMA,
models.Note.reference_id.in_(schema_ids))
else:
note_filter = and_(
models.Note.reference_type ==
models.ReferenceTypeEnum.SCHEMA_ELEMENT,
models.Note.reference_id.in_(element_ids))
return session.query(models.Note).filter(note_filter).order_by(
models.Note.id).all()
def get_note_by_reference_id_and_type(reference_id, reference_type):
return session.query(
models.Note
).filter(
models.Note.reference_type == reference_type,
models.Note.reference_id == reference_id
).first()
def _get_source_by_namespace_id_and_src_name(namespace_id, source):
return session.query(
models.Source
).filter(
models.Source.namespace_id == namespace_id,
models.Source.name == source
).first()
def get_schemas_created_after(created_after,
page_info=None,
include_disabled=False):
# TODO [clin|DATAPIPE-1430] as part of the clean up, merge this function
# into `get_schemas_by_criteira`.
""" Get the Avro schemas (excluding disabled schemas) created after the
specified created_after timestamp and with id greater than or equal to
the min_id. Limits the returned schemas to count. Default it excludes
disabled schemas.
Args:
created_after(datetime): get schemas created after given utc
datetime (inclusive).
page_info(Optional[:class:schematizer.models.tuples.PageInfo]):
limits the schemas to count and those with an id greater than or
equal to min_id.
include_disabled(Optional[bool]): set it to True to include disabled
schemas. Default it excludes disabled ones.
Returns:
(list[:class:schematizer.models.AvroSchema]): List of avro
schemas created after (inclusive) the specified creation
date.
"""
qry = session.query(models.AvroSchema).filter(
models.AvroSchema.created_at >= created_after, )
if not include_disabled:
qry = qry.filter(
models.AvroSchema.status != models.AvroSchemaStatus.DISABLED)
if page_info and page_info.min_id:
qry = qry.filter(models.AvroSchema.id >= page_info.min_id)
qry = qry.order_by(models.AvroSchema.id)
if page_info and page_info.count:
qry = qry.limit(page_info.count)
return qry.all()
def _get_topic_candidates(source_id,
base_schema_id,
contains_pii,
cluster_type,
limit=None,
enabled_schemas_only=True):
""" Get topic candidate(s) for the given args, in order of creation (newest
first).
:param int source_id: The source_id of the topic(s)
:param int|None base_schema_id: The base_schema_id of the schema(s) in the
topic(s). Note that this may be None, as is the case for any schemas
not derived from other schemas.
:param bool contains_pii: Limit to topics which either do or do not
contain PII. Defaults to None, which will not apply any filter.
:param string cluster_type : Limit to topics of same cluster type.
:param int|None limit: Provide a limit to the number of topics returned.
:param bool enabled_schemas_only: Set to True to limit results to schemas
which have not been disabled
:rtype: [schematizer.models.Topic]
"""
query = session.query(models.Topic).join(models.AvroSchema).filter(
models.Topic.source_id == source_id,
models.Topic._contains_pii == int(contains_pii),
models.Topic.cluster_type == cluster_type,
models.AvroSchema.base_schema_id == base_schema_id)
if enabled_schemas_only:
query = query.filter(
models.AvroSchema.status != models.AvroSchemaStatus.DISABLED)
query = query.order_by(models.Topic.id.desc())
if limit:
query = query.limit(limit)
return query.all()
def delete_meta_attribute_mapping_for_entity(entity_model, entity_id,
meta_attr_schema_id):
verify_entity_exists(session, entity_model, entity_id)
verify_entity_exists(session, AvroSchema, meta_attr_schema_id)
mapping_to_delete = MetaAttributeMappingStore.get_by_mapping(
entity_type=entity_model.__name__,
entity_id=entity_id,
meta_attr_schema_id=meta_attr_schema_id)
session.query(MetaAttributeMappingStore).filter(
MetaAttributeMappingStore.entity_type == entity_model.__name__,
MetaAttributeMappingStore.entity_id == entity_id,
MetaAttributeMappingStore.meta_attr_schema_id ==
meta_attr_schema_id).delete()
return mapping_to_delete
def update_refresh(refresh_id, status, offset):
return session.query(
models.Refresh).filter(models.Refresh.id == refresh_id).update({
models.Refresh.status:
status,
models.Refresh.offset:
offset
})
def get_sources(self, page_info=None):
qry = session.query(Source).filter(Source.namespace_id == self.id)
if page_info and page_info.min_id:
qry = qry.filter(Source.id >= page_info.min_id)
qry = qry.order_by(Source.id)
if page_info and page_info.count:
qry = qry.limit(page_info.count)
return qry.all()
def update_source_category(source_id, category):
return session.query(
models.SourceCategory
).filter(
models.SourceCategory.source_id == source_id
).update(
{models.SourceCategory.category: category}
)
def get_meta_attributes_by_schema_id(schema_id):
"""Logic Method to list the schema_ids of all meta attributes registered to
the specified schema id. Invalid schema id will raise an
EntityNotFoundError exception"""
models.AvroSchema.get_by_id(schema_id)
mappings = session.query(SchemaMetaAttributeMapping).filter(
SchemaMetaAttributeMapping.schema_id == schema_id).all()
return [m.meta_attr_schema_id for m in mappings]
def get_latest_schema_by_topic_id(topic_id):
"""Get the latest enabled (Read-Write or Read-Only) schema of given topic.
It returns None if no such schema can be found.
"""
return session.query(models.AvroSchema).filter(
models.AvroSchema.topic_id == topic_id,
models.AvroSchema.status != models.AvroSchemaStatus.DISABLED).order_by(
models.AvroSchema.id.desc()).first()
def list_refreshes_by_source_id(source_id):
return session.query(
models.Refresh
).filter(
models.Refresh.source_id == source_id
).order_by(
models.Refresh.id
).all()
def _get_source_or_create(namespace_id, source_name, owner_email):
try:
return session.query(models.Source).filter(
models.Source.namespace_id == namespace_id,
models.Source.name == source_name).one()
except orm_exc.NoResultFound:
return _create_source_if_not_exist(namespace_id, source_name,
owner_email)
def note(self):
note = session.query(
Note
).filter(
Note.reference_type == ReferenceTypeEnum.SCHEMA_ELEMENT,
Note.reference_id == self.id,
).first()
return note
def test_filter_by_created_timestamp(self, dw_data_target, biz_topic,
foo_topic,
dw_consumer_group_namespace_data_src,
dw_consumer_group_source_data_src):
# set the creation timestamp of foo_topic 10 seconds behind biz_topic
new_created_at = biz_topic.created_at + datetime.timedelta(seconds=10)
session.query(
models.Topic).filter(models.Topic.id == foo_topic.id).update(
{models.Topic.created_at: new_created_at})
actual = reg_repo.get_topics_by_data_target_id(
dw_data_target.id,
created_after=new_created_at + datetime.timedelta(seconds=-1))
asserts.assert_equal_entity_list(
actual_list=actual,
expected_list=[foo_topic],
assert_func=asserts.assert_equal_topic)
def get_schema_elements_by_schema_id(schema_id):
return session.query(
models.AvroSchemaElement
).filter(
models.AvroSchemaElement.avro_schema_id == schema_id
).order_by(
models.AvroSchemaElement.id
).all()
def get_topics_by_source_id(source_id):
return session.query(
models.Topic
).filter(
models.Topic.source_id == source_id
).order_by(
models.Topic.id
).all()
def get_latest_topic_of_source_id(source_id):
return session.query(
models.Topic
).filter(
models.Topic.source_id == source_id
).order_by(
models.Topic.id.desc()
).first()
def get_schema_by_id(schema_id):
"""Get the Avro schema of specified id. It returns None if not found.
"""
return session.query(
models.AvroSchema
).filter(
models.AvroSchema.id == schema_id
).first()
def get_source_by_fullname(namespace_name, source_name):
return session.query(
models.Source
).join(
models.Namespace
).filter(
models.Namespace.name == namespace_name,
models.Source.name == source_name
).first()
def get_topic_by_name(topic_name):
"""Get topic of specified topic name. It returns None if the specified
topic is not found.
"""
return session.query(
models.Topic
).filter(
models.Topic.name == topic_name
).first()
def delete_topics(cls, source_id):
topics = session.query(
models.Topic
).filter(
models.Topic.source_id == source_id
).all()
for topic in topics:
session.delete(topic)
session.flush()
def get_all(cls, pagination=None):
qry = session.query(cls).order_by(cls.id)
# include `id` as part of `where` clause to avoid table scan
# regardless whether the min_id is specified or not.
min_id = pagination.min_id if pagination else 0
qry = qry.filter(cls.id >= min_id)
if pagination and pagination.count > 0:
qry = qry.limit(pagination.count)
return qry.all()
def update_note(id, note_text, last_updated_by):
return session.query(models.Note).filter(models.Note.id == id).update({
models.Note.note:
note_text,
models.Note.last_updated_by:
last_updated_by,
models.Note.updated_at:
datetime.datetime.utcnow()
})
def get_schemas_by_criteria(
namespace_name=None,
source_name=None,
created_after=None,
include_disabled=False,
page_info=None
):
"""Get avro schemas that match the specified criteria, including namespace,
source, schema created timestamp, and/or schema status.
This function supports pagination, i.e. caller can specify minimum schema
id and page size to get single chunk of schemas.
Args:
namespace(Optional[str]): get schemas of given namespace if specified
source(Optional[str]): get schemas of given source name if specified
created_after(Optional[datetime]): get schemas created after given utc
datetime (inclusive) if specified
included_disabled(Optional[bool]): whether to include disabled schemas
page_info(Optional[:class:schematizer.models.page_info.PageInfo]):
limits the topics to count and those with id greater than or
equal to min_id.
Returns:
(list[:class:schematizer.models.AvroSchema]): List of avro schemas
sorted by their ids.
"""
qry = session.query(
models.AvroSchema
).join(
models.Topic,
models.Source,
models.Namespace
).filter(
models.AvroSchema.topic_id == models.Topic.id,
models.Topic.source_id == models.Source.id,
models.Source.namespace_id == models.Namespace.id,
models.Namespace.name == namespace_name
)
if source_name:
qry = qry.filter(models.Source.name == source_name)
if created_after is not None:
qry = qry.filter(models.AvroSchema.created_at >= created_after)
if not include_disabled:
qry = qry.filter(
models.AvroSchema.status != models.AvroSchemaStatus.DISABLED
)
min_id = page_info.min_id if page_info else 0
qry = qry.filter(models.AvroSchema.id >= min_id)
qry = qry.order_by(models.AvroSchema.id)
if page_info and page_info.count:
qry = qry.limit(page_info.count)
return qry.all()
def _get_namespace_or_create(namespace_name):
try:
return session.query(
models.Namespace
).filter(
models.Namespace.name == namespace_name
).one()
except orm_exc.NoResultFound:
return _create_namespace_if_not_exist(namespace_name)
def get_topics_by_data_target_id(data_target_id, created_after=None):
"""Get all the topics that associate to the given data target, and
optionally filtered by topic creation timestamp.
A data target may be associated to multiple consumer groups, and each
consumer group may have multiple data sources (which could be a namespace
or a source). This function returns all the topics under all the data
sources that link to the given data target.
Args:
data_target_id (int): data target id
created_after(Optional[datetime]): get topics created after given utc
datetime (inclusive) if specified.
Returns:
List[:class:schematizer.models.topic.Topic]: List of topic models
sorted by their ids.
"""
data_srcs = get_data_sources_by_data_target_id(data_target_id)
source_ids = {
data_src.data_source_id for data_src in data_srcs
if data_src.data_source_type == models.DataSourceTypeEnum.SOURCE
}
namespace_ids = {
data_src.data_source_id for data_src in data_srcs
if data_src.data_source_type == models.DataSourceTypeEnum.NAMESPACE
}
if namespace_ids:
sources = session.query(models.Source).filter(
models.Source.namespace_id.in_(namespace_ids)
).all()
source_ids.update(source.id for source in sources)
if not source_ids:
return []
qry = session.query(models.Topic).join(models.Source).filter(
models.Source.id.in_(source_ids),
models.Source.id == models.Topic.source_id
)
if created_after:
qry = qry.filter(models.Topic.created_at >= created_after)
return qry.order_by(models.Topic.id).all()
def get_schemas_by_topic_id(topic_id, include_disabled=False):
"""Get all the Avro schemas of specified topic. Default it excludes
disabled schemas. Set `include_disabled` to True to include disabled ones.
"""
qry = session.query(
models.AvroSchema).filter(models.AvroSchema.topic_id == topic_id)
if not include_disabled:
qry = qry.filter(
models.AvroSchema.status != models.AvroSchemaStatus.DISABLED)
return qry.order_by(models.AvroSchema.id).all()
def get_meta_attributes_by_schema_id(schema_id):
"""Logic Method to list the schema_ids of all meta attributes registered to
the specified schema id. Invalid schema id will raise an
EntityNotFoundError exception"""
models.AvroSchema.get_by_id(schema_id)
mappings = session.query(
SchemaMetaAttributeMapping
).filter(
SchemaMetaAttributeMapping.schema_id == schema_id
).all()
return [m.meta_attr_schema_id for m in mappings]
def update_refresh(refresh_id, status, offset):
return session.query(
models.Refresh
).filter(
models.Refresh.id == refresh_id
).update(
{
models.Refresh.status: status,
models.Refresh.offset: offset
}
)
def get_source_categories_by_criteria(namespace_name, source_name=None):
"""Get source_categories by namespace_name, optionally filtering
by source_name.
"""
qry = session.query(models.SourceCategory).join(
models.Source, models.Namespace).filter(
models.SourceCategory.source_id == models.Source.id,
models.Source.namespace_id == models.Namespace.id,
models.Namespace.name == namespace_name)
if source_name:
qry = qry.filter(models.Source.name == source_name)
return qry.order_by(models.SourceCategory.id).all()
def get_by_mapping(cls, entity_type, entity_id, meta_attr_schema_id):
result = session.query(MetaAttributeMappingStore).filter(
cls.entity_type == entity_type, cls.entity_id == entity_id,
cls.meta_attr_schema_id == meta_attr_schema_id).one_or_none()
if result is None:
err_mapping = {
entity_type + '_id': entity_id,
'meta_attribute_schema_id': meta_attr_schema_id
}
raise EntityNotFoundError(entity_desc='{} mapping `{}`'.format(
cls.__name__, err_mapping))
return result
def get_schemas_by_topic_name(topic_name, include_disabled=False):
topic = get_topic_by_name(topic_name)
if not topic:
raise sch_exc.EntityNotFoundException(
'Cannot find topic {0}.'.format(topic_name))
qry = session.query(
models.AvroSchema).filter(models.AvroSchema.topic_id == topic.id)
if not include_disabled:
qry = qry.filter(
models.AvroSchema.status != models.AvroSchemaStatus.DISABLED)
return qry.order_by(models.AvroSchema.id).all()
def get_sources(self, page_info=None):
qry = session.query(
Source
).filter(Source.namespace_id == self.id)
if page_info and page_info.min_id:
qry = qry.filter(
Source.id >= page_info.min_id
)
qry = qry.order_by(Source.id)
if page_info and page_info.count:
qry = qry.limit(page_info.count)
return qry.all()
def get_latest_schema_by_topic_id(topic_id):
"""Get the latest enabled (Read-Write or Read-Only) schema of given topic.
It returns None if no such schema can be found.
"""
return session.query(
models.AvroSchema
).filter(
models.AvroSchema.topic_id == topic_id,
models.AvroSchema.status != models.AvroSchemaStatus.DISABLED
).order_by(
models.AvroSchema.id.desc()
).first()
def test_delete_mapping(self, meta_attr_schema):
self._setup_meta_attribute_mapping(meta_attr_schema, self.entity.id)
actual = meta_attr_logic.delete_meta_attribute_mapping_for_entity(
self.entity_model,
self.entity.id,
meta_attr_schema.id
)
expected = meta_attr_model(
entity_type=self.entity_model.__name__,
entity_id=self.entity.id,
meta_attr_schema_id=meta_attr_schema.id
)
self.assert_equal_meta_attr_partial(expected, actual)
with pytest.raises(orm_exc.NoResultFound):
session.query(
meta_attr_model
).filter(
meta_attr_model.entity_type == self.entity_model.__name__,
meta_attr_model.entity_id == self.entity.id,
meta_attr_model.meta_attr_schema_id == meta_attr_schema.id
).one()
def update_note(id, note_text, last_updated_by):
return session.query(
models.Note
).filter(
models.Note.id == id
).update(
{
models.Note.note: note_text,
models.Note.last_updated_by: last_updated_by,
models.Note.updated_at: datetime.datetime.utcnow()
}
)
def get_latest_schema_by_topic_name(topic_name):
"""Get the latest enabled (Read-Write or Read-Only) schema of given topic.
It returns None if no such schema can be found.
"""
topic = get_topic_by_name(topic_name)
if not topic:
raise sch_exc.EntityNotFoundException(
"Cannot find topic {0}.".format(topic_name))
return session.query(models.AvroSchema).filter(
models.AvroSchema.topic_id == topic.id,
models.AvroSchema.status != models.AvroSchemaStatus.DISABLED).order_by(
models.AvroSchema.id.desc()).first()
def get_latest_topic_of_namespace_source(namespace_name, source_name):
source = get_source_by_fullname(namespace_name, source_name)
if not source:
raise sch_exc.EntityNotFoundException(
"Cannot find namespace {0} source {1}.".format(
namespace_name, source_name))
return session.query(models.Topic).join(
models.Source, models.Namespace).filter(
models.Namespace.id == models.Source.namespace_id,
models.Source.id == models.Topic.source_id,
models.Namespace.name == namespace_name,
models.Source.name == source_name).order_by(
models.Topic.id.desc()).first()
def get_data_targets_by_schema_id(schema_id):
"""Get the data targets of the corresponding schema id.
Since the the data source in ConsumerGroupDataSource can be a
schema, namespace or source, this function first uses the schema_id to
grab corresponding namespace, source id and use them to find
the corresponding consumer group ids. It finally uses these consumer group
ids to fetch the data targets.
Returns:
A list of unique data targets
"""
avro_schema = models.AvroSchema.get_by_id(schema_id)
src_id = avro_schema.topic.source.id
namespace_id = avro_schema.topic.source.namespace_id
results = session.query(
models.ConsumerGroupDataSource.consumer_group_id
).filter(
or_(
_filter_consumer_group_data_src_by_namespace(namespace_id),
_filter_consumer_group_data_src_by_source(src_id),
_filter_consumer_group_data_src_by_schema(schema_id)
)
).all()
consumer_group_ids = set([id[0] for id in results])
data_targets = session.query(
models.DataTarget
).join(
models.ConsumerGroup
).filter(
models.DataTarget.id == models.ConsumerGroup.data_target_id,
models.ConsumerGroup.id.in_(consumer_group_ids)
).all()
return data_targets
def get_schemas_by_criteria(namespace_name=None,
source_name=None,
created_after=None,
include_disabled=False,
page_info=None):
"""Get avro schemas that match the specified criteria, including namespace,
source, schema created timestamp, and/or schema status.
This function supports pagination, i.e. caller can specify minimum schema
id and page size to get single chunk of schemas.
Args:
namespace(Optional[str]): get schemas of given namespace if specified
source(Optional[str]): get schemas of given source name if specified
created_after(Optional[datetime]): get schemas created after given utc
datetime (inclusive) if specified
included_disabled(Optional[bool]): whether to include disabled schemas
page_info(Optional[:class:schematizer.models.page_info.PageInfo]):
limits the topics to count and those with id greater than or
equal to min_id.
Returns:
(list[:class:schematizer.models.AvroSchema]): List of avro schemas
sorted by their ids.
"""
qry = session.query(models.AvroSchema).join(
models.Topic, models.Source, models.Namespace).filter(
models.AvroSchema.topic_id == models.Topic.id,
models.Topic.source_id == models.Source.id,
models.Source.namespace_id == models.Namespace.id,
models.Namespace.name == namespace_name)
if source_name:
qry = qry.filter(models.Source.name == source_name)
if created_after is not None:
qry = qry.filter(models.AvroSchema.created_at >= created_after)
if not include_disabled:
qry = qry.filter(
models.AvroSchema.status != models.AvroSchemaStatus.DISABLED)
min_id = page_info.min_id if page_info else 0
qry = qry.filter(models.AvroSchema.id >= min_id)
qry = qry.order_by(models.AvroSchema.id)
if page_info and page_info.count:
qry = qry.limit(page_info.count)
return qry.all()
def get_topics_by_criteria(namespace=None,
source=None,
created_after=None,
page_info=None):
"""Get all the topics that match given criteria, including namespace,
source, and/or topic created timestamp.
This function supports pagination, i.e. caller can specify miniumum topic
id and page size to get single chunk of topics.
Args:
namespace(Optional[str]): get topics of given namespace if specified
source(Optional[str]): get topics of given source name if specified
created_after(Optional[datetime]): get topics created after given utc
datetime (inclusive) if specified.
page_info(Optional[:class:schematizer.models.page_info.PageInfo]):
limits the topics to count and those with id greater than or
equal to min_id.
Returns:
(list[:class:schematizer.models.Topic]): List of topics sorted by
their ids.
"""
qry = session.query(models.Topic)
if namespace or source:
qry = qry.join(
models.Source).filter(models.Source.id == models.Topic.source_id)
if namespace:
qry = qry.join(models.Namespace).filter(
models.Namespace.name == namespace,
models.Namespace.id == models.Source.namespace_id,
)
if source:
qry = qry.filter(models.Source.name == source)
if created_after is not None:
qry = qry.filter(models.Topic.created_at >= created_after)
min_id = page_info.min_id if page_info else 0
qry = qry.filter(models.Topic.id >= min_id)
qry = qry.order_by(models.Topic.id)
if page_info and page_info.count:
qry = qry.limit(page_info.count)
return qry.all()
def delete_all_children(children, dry_run=False):
for model_id_pair in children:
model = model_id_pair.model
model_name = model.__name__
ids = model_id_pair.ids
if ids:
new_query = session.query(model).filter(model.id.in_(ids))
if dry_run:
all_items = new_query.all()
print "Objects for {} ({} found): {}".format(
model_name, len(all_items), all_items)
else:
print "Deleting {} items of type {}".format(
len(ids), model_name)
new_query.delete(synchronize_session=False)
else:
print "No items found for {}. Not deleting anything".format(
model_name)
def get_consumer_groups_by_data_target_id(data_target_id):
"""Get the list of consumer groups that associate to the given data target.
Args:
data_target_id (int): Id of the data target
Returns:
List[:class: schematizer.models.consumer_group.ConsumerGroup]: List of
consumer group objects.
Raises:
:class:schematizer.models.exceptions.EntityNotFoundError: if specified
data target id is not found.
"""
groups = session.query(models.ConsumerGroup).filter(
models.ConsumerGroup.data_target_id == data_target_id
).all()
if not groups:
verify_entity_exists(session, models.DataTarget, data_target_id)
return groups