def create_avro_schema(
schema_json,
schema_elements=None,
topic_name=None,
namespace=None,
source=None,
status=models.AvroSchemaStatus.READ_AND_WRITE,
base_schema_id=None,
created_at=None
):
topic = get_or_create_topic(
topic_name,
namespace_name=namespace,
source_name=source
)
avro_schema = models.AvroSchema.create(
session,
avro_schema_json=schema_json,
topic_id=topic.id,
status=status,
base_schema_id=base_schema_id,
created_at=created_at
)
schema_elements = (
schema_elements or
AvroSchema.create_schema_elements_from_json(schema_json)
)
for schema_element in schema_elements:
schema_element.avro_schema_id = avro_schema.id
session.add(schema_element)
session.flush()
return avro_schema
def _create_avro_schema(
avro_schema_json,
source_id,
topic_id,
status=models.AvroSchemaStatus.READ_AND_WRITE,
base_schema_id=None
):
avro_schema_elements = models.AvroSchema.create_schema_elements_from_json(
avro_schema_json
)
avro_schema = models.AvroSchema(
avro_schema_json=avro_schema_json,
topic_id=topic_id,
status=status,
base_schema_id=base_schema_id
)
session.add(avro_schema)
session.flush()
for avro_schema_element in avro_schema_elements:
avro_schema_element.avro_schema_id = avro_schema.id
session.add(avro_schema_element)
session.flush()
_add_meta_attribute_mappings(avro_schema.id, source_id)
return avro_schema
def create_note(reference_type, reference_id, note_text, last_updated_by):
note = models.Note(reference_type=reference_type,
reference_id=reference_id,
note=note_text,
last_updated_by=last_updated_by)
session.add(note)
session.flush()
return note
def create_source_category(source_id, category):
source_category = models.SourceCategory(
source_id=source_id,
category=category
)
session.add(source_category)
session.flush()
return source_category
def _add_meta_attribute_mappings(schema_id, source_id):
mappings = []
for meta_attr_schema_id in meta_attr_logic.get_meta_attributes_by_source(
source_id):
new_mapping = SchemaMetaAttributeMapping(
schema_id=schema_id, meta_attr_schema_id=meta_attr_schema_id)
session.add(new_mapping)
mappings.append(new_mapping)
session.flush()
return mappings
def create_note(reference_type, reference_id, note_text, last_updated_by):
note = models.Note(
reference_type=reference_type,
reference_id=reference_id,
note=note_text,
last_updated_by=last_updated_by
)
session.add(note)
session.flush()
return note
def create_refresh(source_id, offset, batch_size, priority, filter_condition,
avg_rows_per_second_cap):
refresh = models.Refresh(source_id=source_id,
offset=offset,
batch_size=batch_size,
priority=priority,
filter_condition=filter_condition,
avg_rows_per_second_cap=avg_rows_per_second_cap)
session.add(refresh)
session.flush()
return refresh
def _create_topic(topic_name, source_id, contains_pii, cluster_type):
"""Create a topic named `topic_name` in the given source.
It returns a newly created topic. If a topic with the same
name already exists, an exception is thrown
"""
topic = models.Topic(name=topic_name,
source_id=source_id,
contains_pii=contains_pii,
cluster_type=cluster_type)
session.add(topic)
session.flush()
return topic
def _add_meta_attribute_mappings(schema_id, source_id):
mappings = []
for meta_attr_schema_id in meta_attr_logic.get_meta_attributes_by_source(
source_id
):
new_mapping = SchemaMetaAttributeMapping(
schema_id=schema_id,
meta_attr_schema_id=meta_attr_schema_id
)
session.add(new_mapping)
mappings.append(new_mapping)
session.flush()
return mappings
def _create_topic(topic_name, source_id, contains_pii, cluster_type):
"""Create a topic named `topic_name` in the given source.
It returns a newly created topic. If a topic with the same
name already exists, an exception is thrown
"""
topic = models.Topic(
name=topic_name,
source_id=source_id,
contains_pii=contains_pii,
cluster_type=cluster_type
)
session.add(topic)
session.flush()
return topic
def _create_namespace_if_not_exist(namespace_name):
try:
# Create a savepoint before trying to create new namespace so that
# in the case which the IntegrityError occurs, the session will
# rollback to savepoint. Upon exiting the nested Context, commit/
# rollback is automatically issued and no need to add it explicitly
with session.begin_nested():
new_namespace = models.Namespace(name=namespace_name)
session.add(new_namespace)
except (IntegrityError, exc.IntegrityError):
# Ignore this error due to trying to create a duplicate namespace
# TODO [clin|DATAPIPE-1471] see if there is a way to only handle one
# exception or the other.
new_namespace = get_namespace_by_name(namespace_name)
return new_namespace
def _create_namespace_if_not_exist(namespace_name):
try:
# Create a savepoint before trying to create new namespace so that
# in the case which the IntegrityError occurs, the session will
# rollback to savepoint. Upon exiting the nested Context, commit/
# rollback is automatically issued and no need to add it explicitly
with session.begin_nested():
new_namespace = models.Namespace(name=namespace_name)
session.add(new_namespace)
except (IntegrityError, exc.IntegrityError):
# Ignore this error due to trying to create a duplicate namespace
# TODO [clin|DATAPIPE-1471] see if there is a way to only handle one
# exception or the other.
new_namespace = get_namespace_by_name(namespace_name)
return new_namespace
def register_meta_attribute_for_entity(entity_model, entity_id,
meta_attr_schema_id):
verify_entity_exists(session, entity_model, entity_id)
verify_entity_exists(session, AvroSchema, meta_attr_schema_id)
try:
with session.begin_nested():
new_mapping = MetaAttributeMappingStore(
entity_type=entity_model.__name__,
entity_id=entity_id,
meta_attr_schema_id=meta_attr_schema_id)
session.add(new_mapping)
except (IntegrityError, exc.IntegrityError):
# Ignore this error due to trying to create a duplicate mapping
new_mapping = MetaAttributeMappingStore.get_by_mapping(
entity_type=entity_model.__name__,
entity_id=entity_id,
meta_attr_schema_id=meta_attr_schema_id)
return new_mapping
def create(cls, session, **kwargs):
"""Create this entity in the database. Note this function will call
`session.flush()`, so do not use this function if there are other
operations that need to happen before the flush is called.
Args:
session (:class:yelp_conn.session.YelpConnScopedSession) global
session manager used to provide sessions.
kwargs (dict): pairs of model attributes and their values.
Returns:
:class:schematizer.models.[cls]: object that is newly created in
the database.
"""
entity = cls(**kwargs)
session.add(entity)
session.flush()
return entity
def create_refresh(
source_id,
offset,
batch_size,
priority,
filter_condition,
avg_rows_per_second_cap
):
refresh = models.Refresh(
source_id=source_id,
offset=offset,
batch_size=batch_size,
priority=priority,
filter_condition=filter_condition,
avg_rows_per_second_cap=avg_rows_per_second_cap
)
session.add(refresh)
session.flush()
return refresh
def _create_avro_schema(avro_schema_json,
source_id,
topic_id,
status=models.AvroSchemaStatus.READ_AND_WRITE,
base_schema_id=None):
avro_schema_elements = models.AvroSchema.create_schema_elements_from_json(
avro_schema_json)
avro_schema = models.AvroSchema(avro_schema_json=avro_schema_json,
topic_id=topic_id,
status=status,
base_schema_id=base_schema_id)
session.add(avro_schema)
session.flush()
for avro_schema_element in avro_schema_elements:
avro_schema_element.avro_schema_id = avro_schema.id
session.add(avro_schema_element)
session.flush()
_add_meta_attribute_mappings(avro_schema.id, source_id)
return avro_schema
def register_meta_attribute_for_entity(
entity_model,
entity_id,
meta_attr_schema_id
):
verify_entity_exists(session, entity_model, entity_id)
verify_entity_exists(session, AvroSchema, meta_attr_schema_id)
try:
with session.begin_nested():
new_mapping = MetaAttributeMappingStore(
entity_type=entity_model.__name__,
entity_id=entity_id,
meta_attr_schema_id=meta_attr_schema_id
)
session.add(new_mapping)
except (IntegrityError, exc.IntegrityError):
# Ignore this error due to trying to create a duplicate mapping
new_mapping = MetaAttributeMappingStore.get_by_mapping(
entity_type=entity_model.__name__,
entity_id=entity_id,
meta_attr_schema_id=meta_attr_schema_id
)
return new_mapping
def create_source_category(source_id, category):
source_category = models.SourceCategory(source_id=source_id,
category=category)
session.add(source_category)
session.flush()
return source_category
def create_in_db(cls, name, namespace):
source = cls.create(name, namespace)
session.add(source)
session.flush()
return source
