def action() -> DataSource:
# noinspection PyTypeChecker
data_source: DataSource = data_source_service.delete(data_source_id)
if data_source is None:
raise_404()
CacheService.data_source().remove(data_source_id)
return data_source
def action() -> Tenant:
# noinspection PyTypeChecker
tenant: Tenant = tenant_service.delete(tenant_id)
if tenant is None:
raise_404()
CacheService.tenant().remove(tenant_id)
return tenant
def ask_topic_storage(
topic_or_schema: Union[Topic, TopicSchema],
principal_service: PrincipalService) -> TopicDataStorageSPI:
topic = topic_or_schema if isinstance(
topic_or_schema, Topic) else topic_or_schema.get_topic()
data_source_id = topic.dataSourceId
if is_blank(data_source_id):
raise DataKernelException(
f'Data source is not defined for topic[id={topic.topicId}, name={topic.name}]'
)
build = CacheService.data_source().get_builder(data_source_id)
if build is not None:
return build()
data_source = get_data_source_service(principal_service).find_by_id(
data_source_id)
if data_source is None:
raise DataKernelException(
f'Data source not declared for topic'
f'[id={topic.topicId}, name={topic.name}, dataSourceId={data_source_id}]'
)
build = build_topic_data_storage(data_source)
CacheService.data_source().put_builder(data_source_id, build)
return build()
def prepare_topic():
data_source = DataSource(dataSourceId='1',
dataSourceCode='ds1',
dataSourceType=DataSourceType.MYSQL,
host='localhost',
port='3306',
username='******',
password='******',
name='watchmen',
tenantId='1')
CacheService.data_source().put(data_source)
return Topic(topicId='1',
name='topic_x',
type=TopicType.DISTINCT,
kind=TopicKind.BUSINESS,
factors=[
Factor(factorId='1',
name='topic1_id',
type=FactorType.SEQUENCE,
indexGroup='u-1'),
Factor(factorId='2',
name='topic1_text',
type=FactorType.TEXT,
precision='64',
indexGroup='u-1')
],
dataSourceId=data_source.dataSourceId,
tenantId='1')
def find_schema_by_id(self, topic_id: TopicId, tenant_id: TenantId) -> Optional[TopicSchema]:
if not self.principalService.is_super_admin():
if self.principalService.get_tenant_id() != tenant_id:
raise Exception('Forbidden')
schema = CacheService.topic().get_schema(topic_id)
if schema is not None:
if schema.get_topic().tenantId != tenant_id:
return None
return schema
storage_service = TopicStorageService(ask_meta_storage(), ask_snowflake_generator(), self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
topic: Topic = storage_service.find_by_id(topic_id)
if topic is None:
return None
CacheService.topic().put(topic)
schema = CacheService.topic().get_schema(topic.topicId)
if schema is not None:
if schema.get_topic().tenantId != tenant_id:
return None
return schema
finally:
storage_service.close_transaction()
def find_by_topic_id(self, topic_id: TopicId) -> List[Pipeline]:
pipeline_ids = CacheService.pipelines_by_topic().get(topic_id)
if pipeline_ids is not None:
pipelines = ArrayHelper(pipeline_ids) \
.map(lambda x: self.find_by_id(x)) \
.filter(lambda x: x is not None).to_list()
if len(pipelines) != len(pipeline_ids):
loaded = ArrayHelper(pipelines).map(
lambda x: x.pipelineId).to_list()
raise Exception(
f'Except pipelines[{pipeline_ids}], but get[{loaded}] only.'
)
return pipelines
storage_service = PipelineStorageService(ask_meta_storage(),
ask_snowflake_generator(),
self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
pipelines: List[Pipeline] = storage_service.find_by_topic_id(
topic_id, self.principalService.get_tenant_id())
if len(pipelines) == 0:
CacheService.pipelines_by_topic().declare_no_pipelines(
topic_id)
return pipelines
return ArrayHelper(pipelines).each(
lambda x: CacheService.pipeline().put(x)).to_list()
finally:
storage_service.close_transaction()
def prepare_data(self):
data_source = DataSource(dataSourceId='1',
dataSourceCode='test',
dataSourceType=DataSourceType.MYSQL,
host='localhost',
port='3306',
username='******',
password='******',
name='watchmen',
tenantId='1')
data_source_service = get_data_source_service(
create_fake_principal_service())
data_source_service.begin_transaction()
data_source_service.create(data_source)
data_source_service.commit_transaction()
CacheService.data_source().put(data_source)
topic1 = Topic(topicId='1',
name='topic1',
type=TopicType.DISTINCT,
kind=TopicKind.BUSINESS,
factors=[
Factor(factorId='1',
name='topic1_id',
type=FactorType.SEQUENCE),
Factor(factorId='2',
name='topic1_enabled',
type=FactorType.BOOLEAN)
],
dataSourceId=data_source.dataSourceId,
tenantId='1')
topic_service = get_topic_service(create_fake_principal_service())
topic_service.begin_transaction()
topic_service.create(topic1)
topic_service.commit_transaction()
CacheService.topic().put(topic1)
space = Space(spaceId='1',
name='Space1',
topicIds=[topic1.topicId],
tenantId='1')
space_service = get_space_service(create_fake_principal_service())
space_service.begin_transaction()
space_service.create(space)
space_service.commit_transaction()
connected_space = ConnectedSpace(connectId='1',
name='ConnectedSpace1',
spaceId=space.spaceId,
isTemplate=False,
userId='1',
tenantId='1')
connected_space_service = get_connected_space_service(
create_fake_principal_service())
connected_space_service.begin_transaction()
connected_space_service.create(connected_space)
connected_space_service.commit_transaction()
def action(a_data_source: DataSource) -> DataSource:
if data_source_service.is_storable_id_faked(
a_data_source.dataSourceId):
data_source_service.redress_storable_id(a_data_source)
# noinspection PyTypeChecker
a_data_source: DataSource = data_source_service.create(
a_data_source)
else:
# noinspection PyTypeChecker
a_data_source: DataSource = data_source_service.update(
a_data_source)
CacheService.data_source().put(a_data_source)
return a_data_source
def ask_topic_data_entity_helper(schema: TopicSchema) -> TopicDataEntityHelper:
"""
ask topic data entity helper, from cache first
"""
data_entity_helper = CacheService.topic().get_entity_helper(
schema.get_topic().topicId)
if data_entity_helper is None:
if is_raw_topic(schema.get_topic()):
data_entity_helper = RawTopicDataEntityHelper(schema)
else:
data_entity_helper = RegularTopicDataEntityHelper(schema)
CacheService.topic().put_entity_helper(data_entity_helper)
return data_entity_helper
def find_by_id(self, tenant_id: TenantId) -> Optional[Tenant]:
tenant = CacheService.tenant().get(tenant_id)
if tenant is not None:
return tenant
storage_service = TenantStorageService(ask_meta_storage(),
ask_snowflake_generator(),
self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
tenant: Tenant = storage_service.find_by_id(tenant_id)
if tenant is None:
return None
CacheService.tenant().put(tenant)
return tenant
finally:
storage_service.close_transaction()
def find_by_type(self, key_type: str,
tenant_id: TenantId) -> Optional[KeyStore]:
hit, key_store = CacheService.key_store().get(key_type, tenant_id)
if hit:
return key_store
storage_service = KeyStoreStorageService(ask_meta_storage())
storage_service.begin_transaction()
try:
key_store: KeyStore = storage_service.find_by_type(
key_type, tenant_id)
if key_store is None:
CacheService.key_store().declare_not_existing(
key_type, tenant_id)
return None
CacheService.key_store().put(key_store)
return key_store
finally:
storage_service.close_transaction()
def find_by_id(self, topic_id: TopicId) -> Optional[Topic]:
topic = CacheService.topic().get(topic_id)
if topic is not None:
if topic.tenantId != self.principalService.get_tenant_id():
raise DataKernelException(
f'Topic[id={topic_id}] not belongs to current tenant[id={self.principalService.get_tenant_id()}].')
return topic
storage_service = TopicStorageService(ask_meta_storage(), ask_snowflake_generator(), self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
topic: Topic = storage_service.find_by_id(topic_id)
if topic is None:
return None
CacheService.topic().put(topic)
return topic
finally:
storage_service.close_transaction()
def action(a_tenant: Tenant) -> Tenant:
if tenant_service.is_storable_id_faked(a_tenant.tenantId):
tenant_service.redress_storable_id(a_tenant)
# noinspection PyTypeChecker
a_tenant: Tenant = tenant_service.create(a_tenant)
if ask_create_pipeline_monitor_topics_on_tenant_create():
topics = ask_pipeline_monitor_topics()
create_topics_and_pipelines(
topics, lambda source_topics:
ask_pipeline_monitor_pipelines(source_topics),
a_tenant.tenantId, tenant_service, principal_service)
if ask_create_dqc_topics_on_tenant_create():
topics = ask_dqc_topics()
create_topics_and_pipelines(
topics,
lambda source_topics: ask_dqc_pipelines(source_topics),
a_tenant.tenantId, tenant_service, principal_service)
else:
# noinspection PyTypeChecker
a_tenant: Tenant = tenant_service.update(a_tenant)
CacheService.tenant().put(a_tenant)
return a_tenant
def find_by_id(self, data_source_id: DataSourceId) -> Optional[DataSource]:
data_source = CacheService.data_source().get(data_source_id)
if data_source is not None:
if data_source.tenantId != self.principalService.get_tenant_id():
raise DataKernelException(
f'Data source[id={data_source_id}] not belongs to '
f'current tenant[id={self.principalService.get_tenant_id()}].')
return data_source
storage_service = DataSourceStorageService(
ask_meta_storage(), ask_snowflake_generator(), self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
data_source: DataSource = storage_service.find_by_id(data_source_id)
if data_source is None:
return None
CacheService.data_source().put(data_source)
return data_source
finally:
storage_service.close_transaction()
def find_by_id(self, writer_id: ExternalWriterId) -> Optional[ExternalWriter]:
external_writer = CacheService.external_writer().get(writer_id)
if external_writer is not None:
if external_writer.tenantId != self.principalService.get_tenant_id():
raise DataKernelException(
f'External writer[id={writer_id}] not belongs to '
f'current tenant[id={self.principalService.get_tenant_id()}].')
register_external_writer(external_writer)
return external_writer
storage_service = ExternalWriterStorageService(
ask_meta_storage(), ask_snowflake_generator(), self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
external_writer: ExternalWriter = storage_service.find_by_id(writer_id)
if external_writer is None:
return None
CacheService.external_writer().put(external_writer)
register_external_writer(external_writer)
return external_writer
finally:
storage_service.close_transaction()
def find_by_id(self, pipeline_id: PipelineId) -> Optional[Pipeline]:
pipeline = CacheService.pipeline().get(pipeline_id)
if pipeline is not None:
if pipeline.tenantId != self.principalService.get_tenant_id():
raise DataKernelException(
f'Pipeline[id={pipeline_id}] not belongs to '
f'current tenant[id={self.principalService.get_tenant_id()}].'
)
return pipeline
storage_service = PipelineStorageService(ask_meta_storage(),
ask_snowflake_generator(),
self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
pipeline: Pipeline = storage_service.find_by_id(pipeline_id)
if pipeline is None:
return None
CacheService.pipeline().put(pipeline)
return pipeline
finally:
storage_service.close_transaction()
def register_topic(self, topic: Topic) -> None:
data_source_id = topic.dataSourceId
if is_blank(data_source_id):
raise InquiryTrinoException(
f'Data source is not defined for topic[id={topic.topicId}, name={topic.name}]'
)
data_source = CacheService.data_source().get(data_source_id)
if data_source is None:
data_source = get_data_source_service(
self.principalService).find_by_id(data_source_id)
if data_source is None:
raise InquiryTrinoException(
f'Data source not declared for topic'
f'[id={topic.topicId}, name={topic.name}, dataSourceId={data_source_id}]'
)
self.schemas.register(
TrinoSchema(catalog=data_source.dataSourceCode,
schema=data_source.name,
topic=topic))
def build_pipeline_cache(pipeline: Pipeline) -> None:
CacheService.pipeline().put(pipeline)
def post_delete_pipeline(pipeline_id: PipelineId,
pipeline_service: PipelineService) -> None:
remove_pipeline_index(pipeline_id, pipeline_service)
CacheService.pipeline().remove(pipeline_id)
def post_update_pipeline_enablement(pipeline: Pipeline,
pipeline_service: PipelineService) -> None:
get_pipeline_index_service(
pipeline_service).update_index_on_enablement_changed(pipeline)
CacheService.pipeline().put(pipeline)
def __init__(self):
# noinspection PyTypeChecker
self.compiledByIdCache = InternalCache(
cache=get_compiled_pipeline_by_id_cache)
CacheService.pipeline().add_cache_listener(self)
async def clear_all_tenants_cache(
principal_service: PrincipalService = Depends(
get_super_admin_principal)) -> None:
CacheService.key_store().clear()
async def clear_all_external_writers_cache(
principal_service: PrincipalService = Depends(
get_super_admin_principal)) -> None:
CacheService.external_writer().clear()
async def clear_all_data_sources_cache(
principal_service: PrincipalService = Depends(
get_super_admin_principal)) -> None:
CacheService.data_source().clear()
async def clear_all_pipelines_cache(
principal_service: PrincipalService = Depends(
get_super_admin_principal)) -> None:
CacheService.pipeline().clear()
def post_delete_topic(topic_id: TopicId, topic_service: TopicService) -> None:
remove_topic_index(topic_id, topic_service)
CacheService.topic().remove(topic_id)
def post_save_topic(topic: Topic, topic_service: TopicService) -> None:
build_topic_index(topic, topic_service)
CacheService.topic().put(topic)
async def clear_all_topic_cache(principal_service: PrincipalService = Depends(
get_super_admin_principal)) -> None:
CacheService.topic().clear()