def ask_topic_storage(
topic_or_schema: Union[Topic, TopicSchema],
principal_service: PrincipalService) -> TopicDataStorageSPI:
topic = topic_or_schema if isinstance(
topic_or_schema, Topic) else topic_or_schema.get_topic()
data_source_id = topic.dataSourceId
if is_blank(data_source_id):
raise DataKernelException(
f'Data source is not defined for topic[id={topic.topicId}, name={topic.name}]'
)
build = CacheService.data_source().get_builder(data_source_id)
if build is not None:
return build()
data_source = get_data_source_service(principal_service).find_by_id(
data_source_id)
if data_source is None:
raise DataKernelException(
f'Data source not declared for topic'
f'[id={topic.topicId}, name={topic.name}, dataSourceId={data_source_id}]'
)
build = build_topic_data_storage(data_source)
CacheService.data_source().put_builder(data_source_id, build)
return build()
def action() -> DataSource:
# noinspection PyTypeChecker
data_source: DataSource = data_source_service.delete(data_source_id)
if data_source is None:
raise_404()
CacheService.data_source().remove(data_source_id)
return data_source
def prepare_topic():
data_source = DataSource(dataSourceId='1',
dataSourceCode='ds1',
dataSourceType=DataSourceType.MYSQL,
host='localhost',
port='3306',
username='******',
password='******',
name='watchmen',
tenantId='1')
CacheService.data_source().put(data_source)
return Topic(topicId='1',
name='topic_x',
type=TopicType.DISTINCT,
kind=TopicKind.BUSINESS,
factors=[
Factor(factorId='1',
name='topic1_id',
type=FactorType.SEQUENCE,
indexGroup='u-1'),
Factor(factorId='2',
name='topic1_text',
type=FactorType.TEXT,
precision='64',
indexGroup='u-1')
],
dataSourceId=data_source.dataSourceId,
tenantId='1')
def prepare_data(self):
data_source = DataSource(dataSourceId='1',
dataSourceCode='test',
dataSourceType=DataSourceType.MYSQL,
host='localhost',
port='3306',
username='******',
password='******',
name='watchmen',
tenantId='1')
data_source_service = get_data_source_service(
create_fake_principal_service())
data_source_service.begin_transaction()
data_source_service.create(data_source)
data_source_service.commit_transaction()
CacheService.data_source().put(data_source)
topic1 = Topic(topicId='1',
name='topic1',
type=TopicType.DISTINCT,
kind=TopicKind.BUSINESS,
factors=[
Factor(factorId='1',
name='topic1_id',
type=FactorType.SEQUENCE),
Factor(factorId='2',
name='topic1_enabled',
type=FactorType.BOOLEAN)
],
dataSourceId=data_source.dataSourceId,
tenantId='1')
topic_service = get_topic_service(create_fake_principal_service())
topic_service.begin_transaction()
topic_service.create(topic1)
topic_service.commit_transaction()
CacheService.topic().put(topic1)
space = Space(spaceId='1',
name='Space1',
topicIds=[topic1.topicId],
tenantId='1')
space_service = get_space_service(create_fake_principal_service())
space_service.begin_transaction()
space_service.create(space)
space_service.commit_transaction()
connected_space = ConnectedSpace(connectId='1',
name='ConnectedSpace1',
spaceId=space.spaceId,
isTemplate=False,
userId='1',
tenantId='1')
connected_space_service = get_connected_space_service(
create_fake_principal_service())
connected_space_service.begin_transaction()
connected_space_service.create(connected_space)
connected_space_service.commit_transaction()
def action(a_data_source: DataSource) -> DataSource:
if data_source_service.is_storable_id_faked(
a_data_source.dataSourceId):
data_source_service.redress_storable_id(a_data_source)
# noinspection PyTypeChecker
a_data_source: DataSource = data_source_service.create(
a_data_source)
else:
# noinspection PyTypeChecker
a_data_source: DataSource = data_source_service.update(
a_data_source)
CacheService.data_source().put(a_data_source)
return a_data_source
def find_by_id(self, data_source_id: DataSourceId) -> Optional[DataSource]:
data_source = CacheService.data_source().get(data_source_id)
if data_source is not None:
if data_source.tenantId != self.principalService.get_tenant_id():
raise DataKernelException(
f'Data source[id={data_source_id}] not belongs to '
f'current tenant[id={self.principalService.get_tenant_id()}].')
return data_source
storage_service = DataSourceStorageService(
ask_meta_storage(), ask_snowflake_generator(), self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
data_source: DataSource = storage_service.find_by_id(data_source_id)
if data_source is None:
return None
CacheService.data_source().put(data_source)
return data_source
finally:
storage_service.close_transaction()
def register_topic(self, topic: Topic) -> None:
data_source_id = topic.dataSourceId
if is_blank(data_source_id):
raise InquiryTrinoException(
f'Data source is not defined for topic[id={topic.topicId}, name={topic.name}]'
)
data_source = CacheService.data_source().get(data_source_id)
if data_source is None:
data_source = get_data_source_service(
self.principalService).find_by_id(data_source_id)
if data_source is None:
raise InquiryTrinoException(
f'Data source not declared for topic'
f'[id={topic.topicId}, name={topic.name}, dataSourceId={data_source_id}]'
)
self.schemas.register(
TrinoSchema(catalog=data_source.dataSourceCode,
schema=data_source.name,
topic=topic))
async def clear_all_data_sources_cache(
principal_service: PrincipalService = Depends(
get_super_admin_principal)) -> None:
CacheService.data_source().clear()
