def find_by_topic_id(self, topic_id: TopicId) -> List[Pipeline]:
pipeline_ids = CacheService.pipelines_by_topic().get(topic_id)
if pipeline_ids is not None:
pipelines = ArrayHelper(pipeline_ids) \
.map(lambda x: self.find_by_id(x)) \
.filter(lambda x: x is not None).to_list()
if len(pipelines) != len(pipeline_ids):
loaded = ArrayHelper(pipelines).map(
lambda x: x.pipelineId).to_list()
raise Exception(
f'Except pipelines[{pipeline_ids}], but get[{loaded}] only.'
)
return pipelines
storage_service = PipelineStorageService(ask_meta_storage(),
ask_snowflake_generator(),
self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
pipelines: List[Pipeline] = storage_service.find_by_topic_id(
topic_id, self.principalService.get_tenant_id())
if len(pipelines) == 0:
CacheService.pipelines_by_topic().declare_no_pipelines(
topic_id)
return pipelines
return ArrayHelper(pipelines).each(
lambda x: CacheService.pipeline().put(x)).to_list()
finally:
storage_service.close_transaction()
def find_by_id(self, pipeline_id: PipelineId) -> Optional[Pipeline]:
pipeline = CacheService.pipeline().get(pipeline_id)
if pipeline is not None:
if pipeline.tenantId != self.principalService.get_tenant_id():
raise DataKernelException(
f'Pipeline[id={pipeline_id}] not belongs to '
f'current tenant[id={self.principalService.get_tenant_id()}].'
)
return pipeline
storage_service = PipelineStorageService(ask_meta_storage(),
ask_snowflake_generator(),
self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
pipeline: Pipeline = storage_service.find_by_id(pipeline_id)
if pipeline is None:
return None
CacheService.pipeline().put(pipeline)
return pipeline
finally:
storage_service.close_transaction()
def build_pipeline_cache(pipeline: Pipeline) -> None:
CacheService.pipeline().put(pipeline)
def post_delete_pipeline(pipeline_id: PipelineId,
pipeline_service: PipelineService) -> None:
remove_pipeline_index(pipeline_id, pipeline_service)
CacheService.pipeline().remove(pipeline_id)
def post_update_pipeline_enablement(pipeline: Pipeline,
pipeline_service: PipelineService) -> None:
get_pipeline_index_service(
pipeline_service).update_index_on_enablement_changed(pipeline)
CacheService.pipeline().put(pipeline)
def __init__(self):
# noinspection PyTypeChecker
self.compiledByIdCache = InternalCache(
cache=get_compiled_pipeline_by_id_cache)
CacheService.pipeline().add_cache_listener(self)
async def clear_all_pipelines_cache(
principal_service: PrincipalService = Depends(
get_super_admin_principal)) -> None:
CacheService.pipeline().clear()