def build_find_user_by_name(
clear_pwd: bool = True) -> Callable[[str], Optional[User]]:
"""
autonomous transaction inside
"""
return lambda username: find_user_by_name(ask_meta_storage(), username,
clear_pwd)
def find_by_topic_id(self, topic_id: TopicId) -> List[Pipeline]:
pipeline_ids = CacheService.pipelines_by_topic().get(topic_id)
if pipeline_ids is not None:
pipelines = ArrayHelper(pipeline_ids) \
.map(lambda x: self.find_by_id(x)) \
.filter(lambda x: x is not None).to_list()
if len(pipelines) != len(pipeline_ids):
loaded = ArrayHelper(pipelines).map(
lambda x: x.pipelineId).to_list()
raise Exception(
f'Except pipelines[{pipeline_ids}], but get[{loaded}] only.'
)
return pipelines
storage_service = PipelineStorageService(ask_meta_storage(),
ask_snowflake_generator(),
self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
pipelines: List[Pipeline] = storage_service.find_by_topic_id(
topic_id, self.principalService.get_tenant_id())
if len(pipelines) == 0:
CacheService.pipelines_by_topic().declare_no_pipelines(
topic_id)
return pipelines
return ArrayHelper(pipelines).each(
lambda x: CacheService.pipeline().put(x)).to_list()
finally:
storage_service.close_transaction()
def find_schema_by_id(self, topic_id: TopicId, tenant_id: TenantId) -> Optional[TopicSchema]:
if not self.principalService.is_super_admin():
if self.principalService.get_tenant_id() != tenant_id:
raise Exception('Forbidden')
schema = CacheService.topic().get_schema(topic_id)
if schema is not None:
if schema.get_topic().tenantId != tenant_id:
return None
return schema
storage_service = TopicStorageService(ask_meta_storage(), ask_snowflake_generator(), self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
topic: Topic = storage_service.find_by_id(topic_id)
if topic is None:
return None
CacheService.topic().put(topic)
schema = CacheService.topic().get_schema(topic.topicId)
if schema is not None:
if schema.get_topic().tenantId != tenant_id:
return None
return schema
finally:
storage_service.close_transaction()
def find_all(self) -> List[Tenant]:
storage_service = TenantStorageService(ask_meta_storage(),
ask_snowflake_generator(),
self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
return storage_service.find_all()
finally:
storage_service.close_transaction()
def find_should_monitored(self, tenant_id: TenantId) -> List[Topic]: storage_service = TopicStorageService(ask_meta_storage(), ask_snowflake_generator(), self.principalService) storage_service.begin_transaction() try: # noinspection PyTypeChecker topics = storage_service.find_all(tenant_id) # only business topics need to be monitored return ArrayHelper(topics).filter(lambda x: x.kind == TopicKind.BUSINESS).to_list() finally: storage_service.close_transaction()
def heart_beat_on_topics() -> None:
topics = CacheService.topic().all()
topic_service = TopicService(ask_meta_storage(), ask_snowflake_generator(),
ask_super_admin())
topic_service.begin_transaction()
try:
for topic in topics:
loaded: Optional[Topic] = topic_service.find_by_id(topic.topicId)
if loaded is None:
CacheService.topic().remove(topic.topicId)
elif loaded.lastModifiedAt > topic.lastModifiedAt or loaded.version > topic.version:
CacheService.topic().put(loaded)
finally:
topic_service.close_transaction()
def heart_beat_on_pipelines() -> None:
pipelines = CacheService.pipeline().all()
pipeline_service = PipelineService(ask_meta_storage(),
ask_snowflake_generator(),
ask_super_admin())
pipeline_service.begin_transaction()
try:
for pipeline in pipelines:
loaded: Optional[Pipeline] = pipeline_service.find_by_id(
pipeline.pipelineId)
if loaded is None:
CacheService.pipeline().remove(pipeline.pipelineId)
elif loaded.lastModifiedAt > pipeline.lastModifiedAt or loaded.version > pipeline.version:
CacheService.pipeline().put(loaded)
finally:
pipeline_service.close_transaction()
def heart_beat_on_data_sources() -> None:
data_sources = CacheService.data_source().all()
data_source_service = DataSourceService(ask_meta_storage(),
ask_snowflake_generator(),
ask_super_admin())
data_source_service.begin_transaction()
try:
for data_source in data_sources:
loaded: Optional[DataSource] = data_source_service.find_by_id(
data_source.dataSourceId)
if loaded is None:
CacheService.data_source().remove(data_source.dataSourceId)
elif loaded.lastModifiedAt > data_source.lastModifiedAt or loaded.version > data_source.version:
CacheService.data_source().put(loaded)
finally:
data_source_service.close_transaction()
def heart_beat_on_tenants() -> None:
tenants = CacheService.tenant().all()
tenant_service = TenantService(ask_meta_storage(),
ask_snowflake_generator(),
ask_super_admin())
tenant_service.begin_transaction()
try:
for tenant in tenants:
loaded: Optional[Tenant] = tenant_service.find_by_id(
tenant.tenantId)
if loaded is None:
CacheService.tenant().remove(tenant.tenantId)
elif loaded.lastModifiedAt > tenant.lastModifiedAt or loaded.version > tenant.version:
CacheService.tenant().put(loaded)
finally:
tenant_service.close_transaction()
async def exchange_user(principal_service: PrincipalService = Depends(get_any_principal)) -> Optional[User]: """ returns current principal """ user_id = principal_service.get_user_id() user_service = UserService(ask_meta_storage(), ask_snowflake_generator(), principal_service) user_service.begin_transaction() try: # noinspection PyTypeChecker user: User = user_service.find_by_id(user_id) if user is None: return None else: del user.password return user finally: user_service.close_transaction()
def heart_beat_on_external_writers() -> None:
external_writers = CacheService.external_writer().all()
external_writer_service = ExternalWriterService(ask_meta_storage(),
ask_snowflake_generator(),
ask_super_admin())
external_writer_service.begin_transaction()
try:
for external_writer in external_writers:
loaded: Optional[
ExternalWriter] = external_writer_service.find_by_id(
external_writer.writerId)
if loaded is None:
CacheService.external_writer().remove(external_writer.writerId)
elif loaded.lastModifiedAt > external_writer.lastModifiedAt or loaded.version > external_writer.version:
CacheService.external_writer().put(loaded)
finally:
external_writer_service.close_transaction()
def find_by_id(self, tenant_id: TenantId) -> Optional[Tenant]:
tenant = CacheService.tenant().get(tenant_id)
if tenant is not None:
return tenant
storage_service = TenantStorageService(ask_meta_storage(),
ask_snowflake_generator(),
self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
tenant: Tenant = storage_service.find_by_id(tenant_id)
if tenant is None:
return None
CacheService.tenant().put(tenant)
return tenant
finally:
storage_service.close_transaction()
def find_by_id(self, topic_id: TopicId) -> Optional[Topic]:
topic = CacheService.topic().get(topic_id)
if topic is not None:
if topic.tenantId != self.principalService.get_tenant_id():
raise DataKernelException(
f'Topic[id={topic_id}] not belongs to current tenant[id={self.principalService.get_tenant_id()}].')
return topic
storage_service = TopicStorageService(ask_meta_storage(), ask_snowflake_generator(), self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
topic: Topic = storage_service.find_by_id(topic_id)
if topic is None:
return None
CacheService.topic().put(topic)
return topic
finally:
storage_service.close_transaction()
def find_by_type(self, key_type: str,
tenant_id: TenantId) -> Optional[KeyStore]:
hit, key_store = CacheService.key_store().get(key_type, tenant_id)
if hit:
return key_store
storage_service = KeyStoreStorageService(ask_meta_storage())
storage_service.begin_transaction()
try:
key_store: KeyStore = storage_service.find_by_type(
key_type, tenant_id)
if key_store is None:
CacheService.key_store().declare_not_existing(
key_type, tenant_id)
return None
CacheService.key_store().put(key_store)
return key_store
finally:
storage_service.close_transaction()
def find_by_id(self, data_source_id: DataSourceId) -> Optional[DataSource]:
data_source = CacheService.data_source().get(data_source_id)
if data_source is not None:
if data_source.tenantId != self.principalService.get_tenant_id():
raise DataKernelException(
f'Data source[id={data_source_id}] not belongs to '
f'current tenant[id={self.principalService.get_tenant_id()}].')
return data_source
storage_service = DataSourceStorageService(
ask_meta_storage(), ask_snowflake_generator(), self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
data_source: DataSource = storage_service.find_by_id(data_source_id)
if data_source is None:
return None
CacheService.data_source().put(data_source)
return data_source
finally:
storage_service.close_transaction()
def find_by_id(self, pipeline_id: PipelineId) -> Optional[Pipeline]:
pipeline = CacheService.pipeline().get(pipeline_id)
if pipeline is not None:
if pipeline.tenantId != self.principalService.get_tenant_id():
raise DataKernelException(
f'Pipeline[id={pipeline_id}] not belongs to '
f'current tenant[id={self.principalService.get_tenant_id()}].'
)
return pipeline
storage_service = PipelineStorageService(ask_meta_storage(),
ask_snowflake_generator(),
self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
pipeline: Pipeline = storage_service.find_by_id(pipeline_id)
if pipeline is None:
return None
CacheService.pipeline().put(pipeline)
return pipeline
finally:
storage_service.close_transaction()
def find_by_id(self, writer_id: ExternalWriterId) -> Optional[ExternalWriter]:
external_writer = CacheService.external_writer().get(writer_id)
if external_writer is not None:
if external_writer.tenantId != self.principalService.get_tenant_id():
raise DataKernelException(
f'External writer[id={writer_id}] not belongs to '
f'current tenant[id={self.principalService.get_tenant_id()}].')
register_external_writer(external_writer)
return external_writer
storage_service = ExternalWriterStorageService(
ask_meta_storage(), ask_snowflake_generator(), self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
external_writer: ExternalWriter = storage_service.find_by_id(writer_id)
if external_writer is None:
return None
CacheService.external_writer().put(external_writer)
register_external_writer(external_writer)
return external_writer
finally:
storage_service.close_transaction()
def find_by_name(self, subject_name: str) -> Optional[Subject]:
storage_service = SubjectStorageService(ask_meta_storage(),
ask_snowflake_generator(),
self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
subject: Subject = storage_service.find_by_name(subject_name)
if subject is None:
return None
if subject.tenantId != self.principalService.get_tenant_id():
raise InquiryKernelException(
f'Subject[name={subject_name}] not belongs to '
f'current tenant[id={self.principalService.get_tenant_id()}].'
)
if not self.principalService.is_admin(
) and subject.userId != self.principalService.get_user_id():
raise InquiryKernelException(
f'Subject[name={subject_name}] not belongs to '
f'current user[id={self.principalService.get_user_id()}].')
return subject
finally:
storage_service.close_transaction()
def get_topic_service(principal_service: PrincipalService) -> TopicService:
return TopicService(ask_meta_storage(), ask_snowflake_generator(),
principal_service)
def get_inspection_service(principal_service: PrincipalService) -> InspectionService: return InspectionService(ask_meta_storage(), ask_snowflake_generator(), principal_service)
def get_pipeline_service(
principal_service: PrincipalService) -> PipelineService:
return PipelineService(ask_meta_storage(), ask_snowflake_generator(),
principal_service)
def get_topic_snapshot_scheduler_service(principal_service: PrincipalService) -> TopicSnapshotSchedulerService: return TopicSnapshotSchedulerService(ask_meta_storage(), ask_snowflake_generator(), principal_service)
def get_subject_service(principal_service: PrincipalService) -> SubjectService:
return SubjectService(ask_meta_storage(), ask_snowflake_generator(),
principal_service)
def get_connected_space_graphic_service(
principal_service: PrincipalService) -> ConnectedSpaceGraphicService:
return ConnectedSpaceGraphicService(ask_meta_storage(),
ask_snowflake_generator(),
principal_service)
def get_lock_service(
principal_service: PrincipalService) -> TopicSnapshotJobLockService:
return TopicSnapshotJobLockService(ask_meta_storage(),
ask_snowflake_generator(),
principal_service)
def get_rule_service(
principal_service: PrincipalService) -> MonitorRuleService:
return MonitorRuleService(ask_meta_storage(), ask_snowflake_generator(),
principal_service)
def get_lock_service(
principal_service: PrincipalService) -> MonitorJobLockService:
return MonitorJobLockService(ask_meta_storage(), ask_snowflake_generator(),
principal_service)
def get_meta_tenant_service(
principal_service: PrincipalService) -> MetaTenantService:
return MetaTenantService(ask_meta_storage(), ask_snowflake_generator(),
principal_service)
def get_external_writer_service(
principal_service: PrincipalService) -> ExternalWriterService:
return ExternalWriterService(ask_meta_storage(), ask_snowflake_generator(),
principal_service)
def get_user_service(principal_service: PrincipalService) -> UserService: return UserService(ask_meta_storage(), ask_snowflake_generator(), principal_service)