def find_schema_by_id(self, topic_id: TopicId, tenant_id: TenantId) -> Optional[TopicSchema]: if not self.principalService.is_super_admin(): if self.principalService.get_tenant_id() != tenant_id: raise Exception('Forbidden') schema = CacheService.topic().get_schema(topic_id) if schema is not None: if schema.get_topic().tenantId != tenant_id: return None return schema storage_service = TopicStorageService(ask_meta_storage(), ask_snowflake_generator(), self.principalService) storage_service.begin_transaction() try: # noinspection PyTypeChecker topic: Topic = storage_service.find_by_id(topic_id) if topic is None: return None CacheService.topic().put(topic) schema = CacheService.topic().get_schema(topic.topicId) if schema is not None: if schema.get_topic().tenantId != tenant_id: return None return schema finally: storage_service.close_transaction()
def handle_scheduler( scheduler: TopicSnapshotScheduler, source_topic: Topic, task_topic: Topic, topic_service: TopicService, scheduler_service: TopicSnapshotSchedulerService, principal_service: PrincipalService) -> Optional[Callable[[], None]]: target_topic_id = scheduler.targetTopicId if is_blank(target_topic_id): # incorrect scheduler, ignored return None should_save_scheduler = False target_topic: Optional[Topic] = topic_service.find_by_id(target_topic_id) if target_topic is None: # create target topic when not found target_topic = create_snapshot_target_topic(scheduler, source_topic) target_topic, target_topic_tail = ask_save_topic_action( topic_service, principal_service)(target_topic) scheduler.targetTopicId = target_topic.topicId should_save_scheduler = True else: # rebuild target topic target_topic = rebuild_snapshot_target_topic(target_topic, source_topic) target_topic, target_topic_tail = ask_save_topic_action( topic_service, principal_service)(target_topic) pipeline_service = get_pipeline_service(topic_service) pipeline_id = scheduler.pipelineId if is_blank(pipeline_id): # create pipeline not declared pipeline = create_snapshot_pipeline(task_topic, target_topic) pipeline = ask_save_pipeline_action(pipeline_service, principal_service)(pipeline) scheduler.pipelineId = pipeline.pipelineId should_save_scheduler = True else: pipeline: Optional[Pipeline] = pipeline_service.find_by_id(pipeline_id) if pipeline is None: # create pipeline when not found pipeline = create_snapshot_pipeline(task_topic, target_topic) pipeline = ask_save_pipeline_action(pipeline_service, principal_service)(pipeline) scheduler.pipelineId = pipeline.pipelineId should_save_scheduler = True else: # rebuild pipeline pipeline = rebuild_snapshot_pipeline(pipeline, task_topic, target_topic) ask_save_pipeline_action(pipeline_service, principal_service)(pipeline) if should_save_scheduler: scheduler_service.update(scheduler) return target_topic_tail
def validate_source_topic(scheduler: TopicSnapshotScheduler, topic_service: TopicService) -> Topic: source_topic: Optional[Topic] = topic_service.find_by_id(scheduler.topicId) if source_topic is None: raise_500(None, f'Topic[id={scheduler.topicId}] not found.') if source_topic.type == TopicType.RAW: raise_500(None, f'Topic[id={scheduler.topicId}] is raw topic.') if source_topic.kind == TopicKind.SYSTEM: raise_500(None, f'Topic[id={scheduler.topicId}] is system topic.') return source_topic
def heart_beat_on_topics() -> None: topics = CacheService.topic().all() topic_service = TopicService(ask_meta_storage(), ask_snowflake_generator(), ask_super_admin()) topic_service.begin_transaction() try: for topic in topics: loaded: Optional[Topic] = topic_service.find_by_id(topic.topicId) if loaded is None: CacheService.topic().remove(topic.topicId) elif loaded.lastModifiedAt > topic.lastModifiedAt or loaded.version > topic.version: CacheService.topic().put(loaded) finally: topic_service.close_transaction()
def try_to_import_topic(topic: Topic, topic_service: TopicService, do_update: bool) -> TopicImportDataResult: if is_blank(topic.topicId): topic_service.redress_storable_id(topic) topic_service.create(topic) else: existing_topic: Optional[Topic] = topic_service.find_by_id(topic.topicId) if existing_topic is None: topic_service.create(topic) elif do_update: topic.version = existing_topic.version topic.dataSourceId = existing_topic.dataSourceId topic_service.update(topic) else: return TopicImportDataResult( topicId=topic.topicId, name=topic.name, passed=False, reason='Topic already exists.') post_save_topic(topic, topic_service) return TopicImportDataResult(topicId=topic.topicId, name=topic.name, passed=True)
def find_by_id(self, topic_id: TopicId) -> Optional[Topic]: topic = CacheService.topic().get(topic_id) if topic is not None: if topic.tenantId != self.principalService.get_tenant_id(): raise DataKernelException( f'Topic[id={topic_id}] not belongs to current tenant[id={self.principalService.get_tenant_id()}].') return topic storage_service = TopicStorageService(ask_meta_storage(), ask_snowflake_generator(), self.principalService) storage_service.begin_transaction() try: # noinspection PyTypeChecker topic: Topic = storage_service.find_by_id(topic_id) if topic is None: return None CacheService.topic().put(topic) return topic finally: storage_service.close_transaction()
def validate_scheduler_on_update(scheduler: TopicSnapshotScheduler, topic_service: TopicService, principal_service: PrincipalService) -> None: if is_blank(scheduler.targetTopicId): raise_500(None, f'TargetTopicId is required on scheduler.') # noinspection DuplicatedCode target_topic: Optional[Topic] = topic_service.find_by_id( scheduler.targetTopicId) if target_topic is None: raise_500(None, f'Topic[id={scheduler.targetTopicId}] not found.') if target_topic.tenantId != principal_service.get_tenant_id(): raise_500(None, f'Topic[id={scheduler.targetTopicId}] not found.') if target_topic.type == TopicType.RAW: raise_500(None, f'Target topic[id={scheduler.targetTopicId}] is raw topic.') if target_topic.kind == TopicKind.SYSTEM: raise_500( None, f'Target topic[id={scheduler.targetTopicId}] is system topic.') if is_not_blank(scheduler.targetTopicName ) and target_topic.name != scheduler.targetTopicName: raise_500( None, f'Target topic[id={scheduler.targetTopicId}, name={target_topic.name}] ' f'has different name with given scheduler[targetTopicName={scheduler.targetTopicName}].' ) if is_blank(scheduler.pipelineId): raise_500(None, f'PipelineId is required on scheduler.') pipeline_service = get_pipeline_service(topic_service) pipeline: Optional[Pipeline] = pipeline_service.find_by_id( scheduler.pipelineId) if pipeline is None: raise_500(None, f'Pipeline[id={scheduler.pipelineId}] not found.') if pipeline.tenantId != principal_service.get_tenant_id(): raise_500(None, f'Pipeline[id={scheduler.pipelineId}] not found.')