def handle_scheduler(
scheduler: TopicSnapshotScheduler, source_topic: Topic,
task_topic: Topic, topic_service: TopicService,
scheduler_service: TopicSnapshotSchedulerService,
principal_service: PrincipalService) -> Optional[Callable[[], None]]:
target_topic_id = scheduler.targetTopicId
if is_blank(target_topic_id):
# incorrect scheduler, ignored
return None
should_save_scheduler = False
target_topic: Optional[Topic] = topic_service.find_by_id(target_topic_id)
if target_topic is None:
# create target topic when not found
target_topic = create_snapshot_target_topic(scheduler, source_topic)
target_topic, target_topic_tail = ask_save_topic_action(
topic_service, principal_service)(target_topic)
scheduler.targetTopicId = target_topic.topicId
should_save_scheduler = True
else:
# rebuild target topic
target_topic = rebuild_snapshot_target_topic(target_topic,
source_topic)
target_topic, target_topic_tail = ask_save_topic_action(
topic_service, principal_service)(target_topic)
pipeline_service = get_pipeline_service(topic_service)
pipeline_id = scheduler.pipelineId
if is_blank(pipeline_id):
# create pipeline not declared
pipeline = create_snapshot_pipeline(task_topic, target_topic)
pipeline = ask_save_pipeline_action(pipeline_service,
principal_service)(pipeline)
scheduler.pipelineId = pipeline.pipelineId
should_save_scheduler = True
else:
pipeline: Optional[Pipeline] = pipeline_service.find_by_id(pipeline_id)
if pipeline is None:
# create pipeline when not found
pipeline = create_snapshot_pipeline(task_topic, target_topic)
pipeline = ask_save_pipeline_action(pipeline_service,
principal_service)(pipeline)
scheduler.pipelineId = pipeline.pipelineId
should_save_scheduler = True
else:
# rebuild pipeline
pipeline = rebuild_snapshot_pipeline(pipeline, task_topic,
target_topic)
ask_save_pipeline_action(pipeline_service,
principal_service)(pipeline)
if should_save_scheduler:
scheduler_service.update(scheduler)
return target_topic_tail
def validate_source_topic(scheduler: TopicSnapshotScheduler,
topic_service: TopicService) -> Topic:
source_topic: Optional[Topic] = topic_service.find_by_id(scheduler.topicId)
if source_topic is None:
raise_500(None, f'Topic[id={scheduler.topicId}] not found.')
if source_topic.type == TopicType.RAW:
raise_500(None, f'Topic[id={scheduler.topicId}] is raw topic.')
if source_topic.kind == TopicKind.SYSTEM:
raise_500(None, f'Topic[id={scheduler.topicId}] is system topic.')
return source_topic
def find_schema_by_id(self, topic_id: TopicId, tenant_id: TenantId) -> Optional[TopicSchema]:
if not self.principalService.is_super_admin():
if self.principalService.get_tenant_id() != tenant_id:
raise Exception('Forbidden')
schema = CacheService.topic().get_schema(topic_id)
if schema is not None:
if schema.get_topic().tenantId != tenant_id:
return None
return schema
storage_service = TopicStorageService(ask_meta_storage(), ask_snowflake_generator(), self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
topic: Topic = storage_service.find_by_id(topic_id)
if topic is None:
return None
CacheService.topic().put(topic)
schema = CacheService.topic().get_schema(topic.topicId)
if schema is not None:
if schema.get_topic().tenantId != tenant_id:
return None
return schema
finally:
storage_service.close_transaction()
def try_to_import_topic(topic: Topic, topic_service: TopicService, do_update: bool) -> TopicImportDataResult: if is_blank(topic.topicId): topic_service.redress_storable_id(topic) topic_service.create(topic) else: existing_topic: Optional[Topic] = topic_service.find_by_id(topic.topicId) if existing_topic is None: topic_service.create(topic) elif do_update: topic.version = existing_topic.version topic.dataSourceId = existing_topic.dataSourceId topic_service.update(topic) else: return TopicImportDataResult( topicId=topic.topicId, name=topic.name, passed=False, reason='Topic already exists.') post_save_topic(topic, topic_service) return TopicImportDataResult(topicId=topic.topicId, name=topic.name, passed=True)
def handle_related_topics_on_create(
scheduler: TopicSnapshotScheduler, topic_service: TopicService,
source_topic: Topic, principal_service: PrincipalService
) -> Tuple[Topic, Callable[[], None]]:
target_topic: Optional[Topic] = topic_service.find_by_name_and_tenant(
scheduler.targetTopicName, scheduler.tenantId)
if target_topic is not None:
raise_500(None,
f'Topic[name={scheduler.targetTopicName}] already exists.')
# create target topic
target_topic = create_snapshot_target_topic(scheduler, source_topic)
target_topic, target_topic_tail = ask_save_topic_action(
topic_service, principal_service)(target_topic)
scheduler.targetTopicId = target_topic.topicId
tail = target_topic_tail
# find task topic, it might be created by another scheduler
task_topic_name = as_snapshot_task_topic_name(source_topic)
task_topic: Optional[Topic] = topic_service.find_by_name_and_tenant(
task_topic_name, scheduler.tenantId)
# create task topic only it is not declared
if task_topic is None:
task_topic = create_snapshot_task_topic(source_topic)
task_topic, task_topic_tail = ask_save_topic_action(
topic_service, principal_service)(task_topic)
tail = combine_tail_actions([target_topic_tail, task_topic_tail])
# create pipeline from task topic to target topic
pipeline = create_snapshot_pipeline(task_topic, target_topic)
pipeline_service = get_pipeline_service(topic_service)
pipeline = ask_save_pipeline_action(pipeline_service,
principal_service)(pipeline)
scheduler.pipelineId = pipeline.pipelineId
return target_topic, tail
def find_should_monitored(self, tenant_id: TenantId) -> List[Topic]: storage_service = TopicStorageService(ask_meta_storage(), ask_snowflake_generator(), self.principalService) storage_service.begin_transaction() try: # noinspection PyTypeChecker topics = storage_service.find_all(tenant_id) # only business topics need to be monitored return ArrayHelper(topics).filter(lambda x: x.kind == TopicKind.BUSINESS).to_list() finally: storage_service.close_transaction()
def heart_beat_on_topics() -> None:
topics = CacheService.topic().all()
topic_service = TopicService(ask_meta_storage(), ask_snowflake_generator(),
ask_super_admin())
topic_service.begin_transaction()
try:
for topic in topics:
loaded: Optional[Topic] = topic_service.find_by_id(topic.topicId)
if loaded is None:
CacheService.topic().remove(topic.topicId)
elif loaded.lastModifiedAt > topic.lastModifiedAt or loaded.version > topic.version:
CacheService.topic().put(loaded)
finally:
topic_service.close_transaction()
def validate_scheduler_on_update(scheduler: TopicSnapshotScheduler,
topic_service: TopicService,
principal_service: PrincipalService) -> None:
if is_blank(scheduler.targetTopicId):
raise_500(None, f'TargetTopicId is required on scheduler.')
# noinspection DuplicatedCode
target_topic: Optional[Topic] = topic_service.find_by_id(
scheduler.targetTopicId)
if target_topic is None:
raise_500(None, f'Topic[id={scheduler.targetTopicId}] not found.')
if target_topic.tenantId != principal_service.get_tenant_id():
raise_500(None, f'Topic[id={scheduler.targetTopicId}] not found.')
if target_topic.type == TopicType.RAW:
raise_500(None,
f'Target topic[id={scheduler.targetTopicId}] is raw topic.')
if target_topic.kind == TopicKind.SYSTEM:
raise_500(
None,
f'Target topic[id={scheduler.targetTopicId}] is system topic.')
if is_not_blank(scheduler.targetTopicName
) and target_topic.name != scheduler.targetTopicName:
raise_500(
None,
f'Target topic[id={scheduler.targetTopicId}, name={target_topic.name}] '
f'has different name with given scheduler[targetTopicName={scheduler.targetTopicName}].'
)
if is_blank(scheduler.pipelineId):
raise_500(None, f'PipelineId is required on scheduler.')
pipeline_service = get_pipeline_service(topic_service)
pipeline: Optional[Pipeline] = pipeline_service.find_by_id(
scheduler.pipelineId)
if pipeline is None:
raise_500(None, f'Pipeline[id={scheduler.pipelineId}] not found.')
if pipeline.tenantId != principal_service.get_tenant_id():
raise_500(None, f'Pipeline[id={scheduler.pipelineId}] not found.')
def find_by_id(self, topic_id: TopicId) -> Optional[Topic]:
topic = CacheService.topic().get(topic_id)
if topic is not None:
if topic.tenantId != self.principalService.get_tenant_id():
raise DataKernelException(
f'Topic[id={topic_id}] not belongs to current tenant[id={self.principalService.get_tenant_id()}].')
return topic
storage_service = TopicStorageService(ask_meta_storage(), ask_snowflake_generator(), self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
topic: Topic = storage_service.find_by_id(topic_id)
if topic is None:
return None
CacheService.topic().put(topic)
return topic
finally:
storage_service.close_transaction()
def get_topic_service(tenant_service: MetaTenantService) -> TopicService:
return TopicService(tenant_service.storage,
tenant_service.snowflakeGenerator,
tenant_service.principalService)
def get_topic_service(user_service: UserService) -> TopicService: return TopicService(user_service.storage, user_service.snowflakeGenerator, user_service.principalService)
def get_topic_service(space_service: SpaceService) -> TopicService: return TopicService(space_service.storage, space_service.snowflakeGenerator, space_service.principalService)
def get_topic_service(catalog_service: CatalogService) -> TopicService:
return TopicService(catalog_service.storage,
catalog_service.snowflakeGenerator,
catalog_service.principalService)
def get_topic_service(subject_service: SubjectService) -> TopicService:
return TopicService(subject_service.storage,
subject_service.snowflakeGenerator,
subject_service.principalService)
def get_topic_service(
scheduler_service: TopicSnapshotSchedulerService) -> TopicService:
return TopicService(scheduler_service.storage,
scheduler_service.snowflakeGenerator,
scheduler_service.principalService)
def get_topic_service(principal_service: PrincipalService) -> TopicService:
return TopicService(ask_meta_storage(), ask_snowflake_generator(),
principal_service)
def get_topic_service(enum_service: EnumService) -> TopicService:
return TopicService(enum_service.storage, enum_service.snowflakeGenerator,
enum_service.principalService)
