def aid_hierarchy(self, data: Dict[str, Any]) -> Dict[str, Any]:
"""
given data might be changed, and returns exactly the given one
"""
if self.should_aid_hierarchy():
aid(data, [], ask_snowflake_generator())
return data
def find_schema_by_id(self, topic_id: TopicId, tenant_id: TenantId) -> Optional[TopicSchema]:
if not self.principalService.is_super_admin():
if self.principalService.get_tenant_id() != tenant_id:
raise Exception('Forbidden')
schema = CacheService.topic().get_schema(topic_id)
if schema is not None:
if schema.get_topic().tenantId != tenant_id:
return None
return schema
storage_service = TopicStorageService(ask_meta_storage(), ask_snowflake_generator(), self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
topic: Topic = storage_service.find_by_id(topic_id)
if topic is None:
return None
CacheService.topic().put(topic)
schema = CacheService.topic().get_schema(topic.topicId)
if schema is not None:
if schema.get_topic().tenantId != tenant_id:
return None
return schema
finally:
storage_service.close_transaction()
def create_jobs(ioScheduler: AsyncIOScheduler) -> None: schedulers = find_enabled_jobs() snowflake_generator = ask_snowflake_generator() ArrayHelper(schedulers) \ .map(lambda x: create_job(ioScheduler, x, snowflake_generator)) \ .filter(lambda x: x is not None) \ .each(lambda x: topic_snapshot_jobs.put_job(x[0].schedulerId, x[0].version, x[1]))
def find_by_topic_id(self, topic_id: TopicId) -> List[Pipeline]:
pipeline_ids = CacheService.pipelines_by_topic().get(topic_id)
if pipeline_ids is not None:
pipelines = ArrayHelper(pipeline_ids) \
.map(lambda x: self.find_by_id(x)) \
.filter(lambda x: x is not None).to_list()
if len(pipelines) != len(pipeline_ids):
loaded = ArrayHelper(pipelines).map(
lambda x: x.pipelineId).to_list()
raise Exception(
f'Except pipelines[{pipeline_ids}], but get[{loaded}] only.'
)
return pipelines
storage_service = PipelineStorageService(ask_meta_storage(),
ask_snowflake_generator(),
self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
pipelines: List[Pipeline] = storage_service.find_by_topic_id(
topic_id, self.principalService.get_tenant_id())
if len(pipelines) == 0:
CacheService.pipelines_by_topic().declare_no_pipelines(
topic_id)
return pipelines
return ArrayHelper(pipelines).each(
lambda x: CacheService.pipeline().put(x)).to_list()
finally:
storage_service.close_transaction()
def __init__(self, schema: TopicSchema,
data_entity_helper: TopicDataEntityHelper,
storage: TopicDataStorageSPI,
principal_service: PrincipalService):
super().__init__(schema, data_entity_helper)
self.storage = storage
self.principalService = principal_service
self.snowflakeGenerator = ask_snowflake_generator()
def register_topic_snapshot_job(scheduler: TopicSnapshotScheduler) -> None: scheduler_id = scheduler.schedulerId topic_snapshot_jobs.remove_job(scheduler_id) if not scheduler.enabled: return job = create_job(topic_snapshot_jobs.get_scheduler(), scheduler, ask_snowflake_generator()) if job is not None: topic_snapshot_jobs.put_job(scheduler_id, scheduler.version, job[1])
async def handle_trigger_data(trigger_data: PipelineTriggerDataWithPAT) -> None:
# TODO should log trigger data
pat = trigger_data.pat
if is_blank(pat):
raise Exception('PAT not found.')
principal_service = get_principal_by_pat(
retrieve_authentication_manager(), pat, [UserRole.ADMIN, UserRole.SUPER_ADMIN])
trace_id: PipelineTriggerTraceId = str(ask_snowflake_generator().next_id())
await try_to_invoke_pipelines(trigger_data, trace_id, principal_service)
def find_all(self) -> List[Tenant]:
storage_service = TenantStorageService(ask_meta_storage(),
ask_snowflake_generator(),
self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
return storage_service.find_all()
finally:
storage_service.close_transaction()
def find_should_monitored(self, tenant_id: TenantId) -> List[Topic]: storage_service = TopicStorageService(ask_meta_storage(), ask_snowflake_generator(), self.principalService) storage_service.begin_transaction() try: # noinspection PyTypeChecker topics = storage_service.find_all(tenant_id) # only business topics need to be monitored return ArrayHelper(topics).filter(lambda x: x.kind == TopicKind.BUSINESS).to_list() finally: storage_service.close_transaction()
async def trigger_pipeline_async(
trigger_data: PipelineTriggerData,
principal_service: PrincipalService = Depends(get_any_admin_principal)
) -> PipelineTriggerResult:
trace_id = trigger_data.traceId if is_not_blank(
trigger_data.traceId) else str(ask_snowflake_generator().next_id())
internal_data_id = await try_to_invoke_pipelines_async(
trigger_data, trace_id, principal_service)
return PipelineTriggerResult(received=True,
traceId=trace_id,
internalDataId=str(internal_data_id))
def run() -> None: snowflake_generator = ask_snowflake_generator() schedulers = find_enabled_jobs() # remove jobs which not exists in enabled schedulers scheduler_ids = ArrayHelper(schedulers).map(lambda x: x.schedulerId).to_list() topic_snapshot_jobs.remove_jobs_but(scheduler_ids) # replace jobs ArrayHelper(schedulers) \ .filter(lambda x: topic_snapshot_jobs.should_put_job(x.schedulerId, x.version)) \ .map(lambda x: create_job(ioScheduler, x, snowflake_generator)) \ .filter(lambda x: x is not None) \ .each(lambda x: topic_snapshot_jobs.put_job(x[0].schedulerId, x[0].version, x[1]))
def heart_beat_on_topics() -> None:
topics = CacheService.topic().all()
topic_service = TopicService(ask_meta_storage(), ask_snowflake_generator(),
ask_super_admin())
topic_service.begin_transaction()
try:
for topic in topics:
loaded: Optional[Topic] = topic_service.find_by_id(topic.topicId)
if loaded is None:
CacheService.topic().remove(topic.topicId)
elif loaded.lastModifiedAt > topic.lastModifiedAt or loaded.version > topic.version:
CacheService.topic().put(loaded)
finally:
topic_service.close_transaction()
def heart_beat_on_pipelines() -> None:
pipelines = CacheService.pipeline().all()
pipeline_service = PipelineService(ask_meta_storage(),
ask_snowflake_generator(),
ask_super_admin())
pipeline_service.begin_transaction()
try:
for pipeline in pipelines:
loaded: Optional[Pipeline] = pipeline_service.find_by_id(
pipeline.pipelineId)
if loaded is None:
CacheService.pipeline().remove(pipeline.pipelineId)
elif loaded.lastModifiedAt > pipeline.lastModifiedAt or loaded.version > pipeline.version:
CacheService.pipeline().put(loaded)
finally:
pipeline_service.close_transaction()
def heart_beat_on_tenants() -> None:
tenants = CacheService.tenant().all()
tenant_service = TenantService(ask_meta_storage(),
ask_snowflake_generator(),
ask_super_admin())
tenant_service.begin_transaction()
try:
for tenant in tenants:
loaded: Optional[Tenant] = tenant_service.find_by_id(
tenant.tenantId)
if loaded is None:
CacheService.tenant().remove(tenant.tenantId)
elif loaded.lastModifiedAt > tenant.lastModifiedAt or loaded.version > tenant.version:
CacheService.tenant().put(loaded)
finally:
tenant_service.close_transaction()
def heart_beat_on_data_sources() -> None:
data_sources = CacheService.data_source().all()
data_source_service = DataSourceService(ask_meta_storage(),
ask_snowflake_generator(),
ask_super_admin())
data_source_service.begin_transaction()
try:
for data_source in data_sources:
loaded: Optional[DataSource] = data_source_service.find_by_id(
data_source.dataSourceId)
if loaded is None:
CacheService.data_source().remove(data_source.dataSourceId)
elif loaded.lastModifiedAt > data_source.lastModifiedAt or loaded.version > data_source.version:
CacheService.data_source().put(loaded)
finally:
data_source_service.close_transaction()
def heart_beat_on_external_writers() -> None:
external_writers = CacheService.external_writer().all()
external_writer_service = ExternalWriterService(ask_meta_storage(),
ask_snowflake_generator(),
ask_super_admin())
external_writer_service.begin_transaction()
try:
for external_writer in external_writers:
loaded: Optional[
ExternalWriter] = external_writer_service.find_by_id(
external_writer.writerId)
if loaded is None:
CacheService.external_writer().remove(external_writer.writerId)
elif loaded.lastModifiedAt > external_writer.lastModifiedAt or loaded.version > external_writer.version:
CacheService.external_writer().put(loaded)
finally:
external_writer_service.close_transaction()
async def exchange_user(principal_service: PrincipalService = Depends(get_any_principal)) -> Optional[User]: """ returns current principal """ user_id = principal_service.get_user_id() user_service = UserService(ask_meta_storage(), ask_snowflake_generator(), principal_service) user_service.begin_transaction() try: # noinspection PyTypeChecker user: User = user_service.find_by_id(user_id) if user is None: return None else: del user.password return user finally: user_service.close_transaction()
def find_by_id(self, tenant_id: TenantId) -> Optional[Tenant]:
tenant = CacheService.tenant().get(tenant_id)
if tenant is not None:
return tenant
storage_service = TenantStorageService(ask_meta_storage(),
ask_snowflake_generator(),
self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
tenant: Tenant = storage_service.find_by_id(tenant_id)
if tenant is None:
return None
CacheService.tenant().put(tenant)
return tenant
finally:
storage_service.close_transaction()
def find_by_id(self, topic_id: TopicId) -> Optional[Topic]:
topic = CacheService.topic().get(topic_id)
if topic is not None:
if topic.tenantId != self.principalService.get_tenant_id():
raise DataKernelException(
f'Topic[id={topic_id}] not belongs to current tenant[id={self.principalService.get_tenant_id()}].')
return topic
storage_service = TopicStorageService(ask_meta_storage(), ask_snowflake_generator(), self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
topic: Topic = storage_service.find_by_id(topic_id)
if topic is None:
return None
CacheService.topic().put(topic)
return topic
finally:
storage_service.close_transaction()
def trigger_pipeline(detected: MonitorRuleDetected,
principal_service: PrincipalService) -> None:
schema = find_topic_schema('dqc_raw_rule_result', principal_service)
trace_id: PipelineTriggerTraceId = str(ask_snowflake_generator().next_id())
asynchronized = ask_monitor_result_pipeline_async()
# noinspection PyUnusedLocal
def handle_monitor_log(monitor_log: PipelineMonitorLog,
is_asynchronized: bool) -> None:
logger.info(monitor_log)
pipeline_trigger = PipelineTrigger(trigger_topic_schema=schema,
trigger_type=PipelineTriggerType.INSERT,
trigger_data=detected.to_dict(),
trace_id=trace_id,
principal_service=principal_service,
asynchronized=asynchronized,
handle_monitor_log=handle_monitor_log)
if asynchronized:
ensure_future(pipeline_trigger.invoke())
else:
run(pipeline_trigger.invoke())
def find_by_id(self, data_source_id: DataSourceId) -> Optional[DataSource]:
data_source = CacheService.data_source().get(data_source_id)
if data_source is not None:
if data_source.tenantId != self.principalService.get_tenant_id():
raise DataKernelException(
f'Data source[id={data_source_id}] not belongs to '
f'current tenant[id={self.principalService.get_tenant_id()}].')
return data_source
storage_service = DataSourceStorageService(
ask_meta_storage(), ask_snowflake_generator(), self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
data_source: DataSource = storage_service.find_by_id(data_source_id)
if data_source is None:
return None
CacheService.data_source().put(data_source)
return data_source
finally:
storage_service.close_transaction()
def find_by_id(self, pipeline_id: PipelineId) -> Optional[Pipeline]:
pipeline = CacheService.pipeline().get(pipeline_id)
if pipeline is not None:
if pipeline.tenantId != self.principalService.get_tenant_id():
raise DataKernelException(
f'Pipeline[id={pipeline_id}] not belongs to '
f'current tenant[id={self.principalService.get_tenant_id()}].'
)
return pipeline
storage_service = PipelineStorageService(ask_meta_storage(),
ask_snowflake_generator(),
self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
pipeline: Pipeline = storage_service.find_by_id(pipeline_id)
if pipeline is None:
return None
CacheService.pipeline().put(pipeline)
return pipeline
finally:
storage_service.close_transaction()
def find_by_id(self, writer_id: ExternalWriterId) -> Optional[ExternalWriter]:
external_writer = CacheService.external_writer().get(writer_id)
if external_writer is not None:
if external_writer.tenantId != self.principalService.get_tenant_id():
raise DataKernelException(
f'External writer[id={writer_id}] not belongs to '
f'current tenant[id={self.principalService.get_tenant_id()}].')
register_external_writer(external_writer)
return external_writer
storage_service = ExternalWriterStorageService(
ask_meta_storage(), ask_snowflake_generator(), self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
external_writer: ExternalWriter = storage_service.find_by_id(writer_id)
if external_writer is None:
return None
CacheService.external_writer().put(external_writer)
register_external_writer(external_writer)
return external_writer
finally:
storage_service.close_transaction()
def find_by_name(self, subject_name: str) -> Optional[Subject]:
storage_service = SubjectStorageService(ask_meta_storage(),
ask_snowflake_generator(),
self.principalService)
storage_service.begin_transaction()
try:
# noinspection PyTypeChecker
subject: Subject = storage_service.find_by_name(subject_name)
if subject is None:
return None
if subject.tenantId != self.principalService.get_tenant_id():
raise InquiryKernelException(
f'Subject[name={subject_name}] not belongs to '
f'current tenant[id={self.principalService.get_tenant_id()}].'
)
if not self.principalService.is_admin(
) and subject.userId != self.principalService.get_user_id():
raise InquiryKernelException(
f'Subject[name={subject_name}] not belongs to '
f'current user[id={self.principalService.get_user_id()}].')
return subject
finally:
storage_service.close_transaction()
def get_topic_service(principal_service: PrincipalService) -> TopicService:
return TopicService(ask_meta_storage(), ask_snowflake_generator(),
principal_service)
def get_inspection_service(principal_service: PrincipalService) -> InspectionService: return InspectionService(ask_meta_storage(), ask_snowflake_generator(), principal_service)
def get_pipeline_service(
principal_service: PrincipalService) -> PipelineService:
return PipelineService(ask_meta_storage(), ask_snowflake_generator(),
principal_service)
def get_topic_snapshot_scheduler_service(principal_service: PrincipalService) -> TopicSnapshotSchedulerService: return TopicSnapshotSchedulerService(ask_meta_storage(), ask_snowflake_generator(), principal_service)
def get_subject_service(principal_service: PrincipalService) -> SubjectService:
return SubjectService(ask_meta_storage(), ask_snowflake_generator(),
principal_service)
def get_connected_space_graphic_service(
principal_service: PrincipalService) -> ConnectedSpaceGraphicService:
return ConnectedSpaceGraphicService(ask_meta_storage(),
ask_snowflake_generator(),
principal_service)