def validate_user(a_tuple: UserBasedTuple, user_service: UserService,
principal_service: PrincipalService) -> None:
if not principal_service.is_admin():
raise_403()
if is_blank(a_tuple.userId):
if principal_service.is_super_admin():
raise_400('User id is required.')
elif principal_service.is_tenant_admin():
a_tuple.userId = principal_service.get_user_id()
else:
raise_403()
else:
if a_tuple.userId == principal_service.get_user_id():
if principal_service.is_super_admin():
raise_400(f'Incorrect user id[{a_tuple.userId}].')
else:
user: Optional[User] = user_service.find_by_id(a_tuple.userId)
if user is None:
raise_400('User id is required.')
if principal_service.is_super_admin():
if user.tenantId == principal_service.get_tenant_id():
raise_400(f'Incorrect user id[{a_tuple.userId}].')
elif principal_service.is_tenant_admin():
if user.tenantId != principal_service.get_tenant_id():
raise_400(f'Incorrect user id[{a_tuple.userId}].')
async def fetch_topic_by_name(
query_name: Optional[str] = None,
tenant_id: Optional[TenantId] = None,
principal_service: PrincipalService = Depends(get_any_admin_principal)
) -> List[Topic]:
if is_blank(query_name):
raise_400('Name criteria is required.')
if principal_service.is_tenant_admin():
if is_not_blank(
tenant_id) and tenant_id != principal_service.get_tenant_id():
raise_400('Tenant id is incorrect.')
else:
tenant_id = principal_service.get_tenant_id()
if principal_service.is_super_admin() and is_blank(tenant_id):
raise_400('Tenant id is required.')
topic_service = get_topic_service(principal_service)
def action() -> List[Topic]:
topic_index_service = get_topic_index_service(topic_service)
factor_index_list = topic_index_service.find(query_name, tenant_id)
if len(factor_index_list) == 0:
return []
topic_ids: List[TopicId] = []
for factor_index in factor_index_list:
if factor_index.topicId not in topic_ids:
topic_ids.append(factor_index.topicId)
return topic_service.find_by_ids(topic_ids, tenant_id)
return trans_readonly(topic_service, action)
async def init_tenant(
tenant_id: Optional[TenantId],
principal_service: PrincipalService = Depends(get_any_admin_principal)
) -> None:
if is_blank(tenant_id):
if principal_service.is_super_admin():
raise_400('Tenant id is required.')
elif principal_service.is_tenant_admin():
tenant_id = principal_service.get_tenant_id()
else:
if principal_service.get_tenant_id(
) != tenant_id and principal_service.is_tenant_admin():
raise_400(f'Tenant[{tenant_id}] does not match principal.')
elif principal_service.is_super_admin():
tenant: Optional[Tenant] = get_tenant_service(
principal_service).find_by_id(tenant_id)
if tenant is None:
raise_404(f'Tenant[id={tenant_id}] not found.')
meta_tenant_service = get_meta_tenant_service(principal_service)
def action() -> None:
topics = ask_pipeline_monitor_topics()
create_topics_and_pipelines(
topics, lambda source_topics: ask_pipeline_monitor_pipelines(
source_topics), tenant_id, meta_tenant_service,
principal_service)
topics = ask_dqc_topics()
create_topics_and_pipelines(
topics, lambda source_topics: ask_dqc_pipelines(source_topics),
tenant_id, meta_tenant_service, principal_service)
trans(meta_tenant_service, action)
def validate_tenant_id_when_tenant_admin( request: MixImportDataRequest, principal_service: PrincipalService) -> TenantId: """ simply assign tenant id of current principal """ fill_tenant_id(request, principal_service.get_tenant_id()) return principal_service.get_tenant_id()
def validate_tenant_id(tenant_id: Optional[TenantId],
principal_service: PrincipalService) -> TenantId:
if principal_service.is_tenant_admin():
if is_not_blank(
tenant_id) and tenant_id != principal_service.get_tenant_id():
raise_400('Tenant id is incorrect.')
return principal_service.get_tenant_id()
elif principal_service.is_super_admin():
if is_blank(tenant_id):
raise_400('Tenant id is required.')
return tenant_id
def find_topic(
self, topic_id: Optional[TopicId], available_schemas: List[TopicSchema],
principal_service: PrincipalService, allow_in_memory_variables: bool) -> Tuple[TopicSchema, bool]:
"""
find topic even it is not found in available list when in-memory variables is allowed.
for the secondary one of return tuple, true when it is found in given available list,
otherwise is false, which means it can be find in definitions
"""
if is_blank(topic_id):
raise DataKernelException(f'Topic not declared.')
schema = ArrayHelper(available_schemas).find(lambda x: x.get_topic().topicId == topic_id)
if schema is None:
if not allow_in_memory_variables:
raise DataKernelException(f'Topic[id={topic_id}] not found.')
else:
# in pipeline, it might be from trigger data
topic_service = get_topic_service(principal_service)
topic: Optional[Topic] = topic_service.find_by_id(topic_id)
if topic is None:
raise DataKernelException(f'Topic[id={topic_id}] not found.')
schema: Optional[TopicSchema] = topic_service.find_schema_by_name(
topic.name, principal_service.get_tenant_id())
if schema is None:
raise DataKernelException(f'Topic schema[id={topic_id}] not found.')
return schema, False
else:
return schema, True
def try_to_lock_topic_for_monitor(
topic: Topic, frequency: MonitorRuleStatisticalInterval,
process_date: date, principal_service: PrincipalService
) -> Tuple[Optional[MonitorJobLock], bool]:
if isinstance(process_date, datetime):
process_date = process_date.date()
lock_service = get_lock_service(principal_service)
lock_service.begin_transaction()
try:
lock = MonitorJobLock(
# lockId: MonitorJobLockId = None
tenantId=principal_service.get_tenant_id(),
topicId=topic.topicId,
frequency=frequency,
processDate=process_date,
status=MonitorJobLockStatus.READY,
userId=principal_service.get_user_id(),
)
lock_service.create(lock)
lock_service.commit_transaction()
return lock, True
except Exception:
lock_service.rollback_transaction()
return None, False
def try_to_lock_scheduler(
scheduler: TopicSnapshotScheduler, process_date: date,
principal_service: PrincipalService
) -> Tuple[Optional[TopicSnapshotJobLock], bool]:
if isinstance(process_date, datetime):
process_date = process_date.date()
lock_service = get_lock_service(principal_service)
lock_service.begin_transaction()
# noinspection PyBroadException
try:
lock = TopicSnapshotJobLock(
tenantId=principal_service.get_tenant_id(),
schedulerId=scheduler.schedulerId,
frequency=scheduler.frequency,
processDate=process_date,
rowCount=0,
status=TopicSnapshotJobLockStatus.READY,
userId=principal_service.get_user_id(),
)
lock_service.create(lock)
lock_service.commit_transaction()
return lock, True
except Exception:
lock_service.rollback_transaction()
return None, False
def validate_reports(
dashboard: Dashboard, dashboard_service: DashboardService, principal_service: PrincipalService) -> None:
reports = dashboard.reports
if reports is None:
dashboard.reports = []
return
if len(reports) == 0:
return
report_ids = ArrayHelper(reports).map(lambda x: x.reportId).distinct().to_list()
if len(report_ids) == 0:
dashboard.reports = []
return
report_service = get_report_service(dashboard_service)
for report_id in report_ids:
existing_one = report_service.find_tenant_and_user(report_id)
if existing_one is None:
raise_400('Report ids do not match.')
existing_tenant_id, existing_user_id = existing_one
if existing_tenant_id != principal_service.get_tenant_id():
raise_403()
if existing_user_id != principal_service.get_user_id():
raise_403()
def ask_topic(topic_id: TopicId, principal_service: PrincipalService) -> Topic:
topic = get_topic_service(principal_service).find_by_id(topic_id)
if topic is None:
raise IndicatorKernelException(f'Topic[id={topic_id}] not found.')
if topic.tenantId != principal_service.get_tenant_id():
raise IndicatorKernelException(f'Topic[id={topic_id}] not found.')
return topic
def run_topics_rules(
topic_name: Optional[str] = None,
frequency: Optional[MonitorRuleStatisticalInterval] = None,
process_date: Optional[str] = None,
tenant_id: Optional[TenantId] = None,
principal_service: PrincipalService = Depends(get_any_admin_principal)
) -> None:
principal_service = ask_principal_service(principal_service, tenant_id)
if is_not_blank(topic_name):
schema = get_topic_service(principal_service).find_schema_by_name(
topic_name, principal_service.get_tenant_id())
if schema is None:
raise_404(f'Topic[name={topic_name}] not found.')
topic_id = schema.get_topic().topicId
else:
topic_id = None
if is_not_blank(process_date):
parsed, parsed_date = is_date(process_date, ask_all_date_formats())
if not parsed:
raise_400(f'Given process date[{process_date}] cannot be parsed.')
process_date = parsed_date
else:
process_date = get_current_time_in_seconds()
process_date = truncate_time(process_date)
now = truncate_time(get_current_time_in_seconds())
if process_date.year > now.year:
raise_400(f'Given process date[{process_date}] cannot be in future.')
if process_date.year == now.year and process_date.month > now.month:
raise_400(f'Given process date[{process_date}] cannot be in future.')
if process_date.year == now.year and process_date.month == now.month and process_date.day > now.day:
raise_400(f'Given process date[{process_date}] cannot be in future.')
if frequency == MonitorRuleStatisticalInterval.MONTHLY:
# given process date is in this month, run previous month
# otherwise, run the given month
if process_date.year == now.year and process_date.month == now.month:
process_date = to_previous_month(process_date)
SelfCleaningMonitorRulesRunner(principal_service) \
.run(process_date, topic_id, MonitorRuleStatisticalInterval.MONTHLY)
elif frequency == MonitorRuleStatisticalInterval.WEEKLY:
# given process date is in this week, run previous week
# otherwise, run the given week
if process_date.year == now.year and int(
process_date.strftime('%U')) == int(now.strftime('%U')):
process_date = to_previous_week(process_date)
SelfCleaningMonitorRulesRunner(principal_service) \
.run(process_date, topic_id, MonitorRuleStatisticalInterval.WEEKLY)
elif frequency == MonitorRuleStatisticalInterval.DAILY:
# given process date is today, run yesterday
# otherwise, run the given day
if process_date.year == now.year and process_date.month == now.month and process_date.day == now.day:
process_date = to_yesterday(process_date)
SelfCleaningMonitorRulesRunner(principal_service) \
.run(process_date, topic_id, MonitorRuleStatisticalInterval.DAILY)
else:
raise_400(f'Given frequency[{frequency}] is not supported.')
def assign_fix_columns_on_create(self, data: Dict[str, Any],
snowflake_generator: SnowflakeGenerator,
principal_service: PrincipalService,
now: datetime) -> None:
self.assign_id_column(data, snowflake_generator.next_id())
self.assign_tenant_id(data, principal_service.get_tenant_id())
self.assign_insert_time(data, now)
self.assign_update_time(data, now)
self.assign_version(data, 1)
def ask_subject(subject_id: SubjectId,
principal_service: PrincipalService) -> Subject:
subject = get_subject_service(principal_service).find_by_id(subject_id)
if subject is None:
raise IndicatorKernelException(f'Subject[id={subject_id}] not found.')
if subject.tenantId != principal_service.get_tenant_id():
raise IndicatorKernelException(f'Subject[id={subject_id}] not found.')
return subject
def validate_tenant_id(a_tuple: Union[TenantBasedTuple, UserBasedTuple],
principal_service: PrincipalService) -> None:
"""
validate tenant id of tuple.\n
a. for super admin, tenant id is required,\
b. for not super admin, tenant id must be same as current principal,\n
c. for not super admin and no tenant id, set as current principal.
"""
tenant_id = a_tuple.tenantId
if principal_service.is_super_admin():
if is_blank(tenant_id):
raise_400('Tenant id is required.')
elif is_not_blank(tenant_id):
if tenant_id != principal_service.get_tenant_id():
raise_403()
else:
# assign tenant id by current principal
a_tuple.tenantId = principal_service.get_tenant_id()
def find_topic_schema(name: str,
principal_service: PrincipalService) -> TopicSchema:
schema = get_topic_service(principal_service).find_schema_by_name(
name, principal_service.get_tenant_id())
if schema is None:
raise PipelineKernelException(
f'Topic schema[name={name}, tenant={principal_service.get_tenant_id()}] not found.'
)
return schema
def get_topic_schema(
topic_id: TopicId, principal_service: PrincipalService) -> TopicSchema:
topic_service = get_topic_service(principal_service)
topic = get_topic_service(principal_service).find_by_id(topic_id)
if topic is None:
raise DataKernelException(f'Topic[id={topic_id}] not found.')
schema = topic_service.find_schema_by_name(topic.name, principal_service.get_tenant_id())
if schema is None:
raise DataKernelException(f'Topic[name={topic.name}] not found.')
return schema
async def find_all_topics(principal_service: PrincipalService = Depends(
get_console_principal)) -> List[Topic]:
tenant_id = principal_service.get_tenant_id()
topic_service = get_topic_service(principal_service)
def action() -> List[Topic]:
return topic_service.find_all(tenant_id)
return trans_readonly(topic_service, action)
def find_topic_schema(topic_id: TopicId,
principal_service: PrincipalService) -> TopicSchema:
topic_service = get_topic_service(principal_service)
topic: Optional[Topic] = topic_service.find_by_id(topic_id)
if topic is None:
raise PipelineKernelException(f'Topic[id={topic_id}] not found.')
schema = topic_service.find_schema_by_name(
topic.name, principal_service.get_tenant_id())
if schema is None:
raise PipelineKernelException(
f'Topic schema[id={topic_id}] not found.')
return schema
async def find_all_external_writers(
principal_service: PrincipalService = Depends(get_any_admin_principal)
) -> List[ExternalWriter]:
tenant_id = None
if principal_service.is_tenant_admin():
tenant_id = principal_service.get_tenant_id()
external_writer_service = get_external_writer_service(principal_service)
def action() -> List[ExternalWriter]:
return external_writer_service.find_all(tenant_id)
return trans_readonly(external_writer_service, action)
def validate_tenant(a_tuple: Union[UserBasedTuple, TenantBasedTuple],
user_service: UserService,
principal_service: PrincipalService) -> None:
if not principal_service.is_admin():
raise_403()
if is_blank(a_tuple.tenantId):
if principal_service.is_super_admin():
raise_400('Tenant id is required.')
elif principal_service.is_tenant_admin():
a_tuple.tenantId = principal_service.get_tenant_id()
else:
if principal_service.is_tenant_admin():
if a_tuple.tenantId != principal_service.get_tenant_id():
raise_403()
elif principal_service.is_super_admin():
if a_tuple.tenantId == principal_service.get_tenant_id():
raise_400(f'Incorrect tenant id[{a_tuple.tenantId}].')
tenant_service = get_tenant_service(user_service)
tenant: Optional[Tenant] = tenant_service.find_by_id(
a_tuple.tenantId)
if tenant is None:
raise_400(f'Incorrect tenant id[{a_tuple.tenantId}].')
def do_load_inspection_by_id(
inspection_id: InspectionId, inspection_service: InspectionService,
principal_service: PrincipalService) -> Inspection:
# noinspection PyTypeChecker
inspection: Inspection = inspection_service.find_by_id(inspection_id)
if inspection is None:
raise_404()
# user id must match current principal's
if inspection.userId != principal_service.get_user_id():
raise_404()
# tenant id must match current principal's
if inspection.tenantId != principal_service.get_tenant_id():
raise_404()
return inspection
def ask_indicator(indicator_id: Optional[IndicatorId], principal_service: PrincipalService) -> Indicator:
if is_blank(indicator_id):
raise IndicatorKernelException('Indicator not declared.')
indicator_service = get_indicator_service(principal_service)
indicator_service.begin_transaction()
try:
# noinspection PyTypeChecker
indicator: Indicator = indicator_service.find_by_id(indicator_id)
if indicator is None:
raise IndicatorKernelException(f'Indicator[id={indicator_id}] not found.')
if indicator.tenantId != principal_service.get_tenant_id():
raise IndicatorKernelException(f'Indicator[id={indicator_id}] not found.')
return indicator
finally:
indicator_service.close_transaction()
def validate_tenant_id_when_super_admin(
request: MixImportDataRequest, user_service: UserService, principal_service: PrincipalService) -> TenantId:
"""
tenant id must be designated by data, because super admin doesn't need any metadata
"""
found_tenant_id = find_tenant_id(request)
if found_tenant_id == principal_service.get_tenant_id():
raise_400('Incorrect tenant id.')
tenant_service = TenantService(
user_service.storage, user_service.snowflakeGenerator, user_service.principalService)
tenant: Optional[Tenant] = tenant_service.find_by_id(found_tenant_id)
if tenant is None:
raise_400('Incorrect tenant id.')
fill_tenant_id(request, found_tenant_id)
return found_tenant_id
def validate_scheduler_on_update(scheduler: TopicSnapshotScheduler,
topic_service: TopicService,
principal_service: PrincipalService) -> None:
if is_blank(scheduler.targetTopicId):
raise_500(None, f'TargetTopicId is required on scheduler.')
# noinspection DuplicatedCode
target_topic: Optional[Topic] = topic_service.find_by_id(
scheduler.targetTopicId)
if target_topic is None:
raise_500(None, f'Topic[id={scheduler.targetTopicId}] not found.')
if target_topic.tenantId != principal_service.get_tenant_id():
raise_500(None, f'Topic[id={scheduler.targetTopicId}] not found.')
if target_topic.type == TopicType.RAW:
raise_500(None,
f'Target topic[id={scheduler.targetTopicId}] is raw topic.')
if target_topic.kind == TopicKind.SYSTEM:
raise_500(
None,
f'Target topic[id={scheduler.targetTopicId}] is system topic.')
if is_not_blank(scheduler.targetTopicName
) and target_topic.name != scheduler.targetTopicName:
raise_500(
None,
f'Target topic[id={scheduler.targetTopicId}, name={target_topic.name}] '
f'has different name with given scheduler[targetTopicName={scheduler.targetTopicName}].'
)
if is_blank(scheduler.pipelineId):
raise_500(None, f'PipelineId is required on scheduler.')
pipeline_service = get_pipeline_service(topic_service)
pipeline: Optional[Pipeline] = pipeline_service.find_by_id(
scheduler.pipelineId)
if pipeline is None:
raise_500(None, f'Pipeline[id={scheduler.pipelineId}] not found.')
if pipeline.tenantId != principal_service.get_tenant_id():
raise_500(None, f'Pipeline[id={scheduler.pipelineId}] not found.')
def ask_bucket(bucket_id: BucketId,
principal_service: PrincipalService) -> Bucket:
bucket_service = get_bucket_service(principal_service)
bucket_service.begin_transaction()
try:
# noinspection PyTypeChecker
bucket: Bucket = bucket_service.find_by_id(bucket_id)
if bucket is None:
raise IndicatorKernelException(
f'Bucket[id={bucket_id}] not found.')
if bucket.tenantId != principal_service.get_tenant_id():
raise IndicatorKernelException(
f'Bucket[id={bucket_id}] not found.')
return bucket
finally:
bucket_service.close_transaction()
def find_topic_schema(topic_id: Optional[TopicId], join: SubjectDatasetJoin,
principal_service: PrincipalService,
where: str) -> TopicSchema:
topic_service = get_topic_service(principal_service)
if is_blank(topic_id):
raise InquiryKernelException(
f'{where} topic of subject join[{join.dict()}] is none.')
topic: Optional[Topic] = topic_service.find_by_id(topic_id)
if topic is None:
raise InquiryKernelException(
f'{where} topic of subject join[{join.dict()}] not found.')
schema = topic_service.find_schema_by_name(
topic.name, principal_service.get_tenant_id())
if schema is None:
raise InquiryKernelException(
f'{where} topic schema of subject join[{join.dict()}] not found.')
return schema
async def rerun_by_topic_data(
scheduler_id: Optional[TopicSnapshotSchedulerId],
process_date: Optional[str],
principal_service: PrincipalService = Depends(get_any_admin_principal)
) -> None:
if is_blank(scheduler_id):
raise_400('Scheduler id is required.')
if is_blank(process_date):
raise_400('Process date is required.')
parsed, parsed_process_date = is_date(process_date, ask_date_formats())
if not parsed:
raise_400('Process date must be date.')
scheduler_service = get_topic_snapshot_scheduler_service(principal_service)
scheduler_service.begin_transaction()
try:
scheduler: Optional[
TopicSnapshotScheduler] = scheduler_service.find_by_id(
scheduler_id)
if scheduler is None:
raise_404(f'Scheduler[id={scheduler_id}] not found.')
if principal_service.is_tenant_admin(
) and scheduler.tenantId != principal_service.get_tenant_id():
raise_404(f'Scheduler[id={scheduler_id}] not found.')
lock_service = get_lock_service(scheduler_service)
lock = lock_service.find_by_scheduler_and_process_date(
scheduler_id, scheduler.frequency, parsed_process_date)
if lock is not None:
raise_406(
f'Scheduler[id={scheduler_id}, processDate={process_date}] run already.'
)
except HTTPException as e:
raise e
except Exception as e:
raise_500(e)
finally:
scheduler_service.close_transaction()
run_job(scheduler_id, parsed_process_date)
async def load_tenant_by_id(
tenant_id: Optional[TenantId] = None,
principal_service: PrincipalService = Depends(get_any_principal)
) -> Tenant:
if is_blank(tenant_id):
raise_400('Tenant id is required.')
if not principal_service.is_super_admin():
if tenant_id != principal_service.get_tenant_id():
raise_403()
tenant_service = get_tenant_service(principal_service)
def action() -> Tenant:
# noinspection PyTypeChecker
tenant: Tenant = tenant_service.find_by_id(tenant_id)
if tenant is None:
raise_404()
return tenant
return trans_readonly(tenant_service, action)
async def load_data_source_by_id(
data_source_id: Optional[DataSourceId] = None,
principal_service: PrincipalService = Depends(get_any_admin_principal)
) -> DataSource:
if is_blank(data_source_id):
raise_400('Data source id is required.')
if not principal_service.is_super_admin():
if data_source_id != principal_service.get_tenant_id():
raise_403()
data_source_service = get_data_source_service(principal_service)
def action() -> DataSource:
# noinspection PyTypeChecker
data_source: DataSource = data_source_service.find_by_id(
data_source_id)
if data_source is None:
raise_404()
return data_source
return trans_readonly(data_source_service, action)
async def load_external_writer_by_id(
writer_id: Optional[ExternalWriterId] = None,
principal_service: PrincipalService = Depends(get_any_admin_principal)
) -> ExternalWriter:
if is_blank(writer_id):
raise_400('External writer id is required.')
if not principal_service.is_super_admin():
if writer_id != principal_service.get_tenant_id():
raise_403()
external_writer_service = get_external_writer_service(principal_service)
def action() -> ExternalWriter:
# noinspection PyTypeChecker
external_writer: ExternalWriter = external_writer_service.find_by_id(
writer_id)
if external_writer is None:
raise_404()
return external_writer
return trans_readonly(external_writer_service, action)
