def try_to_lock_topic_for_monitor(
topic: Topic, frequency: MonitorRuleStatisticalInterval,
process_date: date, principal_service: PrincipalService
) -> Tuple[Optional[MonitorJobLock], bool]:
if isinstance(process_date, datetime):
process_date = process_date.date()
lock_service = get_lock_service(principal_service)
lock_service.begin_transaction()
try:
lock = MonitorJobLock(
# lockId: MonitorJobLockId = None
tenantId=principal_service.get_tenant_id(),
topicId=topic.topicId,
frequency=frequency,
processDate=process_date,
status=MonitorJobLockStatus.READY,
userId=principal_service.get_user_id(),
)
lock_service.create(lock)
lock_service.commit_transaction()
return lock, True
except Exception:
lock_service.rollback_transaction()
return None, False
def validate_user(a_tuple: UserBasedTuple, user_service: UserService,
principal_service: PrincipalService) -> None:
if not principal_service.is_admin():
raise_403()
if is_blank(a_tuple.userId):
if principal_service.is_super_admin():
raise_400('User id is required.')
elif principal_service.is_tenant_admin():
a_tuple.userId = principal_service.get_user_id()
else:
raise_403()
else:
if a_tuple.userId == principal_service.get_user_id():
if principal_service.is_super_admin():
raise_400(f'Incorrect user id[{a_tuple.userId}].')
else:
user: Optional[User] = user_service.find_by_id(a_tuple.userId)
if user is None:
raise_400('User id is required.')
if principal_service.is_super_admin():
if user.tenantId == principal_service.get_tenant_id():
raise_400(f'Incorrect user id[{a_tuple.userId}].')
elif principal_service.is_tenant_admin():
if user.tenantId != principal_service.get_tenant_id():
raise_400(f'Incorrect user id[{a_tuple.userId}].')
async def load_user_by_id(
user_id: Optional[UserId] = None,
principal_service: PrincipalService = Depends(get_any_principal)
) -> User:
if is_blank(user_id):
raise_400('User id is required.')
if not principal_service.is_admin():
# console user cannot visit other users
if user_id != principal_service.get_user_id():
raise_403()
user_service = get_user_service(principal_service)
def action() -> User:
# noinspection PyTypeChecker
user: User = user_service.find_by_id(user_id)
if user is None:
raise_404()
# check tenant id
if not principal_service.is_super_admin():
# tenant id must match current principal's, except current is super admin
if user.tenantId != principal_service.get_tenant_id():
raise_404()
# remove password
clear_pwd(user)
return user
return trans_readonly(user_service, action)
def validate_reports(
dashboard: Dashboard, dashboard_service: DashboardService, principal_service: PrincipalService) -> None:
reports = dashboard.reports
if reports is None:
dashboard.reports = []
return
if len(reports) == 0:
return
report_ids = ArrayHelper(reports).map(lambda x: x.reportId).distinct().to_list()
if len(report_ids) == 0:
dashboard.reports = []
return
report_service = get_report_service(dashboard_service)
for report_id in report_ids:
existing_one = report_service.find_tenant_and_user(report_id)
if existing_one is None:
raise_400('Report ids do not match.')
existing_tenant_id, existing_user_id = existing_one
if existing_tenant_id != principal_service.get_tenant_id():
raise_403()
if existing_user_id != principal_service.get_user_id():
raise_403()
async def fetch_topic_by_name(
query_name: Optional[str] = None,
tenant_id: Optional[TenantId] = None,
principal_service: PrincipalService = Depends(get_any_admin_principal)
) -> List[Topic]:
if is_blank(query_name):
raise_400('Name criteria is required.')
if principal_service.is_tenant_admin():
if is_not_blank(
tenant_id) and tenant_id != principal_service.get_tenant_id():
raise_400('Tenant id is incorrect.')
else:
tenant_id = principal_service.get_tenant_id()
if principal_service.is_super_admin() and is_blank(tenant_id):
raise_400('Tenant id is required.')
topic_service = get_topic_service(principal_service)
def action() -> List[Topic]:
topic_index_service = get_topic_index_service(topic_service)
factor_index_list = topic_index_service.find(query_name, tenant_id)
if len(factor_index_list) == 0:
return []
topic_ids: List[TopicId] = []
for factor_index in factor_index_list:
if factor_index.topicId not in topic_ids:
topic_ids.append(factor_index.topicId)
return topic_service.find_by_ids(topic_ids, tenant_id)
return trans_readonly(topic_service, action)
def validate_tenant_id_when_tenant_admin( request: MixImportDataRequest, principal_service: PrincipalService) -> TenantId: """ simply assign tenant id of current principal """ fill_tenant_id(request, principal_service.get_tenant_id()) return principal_service.get_tenant_id()
def try_to_lock_scheduler(
scheduler: TopicSnapshotScheduler, process_date: date,
principal_service: PrincipalService
) -> Tuple[Optional[TopicSnapshotJobLock], bool]:
if isinstance(process_date, datetime):
process_date = process_date.date()
lock_service = get_lock_service(principal_service)
lock_service.begin_transaction()
# noinspection PyBroadException
try:
lock = TopicSnapshotJobLock(
tenantId=principal_service.get_tenant_id(),
schedulerId=scheduler.schedulerId,
frequency=scheduler.frequency,
processDate=process_date,
rowCount=0,
status=TopicSnapshotJobLockStatus.READY,
userId=principal_service.get_user_id(),
)
lock_service.create(lock)
lock_service.commit_transaction()
return lock, True
except Exception:
lock_service.rollback_transaction()
return None, False
def prepare_and_validate_request(
request: MixImportDataRequest,
user_service: UserService, principal_service: PrincipalService) -> None:
tenant_id = validate_tenant_id(request, user_service, principal_service)
clear_data_source_id(request.topics)
clear_user_group_ids(request.spaces)
def set_user_id_to_report(report: Report, user_id: UserId) -> None:
report.userId = user_id
def set_user_id_to_subject(subject: SubjectWithReports, user_id: UserId) -> None:
subject.userId = user_id
ArrayHelper(subject.reports) \
.flatten() \
.filter(lambda x: x is not None) \
.each(lambda x: set_user_id_to_report(x, user_id))
def set_user_id_to_connected_space(connected_space: ConnectedSpaceWithSubjects, user_id: UserId) -> None:
connected_space.userId = user_id
ArrayHelper(connected_space.subjects) \
.flatten() \
.filter(lambda x: x is not None) \
.each(lambda x: set_user_id_to_subject(x, user_id))
if principal_service.is_super_admin():
# to find a tenant admin
tenant_admin: Optional[User] = user_service.find_admin(tenant_id)
if tenant_admin is None:
raise_400(f'Admin user on tenant[{tenant_id}] to receive imported data is not found.')
elif principal_service.is_tenant_admin():
ArrayHelper(request.connectedSpaces).each(
lambda x: set_user_id_to_connected_space(x, principal_service.get_user_id()))
else:
raise_403()
def validate_tenant_id( request: MixImportDataRequest, user_service: UserService, principal_service: PrincipalService) -> TenantId: if principal_service.is_super_admin(): return validate_tenant_id_when_super_admin(request, user_service, principal_service) elif principal_service.is_tenant_admin(): return validate_tenant_id_when_tenant_admin(request, principal_service) else: raise_403()
def fake_to_tenant(principal_service: PrincipalService,
tenant_id: TenantId) -> PrincipalService:
if principal_service.is_super_admin():
# fake principal as tenant admin
return PrincipalService(
User(userId=principal_service.get_user_id(),
tenantId=tenant_id,
name=principal_service.get_user_name(),
role=UserRole.ADMIN))
else:
return principal_service
def validate_tenant_id(tenant_id: Optional[TenantId],
principal_service: PrincipalService) -> TenantId:
if principal_service.is_tenant_admin():
if is_not_blank(
tenant_id) and tenant_id != principal_service.get_tenant_id():
raise_400('Tenant id is incorrect.')
return principal_service.get_tenant_id()
elif principal_service.is_super_admin():
if is_blank(tenant_id):
raise_400('Tenant id is required.')
return tenant_id
async def find_all_external_writers(
principal_service: PrincipalService = Depends(get_any_admin_principal)
) -> List[ExternalWriter]:
tenant_id = None
if principal_service.is_tenant_admin():
tenant_id = principal_service.get_tenant_id()
external_writer_service = get_external_writer_service(principal_service)
def action() -> List[ExternalWriter]:
return external_writer_service.find_all(tenant_id)
return trans_readonly(external_writer_service, action)
def do_load_inspection_by_id(
inspection_id: InspectionId, inspection_service: InspectionService,
principal_service: PrincipalService) -> Inspection:
# noinspection PyTypeChecker
inspection: Inspection = inspection_service.find_by_id(inspection_id)
if inspection is None:
raise_404()
# user id must match current principal's
if inspection.userId != principal_service.get_user_id():
raise_404()
# tenant id must match current principal's
if inspection.tenantId != principal_service.get_tenant_id():
raise_404()
return inspection
def filter_indicators( indicators: List[Indicator], indicator_service: IndicatorService, principal_service: PrincipalService ) -> List[Indicator]: if principal_service.is_admin(): return indicators user_id = principal_service.get_user_id() user_service = get_user_service(indicator_service) user: Optional[User] = user_service.find_by_id(user_id) if user is None: raise_403() group_ids = user.groupIds return ArrayHelper(indicators).filter(lambda x: ArrayHelper(x.groupIds).some(lambda y: y in group_ids)).to_list()
def find_topic(
self, topic_id: Optional[TopicId], available_schemas: List[TopicSchema],
principal_service: PrincipalService, allow_in_memory_variables: bool) -> Tuple[TopicSchema, bool]:
"""
find topic even it is not found in available list when in-memory variables is allowed.
for the secondary one of return tuple, true when it is found in given available list,
otherwise is false, which means it can be find in definitions
"""
if is_blank(topic_id):
raise DataKernelException(f'Topic not declared.')
schema = ArrayHelper(available_schemas).find(lambda x: x.get_topic().topicId == topic_id)
if schema is None:
if not allow_in_memory_variables:
raise DataKernelException(f'Topic[id={topic_id}] not found.')
else:
# in pipeline, it might be from trigger data
topic_service = get_topic_service(principal_service)
topic: Optional[Topic] = topic_service.find_by_id(topic_id)
if topic is None:
raise DataKernelException(f'Topic[id={topic_id}] not found.')
schema: Optional[TopicSchema] = topic_service.find_schema_by_name(
topic.name, principal_service.get_tenant_id())
if schema is None:
raise DataKernelException(f'Topic schema[id={topic_id}] not found.')
return schema, False
else:
return schema, True
def ask_topic(topic_id: TopicId, principal_service: PrincipalService) -> Topic:
topic = get_topic_service(principal_service).find_by_id(topic_id)
if topic is None:
raise IndicatorKernelException(f'Topic[id={topic_id}] not found.')
if topic.tenantId != principal_service.get_tenant_id():
raise IndicatorKernelException(f'Topic[id={topic_id}] not found.')
return topic
def run_topics_rules(
topic_name: Optional[str] = None,
frequency: Optional[MonitorRuleStatisticalInterval] = None,
process_date: Optional[str] = None,
tenant_id: Optional[TenantId] = None,
principal_service: PrincipalService = Depends(get_any_admin_principal)
) -> None:
principal_service = ask_principal_service(principal_service, tenant_id)
if is_not_blank(topic_name):
schema = get_topic_service(principal_service).find_schema_by_name(
topic_name, principal_service.get_tenant_id())
if schema is None:
raise_404(f'Topic[name={topic_name}] not found.')
topic_id = schema.get_topic().topicId
else:
topic_id = None
if is_not_blank(process_date):
parsed, parsed_date = is_date(process_date, ask_all_date_formats())
if not parsed:
raise_400(f'Given process date[{process_date}] cannot be parsed.')
process_date = parsed_date
else:
process_date = get_current_time_in_seconds()
process_date = truncate_time(process_date)
now = truncate_time(get_current_time_in_seconds())
if process_date.year > now.year:
raise_400(f'Given process date[{process_date}] cannot be in future.')
if process_date.year == now.year and process_date.month > now.month:
raise_400(f'Given process date[{process_date}] cannot be in future.')
if process_date.year == now.year and process_date.month == now.month and process_date.day > now.day:
raise_400(f'Given process date[{process_date}] cannot be in future.')
if frequency == MonitorRuleStatisticalInterval.MONTHLY:
# given process date is in this month, run previous month
# otherwise, run the given month
if process_date.year == now.year and process_date.month == now.month:
process_date = to_previous_month(process_date)
SelfCleaningMonitorRulesRunner(principal_service) \
.run(process_date, topic_id, MonitorRuleStatisticalInterval.MONTHLY)
elif frequency == MonitorRuleStatisticalInterval.WEEKLY:
# given process date is in this week, run previous week
# otherwise, run the given week
if process_date.year == now.year and int(
process_date.strftime('%U')) == int(now.strftime('%U')):
process_date = to_previous_week(process_date)
SelfCleaningMonitorRulesRunner(principal_service) \
.run(process_date, topic_id, MonitorRuleStatisticalInterval.WEEKLY)
elif frequency == MonitorRuleStatisticalInterval.DAILY:
# given process date is today, run yesterday
# otherwise, run the given day
if process_date.year == now.year and process_date.month == now.month and process_date.day == now.day:
process_date = to_yesterday(process_date)
SelfCleaningMonitorRulesRunner(principal_service) \
.run(process_date, topic_id, MonitorRuleStatisticalInterval.DAILY)
else:
raise_400(f'Given frequency[{frequency}] is not supported.')
def validate_tenant_id(a_tuple: Union[TenantBasedTuple, UserBasedTuple],
principal_service: PrincipalService) -> None:
"""
validate tenant id of tuple.\n
a. for super admin, tenant id is required,\
b. for not super admin, tenant id must be same as current principal,\n
c. for not super admin and no tenant id, set as current principal.
"""
tenant_id = a_tuple.tenantId
if principal_service.is_super_admin():
if is_blank(tenant_id):
raise_400('Tenant id is required.')
elif is_not_blank(tenant_id):
if tenant_id != principal_service.get_tenant_id():
raise_403()
else:
# assign tenant id by current principal
a_tuple.tenantId = principal_service.get_tenant_id()
def ask_super_admin() -> PrincipalService:
return PrincipalService(
User(userId=settings.SUPER_ADMIN_USER_ID,
name=settings.SUPER_ADMIN_USER_NAME,
nickname=settings.SUPER_ADMIN_USER_NICKNAME,
isActive=True,
groupIds=[],
tenantId=settings.SUPER_ADMIN_TENANT_ID,
role=UserRole.SUPER_ADMIN))
def find_topic_schema(name: str,
principal_service: PrincipalService) -> TopicSchema:
schema = get_topic_service(principal_service).find_schema_by_name(
name, principal_service.get_tenant_id())
if schema is None:
raise PipelineKernelException(
f'Topic schema[name={name}, tenant={principal_service.get_tenant_id()}] not found.'
)
return schema
def ask_subject(subject_id: SubjectId,
principal_service: PrincipalService) -> Subject:
subject = get_subject_service(principal_service).find_by_id(subject_id)
if subject is None:
raise IndicatorKernelException(f'Subject[id={subject_id}] not found.')
if subject.tenantId != principal_service.get_tenant_id():
raise IndicatorKernelException(f'Subject[id={subject_id}] not found.')
return subject
def assign_fix_columns_on_create(self, data: Dict[str, Any],
snowflake_generator: SnowflakeGenerator,
principal_service: PrincipalService,
now: datetime) -> None:
self.assign_id_column(data, snowflake_generator.next_id())
self.assign_tenant_id(data, principal_service.get_tenant_id())
self.assign_insert_time(data, now)
self.assign_update_time(data, now)
self.assign_version(data, 1)
async def find_all_topics(principal_service: PrincipalService = Depends(
get_console_principal)) -> List[Topic]:
tenant_id = principal_service.get_tenant_id()
topic_service = get_topic_service(principal_service)
def action() -> List[Topic]:
return topic_service.find_all(tenant_id)
return trans_readonly(topic_service, action)
def get_topic_schema(
topic_id: TopicId, principal_service: PrincipalService) -> TopicSchema:
topic_service = get_topic_service(principal_service)
topic = get_topic_service(principal_service).find_by_id(topic_id)
if topic is None:
raise DataKernelException(f'Topic[id={topic_id}] not found.')
schema = topic_service.find_schema_by_name(topic.name, principal_service.get_tenant_id())
if schema is None:
raise DataKernelException(f'Topic[name={topic.name}] not found.')
return schema
async def rerun_by_topic_data(
scheduler_id: Optional[TopicSnapshotSchedulerId],
process_date: Optional[str],
principal_service: PrincipalService = Depends(get_any_admin_principal)
) -> None:
if is_blank(scheduler_id):
raise_400('Scheduler id is required.')
if is_blank(process_date):
raise_400('Process date is required.')
parsed, parsed_process_date = is_date(process_date, ask_date_formats())
if not parsed:
raise_400('Process date must be date.')
scheduler_service = get_topic_snapshot_scheduler_service(principal_service)
scheduler_service.begin_transaction()
try:
scheduler: Optional[
TopicSnapshotScheduler] = scheduler_service.find_by_id(
scheduler_id)
if scheduler is None:
raise_404(f'Scheduler[id={scheduler_id}] not found.')
if principal_service.is_tenant_admin(
) and scheduler.tenantId != principal_service.get_tenant_id():
raise_404(f'Scheduler[id={scheduler_id}] not found.')
lock_service = get_lock_service(scheduler_service)
lock = lock_service.find_by_scheduler_and_process_date(
scheduler_id, scheduler.frequency, parsed_process_date)
if lock is not None:
raise_406(
f'Scheduler[id={scheduler_id}, processDate={process_date}] run already.'
)
except HTTPException as e:
raise e
except Exception as e:
raise_500(e)
finally:
scheduler_service.close_transaction()
run_job(scheduler_id, parsed_process_date)
def validate_tenant_based_tuples(tuples: List[TenantBasedTuple],
user_service: UserService,
principal_service: PrincipalService) -> None:
"""
check tenant of tuple is valid or not.
"""
if not principal_service.is_admin():
raise_403()
ArrayHelper(tuples).each(
lambda x: validate_tenant(x, user_service, principal_service))
async def load_tenant_by_id(
tenant_id: Optional[TenantId] = None,
principal_service: PrincipalService = Depends(get_any_principal)
) -> Tenant:
if is_blank(tenant_id):
raise_400('Tenant id is required.')
if not principal_service.is_super_admin():
if tenant_id != principal_service.get_tenant_id():
raise_403()
tenant_service = get_tenant_service(principal_service)
def action() -> Tenant:
# noinspection PyTypeChecker
tenant: Tenant = tenant_service.find_by_id(tenant_id)
if tenant is None:
raise_404()
return tenant
return trans_readonly(tenant_service, action)
def find_topic_schema(topic_id: TopicId,
principal_service: PrincipalService) -> TopicSchema:
topic_service = get_topic_service(principal_service)
topic: Optional[Topic] = topic_service.find_by_id(topic_id)
if topic is None:
raise PipelineKernelException(f'Topic[id={topic_id}] not found.')
schema = topic_service.find_schema_by_name(
topic.name, principal_service.get_tenant_id())
if schema is None:
raise PipelineKernelException(
f'Topic schema[id={topic_id}] not found.')
return schema
async def load_data_source_by_id(
data_source_id: Optional[DataSourceId] = None,
principal_service: PrincipalService = Depends(get_any_admin_principal)
) -> DataSource:
if is_blank(data_source_id):
raise_400('Data source id is required.')
if not principal_service.is_super_admin():
if data_source_id != principal_service.get_tenant_id():
raise_403()
data_source_service = get_data_source_service(principal_service)
def action() -> DataSource:
# noinspection PyTypeChecker
data_source: DataSource = data_source_service.find_by_id(
data_source_id)
if data_source is None:
raise_404()
return data_source
return trans_readonly(data_source_service, action)
async def load_external_writer_by_id(
writer_id: Optional[ExternalWriterId] = None,
principal_service: PrincipalService = Depends(get_any_admin_principal)
) -> ExternalWriter:
if is_blank(writer_id):
raise_400('External writer id is required.')
if not principal_service.is_super_admin():
if writer_id != principal_service.get_tenant_id():
raise_403()
external_writer_service = get_external_writer_service(principal_service)
def action() -> ExternalWriter:
# noinspection PyTypeChecker
external_writer: ExternalWriter = external_writer_service.find_by_id(
writer_id)
if external_writer is None:
raise_404()
return external_writer
return trans_readonly(external_writer_service, action)
