def update_name(self, dashboard_id: DashboardId, name: str, user_id: UserId, tenant_id: TenantId) -> datetime: """ update name will not increase optimistic lock version """ last_modified_at = self.now() last_modified_by = self.principalService.get_user_id() updated_count = self.storage.update_only( self.get_entity_updater(criteria=[ EntityCriteriaExpression(left=ColumnNameLiteral( columnName=self.get_storable_id_column_name()), right=dashboard_id), EntityCriteriaExpression( left=ColumnNameLiteral(columnName='user_id'), right=user_id), EntityCriteriaExpression( left=ColumnNameLiteral(columnName='tenant_id'), right=tenant_id) ], update={ 'name': name, 'last_modified_at': last_modified_at, 'last_modified_by': last_modified_by })) if updated_count == 0: raise TupleNotFoundException( 'Update 0 row might be caused by tuple not found.') return last_modified_at
def find_last(self, data_id: int, topic_id: TopicId, tenant_id: TenantId) -> Optional[PipelineMonitorLog]: schema = self.get_topic_schema() storage = self.ask_storages().ask_topic_storage(schema) data_service = ask_topic_data_service(schema, storage, self.principalService) # noinspection SpellCheckingInspection page = data_service.page( data_service.get_data_entity_helper().get_entity_pager( criteria=[ EntityCriteriaExpression(left=ColumnNameLiteral( columnName=TopicDataColumnNames.TENANT_ID.value), right=tenant_id), EntityCriteriaExpression( left=ColumnNameLiteral(columnName='topicid'), right=topic_id), EntityCriteriaExpression( left=ColumnNameLiteral(columnName='dataid'), right=data_id) ], sort=[ EntitySortColumn( name=TopicDataColumnNames.INSERT_TIME.value, method=EntitySortMethod.DESC, ) ], pageable=Pageable(pageNumber=1, pageSize=1))) if page.itemCount == 0: return None else: return PipelineMonitorLog(**page.data[0])
def find_numeric_value_by_text( self, text: Optional[str], tenant_id: Optional[TenantId]) -> List[Bucket]: criteria = [ EntityCriteriaExpression(left=ColumnNameLiteral(columnName='type'), right=BucketType.VALUE) ] if text is not None and len(text.strip()) != 0: criteria.append( EntityCriteriaJoint( conjunction=EntityCriteriaJointConjunction.OR, children=[ EntityCriteriaExpression( left=ColumnNameLiteral(columnName='name'), operator=EntityCriteriaOperator.LIKE, right=text), EntityCriteriaExpression( left=ColumnNameLiteral(columnName='description'), operator=EntityCriteriaOperator.LIKE, right=text) ])) if tenant_id is not None and len(tenant_id.strip()) != 0: criteria.append( EntityCriteriaExpression( left=ColumnNameLiteral(columnName='tenant_id'), right=tenant_id)) # noinspection PyTypeChecker return self.storage.find(self.get_entity_finder(criteria=criteria))
def factor_string_length_not_in_range(data_service: TopicDataService, rule: MonitorRule, date_range: Tuple[datetime, datetime], changed_rows_count_in_range: int, total_rows_count: int) -> RuleResult: found, factor = find_factor(data_service, rule.factorId, rule) if not found: return RuleResult.IGNORED count = data_service.count_by_criteria([ EntityCriteriaJoint( conjunction=EntityCriteriaJointConjunction.OR, children=[ EntityCriteriaExpression( left=ComputedLiteral( operator=ComputedLiteralOperator.CHAR_LENGTH, elements=[ build_column_name_literal(factor, data_service) ]), operator=EntityCriteriaOperator.LESS_THAN, right=rule.params.min), EntityCriteriaExpression( left=ComputedLiteral( operator=ComputedLiteralOperator.CHAR_LENGTH, elements=[ build_column_name_literal(factor, data_service) ]), operator=EntityCriteriaOperator.GREATER_THAN, right=rule.params.max) ]), *build_date_range_criteria(date_range) ]) return RuleResult.SUCCESS if count == 0 else RuleResult.FAILED
def find(self, name: str, tenant_id: TenantId) -> List[FactorIndex]: if not ask_engine_index_enabled(): return [] # noinspection PyTypeChecker return self.storage.find( EntityFinder( name=FACTOR_INDEX_ENTITY_NAME, shaper=FACTOR_INDEX_ENTITY_SHAPER, criteria=[ EntityCriteriaExpression( left=ColumnNameLiteral(columnName='tenant_id'), right=tenant_id), EntityCriteriaJoint( conjunction=EntityCriteriaJointConjunction.AND, children=[ EntityCriteriaExpression( left=ColumnNameLiteral( columnName='factor_name'), operator=EntityCriteriaOperator.LIKE, right=name), EntityCriteriaExpression( left=ColumnNameLiteral( columnName='topic_name'), operator=EntityCriteriaOperator.LIKE, right=name) ]) ]))
def find_by_ids(self, graphic_ids: List[PipelineGraphicId], user_id: Optional[UserId], tenant_id: Optional[TenantId]) -> List[PipelineGraphic]: criteria = [ EntityCriteriaExpression( left=ColumnNameLiteral(columnName='pipeline_graphic_id'), operator=EntityCriteriaOperator.IN, right=graphic_ids) ] if is_not_blank(user_id): criteria.append( EntityCriteriaExpression( left=ColumnNameLiteral(columnName='user_id'), right=user_id)) if is_not_blank(tenant_id): criteria.append( EntityCriteriaExpression( left=ColumnNameLiteral(columnName='tenant_id'), right=tenant_id)) # noinspection PyTypeChecker return self.storage.find_distinct_values( self.get_entity_finder_for_columns(distinctColumnNames=[ 'pipeline_graphic_id' ], criteria=criteria))
def find_page_by_text(self, text: Optional[str], tenant_id: Optional[TenantId], pageable: Pageable) -> DataPage: criteria = [] if text is not None and len(text.strip()) != 0: criteria.append( EntityCriteriaJoint( conjunction=EntityCriteriaJointConjunction.OR, children=[ EntityCriteriaExpression( left=ColumnNameLiteral(columnName='name'), operator=EntityCriteriaOperator.LIKE, right=text), EntityCriteriaExpression( left=ColumnNameLiteral(columnName='description'), operator=EntityCriteriaOperator.LIKE, right=text) ])) if tenant_id is not None and len(tenant_id.strip()) != 0: criteria.append( EntityCriteriaExpression( left=ColumnNameLiteral(columnName='tenant_id'), right=tenant_id)) return self.storage.page( self.get_entity_pager(criteria=criteria, pageable=pageable))
def find_all_by_user_id(self, user_id: UserId, tenant_id: TenantId) -> List[UserBasedTuple]: # noinspection PyTypeChecker return self.storage.find(self.get_entity_finder( criteria=[ EntityCriteriaExpression(left=ColumnNameLiteral(columnName='user_id'), right=user_id), EntityCriteriaExpression(left=ColumnNameLiteral(columnName='tenant_id'), right=tenant_id) ] ))
def out_of_range(factor: Factor, data_service: TopicDataService, min_value: Any, max_value: Any) -> EntityCriteriaJoint: left = build_column_name_literal(factor, data_service) return EntityCriteriaJoint( conjunction=EntityCriteriaJointConjunction.OR, children=[ EntityCriteriaExpression(left=left, operator=EntityCriteriaOperator.LESS_THAN, right=min_value), EntityCriteriaExpression(left=left, operator=EntityCriteriaOperator.GREATER_THAN, right=max_value) ] )
def find_by_topic_id(self, topic_id: TopicId, tenant_id: TenantId) -> List[Pipeline]: finder = self.get_entity_finder_for_columns( criteria=[ EntityCriteriaExpression(left=ColumnNameLiteral(columnName='topic_id'), right=topic_id), EntityCriteriaExpression(left=ColumnNameLiteral(columnName='tenant_id'), right=tenant_id), ], distinctColumnNames=['tenant_id'] ) # noinspection PyTypeChecker return self.storage.find(finder)
def find_by_type(self, key_type: str, tenant_id: TenantId) -> Optional[KeyStore]: return self.storage.find_one(EntityFinder( name=self.get_entity_name(), shaper=self.get_entity_shaper(), criteria=[ EntityCriteriaExpression( left=ColumnNameLiteral(columnName='key_type'), right=key_type), EntityCriteriaExpression(left=ColumnNameLiteral(columnName='tenant_id'), right=tenant_id) ] ))
def find_by_name_and_tenant(self, name: str, tenant_id: TenantId) -> Optional[Topic]: return self.storage.find_one( self.get_entity_finder(criteria=[ EntityCriteriaExpression( left=ColumnNameLiteral(columnName='name'), right=name), EntityCriteriaExpression(left=ColumnNameLiteral( columnName='tenant_id'), right=tenant_id) ]))
def find(self, topic_id: TopicId, start_time: datetime, end_time: datetime) -> Optional[TopicProfile]: schema = get_topic_schema(topic_id, self.principalService) if is_raw_topic(schema.get_topic()): raise DqcException(f'Raw topic[name={schema.get_topic().name}] is not supported for profiling.') storage = ask_topic_storage(schema, self.principalService) service = ask_topic_data_service(schema, storage, self.principalService) criteria = [ EntityCriteriaExpression( left=ColumnNameLiteral(columnName=TopicDataColumnNames.TENANT_ID.value), right=self.principalService.get_tenant_id()), EntityCriteriaExpression( left=ColumnNameLiteral(columnName=TopicDataColumnNames.UPDATE_TIME.value), operator=EntityCriteriaOperator.GREATER_THAN_OR_EQUALS, right=start_time), EntityCriteriaExpression( left=ColumnNameLiteral(columnName=TopicDataColumnNames.UPDATE_TIME.value), operator=EntityCriteriaOperator.LESS_THAN_OR_EQUALS, right=end_time) ] data = service.find(criteria) columns = [ TopicDataColumnNames.ID.value, *ArrayHelper(schema.get_topic().factors).map(lambda x: x.name).to_list(), TopicDataColumnNames.TENANT_ID.value, TopicDataColumnNames.INSERT_TIME.value, TopicDataColumnNames.UPDATE_TIME.value ] def row_to_list(row: Dict[str, Any]) -> List[Any]: return ArrayHelper(columns).map(lambda x: row.get(x)).to_list() data_frame = build_data_frame(ArrayHelper(data).map(row_to_list).to_list(), columns) data_frame = convert_data_frame_type_by_topic(data_frame, schema.get_topic()) data_frame.drop([ TopicDataColumnNames.TENANT_ID, TopicDataColumnNames.UPDATE_TIME, TopicDataColumnNames.INSERT_TIME, TopicDataColumnNames.AGGREGATE_ASSIST, TopicDataColumnNames.ID, TopicDataColumnNames.VERSION ], axis=1, inplace=True, errors='ignore') if data_frame.empty or len(data_frame.index) == 1: return None else: logger.info(f'memory_usage {data_frame.memory_usage(deep=True).sum()} bytes') profile = ProfileReport(data_frame, title=f'{schema.get_topic().name} data profile report', minimal=True) json_data = profile.to_json() json_constants_map = { '-Infinity': float('-Infinity'), 'Infinity': float('Infinity'), 'NaN': None, } return loads(json_data, parse_constant=lambda x: json_constants_map[x])
def find_admin(self, tenant_id: TenantId) -> Optional[User]: criteria = [ EntityCriteriaExpression(left=ColumnNameLiteral(columnName='role'), right=UserRole.ADMIN), EntityCriteriaExpression(left=ColumnNameLiteral(columnName='tenant_id'), right=tenant_id) ] # noinspection PyTypeChecker admins: List[User] = self.storage.find(self.get_entity_finder(criteria=criteria)) if len(admins) == 0: return None else: return admins[0]
def find_templates_by_tenant_id( self, tenant_id: TenantId) -> List[ConnectedSpace]: # noinspection PyTypeChecker return self.storage.find( self.get_entity_finder(criteria=[ EntityCriteriaExpression(left=ColumnNameLiteral( columnName='tenant_id'), right=tenant_id), EntityCriteriaExpression(left=ColumnNameLiteral( columnName='is_template'), right=True), ]))
def find_modified_after(self, last_modified_at: datetime, tenant_id: Optional[TenantId]) -> List[Pipeline]: criteria = [ EntityCriteriaExpression( left=ColumnNameLiteral(columnName='last_modified_at'), operator=EntityCriteriaOperator.GREATER_THAN_OR_EQUALS, right=last_modified_at ) ] if is_not_blank(tenant_id): criteria.append(EntityCriteriaExpression(left=ColumnNameLiteral(columnName='tenant_id'), right=tenant_id)) # noinspection PyTypeChecker return self.storage.find(self.get_entity_finder(criteria))
def find_by_name(self, name: str, tenant_id: Optional[TenantId]) -> Optional[Enum]: criteria = [ EntityCriteriaExpression(left=ColumnNameLiteral(columnName='name'), right=name) ] if tenant_id is not None and len(tenant_id.strip()) != 0: criteria.append( EntityCriteriaExpression( left=ColumnNameLiteral(columnName='tenant_id'), right=tenant_id)) return self.storage.find_one(self.get_entity_finder(criteria))
def update(self, a_tuple: Tuple) -> Tuple: """ with optimistic lock logic """ original_last_modified_at, original_last_modified_by = self.try_to_prepare_auditable_on_update( a_tuple) is_optimistic_lock, version = self.try_to_prepare_optimistic_lock_on_update( a_tuple) if is_optimistic_lock: # noinspection PyTypeChecker updated_count = self.storage.update_only( self.get_entity_updater( criteria=[ EntityCriteriaExpression( left=ColumnNameLiteral( columnName=self.get_storable_id_column_name()), right=self.get_storable_id(a_tuple)), EntityCriteriaExpression(left=ColumnNameLiteral( columnName=self.get_optimistic_column_name()), right=version) ], # to avoid update created columns in update update=self.try_to_ignore_created_columns( self.ignore_storable_id( self.get_entity_shaper().serialize(a_tuple))))) if updated_count == 0: a_tuple.version = version self.try_to_recover_auditable_on_update( a_tuple, original_last_modified_at, original_last_modified_by) raise OptimisticLockException( 'Update 0 row might be caused by optimistic lock.') else: updated_count = self.storage.update_only( self.get_entity_updater( criteria=[ EntityCriteriaExpression( left=ColumnNameLiteral( columnName=self.get_storable_id_column_name()), right=self.get_storable_id(a_tuple)) ], # to avoid update created columns in update update=self.try_to_ignore_created_columns( self.ignore_optimistic_keys( self.get_entity_shaper().serialize(a_tuple))))) if updated_count == 0: self.try_to_recover_auditable_on_update( a_tuple, original_last_modified_at, original_last_modified_by) raise TupleNotFoundException( 'Update 0 row might be caused by tuple not found.') return a_tuple
def find_by_user_id(self, user_id: UserId, tenant_id: TenantId) -> Optional[LastSnapshot]: return self.storage.find_one( EntityFinder(name=self.get_entity_name(), shaper=self.get_entity_shaper(), criteria=[ EntityCriteriaExpression( left=ColumnNameLiteral(columnName='user_id'), right=user_id), EntityCriteriaExpression(left=ColumnNameLiteral( columnName='tenant_id'), right=tenant_id) ]))
def find_by_ids(self, user_ids: List[UserId], tenant_id: Optional[TenantId]) -> List[User]: # always ignore super admin criteria = [ EntityCriteriaExpression( left=ColumnNameLiteral(columnName='role'), operator=EntityCriteriaOperator.NOT_EQUALS, right=UserRole.SUPER_ADMIN), EntityCriteriaExpression( left=ColumnNameLiteral(columnName='user_id'), operator=EntityCriteriaOperator.IN, right=user_ids) ] if tenant_id is not None and len(tenant_id.strip()) != 0: criteria.append(EntityCriteriaExpression(left=ColumnNameLiteral(columnName='tenant_id'), right=tenant_id)) # noinspection PyTypeChecker return self.storage.find(self.get_entity_finder(criteria))
def build_date_range_criteria(date_range: Tuple[datetime, datetime]) -> EntityCriteria: return [ EntityCriteriaExpression( left=ColumnNameLiteral(columnName=TopicDataColumnNames.UPDATE_TIME.value), operator=EntityCriteriaOperator.GREATER_THAN_OR_EQUALS, right=date_range[0] ), EntityCriteriaExpression( left=ColumnNameLiteral(columnName=TopicDataColumnNames.UPDATE_TIME.value), operator=EntityCriteriaOperator.LESS_THAN_OR_EQUALS, right=date_range[1] ) ]
def find_by_user_id(self, user_id: UserId, tenant_id: TenantId) -> List[PersonalAccessToken]: # noinspection PyTypeChecker return self.storage.find( EntityFinder(name=self.get_entity_name(), shaper=self.get_entity_shaper(), criteria=[ EntityCriteriaExpression( left=ColumnNameLiteral(columnName='user_id'), right=user_id), EntityCriteriaExpression(left=ColumnNameLiteral( columnName='tenant_id'), right=tenant_id) ]))
def build_index(self, topic: Topic) -> None: if not ask_engine_index_enabled(): return # noinspection PyTypeChecker factor_index_list: List[FactorIndex] = self.storage.find( EntityFinder(name=FACTOR_INDEX_ENTITY_NAME, shaper=FACTOR_INDEX_ENTITY_SHAPER, criteria=[ EntityCriteriaExpression( left=ColumnNameLiteral(columnName='topic_id'), right=topic.topicId) ])) index_map: Dict[FactorId, FactorIndex] = ArrayHelper(factor_index_list) \ .to_map(lambda x: x.factorId, lambda x: x) now = get_current_time_in_seconds() current_index_list = ArrayHelper(topic.factors).map( lambda x: self.build_factor_index(x, topic, now)).to_list() entity_id_helper = EntityIdHelper(name=FACTOR_INDEX_ENTITY_NAME, shaper=FACTOR_INDEX_ENTITY_SHAPER, idColumnName='factor_index_id') for factor_index in current_index_list: if factor_index.factorId in index_map: old_factor_index = index_map[factor_index.factorId] factor_index.factorIndexId = old_factor_index.factorIndexId factor_index.createdAt = old_factor_index.createdAt self.storage.update_one(factor_index, entity_id_helper) else: self.storage.insert_one(factor_index, entity_id_helper) current_factor_ids = ArrayHelper(current_index_list).map( lambda x: x.factorId).to_list() to_remove_list: List[FactorIndex] = ArrayHelper(list(index_map.values())) \ .filter(lambda x: x.factorId not in current_factor_ids).to_list() if len(to_remove_list) != 0: self.storage.delete( EntityDeleter( name=FACTOR_INDEX_ENTITY_NAME, shaper=FACTOR_INDEX_ENTITY_SHAPER, criteria=[ EntityCriteriaExpression( left=ColumnNameLiteral(columnName='topic_id'), right=topic.topicId), EntityCriteriaExpression( left=ColumnNameLiteral( columnName='factor_index_id'), operator=EntityCriteriaOperator.IN, right=ArrayHelper(to_remove_list).map( lambda x: x.factorIndexId).to_list()) ]))
def find_by_ids(self, topic_ids: List[TopicId], tenant_id: Optional[TenantId]) -> List[Topic]: criteria = [ EntityCriteriaExpression( left=ColumnNameLiteral(columnName='topic_id'), operator=EntityCriteriaOperator.IN, right=topic_ids) ] if tenant_id is not None and len(tenant_id.strip()) != 0: criteria.append( EntityCriteriaExpression( left=ColumnNameLiteral(columnName='tenant_id'), right=tenant_id)) # noinspection PyTypeChecker return self.storage.find(self.get_entity_finder(criteria))
def delete_others_by_ids(self, rule_ids: List[MonitorRuleId], tenant_id: TenantId) -> List[MonitorRule]: criteria = [ EntityCriteriaExpression( left=ColumnNameLiteral(columnName='tenant_id'), right=tenant_id), EntityCriteriaExpression( left=ColumnNameLiteral(columnName='rule_id'), operator=EntityCriteriaOperator.NOT_IN, right=rule_ids) ] # noinspection PyTypeChecker return self.storage.delete_and_pull( self.get_entity_deleter(criteria=criteria))
def delete_by_enum_id(self, enum_id: EnumId) -> None: self.storage.delete( self.get_entity_deleter([ EntityCriteriaExpression( left=ColumnNameLiteral(columnName='enum_id'), right=enum_id) ]))
def find_by_id(self, tuple_id: TupleId) -> Optional[UserBasedTuple]: return self.storage.find_one(self.get_entity_finder( criteria=[ EntityCriteriaExpression( left=ColumnNameLiteral(columnName=self.get_storable_id_column_name()), right=tuple_id), ] ))
def find_by_name(self, text: Optional[str], tenant_id: Optional[TenantId]) -> List[Space]: criteria = [] if is_not_blank(text): criteria.append( EntityCriteriaExpression( left=ColumnNameLiteral(columnName='name'), operator=EntityCriteriaOperator.LIKE, right=text.strip())) if tenant_id is not None and len(tenant_id.strip()) != 0: criteria.append( EntityCriteriaExpression( left=ColumnNameLiteral(columnName='tenant_id'), right=tenant_id)) # noinspection PyTypeChecker return self.storage.find(self.get_entity_finder(criteria=criteria))
def run(self, variables: PipelineVariables, principal_service: PrincipalService) -> EntityCriteriaExpression: left_value = self.left.run(variables, principal_service) if self.operator == ParameterExpressionOperator.EMPTY: return EntityCriteriaExpression(left=left_value, operator=EntityCriteriaOperator.IS_EMPTY) elif self.operator == ParameterExpressionOperator.NOT_EMPTY: return EntityCriteriaExpression(left=left_value, operator=EntityCriteriaOperator.IS_NOT_EMPTY) right_value = self.right.run(variables, principal_service) if self.operator == ParameterExpressionOperator.EQUALS: left_value = self.handle_equation_possible_types(left_value, self.right.get_possible_types()) right_value = self.handle_equation_possible_types(right_value, self.left.get_possible_types()) return EntityCriteriaExpression( left=left_value, operator=EntityCriteriaOperator.EQUALS, right=right_value) elif self.operator == ParameterExpressionOperator.NOT_EQUALS: left_value = self.handle_equation_possible_types(left_value, self.right.get_possible_types()) right_value = self.handle_equation_possible_types(right_value, self.left.get_possible_types()) return EntityCriteriaExpression( left=left_value, operator=EntityCriteriaOperator.NOT_EQUALS, right=right_value) elif self.operator == ParameterExpressionOperator.LESS: left_value = self.handle_comparison_possible_types(left_value, self.right.get_possible_types()) right_value = self.handle_comparison_possible_types(right_value, self.left.get_possible_types()) return EntityCriteriaExpression( left=left_value, operator=EntityCriteriaOperator.LESS_THAN, right=right_value) elif self.operator == ParameterExpressionOperator.LESS_EQUALS: left_value = self.handle_comparison_possible_types(left_value, self.right.get_possible_types()) right_value = self.handle_comparison_possible_types(right_value, self.left.get_possible_types()) return EntityCriteriaExpression( left=left_value, operator=EntityCriteriaOperator.LESS_THAN_OR_EQUALS, right=right_value) elif self.operator == ParameterExpressionOperator.MORE: left_value = self.handle_comparison_possible_types(left_value, self.right.get_possible_types()) right_value = self.handle_comparison_possible_types(right_value, self.left.get_possible_types()) return EntityCriteriaExpression( left=left_value, operator=EntityCriteriaOperator.GREATER_THAN, right=right_value) elif self.operator == ParameterExpressionOperator.MORE_EQUALS: left_value = self.handle_comparison_possible_types(left_value, self.right.get_possible_types()) right_value = self.handle_comparison_possible_types(right_value, self.left.get_possible_types()) return EntityCriteriaExpression( left=left_value, operator=EntityCriteriaOperator.GREATER_THAN_OR_EQUALS, right=right_value) elif self.operator == ParameterExpressionOperator.IN: return EntityCriteriaExpression( left=left_value, operator=EntityCriteriaOperator.IN, right=right_value) elif self.operator == ParameterExpressionOperator.NOT_IN: return EntityCriteriaExpression( left=left_value, operator=EntityCriteriaOperator.NOT_IN, right=right_value) else: raise DataKernelException( f'Operator[{self.operator}] is not supported, found from expression[{self.condition.dict()}].')
def find_by_enum_id(self, enum_id: EnumId) -> List[EnumItem]: # noinspection PyTypeChecker return self.storage.find( self.get_entity_finder([ EntityCriteriaExpression( left=ColumnNameLiteral(columnName='enum_id'), right=enum_id) ]))