def find_by_measure_method(self,
measure_methods: List[Tuple[MeasureMethod,
Optional[EnumId]]],
tenant_id: Optional[TenantId]) -> List[Bucket]:
criteria = []
non_enum_measure_methods = ArrayHelper(measure_methods) \
.filter(lambda x: x[0] != MeasureMethod.ENUM and x[1] is None).map(lambda x: x[0]).to_list()
enum_measure_methods = ArrayHelper(measure_methods) \
.filter(lambda x: x[0] == MeasureMethod.ENUM and x[1] is not None).map(lambda x: x[1]).to_list()
if len(non_enum_measure_methods) != 0 and len(
enum_measure_methods) != 0:
criteria.append(
EntityCriteriaJoint(
conjunction=EntityCriteriaJointConjunction.OR,
children=[
EntityCriteriaExpression(
left=ColumnNameLiteral(columnName='measure'),
operator=EntityCriteriaOperator.IN,
right=non_enum_measure_methods),
EntityCriteriaJoint(
conjunction=EntityCriteriaJointConjunction.AND,
children=[
EntityCriteriaExpression(
left=ColumnNameLiteral(
columnName='measure'),
right=MeasureMethod.ENUM),
EntityCriteriaExpression(
left=ColumnNameLiteral(
columnName='enum_id'),
operator=EntityCriteriaOperator.IN,
right=enum_measure_methods)
])
]))
elif len(non_enum_measure_methods) != 0:
criteria.append(
EntityCriteriaExpression(
left=ColumnNameLiteral(columnName='measure'),
operator=EntityCriteriaOperator.IN,
right=non_enum_measure_methods))
elif len(enum_measure_methods) != 0:
criteria.append(
EntityCriteriaJoint(
conjunction=EntityCriteriaJointConjunction.AND,
children=[
EntityCriteriaExpression(
left=ColumnNameLiteral(columnName='measure'),
right=MeasureMethod.ENUM),
EntityCriteriaExpression(
left=ColumnNameLiteral(columnName='enum_id'),
operator=EntityCriteriaOperator.IN,
right=enum_measure_methods)
]))
if tenant_id is not None and len(tenant_id.strip()) != 0:
criteria.append(
EntityCriteriaExpression(
left=ColumnNameLiteral(columnName='tenant_id'),
right=tenant_id))
# noinspection PyTypeChecker
return self.storage.find(self.get_entity_finder(criteria=criteria))
def run(self, variables: PipelineVariables, principal_service: PrincipalService) -> EntityCriteriaJoint: if self.jointType == ParameterJointType.OR: return EntityCriteriaJoint( conjunction=EntityCriteriaJointConjunction.OR, children=ArrayHelper(self.filters).map(lambda x: x.run(variables, principal_service)).to_list() ) else: # and or not given return EntityCriteriaJoint( conjunction=EntityCriteriaJointConjunction.AND, children=ArrayHelper(self.filters).map(lambda x: x.run(variables, principal_service)).to_list() )
def find_page_by_text(self, text: Optional[str],
tenant_id: Optional[TenantId],
pageable: Pageable) -> DataPage:
criteria = []
if text is not None and len(text.strip()) != 0:
criteria.append(
EntityCriteriaJoint(
conjunction=EntityCriteriaJointConjunction.OR,
children=[
EntityCriteriaExpression(
left=ColumnNameLiteral(columnName='name'),
operator=EntityCriteriaOperator.LIKE,
right=text),
EntityCriteriaExpression(
left=ColumnNameLiteral(columnName='description'),
operator=EntityCriteriaOperator.LIKE,
right=text)
]))
if tenant_id is not None and len(tenant_id.strip()) != 0:
criteria.append(
EntityCriteriaExpression(
left=ColumnNameLiteral(columnName='tenant_id'),
right=tenant_id))
return self.storage.page(
self.get_entity_pager(criteria=criteria, pageable=pageable))
def find(self, name: str, tenant_id: TenantId) -> List[FactorIndex]:
if not ask_engine_index_enabled():
return []
# noinspection PyTypeChecker
return self.storage.find(
EntityFinder(
name=FACTOR_INDEX_ENTITY_NAME,
shaper=FACTOR_INDEX_ENTITY_SHAPER,
criteria=[
EntityCriteriaExpression(
left=ColumnNameLiteral(columnName='tenant_id'),
right=tenant_id),
EntityCriteriaJoint(
conjunction=EntityCriteriaJointConjunction.AND,
children=[
EntityCriteriaExpression(
left=ColumnNameLiteral(
columnName='factor_name'),
operator=EntityCriteriaOperator.LIKE,
right=name),
EntityCriteriaExpression(
left=ColumnNameLiteral(
columnName='topic_name'),
operator=EntityCriteriaOperator.LIKE,
right=name)
])
]))
def factor_string_length_not_in_range(data_service: TopicDataService,
rule: MonitorRule,
date_range: Tuple[datetime, datetime],
changed_rows_count_in_range: int,
total_rows_count: int) -> RuleResult:
found, factor = find_factor(data_service, rule.factorId, rule)
if not found:
return RuleResult.IGNORED
count = data_service.count_by_criteria([
EntityCriteriaJoint(
conjunction=EntityCriteriaJointConjunction.OR,
children=[
EntityCriteriaExpression(
left=ComputedLiteral(
operator=ComputedLiteralOperator.CHAR_LENGTH,
elements=[
build_column_name_literal(factor, data_service)
]),
operator=EntityCriteriaOperator.LESS_THAN,
right=rule.params.min),
EntityCriteriaExpression(
left=ComputedLiteral(
operator=ComputedLiteralOperator.CHAR_LENGTH,
elements=[
build_column_name_literal(factor, data_service)
]),
operator=EntityCriteriaOperator.GREATER_THAN,
right=rule.params.max)
]), *build_date_range_criteria(date_range)
])
return RuleResult.SUCCESS if count == 0 else RuleResult.FAILED
def find_numeric_value_by_text(
self, text: Optional[str],
tenant_id: Optional[TenantId]) -> List[Bucket]:
criteria = [
EntityCriteriaExpression(left=ColumnNameLiteral(columnName='type'),
right=BucketType.VALUE)
]
if text is not None and len(text.strip()) != 0:
criteria.append(
EntityCriteriaJoint(
conjunction=EntityCriteriaJointConjunction.OR,
children=[
EntityCriteriaExpression(
left=ColumnNameLiteral(columnName='name'),
operator=EntityCriteriaOperator.LIKE,
right=text),
EntityCriteriaExpression(
left=ColumnNameLiteral(columnName='description'),
operator=EntityCriteriaOperator.LIKE,
right=text)
]))
if tenant_id is not None and len(tenant_id.strip()) != 0:
criteria.append(
EntityCriteriaExpression(
left=ColumnNameLiteral(columnName='tenant_id'),
right=tenant_id))
# noinspection PyTypeChecker
return self.storage.find(self.get_entity_finder(criteria=criteria))
def build_criteria(tables: List[Table], criteria: EntityCriteria): if criteria is None or len(criteria) == 0: return None if len(criteria) == 1: return build_criteria_statement(tables, criteria[0]) else: return build_criteria_statement(tables, EntityCriteriaJoint(children=criteria))
def build_criteria(self, criteria: EntityCriteria):
if criteria is None or len(criteria) == 0:
return None
if len(criteria) == 1:
return self.build_criteria_statement(criteria[0])
else:
return self.build_criteria_statement(
EntityCriteriaJoint(children=criteria))
def out_of_range(factor: Factor, data_service: TopicDataService, min_value: Any, max_value: Any) -> EntityCriteriaJoint: left = build_column_name_literal(factor, data_service) return EntityCriteriaJoint( conjunction=EntityCriteriaJointConjunction.OR, children=[ EntityCriteriaExpression(left=left, operator=EntityCriteriaOperator.LESS_THAN, right=min_value), EntityCriteriaExpression(left=left, operator=EntityCriteriaOperator.GREATER_THAN, right=max_value) ] )
def find_page_by_text(self, text: Optional[str], tenant_id: Optional[TenantId], pageable: Pageable) -> DataPage: # always ignore super admin criteria = [ EntityCriteriaExpression( left=ColumnNameLiteral(columnName='role'), operator=EntityCriteriaOperator.NOT_EQUALS, right=UserRole.SUPER_ADMIN) ] if text is not None and len(text.strip()) != 0: criteria.append(EntityCriteriaJoint( conjunction=EntityCriteriaJointConjunction.OR, children=[ EntityCriteriaExpression( left=ColumnNameLiteral(columnName='name'), operator=EntityCriteriaOperator.LIKE, right=text), EntityCriteriaExpression( left=ColumnNameLiteral(columnName='nickname'), operator=EntityCriteriaOperator.LIKE, right=text) ] )) if tenant_id is not None and len(tenant_id.strip()) != 0: criteria.append(EntityCriteriaExpression(left=ColumnNameLiteral(columnName='tenant_id'), right=tenant_id)) return self.storage.page(self.get_entity_pager(criteria=criteria, pageable=pageable))
def run_task(lock: TopicSnapshotJobLock,
scheduler: TopicSnapshotScheduler,
principal_service: PrincipalService,
after_sleeping: bool = False) -> None:
# scan task topic to fetch task and trigger pipeline to write to target topic
# until there is no data with status ready
# or no task at all and task lock status is success/failed
task_topic_schema, task_topic_service, pipline = get_task_topic_data_service(
scheduler, principal_service)
page = task_topic_service.page_and_unwrap([
EntityCriteriaJoint(
conjunction=EntityCriteriaJointConjunction.AND,
children=[
EntityCriteriaExpression(
left=ColumnNameLiteral(columnName='jobid'),
right=lock.lockId),
EntityCriteriaExpression(
left=ColumnNameLiteral(columnName='schedulerid'),
right=scheduler.schedulerId),
EntityCriteriaExpression(
left=ColumnNameLiteral(columnName='status'),
right='ready'),
])
], Pageable(pageNumber=1, pageSize=1))
if len(page.data) == 0:
if not after_sleeping:
sleep(30)
run_task(lock, scheduler, principal_service, True)
return
else:
# accomplish job
try_to_accomplish_job(lock.lockId, principal_service)
return
# try to update status to process
data = page.data[0]
data['status'] = 'processed'
data_id = data.get(TopicDataColumnNames.ID.value)
tenant_id = data.get(TopicDataColumnNames.TENANT_ID.value)
insert_time = data.get(TopicDataColumnNames.INSERT_TIME.value)
update_time = data.get(TopicDataColumnNames.UPDATE_TIME.value)
data = task_topic_service.try_to_wrap_to_topic_data(data)
data[TopicDataColumnNames.ID.value] = data_id
data[TopicDataColumnNames.TENANT_ID.value] = tenant_id
data[TopicDataColumnNames.INSERT_TIME.value] = insert_time
data[TopicDataColumnNames.UPDATE_TIME.value] = update_time
updated_count, _ = task_topic_service.update_by_id_and_version(
data, [
EntityCriteriaExpression(
left=ColumnNameLiteral(columnName='status'), right='ready')
])
if updated_count == 0:
sleep(1)
run_task(lock, scheduler, principal_service)
return
trace_id = str(ask_snowflake_generator().next_id())
unwrapped_data = data[TopicDataColumnNames.RAW_TOPIC_DATA.value]
run(
PipelineTrigger(
trigger_topic_schema=task_topic_schema,
trigger_type=PipelineTriggerType.INSERT,
trigger_data=unwrapped_data,
trace_id=trace_id,
principal_service=principal_service,
asynchronized=False,
handle_monitor_log=create_monitor_log_pipeline_invoker(
trace_id, principal_service)).start(
TopicTrigger(previous=None,
current=unwrapped_data,
triggerType=PipelineTriggerType.INSERT,
internalDataId=data_id), pipline.pipelineId))
run_task(lock, scheduler, principal_service)