async def find_updated_topics( lastModified: LastModified, principal_service: PrincipalService = Depends(get_admin_principal) ) -> List[Topic]: if lastModified is None or is_blank(lastModified.at): return [] parsed, last_modified_at = is_date(lastModified.at, ask_all_date_formats()) if not parsed: return [] if not isinstance(last_modified_at, datetime): last_modified_at = datetime(year=last_modified_at.year, month=last_modified_at.month, day=last_modified_at.day, hour=0, minute=0, second=0, microsecond=0, tzinfo=None) topic_service = get_topic_service(principal_service) def action() -> List[Topic]: return topic_service.find_modified_after( last_modified_at, principal_service.get_tenant_id()) return trans_readonly(topic_service, action)
def run_topics_rules( topic_name: Optional[str] = None, frequency: Optional[MonitorRuleStatisticalInterval] = None, process_date: Optional[str] = None, tenant_id: Optional[TenantId] = None, principal_service: PrincipalService = Depends(get_any_admin_principal) ) -> None: principal_service = ask_principal_service(principal_service, tenant_id) if is_not_blank(topic_name): schema = get_topic_service(principal_service).find_schema_by_name( topic_name, principal_service.get_tenant_id()) if schema is None: raise_404(f'Topic[name={topic_name}] not found.') topic_id = schema.get_topic().topicId else: topic_id = None if is_not_blank(process_date): parsed, parsed_date = is_date(process_date, ask_all_date_formats()) if not parsed: raise_400(f'Given process date[{process_date}] cannot be parsed.') process_date = parsed_date else: process_date = get_current_time_in_seconds() process_date = truncate_time(process_date) now = truncate_time(get_current_time_in_seconds()) if process_date.year > now.year: raise_400(f'Given process date[{process_date}] cannot be in future.') if process_date.year == now.year and process_date.month > now.month: raise_400(f'Given process date[{process_date}] cannot be in future.') if process_date.year == now.year and process_date.month == now.month and process_date.day > now.day: raise_400(f'Given process date[{process_date}] cannot be in future.') if frequency == MonitorRuleStatisticalInterval.MONTHLY: # given process date is in this month, run previous month # otherwise, run the given month if process_date.year == now.year and process_date.month == now.month: process_date = to_previous_month(process_date) SelfCleaningMonitorRulesRunner(principal_service) \ .run(process_date, topic_id, MonitorRuleStatisticalInterval.MONTHLY) elif frequency == MonitorRuleStatisticalInterval.WEEKLY: # given process date is in this week, run previous week # otherwise, run the given week if process_date.year == now.year and int( process_date.strftime('%U')) == int(now.strftime('%U')): process_date = to_previous_week(process_date) SelfCleaningMonitorRulesRunner(principal_service) \ .run(process_date, topic_id, MonitorRuleStatisticalInterval.WEEKLY) elif frequency == MonitorRuleStatisticalInterval.DAILY: # given process date is today, run yesterday # otherwise, run the given day if process_date.year == now.year and process_date.month == now.month and process_date.day == now.day: process_date = to_yesterday(process_date) SelfCleaningMonitorRulesRunner(principal_service) \ .run(process_date, topic_id, MonitorRuleStatisticalInterval.DAILY) else: raise_400(f'Given frequency[{frequency}] is not supported.')
def try_to_date_type(value: Any) -> Any: if value is None: return None if isinstance(value, date): return value if isinstance(value, str): parsed, dt_value = is_date(value, ask_all_date_formats()) return dt_value if parsed else value else: return value
def get_date_from_variables(variables: PipelineVariables, principal_service: PrincipalService, variable_name: str) -> Tuple[bool, Any, date]: value = get_value_from(variable_name, variable_name.strip().split('.'), create_get_value_from_variables(variables), create_is_list_from_variables(variables)) if isinstance(value, date): return True, value, value parsed, parsed_date = is_date(value, ask_all_date_formats()) return parsed, value, parsed_date
def get_part_of_datetime( variables: PipelineVariables, principal_service: PrincipalService) -> Optional[int]: value = parameter.value(variables, principal_service) if value is None: return None if isinstance(value, date): return func(value) parsed, dt_value = is_date(value, ask_all_date_formats()) if not parsed: raise DataKernelException( f'Cannot parse value[{value}] to datetime.') if dt_value is None: return None return func(dt_value)
def handle_comparison_possible_types( self, a_value: Any, another_possible_types: List[PossibleParameterType]) -> Any: if isinstance(a_value, str): if PossibleParameterType.NUMBER in another_possible_types: parsed, decimal_value = is_decimal(a_value) if parsed: return decimal_value if PossibleParameterType.DATE in another_possible_types or PossibleParameterType.DATETIME in another_possible_types: parsed, date_value = is_date(a_value, ask_all_date_formats()) if parsed: return date_value if PossibleParameterType.TIME in another_possible_types: parsed, time_value = is_time(a_value, ask_time_formats()) if parsed: return time_value return a_value
async def rerun_by_topic_data( scheduler_id: Optional[TopicSnapshotSchedulerId], process_date: Optional[str], principal_service: PrincipalService = Depends(get_any_admin_principal) ) -> None: if is_blank(scheduler_id): raise_400('Scheduler id is required.') if is_blank(process_date): raise_400('Process date is required.') parsed, parsed_process_date = is_date(process_date, ask_date_formats()) if not parsed: raise_400('Process date must be date.') scheduler_service = get_topic_snapshot_scheduler_service(principal_service) scheduler_service.begin_transaction() try: scheduler: Optional[ TopicSnapshotScheduler] = scheduler_service.find_by_id( scheduler_id) if scheduler is None: raise_404(f'Scheduler[id={scheduler_id}] not found.') if principal_service.is_tenant_admin( ) and scheduler.tenantId != principal_service.get_tenant_id(): raise_404(f'Scheduler[id={scheduler_id}] not found.') lock_service = get_lock_service(scheduler_service) lock = lock_service.find_by_scheduler_and_process_date( scheduler_id, scheduler.frequency, parsed_process_date) if lock is not None: raise_406( f'Scheduler[id={scheduler_id}, processDate={process_date}] run already.' ) except HTTPException as e: raise e except Exception as e: raise_500(e) finally: scheduler_service.close_transaction() run_job(scheduler_id, parsed_process_date)
async def find_topic_profile( topic_id: Optional[TopicId] = None, date: Optional[str] = None, principal_service: PrincipalService = Depends(get_admin_principal) ) -> Optional[TopicProfile]: if is_blank(topic_id): raise_400('Topic is is required.') parsed, query_date = is_date(date, ask_all_date_formats()) if not parsed: query_date = get_current_time_in_seconds() start_time = query_date.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=None) end_time = query_date.replace(hour=23, minute=59, second=59, microsecond=999999, tzinfo=None) return TopicProfileService(principal_service).find(topic_id, start_time, end_time)
def page(self, criteria: PipelineMonitorLogCriteria) -> DataPage: schema = self.get_topic_schema() storage = self.ask_storages().ask_topic_storage(schema) data_service = ask_topic_data_service(schema, storage, self.principalService) entity_criteria = [ EntityCriteriaExpression(left=ColumnNameLiteral( columnName=TopicDataColumnNames.TENANT_ID.value), right=criteria.tenantId) ] if is_not_blank(criteria.traceId): # noinspection SpellCheckingInspection entity_criteria.append( EntityCriteriaExpression( left=ColumnNameLiteral(columnName='traceid'), right=criteria.traceId)) if is_not_blank(criteria.topicId): # noinspection SpellCheckingInspection entity_criteria.append( EntityCriteriaExpression( left=ColumnNameLiteral(columnName='topicid'), right=criteria.topicId)) if is_not_blank(criteria.pipelineId): # noinspection SpellCheckingInspection entity_criteria.append( EntityCriteriaExpression( left=ColumnNameLiteral(columnName='pipelineid'), right=criteria.pipelineId)) if is_not_blank(criteria.status): entity_criteria.append( EntityCriteriaExpression( left=ColumnNameLiteral(columnName='status'), right=criteria.status)) start_date_parsed, start_date = is_date(criteria.startDate, ask_datetime_formats()) end_date_parsed, end_date = is_date(criteria.endDate, ask_datetime_formats()) if start_date_parsed: entity_criteria.append( EntityCriteriaExpression( left=ColumnNameLiteral( columnName=TopicDataColumnNames.INSERT_TIME.value), operator=EntityCriteriaOperator.GREATER_THAN_OR_EQUALS, right=start_date)) if end_date_parsed: entity_criteria.append( EntityCriteriaExpression( left=ColumnNameLiteral( columnName=TopicDataColumnNames.INSERT_TIME.value), operator=EntityCriteriaOperator.LESS_THAN_OR_EQUALS, right=end_date)) page = data_service.page( data_service.get_data_entity_helper().get_entity_pager( criteria=entity_criteria, pageable=Pageable(pageNumber=criteria.pageNumber, pageSize=criteria.pageSize))) page.data = ArrayHelper(page.data).map(lambda x: x.get(TopicDataColumnNames.RAW_TOPIC_DATA.value)) \ .filter(lambda x: x is not None) \ .map(lambda x: PipelineMonitorLog(**x)) \ .to_list() return page
def cast_value_for_factor(value: Any, factor: Factor) -> Any: if value is None: return None factor_type = factor.type if factor_type in [ FactorType.SEQUENCE, FactorType.NUMBER, FactorType.UNSIGNED, FactorType.FLOOR, FactorType.RESIDENTIAL_AREA, FactorType.AGE, FactorType.BIZ_SCALE ]: parsed, decimal_value = is_decimal(value) if parsed: return decimal_value else: raise DataKernelException( f'Value[{value}] is incompatible with factor[name={factor.name}, type={factor_type}].' ) elif factor_type == FactorType.TEXT: if isinstance(value, str): return value elif isinstance(value, (int, float, Decimal, bool, date, time)): return str(value) else: raise DataKernelException( f'Value[{value}, type={type(value)}] is incompatible with ' f'factor[name={factor.name}, type={factor_type}].') elif factor_type in [ FactorType.ADDRESS, FactorType.ROAD, FactorType.COMMUNITY, FactorType.EMAIL, FactorType.PHONE, FactorType.MOBILE, FactorType.FAX, FactorType.OCCUPATION, FactorType.ID_NO ]: if isinstance(value, str): return value elif isinstance(value, (int, float, Decimal)): return str(value) else: raise DataKernelException( f'Value[{value}, type={type(value)}] is incompatible with ' f'factor[name={factor.name}, type={factor_type}].') # noinspection PyPep8 elif factor_type in [ FactorType.CONTINENT, FactorType.REGION, FactorType.COUNTRY, FactorType.PROVINCE, FactorType.CITY, FactorType.DISTRICT, FactorType.RESIDENCE_TYPE, FactorType.GENDER, FactorType.RELIGION, FactorType.NATIONALITY, FactorType.BIZ_TRADE, FactorType.ENUM ]: if isinstance(value, str): return value elif isinstance(value, (int, Decimal)): return str(value) else: raise DataKernelException( f'Value[{value}, type={type(value)}] is incompatible with ' f'factor[name={factor.name}, type={factor_type}].') elif factor_type == FactorType.FULL_DATETIME: # noinspection DuplicatedCode if isinstance(value, datetime): return value if isinstance(value, date): return datetime(year=value.year, month=value.month, day=value.day) if isinstance(value, time): raise DataKernelException( f'Value[{value}, type={type(value)}] is incompatible with ' f'factor[name={factor.name}, type={factor_type}].') parsed, date_value = is_date(str(value), ask_full_datetime_formats()) if parsed: return date_value else: raise DataKernelException( f'Value[{value}, type={type(value)}] is incompatible with ' f'factor[name={factor.name}, type={factor_type}].') elif factor_type == FactorType.DATETIME: # noinspection DuplicatedCode if isinstance(value, datetime): return value if isinstance(value, date): return datetime(year=value.year, month=value.month, day=value.day) if isinstance(value, time): raise DataKernelException( f'Value[{value}, type={type(value)}] is incompatible with ' f'factor[name={factor.name}, type={factor_type}].') parsed, date_value = is_date(str(value), ask_all_date_formats()) if parsed: return date_value else: raise DataKernelException( f'Value[{value}, type={type(value)}] is incompatible with ' f'factor[name={factor.name}, type={factor_type}].') elif factor_type in [FactorType.DATE, FactorType.DATE_OF_BIRTH]: if isinstance(value, datetime): return value.date() if isinstance(value, date): return value if isinstance(value, time): raise DataKernelException( f'Value[{value}, type={type(value)}] is incompatible with ' f'factor[name={factor.name}, type={factor_type}].') parsed, date_value = is_date(value, ask_all_date_formats()) if parsed: if isinstance(date_value, datetime): return date_value.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=None) else: return date_value else: raise DataKernelException( f'Value[{value}, type={type(value)}] is incompatible with ' f'factor[name={factor.name}, type={factor_type}].') elif factor_type == FactorType.TIME: if isinstance(value, datetime): return value.time() if isinstance(value, date): raise DataKernelException( f'Value[{value}, type={type(value)}] is incompatible with ' f'factor[name={factor.name}, type={factor_type}].') if isinstance(value, time): return value parsed, time_value = is_time(value, ask_time_formats()) if parsed: return time_value else: raise DataKernelException( f'Value[{value}, type={type(value)}] is incompatible with ' f'factor[name={factor.name}, type={factor_type}].') elif factor_type in [ FactorType.YEAR, FactorType.HALF_YEAR, FactorType.QUARTER, FactorType.MONTH, FactorType.HALF_MONTH, FactorType.TEN_DAYS, FactorType.WEEK_OF_YEAR, FactorType.WEEK_OF_MONTH, FactorType.HALF_WEEK, FactorType.DAY_OF_MONTH, FactorType.DAY_OF_WEEK, FactorType.DAY_KIND, FactorType.HOUR, FactorType.HOUR_KIND, FactorType.MINUTE, FactorType.SECOND, FactorType.MILLISECOND, FactorType.AM_PM ]: # TODO strictly validation is needed or not? parsed, decimal_value = is_decimal(value) if parsed: return decimal_value else: raise DataKernelException( f'Value[{value}] is incompatible with factor[name={factor.name}, type={factor_type}].' ) elif factor_type == FactorType.BOOLEAN: if isinstance(value, bool): return value elif isinstance(value, (int, float, Decimal)): return value != 0 elif isinstance(value, str): v = value.strip().lower() if v == 't' or v == 'y' or v == 'yes' or v == 'true': return True elif v == 'f' or v == 'n' or v == 'no' or v == 'false': return False raise DataKernelException( f'Value[{value}, type={type(value)}] is incompatible with ' f'factor[name={factor.name}, type={factor_type}].') else: raise DataKernelException( f'Factor type[{factor_type}] is not supported.')
def test_date(variable_name: str) -> Tuple[bool, Optional[date]]: if variable_name == VariablePredefineFunctions.NOW: return True, get_current_time_in_seconds() else: return is_date(variable_name, ask_all_date_formats())