async def find_updated_topics(
lastModified: LastModified,
principal_service: PrincipalService = Depends(get_admin_principal)
) -> List[Topic]:
if lastModified is None or is_blank(lastModified.at):
return []
parsed, last_modified_at = is_date(lastModified.at, ask_all_date_formats())
if not parsed:
return []
if not isinstance(last_modified_at, datetime):
last_modified_at = datetime(year=last_modified_at.year,
month=last_modified_at.month,
day=last_modified_at.day,
hour=0,
minute=0,
second=0,
microsecond=0,
tzinfo=None)
topic_service = get_topic_service(principal_service)
def action() -> List[Topic]:
return topic_service.find_modified_after(
last_modified_at, principal_service.get_tenant_id())
return trans_readonly(topic_service, action)
def run_topics_rules(
topic_name: Optional[str] = None,
frequency: Optional[MonitorRuleStatisticalInterval] = None,
process_date: Optional[str] = None,
tenant_id: Optional[TenantId] = None,
principal_service: PrincipalService = Depends(get_any_admin_principal)
) -> None:
principal_service = ask_principal_service(principal_service, tenant_id)
if is_not_blank(topic_name):
schema = get_topic_service(principal_service).find_schema_by_name(
topic_name, principal_service.get_tenant_id())
if schema is None:
raise_404(f'Topic[name={topic_name}] not found.')
topic_id = schema.get_topic().topicId
else:
topic_id = None
if is_not_blank(process_date):
parsed, parsed_date = is_date(process_date, ask_all_date_formats())
if not parsed:
raise_400(f'Given process date[{process_date}] cannot be parsed.')
process_date = parsed_date
else:
process_date = get_current_time_in_seconds()
process_date = truncate_time(process_date)
now = truncate_time(get_current_time_in_seconds())
if process_date.year > now.year:
raise_400(f'Given process date[{process_date}] cannot be in future.')
if process_date.year == now.year and process_date.month > now.month:
raise_400(f'Given process date[{process_date}] cannot be in future.')
if process_date.year == now.year and process_date.month == now.month and process_date.day > now.day:
raise_400(f'Given process date[{process_date}] cannot be in future.')
if frequency == MonitorRuleStatisticalInterval.MONTHLY:
# given process date is in this month, run previous month
# otherwise, run the given month
if process_date.year == now.year and process_date.month == now.month:
process_date = to_previous_month(process_date)
SelfCleaningMonitorRulesRunner(principal_service) \
.run(process_date, topic_id, MonitorRuleStatisticalInterval.MONTHLY)
elif frequency == MonitorRuleStatisticalInterval.WEEKLY:
# given process date is in this week, run previous week
# otherwise, run the given week
if process_date.year == now.year and int(
process_date.strftime('%U')) == int(now.strftime('%U')):
process_date = to_previous_week(process_date)
SelfCleaningMonitorRulesRunner(principal_service) \
.run(process_date, topic_id, MonitorRuleStatisticalInterval.WEEKLY)
elif frequency == MonitorRuleStatisticalInterval.DAILY:
# given process date is today, run yesterday
# otherwise, run the given day
if process_date.year == now.year and process_date.month == now.month and process_date.day == now.day:
process_date = to_yesterday(process_date)
SelfCleaningMonitorRulesRunner(principal_service) \
.run(process_date, topic_id, MonitorRuleStatisticalInterval.DAILY)
else:
raise_400(f'Given frequency[{frequency}] is not supported.')
def try_to_date_type(value: Any) -> Any: if value is None: return None if isinstance(value, date): return value if isinstance(value, str): parsed, dt_value = is_date(value, ask_all_date_formats()) return dt_value if parsed else value else: return value
def get_date_from_variables(variables: PipelineVariables,
principal_service: PrincipalService,
variable_name: str) -> Tuple[bool, Any, date]:
value = get_value_from(variable_name,
variable_name.strip().split('.'),
create_get_value_from_variables(variables),
create_is_list_from_variables(variables))
if isinstance(value, date):
return True, value, value
parsed, parsed_date = is_date(value, ask_all_date_formats())
return parsed, value, parsed_date
def get_part_of_datetime(
variables: PipelineVariables,
principal_service: PrincipalService) -> Optional[int]:
value = parameter.value(variables, principal_service)
if value is None:
return None
if isinstance(value, date):
return func(value)
parsed, dt_value = is_date(value, ask_all_date_formats())
if not parsed:
raise DataKernelException(
f'Cannot parse value[{value}] to datetime.')
if dt_value is None:
return None
return func(dt_value)
def handle_comparison_possible_types( self, a_value: Any, another_possible_types: List[PossibleParameterType]) -> Any: if isinstance(a_value, str): if PossibleParameterType.NUMBER in another_possible_types: parsed, decimal_value = is_decimal(a_value) if parsed: return decimal_value if PossibleParameterType.DATE in another_possible_types or PossibleParameterType.DATETIME in another_possible_types: parsed, date_value = is_date(a_value, ask_all_date_formats()) if parsed: return date_value if PossibleParameterType.TIME in another_possible_types: parsed, time_value = is_time(a_value, ask_time_formats()) if parsed: return time_value return a_value
async def rerun_by_topic_data(
scheduler_id: Optional[TopicSnapshotSchedulerId],
process_date: Optional[str],
principal_service: PrincipalService = Depends(get_any_admin_principal)
) -> None:
if is_blank(scheduler_id):
raise_400('Scheduler id is required.')
if is_blank(process_date):
raise_400('Process date is required.')
parsed, parsed_process_date = is_date(process_date, ask_date_formats())
if not parsed:
raise_400('Process date must be date.')
scheduler_service = get_topic_snapshot_scheduler_service(principal_service)
scheduler_service.begin_transaction()
try:
scheduler: Optional[
TopicSnapshotScheduler] = scheduler_service.find_by_id(
scheduler_id)
if scheduler is None:
raise_404(f'Scheduler[id={scheduler_id}] not found.')
if principal_service.is_tenant_admin(
) and scheduler.tenantId != principal_service.get_tenant_id():
raise_404(f'Scheduler[id={scheduler_id}] not found.')
lock_service = get_lock_service(scheduler_service)
lock = lock_service.find_by_scheduler_and_process_date(
scheduler_id, scheduler.frequency, parsed_process_date)
if lock is not None:
raise_406(
f'Scheduler[id={scheduler_id}, processDate={process_date}] run already.'
)
except HTTPException as e:
raise e
except Exception as e:
raise_500(e)
finally:
scheduler_service.close_transaction()
run_job(scheduler_id, parsed_process_date)
async def find_topic_profile(
topic_id: Optional[TopicId] = None,
date: Optional[str] = None,
principal_service: PrincipalService = Depends(get_admin_principal)
) -> Optional[TopicProfile]:
if is_blank(topic_id):
raise_400('Topic is is required.')
parsed, query_date = is_date(date, ask_all_date_formats())
if not parsed:
query_date = get_current_time_in_seconds()
start_time = query_date.replace(hour=0,
minute=0,
second=0,
microsecond=0,
tzinfo=None)
end_time = query_date.replace(hour=23,
minute=59,
second=59,
microsecond=999999,
tzinfo=None)
return TopicProfileService(principal_service).find(topic_id, start_time,
end_time)
def page(self, criteria: PipelineMonitorLogCriteria) -> DataPage:
schema = self.get_topic_schema()
storage = self.ask_storages().ask_topic_storage(schema)
data_service = ask_topic_data_service(schema, storage,
self.principalService)
entity_criteria = [
EntityCriteriaExpression(left=ColumnNameLiteral(
columnName=TopicDataColumnNames.TENANT_ID.value),
right=criteria.tenantId)
]
if is_not_blank(criteria.traceId):
# noinspection SpellCheckingInspection
entity_criteria.append(
EntityCriteriaExpression(
left=ColumnNameLiteral(columnName='traceid'),
right=criteria.traceId))
if is_not_blank(criteria.topicId):
# noinspection SpellCheckingInspection
entity_criteria.append(
EntityCriteriaExpression(
left=ColumnNameLiteral(columnName='topicid'),
right=criteria.topicId))
if is_not_blank(criteria.pipelineId):
# noinspection SpellCheckingInspection
entity_criteria.append(
EntityCriteriaExpression(
left=ColumnNameLiteral(columnName='pipelineid'),
right=criteria.pipelineId))
if is_not_blank(criteria.status):
entity_criteria.append(
EntityCriteriaExpression(
left=ColumnNameLiteral(columnName='status'),
right=criteria.status))
start_date_parsed, start_date = is_date(criteria.startDate,
ask_datetime_formats())
end_date_parsed, end_date = is_date(criteria.endDate,
ask_datetime_formats())
if start_date_parsed:
entity_criteria.append(
EntityCriteriaExpression(
left=ColumnNameLiteral(
columnName=TopicDataColumnNames.INSERT_TIME.value),
operator=EntityCriteriaOperator.GREATER_THAN_OR_EQUALS,
right=start_date))
if end_date_parsed:
entity_criteria.append(
EntityCriteriaExpression(
left=ColumnNameLiteral(
columnName=TopicDataColumnNames.INSERT_TIME.value),
operator=EntityCriteriaOperator.LESS_THAN_OR_EQUALS,
right=end_date))
page = data_service.page(
data_service.get_data_entity_helper().get_entity_pager(
criteria=entity_criteria,
pageable=Pageable(pageNumber=criteria.pageNumber,
pageSize=criteria.pageSize)))
page.data = ArrayHelper(page.data).map(lambda x: x.get(TopicDataColumnNames.RAW_TOPIC_DATA.value)) \
.filter(lambda x: x is not None) \
.map(lambda x: PipelineMonitorLog(**x)) \
.to_list()
return page
def cast_value_for_factor(value: Any, factor: Factor) -> Any:
if value is None:
return None
factor_type = factor.type
if factor_type in [
FactorType.SEQUENCE, FactorType.NUMBER, FactorType.UNSIGNED,
FactorType.FLOOR, FactorType.RESIDENTIAL_AREA, FactorType.AGE,
FactorType.BIZ_SCALE
]:
parsed, decimal_value = is_decimal(value)
if parsed:
return decimal_value
else:
raise DataKernelException(
f'Value[{value}] is incompatible with factor[name={factor.name}, type={factor_type}].'
)
elif factor_type == FactorType.TEXT:
if isinstance(value, str):
return value
elif isinstance(value, (int, float, Decimal, bool, date, time)):
return str(value)
else:
raise DataKernelException(
f'Value[{value}, type={type(value)}] is incompatible with '
f'factor[name={factor.name}, type={factor_type}].')
elif factor_type in [
FactorType.ADDRESS, FactorType.ROAD, FactorType.COMMUNITY,
FactorType.EMAIL, FactorType.PHONE, FactorType.MOBILE,
FactorType.FAX, FactorType.OCCUPATION, FactorType.ID_NO
]:
if isinstance(value, str):
return value
elif isinstance(value, (int, float, Decimal)):
return str(value)
else:
raise DataKernelException(
f'Value[{value}, type={type(value)}] is incompatible with '
f'factor[name={factor.name}, type={factor_type}].')
# noinspection PyPep8
elif factor_type in [
FactorType.CONTINENT, FactorType.REGION, FactorType.COUNTRY,
FactorType.PROVINCE, FactorType.CITY, FactorType.DISTRICT,
FactorType.RESIDENCE_TYPE, FactorType.GENDER, FactorType.RELIGION,
FactorType.NATIONALITY, FactorType.BIZ_TRADE, FactorType.ENUM
]:
if isinstance(value, str):
return value
elif isinstance(value, (int, Decimal)):
return str(value)
else:
raise DataKernelException(
f'Value[{value}, type={type(value)}] is incompatible with '
f'factor[name={factor.name}, type={factor_type}].')
elif factor_type == FactorType.FULL_DATETIME:
# noinspection DuplicatedCode
if isinstance(value, datetime):
return value
if isinstance(value, date):
return datetime(year=value.year, month=value.month, day=value.day)
if isinstance(value, time):
raise DataKernelException(
f'Value[{value}, type={type(value)}] is incompatible with '
f'factor[name={factor.name}, type={factor_type}].')
parsed, date_value = is_date(str(value), ask_full_datetime_formats())
if parsed:
return date_value
else:
raise DataKernelException(
f'Value[{value}, type={type(value)}] is incompatible with '
f'factor[name={factor.name}, type={factor_type}].')
elif factor_type == FactorType.DATETIME:
# noinspection DuplicatedCode
if isinstance(value, datetime):
return value
if isinstance(value, date):
return datetime(year=value.year, month=value.month, day=value.day)
if isinstance(value, time):
raise DataKernelException(
f'Value[{value}, type={type(value)}] is incompatible with '
f'factor[name={factor.name}, type={factor_type}].')
parsed, date_value = is_date(str(value), ask_all_date_formats())
if parsed:
return date_value
else:
raise DataKernelException(
f'Value[{value}, type={type(value)}] is incompatible with '
f'factor[name={factor.name}, type={factor_type}].')
elif factor_type in [FactorType.DATE, FactorType.DATE_OF_BIRTH]:
if isinstance(value, datetime):
return value.date()
if isinstance(value, date):
return value
if isinstance(value, time):
raise DataKernelException(
f'Value[{value}, type={type(value)}] is incompatible with '
f'factor[name={factor.name}, type={factor_type}].')
parsed, date_value = is_date(value, ask_all_date_formats())
if parsed:
if isinstance(date_value, datetime):
return date_value.replace(hour=0,
minute=0,
second=0,
microsecond=0,
tzinfo=None)
else:
return date_value
else:
raise DataKernelException(
f'Value[{value}, type={type(value)}] is incompatible with '
f'factor[name={factor.name}, type={factor_type}].')
elif factor_type == FactorType.TIME:
if isinstance(value, datetime):
return value.time()
if isinstance(value, date):
raise DataKernelException(
f'Value[{value}, type={type(value)}] is incompatible with '
f'factor[name={factor.name}, type={factor_type}].')
if isinstance(value, time):
return value
parsed, time_value = is_time(value, ask_time_formats())
if parsed:
return time_value
else:
raise DataKernelException(
f'Value[{value}, type={type(value)}] is incompatible with '
f'factor[name={factor.name}, type={factor_type}].')
elif factor_type in [
FactorType.YEAR, FactorType.HALF_YEAR, FactorType.QUARTER,
FactorType.MONTH, FactorType.HALF_MONTH, FactorType.TEN_DAYS,
FactorType.WEEK_OF_YEAR, FactorType.WEEK_OF_MONTH,
FactorType.HALF_WEEK, FactorType.DAY_OF_MONTH,
FactorType.DAY_OF_WEEK, FactorType.DAY_KIND, FactorType.HOUR,
FactorType.HOUR_KIND, FactorType.MINUTE, FactorType.SECOND,
FactorType.MILLISECOND, FactorType.AM_PM
]:
# TODO strictly validation is needed or not?
parsed, decimal_value = is_decimal(value)
if parsed:
return decimal_value
else:
raise DataKernelException(
f'Value[{value}] is incompatible with factor[name={factor.name}, type={factor_type}].'
)
elif factor_type == FactorType.BOOLEAN:
if isinstance(value, bool):
return value
elif isinstance(value, (int, float, Decimal)):
return value != 0
elif isinstance(value, str):
v = value.strip().lower()
if v == 't' or v == 'y' or v == 'yes' or v == 'true':
return True
elif v == 'f' or v == 'n' or v == 'no' or v == 'false':
return False
raise DataKernelException(
f'Value[{value}, type={type(value)}] is incompatible with '
f'factor[name={factor.name}, type={factor_type}].')
else:
raise DataKernelException(
f'Factor type[{factor_type}] is not supported.')
def test_date(variable_name: str) -> Tuple[bool, Optional[date]]:
if variable_name == VariablePredefineFunctions.NOW:
return True, get_current_time_in_seconds()
else:
return is_date(variable_name, ask_all_date_formats())