async def find_updated_topics(
lastModified: LastModified,
principal_service: PrincipalService = Depends(get_admin_principal)
) -> List[Topic]:
if lastModified is None or is_blank(lastModified.at):
return []
parsed, last_modified_at = is_date(lastModified.at, ask_all_date_formats())
if not parsed:
return []
if not isinstance(last_modified_at, datetime):
last_modified_at = datetime(year=last_modified_at.year,
month=last_modified_at.month,
day=last_modified_at.day,
hour=0,
minute=0,
second=0,
microsecond=0,
tzinfo=None)
topic_service = get_topic_service(principal_service)
def action() -> List[Topic]:
return topic_service.find_modified_after(
last_modified_at, principal_service.get_tenant_id())
return trans_readonly(topic_service, action)
def run_topics_rules(
topic_name: Optional[str] = None,
frequency: Optional[MonitorRuleStatisticalInterval] = None,
process_date: Optional[str] = None,
tenant_id: Optional[TenantId] = None,
principal_service: PrincipalService = Depends(get_any_admin_principal)
) -> None:
principal_service = ask_principal_service(principal_service, tenant_id)
if is_not_blank(topic_name):
schema = get_topic_service(principal_service).find_schema_by_name(
topic_name, principal_service.get_tenant_id())
if schema is None:
raise_404(f'Topic[name={topic_name}] not found.')
topic_id = schema.get_topic().topicId
else:
topic_id = None
if is_not_blank(process_date):
parsed, parsed_date = is_date(process_date, ask_all_date_formats())
if not parsed:
raise_400(f'Given process date[{process_date}] cannot be parsed.')
process_date = parsed_date
else:
process_date = get_current_time_in_seconds()
process_date = truncate_time(process_date)
now = truncate_time(get_current_time_in_seconds())
if process_date.year > now.year:
raise_400(f'Given process date[{process_date}] cannot be in future.')
if process_date.year == now.year and process_date.month > now.month:
raise_400(f'Given process date[{process_date}] cannot be in future.')
if process_date.year == now.year and process_date.month == now.month and process_date.day > now.day:
raise_400(f'Given process date[{process_date}] cannot be in future.')
if frequency == MonitorRuleStatisticalInterval.MONTHLY:
# given process date is in this month, run previous month
# otherwise, run the given month
if process_date.year == now.year and process_date.month == now.month:
process_date = to_previous_month(process_date)
SelfCleaningMonitorRulesRunner(principal_service) \
.run(process_date, topic_id, MonitorRuleStatisticalInterval.MONTHLY)
elif frequency == MonitorRuleStatisticalInterval.WEEKLY:
# given process date is in this week, run previous week
# otherwise, run the given week
if process_date.year == now.year and int(
process_date.strftime('%U')) == int(now.strftime('%U')):
process_date = to_previous_week(process_date)
SelfCleaningMonitorRulesRunner(principal_service) \
.run(process_date, topic_id, MonitorRuleStatisticalInterval.WEEKLY)
elif frequency == MonitorRuleStatisticalInterval.DAILY:
# given process date is today, run yesterday
# otherwise, run the given day
if process_date.year == now.year and process_date.month == now.month and process_date.day == now.day:
process_date = to_yesterday(process_date)
SelfCleaningMonitorRulesRunner(principal_service) \
.run(process_date, topic_id, MonitorRuleStatisticalInterval.DAILY)
else:
raise_400(f'Given frequency[{frequency}] is not supported.')
def translate(self, value: Any) -> Optional[Union[date, time, Any]]: factor_type = self.factor.type if factor_type == FactorType.FULL_DATETIME: parsed_value = try_to_date(value, ask_all_date_formats()) elif factor_type == FactorType.DATETIME: parsed_value = try_to_date(value, ask_all_date_formats()) elif factor_type == FactorType.DATE or factor_type == FactorType.DATE_OF_BIRTH: parsed_value = try_to_date(value, ask_all_date_formats()) elif factor_type == FactorType.TIME: parsed_value = try_to_time(value, ask_time_formats()) else: parsed_value = value if parsed_value is not None: return parsed_value elif ask_abandon_date_time_on_parse_fail(): return None else: return value
def greater_than_or_equals(self, one: Any, another: Any) -> bool:
if one is None:
if another is None:
return True
elif is_numeric_instance(another) or is_date_or_time_instance(
another):
return False
elif another is None:
if is_numeric_instance(one) or is_date_or_time_instance(one):
return True
elif isinstance(one, int) or isinstance(one, float) or isinstance(
one, Decimal):
return self.try_compare(
lambda: greater_or_equals_decimal(one, another, True),
lambda: self.raise_cannot_compare(one, another))
elif isinstance(another, int) or isinstance(
another, float) or isinstance(another, Decimal):
return self.try_compare(
lambda: less_or_equals_decimal(one, another, False),
lambda: self.raise_cannot_compare(one, another))
elif isinstance(one, time):
# compare time
return self.try_compare(
lambda: greater_or_equals_time(one, another, ask_time_formats(
), True), lambda: self.raise_cannot_compare(one, another))
elif isinstance(another, time):
# compare time
return self.try_compare(
lambda: less_or_equals_time(another, one, ask_time_formats(
), False), lambda: self.raise_cannot_compare(one, another))
elif isinstance(one, datetime) or isinstance(one, date):
# compare datetime or date
return self.try_compare(
lambda: greater_or_equals_date(another, one,
ask_all_date_formats(), True),
lambda: self.raise_cannot_compare(one, another))
elif isinstance(another, datetime) or isinstance(another, date):
# compare datetime or date
return self.try_compare(
lambda: less_or_equals_date(another, one, ask_all_date_formats(
), False), lambda: self.raise_cannot_compare(one, another))
self.raise_cannot_compare(one, another)
def try_to_date_type(value: Any) -> Any: if value is None: return None if isinstance(value, date): return value if isinstance(value, str): parsed, dt_value = is_date(value, ask_all_date_formats()) return dt_value if parsed else value else: return value
def get_date_from_variables(variables: PipelineVariables,
principal_service: PrincipalService,
variable_name: str) -> Tuple[bool, Any, date]:
value = get_value_from(variable_name,
variable_name.strip().split('.'),
create_get_value_from_variables(variables),
create_is_list_from_variables(variables))
if isinstance(value, date):
return True, value, value
parsed, parsed_date = is_date(value, ask_all_date_formats())
return parsed, value, parsed_date
def get_part_of_datetime(
variables: PipelineVariables,
principal_service: PrincipalService) -> Optional[int]:
value = parameter.value(variables, principal_service)
if value is None:
return None
if isinstance(value, date):
return func(value)
parsed, dt_value = is_date(value, ask_all_date_formats())
if not parsed:
raise DataKernelException(
f'Cannot parse value[{value}] to datetime.')
if dt_value is None:
return None
return func(dt_value)
def handle_comparison_possible_types( self, a_value: Any, another_possible_types: List[PossibleParameterType]) -> Any: if isinstance(a_value, str): if PossibleParameterType.NUMBER in another_possible_types: parsed, decimal_value = is_decimal(a_value) if parsed: return decimal_value if PossibleParameterType.DATE in another_possible_types or PossibleParameterType.DATETIME in another_possible_types: parsed, date_value = is_date(a_value, ask_all_date_formats()) if parsed: return date_value if PossibleParameterType.TIME in another_possible_types: parsed, time_value = is_time(a_value, ask_time_formats()) if parsed: return time_value return a_value
async def find_topic_profile(
topic_id: Optional[TopicId] = None,
date: Optional[str] = None,
principal_service: PrincipalService = Depends(get_admin_principal)
) -> Optional[TopicProfile]:
if is_blank(topic_id):
raise_400('Topic is is required.')
parsed, query_date = is_date(date, ask_all_date_formats())
if not parsed:
query_date = get_current_time_in_seconds()
start_time = query_date.replace(hour=0,
minute=0,
second=0,
microsecond=0,
tzinfo=None)
end_time = query_date.replace(hour=23,
minute=59,
second=59,
microsecond=999999,
tzinfo=None)
return TopicProfileService(principal_service).find(topic_id, start_time,
end_time)
encryptor_registry.register(method, encryptor)
def is_encryptor_registered(method: Union[FactorEncryptMethod, str]) -> bool:
return encryptor_registry.is_registered(method)
def find_encryptor(
method: Union[FactorEncryptMethod, str]) -> Optional[Encryptor]:
return encryptor_registry.ask_encryptor(method)
# register default
aes_key, aes_iv = ask_encrypt_aes_params()
register_encryptor(FactorEncryptMethod.AES256_PKCS5_PADDING,
AESEncryptor(key=aes_key, iv=aes_iv))
register_encryptor(FactorEncryptMethod.MD5, MD5Encryptor())
register_encryptor(FactorEncryptMethod.SHA256, SHA256Encryptor())
register_encryptor(FactorEncryptMethod.MASK_MAIL, MailMasker())
register_encryptor(FactorEncryptMethod.MASK_CENTER_3, CenterMasker(3))
register_encryptor(FactorEncryptMethod.MASK_CENTER_5, CenterMasker(5))
register_encryptor(FactorEncryptMethod.MASK_LAST_3, LastMasker(3))
register_encryptor(FactorEncryptMethod.MASK_LAST_6, LastMasker(6))
date_formats = ask_all_date_formats()
register_encryptor(FactorEncryptMethod.MASK_MONTH_DAY,
DateMasker(month=True, day=True, formats=date_formats))
register_encryptor(FactorEncryptMethod.MASK_MONTH,
DateMasker(month=True, day=False, formats=date_formats))
register_encryptor(FactorEncryptMethod.MASK_DAY,
DateMasker(month=False, day=True, formats=date_formats))
def cast_value_for_factor(value: Any, factor: Factor) -> Any:
if value is None:
return None
factor_type = factor.type
if factor_type in [
FactorType.SEQUENCE, FactorType.NUMBER, FactorType.UNSIGNED,
FactorType.FLOOR, FactorType.RESIDENTIAL_AREA, FactorType.AGE,
FactorType.BIZ_SCALE
]:
parsed, decimal_value = is_decimal(value)
if parsed:
return decimal_value
else:
raise DataKernelException(
f'Value[{value}] is incompatible with factor[name={factor.name}, type={factor_type}].'
)
elif factor_type == FactorType.TEXT:
if isinstance(value, str):
return value
elif isinstance(value, (int, float, Decimal, bool, date, time)):
return str(value)
else:
raise DataKernelException(
f'Value[{value}, type={type(value)}] is incompatible with '
f'factor[name={factor.name}, type={factor_type}].')
elif factor_type in [
FactorType.ADDRESS, FactorType.ROAD, FactorType.COMMUNITY,
FactorType.EMAIL, FactorType.PHONE, FactorType.MOBILE,
FactorType.FAX, FactorType.OCCUPATION, FactorType.ID_NO
]:
if isinstance(value, str):
return value
elif isinstance(value, (int, float, Decimal)):
return str(value)
else:
raise DataKernelException(
f'Value[{value}, type={type(value)}] is incompatible with '
f'factor[name={factor.name}, type={factor_type}].')
# noinspection PyPep8
elif factor_type in [
FactorType.CONTINENT, FactorType.REGION, FactorType.COUNTRY,
FactorType.PROVINCE, FactorType.CITY, FactorType.DISTRICT,
FactorType.RESIDENCE_TYPE, FactorType.GENDER, FactorType.RELIGION,
FactorType.NATIONALITY, FactorType.BIZ_TRADE, FactorType.ENUM
]:
if isinstance(value, str):
return value
elif isinstance(value, (int, Decimal)):
return str(value)
else:
raise DataKernelException(
f'Value[{value}, type={type(value)}] is incompatible with '
f'factor[name={factor.name}, type={factor_type}].')
elif factor_type == FactorType.FULL_DATETIME:
# noinspection DuplicatedCode
if isinstance(value, datetime):
return value
if isinstance(value, date):
return datetime(year=value.year, month=value.month, day=value.day)
if isinstance(value, time):
raise DataKernelException(
f'Value[{value}, type={type(value)}] is incompatible with '
f'factor[name={factor.name}, type={factor_type}].')
parsed, date_value = is_date(str(value), ask_full_datetime_formats())
if parsed:
return date_value
else:
raise DataKernelException(
f'Value[{value}, type={type(value)}] is incompatible with '
f'factor[name={factor.name}, type={factor_type}].')
elif factor_type == FactorType.DATETIME:
# noinspection DuplicatedCode
if isinstance(value, datetime):
return value
if isinstance(value, date):
return datetime(year=value.year, month=value.month, day=value.day)
if isinstance(value, time):
raise DataKernelException(
f'Value[{value}, type={type(value)}] is incompatible with '
f'factor[name={factor.name}, type={factor_type}].')
parsed, date_value = is_date(str(value), ask_all_date_formats())
if parsed:
return date_value
else:
raise DataKernelException(
f'Value[{value}, type={type(value)}] is incompatible with '
f'factor[name={factor.name}, type={factor_type}].')
elif factor_type in [FactorType.DATE, FactorType.DATE_OF_BIRTH]:
if isinstance(value, datetime):
return value.date()
if isinstance(value, date):
return value
if isinstance(value, time):
raise DataKernelException(
f'Value[{value}, type={type(value)}] is incompatible with '
f'factor[name={factor.name}, type={factor_type}].')
parsed, date_value = is_date(value, ask_all_date_formats())
if parsed:
if isinstance(date_value, datetime):
return date_value.replace(hour=0,
minute=0,
second=0,
microsecond=0,
tzinfo=None)
else:
return date_value
else:
raise DataKernelException(
f'Value[{value}, type={type(value)}] is incompatible with '
f'factor[name={factor.name}, type={factor_type}].')
elif factor_type == FactorType.TIME:
if isinstance(value, datetime):
return value.time()
if isinstance(value, date):
raise DataKernelException(
f'Value[{value}, type={type(value)}] is incompatible with '
f'factor[name={factor.name}, type={factor_type}].')
if isinstance(value, time):
return value
parsed, time_value = is_time(value, ask_time_formats())
if parsed:
return time_value
else:
raise DataKernelException(
f'Value[{value}, type={type(value)}] is incompatible with '
f'factor[name={factor.name}, type={factor_type}].')
elif factor_type in [
FactorType.YEAR, FactorType.HALF_YEAR, FactorType.QUARTER,
FactorType.MONTH, FactorType.HALF_MONTH, FactorType.TEN_DAYS,
FactorType.WEEK_OF_YEAR, FactorType.WEEK_OF_MONTH,
FactorType.HALF_WEEK, FactorType.DAY_OF_MONTH,
FactorType.DAY_OF_WEEK, FactorType.DAY_KIND, FactorType.HOUR,
FactorType.HOUR_KIND, FactorType.MINUTE, FactorType.SECOND,
FactorType.MILLISECOND, FactorType.AM_PM
]:
# TODO strictly validation is needed or not?
parsed, decimal_value = is_decimal(value)
if parsed:
return decimal_value
else:
raise DataKernelException(
f'Value[{value}] is incompatible with factor[name={factor.name}, type={factor_type}].'
)
elif factor_type == FactorType.BOOLEAN:
if isinstance(value, bool):
return value
elif isinstance(value, (int, float, Decimal)):
return value != 0
elif isinstance(value, str):
v = value.strip().lower()
if v == 't' or v == 'y' or v == 'yes' or v == 'true':
return True
elif v == 'f' or v == 'n' or v == 'no' or v == 'false':
return False
raise DataKernelException(
f'Value[{value}, type={type(value)}] is incompatible with '
f'factor[name={factor.name}, type={factor_type}].')
else:
raise DataKernelException(
f'Factor type[{factor_type}] is not supported.')
def not_equals(self, one: Any, another: Any) -> bool:
return value_not_equals(one, another, ask_time_formats(),
ask_all_date_formats())
def test_date(variable_name: str) -> Tuple[bool, Optional[date]]:
if variable_name == VariablePredefineFunctions.NOW:
return True, get_current_time_in_seconds()
else:
return is_date(variable_name, ask_all_date_formats())
