def prepare_topic():
data_source = DataSource(dataSourceId='1',
dataSourceCode='ds1',
dataSourceType=DataSourceType.MYSQL,
host='localhost',
port='3306',
username='******',
password='******',
name='watchmen',
tenantId='1')
CacheService.data_source().put(data_source)
return Topic(topicId='1',
name='topic_x',
type=TopicType.DISTINCT,
kind=TopicKind.BUSINESS,
factors=[
Factor(factorId='1',
name='topic1_id',
type=FactorType.SEQUENCE,
indexGroup='u-1'),
Factor(factorId='2',
name='topic1_text',
type=FactorType.TEXT,
precision='64',
indexGroup='u-1')
],
dataSourceId=data_source.dataSourceId,
tenantId='1')
def try_to_import_topic(topic: Topic, topic_service: TopicService, do_update: bool) -> TopicImportDataResult: if is_blank(topic.topicId): topic_service.redress_storable_id(topic) topic_service.create(topic) else: existing_topic: Optional[Topic] = topic_service.find_by_id(topic.topicId) if existing_topic is None: topic_service.create(topic) elif do_update: topic.version = existing_topic.version topic.dataSourceId = existing_topic.dataSourceId topic_service.update(topic) else: return TopicImportDataResult( topicId=topic.topicId, name=topic.name, passed=False, reason='Topic already exists.') post_save_topic(topic, topic_service) return TopicImportDataResult(topicId=topic.topicId, name=topic.name, passed=True)
def prepare_data(self):
data_source = DataSource(dataSourceId='1',
dataSourceCode='test',
dataSourceType=DataSourceType.MYSQL,
host='localhost',
port='3306',
username='******',
password='******',
name='watchmen',
tenantId='1')
data_source_service = get_data_source_service(
create_fake_principal_service())
data_source_service.begin_transaction()
data_source_service.create(data_source)
data_source_service.commit_transaction()
CacheService.data_source().put(data_source)
topic1 = Topic(topicId='1',
name='topic1',
type=TopicType.DISTINCT,
kind=TopicKind.BUSINESS,
factors=[
Factor(factorId='1',
name='topic1_id',
type=FactorType.SEQUENCE),
Factor(factorId='2',
name='topic1_enabled',
type=FactorType.BOOLEAN)
],
dataSourceId=data_source.dataSourceId,
tenantId='1')
topic_service = get_topic_service(create_fake_principal_service())
topic_service.begin_transaction()
topic_service.create(topic1)
topic_service.commit_transaction()
CacheService.topic().put(topic1)
space = Space(spaceId='1',
name='Space1',
topicIds=[topic1.topicId],
tenantId='1')
space_service = get_space_service(create_fake_principal_service())
space_service.begin_transaction()
space_service.create(space)
space_service.commit_transaction()
connected_space = ConnectedSpace(connectId='1',
name='ConnectedSpace1',
spaceId=space.spaceId,
isTemplate=False,
userId='1',
tenantId='1')
connected_space_service = get_connected_space_service(
create_fake_principal_service())
connected_space_service.begin_transaction()
connected_space_service.create(connected_space)
connected_space_service.commit_transaction()
def deserialize(self, row: EntityRow) -> Topic:
# noinspection PyTypeChecker
return TupleShaper.deserialize_tenant_based(
row,
Topic(topicId=row.get('topic_id'),
name=row.get('name'),
description=row.get('description'),
type=row.get('type'),
kind=row.get('kind'),
dataSourceId=row.get('data_source_id'),
factors=row.get('factors')))
def create_snapshot_target_topic(scheduler: TopicSnapshotScheduler, source_topic: Topic) -> Topic:
return Topic(
topicId='f-1',
name=scheduler.targetTopicName,
type=source_topic.type,
kind=source_topic.kind,
dataSourceId=source_topic.dataSourceId,
factors=build_target_topic_factors(source_topic),
description=f'Snapshot of [{source_topic.name}], never change me manually.',
tenantId=source_topic.tenantId,
version=1
)
def create_snapshot_task_topic(source_topic: Topic) -> Topic:
return Topic(
topicId='f-1',
name=as_snapshot_task_topic_name(source_topic),
type=TopicType.RAW,
kind=source_topic.kind,
dataSourceId=source_topic.dataSourceId,
factors=build_task_topic_factors(source_topic),
description=f'Snapshot task of [{source_topic.name}], never change me manually.',
tenantId=source_topic.tenantId,
version=1
)
def prepare_new_topic():
return Topic(topicId='1',
name='topic_x',
type=TopicType.DISTINCT,
kind=TopicKind.BUSINESS,
factors=[
Factor(factorId='1',
name='topic1_id',
type=FactorType.SEQUENCE),
Factor(factorId='2',
name='topic1_text',
type=FactorType.TEXT,
precision='64',
indexGroup='u-1'),
Factor(factorId='2',
name='topic1_text2',
type=FactorType.TEXT,
precision='32',
indexGroup='u-1')
],
dataSourceId='1',
tenantId='1')
def test_criteria(self):
joint = ParameterJoint(
jointType=ParameterJointType.AND,
filters=[
ParameterExpression(
left=ConstantParameter(kind=ParameterKind.CONSTANT,
value="ABC"),
operator=ParameterExpressionOperator.EQUALS,
right=ConstantParameter(
kind=ParameterKind.CONSTANT,
value="{policy.productCode}-{policy.policyNo}"))
])
parsed = parse_condition_for_storage(joint, [
TopicSchema(
Topic(name='policy',
factors=[
Factor(name='productCode', type=FactorType.TEXT),
Factor(name='policyNo', type=FactorType.TEXT)
]))
], create_fake_principal_service(), True)
variables = PipelineVariables(None, {}, None)
variables.put('policy', {'productCode': 'A001', 'policyNo': 'P001'})
result = parsed.run(variables, create_fake_principal_service())
print(result.to_dict())
def set_storable_id(self, storable: Topic, storable_id: TopicId) -> Topic:
storable.topicId = storable_id
return storable
def fill_topic_id(topic: Topic) -> None: old_topic_id = topic.topicId topic.topicId = topic_service.generate_storable_id() topic_id_map[old_topic_id] = topic.topicId ArrayHelper(topic.factors).each(fill_factor_id)
def clear(topic: Topic) -> None: topic.dataSourceId = None
def rebuild_snapshot_target_topic(target_topic: Topic, source_topic: Topic) -> Topic: target_topic.factors = build_target_topic_factors(source_topic) return target_topic
def ask_pipeline_monitor_topics() -> List[Topic]:
# TODO define all pipeline monitor topics
return [
Topic(name='raw_pipeline_monitor_log',
kind=TopicKind.SYSTEM,
type=TopicType.RAW,
factors=[
Factor(factorId='rpml-f-1', name='uid',
type=FactorType.TEXT),
Factor(factorId='rpml-f-2',
name='traceId',
type=FactorType.TEXT,
flatten=True,
indexGroup=FactorIndexGroup.INDEX_1,
precision='50'),
Factor(factorId='rpml-f-3',
name='pipelineId',
type=FactorType.TEXT,
flatten=True,
indexGroup=FactorIndexGroup.INDEX_2,
precision='50'),
Factor(factorId='rpml-f-4',
name='topicId',
type=FactorType.TEXT,
flatten=True,
indexGroup=FactorIndexGroup.INDEX_3,
precision='50'),
Factor(factorId='rpml-f-5',
name='prerequisite',
type=FactorType.BOOLEAN),
Factor(factorId='rpml-f-6',
name='prerequisiteDefinedAs',
type=FactorType.OBJECT),
Factor(factorId='rpml-f-7',
name='status',
type=FactorType.TEXT,
flatten=True),
Factor(factorId='rpml-f-8',
name='startTime',
type=FactorType.FULL_DATETIME,
flatten=True),
Factor(factorId='rpml-f-9',
name='spentInMills',
type=FactorType.UNSIGNED,
flatten=True),
Factor(factorId='rpml-f-10',
name='error',
type=FactorType.TEXT),
Factor(factorId='rpml-f-11',
name='oldValue',
type=FactorType.OBJECT),
Factor(factorId='rpml-f-12',
name='newValue',
type=FactorType.OBJECT),
Factor(factorId='rpml-f-13',
name='stages',
type=FactorType.ARRAY),
Factor(factorId='rpml-f-14',
name='stages.stageId',
type=FactorType.TEXT),
Factor(factorId='rpml-f-15',
name='stages.name',
type=FactorType.TEXT),
Factor(factorId='rpml-f-16',
name='stages.prerequisite',
type=FactorType.BOOLEAN),
Factor(factorId='rpml-f-17',
name='stages.prerequisiteDefinedAs',
type=FactorType.OBJECT),
Factor(factorId='rpml-f-18',
name='stages.status',
type=FactorType.TEXT),
Factor(factorId='rpml-f-19',
name='stages.startTime',
type=FactorType.FULL_DATETIME),
Factor(factorId='rpml-f-20',
name='stages.spentInMills',
type=FactorType.UNSIGNED),
Factor(factorId='rpml-f-21',
name='stages.error',
type=FactorType.TEXT),
Factor(factorId='rpml-f-22',
name='stages.units',
type=FactorType.ARRAY),
Factor(factorId='rpml-f-23',
name='stages.units.unitId',
type=FactorType.TEXT),
Factor(factorId='rpml-f-24',
name='stages.units.name',
type=FactorType.TEXT),
Factor(factorId='rpml-f-25',
name='stages.units.prerequisite',
type=FactorType.BOOLEAN),
Factor(factorId='rpml-f-26',
name='stages.units.prerequisiteDefinedAs',
type=FactorType.OBJECT),
Factor(factorId='rpml-f-27',
name='stages.units.loopVariableName',
type=FactorType.TEXT),
Factor(factorId='rpml-f-28',
name='stages.units.loopVariableValue',
type=FactorType.ARRAY),
Factor(factorId='rpml-f-29',
name='stages.units.status',
type=FactorType.TEXT),
Factor(factorId='rpml-f-30',
name='stages.units.startTime',
type=FactorType.FULL_DATETIME),
Factor(factorId='rpml-f-31',
name='stages.units.spentInMills',
type=FactorType.UNSIGNED),
Factor(factorId='rpml-f-32',
name='stages.units.error',
type=FactorType.TEXT),
Factor(factorId='rpml-f-33',
name='stages.units.actions',
type=FactorType.ARRAY),
Factor(factorId='rpml-f-34',
name='stages.units.actions.uid',
type=FactorType.TEXT),
Factor(factorId='rpml-f-35',
name='stages.units.actions.actionId',
type=FactorType.TEXT),
Factor(factorId='rpml-f-36',
name='stages.units.actions.type',
type=FactorType.TEXT),
Factor(factorId='rpml-f-37',
name='stages.units.actions.insertCount',
type=FactorType.UNSIGNED),
Factor(factorId='rpml-f-38',
name='stages.units.actions.updateCount',
type=FactorType.UNSIGNED),
Factor(factorId='rpml-f-39',
name='stages.units.actions.deleteCount',
type=FactorType.UNSIGNED),
Factor(factorId='rpml-f-40',
name='stages.units.actions.definedAs',
type=FactorType.OBJECT),
Factor(factorId='rpml-f-41',
name='stages.units.actions.findBy',
type=FactorType.OBJECT),
Factor(factorId='rpml-f-42',
name='stages.units.actions.touched',
type=FactorType.OBJECT),
Factor(factorId='rpml-f-43',
name='stages.units.actions.status',
type=FactorType.TEXT),
Factor(factorId='rpml-f-44',
name='stages.units.actions.startTime',
type=FactorType.FULL_DATETIME),
Factor(factorId='rpml-f-45',
name='stages.units.actions.spentInMills',
type=FactorType.UNSIGNED),
Factor(factorId='rpml-f-46',
name='stages.units.actions.error',
type=FactorType.TEXT),
Factor(factorId='rpml-f-47',
name='dataId',
type=FactorType.NUMBER,
flatten=True,
indexGroup=FactorIndexGroup.INDEX_4,
precision='20'),
],
description='Pipeline monitor log raw topic.')
]
def ask_dqc_topics() -> List[Topic]:
# TODO define all dqc topics
return [
Topic(
name='dqc_raw_rule_result',
kind=TopicKind.SYSTEM,
type=TopicType.RAW,
factors=[
Factor(factorId='dra-f-1',
name='ruleCode',
type=FactorType.TEXT,
flatten=True,
indexGroup=FactorIndexGroup.INDEX_1,
precision='50'),
Factor(factorId='dra-f-2',
name='topicId',
type=FactorType.TEXT,
flatten=True,
indexGroup=FactorIndexGroup.INDEX_2,
precision='50'),
Factor(factorId='dra-f-3',
name='topicName',
type=FactorType.TEXT),
Factor(factorId='dra-f-4',
name='factorId',
type=FactorType.TEXT,
flatten=True,
indexGroup=FactorIndexGroup.INDEX_3,
precision='50'),
Factor(factorId='dra-f-5',
name='factorName',
type=FactorType.TEXT),
Factor(factorId='dra-f-6',
name='detected',
type=FactorType.BOOLEAN,
flatten=True),
Factor(factorId='dra-f-7',
name='severity',
type=FactorType.TEXT),
# the start day of date range
# sunday of weekly; 1st of monthly.
Factor(factorId='dra-f-8',
name='processDate',
type=FactorType.DATE,
flatten=True,
indexGroup=FactorIndexGroup.INDEX_4)
],
description='Topic data monitor by rules, raw topic.'),
Topic(name='dqc_rule_daily',
kind=TopicKind.SYSTEM,
type=TopicType.DISTINCT,
factors=[
Factor(factorId='dra-f-1',
name='ruleCode',
type=FactorType.TEXT,
indexGroup=FactorIndexGroup.INDEX_1),
Factor(factorId='dra-f-2',
name='topicId',
type=FactorType.TEXT,
indexGroup=FactorIndexGroup.INDEX_2),
Factor(factorId='dra-f-3',
name='factorId',
type=FactorType.TEXT,
indexGroup=FactorIndexGroup.INDEX_3),
Factor(factorId='dra-f-4',
name='year',
type=FactorType.YEAR,
indexGroup=FactorIndexGroup.INDEX_4),
Factor(factorId='dra-f-5',
name='month',
type=FactorType.MONTH,
indexGroup=FactorIndexGroup.INDEX_5),
Factor(factorId='dra-f-6',
name='day',
type=FactorType.DAY_OF_MONTH,
indexGroup=FactorIndexGroup.INDEX_6),
Factor(factorId='dra-f-7',
name='processDate',
type=FactorType.DATE,
indexGroup=FactorIndexGroup.INDEX_7),
Factor(factorId='dra-f-8',
name='count',
type=FactorType.UNSIGNED,
precision='10')
],
description='Topic data monitor by rules, distinct topic.')
]