def direct_copy_raw_schema_to_topic(model_schema: ModelSchema, topic: Topic):
if topic is None:
topic = Topic()
topic.topicName = model_schema.name
topic.factors = convert_business_fields_to_factors(
model_schema.businessFields)
return topic
def create_topic_schema(topic: Topic) -> Topic:
if topic.topicId is None or check_fake_id(topic.topicId):
topic.topicId = get_surrogate_key()
save_topic(topic)
result = Topic.parse_obj(topic)
if settings.INDEX_ON and topic.type != RAW:
factor_index_service.create_factor_index_data(result, topic.tenantId)
return result
def load_topic_by_name_and_tenant(name, tenant_id, current_user):
headers = build_headers(current_user)
response = requests.get(settings.WATCHMEN_HOST + "topic/name/tenant",
params={
"name": name,
"tenant_id": tenant_id
},
headers=headers)
return Topic.parse_obj(response.json())
def get_another_topic(self, another_topic_id) -> Topic:
headers = build_headers(self.current_user)
response = requests.get(settings.WATCHMEN_HOST + "topic",
params={
"topic_id": another_topic_id,
"tenant_id": self.current_user.tenantId
},
headers=headers)
result = Topic.parse_obj(response.json())
return result
def get_topic(self) -> Topic:
headers = build_headers(self.current_user)
response = requests.get(settings.WATCHMEN_HOST + "topic/name/tenant",
params={
"name": self.topic_name,
"tenant_id": self.current_user.tenantId
},
headers=headers)
result = Topic.parse_obj(response.json())
return result
def build_topic(model_schema_set: ModelSchemaSet, current_user):
topic = Topic()
topic.tenantId = current_user.tenantId
topic.topicId = get_surrogate_key()
topic.name = model_schema_set.code
topic.type = "raw"
topic.factors = []
parent = ""
build_factors(topic.factors, parent, model_schema_set.schemas[topic.name],
model_schema_set)
create_topic_schema(topic)
def insert_monitor_topic():
monitor_topic = get_topic_by_name("raw_pipeline_monitor")
if monitor_topic is None:
topic = Topic()
topic.topicId = get_surrogate_key()
topic.name = "raw_pipeline_monitor"
topic.type = "raw"
topic.kind = "system"
save_topic(topic)
def create_raw_topic(code, data, current_user):
topic = Topic()
topic.topicId = get_surrogate_key()
topic.tenantId = current_user.tenantId
topic.name = code
topic.type = "raw"
topic.factors = []
queue = deque([])
if type(data) == list:
for record in data:
model: dict = {"root": record}
queue.append(model)
create_factors(queue, topic)
create_topic_schema(topic)
def create_raw_topic_v3(code, data, current_user):
topic = Topic()
topic.topicId = get_surrogate_key()
topic.tenantId = current_user.tenantId
topic.name = code
topic.type = "raw"
topic.factors = []
queue = deque([])
rs = Relationship()
if type(data) == list:
for record in data:
model: dict = {"root": record}
queue.append(model)
create_factors(queue, topic, rs)
mp = rs.get_mapping()
parent = []
child = mp.get("root", [])
generate_surrogate_key("root", parent, child, mp, topic)
create_topic_schema(topic)
else:
raise ValueError("the data must be array")
def update_topic_schema(topic_id, topic: Topic):
update_topic(topic_id, topic)
result = Topic.parse_obj(topic)
if settings.INDEX_ON and topic.type != RAW:
factor_index_service.update_factor_index_data(result, topic.tenantId)
return result
def get_topic_by_id(topic_id, current_user: TokenUser):
headers = build_headers(current_user)
response = requests.get(settings.WATCHMEN_HOST + "topic",
params={"topic_id": topic_id},
headers=headers)
return Topic.parse_obj(response.json())
def load_topic_list_without_raw_topic_by_tenant(current_user):
topic_list = load_topic_list_by_tenant(current_user)
filtered = filter(lambda topic: topic["type"] != "raw" and topic.get("kind") == "business", topic_list)
return [Topic.parse_obj(result) for result in list(filtered)]