def __trigger_all_pipeline(pipeline_trigger_merge_list,
current_user=None,
trace_id=None):
after_merge_list = __merge_pipeline_data(pipeline_trigger_merge_list)
for topic_name, item in after_merge_list.items():
merge_data = {}
topic = get_topic_by_name(topic_name, current_user)
if TriggerType.update.value in item:
for update_data in item[TriggerType.update.value]:
old_value = update_data[pipeline_constants.OLD]
pk = old_value[get_id_name_by_datasource(
data_source_container.get_data_source_by_id(
topic.dataSourceId))]
if pk in merge_data:
merge_data[pk][pipeline_constants.NEW].update(
update_data[pipeline_constants.NEW])
else:
merge_data[pk] = {
pipeline_constants.NEW:
update_data[pipeline_constants.NEW],
pipeline_constants.OLD:
update_data[pipeline_constants.OLD]
}
for key, data in merge_data.items():
watchmen.pipeline.index.trigger_pipeline(
topic_name, data, TriggerType.update, current_user,
trace_id)
if TriggerType.insert.value in item:
for insert_data in item[TriggerType.insert.value]:
watchmen.pipeline.index.trigger_pipeline(
topic_name, insert_data, TriggerType.insert, current_user,
trace_id)
def process_date_diff_args(
self, args: str) -> Tuple[CustomFunction, CustomFunction]:
arg_list = args.split(",")
if len(arg_list) != 2:
raise ValueError("Date_diff have invalid args", arg_list)
result = []
for arg in arg_list:
if arg == "now":
date = current_date()
else:
date_fnc = CustomFunction("date", ["col1"])
if "." in arg:
items = arg.split(".")
topic_name = items[0].strip()
topic = get_topic_by_name(topic_name, None)
table = None
if self.topic_space_filter:
if self.topic_space_filter(self.param.topicId):
alias_ = self.topic_space_filter(
self.param.topicId)["alias"]
table = AliasedQuery(alias_)
if table is None:
table = build_table_by_topic_id(topic.topicId)
factor_name = items[1].strip()
date = date_fnc(Field(factor_name, None, table))
else:
date = date_fnc(arg)
result.append(date)
return tuple(result)
async def load_topic_instance(topic_name,
current_user: User = Depends(
deps.get_current_user)):
topic: Topic = get_topic_by_name(topic_name, current_user)
results = get_topic_instances_all(topic)
instances = []
for result in results:
instances.append(TopicInstance(data=result))
return instances
def insert_monitor_topic():
monitor_topic = get_topic_by_name("raw_pipeline_monitor")
if monitor_topic is None:
topic = Topic()
topic.topicId = get_surrogate_key()
topic.name = "raw_pipeline_monitor"
topic.type = "raw"
topic.kind = "system"
save_topic(topic)
async def patch_topic_instance(topic_name,
instance_id,
instance=Body(...),
current_user: User = Depends(
deps.get_current_user)):
topic = get_topic_by_name(topic_name)
result = find_topic_data_by_id_and_topic_name(topic, instance_id)
if result is None:
raise Exception("topic {0} id {1} not found data ".format(
topic_name, instance_id))
else:
update_topic_instance(topic, instance, instance_id)
async def patch_topic_instance(topic_name,
instance_id=None,
instance=Body(...),
current_user: User = Depends(
deps.get_current_user)):
topic = get_topic_by_name(topic_name, current_user)
if instance_id is None:
add_audit_columns(instance, INSERT)
return save_topic_instance(topic, instance, current_user)
else:
result = find_topic_data_by_id_and_topic_name(topic, instance_id)
if result is not None:
add_audit_columns(instance, UPDATE)
return update_topic_instance(topic, instance, instance_id)
else:
raise Exception(
"instance ID {0} could not find any data for update".format(
instance_id))
def get_nodes(code, current_user):
topic = get_topic_by_name(code, current_user)
factors = topic.factors
nodes = {}
for item in factors:
if ".aid_" in item.name:
factor_name = item.name
list_ = factor_name.rsplit(".", 1)
node_name = list_[0]
key_name = list_[1]
node_info = nodes.get(node_name, None)
if node_info:
node_info.aid_keys.append(key_name)
else:
node_info = NodeInfo(
**{
"name": node_name,
"parent": [],
"child": [],
"aid_keys": [key_name]
})
nodes[node_name] = node_info
if "." in node_name:
num_ = node_name.count(".")
parent_name = node_name
for i in range(num_):
parent_name = parent_name.rsplit(".", 1)[0]
parent_node_info = nodes.get(parent_name, None)
if parent_node_info:
parent_node_info.child.append(node_info)
else:
parent_node_info = NodeInfo(
**{
"name": parent_name,
"parent": [],
"chlid": [],
"aid_keys": []
})
nodes[parent_name] = parent_node_info
node_info.parent.append(parent_node_info)
return nodes
def find_monitor_topic(topic_name, current_user):
topic = get_topic_by_name(topic_name, current_user)
if topic is None:
return get_topic_by_name(topic_name)
else:
return topic