def __trigger_all_pipeline(pipeline_trigger_merge_list, current_user=None, trace_id=None): after_merge_list = __merge_pipeline_data(pipeline_trigger_merge_list) for topic_name, item in after_merge_list.items(): merge_data = {} topic = get_topic_by_name(topic_name, current_user) if TriggerType.update.value in item: for update_data in item[TriggerType.update.value]: old_value = update_data[pipeline_constants.OLD] pk = old_value[get_id_name_by_datasource( data_source_container.get_data_source_by_id( topic.dataSourceId))] if pk in merge_data: merge_data[pk][pipeline_constants.NEW].update( update_data[pipeline_constants.NEW]) else: merge_data[pk] = { pipeline_constants.NEW: update_data[pipeline_constants.NEW], pipeline_constants.OLD: update_data[pipeline_constants.OLD] } for key, data in merge_data.items(): watchmen.pipeline.index.trigger_pipeline( topic_name, data, TriggerType.update, current_user, trace_id) if TriggerType.insert.value in item: for insert_data in item[TriggerType.insert.value]: watchmen.pipeline.index.trigger_pipeline( topic_name, insert_data, TriggerType.insert, current_user, trace_id)
def process_date_diff_args( self, args: str) -> Tuple[CustomFunction, CustomFunction]: arg_list = args.split(",") if len(arg_list) != 2: raise ValueError("Date_diff have invalid args", arg_list) result = [] for arg in arg_list: if arg == "now": date = current_date() else: date_fnc = CustomFunction("date", ["col1"]) if "." in arg: items = arg.split(".") topic_name = items[0].strip() topic = get_topic_by_name(topic_name, None) table = None if self.topic_space_filter: if self.topic_space_filter(self.param.topicId): alias_ = self.topic_space_filter( self.param.topicId)["alias"] table = AliasedQuery(alias_) if table is None: table = build_table_by_topic_id(topic.topicId) factor_name = items[1].strip() date = date_fnc(Field(factor_name, None, table)) else: date = date_fnc(arg) result.append(date) return tuple(result)
async def load_topic_instance(topic_name, current_user: User = Depends( deps.get_current_user)): topic: Topic = get_topic_by_name(topic_name, current_user) results = get_topic_instances_all(topic) instances = [] for result in results: instances.append(TopicInstance(data=result)) return instances
def insert_monitor_topic(): monitor_topic = get_topic_by_name("raw_pipeline_monitor") if monitor_topic is None: topic = Topic() topic.topicId = get_surrogate_key() topic.name = "raw_pipeline_monitor" topic.type = "raw" topic.kind = "system" save_topic(topic)
async def patch_topic_instance(topic_name, instance_id, instance=Body(...), current_user: User = Depends( deps.get_current_user)): topic = get_topic_by_name(topic_name) result = find_topic_data_by_id_and_topic_name(topic, instance_id) if result is None: raise Exception("topic {0} id {1} not found data ".format( topic_name, instance_id)) else: update_topic_instance(topic, instance, instance_id)
async def patch_topic_instance(topic_name, instance_id=None, instance=Body(...), current_user: User = Depends( deps.get_current_user)): topic = get_topic_by_name(topic_name, current_user) if instance_id is None: add_audit_columns(instance, INSERT) return save_topic_instance(topic, instance, current_user) else: result = find_topic_data_by_id_and_topic_name(topic, instance_id) if result is not None: add_audit_columns(instance, UPDATE) return update_topic_instance(topic, instance, instance_id) else: raise Exception( "instance ID {0} could not find any data for update".format( instance_id))
def get_nodes(code, current_user): topic = get_topic_by_name(code, current_user) factors = topic.factors nodes = {} for item in factors: if ".aid_" in item.name: factor_name = item.name list_ = factor_name.rsplit(".", 1) node_name = list_[0] key_name = list_[1] node_info = nodes.get(node_name, None) if node_info: node_info.aid_keys.append(key_name) else: node_info = NodeInfo( **{ "name": node_name, "parent": [], "child": [], "aid_keys": [key_name] }) nodes[node_name] = node_info if "." in node_name: num_ = node_name.count(".") parent_name = node_name for i in range(num_): parent_name = parent_name.rsplit(".", 1)[0] parent_node_info = nodes.get(parent_name, None) if parent_node_info: parent_node_info.child.append(node_info) else: parent_node_info = NodeInfo( **{ "name": parent_name, "parent": [], "chlid": [], "aid_keys": [] }) nodes[parent_name] = parent_node_info node_info.parent.append(parent_node_info) return nodes
def find_monitor_topic(topic_name, current_user): topic = get_topic_by_name(topic_name, current_user) if topic is None: return get_topic_by_name(topic_name) else: return topic