def build_factors(factors: list, parent: str, model_schema: ModelSchema, model_schema_set: ModelSchemaSet): for key, value in model_schema.businessFields.items(): if value.type == "array" or value.type == "dict": factor = Factor() if parent == "": factor.name = key factor.label = key build_factors(factors, key, model_schema_set.schemas[key], model_schema_set) else: factor.name = parent + "." + key factor.label = parent + "." + key build_factors(factors, parent + "." + key, model_schema_set.schemas[key], model_schema_set) factor.type = value.type factor.factorId = get_surrogate_key() factors.append(factor) else: factor = Factor() if parent != "": factor.name = parent + "." + key factor.label = parent + "." + key else: factor.name = key factor.label = key factor.type = value.type factor.factorId = get_surrogate_key() factors.append(factor)
def save_console_space(console_space: ConsoleSpace) -> ConsoleSpace: if console_space.connectId is None or check_fake_id( console_space.connectId): console_space.connectId = get_surrogate_key() return create_console_space(console_space) else: return update_console_space(console_space)
async def save_tenant( tenant: Tenant, current_user: User = Depends(deps.get_current_user)) -> Tenant: if check_fake_id(tenant.tenantId): tenant.tenantId = get_surrogate_key() return tenant_service.create(tenant) else: return tenant_service.update(tenant)
async def save_external_writer(external_writer: ExternalWriter, current_user: User = Depends( deps.get_current_user)): if check_fake_id(external_writer.writerId): external_writer.writerId = get_surrogate_key() return external_storage.create(external_writer) else: return external_storage.update(external_writer)
def save_data_source(data_source: DataSource, current_user: User = None): if check_fake_id(data_source.dataSourceId): data_source.dataSourceId = get_surrogate_key() return storage_template.insert_one(data_source, DataSource, DATA_SOURCES) else: return storage_template.update_one(data_source, DataSource, DATA_SOURCES)
def build_query_monitor_report(report: Report, query_type: str): query_monitor = QueryMonitor() query_monitor.queryUid = get_surrogate_key() query_source = QuerySource() query_source.name = report.name query_source.queryType = query_type query_monitor.querySource = query_source return query_monitor
def create_topic_schema(topic: Topic) -> Topic: if topic.topicId is None or check_fake_id(topic.topicId): topic.topicId = get_surrogate_key() save_topic(topic) result = Topic.parse_obj(topic) if settings.INDEX_ON and topic.type != RAW: factor_index_service.create_factor_index_data(result, topic.tenantId) return result
def build_query_monitor(subject: ConsoleSpaceSubject, query_type: str): query_monitor = QueryMonitor() query_monitor.queryUid = get_surrogate_key() query_source = QuerySource() query_source.name = subject.name query_source.queryType = query_type query_monitor.querySource = query_source return query_monitor
def buildFactorLabels(): labels = [] label = Label(**{ 'id': get_surrogate_key(), 'name': 'type', 'value': 'factor' }) labels.append(label) return labels
async def push_pipeline_data_async(topic_event: TopicEvent, current_user: User = Depends(deps.get_current_user)): trace_id = get_surrogate_key() # # create_raw_topic_instance topic = await __load_topic_definition(topic_event.code, current_user) data = get_input_data(topic, topic_event) await save_topic_data(topic, data, current_user) asyncio.ensure_future(run_pipeline(topic_event, current_user, trace_id)) return {"received": True, "trace_id": trace_id}
def insert_monitor_topic(): monitor_topic = get_topic_by_name("raw_pipeline_monitor") if monitor_topic is None: topic = Topic() topic.topicId = get_surrogate_key() topic.name = "raw_pipeline_monitor" topic.type = "raw" topic.kind = "system" save_topic(topic)
def buildPipelineLabels(): labels = [] label = Label(**{ 'id': get_surrogate_key(), 'name': 'type', 'value': 'pipeline' }) labels.append(label) return labels
async def save_pipeline_index(pipeline: Pipeline, current_user): await pipeline_index_storage.delete_pipeline_index_list_by_pipeline_id(pipeline.pipelineId, current_user) pipeline_index_list = await build_pipeline_index_list(pipeline, {}, current_user) for pipeline_index in pipeline_index_list: if pipeline_index.pipelineIndexId is None: pipeline_index.pipelineIndexId = get_surrogate_key() await pipeline_index_storage.create_pipeline_index(pipeline_index) else: await pipeline_index_storage.update_pipeline_index(pipeline_index)
async def push_pipeline_data_async_tenant(topic_event: TopicEvent, current_user: User = Depends(deps.get_current_user)): trace_id = get_surrogate_key() topic = get_topic_by_name_and_tenant_id(topic_event.code, topic_event.tenantId) data = get_input_data(topic, topic_event) current_user.tenantId = topic_event.tenantId await save_topic_data(topic, data, current_user) asyncio.ensure_future(run_pipeline(topic_event, current_user, trace_id)) return {"received": True, "trace_id": trace_id}
def create_raw_data_model_set(code, data): model_schema_set = ModelSchemaSet() model_schema_set.id = get_surrogate_key() model_schema_set.code = code model_schema_set.schemas = {} model_schema_set.relationships = {} create_schema(model_schema_set, code, data, True) # insert_data_schema(model_schema_set.dict()) return model_schema_set
def buildTopicLabels(topic): labels = [] label = Label(**{ 'id': get_surrogate_key(), 'name': 'type', 'value': 'topic' }) labels.append(label) return labels
async def save_monitor_rule(rule_list: List[MonitorRule], current_user=Depends(deps.get_current_user)): for monitor_rule in rule_list: if monitor_rule.ruleId is None: monitor_rule.ruleId = get_surrogate_key() result = load_monitor_rule(monitor_rule, current_user) if result is None: create_monitor_rule(monitor_rule, current_user) else: update_monitor_rule(monitor_rule, current_user) return rule_list
async def save_report(subject_id: str, report: Report, current_user: User = Depends(deps.get_current_user)): report = add_tenant_id_to_model(report, current_user) report.reportId = get_surrogate_key() new_report = create_report(report) subject = load_console_subject_by_id(subject_id, current_user) subject.reportIds.append(new_report.reportId) update_console_subject(subject) return new_report
def buildRelationShip(left, right, properties): relationship = Relationship(**{ 'id': get_surrogate_key(), 'name': 'link', 'direction': 'one-way', 'properties': properties, 'left': left, 'right': right }) return relationship
def buildFactorProperties(properties: dict): property_list = [] for key, value in properties.items(): property_ = Property(**{ 'id': get_surrogate_key(), 'name': key, 'value': value }) property_list.append(property_) return property_list
def generate_surrogate_key(current, parent, child, mp, topic): if current == "root": parent.append("root") for l_node in child: next_ = mp.get(l_node, None) if next_: generate_surrogate_key(l_node, parent, next_, mp, topic) else: factor = Factor() factor.name = current + "." + "aid_me" factor.label = current + "." + "aid_me" factor.type = "number" factor.factorId = get_surrogate_key() topic.factors.append(factor) duplicate = [] for p_node in reversed(parent): if p_node != "root": factor = Factor() if "." in p_node: name_ = p_node.rsplit(".", 1)[1] else: name_ = p_node if check_duplicate_aid_name(name_, duplicate): distance = current.count(".") - p_node.count(".") factor.name = current + "." + "aid_" + name_ + "_" + str( distance) factor.label = current + "." + "aid_" + name_ + "_" + str( distance) else: factor.name = current + "." + "aid_" + name_ factor.label = current + "." + "aid_" + name_ duplicate.append(name_) factor.type = "number" factor.factorId = get_surrogate_key() topic.factors.append(factor) parent.append(current) for l_node in child: next_ = mp.get(l_node, None) if next_: generate_surrogate_key(l_node, parent, next_, mp, topic) else: generate_surrogate_key(l_node, parent, [], mp, topic) parent.remove(current)
def build_topic(model_schema_set: ModelSchemaSet, current_user): topic = Topic() topic.tenantId = current_user.tenantId topic.topicId = get_surrogate_key() topic.name = model_schema_set.code topic.type = "raw" topic.factors = [] parent = "" build_factors(topic.factors, parent, model_schema_set.schemas[topic.name], model_schema_set) create_topic_schema(topic)
def createPAT(note: str, user_id: str, username: str, tenant_id: str): pat = PersonAccessToken() pat.patId = get_surrogate_key() pat.tokenId = create_token() pat.userId = user_id pat.username = username pat.tenantId = tenant_id pat.note = note storage_template.insert_one(pat, PersonAccessToken, "pats") result = {'patId': pat.patId, 'note': pat.note, 'token': pat.tokenId} return result
def build_factor_index(factor, topic): factor_index = FactorIndex() factor_index.factorIndexId = get_surrogate_key() factor_index.type = factor.type factor_index.factorId = factor.factorId factor_index.tenantId = topic.tenantId factor_index.name = factor.name factor_index.label = factor.label factor_index.topicId = topic.topicId factor_index.topicName = topic.name factor_index.description = factor.description factor_index.createTime = datetime.now().replace(tzinfo=None).isoformat() return factor_index
def create_factors(queue, topic): model = queue.popleft() for key, value in model.items(): if not isinstance(value, dict): raise TypeError("create factors need the dict type value, \'{0}\' is not dict".format(value)) else: for factor_name, factor_value in value.items(): result_tup = check_factor_in_topic(key, factor_name, topic) if result_tup[0]: factor = result_tup[1] factor_value_type = check_value_type(factor_value) if factor_value_type != ValueType.ANY and factor.type == ValueType.ANY: factor.type = factor_value_type elif factor_value_type != ValueType.ANY and factor.type != ValueType.ANY and \ factor_value_type != factor.type: raise Exception("factor {0} has different value type: {1} and {2}".format(factor.name, factor_value_type, factor.type)) if (factor_value_type == ValueType.LIST and check_list_element_type_is_object( factor_value)): if key == "root": queue.append({factor_name: factor_value[0]}) else: queue.append({key + "." + factor_name: factor_value[0]}) if factor_value_type == ValueType.DICT: if key == "root": queue.append({factor_name: factor_value}) else: queue.append({key + "." + factor_name: factor_value}) else: factor_value_type = check_value_type(factor_value) factor = Factor() factor.name = factor_name if key == "root" else key + "." + factor_name factor.label = factor_name if key == "root" else key + "." + factor_name factor.type = factor_value_type factor.factorId = get_surrogate_key() topic.factors.append(factor) if factor_value_type == ValueType.LIST and check_list_element_type_is_object(factor_value): if key == "root": queue.append({factor_name: factor_value[0]}) else: queue.append({key + "." + factor_name: factor_value[0]}) if factor_value_type == ValueType.DICT: if key == "root": queue.append({factor_name: factor_value}) else: queue.append({key + "." + factor_name: factor_value}) if len(queue) != 0: create_factors(queue, topic)
def create_raw_topic(code, data, current_user): topic = Topic() topic.topicId = get_surrogate_key() topic.tenantId = current_user.tenantId topic.name = code topic.type = "raw" topic.factors = [] queue = deque([]) if type(data) == list: for record in data: model: dict = {"root": record} queue.append(model) create_factors(queue, topic) create_topic_schema(topic)
async def push_pipeline_data(topic_event: TopicEvent, current_user: User = Depends(deps.get_current_user)): trace_id = get_surrogate_key() enter_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') print(f"The request trace id is {trace_id}, entry_time is {enter_time}.") topic = await __load_topic_definition(topic_event.code, current_user) data = get_input_data(topic, topic_event) before_save_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') print(f"The request trace id is {trace_id}, before_save_time is {before_save_time}.") await save_topic_data(topic, data, current_user) after_save_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') print(f"The request trace id is {trace_id}, after_save_time is {after_save_time}.") await run_pipeline(topic_event, current_user, trace_id) after_run_pipeline_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') print(f"The request trace id is {trace_id}, after_run_pipeline_time is {after_run_pipeline_time}.") return {"received": True, "trace_id": trace_id}
async def rerun_pipeline(topic_name, instance_id, pipeline_id=None, current_user: User = Depends(deps.get_current_user)): topic = get_topic(topic_name) trace_id = get_surrogate_key() instance = find_topic_data_by_id_and_topic_name(topic, instance_id) data = {"new": instance, "old": None} pipeline_list = load_pipeline_by_topic_id(topic.topicId) for pipeline in find_execute_pipeline_list(pipeline_id, pipeline_list): log.info("rerun topic {0} and pipeline {1}".format( topic_name, pipeline.name)) pipeline_context = PipelineContext(pipeline, data, current_user, trace_id) run_pipeline(pipeline_context, current_user) return {"received": True, "trace_id": trace_id}
async def rerun_pipeline(topic_name, instance_id, pipeline_id, current_user: User = Depends(deps.get_current_user)): topic = get_topic(topic_name) trace_id = get_surrogate_key() instance = find_topic_data_by_id_and_topic_name(topic, instance_id) pipeline_list = load_pipeline_by_topic_id(topic.topicId) for pipeline in pipeline_list: if pipeline.pipelineId == pipeline_id: log.info("rerun topic {0} and pipeline {1}".format( topic_name, pipeline.pipelineId)) pipeline_context = PipelineContext(pipeline, instance, current_user, trace_id) run_pipeline(pipeline_context, current_user) return {"received": True}
def save_enum_to_storage(enum: Enum): if check_fake_id(enum.enumId): enum.enumId = get_surrogate_key() # enum.items = [] # result = template.create(ENUMS, enum, Enum) result = storage_template.insert_one(enum, Enum, ENUMS) # items = __add_enum_id(items_copy, result.enumId) # save_enum_items_to_storage(items_copy, enum.name) return result else: # items_copy = enum.items.copy() # # enum.items = [] # # items = __add_enum_id(items_copy, enum.enumId) # save_enum_items_to_storage(items_copy, enum.name) # # return template.update_one(ENUMS, {"enumId": enum.enumId}, enum, Enum) return storage_template.update_one(enum, Enum, ENUMS)