def build_factors(factors: list, parent: str, model_schema: ModelSchema,
model_schema_set: ModelSchemaSet):
for key, value in model_schema.businessFields.items():
if value.type == "array" or value.type == "dict":
factor = Factor()
if parent == "":
factor.name = key
factor.label = key
build_factors(factors, key, model_schema_set.schemas[key],
model_schema_set)
else:
factor.name = parent + "." + key
factor.label = parent + "." + key
build_factors(factors, parent + "." + key,
model_schema_set.schemas[key], model_schema_set)
factor.type = value.type
factor.factorId = get_surrogate_key()
factors.append(factor)
else:
factor = Factor()
if parent != "":
factor.name = parent + "." + key
factor.label = parent + "." + key
else:
factor.name = key
factor.label = key
factor.type = value.type
factor.factorId = get_surrogate_key()
factors.append(factor)
def save_console_space(console_space: ConsoleSpace) -> ConsoleSpace:
if console_space.connectId is None or check_fake_id(
console_space.connectId):
console_space.connectId = get_surrogate_key()
return create_console_space(console_space)
else:
return update_console_space(console_space)
async def save_tenant(
tenant: Tenant,
current_user: User = Depends(deps.get_current_user)) -> Tenant:
if check_fake_id(tenant.tenantId):
tenant.tenantId = get_surrogate_key()
return tenant_service.create(tenant)
else:
return tenant_service.update(tenant)
async def save_external_writer(external_writer: ExternalWriter,
current_user: User = Depends(
deps.get_current_user)):
if check_fake_id(external_writer.writerId):
external_writer.writerId = get_surrogate_key()
return external_storage.create(external_writer)
else:
return external_storage.update(external_writer)
def save_data_source(data_source: DataSource, current_user: User = None):
if check_fake_id(data_source.dataSourceId):
data_source.dataSourceId = get_surrogate_key()
return storage_template.insert_one(data_source, DataSource,
DATA_SOURCES)
else:
return storage_template.update_one(data_source, DataSource,
DATA_SOURCES)
def build_query_monitor_report(report: Report, query_type: str):
query_monitor = QueryMonitor()
query_monitor.queryUid = get_surrogate_key()
query_source = QuerySource()
query_source.name = report.name
query_source.queryType = query_type
query_monitor.querySource = query_source
return query_monitor
def create_topic_schema(topic: Topic) -> Topic:
if topic.topicId is None or check_fake_id(topic.topicId):
topic.topicId = get_surrogate_key()
save_topic(topic)
result = Topic.parse_obj(topic)
if settings.INDEX_ON and topic.type != RAW:
factor_index_service.create_factor_index_data(result, topic.tenantId)
return result
def build_query_monitor(subject: ConsoleSpaceSubject, query_type: str):
query_monitor = QueryMonitor()
query_monitor.queryUid = get_surrogate_key()
query_source = QuerySource()
query_source.name = subject.name
query_source.queryType = query_type
query_monitor.querySource = query_source
return query_monitor
def buildFactorLabels():
labels = []
label = Label(**{
'id': get_surrogate_key(),
'name': 'type',
'value': 'factor'
})
labels.append(label)
return labels
async def push_pipeline_data_async(topic_event: TopicEvent, current_user: User = Depends(deps.get_current_user)):
trace_id = get_surrogate_key()
#
# create_raw_topic_instance
topic = await __load_topic_definition(topic_event.code, current_user)
data = get_input_data(topic, topic_event)
await save_topic_data(topic, data, current_user)
asyncio.ensure_future(run_pipeline(topic_event, current_user, trace_id))
return {"received": True, "trace_id": trace_id}
def insert_monitor_topic():
monitor_topic = get_topic_by_name("raw_pipeline_monitor")
if monitor_topic is None:
topic = Topic()
topic.topicId = get_surrogate_key()
topic.name = "raw_pipeline_monitor"
topic.type = "raw"
topic.kind = "system"
save_topic(topic)
def buildPipelineLabels():
labels = []
label = Label(**{
'id': get_surrogate_key(),
'name': 'type',
'value': 'pipeline'
})
labels.append(label)
return labels
async def save_pipeline_index(pipeline: Pipeline, current_user):
await pipeline_index_storage.delete_pipeline_index_list_by_pipeline_id(pipeline.pipelineId, current_user)
pipeline_index_list = await build_pipeline_index_list(pipeline, {}, current_user)
for pipeline_index in pipeline_index_list:
if pipeline_index.pipelineIndexId is None:
pipeline_index.pipelineIndexId = get_surrogate_key()
await pipeline_index_storage.create_pipeline_index(pipeline_index)
else:
await pipeline_index_storage.update_pipeline_index(pipeline_index)
async def push_pipeline_data_async_tenant(topic_event: TopicEvent, current_user: User = Depends(deps.get_current_user)):
trace_id = get_surrogate_key()
topic = get_topic_by_name_and_tenant_id(topic_event.code,
topic_event.tenantId)
data = get_input_data(topic, topic_event)
current_user.tenantId = topic_event.tenantId
await save_topic_data(topic, data, current_user)
asyncio.ensure_future(run_pipeline(topic_event, current_user, trace_id))
return {"received": True, "trace_id": trace_id}
def create_raw_data_model_set(code, data):
model_schema_set = ModelSchemaSet()
model_schema_set.id = get_surrogate_key()
model_schema_set.code = code
model_schema_set.schemas = {}
model_schema_set.relationships = {}
create_schema(model_schema_set, code, data, True)
# insert_data_schema(model_schema_set.dict())
return model_schema_set
def buildTopicLabels(topic):
labels = []
label = Label(**{
'id': get_surrogate_key(),
'name': 'type',
'value': 'topic'
})
labels.append(label)
return labels
async def save_monitor_rule(rule_list: List[MonitorRule], current_user=Depends(deps.get_current_user)):
for monitor_rule in rule_list:
if monitor_rule.ruleId is None:
monitor_rule.ruleId = get_surrogate_key()
result = load_monitor_rule(monitor_rule, current_user)
if result is None:
create_monitor_rule(monitor_rule, current_user)
else:
update_monitor_rule(monitor_rule, current_user)
return rule_list
async def save_report(subject_id: str,
report: Report,
current_user: User = Depends(deps.get_current_user)):
report = add_tenant_id_to_model(report, current_user)
report.reportId = get_surrogate_key()
new_report = create_report(report)
subject = load_console_subject_by_id(subject_id, current_user)
subject.reportIds.append(new_report.reportId)
update_console_subject(subject)
return new_report
def buildRelationShip(left, right, properties):
relationship = Relationship(**{
'id': get_surrogate_key(),
'name': 'link',
'direction': 'one-way',
'properties': properties,
'left': left,
'right': right
})
return relationship
def buildFactorProperties(properties: dict):
property_list = []
for key, value in properties.items():
property_ = Property(**{
'id': get_surrogate_key(),
'name': key,
'value': value
})
property_list.append(property_)
return property_list
def generate_surrogate_key(current, parent, child, mp, topic):
if current == "root":
parent.append("root")
for l_node in child:
next_ = mp.get(l_node, None)
if next_:
generate_surrogate_key(l_node, parent, next_, mp, topic)
else:
factor = Factor()
factor.name = current + "." + "aid_me"
factor.label = current + "." + "aid_me"
factor.type = "number"
factor.factorId = get_surrogate_key()
topic.factors.append(factor)
duplicate = []
for p_node in reversed(parent):
if p_node != "root":
factor = Factor()
if "." in p_node:
name_ = p_node.rsplit(".", 1)[1]
else:
name_ = p_node
if check_duplicate_aid_name(name_, duplicate):
distance = current.count(".") - p_node.count(".")
factor.name = current + "." + "aid_" + name_ + "_" + str(
distance)
factor.label = current + "." + "aid_" + name_ + "_" + str(
distance)
else:
factor.name = current + "." + "aid_" + name_
factor.label = current + "." + "aid_" + name_
duplicate.append(name_)
factor.type = "number"
factor.factorId = get_surrogate_key()
topic.factors.append(factor)
parent.append(current)
for l_node in child:
next_ = mp.get(l_node, None)
if next_:
generate_surrogate_key(l_node, parent, next_, mp, topic)
else:
generate_surrogate_key(l_node, parent, [], mp, topic)
parent.remove(current)
def build_topic(model_schema_set: ModelSchemaSet, current_user):
topic = Topic()
topic.tenantId = current_user.tenantId
topic.topicId = get_surrogate_key()
topic.name = model_schema_set.code
topic.type = "raw"
topic.factors = []
parent = ""
build_factors(topic.factors, parent, model_schema_set.schemas[topic.name],
model_schema_set)
create_topic_schema(topic)
def createPAT(note: str, user_id: str, username: str, tenant_id: str):
pat = PersonAccessToken()
pat.patId = get_surrogate_key()
pat.tokenId = create_token()
pat.userId = user_id
pat.username = username
pat.tenantId = tenant_id
pat.note = note
storage_template.insert_one(pat, PersonAccessToken, "pats")
result = {'patId': pat.patId, 'note': pat.note, 'token': pat.tokenId}
return result
def build_factor_index(factor, topic):
factor_index = FactorIndex()
factor_index.factorIndexId = get_surrogate_key()
factor_index.type = factor.type
factor_index.factorId = factor.factorId
factor_index.tenantId = topic.tenantId
factor_index.name = factor.name
factor_index.label = factor.label
factor_index.topicId = topic.topicId
factor_index.topicName = topic.name
factor_index.description = factor.description
factor_index.createTime = datetime.now().replace(tzinfo=None).isoformat()
return factor_index
def create_factors(queue, topic):
model = queue.popleft()
for key, value in model.items():
if not isinstance(value, dict):
raise TypeError("create factors need the dict type value, \'{0}\' is not dict".format(value))
else:
for factor_name, factor_value in value.items():
result_tup = check_factor_in_topic(key, factor_name, topic)
if result_tup[0]:
factor = result_tup[1]
factor_value_type = check_value_type(factor_value)
if factor_value_type != ValueType.ANY and factor.type == ValueType.ANY:
factor.type = factor_value_type
elif factor_value_type != ValueType.ANY and factor.type != ValueType.ANY and \
factor_value_type != factor.type:
raise Exception("factor {0} has different value type: {1} and {2}".format(factor.name,
factor_value_type,
factor.type))
if (factor_value_type == ValueType.LIST and check_list_element_type_is_object(
factor_value)):
if key == "root":
queue.append({factor_name: factor_value[0]})
else:
queue.append({key + "." + factor_name: factor_value[0]})
if factor_value_type == ValueType.DICT:
if key == "root":
queue.append({factor_name: factor_value})
else:
queue.append({key + "." + factor_name: factor_value})
else:
factor_value_type = check_value_type(factor_value)
factor = Factor()
factor.name = factor_name if key == "root" else key + "." + factor_name
factor.label = factor_name if key == "root" else key + "." + factor_name
factor.type = factor_value_type
factor.factorId = get_surrogate_key()
topic.factors.append(factor)
if factor_value_type == ValueType.LIST and check_list_element_type_is_object(factor_value):
if key == "root":
queue.append({factor_name: factor_value[0]})
else:
queue.append({key + "." + factor_name: factor_value[0]})
if factor_value_type == ValueType.DICT:
if key == "root":
queue.append({factor_name: factor_value})
else:
queue.append({key + "." + factor_name: factor_value})
if len(queue) != 0:
create_factors(queue, topic)
def create_raw_topic(code, data, current_user):
topic = Topic()
topic.topicId = get_surrogate_key()
topic.tenantId = current_user.tenantId
topic.name = code
topic.type = "raw"
topic.factors = []
queue = deque([])
if type(data) == list:
for record in data:
model: dict = {"root": record}
queue.append(model)
create_factors(queue, topic)
create_topic_schema(topic)
async def push_pipeline_data(topic_event: TopicEvent, current_user: User = Depends(deps.get_current_user)):
trace_id = get_surrogate_key()
enter_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print(f"The request trace id is {trace_id}, entry_time is {enter_time}.")
topic = await __load_topic_definition(topic_event.code, current_user)
data = get_input_data(topic, topic_event)
before_save_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print(f"The request trace id is {trace_id}, before_save_time is {before_save_time}.")
await save_topic_data(topic, data, current_user)
after_save_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print(f"The request trace id is {trace_id}, after_save_time is {after_save_time}.")
await run_pipeline(topic_event, current_user, trace_id)
after_run_pipeline_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print(f"The request trace id is {trace_id}, after_run_pipeline_time is {after_run_pipeline_time}.")
return {"received": True, "trace_id": trace_id}
async def rerun_pipeline(topic_name,
instance_id,
pipeline_id=None,
current_user: User = Depends(deps.get_current_user)):
topic = get_topic(topic_name)
trace_id = get_surrogate_key()
instance = find_topic_data_by_id_and_topic_name(topic, instance_id)
data = {"new": instance, "old": None}
pipeline_list = load_pipeline_by_topic_id(topic.topicId)
for pipeline in find_execute_pipeline_list(pipeline_id, pipeline_list):
log.info("rerun topic {0} and pipeline {1}".format(
topic_name, pipeline.name))
pipeline_context = PipelineContext(pipeline, data, current_user,
trace_id)
run_pipeline(pipeline_context, current_user)
return {"received": True, "trace_id": trace_id}
async def rerun_pipeline(topic_name,
instance_id,
pipeline_id,
current_user: User = Depends(deps.get_current_user)):
topic = get_topic(topic_name)
trace_id = get_surrogate_key()
instance = find_topic_data_by_id_and_topic_name(topic, instance_id)
pipeline_list = load_pipeline_by_topic_id(topic.topicId)
for pipeline in pipeline_list:
if pipeline.pipelineId == pipeline_id:
log.info("rerun topic {0} and pipeline {1}".format(
topic_name, pipeline.pipelineId))
pipeline_context = PipelineContext(pipeline, instance,
current_user, trace_id)
run_pipeline(pipeline_context, current_user)
return {"received": True}
def save_enum_to_storage(enum: Enum):
if check_fake_id(enum.enumId):
enum.enumId = get_surrogate_key()
# enum.items = []
# result = template.create(ENUMS, enum, Enum)
result = storage_template.insert_one(enum, Enum, ENUMS)
# items = __add_enum_id(items_copy, result.enumId)
# save_enum_items_to_storage(items_copy, enum.name)
return result
else:
# items_copy = enum.items.copy()
# # enum.items = []
# # items = __add_enum_id(items_copy, enum.enumId)
# save_enum_items_to_storage(items_copy, enum.name)
# # return template.update_one(ENUMS, {"enumId": enum.enumId}, enum, Enum)
return storage_template.update_one(enum, Enum, ENUMS)
