def nodes_search_neighbour(
meta_node: str, id: Optional[str], limit: int = Query(50, ge=1, le=200)
):
"""Search the neighbour nodes adjacent to the query node."""
log_args(api="/meta/nodes/{meta_node}/search-neighbour", kwargs=locals())
query = nodes_neighbour_query_builder(
meta_node=meta_node, id=id, limit=limit
)
logger.info(query)
response = epigraphdb.run_query(query)
return response
def rels_list(
meta_rel: EpigraphdbMetaRels,
limit: int = Query(10, ge=1, le=2000),
offset: int = 0,
):
"""List relationships under a meta relationship."""
log_args(api="/meta/rels/{meta_rel}/list", kwargs=locals())
query = MetaQueries.get_rels.format(
meta_rel=meta_rel.value, skip=offset, limit=limit
)
logger.info(query)
response = epigraphdb.run_query(query)
return response
def __init__(
self,
df: pd.DataFrame,
node_schemas: List[NetworkNodeSchema],
edge_schemas: List[NetworkEdgeSchema],
limit: Optional[int] = 50,
visjs_option=visjs_option,
):
""""""
self.num_paths_total = len(df)
if limit is not None:
df = df[:limit]
self.num_paths_displayed = len(df)
self.visjs_option = visjs_option
self.node_schemas = node_schemas
self.edge_schemas = edge_schemas
self.nodes_df = nodes_df_from_schema(df, self.node_schemas)
logger.info(f"nodes_df: \n {self.nodes_df.head()}")
self.edges_df = edges_df_from_schema(df, self.nodes_df,
self.edge_schemas)
logger.info(f"edges_df: \n {self.edges_df.head()}")
self.nodes = render_nodes(self.nodes_df)
self.nodes_3d = render_nodes_3d(self.nodes_df)
logger.info(f"nodes: \n {self.nodes[0:2]}")
self.edges = render_edges(self.edges_df)
self.edges_3d = render_edges_3d(self.edges_df)
logger.info(f"edges: \n {self.edges[0:2]}")
def nodes_search(
meta_node: str,
id: Optional[str] = None,
name: Optional[str] = None,
limit: int = Query(10, ge=1, le=200),
full_data: bool = True,
):
"""Use `id` for exact match, and use `name` for fuzzy match.
- full_data: If False, only returns basic info (id, name).
"""
log_args(api="/meta/nodes/{meta_node}/search", kwargs=locals())
validate_at_least_one_not_none(dict(id=id, name=name))
query = nodes_search_query_builder(
meta_node=meta_node, id=id, name=name, limit=limit, full_data=full_data
)
logger.info(query)
response = epigraphdb.run_query(query)
return response
def get_paths_search(
meta_node_source: str,
id_source: str,
meta_node_target: str,
id_target: str,
max_path_length: int = Query(3, ge=1, le=5),
limit: int = Query(100, ge=1, le=500),
):
log_args(api="/meta/paths/search", kwargs=locals())
query = paths_search_query_builder(
meta_node_source=meta_node_source,
id_source=id_source,
meta_node_target=meta_node_target,
id_target=id_target,
max_path_length=max_path_length,
limit=limit,
)
logger.info(query)
response = epigraphdb.run_query(query)
return response
def nodes_list(
meta_node: EpigraphdbMetaNodesFull,
full_data: bool = True,
limit: int = Query(10, ge=1, le=10_000),
offset: int = 0,
):
"""List nodes under a meta node.
- `limit`: If you supply full_data to be True, the limit is 500,
otherwise the limit is 10,000
- `full_data`: When False, only return the id and name fields for
a node.
For the specific id and name fields, refer to /meta/nodes/id-name-schema.
"""
log_args(api="/meta/nodes/{meta_node}/list", kwargs=locals())
if full_data:
full_data_limit = 500
if limit > full_data_limit:
raise HTTPException(
status_code=422,
detail=f"limit should be less equal than {full_data_limit}.",
)
query = MetaQueries.get_nodes.format(
meta_node=meta_node.value, skip=offset, limit=limit
)
else:
id_field = meta_node_id_name_mappings[meta_node.value]["id"]
name_field = meta_node_id_name_mappings[meta_node.value]["name"]
query = MetaQueries.get_node_id_and_name_fields.format(
meta_node=meta_node.value,
id_field=id_field,
name_field=name_field,
limit=limit,
offset=offset,
)
logger.info(query)
response = epigraphdb.run_query(query)
return response
def cache_func_call_json(
cache_name: str,
func: Callable,
params: Optional[Dict[str, Any]] = None,
overwrite: bool = False,
):
"""Write cache to a json file.
- cache_name: name, no extension
"""
cache_file = cache_dir / f"{cache_name}.json"
if overwrite or not cache_file.exists():
if params is None:
res = func()
else:
res = func(**params)
logger.info(f"write to: {cache_file}")
with open(cache_file, "w") as f:
json.dump(res, f)
else:
logger.info(f"reuse: {cache_file}")
with open(cache_file, "r") as f:
res = json.load(f)
return res
def index_data(
input_data: List[Any],
index_name: str,
es_client: Elasticsearch,
indexer_fn: Callable,
batch_size: int = 500,
overwrite: bool = True,
) -> None:
logger.info(f"Creating the index {index_name}.")
indexer_fn(index_name=index_name, overwrite=overwrite, es_client=es_client)
# NOTE: batch_data is a generator
batch_data = batch_by_n(collection=input_data, batch_size=batch_size)
for batch_docs in batch_data:
requests = [
dict(**{
"_op_type": "index",
"_index": index_name
}, **doc) for doc in batch_docs
]
bulk(es_client, requests)
es_client.indices.refresh(index=index_name, request_timeout=300)
logger.info("Done indexing.")
async def auth_login(user_auth: user.UserAuth,
redis_cli: Redis = Depends(get_redis_cli)):
# 验证用户
user_info = await curd_admin.authenticate(account=user_auth.username,
password=user_auth.password)
if not user_info:
logger.info(
f"用户登录认证错误: account: {user_auth.username} password:{user_auth.password}"
)
return ValidationErrorResponse(content="用户不存在或帐户密码错误")
if curd_admin.is_active(user_info):
logger.info(f"用户未激活: account: {user_auth.username}")
return AuthFailedResponse(content="用户未激活")
token_data = {"uid": user_info.id, "uuid": user_info.user_id}
access_token = security.create_access_token(token_data, )
logger.info(f"登录成功: account: {user_auth.username}")
await curd_redis.auth_set_token(redis_cli, user_info.user_id, access_token)
return token.Token(token=access_token)
def get_raw_cyper(
query: str,
db: EpigraphdbGraphsExtended = EpigraphdbGraphsExtended.epigraphdb,
hostname: Optional[str] = None,
bolt_port: Optional[str] = None,
user: Optional[str] = None,
password: Optional[str] = None,
api_key: APIKey = Depends(get_api_key),
):
"""Query neo4j database using cypher command.
"query": A qualified cypher query
"db": epigraphdb (default), or pqtl
"""
log_args(api="/raw_cypher", kwargs=locals())
if db.value == "epigraphdb":
logger.info("Query epigraphdb graph")
neo4j_db = epigraphdb
elif db.value == "pqtl":
logger.info("Query pqtl graph")
neo4j_db = pqtl
elif db.value == "custom":
if (hostname is not None and bolt_port is not None and user is not None
and password is not None):
logger.info("Query custom graphs")
neo4j_db = Neo4jDB(
hostname=hostname,
bolt_port=bolt_port,
user=user,
password=password,
)
if not neo4j_db.check_connection():
raise HTTPException(status_code=422,
detail="Cannot connect to supplied graph")
res = cypher(query=query, neo4j_db=neo4j_db)
return res
def cypher(query: str, neo4j_db: Neo4jDB):
logger.info(query)
data = neo4j_db.run_query(query)
return data
def process_master(self, overwrite: bool = False) -> bool:
# Logics:
# - if not overwrite and the doc is found in all colls,
# do nothing and return True
# - else, send request to api to refresh doc in colls
# - if the returned result is empty, return False
# - else, return True
logger.info(f"Process master for {self.master_name}")
doc_found = sum(
[
mongo_doc_exist(coll, self.doc_name)
for coll in self.mongo_coll_list
]
) == len(self.mongo_coll_list)
if doc_found and not overwrite and use_cache:
logger.info(
f"Do nothing: doc found: {doc_found}; overwrite: {overwrite}"
)
doc_res = self.mongo_coll_query.find_one(
{"doc_name": self.doc_name}
)
empty_results = doc_res["results"]["empty_results"]
if empty_results:
return False
else:
return True
else:
logger.info(
f"Request from api: {self.api_url}, params: {self.params}"
)
r = requests.get(
self.api_url, params=self.params, headers=self.headers
)
r.raise_for_status()
response = r.json()
results = response["results"]
empty_results = len(results) == 0
logger.info("mongo: Write to response collection")
mongo_doc_insert(
collection=self.mongo_coll_response,
doc_name=self.doc_name,
results=response,
)
logger.info("mongo: Write to query collection")
query = render_query(
r,
empty_results=empty_results,
url=self.api_url,
params=self.params,
)
mongo_doc_insert(
collection=self.mongo_coll_query,
doc_name=self.doc_name,
results=query,
)
logger.info("mongo: Write to table collection")
if empty_results:
mongo_doc_insert(
collection=self.mongo_coll_table,
doc_name=self.doc_name,
results=None,
)
else:
results_df = pd.json_normalize(results)[self.table_cols]
if self.table_precaching_hook is not None:
results_df = results_df.pipe(self.table_precaching_hook)
table_data_response = format_table_data_response(
table_data=process_table_data(
results_df, cols_to_round=self.cols_to_round
),
table_docs=self.table_docs,
)
mongo_doc_insert(
collection=self.mongo_coll_table,
doc_name=self.doc_name,
results=table_data_response,
)
if empty_results:
return False
else:
return True