def get_navigation_node_by_name(root_id, node_name):
""" Get the navigation node from a given node name
Args:
root_id:
node_name:
Returns:
"""
navigation_root_node = navigation_api.get_by_id(root_id)
owl_node_name = CQL_NAMESPACE + node_name
navigation_nodes = navigation_api.get_by_name(owl_node_name)
if navigation_root_node in navigation_nodes:
return navigation_root_node
if len(navigation_root_node.children) == 0:
return None
children_nodes = [
get_navigation_node_by_name(child, node_name)
for child in navigation_root_node.children
]
children_nodes = [
child_node for child_node in children_nodes if child_node is not None
]
if len(children_nodes) != 1:
return None
else:
return children_nodes[0]
def retrieve_navigation_filters(navigation_node):
""" retrieve filters from a navigation node
Parameters
navigation_node:
Returns:
"""
filters = []
node_has_parent = True
# If no parent, stops the filter lookup
while node_has_parent:
# Add filter to the list of filters
if 'filter' in navigation_node.options and navigation_node.options[
'filter'] is not None:
filters.append(navigation_node.options['filter'])
# Update parameters
node_has_parent = navigation_node.parent is not None
if node_has_parent:
navigation_node = navigation_api.get_by_id(navigation_node.parent)
return filters
def get_doc_by_nodes(node, dico):
"""
Recursive function that calculates the number of docs under each node of the tree
Args: request:
- node :navigation object
- dico : dict of node and its associated number of docs
Return: Dict[node,number of node under this node]
"""
try: # Type(node) = Navigation
if node.children:
node_doc = sum([
get_doc_by_nodes(child_id, dico) for child_id in node.children
])
dico[node.id] = node_doc
return node_doc
else:
return dico[node.id]
except Exception as e: # node = ID of a Navigation object
nav_child = get_by_id(node)
if nav_child.children:
node_doc = sum([
get_doc_by_nodes(child_id, dico)
for child_id in nav_child.children
])
dico[node] = node_doc
return node_doc
else:
return dico[node]
def render_html_tree(navigation, template_id, tab, navigation_table):
"""
Renders the navigation tree that contains the number of docs under each node of the tree
Args: request:
- navigation root
- template id
- tab: table of the id of the navigation nodes and the number of docs under each node
- nav_table : table of node id and its related information (branch_id, branches) used to build the html tree
Return: navigation tree
"""
nav_tree_html = ""
for navigation_id in navigation.children:
navigation_child = get_by_id(navigation_id)
with open(join(TEMPLATES_PATH, "li.html"), "r") as li_file:
li_content = li_file.read()
li_template = Template(li_content)
# there is a projection, get documents from database
if ("projection" in navigation_child.options
and navigation_child.options["projection"] is not None):
x, y = render_documents(navigation_child, template_id)
navigation_table[navigation_id]["branches"] = x
else:
navigation_table[navigation_id]["branches"] = render_html_tree(
navigation_child, template_id, tab, navigation_table)
nav_tree_html += li_template.render(
Context(navigation_table[navigation_id]))
return nav_tree_html
def get_node_name(node_id):
""" Get the name of the node
Args:
node_id:
Returns:
"""
navigation_name = navigation_api.get_by_id(node_id).name
index = navigation_name.rfind("#")
return navigation_name[index + 1:]
def render_navigation(navigation, template_id, tab, nav_table):
"""
Build the navigation tree with the number of documents under each node that directly contain documents
Args: navigation:
- navigation root
- template id
- tab : table of node_id and the direct number of docs for this node
- nav_table : table of node id and its related information (branch_id, branches)
Return: navigation tree
"""
nav_tree_html = ""
for navigation_id in navigation.children:
navigation_child = get_by_id(navigation_id)
number_of_doc = 0
with open(join(TEMPLATES_PATH, "li.html"), "r") as li_file:
li_content = li_file.read()
li_template = Template(li_content)
name = (navigation_child.name.split("#")[1]
if "#" in navigation_child.name else navigation_child.name)
# there is a projection, get documents from database
if ("projection" in navigation_child.options
and navigation_child.options["projection"] is not None):
_branch, number_of_doc = render_documents(
navigation_child, template_id, number_of_doc)
context = {
"branch_id": navigation_id,
"branches": _branch,
"branch_name": name,
"branch_docs_nb": str(number_of_doc),
}
else:
context = {
"branch_id":
navigation_id,
"branch_name":
str(get_number_of_node_doc(navigation_id, name,
nav_table)),
"branches":
render_navigation(navigation_child, template_id, tab,
nav_table),
}
tab.append((navigation_id, number_of_doc))
if ("view" in navigation_child.options
and navigation_child.options["view"] is not None):
context["branch_view"] = "true"
if ("hidden" in navigation_child.options
and navigation_child.options["hidden"]):
context["hidden"] = True
nav_tree_html += li_template.render(Context(context))
nav_table[navigation_id] = context
return nav_tree_html
def _load_branch_view(request):
""" Load view for a branch
Args:
request:
Returns:
"""
# Retrieve the view annotation
navigation_node = navigation_api.get_by_id(request.POST["node_id"])
filters = navigation_operations.retrieve_navigation_filters(
navigation_node)
# FIXME modified query part to execute query directly
documents = []
query_documents = query.execute_query(filters, "id")
for query_doc in query_documents:
documents.append(query_doc.id)
# Display XML file if "projection_view" annotation is not configured
if "view" not in navigation_node.options:
error = {
"message": "'cql:view' annotation does not exist for this branch."
}
return HttpResponse(json.dumps(error), HTTP_400_BAD_REQUEST)
branch_views = json.loads(navigation_node.options["view"])
name = navigation_node.name.split(
'#')[1] if '#' in navigation_node.name else navigation_node.name
view_data = {"header": name, "type": "branch", "views": []}
for branch_view in branch_views:
result_data = {
"title":
branch_view["title"],
"data":
parser_processview.processviewdocidlist(request.POST["nav_id"],
documents,
branch_view["data"])
}
view_data["views"].append(result_data)
return render(request, "explore_tree/components/view.html", view_data)
def _load_data_view(node_id, nav_id, data_id, from_tree=True):
""" Load view for a data, from a tree or a link
Args:
node_id:
nav_id:
data_id:
from_tree:
Returns:
"""
if not from_tree:
navigation_node = navigation_operations.get_navigation_node_for_document(
node_id, data_id)
else:
navigation_node = navigation_api.get_by_id(node_id)
# Initialize parameters in order to download later some information
xml_document = Data.get_by_id(data_id)
projection_views = json.loads(navigation_node.options["projection_view"])
view_data = {
"header": xml_document.title,
"type": "leaf",
"views": [],
"download": []
}
# Initialize parameters in order to download later some information
# dict of doc_id and queries done of cross documents : {id_doc1: [list of queries1], id_doc2: [list of queries2]}
dict_id_and_queries_cross_docs = dict()
# dict of doc_id and queries results for a cross document : {id_doc: [list of queries results]}
dict_id_and_queryresults_cross_docs = dict()
# dict of queried parameter and associated result for the queries done on the main doc : { queried parameter: value}
dict_tags_values_main_doc = dict()
values_of_items_from_main_doc = []
list_values_of_items_from_main_doc = []
# Send the annotation to the processor and collect the data
for projection_view in projection_views:
result_data = {"title": projection_view["title"], "data": None}
# FIXME better handling of x-queries
# Get info from other doc (without the main queried document)
if "query" in projection_view.keys():
doc_projections = []
# Get the names of the tags tag need to be displayed
for value in projection_view["data"]:
doc_projections.append(value.get('path'))
result_data["data"] = parser_processview.process_cross_query(
nav_id, data_id, projection_view["query"],
projection_view["data"])
# Get all the queried documents (without the main queried document)
queried_docs = parser_processview.ids_docs_to_querys
for id_doc in queried_docs:
other_doc_query = {"_id": ObjectId(id_doc)}
# list of queries done on the current document
query_list = list()
# list of queries results done on the current document
result_list = list()
for projection in doc_projections:
# Get the MongoDB query path for the parameter that need to be displayed
# eg: query_path = dict_content.a.b.c.d.e
query_path = {
doc_projections[doc_projections.index(projection)]: 1
}
# Get the Data corresponding to the id
queried_data = Data.execute_query(other_doc_query).only(
query_path.keys()[0])
# Add the query to the query list for the current doc
query_list.append(query_path.keys()[0].replace(
"dict_content.", ""))
try:
# Get the result of the query
result_query = get_projection(queried_data[0])
# Add the result of the query to the result list for the current doc
result_list.append(str(result_query))
except:
pass
dict_id_and_queries_cross_docs[id_doc] = query_list
dict_id_and_queryresults_cross_docs[id_doc] = result_list
# Get info from main doc
else:
# list of queries done on the current document (Main doc)
query_list = []
doc_projections = [
value.get('path') for value in projection_view["data"]
]
query_list = [
doc_projections[doc_projections.index(projection)]
for projection in doc_projections
]
# Get all results of the queries. type(result_data["data"]) = dict or instance of dict
result_data["data"] = parser_processview.processview(
nav_id, data_id, projection_view["data"])
for query_path, dict_result in zip(query_list,
result_data["data"]):
# eg: query_path = a.b.c.d
# We have only one value as result for the query
dict_result_value = dict_result.get("value", None)
if dict_result_value is not None:
tag = query_path.split(".")[
-1] # Get only d (the needed tag)
if tag in dict_tags_values_main_doc:
v = dict_tags_values_main_doc[tag]
if isinstance(v, list):
dict_tags_values_main_doc[tag].append(
dict_result_value)
else:
dict_tags_values_main_doc[tag] = [
dict_tags_values_main_doc[tag],
dict_result_value
]
else:
dict_tags_values_main_doc[tag] = dict_result_value
# We have multiple values for this result: all the chemical components
# (dict_result[key] is an inclusion of dicts)
dict_result_item, dict_result_items = [
dict_result.get(_, None) for _ in ["item", "items"]
]
if dict_result_item or dict_result_items:
dict_result_item_v = dict_result_item if dict_result_item is not None else dict_result_items
#dict_result_item_v = [dict_result_item, dict_result_items][dict_result_item not None]
# From the inclusion of dict, process the dict into a list and get all the needed values
# values_of_items_from_main_doc = list[list[value1 for dict i,value2 for dict 2, ..]]
# eg: values_of_items_from_main_doc= [l1,l2]
# l1 = [["location", "NIST"], ["Build location X", "59"], "EWI_Build1"]]
# l2 = [["location", "NIST"], ["Build location X", "47"], "EWI_Build2"]]
get_values_items(dict_result_item_v,
values_of_items_from_main_doc)
for list_of_values in values_of_items_from_main_doc:
for value in list_of_values:
# We have a list :value= [displayed parameter, value]
# eg : ["Build location X", "59"]
if len(value) == 2:
# list_tag_of_items_from_main_doc.append(value[0])
list_values_of_items_from_main_doc.append(
value[1]) # Get the value. eg: 59
# We have only one value (last value in the list. eg: EWI_Build1 in l1)
else:
list_values_of_items_from_main_doc.append(
value)
view_data["views"].append(result_data)
# Get the displayed data as an XML format in order to download it later #
# STEP 1: Build the XML based on initial tags for the crossed documents:
# Go into the dict of doc_id and queries of cross documents and build the xml for each document
# dict_id_and_queries_cross_docs = {id_doc1: [list of queries1], id_doc2: [list of queries2]}
xml_cross_queries_string = ""
for key in dict_id_and_queries_cross_docs: # key = doc_id
# Get all queries for the current doc_id.
# eg: query_list = ["a.b.c.d","a.b.c.e","a.b.f.g"]
query_list = dict_id_and_queries_cross_docs[key]
# For the doc_id get all the results of the queries done
# results = ["D","E","G"]
results = dict_id_and_queryresults_cross_docs[key]
# Build a xml string for the doc associated to doc_id thanks to the list of queries and the result list
xml_string = queryNode.tree_to_xml_string(
queryNode.aggregate_query(query_list, results))
xml_object = XSDTree.fromstring(xml_string + "</data>")
# Add the XML part to create an XML resulting of tag and values of crossed documents
xml_cross_queries_string += XSDTree.tostring(xml_object, pretty=True)
# STEP 2: Build the XML for the main document with only the needed tags:
# Get the Data associated to the main document
data = Data.get_by_id(data_id)
# Get the XML content
file_content = data.xml_content
xml_main_doc = XSDTree.fromstring(file_content)
# Transform all the result value into a string to help while testing equality of values with the original XML
for key, value in dict_tags_values_main_doc.items():
if isinstance(value, list):
dict_tags_values_main_doc[key] = map(
lambda x: x if isinstance(x, unicode) else str(x), value)
else:
try:
dict_tags_values_main_doc[key] = str(value)
except:
dict_tags_values_main_doc[key] = u''.join(value).encode(
'utf-8')
v = dict_tags_values_main_doc[key]
# Process the XML structure that represents the main document to keep only the needed tags and information
for child in xml_main_doc.iter():
# Transform all values into a string
try:
text = str(child.text)
except:
text = u''.join(child.text).encode('utf-8')
# If the xml tag is in our dict of tags and values from the main document
# and its value = dict_tags_values_main_doc[child.tag] we keep the text in the XML structure
# else we remove the text
if child.tag in dict_tags_values_main_doc.keys():
# Fixme # if text != str(dict_tags_values_main_doc[child.tag]) or dict_tags_values_main_doc[child.tag] not in text: (caution: does not keep all needed values if we replace by this line)
if isinstance(dict_tags_values_main_doc[child.tag], list):
display_value = False
for value in dict_tags_values_main_doc[child.tag]:
if value == text or value in text:
display_value = True
break
if not display_value:
child.text = ""
else:
if text == str(
dict_tags_values_main_doc[child.tag]
) or dict_tags_values_main_doc[child.tag] in text:
pass
else:
child.text = ""
else:
# If text is in our list of items of the main doc we keep the value and remove it from our list of items
if text in list_values_of_items_from_main_doc:
list_values_of_items_from_main_doc.remove(text)
else:
display_text = False
# v = processed name of the tag as appears in the rendered data after clicking a doc of the tree
# If this name is in our list of items from the main doc we keep the value (text) in the XML tree
# else we remove this value
for v in list_values_of_items_from_main_doc:
if v in text:
display_text = True
break
if not display_text:
child.text = ""
xml_f_main_doc = xml_main_doc
# Remove empty leafs of the tree (all child where child.text="")
while check_empty_nodes(xml_main_doc):
remove_empty_nodes(xml_f_main_doc)
# Build the final XML string result of the main doc and the crossed docs
xml_main_doc = XSDTree.tostring(xml_f_main_doc, pretty="TRUE")
xml = xml_main_doc + xml_cross_queries_string
xml_final = "<xml>\n" + xml + "</xml>"
xml_final = u''.join(xml_final).encode('utf-8')
xml_final = str(xml_final)
view_data["download"] = xml_final
return view_data
def get_navigation_node_for_document(node_id, document_id):
""" Get the navigation node from a given data id
Args:
node_id:
document_id:
Returns:
"""
navigation_node = navigation_api.get_by_id(node_id)
original_node = navigation_api.get_by_id(node_id)
if "projection" in navigation_node.options and navigation_node.options[
'projection'] is not None:
# **************
# FIXME duplicate of parser code
# Get projection
filters = []
while True:
# add filter to the list of filters
if 'filter' in navigation_node.options and navigation_node.options[
'filter'] is not None:
filters.append(navigation_node.options['filter'])
# if no parent, stops the filter lookup
if navigation_node.parent is None:
break
else: # if parent, continue the filter lookup at the parent level
navigation_node = navigation_api.get_by_id(
navigation_node.parent)
# get the documents matching the query
doc = Data.get_by_id(document_id)
template_id = doc.template.id
documents_id = [
str(get_projection(document)) for document in query.execute_query(
template_id, filters, '{"id": 1}')
]
# End fixme
# **************
# If the document is one of the document we return the navigation node, else return None
if document_id in documents_id:
return original_node
else:
return None
else:
navigation_children = navigation_node.children
if len(navigation_children) == 0:
return None
else:
navigation_nodes = [
get_navigation_node_for_document(child_id, document_id)
for child_id in navigation_children
]
navigation_nodes = [
nav_node for nav_node in navigation_nodes
if nav_node is not None
]
if len(navigation_nodes) != 1:
return None
else:
return navigation_nodes[0]
def render_documents(navigation, template_id, number_of_docs=0):
"""
Build the documents in the navigation tree
Args: request:
- navigation root
- template id
- number_of_docs : number of documents under a leaf node
Return: html with the name of the doc
"""
doc_tree_html = ""
number_of_docs = 0
leaf_name = navigation.name
leaf_id = navigation.id
try:
# Get the navigation id
navigation_id = str(navigation.id)
# Get projection
projection = navigation.options["projection"]
# get filters from parents
filters = []
while True:
# add filter to the list of filters
if ("filter" in navigation.options
and navigation.options["filter"] is not None):
filters.append(navigation.options["filter"])
# if no parent, stops the filter lookup
if navigation.parent is None:
break
else: # if parent, continue the filter lookup at the parent level
navigation = get_by_id(navigation.parent)
# get the documents matching the query
documents = query.execute_query(template_id, filters, projection)
for document in documents:
with open(join(TEMPLATES_PATH, "li_document.html"),
"r") as li_file:
li_content = li_file.read()
li_template = Template(li_content)
branch_name = get_projection(document)
context = {
"branch_id": document.id,
"parent_id": navigation_id,
"branch_name": branch_name,
}
doc_tree_html += li_template.render(Context(context))
number_of_docs += 1
leaf_api.upsert_leaf_object(str(leaf_id), str(document.id))
except Exception as e:
with open(join(TEMPLATES_PATH, "li_error.html"), "r") as li_file:
li_content = li_file.read()
li_template = Template(li_content)
context = {"error_message": str(e)}
doc_tree_html = li_template.render(Context(context))
return doc_tree_html, number_of_docs