def get_navigation_node_by_name(root_id, node_name): """ Get the navigation node from a given node name Args: root_id: node_name: Returns: """ navigation_root_node = navigation_api.get_by_id(root_id) owl_node_name = CQL_NAMESPACE + node_name navigation_nodes = navigation_api.get_by_name(owl_node_name) if navigation_root_node in navigation_nodes: return navigation_root_node if len(navigation_root_node.children) == 0: return None children_nodes = [ get_navigation_node_by_name(child, node_name) for child in navigation_root_node.children ] children_nodes = [ child_node for child_node in children_nodes if child_node is not None ] if len(children_nodes) != 1: return None else: return children_nodes[0]
def retrieve_navigation_filters(navigation_node): """ retrieve filters from a navigation node Parameters navigation_node: Returns: """ filters = [] node_has_parent = True # If no parent, stops the filter lookup while node_has_parent: # Add filter to the list of filters if 'filter' in navigation_node.options and navigation_node.options[ 'filter'] is not None: filters.append(navigation_node.options['filter']) # Update parameters node_has_parent = navigation_node.parent is not None if node_has_parent: navigation_node = navigation_api.get_by_id(navigation_node.parent) return filters
def get_doc_by_nodes(node, dico): """ Recursive function that calculates the number of docs under each node of the tree Args: request: - node :navigation object - dico : dict of node and its associated number of docs Return: Dict[node,number of node under this node] """ try: # Type(node) = Navigation if node.children: node_doc = sum([ get_doc_by_nodes(child_id, dico) for child_id in node.children ]) dico[node.id] = node_doc return node_doc else: return dico[node.id] except Exception as e: # node = ID of a Navigation object nav_child = get_by_id(node) if nav_child.children: node_doc = sum([ get_doc_by_nodes(child_id, dico) for child_id in nav_child.children ]) dico[node] = node_doc return node_doc else: return dico[node]
def render_html_tree(navigation, template_id, tab, navigation_table): """ Renders the navigation tree that contains the number of docs under each node of the tree Args: request: - navigation root - template id - tab: table of the id of the navigation nodes and the number of docs under each node - nav_table : table of node id and its related information (branch_id, branches) used to build the html tree Return: navigation tree """ nav_tree_html = "" for navigation_id in navigation.children: navigation_child = get_by_id(navigation_id) with open(join(TEMPLATES_PATH, "li.html"), "r") as li_file: li_content = li_file.read() li_template = Template(li_content) # there is a projection, get documents from database if ("projection" in navigation_child.options and navigation_child.options["projection"] is not None): x, y = render_documents(navigation_child, template_id) navigation_table[navigation_id]["branches"] = x else: navigation_table[navigation_id]["branches"] = render_html_tree( navigation_child, template_id, tab, navigation_table) nav_tree_html += li_template.render( Context(navigation_table[navigation_id])) return nav_tree_html
def get_node_name(node_id): """ Get the name of the node Args: node_id: Returns: """ navigation_name = navigation_api.get_by_id(node_id).name index = navigation_name.rfind("#") return navigation_name[index + 1:]
def render_navigation(navigation, template_id, tab, nav_table): """ Build the navigation tree with the number of documents under each node that directly contain documents Args: navigation: - navigation root - template id - tab : table of node_id and the direct number of docs for this node - nav_table : table of node id and its related information (branch_id, branches) Return: navigation tree """ nav_tree_html = "" for navigation_id in navigation.children: navigation_child = get_by_id(navigation_id) number_of_doc = 0 with open(join(TEMPLATES_PATH, "li.html"), "r") as li_file: li_content = li_file.read() li_template = Template(li_content) name = (navigation_child.name.split("#")[1] if "#" in navigation_child.name else navigation_child.name) # there is a projection, get documents from database if ("projection" in navigation_child.options and navigation_child.options["projection"] is not None): _branch, number_of_doc = render_documents( navigation_child, template_id, number_of_doc) context = { "branch_id": navigation_id, "branches": _branch, "branch_name": name, "branch_docs_nb": str(number_of_doc), } else: context = { "branch_id": navigation_id, "branch_name": str(get_number_of_node_doc(navigation_id, name, nav_table)), "branches": render_navigation(navigation_child, template_id, tab, nav_table), } tab.append((navigation_id, number_of_doc)) if ("view" in navigation_child.options and navigation_child.options["view"] is not None): context["branch_view"] = "true" if ("hidden" in navigation_child.options and navigation_child.options["hidden"]): context["hidden"] = True nav_tree_html += li_template.render(Context(context)) nav_table[navigation_id] = context return nav_tree_html
def _load_branch_view(request): """ Load view for a branch Args: request: Returns: """ # Retrieve the view annotation navigation_node = navigation_api.get_by_id(request.POST["node_id"]) filters = navigation_operations.retrieve_navigation_filters( navigation_node) # FIXME modified query part to execute query directly documents = [] query_documents = query.execute_query(filters, "id") for query_doc in query_documents: documents.append(query_doc.id) # Display XML file if "projection_view" annotation is not configured if "view" not in navigation_node.options: error = { "message": "'cql:view' annotation does not exist for this branch." } return HttpResponse(json.dumps(error), HTTP_400_BAD_REQUEST) branch_views = json.loads(navigation_node.options["view"]) name = navigation_node.name.split( '#')[1] if '#' in navigation_node.name else navigation_node.name view_data = {"header": name, "type": "branch", "views": []} for branch_view in branch_views: result_data = { "title": branch_view["title"], "data": parser_processview.processviewdocidlist(request.POST["nav_id"], documents, branch_view["data"]) } view_data["views"].append(result_data) return render(request, "explore_tree/components/view.html", view_data)
def _load_data_view(node_id, nav_id, data_id, from_tree=True): """ Load view for a data, from a tree or a link Args: node_id: nav_id: data_id: from_tree: Returns: """ if not from_tree: navigation_node = navigation_operations.get_navigation_node_for_document( node_id, data_id) else: navigation_node = navigation_api.get_by_id(node_id) # Initialize parameters in order to download later some information xml_document = Data.get_by_id(data_id) projection_views = json.loads(navigation_node.options["projection_view"]) view_data = { "header": xml_document.title, "type": "leaf", "views": [], "download": [] } # Initialize parameters in order to download later some information # dict of doc_id and queries done of cross documents : {id_doc1: [list of queries1], id_doc2: [list of queries2]} dict_id_and_queries_cross_docs = dict() # dict of doc_id and queries results for a cross document : {id_doc: [list of queries results]} dict_id_and_queryresults_cross_docs = dict() # dict of queried parameter and associated result for the queries done on the main doc : { queried parameter: value} dict_tags_values_main_doc = dict() values_of_items_from_main_doc = [] list_values_of_items_from_main_doc = [] # Send the annotation to the processor and collect the data for projection_view in projection_views: result_data = {"title": projection_view["title"], "data": None} # FIXME better handling of x-queries # Get info from other doc (without the main queried document) if "query" in projection_view.keys(): doc_projections = [] # Get the names of the tags tag need to be displayed for value in projection_view["data"]: doc_projections.append(value.get('path')) result_data["data"] = parser_processview.process_cross_query( nav_id, data_id, projection_view["query"], projection_view["data"]) # Get all the queried documents (without the main queried document) queried_docs = parser_processview.ids_docs_to_querys for id_doc in queried_docs: other_doc_query = {"_id": ObjectId(id_doc)} # list of queries done on the current document query_list = list() # list of queries results done on the current document result_list = list() for projection in doc_projections: # Get the MongoDB query path for the parameter that need to be displayed # eg: query_path = dict_content.a.b.c.d.e query_path = { doc_projections[doc_projections.index(projection)]: 1 } # Get the Data corresponding to the id queried_data = Data.execute_query(other_doc_query).only( query_path.keys()[0]) # Add the query to the query list for the current doc query_list.append(query_path.keys()[0].replace( "dict_content.", "")) try: # Get the result of the query result_query = get_projection(queried_data[0]) # Add the result of the query to the result list for the current doc result_list.append(str(result_query)) except: pass dict_id_and_queries_cross_docs[id_doc] = query_list dict_id_and_queryresults_cross_docs[id_doc] = result_list # Get info from main doc else: # list of queries done on the current document (Main doc) query_list = [] doc_projections = [ value.get('path') for value in projection_view["data"] ] query_list = [ doc_projections[doc_projections.index(projection)] for projection in doc_projections ] # Get all results of the queries. type(result_data["data"]) = dict or instance of dict result_data["data"] = parser_processview.processview( nav_id, data_id, projection_view["data"]) for query_path, dict_result in zip(query_list, result_data["data"]): # eg: query_path = a.b.c.d # We have only one value as result for the query dict_result_value = dict_result.get("value", None) if dict_result_value is not None: tag = query_path.split(".")[ -1] # Get only d (the needed tag) if tag in dict_tags_values_main_doc: v = dict_tags_values_main_doc[tag] if isinstance(v, list): dict_tags_values_main_doc[tag].append( dict_result_value) else: dict_tags_values_main_doc[tag] = [ dict_tags_values_main_doc[tag], dict_result_value ] else: dict_tags_values_main_doc[tag] = dict_result_value # We have multiple values for this result: all the chemical components # (dict_result[key] is an inclusion of dicts) dict_result_item, dict_result_items = [ dict_result.get(_, None) for _ in ["item", "items"] ] if dict_result_item or dict_result_items: dict_result_item_v = dict_result_item if dict_result_item is not None else dict_result_items #dict_result_item_v = [dict_result_item, dict_result_items][dict_result_item not None] # From the inclusion of dict, process the dict into a list and get all the needed values # values_of_items_from_main_doc = list[list[value1 for dict i,value2 for dict 2, ..]] # eg: values_of_items_from_main_doc= [l1,l2] # l1 = [["location", "NIST"], ["Build location X", "59"], "EWI_Build1"]] # l2 = [["location", "NIST"], ["Build location X", "47"], "EWI_Build2"]] get_values_items(dict_result_item_v, values_of_items_from_main_doc) for list_of_values in values_of_items_from_main_doc: for value in list_of_values: # We have a list :value= [displayed parameter, value] # eg : ["Build location X", "59"] if len(value) == 2: # list_tag_of_items_from_main_doc.append(value[0]) list_values_of_items_from_main_doc.append( value[1]) # Get the value. eg: 59 # We have only one value (last value in the list. eg: EWI_Build1 in l1) else: list_values_of_items_from_main_doc.append( value) view_data["views"].append(result_data) # Get the displayed data as an XML format in order to download it later # # STEP 1: Build the XML based on initial tags for the crossed documents: # Go into the dict of doc_id and queries of cross documents and build the xml for each document # dict_id_and_queries_cross_docs = {id_doc1: [list of queries1], id_doc2: [list of queries2]} xml_cross_queries_string = "" for key in dict_id_and_queries_cross_docs: # key = doc_id # Get all queries for the current doc_id. # eg: query_list = ["a.b.c.d","a.b.c.e","a.b.f.g"] query_list = dict_id_and_queries_cross_docs[key] # For the doc_id get all the results of the queries done # results = ["D","E","G"] results = dict_id_and_queryresults_cross_docs[key] # Build a xml string for the doc associated to doc_id thanks to the list of queries and the result list xml_string = queryNode.tree_to_xml_string( queryNode.aggregate_query(query_list, results)) xml_object = XSDTree.fromstring(xml_string + "</data>") # Add the XML part to create an XML resulting of tag and values of crossed documents xml_cross_queries_string += XSDTree.tostring(xml_object, pretty=True) # STEP 2: Build the XML for the main document with only the needed tags: # Get the Data associated to the main document data = Data.get_by_id(data_id) # Get the XML content file_content = data.xml_content xml_main_doc = XSDTree.fromstring(file_content) # Transform all the result value into a string to help while testing equality of values with the original XML for key, value in dict_tags_values_main_doc.items(): if isinstance(value, list): dict_tags_values_main_doc[key] = map( lambda x: x if isinstance(x, unicode) else str(x), value) else: try: dict_tags_values_main_doc[key] = str(value) except: dict_tags_values_main_doc[key] = u''.join(value).encode( 'utf-8') v = dict_tags_values_main_doc[key] # Process the XML structure that represents the main document to keep only the needed tags and information for child in xml_main_doc.iter(): # Transform all values into a string try: text = str(child.text) except: text = u''.join(child.text).encode('utf-8') # If the xml tag is in our dict of tags and values from the main document # and its value = dict_tags_values_main_doc[child.tag] we keep the text in the XML structure # else we remove the text if child.tag in dict_tags_values_main_doc.keys(): # Fixme # if text != str(dict_tags_values_main_doc[child.tag]) or dict_tags_values_main_doc[child.tag] not in text: (caution: does not keep all needed values if we replace by this line) if isinstance(dict_tags_values_main_doc[child.tag], list): display_value = False for value in dict_tags_values_main_doc[child.tag]: if value == text or value in text: display_value = True break if not display_value: child.text = "" else: if text == str( dict_tags_values_main_doc[child.tag] ) or dict_tags_values_main_doc[child.tag] in text: pass else: child.text = "" else: # If text is in our list of items of the main doc we keep the value and remove it from our list of items if text in list_values_of_items_from_main_doc: list_values_of_items_from_main_doc.remove(text) else: display_text = False # v = processed name of the tag as appears in the rendered data after clicking a doc of the tree # If this name is in our list of items from the main doc we keep the value (text) in the XML tree # else we remove this value for v in list_values_of_items_from_main_doc: if v in text: display_text = True break if not display_text: child.text = "" xml_f_main_doc = xml_main_doc # Remove empty leafs of the tree (all child where child.text="") while check_empty_nodes(xml_main_doc): remove_empty_nodes(xml_f_main_doc) # Build the final XML string result of the main doc and the crossed docs xml_main_doc = XSDTree.tostring(xml_f_main_doc, pretty="TRUE") xml = xml_main_doc + xml_cross_queries_string xml_final = "<xml>\n" + xml + "</xml>" xml_final = u''.join(xml_final).encode('utf-8') xml_final = str(xml_final) view_data["download"] = xml_final return view_data
def get_navigation_node_for_document(node_id, document_id): """ Get the navigation node from a given data id Args: node_id: document_id: Returns: """ navigation_node = navigation_api.get_by_id(node_id) original_node = navigation_api.get_by_id(node_id) if "projection" in navigation_node.options and navigation_node.options[ 'projection'] is not None: # ************** # FIXME duplicate of parser code # Get projection filters = [] while True: # add filter to the list of filters if 'filter' in navigation_node.options and navigation_node.options[ 'filter'] is not None: filters.append(navigation_node.options['filter']) # if no parent, stops the filter lookup if navigation_node.parent is None: break else: # if parent, continue the filter lookup at the parent level navigation_node = navigation_api.get_by_id( navigation_node.parent) # get the documents matching the query doc = Data.get_by_id(document_id) template_id = doc.template.id documents_id = [ str(get_projection(document)) for document in query.execute_query( template_id, filters, '{"id": 1}') ] # End fixme # ************** # If the document is one of the document we return the navigation node, else return None if document_id in documents_id: return original_node else: return None else: navigation_children = navigation_node.children if len(navigation_children) == 0: return None else: navigation_nodes = [ get_navigation_node_for_document(child_id, document_id) for child_id in navigation_children ] navigation_nodes = [ nav_node for nav_node in navigation_nodes if nav_node is not None ] if len(navigation_nodes) != 1: return None else: return navigation_nodes[0]
def render_documents(navigation, template_id, number_of_docs=0): """ Build the documents in the navigation tree Args: request: - navigation root - template id - number_of_docs : number of documents under a leaf node Return: html with the name of the doc """ doc_tree_html = "" number_of_docs = 0 leaf_name = navigation.name leaf_id = navigation.id try: # Get the navigation id navigation_id = str(navigation.id) # Get projection projection = navigation.options["projection"] # get filters from parents filters = [] while True: # add filter to the list of filters if ("filter" in navigation.options and navigation.options["filter"] is not None): filters.append(navigation.options["filter"]) # if no parent, stops the filter lookup if navigation.parent is None: break else: # if parent, continue the filter lookup at the parent level navigation = get_by_id(navigation.parent) # get the documents matching the query documents = query.execute_query(template_id, filters, projection) for document in documents: with open(join(TEMPLATES_PATH, "li_document.html"), "r") as li_file: li_content = li_file.read() li_template = Template(li_content) branch_name = get_projection(document) context = { "branch_id": document.id, "parent_id": navigation_id, "branch_name": branch_name, } doc_tree_html += li_template.render(Context(context)) number_of_docs += 1 leaf_api.upsert_leaf_object(str(leaf_id), str(document.id)) except Exception as e: with open(join(TEMPLATES_PATH, "li_error.html"), "r") as li_file: li_content = li_file.read() li_template = Template(li_content) context = {"error_message": str(e)} doc_tree_html = li_template.render(Context(context)) return doc_tree_html, number_of_docs