Example #1
0
    def get_label_nodes(cls,
                        team_id,
                        label,
                        name=None,
                        limit=None,
                        random=False):
        team = TeamController._get({"Team": {"id": team_id}})

        neo = Neo4jClient()
        query = "MATCH (n:{}_{}) WITH n".format(team.kafka_topic, label)

        if random:
            query += ", rand() as r ORDER BY r"

        if name:
            query += " WHERE n.name CONTAINS '{0}'".format(name)

        query += " RETURN distinct(n.name)"

        try:
            limit = int(limit)
            query += " LIMIT {0}".format(limit)
        except (TypeError, ValueError):
            pass

        results = neo.query(query, returns=(str, ))
        return [r[0] for r in results]
Example #2
0
    def delete_node(cls, team_id, label, node, detach=False):
        team = TeamController._get({"Team": {"id": team_id}})
        neo = Neo4jClient()
        query = 'MATCH(n:{topic}_{label}{{name: "{name}"}})'.format(
            topic=team.kafka_topic, label=label, name=node)

        # First we check if the node has dependencies
        has_deps = query + "-[r:DEPENDS_ON]->() RETURN count(r)"
        count = neo.query(has_deps,
                          returns=(int, ))[0][0]  # return is weird : [[100]]

        if count and not detach:
            msg = (
                "Can not delete node {node} because it still has {count} relationships"
            )
            raise RequirementsNotSatisfiedError(
                msg.format(node=node, count=count))

        # We can delete the node and its relationships
        query += " DETACH DELETE n"

        try:
            # Neo4j returns nothing when deleting a node
            neo.query(query)
        except TransactionException as e:
            raise RequirementsNotSatisfiedError(str(e))

        return {}
Example #3
0
    def count_node_dependencies(cls, team_id, label, node):
        team = TeamController._get({"Team": {"id": team_id}})

        neo = Neo4jClient()
        query = ("MATCH(n:{0}_{1}{{name: '{2}'}}) "
                 "OPTIONAL MATCH (n)-[:DEPENDS_ON]->(m) "
                 "RETURN count(m)").format(team.kafka_topic, label, node)
        sequence = neo.query(query, data_contents=DATA_GRAPH)

        return {"count": sequence[0][0]}
Example #4
0
    def get_labels(cls, team_id):
        """
        This function returns the list of labels for a team. The labels can be
        found in 2 places : in the configuration or directly in Neo4j.

        Having a label in Neo4j does not mean it's present in the config, and
        having a label in the config does not mean it has been created in Neo4j.
        """
        team = TeamController._get({"Team": {"id": team_id}})

        # Get labels per team
        neo = Neo4jClient()
        query = (
            "CALL db.labels() YIELD label "
            "WITH split(label, '_')[0] AS Team, split(label, '_')[1] AS Label "
            "RETURN Team, collect(Label) AS Labels "
            "ORDER BY Team")
        results = neo.query(query, returns=(str, list))

        # Get the label for the current team
        labels_per_teams = {r[0]: r[1] for r in results}

        # Pick the wanted team's labels
        team_labels = labels_per_teams.get(team.kafka_topic, [])

        # Get the number of nodes per label
        labels = {}
        if team_labels:
            query = cls._build_query_count_nodes(team.kafka_topic, team_labels)
            results = neo.query(query, returns=(str, int))

            # Merge the Neo4j and DB labels
            labels = {
                r[0]: {
                    "name": r[0],
                    "nodes_count": r[1],
                    "qos_query": None
                }
                for r in results
            }

        # No config: return the list of labels
        try:
            config = ConfigController.get_current_config(team_id)
        except NotFoundError:
            return list(labels.values())

        # Add the QoS query for each label
        qos_queries = {l: d["qos"] for l, d in config["data"].items()}
        for label, query in qos_queries.items():
            if label not in labels:
                labels[label] = {"name": label, "nodes_count": 0}
            labels[label]["qos_query"] = query

        return list(labels.values())
Example #5
0
    def _get_team(cls, team_id):
        obj = TeamController._get(filters={"Team": {"id": team_id}})

        # Add the members of the team
        team = TeamController.resource_to_dict(obj)
        team.update({
            "members": [m.name for m in obj.members],
            "editors": [m.name for m in obj.editors],
            "managers": [m.name for m in obj.managers],
        })
        return team
Example #6
0
    def get_impacted_nodes_all(cls, team_id, label, node, impacted_label, ts,
                               with_inactive_nodes):
        try:
            ts = int(ts)
        except (TypeError, ValueError):
            raise IntegrityError("'ts' parameter must be a positive number")

        if ts < 0:
            raise IntegrityError("'ts' parameter must be a positive number")

        if not impacted_label:
            raise IntegrityError("'impactedLabel' parameter must not be empty")

        team = TeamController._get({"Team": {"id": team_id}})

        json_string = "["
        nodes_batch = 50000
        skip = 0
        total_count = cls.get_impacted_nodes_count(team_id, label, node,
                                                   impacted_label)["count"]

        # Load all impacted nodes data by batch inside a list
        while skip < total_count:
            query = cls._build_impacted_nodes_queries(
                topic=team.kafka_topic,
                label=label,
                node=node,
                impacted_label=impacted_label,
                skip=skip,
                limit=nodes_batch,
                count=False,
            )

            new_json_string_data = json.dumps(
                cls._compute_impacted_nodes_from_data(
                    get_records(query).data(),
                    ts,
                    with_inactive_nodes=with_inactive_nodes,
                ),
                indent=4,
            )

            skip += nodes_batch

            # if this is not the last loop, add a "," to the last element
            # of the array to help easy concatenation at the next loop
            if skip < total_count:
                new_json_string_data = new_json_string_data[:-2] + ",\n]"
            json_string += new_json_string_data[1:-2]

        json_string += "\n]"

        return json_string
Example #7
0
    def get_label_nodes(cls,
                        team_id,
                        label,
                        name=None,
                        limit=None,
                        random=False):
        team = TeamController._get({"Team": {"id": team_id}})

        neo = Neo4jClient()
        query = cls._build_query_nodes(team.kafka_topic, label, random, name,
                                       limit)

        results = neo.query(query, returns=(str, ))
        return [r[0] for r in results]
Example #8
0
    def get_label_node(cls, team_id, label, node):
        team = TeamController._get({"Team": {"id": team_id}})

        neo = Neo4jClient()
        query = "MATCH(n:{0}_{1}{{name: '{2}'}}) RETURN n".format(
            team.kafka_topic, label, node)
        result = neo.query(query)

        try:
            data = list(result)[0][0]["data"]
        except IndexError:
            raise NotFoundError("Node {} does not exist".format(node))

        return data
Example #9
0
    def get_impacted_nodes_count(cls, team_id, label, node, impacted_label):
        if not impacted_label:
            raise IntegrityError("'impactedLabel' parameter must not be empty")

        team = TeamController._get({"Team": {"id": team_id}})

        query = cls._build_impacted_nodes_queries(
            topic=team.kafka_topic,
            label=label,
            node=node,
            impacted_label=impacted_label,
            count=True,
        )

        results = get_records(query)
        return {"count": results.value()[0]}
Example #10
0
    def get_impacted_nodes(cls, team_id, label, node, impacted_label, skip,
                           limit, ts):
        try:
            skip = int(skip)
            limit = int(limit)
            ts = int(ts)
        except (TypeError, ValueError):
            raise IntegrityError(
                "'skip', 'limit' and 'ts' parameters must be positive numbers")

        if skip < 0 or limit < 0 or ts < 0:
            raise IntegrityError(
                "'skip', 'limit' and 'ts' parameters must be positive numbers")

        if not impacted_label:
            raise IntegrityError("'impactedLabel' parameter must not be empty")

        team = TeamController._get({"Team": {"id": team_id}})

        query = cls._build_impacted_nodes_queries(
            topic=team.kafka_topic,
            label=label,
            node=node,
            impacted_label=impacted_label,
            skip=skip,
            limit=limit,
            count=False,
        )

        results = get_records(query)
        impacted_nodes_data = results.data()

        # Return all impacted nodes (active and inactive) with metadata indicating if they are active or not
        return cls._compute_impacted_nodes_from_data(impacted_nodes_data,
                                                     ts,
                                                     with_inactive_nodes=True)
Example #11
0
    def get_node_dependencies(
        cls,
        team_id,
        label,
        node,
        day=None,
        filter_on_config=False,
        include_inactive=False,
    ):
        topic = TeamController._get({"Team": {"id": team_id}}).kafka_topic
        query = cls._build_dependencies_query(team_id, topic, label, node,
                                              filter_on_config)
        dependencies = {
            "dependencies": {},
            "graph": {
                "nodes": [],
                "relationships": []
            }
        }
        records = get_records(query)

        # Loop on all relationships
        for idx, record in enumerate(records):

            # Handle the main node
            if idx == 0:
                node = record.get("n")
                title = list(node.labels)[0][len(topic) + 1:]

                if title not in dependencies["dependencies"]:
                    dependencies["dependencies"][title] = []
                dependencies["dependencies"][title].append(dict(node.items()))

                dependencies["graph"]["nodes"].append({
                    "id":
                    node.id,
                    "label":
                    dict(node.items())["name"],
                    "title":
                    title
                })

            # Handle the relationship
            rel = record.get("r")
            if not rel:
                continue

            # Check inactive nodes
            start_node = rel.start_node
            end_node = rel.end_node
            start = arrow.get(day, "YYYY-MM-DD").floor("day").timestamp
            end = arrow.get(day, "YYYY-MM-DD").ceil("day").timestamp

            if (not is_active_node(start, end,
                                   end_node)) or (not has_active_relationship(
                                       start, end, rel.get("periods"))):
                if not include_inactive:
                    continue
                else:
                    setattr(end_node, "inactive", True)

            # The label is 'acme_Mylabel', we just want 'Mylabel'
            title = list(end_node.labels)[0][len(topic) + 1:]

            if title not in dependencies["dependencies"]:
                dependencies["dependencies"][title] = []
            dependencies["dependencies"][title].append({
                **dict(end_node.items()),
                **{
                    "periods": list(rel.get("periods")),
                    "inactive": getattr(end_node, "inactive", False),
                },
            })

            dependencies["graph"]["nodes"].append({
                "id":
                end_node.id,
                "label":
                dict(end_node.items())["name"],
                "title":
                title,
            })

            dependencies["graph"]["relationships"].append({
                "id":
                rel.id,
                "from":
                start_node.id,
                "to":
                end_node.id,
                "arrows":
                "to",
                "periods":
                list(rel.get("periods")),
            })

        return dependencies
Example #12
0
    def get_labels(cls, team_id):
        """
        This function returns the list of labels for a team. The labels can be
        found in 2 places : in the configuration or directly in Neo4j.

        Having a label in Neo4j does not mean it's present in the config, and
        having a label in the config does not mean it has been created in Neo4j.
        """
        team = TeamController._get({"Team": {"id": team_id}})

        # Get labels per team
        neo = Neo4jClient()
        query = """CALL db.labels() YIELD label
        WITH split(label, '_')[0] AS Team, split(label, '_')[1] AS Label
        RETURN Team, collect(Label) AS Labels
        ORDER BY Team"""
        results = neo.query(query, returns=(str, list))

        # Get the label for the current team
        labels_per_teams = {r[0]: r[1] for r in results}

        # Pick the wanted team's labels
        team_labels = labels_per_teams.get(team.kafka_topic, [])

        # Construct the cypher to get the number of nodes per label
        # TODO : add cache for this feature
        labels = {}
        if team_labels:
            query = ("MATCH (n:{team}_{label}) "
                     'WITH "{label}" AS Label, count(n) AS Count '
                     "RETURN Label, Count")
            query = query.format(team=team.kafka_topic, label=team_labels[0])

            for label in team_labels[1:]:
                union = ("\nUNION MATCH (n:{team}_{label}) "
                         'WITH "{label}" AS Label, count(n) AS Count '
                         "RETURN Label, Count")
                query += union.format(team=team.kafka_topic, label=label)

            results = neo.query(query, returns=(str, int))

            # Merge the Neo4j and DB labels
            labels = {
                r[0]: {
                    "name": r[0],
                    "nodes_count": r[1],
                    "qos_query": None
                }
                for r in results
            }
        try:
            config = ConfigController.get_current_config(team_id)
        except NotFoundError:
            return list(labels.values())

        qos_queries = {l: d["qos"] for l, d in config["data"].items()}

        for label, query in qos_queries.items():
            if label not in labels:
                labels[label] = {"name": label, "nodes_count": 0}
            labels[label]["qos_query"] = query

        return list(labels.values())
Example #13
0
    def get_node_dependencies(cls,
                              team_id,
                              label,
                              node,
                              filter_on_config=False):
        team = TeamController._get({"Team": {"id": team_id}})
        query = """
        MATCH(n:{topic}_{label}{{name: "{name}"}})
        OPTIONAL MATCH (n)-[r]->(m) {where}
        RETURN n,r,m
        ORDER BY m.name LIMIT 10
        """

        # Filter the dependencies with the labels declared in the configuration
        where = ""
        if filter_on_config:
            label_config = ConfigController.get_label_config(team_id, label)
            regex = r"^.*(\[[A-Za-z]+(, [A-Za-z]+)*?\])$"
            match = re.search(regex, label_config["qos"])
            if match:
                deps = match.group(1)[1:-1].split(", ")
                where += "WHERE '{}_{}' IN LABELS(m)".format(
                    team.kafka_topic, deps[0])
                for dep in deps[1:]:
                    where += " OR '{}_{}' IN LABELS(m)".format(
                        team.kafka_topic, dep)

        neo = Neo4jClient()
        query = query.format(topic=team.kafka_topic,
                             label=label,
                             name=node,
                             where=where)
        sequence = neo.query(query, data_contents=DATA_GRAPH)

        dependencies = {
            "dependencies": {},
            "graph": {
                "nodes": [],
                "relationships": []
            }
        }

        if sequence.graph:
            for g in sequence.graph:

                # Handle the nodes
                for node in g["nodes"]:
                    title = node["labels"][0][len(team.kafka_topic) + 1:]
                    if title not in dependencies["dependencies"]:
                        dependencies["dependencies"][title] = []

                    # Id is required to make data unique later
                    node["properties"]["id"] = node["id"]
                    dependencies["dependencies"][title].append(
                        node["properties"])

                    # Add the node data in the whole graph
                    dependencies["graph"]["nodes"].append({
                        "id":
                        node["id"],
                        "label":
                        node["properties"].get("name", "unknown"),
                        "title":
                        title,
                    })

                # Handle the relationships
                for rel in g["relationships"]:
                    dependencies["graph"]["relationships"].append({
                        "id":
                        rel["id"],
                        "from":
                        rel["startNode"],
                        "to":
                        rel["endNode"],
                        "arrows":
                        "to",
                    })

        dependencies = cls.make_unique(dependencies)
        return dependencies