Exemplo n.º 1
0
    def find_edges(self, props: Dict = None) -> List[Edge]:
        """
        Find all edges which have specified properties
        :param props: a dict containing desirable properties of an edge
        :return: a list of edges
        """
        def find_edges_in_worker(worker):
            url = f'{worker}/findEdges?props={json.dumps(props)}'
            try:
                response = requests.get(url)
                response.raise_for_status()

                edges = response.json().get('edges')
                edges_modified = []
                if edges:
                    for edge in edges:
                        if not edge:
                            continue
                        edges_modified.append(
                            self.change_ids_in_edge(edge, worker))
                return edges_modified
            except Exception as e:
                print(
                    f'Could not find edges in worker {worker}. Response: {e}')
                return []

        result = execute_function_in_parallel(find_edges_in_worker,
                                              [(worker, )
                                               for worker in self.workers])

        edges = []
        for res in result:
            if res:
                edges.extend(res)
        return edges
Exemplo n.º 2
0
    def get_edges_to(self,
                     node_id: NodeId,
                     properties: Dict = None,
                     parallel: bool = False) -> List[Edge]:
        """
        Find all edges to a node
        :param parallel: execute function in parallel
        :param node_id: an id of the node
        :param properties: a dict containing desirable properties of an edge
        :return: a list of edges
        """

        node = self.get_node(node_id)
        if not node:
            return []
        edges_ids = self.storage.edges_to(node_id)
        if parallel:
            if not properties:
                results = execute_function_in_parallel(
                    self.get_edge,
                    [(edges_id, False, True) for edges_id in edges_ids])
            else:
                results = execute_function_in_parallel(
                    self.check_property_in_edge,
                    [(edges_id, properties, False, True)
                     for edges_id in edges_ids])
        else:
            results = []
            for edges_id in edges_ids:
                if not properties:
                    res = self.get_edge(edges_id, False, True)
                else:
                    res = self.check_property_in_edge(edges_id, properties,
                                                      False, True)
                if res:
                    results.append(res)
        return results
Exemplo n.º 3
0
    def get_edges_by_properties(self,
                                properties: Dict,
                                parallel: bool = False) -> List[Edge]:
        """
        Find all edges which have specified properties
        :param parallel: execute function in parallel
        :param properties: a dict containing desirable properties of an edge
        :return: a list of edges
        """
        all_edges = self.storage.get_edge_ids()
        if parallel:
            return execute_function_in_parallel(self.check_property_in_edge,
                                                [(edge_id, properties)
                                                 for edge_id in all_edges])

        else:
            edges = []
            for edge_id in all_edges:
                res = self.check_property_in_edge(edge_id, properties)
                if res:
                    edges.append(res)
            return edges
Exemplo n.º 4
0
    def find_nodes(self, props: Dict = None) -> List[Node]:
        """
        Find all nodes which have specified properties
        :param props: a dict containing desirable properties of a node
        :return: a list of nodes
        """
        def find_nodes_in_worker(worker):
            url = f'{worker}/findNodes?props={json.dumps(props)}'
            try:
                response = requests.get(url)
                response.raise_for_status()

                nodes = response.json().get('nodes')
                nodes_modified = []
                if nodes:
                    for node in nodes:
                        if not node:
                            continue
                        if 'remote_node' in node['props']:
                            nodes_modified.append(
                                self.get_node(node['props']['remote_node_id']))
                        else:
                            node['node_id'] = f'{worker}${node["node_id"]}'
                            nodes_modified.append(node)
                return nodes_modified
            except Exception as e:
                print(
                    f'Could not find nodes in worker {worker}. Response: {e}')
                return []

        result = execute_function_in_parallel(find_nodes_in_worker,
                                              [(worker, )
                                               for worker in self.workers])
        nodes = []
        for res in result:
            if res:
                nodes.extend(res)
        return nodes
Exemplo n.º 5
0
    def find_neighbours(self,
                        node_id_str: str,
                        hops: int,
                        node_props: Dict = None,
                        edge_props: Dict = None) -> Iterable[Node]:
        """
        Get all neighbours of a node
        :return: a list of neighbours
        """
        query_id = ''.join(
            random.choices(string.ascii_uppercase + string.digits, k=10))

        def find_neighbours_in_worker(node_id_str_to_check, hops_left):
            split_node_id = node_id_str_to_check.split('$')
            worker = split_node_id[0]
            node_id = split_node_id[1]

            if worker not in self.workers:
                return None

            url = f'{worker}/findNeighbours?node_id={node_id}&hops={hops_left}&query_id={query_id}'
            if node_props:
                url += f'&node_props={json.dumps(node_props)}'
            if edge_props:
                url += f'&edge_props={json.dumps(edge_props)}'
            try:
                response = requests.get(url)
                response.raise_for_status()

                res = response.json()
                if res['neighbours']:
                    for neighbour in res['neighbours']:
                        neighbour[
                            'node_id'] = f'{worker}${neighbour["node_id"]}'
                return res
            except Exception as e:
                print(f'Could not find neighbours in {worker}. Response: {e}')
                return None

        def clear_history(worker):
            url = f'{worker}/clearVisitedNodes?query_id={query_id}'
            try:
                response = requests.put(url)
                response.raise_for_status()
            except Exception as e:
                print(
                    f'Could not clear visited nodes for worker {worker}. Response: {e}'
                )

        nodes_to_process = [(node_id_str, hops)]
        neighbours = {}
        while len(nodes_to_process) > 0:
            results = execute_function_in_parallel(
                find_neighbours_in_worker,
                [(node_id, hops_left)
                 for node_id, hops_left in nodes_to_process])
            nodes_to_process = []
            for result in results:
                new_neighbours = result['neighbours']
                if new_neighbours:
                    for new_neighbour in new_neighbours:
                        neighbours[new_neighbour['node_id']] = new_neighbour
                remote_nodes = result['remote_nodes']
                if remote_nodes:
                    for remote_node, hops_left in remote_nodes:
                        if remote_node not in neighbours:
                            neighbours[remote_node] = self.get_node(
                                remote_node)
                            nodes_to_process.append((remote_node, hops_left))

        execute_function_in_parallel(clear_history,
                                     [(worker, ) for worker in self.workers])

        return neighbours.values()
Exemplo n.º 6
0
def activity_add():
    """
    Add an activity
    ---
    post:
        summary: Add an activity.
        description: Add an activity or multiple activities to the current user.
        parameters:
            -   name: activity
                in: formData
                required: true
                description: json containing all specified parameters
                type: string
            -   name: activities
                in: formData
                required: false
                description: List containing activity_data
                type: array
                items:
                    type: string
            -   name: start_time
                in: formData
                required: true
                type: string
                description: a start time of the activity
            -   name: end_time
                in: formData
                required: true
                type: string
                description: an end time of the activity
            -   name: executable_name
                in: formData
                required: true
                type: string
                description: a name of the current executable
            -   name: browser_url
                in: formData
                required: false
                type: string
                description: a url opened during the activity
            -   name: browser_title
                in: formData
                required: false
                type: string
                description: a title of the browsing window
            -   name: ip_address
                in: formData
                required: true
                type: string
                description: an ip address of the user
            -   name: mac_address
                in: formData
                required: true
                type: string
                description: an mac address of the user
            -   name: idle_activity
                in: formData
                required: false
                type: boolean
                description: if activity is an idle one
            -   name: activity_type
                in: formData
                required: false
                type: string
                description: a type of activity collected (os, eclipse tab and etc)
        responses:
            400:
                description: Parameters are not correct
            201:
                description: Activity was added
    """
    data = flask.request.json if flask.request.json else flask.request.form
    activity_data = data.get(ACTIVITY_KEY)
    if not isinstance(activity_data, dict):
        try:
            activity_data = json.loads(activity_data)
        except Exception:
            return make_response(jsonify({MESSAGE_KEY: 'Wrong format'}),
                                 HTTPStatus.BAD_REQUEST)

    if ACTIVITIES_KEY in activity_data:
        #  Add multiple activities
        activities = [(activity, current_user.to_dbref())
                      for activity in activity_data.get(ACTIVITIES_KEY, [])]
        all_result = execute_function_in_parallel(add_activity, activities)
        result = 1
        for part_result in all_result:
            if not part_result:
                result = part_result
        if result:
            result = all_result
        else:
            # Delete those activities that were added
            for part_result in all_result:
                if part_result:
                    delete_activity(part_result)
    else:
        result = add_activity(activity_data, current_user.to_dbref())

    if not result:
        return make_response(
            jsonify({MESSAGE_KEY: 'Failed to create activity'}),
            HTTPStatus.INTERNAL_SERVER_ERROR)

    return make_response(
        jsonify({
            MESSAGE_KEY: 'Success',
            ACTIVITY_ID_KEY: result
        }), HTTPStatus.CREATED)