def find_edges(self, props: Dict = None) -> List[Edge]:
"""
Find all edges which have specified properties
:param props: a dict containing desirable properties of an edge
:return: a list of edges
"""
def find_edges_in_worker(worker):
url = f'{worker}/findEdges?props={json.dumps(props)}'
try:
response = requests.get(url)
response.raise_for_status()
edges = response.json().get('edges')
edges_modified = []
if edges:
for edge in edges:
if not edge:
continue
edges_modified.append(
self.change_ids_in_edge(edge, worker))
return edges_modified
except Exception as e:
print(
f'Could not find edges in worker {worker}. Response: {e}')
return []
result = execute_function_in_parallel(find_edges_in_worker,
[(worker, )
for worker in self.workers])
edges = []
for res in result:
if res:
edges.extend(res)
return edges
def get_edges_to(self,
node_id: NodeId,
properties: Dict = None,
parallel: bool = False) -> List[Edge]:
"""
Find all edges to a node
:param parallel: execute function in parallel
:param node_id: an id of the node
:param properties: a dict containing desirable properties of an edge
:return: a list of edges
"""
node = self.get_node(node_id)
if not node:
return []
edges_ids = self.storage.edges_to(node_id)
if parallel:
if not properties:
results = execute_function_in_parallel(
self.get_edge,
[(edges_id, False, True) for edges_id in edges_ids])
else:
results = execute_function_in_parallel(
self.check_property_in_edge,
[(edges_id, properties, False, True)
for edges_id in edges_ids])
else:
results = []
for edges_id in edges_ids:
if not properties:
res = self.get_edge(edges_id, False, True)
else:
res = self.check_property_in_edge(edges_id, properties,
False, True)
if res:
results.append(res)
return results
def get_edges_by_properties(self,
properties: Dict,
parallel: bool = False) -> List[Edge]:
"""
Find all edges which have specified properties
:param parallel: execute function in parallel
:param properties: a dict containing desirable properties of an edge
:return: a list of edges
"""
all_edges = self.storage.get_edge_ids()
if parallel:
return execute_function_in_parallel(self.check_property_in_edge,
[(edge_id, properties)
for edge_id in all_edges])
else:
edges = []
for edge_id in all_edges:
res = self.check_property_in_edge(edge_id, properties)
if res:
edges.append(res)
return edges
def find_nodes(self, props: Dict = None) -> List[Node]:
"""
Find all nodes which have specified properties
:param props: a dict containing desirable properties of a node
:return: a list of nodes
"""
def find_nodes_in_worker(worker):
url = f'{worker}/findNodes?props={json.dumps(props)}'
try:
response = requests.get(url)
response.raise_for_status()
nodes = response.json().get('nodes')
nodes_modified = []
if nodes:
for node in nodes:
if not node:
continue
if 'remote_node' in node['props']:
nodes_modified.append(
self.get_node(node['props']['remote_node_id']))
else:
node['node_id'] = f'{worker}${node["node_id"]}'
nodes_modified.append(node)
return nodes_modified
except Exception as e:
print(
f'Could not find nodes in worker {worker}. Response: {e}')
return []
result = execute_function_in_parallel(find_nodes_in_worker,
[(worker, )
for worker in self.workers])
nodes = []
for res in result:
if res:
nodes.extend(res)
return nodes
def find_neighbours(self,
node_id_str: str,
hops: int,
node_props: Dict = None,
edge_props: Dict = None) -> Iterable[Node]:
"""
Get all neighbours of a node
:return: a list of neighbours
"""
query_id = ''.join(
random.choices(string.ascii_uppercase + string.digits, k=10))
def find_neighbours_in_worker(node_id_str_to_check, hops_left):
split_node_id = node_id_str_to_check.split('$')
worker = split_node_id[0]
node_id = split_node_id[1]
if worker not in self.workers:
return None
url = f'{worker}/findNeighbours?node_id={node_id}&hops={hops_left}&query_id={query_id}'
if node_props:
url += f'&node_props={json.dumps(node_props)}'
if edge_props:
url += f'&edge_props={json.dumps(edge_props)}'
try:
response = requests.get(url)
response.raise_for_status()
res = response.json()
if res['neighbours']:
for neighbour in res['neighbours']:
neighbour[
'node_id'] = f'{worker}${neighbour["node_id"]}'
return res
except Exception as e:
print(f'Could not find neighbours in {worker}. Response: {e}')
return None
def clear_history(worker):
url = f'{worker}/clearVisitedNodes?query_id={query_id}'
try:
response = requests.put(url)
response.raise_for_status()
except Exception as e:
print(
f'Could not clear visited nodes for worker {worker}. Response: {e}'
)
nodes_to_process = [(node_id_str, hops)]
neighbours = {}
while len(nodes_to_process) > 0:
results = execute_function_in_parallel(
find_neighbours_in_worker,
[(node_id, hops_left)
for node_id, hops_left in nodes_to_process])
nodes_to_process = []
for result in results:
new_neighbours = result['neighbours']
if new_neighbours:
for new_neighbour in new_neighbours:
neighbours[new_neighbour['node_id']] = new_neighbour
remote_nodes = result['remote_nodes']
if remote_nodes:
for remote_node, hops_left in remote_nodes:
if remote_node not in neighbours:
neighbours[remote_node] = self.get_node(
remote_node)
nodes_to_process.append((remote_node, hops_left))
execute_function_in_parallel(clear_history,
[(worker, ) for worker in self.workers])
return neighbours.values()
def activity_add():
"""
Add an activity
---
post:
summary: Add an activity.
description: Add an activity or multiple activities to the current user.
parameters:
- name: activity
in: formData
required: true
description: json containing all specified parameters
type: string
- name: activities
in: formData
required: false
description: List containing activity_data
type: array
items:
type: string
- name: start_time
in: formData
required: true
type: string
description: a start time of the activity
- name: end_time
in: formData
required: true
type: string
description: an end time of the activity
- name: executable_name
in: formData
required: true
type: string
description: a name of the current executable
- name: browser_url
in: formData
required: false
type: string
description: a url opened during the activity
- name: browser_title
in: formData
required: false
type: string
description: a title of the browsing window
- name: ip_address
in: formData
required: true
type: string
description: an ip address of the user
- name: mac_address
in: formData
required: true
type: string
description: an mac address of the user
- name: idle_activity
in: formData
required: false
type: boolean
description: if activity is an idle one
- name: activity_type
in: formData
required: false
type: string
description: a type of activity collected (os, eclipse tab and etc)
responses:
400:
description: Parameters are not correct
201:
description: Activity was added
"""
data = flask.request.json if flask.request.json else flask.request.form
activity_data = data.get(ACTIVITY_KEY)
if not isinstance(activity_data, dict):
try:
activity_data = json.loads(activity_data)
except Exception:
return make_response(jsonify({MESSAGE_KEY: 'Wrong format'}),
HTTPStatus.BAD_REQUEST)
if ACTIVITIES_KEY in activity_data:
# Add multiple activities
activities = [(activity, current_user.to_dbref())
for activity in activity_data.get(ACTIVITIES_KEY, [])]
all_result = execute_function_in_parallel(add_activity, activities)
result = 1
for part_result in all_result:
if not part_result:
result = part_result
if result:
result = all_result
else:
# Delete those activities that were added
for part_result in all_result:
if part_result:
delete_activity(part_result)
else:
result = add_activity(activity_data, current_user.to_dbref())
if not result:
return make_response(
jsonify({MESSAGE_KEY: 'Failed to create activity'}),
HTTPStatus.INTERNAL_SERVER_ERROR)
return make_response(
jsonify({
MESSAGE_KEY: 'Success',
ACTIVITY_ID_KEY: result
}), HTTPStatus.CREATED)
