class GS_Graph:
def __init__(self):
self.api_issues = API_Issues()
self.puml = Puml().startuml()
self.puml_options = {
'node-text-value': "Summary",
'only-from-projects': [],
'link-types-to-add': [],
'link-types-to-ignore': [],
'left-to-right': True,
'show-key-in-text': True,
'show-edge-labels': True,
'width': None,
'height': None
}
self.nodes = []
self.edges = []
self.issues = None
self._link_types = None
self.use_cache = False
self.notes = []
self.node_type = {}
self.skin_params = []
self.create_params = []
self.risk_links_paths_down = [
'is affected by', 'Requires', 'is used by', 'delivered by',
'causes', 'is Technical Owner', 'is Management Owner',
'is Business Owner', 'RISK affects', 'is fixed by',
'is mitigated by', 'is created by VULN', 'risk reduced by',
'is Vulnerable to', 'has VULN', 'RISK supported by',
'VULN is created by', 'is Stakeholder', 'is parent of',
'is manager of', 'is created by R2', 'is created by R3',
'is created by R4'
]
self.risk_links_paths_up = [
'affects', 'Is Required by', 'uses', 'delivers', 'is caused by',
'has Technical Owner', 'has Management Owner',
'has Business Owner', 'has RISK', 'fixes', 'mitigates',
'creates RISK', 'reduces risk of', 'is Vulnerability of',
'VULN affects', 'supports RISK', 'creates VULN', 'has Stakeholder',
'is child of', 'is managed by', 'creates R1', 'creates R2',
'creates R3'
]
#, 'has to be done before',
#, 'has to be done after',
self.link_paths_other = [
'is blocked by', 'VULNThreat is created by', 'is budget item',
'Failure Implicated by', 'Data touches', 'Data sources used in',
'Needs', 'relates to', 'Missing High', 'Is missing',
'has budget item', 'Has VULNThreat', 'Is Needed by', 'Present',
'requires answer to ', 'Missing From (high)', 'Is Required by',
'is cloned by', 'Data sources', 'duplicates', 'blocks',
'Implicates Failure of', 'clones', 'is duplicated by',
'System used for', 'is answer to'
]
self.link_paths_mappings = {
'r0_up': [
'creates VULN', 'has RISK', 'creates RISK', 'creates R3',
'creates R2', 'creates R1', 'creates RISK', 'creates VULN'
],
'r0_down': [
'VULN is created by', 'is created by R4', 'is created by R3',
'is created by R2', 'is created by VULN', 'VULN is created by'
],
'risks_up': [
'creates VULN', 'fixes', 'reduces risk of', 'has RISK',
'creates RISK', 'creates R3', 'creates R2', 'creates R1'
],
'risks_down': [
'VULN is created by', 'is fixed by', 'risk reduced by',
'RISK affects', 'is created by R4', 'is created by R3',
'is created by R2'
],
'stakeholders_up': [
'has Technical Owner', 'has Management Owner',
'has Business Owner', 'has Stakeholder', 'is managed by'
],
'stakeholders_down': [
'is Technical Owner', 'is Management Owner',
'is Business Owner', 'is Stakeholder', 'is manager of'
]
}
def add_issue(self, key, issue):
if issue:
if self.issues is None:
self.issues = {}
self.issues[key] = issue
def add_node(self, key, issue=None):
if key not in self.nodes:
self.nodes.append(key)
self.add_issue(key, issue)
return self
def add_nodes(self, keys):
for key in keys:
self.add_node(key)
return self
def add_edge(self, from_key, link_type, to_key):
edge = (from_key, link_type, to_key)
if edge not in self.edges:
self.edges.append(
edge
) # this operation is very expensive with 10k+ edges (I wonder how does pydot performance looks like
return self
def add_edges(self, edges):
for edge in edges:
self.add_edge(edge[0], edge[1], edge[2])
# this needs to be refactored and improved (since there are now more than 10k issues)
# and this is called by the .add_all_linked_issues method
def all_link_types_per_key(self):
link_types = self.all_link_types()
result = {}
for link_type, data in link_types.items():
for key, items in data.items():
if result.get(key) is None: result[key] = {}
if result[key].get(link_type) is None:
result[key][link_type] = []
result[key][link_type] += items
return result
def add_all_linked_issues(self, keys=None, depth=1):
if keys is None:
keys = []
self.expand_link_types_to_add()
link_types_per_key = self.all_link_types_per_key(
) # this needs to be refactored and improved (since there are now more than 10k issues)
only_from_projects = self.puml_options['only-from-projects']
link_types_to_add = self.puml_options['link-types-to-add']
link_types_to_ignore = self.puml_options['link-types-to-ignore']
self.add_nodes(keys)
for i in range(0, depth):
for key in list(self.nodes):
data = link_types_per_key.get(key)
if data:
for issue_type, items in data.items():
for item in items:
if only_from_projects and item.split('-').pop(
0) not in only_from_projects:
continue
if link_types_to_add and issue_type not in link_types_to_add:
continue
if link_types_to_ignore and issue_type in link_types_to_ignore:
continue
self.add_edge(key, issue_type, item)
self.add_node(item)
return self
def add_linked_issues_of_types(self, link_types):
for link_type in link_types:
self.add_linked_issues_of_type(link_type)
return self
def add_linked_issues_of_type(self, link_type):
link_type = link_type.strip() # remove any spaces
mappings = self.all_link_types().get(link_type)
if mappings:
for key in list(
self.nodes
): # for each key provided (it is important to pin the self.nodes here since that value is changed below)
linked_issues = mappings.get(key)
if linked_issues:
for linked_issue in linked_issues:
edge = (key, link_type, linked_issue)
if not edge in self.edges:
self.edges.append(edge)
self.nodes.append(linked_issue)
return self
def add_link_types_as_nodes(self, issue_types_to_ignore=None):
if issue_types_to_ignore is None:
issue_types_to_ignore = []
if self.issues is None:
self.issues = self.api_issues.issues(self.nodes)
issue_types_to_ignore.append('_all')
for key in self.nodes:
issue = self.issues.get(key)
if issue:
for link_type, items in issue['Issue Links'].items():
if link_type not in issue_types_to_ignore:
link_type_node_key = "{0} - {1}".format(key, link_type)
for item in items:
self.add_edge(key, "", link_type_node_key)
self.add_edge(link_type_node_key, "", item)
return self
def add_nodes_from_epics(self):
issues = self.get_nodes_issues()
self.api_issues.set_default_indexes()
for key, issue in issues.items():
if issue and issue.get('Issue Type') == 'Epic':
for epic_key in self.api_issues.epic_issues(key):
self.add_edge(key, 'epic issue', epic_key)
self.add_node(epic_key)
return self
def edges__link_types(self):
return sorted(list(set([edge[1] for edge in self.edges])))
def expand_link_types_to_add(self):
link_types_to_add = self.puml_options['link-types-to-add']
extra_link_types = []
for link in link_types_to_add:
extra_links = self.link_paths_mappings.get(link)
if extra_links:
extra_link_types.extend(extra_links)
link_types_to_add.extend(extra_link_types)
self.puml_options['link-types-to-add'] = link_types_to_add
return self
def get_graph_data(self):
return {"nodes": self.get_nodes_issues(True), "edges": self.edges}
@index_by
def get_issues(self, reload=False): # better name for the method
return list(self.get_nodes_issues(reload).values())
def get_nodes_issues(self, reload=False): # depreciate
if self.issues is None or reload is True:
self.issues = self.api_issues.issues(self.nodes)
return self.issues
def get_puml(self):
return self.puml.puml
def graph(self):
return Graph().add_nodes(self.nodes) \
.add_edges(self.edges)
def issues__values_by_field(self, field_name):
results = []
issues = self.get_nodes_issues()
for node in self.nodes:
issue = issues.get(node)
if issue:
results.append(issue.get(field_name))
return list(set(results))
def issues__issue_types(self):
return self.issues__values_by_field('Issue Type')
# this needs to be refactored since it is quite an expensive call (invoked quite often)
def all_link_types(self, index="all"):
if self._link_types is None:
self._link_types = self.api_issues.link_types(index)
return self._link_types
def set_link_types_from_issues(self, issues):
self._link_types = self.api_issues.link_types_from_issues(
issues.values(), issues.keys())
return self
def load(self, path):
data = Json.load_file(path)
if data:
self.nodes = data['nodes']
self.edges = data['edges']
return self
def nodes__ratings(self):
return self.nodes__field_values('Rating')
def nodes__field_values(self, field):
values = []
issues = self.get_nodes_issues()
for node in self.nodes:
issue = issues.get(node)
if issue:
value = issue.get(field)
if value:
values.append(value)
return sorted(list(set(values)))
def remove_link_type(self, link_type_to_remove):
for edge in list(
self.edges
): # create new list so that it is not affected by the remove action
(from_key, link_type, to_key) = edge
if link_type in link_type_to_remove:
self.edges.remove(edge)
return self
def remove_no_links(self):
nodes_with_links = []
nodes_with_no_links = []
for edge in list(
self.edges): # get list of all nodes that have an edge
(from_key, link_type, to_key) = edge
nodes_with_links.append(from_key)
nodes_with_links.append(to_key)
for key in self.nodes: # diff it with the current list of nodes
if key not in nodes_with_links:
nodes_with_no_links.append(key)
self.remove_nodes(nodes_with_no_links) # remove nodes
return self
def remove_with_links(self):
nodes_with_links = []
for edge in list(
self.edges): # get list of all nodes that have an edge
(from_key, link_type, to_key) = edge
nodes_with_links.append(from_key)
nodes_with_links.append(to_key)
new_nodes = []
for key in self.nodes: # only add the nodes that are not in the
if key not in nodes_with_links: # nodes_with_links array
new_nodes.append(key)
self.nodes = new_nodes
self.edges = []
return self
# replace with filter only_edges_with_both_nodes
def remove_no_links_with_no_nodes(self):
new_edges = []
nodes = self.nodes
for edge in list(self.edges):
if edge[0] in nodes and edge[
2] in nodes: # if both edges exist in the current list of nodes
new_edges.append(edge) # keep the edge
self.edges = new_edges
return self
def remove_node(self, key):
if key in self.nodes:
self.nodes.remove(key)
for edge in list(self.edges):
(from_key, link_type, to_key) = edge
if from_key == key or to_key == key:
self.edges.remove(edge)
if self.issues and self.issues.get(key) is not None:
del self.issues[key]
return self
def remove_nodes(self, keys):
for key in keys:
self.remove_node(key)
return self
def remove_node_and_its_childen(self, key):
nodes_to_remove = [key]
for edge in list(self.edges):
(from_key, link_type, to_key) = edge
if from_key in nodes_to_remove: # check if there are blind spots in this logic
nodes_to_remove.append(to_key)
print('nodes_to_remove: {0}'.format(nodes_to_remove))
self.remove_nodes(nodes_to_remove)
return self
def save(self, path=None):
if path is None: path = Files.temp_file('graph.json')
data = {'nodes': self.nodes, 'edges': self.edges}
return Json.save_file_pretty(path, data)
def render_and_save_to_elk(
self,
graph_name=None,
graph_type=None,
channel=None,
user=None): # might need to move this to a Lambda function
from osbot_jira.api.graph.Lambda_Graph import Lambda_Graph # todo: find out why this needs to be here of it fail to load the dependency (could be caused by a cyclic dependency)
lambda_graph = Lambda_Graph()
graph_name = lambda_graph.save_gs_graph(self, graph_name, graph_type,
channel, user)
if lambda_graph.wait_for_elk_to_index_graph(
graph_name
): # wait for ELK to index the graph (to prevent a fetch before the data is available in ELK)
return graph_name
return None
def render_puml(self):
self.reset_puml()
return GS_Graph_Puml(self).render_puml()
# node_text_value = self.puml_options['node-text-value' ]
#
# if self.puml_options['left-to-right']: self.puml.add_line('left to right direction')
# if self.puml_options['width' ]: self.puml.add_line('scale {0} width '.format(self.puml_options['width' ]))
# if self.puml_options['height' ]: self.puml.add_line('scale {0} height'.format(self.puml_options['height']))
#
# if self.issues is None:
# self.issues = self.get_nodes_issues()
# for key in self.nodes:
# if node_text_value is None:
# self.puml.add_card(key, key)
# else:
# issue = self.issues.get(key)
# if issue:
# key_text = issue.get(node_text_value) #[0:30]
# if self.puml_options['show-key-in-text']:
# line = '{1} \\n<font:10><i>{0}</i></font>'.format(key,key_text)
# if self.node_type.get(key):
# self.puml.add_node(self.node_type.get(key),line, key)
# else:
# self.puml.add_card(line, key)
# else:
# line = '{0}'.format(key_text)
# if self.node_type.get(key):
# self.puml.add_node(self.node_type.get(key), line, key)
# else:
# self.puml.add_card(line, key)
# else:
# self.puml.add_card(key, key)
#
# for edge in self.edges:
# from_key = edge[0]
# link_type = edge[1]
# to_key = edge[2]
# if self.puml_options['show-edge-labels'] is False:
# link_type = ""
# self.puml.add_edge(from_key, to_key, 'down', link_type)
#
# for note in self.notes:
# position = note[0]
# key = note[1]
# text = note[2]
# line = "\n\n\t note {0} of {1} \n {2} \n end note".format(position, key, text)
# self.puml.add_line(line)
#
# self.puml.add_line('')
# for skin_param in self.skin_params:
# line = "skinparam {0} {1}".format(skin_param[0],skin_param[1])
# self.puml.add_line(line)
#
# self.puml.enduml()
#return self.puml
def render_puml_and_save_tmp(self):
self.render_puml()
return self.puml.save_tmp()
def render_puml_save_to_elk_and_to_tmp(self, graph_name=None):
self.render_and_save_to_elk(graph_name)
return self.puml.save_tmp()
def reset_puml(self):
self.puml.puml = "@startuml\n"
return self
def to_json(self, puml_config=True, store_issues=False):
if puml_config:
data = {
"nodes": self.nodes,
"edges": self.edges,
"notes": self.notes,
"node_type": self.node_type,
"skin_params": self.skin_params,
"create_params": self.create_params,
"puml_options": self.puml_options,
"puml_config": True
}
else:
data = {"nodes": self.nodes, "edges": self.edges}
if store_issues:
data['issues'] = self.get_nodes_issues()
return json.dumps(data)
def from_json(self, json_data):
data = json.loads(json_data)
if data.get('puml_config') is True:
self.nodes = data.get("nodes")
self.edges = data.get("edges")
self.notes = data.get("notes")
self.node_type = data.get("node_type")
self.skin_params = data.get("skin_params")
self.create_params = data.get("create_params")
self.puml_options = data.get("puml_options")
else:
self.nodes = data.get("nodes")
self.edges = data.get("edges")
self.issues = data.get('issues')
return self
def set_puml_node_text_value(self, value):
self.puml_options['node-text-value'] = value
return self
def set_puml_only_from_projects(self, value):
self.puml_options['only-from-projects'] = value
return self
def set_puml_link_types_to_add(self, value):
self.puml_options['link-types-to-add'] = value
return self
def set_puml_left_to_right(self, value):
self.puml_options['left-to-right'] = value
return self
def set_puml_direction_top_down(self):
self.puml_options['left-to-right'] = False
return self
def set_puml_show_key_in_text(self, value):
self.puml_options['show-key-in-text'] = value
return self
def set_puml_show_edge_labels(self, value):
self.puml_options['show-edge-labels'] = value
return self
def set_puml_width(self, value):
self.puml_options['width'] = value
return self
def set_puml_height(self, value):
self.puml_options['height'] = value
return self
def set_puml_on_add_node(self, callback):
self.puml.set_on_add_node(callback)
return self
def set_links_path_mode_to_down(self):
self.set_puml_link_types_to_add(self.risk_links_paths_down)
return self
def set_links_path_mode_to_up(self):
self.set_puml_link_types_to_add(self.risk_links_paths_up)
return self
def set_nodes_and_edges(self, nodes, edges):
self.nodes = nodes
self.edges = edges
return self
def set_link_paths_to_ignore(self, value):
self.puml_options['link-types-to-ignore'] = value
return self
def set_skin_param(self, name, value):
self.skin_params.append((name, value))
return self
def set_create_params(self, value):
self.create_params = value
return self
def set_edges(self, edges):
self.edges = edges
return self
def set_nodes(self, nodes):
self.nodes = nodes
return self
def stats(self):
return {
"count_nodes": len(self.nodes),
"count_edges": len(self.edges),
"size_puml": len(self.puml.puml)
}
def filter(self):
from osbot_jira.api.graph.Filters import Filters
return Filters().setup(graph=self)
# view
def view_nodes(self,
label_key=None,
show_key=False,
key_id='id',
label_id='label'):
nodes = []
issues = self.issues or {}
# if issues is None:
# issues = {}
for key in self.nodes:
issue = issues.get(key)
if issue and label_key is not None:
label = issue.get(label_key)
if label is None:
label = key
if show_key:
label = f'{key} - {label}'
else:
label = key
nodes.append({key_id: key, label_id: label})
return nodes
def view_nodes_and_edges(self, label_key=None, show_key=False):
nodes = self.view_nodes(label_key, show_key)
edges = []
for edge in self.edges:
edges.append({'from': edge[0], 'to': edge[2]})
return nodes, edges
# LEGACY (Check and delete)
def create_sub_graph_from_start_node(self, graph_nodes, start_node,
issue_types_paths):
issues = self.api_issues.issues(graph_nodes)
link_types = self.api_issues.link_types_from_issues(
issues.values(), graph_nodes)
self.issues = issues
self._link_types = link_types
self.puml_options['left-to-right'] = True
self.puml.add_line("scale 1")
self.add_node(start_node)
self.add_linked_issues_of_types(issue_types_paths)
self.render_puml()
class Test_API_Issues(unittest.TestCase):
def setUp(self):
self.key = 'RISK-1515'
self.api = API_Issues()
def test_epic_issues(self):
results = self.api.epic_issues('SEC-2024')
assert len(list(results)) > 10
def test_issue(self):
issue = self.api.issue(self.key)
assert issue.get('Key') == self.key
@unittest.skip('todo: fix: error on "if item[\'found\'] is False:" line')
def test_issues(self):
keys = ['RISK-1499', 'RISK-1496', 'AAA']
#result = self.api.elastic.es.mget(index='jira',
# doc_type='item',
# body={'ids': keys})
results = self.api.elastic().get_many(keys)
#Dev.pprint(results)
def test_issues_all(self):
assert len(self.api.issues_all('jira')) > 2000
def test_issues_created_in_last(self):
result = self.api.issues_created_in_last("3d")
assert len(result) > 10
def test_issues_updated_in_last(self):
result = self.api.issues_updated_in_last("3d")
assert len(result) > 10
def test_issues_created_in_last_nnn(self):
assert len(self.api.issues_created_in_last_seconds(
1 * 24 * 60 * 60)) > 0
assert len(self.api.issues_created_in_last_minutes(1 * 24 * 60)) > 0
assert len(self.api.issues_created_in_last_hours(1 * 24)) > 0
assert len(self.api.issues_created_in_last_days(1)) > 0
assert len(self.api.issues_created_in_last_weeks(1)) > 0
assert len(self.api.issues_created_in_last_months(1)) > 0
@unittest.skip('fails in CodeBuild')
def test_create_issue_table(self):
table = self.api.create_issue_table('RISK-424')
table.save_tmp()
#Dev.print(table.puml)
def test_labels(self):
result = self.api.labels()
assert 'VULN-2160' in result['AWSrisk']
assert len(result) > 650
def test_link_types(self):
result = self.api.link_types()
assert len(result) > 100
def test_keys(self):
keys = self.api.keys()
assert 'VULN-10' in keys
assert len(keys) > 15000
def test_search(self):
assert len(self.api.search("aw", "Summary", 100)) > 0
assert len(self.api.search("R1", "Labels", 10)) > 0
assert self.api.search("GSP-95", "Key",
10).pop().get('Key') == "GSP-95"
def test_search_using_lucene(self):
query = 'Project:RISK AND Status:"Fixed"'
results = self.api.search_using_lucene(query)
assert len(results) > 200
# def test_stakeholders(self):
# result = self.api.stakeholders()
# #Dev.pprint(result)
# #Dev.pprint(len(set(result)))
### move these to separate analysis file
@unittest.skip('Fails in CodeBuild')
def test_graph_issue_links_plant_uml(self):
puml = self.api.graph_issue_links_plant_uml(self.key)
puml.save_tmp()
puml._dev_send_to_slack()