def get_github_resolve_alert_status():
today = datetime.date.today().isoformat()
by_alert_status = defaultdict(list)
repositories = storage.read_json(f"{today}/data/repositories.json")
for repo in repositories["active"]:
response = github_rest_client.get(
f"/repos/{repo.owner.login}/{repo.name}/vulnerability-alerts")
alerts_enabled = response.status_code == 204
vulnerable = repo.vulnerabilityAlerts.edges
if vulnerable:
status = "vulnerable"
elif alerts_enabled:
status = "clean"
else:
status = "disabled"
# append status to repo in original repositories file
repo.securityAdvisoriesEnabledStatus = alerts_enabled
by_alert_status[status].append(repo)
storage.save_json(f"{today}/data/repositories.json", repositories)
status = storage.save_json(f"{today}/data/alert_status.json",
by_alert_status)
return status
def test_delete_json(self):
test_data = '{"chessfield": [[null, null, null, null, null, null, null, null], [null, "b", "b", "b", "b", "b", "b", "b"], [null, null, null, null, null, null, null, null], ["b", null, null, null, null, null, null, null], ["w", null, null, null, null, null, null, null], [null, null, null, null, null, null, null, null], [null, "w", "w", "w", "w", "w", "w", "w"], [null, null, null, null, null, null, null, null]], "player1": {"name": "1", "color": "b", "turn": false}, "player2": {"name": "2", "color": "w", "turn": true}}'
storage.save_json(test_data)
storage.delete_json()
self.assertIsNone(storage.load_json())
def test_save_load(self):
test_data = '{"chessfield": [[null, null, null, null, null, null, null, null], [null, "b", "b", "b", "b", "b", "b", "b"], [null, null, null, null, null, null, null, null], ["b", null, null, null, null, null, null, null], ["w", null, null, null, null, null, null, null], [null, null, null, null, null, null, null, null], [null, "w", "w", "w", "w", "w", "w", "w"], [null, null, null, null, null, null, null, null]], "player1": {"name": "1", "color": "b", "turn": false}, "player2": {"name": "2", "color": "w", "turn": true}}'
storage.save_json(test_data)
data_output = storage.load_json()
self.assertEqual(test_data, data_output)
def get_dependabot_status(org, today):
dependabot_status = dependabot_api.get_active_repos(org)
storage.save_json(f"{today}/data/dependabot_status.json",
dependabot_status)
repositories = storage.read_json(f"{today}/data/repositories.json")
dependabot_repo_names = [r.attributes.name for r in dependabot_status]
for r in repositories["active"]:
r.dependabotEnabledStatus = r.name in dependabot_repo_names
storage.save_json(f"{today}/data/repositories.json", repositories)
def analyse_repo_ownership(today):
list_owners = defaultdict(list)
list_topics = defaultdict(list)
repositories = storage.read_json(f"{today}/data/repositories.json")
for repo in repositories["active"]:
if repo.repositoryTopics.edges:
for topics in repo.repositoryTopics.edges:
list_owners[repo.name].append(topics.node.topic.name)
list_topics[topics.node.topic.name].append(repo.name)
owner_status = storage.save_json(f"{today}/data/owners.json", list_owners)
topic_status = storage.save_json(f"{today}/data/topics.json", list_topics)
return owner_status and topic_status
def route_data_overview_vulnerable_repositories(today):
alert_status = storage.read_json(f"{today}/routes/count_alert_status.json")
vulnerable_by_severity = storage.read_json(
f"{today}/data/vulnerable_by_severity.json")
severity_counts = stats.count_types(vulnerable_by_severity)
vulnerable_count = sum(severity_counts.values())
severities = vulnerability_summarizer.SEVERITIES
template_data = {
"content": {
"title": "Overview - Vulnerable repositories",
"org": config.get_value("github_org"),
"vulnerable": {
"severities": severities,
"all": vulnerable_count,
"by_severity": severity_counts,
"repositories": vulnerable_by_severity,
},
"alert_status": alert_status,
},
"footer": {
"updated": today
},
}
template_status = storage.save_json(
f"{today}/routes/overview_vulnerable_repositories.json", template_data)
return template_status
def route_data_overview_activity(today):
repositories = storage.read_json(f"{today}/data/repositories.json")
counts = defaultdict(int)
repositories_by_activity = defaultdict(list)
for repo in repositories["active"]:
if "recentCommitDaysAgo" in repo:
currency = repo.currencyBand
counts[currency] += 1
repositories_by_activity[currency].append(repo)
bands = ["within a month", "within a quarter", "within a year", "older"]
template_data = {
"content": {
"title": "Overview - Activity",
"org": config.get_value("github_org"),
"activity": {
"bands": bands,
"counts": counts,
"repositories": repositories_by_activity,
},
},
"footer": {
"updated": today
},
}
overview_activity_status = storage.save_json(
f"{today}/routes/overview_activity.json", template_data)
return overview_activity_status
def route_data_overview_repositories_by_status(today):
repositories_by_status = storage.read_json(
f"{today}/data/repositories.json")
status_counts = stats.count_types(repositories_by_status)
repo_count = sum(status_counts.values())
vulnerable_by_severity = storage.read_json(
f"{today}/data/vulnerable_by_severity.json")
template_data = {
"content": {
"title": "Overview - Repositories by status",
"org": config.get_value("github_org"),
"repositories": {
"all": repo_count,
"by_status": status_counts,
"repos_by_status": repositories_by_status,
},
"vulnerable_by_severity": vulnerable_by_severity,
},
"footer": {
"updated": today
},
}
overview_status = storage.save_json(
f"{today}/routes/overview_repositories_by_status.json", template_data)
return overview_status
def analyse_team_membership(today):
storage.read_json(f"{today}/data/topics.json")
repositories = storage.read_json(f"{today}/data/repositories.json")
# This one can't currently use storage because it's
# outside the output folder and not written to S3.
with open("teams.json", "r") as teams_file:
teams = json.loads(teams_file.read())
for status, repo_list in repositories.items():
for repo in repo_list:
repo_team = "unknown"
repo_topics = []
if repo.repositoryTopics:
repo_topics = [
topic_edge.node.topic.name
for topic_edge in repo.repositoryTopics.edges
]
for team, team_topics in teams.items():
for topic in team_topics:
if topic in repo_topics:
repo_team = team
repo.team = repo_team
updated = storage.save_json(f"{today}/data/repositories.json",
repositories)
def save_game(chessfield, player1, player2):
'''Puts the game objects into a dict and calls the save_json() method of the storage module
Arguments:
chessfield {list} -- current chessfield as list
player1 {obj} -- 1. player
player2 {obj} -- 2. player
Returns:
---
'''
data = {}
data['chessfield'] = chessfield
data['player1'] = player1.to_dict()
data['player2'] = player2.to_dict()
storage.save_json(data)
def route_data_overview_alert_status(today):
by_alert_status = {"public": {}}
alert_statuses = storage.read_json(f"{today}/data/alert_status.json")
for status, repos in alert_statuses.items():
by_alert_status["public"][status] = len(repos)
status = storage.save_json(f"{today}/routes/count_alert_status.json",
by_alert_status)
return status
def analyse_vulnerability_patch_recommendations(today):
repositories = storage.read_json(f"{today}/data/repositories.json")
vulnerable_list = [
node for node in repositories["active"]
if node.vulnerabilityAlerts.edges
]
vulnerable_by_severity = vulnerability_summarizer.group_by_severity(
vulnerable_list)
storage.save_json(f"{today}/data/vulnerable_by_severity.json",
vulnerable_by_severity)
for repo in repositories["active"]:
for severity, vuln_repos in vulnerable_by_severity.items():
for vuln_repo in vuln_repos:
if repo.name == vuln_repo.name:
log.debug(repo.name)
repo.maxSeverity = severity
repo.patches = vulnerability_summarizer.get_patch_list(
repo)
repo.vulnerabiltyCounts = (
vulnerability_summarizer.
get_repository_severity_counts(repo))
updated = storage.save_json(f"{today}/data/repositories.json",
repositories)
# TODO Remove this redundant rebuilding step
repositories = storage.read_json(f"{today}/data/repositories.json")
vulnerable_list = [
node for node in repositories["active"]
if node.vulnerabilityAlerts.edges
]
vulnerable_by_severity = vulnerability_summarizer.group_by_severity(
vulnerable_list)
storage.save_json(f"{today}/data/vulnerable_by_severity.json",
vulnerable_by_severity)
return updated
def update_github_advisories_status():
log.debug("update_github_advisories_status")
today = datetime.date.today().isoformat()
current = get_current_audit()
by_alert_status = defaultdict(list)
today_repositories = storage.read_json(f"{today}/data/repositories.json")
current_repositories = storage.read_json(
f"{current}/data/repositories.json")
for state in today_repositories.keys():
log.debug(f"{state} repos: {len(today_repositories[state])}")
with ThreadPoolExecutor(max_workers=20) as executor:
for r in executor.map(get_adv_status, today_repositories[state]):
by_alert_status[r.status].append(r)
storage.save_json(f"{today}/data/repositories.json", today_repositories)
status = storage.save_json(f"{today}/data/alert_status.json",
by_alert_status)
return status
def get_github_repositories_and_classify_by_status(org, today):
cursor = None
last = False
repository_list = []
while not last:
page = pgraph.query("all", org=org, nth=100, after=cursor)
repository_list.extend(page.organization.repositories.nodes)
last = not page.organization.repositories.pageInfo.hasNextPage
cursor = page.organization.repositories.pageInfo.endCursor
repositories_by_status = repository_summarizer.group_by_status(
repository_list)
save_to = f"{today}/data/repositories.json"
updated = storage.save_json(save_to, repositories_by_status)
def analyse_activity_refs(today):
now = arrow.utcnow()
refs = storage.read_json(f"{today}/data/activity_refs.json")
repositories = storage.read_json(f"{today}/data/repositories.json")
for repo_name, repo in refs.items():
repo_commit_dates = []
for ref in repo.refs.edges:
branch = ref.node
for commit_edge in branch.target.history.edges:
commit = commit_edge.node
committed_date = arrow.get(commit.committedDate)
repo_commit_dates.append(committed_date)
repo_commit_dates.sort()
if len(repo_commit_dates) > 0:
first_commit = repo_commit_dates[0]
last_commit = repo_commit_dates[-1]
delta = last_commit - first_commit
average_delta = delta / len(repo_commit_dates)
average_days = average_delta.days
currency_delta = now - last_commit
currency_days = currency_delta.days
for repo in repositories["active"]:
if repo.name == repo_name:
repo.recentCommitDaysAgo = currency_days
repo.averageCommitFrequency = average_days
repo.isActive = currency_days < 365 and average_days < 180
currency_band = "older"
if currency_days <= 28:
currency_band = "within a month"
elif currency_days <= 91:
currency_band = "within a quarter"
elif currency_days <= 365:
currency_band = "within a year"
repo.currencyBand = currency_band
updated = storage.save_json(f"{today}/data/repositories.json",
repositories)
return updated
def analyse_pull_request_status(today):
now = arrow.utcnow()
pull_requests = storage.read_json(f"{today}/data/activity_prs.json")
repositories = storage.read_json(f"{today}/data/repositories.json")
for repo in repositories["active"]:
repo_prs = pull_requests[repo.name]
pr_final_status = "No pull requests in this repository"
if repo_prs.pullRequests.edges:
node = repo_prs.pullRequests.edges[0].node
if node.merged:
reference_date = arrow.get(node.mergedAt)
pr_status = "merged"
elif node.closed:
reference_date = arrow.get(node.closedAt)
pr_status = "closed"
else:
reference_date = arrow.get(node.createdAt)
pr_status = "open"
if reference_date < now.shift(years=-1):
pr_final_status = (
f"Last pull request more than a year ago ({pr_status})")
elif reference_date < now.shift(months=-1):
pr_final_status = (
f"Last pull request more than a month ago ({pr_status})")
elif reference_date < now.shift(weeks=-1):
pr_final_status = (
f"Last pull request more than a week ago ({pr_status})")
else:
pr_final_status = f"Last pull request this week ({pr_status})"
repo.recentPullRequestStatus = pr_final_status
status = storage.save_json(f"{today}/data/repositories.json", repositories)
return status
def get_github_activity_prs_audit(org, today):
cursor = None
last = False
repository_list = []
while not last:
page = pgraph.query("prs", org=org, nth=100, after=cursor)
repository_list.extend(page.organization.repositories.nodes)
last = not page.organization.repositories.pageInfo.hasNextPage
cursor = page.organization.repositories.pageInfo.endCursor
total = len(repository_list)
log.debug(f"Repository list count: {total}")
repository_lookup = {repo.name: repo for repo in repository_list}
total = len(repository_lookup.keys())
log.debug(f"Repository lookup count: {total}")
updated = storage.save_json(f"{today}/data/activity_prs.json",
repository_lookup)
def route_data_overview_monitoring_status(today):
monitoring_disabled = storage.read_json(f"{today}/data/alert_status.json")
repositories = storage.read_json(f"{today}/data/repositories.json")
monitoring_alert = len(monitoring_disabled["disabled"])
dependabot_count = 0
advisory_count = 0
for repo in repositories["active"]:
dependabot_status = repo.dependabotEnabledStatus
advisory_status = repo.securityAdvisoriesEnabledStatus
if dependabot_status:
dependabot_count += 1
elif advisory_status:
advisory_count += 1
template_data = {
"content": {
"title": "Overview - Monitoring status",
"org": config.get_value("github_org"),
"monitoring": {
"dependabot_enabled": dependabot_count,
"advisory_enabled": advisory_count,
"disabled": monitoring_alert,
},
},
"footer": {
"updated": today
},
}
monitoring_status = storage.save_json(
f"{today}/routes/overview_monitoring_status.json", template_data)
return monitoring_status
def update_history(history):
return storage.save_json("all/data/history.json", history)
def test_save_json_error(self):
game_data = Player('1', 'b', False)
storage.save_json(game_data)
self.assertRaises(ValueError)
