def test_add_partitioned_evidence(self):
"""Test that partitioned evidence is added to locker as expected."""
with Locker(name=REPO_DIR) as locker:
evidence = RawEvidence('foo.json',
'cat_foo',
DAY,
'Partitioned evidence',
partition={'fields': ['lname']})
data = [{
'fname': 'simon',
'lname': 'metson'
}, {
'fname': 'al',
'lname': 'finkel'
}]
evidence.set_content(json.dumps(data))
locker.add_evidence(evidence)
location = os.path.join(locker.local_path, evidence.dir_path)
self.assertTrue(location.endswith('/raw/cat_foo'))
finkel_hash = get_sha256_hash('finkel', 10)
metson_hash = get_sha256_hash('metson', 10)
expected = [
'index.json', f'{finkel_hash}_foo.json',
f'{metson_hash}_foo.json'
]
self.assertEqual(set(os.listdir(location)), set(expected))
meta = json.loads(open(os.path.join(location, expected[0])).read())
self.assertEqual(len(meta.keys()), 1)
self.assertEqual(
set(meta['foo.json'].keys()), {
'description', 'last_update', 'partition_fields',
'partition_root', 'partitions', 'ttl'
})
self.assertEqual(meta['foo.json']['partition_fields'], ['lname'])
self.assertIsNone(meta['foo.json']['partition_root'])
self.assertEqual(len(meta['foo.json']['partitions']), 2)
self.assertEqual(meta['foo.json']['partitions'][finkel_hash],
['finkel'])
self.assertEqual(meta['foo.json']['partitions'][metson_hash],
['metson'])
self.assertEqual(
json.loads(open(os.path.join(location, expected[1])).read()),
[{
'fname': 'al',
'lname': 'finkel'
}])
self.assertEqual(
json.loads(open(os.path.join(location, expected[2])).read()),
[{
'fname': 'simon',
'lname': 'metson'
}])
def fetch_workspaces(self):
"""Fetch Github repository Zenhub workspaces."""
for config in self.configs:
gh_host = config.get('github_host', GH_HOST_URL)
zh_root = config.get('api_root', ZH_API_ROOT)
repo = config['github_repo']
repo_hash = get_sha256_hash([gh_host, repo], 10)
fname = f'zh_repo_{repo_hash}_workspaces.json'
self.config.add_evidences([
RawEvidence(
fname, 'issues', DAY,
f'Zenhub workspaces for {gh_host}/{repo} repository')
])
with raw_evidence(self.locker, f'issues/{fname}') as evidence:
if evidence:
if gh_host not in self.gh_pool.keys():
self.gh_pool[gh_host] = Github(base_url=gh_host)
if zh_root not in self.zh_pool.keys():
self.zh_pool[zh_root] = BaseSession(zh_root)
service = 'zenhub'
if zh_root != ZH_API_ROOT:
service = 'zenhub_enterprise'
token = self.config.creds[service].token
self.zh_pool[zh_root].headers.update({
'Content-Type':
'application/json',
'X-Authentication-Token':
token
})
workspaces = self._get_workspaces(repo,
config.get('workspaces'),
gh_host, zh_root)
evidence.set_content(json.dumps(workspaces))
def fetch_gh_org_collaborators(self):
"""Fetch collaborators from GH organization repositories."""
for config in self.config.get('org.permissions.org_integrity.orgs'):
host, org = config['url'].rsplit('/', 1)
for aff in config.get('collaborator_types', GH_ALL_COLLABORATORS):
url_hash = get_sha256_hash([config['url']], 10)
json_file = f'gh_{aff}_collaborators_{url_hash}.json'
path = ['permissions', json_file]
description = (
f'{aff.title()} collaborators of the {org} GH org')
self.config.add_evidences(
[RawEvidence(path[1], path[0], DAY, description)])
with raw_evidence(self.locker, '/'.join(path)) as evidence:
if evidence:
if host not in self.gh_pool:
self.gh_pool[host] = Github(base_url=host)
if not config.get('repos'):
repos = self.gh_pool[host].paginate_api(
f'orgs/{org}/repos')
config['repos'] = [repo['name'] for repo in repos]
collabs = {}
for repo in config['repos']:
collabs_url = f'repos/{org}/{repo}/collaborators'
collabs[repo] = self.gh_pool[host].paginate_api(
collabs_url, affiliation=aff)
evidence.set_content(json.dumps(collabs))
def test_org_direct_collaborators(self):
"""Check that there are no direct collaborators in the org repos."""
orgs = self.config.get('org.permissions.org_integrity.orgs')
evidence_paths = {}
exceptions = {}
for org in orgs:
if 'direct' not in org.get('collaborator_types', []):
continue
host, org_name = org['url'].rsplit('/', 1)
service = 'gh'
if 'gitlab' in host:
service = 'gl'
elif 'bitbucket' in host:
service = 'bb'
url_hash = get_sha256_hash([org['url']], 10)
filename = f'{service}_direct_collaborators_{url_hash}.json'
path = f'raw/permissions/{filename}'
evidence_paths[org_name] = path
exceptions[org_name] = org.get('exceptions', [])
with evidences(self, evidence_paths) as raws:
self._generate_results(raws, exceptions)
def index(self, evidence, checks=None, evidence_used=None):
"""
Add an evidence to the git index.
:param evidence: the evidence object.
:param checks: A list of checks used to generate report content. Only
applicable for check generated ReportEvidence.
:param evidence_used: metadata for evidence used by a check. Only
applicable for check generated ReportEvidence.
"""
with self.lock:
index_file = self.get_index_file(evidence)
repo_files = [index_file]
if not os.path.exists(index_file):
metadata = {}
else:
metadata = json.loads(open(index_file).read())
ev_meta = metadata.get(evidence.name, {})
old_parts = ev_meta.get('partitions', {}).keys()
metadata[evidence.name] = {
'last_update': self.commit_date,
'ttl': evidence.ttl,
'description': evidence.description
}
tombstones = None
if getattr(evidence, 'is_partitioned', False):
unpartitioned = self.get_file(evidence.path)
if os.path.isfile(unpartitioned):
# Remove/tombstone unpartitioned evidence file
# replaced by partitioned evidence files
self.repo.index.remove([unpartitioned], working_tree=True)
tombstones = self.create_tombstone_metadata(
evidence.name, ev_meta, 'Evidence is partitioned')
parts = {}
for key in evidence.partition_keys:
sha256_hash = get_sha256_hash(key, 10)
parts[sha256_hash] = key
repo_file = self.get_file(
f'{evidence.dir_path}/{sha256_hash}_{evidence.name}')
repo_files.append(repo_file)
dead_parts = set(old_parts) - set(parts.keys())
if dead_parts:
# Remove/tombstone partitioned evidence files
# no longer part of the evidence content
self.remove_partitions(evidence, dead_parts)
tombstones = self.create_tombstone_metadata(
dead_parts, ev_meta,
'Partition no longer part of evidence')
metadata[evidence.name].update({
'partition_fields': evidence.part_fields,
'partition_root': evidence.part_root,
'partitions': parts
})
if tombstones is None:
# Preserve prior tombstones
tombstones = ev_meta.get('tombstones')
else:
# Remove/tombstone partitioned evidence files
# replaced by unpartitioned evidence file
self.remove_partitions(evidence, old_parts)
tombstones = self.create_tombstone_metadata(
old_parts, ev_meta, 'Evidence no longer partitioned')
repo_files.append(self.get_file(evidence.path))
if tombstones:
metadata[evidence.name]['tombstones'] = tombstones
if checks is not None:
metadata[evidence.name]['checks'] = checks
if evidence_used is not None:
metadata[evidence.name]['evidence'] = evidence_used
with open(index_file, 'w') as f:
f.write(format_json(metadata))
self.repo.index.add(repo_files)
def test_sha256_hash_multiple_keys(self):
"""Test hash is correct when a list of keys is provided."""
self.assertEqual(get_sha256_hash(['foo', 'bar', 'baz', 1234]),
hashlib.sha256(b'foobarbaz1234').hexdigest())
def test_sha256_hash_sized(self):
"""Test the first 'size' number of characters are returned."""
actual = get_sha256_hash(['foo'], 10)
self.assertEqual(len(actual), 10)
self.assertTrue(hashlib.sha256(b'foo').hexdigest().startswith(actual))
def test_sha256_hash_oversized(self):
"""Test when size is too big, the full hash is returned."""
expected = hashlib.sha256(b'foo').hexdigest()
self.assertEqual(len(expected), 64)
self.assertEqual(get_sha256_hash(['foo'], 1000), expected)
def test_sha256_hash_no_size(self):
"""Test when no size is provided, the full hash is returned."""
self.assertEqual(get_sha256_hash(['foo']),
hashlib.sha256(b'foo').hexdigest())