def test_get_partition_default_root(self):
"""Ensure partition returned for default root."""
evidence = RawEvidence(
'foo.json', 'foo', partition={'fields': ['foo', 'bar']}
)
evidence.set_content(
json.dumps(
[
{
'foo': 'FOO', 'bar': 'BAR', 'baz': 'BAZ'
}, {
'foo': 'FOO', 'bar': 'BAR', 'baz': 'BLAH'
}, {
'foo': 'FOO', 'bar': 'BOO', 'baz': 'BAZ'
}
]
)
)
self.assertEqual(
json.loads(evidence.get_partition(['FOO', 'BAR'])),
[
{
'foo': 'FOO', 'bar': 'BAR', 'baz': 'BAZ'
}, {
'foo': 'FOO', 'bar': 'BAR', 'baz': 'BLAH'
}
]
)
def test_partition_keys_nested_root(self):
"""Ensure keys returned for a defined nested root."""
evidence = RawEvidence(
'foo.json',
'foo',
partition={
'fields': ['foo', 'bar'], 'root': 'nested.root'
}
)
evidence.set_content(
json.dumps(
{
'stuff': 'who cares?',
'nested': {
'nested_stuff': 'who cares?',
'root': [
{
'foo': 'FOO', 'bar': 'BAR', 'baz': 'BAZ'
}, {
'foo': 'FOO', 'bar': 'BAR', 'baz': 'BLAH'
}, {
'foo': 'FOO', 'bar': 'BOO', 'baz': 'BAZ'
}
],
'nested_other_stuff': 'so what?'
},
'other_stuff': 'so what?'
}
)
)
keys = evidence.partition_keys
self.assertEqual(len(keys), 2)
self.assertTrue(['FOO', 'BAR'] in keys)
self.assertTrue(['FOO', 'BOO'] in keys)
def test_is_partitioned(self):
"""Ensure is_partitioned property works as expected."""
evidence = RawEvidence('foo.json', 'foo')
self.assertFalse(evidence.is_partitioned)
evidence = RawEvidence(
'foo.json', 'foo', partition={'fields': ['foo']}
)
self.assertTrue(evidence.is_partitioned)
evidence = RawEvidence('foo.txt', 'foo', partition={'fields': ['foo']})
self.assertFalse(evidence.is_partitioned)
def fetch_compliance_configuration(self):
"""Fetch the compliance tooling configuration.
- Evidence is refreshed to the locker regardless of TTL
- Evidence is valid for the next two hours (for the check)
"""
evidence = RawEvidence('compliance_config.json', 'auditree', 2 * HOUR,
'Compliance Configuration')
evidence.set_content(json.dumps(self.config.raw_config))
self.locker.add_evidence(evidence)
def test_get_hash(self):
"""Test that get_hash returns a valid commit hash."""
with Locker(name=REPO_DIR, do_push=False) as locker:
evidence = RawEvidence('test.json', 'test_category', DAY,
'This tests evidence.py')
test_content = '{"key": "value"}'
evidence.set_content(test_content)
locker.add_evidence(evidence)
self.assertIsNone(locker.get_hash())
commit = locker.get_hash()
self.assertEqual(len(commit), 40)
def test_add_partitioned_evidence(self):
"""Test that partitioned evidence is added to locker as expected."""
with Locker(name=REPO_DIR) as locker:
evidence = RawEvidence('foo.json',
'cat_foo',
DAY,
'Partitioned evidence',
partition={'fields': ['lname']})
data = [{
'fname': 'simon',
'lname': 'metson'
}, {
'fname': 'al',
'lname': 'finkel'
}]
evidence.set_content(json.dumps(data))
locker.add_evidence(evidence)
location = os.path.join(locker.local_path, evidence.dir_path)
self.assertTrue(location.endswith('/raw/cat_foo'))
finkel_hash = get_sha256_hash('finkel', 10)
metson_hash = get_sha256_hash('metson', 10)
expected = [
'index.json', f'{finkel_hash}_foo.json',
f'{metson_hash}_foo.json'
]
self.assertEqual(set(os.listdir(location)), set(expected))
meta = json.loads(open(os.path.join(location, expected[0])).read())
self.assertEqual(len(meta.keys()), 1)
self.assertEqual(
set(meta['foo.json'].keys()), {
'description', 'last_update', 'partition_fields',
'partition_root', 'partitions', 'ttl'
})
self.assertEqual(meta['foo.json']['partition_fields'], ['lname'])
self.assertIsNone(meta['foo.json']['partition_root'])
self.assertEqual(len(meta['foo.json']['partitions']), 2)
self.assertEqual(meta['foo.json']['partitions'][finkel_hash],
['finkel'])
self.assertEqual(meta['foo.json']['partitions'][metson_hash],
['metson'])
self.assertEqual(
json.loads(open(os.path.join(location, expected[1])).read()),
[{
'fname': 'al',
'lname': 'finkel'
}])
self.assertEqual(
json.loads(open(os.path.join(location, expected[2])).read()),
[{
'fname': 'simon',
'lname': 'metson'
}])
def test_get_partition_with_nested_root(self):
"""Ensure partition returned for a defined nested root."""
evidence = RawEvidence(
'foo.json',
'foo',
partition={
'fields': ['foo', 'bar'], 'root': 'nested.root'
}
)
evidence.set_content(
json.dumps(
{
'stuff': 'who cares?',
'nested': {
'nested_stuff': 'who cares?',
'root': [
{
'foo': 'FOO', 'bar': 'BAR', 'baz': 'BAZ'
}, {
'foo': 'FOO', 'bar': 'BAR', 'baz': 'BLAH'
}, {
'foo': 'FOO', 'bar': 'BOO', 'baz': 'BAZ'
}
],
'nested_other_stuff': 'so what?'
},
'other_stuff': 'so what?'
}
)
)
self.assertEqual(
json.loads(evidence.get_partition(['FOO', 'BAR'])),
{
'stuff': 'who cares?',
'nested': {
'nested_stuff': 'who cares?',
'root': [
{
'foo': 'FOO', 'bar': 'BAR', 'baz': 'BAZ'
}, {
'foo': 'FOO', 'bar': 'BAR', 'baz': 'BLAH'
}
],
'nested_other_stuff': 'so what?'
},
'other_stuff': 'so what?'
}
)
def test_raw_evidence(self):
"""Test putting RawEvidence into the locker."""
locker = Locker(name=REPO_DIR)
with locker:
f = tempfile.mkstemp(prefix='testfile',
suffix='.json',
dir=FILES_DIR)
evidence_name = os.path.split(f[1])[1]
evidence = RawEvidence(evidence_name, 'test_category', DAY,
'This tests evidence.py')
test_content = '{"Squirtle": true, "Pikachu": 1, "Bulbasaur": 2}'
evidence.set_content(test_content)
locker.add_evidence(evidence)
self.assertEqual(len(locker.touched_files), 1)
def fetch_gh_org_collaborators(self):
"""Fetch collaborators from GH organization repositories."""
for config in self.config.get('org.permissions.org_integrity.orgs'):
host, org = config['url'].rsplit('/', 1)
for aff in config.get('collaborator_types', GH_ALL_COLLABORATORS):
url_hash = get_sha256_hash([config['url']], 10)
json_file = f'gh_{aff}_collaborators_{url_hash}.json'
path = ['permissions', json_file]
description = (
f'{aff.title()} collaborators of the {org} GH org')
self.config.add_evidences(
[RawEvidence(path[1], path[0], DAY, description)])
with raw_evidence(self.locker, '/'.join(path)) as evidence:
if evidence:
if host not in self.gh_pool:
self.gh_pool[host] = Github(base_url=host)
if not config.get('repos'):
repos = self.gh_pool[host].paginate_api(
f'orgs/{org}/repos')
config['repos'] = [repo['name'] for repo in repos]
collabs = {}
for repo in config['repos']:
collabs_url = f'repos/{org}/{repo}/collaborators'
collabs[repo] = self.gh_pool[host].paginate_api(
collabs_url, affiliation=aff)
evidence.set_content(json.dumps(collabs))
def fetch_workspaces(self):
"""Fetch Github repository Zenhub workspaces."""
for config in self.configs:
gh_host = config.get('github_host', GH_HOST_URL)
zh_root = config.get('api_root', ZH_API_ROOT)
repo = config['github_repo']
repo_hash = get_sha256_hash([gh_host, repo], 10)
fname = f'zh_repo_{repo_hash}_workspaces.json'
self.config.add_evidences([
RawEvidence(
fname, 'issues', DAY,
f'Zenhub workspaces for {gh_host}/{repo} repository')
])
with raw_evidence(self.locker, f'issues/{fname}') as evidence:
if evidence:
if gh_host not in self.gh_pool.keys():
self.gh_pool[gh_host] = Github(base_url=gh_host)
if zh_root not in self.zh_pool.keys():
self.zh_pool[zh_root] = BaseSession(zh_root)
service = 'zenhub'
if zh_root != ZH_API_ROOT:
service = 'zenhub_enterprise'
token = self.config.creds[service].token
self.zh_pool[zh_root].headers.update({
'Content-Type':
'application/json',
'X-Authentication-Token':
token
})
workspaces = self._get_workspaces(repo,
config.get('workspaces'),
gh_host, zh_root)
evidence.set_content(json.dumps(workspaces))
def setUpClass(cls):
"""Initialize the fetcher object with configuration settings."""
cls.config.add_evidences([
RawEvidence('abandoned_evidence.json', 'auditree', DAY,
'Abandoned evidence')
])
return cls
def test_partition_keys_dot_notation(self):
"""Ensure keys returned when using dot notation for key fields."""
evidence = RawEvidence(
'foo.json',
'foo',
partition={
'fields': ['dot_nota.foo', 'dot_nota.bar'],
'root': 'nested.root'
}
)
evidence.set_content(
json.dumps(
{
'stuff': 'who cares?',
'nested': {
'nested_stuff': 'who cares?',
'root': [
{
'dot_nota': {
'foo': 'FOO', 'bar': 'BAR'
},
'baz': 'BAZ'
},
{
'dot_nota': {
'foo': 'FOO', 'bar': 'BAR'
},
'baz': 'BLAH'
},
{
'dot_nota': {
'foo': 'FOO', 'bar': 'BOO'
},
'baz': 'BAZ'
}
],
'nested_other_stuff': 'so what?'
},
'other_stuff': 'so what?'
}
)
)
keys = evidence.partition_keys
self.assertEqual(len(keys), 2)
self.assertTrue(['FOO', 'BAR'] in keys)
self.assertTrue(['FOO', 'BOO'] in keys)
def setUpClass(cls):
"""Initialize the fetcher object with configuration settings."""
cls.config.add_evidences([
RawEvidence('cluster_resources.json', 'kubernetes', DAY,
'Kubernetes cluster resources')
])
cls.resource_types = cls.config.get(
'org.kubernetes.cluster_resources.types', RESOURCE_TYPES_DEFAULT)
return cls
def test_tmp_evidence(self):
"""Test putting TmpEvidence into the locker."""
locker = Locker(name=REPO_DIR)
with locker:
f = tempfile.mkstemp(prefix='testfile',
suffix='.json',
dir=FILES_DIR)
evidence_name = os.path.split(f[1])[1]
tmp_evidence = TmpEvidence(evidence_name, 'test_category', DAY,
'This tests evidence.py')
raw_evidence = RawEvidence(evidence_name, 'test_category', DAY,
'This tests evidence.py')
test_content = '{"BLAH": "Test"}'
tmp_evidence.set_content(test_content)
raw_evidence.set_content(test_content)
locker.add_evidence(tmp_evidence)
locker.add_evidence(raw_evidence)
self.assertEqual(len(locker.touched_files), 1)
def test_get_unpartitioned_evidence(self):
"""Test that unpartitioned evidence is retrieved from the locker."""
with Locker(name=REPO_DIR) as locker:
evidence = RawEvidence('foo.json', 'cat_foo', DAY,
'Partitioned evidence')
data = [{
'fname': 'simon',
'lname': 'metson'
}, {
'fname': 'al',
'lname': 'finkel'
}]
evidence.set_content(json.dumps(data))
locker.add_evidence(evidence)
locker.checkin()
partitioned = locker.get_evidence(evidence.path)
self.assertFalse(partitioned.is_partitioned)
content = json.loads(partitioned.content)
self.assertEqual(len(content), 2)
self.assertIn({'fname': 'simon', 'lname': 'metson'}, content)
self.assertIn({'fname': 'al', 'lname': 'finkel'}, content)
def test_partition_keys_default_root(self):
"""Ensure keys returned for default root."""
evidence = RawEvidence(
'foo.json', 'foo', partition={'fields': ['foo', 'bar']}
)
evidence.set_content(
json.dumps(
[
{
'foo': 'FOO', 'bar': 'BAR', 'baz': 'BAZ'
}, {
'foo': 'FOO', 'bar': 'BAR', 'baz': 'BLAH'
}, {
'foo': 'FOO', 'bar': 'BOO', 'baz': 'BAZ'
}
]
)
)
keys = evidence.partition_keys
self.assertEqual(len(keys), 2)
self.assertTrue(['FOO', 'BAR'] in keys)
self.assertTrue(['FOO', 'BOO'] in keys)
def test_abandoned_partitioned_evidence(self):
"""Test that ensures abandoned partitioned evidence is found."""
with Locker(name=REPO_DIR) as locker:
evidence = RawEvidence('foo.json',
'cat_foo',
DAY,
'Partitioned evidence',
partition={'fields': ['lname']})
data = [{
'fname': 'simon',
'lname': 'metson'
}, {
'fname': 'al',
'lname': 'finkel'
}]
evidence.set_content(json.dumps(data))
locker.add_evidence(evidence)
locker.checkin()
# Test abandoned
self.assertEqual(set(), locker.get_abandoned_evidences())
abandoned_meta = {
'foo.json': {
'last_update': '2017-10-01T00:00:00.0',
'partitions': {
'1197ced566_foo.json': 'meh',
'3eeaf57767_foo.json': 'bleh'
}
}
}
with open(locker.get_index_file(evidence), 'w') as f:
f.write(json.dumps(abandoned_meta))
locker.checkin()
self.assertEqual(
{
'raw/cat_foo/1197ced566_foo.json',
'raw/cat_foo/3eeaf57767_foo.json'
}, locker.get_abandoned_evidences())
def test_validate_raw_evidence(self, validation, check_mock,
evidence_mock):
"""Test validate method in locker for an evidence."""
config = ComplianceConfig()
check_mock.return_value = config
evidence_mock.return_value = config
with Locker(name=REPO_DIR, do_push=True) as locker:
f = tempfile.mkstemp(prefix='testfile',
suffix='.json',
dir=FILES_DIR)
test_evidence_name = os.path.split(f[1])[1]
evidence = RawEvidence(test_evidence_name, 'test_category', DAY,
'This tests if the validation works')
config.add_evidences([evidence])
test_content = '{"Squirtle": true, "Pikachu": 1, "Bulbasaur": 2}'
evidence.set_content(test_content)
locker.add_evidence(evidence)
evidence = get_evidence_by_path(
os.path.join(evidence.rootdir, evidence.category,
evidence.name))
self.assertTrue(isinstance(evidence, RawEvidence))
validation.return_value = True
locker.get_evidence(
os.path.join('raw', 'test_category', test_evidence_name))
def setUpClass(cls):
"""Initialize the fetcher object with configuration settings."""
cls.config.add_evidences(
[
RawEvidence(
'cluster_list.json',
'ibm_cloud',
DAY,
'IBM Cloud cluster list inventory'
)
]
)
headers = {'Accept': 'application/json'}
cls.session(IC_CONTAINERS_BASE_URL, **headers)
return cls
def setUpClass(cls):
"""Initialize the fetcher object with configuration settings."""
cls.config.add_evidences([
RawEvidence('python_packages.json', 'auditree', DAY,
'Python Package List'),
PackageReleaseEvidence('auditree_arboretum_releases.xml',
'auditree', DAY,
'Auditree Arboretum PyPI releases'),
PackageReleaseEvidence('auditree_framework_releases.xml',
'auditree', DAY,
'Auditree Framework PyPI releases')
])
headers = {
'Content-Type': 'application/xml',
'Accept': 'application/xml'
}
cls.session(PYPI_RSS_BASE_URL, **headers)
return cls
def fetch_issues(self):
"""Fetch Github repository issues."""
for config in self.configs:
host = config.get('host', GH_HOST_URL)
repo = config['repo']
fname = f'gh_repo_{get_sha256_hash([host, repo], 10)}_issues.json'
self.config.add_evidences([
RawEvidence(fname, 'issues', DAY,
f'Github issues for {host}/{repo} repository')
])
with raw_evidence(self.locker, f'issues/{fname}') as evidence:
if evidence:
if host not in self.gh_pool.keys():
self.gh_pool[host] = Github(base_url=host)
issues = []
for search in self._compose_searches(config, host):
issue_ids = [i['id'] for i in issues]
for result in self.gh_pool[host].search_issues(search):
if result['id'] not in issue_ids:
issues.append(result)
evidence.set_content(json.dumps(issues))
def test_locker_as_expected(self):
"""Test that checks contents of the locker are as expected."""
with Locker(name=REPO_DIR) as locker:
test_content_one = '{"key": "value1"}'
evidence_one = RawEvidence('test_one.json', 'test_category', DAY,
'This tests locker')
evidence_one.set_content(test_content_one)
locker.add_evidence(evidence_one)
evidence_two = RawEvidence('test_two.json', 'test_category', DAY,
'This tests locker')
evidence_two.set_content(test_content_one)
locker.add_evidence(evidence_two)
evidence_three = RawEvidence('test_three.json',
'test_category_two', DAY,
'This tests locker')
evidence_three.set_content(test_content_one)
locker.add_evidence(evidence_three)
locker.checkin()
self.assertTrue(
locker.get_index_file(evidence_one).endswith(
f'/raw/test_category/{INDEX_FILE}'))
self.assertTrue(
locker.get_index_file(evidence_two).endswith(
f'/raw/test_category/{INDEX_FILE}'))
self.assertTrue(
locker.get_index_file(evidence_three).endswith(
f'/raw/test_category_two/{INDEX_FILE}'))
# Test abandoned
self.assertEqual(set(), locker.get_abandoned_evidences())
with open(locker.get_index_file(evidence_three), 'w') as f:
meta = ('{"test_three.json":'
'{"last_update":"2017-10-01T00:00:00.0"} }')
f.write(meta)
locker.checkin()
self.assertEqual({'raw/test_category_two/test_three.json'},
locker.get_abandoned_evidences())
# -*- mode:python; coding:utf-8 -*-
# Copyright (c) 2020 IBM Corp. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from compliance.config import get_config
from compliance.evidence import DAY, ReportEvidence, RawEvidence
get_config().add_evidences([
RawEvidence('world_clock_utc.json', 'time', DAY,
'Coordinated Universal Time'),
ReportEvidence('world_clock.md', 'time', DAY,
'World Clock Analysis report.')
])
