def to_json(self):
j = super().to_json()
j["status"] = self.status.value
j["expires_at"] = datetime_to_rfc3339(self.expires_at)
j["created_at"] = datetime_to_rfc3339(self.created_at)
j["last_update"] = datetime_to_rfc3339(self.last_updated)
return j
class TestExecutableWhitelistItem:
ten_min_ago = datetime_to_rfc3339(
anchore_now_datetime() - datetime.timedelta(minutes=10)
)
ten_min_from_now = datetime_to_rfc3339(
anchore_now_datetime() + datetime.timedelta(minutes=10)
)
@pytest.mark.parametrize(
"expires_on,expected", [(ten_min_ago, True), (ten_min_from_now, False)]
)
def test_is_expired(self, expires_on, expected):
item_json = {"expires_on": expires_on, "gate": "fake_gate"}
item = ExecutableWhitelistItem(item_json, None)
assert item.is_expired() == expected
@pytest.mark.parametrize(
"expires_on,exception,expected",
[(None, None, False), ("", None, False), ("2020-10-15", Exception, None)],
)
def test_is_expired_bad_input(self, expires_on, exception, expected):
if exception:
with pytest.raises(exception):
item_json = {"expires_on": expires_on, "gate": "fake_gate"}
ExecutableWhitelistItem(item_json, None)
else:
item_json = {"expires_on": expires_on, "gate": "fake_gate"}
item = ExecutableWhitelistItem(item_json, None)
assert item.is_expired() == expected
def user_db_to_msg(user):
if user is None:
return None
return {
'username': user['username'],
'created_at': datetime_to_rfc3339(datetime.datetime.utcfromtimestamp(user['created_at'])),
'last_updated': datetime_to_rfc3339(datetime.datetime.utcfromtimestamp(user['last_updated']))
}
def account_db_to_msg(account):
if account is None:
return None
return {
'name': account['name'],
'email': account['email'],
'state': account['state'].value if type(account['state']) != str else account['state'],
'type': account['type'] if type(account['type']) == str else account['type'].value ,
'created_at': datetime_to_rfc3339(datetime.datetime.utcfromtimestamp(account['created_at'])),
'last_updated': datetime_to_rfc3339(datetime.datetime.utcfromtimestamp(account['last_updated']))
}
def test_rfc3339():
# parsing/validation and conversion symmetry
for rfc_str, assert_targets in rfc3339_examples:
print("testing input string: {}".format(rfc_str))
rc = utils.rfc3339str_to_epoch(rfc_str)
print("\trfc3339_to_epoch: {}".format(rc))
assert (rc == assert_targets['epoch'])
print("\tepoch assertion passed")
rc = utils.rfc3339str_to_datetime(rfc_str)
print("\trfc3339_to_datetime: {}".format(rc))
assert (rc == assert_targets['dt'])
print("\tdatetime assertion passed")
for epoch, assert_targets in epoch_examples:
print("testing input epoch: {}".format(epoch))
rc = utils.epoch_to_rfc3339(epoch)
print("\tepoch_to_rfc3339: {}".format(rc))
assert (rc == assert_targets['rfc3339'])
print("\tdatetime assertion passed")
for dt, assert_targets in dt_examples:
print("testing input datetime: {}".format(dt))
rc = utils.datetime_to_rfc3339(dt)
print("\tdatetime_to_rfc3339: {}".format(rc))
assert (rc == assert_targets['rfc3339'])
print("\tdatetime assertion passed")
def list_import_content(operation_id: str, account: str, content_type: str):
"""
Generic way to list content of a given type from the db entries
:param operation_id:
:param account:
:param content_type:
:return:
"""
try:
with session_scope() as db_session:
resp = []
for x in (db_session.query(ImageImportContent).join(
ImageImportContent.operation).filter(
ImageImportOperation.account == account,
ImageImportOperation.uuid == operation_id,
ImageImportContent.content_type == content_type,
)):
resp.append({
"created_at": datetime_to_rfc3339(x.created_at),
"digest": x.digest,
})
return resp, 200
except Exception as ex:
return make_response_error(ex, in_httpcode=500), 500
def credential_db_to_msg(credential):
if credential is None:
return None
return {
'type': credential['type'].value,
'value': ''.join(['*' for _ in credential['value']]),
'created_at': datetime_to_rfc3339(datetime.datetime.utcfromtimestamp(credential['created_at']))
}
def test_groups():
d1 = datetime.datetime.utcnow()
d2 = datetime.datetime.utcnow() - datetime.timedelta(days=1)
g = FeedGroupMetadata()
g.name = "group"
g.enabled = True
g.created_at = d2
g.updated_at = d1
g.last_sync = d1
g.record_count = 0
assert g.to_json() == {
"name": "group",
"enabled": True,
"created_at": datetime_to_rfc3339(d2),
"updated_at": datetime_to_rfc3339(d1),
"last_sync": datetime_to_rfc3339(d1),
"record_count": 0,
}
def user_db_to_msg(user):
if user is None:
return None
return {
"username":
user["username"],
"type":
user["type"].value,
"source":
user["source"],
"created_at":
datetime_to_rfc3339(
datetime.datetime.utcfromtimestamp(user["created_at"])),
"last_updated":
datetime_to_rfc3339(
datetime.datetime.utcfromtimestamp(user["last_updated"])),
}
def _map_type(obj):
if type(obj) == datetime.datetime:
return datetime_to_rfc3339(obj)
elif type(obj) in [list, set]:
return [SimpleJsonModel._map_type(i) for i in obj]
elif type(obj) == dict:
return {k: SimpleJsonModel._map_type(v) for k, v in obj.items()}
elif isinstance(obj, SimpleJsonModel):
return obj.to_json()
else:
return obj
def test_feeds():
f = FeedMetadata()
f.name = "feed1"
d1 = datetime.datetime.utcnow()
f.updated_at = d1
assert f.to_json() == {
"name": "feed1",
"updated_at": datetime_to_rfc3339(d1),
"groups": None,
"enabled": None,
"last_full_sync": None,
"created_at": None,
}
f.groups = []
g = FeedGroupMetadata()
g.name = "group1"
g.record_count = 10
g.enabled = True
f.groups.append(g)
assert f.to_json() == {
"name":
"feed1",
"updated_at":
datetime_to_rfc3339(d1),
"enabled":
None,
"last_full_sync":
None,
"created_at":
None,
"groups": [{
"name": "group1",
"enabled": True,
"record_count": 10,
"created_at": None,
"updated_at": None,
"last_sync": None,
}],
}
def content_upload(operation_id, content_type, request):
"""
Generic handler for multiple types of content uploads. Still operates at the API layer
:param operation_id:
:param content_type:
:param request:
:return:
"""
try:
with session_scope() as db_session:
record = (db_session.query(ImageImportOperation).filter_by(
account=ApiRequestContextProxy.namespace(),
uuid=operation_id).one_or_none())
if not record:
raise api_exceptions.ResourceNotFound(resource=operation_id,
detail={})
if not record.status.is_active():
raise api_exceptions.ConflictingRequest(
message="import operation status does not allow uploads",
detail={"status": record.status},
)
if not request.content_length:
raise api_exceptions.BadRequest(
message="Request must contain content-length header",
detail={})
elif request.content_length > MAX_UPLOAD_SIZE:
raise api_exceptions.BadRequest(
message=
"too large. Max size of 100MB supported for content",
detail={"content-length": request.content_length},
)
digest, created_at = save_import_content(db_session, operation_id,
request.data,
content_type)
resp = {
"digest": digest,
"created_at": datetime_to_rfc3339(created_at)
}
return resp, 200
except api_exceptions.AnchoreApiError as ex:
return (
make_response_error(ex, in_httpcode=ex.__response_code__),
ex.__response_code__,
)
except Exception as ex:
logger.exception("Unexpected error in api processing")
return make_response_error(ex, in_httpcode=500), 500
def account_db_to_msg(account):
if account is None:
return None
return {
"name":
account["name"],
"email":
account["email"],
"state":
account["state"].value
if type(account["state"]) != str else account["state"],
"type":
account["type"]
if type(account["type"]) == str else account["type"].value,
"created_at":
datetime_to_rfc3339(
datetime.datetime.utcfromtimestamp(account["created_at"])),
"last_updated":
datetime_to_rfc3339(
datetime.datetime.utcfromtimestamp(account["last_updated"])),
}
def credential_db_to_msg(credential):
if credential is None:
return None
return {
"type":
credential["type"].value,
"value":
"".join(["*" for _ in credential["value"]]),
"created_at":
datetime_to_rfc3339(
datetime.datetime.utcfromtimestamp(credential["created_at"])),
}
def archive_required(self, src_mgr: ObjectStorageManager, artifacts: list, img_archive: ImageArchive) -> list:
"""
:return: tuple of (manifest (dict), bucket (str), key (str))
"""
for artifact in artifacts:
data = src_mgr.get(self.account, artifact.source.bucket, artifact.source.key)
if not data:
raise Exception('Required artifact not found for migration: {}'.format(artifact.name))
artifact.metadata['completed_at'] = datetime_to_rfc3339(datetime.datetime.utcnow())
artifact.metadata['bytes_copied'] = len(data)
artifact.dest = TarballLocation(tarfile_path=artifact.name)
img_archive.add_artifact(artifact.name, source=artifact.source, data=data, metadata=artifact.metadata)
return img_archive.manifest.artifacts
def archive_policy_evaluations(self, src_obj_mgr: ObjectStorageManager, img_archive: ImageArchive, session) -> list:
"""
Returns a dict of the following structure:
{
tag1 -> {policyId1 -> [eval history], policyId2 -> [eval history], ... },
tag2 -> {policyId1...}, ... }
}
Policy evaluation histories are only moved, not generated, so online previously generated evaluations are migrated.
:param dest_archive_mgr:
:param dest_bucket:
:return: dict mapping tags (full pull tags) to policy eval histories for the migrating image
"""
logger.debug("Copying policy evaluation history to archive")
policy_evaluations = db_policyeval.get_all_bydigest(self.account, self.image_digest, session)
artifacts = []
for eval_rec in policy_evaluations:
artifact = Artifact(name='policy_evaluation-' + eval_rec['evalId'], metadata=None, source=None, dest=None)
artifact.source = ObjectStoreLocation(bucket='policy_evaluations', key=eval_rec['evalId'])
eval_content = src_obj_mgr.get(self.account, artifact.source.bucket, artifact.source.key)
meta = {
'record': eval_rec,
'record_type': 'policy_evaluation',
'completed_at': datetime_to_rfc3339(datetime.datetime.utcnow()),
'bytes_copied': len(eval_content)
}
artifacts.append(artifact)
img_archive.add_artifact(artifact.name, source=None, data=eval_content, metadata=meta)
return artifacts
def archive_vuln_history(self, img_archive: ImageArchive) -> list:
logger.debug("Migrating image vulnerability history to archive")
image_id = img_archive.manifest.metadata.get('image_id')
if not image_id:
logger.warn('No image id found in archive metadata for getting vuln history.')
return []
try:
pe_client = internal_client_for(PolicyEngineClient, userId=self.account)
vuln_report = pe_client.get_image_vulnerabilities(self.account, image_id)
data = ensure_bytes(json.dumps(vuln_report, sort_keys=True))
metadata = {'completed_at': datetime_to_rfc3339(datetime.datetime.utcnow()),
'bytes_copied': len(data)}
a = img_archive.add_artifact('vulnerabilities', source=None, data=data, metadata=metadata)
return [a]
except Exception as ex:
logger.exception("Error flushing policy engine state for image")
raise ex
def to_json(self):
j = super().to_json()
j["created_at"] = datetime_to_rfc3339(self.created_at)
j["last_update"] = datetime_to_rfc3339(self.last_updated)
return j
class TestImageAddWorkflow:
now_str = datetime_to_rfc3339(datetime.now())
@pytest.mark.parametrize(
"param",
[
pytest.param(
{
"input": {},
"expected_dockerfile": None,
"expected_dockerfile_mode": None,
"expected_error_prefix": None,
},
id="no-dockerfile-data",
),
pytest.param(
{
"input": {
"dockerfile": "not-encoded!"
},
"expected_dockerfile":
None,
"expected_dockerfile_mode":
None,
"expected_error_prefix":
"input dockerfile data must be base64 encoded - exception on decode",
},
id="not-encoded",
),
pytest.param(
{
"input": {
"dockerfile":
str(
base64.b64encode(
"dockerfile contents".encode("utf-8")),
"utf-8",
)
},
"expected_dockerfile":
str(
base64.b64encode(
"dockerfile contents".encode("utf-8")), "utf-8"),
"expected_dockerfile_mode":
"Actual",
"expected_error_prefix":
None,
},
id="success",
),
],
)
def test_get_dockerfile_info(self, param):
if param["expected_error_prefix"] is not None:
with pytest.raises(Exception) as err:
catalog_impl.get_dockerfile_info(param["input"])
assert str(err).startswith(param["expected_error_prefix"])
else:
(
actual_dockerfile,
actual_dockerfile_mode,
) = catalog_impl.get_dockerfile_info(param["input"])
assert actual_dockerfile == param["expected_dockerfile"]
assert actual_dockerfile_mode == param["expected_dockerfile_mode"]
@pytest.mark.parametrize(
"param",
[
pytest.param(
{
"params": {
"digest":
"sha256:714511030a442b48f37791a817ce6e124d9ea4b0158f93ce914520549bd6bc30",
},
"image_info": {
"repo": "anchore/kai",
"registry": "docker.io",
"tag": None,
},
"input_string": "test",
"expected_overrides": {},
"expected_input_string": "test",
},
id="missing-tag",
),
pytest.param(
{
"params": {
"tag": "anchore/kai:latest"
},
"image_info": {
"repo": "anchore/kai",
"registry": "docker.io",
"tag": None,
},
"input_string": "test",
"expected_overrides": {},
"expected_input_string": "test",
},
id="missing-digest",
),
pytest.param(
{
"params": {
"tag":
"anchore/kai:latest",
"digest":
"sha256:714511030a442b48f37791a817ce6e124d9ea4b0158f93ce914520549bd6bc30",
},
"image_info": {
"repo": "anchore/kai",
"registry": "docker.io",
"tag": "latest",
},
"input_string":
"test",
"expected_overrides": {
"fulltag": "anchore/kai:latest",
"tag": "latest",
},
"expected_input_string":
"docker.io/anchore/kai@sha256:714511030a442b48f37791a817ce6e124d9ea4b0158f93ce914520549bd6bc30",
},
id="success-no-created-at",
),
pytest.param(
{
"params": {
"tag": "anchore/kai:latest",
"digest":
"sha256:714511030a442b48f37791a817ce6e124d9ea4b0158f93ce914520549bd6bc30",
"created_at": now_str,
},
"image_info": {
"repo": "anchore/kai",
"registry": "docker.io",
"tag": "latest",
},
"input_string":
"test",
"expected_overrides": {
"fulltag": "anchore/kai:latest",
"tag": "latest",
"created_at_override": now_str,
},
"expected_input_string":
"docker.io/anchore/kai@sha256:714511030a442b48f37791a817ce6e124d9ea4b0158f93ce914520549bd6bc30",
},
id="success-with-created-at",
),
],
)
def test_resolve_image_info_overrides_and_input_string(self, param):
(
actual_overrides,
actual_input_string,
) = catalog_impl.resolve_image_info_overrides_and_input_string(
param["params"], param["image_info"], param["input_string"])
assert actual_overrides == param["expected_overrides"]
assert actual_input_string == param["expected_input_string"]
@pytest.mark.parametrize(
"param",
[
pytest.param(
{
"image_info": {},
"expected_manifest":
None,
"expected_error_prefix":
"could not fetch/parse manifest - exception: no manifest from get_image_info",
},
id="no-manifest",
),
pytest.param(
{
"image_info": {
"manifest": "hey there"
},
"expected_manifest":
None,
"expected_error_prefix":
"could not fetch/parse manifest - exception: ",
},
id="non-json-manifest",
),
pytest.param(
{
"image_info": {
"manifest": {
"layers": [{
"this": "is a layer"
}]
}
},
"expected_manifest":
json.dumps({"layers": [{
"this": "is a layer"
}]}),
"expected_error_prefix":
"could not fetch/parse manifest - exception: no manifest from get_image_info",
},
id="valid-manifest",
),
],
)
def test_get_manifest(self, param):
if param["expected_error_prefix"] is not None:
with pytest.raises(Exception) as err:
catalog_impl.get_manifest(param["image_info"])
assert str(err).startswith(param["expected_error_prefix"])
else:
actual_manifest = catalog_impl.get_manifest(param["image_info"])
assert actual_manifest == param["expected_manifest"]
@pytest.mark.parametrize(
"param",
[
pytest.param(
{
"image_key":
catalog_impl.ImageKey(
tag="docker.io/anchore/test_images:centos8",
digest=""),
"expected":
"docker.io/anchore/test_images:centos8",
},
id="no-digest",
),
pytest.param(
{
"image_key":
catalog_impl.ImageKey(
tag="docker.io/anchore/test_images:centos8",
digest="unknown"),
"expected":
"docker.io/anchore/test_images:centos8",
},
id="digest-unknown",
),
pytest.param(
{
"image_key": catalog_impl.ImageKey(tag="", digest=""),
"expected": "",
},
id="no-digest-no-tag",
),
pytest.param(
{
"image_key":
catalog_impl.ImageKey(
tag="docker.io/anchore/test_images:centos8",
digest="sha256:1234abcd",
),
"expected":
"docker.io/anchore/test_images@sha256:1234abcd",
},
id="valid-digest-valid-tag",
),
],
)
def test_get_input_string(self, param):
actual = catalog_impl.get_input_string(param["image_key"])
assert actual == param["expected"]
