def __init__(self, kind, missing=False, user=None, token=None, oauthtoken=None,
robot=None, appspecifictoken=None, signed_data=None, error_message=None):
self.kind = kind
self.missing = missing
self.error_message = error_message
self.context = ValidatedAuthContext(user=user, token=token, oauthtoken=oauthtoken, robot=robot,
appspecifictoken=appspecifictoken, signed_data=signed_data)
def _token_data(
access=[],
context=None,
audience=TEST_AUDIENCE,
user=TEST_USER,
iat=None,
exp=None,
nbf=None,
iss=None,
subject=None,
):
if subject is None:
_, subject = build_context_and_subject(ValidatedAuthContext(user=user))
return {
"iss":
iss or instance_keys.service_name,
"aud":
audience,
"nbf":
nbf if nbf is not None else int(time.time()),
"iat":
iat if iat is not None else int(time.time()),
"exp":
exp if exp is not None else int(time.time() +
TOKEN_VALIDITY_LIFETIME_S),
"sub":
subject,
"access":
access,
"context":
context,
}
def _token_data(access=[],
context=None,
audience=TEST_AUDIENCE,
user=TEST_USER,
iat=None,
exp=None,
nbf=None,
iss=None,
subject=None):
if subject is None:
_, subject = build_context_and_subject(ValidatedAuthContext(user=user))
return {
'iss':
iss or instance_keys.service_name,
'aud':
audience,
'nbf':
nbf if nbf is not None else int(time.time()),
'iat':
iat if iat is not None else int(time.time()),
'exp':
exp if exp is not None else int(time.time() +
TOKEN_VALIDITY_LIFETIME_S),
'sub':
subject,
'access':
access,
'context':
context,
}
def test_blob_caching(method, endpoint, client, app):
digest = "sha256:" + hashlib.sha256(b"a").hexdigest()
location = ImageStorageLocation.get(name="local_us")
model.blob.store_blob_record_and_temp_link("devtable", "simple", digest,
location, 1, 10000000)
params = {
"repository": "devtable/simple",
"digest": digest,
}
user = model.user.get_user("devtable")
access = [{
"type": "repository",
"name": "devtable/simple",
"actions": ["pull"],
}]
context, subject = build_context_and_subject(
ValidatedAuthContext(user=user))
token = generate_bearer_token(realapp.config["SERVER_HOSTNAME"], subject,
context, access, 600, instance_keys)
headers = {
"Authorization": "Bearer %s" % token.decode("ascii"),
}
# Run without caching to make sure the request works. This also preloads some of
# our global model caches.
conduct_call(client,
"v2." + endpoint,
url_for,
method,
params,
expected_code=200,
headers=headers)
with patch("endpoints.v2.blob.model_cache",
InMemoryDataModelCache(TEST_CACHE_CONFIG)):
# First request should make a DB query to retrieve the blob.
conduct_call(client,
"v2." + endpoint,
url_for,
method,
params,
expected_code=200,
headers=headers)
# Subsequent requests should use the cached blob.
with assert_query_count(0):
conduct_call(
client,
"v2." + endpoint,
url_for,
method,
params,
expected_code=200,
headers=headers,
)
def test_e2e_query_count_manifest_norewrite(client, app):
repo_ref = registry_model.lookup_repository("devtable", "simple")
tag = registry_model.get_repo_tag(repo_ref, "latest")
manifest = registry_model.get_manifest_for_tag(tag)
params = {
"repository": "devtable/simple",
"manifest_ref": manifest.digest,
}
user = model.user.get_user("devtable")
access = [{
"type": "repository",
"name": "devtable/simple",
"actions": ["pull", "push"],
}]
context, subject = build_context_and_subject(
ValidatedAuthContext(user=user))
token = generate_bearer_token(realapp.config["SERVER_HOSTNAME"], subject,
context, access, 600, instance_keys)
headers = {
"Authorization": "Bearer %s" % token.decode("ascii"),
}
# Conduct a call to prime the instance key and other caches.
conduct_call(
client,
"v2.write_manifest_by_digest",
url_for,
"PUT",
params,
expected_code=201,
headers=headers,
raw_body=manifest.internal_manifest_bytes.as_encoded_str(),
)
timecode = time.time()
def get_time():
return timecode + 10
with patch("time.time", get_time):
# Necessary in order to have the tag updates not occur in the same second, which is the
# granularity supported currently.
with count_queries() as counter:
conduct_call(
client,
"v2.write_manifest_by_digest",
url_for,
"PUT",
params,
expected_code=201,
headers=headers,
raw_body=manifest.internal_manifest_bytes.as_encoded_str(),
)
assert counter.count <= 27
def test_signed_auth_context(kind, entity_reference, loader, v1_dict_format,
initialized_db):
if kind == ContextEntityKind.anonymous:
validated = ValidatedAuthContext()
assert validated.is_anonymous
else:
ref = loader(entity_reference)
validated = ValidatedAuthContext(**{kind.value: ref})
assert not validated.is_anonymous
assert validated.entity_kind == kind
assert validated.unique_key
signed = SignedAuthContext.build_from_signed_dict(
validated.to_signed_dict(), v1_dict_format=v1_dict_format)
if not v1_dict_format:
# Under legacy V1 format, we don't track the app specific token, merely its associated user.
assert signed.entity_kind == kind
assert signed.description == validated.description
assert signed.credential_username == validated.credential_username
assert (signed.analytics_id_and_public_metadata() ==
validated.analytics_id_and_public_metadata())
assert signed.unique_key == validated.unique_key
assert signed.is_anonymous == validated.is_anonymous
assert signed.authed_user == validated.authed_user
assert signed.has_nonrobot_user == validated.has_nonrobot_user
assert signed.to_signed_dict() == validated.to_signed_dict()
class ValidateResult(object):
""" A result of validating auth in one form or another. """
def __init__(self, kind, missing=False, user=None, token=None, oauthtoken=None,
robot=None, appspecifictoken=None, signed_data=None, error_message=None):
self.kind = kind
self.missing = missing
self.error_message = error_message
self.context = ValidatedAuthContext(user=user, token=token, oauthtoken=oauthtoken, robot=robot,
appspecifictoken=appspecifictoken, signed_data=signed_data)
def tuple(self):
return (self.kind, self.missing, self.error_message, self.context.tuple())
def __eq__(self, other):
return self.tuple() == other.tuple()
def apply_to_context(self):
""" Applies this auth result to the auth context and Flask-Principal. """
self.context.apply_to_request_context()
def with_kind(self, kind):
""" Returns a copy of this result, but with the kind replaced. """
result = ValidateResult(kind, missing=self.missing, error_message=self.error_message)
result.context = self.context
return result
def __repr__(self):
return 'ValidateResult: %s (missing: %s, error: %s)' % (self.kind, self.missing,
self.error_message)
@property
def authed_user(self):
""" Returns the authenticated user, whether directly, or via an OAuth token. """
return self.context.authed_user
@property
def has_nonrobot_user(self):
""" Returns whether a user (not a robot) was authenticated successfully. """
return self.context.has_nonrobot_user
@property
def auth_valid(self):
""" Returns whether authentication successfully occurred. """
return self.context.entity_kind != ContextEntityKind.anonymous
def test_blob_caching(method, endpoint, client, app):
digest = 'sha256:' + hashlib.sha256("a").hexdigest()
location = ImageStorageLocation.get(name='local_us')
model.blob.store_blob_record_and_temp_link('devtable', 'simple', digest,
location, 1, 10000000)
params = {
'repository': 'devtable/simple',
'digest': digest,
}
user = model.user.get_user('devtable')
access = [{
'type': 'repository',
'name': 'devtable/simple',
'actions': ['pull'],
}]
context, subject = build_context_and_subject(
ValidatedAuthContext(user=user))
token = generate_bearer_token(realapp.config['SERVER_HOSTNAME'], subject,
context, access, 600, instance_keys)
headers = {
'Authorization': 'Bearer %s' % token,
}
# Run without caching to make sure the request works. This also preloads some of
# our global model caches.
conduct_call(client,
'v2.' + endpoint,
url_for,
method,
params,
expected_code=200,
headers=headers)
with patch('endpoints.v2.blob.model_cache', InMemoryDataModelCache()):
# First request should make a DB query to retrieve the blob.
conduct_call(client,
'v2.' + endpoint,
url_for,
method,
params,
expected_code=200,
headers=headers)
# Subsequent requests should use the cached blob.
with assert_query_count(0):
conduct_call(client,
'v2.' + endpoint,
url_for,
method,
params,
expected_code=200,
headers=headers)
def test_e2e_query_count_manifest_norewrite(client, app):
tag_manifest = model.tag.load_tag_manifest('devtable', 'simple', 'latest')
params = {
'repository': 'devtable/simple',
'manifest_ref': tag_manifest.digest,
}
user = model.user.get_user('devtable')
access = [{
'type': 'repository',
'name': 'devtable/simple',
'actions': ['pull', 'push'],
}]
context, subject = build_context_and_subject(
ValidatedAuthContext(user=user))
token = generate_bearer_token(realapp.config['SERVER_HOSTNAME'], subject,
context, access, 600, instance_keys)
headers = {
'Authorization': 'Bearer %s' % token,
}
# Conduct a call to prime the instance key and other caches.
conduct_call(client,
'v2.write_manifest_by_digest',
url_for,
'PUT',
params,
expected_code=202,
headers=headers,
raw_body=tag_manifest.json_data)
timecode = time.time()
def get_time():
return timecode + 10
with patch('time.time', get_time):
# Necessary in order to have the tag updates not occur in the same second, which is the
# granularity supported currently.
with count_queries() as counter:
conduct_call(client,
'v2.write_manifest_by_digest',
url_for,
'PUT',
params,
expected_code=202,
headers=headers,
raw_body=tag_manifest.json_data)
assert counter.count <= 27
def test_blob_upload_offset(client, app):
user = model.user.get_user("devtable")
access = [{
"type": "repository",
"name": "devtable/simple",
"actions": ["pull", "push"],
}]
context, subject = build_context_and_subject(
ValidatedAuthContext(user=user))
token = generate_bearer_token(realapp.config["SERVER_HOSTNAME"], subject,
context, access, 600, instance_keys)
headers = {
"Authorization": "Bearer %s" % token.decode("ascii"),
}
# Create a blob upload request.
params = {
"repository": "devtable/simple",
}
response = conduct_call(client,
"v2.start_blob_upload",
url_for,
"POST",
params,
expected_code=202,
headers=headers)
upload_uuid = response.headers["Docker-Upload-UUID"]
# Attempt to start an upload past index zero.
params = {
"repository": "devtable/simple",
"upload_uuid": upload_uuid,
}
headers = {
"Authorization": "Bearer %s" % token.decode("ascii"),
"Content-Range": "13-50",
}
conduct_call(
client,
"v2.upload_chunk",
url_for,
"PATCH",
params,
expected_code=416,
headers=headers,
body="something",
)
def setup(self, client, app):
self.client = client
self.user = model.user.get_user("devtable")
context, subject = build_context_and_subject(
ValidatedAuthContext(user=self.user))
access = [{
"type": "repository",
"name": self.repository,
"actions": ["pull"],
}]
token = generate_bearer_token(realapp.config["SERVER_HOSTNAME"],
subject, context, access, 600,
instance_keys)
self.headers = {
"Authorization": f"Bearer {token.decode('ascii')}",
}
try:
model.organization.get(self.org)
except Exception:
org = model.organization.create_organization(
self.org, "*****@*****.**", self.user)
org.save()
if self.config is None:
self.config = model.proxy_cache.create_proxy_cache_config(
org_name=self.org,
upstream_registry=self.registry,
expiration_s=3600,
)
if self.repo_ref is None:
r = model.repository.create_repository(self.org, self.image_name,
self.user)
assert r is not None
self.repo_ref = registry_model.lookup_repository(
self.org, self.image_name)
assert self.repo_ref is not None
if self.blob_digest is None:
proxy_model = ProxyModel(self.org, self.image_name, self.user)
manifest = proxy_model.lookup_manifest_by_digest(
self.repo_ref, self.manifest_digest)
self.blob_digest = manifest.get_parsed_manifest().blob_digests[0]
def setup(self, client, app):
self.client = client
self.user = model.user.get_user("devtable")
context, subject = build_context_and_subject(
ValidatedAuthContext(user=self.user))
self.ctx = context
self.sub = subject
if self.org is None:
self.org = model.organization.create_organization(
self.orgname, "{self.orgname}@devtable.com", self.user)
self.org.save()
self.config = model.proxy_cache.create_proxy_cache_config(
org_name=self.orgname,
upstream_registry=self.registry,
expiration_s=3600,
)
def setup(self, client, app):
self.client = client
self.user = model.user.get_user("devtable")
context, sub = build_context_and_subject(
ValidatedAuthContext(user=self.user))
self.ctx = context
self.sub = sub
try:
model.organization.get(self.org)
except Exception:
org = model.organization.create_organization(
self.org, "*****@*****.**", self.user)
org.save()
try:
model.organization.get(self.org2)
except Exception:
org = model.organization.create_organization(
self.org2,
"*****@*****.**",
self.user,
)
org.save()
try:
model.proxy_cache.get_proxy_cache_config_for_org(self.org)
except Exception:
model.proxy_cache.create_proxy_cache_config(
org_name=self.org,
upstream_registry=self.registry,
expiration_s=3600,
)
try:
model.proxy_cache.get_proxy_cache_config_for_org(self.org2)
except Exception:
model.proxy_cache.create_proxy_cache_config(
org_name=self.org2,
upstream_registry=self.registry + "/library",
expiration_s=3600,
)
def test_blob_mounting(
mount_digest, source_repo, username, include_from_param, expected_code, client, app
):
location = ImageStorageLocation.get(name="local_us")
# Store and link some blobs.
digest = "sha256:" + hashlib.sha256(b"a").hexdigest()
model.blob.store_blob_record_and_temp_link("devtable", "simple", digest, location, 1, 10000000)
digest = "sha256:" + hashlib.sha256(b"b").hexdigest()
model.blob.store_blob_record_and_temp_link("devtable", "complex", digest, location, 1, 10000000)
digest = "sha256:" + hashlib.sha256(b"c").hexdigest()
model.blob.store_blob_record_and_temp_link(
"public", "publicrepo", digest, location, 1, 10000000
)
params = {
"repository": "devtable/building",
"mount": mount_digest,
}
if include_from_param:
params["from"] = source_repo
user = model.user.get_user(username)
access = [
{
"type": "repository",
"name": "devtable/building",
"actions": ["pull", "push"],
}
]
if source_repo.find(username) == 0:
access.append(
{
"type": "repository",
"name": source_repo,
"actions": ["pull"],
}
)
context, subject = build_context_and_subject(ValidatedAuthContext(user=user))
token = generate_bearer_token(
realapp.config["SERVER_HOSTNAME"], subject, context, access, 600, instance_keys
)
headers = {
"Authorization": "Bearer %s" % token.decode("ascii"),
}
conduct_call(
client,
"v2.start_blob_upload",
url_for,
"POST",
params,
expected_code=expected_code,
headers=headers,
)
repository = model.repository.get_repository("devtable", "building")
if expected_code == 201:
# Ensure the blob now exists under the repo.
assert model.oci.blob.get_repository_blob_by_digest(repository, mount_digest)
else:
assert model.oci.blob.get_repository_blob_by_digest(repository, mount_digest) is None
def test_blob_mounting(mount_digest, source_repo, username, expect_success,
client, app):
location = ImageStorageLocation.get(name='local_us')
# Store and link some blobs.
digest = 'sha256:' + hashlib.sha256("a").hexdigest()
model.blob.store_blob_record_and_temp_link('devtable', 'simple', digest,
location, 1, 10000000)
digest = 'sha256:' + hashlib.sha256("b").hexdigest()
model.blob.store_blob_record_and_temp_link('devtable', 'complex', digest,
location, 1, 10000000)
digest = 'sha256:' + hashlib.sha256("c").hexdigest()
model.blob.store_blob_record_and_temp_link('public', 'publicrepo', digest,
location, 1, 10000000)
params = {
'repository': 'devtable/building',
'mount': mount_digest,
'from': source_repo,
}
user = model.user.get_user(username)
access = [{
'type': 'repository',
'name': 'devtable/building',
'actions': ['pull', 'push'],
}]
if source_repo.find(username) == 0:
access.append({
'type': 'repository',
'name': source_repo,
'actions': ['pull'],
})
context, subject = build_context_and_subject(
ValidatedAuthContext(user=user))
token = generate_bearer_token(realapp.config['SERVER_HOSTNAME'], subject,
context, access, 600, instance_keys)
headers = {
'Authorization': 'Bearer %s' % token,
}
expected_code = 201 if expect_success else 202
conduct_call(client,
'v2.start_blob_upload',
url_for,
'POST',
params,
expected_code=expected_code,
headers=headers)
if expect_success:
# Ensure the blob now exists under the repo.
model.blob.get_repo_blob_by_digest('devtable', 'building',
mount_digest)
else:
with pytest.raises(model.blob.BlobDoesNotExist):
model.blob.get_repo_blob_by_digest('devtable', 'building',
mount_digest)
def setup(self, client, app, proxy_manifest_response):
self.client = client
self.user = model.user.get_user("devtable")
context, subject = build_context_and_subject(
ValidatedAuthContext(user=self.user))
access = [{
"type": "repository",
"name": self.repository,
"actions": ["pull"],
}]
token = generate_bearer_token(realapp.config["SERVER_HOSTNAME"],
subject, context, access, 600,
instance_keys)
self.headers = {
"Authorization": f"Bearer {token.decode('ascii')}",
}
if self.org is None:
self.org = model.organization.create_organization(
self.orgname, "{self.orgname}@devtable.com", self.user)
self.org.save()
self.config = model.proxy_cache.create_proxy_cache_config(
org_name=self.orgname,
upstream_registry=self.registry,
expiration_s=3600,
)
if self.repo_ref is None:
r = model.repository.create_repository(self.orgname,
self.image_name, self.user)
assert r is not None
self.repo_ref = registry_model.lookup_repository(
self.orgname, self.image_name)
assert self.repo_ref is not None
def get_blob(layer):
content = Bytes.for_string_or_unicode(layer).as_encoded_str()
digest = str(sha256_digest(content))
blob = model.blob.store_blob_record_and_temp_link(
self.orgname,
self.image_name,
digest,
ImageStorageLocation.get(name="local_us"),
len(content),
120,
)
storage.put_content(["local_us"], get_layer_path(blob), content)
return blob, digest
if self.manifest is None:
layer1 = json.dumps({
"config": {},
"rootfs": {
"type": "layers",
"diff_ids": []
},
"history": [{}],
})
_, config_digest = get_blob(layer1)
layer2 = "test"
_, blob_digest = get_blob(layer2)
builder = DockerSchema2ManifestBuilder()
builder.set_config_digest(config_digest,
len(layer1.encode("utf-8")))
builder.add_layer(blob_digest, len(layer2.encode("utf-8")))
manifest = builder.build()
created_manifest = model.oci.manifest.get_or_create_manifest(
self.repo_ref.id, manifest, storage)
self.manifest = created_manifest.manifest
assert self.digest == blob_digest
assert self.manifest is not None
if self.blob is None:
self.blob = ImageStorage.filter(
ImageStorage.content_checksum == self.digest).get()
