def delete_channel(namespace, package_name, channel_name):
_check_channel_name(channel_name)
reponame = repo_name(namespace, package_name)
result = cnr_registry.delete_channel(reponame, channel_name, channel_class=Channel)
logs_model.log_action('delete_tag', namespace, repository_name=package_name,
metadata={'channel': channel_name})
return jsonify(result)
def emit_log(mirror,
log_kind,
verb,
message,
tag=None,
tags=None,
stdout=None,
stderr=None):
logs_model.log_action(
log_kind,
namespace_name=mirror.repository.namespace_user.username,
repository_name=mirror.repository.name,
metadata={
"verb": verb,
"namespace": mirror.repository.namespace_user.username,
"repo": mirror.repository.name,
"message": message,
"tag": tag,
"tags": tags,
"stdout": stdout,
"stderr": stderr,
},
)
if log_kind in (
"repo_mirror_sync_started",
"repo_mirror_sync_failed",
"repo_mirror_sync_success",
):
spawn_notification(wrap_repository(mirror.repository), log_kind,
{"message": message})
def generate_key(service, name, expiration_date=None, notes=None):
metadata = {
'created_by': 'CLI tool',
}
# Generate a key with a private key that we *never save*.
(private_key, key) = model.service_keys.generate_service_key(service, expiration_date,
metadata=metadata,
name=name)
# Auto-approve the service key.
model.service_keys.approve_service_key(key.kid, ServiceKeyApprovalType.AUTOMATIC, notes=notes or '')
# Log the creation and auto-approval of the service key.
key_log_metadata = {
'kid': key.kid,
'preshared': True,
'service': service,
'name': name,
'expiration_date': expiration_date,
'auto_approved': True,
}
logs_model.log_action('service_key_create', metadata=key_log_metadata)
logs_model.log_action('service_key_approve', metadata=key_log_metadata)
return private_key, key.kid
def log_action(kind,
user_or_orgname,
metadata=None,
repo=None,
repo_name=None):
if not metadata:
metadata = {}
oauth_token = get_validated_oauth_token()
if oauth_token:
metadata["oauth_token_id"] = oauth_token.id
metadata[
"oauth_token_application_id"] = oauth_token.application.client_id
metadata["oauth_token_application"] = oauth_token.application.name
performer = get_authenticated_user()
if repo_name is not None:
repo = data_model.repository.get_repository(user_or_orgname, repo_name)
logs_model.log_action(
kind,
user_or_orgname,
repository=repo,
performer=performer,
ip=get_request_ip(),
metadata=metadata,
)
def generate_key(service, name, expiration_date=None, notes=None):
metadata = {
"created_by": "CLI tool",
}
# Generate a key with a private key that we *never save*.
(private_key,
key) = model.service_keys.generate_service_key(service,
expiration_date,
metadata=metadata,
name=name)
# Auto-approve the service key.
model.service_keys.approve_service_key(key.kid,
ServiceKeyApprovalType.AUTOMATIC,
notes=notes or "")
# Log the creation and auto-approval of the service key.
key_log_metadata = {
"kid": key.kid,
"preshared": True,
"service": service,
"name": name,
"expiration_date": expiration_date,
"auto_approved": True,
}
logs_model.log_action("service_key_create", metadata=key_log_metadata)
logs_model.log_action("service_key_approve", metadata=key_log_metadata)
return private_key, key.kid
def push(namespace, package_name):
reponame = repo_name(namespace, package_name)
if not REPOSITORY_NAME_REGEX.match(package_name):
logger.debug("Found invalid repository name CNR push: %s", reponame)
raise InvalidUsage("invalid repository name: %s" % reponame)
values = request.get_json(force=True, silent=True) or {}
private = values.get("visibility", "private")
owner = get_authenticated_user()
if not Package.exists(reponame):
if not CreateRepositoryPermission(namespace).can():
raise Forbidden(
"Unauthorized access for: %s" % reponame,
{
"package": reponame,
"scopes": ["create"]
},
)
Package.create_repository(reponame, private, owner)
logs_model.log_action("create_repo",
namespace,
repository_name=package_name)
if not ModifyRepositoryPermission(namespace, package_name).can():
raise Forbidden("Unauthorized access for: %s" % reponame, {
"package": reponame,
"scopes": ["push"]
})
if not "release" in values:
raise InvalidUsage("Missing release")
if not "media_type" in values:
raise InvalidUsage("Missing media_type")
if not "blob" in values:
raise InvalidUsage("Missing blob")
release_version = str(values["release"])
media_type = values["media_type"]
force = request.args.get("force", "false") == "true"
blob = Blob(reponame, values["blob"])
app_release = cnr_registry.push(
reponame,
release_version,
media_type,
blob,
force,
package_class=Package,
user=owner,
visibility=private,
)
logs_model.log_action("push_repo",
namespace,
repository_name=package_name,
metadata={"release": release_version})
return jsonify(app_release)
def pull(namespace, package_name, release, media_type):
logger.debug('Pull of release %s of app repository %s/%s', release, namespace, package_name)
reponame = repo_name(namespace, package_name)
data = cnr_registry.pull(reponame, release, media_type, Package, blob_class=Blob)
logs_model.log_action('pull_repo', namespace, repository_name=package_name,
metadata={'release': release, 'mediatype': media_type})
json_format = request.args.get('format', None) == 'json'
return _pull(data, json_format)
def add_channel_release(namespace, package_name, channel_name, release):
_check_channel_name(channel_name, release)
reponame = repo_name(namespace, package_name)
result = cnr_registry.add_channel_release(reponame, channel_name, release, channel_class=Channel,
package_class=Package)
logs_model.log_action('create_tag', namespace, repository_name=package_name,
metadata={'channel': channel_name, 'release': release})
return jsonify(result)
def delete_package(namespace, package_name, release, media_type):
reponame = repo_name(namespace, package_name)
result = cnr_registry.delete_package(reponame, release, media_type, package_class=Package)
logs_model.log_action(
"delete_tag",
namespace,
repository_name=package_name,
metadata={"release": release, "mediatype": media_type},
)
return jsonify(result)
def log_action(self, kind, namespace_name, repository_name, performer, ip, metadata):
repository = model.repository.get_repository(namespace_name, repository_name)
logs_model.log_action(
kind,
namespace_name,
performer=performer,
ip=ip,
metadata=metadata,
repository=repository,
)
def pull(namespace, package_name, release, media_type):
logger.debug("Pull of release %s of app repository %s/%s", release, namespace, package_name)
reponame = repo_name(namespace, package_name)
data = cnr_registry.pull(reponame, release, media_type, Package, blob_class=Blob)
logs_model.log_action(
"pull_repo",
namespace,
repository_name=package_name,
metadata={"release": release, "mediatype": media_type},
)
json_format = request.args.get("format", None) == "json"
return _pull(data, json_format)
def delete_channel_release(namespace, package_name, channel_name, release):
_check_channel_name(channel_name, release)
reponame = repo_name(namespace, package_name)
result = cnr_registry.delete_channel_release(
reponame, channel_name, release, channel_class=Channel, package_class=Package
)
logs_model.log_action(
"delete_tag",
namespace,
repository_name=package_name,
metadata={"channel": channel_name, "release": release},
)
return jsonify(result)
def __generate_service_key(
kid,
name,
user,
timestamp,
approval_type,
expiration=None,
metadata=None,
service="sample_service",
rotation_duration=None,
):
_, key = model.service_keys.generate_service_key(
service,
expiration,
kid=kid,
name=name,
metadata=metadata,
rotation_duration=rotation_duration,
)
if approval_type is not None:
model.service_keys.approve_service_key(key.kid,
approval_type,
notes="The **test** approval")
key_metadata = {
"kid": kid,
"preshared": True,
"service": service,
"name": name,
"expiration_date": expiration,
"auto_approved": True,
}
logs_model.log_action("service_key_approve",
None,
performer=user,
timestamp=timestamp,
metadata=key_metadata)
logs_model.log_action("service_key_create",
None,
performer=user,
timestamp=timestamp,
metadata=key_metadata)
def push(namespace, package_name):
reponame = repo_name(namespace, package_name)
if not REPOSITORY_NAME_REGEX.match(package_name):
logger.debug('Found invalid repository name CNR push: %s', reponame)
raise InvalidUsage('invalid repository name: %s' % reponame)
values = request.get_json(force=True, silent=True) or {}
private = values.get('visibility', 'private')
owner = get_authenticated_user()
if not Package.exists(reponame):
if not CreateRepositoryPermission(namespace).can():
raise Forbidden("Unauthorized access for: %s" % reponame,
{"package": reponame,
"scopes": ['create']})
Package.create_repository(reponame, private, owner)
logs_model.log_action('create_repo', namespace, repository_name=package_name)
if not ModifyRepositoryPermission(namespace, package_name).can():
raise Forbidden("Unauthorized access for: %s" % reponame,
{"package": reponame,
"scopes": ['push']})
if not 'release' in values:
raise InvalidUsage('Missing release')
if not 'media_type' in values:
raise InvalidUsage('Missing media_type')
if not 'blob' in values:
raise InvalidUsage('Missing blob')
release_version = str(values['release'])
media_type = values['media_type']
force = request.args.get('force', 'false') == 'true'
blob = Blob(reponame, values['blob'])
app_release = cnr_registry.push(reponame, release_version, media_type, blob, force,
package_class=Package, user=owner, visibility=private)
logs_model.log_action('push_repo', namespace, repository_name=package_name,
metadata={'release': release_version})
return jsonify(app_release)
def __generate_service_key(kid,
name,
user,
timestamp,
approval_type,
expiration=None,
metadata=None,
service='sample_service',
rotation_duration=None):
_, key = model.service_keys.generate_service_key(
service,
expiration,
kid=kid,
name=name,
metadata=metadata,
rotation_duration=rotation_duration)
if approval_type is not None:
model.service_keys.approve_service_key(key.kid,
approval_type,
notes='The **test** approval')
key_metadata = {
'kid': kid,
'preshared': True,
'service': service,
'name': name,
'expiration_date': expiration,
'auto_approved': True
}
logs_model.log_action('service_key_approve',
None,
performer=user,
timestamp=timestamp,
metadata=key_metadata)
logs_model.log_action('service_key_create',
None,
performer=user,
timestamp=timestamp,
metadata=key_metadata)
def delete_service_key(service, kid):
jwt_header = request.headers.get(JWT_HEADER_NAME, "")
match = jwtutil.TOKEN_REGEX.match(jwt_header)
if match is None:
abort(400)
encoded_jwt = match.group(1)
signer_kid = _signer_kid(encoded_jwt)
signer_key = _lookup_service_key(service, signer_kid, approved_only=False)
self_signed = kid == signer_kid
approved_key_for_service = signer_key.approval is not None
if self_signed or approved_key_for_service:
_validate_jwt(encoded_jwt, signer_key.jwk, service)
try:
model.delete_service_key(kid)
except ServiceKeyDoesNotExist:
abort(404)
logs_model.log_action(
"service_key_delete",
ip=get_request_ip(),
metadata={
"kid": kid,
"signer_kid": signer_key.kid,
"service": service,
"name": signer_key.name,
"user_agent": request.headers.get("User-Agent"),
"ip": get_request_ip(),
},
)
return make_response("", 204)
abort(403)
def delete_service_key(service, kid):
jwt_header = request.headers.get(JWT_HEADER_NAME, '')
match = jwtutil.TOKEN_REGEX.match(jwt_header)
if match is None:
abort(400)
encoded_jwt = match.group(1)
signer_kid = _signer_kid(encoded_jwt)
signer_key = _lookup_service_key(service, signer_kid, approved_only=False)
self_signed = kid == signer_kid
approved_key_for_service = signer_key.approval is not None
if self_signed or approved_key_for_service:
_validate_jwt(encoded_jwt, signer_key.jwk, service)
try:
model.delete_service_key(kid)
except ServiceKeyDoesNotExist:
abort(404)
logs_model.log_action('service_key_delete',
ip=get_request_ip(),
metadata={
'kid': kid,
'signer_kid': signer_key.kid,
'service': service,
'name': signer_key.name,
'user_agent':
request.headers.get('User-Agent'),
'ip': get_request_ip(),
})
return make_response('', 204)
abort(403)
def put_service_key(service, kid):
metadata = {"ip": get_request_ip()}
rotation_duration = request.args.get("rotation", None)
expiration_date = request.args.get("expiration", None)
if expiration_date is not None:
try:
expiration_date = datetime.utcfromtimestamp(float(expiration_date))
except ValueError:
logger.exception("Error parsing expiration date on key")
abort(400)
try:
jwk = request.get_json()
except ValueError:
logger.exception("Error parsing JWK")
abort(400)
jwt_header = request.headers.get(JWT_HEADER_NAME, "")
match = jwtutil.TOKEN_REGEX.match(jwt_header)
if match is None:
logger.error("Could not find matching bearer token")
abort(400)
encoded_jwt = match.group(1)
_validate_jwk(jwk)
signer_kid = _signer_kid(encoded_jwt, allow_none=True)
if kid == signer_kid or signer_kid is None:
# The key is self-signed. Create a new instance and await approval.
_validate_jwt(encoded_jwt, jwk, service)
model.create_service_key("",
kid,
service,
jwk,
metadata,
expiration_date,
rotation_duration=rotation_duration)
logs_model.log_action(
"service_key_create",
ip=get_request_ip(),
metadata={
"kid": kid,
"preshared": False,
"service": service,
"name": "",
"expiration_date": expiration_date,
"user_agent": request.headers.get("User-Agent"),
"ip": get_request_ip(),
},
)
return make_response("", 202)
# Key is going to be rotated.
metadata.update({"created_by": "Key Rotation"})
signer_key = _lookup_service_key(service, signer_kid)
signer_jwk = signer_key.jwk
_validate_jwt(encoded_jwt, signer_jwk, service)
try:
model.replace_service_key(signer_key.kid, kid, jwk, metadata,
expiration_date)
except ServiceKeyDoesNotExist:
abort(404)
logs_model.log_action(
"service_key_rotate",
ip=get_request_ip(),
metadata={
"kid": kid,
"signer_kid": signer_key.kid,
"service": service,
"name": signer_key.name,
"expiration_date": expiration_date,
"user_agent": request.headers.get("User-Agent"),
"ip": get_request_ip(),
},
)
return make_response("", 200)
def populate_database(minimal=False):
logger.debug("Populating the DB with test data.")
# Check if the data already exists. If so, we skip. This can happen between calls from the
# "old style" tests and the new py.test's.
try:
User.get(username="******")
logger.debug("DB already populated")
return
except User.DoesNotExist:
pass
# Note: databases set up with "real" schema (via Alembic) will not have these types
# type, so we it here it necessary.
try:
ImageStorageLocation.get(name="local_eu")
ImageStorageLocation.get(name="local_us")
except ImageStorageLocation.DoesNotExist:
ImageStorageLocation.create(name="local_eu")
ImageStorageLocation.create(name="local_us")
try:
NotificationKind.get(name="test_notification")
except NotificationKind.DoesNotExist:
NotificationKind.create(name="test_notification")
new_user_1 = model.user.create_user("devtable", "password",
"*****@*****.**")
new_user_1.verified = True
new_user_1.stripe_id = TEST_STRIPE_ID
new_user_1.save()
if minimal:
logger.debug(
"Skipping most db population because user requested mininal db")
return
UserRegion.create(user=new_user_1,
location=ImageStorageLocation.get(name="local_us"))
model.release.set_region_release("quay", "us", "v0.1.2")
model.user.create_confirm_email_code(new_user_1,
new_email="*****@*****.**")
disabled_user = model.user.create_user("disabled", "password",
"*****@*****.**")
disabled_user.verified = True
disabled_user.enabled = False
disabled_user.save()
dtrobot = model.user.create_robot("dtrobot", new_user_1)
dtrobot2 = model.user.create_robot("dtrobot2", new_user_1)
new_user_2 = model.user.create_user("public", "password",
"*****@*****.**")
new_user_2.verified = True
new_user_2.save()
new_user_3 = model.user.create_user("freshuser", "password",
"*****@*****.**")
new_user_3.verified = True
new_user_3.save()
another_robot = model.user.create_robot("anotherrobot", new_user_3)
new_user_4 = model.user.create_user("randomuser", "password",
"*****@*****.**")
new_user_4.verified = True
new_user_4.save()
new_user_5 = model.user.create_user("unverified", "password",
"*****@*****.**")
new_user_5.save()
reader = model.user.create_user("reader", "password", "*****@*****.**")
reader.verified = True
reader.save()
creatoruser = model.user.create_user("creator", "password",
"*****@*****.**")
creatoruser.verified = True
creatoruser.save()
outside_org = model.user.create_user("outsideorg", "password",
"*****@*****.**")
outside_org.verified = True
outside_org.save()
model.notification.create_notification(
"test_notification",
new_user_1,
metadata={
"some": "value",
"arr": [1, 2, 3],
"obj": {
"a": 1,
"b": 2
}
},
)
from_date = datetime.utcnow()
to_date = from_date + timedelta(hours=1)
notification_metadata = {
"from_date": formatdate(calendar.timegm(from_date.utctimetuple())),
"to_date": formatdate(calendar.timegm(to_date.utctimetuple())),
"reason": "database migration",
}
model.notification.create_notification("maintenance",
new_user_1,
metadata=notification_metadata)
__generate_repository(
new_user_4,
"randomrepo",
"Random repo repository.",
False,
[],
(4, [], ["latest", "prod"]),
)
simple_repo = __generate_repository(
new_user_1,
"simple",
"Simple repository.",
False,
[],
(4, [], ["latest", "prod"]),
)
# Add some labels to the latest tag's manifest.
repo_ref = RepositoryReference.for_repo_obj(simple_repo)
tag = registry_model.get_repo_tag(repo_ref, "latest")
manifest = registry_model.get_manifest_for_tag(tag)
assert manifest
first_label = registry_model.create_manifest_label(manifest, "foo", "bar",
"manifest")
registry_model.create_manifest_label(manifest, "foo", "baz", "api")
registry_model.create_manifest_label(manifest, "anotherlabel", "1234",
"internal")
registry_model.create_manifest_label(manifest, "jsonlabel",
'{"hey": "there"}', "internal",
"application/json")
label_metadata = {
"key": "foo",
"value": "bar",
"id": first_label._db_id,
"manifest_digest": manifest.digest,
}
logs_model.log_action(
"manifest_label_add",
new_user_1.username,
performer=new_user_1,
timestamp=datetime.now(),
metadata=label_metadata,
repository=simple_repo,
)
model.blob.initiate_upload(new_user_1.username, simple_repo.name,
str(uuid4()), "local_us", {})
model.notification.create_repo_notification(simple_repo, "repo_push",
"quay_notification", {}, {})
__generate_repository(
new_user_1,
"sharedtags",
"Shared tags repository",
False,
[(new_user_2, "read"), (dtrobot[0], "read")],
(
2,
[
(3, [], ["v2.0", "v2.1", "v2.2"]),
(
1,
[(1, [(1, [], ["prod", "581a284"])
], ["staging", "8423b58"]), (1, [], None)],
None,
),
],
None,
),
)
__generate_repository(
new_user_1,
"history",
"Historical repository.",
False,
[],
(4, [(2, [], "#latest"), (3, [], "latest")], None),
)
__generate_repository(
new_user_1,
"complex",
"Complex repository with many branches and tags.",
False,
[(new_user_2, "read"), (dtrobot[0], "read")],
(
2,
[(3, [], "v2.0"),
(1, [(1, [(2, [], ["prod"])], "staging"), (1, [], None)], None)],
None,
),
)
__generate_repository(
new_user_1,
"gargantuan",
None,
False,
[],
(
2,
[
(3, [], "v2.0"),
(1, [(1, [(1, [], ["latest", "prod"])], "staging"),
(1, [], None)], None),
(20, [], "v3.0"),
(5, [], "v4.0"),
(1, [(1, [], "v5.0"), (1, [], "v6.0")], None),
],
None,
),
)
trusted_repo = __generate_repository(
new_user_1,
"trusted",
"Trusted repository.",
False,
[],
(4, [], ["latest", "prod"]),
)
trusted_repo.trust_enabled = True
trusted_repo.save()
publicrepo = __generate_repository(
new_user_2,
"publicrepo",
"Public repository pullable by the world.",
True,
[],
(10, [], "latest"),
)
__generate_repository(outside_org, "coolrepo", "Some cool repo.", False,
[], (5, [], "latest"))
__generate_repository(
new_user_1,
"shared",
"Shared repository, another user can write.",
False,
[(new_user_2, "write"), (reader, "read")],
(5, [], "latest"),
)
__generate_repository(
new_user_1,
"text-full-repo",
"This is a repository for testing text search",
False,
[(new_user_2, "write"), (reader, "read")],
(5, [], "latest"),
)
building = __generate_repository(
new_user_1,
"building",
"Empty repository which is building.",
False,
[(new_user_2, "write"), (reader, "read")],
(0, [], None),
)
new_token = model.token.create_access_token(building, "write",
"build-worker")
trigger = model.build.create_build_trigger(building,
"github",
"123authtoken",
new_user_1,
pull_robot=dtrobot[0])
trigger.config = json.dumps({
"build_source": "jakedt/testconnect",
"subdir": "",
"dockerfile_path": "Dockerfile",
"context": "/",
})
trigger.save()
repo = "ci.devtable.com:5000/%s/%s" % (building.namespace_user.username,
building.name)
job_config = {
"repository": repo,
"docker_tags": ["latest"],
"build_subdir": "",
"trigger_metadata": {
"commit": "3482adc5822c498e8f7db2e361e8d57b3d77ddd9",
"ref": "refs/heads/master",
"default_branch": "master",
},
}
model.repository.star_repository(new_user_1, simple_repo)
record = model.repository.create_email_authorization_for_repo(
new_user_1.username, "simple", "*****@*****.**")
record.confirmed = True
record.save()
model.repository.create_email_authorization_for_repo(
new_user_1.username, "simple", "*****@*****.**")
build2 = model.build.create_repository_build(
building,
new_token,
job_config,
"68daeebd-a5b9-457f-80a0-4363b882f8ea",
"build-name",
trigger,
)
build2.uuid = "deadpork-dead-pork-dead-porkdeadpork"
build2.save()
build3 = model.build.create_repository_build(
building,
new_token,
job_config,
"f49d07f9-93da-474d-ad5f-c852107c3892",
"build-name",
trigger,
)
build3.uuid = "deadduck-dead-duck-dead-duckdeadduck"
build3.save()
build1 = model.build.create_repository_build(
building, new_token, job_config, "701dcc3724fb4f2ea6c31400528343cd",
"build-name", trigger)
build1.uuid = "deadbeef-dead-beef-dead-beefdeadbeef"
build1.save()
org = model.organization.create_organization("buynlarge",
"*****@*****.**",
new_user_1)
org.stripe_id = TEST_STRIPE_ID
org.save()
liborg = model.organization.create_organization(
"library", "*****@*****.**", new_user_1)
liborg.save()
titiorg = model.organization.create_organization("titi",
"*****@*****.**",
new_user_1)
titiorg.save()
thirdorg = model.organization.create_organization(
"sellnsmall", "*****@*****.**", new_user_1)
thirdorg.save()
model.user.create_robot("coolrobot", org)
oauth_app_1 = model.oauth.create_application(
org,
"Some Test App",
"http://localhost:8000",
"http://localhost:8000/o2c.html",
client_id="deadbeef",
)
model.oauth.create_application(
org,
"Some Other Test App",
"http://quay.io",
"http://localhost:8000/o2c.html",
client_id="deadpork",
description="This is another test application",
)
model.oauth.create_user_access_token(new_user_1,
"deadbeef",
"repo:admin",
access_token="%s%s" %
("b" * 40, "c" * 40))
oauth_credential = Credential.from_string("dswfhasdf1")
OAuthAuthorizationCode.create(
application=oauth_app_1,
code="Z932odswfhasdf1",
scope="repo:admin",
data='{"somejson": "goeshere"}',
code_name="Z932odswfhasdf1Z932o",
code_credential=oauth_credential,
)
model.user.create_robot("neworgrobot", org)
ownerbot = model.user.create_robot("ownerbot", org)[0]
creatorbot = model.user.create_robot("creatorbot", org)[0]
owners = model.team.get_organization_team("buynlarge", "owners")
owners.description = "Owners have unfetterd access across the entire org."
owners.save()
org_repo = __generate_repository(
org,
"orgrepo",
"Repository owned by an org.",
False,
[(outside_org, "read")],
(4, [], ["latest", "prod"]),
)
__generate_repository(
org,
"anotherorgrepo",
"Another repository owned by an org.",
False,
[],
(4, [], ["latest", "prod"]),
)
creators = model.team.create_team("creators", org, "creator",
"Creators of orgrepo.")
reader_team = model.team.create_team("readers", org, "member",
"Readers of orgrepo.")
model.team.add_or_invite_to_team(new_user_1, reader_team, outside_org)
model.permission.set_team_repo_permission(reader_team.name,
org_repo.namespace_user.username,
org_repo.name, "read")
model.team.add_user_to_team(new_user_2, reader_team)
model.team.add_user_to_team(reader, reader_team)
model.team.add_user_to_team(ownerbot, owners)
model.team.add_user_to_team(creatorbot, creators)
model.team.add_user_to_team(creatoruser, creators)
sell_owners = model.team.get_organization_team("sellnsmall", "owners")
sell_owners.description = "Owners have unfettered access across the entire org."
sell_owners.save()
model.team.add_user_to_team(new_user_4, sell_owners)
sync_config = {
"group_dn": "cn=Test-Group,ou=Users",
"group_id": "somegroupid"
}
synced_team = model.team.create_team("synced", org, "member",
"Some synced team.")
model.team.set_team_syncing(synced_team, "ldap", sync_config)
another_synced_team = model.team.create_team("synced", thirdorg, "member",
"Some synced team.")
model.team.set_team_syncing(another_synced_team, "ldap",
{"group_dn": "cn=Test-Group,ou=Users"})
__generate_repository(
new_user_1,
"superwide",
None,
False,
[],
[
(10, [], "latest2"),
(2, [], "latest3"),
(2, [(1, [], "latest11"), (2, [], "latest12")], "latest4"),
(2, [], "latest5"),
(2, [], "latest6"),
(2, [], "latest7"),
(2, [], "latest8"),
(2, [], "latest9"),
(2, [], "latest10"),
(2, [], "latest13"),
(2, [], "latest14"),
(2, [], "latest15"),
(2, [], "latest16"),
(2, [], "latest17"),
(2, [], "latest18"),
],
)
mirror_repo = __generate_repository(
new_user_1,
"mirrored",
"Mirrored repository.",
False,
[(dtrobot[0], "write"), (dtrobot2[0], "write")],
(4, [], ["latest", "prod"]),
)
mirror_rule = model.repo_mirror.create_mirroring_rule(
mirror_repo, ["latest", "3.3*"])
mirror_args = (mirror_repo, mirror_rule, dtrobot[0], "quay.io/coreos/etcd",
60 * 60 * 24)
mirror_kwargs = {
"external_registry_username": "******",
"external_registry_password": "******",
"external_registry_config": {},
"is_enabled": True,
"sync_start_date": datetime.utcnow(),
}
mirror = model.repo_mirror.enable_mirroring_for_repository(
*mirror_args, **mirror_kwargs)
read_only_repo = __generate_repository(
new_user_1,
"readonly",
"Read-Only Repo.",
False,
[],
(4, [], ["latest", "prod"]),
)
read_only_repo.state = RepositoryState.READ_ONLY
read_only_repo.save()
model.permission.add_prototype_permission(org,
"read",
activating_user=new_user_1,
delegate_user=new_user_2)
model.permission.add_prototype_permission(org,
"read",
activating_user=new_user_1,
delegate_team=reader_team)
model.permission.add_prototype_permission(org,
"write",
activating_user=new_user_2,
delegate_user=new_user_1)
today = datetime.today()
week_ago = today - timedelta(6)
six_ago = today - timedelta(5)
four_ago = today - timedelta(4)
yesterday = datetime.combine(date.today(),
datetime.min.time()) - timedelta(hours=6)
__generate_service_key("kid1", "somesamplekey", new_user_1, today,
ServiceKeyApprovalType.SUPERUSER)
__generate_service_key(
"kid2",
"someexpiringkey",
new_user_1,
week_ago,
ServiceKeyApprovalType.SUPERUSER,
today + timedelta(days=14),
)
__generate_service_key("kid3", "unapprovedkey", new_user_1, today, None)
__generate_service_key(
"kid4",
"autorotatingkey",
new_user_1,
six_ago,
ServiceKeyApprovalType.KEY_ROTATION,
today + timedelta(days=1),
rotation_duration=timedelta(hours=12).total_seconds(),
)
__generate_service_key(
"kid5",
"key for another service",
new_user_1,
today,
ServiceKeyApprovalType.SUPERUSER,
today + timedelta(days=14),
service="different_sample_service",
)
__generate_service_key(
"kid6",
"someexpiredkey",
new_user_1,
week_ago,
ServiceKeyApprovalType.SUPERUSER,
today - timedelta(days=1),
)
__generate_service_key(
"kid7",
"somewayexpiredkey",
new_user_1,
week_ago,
ServiceKeyApprovalType.SUPERUSER,
today - timedelta(days=30),
)
# Add the test pull key as pre-approved for local and unittest registry testing.
# Note: this must match the private key found in the local/test config.
_TEST_JWK = {
"e":
"AQAB",
"kty":
"RSA",
"n":
"yqdQgnelhAPMSeyH0kr3UGePK9oFOmNfwD0Ymnh7YYXr21VHWwyM2eVW3cnLd9KXywDFtGSe9oFDbnOuMCdUowdkBcaHju-isbv5KEbNSoy_T2Rip-6L0cY63YzcMJzv1nEYztYXS8wz76pSK81BKBCLapqOCmcPeCvV9yaoFZYvZEsXCl5jjXN3iujSzSF5Z6PpNFlJWTErMT2Z4QfbDKX2Nw6vJN6JnGpTNHZvgvcyNX8vkSgVpQ8DFnFkBEx54PvRV5KpHAq6AsJxKONMo11idQS2PfCNpa2hvz9O6UZe-eIX8jPo5NW8TuGZJumbdPT_nxTDLfCqfiZboeI0Pw",
}
key = model.service_keys.create_service_key("test_service_key",
"test_service_key", "quay",
_TEST_JWK, {}, None)
model.service_keys.approve_service_key(
key.kid,
ServiceKeyApprovalType.SUPERUSER,
notes="Test service key for local/test registry testing",
)
# Add an app specific token.
token = model.appspecifictoken.create_token(new_user_1, "some app")
token.token_name = "a" * 60
token.token_secret = "b" * 60
token.save()
logs_model.log_action(
"org_create_team",
org.username,
performer=new_user_1,
timestamp=week_ago,
metadata={"team": "readers"},
)
logs_model.log_action(
"org_set_team_role",
org.username,
performer=new_user_1,
timestamp=week_ago,
metadata={
"team": "readers",
"role": "read"
},
)
logs_model.log_action(
"create_repo",
org.username,
performer=new_user_1,
repository=org_repo,
timestamp=week_ago,
metadata={
"namespace": org.username,
"repo": "orgrepo"
},
)
logs_model.log_action(
"change_repo_permission",
org.username,
performer=new_user_2,
repository=org_repo,
timestamp=six_ago,
metadata={
"username": new_user_1.username,
"repo": "orgrepo",
"role": "admin"
},
)
logs_model.log_action(
"change_repo_permission",
org.username,
performer=new_user_1,
repository=org_repo,
timestamp=six_ago,
metadata={
"username": new_user_2.username,
"repo": "orgrepo",
"role": "read"
},
)
logs_model.log_action(
"add_repo_accesstoken",
org.username,
performer=new_user_1,
repository=org_repo,
timestamp=four_ago,
metadata={
"repo": "orgrepo",
"token": "deploytoken"
},
)
logs_model.log_action(
"push_repo",
org.username,
performer=new_user_2,
repository=org_repo,
timestamp=today,
metadata={
"username": new_user_2.username,
"repo": "orgrepo"
},
)
logs_model.log_action(
"pull_repo",
org.username,
performer=new_user_2,
repository=org_repo,
timestamp=today,
metadata={
"username": new_user_2.username,
"repo": "orgrepo"
},
)
logs_model.log_action(
"pull_repo",
org.username,
repository=org_repo,
timestamp=today,
metadata={
"token": "sometoken",
"token_code": "somecode",
"repo": "orgrepo"
},
)
logs_model.log_action(
"delete_tag",
org.username,
performer=new_user_2,
repository=org_repo,
timestamp=today,
metadata={
"username": new_user_2.username,
"repo": "orgrepo",
"tag": "sometag"
},
)
logs_model.log_action(
"pull_repo",
org.username,
repository=org_repo,
timestamp=today,
metadata={
"token_code": "somecode",
"repo": "orgrepo"
},
)
logs_model.log_action(
"pull_repo",
new_user_2.username,
repository=publicrepo,
timestamp=yesterday,
metadata={
"token_code": "somecode",
"repo": "publicrepo"
},
)
logs_model.log_action(
"build_dockerfile",
new_user_1.username,
repository=building,
timestamp=today,
metadata={
"repo": "building",
"namespace": new_user_1.username,
"trigger_id": trigger.uuid,
"config": json.loads(trigger.config),
"service": trigger.service.name,
},
)
model.message.create([{
"content": "We love you, Quay customers!",
"severity": "info",
"media_type": "text/plain",
}])
model.message.create([{
"content": "This is a **development** install of Quay",
"severity": "warning",
"media_type": "text/markdown",
}])
fake_queue = WorkQueue("fakequeue", tf)
fake_queue.put(["canonical", "job", "name"], "{}")
model.user.create_user_prompt(new_user_4, "confirm_username")
for to_count in Repository.select():
model.repositoryactioncount.count_repository_actions(
to_count, datetime.utcnow())
model.repositoryactioncount.update_repository_score(to_count)
def test_export_logs(initialized_db, storage_engine, has_logs):
# Delete all existing logs.
database.LogEntry3.delete().execute()
repo = model.repository.get_repository("devtable", "simple")
user = model.user.get_user("devtable")
now = datetime.now()
if has_logs:
# Add new logs over a multi-day period.
for index in range(-10, 10):
logs_model.log_action(
"push_repo",
"devtable",
user,
"0.0.0.0",
{"index": index},
repo,
timestamp=now + timedelta(days=index),
)
worker = ExportActionLogsWorker(None)
called = [{}]
@urlmatch(netloc=r"testcallback")
def handle_request(url, request):
called[0] = json.loads(request.body)
return {"status_code": 200, "content": "{}"}
def format_date(datetime):
return datetime.strftime("%m/%d/%Y")
with HTTMock(handle_request):
worker._process_queue_item(
{
"export_id": "someid",
"repository_id": repo.id,
"namespace_id": repo.namespace_user.id,
"namespace_name": "devtable",
"repository_name": "simple",
"start_time": format_date(now + timedelta(days=-10)),
"end_time": format_date(now + timedelta(days=10)),
"callback_url": "http://testcallback/",
"callback_email": None,
},
storage_engine,
)
assert called[0]
assert called[0]["export_id"] == "someid"
assert called[0]["status"] == "success"
url = called[0]["exported_data_url"]
if url.find("http://localhost:5000/exportedlogs/") == 0:
storage_id = url[len("http://localhost:5000/exportedlogs/"):]
else:
assert (url.find(
"https://some_bucket.s3.amazonaws.com:443/some/path/exportedactionlogs/"
) == 0)
storage_id, _ = url[len(
"https://some_bucket.s3.amazonaws.com:443/some/path/exportedactionlogs/"
):].split("?")
created = storage_engine.get_content(storage_engine.preferred_locations,
"exportedactionlogs/" + storage_id)
created_json = json.loads(created)
if has_logs:
found = set()
for log in created_json["logs"]:
if log.get("terminator"):
continue
found.add(log["metadata"]["index"])
for index in range(-10, 10):
assert index in found
else:
assert created_json["logs"] == [{"terminator": True}]
def start_build(repository, prepared_build, pull_robot_name=None):
# Ensure that builds are only run in image repositories.
if repository.kind.name != "image":
raise Exception(
"Attempt to start a build for application repository %s" %
repository.id)
# Ensure the repository isn't in mirror or read-only mode.
if repository.state != RepositoryState.NORMAL:
raise Exception(
("Attempt to start a build for a non-normal repository: %s %s" %
(repository.id, repository.state)))
# Ensure that disabled triggers are not run.
if prepared_build.trigger is not None and not prepared_build.trigger.enabled:
raise BuildTriggerDisabledException
if repository.namespace_user.maximum_queued_builds_count is not None:
queue_item_canonical_name = [repository.namespace_user.username]
alive_builds = dockerfile_build_queue.num_alive_jobs(
queue_item_canonical_name)
if alive_builds >= repository.namespace_user.maximum_queued_builds_count:
logger.debug(
"Prevented queueing of build under namespace %s due to reaching max: %s",
repository.namespace_user.username,
repository.namespace_user.maximum_queued_builds_count,
)
raise MaximumBuildsQueuedException()
host = app.config["SERVER_HOSTNAME"]
repo_path = "%s/%s/%s" % (host, repository.namespace_user.username,
repository.name)
new_token = model.token.create_access_token(
repository,
"write",
kind="build-worker",
friendly_name="Repository Build Token")
logger.debug(
"Creating build %s with repo %s tags %s",
prepared_build.build_name,
repo_path,
prepared_build.tags,
)
job_config = {
"docker_tags":
prepared_build.tags,
"registry":
host,
"build_subdir":
prepared_build.subdirectory,
"context":
prepared_build.context,
"trigger_metadata":
prepared_build.metadata or {},
"is_manual":
prepared_build.is_manual,
"manual_user":
get_authenticated_user().username
if get_authenticated_user() else None,
"archive_url":
prepared_build.archive_url,
}
with app.config["DB_TRANSACTION_FACTORY"](db):
build_request = model.build.create_repository_build(
repository,
new_token,
job_config,
prepared_build.dockerfile_id,
prepared_build.build_name,
prepared_build.trigger,
pull_robot_name=pull_robot_name,
)
pull_creds = model.user.get_pull_credentials(
pull_robot_name) if pull_robot_name else None
json_data = json.dumps({
"build_uuid": build_request.uuid,
"pull_credentials": pull_creds
})
queue_id = dockerfile_build_queue.put(
[repository.namespace_user.username, repository.name],
json_data,
retries_remaining=3)
build_request.queue_id = queue_id
build_request.save()
# Add the build to the repo's log and spawn the build_queued notification.
event_log_metadata = {
"build_id":
build_request.uuid,
"docker_tags":
prepared_build.tags,
"repo":
repository.name,
"namespace":
repository.namespace_user.username,
"is_manual":
prepared_build.is_manual,
"manual_user":
get_authenticated_user().username
if get_authenticated_user() else None,
}
if prepared_build.trigger:
event_log_metadata["trigger_id"] = prepared_build.trigger.uuid
event_log_metadata[
"trigger_kind"] = prepared_build.trigger.service.name
event_log_metadata["trigger_metadata"] = prepared_build.metadata or {}
logs_model.log_action(
"build_dockerfile",
repository.namespace_user.username,
ip=get_request_ip(),
metadata=event_log_metadata,
repository=repository,
)
# TODO: remove when more endpoints have been converted to using interfaces
repo = AttrDict({
"namespace_name": repository.namespace_user.username,
"name": repository.name,
})
spawn_notification(
repo,
"build_queued",
event_log_metadata,
subpage="build/%s" % build_request.uuid,
pathargs=["build", build_request.uuid],
)
return build_request
def populate_database(minimal=False, with_storage=False):
logger.debug('Populating the DB with test data.')
# Check if the data already exists. If so, we skip. This can happen between calls from the
# "old style" tests and the new py.test's.
try:
User.get(username='******')
logger.debug('DB already populated')
return
except User.DoesNotExist:
pass
# Note: databases set up with "real" schema (via Alembic) will not have these types
# type, so we it here it necessary.
try:
ImageStorageLocation.get(name='local_eu')
ImageStorageLocation.get(name='local_us')
except ImageStorageLocation.DoesNotExist:
ImageStorageLocation.create(name='local_eu')
ImageStorageLocation.create(name='local_us')
try:
NotificationKind.get(name='test_notification')
except NotificationKind.DoesNotExist:
NotificationKind.create(name='test_notification')
new_user_1 = model.user.create_user('devtable', 'password',
'*****@*****.**')
new_user_1.verified = True
new_user_1.stripe_id = TEST_STRIPE_ID
new_user_1.save()
if minimal:
logger.debug(
'Skipping most db population because user requested mininal db')
return
UserRegion.create(user=new_user_1,
location=ImageStorageLocation.get(name='local_us'))
model.release.set_region_release('quay', 'us', 'v0.1.2')
model.user.create_confirm_email_code(new_user_1,
new_email='*****@*****.**')
disabled_user = model.user.create_user('disabled', 'password',
'*****@*****.**')
disabled_user.verified = True
disabled_user.enabled = False
disabled_user.save()
dtrobot = model.user.create_robot('dtrobot', new_user_1)
dtrobot2 = model.user.create_robot('dtrobot2', new_user_1)
new_user_2 = model.user.create_user('public', 'password',
'*****@*****.**')
new_user_2.verified = True
new_user_2.save()
new_user_3 = model.user.create_user('freshuser', 'password',
'*****@*****.**')
new_user_3.verified = True
new_user_3.save()
another_robot = model.user.create_robot('anotherrobot', new_user_3)
new_user_4 = model.user.create_user('randomuser', 'password',
'*****@*****.**')
new_user_4.verified = True
new_user_4.save()
new_user_5 = model.user.create_user('unverified', 'password',
'*****@*****.**')
new_user_5.save()
reader = model.user.create_user('reader', 'password', '*****@*****.**')
reader.verified = True
reader.save()
creatoruser = model.user.create_user('creator', 'password',
'*****@*****.**')
creatoruser.verified = True
creatoruser.save()
outside_org = model.user.create_user('outsideorg', 'password',
'*****@*****.**')
outside_org.verified = True
outside_org.save()
model.notification.create_notification('test_notification',
new_user_1,
metadata={
'some': 'value',
'arr': [1, 2, 3],
'obj': {
'a': 1,
'b': 2
}
})
from_date = datetime.utcnow()
to_date = from_date + timedelta(hours=1)
notification_metadata = {
'from_date': formatdate(calendar.timegm(from_date.utctimetuple())),
'to_date': formatdate(calendar.timegm(to_date.utctimetuple())),
'reason': 'database migration'
}
model.notification.create_notification('maintenance',
new_user_1,
metadata=notification_metadata)
__generate_repository(with_storage, new_user_4, 'randomrepo',
'Random repo repository.', False, [],
(4, [], ['latest', 'prod']))
simple_repo = __generate_repository(with_storage, new_user_1, 'simple',
'Simple repository.', False, [],
(4, [], ['latest', 'prod']))
# Add some labels to the latest tag's manifest.
tag_manifest = model.tag.load_tag_manifest(new_user_1.username, 'simple',
'latest')
first_label = model.label.create_manifest_label(tag_manifest, 'foo', 'bar',
'manifest')
model.label.create_manifest_label(tag_manifest, 'foo', 'baz', 'api')
model.label.create_manifest_label(tag_manifest, 'anotherlabel', '1234',
'internal')
model.label.create_manifest_label(tag_manifest, 'jsonlabel',
'{"hey": "there"}', 'internal')
label_metadata = {
'key': 'foo',
'value': 'bar',
'id': first_label.id,
'manifest_digest': tag_manifest.digest
}
logs_model.log_action('manifest_label_add',
new_user_1.username,
performer=new_user_1,
timestamp=datetime.now(),
metadata=label_metadata,
repository=tag_manifest.tag.repository)
model.blob.initiate_upload(new_user_1.username, simple_repo.name,
str(uuid4()), 'local_us', {})
model.notification.create_repo_notification(simple_repo, 'repo_push',
'quay_notification', {}, {})
__generate_repository(
with_storage, new_user_1, 'sharedtags', 'Shared tags repository',
False, [(new_user_2, 'read'), (dtrobot[0], 'read')],
(2, [(3, [], ['v2.0', 'v2.1', 'v2.2']),
(1, [(1, [(1, [], ['prod', '581a284'])], ['staging', '8423b58']),
(1, [], None)], None)], None))
__generate_repository(with_storage, new_user_1, 'history',
'Historical repository.', False, [],
(4, [(2, [], '#latest'), (3, [], 'latest')], None))
__generate_repository(with_storage, new_user_1, 'complex',
'Complex repository with many branches and tags.',
False, [(new_user_2, 'read'), (dtrobot[0], 'read')],
(2, [(3, [], 'v2.0'),
(1, [(1, [(2, [], ['prod'])], 'staging'),
(1, [], None)], None)], None))
__generate_repository(
with_storage, new_user_1, 'gargantuan', None, False,
[], (2, [(3, [], 'v2.0'),
(1, [(1, [(1, [], ['latest', 'prod'])], 'staging'),
(1, [], None)], None), (20, [], 'v3.0'), (5, [], 'v4.0'),
(1, [(1, [], 'v5.0'), (1, [], 'v6.0')], None)], None))
trusted_repo = __generate_repository(with_storage, new_user_1, 'trusted',
'Trusted repository.', False, [],
(4, [], ['latest', 'prod']))
trusted_repo.trust_enabled = True
trusted_repo.save()
publicrepo = __generate_repository(
with_storage, new_user_2, 'publicrepo',
'Public repository pullable by the world.', True, [],
(10, [], 'latest'))
__generate_repository(with_storage, outside_org, 'coolrepo',
'Some cool repo.', False, [], (5, [], 'latest'))
__generate_repository(with_storage, new_user_1, 'shared',
'Shared repository, another user can write.', False,
[(new_user_2, 'write'),
(reader, 'read')], (5, [], 'latest'))
__generate_repository(with_storage, new_user_1, 'text-full-repo',
'This is a repository for testing text search',
False, [(new_user_2, 'write'),
(reader, 'read')], (5, [], 'latest'))
building = __generate_repository(with_storage, new_user_1, 'building',
'Empty repository which is building.',
False, [(new_user_2, 'write'),
(reader, 'read')], (0, [], None))
new_token = model.token.create_access_token(building, 'write',
'build-worker')
trigger = model.build.create_build_trigger(building,
'github',
'123authtoken',
new_user_1,
pull_robot=dtrobot[0])
trigger.config = json.dumps({
'build_source': 'jakedt/testconnect',
'subdir': '',
'dockerfile_path': 'Dockerfile',
'context': '/',
})
trigger.save()
repo = 'ci.devtable.com:5000/%s/%s' % (building.namespace_user.username,
building.name)
job_config = {
'repository': repo,
'docker_tags': ['latest'],
'build_subdir': '',
'trigger_metadata': {
'commit': '3482adc5822c498e8f7db2e361e8d57b3d77ddd9',
'ref': 'refs/heads/master',
'default_branch': 'master'
}
}
model.repository.star_repository(new_user_1, simple_repo)
record = model.repository.create_email_authorization_for_repo(
new_user_1.username, 'simple', '*****@*****.**')
record.confirmed = True
record.save()
model.repository.create_email_authorization_for_repo(
new_user_1.username, 'simple', '*****@*****.**')
build2 = model.build.create_repository_build(
building, new_token, job_config,
'68daeebd-a5b9-457f-80a0-4363b882f8ea', 'build-name', trigger)
build2.uuid = 'deadpork-dead-pork-dead-porkdeadpork'
build2.save()
build3 = model.build.create_repository_build(
building, new_token, job_config,
'f49d07f9-93da-474d-ad5f-c852107c3892', 'build-name', trigger)
build3.uuid = 'deadduck-dead-duck-dead-duckdeadduck'
build3.save()
build1 = model.build.create_repository_build(
building, new_token, job_config, '701dcc3724fb4f2ea6c31400528343cd',
'build-name', trigger)
build1.uuid = 'deadbeef-dead-beef-dead-beefdeadbeef'
build1.save()
org = model.organization.create_organization('buynlarge',
'*****@*****.**',
new_user_1)
org.stripe_id = TEST_STRIPE_ID
org.save()
liborg = model.organization.create_organization(
'library', '*****@*****.**', new_user_1)
liborg.save()
titiorg = model.organization.create_organization('titi',
'*****@*****.**',
new_user_1)
titiorg.save()
thirdorg = model.organization.create_organization(
'sellnsmall', '*****@*****.**', new_user_1)
thirdorg.save()
model.user.create_robot('coolrobot', org)
oauth_app_1 = model.oauth.create_application(
org,
'Some Test App',
'http://localhost:8000',
'http://localhost:8000/o2c.html',
client_id='deadbeef')
model.oauth.create_application(
org,
'Some Other Test App',
'http://quay.io',
'http://localhost:8000/o2c.html',
client_id='deadpork',
description='This is another test application')
model.oauth.create_access_token_for_testing(new_user_1,
'deadbeef',
'repo:admin',
access_token='%s%s' %
('b' * 40, 'c' * 40))
oauth_credential = Credential.from_string('dswfhasdf1')
OAuthAuthorizationCode.create(application=oauth_app_1,
code='Z932odswfhasdf1',
scope='repo:admin',
data='{"somejson": "goeshere"}',
code_name='Z932odswfhasdf1Z932o',
code_credential=oauth_credential)
model.user.create_robot('neworgrobot', org)
ownerbot = model.user.create_robot('ownerbot', org)[0]
creatorbot = model.user.create_robot('creatorbot', org)[0]
owners = model.team.get_organization_team('buynlarge', 'owners')
owners.description = 'Owners have unfetterd access across the entire org.'
owners.save()
org_repo = __generate_repository(with_storage, org, 'orgrepo',
'Repository owned by an org.', False,
[(outside_org, 'read')],
(4, [], ['latest', 'prod']))
__generate_repository(with_storage, org, 'anotherorgrepo',
'Another repository owned by an org.', False, [],
(4, [], ['latest', 'prod']))
creators = model.team.create_team('creators', org, 'creator',
'Creators of orgrepo.')
reader_team = model.team.create_team('readers', org, 'member',
'Readers of orgrepo.')
model.team.add_or_invite_to_team(new_user_1, reader_team, outside_org)
model.permission.set_team_repo_permission(reader_team.name,
org_repo.namespace_user.username,
org_repo.name, 'read')
model.team.add_user_to_team(new_user_2, reader_team)
model.team.add_user_to_team(reader, reader_team)
model.team.add_user_to_team(ownerbot, owners)
model.team.add_user_to_team(creatorbot, creators)
model.team.add_user_to_team(creatoruser, creators)
sell_owners = model.team.get_organization_team('sellnsmall', 'owners')
sell_owners.description = 'Owners have unfettered access across the entire org.'
sell_owners.save()
model.team.add_user_to_team(new_user_4, sell_owners)
sync_config = {
'group_dn': 'cn=Test-Group,ou=Users',
'group_id': 'somegroupid'
}
synced_team = model.team.create_team('synced', org, 'member',
'Some synced team.')
model.team.set_team_syncing(synced_team, 'ldap', sync_config)
another_synced_team = model.team.create_team('synced', thirdorg, 'member',
'Some synced team.')
model.team.set_team_syncing(another_synced_team, 'ldap',
{'group_dn': 'cn=Test-Group,ou=Users'})
__generate_repository(with_storage, new_user_1, 'superwide', None,
False, [], [(10, [], 'latest2'), (2, [], 'latest3'),
(2, [(1, [], 'latest11'),
(2, [], 'latest12')], 'latest4'),
(2, [], 'latest5'), (2, [], 'latest6'),
(2, [], 'latest7'), (2, [], 'latest8'),
(2, [], 'latest9'), (2, [], 'latest10'),
(2, [], 'latest13'), (2, [], 'latest14'),
(2, [], 'latest15'), (2, [], 'latest16'),
(2, [], 'latest17'),
(2, [], 'latest18')])
mirror_repo = __generate_repository(with_storage, new_user_1, 'mirrored',
'Mirrored repository.', False,
[(dtrobot[0], 'write'),
(dtrobot2[0], 'write')],
(4, [], ['latest', 'prod']))
mirror_rule = model.repo_mirror.create_mirroring_rule(
mirror_repo, ['latest', '3.3*'])
mirror_args = (mirror_repo, mirror_rule, dtrobot[0], 'quay.io/coreos/etcd',
60 * 60 * 24)
mirror_kwargs = {
'external_registry_username': '******',
'external_registry_password': '******',
'external_registry_config': {},
'is_enabled': True,
'sync_start_date': datetime.utcnow()
}
mirror = model.repo_mirror.enable_mirroring_for_repository(
*mirror_args, **mirror_kwargs)
read_only_repo = __generate_repository(with_storage, new_user_1,
'readonly', 'Read-Only Repo.',
False, [],
(4, [], ['latest', 'prod']))
read_only_repo.state = RepositoryState.READ_ONLY
read_only_repo.save()
model.permission.add_prototype_permission(org,
'read',
activating_user=new_user_1,
delegate_user=new_user_2)
model.permission.add_prototype_permission(org,
'read',
activating_user=new_user_1,
delegate_team=reader_team)
model.permission.add_prototype_permission(org,
'write',
activating_user=new_user_2,
delegate_user=new_user_1)
today = datetime.today()
week_ago = today - timedelta(6)
six_ago = today - timedelta(5)
four_ago = today - timedelta(4)
yesterday = datetime.combine(date.today(),
datetime.min.time()) - timedelta(hours=6)
__generate_service_key('kid1', 'somesamplekey', new_user_1, today,
ServiceKeyApprovalType.SUPERUSER)
__generate_service_key('kid2', 'someexpiringkey', new_user_1, week_ago,
ServiceKeyApprovalType.SUPERUSER,
today + timedelta(days=14))
__generate_service_key('kid3', 'unapprovedkey', new_user_1, today, None)
__generate_service_key(
'kid4',
'autorotatingkey',
new_user_1,
six_ago,
ServiceKeyApprovalType.KEY_ROTATION,
today + timedelta(days=1),
rotation_duration=timedelta(hours=12).total_seconds())
__generate_service_key('kid5',
'key for another service',
new_user_1,
today,
ServiceKeyApprovalType.SUPERUSER,
today + timedelta(days=14),
service='different_sample_service')
__generate_service_key('kid6', 'someexpiredkey', new_user_1, week_ago,
ServiceKeyApprovalType.SUPERUSER,
today - timedelta(days=1))
__generate_service_key('kid7', 'somewayexpiredkey', new_user_1, week_ago,
ServiceKeyApprovalType.SUPERUSER,
today - timedelta(days=30))
# Add the test pull key as pre-approved for local and unittest registry testing.
# Note: this must match the private key found in the local/test config.
_TEST_JWK = {
'e':
'AQAB',
'kty':
'RSA',
'n':
'yqdQgnelhAPMSeyH0kr3UGePK9oFOmNfwD0Ymnh7YYXr21VHWwyM2eVW3cnLd9KXywDFtGSe9oFDbnOuMCdUowdkBcaHju-isbv5KEbNSoy_T2Rip-6L0cY63YzcMJzv1nEYztYXS8wz76pSK81BKBCLapqOCmcPeCvV9yaoFZYvZEsXCl5jjXN3iujSzSF5Z6PpNFlJWTErMT2Z4QfbDKX2Nw6vJN6JnGpTNHZvgvcyNX8vkSgVpQ8DFnFkBEx54PvRV5KpHAq6AsJxKONMo11idQS2PfCNpa2hvz9O6UZe-eIX8jPo5NW8TuGZJumbdPT_nxTDLfCqfiZboeI0Pw'
}
key = model.service_keys.create_service_key('test_service_key',
'test_service_key', 'quay',
_TEST_JWK, {}, None)
model.service_keys.approve_service_key(
key.kid,
ServiceKeyApprovalType.SUPERUSER,
notes='Test service key for local/test registry testing')
# Add an app specific token.
token = model.appspecifictoken.create_token(new_user_1, 'some app')
token.token_name = 'a' * 60
token.token_secret = 'b' * 60
token.save()
logs_model.log_action('org_create_team',
org.username,
performer=new_user_1,
timestamp=week_ago,
metadata={'team': 'readers'})
logs_model.log_action('org_set_team_role',
org.username,
performer=new_user_1,
timestamp=week_ago,
metadata={
'team': 'readers',
'role': 'read'
})
logs_model.log_action('create_repo',
org.username,
performer=new_user_1,
repository=org_repo,
timestamp=week_ago,
metadata={
'namespace': org.username,
'repo': 'orgrepo'
})
logs_model.log_action('change_repo_permission',
org.username,
performer=new_user_2,
repository=org_repo,
timestamp=six_ago,
metadata={
'username': new_user_1.username,
'repo': 'orgrepo',
'role': 'admin'
})
logs_model.log_action('change_repo_permission',
org.username,
performer=new_user_1,
repository=org_repo,
timestamp=six_ago,
metadata={
'username': new_user_2.username,
'repo': 'orgrepo',
'role': 'read'
})
logs_model.log_action('add_repo_accesstoken',
org.username,
performer=new_user_1,
repository=org_repo,
timestamp=four_ago,
metadata={
'repo': 'orgrepo',
'token': 'deploytoken'
})
logs_model.log_action('push_repo',
org.username,
performer=new_user_2,
repository=org_repo,
timestamp=today,
metadata={
'username': new_user_2.username,
'repo': 'orgrepo'
})
logs_model.log_action('pull_repo',
org.username,
performer=new_user_2,
repository=org_repo,
timestamp=today,
metadata={
'username': new_user_2.username,
'repo': 'orgrepo'
})
logs_model.log_action('pull_repo',
org.username,
repository=org_repo,
timestamp=today,
metadata={
'token': 'sometoken',
'token_code': 'somecode',
'repo': 'orgrepo'
})
logs_model.log_action('delete_tag',
org.username,
performer=new_user_2,
repository=org_repo,
timestamp=today,
metadata={
'username': new_user_2.username,
'repo': 'orgrepo',
'tag': 'sometag'
})
logs_model.log_action('pull_repo',
org.username,
repository=org_repo,
timestamp=today,
metadata={
'token_code': 'somecode',
'repo': 'orgrepo'
})
logs_model.log_action('pull_repo',
new_user_2.username,
repository=publicrepo,
timestamp=yesterday,
metadata={
'token_code': 'somecode',
'repo': 'publicrepo'
})
logs_model.log_action('build_dockerfile',
new_user_1.username,
repository=building,
timestamp=today,
metadata={
'repo': 'building',
'namespace': new_user_1.username,
'trigger_id': trigger.uuid,
'config': json.loads(trigger.config),
'service': trigger.service.name
})
model.message.create([{
'content': 'We love you, Quay customers!',
'severity': 'info',
'media_type': 'text/plain'
}])
model.message.create([{
'content': 'This is a **development** install of Quay',
'severity': 'warning',
'media_type': 'text/markdown'
}])
fake_queue = WorkQueue('fakequeue', tf)
fake_queue.put(['canonical', 'job', 'name'], '{}')
model.user.create_user_prompt(new_user_4, 'confirm_username')
while True:
to_count = model.repositoryactioncount.find_uncounted_repository()
if not to_count:
break
model.repositoryactioncount.count_repository_actions(to_count)
model.repositoryactioncount.update_repository_score(to_count)
def track_and_log(event_name,
repo_obj,
analytics_name=None,
analytics_sample=1,
**kwargs):
repo_name = repo_obj.name
namespace_name = repo_obj.namespace_name
metadata = {
"repo": repo_name,
"namespace": namespace_name,
"user-agent": request.user_agent.string,
}
metadata.update(kwargs)
is_free_namespace = False
if hasattr(repo_obj, "is_free_namespace"):
is_free_namespace = repo_obj.is_free_namespace
# Add auth context metadata.
analytics_id = "anonymous"
auth_context = get_authenticated_context()
if auth_context is not None:
analytics_id, context_metadata = auth_context.analytics_id_and_public_metadata(
)
metadata.update(context_metadata)
# Publish the user event (if applicable)
logger.debug("Checking publishing %s to the user events system",
event_name)
if auth_context and auth_context.has_nonrobot_user:
logger.debug("Publishing %s to the user events system", event_name)
user_event_data = {
"action": event_name,
"repository": repo_name,
"namespace": namespace_name,
}
event = userevents.get_event(auth_context.authed_user.username)
event.publish_event_data("docker-cli", user_event_data)
# Save the action to mixpanel.
if random.random() < analytics_sample:
if analytics_name is None:
analytics_name = event_name
logger.debug("Logging the %s to analytics engine", analytics_name)
request_parsed = urlparse(request.url_root)
extra_params = {
"repository": "%s/%s" % (namespace_name, repo_name),
"user-agent": request.user_agent.string,
"hostname": request_parsed.hostname,
}
analytics.track(analytics_id, analytics_name, extra_params)
# Add the resolved information to the metadata.
logger.debug("Resolving IP address %s", get_request_ip())
resolved_ip = ip_resolver.resolve_ip(get_request_ip())
if resolved_ip is not None:
metadata["resolved_ip"] = resolved_ip._asdict()
logger.debug("Resolved IP address %s", get_request_ip())
# Log the action to the database.
logger.debug("Logging the %s to logs system", event_name)
try:
logs_model.log_action(
event_name,
namespace_name,
performer=get_authenticated_user(),
ip=get_request_ip(),
metadata=metadata,
repository=repo_obj,
is_free_namespace=is_free_namespace,
)
logger.debug("Track and log of %s complete", event_name)
except ReadOnlyModeException:
pass
def start_build(repository, prepared_build, pull_robot_name=None):
# Ensure that builds are only run in image repositories.
if repository.kind.name != 'image':
raise Exception(
'Attempt to start a build for application repository %s' %
repository.id)
# Ensure the repository isn't in mirror or read-only mode.
if repository.state != RepositoryState.NORMAL:
raise Exception(
('Attempt to start a build for a non-normal repository: %s %s' %
(repository.id, repository.state)))
# Ensure that disabled triggers are not run.
if prepared_build.trigger is not None and not prepared_build.trigger.enabled:
raise BuildTriggerDisabledException
if repository.namespace_user.maximum_queued_builds_count is not None:
queue_item_canonical_name = [repository.namespace_user.username]
alive_builds = dockerfile_build_queue.num_alive_jobs(
queue_item_canonical_name)
if alive_builds >= repository.namespace_user.maximum_queued_builds_count:
logger.debug(
'Prevented queueing of build under namespace %s due to reaching max: %s',
repository.namespace_user.username,
repository.namespace_user.maximum_queued_builds_count)
raise MaximumBuildsQueuedException()
host = app.config['SERVER_HOSTNAME']
repo_path = '%s/%s/%s' % (host, repository.namespace_user.username,
repository.name)
new_token = model.token.create_access_token(
repository,
'write',
kind='build-worker',
friendly_name='Repository Build Token')
logger.debug('Creating build %s with repo %s tags %s',
prepared_build.build_name, repo_path, prepared_build.tags)
job_config = {
'docker_tags':
prepared_build.tags,
'registry':
host,
'build_subdir':
prepared_build.subdirectory,
'context':
prepared_build.context,
'trigger_metadata':
prepared_build.metadata or {},
'is_manual':
prepared_build.is_manual,
'manual_user':
get_authenticated_user().username
if get_authenticated_user() else None,
'archive_url':
prepared_build.archive_url
}
with app.config['DB_TRANSACTION_FACTORY'](db):
build_request = model.build.create_repository_build(
repository,
new_token,
job_config,
prepared_build.dockerfile_id,
prepared_build.build_name,
prepared_build.trigger,
pull_robot_name=pull_robot_name)
pull_creds = model.user.get_pull_credentials(
pull_robot_name) if pull_robot_name else None
json_data = json.dumps({
'build_uuid': build_request.uuid,
'pull_credentials': pull_creds
})
queue_id = dockerfile_build_queue.put(
[repository.namespace_user.username, repository.name],
json_data,
retries_remaining=3)
build_request.queue_id = queue_id
build_request.save()
# Add the queueing of the build to the metrics queue.
metric_queue.repository_build_queued.Inc(
labelvalues=[repository.namespace_user.username, repository.name])
# Add the build to the repo's log and spawn the build_queued notification.
event_log_metadata = {
'build_id':
build_request.uuid,
'docker_tags':
prepared_build.tags,
'repo':
repository.name,
'namespace':
repository.namespace_user.username,
'is_manual':
prepared_build.is_manual,
'manual_user':
get_authenticated_user().username if get_authenticated_user() else None
}
if prepared_build.trigger:
event_log_metadata['trigger_id'] = prepared_build.trigger.uuid
event_log_metadata[
'trigger_kind'] = prepared_build.trigger.service.name
event_log_metadata['trigger_metadata'] = prepared_build.metadata or {}
logs_model.log_action('build_dockerfile',
repository.namespace_user.username,
ip=get_request_ip(),
metadata=event_log_metadata,
repository=repository)
# TODO: remove when more endpoints have been converted to using interfaces
repo = AttrDict({
'namespace_name': repository.namespace_user.username,
'name': repository.name,
})
spawn_notification(repo,
'build_queued',
event_log_metadata,
subpage='build/%s' % build_request.uuid,
pathargs=['build', build_request.uuid])
return build_request
def put_service_key(service, kid):
metadata = {'ip': get_request_ip()}
rotation_duration = request.args.get('rotation', None)
expiration_date = request.args.get('expiration', None)
if expiration_date is not None:
try:
expiration_date = datetime.utcfromtimestamp(float(expiration_date))
except ValueError:
logger.exception('Error parsing expiration date on key')
abort(400)
try:
jwk = request.get_json()
except ValueError:
logger.exception('Error parsing JWK')
abort(400)
jwt_header = request.headers.get(JWT_HEADER_NAME, '')
match = jwtutil.TOKEN_REGEX.match(jwt_header)
if match is None:
logger.error('Could not find matching bearer token')
abort(400)
encoded_jwt = match.group(1)
_validate_jwk(jwk)
signer_kid = _signer_kid(encoded_jwt, allow_none=True)
if kid == signer_kid or signer_kid is None:
# The key is self-signed. Create a new instance and await approval.
_validate_jwt(encoded_jwt, jwk, service)
model.create_service_key('',
kid,
service,
jwk,
metadata,
expiration_date,
rotation_duration=rotation_duration)
logs_model.log_action('service_key_create',
ip=get_request_ip(),
metadata={
'kid': kid,
'preshared': False,
'service': service,
'name': '',
'expiration_date': expiration_date,
'user_agent':
request.headers.get('User-Agent'),
'ip': get_request_ip(),
})
return make_response('', 202)
# Key is going to be rotated.
metadata.update({'created_by': 'Key Rotation'})
signer_key = _lookup_service_key(service, signer_kid)
signer_jwk = signer_key.jwk
_validate_jwt(encoded_jwt, signer_jwk, service)
try:
model.replace_service_key(signer_key.kid, kid, jwk, metadata,
expiration_date)
except ServiceKeyDoesNotExist:
abort(404)
logs_model.log_action('service_key_rotate',
ip=get_request_ip(),
metadata={
'kid': kid,
'signer_kid': signer_key.kid,
'service': service,
'name': signer_key.name,
'expiration_date': expiration_date,
'user_agent': request.headers.get('User-Agent'),
'ip': get_request_ip(),
})
return make_response('', 200)
def test_export_logs(initialized_db, storage_engine, has_logs):
# Delete all existing logs.
database.LogEntry3.delete().execute()
repo = model.repository.get_repository('devtable', 'simple')
user = model.user.get_user('devtable')
now = datetime.now()
if has_logs:
# Add new logs over a multi-day period.
for index in range(-10, 10):
logs_model.log_action('push_repo',
'devtable',
user,
'0.0.0.0', {'index': index},
repo,
timestamp=now + timedelta(days=index))
worker = ExportActionLogsWorker(None)
called = [{}]
@urlmatch(netloc=r'testcallback')
def handle_request(url, request):
called[0] = json.loads(request.body)
return {'status_code': 200, 'content': '{}'}
def format_date(datetime):
return datetime.strftime("%m/%d/%Y")
with HTTMock(handle_request):
worker._process_queue_item(
{
'export_id': 'someid',
'repository_id': repo.id,
'namespace_id': repo.namespace_user.id,
'namespace_name': 'devtable',
'repository_name': 'simple',
'start_time': format_date(now + timedelta(days=-10)),
'end_time': format_date(now + timedelta(days=10)),
'callback_url': 'http://testcallback/',
'callback_email': None,
}, storage_engine)
assert called[0]
assert called[0][u'export_id'] == 'someid'
assert called[0][u'status'] == 'success'
url = called[0][u'exported_data_url']
if url.find('http://localhost:5000/exportedlogs/') == 0:
storage_id = url[len('http://localhost:5000/exportedlogs/'):]
else:
assert url.find(
'https://some_bucket.s3.amazonaws.com/some/path/exportedactionlogs/'
) == 0
storage_id, _ = url[len(
'https://some_bucket.s3.amazonaws.com/some/path/exportedactionlogs/'
):].split('?')
created = storage_engine.get_content(storage_engine.preferred_locations,
'exportedactionlogs/' + storage_id)
created_json = json.loads(created)
if has_logs:
found = set()
for log in created_json['logs']:
if log.get('terminator'):
continue
found.add(log['metadata']['index'])
for index in range(-10, 10):
assert index in found
else:
assert created_json['logs'] == [{'terminator': True}]
