def process(resources):
response = []
changed = True
for resource in resources:
p_state = resource["state"]
p_name = resource["name"]
try:
image_storage_location = ImageStorageLocation.get(name=p_name)
except ImageStorageLocation.DoesNotExist:
image_storage_location = None
if p_state == "absent":
if image_storage_location is not None:
changed = True
image_storage_location.delete_instance()
response.append("Image Storage Location '%s' deleted" % p_name)
changed = True
else:
response.append("Image Storage Location '%s' does not exist" %
p_name)
else:
if image_storage_location is None:
changed = True
image_storage_location = ImageStorageLocation.create(
name=p_name)
response.append("Image Storage Location '%s' created" % p_name)
else:
response.append("Image Storage Location '%s' exists" % p_name)
return {"failed": False, "changed": changed, "meta": response}, 200
def ensure_image_locations(*names):
with db_transaction():
locations = ImageStorageLocation.select().where(ImageStorageLocation.name << names)
insert_names = list(names)
for location in locations:
insert_names.remove(location.name)
if not insert_names:
return
data = [{"name": name} for name in insert_names]
ImageStorageLocation.insert_many(data).execute()
def get_or_create_shared_blob(digest, byte_data, storage):
"""
Returns the ImageStorage blob with the given digest or, if not present, adds a row and writes
the given byte data to the storage engine.
This method is *only* to be used for shared blobs that are globally accessible, such as the
special empty gzipped tar layer that Docker no longer pushes to us.
"""
assert digest
assert byte_data is not None and isinstance(byte_data, bytes)
assert storage
try:
return ImageStorage.get(content_checksum=digest)
except ImageStorage.DoesNotExist:
preferred = storage.preferred_locations[0]
location_obj = ImageStorageLocation.get(name=preferred)
record = ImageStorage.create(image_size=len(byte_data),
content_checksum=digest)
try:
storage.put_content([preferred],
storage_model.get_layer_path(record),
byte_data)
ImageStoragePlacement.create(storage=record, location=location_obj)
except:
logger.exception("Exception when trying to write special layer %s",
digest)
record.delete_instance()
raise
return record
def _populate_blob(content):
digest = str(sha256_digest(content))
location = ImageStorageLocation.get(name="local_us")
blob = store_blob_record_and_temp_link("devtable", "newrepo", digest,
location, len(content), 120)
storage.put_content(["local_us"], get_layer_path(blob), content)
return blob, digest
def _populate_blob(repo, content):
digest = str(sha256_digest(content))
location = ImageStorageLocation.get(name="local_us")
storage.put_content(["local_us"], storage.blob_path(digest), "somecontent")
blob = model.blob.store_blob_record_and_temp_link_in_repo(
repo, digest, location, len(content), 120)
return blob, digest
def _get_location_row(location):
if location in LOCATION_MAP:
return LOCATION_MAP[location]
location_row = ImageStorageLocation.get(name=location)
LOCATION_MAP[location] = location_row
return location_row
def test_blob_caching(method, endpoint, client, app):
digest = "sha256:" + hashlib.sha256(b"a").hexdigest()
location = ImageStorageLocation.get(name="local_us")
model.blob.store_blob_record_and_temp_link("devtable", "simple", digest,
location, 1, 10000000)
params = {
"repository": "devtable/simple",
"digest": digest,
}
user = model.user.get_user("devtable")
access = [{
"type": "repository",
"name": "devtable/simple",
"actions": ["pull"],
}]
context, subject = build_context_and_subject(
ValidatedAuthContext(user=user))
token = generate_bearer_token(realapp.config["SERVER_HOSTNAME"], subject,
context, access, 600, instance_keys)
headers = {
"Authorization": "Bearer %s" % token.decode("ascii"),
}
# Run without caching to make sure the request works. This also preloads some of
# our global model caches.
conduct_call(client,
"v2." + endpoint,
url_for,
method,
params,
expected_code=200,
headers=headers)
with patch("endpoints.v2.blob.model_cache",
InMemoryDataModelCache(TEST_CACHE_CONFIG)):
# First request should make a DB query to retrieve the blob.
conduct_call(client,
"v2." + endpoint,
url_for,
method,
params,
expected_code=200,
headers=headers)
# Subsequent requests should use the cached blob.
with assert_query_count(0):
conduct_call(
client,
"v2." + endpoint,
url_for,
method,
params,
expected_code=200,
headers=headers,
)
def get_image_locations():
location_map = {}
for location in ImageStorageLocation.select():
location_tuple = _Location(location.id, location.name)
location_map[location.id] = location_tuple
location_map[location.name] = location_tuple
return location_map
def _populate_blob(repo, content):
assert isinstance(content, bytes)
digest = sha256_digest(content)
location = ImageStorageLocation.get(name="local_us")
storage.put_content(["local_us"], storage.blob_path(digest), content)
blob = model.blob.store_blob_record_and_temp_link_in_repo(
repo, digest, location, len(content), 120)
return blob, digest
def _populate_blob(repo, content):
assert isinstance(repo, Repository)
assert isinstance(content, bytes)
digest = str(sha256_digest(content))
location = ImageStorageLocation.get(name="local_us")
blob = model.blob.store_blob_record_and_temp_link_in_repo(
repo, digest, location, len(content), TEMP_BLOB_EXPIRATION)
return blob, digest
def test_blob_caching(method, endpoint, client, app):
digest = 'sha256:' + hashlib.sha256("a").hexdigest()
location = ImageStorageLocation.get(name='local_us')
model.blob.store_blob_record_and_temp_link('devtable', 'simple', digest,
location, 1, 10000000)
params = {
'repository': 'devtable/simple',
'digest': digest,
}
user = model.user.get_user('devtable')
access = [{
'type': 'repository',
'name': 'devtable/simple',
'actions': ['pull'],
}]
context, subject = build_context_and_subject(
ValidatedAuthContext(user=user))
token = generate_bearer_token(realapp.config['SERVER_HOSTNAME'], subject,
context, access, 600, instance_keys)
headers = {
'Authorization': 'Bearer %s' % token,
}
# Run without caching to make sure the request works. This also preloads some of
# our global model caches.
conduct_call(client,
'v2.' + endpoint,
url_for,
method,
params,
expected_code=200,
headers=headers)
with patch('endpoints.v2.blob.model_cache', InMemoryDataModelCache()):
# First request should make a DB query to retrieve the blob.
conduct_call(client,
'v2.' + endpoint,
url_for,
method,
params,
expected_code=200,
headers=headers)
# Subsequent requests should use the cached blob.
with assert_query_count(0):
conduct_call(client,
'v2.' + endpoint,
url_for,
method,
params,
expected_code=200,
headers=headers)
def get_blob(layer):
content = Bytes.for_string_or_unicode(layer).as_encoded_str()
digest = str(sha256_digest(content))
blob = store_blob_record_and_temp_link(
self.orgname,
self.upstream_repository,
digest,
ImageStorageLocation.get(name="local_us"),
len(content),
120,
)
storage.put_content(["local_us"], get_layer_path(blob), content)
return blob, digest
def initialize_database():
db_encrypter.initialize(FieldEncrypter("anothercrazykey!"))
db.create_tables(all_models)
Role.create(name="admin")
Role.create(name="write")
Role.create(name="read")
TeamRole.create(name="admin")
TeamRole.create(name="creator")
TeamRole.create(name="member")
Visibility.create(name="public")
Visibility.create(name="private")
LoginService.create(name="google")
LoginService.create(name="github")
LoginService.create(name="quayrobot")
LoginService.create(name="ldap")
LoginService.create(name="jwtauthn")
LoginService.create(name="keystone")
LoginService.create(name="dex")
LoginService.create(name="oidc")
BuildTriggerService.create(name="github")
BuildTriggerService.create(name="custom-git")
BuildTriggerService.create(name="bitbucket")
BuildTriggerService.create(name="gitlab")
AccessTokenKind.create(name="build-worker")
AccessTokenKind.create(name="pushpull-token")
LogEntryKind.create(name="account_change_plan")
LogEntryKind.create(name="account_change_cc")
LogEntryKind.create(name="account_change_password")
LogEntryKind.create(name="account_convert")
LogEntryKind.create(name="create_robot")
LogEntryKind.create(name="delete_robot")
LogEntryKind.create(name="create_repo")
LogEntryKind.create(name="push_repo")
LogEntryKind.create(name="pull_repo")
LogEntryKind.create(name="delete_repo")
LogEntryKind.create(name="create_tag")
LogEntryKind.create(name="move_tag")
LogEntryKind.create(name="delete_tag")
LogEntryKind.create(name="revert_tag")
LogEntryKind.create(name="add_repo_permission")
LogEntryKind.create(name="change_repo_permission")
LogEntryKind.create(name="delete_repo_permission")
LogEntryKind.create(name="change_repo_visibility")
LogEntryKind.create(name="change_repo_trust")
LogEntryKind.create(name="add_repo_accesstoken")
LogEntryKind.create(name="delete_repo_accesstoken")
LogEntryKind.create(name="set_repo_description")
LogEntryKind.create(name="change_repo_state")
LogEntryKind.create(name="build_dockerfile")
LogEntryKind.create(name="org_create_team")
LogEntryKind.create(name="org_delete_team")
LogEntryKind.create(name="org_invite_team_member")
LogEntryKind.create(name="org_delete_team_member_invite")
LogEntryKind.create(name="org_add_team_member")
LogEntryKind.create(name="org_team_member_invite_accepted")
LogEntryKind.create(name="org_team_member_invite_declined")
LogEntryKind.create(name="org_remove_team_member")
LogEntryKind.create(name="org_set_team_description")
LogEntryKind.create(name="org_set_team_role")
LogEntryKind.create(name="create_prototype_permission")
LogEntryKind.create(name="modify_prototype_permission")
LogEntryKind.create(name="delete_prototype_permission")
LogEntryKind.create(name="setup_repo_trigger")
LogEntryKind.create(name="delete_repo_trigger")
LogEntryKind.create(name="create_application")
LogEntryKind.create(name="update_application")
LogEntryKind.create(name="delete_application")
LogEntryKind.create(name="reset_application_client_secret")
# Note: These next two are deprecated.
LogEntryKind.create(name="add_repo_webhook")
LogEntryKind.create(name="delete_repo_webhook")
LogEntryKind.create(name="add_repo_notification")
LogEntryKind.create(name="delete_repo_notification")
LogEntryKind.create(name="reset_repo_notification")
LogEntryKind.create(name="regenerate_robot_token")
LogEntryKind.create(name="repo_verb")
LogEntryKind.create(name="repo_mirror_enabled")
LogEntryKind.create(name="repo_mirror_disabled")
LogEntryKind.create(name="repo_mirror_config_changed")
LogEntryKind.create(name="repo_mirror_sync_started")
LogEntryKind.create(name="repo_mirror_sync_failed")
LogEntryKind.create(name="repo_mirror_sync_success")
LogEntryKind.create(name="repo_mirror_sync_now_requested")
LogEntryKind.create(name="repo_mirror_sync_tag_success")
LogEntryKind.create(name="repo_mirror_sync_tag_failed")
LogEntryKind.create(name="repo_mirror_sync_test_success")
LogEntryKind.create(name="repo_mirror_sync_test_failed")
LogEntryKind.create(name="repo_mirror_sync_test_started")
LogEntryKind.create(name="service_key_create")
LogEntryKind.create(name="service_key_approve")
LogEntryKind.create(name="service_key_delete")
LogEntryKind.create(name="service_key_modify")
LogEntryKind.create(name="service_key_extend")
LogEntryKind.create(name="service_key_rotate")
LogEntryKind.create(name="take_ownership")
LogEntryKind.create(name="manifest_label_add")
LogEntryKind.create(name="manifest_label_delete")
LogEntryKind.create(name="change_tag_expiration")
LogEntryKind.create(name="toggle_repo_trigger")
LogEntryKind.create(name="create_app_specific_token")
LogEntryKind.create(name="revoke_app_specific_token")
ImageStorageLocation.create(name="local_eu")
ImageStorageLocation.create(name="local_us")
ApprBlobPlacementLocation.create(name="local_eu")
ApprBlobPlacementLocation.create(name="local_us")
ImageStorageTransformation.create(name="squash")
ImageStorageTransformation.create(name="aci")
ImageStorageSignatureKind.create(name="gpg2")
# NOTE: These MUST be copied over to NotificationKind, since every external
# notification can also generate a Quay.io notification.
ExternalNotificationEvent.create(name="repo_push")
ExternalNotificationEvent.create(name="build_queued")
ExternalNotificationEvent.create(name="build_start")
ExternalNotificationEvent.create(name="build_success")
ExternalNotificationEvent.create(name="build_cancelled")
ExternalNotificationEvent.create(name="build_failure")
ExternalNotificationEvent.create(name="vulnerability_found")
ExternalNotificationEvent.create(name="repo_mirror_sync_started")
ExternalNotificationEvent.create(name="repo_mirror_sync_success")
ExternalNotificationEvent.create(name="repo_mirror_sync_failed")
ExternalNotificationMethod.create(name="quay_notification")
ExternalNotificationMethod.create(name="email")
ExternalNotificationMethod.create(name="webhook")
ExternalNotificationMethod.create(name="flowdock")
ExternalNotificationMethod.create(name="hipchat")
ExternalNotificationMethod.create(name="slack")
NotificationKind.create(name="repo_push")
NotificationKind.create(name="build_queued")
NotificationKind.create(name="build_start")
NotificationKind.create(name="build_success")
NotificationKind.create(name="build_cancelled")
NotificationKind.create(name="build_failure")
NotificationKind.create(name="vulnerability_found")
NotificationKind.create(name="service_key_submitted")
NotificationKind.create(name="password_required")
NotificationKind.create(name="over_private_usage")
NotificationKind.create(name="expiring_license")
NotificationKind.create(name="maintenance")
NotificationKind.create(name="org_team_invite")
NotificationKind.create(name="repo_mirror_sync_started")
NotificationKind.create(name="repo_mirror_sync_success")
NotificationKind.create(name="repo_mirror_sync_failed")
NotificationKind.create(name="test_notification")
QuayRegion.create(name="us")
QuayService.create(name="quay")
MediaType.create(name="text/plain")
MediaType.create(name="application/json")
MediaType.create(name="text/markdown")
MediaType.create(name="application/vnd.cnr.blob.v0.tar+gzip")
MediaType.create(name="application/vnd.cnr.package-manifest.helm.v0.json")
MediaType.create(name="application/vnd.cnr.package-manifest.kpm.v0.json")
MediaType.create(
name="application/vnd.cnr.package-manifest.docker-compose.v0.json")
MediaType.create(name="application/vnd.cnr.package.kpm.v0.tar+gzip")
MediaType.create(name="application/vnd.cnr.package.helm.v0.tar+gzip")
MediaType.create(
name="application/vnd.cnr.package.docker-compose.v0.tar+gzip")
MediaType.create(name="application/vnd.cnr.manifests.v0.json")
MediaType.create(name="application/vnd.cnr.manifest.list.v0.json")
for media_type in DOCKER_SCHEMA1_CONTENT_TYPES:
MediaType.create(name=media_type)
for media_type in DOCKER_SCHEMA2_CONTENT_TYPES:
MediaType.create(name=media_type)
for media_type in OCI_CONTENT_TYPES:
MediaType.create(name=media_type)
LabelSourceType.create(name="manifest")
LabelSourceType.create(name="api", mutable=True)
LabelSourceType.create(name="internal")
UserPromptKind.create(name="confirm_username")
UserPromptKind.create(name="enter_name")
UserPromptKind.create(name="enter_company")
RepositoryKind.create(name="image")
RepositoryKind.create(name="application")
ApprTagKind.create(name="tag")
ApprTagKind.create(name="release")
ApprTagKind.create(name="channel")
DisableReason.create(name="user_toggled")
DisableReason.create(name="successive_build_failures")
DisableReason.create(name="successive_build_internal_errors")
TagKind.create(name="tag")
def test_blob_mounting(mount_digest, source_repo, username, expect_success,
client, app):
location = ImageStorageLocation.get(name='local_us')
# Store and link some blobs.
digest = 'sha256:' + hashlib.sha256("a").hexdigest()
model.blob.store_blob_record_and_temp_link('devtable', 'simple', digest,
location, 1, 10000000)
digest = 'sha256:' + hashlib.sha256("b").hexdigest()
model.blob.store_blob_record_and_temp_link('devtable', 'complex', digest,
location, 1, 10000000)
digest = 'sha256:' + hashlib.sha256("c").hexdigest()
model.blob.store_blob_record_and_temp_link('public', 'publicrepo', digest,
location, 1, 10000000)
params = {
'repository': 'devtable/building',
'mount': mount_digest,
'from': source_repo,
}
user = model.user.get_user(username)
access = [{
'type': 'repository',
'name': 'devtable/building',
'actions': ['pull', 'push'],
}]
if source_repo.find(username) == 0:
access.append({
'type': 'repository',
'name': source_repo,
'actions': ['pull'],
})
context, subject = build_context_and_subject(
ValidatedAuthContext(user=user))
token = generate_bearer_token(realapp.config['SERVER_HOSTNAME'], subject,
context, access, 600, instance_keys)
headers = {
'Authorization': 'Bearer %s' % token,
}
expected_code = 201 if expect_success else 202
conduct_call(client,
'v2.start_blob_upload',
url_for,
'POST',
params,
expected_code=expected_code,
headers=headers)
if expect_success:
# Ensure the blob now exists under the repo.
model.blob.get_repo_blob_by_digest('devtable', 'building',
mount_digest)
else:
with pytest.raises(model.blob.BlobDoesNotExist):
model.blob.get_repo_blob_by_digest('devtable', 'building',
mount_digest)
def _populate_blob(digest):
location = ImageStorageLocation.get(name='local_us')
store_blob_record_and_temp_link('devtable', 'simple', digest, location, 1,
120)
def _populate_blob(digest):
location = ImageStorageLocation.get(name="local_us")
store_blob_record_and_temp_link("devtable", "simple", digest, location, 1,
120)
def populate_database(minimal=False, with_storage=False):
logger.debug('Populating the DB with test data.')
# Check if the data already exists. If so, we skip. This can happen between calls from the
# "old style" tests and the new py.test's.
try:
User.get(username='******')
logger.debug('DB already populated')
return
except User.DoesNotExist:
pass
# Note: databases set up with "real" schema (via Alembic) will not have these types
# type, so we it here it necessary.
try:
ImageStorageLocation.get(name='local_eu')
ImageStorageLocation.get(name='local_us')
except ImageStorageLocation.DoesNotExist:
ImageStorageLocation.create(name='local_eu')
ImageStorageLocation.create(name='local_us')
try:
NotificationKind.get(name='test_notification')
except NotificationKind.DoesNotExist:
NotificationKind.create(name='test_notification')
new_user_1 = model.user.create_user('devtable', 'password',
'*****@*****.**')
new_user_1.verified = True
new_user_1.stripe_id = TEST_STRIPE_ID
new_user_1.save()
if minimal:
logger.debug(
'Skipping most db population because user requested mininal db')
return
UserRegion.create(user=new_user_1,
location=ImageStorageLocation.get(name='local_us'))
model.release.set_region_release('quay', 'us', 'v0.1.2')
model.user.create_confirm_email_code(new_user_1,
new_email='*****@*****.**')
disabled_user = model.user.create_user('disabled', 'password',
'*****@*****.**')
disabled_user.verified = True
disabled_user.enabled = False
disabled_user.save()
dtrobot = model.user.create_robot('dtrobot', new_user_1)
dtrobot2 = model.user.create_robot('dtrobot2', new_user_1)
new_user_2 = model.user.create_user('public', 'password',
'*****@*****.**')
new_user_2.verified = True
new_user_2.save()
new_user_3 = model.user.create_user('freshuser', 'password',
'*****@*****.**')
new_user_3.verified = True
new_user_3.save()
another_robot = model.user.create_robot('anotherrobot', new_user_3)
new_user_4 = model.user.create_user('randomuser', 'password',
'*****@*****.**')
new_user_4.verified = True
new_user_4.save()
new_user_5 = model.user.create_user('unverified', 'password',
'*****@*****.**')
new_user_5.save()
reader = model.user.create_user('reader', 'password', '*****@*****.**')
reader.verified = True
reader.save()
creatoruser = model.user.create_user('creator', 'password',
'*****@*****.**')
creatoruser.verified = True
creatoruser.save()
outside_org = model.user.create_user('outsideorg', 'password',
'*****@*****.**')
outside_org.verified = True
outside_org.save()
model.notification.create_notification('test_notification',
new_user_1,
metadata={
'some': 'value',
'arr': [1, 2, 3],
'obj': {
'a': 1,
'b': 2
}
})
from_date = datetime.utcnow()
to_date = from_date + timedelta(hours=1)
notification_metadata = {
'from_date': formatdate(calendar.timegm(from_date.utctimetuple())),
'to_date': formatdate(calendar.timegm(to_date.utctimetuple())),
'reason': 'database migration'
}
model.notification.create_notification('maintenance',
new_user_1,
metadata=notification_metadata)
__generate_repository(with_storage, new_user_4, 'randomrepo',
'Random repo repository.', False, [],
(4, [], ['latest', 'prod']))
simple_repo = __generate_repository(with_storage, new_user_1, 'simple',
'Simple repository.', False, [],
(4, [], ['latest', 'prod']))
# Add some labels to the latest tag's manifest.
tag_manifest = model.tag.load_tag_manifest(new_user_1.username, 'simple',
'latest')
first_label = model.label.create_manifest_label(tag_manifest, 'foo', 'bar',
'manifest')
model.label.create_manifest_label(tag_manifest, 'foo', 'baz', 'api')
model.label.create_manifest_label(tag_manifest, 'anotherlabel', '1234',
'internal')
model.label.create_manifest_label(tag_manifest, 'jsonlabel',
'{"hey": "there"}', 'internal')
label_metadata = {
'key': 'foo',
'value': 'bar',
'id': first_label.id,
'manifest_digest': tag_manifest.digest
}
logs_model.log_action('manifest_label_add',
new_user_1.username,
performer=new_user_1,
timestamp=datetime.now(),
metadata=label_metadata,
repository=tag_manifest.tag.repository)
model.blob.initiate_upload(new_user_1.username, simple_repo.name,
str(uuid4()), 'local_us', {})
model.notification.create_repo_notification(simple_repo, 'repo_push',
'quay_notification', {}, {})
__generate_repository(
with_storage, new_user_1, 'sharedtags', 'Shared tags repository',
False, [(new_user_2, 'read'), (dtrobot[0], 'read')],
(2, [(3, [], ['v2.0', 'v2.1', 'v2.2']),
(1, [(1, [(1, [], ['prod', '581a284'])], ['staging', '8423b58']),
(1, [], None)], None)], None))
__generate_repository(with_storage, new_user_1, 'history',
'Historical repository.', False, [],
(4, [(2, [], '#latest'), (3, [], 'latest')], None))
__generate_repository(with_storage, new_user_1, 'complex',
'Complex repository with many branches and tags.',
False, [(new_user_2, 'read'), (dtrobot[0], 'read')],
(2, [(3, [], 'v2.0'),
(1, [(1, [(2, [], ['prod'])], 'staging'),
(1, [], None)], None)], None))
__generate_repository(
with_storage, new_user_1, 'gargantuan', None, False,
[], (2, [(3, [], 'v2.0'),
(1, [(1, [(1, [], ['latest', 'prod'])], 'staging'),
(1, [], None)], None), (20, [], 'v3.0'), (5, [], 'v4.0'),
(1, [(1, [], 'v5.0'), (1, [], 'v6.0')], None)], None))
trusted_repo = __generate_repository(with_storage, new_user_1, 'trusted',
'Trusted repository.', False, [],
(4, [], ['latest', 'prod']))
trusted_repo.trust_enabled = True
trusted_repo.save()
publicrepo = __generate_repository(
with_storage, new_user_2, 'publicrepo',
'Public repository pullable by the world.', True, [],
(10, [], 'latest'))
__generate_repository(with_storage, outside_org, 'coolrepo',
'Some cool repo.', False, [], (5, [], 'latest'))
__generate_repository(with_storage, new_user_1, 'shared',
'Shared repository, another user can write.', False,
[(new_user_2, 'write'),
(reader, 'read')], (5, [], 'latest'))
__generate_repository(with_storage, new_user_1, 'text-full-repo',
'This is a repository for testing text search',
False, [(new_user_2, 'write'),
(reader, 'read')], (5, [], 'latest'))
building = __generate_repository(with_storage, new_user_1, 'building',
'Empty repository which is building.',
False, [(new_user_2, 'write'),
(reader, 'read')], (0, [], None))
new_token = model.token.create_access_token(building, 'write',
'build-worker')
trigger = model.build.create_build_trigger(building,
'github',
'123authtoken',
new_user_1,
pull_robot=dtrobot[0])
trigger.config = json.dumps({
'build_source': 'jakedt/testconnect',
'subdir': '',
'dockerfile_path': 'Dockerfile',
'context': '/',
})
trigger.save()
repo = 'ci.devtable.com:5000/%s/%s' % (building.namespace_user.username,
building.name)
job_config = {
'repository': repo,
'docker_tags': ['latest'],
'build_subdir': '',
'trigger_metadata': {
'commit': '3482adc5822c498e8f7db2e361e8d57b3d77ddd9',
'ref': 'refs/heads/master',
'default_branch': 'master'
}
}
model.repository.star_repository(new_user_1, simple_repo)
record = model.repository.create_email_authorization_for_repo(
new_user_1.username, 'simple', '*****@*****.**')
record.confirmed = True
record.save()
model.repository.create_email_authorization_for_repo(
new_user_1.username, 'simple', '*****@*****.**')
build2 = model.build.create_repository_build(
building, new_token, job_config,
'68daeebd-a5b9-457f-80a0-4363b882f8ea', 'build-name', trigger)
build2.uuid = 'deadpork-dead-pork-dead-porkdeadpork'
build2.save()
build3 = model.build.create_repository_build(
building, new_token, job_config,
'f49d07f9-93da-474d-ad5f-c852107c3892', 'build-name', trigger)
build3.uuid = 'deadduck-dead-duck-dead-duckdeadduck'
build3.save()
build1 = model.build.create_repository_build(
building, new_token, job_config, '701dcc3724fb4f2ea6c31400528343cd',
'build-name', trigger)
build1.uuid = 'deadbeef-dead-beef-dead-beefdeadbeef'
build1.save()
org = model.organization.create_organization('buynlarge',
'*****@*****.**',
new_user_1)
org.stripe_id = TEST_STRIPE_ID
org.save()
liborg = model.organization.create_organization(
'library', '*****@*****.**', new_user_1)
liborg.save()
titiorg = model.organization.create_organization('titi',
'*****@*****.**',
new_user_1)
titiorg.save()
thirdorg = model.organization.create_organization(
'sellnsmall', '*****@*****.**', new_user_1)
thirdorg.save()
model.user.create_robot('coolrobot', org)
oauth_app_1 = model.oauth.create_application(
org,
'Some Test App',
'http://localhost:8000',
'http://localhost:8000/o2c.html',
client_id='deadbeef')
model.oauth.create_application(
org,
'Some Other Test App',
'http://quay.io',
'http://localhost:8000/o2c.html',
client_id='deadpork',
description='This is another test application')
model.oauth.create_access_token_for_testing(new_user_1,
'deadbeef',
'repo:admin',
access_token='%s%s' %
('b' * 40, 'c' * 40))
oauth_credential = Credential.from_string('dswfhasdf1')
OAuthAuthorizationCode.create(application=oauth_app_1,
code='Z932odswfhasdf1',
scope='repo:admin',
data='{"somejson": "goeshere"}',
code_name='Z932odswfhasdf1Z932o',
code_credential=oauth_credential)
model.user.create_robot('neworgrobot', org)
ownerbot = model.user.create_robot('ownerbot', org)[0]
creatorbot = model.user.create_robot('creatorbot', org)[0]
owners = model.team.get_organization_team('buynlarge', 'owners')
owners.description = 'Owners have unfetterd access across the entire org.'
owners.save()
org_repo = __generate_repository(with_storage, org, 'orgrepo',
'Repository owned by an org.', False,
[(outside_org, 'read')],
(4, [], ['latest', 'prod']))
__generate_repository(with_storage, org, 'anotherorgrepo',
'Another repository owned by an org.', False, [],
(4, [], ['latest', 'prod']))
creators = model.team.create_team('creators', org, 'creator',
'Creators of orgrepo.')
reader_team = model.team.create_team('readers', org, 'member',
'Readers of orgrepo.')
model.team.add_or_invite_to_team(new_user_1, reader_team, outside_org)
model.permission.set_team_repo_permission(reader_team.name,
org_repo.namespace_user.username,
org_repo.name, 'read')
model.team.add_user_to_team(new_user_2, reader_team)
model.team.add_user_to_team(reader, reader_team)
model.team.add_user_to_team(ownerbot, owners)
model.team.add_user_to_team(creatorbot, creators)
model.team.add_user_to_team(creatoruser, creators)
sell_owners = model.team.get_organization_team('sellnsmall', 'owners')
sell_owners.description = 'Owners have unfettered access across the entire org.'
sell_owners.save()
model.team.add_user_to_team(new_user_4, sell_owners)
sync_config = {
'group_dn': 'cn=Test-Group,ou=Users',
'group_id': 'somegroupid'
}
synced_team = model.team.create_team('synced', org, 'member',
'Some synced team.')
model.team.set_team_syncing(synced_team, 'ldap', sync_config)
another_synced_team = model.team.create_team('synced', thirdorg, 'member',
'Some synced team.')
model.team.set_team_syncing(another_synced_team, 'ldap',
{'group_dn': 'cn=Test-Group,ou=Users'})
__generate_repository(with_storage, new_user_1, 'superwide', None,
False, [], [(10, [], 'latest2'), (2, [], 'latest3'),
(2, [(1, [], 'latest11'),
(2, [], 'latest12')], 'latest4'),
(2, [], 'latest5'), (2, [], 'latest6'),
(2, [], 'latest7'), (2, [], 'latest8'),
(2, [], 'latest9'), (2, [], 'latest10'),
(2, [], 'latest13'), (2, [], 'latest14'),
(2, [], 'latest15'), (2, [], 'latest16'),
(2, [], 'latest17'),
(2, [], 'latest18')])
mirror_repo = __generate_repository(with_storage, new_user_1, 'mirrored',
'Mirrored repository.', False,
[(dtrobot[0], 'write'),
(dtrobot2[0], 'write')],
(4, [], ['latest', 'prod']))
mirror_rule = model.repo_mirror.create_mirroring_rule(
mirror_repo, ['latest', '3.3*'])
mirror_args = (mirror_repo, mirror_rule, dtrobot[0], 'quay.io/coreos/etcd',
60 * 60 * 24)
mirror_kwargs = {
'external_registry_username': '******',
'external_registry_password': '******',
'external_registry_config': {},
'is_enabled': True,
'sync_start_date': datetime.utcnow()
}
mirror = model.repo_mirror.enable_mirroring_for_repository(
*mirror_args, **mirror_kwargs)
read_only_repo = __generate_repository(with_storage, new_user_1,
'readonly', 'Read-Only Repo.',
False, [],
(4, [], ['latest', 'prod']))
read_only_repo.state = RepositoryState.READ_ONLY
read_only_repo.save()
model.permission.add_prototype_permission(org,
'read',
activating_user=new_user_1,
delegate_user=new_user_2)
model.permission.add_prototype_permission(org,
'read',
activating_user=new_user_1,
delegate_team=reader_team)
model.permission.add_prototype_permission(org,
'write',
activating_user=new_user_2,
delegate_user=new_user_1)
today = datetime.today()
week_ago = today - timedelta(6)
six_ago = today - timedelta(5)
four_ago = today - timedelta(4)
yesterday = datetime.combine(date.today(),
datetime.min.time()) - timedelta(hours=6)
__generate_service_key('kid1', 'somesamplekey', new_user_1, today,
ServiceKeyApprovalType.SUPERUSER)
__generate_service_key('kid2', 'someexpiringkey', new_user_1, week_ago,
ServiceKeyApprovalType.SUPERUSER,
today + timedelta(days=14))
__generate_service_key('kid3', 'unapprovedkey', new_user_1, today, None)
__generate_service_key(
'kid4',
'autorotatingkey',
new_user_1,
six_ago,
ServiceKeyApprovalType.KEY_ROTATION,
today + timedelta(days=1),
rotation_duration=timedelta(hours=12).total_seconds())
__generate_service_key('kid5',
'key for another service',
new_user_1,
today,
ServiceKeyApprovalType.SUPERUSER,
today + timedelta(days=14),
service='different_sample_service')
__generate_service_key('kid6', 'someexpiredkey', new_user_1, week_ago,
ServiceKeyApprovalType.SUPERUSER,
today - timedelta(days=1))
__generate_service_key('kid7', 'somewayexpiredkey', new_user_1, week_ago,
ServiceKeyApprovalType.SUPERUSER,
today - timedelta(days=30))
# Add the test pull key as pre-approved for local and unittest registry testing.
# Note: this must match the private key found in the local/test config.
_TEST_JWK = {
'e':
'AQAB',
'kty':
'RSA',
'n':
'yqdQgnelhAPMSeyH0kr3UGePK9oFOmNfwD0Ymnh7YYXr21VHWwyM2eVW3cnLd9KXywDFtGSe9oFDbnOuMCdUowdkBcaHju-isbv5KEbNSoy_T2Rip-6L0cY63YzcMJzv1nEYztYXS8wz76pSK81BKBCLapqOCmcPeCvV9yaoFZYvZEsXCl5jjXN3iujSzSF5Z6PpNFlJWTErMT2Z4QfbDKX2Nw6vJN6JnGpTNHZvgvcyNX8vkSgVpQ8DFnFkBEx54PvRV5KpHAq6AsJxKONMo11idQS2PfCNpa2hvz9O6UZe-eIX8jPo5NW8TuGZJumbdPT_nxTDLfCqfiZboeI0Pw'
}
key = model.service_keys.create_service_key('test_service_key',
'test_service_key', 'quay',
_TEST_JWK, {}, None)
model.service_keys.approve_service_key(
key.kid,
ServiceKeyApprovalType.SUPERUSER,
notes='Test service key for local/test registry testing')
# Add an app specific token.
token = model.appspecifictoken.create_token(new_user_1, 'some app')
token.token_name = 'a' * 60
token.token_secret = 'b' * 60
token.save()
logs_model.log_action('org_create_team',
org.username,
performer=new_user_1,
timestamp=week_ago,
metadata={'team': 'readers'})
logs_model.log_action('org_set_team_role',
org.username,
performer=new_user_1,
timestamp=week_ago,
metadata={
'team': 'readers',
'role': 'read'
})
logs_model.log_action('create_repo',
org.username,
performer=new_user_1,
repository=org_repo,
timestamp=week_ago,
metadata={
'namespace': org.username,
'repo': 'orgrepo'
})
logs_model.log_action('change_repo_permission',
org.username,
performer=new_user_2,
repository=org_repo,
timestamp=six_ago,
metadata={
'username': new_user_1.username,
'repo': 'orgrepo',
'role': 'admin'
})
logs_model.log_action('change_repo_permission',
org.username,
performer=new_user_1,
repository=org_repo,
timestamp=six_ago,
metadata={
'username': new_user_2.username,
'repo': 'orgrepo',
'role': 'read'
})
logs_model.log_action('add_repo_accesstoken',
org.username,
performer=new_user_1,
repository=org_repo,
timestamp=four_ago,
metadata={
'repo': 'orgrepo',
'token': 'deploytoken'
})
logs_model.log_action('push_repo',
org.username,
performer=new_user_2,
repository=org_repo,
timestamp=today,
metadata={
'username': new_user_2.username,
'repo': 'orgrepo'
})
logs_model.log_action('pull_repo',
org.username,
performer=new_user_2,
repository=org_repo,
timestamp=today,
metadata={
'username': new_user_2.username,
'repo': 'orgrepo'
})
logs_model.log_action('pull_repo',
org.username,
repository=org_repo,
timestamp=today,
metadata={
'token': 'sometoken',
'token_code': 'somecode',
'repo': 'orgrepo'
})
logs_model.log_action('delete_tag',
org.username,
performer=new_user_2,
repository=org_repo,
timestamp=today,
metadata={
'username': new_user_2.username,
'repo': 'orgrepo',
'tag': 'sometag'
})
logs_model.log_action('pull_repo',
org.username,
repository=org_repo,
timestamp=today,
metadata={
'token_code': 'somecode',
'repo': 'orgrepo'
})
logs_model.log_action('pull_repo',
new_user_2.username,
repository=publicrepo,
timestamp=yesterday,
metadata={
'token_code': 'somecode',
'repo': 'publicrepo'
})
logs_model.log_action('build_dockerfile',
new_user_1.username,
repository=building,
timestamp=today,
metadata={
'repo': 'building',
'namespace': new_user_1.username,
'trigger_id': trigger.uuid,
'config': json.loads(trigger.config),
'service': trigger.service.name
})
model.message.create([{
'content': 'We love you, Quay customers!',
'severity': 'info',
'media_type': 'text/plain'
}])
model.message.create([{
'content': 'This is a **development** install of Quay',
'severity': 'warning',
'media_type': 'text/markdown'
}])
fake_queue = WorkQueue('fakequeue', tf)
fake_queue.put(['canonical', 'job', 'name'], '{}')
model.user.create_user_prompt(new_user_4, 'confirm_username')
while True:
to_count = model.repositoryactioncount.find_uncounted_repository()
if not to_count:
break
model.repositoryactioncount.count_repository_actions(to_count)
model.repositoryactioncount.update_repository_score(to_count)
def populate_database(minimal=False):
logger.debug("Populating the DB with test data.")
# Check if the data already exists. If so, we skip. This can happen between calls from the
# "old style" tests and the new py.test's.
try:
User.get(username="******")
logger.debug("DB already populated")
return
except User.DoesNotExist:
pass
# Note: databases set up with "real" schema (via Alembic) will not have these types
# type, so we it here it necessary.
try:
ImageStorageLocation.get(name="local_eu")
ImageStorageLocation.get(name="local_us")
except ImageStorageLocation.DoesNotExist:
ImageStorageLocation.create(name="local_eu")
ImageStorageLocation.create(name="local_us")
try:
NotificationKind.get(name="test_notification")
except NotificationKind.DoesNotExist:
NotificationKind.create(name="test_notification")
new_user_1 = model.user.create_user("devtable", "password",
"*****@*****.**")
new_user_1.verified = True
new_user_1.stripe_id = TEST_STRIPE_ID
new_user_1.save()
if minimal:
logger.debug(
"Skipping most db population because user requested mininal db")
return
UserRegion.create(user=new_user_1,
location=ImageStorageLocation.get(name="local_us"))
model.release.set_region_release("quay", "us", "v0.1.2")
model.user.create_confirm_email_code(new_user_1,
new_email="*****@*****.**")
disabled_user = model.user.create_user("disabled", "password",
"*****@*****.**")
disabled_user.verified = True
disabled_user.enabled = False
disabled_user.save()
dtrobot = model.user.create_robot("dtrobot", new_user_1)
dtrobot2 = model.user.create_robot("dtrobot2", new_user_1)
new_user_2 = model.user.create_user("public", "password",
"*****@*****.**")
new_user_2.verified = True
new_user_2.save()
new_user_3 = model.user.create_user("freshuser", "password",
"*****@*****.**")
new_user_3.verified = True
new_user_3.save()
another_robot = model.user.create_robot("anotherrobot", new_user_3)
new_user_4 = model.user.create_user("randomuser", "password",
"*****@*****.**")
new_user_4.verified = True
new_user_4.save()
new_user_5 = model.user.create_user("unverified", "password",
"*****@*****.**")
new_user_5.save()
reader = model.user.create_user("reader", "password", "*****@*****.**")
reader.verified = True
reader.save()
creatoruser = model.user.create_user("creator", "password",
"*****@*****.**")
creatoruser.verified = True
creatoruser.save()
outside_org = model.user.create_user("outsideorg", "password",
"*****@*****.**")
outside_org.verified = True
outside_org.save()
model.notification.create_notification(
"test_notification",
new_user_1,
metadata={
"some": "value",
"arr": [1, 2, 3],
"obj": {
"a": 1,
"b": 2
}
},
)
from_date = datetime.utcnow()
to_date = from_date + timedelta(hours=1)
notification_metadata = {
"from_date": formatdate(calendar.timegm(from_date.utctimetuple())),
"to_date": formatdate(calendar.timegm(to_date.utctimetuple())),
"reason": "database migration",
}
model.notification.create_notification("maintenance",
new_user_1,
metadata=notification_metadata)
__generate_repository(
new_user_4,
"randomrepo",
"Random repo repository.",
False,
[],
(4, [], ["latest", "prod"]),
)
simple_repo = __generate_repository(
new_user_1,
"simple",
"Simple repository.",
False,
[],
(4, [], ["latest", "prod"]),
)
# Add some labels to the latest tag's manifest.
repo_ref = RepositoryReference.for_repo_obj(simple_repo)
tag = registry_model.get_repo_tag(repo_ref, "latest")
manifest = registry_model.get_manifest_for_tag(tag)
assert manifest
first_label = registry_model.create_manifest_label(manifest, "foo", "bar",
"manifest")
registry_model.create_manifest_label(manifest, "foo", "baz", "api")
registry_model.create_manifest_label(manifest, "anotherlabel", "1234",
"internal")
registry_model.create_manifest_label(manifest, "jsonlabel",
'{"hey": "there"}', "internal",
"application/json")
label_metadata = {
"key": "foo",
"value": "bar",
"id": first_label._db_id,
"manifest_digest": manifest.digest,
}
logs_model.log_action(
"manifest_label_add",
new_user_1.username,
performer=new_user_1,
timestamp=datetime.now(),
metadata=label_metadata,
repository=simple_repo,
)
model.blob.initiate_upload(new_user_1.username, simple_repo.name,
str(uuid4()), "local_us", {})
model.notification.create_repo_notification(simple_repo, "repo_push",
"quay_notification", {}, {})
__generate_repository(
new_user_1,
"sharedtags",
"Shared tags repository",
False,
[(new_user_2, "read"), (dtrobot[0], "read")],
(
2,
[
(3, [], ["v2.0", "v2.1", "v2.2"]),
(
1,
[(1, [(1, [], ["prod", "581a284"])
], ["staging", "8423b58"]), (1, [], None)],
None,
),
],
None,
),
)
__generate_repository(
new_user_1,
"history",
"Historical repository.",
False,
[],
(4, [(2, [], "#latest"), (3, [], "latest")], None),
)
__generate_repository(
new_user_1,
"complex",
"Complex repository with many branches and tags.",
False,
[(new_user_2, "read"), (dtrobot[0], "read")],
(
2,
[(3, [], "v2.0"),
(1, [(1, [(2, [], ["prod"])], "staging"), (1, [], None)], None)],
None,
),
)
__generate_repository(
new_user_1,
"gargantuan",
None,
False,
[],
(
2,
[
(3, [], "v2.0"),
(1, [(1, [(1, [], ["latest", "prod"])], "staging"),
(1, [], None)], None),
(20, [], "v3.0"),
(5, [], "v4.0"),
(1, [(1, [], "v5.0"), (1, [], "v6.0")], None),
],
None,
),
)
trusted_repo = __generate_repository(
new_user_1,
"trusted",
"Trusted repository.",
False,
[],
(4, [], ["latest", "prod"]),
)
trusted_repo.trust_enabled = True
trusted_repo.save()
publicrepo = __generate_repository(
new_user_2,
"publicrepo",
"Public repository pullable by the world.",
True,
[],
(10, [], "latest"),
)
__generate_repository(outside_org, "coolrepo", "Some cool repo.", False,
[], (5, [], "latest"))
__generate_repository(
new_user_1,
"shared",
"Shared repository, another user can write.",
False,
[(new_user_2, "write"), (reader, "read")],
(5, [], "latest"),
)
__generate_repository(
new_user_1,
"text-full-repo",
"This is a repository for testing text search",
False,
[(new_user_2, "write"), (reader, "read")],
(5, [], "latest"),
)
building = __generate_repository(
new_user_1,
"building",
"Empty repository which is building.",
False,
[(new_user_2, "write"), (reader, "read")],
(0, [], None),
)
new_token = model.token.create_access_token(building, "write",
"build-worker")
trigger = model.build.create_build_trigger(building,
"github",
"123authtoken",
new_user_1,
pull_robot=dtrobot[0])
trigger.config = json.dumps({
"build_source": "jakedt/testconnect",
"subdir": "",
"dockerfile_path": "Dockerfile",
"context": "/",
})
trigger.save()
repo = "ci.devtable.com:5000/%s/%s" % (building.namespace_user.username,
building.name)
job_config = {
"repository": repo,
"docker_tags": ["latest"],
"build_subdir": "",
"trigger_metadata": {
"commit": "3482adc5822c498e8f7db2e361e8d57b3d77ddd9",
"ref": "refs/heads/master",
"default_branch": "master",
},
}
model.repository.star_repository(new_user_1, simple_repo)
record = model.repository.create_email_authorization_for_repo(
new_user_1.username, "simple", "*****@*****.**")
record.confirmed = True
record.save()
model.repository.create_email_authorization_for_repo(
new_user_1.username, "simple", "*****@*****.**")
build2 = model.build.create_repository_build(
building,
new_token,
job_config,
"68daeebd-a5b9-457f-80a0-4363b882f8ea",
"build-name",
trigger,
)
build2.uuid = "deadpork-dead-pork-dead-porkdeadpork"
build2.save()
build3 = model.build.create_repository_build(
building,
new_token,
job_config,
"f49d07f9-93da-474d-ad5f-c852107c3892",
"build-name",
trigger,
)
build3.uuid = "deadduck-dead-duck-dead-duckdeadduck"
build3.save()
build1 = model.build.create_repository_build(
building, new_token, job_config, "701dcc3724fb4f2ea6c31400528343cd",
"build-name", trigger)
build1.uuid = "deadbeef-dead-beef-dead-beefdeadbeef"
build1.save()
org = model.organization.create_organization("buynlarge",
"*****@*****.**",
new_user_1)
org.stripe_id = TEST_STRIPE_ID
org.save()
liborg = model.organization.create_organization(
"library", "*****@*****.**", new_user_1)
liborg.save()
titiorg = model.organization.create_organization("titi",
"*****@*****.**",
new_user_1)
titiorg.save()
thirdorg = model.organization.create_organization(
"sellnsmall", "*****@*****.**", new_user_1)
thirdorg.save()
model.user.create_robot("coolrobot", org)
oauth_app_1 = model.oauth.create_application(
org,
"Some Test App",
"http://localhost:8000",
"http://localhost:8000/o2c.html",
client_id="deadbeef",
)
model.oauth.create_application(
org,
"Some Other Test App",
"http://quay.io",
"http://localhost:8000/o2c.html",
client_id="deadpork",
description="This is another test application",
)
model.oauth.create_user_access_token(new_user_1,
"deadbeef",
"repo:admin",
access_token="%s%s" %
("b" * 40, "c" * 40))
oauth_credential = Credential.from_string("dswfhasdf1")
OAuthAuthorizationCode.create(
application=oauth_app_1,
code="Z932odswfhasdf1",
scope="repo:admin",
data='{"somejson": "goeshere"}',
code_name="Z932odswfhasdf1Z932o",
code_credential=oauth_credential,
)
model.user.create_robot("neworgrobot", org)
ownerbot = model.user.create_robot("ownerbot", org)[0]
creatorbot = model.user.create_robot("creatorbot", org)[0]
owners = model.team.get_organization_team("buynlarge", "owners")
owners.description = "Owners have unfetterd access across the entire org."
owners.save()
org_repo = __generate_repository(
org,
"orgrepo",
"Repository owned by an org.",
False,
[(outside_org, "read")],
(4, [], ["latest", "prod"]),
)
__generate_repository(
org,
"anotherorgrepo",
"Another repository owned by an org.",
False,
[],
(4, [], ["latest", "prod"]),
)
creators = model.team.create_team("creators", org, "creator",
"Creators of orgrepo.")
reader_team = model.team.create_team("readers", org, "member",
"Readers of orgrepo.")
model.team.add_or_invite_to_team(new_user_1, reader_team, outside_org)
model.permission.set_team_repo_permission(reader_team.name,
org_repo.namespace_user.username,
org_repo.name, "read")
model.team.add_user_to_team(new_user_2, reader_team)
model.team.add_user_to_team(reader, reader_team)
model.team.add_user_to_team(ownerbot, owners)
model.team.add_user_to_team(creatorbot, creators)
model.team.add_user_to_team(creatoruser, creators)
sell_owners = model.team.get_organization_team("sellnsmall", "owners")
sell_owners.description = "Owners have unfettered access across the entire org."
sell_owners.save()
model.team.add_user_to_team(new_user_4, sell_owners)
sync_config = {
"group_dn": "cn=Test-Group,ou=Users",
"group_id": "somegroupid"
}
synced_team = model.team.create_team("synced", org, "member",
"Some synced team.")
model.team.set_team_syncing(synced_team, "ldap", sync_config)
another_synced_team = model.team.create_team("synced", thirdorg, "member",
"Some synced team.")
model.team.set_team_syncing(another_synced_team, "ldap",
{"group_dn": "cn=Test-Group,ou=Users"})
__generate_repository(
new_user_1,
"superwide",
None,
False,
[],
[
(10, [], "latest2"),
(2, [], "latest3"),
(2, [(1, [], "latest11"), (2, [], "latest12")], "latest4"),
(2, [], "latest5"),
(2, [], "latest6"),
(2, [], "latest7"),
(2, [], "latest8"),
(2, [], "latest9"),
(2, [], "latest10"),
(2, [], "latest13"),
(2, [], "latest14"),
(2, [], "latest15"),
(2, [], "latest16"),
(2, [], "latest17"),
(2, [], "latest18"),
],
)
mirror_repo = __generate_repository(
new_user_1,
"mirrored",
"Mirrored repository.",
False,
[(dtrobot[0], "write"), (dtrobot2[0], "write")],
(4, [], ["latest", "prod"]),
)
mirror_rule = model.repo_mirror.create_mirroring_rule(
mirror_repo, ["latest", "3.3*"])
mirror_args = (mirror_repo, mirror_rule, dtrobot[0], "quay.io/coreos/etcd",
60 * 60 * 24)
mirror_kwargs = {
"external_registry_username": "******",
"external_registry_password": "******",
"external_registry_config": {},
"is_enabled": True,
"sync_start_date": datetime.utcnow(),
}
mirror = model.repo_mirror.enable_mirroring_for_repository(
*mirror_args, **mirror_kwargs)
read_only_repo = __generate_repository(
new_user_1,
"readonly",
"Read-Only Repo.",
False,
[],
(4, [], ["latest", "prod"]),
)
read_only_repo.state = RepositoryState.READ_ONLY
read_only_repo.save()
model.permission.add_prototype_permission(org,
"read",
activating_user=new_user_1,
delegate_user=new_user_2)
model.permission.add_prototype_permission(org,
"read",
activating_user=new_user_1,
delegate_team=reader_team)
model.permission.add_prototype_permission(org,
"write",
activating_user=new_user_2,
delegate_user=new_user_1)
today = datetime.today()
week_ago = today - timedelta(6)
six_ago = today - timedelta(5)
four_ago = today - timedelta(4)
yesterday = datetime.combine(date.today(),
datetime.min.time()) - timedelta(hours=6)
__generate_service_key("kid1", "somesamplekey", new_user_1, today,
ServiceKeyApprovalType.SUPERUSER)
__generate_service_key(
"kid2",
"someexpiringkey",
new_user_1,
week_ago,
ServiceKeyApprovalType.SUPERUSER,
today + timedelta(days=14),
)
__generate_service_key("kid3", "unapprovedkey", new_user_1, today, None)
__generate_service_key(
"kid4",
"autorotatingkey",
new_user_1,
six_ago,
ServiceKeyApprovalType.KEY_ROTATION,
today + timedelta(days=1),
rotation_duration=timedelta(hours=12).total_seconds(),
)
__generate_service_key(
"kid5",
"key for another service",
new_user_1,
today,
ServiceKeyApprovalType.SUPERUSER,
today + timedelta(days=14),
service="different_sample_service",
)
__generate_service_key(
"kid6",
"someexpiredkey",
new_user_1,
week_ago,
ServiceKeyApprovalType.SUPERUSER,
today - timedelta(days=1),
)
__generate_service_key(
"kid7",
"somewayexpiredkey",
new_user_1,
week_ago,
ServiceKeyApprovalType.SUPERUSER,
today - timedelta(days=30),
)
# Add the test pull key as pre-approved for local and unittest registry testing.
# Note: this must match the private key found in the local/test config.
_TEST_JWK = {
"e":
"AQAB",
"kty":
"RSA",
"n":
"yqdQgnelhAPMSeyH0kr3UGePK9oFOmNfwD0Ymnh7YYXr21VHWwyM2eVW3cnLd9KXywDFtGSe9oFDbnOuMCdUowdkBcaHju-isbv5KEbNSoy_T2Rip-6L0cY63YzcMJzv1nEYztYXS8wz76pSK81BKBCLapqOCmcPeCvV9yaoFZYvZEsXCl5jjXN3iujSzSF5Z6PpNFlJWTErMT2Z4QfbDKX2Nw6vJN6JnGpTNHZvgvcyNX8vkSgVpQ8DFnFkBEx54PvRV5KpHAq6AsJxKONMo11idQS2PfCNpa2hvz9O6UZe-eIX8jPo5NW8TuGZJumbdPT_nxTDLfCqfiZboeI0Pw",
}
key = model.service_keys.create_service_key("test_service_key",
"test_service_key", "quay",
_TEST_JWK, {}, None)
model.service_keys.approve_service_key(
key.kid,
ServiceKeyApprovalType.SUPERUSER,
notes="Test service key for local/test registry testing",
)
# Add an app specific token.
token = model.appspecifictoken.create_token(new_user_1, "some app")
token.token_name = "a" * 60
token.token_secret = "b" * 60
token.save()
logs_model.log_action(
"org_create_team",
org.username,
performer=new_user_1,
timestamp=week_ago,
metadata={"team": "readers"},
)
logs_model.log_action(
"org_set_team_role",
org.username,
performer=new_user_1,
timestamp=week_ago,
metadata={
"team": "readers",
"role": "read"
},
)
logs_model.log_action(
"create_repo",
org.username,
performer=new_user_1,
repository=org_repo,
timestamp=week_ago,
metadata={
"namespace": org.username,
"repo": "orgrepo"
},
)
logs_model.log_action(
"change_repo_permission",
org.username,
performer=new_user_2,
repository=org_repo,
timestamp=six_ago,
metadata={
"username": new_user_1.username,
"repo": "orgrepo",
"role": "admin"
},
)
logs_model.log_action(
"change_repo_permission",
org.username,
performer=new_user_1,
repository=org_repo,
timestamp=six_ago,
metadata={
"username": new_user_2.username,
"repo": "orgrepo",
"role": "read"
},
)
logs_model.log_action(
"add_repo_accesstoken",
org.username,
performer=new_user_1,
repository=org_repo,
timestamp=four_ago,
metadata={
"repo": "orgrepo",
"token": "deploytoken"
},
)
logs_model.log_action(
"push_repo",
org.username,
performer=new_user_2,
repository=org_repo,
timestamp=today,
metadata={
"username": new_user_2.username,
"repo": "orgrepo"
},
)
logs_model.log_action(
"pull_repo",
org.username,
performer=new_user_2,
repository=org_repo,
timestamp=today,
metadata={
"username": new_user_2.username,
"repo": "orgrepo"
},
)
logs_model.log_action(
"pull_repo",
org.username,
repository=org_repo,
timestamp=today,
metadata={
"token": "sometoken",
"token_code": "somecode",
"repo": "orgrepo"
},
)
logs_model.log_action(
"delete_tag",
org.username,
performer=new_user_2,
repository=org_repo,
timestamp=today,
metadata={
"username": new_user_2.username,
"repo": "orgrepo",
"tag": "sometag"
},
)
logs_model.log_action(
"pull_repo",
org.username,
repository=org_repo,
timestamp=today,
metadata={
"token_code": "somecode",
"repo": "orgrepo"
},
)
logs_model.log_action(
"pull_repo",
new_user_2.username,
repository=publicrepo,
timestamp=yesterday,
metadata={
"token_code": "somecode",
"repo": "publicrepo"
},
)
logs_model.log_action(
"build_dockerfile",
new_user_1.username,
repository=building,
timestamp=today,
metadata={
"repo": "building",
"namespace": new_user_1.username,
"trigger_id": trigger.uuid,
"config": json.loads(trigger.config),
"service": trigger.service.name,
},
)
model.message.create([{
"content": "We love you, Quay customers!",
"severity": "info",
"media_type": "text/plain",
}])
model.message.create([{
"content": "This is a **development** install of Quay",
"severity": "warning",
"media_type": "text/markdown",
}])
fake_queue = WorkQueue("fakequeue", tf)
fake_queue.put(["canonical", "job", "name"], "{}")
model.user.create_user_prompt(new_user_4, "confirm_username")
for to_count in Repository.select():
model.repositoryactioncount.count_repository_actions(
to_count, datetime.utcnow())
model.repositoryactioncount.update_repository_score(to_count)
def test_blob_mounting(
mount_digest, source_repo, username, include_from_param, expected_code, client, app
):
location = ImageStorageLocation.get(name="local_us")
# Store and link some blobs.
digest = "sha256:" + hashlib.sha256(b"a").hexdigest()
model.blob.store_blob_record_and_temp_link("devtable", "simple", digest, location, 1, 10000000)
digest = "sha256:" + hashlib.sha256(b"b").hexdigest()
model.blob.store_blob_record_and_temp_link("devtable", "complex", digest, location, 1, 10000000)
digest = "sha256:" + hashlib.sha256(b"c").hexdigest()
model.blob.store_blob_record_and_temp_link(
"public", "publicrepo", digest, location, 1, 10000000
)
params = {
"repository": "devtable/building",
"mount": mount_digest,
}
if include_from_param:
params["from"] = source_repo
user = model.user.get_user(username)
access = [
{
"type": "repository",
"name": "devtable/building",
"actions": ["pull", "push"],
}
]
if source_repo.find(username) == 0:
access.append(
{
"type": "repository",
"name": source_repo,
"actions": ["pull"],
}
)
context, subject = build_context_and_subject(ValidatedAuthContext(user=user))
token = generate_bearer_token(
realapp.config["SERVER_HOSTNAME"], subject, context, access, 600, instance_keys
)
headers = {
"Authorization": "Bearer %s" % token.decode("ascii"),
}
conduct_call(
client,
"v2.start_blob_upload",
url_for,
"POST",
params,
expected_code=expected_code,
headers=headers,
)
repository = model.repository.get_repository("devtable", "building")
if expected_code == 201:
# Ensure the blob now exists under the repo.
assert model.oci.blob.get_repository_blob_by_digest(repository, mount_digest)
else:
assert model.oci.blob.get_repository_blob_by_digest(repository, mount_digest) is None
def initialize_database():
db.create_tables(all_models)
Role.create(name='admin')
Role.create(name='write')
Role.create(name='read')
TeamRole.create(name='admin')
TeamRole.create(name='creator')
TeamRole.create(name='member')
Visibility.create(name='public')
Visibility.create(name='private')
LoginService.create(name='google')
LoginService.create(name='github')
LoginService.create(name='quayrobot')
LoginService.create(name='ldap')
LoginService.create(name='jwtauthn')
LoginService.create(name='keystone')
LoginService.create(name='dex')
LoginService.create(name='oidc')
BuildTriggerService.create(name='github')
BuildTriggerService.create(name='custom-git')
BuildTriggerService.create(name='bitbucket')
BuildTriggerService.create(name='gitlab')
AccessTokenKind.create(name='build-worker')
AccessTokenKind.create(name='pushpull-token')
LogEntryKind.create(name='account_change_plan')
LogEntryKind.create(name='account_change_cc')
LogEntryKind.create(name='account_change_password')
LogEntryKind.create(name='account_convert')
LogEntryKind.create(name='create_robot')
LogEntryKind.create(name='delete_robot')
LogEntryKind.create(name='create_repo')
LogEntryKind.create(name='push_repo')
LogEntryKind.create(name='pull_repo')
LogEntryKind.create(name='delete_repo')
LogEntryKind.create(name='create_tag')
LogEntryKind.create(name='move_tag')
LogEntryKind.create(name='delete_tag')
LogEntryKind.create(name='revert_tag')
LogEntryKind.create(name='add_repo_permission')
LogEntryKind.create(name='change_repo_permission')
LogEntryKind.create(name='delete_repo_permission')
LogEntryKind.create(name='change_repo_visibility')
LogEntryKind.create(name='change_repo_trust')
LogEntryKind.create(name='add_repo_accesstoken')
LogEntryKind.create(name='delete_repo_accesstoken')
LogEntryKind.create(name='set_repo_description')
LogEntryKind.create(name='change_repo_state')
LogEntryKind.create(name='build_dockerfile')
LogEntryKind.create(name='org_create_team')
LogEntryKind.create(name='org_delete_team')
LogEntryKind.create(name='org_invite_team_member')
LogEntryKind.create(name='org_delete_team_member_invite')
LogEntryKind.create(name='org_add_team_member')
LogEntryKind.create(name='org_team_member_invite_accepted')
LogEntryKind.create(name='org_team_member_invite_declined')
LogEntryKind.create(name='org_remove_team_member')
LogEntryKind.create(name='org_set_team_description')
LogEntryKind.create(name='org_set_team_role')
LogEntryKind.create(name='create_prototype_permission')
LogEntryKind.create(name='modify_prototype_permission')
LogEntryKind.create(name='delete_prototype_permission')
LogEntryKind.create(name='setup_repo_trigger')
LogEntryKind.create(name='delete_repo_trigger')
LogEntryKind.create(name='create_application')
LogEntryKind.create(name='update_application')
LogEntryKind.create(name='delete_application')
LogEntryKind.create(name='reset_application_client_secret')
# Note: These next two are deprecated.
LogEntryKind.create(name='add_repo_webhook')
LogEntryKind.create(name='delete_repo_webhook')
LogEntryKind.create(name='add_repo_notification')
LogEntryKind.create(name='delete_repo_notification')
LogEntryKind.create(name='reset_repo_notification')
LogEntryKind.create(name='regenerate_robot_token')
LogEntryKind.create(name='repo_verb')
LogEntryKind.create(name='repo_mirror_enabled')
LogEntryKind.create(name='repo_mirror_disabled')
LogEntryKind.create(name='repo_mirror_config_changed')
LogEntryKind.create(name='repo_mirror_sync_started')
LogEntryKind.create(name='repo_mirror_sync_failed')
LogEntryKind.create(name='repo_mirror_sync_success')
LogEntryKind.create(name='repo_mirror_sync_now_requested')
LogEntryKind.create(name='repo_mirror_sync_tag_success')
LogEntryKind.create(name='repo_mirror_sync_tag_failed')
LogEntryKind.create(name='repo_mirror_sync_test_success')
LogEntryKind.create(name='repo_mirror_sync_test_failed')
LogEntryKind.create(name='repo_mirror_sync_test_started')
LogEntryKind.create(name='service_key_create')
LogEntryKind.create(name='service_key_approve')
LogEntryKind.create(name='service_key_delete')
LogEntryKind.create(name='service_key_modify')
LogEntryKind.create(name='service_key_extend')
LogEntryKind.create(name='service_key_rotate')
LogEntryKind.create(name='take_ownership')
LogEntryKind.create(name='manifest_label_add')
LogEntryKind.create(name='manifest_label_delete')
LogEntryKind.create(name='change_tag_expiration')
LogEntryKind.create(name='toggle_repo_trigger')
LogEntryKind.create(name='create_app_specific_token')
LogEntryKind.create(name='revoke_app_specific_token')
ImageStorageLocation.create(name='local_eu')
ImageStorageLocation.create(name='local_us')
ApprBlobPlacementLocation.create(name='local_eu')
ApprBlobPlacementLocation.create(name='local_us')
ImageStorageTransformation.create(name='squash')
ImageStorageTransformation.create(name='aci')
ImageStorageSignatureKind.create(name='gpg2')
# NOTE: These MUST be copied over to NotificationKind, since every external
# notification can also generate a Quay.io notification.
ExternalNotificationEvent.create(name='repo_push')
ExternalNotificationEvent.create(name='build_queued')
ExternalNotificationEvent.create(name='build_start')
ExternalNotificationEvent.create(name='build_success')
ExternalNotificationEvent.create(name='build_cancelled')
ExternalNotificationEvent.create(name='build_failure')
ExternalNotificationEvent.create(name='vulnerability_found')
ExternalNotificationEvent.create(name='repo_mirror_sync_started')
ExternalNotificationEvent.create(name='repo_mirror_sync_success')
ExternalNotificationEvent.create(name='repo_mirror_sync_failed')
ExternalNotificationMethod.create(name='quay_notification')
ExternalNotificationMethod.create(name='email')
ExternalNotificationMethod.create(name='webhook')
ExternalNotificationMethod.create(name='flowdock')
ExternalNotificationMethod.create(name='hipchat')
ExternalNotificationMethod.create(name='slack')
NotificationKind.create(name='repo_push')
NotificationKind.create(name='build_queued')
NotificationKind.create(name='build_start')
NotificationKind.create(name='build_success')
NotificationKind.create(name='build_cancelled')
NotificationKind.create(name='build_failure')
NotificationKind.create(name='vulnerability_found')
NotificationKind.create(name='service_key_submitted')
NotificationKind.create(name='password_required')
NotificationKind.create(name='over_private_usage')
NotificationKind.create(name='expiring_license')
NotificationKind.create(name='maintenance')
NotificationKind.create(name='org_team_invite')
NotificationKind.create(name='repo_mirror_sync_started')
NotificationKind.create(name='repo_mirror_sync_success')
NotificationKind.create(name='repo_mirror_sync_failed')
NotificationKind.create(name='test_notification')
QuayRegion.create(name='us')
QuayService.create(name='quay')
MediaType.create(name='text/plain')
MediaType.create(name='application/json')
MediaType.create(name='text/markdown')
MediaType.create(name='application/vnd.cnr.blob.v0.tar+gzip')
MediaType.create(name='application/vnd.cnr.package-manifest.helm.v0.json')
MediaType.create(name='application/vnd.cnr.package-manifest.kpm.v0.json')
MediaType.create(
name='application/vnd.cnr.package-manifest.docker-compose.v0.json')
MediaType.create(name='application/vnd.cnr.package.kpm.v0.tar+gzip')
MediaType.create(name='application/vnd.cnr.package.helm.v0.tar+gzip')
MediaType.create(
name='application/vnd.cnr.package.docker-compose.v0.tar+gzip')
MediaType.create(name='application/vnd.cnr.manifests.v0.json')
MediaType.create(name='application/vnd.cnr.manifest.list.v0.json')
for media_type in DOCKER_SCHEMA1_CONTENT_TYPES:
MediaType.create(name=media_type)
for media_type in DOCKER_SCHEMA2_CONTENT_TYPES:
MediaType.create(name=media_type)
LabelSourceType.create(name='manifest')
LabelSourceType.create(name='api', mutable=True)
LabelSourceType.create(name='internal')
UserPromptKind.create(name='confirm_username')
UserPromptKind.create(name='enter_name')
UserPromptKind.create(name='enter_company')
RepositoryKind.create(name='image')
RepositoryKind.create(name='application')
ApprTagKind.create(name='tag')
ApprTagKind.create(name='release')
ApprTagKind.create(name='channel')
DisableReason.create(name='user_toggled')
DisableReason.create(name='successive_build_failures')
DisableReason.create(name='successive_build_internal_errors')
TagKind.create(name='tag')
