def _get_sorted_matching_repositories(lookup_value,
repo_kind="image",
include_private=False,
search_fields=None,
ids_only=False):
"""
Returns a query of repositories matching the given lookup string, with optional inclusion of
private repositories.
Note that this method does *not* filter results based on visibility to users.
"""
select_fields = [Repository.id] if ids_only else [Repository, Namespace]
if not lookup_value:
# This is a generic listing of repositories. Simply return the sorted repositories based
# on RepositorySearchScore.
query = (Repository.select(
*select_fields).join(RepositorySearchScore).where(
Repository.state != RepositoryState.MARKED_FOR_DELETION).
order_by(RepositorySearchScore.score.desc(),
RepositorySearchScore.id))
else:
if search_fields is None:
search_fields = set(
[SEARCH_FIELDS.description.name, SEARCH_FIELDS.name.name])
# Always search at least on name (init clause)
clause = Repository.name.match(lookup_value)
computed_score = RepositorySearchScore.score.alias("score")
# If the description field is in the search fields, then we need to compute a synthetic score
# to discount the weight of the description more than the name.
if SEARCH_FIELDS.description.name in search_fields:
clause = Repository.description.match(lookup_value) | clause
cases = [
(Repository.name.match(lookup_value),
100 * RepositorySearchScore.score),
]
computed_score = Case(None, cases,
RepositorySearchScore.score).alias("score")
select_fields.append(computed_score)
query = (Repository.select(
*select_fields).join(RepositorySearchScore).where(clause).where(
Repository.state != RepositoryState.MARKED_FOR_DELETION).
order_by(SQL("score").desc(), RepositorySearchScore.id))
if repo_kind is not None:
query = query.where(
Repository.kind == Repository.kind.get_id(repo_kind))
if not include_private:
query = query.where(
Repository.visibility == _basequery.get_public_repo_visibility())
if not ids_only:
query = query.switch(Repository).join(
Namespace, on=(Namespace.id == Repository.namespace_user))
return query
def create_repository(namespace,
name,
creating_user,
visibility="private",
repo_kind="image",
description=None):
namespace_user = User.get(username=namespace)
yesterday = datetime.now() - timedelta(days=1)
try:
with db_transaction():
# Check if the repository exists to avoid an IntegrityError if possible.
existing = get_repository(namespace, name)
if existing is not None:
return None
try:
repo = Repository.create(
name=name,
visibility=Repository.visibility.get_id(visibility),
namespace_user=namespace_user,
kind=Repository.kind.get_id(repo_kind),
description=description,
)
except IntegrityError as ie:
raise _RepositoryExistsException(ie)
RepositoryActionCount.create(repository=repo,
count=0,
date=yesterday)
RepositorySearchScore.create(repository=repo, score=0)
# Note: We put the admin create permission under the transaction to ensure it is created.
if creating_user and not creating_user.organization:
admin = Role.get(name="admin")
RepositoryPermission.create(user=creating_user,
repository=repo,
role=admin)
except _RepositoryExistsException as ree:
try:
return Repository.get(namespace_user=namespace_user, name=name)
except Repository.DoesNotExist:
logger.error(
"Got integrity error when trying to create repository %s/%s: %s",
namespace,
name,
ree.internal_exception,
)
return None
# Apply default permissions (only occurs for repositories under organizations)
if creating_user and not creating_user.organization and creating_user.username != namespace:
permission.apply_default_permissions(repo, creating_user)
return repo
def get_visible_repositories(username,
namespace=None,
kind_filter="image",
include_public=False,
start_id=None,
limit=None):
"""
Returns the repositories visible to the given user (if any).
"""
if not include_public and not username:
# Short circuit by returning a query that will find no repositories. We need to return a query
# here, as it will be modified by other queries later on.
return Repository.select(
Repository.id.alias("rid")).where(Repository.id == -1)
query = (Repository.select(
Repository.name,
Repository.id.alias("rid"),
Repository.description,
Namespace.username,
Repository.visibility,
Repository.kind,
Repository.state,
).switch(Repository).join(
Namespace, on=(Repository.namespace_user == Namespace.id)).where(
Repository.state != RepositoryState.MARKED_FOR_DELETION))
user_id = None
if username:
# Note: We only need the permissions table if we will filter based on a user's permissions.
query = query.switch(Repository).distinct().join(
RepositoryPermission, JOIN.LEFT_OUTER)
found_namespace = _get_namespace_user(username)
if not found_namespace:
return Repository.select(
Repository.id.alias("rid")).where(Repository.id == -1)
user_id = found_namespace.id
query = _basequery.filter_to_repos_for_user(query,
user_id,
namespace,
kind_filter,
include_public,
start_id=start_id)
if limit is not None:
query = query.limit(limit).order_by(SQL("rid"))
return query
def find_repository_with_garbage(limit_to_gc_policy_s):
expiration_timestamp = get_epoch_timestamp() - limit_to_gc_policy_s
try:
candidates = (RepositoryTag.select(
RepositoryTag.repository).join(Repository).join(
Namespace,
on=(Repository.namespace_user == Namespace.id)).where(
~(RepositoryTag.lifetime_end_ts >> None),
(RepositoryTag.lifetime_end_ts <= expiration_timestamp),
(Namespace.removed_tag_expiration_s
== limit_to_gc_policy_s),
).limit(500).distinct().alias("candidates"))
found = (RepositoryTag.select(
candidates.c.repository_id).from_(candidates).order_by(
db_random_func()).get())
if found is None:
return
return Repository.get(Repository.id == found.repository_id)
except RepositoryTag.DoesNotExist:
return None
except Repository.DoesNotExist:
return None
def test_does_not_create_repo_when_upstream_repo_does_not_exist(
self, test_name, proxy_manifest_response):
test_params = storage_test_cases[test_name]
repo = f"{self.orgname}/{test_params['image_name']}"
params = {
"repository": repo,
"manifest_ref": test_params["manifest_ref"],
}
proxy_mock = proxy_manifest_response("not-existing-ref", "", "")
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
headers = _get_auth_headers(self.sub, self.ctx, repo)
headers["Accept"] = ", ".join(
DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union(
DOCKER_SCHEMA1_CONTENT_TYPES))
conduct_call(
self.client,
test_params["view_name"],
url_for,
"GET",
params,
expected_code=404,
headers=headers,
)
count = Repository.filter(
Repository.name == test_params["image_name"],
Repository.namespace_user == self.org.id).count()
assert count == 0
def test_lookup_unrecoverable_tags(initialized_db):
# Ensure no existing tags are found.
for repo in Repository.select():
assert not list(lookup_unrecoverable_tags(repo))
# Mark a tag as outside the expiration window and ensure it is found.
repo = get_repository("devtable", "history")
results, _ = list_repository_tag_history(repo,
1,
100,
specific_tag_name="latest")
assert len(results) == 2
results[1].lifetime_end_ms = 1
results[1].save()
# Ensure the tag is now found.
found = list(lookup_unrecoverable_tags(repo))
assert found
assert len(found) == 1
assert found[0] == results[1]
# Mark the tag as expiring in the future and ensure it is no longer found.
results[1].lifetime_end_ms = get_epoch_timestamp_ms() + 1000000
results[1].save()
found = list(lookup_unrecoverable_tags(repo))
assert not found
def create_repository(
namespace, name, creating_user, visibility="private", repo_kind="image", description=None
):
namespace_user = User.get(username=namespace)
yesterday = datetime.now() - timedelta(days=1)
with db_transaction():
repo = Repository.create(
name=name,
visibility=Repository.visibility.get_id(visibility),
namespace_user=namespace_user,
kind=Repository.kind.get_id(repo_kind),
description=description,
)
RepositoryActionCount.create(repository=repo, count=0, date=yesterday)
RepositorySearchScore.create(repository=repo, score=0)
# Note: We put the admin create permission under the transaction to ensure it is created.
if creating_user and not creating_user.organization:
admin = Role.get(name="admin")
RepositoryPermission.create(user=creating_user, repository=repo, role=admin)
# Apply default permissions (only occurs for repositories under organizations)
if creating_user and not creating_user.organization and creating_user.username != namespace:
permission.apply_default_permissions(repo, creating_user)
return repo
def get_private_repo_count(username):
return (Repository.select().join(Visibility).switch(Repository).join(
Namespace, on=(Repository.namespace_user == Namespace.id)).where(
Namespace.username == username,
Visibility.name == "private").where(
Repository.state != RepositoryState.MARKED_FOR_DELETION).count(
))
def _garbage_collection_repos(self, skip_lock_for_testing=False):
"""
Performs garbage collection on repositories.
"""
with UseThenDisconnect(app.config):
policy = get_random_gc_policy()
if policy is None:
logger.debug("No GC policies found")
return
repo_ref = registry_model.find_repository_with_garbage(policy)
if repo_ref is None:
logger.debug("No repository with garbage found")
return
assert features.GARBAGE_COLLECTION
try:
with GlobalLock(
"REPO_GARBAGE_COLLECTION_%s" % repo_ref.id,
lock_ttl=REPOSITORY_GC_TIMEOUT + LOCK_TIMEOUT_PADDING,
) if not skip_lock_for_testing else empty_context():
try:
repository = Repository.get(id=repo_ref.id)
except Repository.DoesNotExist:
return
logger.debug("Starting GC of repository #%s (%s)",
repository.id, repository.name)
garbage_collect_repo(repository)
logger.debug("Finished GC of repository #%s (%s)",
repository.id, repository.name)
except LockNotAcquiredException:
logger.debug(
"Could not acquire repo lock for garbage collection")
def test_filter_repositories(username, include_public, filter_to_namespace,
repo_kind, initialized_db):
namespace = username if filter_to_namespace else None
if '+' in username and filter_to_namespace:
namespace, _ = parse_robot_username(username)
user = get_namespace_user(username)
query = (Repository.select().distinct().join(
Namespace, on=(Repository.namespace_user == Namespace.id
)).switch(Repository).join(RepositoryPermission,
JOIN.LEFT_OUTER))
# Prime the cache.
Repository.kind.get_id('image')
with assert_query_count(1):
found = list(
filter_to_repos_for_user(query,
user.id,
namespace=namespace,
include_public=include_public,
repo_kind=repo_kind))
expected = list(
_get_visible_repositories_for_user(user,
repo_kind=repo_kind,
namespace=namespace,
include_public=include_public))
assert len(found) == len(expected)
assert {r.id for r in found} == {r.id for r in expected}
def create_temporary_hidden_tag(repo, image_obj, expiration_s):
"""
Create a tag with a defined timeline, that will not appear in the UI or CLI.
Returns the name of the temporary tag or None on error.
"""
now_ts = get_epoch_timestamp()
expire_ts = now_ts + expiration_s
tag_name = str(uuid4())
# Ensure the repository is not marked for deletion.
with db_transaction():
current = Repository.get(id=repo)
if current.state == RepositoryState.MARKED_FOR_DELETION:
return None
RepositoryTag.create(
repository=repo,
image=image_obj,
name=tag_name,
lifetime_start_ts=now_ts,
lifetime_end_ts=expire_ts,
hidden=True,
)
return tag_name
def find_repository_with_garbage(limit_to_gc_policy_s):
""" Returns a repository that has garbage (defined as an expired Tag that is past
the repo's namespace's expiration window) or None if none.
"""
expiration_timestamp = get_epoch_timestamp_ms() - (limit_to_gc_policy_s *
1000)
try:
candidates = (Tag.select(Tag.repository).join(Repository).join(
Namespace, on=(Repository.namespace_user == Namespace.id)).where(
~(Tag.lifetime_end_ms >> None),
(Tag.lifetime_end_ms <= expiration_timestamp),
(Namespace.removed_tag_expiration_s == limit_to_gc_policy_s),
(Namespace.enabled == True),
(Repository.state != RepositoryState.MARKED_FOR_DELETION),
).limit(GC_CANDIDATE_COUNT).distinct().alias("candidates"))
found = (Tag.select(
candidates.c.repository_id).from_(candidates).order_by(
db_random_func()).get())
if found is None:
return
return Repository.get(Repository.id == found.repository_id)
except Tag.DoesNotExist:
return None
except Repository.DoesNotExist:
return None
def filter_to_repos_for_user(query,
user_id=None,
namespace=None,
repo_kind="image",
include_public=True,
start_id=None):
if not include_public and not user_id:
return Repository.select().where(Repository.id == "-1")
# Filter on the type of repository.
if repo_kind is not None:
try:
query = query.where(
Repository.kind == Repository.kind.get_id(repo_kind))
except RepositoryKind.DoesNotExist:
raise DataModelException("Unknown repository kind")
# Add the start ID if necessary.
if start_id is not None:
query = query.where(Repository.id >= start_id)
# Add a namespace filter if necessary.
if namespace:
query = query.where(Namespace.username == namespace)
# Build a set of queries that, when unioned together, return the full set of visible repositories
# for the filters specified.
queries = []
if include_public:
queries.append(
query.where(Repository.visibility == get_public_repo_visibility()))
if user_id is not None:
AdminTeam = Team.alias()
AdminTeamMember = TeamMember.alias()
# Add repositories in which the user has permission.
queries.append(
query.switch(RepositoryPermission).where(
RepositoryPermission.user == user_id))
# Add repositories in which the user is a member of a team that has permission.
queries.append(
query.switch(RepositoryPermission).join(Team).join(
TeamMember).where(TeamMember.user == user_id))
# Add repositories under namespaces in which the user is the org admin.
queries.append(
query.switch(Repository).join(
AdminTeam,
on=(Repository.namespace_user == AdminTeam.organization)).join(
AdminTeamMember,
on=(AdminTeam.id == AdminTeamMember.team)).where(
AdminTeam.role == _lookup_team_role("admin")).where(
AdminTeamMember.user == user_id))
return reduce(lambda l, r: l | r, queries)
def purge_repository(repo, force=False):
"""
Completely delete all traces of the repository.
Will return True upon complete success, and False upon partial or total failure. Garbage
collection is incremental and repeatable, so this return value does not need to be checked or
responded to.
"""
assert repo.state == RepositoryState.MARKED_FOR_DELETION or force
# Delete the repository of all Appr-referenced entries.
# Note that new-model Tag's must be deleted in *two* passes, as they can reference parent tags,
# and MySQL is... particular... about such relationships when deleting.
if repo.kind.name == "application":
ApprTag.delete().where(ApprTag.repository == repo,
~(ApprTag.linked_tag >> None)).execute()
ApprTag.delete().where(ApprTag.repository == repo).execute()
else:
# GC to remove the images and storage.
_purge_repository_contents(repo)
# Ensure there are no additional tags, manifests, images or blobs in the repository.
assert ApprTag.select().where(ApprTag.repository == repo).count() == 0
assert Tag.select().where(Tag.repository == repo).count() == 0
assert RepositoryTag.select().where(
RepositoryTag.repository == repo).count() == 0
assert Manifest.select().where(Manifest.repository == repo).count() == 0
assert ManifestBlob.select().where(
ManifestBlob.repository == repo).count() == 0
assert Image.select().where(Image.repository == repo).count() == 0
# Delete any repository build triggers, builds, and any other large-ish reference tables for
# the repository.
_chunk_delete_all(repo, RepositoryPermission, force=force)
_chunk_delete_all(repo, RepositoryBuild, force=force)
_chunk_delete_all(repo, RepositoryBuildTrigger, force=force)
_chunk_delete_all(repo, RepositoryActionCount, force=force)
_chunk_delete_all(repo, Star, force=force)
_chunk_delete_all(repo, AccessToken, force=force)
_chunk_delete_all(repo, RepositoryNotification, force=force)
_chunk_delete_all(repo, BlobUpload, force=force)
_chunk_delete_all(repo, RepoMirrorConfig, force=force)
_chunk_delete_all(repo, RepositoryAuthorizedEmail, force=force)
# Delete any marker rows for the repository.
DeletedRepository.delete().where(
DeletedRepository.repository == repo).execute()
# Delete the rest of the repository metadata.
try:
# Make sure the repository still exists.
fetched = Repository.get(id=repo.id)
except Repository.DoesNotExist:
return False
fetched.delete_instance(recursive=True, delete_nullable=False, force=force)
return True
def get_repository_for_resource(resource_key):
try:
return (Repository.select(Repository, Namespace).join(
Namespace,
on=(Repository.namespace_user == Namespace.id
)).switch(Repository).join(RepositoryBuild).where(
RepositoryBuild.resource_key == resource_key).get())
except Repository.DoesNotExist:
return None
def enable_mirroring_for_repository(
repository,
root_rule,
internal_robot,
external_reference,
sync_interval,
external_registry_username=None,
external_registry_password=None,
external_registry_config=None,
is_enabled=True,
sync_start_date=None,
):
"""
Create a RepoMirrorConfig and set the Repository to the MIRROR state.
"""
assert internal_robot.robot
namespace, _ = parse_robot_username(internal_robot.username)
if namespace != repository.namespace_user.username:
raise DataModelException("Cannot use robot for mirroring")
with db_transaction():
# Create the RepoMirrorConfig
try:
username = (
DecryptedValue(external_registry_username) if external_registry_username else None
)
password = (
DecryptedValue(external_registry_password) if external_registry_password else None
)
mirror = RepoMirrorConfig.create(
repository=repository,
root_rule=root_rule,
is_enabled=is_enabled,
internal_robot=internal_robot,
external_reference=external_reference,
external_registry_username=username,
external_registry_password=password,
external_registry_config=external_registry_config or {},
sync_interval=sync_interval,
sync_start_date=sync_start_date or datetime.utcnow(),
)
except IntegrityError:
return RepoMirrorConfig.get(repository=repository)
# Change Repository state to mirroring mode as needed
if repository.state != RepositoryState.MIRROR:
query = Repository.update(state=RepositoryState.MIRROR).where(
Repository.id == repository.id
)
if not query.execute():
raise DataModelException("Could not change the state of the repository")
return mirror
def repository_is_public(namespace_name, repository_name):
try:
(Repository.select().join(
Namespace, on=(Repository.namespace_user == Namespace.id
)).switch(Repository).join(Visibility).where(
Namespace.username == namespace_name,
Repository.name == repository_name,
Visibility.name == 'public').get())
return True
except Repository.DoesNotExist:
return False
def missing_counts_query(date):
""" Returns a query to find all Repository's with missing RAC entries for the given date. """
subquery = (RepositoryActionCount.select(
RepositoryActionCount.id, RepositoryActionCount.repository).where(
RepositoryActionCount.date == date).alias("rac"))
return (Repository.select().join(
subquery,
JOIN.LEFT_OUTER,
on=(Repository.id == subquery.c.repository_id
)).where(subquery.c.id >> None))
def repository_is_public(namespace_name, repository_name):
try:
(Repository.select().join(
Namespace, on=(Repository.namespace_user == Namespace.id)
).switch(Repository).join(Visibility).where(
Namespace.username == namespace_name,
Repository.name == repository_name,
Visibility.name == "public",
).where(Repository.state != RepositoryState.MARKED_FOR_DELETION).get())
return True
except Repository.DoesNotExist:
return False
def get_user_starred_repositories(user, kind_filter="image"):
""" Retrieves all of the repositories a user has starred. """
try:
repo_kind = Repository.kind.get_id(kind_filter)
except RepositoryKind.DoesNotExist:
raise DataModelException("Unknown kind of repository")
query = (Repository.select(
Repository, User, Visibility,
Repository.id.alias("rid")).join(Star).switch(Repository).join(
User).switch(Repository).join(Visibility).where(
Star.user == user, Repository.kind == repo_kind))
return query
def test_tag_names_for_manifest(initialized_db, registry_model):
verified_tag = False
for repository in Repository.select():
repo_ref = RepositoryReference.for_repo_obj(repository)
for tag in registry_model.list_all_active_repository_tags(repo_ref):
manifest = registry_model.get_manifest_for_tag(tag)
tag_names = set(registry_model.tag_names_for_manifest(manifest, 1000))
assert tag.name in tag_names
verified_tag = True
for found_name in tag_names:
found_tag = registry_model.get_repo_tag(repo_ref, found_name)
assert registry_model.get_manifest_for_tag(found_tag) == manifest
assert verified_tag
def _filter_repositories_visible_to_user(unfiltered_query, filter_user_id,
limit, repo_kind):
encountered = set()
chunk_count = limit * 2
unfiltered_page = 0
iteration_count = 0
while iteration_count < 10: # Just to be safe
# Find the next chunk's worth of repository IDs, paginated by the chunk size.
unfiltered_page = unfiltered_page + 1
found_ids = [
r.id
for r in unfiltered_query.paginate(unfiltered_page, chunk_count)
]
# Make sure we haven't encountered these results before. This code is used to handle
# the case where we've previously seen a result, as pagination is not necessary
# stable in SQL databases.
unfiltered_repository_ids = set(found_ids)
new_unfiltered_ids = unfiltered_repository_ids - encountered
if not new_unfiltered_ids:
break
encountered.update(new_unfiltered_ids)
# Filter the repositories found to only those visible to the current user.
query = (Repository.select(Repository, Namespace).distinct().join(
Namespace,
on=(Namespace.id == Repository.namespace_user
)).switch(Repository).join(RepositoryPermission).where(
Repository.id << list(new_unfiltered_ids)))
filtered = _basequery.filter_to_repos_for_user(query,
filter_user_id,
repo_kind=repo_kind)
# Sort the filtered repositories by their initial order.
all_filtered_repos = list(filtered)
all_filtered_repos.sort(key=lambda repo: found_ids.index(repo.id))
# Yield the repositories in sorted order.
for filtered_repo in all_filtered_repos:
yield filtered_repo
# If the number of found IDs is less than the chunk count, then we're done.
if len(found_ids) < chunk_count:
break
iteration_count = iteration_count + 1
def _temp_link_blob(repository_id, storage, link_expiration_s):
""" Note: Should *always* be called by a parent under a transaction. """
try:
repository = Repository.get(id=repository_id)
except Repository.DoesNotExist:
return None
if repository.state == RepositoryState.MARKED_FOR_DELETION:
return None
return UploadedBlob.create(
repository=repository_id,
blob=storage,
expires_at=datetime.utcnow() + timedelta(seconds=link_expiration_s),
)
def get_existing_repository(namespace_name,
repository_name,
for_update=False,
kind_filter=None):
query = (Repository.select(Repository, Namespace).join(
Namespace, on=(Repository.namespace_user == Namespace.id)).where(
Namespace.username == namespace_name,
Repository.name == repository_name))
if kind_filter:
query = (query.switch(Repository).join(RepositoryKind).where(
RepositoryKind.name == kind_filter))
if for_update:
query = db_for_update(query)
return query.get()
def delete_user(user, queues):
""" Deletes a user/organization/robot. Should *not* be called by any user-facing API. Instead,
mark_namespace_for_deletion should be used, and the queue should call this method.
"""
# Delete all queue items for the user.
for queue in queues:
queue.delete_namespaced_items(user.username)
# Delete any repositories under the user's namespace.
for repo in list(Repository.select().where(Repository.namespace_user == user)):
gc.purge_repository(user.username, repo.name)
# Delete non-repository related items.
_delete_user_linked_data(user)
# Delete the user itself.
user.delete_instance(recursive=True, delete_nullable=True)
def find_uncounted_repository():
""" Returns a repository that has not yet had an entry added into the RepositoryActionCount
table for yesterday.
"""
try:
# Get a random repository to count.
today = date.today()
yesterday = today - timedelta(days=1)
has_yesterday_actions = RepositoryActionCount.select(
RepositoryActionCount.repository).where(
RepositoryActionCount.date == yesterday)
to_count = (Repository.select().where(
~(Repository.id << (has_yesterday_actions))).order_by(
db_random_func()).get())
return to_count
except Repository.DoesNotExist:
return None
def _get_visible_repositories_for_user(user,
repo_kind="image",
include_public=False,
namespace=None):
"""
Returns all repositories directly visible to the given user, by either repo permission, or the
user being the admin of a namespace.
"""
for repo in Repository.select():
if repo_kind is not None and repo.kind.name != repo_kind:
continue
if namespace is not None and repo.namespace_user.username != namespace:
continue
if include_public and repo.visibility.name == "public":
yield repo
continue
# Direct repo permission.
try:
RepositoryPermission.get(repository=repo, user=user).get()
yield repo
continue
except RepositoryPermission.DoesNotExist:
pass
# Team permission.
found_in_team = False
for perm in RepositoryPermission.select().where(
RepositoryPermission.repository == repo):
if perm.team and _is_team_member(perm.team, user):
found_in_team = True
break
if found_in_team:
yield repo
continue
# Org namespace admin permission.
if user in get_admin_users(repo.namespace_user):
yield repo
continue
def delete_user(user, queues):
"""
Deletes a user/organization/robot.
Should *not* be called by any user-facing API. Instead, mark_namespace_for_deletion should be
used, and the queue should call this method.
Returns True on success and False otherwise.
"""
# Ensure the user is disabled before beginning the deletion process.
if user.enabled:
user.enabled = False
user.save()
# Delete all queue items for the user.
for queue in queues:
queue.delete_namespaced_items(user.username)
# Delete any repositories under the user's namespace.
while True:
is_progressing = False
repositories = list(
Repository.select().where(Repository.namespace_user == user))
if not repositories:
break
for repo in repositories:
if gc.purge_repository(repo, force=True):
is_progressing = True
if not is_progressing:
return False
# Delete non-repository related items.
_delete_user_linked_data(user)
# Delete the user itself.
try:
user.delete_instance(recursive=True, delete_nullable=True)
return True
except IntegrityError:
return False
def lookup_secscan_notification_severities(repository_id):
"""
Returns the configured security scanner notification severities for the repository
or None if none.
"""
try:
repo = Repository.get(id=repository_id)
except Repository.DoesNotExist:
return None
event_kind = ExternalNotificationEvent.get(name="vulnerability_found")
for event in RepositoryNotification.select().where(
RepositoryNotification.repository == repository_id,
RepositoryNotification.event == event_kind,
):
severity = json.loads(event.event_config_json).get(
"vulnerability", {}).get("priority")
if severity:
yield severity
def populate_database(minimal=False):
logger.debug("Populating the DB with test data.")
# Check if the data already exists. If so, we skip. This can happen between calls from the
# "old style" tests and the new py.test's.
try:
User.get(username="******")
logger.debug("DB already populated")
return
except User.DoesNotExist:
pass
# Note: databases set up with "real" schema (via Alembic) will not have these types
# type, so we it here it necessary.
try:
ImageStorageLocation.get(name="local_eu")
ImageStorageLocation.get(name="local_us")
except ImageStorageLocation.DoesNotExist:
ImageStorageLocation.create(name="local_eu")
ImageStorageLocation.create(name="local_us")
try:
NotificationKind.get(name="test_notification")
except NotificationKind.DoesNotExist:
NotificationKind.create(name="test_notification")
new_user_1 = model.user.create_user("devtable", "password",
"*****@*****.**")
new_user_1.verified = True
new_user_1.stripe_id = TEST_STRIPE_ID
new_user_1.save()
if minimal:
logger.debug(
"Skipping most db population because user requested mininal db")
return
UserRegion.create(user=new_user_1,
location=ImageStorageLocation.get(name="local_us"))
model.release.set_region_release("quay", "us", "v0.1.2")
model.user.create_confirm_email_code(new_user_1,
new_email="*****@*****.**")
disabled_user = model.user.create_user("disabled", "password",
"*****@*****.**")
disabled_user.verified = True
disabled_user.enabled = False
disabled_user.save()
dtrobot = model.user.create_robot("dtrobot", new_user_1)
dtrobot2 = model.user.create_robot("dtrobot2", new_user_1)
new_user_2 = model.user.create_user("public", "password",
"*****@*****.**")
new_user_2.verified = True
new_user_2.save()
new_user_3 = model.user.create_user("freshuser", "password",
"*****@*****.**")
new_user_3.verified = True
new_user_3.save()
another_robot = model.user.create_robot("anotherrobot", new_user_3)
new_user_4 = model.user.create_user("randomuser", "password",
"*****@*****.**")
new_user_4.verified = True
new_user_4.save()
new_user_5 = model.user.create_user("unverified", "password",
"*****@*****.**")
new_user_5.save()
reader = model.user.create_user("reader", "password", "*****@*****.**")
reader.verified = True
reader.save()
creatoruser = model.user.create_user("creator", "password",
"*****@*****.**")
creatoruser.verified = True
creatoruser.save()
outside_org = model.user.create_user("outsideorg", "password",
"*****@*****.**")
outside_org.verified = True
outside_org.save()
model.notification.create_notification(
"test_notification",
new_user_1,
metadata={
"some": "value",
"arr": [1, 2, 3],
"obj": {
"a": 1,
"b": 2
}
},
)
from_date = datetime.utcnow()
to_date = from_date + timedelta(hours=1)
notification_metadata = {
"from_date": formatdate(calendar.timegm(from_date.utctimetuple())),
"to_date": formatdate(calendar.timegm(to_date.utctimetuple())),
"reason": "database migration",
}
model.notification.create_notification("maintenance",
new_user_1,
metadata=notification_metadata)
__generate_repository(
new_user_4,
"randomrepo",
"Random repo repository.",
False,
[],
(4, [], ["latest", "prod"]),
)
simple_repo = __generate_repository(
new_user_1,
"simple",
"Simple repository.",
False,
[],
(4, [], ["latest", "prod"]),
)
# Add some labels to the latest tag's manifest.
repo_ref = RepositoryReference.for_repo_obj(simple_repo)
tag = registry_model.get_repo_tag(repo_ref, "latest")
manifest = registry_model.get_manifest_for_tag(tag)
assert manifest
first_label = registry_model.create_manifest_label(manifest, "foo", "bar",
"manifest")
registry_model.create_manifest_label(manifest, "foo", "baz", "api")
registry_model.create_manifest_label(manifest, "anotherlabel", "1234",
"internal")
registry_model.create_manifest_label(manifest, "jsonlabel",
'{"hey": "there"}', "internal",
"application/json")
label_metadata = {
"key": "foo",
"value": "bar",
"id": first_label._db_id,
"manifest_digest": manifest.digest,
}
logs_model.log_action(
"manifest_label_add",
new_user_1.username,
performer=new_user_1,
timestamp=datetime.now(),
metadata=label_metadata,
repository=simple_repo,
)
model.blob.initiate_upload(new_user_1.username, simple_repo.name,
str(uuid4()), "local_us", {})
model.notification.create_repo_notification(simple_repo, "repo_push",
"quay_notification", {}, {})
__generate_repository(
new_user_1,
"sharedtags",
"Shared tags repository",
False,
[(new_user_2, "read"), (dtrobot[0], "read")],
(
2,
[
(3, [], ["v2.0", "v2.1", "v2.2"]),
(
1,
[(1, [(1, [], ["prod", "581a284"])
], ["staging", "8423b58"]), (1, [], None)],
None,
),
],
None,
),
)
__generate_repository(
new_user_1,
"history",
"Historical repository.",
False,
[],
(4, [(2, [], "#latest"), (3, [], "latest")], None),
)
__generate_repository(
new_user_1,
"complex",
"Complex repository with many branches and tags.",
False,
[(new_user_2, "read"), (dtrobot[0], "read")],
(
2,
[(3, [], "v2.0"),
(1, [(1, [(2, [], ["prod"])], "staging"), (1, [], None)], None)],
None,
),
)
__generate_repository(
new_user_1,
"gargantuan",
None,
False,
[],
(
2,
[
(3, [], "v2.0"),
(1, [(1, [(1, [], ["latest", "prod"])], "staging"),
(1, [], None)], None),
(20, [], "v3.0"),
(5, [], "v4.0"),
(1, [(1, [], "v5.0"), (1, [], "v6.0")], None),
],
None,
),
)
trusted_repo = __generate_repository(
new_user_1,
"trusted",
"Trusted repository.",
False,
[],
(4, [], ["latest", "prod"]),
)
trusted_repo.trust_enabled = True
trusted_repo.save()
publicrepo = __generate_repository(
new_user_2,
"publicrepo",
"Public repository pullable by the world.",
True,
[],
(10, [], "latest"),
)
__generate_repository(outside_org, "coolrepo", "Some cool repo.", False,
[], (5, [], "latest"))
__generate_repository(
new_user_1,
"shared",
"Shared repository, another user can write.",
False,
[(new_user_2, "write"), (reader, "read")],
(5, [], "latest"),
)
__generate_repository(
new_user_1,
"text-full-repo",
"This is a repository for testing text search",
False,
[(new_user_2, "write"), (reader, "read")],
(5, [], "latest"),
)
building = __generate_repository(
new_user_1,
"building",
"Empty repository which is building.",
False,
[(new_user_2, "write"), (reader, "read")],
(0, [], None),
)
new_token = model.token.create_access_token(building, "write",
"build-worker")
trigger = model.build.create_build_trigger(building,
"github",
"123authtoken",
new_user_1,
pull_robot=dtrobot[0])
trigger.config = json.dumps({
"build_source": "jakedt/testconnect",
"subdir": "",
"dockerfile_path": "Dockerfile",
"context": "/",
})
trigger.save()
repo = "ci.devtable.com:5000/%s/%s" % (building.namespace_user.username,
building.name)
job_config = {
"repository": repo,
"docker_tags": ["latest"],
"build_subdir": "",
"trigger_metadata": {
"commit": "3482adc5822c498e8f7db2e361e8d57b3d77ddd9",
"ref": "refs/heads/master",
"default_branch": "master",
},
}
model.repository.star_repository(new_user_1, simple_repo)
record = model.repository.create_email_authorization_for_repo(
new_user_1.username, "simple", "*****@*****.**")
record.confirmed = True
record.save()
model.repository.create_email_authorization_for_repo(
new_user_1.username, "simple", "*****@*****.**")
build2 = model.build.create_repository_build(
building,
new_token,
job_config,
"68daeebd-a5b9-457f-80a0-4363b882f8ea",
"build-name",
trigger,
)
build2.uuid = "deadpork-dead-pork-dead-porkdeadpork"
build2.save()
build3 = model.build.create_repository_build(
building,
new_token,
job_config,
"f49d07f9-93da-474d-ad5f-c852107c3892",
"build-name",
trigger,
)
build3.uuid = "deadduck-dead-duck-dead-duckdeadduck"
build3.save()
build1 = model.build.create_repository_build(
building, new_token, job_config, "701dcc3724fb4f2ea6c31400528343cd",
"build-name", trigger)
build1.uuid = "deadbeef-dead-beef-dead-beefdeadbeef"
build1.save()
org = model.organization.create_organization("buynlarge",
"*****@*****.**",
new_user_1)
org.stripe_id = TEST_STRIPE_ID
org.save()
liborg = model.organization.create_organization(
"library", "*****@*****.**", new_user_1)
liborg.save()
titiorg = model.organization.create_organization("titi",
"*****@*****.**",
new_user_1)
titiorg.save()
thirdorg = model.organization.create_organization(
"sellnsmall", "*****@*****.**", new_user_1)
thirdorg.save()
model.user.create_robot("coolrobot", org)
oauth_app_1 = model.oauth.create_application(
org,
"Some Test App",
"http://localhost:8000",
"http://localhost:8000/o2c.html",
client_id="deadbeef",
)
model.oauth.create_application(
org,
"Some Other Test App",
"http://quay.io",
"http://localhost:8000/o2c.html",
client_id="deadpork",
description="This is another test application",
)
model.oauth.create_user_access_token(new_user_1,
"deadbeef",
"repo:admin",
access_token="%s%s" %
("b" * 40, "c" * 40))
oauth_credential = Credential.from_string("dswfhasdf1")
OAuthAuthorizationCode.create(
application=oauth_app_1,
code="Z932odswfhasdf1",
scope="repo:admin",
data='{"somejson": "goeshere"}',
code_name="Z932odswfhasdf1Z932o",
code_credential=oauth_credential,
)
model.user.create_robot("neworgrobot", org)
ownerbot = model.user.create_robot("ownerbot", org)[0]
creatorbot = model.user.create_robot("creatorbot", org)[0]
owners = model.team.get_organization_team("buynlarge", "owners")
owners.description = "Owners have unfetterd access across the entire org."
owners.save()
org_repo = __generate_repository(
org,
"orgrepo",
"Repository owned by an org.",
False,
[(outside_org, "read")],
(4, [], ["latest", "prod"]),
)
__generate_repository(
org,
"anotherorgrepo",
"Another repository owned by an org.",
False,
[],
(4, [], ["latest", "prod"]),
)
creators = model.team.create_team("creators", org, "creator",
"Creators of orgrepo.")
reader_team = model.team.create_team("readers", org, "member",
"Readers of orgrepo.")
model.team.add_or_invite_to_team(new_user_1, reader_team, outside_org)
model.permission.set_team_repo_permission(reader_team.name,
org_repo.namespace_user.username,
org_repo.name, "read")
model.team.add_user_to_team(new_user_2, reader_team)
model.team.add_user_to_team(reader, reader_team)
model.team.add_user_to_team(ownerbot, owners)
model.team.add_user_to_team(creatorbot, creators)
model.team.add_user_to_team(creatoruser, creators)
sell_owners = model.team.get_organization_team("sellnsmall", "owners")
sell_owners.description = "Owners have unfettered access across the entire org."
sell_owners.save()
model.team.add_user_to_team(new_user_4, sell_owners)
sync_config = {
"group_dn": "cn=Test-Group,ou=Users",
"group_id": "somegroupid"
}
synced_team = model.team.create_team("synced", org, "member",
"Some synced team.")
model.team.set_team_syncing(synced_team, "ldap", sync_config)
another_synced_team = model.team.create_team("synced", thirdorg, "member",
"Some synced team.")
model.team.set_team_syncing(another_synced_team, "ldap",
{"group_dn": "cn=Test-Group,ou=Users"})
__generate_repository(
new_user_1,
"superwide",
None,
False,
[],
[
(10, [], "latest2"),
(2, [], "latest3"),
(2, [(1, [], "latest11"), (2, [], "latest12")], "latest4"),
(2, [], "latest5"),
(2, [], "latest6"),
(2, [], "latest7"),
(2, [], "latest8"),
(2, [], "latest9"),
(2, [], "latest10"),
(2, [], "latest13"),
(2, [], "latest14"),
(2, [], "latest15"),
(2, [], "latest16"),
(2, [], "latest17"),
(2, [], "latest18"),
],
)
mirror_repo = __generate_repository(
new_user_1,
"mirrored",
"Mirrored repository.",
False,
[(dtrobot[0], "write"), (dtrobot2[0], "write")],
(4, [], ["latest", "prod"]),
)
mirror_rule = model.repo_mirror.create_mirroring_rule(
mirror_repo, ["latest", "3.3*"])
mirror_args = (mirror_repo, mirror_rule, dtrobot[0], "quay.io/coreos/etcd",
60 * 60 * 24)
mirror_kwargs = {
"external_registry_username": "******",
"external_registry_password": "******",
"external_registry_config": {},
"is_enabled": True,
"sync_start_date": datetime.utcnow(),
}
mirror = model.repo_mirror.enable_mirroring_for_repository(
*mirror_args, **mirror_kwargs)
read_only_repo = __generate_repository(
new_user_1,
"readonly",
"Read-Only Repo.",
False,
[],
(4, [], ["latest", "prod"]),
)
read_only_repo.state = RepositoryState.READ_ONLY
read_only_repo.save()
model.permission.add_prototype_permission(org,
"read",
activating_user=new_user_1,
delegate_user=new_user_2)
model.permission.add_prototype_permission(org,
"read",
activating_user=new_user_1,
delegate_team=reader_team)
model.permission.add_prototype_permission(org,
"write",
activating_user=new_user_2,
delegate_user=new_user_1)
today = datetime.today()
week_ago = today - timedelta(6)
six_ago = today - timedelta(5)
four_ago = today - timedelta(4)
yesterday = datetime.combine(date.today(),
datetime.min.time()) - timedelta(hours=6)
__generate_service_key("kid1", "somesamplekey", new_user_1, today,
ServiceKeyApprovalType.SUPERUSER)
__generate_service_key(
"kid2",
"someexpiringkey",
new_user_1,
week_ago,
ServiceKeyApprovalType.SUPERUSER,
today + timedelta(days=14),
)
__generate_service_key("kid3", "unapprovedkey", new_user_1, today, None)
__generate_service_key(
"kid4",
"autorotatingkey",
new_user_1,
six_ago,
ServiceKeyApprovalType.KEY_ROTATION,
today + timedelta(days=1),
rotation_duration=timedelta(hours=12).total_seconds(),
)
__generate_service_key(
"kid5",
"key for another service",
new_user_1,
today,
ServiceKeyApprovalType.SUPERUSER,
today + timedelta(days=14),
service="different_sample_service",
)
__generate_service_key(
"kid6",
"someexpiredkey",
new_user_1,
week_ago,
ServiceKeyApprovalType.SUPERUSER,
today - timedelta(days=1),
)
__generate_service_key(
"kid7",
"somewayexpiredkey",
new_user_1,
week_ago,
ServiceKeyApprovalType.SUPERUSER,
today - timedelta(days=30),
)
# Add the test pull key as pre-approved for local and unittest registry testing.
# Note: this must match the private key found in the local/test config.
_TEST_JWK = {
"e":
"AQAB",
"kty":
"RSA",
"n":
"yqdQgnelhAPMSeyH0kr3UGePK9oFOmNfwD0Ymnh7YYXr21VHWwyM2eVW3cnLd9KXywDFtGSe9oFDbnOuMCdUowdkBcaHju-isbv5KEbNSoy_T2Rip-6L0cY63YzcMJzv1nEYztYXS8wz76pSK81BKBCLapqOCmcPeCvV9yaoFZYvZEsXCl5jjXN3iujSzSF5Z6PpNFlJWTErMT2Z4QfbDKX2Nw6vJN6JnGpTNHZvgvcyNX8vkSgVpQ8DFnFkBEx54PvRV5KpHAq6AsJxKONMo11idQS2PfCNpa2hvz9O6UZe-eIX8jPo5NW8TuGZJumbdPT_nxTDLfCqfiZboeI0Pw",
}
key = model.service_keys.create_service_key("test_service_key",
"test_service_key", "quay",
_TEST_JWK, {}, None)
model.service_keys.approve_service_key(
key.kid,
ServiceKeyApprovalType.SUPERUSER,
notes="Test service key for local/test registry testing",
)
# Add an app specific token.
token = model.appspecifictoken.create_token(new_user_1, "some app")
token.token_name = "a" * 60
token.token_secret = "b" * 60
token.save()
logs_model.log_action(
"org_create_team",
org.username,
performer=new_user_1,
timestamp=week_ago,
metadata={"team": "readers"},
)
logs_model.log_action(
"org_set_team_role",
org.username,
performer=new_user_1,
timestamp=week_ago,
metadata={
"team": "readers",
"role": "read"
},
)
logs_model.log_action(
"create_repo",
org.username,
performer=new_user_1,
repository=org_repo,
timestamp=week_ago,
metadata={
"namespace": org.username,
"repo": "orgrepo"
},
)
logs_model.log_action(
"change_repo_permission",
org.username,
performer=new_user_2,
repository=org_repo,
timestamp=six_ago,
metadata={
"username": new_user_1.username,
"repo": "orgrepo",
"role": "admin"
},
)
logs_model.log_action(
"change_repo_permission",
org.username,
performer=new_user_1,
repository=org_repo,
timestamp=six_ago,
metadata={
"username": new_user_2.username,
"repo": "orgrepo",
"role": "read"
},
)
logs_model.log_action(
"add_repo_accesstoken",
org.username,
performer=new_user_1,
repository=org_repo,
timestamp=four_ago,
metadata={
"repo": "orgrepo",
"token": "deploytoken"
},
)
logs_model.log_action(
"push_repo",
org.username,
performer=new_user_2,
repository=org_repo,
timestamp=today,
metadata={
"username": new_user_2.username,
"repo": "orgrepo"
},
)
logs_model.log_action(
"pull_repo",
org.username,
performer=new_user_2,
repository=org_repo,
timestamp=today,
metadata={
"username": new_user_2.username,
"repo": "orgrepo"
},
)
logs_model.log_action(
"pull_repo",
org.username,
repository=org_repo,
timestamp=today,
metadata={
"token": "sometoken",
"token_code": "somecode",
"repo": "orgrepo"
},
)
logs_model.log_action(
"delete_tag",
org.username,
performer=new_user_2,
repository=org_repo,
timestamp=today,
metadata={
"username": new_user_2.username,
"repo": "orgrepo",
"tag": "sometag"
},
)
logs_model.log_action(
"pull_repo",
org.username,
repository=org_repo,
timestamp=today,
metadata={
"token_code": "somecode",
"repo": "orgrepo"
},
)
logs_model.log_action(
"pull_repo",
new_user_2.username,
repository=publicrepo,
timestamp=yesterday,
metadata={
"token_code": "somecode",
"repo": "publicrepo"
},
)
logs_model.log_action(
"build_dockerfile",
new_user_1.username,
repository=building,
timestamp=today,
metadata={
"repo": "building",
"namespace": new_user_1.username,
"trigger_id": trigger.uuid,
"config": json.loads(trigger.config),
"service": trigger.service.name,
},
)
model.message.create([{
"content": "We love you, Quay customers!",
"severity": "info",
"media_type": "text/plain",
}])
model.message.create([{
"content": "This is a **development** install of Quay",
"severity": "warning",
"media_type": "text/markdown",
}])
fake_queue = WorkQueue("fakequeue", tf)
fake_queue.put(["canonical", "job", "name"], "{}")
model.user.create_user_prompt(new_user_4, "confirm_username")
for to_count in Repository.select():
model.repositoryactioncount.count_repository_actions(
to_count, datetime.utcnow())
model.repositoryactioncount.update_repository_score(to_count)
