def get_user_starred_repositories(user, kind_filter="image"):
"""
Retrieves all of the repositories a user has starred.
"""
try:
repo_kind = Repository.kind.get_id(kind_filter)
except RepositoryKind.DoesNotExist:
raise DataModelException("Unknown kind of repository")
query = (Repository.select(
Repository, User, Visibility,
Repository.id.alias("rid")).join(Star).switch(Repository).join(
User).switch(Repository).join(Visibility).where(
Star.user == user, Repository.kind == repo_kind).where(
Repository.state != RepositoryState.MARKED_FOR_DELETION))
return query
def store_tag_manifest_for_testing(namespace_name, repository_name, tag_name,
manifest, leaf_layer_id, storage_id_map):
"""
Stores a tag manifest for a specific tag name in the database.
Returns the TagManifest object, as well as a boolean indicating whether the TagManifest was
created.
"""
try:
repo = _basequery.get_existing_repository(namespace_name,
repository_name)
except Repository.DoesNotExist:
raise DataModelException("Invalid repository %s/%s" %
(namespace_name, repository_name))
return store_tag_manifest_for_repo(repo.id, tag_name, manifest,
leaf_layer_id, storage_id_map)
def set_image_metadata(
docker_image_id,
namespace_name,
repository_name,
created_date_str,
comment,
command,
v1_json_metadata,
parent=None,
):
""" Sets metadata that is specific to how a binary piece of storage fits into the layer tree.
"""
with db_transaction():
try:
fetched = (Image.select(Image, ImageStorage).join(Repository).join(
Namespace, on=(Repository.namespace_user == Namespace.id
)).switch(Image).join(ImageStorage).where(
Repository.name == repository_name,
Namespace.username == namespace_name,
Image.docker_image_id == docker_image_id,
).get())
except Image.DoesNotExist:
raise DataModelException(
"No image with specified id and repository")
fetched.created = datetime.now()
if created_date_str is not None:
try:
fetched.created = dateutil.parser.parse(
created_date_str).replace(tzinfo=None)
except:
# parse raises different exceptions, so we cannot use a specific kind of handler here.
pass
# We cleanup any old checksum in case it's a retry after a fail
fetched.v1_checksum = None
fetched.comment = comment
fetched.command = command
fetched.v1_json_metadata = v1_json_metadata
if parent:
fetched.ancestors = "%s%s/" % (parent.ancestors, parent.id)
fetched.parent = parent
fetched.save()
return fetched
def confirm_email_authorization_for_repo(code):
try:
found = (
RepositoryAuthorizedEmail.select(RepositoryAuthorizedEmail, Repository, Namespace)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(RepositoryAuthorizedEmail.code == code)
.where(Repository.state != RepositoryState.MARKED_FOR_DELETION)
.get()
)
except RepositoryAuthorizedEmail.DoesNotExist:
raise DataModelException("Invalid confirmation code.")
found.confirmed = True
found.save()
return found
def set_team_org_permission(team, team_role_name, set_by_username):
if team.role.name == 'admin' and team_role_name != 'admin':
# We need to make sure we're not removing the users only admin role
user_admin_teams = __get_user_admin_teams(team.organization.username,
set_by_username)
admin_team_set = {admin_team.name for admin_team in user_admin_teams}
if team.name in admin_team_set and len(admin_team_set) <= 1:
msg = (
('Cannot remove admin from team \'%s\' because calling user ' +
'would no longer have admin on org \'%s\'') %
(team.name, team.organization.username))
raise DataModelException(msg)
new_role = TeamRole.get(TeamRole.name == team_role_name)
team.role = new_role
team.save()
return team
def create_or_update_tag(namespace_name,
repository_name,
tag_name,
tag_docker_image_id,
reversion=False,
now_ms=None):
try:
repo = _basequery.get_existing_repository(namespace_name,
repository_name)
except Repository.DoesNotExist:
raise DataModelException("Invalid repository %s/%s" %
(namespace_name, repository_name))
return create_or_update_tag_for_repo(repo.id,
tag_name,
tag_docker_image_id,
reversion=reversion,
now_ms=now_ms)
def create_robot(robot_shortname,
parent,
description="",
unstructured_metadata=None):
(username_valid, username_issue) = validate_username(robot_shortname)
if not username_valid:
raise InvalidRobotException(
"The name for the robot '%s' is invalid: %s" %
(robot_shortname, username_issue))
username = format_robot_username(parent.username, robot_shortname)
try:
User.get(User.username == username)
msg = "Existing robot with name: %s" % username
logger.debug(msg)
raise InvalidRobotException(msg)
except User.DoesNotExist:
pass
service = LoginService.get(name="quayrobot")
try:
with db_transaction():
created = User.create(username=username,
email=str(uuid.uuid4()),
robot=True)
token = random_string_generator(length=64)()
RobotAccountToken.create(robot_account=created,
token=token,
fully_migrated=True)
FederatedLogin.create(user=created,
service=service,
service_ident="robot:%s" % created.id)
RobotAccountMetadata.create(
robot_account=created,
description=description[0:255],
unstructured_json=unstructured_metadata or {},
)
return created, token
except Exception as ex:
raise DataModelException(ex)
def calculate_image_aggregate_size(ancestors_str, image_size, parent_image):
ancestors = ancestors_str.split("/")[1:-1]
if not ancestors:
return image_size
if parent_image is None:
raise DataModelException("Could not load parent image")
ancestor_size = parent_image.aggregate_size
if ancestor_size is not None:
return ancestor_size + image_size
# Fallback to a slower path if the parent doesn't have an aggregate size saved.
# TODO: remove this code if/when we do a full backfill.
ancestor_size = (ImageStorage.select(fn.Sum(ImageStorage.image_size)).join(
Image).where(Image.id << ancestors).scalar())
if ancestor_size is None:
return None
return ancestor_size + image_size
def restore_tag_to_manifest(repo_obj, tag_name, manifest_digest):
"""
Restores a tag to a specific manifest digest.
"""
with db_transaction():
# Verify that the manifest digest already existed under this repository under the
# tag.
try:
tag_manifest = (TagManifest.select(
TagManifest, RepositoryTag,
Image).join(RepositoryTag).join(Image).where(
RepositoryTag.repository == repo_obj).where(
RepositoryTag.name == tag_name).where(
TagManifest.digest == manifest_digest).get())
except TagManifest.DoesNotExist:
raise DataModelException(
"Cannot restore to unknown or invalid digest")
# Lookup the existing image, if any.
try:
existing_image = get_repo_tag_image(repo_obj, tag_name)
except DataModelException:
existing_image = None
docker_image_id = tag_manifest.tag.image.docker_image_id
oci_manifest = None
try:
oci_manifest = Manifest.get(repository=repo_obj,
digest=manifest_digest)
except Manifest.DoesNotExist:
pass
# Change the tag and tag manifest to point to the updated image.
updated_tag = create_or_update_tag_for_repo(repo_obj,
tag_name,
docker_image_id,
reversion=True,
oci_manifest=oci_manifest)
tag_manifest.tag = updated_tag
tag_manifest.save()
return existing_image
def _logs_query(
selections,
start_time=None,
end_time=None,
performer=None,
repository=None,
namespace=None,
ignore=None,
model=LogEntry3,
id_range=None,
):
""" Returns a query for selecting logs from the table, with various options and filters. """
assert (start_time is not None and end_time is not None) or (id_range
is not None)
joined = model.select(*selections).switch(model)
if id_range is not None:
joined = joined.where(model.id >= id_range[0], model.id <= id_range[1])
else:
joined = joined.where(model.datetime >= start_time,
model.datetime < end_time)
if repository:
joined = joined.where(model.repository == repository)
if performer:
joined = joined.where(model.performer == performer)
if namespace and not repository:
namespace_user = user.get_user_or_org(namespace)
if namespace_user is None:
raise DataModelException("Invalid namespace requested")
joined = joined.where(model.account == namespace_user.id)
if ignore:
kind_map = get_log_entry_kinds()
ignore_ids = [kind_map[kind_name] for kind_name in ignore]
joined = joined.where(~(model.kind << ignore_ids))
return joined
def remove_team(org_name, team_name, removed_by_username):
joined = Team.select(Team, TeamRole).join(User).switch(Team).join(TeamRole)
found = list(
joined.where(User.organization == True, User.username == org_name,
Team.name == team_name))
if not found:
raise InvalidTeamException("Team '%s' is not a team in org '%s'" %
(team_name, org_name))
team = found[0]
if team.role.name == "admin":
admin_teams = list(
__get_user_admin_teams(org_name, removed_by_username))
if len(admin_teams) <= 1:
# The team we are trying to remove is the only admin team containing this user.
msg = "Deleting team '%s' would remove admin ability for user '%s' in organization '%s'"
raise DataModelException(
msg % (team_name, removed_by_username, org_name))
team.delete_instance(recursive=True, delete_nullable=True)
def delete_tag(namespace_name, repository_name, tag_name, now_ms=None):
now_ms = now_ms or get_epoch_timestamp_ms()
now_ts = int(now_ms / 1000)
with db_transaction():
try:
query = _tag_alive(
RepositoryTag.select(
RepositoryTag, Repository).join(Repository).join(
Namespace,
on=(Repository.namespace_user == Namespace.id)).where(
Repository.name == repository_name,
Namespace.username == namespace_name,
RepositoryTag.name == tag_name,
),
now_ts,
)
found = db_for_update(query).get()
except RepositoryTag.DoesNotExist:
msg = "Invalid repository tag '%s' on repository '%s/%s'" % (
tag_name,
namespace_name,
repository_name,
)
raise DataModelException(msg)
found.lifetime_end_ts = now_ts
found.save()
try:
oci_tag_query = TagToRepositoryTag.select().where(
TagToRepositoryTag.repository_tag == found)
oci_tag = db_for_update(oci_tag_query).get().tag
oci_tag.lifetime_end_ms = now_ms
oci_tag.save()
except TagToRepositoryTag.DoesNotExist:
pass
return found
def set_image_storage_metadata(
docker_image_id, namespace_name, repository_name, image_size, uncompressed_size
):
"""
Sets metadata that is specific to the binary storage of the data, irrespective of how it is used
in the layer tree.
"""
if image_size is None:
raise DataModelException("Empty image size field")
try:
image = (
Image.select(Image, ImageStorage)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(Image)
.join(ImageStorage)
.where(
Repository.name == repository_name,
Namespace.username == namespace_name,
Image.docker_image_id == docker_image_id,
)
.get()
)
except ImageStorage.DoesNotExist:
raise InvalidImageException("No image with specified id and repository")
# We MUST do this here, it can't be done in the corresponding image call because the storage
# has not yet been pushed
image.aggregate_size = _basequery.calculate_image_aggregate_size(
image.ancestors, image_size, image.parent
)
image.save()
image.storage.image_size = image_size
image.storage.uncompressed_size = uncompressed_size
image.storage.save()
return image.storage
def _latest_logs_query(
selections,
performer=None,
repository=None,
namespace=None,
ignore=None,
model=LogEntry3,
size=None,
):
"""
Returns a query for selecting the latest logs from the table, with various options and filters.
"""
query = model.select(*selections).switch(model)
if repository:
query = query.where(model.repository == repository)
if performer:
query = query.where(model.repository == repository)
if namespace and not repository:
namespace_user = user.get_user_or_org(namespace)
if namespace_user is None:
raise DataModelException("Invalid namespace requested")
query = query.where(model.account == namespace_user.id)
if ignore:
kind_map = get_log_entry_kinds()
ignore_ids = [kind_map[kind_name] for kind_name in ignore]
query = query.where(~(model.kind << ignore_ids))
query = query.order_by(model.datetime.desc(), model.id)
if size:
query = query.limit(size)
return query
def convert_user_to_organization(user_obj, admin_user):
if user_obj.robot:
raise DataModelException("Cannot convert a robot into an organization")
with db_transaction():
# Change the user to an organization and disable this account for login.
user_obj.organization = True
user_obj.password_hash = None
user_obj.save()
# Clear any federated auth pointing to this user.
FederatedLogin.delete().where(FederatedLogin.user == user_obj).execute()
# Delete any user-specific permissions on repositories.
(RepositoryPermission.delete().where(RepositoryPermission.user == user_obj).execute())
# Create a team for the owners
owners_team = team.create_team("owners", user_obj, "admin")
# Add the user who will admin the org to the owners team
team.add_user_to_team(admin_user, owners_team)
return user_obj
def mark_namespace_for_deletion(user, queues, namespace_gc_queue, force=False):
"""
Marks a namespace (as referenced by the given user) for deletion.
A queue item will be added to delete the namespace's repositories and storage, while the
namespace itself will be renamed, disabled, and delinked from other tables.
"""
if not user.enabled:
return None
if not force and not user.organization:
# Ensure that the user is not the sole admin for any organizations. If so, then the user
# cannot be deleted before those organizations are deleted or reassigned.
organizations = get_solely_admined_organizations(user)
if len(organizations) > 0:
message = (
"Cannot delete %s as you are the only admin for organizations: "
% user.username)
for index, org in enumerate(organizations):
if index > 0:
message = message + ", "
message = message + org.username
raise DataModelException(message)
# Delete all queue items for the user.
for queue in queues:
queue.delete_namespaced_items(user.username)
# Delete non-repository related items. This operation is very quick, so we can do so here.
_delete_user_linked_data(user)
with db_transaction():
original_username = user.username
user = db_for_update(User.select().where(User.id == user.id)).get()
# Mark the namespace as deleted and ready for GC.
try:
marker = DeletedNamespace.create(
namespace=user,
original_username=original_username,
original_email=user.email)
except IntegrityError:
return
# Disable the namespace itself, and replace its various unique fields with UUIDs.
user.enabled = False
user.username = str(uuid4())
user.email = str(uuid4())
user.save()
# Add a queueitem to delete the namespace.
marker.queue_id = namespace_gc_queue.put(
[str(user.id)],
json.dumps({
"marker_id": marker.id,
"original_username": original_username,
}),
)
marker.save()
return marker.id
def create_user_noverify(username,
email,
email_required=True,
prompts=tuple(),
is_possible_abuser=False):
if email_required:
if not validate_email(email):
raise InvalidEmailAddressException("Invalid email address: %s" %
email)
else:
# If email addresses are not required and none was specified, then we just use a unique
# ID to ensure that the database consistency check remains intact.
email = email or str(uuid.uuid4())
(username_valid, username_issue) = validate_username(username)
if not username_valid:
raise InvalidUsernameException("Invalid namespace %s: %s" %
(username, username_issue))
try:
existing = User.get((User.username == username)
| (User.email == email))
logger.debug("Existing user with same username or email.")
# A user already exists with either the same username or email
if existing.username == username:
assert not existing.robot
msg = (
"Username has already been taken by an organization and cannot be reused: %s"
% username)
if not existing.organization:
msg = "Username has already been taken by user cannot be reused: %s" % username
raise InvalidUsernameException(msg)
raise InvalidEmailAddressException("Email has already been used: %s" %
email)
except User.DoesNotExist:
# This is actually the happy path
logger.debug("Email and username are unique!")
# Create the user.
try:
default_expr_s = _convert_to_s(
config.app_config["DEFAULT_TAG_EXPIRATION"])
default_max_builds = config.app_config.get(
"DEFAULT_NAMESPACE_MAXIMUM_BUILD_COUNT")
threat_max_builds = config.app_config.get(
"THREAT_NAMESPACE_MAXIMUM_BUILD_COUNT")
if is_possible_abuser and threat_max_builds is not None:
default_max_builds = threat_max_builds
new_user = User.create(
username=username,
email=email,
removed_tag_expiration_s=default_expr_s,
maximum_queued_builds_count=default_max_builds,
)
for prompt in prompts:
create_user_prompt(new_user, prompt)
return new_user
except Exception as ex:
raise DataModelException(ex)
def create_or_update_tag_for_repo(repository_id,
tag_name,
tag_docker_image_id,
reversion=False,
oci_manifest=None,
now_ms=None):
now_ms = now_ms or get_epoch_timestamp_ms()
now_ts = int(now_ms / 1000)
with db_transaction():
try:
tag = db_for_update(
_tag_alive(
RepositoryTag.select().where(
RepositoryTag.repository == repository_id,
RepositoryTag.name == tag_name),
now_ts,
)).get()
tag.lifetime_end_ts = now_ts
tag.save()
# Check for an OCI tag.
try:
oci_tag = db_for_update(
Tag.select().join(TagToRepositoryTag).where(
TagToRepositoryTag.repository_tag == tag)).get()
oci_tag.lifetime_end_ms = now_ms
oci_tag.save()
except Tag.DoesNotExist:
pass
except RepositoryTag.DoesNotExist:
pass
except IntegrityError:
msg = "Tag with name %s was stale when we tried to update it; Please retry the push"
raise StaleTagException(msg % tag_name)
try:
image_obj = Image.get(Image.docker_image_id == tag_docker_image_id,
Image.repository == repository_id)
except Image.DoesNotExist:
raise DataModelException("Invalid image with id: %s" %
tag_docker_image_id)
try:
created = RepositoryTag.create(
repository=repository_id,
image=image_obj,
name=tag_name,
lifetime_start_ts=now_ts,
reversion=reversion,
)
if oci_manifest:
# Create the OCI tag as well.
oci_tag = Tag.create(
repository=repository_id,
manifest=oci_manifest,
name=tag_name,
lifetime_start_ms=now_ms,
reversion=reversion,
tag_kind=Tag.tag_kind.get_id("tag"),
)
TagToRepositoryTag.create(tag=oci_tag,
repository_tag=created,
repository=repository_id)
return created
except IntegrityError:
msg = "Tag with name %s and lifetime start %s already exists"
raise TagAlreadyCreatedException(msg % (tag_name, now_ts))