def set_mirroring_robot(repository, robot):
"""
Sets the mirroring robot for the repository.
"""
assert robot.robot
namespace, _ = parse_robot_username(robot.username)
if namespace != repository.namespace_user.username:
raise DataModelException("Cannot use robot for mirroring")
mirror = get_mirror(repository)
mirror.internal_robot = robot
mirror.save()
def find_matching_team_invite(code, user_obj):
""" Finds a team invite with the given code that applies to the given user and returns it or
raises a DataModelException if not found. """
found = lookup_team_invite(code)
# If the invite is for a specific user, we have to confirm that here.
if found.user is not None and found.user != user_obj:
message = """This invite is intended for user "%s".
Please login to that account and try again.""" % found.user.username
raise DataModelException(message)
return found
def create_email_authorization_for_repo(namespace_name, repository_name,
email):
try:
repo = _basequery.get_existing_repository(namespace_name,
repository_name)
except Repository.DoesNotExist:
raise DataModelException("Invalid repository %s/%s" %
(namespace_name, repository_name))
return RepositoryAuthorizedEmail.create(repository=repo,
email=email,
confirmed=False)
def delete_tag(self, repository_ref, tag_name):
"""
Deletes the latest, *active* tag with the given name in the repository.
"""
deleted_tag = oci.tag.delete_tag(repository_ref._db_id, tag_name)
if deleted_tag is None:
# TODO: This is only needed because preoci raises an exception. Remove and fix
# expected status codes once PreOCIModel is gone.
msg = ('Invalid repository tag \'%s\' on repository' % tag_name)
raise DataModelException(msg)
return Tag.for_tag(deleted_tag)
def confirm_team_invite(code, user_obj):
"""
Confirms the given team invite code for the given user by adding the user to the team and
deleting the code.
Raises a DataModelException if the code was not found or does not apply to the given user. If
the user is invited to two or more teams under the same organization, they are automatically
confirmed for all of them.
"""
found = find_matching_team_invite(code, user_obj)
# Find all matching invitations for the user under the organization.
code_found = False
for invite in find_organization_invites(found.team.organization, user_obj):
# Add the user to the team.
try:
code_found = True
add_user_to_team(user_obj, invite.team)
except UserAlreadyInTeam:
# Ignore.
pass
# Delete the invite and return the team.
invite.delete_instance()
if not code_found:
if found.user:
message = ("""This invite is intended for user "%s".
Please login to that account and try again.""" %
found.user.username)
raise DataModelException(message)
else:
message = ("""This invite is intended for email "%s".
Please login to that account and try again.""" %
found.email)
raise DataModelException(message)
team = found.team
inviter = found.inviter
return (team, inviter)
def _logs_query(
selections,
start_time=None,
end_time=None,
performer=None,
repository=None,
namespace=None,
ignore=None,
model=LogEntry3,
id_range=None,
namespace_id=None,
):
""" Returns a query for selecting logs from the table, with various options and filters. """
if namespace is not None:
assert namespace_id is None
if namespace_id is not None:
assert namespace is None
assert (start_time is not None and end_time is not None) or (id_range
is not None)
joined = model.select(*selections).switch(model)
if id_range is not None:
joined = joined.where(model.id >= id_range[0], model.id <= id_range[1])
else:
joined = joined.where(model.datetime >= start_time,
model.datetime < end_time)
if repository:
joined = joined.where(model.repository == repository)
if performer:
joined = joined.where(model.performer == performer)
if namespace and not repository:
namespace_user = user.get_user_or_org(namespace)
if namespace_user is None:
raise DataModelException("Invalid namespace requested: %s" %
namespace)
joined = joined.where(model.account == namespace_user.id)
if namespace_id is not None and not repository:
joined = joined.where(model.account == namespace_id)
if ignore:
kind_map = get_log_entry_kinds()
ignore_ids = [kind_map[kind_name] for kind_name in ignore]
joined = joined.where(~(model.kind << ignore_ids))
return joined
def _get_repo_tag_image(tag_name, include_storage, modifier):
query = Image.select().join(RepositoryTag)
if include_storage:
query = (Image.select(
Image,
ImageStorage).join(ImageStorage).switch(Image).join(RepositoryTag))
images = _tag_alive(modifier(query.where(RepositoryTag.name == tag_name)))
if not images:
raise DataModelException("Unable to find image for tag.")
else:
return images[0]
def delete_tag(self, repository_ref, tag_name):
"""
Deletes the latest, *active* tag with the given name in the repository.
"""
with db_disallow_replica_use():
deleted_tag = oci.tag.delete_tag(repository_ref._db_id, tag_name)
if deleted_tag is None:
# TODO: This is only needed because preoci raises an exception. Remove and fix
# expected status codes once PreOCIModel is gone.
msg = "Invalid repository tag '%s' on repository" % tag_name
raise DataModelException(msg)
return Tag.for_tag(deleted_tag, self._legacy_image_id_handler)
def set_user_repo_permission(username, namespace_name, repository_name,
role_name):
if username == namespace_name:
raise DataModelException("Namespace owner must always be admin.")
try:
user = User.get(User.username == username)
except User.DoesNotExist:
raise DataModelException("Invalid username: %s" % username)
if user.robot:
parts = parse_robot_username(user.username)
if not parts:
raise DataModelException("Invalid robot: %s" % username)
robot_namespace, _ = parts
if robot_namespace != namespace_name:
raise DataModelException("Cannot add robot %s under namespace %s" %
(username, namespace_name))
return __set_entity_repo_permission(user, "user", namespace_name,
repository_name, role_name)
def change_user_tag_expiration(user, tag_expiration_s):
"""
Changes the tag expiration on the given user/org.
Note that the specified expiration must be within the configured TAG_EXPIRATION_OPTIONS or this
method will raise a DataModelException.
"""
allowed_options = [_convert_to_s(o) for o in config.app_config["TAG_EXPIRATION_OPTIONS"]]
if tag_expiration_s not in allowed_options:
raise DataModelException("Invalid tag expiration option")
user.removed_tag_expiration_s = tag_expiration_s
user.save()
def set_image_metadata(
docker_image_id,
namespace_name,
repository_name,
created_date_str,
comment,
command,
v1_json_metadata,
parent=None,
):
"""
Sets metadata that is specific to how a binary piece of storage fits into the layer tree.
"""
with db_transaction():
try:
fetched = (
Image.select(Image, ImageStorage)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(Image)
.join(ImageStorage)
.where(
Repository.name == repository_name,
Namespace.username == namespace_name,
Image.docker_image_id == docker_image_id,
)
.get()
)
except Image.DoesNotExist:
raise DataModelException("No image with specified id and repository")
fetched.created = datetime.now()
if created_date_str is not None:
try:
fetched.created = dateutil.parser.parse(created_date_str).replace(tzinfo=None)
except:
# parse raises different exceptions, so we cannot use a specific kind of handler here.
pass
# We cleanup any old checksum in case it's a retry after a fail
fetched.v1_checksum = None
fetched.comment = comment
fetched.command = command
fetched.v1_json_metadata = v1_json_metadata
if parent:
fetched.ancestors = "%s%s/" % (parent.ancestors, parent.id)
fetched.parent = parent
fetched.save()
return fetched
def create_user_noverify(username, email, email_required=True, prompts=tuple(),
is_possible_abuser=False):
if email_required:
if not validate_email(email):
raise InvalidEmailAddressException('Invalid email address: %s' % email)
else:
# If email addresses are not required and none was specified, then we just use a unique
# ID to ensure that the database consistency check remains intact.
email = email or str(uuid.uuid4())
(username_valid, username_issue) = validate_username(username)
if not username_valid:
raise InvalidUsernameException('Invalid namespace %s: %s' % (username, username_issue))
try:
existing = User.get((User.username == username) | (User.email == email))
logger.info('Existing user with same username or email.')
# A user already exists with either the same username or email
if existing.username == username:
assert not existing.robot
msg = 'Username has already been taken by an organization and cannot be reused: %s' % username
if not existing.organization:
msg = 'Username has already been taken by user cannot be reused: %s' % username
raise InvalidUsernameException(msg)
raise InvalidEmailAddressException('Email has already been used: %s' % email)
except User.DoesNotExist:
# This is actually the happy path
logger.debug('Email and username are unique!')
# Create the user.
try:
default_expr_s = _convert_to_s(config.app_config['DEFAULT_TAG_EXPIRATION'])
default_max_builds = config.app_config.get('DEFAULT_NAMESPACE_MAXIMUM_BUILD_COUNT')
threat_max_builds = config.app_config.get('THREAT_NAMESPACE_MAXIMUM_BUILD_COUNT')
if is_possible_abuser and threat_max_builds is not None:
default_max_builds = threat_max_builds
new_user = User.create(username=username, email=email, removed_tag_expiration_s=default_expr_s,
maximum_queued_builds_count=default_max_builds)
for prompt in prompts:
create_user_prompt(new_user, prompt)
return new_user
except Exception as ex:
raise DataModelException(ex.message)
def remove_user_from_team(org_name, team_name, username, removed_by_username):
Org = User.alias()
joined = TeamMember.select().join(User).switch(TeamMember).join(Team)
with_role = joined.join(TeamRole)
with_org = with_role.switch(Team).join(Org,
on=(Org.id == Team.organization))
found = list(
with_org.where(User.username == username, Org.username == org_name,
Team.name == team_name))
if not found:
raise DataModelException("User %s does not belong to team %s" %
(username, team_name))
if username == removed_by_username:
admin_team_query = __get_user_admin_teams(org_name, username)
admin_team_names = {team.name for team in admin_team_query}
if team_name in admin_team_names and len(admin_team_names) <= 1:
msg = "User cannot remove themselves from their only admin team."
raise DataModelException(msg)
user_in_team = found[0]
user_in_team.delete_instance()
def store_tag_manifest_for_testing(namespace_name, repository_name, tag_name,
manifest, leaf_layer_id, storage_id_map):
""" Stores a tag manifest for a specific tag name in the database. Returns the TagManifest
object, as well as a boolean indicating whether the TagManifest was created.
"""
try:
repo = _basequery.get_existing_repository(namespace_name,
repository_name)
except Repository.DoesNotExist:
raise DataModelException("Invalid repository %s/%s" %
(namespace_name, repository_name))
return store_tag_manifest_for_repo(repo.id, tag_name, manifest,
leaf_layer_id, storage_id_map)
def get_user_starred_repositories(user, kind_filter="image"):
""" Retrieves all of the repositories a user has starred. """
try:
repo_kind = Repository.kind.get_id(kind_filter)
except RepositoryKind.DoesNotExist:
raise DataModelException("Unknown kind of repository")
query = (Repository.select(
Repository, User, Visibility,
Repository.id.alias("rid")).join(Star).switch(Repository).join(
User).switch(Repository).join(Visibility).where(
Star.user == user, Repository.kind == repo_kind))
return query
def confirm_user_email(token):
# TODO(remove-unenc): Remove allow_public_only once migrated.
allow_public_only = ActiveDataMigration.has_flag(
ERTMigrationFlags.READ_OLD_FIELDS)
result = decode_public_private_token(token,
allow_public_only=allow_public_only)
if not result:
raise DataModelException("Invalid email confirmation code")
try:
code = EmailConfirmation.get(
EmailConfirmation.code == result.public_code,
EmailConfirmation.email_confirm == True)
except EmailConfirmation.DoesNotExist:
raise DataModelException("Invalid email confirmation code")
if result.private_token and not code.verification_code.matches(
result.private_token):
raise DataModelException("Invalid email confirmation code")
user = code.user
user.verified = True
old_email = None
new_email = code.new_email
if new_email and new_email != old_email:
if find_user_by_email(new_email):
raise DataModelException("E-mail address already used")
old_email = user.email
user.email = new_email
with db_transaction():
user.save()
code.delete_instance()
return user, new_email, old_email
def confirm_email_authorization_for_repo(code):
try:
found = (RepositoryAuthorizedEmail.select(
RepositoryAuthorizedEmail, Repository,
Namespace).join(Repository).join(
Namespace, on=(Repository.namespace_user == Namespace.id)
).where(RepositoryAuthorizedEmail.code == code).where(
Repository.state != RepositoryState.MARKED_FOR_DELETION).get())
except RepositoryAuthorizedEmail.DoesNotExist:
raise DataModelException("Invalid confirmation code.")
found.confirmed = True
found.save()
return found
def set_team_org_permission(team, team_role_name, set_by_username):
if team.role.name == "admin" and team_role_name != "admin":
# We need to make sure we're not removing the users only admin role
user_admin_teams = __get_user_admin_teams(team.organization.username,
set_by_username)
admin_team_set = {admin_team.name for admin_team in user_admin_teams}
if team.name in admin_team_set and len(admin_team_set) <= 1:
msg = ("Cannot remove admin from team '%s' because calling user " +
"would no longer have admin on org '%s'") % (
team.name, team.organization.username)
raise DataModelException(msg)
new_role = TeamRole.get(TeamRole.name == team_role_name)
team.role = new_role
team.save()
return team
def create_or_update_tag(namespace_name,
repository_name,
tag_name,
tag_docker_image_id,
reversion=False,
now_ms=None):
try:
repo = _basequery.get_existing_repository(namespace_name,
repository_name)
except Repository.DoesNotExist:
raise DataModelException("Invalid repository %s/%s" %
(namespace_name, repository_name))
return create_or_update_tag_for_repo(repo.id,
tag_name,
tag_docker_image_id,
reversion=reversion,
now_ms=now_ms)
def create_robot(robot_shortname,
parent,
description="",
unstructured_metadata=None):
(username_valid, username_issue) = validate_username(robot_shortname)
if not username_valid:
raise InvalidRobotException(
"The name for the robot '%s' is invalid: %s" %
(robot_shortname, username_issue))
username = format_robot_username(parent.username, robot_shortname)
try:
User.get(User.username == username)
msg = "Existing robot with name: %s" % username
logger.debug(msg)
raise InvalidRobotException(msg)
except User.DoesNotExist:
pass
service = LoginService.get(name="quayrobot")
try:
with db_transaction():
created = User.create(username=username,
email=str(uuid.uuid4()),
robot=True)
token = random_string_generator(length=64)()
RobotAccountToken.create(robot_account=created,
token=token,
fully_migrated=True)
FederatedLogin.create(user=created,
service=service,
service_ident="robot:%s" % created.id)
RobotAccountMetadata.create(
robot_account=created,
description=description[0:255],
unstructured_json=unstructured_metadata or {},
)
return created, token
except Exception as ex:
raise DataModelException(ex)
def calculate_image_aggregate_size(ancestors_str, image_size, parent_image):
ancestors = ancestors_str.split("/")[1:-1]
if not ancestors:
return image_size
if parent_image is None:
raise DataModelException("Could not load parent image")
ancestor_size = parent_image.aggregate_size
if ancestor_size is not None:
return ancestor_size + image_size
# Fallback to a slower path if the parent doesn't have an aggregate size saved.
# TODO: remove this code if/when we do a full backfill.
ancestor_size = (ImageStorage.select(fn.Sum(ImageStorage.image_size)).join(
Image).where(Image.id << ancestors).scalar())
if ancestor_size is None:
return None
return ancestor_size + image_size
def restore_tag_to_manifest(repo_obj, tag_name, manifest_digest):
"""
Restores a tag to a specific manifest digest.
"""
with db_transaction():
# Verify that the manifest digest already existed under this repository under the
# tag.
try:
tag_manifest = (TagManifest.select(
TagManifest, RepositoryTag,
Image).join(RepositoryTag).join(Image).where(
RepositoryTag.repository == repo_obj).where(
RepositoryTag.name == tag_name).where(
TagManifest.digest == manifest_digest).get())
except TagManifest.DoesNotExist:
raise DataModelException(
"Cannot restore to unknown or invalid digest")
# Lookup the existing image, if any.
try:
existing_image = get_repo_tag_image(repo_obj, tag_name)
except DataModelException:
existing_image = None
docker_image_id = tag_manifest.tag.image.docker_image_id
oci_manifest = None
try:
oci_manifest = Manifest.get(repository=repo_obj,
digest=manifest_digest)
except Manifest.DoesNotExist:
pass
# Change the tag and tag manifest to point to the updated image.
updated_tag = create_or_update_tag_for_repo(repo_obj,
tag_name,
docker_image_id,
reversion=True,
oci_manifest=oci_manifest)
tag_manifest.tag = updated_tag
tag_manifest.save()
return existing_image
def remove_team(org_name, team_name, removed_by_username):
joined = Team.select(Team, TeamRole).join(User).switch(Team).join(TeamRole)
found = list(
joined.where(User.organization == True, User.username == org_name,
Team.name == team_name))
if not found:
raise InvalidTeamException("Team '%s' is not a team in org '%s'" %
(team_name, org_name))
team = found[0]
if team.role.name == "admin":
admin_teams = list(
__get_user_admin_teams(org_name, removed_by_username))
if len(admin_teams) <= 1:
# The team we are trying to remove is the only admin team containing this user.
msg = "Deleting team '%s' would remove admin ability for user '%s' in organization '%s'"
raise DataModelException(
msg % (team_name, removed_by_username, org_name))
team.delete_instance(recursive=True, delete_nullable=True)
def delete_tag(namespace_name, repository_name, tag_name, now_ms=None):
now_ms = now_ms or get_epoch_timestamp_ms()
now_ts = int(now_ms / 1000)
with db_transaction():
try:
query = _tag_alive(
RepositoryTag.select(
RepositoryTag, Repository).join(Repository).join(
Namespace,
on=(Repository.namespace_user == Namespace.id)).where(
Repository.name == repository_name,
Namespace.username == namespace_name,
RepositoryTag.name == tag_name,
),
now_ts,
)
found = db_for_update(query).get()
except RepositoryTag.DoesNotExist:
msg = "Invalid repository tag '%s' on repository '%s/%s'" % (
tag_name,
namespace_name,
repository_name,
)
raise DataModelException(msg)
found.lifetime_end_ts = now_ts
found.save()
try:
oci_tag_query = TagToRepositoryTag.select().where(
TagToRepositoryTag.repository_tag == found)
oci_tag = db_for_update(oci_tag_query).get().tag
oci_tag.lifetime_end_ms = now_ms
oci_tag.save()
except TagToRepositoryTag.DoesNotExist:
pass
return found
def set_image_storage_metadata(
docker_image_id, namespace_name, repository_name, image_size, uncompressed_size
):
"""
Sets metadata that is specific to the binary storage of the data, irrespective of how it is used
in the layer tree.
"""
if image_size is None:
raise DataModelException("Empty image size field")
try:
image = (
Image.select(Image, ImageStorage)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(Image)
.join(ImageStorage)
.where(
Repository.name == repository_name,
Namespace.username == namespace_name,
Image.docker_image_id == docker_image_id,
)
.get()
)
except ImageStorage.DoesNotExist:
raise InvalidImageException("No image with specified id and repository")
# We MUST do this here, it can't be done in the corresponding image call because the storage
# has not yet been pushed
image.aggregate_size = _basequery.calculate_image_aggregate_size(
image.ancestors, image_size, image.parent
)
image.save()
image.storage.image_size = image_size
image.storage.uncompressed_size = uncompressed_size
image.storage.save()
return image.storage
def _latest_logs_query(
selections,
performer=None,
repository=None,
namespace=None,
ignore=None,
model=LogEntry3,
size=None,
):
"""
Returns a query for selecting the latest logs from the table, with various options and filters.
"""
query = model.select(*selections).switch(model)
if repository:
query = query.where(model.repository == repository)
if performer:
query = query.where(model.repository == repository)
if namespace and not repository:
namespace_user = user.get_user_or_org(namespace)
if namespace_user is None:
raise DataModelException("Invalid namespace requested")
query = query.where(model.account == namespace_user.id)
if ignore:
kind_map = get_log_entry_kinds()
ignore_ids = [kind_map[kind_name] for kind_name in ignore]
query = query.where(~(model.kind << ignore_ids))
query = query.order_by(model.datetime.desc(), model.id)
if size:
query = query.limit(size)
return query
def convert_user_to_organization(user_obj, admin_user):
if user_obj.robot:
raise DataModelException("Cannot convert a robot into an organization")
with db_transaction():
# Change the user to an organization and disable this account for login.
user_obj.organization = True
user_obj.password_hash = None
user_obj.save()
# Clear any federated auth pointing to this user.
FederatedLogin.delete().where(FederatedLogin.user == user_obj).execute()
# Delete any user-specific permissions on repositories.
(RepositoryPermission.delete().where(RepositoryPermission.user == user_obj).execute())
# Create a team for the owners
owners_team = team.create_team("owners", user_obj, "admin")
# Add the user who will admin the org to the owners team
team.add_user_to_team(admin_user, owners_team)
return user_obj
def mark_namespace_for_deletion(user, queues, namespace_gc_queue, force=False):
"""
Marks a namespace (as referenced by the given user) for deletion.
A queue item will be added to delete the namespace's repositories and storage, while the
namespace itself will be renamed, disabled, and delinked from other tables.
"""
if not user.enabled:
return None
if not force and not user.organization:
# Ensure that the user is not the sole admin for any organizations. If so, then the user
# cannot be deleted before those organizations are deleted or reassigned.
organizations = get_solely_admined_organizations(user)
if len(organizations) > 0:
message = (
"Cannot delete %s as you are the only admin for organizations: "
% user.username)
for index, org in enumerate(organizations):
if index > 0:
message = message + ", "
message = message + org.username
raise DataModelException(message)
# Delete all queue items for the user.
for queue in queues:
queue.delete_namespaced_items(user.username)
# Delete non-repository related items. This operation is very quick, so we can do so here.
_delete_user_linked_data(user)
with db_transaction():
original_username = user.username
user = db_for_update(User.select().where(User.id == user.id)).get()
# Mark the namespace as deleted and ready for GC.
try:
marker = DeletedNamespace.create(
namespace=user,
original_username=original_username,
original_email=user.email)
except IntegrityError:
return
# Disable the namespace itself, and replace its various unique fields with UUIDs.
user.enabled = False
user.username = str(uuid4())
user.email = str(uuid4())
user.save()
# Add a queueitem to delete the namespace.
marker.queue_id = namespace_gc_queue.put(
[str(user.id)],
json.dumps({
"marker_id": marker.id,
"original_username": original_username,
}),
)
marker.save()
return marker.id
def create_or_update_tag_for_repo(repository_id,
tag_name,
tag_docker_image_id,
reversion=False,
oci_manifest=None,
now_ms=None):
now_ms = now_ms or get_epoch_timestamp_ms()
now_ts = int(now_ms / 1000)
with db_transaction():
try:
tag = db_for_update(
_tag_alive(
RepositoryTag.select().where(
RepositoryTag.repository == repository_id,
RepositoryTag.name == tag_name),
now_ts,
)).get()
tag.lifetime_end_ts = now_ts
tag.save()
# Check for an OCI tag.
try:
oci_tag = db_for_update(
Tag.select().join(TagToRepositoryTag).where(
TagToRepositoryTag.repository_tag == tag)).get()
oci_tag.lifetime_end_ms = now_ms
oci_tag.save()
except Tag.DoesNotExist:
pass
except RepositoryTag.DoesNotExist:
pass
except IntegrityError:
msg = "Tag with name %s was stale when we tried to update it; Please retry the push"
raise StaleTagException(msg % tag_name)
try:
image_obj = Image.get(Image.docker_image_id == tag_docker_image_id,
Image.repository == repository_id)
except Image.DoesNotExist:
raise DataModelException("Invalid image with id: %s" %
tag_docker_image_id)
try:
created = RepositoryTag.create(
repository=repository_id,
image=image_obj,
name=tag_name,
lifetime_start_ts=now_ts,
reversion=reversion,
)
if oci_manifest:
# Create the OCI tag as well.
oci_tag = Tag.create(
repository=repository_id,
manifest=oci_manifest,
name=tag_name,
lifetime_start_ms=now_ms,
reversion=reversion,
tag_kind=Tag.tag_kind.get_id("tag"),
)
TagToRepositoryTag.create(tag=oci_tag,
repository_tag=created,
repository=repository_id)
return created
except IntegrityError:
msg = "Tag with name %s and lifetime start %s already exists"
raise TagAlreadyCreatedException(msg % (tag_name, now_ts))
