def verify_replication_for(namespace, repo_name, tag_name):
repo_ref = registry_model.lookup_repository(namespace, repo_name)
assert repo_ref
tag = registry_model.get_repo_tag(repo_ref, tag_name)
assert tag
manifest = registry_model.get_manifest_for_tag(tag)
assert manifest
for layer in registry_model.list_manifest_layers(manifest, storage):
if layer.blob.digest != EMPTY_LAYER_BLOB_DIGEST:
QueueItem.select().where(
QueueItem.queue_name**("%" + layer.blob.uuid + "%")).get()
return "OK"
def _select_available_item(self, ordering_required, now):
""" Selects an available queue item from the queue table and returns it, if any. If none,
return None.
"""
name_match_query = self._name_match_query()
try:
if ordering_required:
# The previous solution to this used a select for update in a
# transaction to prevent multiple instances from processing the
# same queue item. This suffered performance problems. This solution
# instead has instances attempt to update the potential queue item to be
# unavailable. However, since their update clause is restricted to items
# that are available=False, only one instance's update will succeed, and
# it will have a changed row count of 1. Instances that have 0 changed
# rows know that another instance is already handling that item.
running = self._running_jobs(now, name_match_query)
avail = self._available_jobs_not_running(
now, name_match_query, running)
return avail.order_by(QueueItem.id).get()
else:
# If we don't require ordering, we grab a random item from any of the first 50 available.
subquery = self._available_jobs(
now, name_match_query).limit(50).alias("j1")
return (QueueItem.select().join(
subquery, on=QueueItem.id == subquery.c.id).order_by(
db_random_func()).get())
except QueueItem.DoesNotExist:
# No available queue item was found.
return None
def num_alive_jobs(self, canonical_name_list):
""" Returns the number of alive queue items with a given prefix. """
def strip_slash(name):
return name.lstrip("/")
canonical_name_list = map(strip_slash, canonical_name_list)
canonical_name_query = "/".join([self._queue_name] +
canonical_name_list) + "%"
return (QueueItem.select().where(
QueueItem.queue_name**canonical_name_query).where(
QueueItem.retries_remaining > 0).count())
def delete_expired(expiration_threshold, deletion_threshold, batch_size):
"""
Deletes all queue items that are older than the provided expiration threshold in batches of the
provided size. If there are less items than the deletion threshold, this method does nothing.
Returns the number of items deleted.
"""
to_delete = list(QueueItem.select().where(
QueueItem.processing_expires <= expiration_threshold).limit(
batch_size))
if len(to_delete) < deletion_threshold:
return 0
QueueItem.delete().where(QueueItem.id << to_delete).execute()
return len(to_delete)
def alive(self, canonical_name_list):
"""
Returns True if a job matching the canonical name list is currently processing
or available.
"""
canonical_name = self._canonical_name([self._queue_name] + canonical_name_list)
try:
select_query = QueueItem.select().where(QueueItem.queue_name == canonical_name)
now = datetime.utcnow()
overall_query = (self._available_jobs_where(select_query.clone(), now) |
self._running_jobs_where(select_query.clone(), now))
overall_query.get()
return True
except QueueItem.DoesNotExist:
return False
def _available_jobs(cls, now, name_match_query):
return cls._available_jobs_where(QueueItem.select(), now).where(
QueueItem.queue_name**name_match_query)
def _running_jobs(cls, now, name_match_query):
return cls._running_jobs_where(
QueueItem.select(QueueItem.queue_name),
now).where(QueueItem.queue_name**name_match_query)
def _item_by_id_for_update(queue_id):
return db_for_update(
QueueItem.select().where(QueueItem.id == queue_id)).get()
def get_storage_replication_entry(image_id):
image = Image.get(docker_image_id=image_id)
QueueItem.select().where(
QueueItem.queue_name**("%" + image.storage.uuid + "%")).get()
return "OK"
def ask_disable_namespace(username, queue_name):
user = model.user.get_namespace_user(username)
if user is None:
raise Exception("Unknown user or organization %s" % username)
if not user.enabled:
print("NOTE: Namespace %s is already disabled" % username)
queue_prefix = "%s/%s/%%" % (queue_name, username)
existing_queue_item_count = (QueueItem.select().where(
QueueItem.queue_name**queue_prefix).where(
QueueItem.available == 1,
QueueItem.retries_remaining > 0,
QueueItem.processing_expires > datetime.now(),
).count())
repository_trigger_count = (
RepositoryBuildTrigger.select().join(Repository).where(
Repository.namespace_user == user).count())
print("=============================================")
print("For namespace %s" % username)
print("=============================================")
print("User %s has email address %s" % (username, user.email))
print("User %s has %s queued builds in their namespace" %
(username, existing_queue_item_count))
print("User %s has %s build triggers in their namespace" %
(username, repository_trigger_count))
confirm_msg = (
"Would you like to disable this user and delete their triggers and builds? [y/N]> "
)
letter = str(input(confirm_msg))
if letter.lower() != "y":
print("Action canceled")
return
print("=============================================")
triggers = []
count_removed = 0
with db_transaction():
user.enabled = False
user.save()
repositories_query = Repository.select().where(
Repository.namespace_user == user)
if len(repositories_query.clone()):
builds = list(RepositoryBuild.select().where(
RepositoryBuild.repository << list(repositories_query)))
triggers = list(RepositoryBuildTrigger.select().where(
RepositoryBuildTrigger.repository << list(repositories_query)))
mirrors = list(RepoMirrorConfig.select().where(
RepoMirrorConfig.repository << list(repositories_query)))
# Delete all builds for the user's repositories.
if builds:
RepositoryBuild.delete().where(
RepositoryBuild.id << builds).execute()
# Delete all build triggers for the user's repositories.
if triggers:
RepositoryBuildTrigger.delete().where(
RepositoryBuildTrigger.id << triggers).execute()
# Delete all mirrors for the user's repositories.
if mirrors:
RepoMirrorConfig.delete().where(
RepoMirrorConfig.id << mirrors).execute()
# Delete all queue items for the user's namespace.
dockerfile_build_queue = WorkQueue(queue_name,
tf,
has_namespace=True)
count_removed = dockerfile_build_queue.delete_namespaced_items(
user.username)
info = (user.username, len(triggers), count_removed, len(mirrors))
print(
"Namespace %s disabled, %s triggers deleted, %s queued builds removed, %s mirrors deleted"
% info)
return user
