def get_archivable_build():
presumed_dead_date = datetime.utcnow() - PRESUMED_DEAD_BUILD_AGE
candidates = (RepositoryBuild.select(RepositoryBuild.id).where(
(RepositoryBuild.phase << ARCHIVABLE_BUILD_PHASES) |
(RepositoryBuild.started < presumed_dead_date),
RepositoryBuild.logs_archived == False).limit(50).alias('candidates'))
try:
found_id = (RepositoryBuild.select(
candidates.c.id).from_(candidates).order_by(
db_random_func()).get())
return RepositoryBuild.get(id=found_id)
except RepositoryBuild.DoesNotExist:
return None
def mark_build_archived(build_uuid):
"""
Mark a build as archived, and return True if we were the ones who actually updated the row.
"""
return (RepositoryBuild.update(logs_archived=True).where(
RepositoryBuild.uuid == build_uuid, RepositoryBuild.logs_archived
== False).execute()) > 0
def test_archivable_build_logs(initialized_db):
# Make sure there are no archivable logs.
result = model.build.get_archivable_build()
assert result is None
# Add a build that cannot (yet) be archived.
repo = model.repository.get_repository("devtable", "simple")
token = model.token.create_access_token(repo, "write")
created = RepositoryBuild.create(
repository=repo,
access_token=token,
phase=model.build.BUILD_PHASE.WAITING,
logs_archived=False,
job_config="{}",
display_name="",
)
# Make sure there are no archivable logs.
result = model.build.get_archivable_build()
assert result is None
# Change the build to being complete.
created.phase = model.build.BUILD_PHASE.COMPLETE
created.save()
# Make sure we now find an archivable build.
result = model.build.get_archivable_build()
assert result.id == created.id
def delete_temporary_access_tokens(older_than):
# Find the highest ID up to which we should delete
up_to_id = (AccessToken.select(AccessToken.id).where(
AccessToken.created < older_than).limit(1).order_by(
AccessToken.id.desc()).get().id)
logger.debug('Deleting temporary access tokens with ids lower than: %s',
up_to_id)
access_tokens_in_builds = (RepositoryBuild.select(
RepositoryBuild.access_token).distinct())
while up_to_id > 0:
starting_at_id = max(up_to_id - BATCH_SIZE, 0)
logger.debug('Deleting tokens with ids between %s and %s',
starting_at_id, up_to_id)
start_time = datetime.utcnow()
(AccessToken.delete().where(
AccessToken.id >= starting_at_id, AccessToken.id < up_to_id,
AccessToken.temporary == True,
~(AccessToken.id << access_tokens_in_builds)).execute())
time_to_delete = datetime.utcnow() - start_time
up_to_id -= BATCH_SIZE
logger.debug('Sleeping for %s seconds', time_to_delete.total_seconds())
time.sleep(time_to_delete.total_seconds())
def _get_build_base_query():
return (RepositoryBuild.select(
RepositoryBuild, RepositoryBuildTrigger, BuildTriggerService,
Repository, Namespace, User).join(Repository).join(
Namespace,
on=(Repository.namespace_user == Namespace.id
)).switch(RepositoryBuild).join(
User, JOIN.LEFT_OUTER).switch(RepositoryBuild).join(
RepositoryBuildTrigger, JOIN.LEFT_OUTER).join(
BuildTriggerService, JOIN.LEFT_OUTER).order_by(
RepositoryBuild.started.desc()))
def update_phase_then_close(build_uuid, phase):
""" A function to change the phase of a build """
with UseThenDisconnect(config.app_config):
try:
build = _get_build_row(build_uuid)
except RepositoryBuild.DoesNotExist:
return False
# Can't update a cancelled build
if build.phase == BUILD_PHASE.CANCELLED:
return False
updated = (RepositoryBuild.update(phase=phase).where(
RepositoryBuild.id == build.id,
RepositoryBuild.phase == build.phase).execute())
return updated > 0
def create_repository_build(repo,
access_token,
job_config_obj,
dockerfile_id,
display_name,
trigger=None,
pull_robot_name=None):
pull_robot = None
if pull_robot_name:
pull_robot = user_model.lookup_robot(pull_robot_name)
return RepositoryBuild.create(repository=repo,
access_token=access_token,
job_config=json.dumps(job_config_obj),
display_name=display_name,
trigger=trigger,
resource_key=dockerfile_id,
pull_robot=pull_robot)
def _get_build_row(build_uuid):
return RepositoryBuild.select().where(
RepositoryBuild.uuid == build_uuid).get()
def ask_disable_namespace(username, queue_name):
user = model.user.get_namespace_user(username)
if user is None:
raise Exception("Unknown user or organization %s" % username)
if not user.enabled:
print("NOTE: Namespace %s is already disabled" % username)
queue_prefix = "%s/%s/%%" % (queue_name, username)
existing_queue_item_count = (QueueItem.select().where(
QueueItem.queue_name**queue_prefix).where(
QueueItem.available == 1,
QueueItem.retries_remaining > 0,
QueueItem.processing_expires > datetime.now(),
).count())
repository_trigger_count = (
RepositoryBuildTrigger.select().join(Repository).where(
Repository.namespace_user == user).count())
print("=============================================")
print("For namespace %s" % username)
print("=============================================")
print("User %s has email address %s" % (username, user.email))
print("User %s has %s queued builds in their namespace" %
(username, existing_queue_item_count))
print("User %s has %s build triggers in their namespace" %
(username, repository_trigger_count))
confirm_msg = (
"Would you like to disable this user and delete their triggers and builds? [y/N]> "
)
letter = str(input(confirm_msg))
if letter.lower() != "y":
print("Action canceled")
return
print("=============================================")
triggers = []
count_removed = 0
with db_transaction():
user.enabled = False
user.save()
repositories_query = Repository.select().where(
Repository.namespace_user == user)
if len(repositories_query.clone()):
builds = list(RepositoryBuild.select().where(
RepositoryBuild.repository << list(repositories_query)))
triggers = list(RepositoryBuildTrigger.select().where(
RepositoryBuildTrigger.repository << list(repositories_query)))
mirrors = list(RepoMirrorConfig.select().where(
RepoMirrorConfig.repository << list(repositories_query)))
# Delete all builds for the user's repositories.
if builds:
RepositoryBuild.delete().where(
RepositoryBuild.id << builds).execute()
# Delete all build triggers for the user's repositories.
if triggers:
RepositoryBuildTrigger.delete().where(
RepositoryBuildTrigger.id << triggers).execute()
# Delete all mirrors for the user's repositories.
if mirrors:
RepoMirrorConfig.delete().where(
RepoMirrorConfig.id << mirrors).execute()
# Delete all queue items for the user's namespace.
dockerfile_build_queue = WorkQueue(queue_name,
tf,
has_namespace=True)
count_removed = dockerfile_build_queue.delete_namespaced_items(
user.username)
info = (user.username, len(triggers), count_removed, len(mirrors))
print(
"Namespace %s disabled, %s triggers deleted, %s queued builds removed, %s mirrors deleted"
% info)
return user
