def _cleanup_uploads(self):
"""
Performs cleanup on the blobupload table.
"""
logger.debug("Performing blob upload cleanup")
while True:
# Find all blob uploads older than the threshold (typically a week) and delete them.
with UseThenDisconnect(app.config):
stale_upload = model.get_stale_blob_upload(DELETION_DATE_THRESHOLD)
if stale_upload is None:
logger.debug("No additional stale blob uploads found")
return
# Remove the stale upload from storage.
logger.debug("Removing stale blob upload %s", stale_upload.uuid)
assert stale_upload.created <= (datetime.utcnow() - DELETION_DATE_THRESHOLD)
try:
storage.cancel_chunked_upload(
[stale_upload.location_name], stale_upload.uuid, stale_upload.storage_metadata
)
except Exception as ex:
logger.debug(
"Got error when trying to cancel chunked upload %s: %s",
stale_upload.uuid,
ex.message,
)
# Delete the stale upload's row.
with UseThenDisconnect(app.config):
model.delete_blob_upload(stale_upload)
logger.debug("Removed stale blob upload %s", stale_upload.uuid)
def index_images(target_version, analyzer, token=None):
""" Performs security indexing of all images in the database not scanned at the target version.
If a token is provided, scanning will begin where the token indicates it previously completed.
"""
iterator, next_token = model.candidates_to_scan(target_version,
start_token=token)
if iterator is None:
logger.debug("Found no additional images to scan")
return None
with UseThenDisconnect(app.config):
for candidate, abt, num_remaining in iterator:
try:
analyzer.analyze_recursively(candidate)
except PreemptedException:
logger.info("Another worker pre-empted us for layer: %s",
candidate.id)
abt.set()
except APIRequestFailure:
logger.exception("Security scanner service unavailable")
return
unscanned_images_gauge.Set(num_remaining)
return next_token
def _garbage_collection_repos(self, skip_lock_for_testing=False):
"""
Performs garbage collection on repositories.
"""
with UseThenDisconnect(app.config):
policy = get_random_gc_policy()
if policy is None:
logger.debug("No GC policies found")
return
repo_ref = registry_model.find_repository_with_garbage(policy)
if repo_ref is None:
logger.debug("No repository with garbage found")
return
assert features.GARBAGE_COLLECTION
try:
with GlobalLock(
"REPO_GARBAGE_COLLECTION_%s" % repo_ref.id,
lock_ttl=REPOSITORY_GC_TIMEOUT + LOCK_TIMEOUT_PADDING,
) if not skip_lock_for_testing else empty_context():
try:
repository = Repository.get(id=repo_ref.id)
except Repository.DoesNotExist:
return
logger.debug("Starting GC of repository #%s (%s)",
repository.id, repository.name)
garbage_collect_repo(repository)
logger.debug("Finished GC of repository #%s (%s)",
repository.id, repository.name)
except LockNotAcquiredException:
logger.debug(
"Could not acquire repo lock for garbage collection")
def perform_indexing(self, start_token=None):
"""
Performs indexing of the next set of unindexed manifests/images.
If start_token is given, the indexing should resume from that point. Returns a new start
index for the next iteration of indexing. The tokens returned and given are assumed to be
opaque outside of this implementation and should not be relied upon by the caller to conform
to any particular format.
"""
# NOTE: This import is in here because otherwise this class would depend upon app.
# Its not great, but as this is intended to be legacy until its removed, its okay.
from util.secscan.analyzer import PreemptedException
iterator, next_token = self._candidates_to_scan(start_token)
if iterator is None:
logger.debug("Found no additional images to scan")
return None
with UseThenDisconnect(self.app.config):
for candidate, abt, num_remaining in iterator:
try:
self._analyzer.analyze_recursively(candidate)
except PreemptedException:
logger.debug("Another worker pre-empted us for layer: %s",
candidate.id)
abt.set()
except APIRequestFailure:
logger.exception("Security scanner service unavailable")
return
unscanned_images.set(num_remaining)
return next_token
def _report_stats(self):
logger.debug("Reporting global stats")
with UseThenDisconnect(app.config):
repository_rows.set(get_repository_count())
user_rows.set(get_active_user_count())
org_rows.set(get_active_org_count())
robot_rows.set(get_robot_count())
def yield_log_rotation_context(self, cutoff_date, min_logs_per_rotation):
"""
Yield a context manager for a group of outdated logs.
"""
for log_model in LOG_MODELS:
while True:
with UseThenDisconnect(config.app_config):
start_id = get_stale_logs_start_id(log_model)
if start_id is None:
logger.warning("Failed to find start id")
break
logger.debug("Found starting ID %s", start_id)
lookup_end_id = start_id + min_logs_per_rotation
logs = [
log for log in get_stale_logs(start_id, lookup_end_id,
log_model, cutoff_date)
]
if not logs:
logger.debug("No further logs found")
break
end_id = max([log.id for log in logs])
context = DatabaseLogRotationContext(logs, log_model, start_id,
end_id)
yield context
def _load_repo_build(self):
with UseThenDisconnect(app.config):
try:
return model.build.get_repository_build(self.build_uuid)
except model.InvalidRepositoryBuildException:
raise BuildJobLoadException(
"Could not load repository build with ID %s" % self.build_uuid
)
def _operation_func():
try:
with UseThenDisconnect(app.config):
return operation_func()
except Exception:
logger.exception('Operation raised exception')
if self._raven_client:
logger.debug('Logging exception to Sentry')
self._raven_client.captureException()
def _report_stats(self):
logger.debug('Reporting global stats')
with UseThenDisconnect(app.config):
# Repository count.
metric_queue.repository_count.Set(model.get_repository_count())
# User counts.
metric_queue.user_count.Set(model.get_active_user_count())
metric_queue.org_count.Set(model.get_active_org_count())
metric_queue.robot_count.Set(model.get_robot_count())
def _cleanup_queue(self):
""" Performs garbage collection on the queueitem table. """
with UseThenDisconnect(app.config):
while True:
# Find all queue items older than the threshold (typically a week) and delete them.
expiration_threshold = datetime.now() - DELETION_DATE_THRESHOLD
deleted_count = delete_expired(expiration_threshold,
DELETION_COUNT_THRESHOLD,
BATCH_SIZE)
if deleted_count == 0:
return
def _backfill_labels(self):
with UseThenDisconnect(app.config):
iterator = self._candidates_to_backfill()
if iterator is None:
logger.debug("Found no additional labels to backfill")
time.sleep(10000)
return None
for candidate, abt, _ in iterator:
if not backfill_label(candidate):
logger.info("Another worker pre-empted us for label: %s", candidate.id)
abt.set()
def _backfill_tags(self):
with UseThenDisconnect(app.config):
iterator = self._candidates_to_backfill()
if iterator is None:
logger.debug('Found no additional tags to backfill')
time.sleep(10000)
return None
for candidate, abt, _ in iterator:
if not backfill_tag(candidate):
logger.info('Another worker pre-empted us for tag: %s',
candidate.id)
abt.set()
def _garbage_collection_repos(self):
""" Performs garbage collection on repositories. """
with UseThenDisconnect(app.config):
repository = find_repository_with_garbage(get_random_gc_policy())
if repository is None:
logger.debug('No repository with garbage found')
return
assert features.GARBAGE_COLLECTION
logger.debug('Starting GC of repository #%s (%s)', repository.id,
repository.name)
garbage_collect_repo(repository)
logger.debug('Finished GC of repository #%s (%s)', repository.id,
repository.name)
def _cleanup_uploads(self):
""" Performs garbage collection on the blobupload table. """
while True:
# Find all blob uploads older than the threshold (typically a week) and delete them.
with UseThenDisconnect(app.config):
stale_upload = model.get_stale_blob_upload(DELETION_DATE_THRESHOLD)
if stale_upload is None:
logger.debug('No additional stale blob uploads found')
return
# Remove the stale upload from storage.
logger.debug('Removing stale blob upload %s', stale_upload.uuid)
try:
storage.cancel_chunked_upload([stale_upload.location_name], stale_upload.uuid,
stale_upload.storage_metadata)
except Exception as ex:
logger.debug('Got error when trying to cancel chunked upload %s: %s', stale_upload.uuid,
ex.message)
# Delete the stale upload's row.
with UseThenDisconnect(app.config):
model.delete_blob_upload(stale_upload)
logger.debug('Removed stale blob upload %s', stale_upload.uuid)
def _determine_cached_tag_by_tag(self):
""" Determines the cached tag by looking for one of the tags being built, and seeing if it
exists in the repository. This is a fallback for when no comment information is available.
"""
with UseThenDisconnect(app.config):
tags = self.build_config.get("docker_tags", ["latest"])
repository = RepositoryReference.for_repo_obj(self.repo_build.repository)
matching_tag = registry_model.find_matching_tag(repository, tags)
if matching_tag is not None:
return matching_tag.name
most_recent_tag = registry_model.get_most_recent_tag(repository)
if most_recent_tag is not None:
return most_recent_tag.name
return None
def send_notification(self,
kind,
error_message=None,
image_id=None,
manifest_digests=None):
with UseThenDisconnect(app.config):
tags = self.build_config.get("docker_tags", ["latest"])
trigger = self.repo_build.trigger
if trigger is not None and trigger.id is not None:
trigger_kind = trigger.service.name
else:
trigger_kind = None
event_data = {
"build_id": self.repo_build.uuid,
"build_name": self.repo_build.display_name,
"docker_tags": tags,
"trigger_id": trigger.uuid if trigger is not None else None,
"trigger_kind": trigger_kind,
"trigger_metadata":
self.build_config.get("trigger_metadata", {}),
}
if image_id is not None:
event_data["image_id"] = image_id
if manifest_digests:
event_data["manifest_digests"] = manifest_digests
if error_message is not None:
event_data["error_message"] = error_message
# TODO: remove when more endpoints have been converted to using
# interfaces
repo = AttrDict({
"namespace_name":
self.repo_build.repository.namespace_user.username,
"name":
self.repo_build.repository.name,
})
spawn_notification(
repo,
kind,
event_data,
subpage="build/%s" % self.repo_build.uuid,
pathargs=["build", self.repo_build.uuid],
)
def update_phase_then_close(build_uuid, phase):
""" A function to change the phase of a build """
with UseThenDisconnect(config.app_config):
try:
build = _get_build_row(build_uuid)
except RepositoryBuild.DoesNotExist:
return False
# Can't update a cancelled build
if build.phase == BUILD_PHASE.CANCELLED:
return False
updated = (RepositoryBuild.update(phase=phase).where(
RepositoryBuild.id == build.id,
RepositoryBuild.phase == build.phase).execute())
return updated > 0
def send_notification(self,
kind,
error_message=None,
image_id=None,
manifest_digests=None):
with UseThenDisconnect(app.config):
tags = self.build_config.get('docker_tags', ['latest'])
trigger = self.repo_build.trigger
if trigger is not None and trigger.id is not None:
trigger_kind = trigger.service.name
else:
trigger_kind = None
event_data = {
'build_id': self.repo_build.uuid,
'build_name': self.repo_build.display_name,
'docker_tags': tags,
'trigger_id': trigger.uuid if trigger is not None else None,
'trigger_kind': trigger_kind,
'trigger_metadata':
self.build_config.get('trigger_metadata', {})
}
if image_id is not None:
event_data['image_id'] = image_id
if manifest_digests:
event_data['manifest_digests'] = manifest_digests
if error_message is not None:
event_data['error_message'] = error_message
# TODO: remove when more endpoints have been converted to using
# interfaces
repo = AttrDict({
'namespace_name':
self.repo_build.repository.namespace_user.username,
'name':
self.repo_build.repository.name,
})
spawn_notification(repo,
kind,
event_data,
subpage='build/%s' % self.repo_build.uuid,
pathargs=['build', self.repo_build.uuid])
def __exit__(self, ex_type, ex_value, ex_traceback):
if ex_type is None and ex_value is None and ex_traceback is None:
with UseThenDisconnect(config.app_config):
logger.debug("Deleting logs from IDs %s to %s", self.start_id,
self.end_id)
delete_stale_logs(self.start_id, self.end_id, self.log_model)
def _build_complete(self, result):
""" Wraps up a completed build. Handles any errors and calls self._build_finished. """
build_id = self._current_job.repo_build.uuid
try:
# Retrieve the result. This will raise an ApplicationError on any error that occurred.
result_value = result.result()
kwargs = {}
# Note: If we are hitting an older builder that didn't return ANY map data, then the result
# value will be a bool instead of a proper CallResult object.
# Therefore: we have a try-except guard here to ensure we don't hit this pitfall.
try:
kwargs = result_value.kwresults
except:
pass
try:
yield From(self._build_status.set_phase(BUILD_PHASE.COMPLETE))
except InvalidRepositoryBuildException:
logger.warning(
'Build %s was not found; repo was probably deleted',
build_id)
raise Return()
yield From(self._build_finished(BuildJobResult.COMPLETE))
# Label the pushed manifests with the build metadata.
manifest_digests = kwargs.get('digests') or []
repository = registry_model.lookup_repository(
self._current_job.namespace, self._current_job.repo_name)
if repository is not None:
for digest in manifest_digests:
with UseThenDisconnect(app.config):
manifest = registry_model.lookup_manifest_by_digest(
repository, digest, require_available=True)
if manifest is None:
continue
registry_model.create_manifest_label(
manifest, INTERNAL_LABEL_BUILD_UUID, build_id,
'internal', 'text/plain')
# Send the notification that the build has completed successfully.
self._current_job.send_notification(
'build_success',
image_id=kwargs.get('image_id'),
manifest_digests=manifest_digests)
except ApplicationError as aex:
worker_error = WorkerError(aex.error, aex.kwargs.get('base_error'))
# Write the error to the log.
yield From(
self._build_status.set_error(
worker_error.public_message(),
worker_error.extra_data(),
internal_error=worker_error.is_internal_error(),
requeued=self._current_job.has_retries_remaining()))
# Send the notification that the build has failed.
self._current_job.send_notification(
'build_failure', error_message=worker_error.public_message())
# Mark the build as completed.
if worker_error.is_internal_error():
logger.exception(
'[BUILD INTERNAL ERROR: Remote] Build ID: %s: %s',
build_id, worker_error.public_message())
yield From(self._build_finished(BuildJobResult.INCOMPLETE))
else:
logger.debug('Got remote failure exception for build %s: %s',
build_id, aex)
yield From(self._build_finished(BuildJobResult.ERROR))
# Remove the current job.
self._current_job = None
