def archive_builds_task(days=14, limit=200, include_cold=False, delete=False):
"""
Find stale builds and remove build paths.
:arg days: Find builds older than `days` days.
:arg include_cold: If True, include builds that are already in cold storage
:arg delete: If True, deletes BuildCommand objects after archiving them
"""
max_date = datetime.now() - timedelta(days=days)
queryset = Build.objects.exclude(commands__isnull=True)
if not include_cold:
queryset = queryset.exclude(cold_storage=True)
queryset = queryset.filter(date__lt=max_date)[:limit]
for build in queryset:
data = BuildSerializer(build).data['commands']
if data:
for cmd in data:
if len(cmd['output']) > MAX_BUILD_COMMAND_SIZE:
cmd['output'] = cmd['output'][-MAX_BUILD_COMMAND_SIZE:]
cmd['output'] = "... (truncated) ...\n\nCommand output too long. Truncated to last 1MB.\n\n" + cmd['output'] # noqa
log.warning('Truncating build command for build. build=%s', build.pk)
output = BytesIO()
output.write(json.dumps(data).encode('utf8'))
output.seek(0)
filename = '{date}/{id}.json'.format(date=str(build.date.date()), id=build.id)
try:
build_commands_storage.save(name=filename, content=output)
build.cold_storage = True
build.save()
if delete:
build.commands.all().delete()
except IOError:
log.exception('Cold Storage save failure')
def archive_builds_task(self, days=14, limit=200, delete=False):
"""
Task to archive old builds to cold storage.
:arg days: Find builds older than `days` days.
:arg delete: If True, deletes BuildCommand objects after archiving them
"""
if not settings.RTD_SAVE_BUILD_COMMANDS_TO_STORAGE:
return
lock_id = '{0}-lock'.format(self.name)
with memcache_lock(lock_id, self.app.oid) as acquired:
if not acquired:
log.warning('Archive Builds Task still locked')
return False
max_date = timezone.now() - timezone.timedelta(days=days)
queryset = (Build.objects.exclude(cold_storage=True).filter(
date__lt=max_date).prefetch_related('commands').only(
'date', 'cold_storage')[:limit])
for build in queryset:
commands = BuildCommandSerializer(build.commands, many=True).data
if commands:
for cmd in commands:
if len(cmd['output']) > MAX_BUILD_COMMAND_SIZE:
cmd['output'] = cmd['output'][-MAX_BUILD_COMMAND_SIZE:]
cmd['output'] = "... (truncated) ...\n\nCommand output too long. Truncated to last 1MB.\n\n" + cmd[
'output'] # noqa
log.warning('Truncating build command for build.',
build_id=build.id)
output = BytesIO(json.dumps(commands).encode('utf8'))
filename = '{date}/{id}.json'.format(date=str(
build.date.date()),
id=build.id)
try:
build_commands_storage.save(name=filename, content=output)
if delete:
build.commands.all().delete()
except IOError:
log.exception('Cold Storage save failure')
continue
build.cold_storage = True
build.save()