def test_aborts_on_missing_term(self, app, caplog):
from nessie.jobs.create_sis_schema import CreateSisSchema
with mock_s3(app):
daily_path = get_s3_sis_daily_path()
historical_path = app.config[
'LOCH_S3_SIS_DATA_PATH'] + '/historical'
self._upload_data_to_s3(daily_path, historical_path)
s3.delete_objects(
[f'{daily_path}/enrollments/enrollments-2178.gz'])
with capture_app_logs(app):
with pytest.raises(BackgroundJobError) as e:
CreateSisSchema().update_manifests()
assert 'Expected filename enrollments-2178.gz not found in S3, aborting' in str(
e.value)
def delete_old_incrementals(self):
old_incrementals = s3.get_keys_with_prefix(self.transient_path, bucket=self.transient_bucket)
if old_incrementals is None:
raise BackgroundJobError('Error listing old incrementals, aborting job.')
if len(old_incrementals) > 0:
delete_response = s3.delete_objects(old_incrementals, bucket=self.transient_bucket)
if not delete_response:
raise BackgroundJobError(f'Error deleting old incremental files from {self.transient_bucket}, aborting job.')
else:
app.logger.info(f'Deleted {len(old_incrementals)} old incremental files from {self.transient_bucket}.')
def delete_old_unloads(self):
old_unloads = s3.get_keys_with_prefix(app.config['LRS_CANVAS_INCREMENTAL_ETL_PATH_REDSHIFT'], bucket=self.transient_bucket)
if old_unloads is None:
raise BackgroundJobError('Error listing old unloads, aborting job.')
if len(old_unloads) > 0:
delete_response = s3.delete_objects(old_unloads, bucket=self.transient_bucket)
if not delete_response:
raise BackgroundJobError(f'Error deleting old unloads from {self.transient_bucket}, aborting job.')
else:
app.logger.info(f'Deleted {len(old_unloads)} old unloads from {self.transient_bucket}.')
def run(self, cleanup=True):
job_id = self.generate_job_id()
app.logger.info(f'Starting Canvas snapshot resync job... (id={job_id})')
md = metadata.get_failures_from_last_sync()
if not md['failures']:
return f"No failures found for job_id {md['job_id']}, skipping resync."
app.logger.info(f"Found {len(md['failures'])} failures for job_id {md['job_id']}, attempting resync.")
failures = 0
successes = 0
for failure in md['failures']:
if cleanup and failure['destination_url']:
destination_key = failure['destination_url'].split(app.config['LOCH_S3_BUCKET'] + '/')[1]
if s3.delete_objects([destination_key]):
metadata.delete_canvas_snapshots([destination_key])
else:
app.logger.error(f'Could not delete failed snapshot from S3 (url={failure.destination_url})')
metadata.create_canvas_sync_status(
job_id=job_id,
filename=failure['filename'],
canvas_table=failure['canvas_table'],
# The original signed source URL will remain valid if the resync job is run within an hour of the sync job.
# TODO Add logic to fetch a new signed URL from the Canvas Data API for older jobs.
source_url=failure['source_url'],
)
# Regenerate the S3 key, since the failed job may not have progressed far enough to store a destination URL in its metadata.
if failure['canvas_table'] == 'requests':
key_components = [berkeley.s3_canvas_data_path_current_term(), failure['canvas_table'], failure['filename']]
else:
key_components = [get_s3_canvas_daily_path(), failure['canvas_table'], failure['filename']]
key = '/'.join(key_components)
response = dispatch('sync_file_to_s3', data={'canvas_sync_job_id': job_id, 'url': failure['source_url'], 'key': key})
if not response:
app.logger.error('Failed to dispatch S3 resync of snapshot ' + failure['filename'])
metadata.update_canvas_sync_status(job_id, key, 'error', details=f'Failed to dispatch: {response}')
failures += 1
else:
app.logger.info('Dispatched S3 resync of snapshot ' + failure['filename'])
successes += 1
return f'Canvas snapshot resync job dispatched to workers ({successes} successful dispatches, {failures} failures).'
def delete_objects_with_prefix(prefix, whitelist=[]):
keys_to_delete = []
existing_keys = s3.get_keys_with_prefix(prefix)
if existing_keys is None:
app.logger.error('Error listing keys, aborting job.')
return False
for key in existing_keys:
filename = key.split('/')[-1]
if filename not in whitelist:
keys_to_delete.append(key)
app.logger.info(
f'Found {len(existing_keys)} key(s) matching prefix "{prefix}", {len(existing_keys) - len(keys_to_delete)} '
f'key(s) in whitelist, will delete {len(keys_to_delete)} object(s)')
if not keys_to_delete:
return True
if s3.delete_objects(keys_to_delete):
metadata.delete_canvas_snapshots(keys_to_delete)
return True
else:
return False
def cleanup_s3(app):
yield
from nessie.externals import s3
keys = s3.get_keys_with_prefix(app.config['LOCH_S3_PREFIX_TESTEXT'])
s3.delete_objects(keys)