def generate_all_tables():
chained_job = ChainedBackgroundJob(steps=[
CreateCanvasSchema(),
CreateSisSchema(),
GenerateIntermediateTables(),
GenerateBoacAnalytics(),
], )
job_started = chained_job.run_async()
return respond_with_status(job_started)
def test_update_manifests(self, app):
"""Updates manifests in S3."""
from nessie.jobs.create_sis_schema import CreateSisSchema
with mock_s3(app):
daily_path = get_s3_sis_daily_path()
historical_path = app.config[
'LOCH_S3_SIS_DATA_PATH'] + '/historical'
self._upload_data_to_s3(daily_path, historical_path)
assert CreateSisSchema().update_manifests()
self._assert_complete_manifest(app, daily_path, historical_path)
def test_fallback_update_manifests(self, app):
"""Uses yesterday's news if today's is unavailable."""
from nessie.jobs.create_sis_schema import CreateSisSchema
with mock_s3(app):
yesterday = datetime.now() - timedelta(days=1)
daily_path = get_s3_sis_daily_path(yesterday)
historical_path = app.config[
'LOCH_S3_SIS_DATA_PATH'] + '/historical'
self._upload_data_to_s3(daily_path, historical_path)
assert CreateSisSchema().update_manifests()
self._assert_complete_manifest(app, daily_path, historical_path)
def __init__(self):
steps = [
CreateCoeSchema(),
ImportAscAthletes(),
GenerateAscProfiles(),
CreateUndergradsSchema(),
ImportNonCurrentStudents(),
CreateSisSchema(),
ImportCalNetData(),
CreateCalNetSchema(),
ImportStudentPhotos(),
]
super().__init__(steps=steps)
def test_aborts_on_missing_term(self, app, caplog):
from nessie.jobs.create_sis_schema import CreateSisSchema
with mock_s3(app):
daily_path = get_s3_sis_daily_path()
historical_path = app.config[
'LOCH_S3_SIS_DATA_PATH'] + '/historical'
self._upload_data_to_s3(daily_path, historical_path)
s3.delete_objects(
[f'{daily_path}/enrollments/enrollments-2178.gz'])
with capture_app_logs(app):
with pytest.raises(BackgroundJobError) as e:
CreateSisSchema().update_manifests()
assert 'Expected filename enrollments-2178.gz not found in S3, aborting' in str(
e.value)
def test_fallback_update_manifests(self, app):
"""Uses yesterday's news if today's is unavailable."""
with mock_s3(app):
yesterday = datetime.now() - timedelta(days=1)
daily_path = get_s3_sis_daily_path(yesterday)
historical_path = app.config[
'LOCH_S3_SIS_DATA_PATH'] + '/historical'
manifest_path = app.config['LOCH_S3_SIS_DATA_PATH'] + '/manifests'
s3.upload_data('some new course data',
f'{daily_path}/courses/courses-aaa.gz')
s3.upload_data('some more new course data',
f'{daily_path}/courses/courses-bbb.gz')
s3.upload_data('some new enrollment data',
f'{daily_path}/enrollments/enrollments-ccc.gz')
s3.upload_data('some old course data',
f'{historical_path}/courses/courses-ddd.gz')
s3.upload_data(
'some old enrollment data',
f'{historical_path}/enrollments/enrollments-eee.gz')
s3.upload_data(
'some perfectly antique enrollment data',
f'{historical_path}/enrollments/enrollments-fff.gz')
assert CreateSisSchema().update_manifests()
courses_manifest = json.loads(
s3.get_object_text(manifest_path + '/courses.json'))
assert len(courses_manifest['entries']) == 3
assert courses_manifest['entries'][0][
'url'] == f's3://{app.config["LOCH_S3_BUCKET"]}/{daily_path}/courses/courses-aaa.gz'
assert courses_manifest['entries'][0]['meta'][
'content_length'] == 20
enrollments_manifest = json.loads(
s3.get_object_text(manifest_path + '/enrollments.json'))
assert len(enrollments_manifest['entries']) == 3
assert (
enrollments_manifest['entries'][2]['url'] ==
f's3://{app.config["LOCH_S3_BUCKET"]}/{historical_path}/enrollments/enrollments-fff.gz'
)
assert enrollments_manifest['entries'][2]['meta'][
'content_length'] == 38
def create_sis_schema():
job_started = CreateSisSchema().run_async()
return respond_with_status(job_started)