def test_submit_restore_version_cancel(mock_tabulator_stream, is_authenticated,
app_with_db, captured_templates):
pipeline = PipelineFactory()
# Latest version
PipelineDataFileFactory(
pipeline=pipeline,
state=DataUploaderFileState.COMPLETED.value,
processed_at=datetime.datetime(2020, 7, 1),
)
# Version that can be restored
data_file_2 = PipelineDataFileFactory(
pipeline=pipeline,
state=DataUploaderFileState.COMPLETED.value,
processed_at=datetime.datetime(2020, 6, 1),
)
client = get_client(app_with_db)
url = url_for('uploader_views.pipeline_restore_version',
slug=pipeline.slug,
file_id=data_file_2.id)
form_data = {'proceed': NO}
response = client.post(url,
data=form_data,
follow_redirects=True,
content_type='multipart/form-data')
html = response.get_data(as_text=True)
assert 'Existing versions' in html
template, template_context = captured_templates.pop()
assert template.name == 'pipeline_data_upload.html'
assert template_context['request'].path == url_for(
'uploader_views.pipeline_data_upload', slug=pipeline.slug)
def test_submit_restore_version_proceed(
mock_tabulator_stream,
mock_process_pipeline_data_file,
is_authenticated,
app_with_db,
captured_templates,
):
mock_thread = mock.Mock()
mock_process_pipeline_data_file.return_value = mock_thread
csv_string = 'hello,goodbye\n1,2\n3,4'
# The view first gets the s3 sample of the latest version and then the version to restore
_mock_stream_return_values(mock_tabulator_stream, [csv_string, csv_string])
pipeline = PipelineFactory()
# Latest version
PipelineDataFileFactory(
pipeline=pipeline,
state=DataUploaderFileState.COMPLETED.value,
processed_at=datetime.datetime(2020, 7, 1),
delimiter=',',
)
# Version that can be restored
data_file_2 = PipelineDataFileFactory(
pipeline=pipeline,
state=DataUploaderFileState.COMPLETED.value,
processed_at=datetime.datetime(2020, 6, 1),
delimiter=',',
)
client = get_client(app_with_db)
url = url_for('uploader_views.pipeline_restore_version',
slug=pipeline.slug,
file_id=data_file_2.id)
form_data = {'proceed': YES}
response = client.post(url,
data=form_data,
follow_redirects=True,
content_type='multipart/form-data')
html = response.get_data(as_text=True)
assert 'Data now being processed' in html
template, template_context = captured_templates.pop()
assert template.name == 'pipeline_data_uploaded.html'
assert template_context['request'].path == url_for(
'uploader_views.pipeline_data_uploaded',
slug=pipeline.slug,
file_id=data_file_2.id)
assert mock_process_pipeline_data_file.called is True
assert mock_thread.start.called is True
def test_submit_data_using_reserved_column_is_rejected_early(
mock_tabulator_stream, mock_delete_file, is_authenticated, app_with_db,
captured_templates):
csv_string = 'id\n1\n3'
_mock_stream_return_values(mock_tabulator_stream, [csv_string])
data_file = PipelineDataFileFactory()
client = get_client(app_with_db)
url = url_for('uploader_views.pipeline_data_verify',
slug=data_file.pipeline.slug,
file_id=data_file.id)
response, template_context = _test_view(
client,
url,
'pipeline_data_verify.html',
captured_templates,
)
html = response.get_data(as_text=True)
assert 'Try again' in html
form = template_context['form']
assert form.errors == {
'non_field_errors':
[("Unable to process CSV file: “id” in the uploaded file is a reserved column name. "
"You must rename that column in the data file.")]
}
def test_submit_data_verify_proceed_yes(
mock_tabulator_stream,
mock_process_pipeline_data_file,
is_authenticated,
app_with_db,
captured_templates,
):
mock_thread = mock.Mock()
mock_process_pipeline_data_file.return_value = mock_thread
_mock_stream_return_values(mock_tabulator_stream,
'hello,goodbye\n1,2\n3,4')
data_file = PipelineDataFileFactory(delimiter=',')
client = get_client(app_with_db)
url = url_for('uploader_views.pipeline_data_verify',
slug=data_file.pipeline.slug,
file_id=data_file.id)
form_data = {'proceed': YES, 'hello': 'integer', 'goodbye': 'integer'}
response = client.post(url,
data=form_data,
follow_redirects=True,
content_type='multipart/form-data')
html = response.get_data(as_text=True)
assert 'Data now being processed' in html
template, template_context = captured_templates.pop()
assert template.name == 'pipeline_data_uploaded.html'
assert template_context['request'].path == url_for(
'uploader_views.pipeline_data_uploaded',
slug=data_file.pipeline.slug,
file_id=data_file.id)
assert PipelineDataFile.query.get(data_file.id)
assert mock_process_pipeline_data_file.called is True
assert mock_thread.start.called is True
def test_mark_old_processing_data_files_as_failed(app_with_db):
recent_uuid = 'ed67ecea-9567-4f87-a5b5-eb517d84ce6c'
PipelineDataFileFactory(
id=recent_uuid,
state=DataUploaderFileState.VERIFIED.value,
started_processing_at=datetime.now() - timedelta(hours=1),
)
old_uuid = '7849ab8c-96e4-40ff-afc7-40ae1decd6b2'
PipelineDataFileFactory(
id=old_uuid,
state=DataUploaderFileState.VERIFIED.value,
started_processing_at=datetime.now() - timedelta(days=2),
)
mark_old_processing_data_files_as_failed(app_with_db)
assert PipelineDataFile.query.get(
recent_uuid).state == DataUploaderFileState.VERIFIED.value
assert PipelineDataFile.query.get(
old_uuid).state == DataUploaderFileState.FAILED.value
def test_get_restore_version_view(mock_tabulator_stream, is_authenticated,
app_with_db, captured_templates):
csv_string_1 = 'hello,goodbye\n1,2\n3,4' # latest
csv_string_2 = 'hi,bye\n1,2\n3,4' # version to restore
# The view first gets the s3 sample of the latest version and then the version to restore
_mock_stream_return_values(mock_tabulator_stream,
[csv_string_1, csv_string_2])
pipeline = PipelineFactory()
# Latest version
PipelineDataFileFactory(
pipeline=pipeline,
state=DataUploaderFileState.COMPLETED.value,
processed_at=datetime.datetime(2020, 7, 1),
column_types=[('hello', 'integer'), ('goodbye', 'integer')],
)
# Version that can be restored
data_file_2 = PipelineDataFileFactory(
pipeline=pipeline,
state=DataUploaderFileState.COMPLETED.value,
processed_at=datetime.datetime(2020, 6, 1),
column_types=[('hi', 'integer'), ('bye', 'integer')],
)
client = get_client(app_with_db)
url = url_for('uploader_views.pipeline_restore_version',
slug=pipeline.slug,
file_id=data_file_2.id)
response, template_context = _test_view(
client,
url,
'pipeline_restore_version.html',
captured_templates,
)
html = response.get_data(as_text=True)
assert 'Latest version' in html # data_file_1
assert 'hello' in html
assert 'goodbye' in html
assert 'Version to restore' in html # data_file_2
assert 'hi' in html
assert 'bye' in html
def test_get_data_upload_view_existing_versions(is_authenticated, app_with_db,
captured_templates):
pipeline = PipelineFactory()
# Latest version
PipelineDataFileFactory(
pipeline=pipeline,
state=DataUploaderFileState.COMPLETED.value,
processed_at=datetime.datetime(2020, 7, 1),
)
# Version that can be restored
PipelineDataFileFactory(
pipeline=pipeline,
state=DataUploaderFileState.COMPLETED.value,
processed_at=datetime.datetime(2020, 6, 1),
)
# Version that is in the middle of processing
PipelineDataFileFactory(
pipeline=pipeline,
state=DataUploaderFileState.PROCESSING_DATAFLOW.value,
processed_at=None)
# Version that is yet to be processed
PipelineDataFileFactory(pipeline=pipeline,
state=DataUploaderFileState.UPLOADED.value,
processed_at=None)
client = get_client(app_with_db)
url = url_for('uploader_views.pipeline_data_upload', slug=pipeline.slug)
response, template_context = _test_view(
client,
url,
'pipeline_data_upload.html',
captured_templates,
)
html = response.get_data(as_text=True)
assert 'Latest' in html # data_file_1
assert 'Restore' in html # data_file_2
assert 'View progress' in html # data_file_3
assert 'Process' in html # data_file_4
def test_get_data_uploaded_view(is_authenticated, process_pipeline_data_file,
app_with_db, captured_templates):
data_file = PipelineDataFileFactory()
client = get_client(app_with_db)
url = url_for('uploader_views.pipeline_data_uploaded',
slug=data_file.pipeline.slug,
file_id=data_file.id)
response, template_context = _test_view(
client,
url,
'pipeline_data_uploaded.html',
captured_templates,
)
html = response.get_data(as_text=True)
assert 'Data now being processed' in html
def test_data_uploaded_view_immediately_redirects_to_failed_if_pipeline_failed(
is_authenticated, process_pipeline_data_file, app_with_db,
captured_templates):
data_file = PipelineDataFileFactory.create(
state=DataUploaderFileState.FAILED.value)
client = get_client(app_with_db)
url = url_for('uploader_views.pipeline_data_uploaded',
slug=data_file.pipeline.slug,
file_id=data_file.id)
response = make_sso_request(client, url)
assert response.status_code == 302
assert response.location.endswith(
url_for(
'uploader_views.pipeline_data_upload_failed',
slug=data_file.pipeline.slug,
file_id=data_file.id,
))
def test_submit_data_verify_proceed_blank(mock_open, mock_delete_file,
is_authenticated, app_with_db,
captured_templates):
csv_string = 'hello,goodbye\n1,2\n3,4'
mock_open.side_effect = [io.StringIO(csv_string), io.StringIO(csv_string)]
data_file = PipelineDataFileFactory()
client = get_client(app_with_db)
url = url_for('uploader_views.pipeline_data_verify',
slug=data_file.pipeline.slug,
file_id=data_file.id)
form_data = {}
response = client.post(url,
data=form_data,
follow_redirects=True,
content_type='multipart/form-data')
html = response.get_data(as_text=True)
assert 'Confirm or reject the data file contents' in html
def test_get_data_verify_view(mock_tabulator_stream, is_authenticated,
app_with_db, captured_templates):
_mock_stream_return_values(mock_tabulator_stream,
['hello,goodbye\n1,2\n3,4'])
data_file = PipelineDataFileFactory(delimiter=',')
client = get_client(app_with_db)
url = url_for('uploader_views.pipeline_data_verify',
slug=data_file.pipeline.slug,
file_id=data_file.id)
response, template_context = _test_view(
client,
url,
'pipeline_data_verify.html',
captured_templates,
)
html = response.get_data(as_text=True)
assert 'Data successfully uploaded' in html
def test_data_upload_failed_view_has_link_to_try_again(
is_authenticated, process_pipeline_data_file, app_with_db,
captured_templates):
data_file = PipelineDataFileFactory.create(
state=DataUploaderFileState.FAILED.value)
client = get_client(app_with_db)
url = url_for(
'uploader_views.pipeline_data_upload_failed',
slug=data_file.pipeline.slug,
file_id=data_file.id,
)
response = make_sso_request(client, url)
body = response.get_data(as_text=True)
assert response.status_code == 200
assert "Try again" in body
assert (url_for(
'uploader_views.pipeline_data_upload',
slug=data_file.pipeline.slug,
) in body)
def test_submit_data_verify_proceed_no(mock_delete_file, is_authenticated,
app_with_db, captured_templates):
data_file = PipelineDataFileFactory()
client = get_client(app_with_db)
url = url_for('uploader_views.pipeline_data_verify',
slug=data_file.pipeline.slug,
file_id=data_file.id)
form_data = {'proceed': NO}
response = client.post(url,
data=form_data,
follow_redirects=True,
content_type='multipart/form-data')
html = response.get_data(as_text=True)
assert 'Continue to upload data' in html
template, template_context = captured_templates.pop()
assert template.name == 'pipeline_select.html'
assert template_context['request'].path == url_for(
'uploader_views.pipeline_select')
assert not PipelineDataFile.query.get(data_file.id)
assert mock_delete_file.called is True
def test_get_data_verify_error_view(mock_tabulator_stream, is_authenticated,
app_with_db, captured_templates):
mock_tabulator_stream.side_effect = EncodingError('invalid encoding')
data_file = PipelineDataFileFactory()
client = get_client(app_with_db)
url = url_for('uploader_views.pipeline_data_verify',
slug=data_file.pipeline.slug,
file_id=data_file.id)
response, template_context = _test_view(
client,
url,
'pipeline_data_verify.html',
captured_templates,
)
html = response.get_data(as_text=True)
assert 'Try again' in html
form = template_context['form']
assert form.errors == {
'non_field_errors':
[("Unable to process CSV file: the CSV file could not be opened. "
"(Technical details: invalid encoding)")]
}