def test_submit_restore_version_cancel(mock_tabulator_stream, is_authenticated, app_with_db, captured_templates): pipeline = PipelineFactory() # Latest version PipelineDataFileFactory( pipeline=pipeline, state=DataUploaderFileState.COMPLETED.value, processed_at=datetime.datetime(2020, 7, 1), ) # Version that can be restored data_file_2 = PipelineDataFileFactory( pipeline=pipeline, state=DataUploaderFileState.COMPLETED.value, processed_at=datetime.datetime(2020, 6, 1), ) client = get_client(app_with_db) url = url_for('uploader_views.pipeline_restore_version', slug=pipeline.slug, file_id=data_file_2.id) form_data = {'proceed': NO} response = client.post(url, data=form_data, follow_redirects=True, content_type='multipart/form-data') html = response.get_data(as_text=True) assert 'Existing versions' in html template, template_context = captured_templates.pop() assert template.name == 'pipeline_data_upload.html' assert template_context['request'].path == url_for( 'uploader_views.pipeline_data_upload', slug=pipeline.slug)
def test_submit_restore_version_proceed( mock_tabulator_stream, mock_process_pipeline_data_file, is_authenticated, app_with_db, captured_templates, ): mock_thread = mock.Mock() mock_process_pipeline_data_file.return_value = mock_thread csv_string = 'hello,goodbye\n1,2\n3,4' # The view first gets the s3 sample of the latest version and then the version to restore _mock_stream_return_values(mock_tabulator_stream, [csv_string, csv_string]) pipeline = PipelineFactory() # Latest version PipelineDataFileFactory( pipeline=pipeline, state=DataUploaderFileState.COMPLETED.value, processed_at=datetime.datetime(2020, 7, 1), delimiter=',', ) # Version that can be restored data_file_2 = PipelineDataFileFactory( pipeline=pipeline, state=DataUploaderFileState.COMPLETED.value, processed_at=datetime.datetime(2020, 6, 1), delimiter=',', ) client = get_client(app_with_db) url = url_for('uploader_views.pipeline_restore_version', slug=pipeline.slug, file_id=data_file_2.id) form_data = {'proceed': YES} response = client.post(url, data=form_data, follow_redirects=True, content_type='multipart/form-data') html = response.get_data(as_text=True) assert 'Data now being processed' in html template, template_context = captured_templates.pop() assert template.name == 'pipeline_data_uploaded.html' assert template_context['request'].path == url_for( 'uploader_views.pipeline_data_uploaded', slug=pipeline.slug, file_id=data_file_2.id) assert mock_process_pipeline_data_file.called is True assert mock_thread.start.called is True
def test_submit_data_using_reserved_column_is_rejected_early( mock_tabulator_stream, mock_delete_file, is_authenticated, app_with_db, captured_templates): csv_string = 'id\n1\n3' _mock_stream_return_values(mock_tabulator_stream, [csv_string]) data_file = PipelineDataFileFactory() client = get_client(app_with_db) url = url_for('uploader_views.pipeline_data_verify', slug=data_file.pipeline.slug, file_id=data_file.id) response, template_context = _test_view( client, url, 'pipeline_data_verify.html', captured_templates, ) html = response.get_data(as_text=True) assert 'Try again' in html form = template_context['form'] assert form.errors == { 'non_field_errors': [("Unable to process CSV file: “id” in the uploaded file is a reserved column name. " "You must rename that column in the data file.")] }
def test_submit_data_verify_proceed_yes( mock_tabulator_stream, mock_process_pipeline_data_file, is_authenticated, app_with_db, captured_templates, ): mock_thread = mock.Mock() mock_process_pipeline_data_file.return_value = mock_thread _mock_stream_return_values(mock_tabulator_stream, 'hello,goodbye\n1,2\n3,4') data_file = PipelineDataFileFactory(delimiter=',') client = get_client(app_with_db) url = url_for('uploader_views.pipeline_data_verify', slug=data_file.pipeline.slug, file_id=data_file.id) form_data = {'proceed': YES, 'hello': 'integer', 'goodbye': 'integer'} response = client.post(url, data=form_data, follow_redirects=True, content_type='multipart/form-data') html = response.get_data(as_text=True) assert 'Data now being processed' in html template, template_context = captured_templates.pop() assert template.name == 'pipeline_data_uploaded.html' assert template_context['request'].path == url_for( 'uploader_views.pipeline_data_uploaded', slug=data_file.pipeline.slug, file_id=data_file.id) assert PipelineDataFile.query.get(data_file.id) assert mock_process_pipeline_data_file.called is True assert mock_thread.start.called is True
def test_mark_old_processing_data_files_as_failed(app_with_db): recent_uuid = 'ed67ecea-9567-4f87-a5b5-eb517d84ce6c' PipelineDataFileFactory( id=recent_uuid, state=DataUploaderFileState.VERIFIED.value, started_processing_at=datetime.now() - timedelta(hours=1), ) old_uuid = '7849ab8c-96e4-40ff-afc7-40ae1decd6b2' PipelineDataFileFactory( id=old_uuid, state=DataUploaderFileState.VERIFIED.value, started_processing_at=datetime.now() - timedelta(days=2), ) mark_old_processing_data_files_as_failed(app_with_db) assert PipelineDataFile.query.get( recent_uuid).state == DataUploaderFileState.VERIFIED.value assert PipelineDataFile.query.get( old_uuid).state == DataUploaderFileState.FAILED.value
def test_get_restore_version_view(mock_tabulator_stream, is_authenticated, app_with_db, captured_templates): csv_string_1 = 'hello,goodbye\n1,2\n3,4' # latest csv_string_2 = 'hi,bye\n1,2\n3,4' # version to restore # The view first gets the s3 sample of the latest version and then the version to restore _mock_stream_return_values(mock_tabulator_stream, [csv_string_1, csv_string_2]) pipeline = PipelineFactory() # Latest version PipelineDataFileFactory( pipeline=pipeline, state=DataUploaderFileState.COMPLETED.value, processed_at=datetime.datetime(2020, 7, 1), column_types=[('hello', 'integer'), ('goodbye', 'integer')], ) # Version that can be restored data_file_2 = PipelineDataFileFactory( pipeline=pipeline, state=DataUploaderFileState.COMPLETED.value, processed_at=datetime.datetime(2020, 6, 1), column_types=[('hi', 'integer'), ('bye', 'integer')], ) client = get_client(app_with_db) url = url_for('uploader_views.pipeline_restore_version', slug=pipeline.slug, file_id=data_file_2.id) response, template_context = _test_view( client, url, 'pipeline_restore_version.html', captured_templates, ) html = response.get_data(as_text=True) assert 'Latest version' in html # data_file_1 assert 'hello' in html assert 'goodbye' in html assert 'Version to restore' in html # data_file_2 assert 'hi' in html assert 'bye' in html
def test_get_data_upload_view_existing_versions(is_authenticated, app_with_db, captured_templates): pipeline = PipelineFactory() # Latest version PipelineDataFileFactory( pipeline=pipeline, state=DataUploaderFileState.COMPLETED.value, processed_at=datetime.datetime(2020, 7, 1), ) # Version that can be restored PipelineDataFileFactory( pipeline=pipeline, state=DataUploaderFileState.COMPLETED.value, processed_at=datetime.datetime(2020, 6, 1), ) # Version that is in the middle of processing PipelineDataFileFactory( pipeline=pipeline, state=DataUploaderFileState.PROCESSING_DATAFLOW.value, processed_at=None) # Version that is yet to be processed PipelineDataFileFactory(pipeline=pipeline, state=DataUploaderFileState.UPLOADED.value, processed_at=None) client = get_client(app_with_db) url = url_for('uploader_views.pipeline_data_upload', slug=pipeline.slug) response, template_context = _test_view( client, url, 'pipeline_data_upload.html', captured_templates, ) html = response.get_data(as_text=True) assert 'Latest' in html # data_file_1 assert 'Restore' in html # data_file_2 assert 'View progress' in html # data_file_3 assert 'Process' in html # data_file_4
def test_get_data_uploaded_view(is_authenticated, process_pipeline_data_file, app_with_db, captured_templates): data_file = PipelineDataFileFactory() client = get_client(app_with_db) url = url_for('uploader_views.pipeline_data_uploaded', slug=data_file.pipeline.slug, file_id=data_file.id) response, template_context = _test_view( client, url, 'pipeline_data_uploaded.html', captured_templates, ) html = response.get_data(as_text=True) assert 'Data now being processed' in html
def test_data_uploaded_view_immediately_redirects_to_failed_if_pipeline_failed( is_authenticated, process_pipeline_data_file, app_with_db, captured_templates): data_file = PipelineDataFileFactory.create( state=DataUploaderFileState.FAILED.value) client = get_client(app_with_db) url = url_for('uploader_views.pipeline_data_uploaded', slug=data_file.pipeline.slug, file_id=data_file.id) response = make_sso_request(client, url) assert response.status_code == 302 assert response.location.endswith( url_for( 'uploader_views.pipeline_data_upload_failed', slug=data_file.pipeline.slug, file_id=data_file.id, ))
def test_submit_data_verify_proceed_blank(mock_open, mock_delete_file, is_authenticated, app_with_db, captured_templates): csv_string = 'hello,goodbye\n1,2\n3,4' mock_open.side_effect = [io.StringIO(csv_string), io.StringIO(csv_string)] data_file = PipelineDataFileFactory() client = get_client(app_with_db) url = url_for('uploader_views.pipeline_data_verify', slug=data_file.pipeline.slug, file_id=data_file.id) form_data = {} response = client.post(url, data=form_data, follow_redirects=True, content_type='multipart/form-data') html = response.get_data(as_text=True) assert 'Confirm or reject the data file contents' in html
def test_get_data_verify_view(mock_tabulator_stream, is_authenticated, app_with_db, captured_templates): _mock_stream_return_values(mock_tabulator_stream, ['hello,goodbye\n1,2\n3,4']) data_file = PipelineDataFileFactory(delimiter=',') client = get_client(app_with_db) url = url_for('uploader_views.pipeline_data_verify', slug=data_file.pipeline.slug, file_id=data_file.id) response, template_context = _test_view( client, url, 'pipeline_data_verify.html', captured_templates, ) html = response.get_data(as_text=True) assert 'Data successfully uploaded' in html
def test_data_upload_failed_view_has_link_to_try_again( is_authenticated, process_pipeline_data_file, app_with_db, captured_templates): data_file = PipelineDataFileFactory.create( state=DataUploaderFileState.FAILED.value) client = get_client(app_with_db) url = url_for( 'uploader_views.pipeline_data_upload_failed', slug=data_file.pipeline.slug, file_id=data_file.id, ) response = make_sso_request(client, url) body = response.get_data(as_text=True) assert response.status_code == 200 assert "Try again" in body assert (url_for( 'uploader_views.pipeline_data_upload', slug=data_file.pipeline.slug, ) in body)
def test_submit_data_verify_proceed_no(mock_delete_file, is_authenticated, app_with_db, captured_templates): data_file = PipelineDataFileFactory() client = get_client(app_with_db) url = url_for('uploader_views.pipeline_data_verify', slug=data_file.pipeline.slug, file_id=data_file.id) form_data = {'proceed': NO} response = client.post(url, data=form_data, follow_redirects=True, content_type='multipart/form-data') html = response.get_data(as_text=True) assert 'Continue to upload data' in html template, template_context = captured_templates.pop() assert template.name == 'pipeline_select.html' assert template_context['request'].path == url_for( 'uploader_views.pipeline_select') assert not PipelineDataFile.query.get(data_file.id) assert mock_delete_file.called is True
def test_get_data_verify_error_view(mock_tabulator_stream, is_authenticated, app_with_db, captured_templates): mock_tabulator_stream.side_effect = EncodingError('invalid encoding') data_file = PipelineDataFileFactory() client = get_client(app_with_db) url = url_for('uploader_views.pipeline_data_verify', slug=data_file.pipeline.slug, file_id=data_file.id) response, template_context = _test_view( client, url, 'pipeline_data_verify.html', captured_templates, ) html = response.get_data(as_text=True) assert 'Try again' in html form = template_context['form'] assert form.errors == { 'non_field_errors': [("Unable to process CSV file: the CSV file could not be opened. " "(Technical details: invalid encoding)")] }