예제 #1
0
    def test_job_local_paths_are_hidden(self, mock_open):

        invalid_csv = 'id,type\n' + '1,a,\n' * 1010
        invalid_file = StringIO.StringIO()

        invalid_file.write(invalid_csv)

        mock_upload = MockFieldStorage(invalid_file, 'invalid.csv')

        resource = factories.Resource(format='csv', upload=mock_upload)

        invalid_stream = io.BufferedReader(io.BytesIO(invalid_csv))

        with mock.patch('io.open', return_value=invalid_stream):

            run_validation_job(resource)

        validation = Session.query(Validation).filter(
            Validation.resource_id == resource['id']).one()

        source = validation.report['tables'][0]['source']
        assert source.startswith('http')
        assert source.endswith('invalid.csv')

        warning = validation.report['warnings'][0]
        assert_equals(warning,
                      'Table inspection has reached 1000 row(s) limit')
예제 #2
0
    def test_job_pass_validation_options_string(self, mock_open):

        invalid_csv = '''

a;b;c
#comment
1;2;3
'''

        validation_options = '''{
            "headers": 3,
            "skip_rows": ["#"]
        }'''

        invalid_file = StringIO.StringIO()

        invalid_file.write(invalid_csv)

        mock_upload = MockFieldStorage(invalid_file, 'invalid.csv')

        resource = factories.Resource(format='csv',
                                      upload=mock_upload,
                                      validation_options=validation_options)

        invalid_stream = io.BufferedReader(io.BytesIO(invalid_csv))

        with mock.patch('io.open', return_value=invalid_stream):

            run_validation_job(resource)

        validation = Session.query(Validation).filter(
            Validation.resource_id == resource['id']).one()

        assert_equals(validation.report['valid'], True)
예제 #3
0
    def test_job_pass_validation_options(self, mock_open):

        invalid_csv = b'''

a,b,c
#comment
1,2,3
'''

        validation_options = {'headers': 3, 'skip_rows': ['#']}

        invalid_file = io.BytesIO(invalid_csv)

        mock_upload = MockFieldStorage(invalid_file, 'invalid.csv')

        resource = factories.Resource(format='csv',
                                      upload=mock_upload,
                                      validation_options=validation_options)

        invalid_stream = io.BufferedReader(io.BytesIO(invalid_csv))

        with mock.patch('io.open', return_value=invalid_stream):

            run_validation_job(resource)

        validation = Session.query(Validation).filter(
            Validation.resource_id == resource['id']).one()

        assert validation.report['valid']
예제 #4
0
    def test_job_run_uploaded_file_replaces_paths(self, mock_uploader,
                                                  mock_validate):

        resource = factories.Resource(url='__upload',
                                      url_type='upload',
                                      format='csv')

        run_validation_job(resource)

        validation = Session.query(Validation).filter(
            Validation.resource_id == resource['id']).one()

        assert validation.report['tables'][0]['source'].startswith('http')
예제 #5
0
    def test_job_run_invalid_stores_validation_object(self, mock_validate):

        resource = factories.Resource(url='http://example.com/file.csv',
                                      format='csv')

        run_validation_job(resource)

        validation = Session.query(Validation).filter(
            Validation.resource_id == resource['id']).one()

        assert_equals(validation.status, 'failure')
        assert_equals(validation.report, INVALID_REPORT)
        assert validation.finished
예제 #6
0
    def test_job_run_error_stores_validation_object(self, mock_validate):

        resource = factories.Resource(url='http://example.com/file.csv',
                                      format='csv')

        run_validation_job(resource)

        validation = Session.query(Validation).filter(
            Validation.resource_id == resource['id']).one()

        assert_equals(validation.status, 'error')
        assert_equals(validation.report, None)
        assert_equals(validation.error, {'message': 'Some warning'})
        assert validation.finished
예제 #7
0
    def test_job_run_valid_stores_status_in_resource(self, mock_validate):

        resource = factories.Resource(url='http://example.com/file.csv',
                                      format='csv')

        run_validation_job(resource)

        validation = Session.query(Validation).filter(
            Validation.resource_id == resource['id']).one()

        updated_resource = call_action('resource_show', id=resource['id'])

        assert_equals(updated_resource['validation_status'], validation.status)
        assert_equals(updated_resource['validation_timestamp'],
                      validation.finished.isoformat())