def test_job_local_paths_are_hidden(self, mock_open):
invalid_csv = 'id,type\n' + '1,a,\n' * 1010
invalid_file = StringIO.StringIO()
invalid_file.write(invalid_csv)
mock_upload = MockFieldStorage(invalid_file, 'invalid.csv')
resource = factories.Resource(format='csv', upload=mock_upload)
invalid_stream = io.BufferedReader(io.BytesIO(invalid_csv))
with mock.patch('io.open', return_value=invalid_stream):
run_validation_job(resource)
validation = Session.query(Validation).filter(
Validation.resource_id == resource['id']).one()
source = validation.report['tables'][0]['source']
assert source.startswith('http')
assert source.endswith('invalid.csv')
warning = validation.report['warnings'][0]
assert_equals(warning,
'Table inspection has reached 1000 row(s) limit')
def test_job_pass_validation_options_string(self, mock_open):
invalid_csv = '''
a;b;c
#comment
1;2;3
'''
validation_options = '''{
"headers": 3,
"skip_rows": ["#"]
}'''
invalid_file = StringIO.StringIO()
invalid_file.write(invalid_csv)
mock_upload = MockFieldStorage(invalid_file, 'invalid.csv')
resource = factories.Resource(format='csv',
upload=mock_upload,
validation_options=validation_options)
invalid_stream = io.BufferedReader(io.BytesIO(invalid_csv))
with mock.patch('io.open', return_value=invalid_stream):
run_validation_job(resource)
validation = Session.query(Validation).filter(
Validation.resource_id == resource['id']).one()
assert_equals(validation.report['valid'], True)
def test_job_pass_validation_options(self, mock_open):
invalid_csv = b'''
a,b,c
#comment
1,2,3
'''
validation_options = {'headers': 3, 'skip_rows': ['#']}
invalid_file = io.BytesIO(invalid_csv)
mock_upload = MockFieldStorage(invalid_file, 'invalid.csv')
resource = factories.Resource(format='csv',
upload=mock_upload,
validation_options=validation_options)
invalid_stream = io.BufferedReader(io.BytesIO(invalid_csv))
with mock.patch('io.open', return_value=invalid_stream):
run_validation_job(resource)
validation = Session.query(Validation).filter(
Validation.resource_id == resource['id']).one()
assert validation.report['valid']
def test_job_run_uploaded_file_replaces_paths(self, mock_uploader,
mock_validate):
resource = factories.Resource(url='__upload',
url_type='upload',
format='csv')
run_validation_job(resource)
validation = Session.query(Validation).filter(
Validation.resource_id == resource['id']).one()
assert validation.report['tables'][0]['source'].startswith('http')
def test_job_run_invalid_stores_validation_object(self, mock_validate):
resource = factories.Resource(url='http://example.com/file.csv',
format='csv')
run_validation_job(resource)
validation = Session.query(Validation).filter(
Validation.resource_id == resource['id']).one()
assert_equals(validation.status, 'failure')
assert_equals(validation.report, INVALID_REPORT)
assert validation.finished
def test_job_run_error_stores_validation_object(self, mock_validate):
resource = factories.Resource(url='http://example.com/file.csv',
format='csv')
run_validation_job(resource)
validation = Session.query(Validation).filter(
Validation.resource_id == resource['id']).one()
assert_equals(validation.status, 'error')
assert_equals(validation.report, None)
assert_equals(validation.error, {'message': 'Some warning'})
assert validation.finished
def test_job_run_valid_stores_status_in_resource(self, mock_validate):
resource = factories.Resource(url='http://example.com/file.csv',
format='csv')
run_validation_job(resource)
validation = Session.query(Validation).filter(
Validation.resource_id == resource['id']).one()
updated_resource = call_action('resource_show', id=resource['id'])
assert_equals(updated_resource['validation_status'], validation.status)
assert_equals(updated_resource['validation_timestamp'],
validation.finished.isoformat())